Merge remote-tracking branch 'origin/main' into refactor/track-edm-co…

…py-dynamic-backends

.github/workflows/checks.yml

@@ -150,3 +150,13 @@ jobs:
       - name: Check
         run: >
           CI/check_fpe_masks.py --token ${{ secrets.GITHUB_TOKEN }}
+  unused_files:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-python@v4
+        with:
+          python-version: '3.10'
+      - name: Check
+        run: >
+          CI/check_unused_files.py

.github/workflows/docs.yml

@@ -8,7 +8,7 @@ on:
       - 'release/**'
       - 'develop/**'
 
-concurrency: 
+concurrency:
   group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
   cancel-in-progress: true
 
@@ -17,7 +17,7 @@ jobs:
     runs-on: ubuntu-latest
     env:
         DOXYGEN_WARN_AS_ERROR: FAIL_ON_WARNINGS
-        DOXYGEN_VERSION: 1.9.4
+        DOXYGEN_VERSION: 1.9.8
     steps:
       - uses: actions/checkout@v3
 

.github/workflows/iwyu.yml

@@ -55,7 +55,7 @@ jobs:
           -DACTS_BUILD_EXAMPLES_EDM4HEP=ON
 
       - name: Run IWYU
-        run: python3 iwyu-install/bin/iwyu_tool.py -p acts-build/ -j2 -- --mapping_file=acts/CI/iwyu/mapping.imp | tee iwyu-output.txt
+        run: python3 iwyu-install/bin/iwyu_tool.py -p acts-build/ -j2 -- -Xiwyu --mapping_file=$(pwd)/acts/CI/iwyu/mapping.imp | tee iwyu-output.txt
       - name: Filter IWYU output
         run: python3 acts/CI/iwyu/filter.py acts/CI/iwyu/filter.yaml iwyu-output.txt iwyu-filtered.txt
       - name: Apply IWYU

.pre-commit-config.yaml

@@ -0,0 +1,12 @@
+repos:
+  - repo: https://github.com/pre-commit/mirrors-clang-format
+    rev: 'v14.0.0'
+    hooks:
+      - id: clang-format
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v3.2.0
+    hooks:
+    - id: trailing-whitespace
+    - id: end-of-file-fixer
+    - id: check-yaml
+    - id: check-added-large-files

CI/check_unused_files.py

@@ -0,0 +1,245 @@
+#!/usr/bin/env python3
+
+from pathlib import Path
+import os
+import sys
+import subprocess
+
+
+def file_can_be_removed(searchstring, scope):
+    cmd = "grep -IR '" + searchstring + "' " + " ".join(scope)
+
+    p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE)
+    output, _ = p.communicate()
+    return output == b""
+
+
+def count_files(path=".", exclude_dirs=(), exclude_files=()):
+    count = 0
+    for root, dirs, files in os.walk(path):
+        dirs[:] = [d for d in dirs if d not in exclude_dirs]
+        files[:] = [f for f in files if f not in exclude_files]
+        count += len(files)
+
+    return count
+
+
+def main():
+    print("\033[32mINFO\033[0m Start check_unused_files.py ...")
+    exclude_dirs = (
+        "Scripts",
+        "thirdparty",
+        "CI",
+        "git",
+        "cmake",
+        ".git",
+        ".github",
+        ".idea",
+    )
+    exclude_files = (
+        "acts_logo_colored.svg",
+        ".gitignore",
+        "README.md",
+        "CMakeLists.txt",
+        "DetUtils.h",
+        "CommandLineArguments.h",
+        # Filename not completed in source
+        "vertexing_event_mu20_beamspot.csv",
+        "vertexing_event_mu20_tracks.csv",
+        "vertexing_event_mu20_vertices_AMVF.csv",
+        # TODO Move the following files to a better place?
+        "Magfield.ipynb",
+        "SolenoidField.ipynb",
+        # TODO Add README next to the following files?
+        "default-input-config-generic.json",
+        "geoSelection-openDataDetector.json",
+        "alignment-geo-contextualDetector.json",
+        # TODO Mention these files somewhere?
+        "generate_particle_data_table.py",
+        "lazy_autodoc.py",
+    )
+
+    suffix_header = (
+        ".hpp",
+        ".cuh",
+    )
+    suffix_source = (
+        ".ipp",
+        ".cpp",
+        ".cu",
+    )
+    suffix_image = (
+        ".png",
+        ".svg",
+        ".jpg",
+        ".gif",
+    )
+    suffix_python = (".py",)
+    suffix_doc = (
+        ".md",
+        ".rst",
+    )
+    suffix_other = (
+        "",
+        ".csv",
+        ".css",
+        ".gdml",
+        ".hepmc3",
+        ".in",
+        ".ipynb",
+        ".json",
+        ".j2",
+        ".onnx",
+        ".root",
+        ".toml",
+        ".txt",
+        ".yml",
+    )
+    suffix_allowed = (
+        suffix_header
+        + suffix_source
+        + suffix_image
+        + suffix_python
+        + suffix_doc
+        + suffix_other
+    )
+
+    exit = 0
+
+    dirs_base = next(os.walk("."))[1]
+    dirs_base.append(".")
+    dirs_base[:] = [d for d in dirs_base if d not in exclude_dirs]
+    dirs_base_docs = ("docs",)
+    dirs_base_code = [d for d in dirs_base if d not in dirs_base_docs]
+
+    # Collectors
+    wrong_extension = ()
+    unused_files = ()
+
+    # walk over all files
+    for root, dirs, files in os.walk("."):
+        dirs[:] = [d for d in dirs if d not in exclude_dirs]
+        files[:] = [f for f in files if f not in exclude_files]
+
+        # Skip base-directory
+        if str(Path(root)) == ".":
+            continue
+
+        # Print progress
+        if root[2:] in dirs_base:
+            processed_files = 0
+            current_base_dir = root
+            number_files = count_files(root, exclude_dirs, exclude_files)
+            # print empty to start a new line
+            print("")
+
+        # Skip "white-paper-figures"
+        # TODO Find a more elegant way
+        if str(root).find("white_papers/figures") != -1:
+            processed_files += count_files(root, exclude_dirs, exclude_files)
+            continue
+
+        # Skip "DD4hep-tests" since their cmake looks a bit different
+        # TODO Find a more elegant way
+        if str(root).find("Tests/UnitTests/Plugins/DD4hep") != -1:
+            processed_files += count_files(root, exclude_dirs, exclude_files)
+            continue
+
+        root = Path(root)
+        for filename in files:
+            processed_files += 1
+            # get the full path of the file
+            filepath = root / filename
+
+            # Check for wrong extensions
+            if filepath.suffix not in suffix_allowed:
+                wrong_extension += (str(filepath),)
+
+            # Check header files and remove
+            elif filepath.suffix in suffix_header + suffix_source:
+                if file_can_be_removed(filepath.stem, dirs_base_code):
+                    unused_files += (str(filepath),)
+                    remove_cmd = "rm " + str(filepath)
+                    os.system(remove_cmd)
+
+            elif filepath.suffix in suffix_python:
+                # Skip the python tests folder
+                if str(root).find("Examples/Python") != -1:
+                    continue
+
+                if not file_can_be_removed("import .*" + filepath.stem, dirs_base):
+                    continue
+
+                if not file_can_be_removed(
+                    "from " + filepath.stem + " import", dirs_base
+                ):
+                    continue
+
+                if file_can_be_removed(filename, dirs_base):
+                    unused_files += (str(filepath),)
+                    remove_cmd = "rm " + str(filepath)
+                    os.system(remove_cmd)
+
+            # Check documentation files (weak tests)
+            # TODO find more reliable test for this
+            elif filepath.suffix in suffix_doc:
+                if file_can_be_removed(filepath.stem, dirs_base_docs):
+                    unused_files += (str(filepath),)
+                    remove_cmd = "rm " + str(filepath)
+                    os.system(remove_cmd)
+
+            # Check and print other files
+            elif filepath.suffix in suffix_image + suffix_other:
+                if file_can_be_removed(filename, dirs_base):
+                    unused_files += (str(filepath),)
+                    remove_cmd = "rm " + str(filepath)
+                    os.system(remove_cmd)
+
+        # Print the progress in place
+        progress = int(20 * processed_files / number_files)
+        sys.stdout.write("\r")
+        sys.stdout.write(
+            "Checked [%-20s] %d/%d files in %s"
+            % ("=" * progress, processed_files, number_files, current_base_dir)
+        )
+        sys.stdout.flush()
+
+    if len(wrong_extension) != 0:
+        print(
+            "\n\n\033[31mERROR\033[0m "
+            + f"The following {len(wrong_extension)} files have an unsupported extension:\n\n"
+            + "\033[31m"
+            + "\n".join(wrong_extension)
+            + "\033[0m"
+            + "\nCheck if you can change the format to one of the following:\n"
+            + "\n".join(suffix_allowed)
+            + "\nIf you really need that specific extension, add it to the list above.\n"
+        )
+
+        exit += 1
+
+    if len(unused_files) != 0:
+        print(
+            "\n\n\033[31mERROR\033[0m "
+            + f"The following {len(unused_files)} files seem to be unused:\n"
+            + "\033[31m"
+            + "\n".join(unused_files)
+            + "\033[0m"
+            + "\nYou have 3 options:"
+            + "\n\t- Remove them"
+            + "\n\t- Use them (check proper include)"
+            + "\n\t- Modify the ignore list of this check\n"
+        )
+
+        exit += 1
+
+    if exit == 0:
+        print(
+            "\n\n\033[32mINFO\033[0m Finished check_unused_files.py without any errors."
+        )
+
+    return exit
+
+
+if "__main__" == __name__:
+    sys.exit(main())

CI/iwyu/README.md

@@ -0,0 +1,16 @@
+# IWYU = Include what you use
+
+This tool finds unused includes and suggestes additional includes and declarations.
+
+It is not very stable at the moment and takes a few hours to complete within ACTS therefor it only runs once a week and can be triggered manually.
+
+There is also not sufficient filtering offered by IWYU at the moment. For that reason there is a specific filtering script for now which tries to get rid of unwanted changes.
+
+offline resources
+- [GitHub Actions Workflow](../../.github/workflows/iwyu.yml)
+- [Custom filter script](./filter.py)
+
+online resources
+
+- https://include-what-you-use.org/
+- https://github.com/include-what-you-use/include-what-you-use

CI/iwyu/filter.yaml

@@ -1,5 +1,5 @@
 remove_lines:
-  # ignore unexiting std
+  # ignore if not existing in std
   - "^#include <new>"
   - "^#include <bits/"
   - "^#include <ext/"
@@ -8,7 +8,7 @@ remove_lines:
   - "namespace Eigen {"
   # ignore boost
   - "^(- )?#include <boost/"
-  # don remove ipp
+  # don't remove ipp
   - "^- #include [<\"].*\\.ipp"
   # ignore pybind11
   - "^(- )?#include <pybind11/"
@@ -21,7 +21,7 @@ replace_lines:
   - "^#include <unistd\\.h>": "#include <cunistd>"
   - "^#include <stdint\\.h>": "#include <cstdint>"
   - "^#include <stdlib.h>": "#include <cstdlib>"
-  # don use ipp
+  # don't use ipp
   - "^#include ([<\"].*)\\.ipp": "#include \\1.hpp"
 
 ignore_files:

CI/physmon/summary.py

@@ -7,7 +7,7 @@
 import os
 import csv
 
-HERALD_URL = "https://herald.dokku.paulgessinger.com/view/{repo}/runs/{run_id}/artifacts/{artifact_name}/{path}"
+HERALD_URL = "https://acts-herald.app.cern.ch/view/{repo}/runs/{run_id}/artifacts/{artifact_name}/{path}"
 IS_CI = "GITHUB_ACTIONS" in os.environ
 
 

CI/physmon/vertexing_config.yml

@@ -84,6 +84,11 @@ histograms:
     nbins: 100
     min: 0.999
     max: 1
+
+  "trk_weight":
+    nbins: 100
+    min: -0.01
+    max: 1.01
 
 extra_histograms:
   - expression: df["nRecoVtx"] / df["nTrueVtx"]