Enhance add_dirs with exclusion feature

.github/workflows/tests.yml

@@ -105,6 +105,7 @@ jobs:
             bash \
             coreutils \
             dos2unix \
+            gnu-tar \
             grep
 
       - name: Install CMSIS-Toolbox

README.md

@@ -28,6 +28,7 @@ This library is written for Bash v5 or later and uses a couple of standard
 - realpath
 - sed
 - sha1sum
+- tar
 - test
 - unix2dos (optional)
 - unix2mac (optional)
@@ -54,6 +55,7 @@ This library requires Bash v5 and some additional GNU tools to be installed usin
 $ brew install \
     bash \ 
     coreutils \
+    gnu-tar \
     grep
 ```
 
@@ -106,39 +108,57 @@ can use the same relative file names as within the `.pdsc` file.
 1. **Windows only**! Run `git update-index --chmod=+x gen_pack.sh` to set the eXecute permission. Otherwise the script will not be executable in a Linux/Mac checkout by default such as running in a GitHub Action.
 
 1. Replace `<pin lib version here>` with the version of the library you want to use, e.g. `1.0.0`.
+
    For available versions see [Open-CMSIS-Pack/gen-pack/tags](https://github.com/Open-CMSIS-Pack/gen-pack/tags).
+
    Use the tag name without the prefix "v", e.g., 0.7.0
 
 1. Add any required default command line arguments to the line `DEFAULT_ARGS=()`.
+
    For example, add `-c [<prefix>]` here to force creating release history from Git.
+
    The `<prefix>` is the version prefixed used for release tags if any.
 
 1. Replace `<list directories here>` with a list of directories that shall be included in the pack.
+
    The directories are included recursively with all contained files. If left empty (i.e. `PACK_DIRS=""`),
    all folders next to the `.pdsc` file are copied.
+
    Subdirectories (e.g., `path/to/folder`) are copied with same hierarchy
    (i.e., resulting in `<build>/path/to/folder/**/*`).
-   Folder from outside the pack root (e.g., `../path/to/src`) are copied without hierarchy into the build
-   folder (i.e., resulting in `<build>/src`). For customizing the layout consider the `postprocess` hook.
+
+   Folders from outside the pack root (e.g., `../path/to/src`) are copied without hierarchy into the build
+   folder (i.e., resulting in `<build>/src`).
+
+   Files and folders used by common version control system (vsc) are ignored by default (`--exclude-vcs`).
+   By providing a `.gpignore` file in any folder additional patterns can be excluded (`--exclude-ignore`).
+   Find more details on the [tar manpage](https://www.gnu.org/software/tar/manual/html_node/exclude.html).
+
+   For customizing the layout any further consider the `postprocess` hook.
 
 1. Replace `<list files here>` with a list of files that shall be included in the pack.
+
    This can be used as an alternative to including whole directories.
    Files from subdirectories (e.g., `path/to/file`) are copied with same hierarchy
    (i.e., resulting in `<build>/path/to/file`).
+
    Files from outside the pack root (e.g., `../path/to/file`) are copied without hierarchy into the build
    folder (i.e., resulting in `<build>/file`). For customizing the layout consider the `postprocess` hook.
 
 1. Replace `<list files here>` with a list of files to be removed again.
+
    This can be used to copy whole directories and remove files afterwards.
 
 1. Replace `<list patches here>` with a list of patches that shall be applied.
 
 1. Add additional required command line arguments for packchk to the line `PACKCHK_ARGS=()`.
+
    For example, add `-x M353` to suppress this warning.
 
 1. Replace `<list pdsc files here>` with additional `.pdsc` files required to resolve references into other packs during `packchk`.
 
    The following formats can be used:
+
    - Plain `.pdsc` file looked up via `index.pidx`. File will be downloaded if not already in cache.
      E.g., `ARM.CMSIS.pdsc`.
 
@@ -166,6 +186,7 @@ can use the same relative file names as within the `.pdsc` file.
    into the `preprocess` and `postprocess` functions. The working directory (`pwd`) for
    both functions is the base folder containing the pack description file. The first
    parameter of the functions (`$1`) points to the build folder.
+
    For example, if you need to customize the folder structure generated by default, add
    some move commands into the `postprocess` hook:
 

gen-pack

@@ -79,10 +79,11 @@ function add_dirs {
   echo_log " "
   for d in ${PACK_DIRS}; do
     d=$(realpath --relative-to="$(pwd)" "$d")
+    local tar_opts=(--exclude-vcs --exclude-ignore=.gpignore --exclude=.gpignore)
     if [[ $d = ../* ]]; then
-      cp -r "$d" "$1"
+      tar "${tar_opts[@]}" -C "$(dirname "$d")" -cf - "$(basename "$d")" | tar -C "$1" -xf -
     else
-      cp -r --parents "$d" "$1"
+      tar "${tar_opts[@]}" -cf - "$d" | tar -C "$1" -xf -
     fi
   done
 }

lib/patches

@@ -26,10 +26,15 @@ case $(uname -s) in
       echo "GNU stat is required, run brew install coreutils!" >&2
       exit 1
     fi
-        alias "cp"="gcp"
+    if ! which gtar > /dev/null; then
+      echo "GNU tar is required, run brew install gnu-tar!" >&2
+      exit 1
+    fi
+    alias "cp"="gcp"
     alias "grep"="ggrep"
     alias "realpath"="grealpath"
-        alias "stat"="gstat"
+    alias "stat"="gstat"
+    alias "tar=gtar"
     ;;
 esac
 

test/tests_gen_pack.sh

@@ -101,6 +101,54 @@ test_add_dirs_default() {
   assertTrue  "[ -f \"${PACK_BUILD}/input3/file33.txt\" ]"
 }
 
+test_add_dirs_with_exclude_vcs() { 
+  mkdir -p "input1/.svn"
+  mkdir -p "input1/sub/.svn"
+  mkdir -p "../input5/.svn"
+
+  # run add_dirs
+  PACK_DIRS="
+    input1
+     ../input5
+  "
+  add_dirs "${PACK_BUILD}"
+
+  assertFalse "[ -d \"${PACK_BUILD}/input1/.svn\" ]"
+  assertFalse "[ -d \"${PACK_BUILD}/input1/sub/.svn\" ]"
+  assertFalse "[ -d \"${PACK_BUILD}/input5/.svn\" ]"
+}
+
+test_add_dirs_with_exclude_gpignore() { 
+  createTestData
+
+  touch "input1/do_not_copy.txt"
+  cat > input1/.gpignore <<EOF
+    do_not_copy.txt
+    file12.txt
+EOF
+
+  cat > ../input5/.gpignore <<EOF
+     file51.txt
+EOF
+
+  # run add_dirs
+  PACK_DIRS="
+    input1
+    ../input5
+  "
+  add_dirs "${PACK_BUILD}"
+
+  assertTrue  "[ -d \"${PACK_BUILD}/input1\" ]"
+  assertTrue  "[ -f \"${PACK_BUILD}/input1/file11.txt\" ]"
+  assertFalse "[ -f \"${PACK_BUILD}/input1/file12.txt\" ]"
+  assertTrue  "[ -f \"${PACK_BUILD}/input1/file13.txt\" ]"
+  assertFalse "[ -f \"${PACK_BUILD}/input5/file51.txt\" ]"
+
+  assertFalse "[ -f \"${PACK_BUILD}/input1/do_not_copy.txt\" ]"
+  assertFalse "[ -f \"${PACK_BUILD}/input1/.gpignore\" ]"
+  assertFalse "[ -f \"${PACK_BUILD}/input5/.gpignore\" ]"
+}
+
 test_add_files() {
   createTestData