Release v2.0.0: Run ASH as non-root user, add explicit CI stage

.github/workflows/ash-build-and-scan.yml

@@ -1,4 +1,4 @@
-name: Build & run ASH against itself
+name: ASH - Core Pipeline
 on:
   push:
     branches:
@@ -15,6 +15,8 @@ permissions:
   id-token: write
   security-events: write
   pull-requests: write
+env:
+  PYTHON_VERSION: "3.12"
 jobs:
   build:
     strategy:
@@ -69,7 +71,7 @@ jobs:
           set +e
 
           # Run ASH against itself
-          ./ash --source-dir $(pwd) --output-dir ash_output --debug | \
+          ./ash --source-dir $(pwd) --output-dir ash_output --container-uid 1001 --container-gid 123 --debug | \
               tee ash_stdout.txt
 
           # cat the output contents to build the summary markdown
@@ -152,3 +154,45 @@ jobs:
           name: ash_output
           path: ash_output
           if-no-files-found: error
+
+  build-docs:
+    name: Build documentation
+    needs: []
+    runs-on: ubuntu-latest
+    if: github.event_name == 'pull_request' || (github.event_name == 'push' && github.ref != 'refs/heads/main')
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ env.PYTHON_VERSION }}
+          cache: 'pip'
+
+      - name: Install dependencies
+        run: pip install -r requirements.txt
+
+      - name: Build documentation
+        run: mkdocs build --clean
+
+  deploy-docs:
+    name: Deploy documentation
+    needs: []
+    runs-on: ubuntu-latest
+    if: github.event_name == 'push' && github.ref == 'refs/heads/main'
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ env.PYTHON_VERSION }}
+          cache: 'pip'
+
+      - name: Install dependencies
+        run: pip install -r requirements.txt
+
+      - name: Deploy documentation
+        run: mkdocs gh-deploy --clean --force

Dockerfile

@@ -1,44 +1,27 @@
 #checkov:skip=CKV_DOCKER_7: Base image is using a non-latest version tag by default, Checkov is unable to parse due to the use of ARG
-#checkov:skip=CKV_DOCKER_3: ASH is focused on mounting source code into the container and scanning it, not running services. Setting USER breaks the ability for certain scanners to work correctly.
-#
-# Enable BASE_IMAGE as an overrideable ARG for proxy cache + private registry support
-#
 ARG BASE_IMAGE=public.ecr.aws/docker/library/python:3.10-bullseye
 
-
+# First stage: Build poetry requirements
 FROM ${BASE_IMAGE} as poetry-reqs
-
 ENV PYTHONDONTWRITEBYTECODE 1
-
 RUN apt-get update && \
     apt-get upgrade -y && \
-    apt-get install -y \
-        python3-venv && \
+    apt-get install -y python3-venv && \
     rm -rf /var/lib/apt/lists/*
-
 RUN python3 -m pip install -U pip poetry
-
 WORKDIR /src
-
-COPY pyproject.toml pyproject.toml
-COPY poetry.lock poetry.lock
-COPY README.md README.md
+COPY pyproject.toml poetry.lock README.md ./
 COPY src/ src/
-
 RUN poetry build
 
-
-FROM ${BASE_IMAGE} as ash
+# Second stage: Core ASH image
+FROM ${BASE_IMAGE} as core
 SHELL ["/bin/bash", "-c"]
 ARG OFFLINE="NO"
 ARG OFFLINE_SEMGREP_RULESETS="p/ci"
-
 ENV OFFLINE="${OFFLINE}"
 ENV OFFLINE_AT_BUILD_TIME="${OFFLINE}"
 ENV OFFLINE_SEMGREP_RULESETS="${OFFLINE_SEMGREP_RULESETS}"
-#
-# Setting timezone in the container to UTC to ensure logged times are universal.
-#
 ENV TZ=UTC
 RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
 
@@ -117,11 +100,11 @@ RUN echo "gem: --no-document" >> /etc/gemrc && \
 #
 
 #
-# Grype/Syft/Semgrep
+# Grype/Syft/Semgrep - Also sets default location env vars for root user for CI compat
 #
-ENV HOME="/root"
-ENV GRYPE_DB_CACHE_DIR="${HOME}/.grype"
-ENV SEMGREP_RULES_CACHE_DIR="${HOME}/.semgrep"
+ENV GRYPE_DB_CACHE_DIR="/deps/.grype"
+ENV SEMGREP_RULES_CACHE_DIR="/deps/.semgrep"
+RUN mkdir -p ${GRYPE_DB_CACHE_DIR} ${SEMGREP_RULES_CACHE_DIR}
 
 RUN curl -sSfL https://raw.githubusercontent.com/anchore/syft/main/install.sh | \
     sh -s -- -b /usr/local/bin
@@ -180,7 +163,7 @@ RUN python3 -m pip install *.whl && rm *.whl
 #
 # Make sure the ash script is executable
 #
-RUN chmod +x /ash/ash
+RUN chmod -R 755 /ash && chmod -R 777 /src /out /deps
 
 #
 # Flag ASH as local execution mode since we are running in a container already
@@ -192,20 +175,66 @@ ENV _ASH_EXEC_MODE="local"
 #
 ENV PATH="$PATH:/ash"
 
-# nosemgrep
-HEALTHCHECK --interval=12s --timeout=12s --start-period=30s \
-    CMD type ash || exit 1
+
+# CI stage -- any customizations specific to CI platform compatibility should be added
+# in this stage if it is not applicable to ASH outside of CI usage
+FROM core as ci
+
+ENV ASH_TARGET=ci
+
+
+# Final stage: Non-root user final version. This image contains all dependencies
+# for ASH from the `core` stage, but ensures it is launched as a non-root user.
+# Running as a non-root user impacts the ability to run ASH reliably across CI
+# platforms and other orchestrators where the initialization and launch of the image
+# is not configurable for customizing the running UID/GID.
+FROM core as non-root
+
+ENV ASH_TARGET=non-root
+
+ARG UID=500
+ARG GID=100
+ARG ASH_USER=ash-user
+ARG ASH_GROUP=ash-group
+ARG ASHUSER_HOME=/home/${ASH_USER}
 
 #
-# The ENTRYPOINT needs to be NULL for CI platform support
-# This needs to be an empty array ([ ]), as nerdctl-based runners will attempt to
-# resolve an empty string in PATH, unlike Docker which treats an empty string the
-# same as a literal NULL
+# Create a non-root user in the container and run as this user
 #
-ENTRYPOINT [ ]
+# And add GitHub's public fingerprints to known_hosts inside the image to prevent fingerprint
+# confirmation requests unexpectedly
+#
+# ignore a failure to add the group
+RUN addgroup --gid ${GID} ${ASH_GROUP} || :
+RUN adduser --disabled-password --disabled-login \
+        --uid ${UID} --gid ${GID} \
+        ${ASH_USER} && \
+    mkdir -p ${ASHUSER_HOME}/.ssh && \
+    echo "github.com ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOMqqnkVzrm0SdG6UOoqKLsabgH5C9okWi0dh2l9GKJl" >> ${ASHUSER_HOME}/.ssh/known_hosts && \
+    echo "github.com ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBEmKSENjQEezOmxkZMy7opKgwFB9nkt5YRrYMjNuG5N87uRgg6CLrbo5wAdT/y6v0mKV0U2w0WZ2YB/++Tpockg=" >> ${ASHUSER_HOME}/.ssh/known_hosts && \
+    echo "github.com ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCj7ndNxQowgcQnjshcLrqPEiiphnt+VTTvDP6mHBL9j1aNUkY4Ue1gvwnGLVlOhGeYrnZaMgRK6+PKCUXaDbC7qtbW8gIkhL7aGCsOr/C56SJMy/BCZfxd1nWzAOxSDPgVsmerOBYfNqltV9/hWCqBywINIR+5dIg6JTJ72pcEpEjcYgXkE2YEFXV1JHnsKgbLWNlhScqb2UmyRkQyytRLtL+38TGxkxCflmO+5Z8CSSNY7GidjMIZ7Q4zMjA2n1nGrlTDkzwDCsw+wqFPGQA179cnfGWOWRVruj16z6XyvxvjJwbz0wQZ75XK5tKSb7FNyeIEs4TT4jk+S4dhPeAUC5y+bDYirYgM4GC7uEnztnZyaVWQ7B381AK4Qdrwt51ZqExKbQpTUNn+EjqoTwvqNj4kqx5QUCI0ThS/YkOxJCXmPUWZbhjpCg56i+2aB6CmK2JGhn57K5mj0MNdBXA4/WnwH6XoPWJzK5Nyu2zB3nAZp+S5hpQs+p1vN1/wsjk=" >> ${ASHUSER_HOME}/.ssh/known_hosts
+
+# Change ownership and permissions now that we are running with a non-root
+# user by default.
+RUN chown -R ${UID}:${GID} ${ASHUSER_HOME} /src /out /deps && \
+    chgrp -R ${GID} ${ASHUSER_HOME} /src /out /deps && \
+    chmod 750 -R ${ASHUSER_HOME} /src /out /deps
+
+# Setting default WORKDIR to ${ASHUSER_HOME}
+WORKDIR ${ASHUSER_HOME}
+
+USER ${UID}:${GID}
 
 #
-# CMD will be run when invoking it via `$OCI_RUNNER run ...`, but will
-# be overridden during CI execution when used as the job image directly.
+# Set the HOME environment variable to be the HOME folder for the non-root user,
+# along with any additional details that were set to root user values by default
 #
+ENV HOME=${ASHUSER_HOME}
+ENV ASH_USER=${ASH_USER}
+ENV ASH_GROUP=${ASH_GROUP}
+
+HEALTHCHECK --interval=12s --timeout=12s --start-period=30s \
+    CMD type ash || exit 1
+
+ENTRYPOINT [ ]
 CMD [ "ash" ]

README.md

@@ -207,26 +207,43 @@ ash --source-dir . --ext py
 ## Synopsis
 
 ```text
+$ ash --help
 NAME:
-        ash
+    ash
 SYNOPSIS:
-        ash [OPTIONS] --source-dir /path/to/dir --output-dir /path/to/dir
+    ash [OPTIONS] --source-dir /path/to/dir --output-dir /path/to/dir
 OPTIONS:
-        -v | --version           Prints version number.
-
-        -p | --preserve-report   Add timestamp to the final report file to avoid overwriting it after multiple executions.
-        --source-dir             Path to the directory containing the code/files you wish to scan. Defaults to $(pwd)
-        --output-dir             Path to the directory that will contain the report of the scans. Defaults to $(pwd)
-        --ext | -extension       Force a file extension to scan. Defaults to identify files automatically.
-        --offline                      Build ASH for offline execution. Defaults to false.
-        --offline-semgrep-rulesets     Specify Semgrep rulesets for use in ASH offline mode. Defaults to 'p/ci'.
-        --force                  Rebuild the Docker images of the scanning tools, to make sure software is up-to-date.
-        --no-cleanup             Don't cleanup the work directory where temp reports are stored during scans.
-        --debug                  Print ASH debug log information where applicable.
-        -q | --quiet             Don't print verbose text about the build process.
-        -c | --no-color          Don't print colorized output.
-        -s | --single-process    Run ash scanners serially rather than as separate, parallel sub-processes.
-        -o | --oci-runner        Use the specified OCI runner instead of docker to run the containerized tools.
+    --source-dir                    Path to the directory containing the code/files you wish to scan. Defaults to $(pwd)
+    --output-dir                    Path to the directory that will contain the report of the scans. Defaults to $(pwd)
+
+    --format                        Output format of the aggregated_results file segments.
+                                    Options: text, json
+                                    Default: text
+
+    --target                        Specify the target stage of the ASH image to build.
+                                    Options: non-root, ci
+                                    Default: non-root
+
+    --offline                       Build ASH for offline execution.
+                                    Default: false
+
+    --offline-semgrep-rulesets      Specify Semgrep rulesets for use in ASH offline mode.
+                                    Default: p/ci
+
+    --no-cleanup                    Don't cleanup the work directory where temp reports are stored during scans.
+    --ext | -extension              Force a file extension to scan. Defaults to identify files automatically.
+
+    -c    | --no-color              Don't print colorized output.
+    -s    | --single-process        Run ash scanners serially rather than as separate, parallel sub-processes.
+    -o    | --oci-runner            Use the specified OCI runner instead of docker to run the containerized tools.
+    -p    | --preserve-report       Add timestamp to the final report file to avoid overwriting it after multiple executions.
+
+    -d    | --debug                 Print ASH debug log information where applicable.
+    -q    | --quiet                 Don't print verbose text about the build process.
+    -v    | --version               Prints version number.
+
+INFO:
+    For more information, please visit https://github.com/awslabs/automated-security-helper
 ```
 
 ## FAQ
@@ -260,7 +277,7 @@ OPTIONS:
 
 - Q: How to run `ash` in an environment without internet connectivity/with an airgap?
 
-  A: From your environment which does have internet connectivity, build the ASH image using `--offline` and `--offline-semgrep-rulesets` to specify what resources to package into the image.  Environment variable `$ASH_IMAGE_NAME` controls the name of the image.  After building, push to your container repository of choice which will be available within the airgapped environment.  When you go to execute ASH in your offline environment, passing `--no-build` to `ash` alongside `--offline` and `--offline-semgrep-rulesets` will use your offline image and skip the build.  Specify `$ASH_IMAGE_NAME` to override ASH's container image to the previously-built image available within your airgapped environment. 
+  A: From your environment which does have internet connectivity, build the ASH image using `--offline` and `--offline-semgrep-rulesets` to specify what resources to package into the image.  Environment variable `$ASH_IMAGE_NAME` controls the name of the image.  After building, push to your container repository of choice which will be available within the airgapped environment.  When you go to execute ASH in your offline environment, passing `--no-build` to `ash` alongside `--offline` and `--offline-semgrep-rulesets` will use your offline image and skip the build.  Specify `$ASH_IMAGE_NAME` to override ASH's container image to the previously-built image available within your airgapped environment.
 
 ## Feedback