Add JSON output format as non-default output option via new --format parameter

.gitignore

@@ -4,6 +4,7 @@ utils/cfn-to-cdk/cdk.out/
 /**/__pycache__
 utils/cfn-to-cdk/cfn_to_cdk/cfn_to_cdk_stack.py
 ash_output
+*.bak
 
 ### macOS ###
 # General

CHANGELOG.md

@@ -1,5 +1,12 @@
 # Automated Security Helper - CHANGELOG
 
+- [v1.4.0](#v140)
+    - [What's Changed](#whats-changed)
+- [v1.3.3](#v133)
+    - [What's Changed](#whats-changed-1)
+- [v1.3.2](#v132)
+    - [What's Changed](#whats-changed-2)
+- [New Contributors](#new-contributors)
 - [1.3.0 - 2024-04-17](#130---2024-04-17)
     - [Features](#features)
     - [Fixes](#fixes)
@@ -11,6 +18,46 @@
 - [1.0.5-e-06Mar2023](#105-e-06mar2023)
 - [1.0.1-e-10Jan2023](#101-e-10jan2023)
 
+## v1.4.0
+
+### What's Changed
+
+- Adds `--format` parameter to `ash`/`ash-multi` scripts to enable additional output integrations, beginning with ASHARP (Automated Security Helper Aggregated Report Parser) as the intermediary data model to enable subsequent conversion from there.
+- Adds `automated_security_helper` Python code as a module of the same name from within new `src` directory, including poetry.lock and pyproject.toml files to support. This module includes the `asharp` script (CLI tool) that enabled programmatic parsing of the aggregated_results content in conjunction with the JSON output changes.
+- Adds pre-stage build of `automated_security_helper` module to Dockerfile
+- Adds support to handle when `--format` is a value other than the current default of `text` so scanners switch output to programmatically parseable output formats and `asharp` is called to parse the `aggregated_results.txt` file into `aggregated_results.txt.json`.
+- Moved source of version string truth into `pyproject.toml` for all projects, removed `__version__` file to coincide with this.
+
+**Full Changelog**: https://github.com/awslabs/automated-security-helper/compare/v1.3.3...v1.4.0
+
+## v1.3.3
+
+### What's Changed
+* fix(ash): adjust where/when output-dir is created, if necessary by @climbertjh2 in https://github.com/awslabs/automated-security-helper/pull/74
+* fix(ash): set execute permission on ash script in the container by @climbertjh2 in https://github.com/awslabs/automated-security-helper/pull/81
+* fix: update __version__ file to match release tag format in github.com by @climbertjh2 in https://github.com/awslabs/automated-security-helper/pull/84
+
+
+**Full Changelog**: https://github.com/awslabs/automated-security-helper/compare/v1.3.2...v1.3.3
+
+## v1.3.2
+
+### What's Changed
+* added get-scan-set.py to utils scripts to return a list of non-ignored files for processing by @scrthq in https://github.com/awslabs/automated-security-helper/pull/47
+* fix/codebuild shared bindmount issue by @scrthq in https://github.com/awslabs/automated-security-helper/pull/49
+* fix error in reflecting return code in ash script by @climbertjh2 in https://github.com/awslabs/automated-security-helper/pull/51
+* Issue 58: missing double quotes by @awsntheule in https://github.com/awslabs/automated-security-helper/pull/64
+* fixed cdk nag scanner, added unique stack names based on input filenames. corrected guards on git clone calls within the scanner scripts to ensure those happen in the container image by @scrthq in https://github.com/awslabs/automated-security-helper/pull/54
+* Add support for pnpm audit by @awsntheule in https://github.com/awslabs/automated-security-helper/pull/66
+* fix(cdk-nag-scan): copy output files to separate folders by @climbertjh2 in https://github.com/awslabs/automated-security-helper/pull/69
+* fix(ash): use /tmp rather than tmpfs for scratch area by @climbertjh2 in https://github.com/awslabs/automated-security-helper/pull/73
+* Fix CTRL-C cancelling by @awsntheule in https://github.com/awslabs/automated-security-helper/pull/71
+
+## New Contributors
+* @awsntheule made their first contribution in https://github.com/awslabs/automated-security-helper/pull/64
+
+**Full Changelog**: https://github.com/awslabs/automated-security-helper/compare/1.2.0-e-06Mar2024...v1.3.2
+
 ## 1.3.0 - 2024-04-17
 
 ### Features

Dockerfile

@@ -4,7 +4,29 @@
 # Enable BASE_IMAGE as an overrideable ARG for proxy cache + private registry support
 #
 ARG BASE_IMAGE=public.ecr.aws/docker/library/python:3.10-bullseye
-FROM ${BASE_IMAGE}
+FROM ${BASE_IMAGE} as poetry-reqs
+
+ENV PYTHONDONTWRITEBYTECODE 1
+
+RUN apt-get update && \
+    apt-get upgrade -y && \
+    apt-get install -y \
+        python3-venv && \
+    rm -rf /var/lib/apt/lists/*
+
+RUN python3 -m pip install -U pip poetry
+
+WORKDIR /src
+
+COPY pyproject.toml pyproject.toml
+COPY poetry.lock poetry.lock
+COPY README.md README.md
+COPY src/ src/
+
+RUN poetry build
+
+
+FROM ${BASE_IMAGE} as ash
 
 #
 # Setting timezone in the container to UTC to ensure logged times are universal.
@@ -132,7 +154,10 @@ COPY ./utils/cfn-to-cdk /ash/utils/cfn-to-cdk/
 COPY ./utils/*.* /ash/utils/
 COPY ./appsec_cfn_rules /ash/appsec_cfn_rules/
 COPY ./ash-multi /ash/ash
-COPY ./__version__ /ash/__version__
+COPY ./pyproject.toml /ash/pyproject.toml
+
+COPY --from=poetry-reqs /src/dist/*.whl .
+RUN python3 -m pip install *.whl && rm *.whl
 
 #
 # Make sure the ash script is executable

ash

@@ -1,4 +1,6 @@
 #!/bin/bash
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# SPDX-License-Identifier: Apache-2.0
 
 # Resolve the absolute path of the parent of the script directory (ASH repo root)
 export ASH_ROOT_DIR="$(cd "$(dirname "$0")"; pwd)"
@@ -9,6 +11,7 @@ export ASH_IMAGE_NAME=${ASH_IMAGE_NAME:-"automated-security-helper:local"}
 SOURCE_DIR=""
 OUTPUT_DIR=""
 OUTPUT_DIR_SPECIFIED="NO"
+OUTPUT_FORMAT="text"
 DOCKER_EXTRA_ARGS=""
 ASH_ARGS=""
 NO_BUILD="NO"
@@ -46,6 +49,10 @@ while (("$#")); do
     --debug)
       DEBUG="YES"
       ;;
+    --format)
+      shift
+      OUTPUT_FORMAT="$1"
+      ;;
     --help | -h)
       source "${ASH_ROOT_DIR}/ash-multi" --help
       exit 0
@@ -54,7 +61,7 @@ while (("$#")); do
       source "${ASH_ROOT_DIR}/ash-multi" --version
       exit 0
       ;;
-    --finch | -f)
+    --finch|-f)
       # Show colored deprecation warning from entrypoint script and exit 1
       source "${ASH_ROOT_DIR}/ash-multi" --finch
       exit 1
@@ -118,6 +125,7 @@ else
         --rm \
         -e ACTUAL_SOURCE_DIR="${SOURCE_DIR}" \
         -e ASH_DEBUG=${DEBUG} \
+        -e ASH_OUTPUT_FORMAT=${OUTPUT_FORMAT} \
         ${MOUNT_SOURCE_DIR} \
         ${MOUNT_OUTPUT_DIR} \
         ${ASH_IMAGE_NAME} \

ash-multi

@@ -39,7 +39,8 @@ version_check() {
 
   if [ -n "${_REPO_VERSION}" ]; then # found a version
     if [ "${_REPO_VERSION}" != "${_SCRIPT_VERSION}" ]; then
-      echo -e "${YELLOW}ASH version ${_SCRIPT_VERSION} is different from repository version ${_REPO_VERSION} ... consider upgrading${NC}"
+      echo -e "${YELLOW}ASH version ${_SCRIPT_VERSION} is different from the latest tagged version in the repository: ${_REPO_VERSION} -- consider upgrading!${NC}"
+      echo -e "${YELLOW}View changes: https://github.com/awslabs/automated-security-helper/compare/${_REPO_VERSION}...${_SCRIPT_VERSION}${NC}"
     else
       # the ":" below allows the else/fi clause to remain, even if there is no operation listed
       : #   echo "repo version is ${_REPO_VERSION}, current version is ${_SCRIPT_VERSION}"
@@ -57,6 +58,7 @@ print_usage() {
   echo -e "\t-p | --preserve-report   Add timestamp to the final report file to avoid overwriting it after multiple executions."
   echo -e "\t--source-dir             Path to the directory containing the code/files you wish to scan. Defaults to \$(pwd)"
   echo -e "\t--output-dir             Path to the directory that will contain the report of the scans. Defaults to \$(pwd)"
+  echo -e "\t--format                 Output format of the aggregated_results file segments. Defaults to text. Use json instead to enable parseable output."
   echo -e "\t--ext | -extension       Force a file extension to scan. Defaults to identify files automatically."
   echo -e "\t--force                  Rebuild the Docker images of the scanning tools, to make sure software is up-to-date."
   echo -e "\t--no-cleanup             Don't cleanup the work directory where temp reports are stored during scans."
@@ -277,8 +279,9 @@ run_security_check() {
 
 set -e
 START_TIME=$(date +%s)
-VERSION=$(cat "$(dirname "${BASH_SOURCE[0]}")"/"__version__")
+VERSION=v$(cat "$(dirname "${BASH_SOURCE[0]}")"/"pyproject.toml" | sed -n 's/^version = "\(.*\)"$/\1/p' | head -n 1)
 OCI_RUNNER="docker"
+ASH_OUTPUT_FORMAT=${ASH_OUTPUT_FORMAT:-text}
 
 # Overrides default OCI Runner used by ASH
 [ ! -z "$ASH_OCI_RUNNER" ] && OCI_RUNNER="$ASH_OCI_RUNNER"
@@ -357,6 +360,10 @@ while (("$#")); do
     ASH_DEBUG_ENABLED="true"
     ASH_SCANSET_ARGS="${ASH_SCANSET_ARGS} --debug"
     ;;
+  --format)
+    shift
+    ASH_OUTPUT_FORMAT="$1"
+    ;;
   --no-color | -c)
     COLOR_OUTPUT="false"
     #
@@ -611,6 +618,11 @@ then
 
   RESOLVED_OUTPUT_DIR=${ACTUAL_OUTPUT_DIR:-${OUTPUT_DIR}}
   echo -e "${GREEN}\nYour final report can be found here:${NC} ${RESOLVED_OUTPUT_DIR}/${AGGREGATED_RESULTS_REPORT_FILENAME}"
+
+  if [[ "${ASH_OUTPUT_FORMAT:-text}" != "text" ]]; then
+    echo -e "${GREEN}Converting${NC} ${RESOLVED_OUTPUT_DIR}/${AGGREGATED_RESULTS_REPORT_FILENAME}${GREEN} to ASHARP JSON model${NC}"
+    asharp --input "${RESOLVED_OUTPUT_DIR}/${AGGREGATED_RESULTS_REPORT_FILENAME}" --output "${RESOLVED_OUTPUT_DIR}/${AGGREGATED_RESULTS_REPORT_FILENAME}.json"
+  fi
 else
   echo -e "${GREEN}No extensions were found, nothing to scan at the moment.${NC}"
 fi