From ffb54b7e1b3693e181a5901f172141f4a742c105 Mon Sep 17 00:00:00 2001 From: Mathieu Leplatre Date: Fri, 3 Mar 2023 11:07:31 +0100 Subject: [PATCH 1/6] Add `make format` command --- Makefile | 6 ++++ bin/lint.sh | 89 ++++++++++++++++++++++++++++++++++++++++++++--------- 2 files changed, 80 insertions(+), 15 deletions(-) diff --git a/Makefile b/Makefile index a44c3cd2..05da445c 100644 --- a/Makefile +++ b/Makefile @@ -20,6 +20,7 @@ help: @echo " build - build docker containers" @echo " db-only - run PostgreSQL server" @echo " lint - lint check for code" + @echo " format - run formatters (black, isort), fix in place" @echo " setup - (re)create the database" @echo " shell - open a shell in the web container" @echo " start - run the API service" @@ -47,6 +48,11 @@ build: .env lint: $(INSTALL_STAMP) bin/lint.sh +.PHONY: format +format: $(INSTALL_STAMP) + bin/lint.sh black --fix + bin/lint.sh isort --fix + .PHONY: db-only db-only: .env docker-compose up postgres-admin diff --git a/bin/lint.sh b/bin/lint.sh index 3880a718..3addbe56 100755 --- a/bin/lint.sh +++ b/bin/lint.sh @@ -1,23 +1,82 @@ -#!/bin/bash +#!/bin/sh set -euo pipefail POETRY_RUN="poetry run" -CURRENT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) -BASE_DIR="$(dirname "$CURRENT_DIR")" -HAS_GIT="$(command -v git || echo '')" -echo $HAS_GIT +bandit () { + $POETRY_RUN bandit -lll --recursive "${BASE_DIR}" --exclude "${BASE_DIR}/poetry.lock,${BASE_DIR}/.venv,${BASE_DIR}/.mypy,${BASE_DIR}/build" +} +black () { + $POETRY_RUN black ${check:+--check} "${BASE_DIR}" +} +detect_secrets () { + if [ -n "$HAS_GIT" ]; then + # Scan only files fixed into the repo, omit poetry.lock + FILES_TO_SCAN=`git ls-tree --full-tree -r --name-only HEAD | grep -v poetry.lock` + $POETRY_RUN detect-secrets-hook $FILES_TO_SCAN --baseline .secrets.baseline + fi +} +isort () { + $POETRY_RUN isort ${check:+--check-only} "${BASE_DIR}" +} +pylint () { + $POETRY_RUN pylint "${BASE_DIR}/ctms" "${BASE_DIR}/tests/unit" +} +mypy () { + $POETRY_RUN mypy "${BASE_DIR}/ctms" +} +all () { + echo "running black" + black + echo "running isort" + isort + echo "running mypy" + mypy + echo "running pylint" + pylint + echo "running bandit" + bandit + echo "running detect_secrets" + detect_secrets +} -$POETRY_RUN bandit -lll --recursive "${BASE_DIR}" --exclude "${BASE_DIR}/poetry.lock,${BASE_DIR}/.venv,${BASE_DIR}/.mypy,${BASE_DIR}/build" +usage () { + echo "Usage: bin/lint.sh [OPTION]" + echo " run linting checks" + echo "Options": + echo " bandit" + echo " black [--fix]" + echo " detect-secrets" + echo " isort [--fix]" + echo " mypy" + echo " pylint" + echo " yamllint" +} -if [ -n "$HAS_GIT" ]; then - # Scan only files checked into the repo, omit poetry.lock - SECRETS_TO_SCAN=`git ls-tree --full-tree -r --name-only HEAD | grep -v poetry.lock` - $POETRY_RUN detect-secrets-hook $SECRETS_TO_SCAN --baseline .secrets.baseline -fi +subcommand=''; +check="true" +if [ -z $1 ]; then + all +else + subcommand=$1; shift + case $subcommand in + "black" | "isort") + case $1 in + "--fix") + check="" + ;; + esac + case $subcommand in + "isort") isort;; + "black") black;; + esac + ;; -$POETRY_RUN isort --check-only "${BASE_DIR}" -$POETRY_RUN black --check "${BASE_DIR}" -$POETRY_RUN mypy "${BASE_DIR}/ctms" -$POETRY_RUN pylint "${BASE_DIR}/ctms" "${BASE_DIR}/tests/unit" + "pylint") pylint;; + "mypy") mypy;; + "bandit") bandit;; + "detect-secrets") detect_secrets;; + *) usage;; + esac +fi From f30ebd5f0e0c50f6c059b7680767856ec16c8543 Mon Sep 17 00:00:00 2001 From: Mathieu Leplatre Date: Fri, 3 Mar 2023 11:54:49 +0100 Subject: [PATCH 2/6] Remove bash option --- bin/lint.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/lint.sh b/bin/lint.sh index 3addbe56..2cea1b8d 100755 --- a/bin/lint.sh +++ b/bin/lint.sh @@ -1,6 +1,6 @@ #!/bin/sh -set -euo pipefail +set -e POETRY_RUN="poetry run" From 412a0fc63a2c8e95ce3adb802fd4a4735a5f7659 Mon Sep 17 00:00:00 2001 From: Mathieu Leplatre Date: Fri, 3 Mar 2023 15:15:58 +0100 Subject: [PATCH 3/6] Restore variables --- bin/lint.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/bin/lint.sh b/bin/lint.sh index 2cea1b8d..12cbc568 100755 --- a/bin/lint.sh +++ b/bin/lint.sh @@ -3,6 +3,10 @@ set -e POETRY_RUN="poetry run" +CURRENT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +BASE_DIR="$(dirname "$CURRENT_DIR")" +HAS_GIT="$(command -v git || echo '')" +echo $HAS_GIT bandit () { $POETRY_RUN bandit -lll --recursive "${BASE_DIR}" --exclude "${BASE_DIR}/poetry.lock,${BASE_DIR}/.venv,${BASE_DIR}/.mypy,${BASE_DIR}/build" @@ -15,6 +19,8 @@ detect_secrets () { # Scan only files fixed into the repo, omit poetry.lock FILES_TO_SCAN=`git ls-tree --full-tree -r --name-only HEAD | grep -v poetry.lock` $POETRY_RUN detect-secrets-hook $FILES_TO_SCAN --baseline .secrets.baseline + else + echo "Git not available. Skip detect-secrets" fi } isort () { From 37403e51b6cecd76997012320268a9443227dc0f Mon Sep 17 00:00:00 2001 From: Mathieu Leplatre Date: Fri, 3 Mar 2023 15:33:15 +0100 Subject: [PATCH 4/6] Update secrets baseline --- .secrets.baseline | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/.secrets.baseline b/.secrets.baseline index e867c459..448f4d9b 100644 --- a/.secrets.baseline +++ b/.secrets.baseline @@ -150,15 +150,6 @@ "line_number": 39 } ], - "bin/lint.sh": [ - { - "type": "Secret Keyword", - "filename": "bin/lint.sh", - "hashed_secret": "fd336c21216d202878615f06d5fd8528d187f37e", - "is_verified": false, - "line_number": 16 - } - ], "docs/developer_setup.md": [ { "type": "Secret Keyword", @@ -203,5 +194,5 @@ } ] }, - "generated_at": "2023-01-30T15:11:07Z" + "generated_at": "2023-03-03T14:31:04Z" } From e7a76dd44c8dd9c60ca40c314bcfacf29a80841f Mon Sep 17 00:00:00 2001 From: Mathieu Leplatre Date: Tue, 7 Mar 2023 11:44:42 +0100 Subject: [PATCH 5/6] Remove yamllint from help --- bin/lint.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/bin/lint.sh b/bin/lint.sh index 12cbc568..cc1ede1a 100755 --- a/bin/lint.sh +++ b/bin/lint.sh @@ -57,7 +57,6 @@ usage () { echo " isort [--fix]" echo " mypy" echo " pylint" - echo " yamllint" } subcommand=''; From af57006e2a1511c4925a976baed9d6287a47dc0e Mon Sep 17 00:00:00 2001 From: Mathieu Leplatre Date: Tue, 7 Mar 2023 11:45:41 +0100 Subject: [PATCH 6/6] Remove safety check for Git and detect-secrets --- bin/lint.sh | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/bin/lint.sh b/bin/lint.sh index cc1ede1a..d5ca2de4 100755 --- a/bin/lint.sh +++ b/bin/lint.sh @@ -5,8 +5,6 @@ set -e POETRY_RUN="poetry run" CURRENT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) BASE_DIR="$(dirname "$CURRENT_DIR")" -HAS_GIT="$(command -v git || echo '')" -echo $HAS_GIT bandit () { $POETRY_RUN bandit -lll --recursive "${BASE_DIR}" --exclude "${BASE_DIR}/poetry.lock,${BASE_DIR}/.venv,${BASE_DIR}/.mypy,${BASE_DIR}/build" @@ -15,13 +13,9 @@ black () { $POETRY_RUN black ${check:+--check} "${BASE_DIR}" } detect_secrets () { - if [ -n "$HAS_GIT" ]; then - # Scan only files fixed into the repo, omit poetry.lock - FILES_TO_SCAN=`git ls-tree --full-tree -r --name-only HEAD | grep -v poetry.lock` - $POETRY_RUN detect-secrets-hook $FILES_TO_SCAN --baseline .secrets.baseline - else - echo "Git not available. Skip detect-secrets" - fi + # Scan only files fixed into the repo, omit poetry.lock + FILES_TO_SCAN=`git ls-tree --full-tree -r --name-only HEAD | grep -v poetry.lock` + $POETRY_RUN detect-secrets-hook $FILES_TO_SCAN --baseline .secrets.baseline } isort () { $POETRY_RUN isort ${check:+--check-only} "${BASE_DIR}"