From 7d19cb12d333a74b42846172b88aa1e5e19e8b79 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 2 Oct 2024 04:12:31 +0200 Subject: [PATCH] Initial commit --- .editorconfig | 85 +++++++++++ .envrc | 203 +++++++++++++++++++++++++++ .envrc-python | 43 ++++++ .github/workflows/grype.yaml | 60 ++++++++ .github/workflows/kics.yaml | 60 ++++++++ .github/workflows/semgrep-cloud.yaml | 62 ++++++++ .github/workflows/semgrep.yaml | 64 +++++++++ .github/workflows/sonarcloud.yaml | 48 +++++++ .github/workflows/trivy.yaml | 62 ++++++++ .github/workflows/validate.yaml | 60 ++++++++ .gitlab-ci.yml | 30 ++++ .gitmodules | 3 + .mdl.rb | 30 ++++ .mdlrc | 5 + .pre-commit-config.yaml | 73 ++++++++++ LICENSE | 7 + Makefile | 85 +++++++++++ README.md | 138 ++++++++++++++++++ azure-pipelines.yml | 70 +++++++++ bash-tools | 1 + bitbucket-pipelines.yml | 38 +++++ customize.sh | 54 +++++++ setup/ci_bootstrap.sh | 93 ++++++++++++ setup/ci_git_set_dir_safe.sh | 39 +++++ sonar-project.properties | 42 ++++++ 25 files changed, 1455 insertions(+) create mode 100644 .editorconfig create mode 100644 .envrc create mode 100644 .envrc-python create mode 100644 .github/workflows/grype.yaml create mode 100644 .github/workflows/kics.yaml create mode 100644 .github/workflows/semgrep-cloud.yaml create mode 100644 .github/workflows/semgrep.yaml create mode 100644 .github/workflows/sonarcloud.yaml create mode 100644 .github/workflows/trivy.yaml create mode 100644 .github/workflows/validate.yaml create mode 100644 .gitlab-ci.yml create mode 100644 .gitmodules create mode 100644 .mdl.rb create mode 100644 .mdlrc create mode 100644 .pre-commit-config.yaml create mode 100644 LICENSE create mode 100644 Makefile create mode 100644 README.md create mode 100644 azure-pipelines.yml create mode 160000 bash-tools create mode 100644 bitbucket-pipelines.yml create mode 100755 customize.sh create mode 100755 setup/ci_bootstrap.sh create mode 100755 setup/ci_git_set_dir_safe.sh create mode 100644 sonar-project.properties diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..b83a3cd --- /dev/null +++ b/.editorconfig @@ -0,0 +1,85 @@ +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: 2015-10-31 19:04:34 +0000 (Sat, 31 Oct 2015) +# +# https://github.com/HariSekhon/Template-repo +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# to help improve or steer this or other code I publish +# +# https://www.linkedin.com/in/HariSekhon +# + +# http://EditorConfig.org + +# stop recursing upwards for other .editorconfig files +root = true + +# Unix-style newlines with a newline ending every file +[*] +indent_size = 4 +indent_style = space +end_of_line = lf +trim_trailing_whitespace = true +insert_final_newline = true + +[*.go] +indent_size = 4 +indent_style = tab +end_of_line = lf +trim_trailing_whitespace = true +insert_final_newline = true + +[Makefile] +indent_size = 4 +indent_style = tab +end_of_line = lf +trim_trailing_whitespace = true +insert_final_newline = true + +[{*.md,*.hcl,*.tf,*.tfvars}] +indent_size = 2 +indent_style = space +end_of_line = lf +trim_trailing_whitespace = true +insert_final_newline = true + +[*.yml,*.yaml] +indent_size = 2 +indent_style = space +end_of_line = lf +trim_trailing_whitespace = true +insert_final_newline = true + +[.*] +indent_size = 4 +indent_style = space +end_of_line = lf +trim_trailing_whitespace = true +insert_final_newline = true + +# ============================================================================ # +# Older Stuff, don't think I use this any more +# ============================================================================ # + +# Matches multiple files with brace expansion notation +# Set default charset +#[*.{js,py}] +#charset = utf-8 + +# Indentation override for all JS under lib directory +#[lib/**.js] +#indent_style = space +#indent_size = 2 + +# Matches the exact files either package.json or .travis.yml +#[{package.json,.travis.yml}] +#indent_style = space +#indent_size = 2 + +#[*.xml] +#indent_style = space +#indent_size = 2 diff --git a/.envrc b/.envrc new file mode 100644 index 0000000..c4a1741 --- /dev/null +++ b/.envrc @@ -0,0 +1,203 @@ +#!/usr/bin/env bash +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: Mon Feb 22 17:42:01 2021 +0000 +# +# https://github.com/HariSekhon/Template-repo +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/HariSekhon +# + +# ============================================================================ # +# D i r E n v +# ============================================================================ # + +# https://direnv.net/man/direnv-stdlib.1.html + +# See Also: +# +# .envrc-aws +# .envrc-gcp +# .envrc-kubernetes + +# direnv stdlib - loads .envrc from parent dir up to / +# +# useful to accumulate parent and child directory .envrc settings eg. adding Kubernetes namespace, ArgoCD app etc. +# +# bypasses security authorization though - use with care +#source_up +# +# source_up must be loaded before set -u otherwise gets this error: +# +# direnv: loading .envrc +# /bin/bash: line 226: $1: unbound variable +# +# source_up causes this error is up .envrc is found in parent directories: +# +# direnv: No ancestor .envrc found + +set -euo pipefail +[ -n "${DEBUG:-}" ] && set -x +src="$(readlink -f "${BASH_SOURCE[0]}")" +srcdir="$(cd "$(dirname "$src")" && pwd)" + +# ============================================================================ # +# P r e - C o m m i t +# ============================================================================ # + +# Automatically install Pre-Commit Git hooks if not already present + +if ! type -P pre-commit &>/dev/null; then + if uname -s | grep -q Darwin && + type -P brew &>/dev/null; then + echo + echo "Pre-commit is not installed - installing now using Homebrew..." + echo + brew install pre-commit + echo + elif type -P pip &>/dev/null; then + echo + echo "Pre-commit is not installed - installing now using Pip..." + echo + pip install pre-commit + fi +fi + +if [ -f .pre-commit-config.yaml ] && + type -P pre-commit &>/dev/null && + git rev-parse --is-inside-work-tree &>/dev/null; then + if ! [ -f "$(git rev-parse --show-toplevel)/.git/hooks/pre-commit" ]; then + echo + echo "Pre-commit hook is not installed in local Git repo checkout - installing now..." + echo + pre-commit install + fi +fi + +# ============================================================================ # +# D o c k e r C o m p o s e +# ============================================================================ # + +export COMPOSE_PROJECT_NAME="Template-repo" + +# ============================================================================ # +# G i t H u b +# ============================================================================ # + +#export GITHUB_ORGANIZATION=HariSekhon + +# ============================================================================ # +# A n s i b l e +# ============================================================================ # + +# use the local repo's ansible.cfg rather than: +# +# $PWD/ansible.cfg +# ~/.ansible.cfg +# /etc/ansible/ansible.cfg +# +# set this in project repos to ensure user environment ANSIBLE_CONFIG doesn't get used +#export ANSIBLE_CONFIG="/path/to/ansible.cfg" + +# ============================================================================ # +# C l o u d f l a r e +# ============================================================================ # + +#export CLOUDFLARE_EMAIL=hari@... +#export CLOUDFLARE_API_KEY=... # generate here: https://dash.cloudflare.com/profile/api-tokens +#export CLOUDFLARE_TOKEN=... # used by cloudflare_api.sh but not by terraform module + +# export the variables for terraform +#export TF_VAR_cloudflare_email="$CLOUDFLARE_EMAIL" +#export TF_VAR_cloudflare_api_key="$CLOUDFLARE_API_KEY" # must be a key, not a token using the link above + +# ============================================================================ # +# Load External Envrc Files If Present +# ============================================================================ # + +# XXX: safer to bring all these external .envrc inline if you're worried about changes +# to it bypassing 'direnv allow' authorization +load_if_exists(){ + # first arg is a path to a .envrc + # all other args are passed to the sourcing of .envrc - used by .envrc-kubernetes + # to pass the context name 'docker-desktop' to switch to + local envrc="$1" + shift + if ! [[ "$envrc" =~ ^/ ]]; then + envrc="$srcdir/$envrc" + fi + if [ -f "$envrc" ]; then + # prevent looping on symlinks to this .envrc if given + if [ "$(readlink "$envrc")" = "$src" ]; then + return + fi + echo + echo "Loading $envrc" + # shellcheck disable=SC1090,SC1091 + . "$envrc" "$@" + fi +} + +# don't do this it may lead to an infinite loop if 'make link' symlinking ~/.envrc to this repo's .envrc +# (which I do to keep Python virtual automatically loaded at all times because recent pip on Python refuses +# to install to system Python) +#load_if_exists ~/.envrc + +# ============================================================================ # +# P y t h o n +# ============================================================================ # + + #.envrc-aws \ + #.envrc-gcp \ + #.envrc-terraform \ +# shellcheck disable=SC2043 +for envrc in \ + .envrc-python \ + ; do + load_if_exists "$envrc" +done + +# ============================================================================ # +# A W S +# ============================================================================ # + +if [[ "$PWD" =~ /aws/ ]]; then + load_if_exists .envrc-aws +fi + +# ============================================================================ # +# G C P +# ============================================================================ # + +if [[ "$PWD" =~ /gcp/ ]]; then + load_if_exists .envrc-gcp +fi + +# ============================================================================ # +# T e r r a f o r m +# ============================================================================ # + +if [[ "$PWD" =~ /(terra(form)?|tf)(/|$) ]]; then + load_if_exists .envrc-terraform +fi + +# ============================================================================ # +# K u b e r n e t e s +# ============================================================================ # + +if [ -f "$srcdir/.envrc-kubernetes" ]; then + load_if_exists .envrc-kubernetes docker-desktop +fi + +# ============================================================================ # +# . E n v +# ============================================================================ # + +echo +# read .env too +#dotenv diff --git a/.envrc-python b/.envrc-python new file mode 100644 index 0000000..4766a41 --- /dev/null +++ b/.envrc-python @@ -0,0 +1,43 @@ +#!/usr/bin/env bash +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: Mon Feb 22 17:42:01 2021 +0000 +# +# https://github.com/HariSekhon/Template-repo +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/HariSekhon +# + +# ============================================================================ # +# P y t h o n D i r E n v +# ============================================================================ # + +# .envrc to auto-load the virtualenv inside the 'venv' directory if present + +# https://direnv.net/man/direnv-stdlib.1.html + +set -euo pipefail +[ -n "${DEBUG:-}" ] && set -x +#srcdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +# this is necessary because newer versions of pip no longer allow you to install PyPI packages in system-packages by default +for venv in "$PWD/venv" "$HOME/venv"; do + if [ -f "$venv/bin/activate" ]; then + echo + echo "Virtualenv directory found in: $venv" + echo + echo "Activating Virtualenv inside the directory: $venv" + + # shellcheck disable=SC1091 + source "$venv/bin/activate" + break + fi +done + +# read .env too +#dotenv diff --git a/.github/workflows/grype.yaml b/.github/workflows/grype.yaml new file mode 100644 index 0000000..06afe76 --- /dev/null +++ b/.github/workflows/grype.yaml @@ -0,0 +1,60 @@ +# +# Author: Hari Sekhon +# Date: 2023-05-13 01:07:56 +0100 (Sat, 13 May 2023) +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/HariSekhon/Template-repo +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/HariSekhon +# + +# ============================================================================ # +# G r y p e +# ============================================================================ # + +--- +name: Grype + +on: + push: + branches: + - master + - main + paths-ignore: + - '**/*.md' + pull_request: + branches: + - master + - main + paths-ignore: + - '**/*.md' + workflow_dispatch: + inputs: + debug: + type: boolean + required: false + default: false + schedule: + - cron: '0 0 * * 1' + +permissions: + actions: read + contents: read + security-events: write + +concurrency: + group: ${{ github.ref }}-${{ github.workflow }} + cancel-in-progress: true + +jobs: + Grype: + # github.event.repository context not available in scheduled workflows + #if: github.event.repository.fork == false + if: github.repository_owner == 'HariSekhon' + name: Grype + uses: HariSekhon/GitHub-Actions/.github/workflows/grype.yaml@master + with: + debug: ${{ github.event.inputs.debug }} diff --git a/.github/workflows/kics.yaml b/.github/workflows/kics.yaml new file mode 100644 index 0000000..11d02e5 --- /dev/null +++ b/.github/workflows/kics.yaml @@ -0,0 +1,60 @@ +# +# Author: Hari Sekhon +# Date: 2022-02-01 19:36:08 +0000 (Tue, 01 Feb 2022) +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/HariSekhon/Template-repo +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/HariSekhon +# + +# ============================================================================ # +# K i c s +# ============================================================================ # + +--- +name: Kics + +on: + push: + branches: + - master + - main + paths-ignore: + - '**/*.md' + pull_request: + branches: + - master + - main + paths-ignore: + - '**/*.md' + workflow_dispatch: + inputs: + debug: + type: boolean + required: false + default: false + schedule: + - cron: '0 0 * * 1' + +permissions: + actions: read + contents: read + security-events: write + +concurrency: + group: ${{ github.ref }}-${{ github.workflow }} + cancel-in-progress: true + +jobs: + kics: + # github.event.repository context not available in scheduled workflows + #if: github.event.repository.fork == false + if: github.repository_owner == 'HariSekhon' + name: Kics + uses: HariSekhon/GitHub-Actions/.github/workflows/kics.yaml@master + with: + debug: ${{ github.event.inputs.debug }} diff --git a/.github/workflows/semgrep-cloud.yaml b/.github/workflows/semgrep-cloud.yaml new file mode 100644 index 0000000..1981b05 --- /dev/null +++ b/.github/workflows/semgrep-cloud.yaml @@ -0,0 +1,62 @@ +# +# Author: Hari Sekhon +# Date: Tue Feb 4 09:53:28 2020 +0000 +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/HariSekhon/Template-repo +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/HariSekhon +# + +# ============================================================================ # +# S e m g r e p C l o u d W o r k f l o w +# ============================================================================ # + +# Logs results to https://semgrep.dev/ + +--- +name: Semgrep Cloud + +on: + push: + branches: + - master + - main + paths-ignore: + - '**/*.md' + pull_request: + branches: + - master + - main + paths-ignore: + - '**/*.md' + workflow_dispatch: + inputs: + debug: + type: boolean + required: false + default: false + schedule: + - cron: '0 0 * * 1' + +permissions: + contents: read + +concurrency: + group: ${{ github.ref }}-${{ github.workflow }} + cancel-in-progress: true + +jobs: + semgrep: + # github.event.repository context not available in scheduled workflows + #if: github.event.repository.fork == false + if: github.repository_owner == 'HariSekhon' + name: Semgrep Cloud + uses: HariSekhon/GitHub-Actions/.github/workflows/semgrep-cloud.yaml@master + secrets: + SEMGREP_APP_TOKEN: ${{ secrets.SEMGREP_APP_TOKEN }} + with: + debug: ${{ github.event.inputs.debug }} diff --git a/.github/workflows/semgrep.yaml b/.github/workflows/semgrep.yaml new file mode 100644 index 0000000..1e38e7f --- /dev/null +++ b/.github/workflows/semgrep.yaml @@ -0,0 +1,64 @@ +# +# Author: Hari Sekhon +# Date: Tue Feb 4 09:53:28 2020 +0000 +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/HariSekhon/Template-repo +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/HariSekhon +# + +# ============================================================================ # +# S e m g r e p G i t H u b W o r k f l o w +# ============================================================================ # + +# Generates code scanning alerts in GitHub's Security tab -> Code scanning alerts + +# https://semgrep.dev/docs/semgrep-ci/sample-ci-configs/#github-actions + +--- +name: Semgrep + +on: + push: + branches: + - master + - main + paths-ignore: + - '**/*.md' + pull_request: + branches: + - master + - main + paths-ignore: + - '**/*.md' + workflow_dispatch: + inputs: + debug: + type: boolean + required: false + default: false + schedule: + - cron: '0 0 * * 1' + +permissions: + actions: read + contents: read + security-events: write + +concurrency: + group: ${{ github.ref }}-${{ github.workflow }} + cancel-in-progress: true + +jobs: + semgrep: + # github.event.repository context not available in scheduled workflows + #if: github.event.repository.fork == false + if: github.repository_owner == 'HariSekhon' + name: Semgrep GitHub Security Tab + uses: HariSekhon/GitHub-Actions/.github/workflows/semgrep.yaml@master + with: + debug: ${{ github.event.inputs.debug }} diff --git a/.github/workflows/sonarcloud.yaml b/.github/workflows/sonarcloud.yaml new file mode 100644 index 0000000..89de924 --- /dev/null +++ b/.github/workflows/sonarcloud.yaml @@ -0,0 +1,48 @@ +# +# Author: Hari Sekhon +# Date: 2023-04-14 23:53:43 +0100 (Fri, 14 Apr 2023) +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/HariSekhon/Template-repo +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/HariSekhon +# + +# ============================================================================ # +# S o n a r C l o u d +# ============================================================================ # + +--- +name: SonarCloud + +on: + push: + branches: + - master + - main + paths-ignore: + - '**/*.md' + pull_request: + branches: + - master + - main + paths-ignore: + - '**/*.md' + workflow_dispatch: + +permissions: + contents: read + pull-requests: read + +jobs: + SonarCloud: + # github.event.repository context not available in scheduled workflows + #if: github.event.repository.fork == false + if: github.repository_owner == 'HariSekhon' + name: SonarCloud + uses: HariSekhon/GitHub-Actions/.github/workflows/sonarcloud.yaml@master + secrets: + SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} diff --git a/.github/workflows/trivy.yaml b/.github/workflows/trivy.yaml new file mode 100644 index 0000000..6510d24 --- /dev/null +++ b/.github/workflows/trivy.yaml @@ -0,0 +1,62 @@ +# +# Author: Hari Sekhon +# Date: 2022-02-02 11:27:37 +0000 (Wed, 02 Feb 2022) +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/HariSekhon/Template-repo +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/HariSekhon +# + +# ============================================================================ # +# T r i v y +# ============================================================================ # + +# Scan files in the local repo + +--- +name: Trivy + +on: + push: + branches: + - master + - main + paths-ignore: + - '**/*.md' + pull_request: + branches: + - master + - main + paths-ignore: + - '**/*.md' + workflow_dispatch: + inputs: + debug: + type: boolean + required: false + default: false + schedule: + - cron: '0 0 * * 1' + +permissions: + actions: read + contents: read + security-events: write + +concurrency: + group: ${{ github.ref }}-${{ github.workflow }} + cancel-in-progress: true + +jobs: + trivy: + # github.event.repository context not available in scheduled workflows + #if: github.event.repository.fork == false + if: github.repository_owner == 'HariSekhon' + name: Trivy + uses: HariSekhon/GitHub-Actions/.github/workflows/trivy.yaml@master + with: + debug: ${{ github.event.inputs.debug }} diff --git a/.github/workflows/validate.yaml b/.github/workflows/validate.yaml new file mode 100644 index 0000000..2b28a73 --- /dev/null +++ b/.github/workflows/validate.yaml @@ -0,0 +1,60 @@ +# +# Author: Hari Sekhon +# Date: Tue Feb 4 09:53:28 2020 +0000 +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/HariSekhon/Template-repo +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/HariSekhon +# + +# ============================================================================ # +# V a l i d a t i o n +# ============================================================================ # + +# Run all custom validations against files in the repo + +--- +name: Validation + +on: + push: + branches: + - master + - main + paths-ignore: + - '**/*.md' + pull_request: + branches: + - master + - main + paths-ignore: + - '**/*.md' + workflow_dispatch: + inputs: + debug: + type: boolean + required: false + default: false + #schedule: + # - cron: '0 0 * * 1' + +permissions: + contents: read + +concurrency: + group: ${{ github.ref }}-${{ github.workflow }} + cancel-in-progress: true + +jobs: + validate: + # github.event.repository context not available in scheduled workflows + #if: github.event.repository.fork == false + if: github.repository_owner == 'HariSekhon' + name: Validate + uses: HariSekhon/GitHub-Actions/.github/workflows/validate.yaml@master + with: + debug: ${{ github.event.inputs.debug }} diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 0000000..182d857 --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,30 @@ +# vim:ts=2:sts=2:sw=2:et +# +# Author: Hari Sekhon +# Date: Sun Feb 23 19:02:10 2020 +0000 +# +# https://github.com/HariSekhon/Template-repo +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# to help improve or steer this or other code I publish +# +# https://www.linkedin.com/in/HariSekhon +# + +# ============================================================================ # +# G i t L a b C I +# ============================================================================ # + +# https://docs.gitlab.com/ee/ci/yaml/README.html + +#include: '.gitlab/*.y*ml' + +image: ubuntu:18.04 + +job: + before_script: + - setup/ci_bootstrap.sh + script: + - make init && make ci test diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..a6026cb --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "bash-tools"] + path = bash-tools + url = https://github.com/HariSekhon/DevOps-Bash-tools diff --git a/.mdl.rb b/.mdl.rb new file mode 100644 index 0000000..4409cf8 --- /dev/null +++ b/.mdl.rb @@ -0,0 +1,30 @@ +#!/usr/bin/env ruby +# vim:ts=4:sts=4:sw=4:et:filetype=ruby +# +# Author: Hari Sekhon +# Date: 2024-08-22 01:58:12 +0200 (Thu, 22 Aug 2024) +# +# https///github.com/HariSekhon/Template-repo +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/HariSekhon +# + +all +#exclude_rule 'MD001' +#exclude_rule 'MD003' +#exclude_rule 'MD005' +exclude_rule 'MD007' # leave 2 space indentation for lists, 3 space is ugly af +#exclude_rule 'MD012' +exclude_rule 'MD013' # long lines cannot be split if they are URLs +#exclude_rule 'MD022' +#exclude_rule 'MD025' +#exclude_rule 'MD031' +#exclude_rule 'MD032' +exclude_rule 'MD033' # inline HTML is important for formatting +exclude_rule 'MD036' # emphasis used instead of header for footer Ported from lines +#exclude_rule 'MD039' +#exclude_rule 'MD056' diff --git a/.mdlrc b/.mdlrc new file mode 100644 index 0000000..27e5b68 --- /dev/null +++ b/.mdlrc @@ -0,0 +1,5 @@ +mdlrc_dir = File.expand_path('..', __FILE__) + +style_file = File.join(mdlrc_dir, '.mdl.rb') + +style style_file diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..38aa145 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,73 @@ +# +# Author: Hari Sekhon +# Date: 2024-08-08 17:34:56 +0300 (Thu, 08 Aug 2024) +# +# vim:ts=2:sts=2:sw=2:et +# +# https///github.com/HariSekhon/Template-repo +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/HariSekhon +# + +# ============================================================================ # +# P r e - C o m m i t +# ============================================================================ # + +--- +fail_fast: false +#exclude: *.tmp$ + +repos: + + # will accept anything that 'git clone' understands + # this means you can set this to a local git repo to develop your own hook repos interactively + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.6.0 + hooks: + - id: check-yaml + # Common errors + - id: end-of-file-fixer + - id: trailing-whitespace + args: [--markdown-linebreak-ext=md] + # Git style + - id: check-added-large-files + - id: check-merge-conflict + - id: check-vcs-permalinks + - id: forbid-new-submodules + # Cross platform + - id: check-case-conflict + - id: mixed-line-ending + args: [--fix=lf] + # Security + - id: detect-aws-credentials + args: ['--allow-missing-credentials'] + + - repo: https://github.com/psf/black + rev: 24.8.0 + hooks: + - id: black + + # Git secrets Leaks + - repo: https://github.com/awslabs/git-secrets.git + # the release tags for 1.2.0, 1.2.1 and 1.3.0 are broken with this error: + # + # /Users/hari/.cache/pre-commit/repo......./.pre-commit-hooks.yaml is not a file + # + rev: 5357e18 + hooks: + - id: git-secrets + + - repo: https://github.com/markdownlint/markdownlint + rev: v0.12.0 + hooks: + - id: markdownlint + name: Markdownlint + description: Run markdownlint on your Markdown files + entry: mdl + args: [-s, .mdl.rb] + language: ruby + files: \.(md|mdown|markdown)$ diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..7e9e2fb --- /dev/null +++ b/LICENSE @@ -0,0 +1,7 @@ +Copyright 2016 Hari Sekhon + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..010db9d --- /dev/null +++ b/Makefile @@ -0,0 +1,85 @@ +# +# Author: Hari Sekhon +# Date: 2016-01-17 12:56:53 +0000 (Sun, 17 Jan 2016) +# +# vim:ts=4:sts=4:sw=4:noet +# +# https://github.com/HariSekhon/Template-Repo +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/HariSekhon +# + +# =================== +# bootstrap commands: + +# setup/bootstrap.sh +# +# OR +# +# Alpine: +# +# apk add --no-cache git make && git clone https://github.com/HariSekhon/Template-repo repo && cd repo && make +# +# Debian / Ubuntu: +# +# apt-get update && apt-get install -y make git && git clone https://github.com/HariSekhon/Template-repo repo && cd repo && make +# +# RHEL / CentOS: +# +# yum install -y make git && git clone https://github.com/HariSekhon/Template-repo repo && cd repo && make + +# =================== + +ifneq ("$(wildcard bash-tools/Makefile.in)", "") + include bash-tools/Makefile.in +endif + +REPO := HariSekhon/Template-Repo + +CODE_FILES := $(shell git ls-files | grep -E -e '\.sh$$' -e '\.py$$' | sort) + +.PHONY: build +build: init + @echo ================ + @echo Template-repo Builds + @echo ================ + @$(MAKE) git-summary + @echo + # defer via external sub-call, otherwise will result in error like + # make: *** No rule to make target 'python-version', needed by 'build'. Stop. + @$(MAKE) python-version + + if [ -z "$(CPANM)" ]; then make; exit $$?; fi + $(MAKE) system-packages-python + + # TODO: uncomment if adding requirements.txt with pip modules + #$(MAKE) python + +.PHONY: init +init: + @echo + @echo "running init:" + git submodule update --init --recursive + @echo + +.PHONY: install +install: build + @: + +.PHONY: python +python: + @PIP=$(PIP) PIP_OPTS="--ignore-installed" bash-tools/python/python_pip_install_if_absent.sh requirements.txt + @echo + $(MAKE) pycompile + @echo + @echo 'BUILD SUCCESSFUL (Template-Repo)' + +.PHONY: test +test: + bash-tools/checks/check_all.sh + +.PHONY: clean +clean: + @rm -fv -- *.pyc *.pyo diff --git a/README.md b/README.md new file mode 100644 index 0000000..f173497 --- /dev/null +++ b/README.md @@ -0,0 +1,138 @@ +# Hari Sekhon - Template Repo + +[![GitHub stars](https://img.shields.io/github/stars/HariSekhon/Template-Repo?logo=github)](https://github.com/HariSekhon/Template-Repo/stargazers) +[![GitHub forks](https://img.shields.io/github/forks/HariSekhon/Template-Repo?logo=github)](https://github.com/HariSekhon/Template-Repo/network) +[![Lines of Code](https://img.shields.io/badge/lines%20of%20code-1k-lightgrey?logo=codecademy)](https://github.com/HariSekhon/Template-Repo#Hari-Sekhon---Template-Repo) +[![License](https://img.shields.io/badge/license-MIT-green)](https://github.com/HariSekhon/Template-Repo/blob/master/LICENSE) +[![My LinkedIn](https://img.shields.io/badge/LinkedIn%20Profile-HariSekhon-blue?logo=data:image/svg%2bxml;base64,PHN2ZyByb2xlPSJpbWciIGZpbGw9IiNmZmZmZmYiIHZpZXdCb3g9IjAgMCAyNCAyNCIgeG1sbnM9Imh0dHA6Ly93d3cudzMub3JnLzIwMDAvc3ZnIj48dGl0bGU+TGlua2VkSW48L3RpdGxlPjxwYXRoIGQ9Ik0yMC40NDcgMjAuNDUyaC0zLjU1NHYtNS41NjljMC0xLjMyOC0uMDI3LTMuMDM3LTEuODUyLTMuMDM3LTEuODUzIDAtMi4xMzYgMS40NDUtMi4xMzYgMi45Mzl2NS42NjdIOS4zNTFWOWgzLjQxNHYxLjU2MWguMDQ2Yy40NzctLjkgMS42MzctMS44NSAzLjM3LTEuODUgMy42MDEgMCA0LjI2NyAyLjM3IDQuMjY3IDUuNDU1djYuMjg2ek01LjMzNyA3LjQzM2MtMS4xNDQgMC0yLjA2My0uOTI2LTIuMDYzLTIuMDY1IDAtMS4xMzguOTItMi4wNjMgMi4wNjMtMi4wNjMgMS4xNCAwIDIuMDY0LjkyNSAyLjA2NCAyLjA2MyAwIDEuMTM5LS45MjUgMi4wNjUtMi4wNjQgMi4wNjV6bTEuNzgyIDEzLjAxOUgzLjU1NVY5aDMuNTY0djExLjQ1MnpNMjIuMjI1IDBIMS43NzFDLjc5MiAwIDAgLjc3NCAwIDEuNzI5djIwLjU0MkMwIDIzLjIyNy43OTIgMjQgMS43NzEgMjRoMjAuNDUxQzIzLjIgMjQgMjQgMjMuMjI3IDI0IDIyLjI3MVYxLjcyOUMyNCAuNzc0IDIzLjIgMCAyMi4yMjIgMGguMDAzeiIvPjwvc3ZnPgo=)](https://www.linkedin.com/in/HariSekhon/) +[![GitHub Last Commit](https://img.shields.io/github/last-commit/HariSekhon/Template-Repo?logo=github)](https://github.com/HariSekhon/Template-Repo/commits/master) + +TODO: update Codacy grade link below - it's a hash so cannot be predicted to replace via `./customize.sh` + +[![Codacy](https://app.codacy.com/project/badge/Grade/abce2438224948fd93d462cd6aba3aaf)](https://www.codacy.com/gh/HariSekhon/Template-Repo/dashboard) +[![CodeFactor](https://www.codefactor.io/repository/github/harisekhon/Template-Repo/badge)](https://www.codefactor.io/repository/github/harisekhon/Template-Repo) +[![Quality Gate Status](https://sonarcloud.io/api/project_badges/measure?project=HariSekhon_Template-Repo&metric=alert_status)](https://sonarcloud.io/dashboard?id=HariSekhon_Template-Repo) +[![Maintainability Rating](https://sonarcloud.io/api/project_badges/measure?project=HariSekhon_Template-Repo&metric=sqale_rating)](https://sonarcloud.io/dashboard?id=HariSekhon_Template-Repo) +[![Reliability Rating](https://sonarcloud.io/api/project_badges/measure?project=HariSekhon_Template-Repo&metric=reliability_rating)](https://sonarcloud.io/dashboard?id=HariSekhon_Template-Repo) +[![Security Rating](https://sonarcloud.io/api/project_badges/measure?project=HariSekhon_Template-Repo&metric=security_rating)](https://sonarcloud.io/dashboard?id=HariSekhon_Template-Repo) + +![Python](https://img.shields.io/badge/Python-3-blue?logo=python&logoColor=white) +[![Linux](https://img.shields.io/badge/OS-Linux-blue?logo=linux)](https://github.com/HariSekhon/DevOps-Bash-tools#hari-sekhon---devops-bash-tools) +[![Mac](https://img.shields.io/badge/OS-Mac-blue?logo=apple)](https://github.com/HariSekhon/DevOps-Bash-tools#hari-sekhon---devops-bash-tools) + +[![Mac Homebrew](https://img.shields.io/badge/Mac-Homebrew-999999?logo=apple&logoColor=white)](https://brew.sh/) +[![Alpine](https://img.shields.io/badge/Linux-Alpine-0D597F?logo=alpine%20linux)](https://alpinelinux.org/) +[![CentOS](https://img.shields.io/badge/Linux-CentOS-262577?logo=centos&logoColor=white)](https://www.centos.org/) +[![Debian](https://img.shields.io/badge/Linux-Debian-A81D33?logo=debian)](https://www.debian.org/) +[![Fedora](https://img.shields.io/badge/Linux-Fedora-294172?logo=fedora&logoColor=white)](https://getfedora.org/) +[![Redhat](https://img.shields.io/badge/Linux-Redhat-EE0000?logo=red%20hat)](https://www.redhat.com/en) +[![Rocky](https://img.shields.io/badge/Linux-Rocky-10B981?logo=rockylinux&logoColor=white)](https://rockylinux.org/) +[![Ubuntu](https://img.shields.io/badge/Linux-Ubuntu-E95420?logo=ubuntu&logoColor=white)](https://ubuntu.com/) + + +[![CI Builds Overview](https://img.shields.io/badge/CI%20Builds-Overview%20Page-blue?logo=circleci)](https://harisekhon.github.io/CI-CD/) +[![Azure DevOps Pipeline](https://dev.azure.com/harisekhon/GitHub/_apis/build/status%2FTemplate-Repo?branchName=master)](https://dev.azure.com/harisekhon/GitHub/_build/latest?definitionId=15&branchName=master) +[![GitLab Pipeline](https://img.shields.io/badge/GitLab%20CI-legacy-lightgrey?logo=gitlab)](https://gitlab.com/HariSekhon/Template-Repo/pipelines) +[![BitBucket Pipeline](https://img.shields.io/badge/Bitbucket%20CI-legacy-lightgrey?logo=bitbucket)](https://bitbucket.org/HariSekhon/Template-Repo/addon/pipelines/home#!/) + +[![Repo on GitHub](https://img.shields.io/badge/repo-GitHub-2088FF?logo=github)](https://github.com/HariSekhon/Template-Repo) +[![Repo on GitLab](https://img.shields.io/badge/repo-GitLab-FCA121?logo=gitlab)](https://gitlab.com/HariSekhon/Template-Repo) +[![Repo on Azure DevOps](https://img.shields.io/badge/repo-Azure%20DevOps-0078D7?logo=azure%20devops)](https://dev.azure.com/HariSekhon/GitHub/_git/Template-Repo) +[![Repo on BitBucket](https://img.shields.io/badge/repo-BitBucket-0052CC?logo=bitbucket)](https://bitbucket.org/HariSekhon/Template-Repo) + +[![Validation](https://github.com/HariSekhon/Template-Repo/actions/workflows/validate.yaml/badge.svg)](https://github.com/HariSekhon/Template-Repo/actions/workflows/validate.yaml) +[![Kics](https://github.com/HariSekhon/Template-Repo/actions/workflows/kics.yaml/badge.svg)](https://github.com/HariSekhon/Template-Repo/actions/workflows/kics.yaml) +[![Grype](https://github.com/HariSekhon/Template-Repo/actions/workflows/grype.yaml/badge.svg)](https://github.com/HariSekhon/Template-Repo/actions/workflows/grype.yaml) +[![Semgrep](https://github.com/HariSekhon/Template-Repo/actions/workflows/semgrep.yaml/badge.svg)](https://github.com/HariSekhon/Template-Repo/actions/workflows/semgrep.yaml) +[![Semgrep Cloud](https://github.com/HariSekhon/Template-Repo/actions/workflows/semgrep-cloud.yaml/badge.svg)](https://github.com/HariSekhon/Template-Repo/actions/workflows/semgrep-cloud.yaml) +[![SonarCloud](https://github.com/HariSekhon/Template-Repo/actions/workflows/sonarcloud.yaml/badge.svg)](https://github.com/HariSekhon/Template-Repo/actions/workflows/sonarcloud.yaml) +[![Trivy](https://github.com/HariSekhon/Template-Repo/actions/workflows/trivy.yaml/badge.svg)](https://github.com/HariSekhon/Template-Repo/actions/workflows/trivy.yaml) + +## PUT CONTENT HERE + +### Customize Repo in 1 Command + +Run this script to replace the repo name and workflow URL references across the configurations for CI/CD and +other variety of files in this repo: + +```shell +./customize.sh +``` + +Then delete the script and edit this README. + +#### Ensure to update Codacy Grade link manually after adding this repo to Codacy since the link is a hash and therefore cannot be predicted for replacement by `./customize.sh`. + +## More Core Repos + + + +### Knowledge + +[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Knowledge-Base&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Knowledge-Base) +[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Diagrams-as-Code&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Diagrams-as-Code) + + + +### DevOps Code + +[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=DevOps-Bash-tools&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/DevOps-Bash-tools) +[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=DevOps-Python-tools&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/DevOps-Python-tools) +[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=DevOps-Perl-tools&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/DevOps-Perl-tools) +[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=DevOps-Golang-tools&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/DevOps-Golang-tools) + + + +### Containerization + +[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Kubernetes-configs&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Kubernetes-configs) +[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Dockerfiles&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Dockerfiles) + +### CI/CD + +[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=GitHub-Actions&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/GitHub-Actions) +[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Jenkins&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Jenkins) + +### DBA - SQL + +[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=SQL-scripts&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/SQL-scripts) + +### DevOps Reloaded + +[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Nagios-Plugins&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Nagios-Plugins) +[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=HAProxy-configs&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/HAProxy-configs) +[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Terraform&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Terraform) +[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Packer-templates&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Packer-templates) +[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Nagios-Plugin-Kafka&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Nagios-Plugin-Kafka) + +### Templates + +[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Templates&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Templates) +[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Template-repo&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Template-repo) + +### Misc + +[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Spotify-tools&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Spotify-tools) +[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Spotify-playlists&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Spotify-playlists) + +The rest of my original source repos are +[here](https://github.com/HariSekhon?tab=repositories&q=&type=source&language=&sort=stargazers). + +Pre-built Docker images are available on my [DockerHub](https://hub.docker.com/u/harisekhon/). + + +![](https://hit.yhype.me/github/profile?user_id=2211051) + + diff --git a/azure-pipelines.yml b/azure-pipelines.yml new file mode 100644 index 0000000..2420802 --- /dev/null +++ b/azure-pipelines.yml @@ -0,0 +1,70 @@ +# vim:ts=2:sts=2:sw=2:et +# +# Author: Hari Sekhon +# Date: Sun Feb 23 19:02:10 2020 +0000 +# +# https://github.com/HariSekhon/Template-repo +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# to help improve or steer this or other code I publish +# +# https://www.linkedin.com/in/HariSekhon +# + +# ============================================================================ # +# A z u r e D e v O p s P i p e l i n e +# ============================================================================ # + +# https://aka.ms/yaml + +trigger: + - master + +variables: + # ubuntu version + os_version: '22.04' + +pool: + # there is no /dev/stderr on this azure build! + #vmImage: 'ubuntu-latest' + #vmImage: 'ubuntu-22.04' + vmImage: 'ubuntu-$(os_version)' + +# unprivileged container without sudo, cannot install dependencies +#container: ubuntu:22.04 + +steps: + - script: cat /etc/*-release + displayName: OS Release + + # requires script as first key, otherwise parsing breaks with error message: Unexpected value 'displayName' + - script: env | sort + displayName: Environment + + # doesn't work in container due to unprivileged execution and lack of sudo + #- script: sudo apt-get update && sudo apt-get install -y git make + # displayName: install git & make + + #- script: make + # displayName: build + + # doesn't work in vmImage build due to lack of access to normal /dev/stderr device + # tee: /dev/stderr: No such device or address + #- script: make test + # displayName: test + + # hacky workaround to Azure Pipelines ubuntu environment limitations of unprivileged container and no /dev/stderr in vmImage :-( + - script: | + sudo docker run -v "$PWD":/code "ubuntu:$(os_version)" /bin/bash -c ' + set -ex + cd /code + setup/ci_bootstrap.sh + if [ -x setup/ci_git_set_dir_safe.sh ]; then + setup/ci_git_set_dir_safe.sh + fi + make init + make ci test + ' + displayName: docker build diff --git a/bash-tools b/bash-tools new file mode 160000 index 0000000..adc18db --- /dev/null +++ b/bash-tools @@ -0,0 +1 @@ +Subproject commit adc18db20909106dbf4d6ededa1c6c66d5c28e8c diff --git a/bitbucket-pipelines.yml b/bitbucket-pipelines.yml new file mode 100644 index 0000000..11a19bd --- /dev/null +++ b/bitbucket-pipelines.yml @@ -0,0 +1,38 @@ +# +# Author: Hari Sekhon +# Date: 2020-02-24 17:08:57 +0000 (Mon, 24 Feb 2020) +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/HariSekhon/Template-repo +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/HariSekhon +# + +# ============================================================================ # +# B i t b u c k e t C I / C D P i p e l i n e +# ============================================================================ # + +# Reference: +# +# https://support.atlassian.com/bitbucket-cloud/docs/configure-bitbucket-pipelinesyml/ + +# Languages: +# +# https://confluence.atlassian.com/x/5Q4SMw + +# You can specify a custom docker image from Docker Hub as your build environment. +image: atlassian/default-image:2 + +pipelines: + default: + - step: + script: + - setup/ci_bootstrap.sh + - make init + - make ci + - make test diff --git a/customize.sh b/customize.sh new file mode 100755 index 0000000..6f3cdf0 --- /dev/null +++ b/customize.sh @@ -0,0 +1,54 @@ +#!/usr/bin/env bash +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: 2023-06-02 01:29:52 +0100 (Fri, 02 Jun 2023) +# +# https://github.com/HariSekhon/Template-repo +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/HariSekhon +# + +# Replaces references in this repo with the new name of your choice +# +# The delete this script in the new templated repo and carry on + +set -euo pipefail +[ -n "${DEBUG:-}" ] && set -x +srcdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +cd "$srcdir" + +if [ $# -ne 1 ]; then + echo "usage: " + exit 3 +fi + +repo="$1" + +if uname -s | grep -q Darwin; then + sed(){ + gsed "$@" + } +fi + +sed -i "s/- Template Repo$/- $repo/" README.md + +sed -i "s/Template-Repo/$repo/gi" \ + README.md \ + Makefile \ + azure-pipelines.yml \ + bitbucket-pipelines.yml \ + sonar-project.properties \ + .github/workflows/*.y*ml + +# can't commit without the submodules checked out +# +# error: 'bash-tools' does not have a commit checked out +# fatal: updating files failed +# +make init diff --git a/setup/ci_bootstrap.sh b/setup/ci_bootstrap.sh new file mode 100755 index 0000000..e01a244 --- /dev/null +++ b/setup/ci_bootstrap.sh @@ -0,0 +1,93 @@ +#!/bin/sh +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: 2020-06-02 17:43:35 +0100 (Tue, 02 Jun 2020) +# +# https://github.com/HariSekhon/Template-repo +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/HariSekhon +# + +# Designed to bootstrap all CI systems with retries to make sure the networking, package lists and package repos works before proceeding +# +# Minimizes CI build failures due to temporary networking blips, which happens more often than you would think when you have a large number of CI builds across a lot of disparate systems + +set -eu +[ -n "${DEBUG:-}" ] && set -x + +max_tries=10 +interval=60 # secs + +sudo="" +# EUID undefined in posix sh +#[ $EUID = 0 ] || sudo=sudo +[ "$(whoami)" = root ] || sudo=sudo + +retry(){ + # no local in posix sh + count=0 + while true; do + # no let or bare (()) in posix sh, must discard output rather than execute it + _=$((count+=1)) + printf "%s try %d: " "$(date '+%F %T')" "$count" + echo "$*" + "$@" && + break; + echo + if [ "$count" -ge "$max_tries" ]; then + echo "$count tries failed, aborting..." + exit 1 + fi + echo "sleeping for $interval secs before retrying" + sleep "$interval" + echo + done +} + +if [ "$(uname -s)" = Darwin ]; then + echo "Bootstrapping Mac" + # removing adjacent dependency to be able to curl from github to avoid submodule circular dependency (git / submodule / install git & make) + #retry "$srcdir/../install/install_homebrew.sh" + if command -v brew 2>&1; then + # fix for CI runners on Mac with shallow homebrew clone - which is failing all the BuildKite builds + for git_root in /usr/local/Homebrew/Library/Taps/homebrew/homebrew-core /usr/local/Homebrew/Library/Taps/homebrew/homebrew-cask; do + if [ -d "$git_root" ]; then + # find out if Homebrew is a shallow git checkout and if so fix it + if [ -f "$(git -C "$git_root" rev-parse --git-dir)/shallow" ] || + [ "$(git -C "$git_root" rev-parse --is-shallow-repository)" = true ]; then + git -C "$git_root" fetch --unshallow + fi + fi + done + retry brew update + fi +elif [ "$(uname -s)" = Linux ]; then + echo "Bootstrapping Linux" + if type apk >/dev/null 2>&1; then + retry $sudo apk update + retry $sudo apk add --no-progress bash git make + elif type apt-get >/dev/null 2>&1; then + retry $sudo apt-get update -q + retry $sudo apt-get install -qy git make + elif type yum >/dev/null 2>&1; then + #retry $sudo yum makecache + retry $sudo yum install -qy git make + else + echo "Package Manager not found on Linux, cannot bootstrap" + exit 1 + fi +else + echo "Only Mac & Linux are supported for conveniently bootstrapping all install scripts at this time" + exit 1 +fi + +#retry make init + +# not calling make because in some CI systems we call 'make ci' which includes retries but in others with more restrictive build minutes we only run 'make' for a single shot build +# +#make diff --git a/setup/ci_git_set_dir_safe.sh b/setup/ci_git_set_dir_safe.sh new file mode 100755 index 0000000..c3f2ee9 --- /dev/null +++ b/setup/ci_git_set_dir_safe.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: 2022-08-03 20:07:09 +0100 (Wed, 03 Aug 2022) +# +# https://github.com/HariSekhon/DevOps-Bash-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/HariSekhon +# + +# Necessary for some CI/CD systems like Azure DevOps Pipelines which have incorrect ownership on the git checkout dir triggering this error: +# +# fatal: detected dubious ownership in repository at '/code/sql' + +# standalone script without lib dependency so it can be called directly from bootstrapped CI before submodules, since that is the exact problem that needs to be solved to allow CI/CD systems with incorrect ownership of the checkout directory to be able to checkout the necessary git submodules + +set -euo pipefail +[ -n "${DEBUG:-}" ] && set -x +srcdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +dir="${1:-$srcdir/..}" + +cd "$dir" + +echo "Setting directory as safe: $PWD" +git config --global --add safe.directory "$PWD" + +while read -r submodule_dir; do + dir="$PWD/$submodule_dir" + echo "Setting directory as safe: $dir" + git config --global --add safe.directory "$dir" +done < <(git submodule | awk '{print $2}') + +echo "Done" diff --git a/sonar-project.properties b/sonar-project.properties new file mode 100644 index 0000000..c02f5cc --- /dev/null +++ b/sonar-project.properties @@ -0,0 +1,42 @@ +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: 2016-07-19 18:31:17 +0100 (Tue, 19 Jul 2016) +# +# https://github.com/HariSekhon/Template-repo +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/HariSekhon +# + +# ============================================================================ # +# S o n a r Q u b e +# ============================================================================ # + +sonar.host.url=https://sonarcloud.io + +# Required metadata +sonar.organization=harisekhon +sonar.projectName=Template-repo +sonar.projectKey=HariSekhon_Template-repo +sonar.projectVersion=1.0 + +sonar.projectDescription=Template-repo + +sonar.links.homepage=https://github.com/HariSekhon/Template-repo +sonar.links.scm=https://github.com/HariSekhon/Template-repo +sonar.links.issue=https://github.com/HariSekhon/Template-repo/issues +sonar.links.ci=https://github.com/HariSekhon/Template-repo/actions + +# directories to scan (defaults to sonar-project.properties dir otherwise) +sonar.sources=. + +#sonar.language=py + +sonar.sourceEncoding=UTF-8 + +#sonar.exclusions=**/tests/** +sonar.exclusions=**/zookeeper-*/**/*