From 971bc55bb7a7d7571114f5a8abae195260d3216a Mon Sep 17 00:00:00 2001 From: Will Boyce Date: Sun, 23 Sep 2018 20:08:45 +0100 Subject: [PATCH 1/4] openssl: Improve cert generation and generate vpn certs * use easyrsa for cert generation/management * generate vpn requirements (ca, crt, dh) * remove `node` requirement for generating jwt kid * fix root ca cert generation * fix haproxy certificate generation Signed-off-by: Will Boyce --- compose/services.yml | 6 +++ haproxy/entry.sh | 17 ++++---- haproxy/haproxy.cfg | 2 +- scripts/_keyid.js | 79 ------------------------------------- scripts/gen-root-ca | 32 +++++++++++++++ scripts/gen-root-ca-cert | 55 -------------------------- scripts/gen-root-cert | 33 ++++++++++++++++ scripts/gen-token-auth-cert | 60 ++++++++++++++-------------- scripts/gen-vpn-certs | 52 ++++++++++++++++++++++++ scripts/make-env | 56 +++++++++++++++++--------- scripts/ssl-common.sh | 22 +++++++++++ scripts/start-project | 31 +++++++-------- 12 files changed, 236 insertions(+), 209 deletions(-) delete mode 100644 scripts/_keyid.js create mode 100755 scripts/gen-root-ca delete mode 100755 scripts/gen-root-ca-cert create mode 100755 scripts/gen-root-cert create mode 100755 scripts/gen-vpn-certs create mode 100644 scripts/ssl-common.sh diff --git a/compose/services.yml b/compose/services.yml index 74f5db65a..6c8febaf1 100644 --- a/compose/services.yml +++ b/compose/services.yml @@ -134,6 +134,10 @@ services: SENTRY_DSN: VPN_HAPROXY_USEPROXYPROTOCOL: 'true' VPN_SERVICE_API_KEY: ${OPENBALENA_VPN_SERVICE_API_KEY} + VPN_OPENVPN_CA_CRT: ${OPENBALENA_VPN_CA} + VPN_OPENVPN_SERVER_CRT: ${OPENBALENA_VPN_SERVER_CRT} + VPN_OPENVPN_SERVER_KEY: ${OPENBALENA_VPN_SERVER_KEY} + VPN_OPENVPN_SERVER_DH: ${OPENBALENA_VPN_SERVER_DH} # FIXME: remove all of the following API_SERVICE_API_KEY: __unused__ PROXY_SERVICE_API_KEY: __unused__5 @@ -176,6 +180,8 @@ services: - img.${OPENBALENA_HOST_NAME} environment: BALENA_ROOT_CA: ${OPENBALENA_ROOT_CA} + BALENA_HAPROXY_CRT: ${OPENBALENA_ROOT_CRT} + BALENA_HAPROXY_KEY: ${OPENBALENA_ROOT_KEY} HAPROXY_HOSTNAME: ${OPENBALENA_HOST_NAME} # FIXME: remove the following diff --git a/haproxy/entry.sh b/haproxy/entry.sh index 62b49bb5d..443fced27 100755 --- a/haproxy/entry.sh +++ b/haproxy/entry.sh @@ -1,9 +1,10 @@ -#!/bin/sh +#!/bin/bash -eu -CA_B64="$BALENA_ROOT_CA" -CA_FILE=/etc/ssl/private/root.chain.pem - -mkdir -p $(dirname "$CA_FILE") -echo "$CA_B64" | base64 -d >"$CA_FILE" - -exec haproxy -f /usr/local/etc/haproxy/haproxy.cfg +HAPROXY_CHAIN=/etc/ssl/private/open-balena.pem +mkdir -p "$(dirname "${HAPROXY_CHAIN}")" +( + echo "${BALENA_HAPROXY_CRT}" | base64 -d + echo "${BALENA_HAPROXY_KEY}" | base64 -d + echo "${BALENA_ROOT_CA}" | base64 -d +) > "${HAPROXY_CHAIN}" +exec haproxy -f /usr/local/etc/haproxy/haproxy.cfg \ No newline at end of file diff --git a/haproxy/haproxy.cfg b/haproxy/haproxy.cfg index d84190f64..5e0bed928 100644 --- a/haproxy/haproxy.cfg +++ b/haproxy/haproxy.cfg @@ -47,7 +47,7 @@ backend redirect-to-https-in frontend https-in mode http option forwardfor - bind 127.0.0.1:444 ssl crt /etc/ssl/private/root.chain.pem accept-proxy + bind 127.0.0.1:444 ssl crt /etc/ssl/private/open-balena.pem accept-proxy reqadd X-Forwarded-Proto:\ https acl host_api hdr_dom(host) -i "api.${HAPROXY_HOSTNAME}" diff --git a/scripts/_keyid.js b/scripts/_keyid.js deleted file mode 100644 index 61ee4190a..000000000 --- a/scripts/_keyid.js +++ /dev/null @@ -1,79 +0,0 @@ -'use strict'; - -var crypto = require('crypto'); -var fs = require('fs'); - -var base32 = (function() { - // Extracted from https://github.com/chrisumbel/thirty-two - // to avoid having to install packages for this script. - var charTable = "ABCDEFGHIJKLMNOPQRSTUVWXYZ234567"; - var byteTable = [ - 0xff, 0xff, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, - 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, - 0x0f, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, - 0x17, 0x18, 0x19, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, - 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, - 0x0f, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, - 0x17, 0x18, 0x19, 0xff, 0xff, 0xff, 0xff, 0xff - ]; - - function quintetCount(buff) { - var quintets = Math.floor(buff.length / 5); - return buff.length % 5 == 0 ? quintets: quintets + 1; - } - - return function(plain) { - if (!Buffer.isBuffer(plain)) { - plain = new Buffer(plain); - } - var i = 0; - var j = 0; - var shiftIndex = 0; - var digit = 0; - var encoded = new Buffer(quintetCount(plain) * 8); - - /* byte by byte isn't as pretty as quintet by quintet but tests a bit - faster. will have to revisit. */ - while(i < plain.length) { - var current = plain[i]; - - if(shiftIndex > 3) { - digit = current & (0xff >> shiftIndex); - shiftIndex = (shiftIndex + 5) % 8; - digit = (digit << shiftIndex) | ((i + 1 < plain.length) ? - plain[i + 1] : 0) >> (8 - shiftIndex); - i++; - } else { - digit = (current >> (8 - (shiftIndex + 5))) & 0x1f; - shiftIndex = (shiftIndex + 5) % 8; - if(shiftIndex == 0) i++; - } - - encoded[j] = charTable.charCodeAt(digit); - j++; - } - - for (i = j; i < encoded.length; i++) { - encoded[i] = 0x3d; //'='.charCodeAt(0) - } - return encoded; - } -})(); - -function joseKeyId(der) { - var hasher = crypto.createHash('sha256'); - hasher.update(der); - var b32 = base32(hasher.digest().slice(0, 30)).toString('ascii'); - var chunks = []; - for (var i = 0; i < b32.length; i += 4) { - chunks.push(b32.substr(i, 4)); - } - return chunks.join(':'); -} - -var derFilePath = process.argv[2]; -var der = fs.readFileSync(derFilePath); -process.stdout.write(joseKeyId(der)); \ No newline at end of file diff --git a/scripts/gen-root-ca b/scripts/gen-root-ca new file mode 100755 index 000000000..fba233bba --- /dev/null +++ b/scripts/gen-root-ca @@ -0,0 +1,32 @@ +#!/bin/bash -eu + +usage() { + echo "usage: $0 COMMON_NAME [OUT]" + echo + echo " COMMON_NAME the domain name the certificate is valid for, eg. example.com" + echo " OUT path to output directory generated files will be placed in" + echo +} + +if [ -z "$1" ]; then + usage + exit 1 +fi + +CMD="$(realpath "$0")" +DIR="$(dirname "${CMD}")" + +CN="$1" +OUT="$(realpath "${2:-.}")" + +source "${DIR}/ssl-common.sh" + +# Create a secret key and CA file for the self-signed CA +"$easyrsa_bin" --pki-dir="${ROOT_PKI}" init-pki 2>/dev/null +"$easyrsa_bin" --pki-dir="${ROOT_PKI}" --days="${CA_EXPIRY_DAYS}" --req-cn="ca.${CN}" build-ca nopass 2>/dev/null +ROOT_CA="${ROOT_PKI}/ca.crt" +echo "ROOT_CA=${ROOT_CA//$OUT/\$OUT}" + +# update indexes and generate CRLs +"$easyrsa_bin" --pki-dir="${ROOT_PKI}" update-db 2>/dev/null +"$easyrsa_bin" --pki-dir="${ROOT_PKI}" gen-crl 2>/dev/null \ No newline at end of file diff --git a/scripts/gen-root-ca-cert b/scripts/gen-root-ca-cert deleted file mode 100755 index d4086ef77..000000000 --- a/scripts/gen-root-ca-cert +++ /dev/null @@ -1,55 +0,0 @@ -#!/bin/sh - -set -e - -CN=$1 -OUT=${2:-.} - -CA="${CN}-ca" -CA_FILE="${OUT}/ca" -CN_FILE="${OUT}/root" - -CN_EXPIRY_DAYS=730 -CA_EXPIRY_DAYS=3650 - -usage() { - echo "usage: $0 HOST_NAME [OUT]" - echo - echo " HOST_NAME the domain name the certificate is valid for, eg. example.com" - echo " OUT path to output directory generated files will be placed in" - echo -} - -if [ -z "$CN" ]; then - usage - exit 1 -fi - -cat > "${CN_FILE}.v3.ext" </dev/null -openssl req -x509 -new -nodes -sha256 -days $CA_EXPIRY_DAYS -key "${CA_FILE}.key" -subj "/CN=${CA}" -out "${CA_FILE}.pem" 2>/dev/null - -# Create a secret key and Certificate Signing Request (CSR) for the domain -openssl genrsa -out "${CN_FILE}.key" 2048 2>/dev/null -openssl req -new -key "${CN_FILE}.key" -subj "/CN=${CN}" -out "${CN_FILE}.csr" 2>/dev/null - -# Sign the request with the self-signed CA and extension file -openssl x509 -req -sha256 -days $CN_EXPIRY_DAYS -in "${CN_FILE}.csr" -CA "${CA_FILE}.pem" -CAkey "${CA_FILE}.key" -CAcreateserial -out "${CN_FILE}.pem" -extfile "${CN_FILE}.v3.ext" 2>/dev/null - -# Create the custom certificate chain file -cat "${CN_FILE}.pem" "${CA_FILE}.pem" "${CN_FILE}.key" >"${CN_FILE}.chain.pem" - -# Cleanup -rm "${CA_FILE}.key" "${CA_FILE}.pem" "${CN_FILE}.key" "${CN_FILE}.pem" "${CN_FILE}.csr" "${CN_FILE}.v3.ext" - -echo "ROOT_CA=${CN_FILE}.chain.pem" diff --git a/scripts/gen-root-cert b/scripts/gen-root-cert new file mode 100755 index 000000000..f310a60c7 --- /dev/null +++ b/scripts/gen-root-cert @@ -0,0 +1,33 @@ +#!/bin/bash -eu + +usage() { + echo "usage: $0 COMMON_NAME [OUT]" + echo + echo " COMMON_NAME the domain name the certificate is valid for, eg. example.com" + echo " OUT path to output directory generated files will be placed in" + echo +} + +if [ -z "$1" ]; then + usage + exit 1 +fi + +CMD="$(realpath "$0")" +DIR="$(dirname "${CMD}")" + +CN="$1" +OUT="$(realpath "${2:-.}")" + +source "${DIR}/ssl-common.sh" + +# generate default CSR and sign (root + wildcard) +"$easyrsa_bin" --pki-dir="${ROOT_PKI}" --days="${CRT_EXPIRY_DAYS}" --subject-alt-name="DNS:${CN}" build-server-full "*.${CN}" nopass 2>/dev/null +ROOT_CRT="${ROOT_PKI}"'/issued/*.'"${CN}"'.crt' +ROOT_KEY="${ROOT_PKI}"'/private/*.'"${CN}"'.key' +echo "ROOT_CRT=${ROOT_CRT//$OUT/\$OUT}" +echo "ROOT_KEY=${ROOT_KEY//$OUT/\$OUT}" + +# update indexes and generate CRLs +"$easyrsa_bin" --pki-dir="${ROOT_PKI}" update-db 2>/dev/null +"$easyrsa_bin" --pki-dir="${ROOT_PKI}" gen-crl 2>/dev/null \ No newline at end of file diff --git a/scripts/gen-token-auth-cert b/scripts/gen-token-auth-cert index 29197c216..60ae2a943 100755 --- a/scripts/gen-token-auth-cert +++ b/scripts/gen-token-auth-cert @@ -1,42 +1,42 @@ -#!/bin/sh - -set -e - -CMD=$0 -DIR=$(dirname "$CMD") - -CN=$1 -OUT=${2:-.} - -CERT_FILE="${OUT}/token-auth" -EXPIRY_DAYS=730 +#!/bin/bash -eu usage() { - echo "usage: $0 HOST_NAME [OUT]" + echo "usage: $0 COMMON_NAME [OUT]" echo - echo " HOST_NAME the domain name the certificate is valid for, eg. example.com" - echo " OUT path to output directory generated files will be placed in" + echo " COMMON_NAME the domain name the certificate is valid for, eg. example.com" + echo " OUT path to output directory generated files will be placed in" echo } -keyid() { - # FIXME: do this in bash or python, not node - nodejs "${DIR}/_keyid.js" "$1" -} - -if [ -z "$CN" ]; then +if [ -z "$1" ]; then usage exit 1 fi -openssl ecparam -name prime256v1 -genkey -noout -out "${CERT_FILE}.pem" 2>/dev/null -openssl req -x509 -new -nodes -days "${EXPIRY_DAYS}" -key "${CERT_FILE}.pem" -subj "/CN=api.${CN}" -out "${CERT_FILE}.crt" 2>/dev/null -openssl ec -in "${CERT_FILE}.pem" -pubout -outform DER -out "${CERT_FILE}".der 2>/dev/null -keyid "${CERT_FILE}".der >"${CERT_FILE}".kid +CMD="$(realpath "$0")" +DIR="$(dirname "${CMD}")" + +CN="$1" +OUT="$(realpath "${2:-.}")" + +source "${DIR}/ssl-common.sh" + +keyid() { + local der="$(openssl ec -in "$1" -pubout -outform DER 2>/dev/null)" + python -c "import sys as S; from base64 import b32encode as B; import hashlib as H; h = H.sha256(); h.update(S.argv[1].encode('ascii')); s = B(h.digest()[:30]).decode('ascii'); S.stdout.write(':'.join([s[i:i+4] for i in range(0, len(s), 4)]))" "${der}" +} + +# generate api CSR and sign +"$easyrsa_bin" --pki-dir="${ROOT_PKI}" --days=730 --use-algo=ec --curve=prime256v1 build-server-full "api.${CN}" nopass 2>/dev/null +JWT_CRT="${ROOT_PKI}/issued/api.${CN}.crt" +JWT_KEY="${ROOT_PKI}/private/api.${CN}.key" +echo "JWT_CRT=${JWT_CRT//$OUT/\$OUT}" +echo "JWT_KEY=${JWT_KEY//$OUT/\$OUT}" -# Cleanup -rm "${CERT_FILE}.der" +# update indexes and generate CRLs +"$easyrsa_bin" --pki-dir="${ROOT_PKI}" update-db 2>/dev/null +"$easyrsa_bin" --pki-dir="${ROOT_PKI}" gen-crl 2>/dev/null -echo "PUB=${CERT_FILE}.crt" -echo "KEY=${CERT_FILE}.pem" -echo "KID=${CERT_FILE}.kid" +# generate key ID +JWT_KID="$(keyid "${JWT_CRT}")" +echo "JWT_KID=${JWT_KID//$OUT/\$OUT}" diff --git a/scripts/gen-vpn-certs b/scripts/gen-vpn-certs new file mode 100755 index 000000000..af217358b --- /dev/null +++ b/scripts/gen-vpn-certs @@ -0,0 +1,52 @@ +#!/bin/bash -eu + +usage() { + echo "usage: $0 COMMON_NAME [OUT]" + echo + echo " COMMON_NAME the domain name the certificate is valid for, eg. example.com" + echo " OUT path to output directory generated files will be placed in" + echo +} + +if [ -z "$1" ]; then + usage + exit 1 +fi + +CMD="$(realpath "$0")" +DIR="$(dirname "${CMD}")" + +CN="$1" +OUT="$(realpath "${2:-.}")" + +source "${DIR}/ssl-common.sh" +VPN_PKI="$(realpath "${OUT}/vpn")" + +# generate VPN sub-CA +"$easyrsa_bin" --pki-dir="${VPN_PKI}" init-pki 2>/dev/null +"$easyrsa_bin" --pki-dir="${VPN_PKI}" --days="${CA_EXPIRY_DAYS}" --req-cn="vpn-ca.${CN}" build-ca nopass subca 2>/dev/null + +# import sub-CA CSR into root PKI, sign, and copy back to vpn PKI +"$easyrsa_bin" --pki-dir="${ROOT_PKI}" import-req "${VPN_PKI}/reqs/ca.req" "vpn-ca" 2>/dev/null +"$easyrsa_bin" --pki-dir="${ROOT_PKI}" sign-req ca "vpn-ca" 2>/dev/null +cp "${ROOT_PKI}/issued/vpn-ca.crt" "${VPN_PKI}/ca.crt" +VPN_CA="${VPN_PKI}/ca.crt" +echo "VPN_CA=${VPN_CA//$OUT/\$OUT}" + +# generate and sign vpn server certificate +"$easyrsa_bin" --pki-dir="${VPN_PKI}" --days="${CRT_EXPIRY_DAYS}" build-server-full "vpn.${CN}" nopass 2>/dev/null +VPN_CRT="${VPN_PKI}/issued/vpn.${CN}.crt" +VPN_KEY="${VPN_PKI}/private/vpn.${CN}.key" +echo "VPN_CRT=${VPN_CRT//$OUT/\$OUT}" +echo "VPN_KEY=${VPN_KEY//$OUT/\$OUT}" + +# generate vpn dhparams (keysize of 2048 will do, 4096 can wind up taking hours to generate) +"$easyrsa_bin" --pki-dir="${VPN_PKI}" --keysize=2048 gen-dh 2>/dev/null +VPN_DH="${VPN_PKI}/dh.pem" +echo "VPN_DH=${VPN_DH//$OUT/\$OUT}" + +# update indexes and generate CRLs +"$easyrsa_bin" --pki-dir="${ROOT_PKI}" update-db 2>/dev/null +"$easyrsa_bin" --pki-dir="${VPN_PKI}" update-db 2>/dev/null +"$easyrsa_bin" --pki-dir="${ROOT_PKI}" gen-crl 2>/dev/null +"$easyrsa_bin" --pki-dir="${VPN_PKI}" gen-crl 2>/dev/null \ No newline at end of file diff --git a/scripts/make-env b/scripts/make-env index 8f303f4b8..9de2c2dd6 100755 --- a/scripts/make-env +++ b/scripts/make-env @@ -1,30 +1,42 @@ -#!/bin/sh - -set -e - -HOST_NAME="$1" -ROOT_CA="$2" -TOKEN_AUTH_PUB="$3" -TOKEN_AUTH_KEY="$4" -TOKEN_AUTH_KID="$5" +#!/bin/bash -eu usage() { - echo "usage: $0 HOST_NAME ROOT_CA TOKEN_AUTH_PUB TOKEN_AUTH_KEY TOKEN_AUTH_KID" + echo "usage: $0" + echo + echo "Required Variables:" + echo + echo " HOST_NAME" + echo " ROOT_CA Path to root CA certificate" + echo " ROOT_CRT Path to root/wildcard certificate" + echo " ROOT_KEY Path to root/wildcard private key" + echo " JWT_CRT Path to Token Auth certificate" + echo " JWT_KEY Path to Token Auth private key" + echo " JWT_KID The KeyID for the Token Auth certificate" + echo " VPN_CA Path to the VPN sub-CA certificate" + echo " VPN_CRT Path to the VPN server certificate" + echo " VPN_KEY Path to the VPN server private key" + echo " VPN_DH Path to the VPN server Diffie Hellman parameters" echo } +for var in HOST_NAME ROOT_CA ROOT_CRT ROOT_KEY JWT_CRT JWT_KEY JWT_KID VPN_CA VPN_CRT VPN_KEY VPN_DH; do + if [ -z "${!var-}" ]; then + usage + exit 1 + fi +done + randstr() { - LC_CTYPE=C tr -dc A-Za-z0-9 < /dev/urandom | fold -w ${1:-32} | head -n 1 + LC_CTYPE=C tr -dc A-Za-z0-9 < /dev/urandom | fold -w "${1:-32}" | head -n 1 } b64encode() { - cat "$1" | base64 --wrap=0 2>/dev/null || cat "$1" | base64 --break=0 + cat "$@" | base64 --wrap=0 2>/dev/null || cat "$@" | base64 --break=0 } -if [ -z "$HOST_NAME" ] || [ -z "$ROOT_CA" ] || [ -z "$TOKEN_AUTH_PUB" ] || [ -z "$TOKEN_AUTH_KEY" ] || [ -z "$TOKEN_AUTH_KID" ]; then - usage - exit 1 -fi +b64encode_str() { + echo -n "$@" | base64 --wrap=0 - 2>/dev/null || echo -n "$@" | base64 --break=0 - +} cat < Creating new project at: $PROJECT_DIR" mkdir -p "$PROJECT_DIR" "$CERTS_DIR" -echo_bold "==> Creating root CA cert..." -"${DIR}/gen-root-ca-cert" $HOST_NAME "$CERTS_DIR" +echo_bold "==> Generating root CA cert..." +source "${DIR}/gen-root-ca" "${HOST_NAME}" "${CERTS_DIR}" + +echo_bold "==> Generating root cert chain for haproxy..." +source "${DIR}/gen-root-cert" "${HOST_NAME}" "${CERTS_DIR}" + +echo_bold "==> Generating token auth cert..." +source "${DIR}/gen-token-auth-cert" "${HOST_NAME}" "${CERTS_DIR}" -echo_bold "==> Creating token auth cert..." -"${DIR}/gen-token-auth-cert" $HOST_NAME "$CERTS_DIR" +echo_bold "==> Generating VPN CA, cert and dhparam (this may take a while)..." +source "${DIR}/gen-vpn-certs" "${HOST_NAME}" "${CERTS_DIR}" echo_bold "==> Setting up environment..." -cat >"${PROJECT_DIR}/activate" <"${PROJECT_DIR}/activate" <(source "${DIR}/make-env") echo_bold "==> Adding default compose file..." cp "${BASE_DIR}/compose/template.yml" "${PROJECT_DIR}/docker-compose.yml" @@ -56,4 +53,4 @@ echo_bold "==> Patching /etc/hosts..." "${DIR}/patch-hosts" $HOST_NAME echo_bold "==> Activating project..." -"${DIR}/select-project" "$PROJECT_DIR" +"${DIR}/select-project" "${PROJECT_DIR}" \ No newline at end of file From 08e2168cc04e878948357cb210a59b6bb1fba3b4 Mon Sep 17 00:00:00 2001 From: Will Boyce Date: Thu, 27 Sep 2018 18:49:16 +0100 Subject: [PATCH 2/4] start-project: Use getopts in place of positional args Signed-off-by: Will Boyce --- Vagrantfile | 3 ++- scripts/make-env | 6 +++--- scripts/patch-hosts | 10 +++++----- scripts/run-fig-command | 4 ++-- scripts/start-project | 41 ++++++++++++++++++++++++++++++----------- 5 files changed, 42 insertions(+), 22 deletions(-) diff --git a/Vagrantfile b/Vagrantfile index 870850198..d690ede3b 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -27,7 +27,8 @@ Vagrant.configure('2') do |config| config.vm.provision :shell, inline: "docker login --username resindev --password #{ENV.fetch('DOCKERHUB_PASSWORD')}" config.vm.provision :shell, privileged: false, - inline: "cd /home/vagrant/open-balena && ./scripts/start-project #{ENV.fetch('OPENBALENA_PROJECT_NAME', '')} #{ENV.fetch('OPENBALENA_HOST_NAME', '')}" + # FIXME: -n/-d should only be passed if the relevant ENV var is set + inline: "cd /home/vagrant/open-balena && ./scripts/start-project -p -n #{ENV.fetch('OPENBALENA_PROJECT_NAME', 'demo')} -d #{ENV.fetch('OPENBALENA_DOMAIN', 'openbalena.local')}" config.vm.provision :shell, privileged: false, inline: 'cd /home/vagrant/open-balena && ./scripts/run-fig-command up -d || true', run: 'always' diff --git a/scripts/make-env b/scripts/make-env index 9de2c2dd6..83a57194e 100755 --- a/scripts/make-env +++ b/scripts/make-env @@ -5,7 +5,7 @@ usage() { echo echo "Required Variables:" echo - echo " HOST_NAME" + echo " DOMAIN" echo " ROOT_CA Path to root CA certificate" echo " ROOT_CRT Path to root/wildcard certificate" echo " ROOT_KEY Path to root/wildcard private key" @@ -19,7 +19,7 @@ usage() { echo } -for var in HOST_NAME ROOT_CA ROOT_CRT ROOT_KEY JWT_CRT JWT_KEY JWT_KID VPN_CA VPN_CRT VPN_KEY VPN_DH; do +for var in DOMAIN ROOT_CA ROOT_CRT ROOT_KEY JWT_CRT JWT_KEY JWT_KID VPN_CA VPN_CRT VPN_KEY VPN_DH; do if [ -z "${!var-}" ]; then usage exit 1 @@ -41,7 +41,7 @@ b64encode_str() { cat </dev/null 2>&1 ; then echo "adding $name" echo "127.0.0.1 $name" >>$tmp diff --git a/scripts/run-fig-command b/scripts/run-fig-command index 3332ca867..fa8ec1528 100755 --- a/scripts/run-fig-command +++ b/scripts/run-fig-command @@ -7,7 +7,7 @@ DIR=$(dirname "$CMD") BASE_DIR=$(dirname "$DIR") echo_bold() { - printf "\033[1m${@}\033[0m\n" + printf "\033[1m%s\033[0m\n" "$@" } PROJECT_FILE="${BASE_DIR}/.project" @@ -28,4 +28,4 @@ PROJECT_NAME=$(basename "$PROJECT") --project-name $PROJECT_NAME \ -f "${BASE_DIR}/compose/services.yml" \ -f "${PROJECT}/docker-compose.yml" \ - "$@" + "$@" \ No newline at end of file diff --git a/scripts/start-project b/scripts/start-project index 1a4991ba6..b05089c9d 100755 --- a/scripts/start-project +++ b/scripts/start-project @@ -4,20 +4,38 @@ CMD=$0 DIR=$(dirname "$CMD") BASE_DIR=$(dirname "$DIR") -PROJECT_NAME=${1:-demo} -HOST_NAME=${2:-openbalena.local} +PROJECT_NAME=demo +DOMAIN=openbalena.local + +show_help=false +patch_hosts=false +while getopts ":hpn:d:" opt; do + case "${opt}" in + h) show_help=true;; + p) patch_hosts=true;; + P) PROJECT_NAME="${OPTARG}";; + H) DOMAIN="${OPTARG}";; + esac +done +shift $((OPTIND-1)) PROJECT_DIR="$(pwd)/${PROJECT_NAME}" CERTS_DIR="${PROJECT_DIR}/certs" usage() { - echo "usage: $0 [PROJECT_NAME [HOST_NAME]]" + echo "usage: $0 [-h] [-p] [-n PROJECT_NAME] [-d DOMAIN]" echo + echo " -p patch hosts - patch the host /etc/hosts file" echo " PROJECT_NAME a name for the deployment, eg. staging. Default is 'demo'" - echo " HOST_NAME the domain name this deployment will run as, eg. example.com. Default is 'openbalena.local'" + echo " DOMAIN the domain name this deployment will run as, eg. example.com. Default is 'openbalena.local'" echo } +if [ "$show_help" = "true" ]; then + usage + exit 1 +fi + echo_bold() { printf "\033[1m%s\033[0m\n" "${@}" } @@ -31,16 +49,16 @@ echo_bold "==> Creating new project at: $PROJECT_DIR" mkdir -p "$PROJECT_DIR" "$CERTS_DIR" echo_bold "==> Generating root CA cert..." -source "${DIR}/gen-root-ca" "${HOST_NAME}" "${CERTS_DIR}" +source "${DIR}/gen-root-ca" "${DOMAIN}" "${CERTS_DIR}" echo_bold "==> Generating root cert chain for haproxy..." -source "${DIR}/gen-root-cert" "${HOST_NAME}" "${CERTS_DIR}" +source "${DIR}/gen-root-cert" "${DOMAIN}" "${CERTS_DIR}" echo_bold "==> Generating token auth cert..." -source "${DIR}/gen-token-auth-cert" "${HOST_NAME}" "${CERTS_DIR}" +source "${DIR}/gen-token-auth-cert" "${DOMAIN}" "${CERTS_DIR}" echo_bold "==> Generating VPN CA, cert and dhparam (this may take a while)..." -source "${DIR}/gen-vpn-certs" "${HOST_NAME}" "${CERTS_DIR}" +source "${DIR}/gen-vpn-certs" "${DOMAIN}" "${CERTS_DIR}" echo_bold "==> Setting up environment..." cat >"${PROJECT_DIR}/activate" <(source "${DIR}/make-env") @@ -48,9 +66,10 @@ cat >"${PROJECT_DIR}/activate" <(source "${DIR}/make-env") echo_bold "==> Adding default compose file..." cp "${BASE_DIR}/compose/template.yml" "${PROJECT_DIR}/docker-compose.yml" -# FIXME: should be explicitly requested via a flag -echo_bold "==> Patching /etc/hosts..." -"${DIR}/patch-hosts" $HOST_NAME +if [ "${patch_hosts}" = "true" ]; then + echo_bold "==> Patching /etc/hosts..." + source "${DIR}/patch-hosts" "${DOMAIN}" +fi echo_bold "==> Activating project..." "${DIR}/select-project" "${PROJECT_DIR}" \ No newline at end of file From 36344342649a63901240758e402f82db79d951d5 Mon Sep 17 00:00:00 2001 From: Will Boyce Date: Thu, 27 Sep 2018 19:26:37 +0100 Subject: [PATCH 3/4] vagrant: Run docker login provision step as vagrant user Signed-off-by: Will Boyce --- Vagrantfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Vagrantfile b/Vagrantfile index d690ede3b..e1415a251 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -24,7 +24,7 @@ Vagrant.configure('2') do |config| config.vm.provision :shell, inline: 'apt-get update && apt-get install -y nodejs && rm -rf /var/lib/apt/lists/*' # FIXME: remove `docker login` - config.vm.provision :shell, inline: "docker login --username resindev --password #{ENV.fetch('DOCKERHUB_PASSWORD')}" + config.vm.provision :shell, privileged: false, inline: "docker login --username resindev --password #{ENV.fetch('DOCKERHUB_PASSWORD')}" config.vm.provision :shell, privileged: false, # FIXME: -n/-d should only be passed if the relevant ENV var is set From 5d498720bf19fbaa165b9904c6f914161382afa3 Mon Sep 17 00:00:00 2001 From: Will Boyce Date: Thu, 27 Sep 2018 20:08:46 +0100 Subject: [PATCH 4/4] scripts: Lint fixes, add Makefile for linting Signed-off-by: Will Boyce --- Makefile | 4 ++++ scripts/gen-root-ca | 1 + scripts/gen-root-cert | 1 + scripts/gen-token-auth-cert | 4 +++- scripts/gen-vpn-certs | 1 + scripts/patch-hosts | 24 +++++++++------------- scripts/run-fig-command | 21 +++++++++---------- scripts/select-project | 20 ++++++++---------- scripts/ssl-common.sh | 2 ++ scripts/start-project | 41 +++++++++++++++++++++++-------------- 10 files changed, 67 insertions(+), 52 deletions(-) create mode 100644 Makefile diff --git a/Makefile b/Makefile new file mode 100644 index 000000000..725f06f54 --- /dev/null +++ b/Makefile @@ -0,0 +1,4 @@ +.PHONY: lint + +lint: + shellcheck scripts/* diff --git a/scripts/gen-root-ca b/scripts/gen-root-ca index fba233bba..ecafe798e 100755 --- a/scripts/gen-root-ca +++ b/scripts/gen-root-ca @@ -19,6 +19,7 @@ DIR="$(dirname "${CMD}")" CN="$1" OUT="$(realpath "${2:-.}")" +# shellcheck source=scripts/ssl-common.sh source "${DIR}/ssl-common.sh" # Create a secret key and CA file for the self-signed CA diff --git a/scripts/gen-root-cert b/scripts/gen-root-cert index f310a60c7..0377d722d 100755 --- a/scripts/gen-root-cert +++ b/scripts/gen-root-cert @@ -19,6 +19,7 @@ DIR="$(dirname "${CMD}")" CN="$1" OUT="$(realpath "${2:-.}")" +# shellcheck source=scripts/ssl-common.sh source "${DIR}/ssl-common.sh" # generate default CSR and sign (root + wildcard) diff --git a/scripts/gen-token-auth-cert b/scripts/gen-token-auth-cert index 60ae2a943..48cbf60d3 100755 --- a/scripts/gen-token-auth-cert +++ b/scripts/gen-token-auth-cert @@ -19,10 +19,12 @@ DIR="$(dirname "${CMD}")" CN="$1" OUT="$(realpath "${2:-.}")" +# shellcheck source=scripts/ssl-common.sh source "${DIR}/ssl-common.sh" keyid() { - local der="$(openssl ec -in "$1" -pubout -outform DER 2>/dev/null)" + local der + der="$(openssl ec -in "$1" -pubout -outform DER 2>/dev/null)" python -c "import sys as S; from base64 import b32encode as B; import hashlib as H; h = H.sha256(); h.update(S.argv[1].encode('ascii')); s = B(h.digest()[:30]).decode('ascii'); S.stdout.write(':'.join([s[i:i+4] for i in range(0, len(s), 4)]))" "${der}" } diff --git a/scripts/gen-vpn-certs b/scripts/gen-vpn-certs index af217358b..df755f47c 100755 --- a/scripts/gen-vpn-certs +++ b/scripts/gen-vpn-certs @@ -19,6 +19,7 @@ DIR="$(dirname "${CMD}")" CN="$1" OUT="$(realpath "${2:-.}")" +# shellcheck source=scripts/ssl-common.sh source "${DIR}/ssl-common.sh" VPN_PKI="$(realpath "${OUT}/vpn")" diff --git a/scripts/patch-hosts b/scripts/patch-hosts index b79e6a6fd..8b03cc58c 100755 --- a/scripts/patch-hosts +++ b/scripts/patch-hosts @@ -1,11 +1,4 @@ -#!/bin/sh - -set -e - -CMD=$0 -DIR=$(dirname "$CMD") - -DOMAIN=$1 +#!/bin/bash -eu SERVICES="api registry vpn db" SERVICES="${SERVICES} img devices" # FIXME: remove @@ -17,20 +10,23 @@ usage() { echo } -if [ -z "$DOMAIN" ]; then +if [ -z "$1" ]; then usage exit 1 fi +DOMAIN="$1" + # We need sudo to write to /etc/hosts, so first write to a temp file and then # append all entries to hosts file. -tmp=$(mktemp --tmpdir openbalena.XXXX) +tmp="$(mktemp --tmpdir openbalena.XXXX)" for service in $SERVICES; do name="${service}.${DOMAIN}" - if ! grep "\s$name" /etc/hosts >/dev/null 2>&1 ; then + if ! grep "\\s$name" /etc/hosts >/dev/null 2>&1 ; then echo "adding $name" - echo "127.0.0.1 $name" >>$tmp + echo "127.0.0.1 $name" >>"${tmp}" fi done -cat $tmp | sudo tee -a /etc/hosts >/dev/null -rm -f $tmp +# shellcheck disable=SC2024 +sudo tee -a /etc/hosts >/dev/null <"${tmp}" +rm -f "${tmp}" diff --git a/scripts/run-fig-command b/scripts/run-fig-command index fa8ec1528..edf8446a8 100755 --- a/scripts/run-fig-command +++ b/scripts/run-fig-command @@ -1,13 +1,11 @@ -#!/bin/sh +#!/bin/bash -eu -set -e - -CMD=$0 -DIR=$(dirname "$CMD") -BASE_DIR=$(dirname "$DIR") +CMD="$0" +DIR="$(dirname "$CMD")" +BASE_DIR="$(dirname "$DIR")" echo_bold() { - printf "\033[1m%s\033[0m\n" "$@" + printf "\\033[1m%s\\033[0m\\n" "$@" } PROJECT_FILE="${BASE_DIR}/.project" @@ -16,16 +14,17 @@ if [ ! -f "$PROJECT_FILE" ]; then echo_bold 'See README.md for help.' exit 1 fi -PROJECT=$(cat "$PROJECT_FILE") +PROJECT="$(cat "$PROJECT_FILE")" if [ ! -f "${PROJECT}/activate" ]; then echo_bold 'No project activated. Please create or select an existing one first.' echo_bold 'See README.md for help.' exit 1 fi -PROJECT_NAME=$(basename "$PROJECT") +PROJECT_NAME="$(basename "$PROJECT")" -. "${PROJECT}/activate"; docker-compose \ - --project-name $PROJECT_NAME \ +# shellcheck source=/dev/null +source "${PROJECT}/activate"; docker-compose \ + --project-name "${PROJECT_NAME}" \ -f "${BASE_DIR}/compose/services.yml" \ -f "${PROJECT}/docker-compose.yml" \ "$@" \ No newline at end of file diff --git a/scripts/select-project b/scripts/select-project index 451f1193a..e582b2a85 100755 --- a/scripts/select-project +++ b/scripts/select-project @@ -1,11 +1,8 @@ -#!/bin/sh +#!/bin/bash -eu -set -e - -CMD=$0 -DIR=$(dirname "$CMD") -BASE_DIR=$(dirname "$DIR") -PROJECT_PATH="$1" +CMD="$0" +DIR="$(dirname "$CMD")" +BASE_DIR="$(dirname "$DIR")" usage() { echo "usage: $0 PROJECT_PATH" @@ -14,12 +11,13 @@ usage() { echo } -if [ -z "$PROJECT_PATH" ]; then +if [ -z "$1" ]; then usage exit 1 fi -PROJECT_DIR=$(realpath "$PROJECT_PATH") +PROJECT_PATH="$1" +PROJECT_DIR="$(realpath "$PROJECT_PATH")" if [ ! -d "$PROJECT_DIR" ]; then echo 'Project path refers to a directory that does not exist.' @@ -27,8 +25,8 @@ if [ ! -d "$PROJECT_DIR" ]; then fi if [ ! -f "${PROJECT_DIR}/activate" ]; then - echo 'Project path refers to a directory that is not a valid porject.' + echo 'Project path refers to a directory that is not a valid project.' exit 1 fi -echo -n $PROJECT_DIR >"${BASE_DIR}/.project" +echo -n "${PROJECT_DIR}" >"${BASE_DIR}/.project" diff --git a/scripts/ssl-common.sh b/scripts/ssl-common.sh index 7f4479ebe..0f6a690e0 100644 --- a/scripts/ssl-common.sh +++ b/scripts/ssl-common.sh @@ -1,4 +1,5 @@ #!/bin/bash -eu +# shellcheck disable=SC2034 # ensure we have `easyrsa` available if [ -z "${easyrsa_bin-}" ] || [ ! -x "${easyrsa_bin}" ]; then @@ -8,6 +9,7 @@ if [ -z "${easyrsa_bin-}" ] || [ ! -x "${easyrsa_bin}" ]; then easyrsa_url="https://github.com/OpenVPN/easy-rsa/releases/download/v3.0.5/EasyRSA-nix-3.0.5.tgz" (cd "${easyrsa_dir}"; curl -sL "${easyrsa_url}" | tar xz --strip-components=1) easyrsa_bin="${easyrsa_dir}/easyrsa" + # shellcheck disable=SC2064 trap "rm -rf \"${easyrsa_dir}\"" EXIT fi export EASYRSA_BATCH=1 diff --git a/scripts/start-project b/scripts/start-project index b05089c9d..2a0ef4517 100755 --- a/scripts/start-project +++ b/scripts/start-project @@ -7,14 +7,28 @@ BASE_DIR=$(dirname "$DIR") PROJECT_NAME=demo DOMAIN=openbalena.local +usage() { + echo "usage: $0 [-h] [-p] [-n PROJECT_NAME] [-d DOMAIN]" + echo + echo " -p patch hosts - patch the host /etc/hosts file" + echo " PROJECT_NAME a name for the deployment, eg. staging. Default is 'demo'" + echo " DOMAIN the domain name this deployment will run as, eg. example.com. Default is 'openbalena.local'" + echo +} + show_help=false patch_hosts=false while getopts ":hpn:d:" opt; do case "${opt}" in h) show_help=true;; p) patch_hosts=true;; - P) PROJECT_NAME="${OPTARG}";; - H) DOMAIN="${OPTARG}";; + n) PROJECT_NAME="${OPTARG}";; + d) DOMAIN="${OPTARG}";; + *) + echo "Invalid argument: -${OPTARG}" + usage + exit 1 + ;; esac done shift $((OPTIND-1)) @@ -22,22 +36,13 @@ shift $((OPTIND-1)) PROJECT_DIR="$(pwd)/${PROJECT_NAME}" CERTS_DIR="${PROJECT_DIR}/certs" -usage() { - echo "usage: $0 [-h] [-p] [-n PROJECT_NAME] [-d DOMAIN]" - echo - echo " -p patch hosts - patch the host /etc/hosts file" - echo " PROJECT_NAME a name for the deployment, eg. staging. Default is 'demo'" - echo " DOMAIN the domain name this deployment will run as, eg. example.com. Default is 'openbalena.local'" - echo -} - if [ "$show_help" = "true" ]; then usage exit 1 fi echo_bold() { - printf "\033[1m%s\033[0m\n" "${@}" + printf "\\033[1m%s\\033[0m\\n" "${@}" } if [ -d "$PROJECT_DIR" ]; then @@ -49,27 +54,33 @@ echo_bold "==> Creating new project at: $PROJECT_DIR" mkdir -p "$PROJECT_DIR" "$CERTS_DIR" echo_bold "==> Generating root CA cert..." +# shellcheck source=scripts/gen-root-ca source "${DIR}/gen-root-ca" "${DOMAIN}" "${CERTS_DIR}" echo_bold "==> Generating root cert chain for haproxy..." +# shellcheck source=scripts/gen-root-cert source "${DIR}/gen-root-cert" "${DOMAIN}" "${CERTS_DIR}" echo_bold "==> Generating token auth cert..." +# shellcheck source=scripts/gen-token-auth-cert source "${DIR}/gen-token-auth-cert" "${DOMAIN}" "${CERTS_DIR}" echo_bold "==> Generating VPN CA, cert and dhparam (this may take a while)..." +# shellcheck source=scripts/gen-vpn-certs source "${DIR}/gen-vpn-certs" "${DOMAIN}" "${CERTS_DIR}" echo_bold "==> Setting up environment..." +# shellcheck source=scripts/make-env cat >"${PROJECT_DIR}/activate" <(source "${DIR}/make-env") echo_bold "==> Adding default compose file..." cp "${BASE_DIR}/compose/template.yml" "${PROJECT_DIR}/docker-compose.yml" if [ "${patch_hosts}" = "true" ]; then - echo_bold "==> Patching /etc/hosts..." - source "${DIR}/patch-hosts" "${DOMAIN}" + echo_bold "==> Patching /etc/hosts..." + # shellcheck source=scripts/patch-hosts + source "${DIR}/patch-hosts" "${DOMAIN}" fi echo_bold "==> Activating project..." -"${DIR}/select-project" "${PROJECT_DIR}" \ No newline at end of file +"${DIR}/select-project" "${PROJECT_DIR}"