From 7844e7aa2685994011100cbc49624674777bb31d Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Wed, 24 Apr 2019 14:12:17 +0300 Subject: [PATCH 1/5] Retry SSH keyscan command Prevent CircleCI build from failing due to transient network errors --- Makefile | 4 ++-- docker/Dockerfile.flux | 7 ++++--- docker/Dockerfile.helm-operator | 7 ++++--- docker/known_hosts.sh | 20 ++++++++++++++++++++ 4 files changed, 30 insertions(+), 8 deletions(-) create mode 100755 docker/known_hosts.sh diff --git a/Makefile b/Makefile index 04723e65f..788134e4e 100644 --- a/Makefile +++ b/Makefile @@ -69,8 +69,8 @@ build/.%.done: docker/Dockerfile.% -f build/docker/$*/Dockerfile.$* ./build/docker/$* touch $@ -build/.flux.done: build/fluxd build/kubectl docker/ssh_config docker/kubeconfig docker/verify_known_hosts.sh -build/.helm-operator.done: build/helm-operator build/kubectl build/helm docker/ssh_config docker/verify_known_hosts.sh docker/helm-repositories.yaml +build/.flux.done: build/fluxd build/kubectl docker/ssh_config docker/kubeconfig docker/verify_known_hosts.sh docker/known_hosts.sh +build/.helm-operator.done: build/helm-operator build/kubectl build/helm docker/ssh_config docker/verify_known_hosts.sh docker/known_hosts.sh docker/helm-repositories.yaml build/fluxd: $(FLUXD_DEPS) build/fluxd: cmd/fluxd/*.go diff --git a/docker/Dockerfile.flux b/docker/Dockerfile.flux index 2aa23fcca..6414b4919 100644 --- a/docker/Dockerfile.flux +++ b/docker/Dockerfile.flux @@ -7,9 +7,10 @@ RUN apk add --no-cache openssh ca-certificates tini 'git>=2.3.0' gnupg # Add git hosts to known hosts file so we can use # StrickHostKeyChecking with git+ssh ADD ./verify_known_hosts.sh /home/flux/verify_known_hosts.sh -RUN ssh-keyscan github.com gitlab.com bitbucket.org ssh.dev.azure.com vs-ssh.visualstudio.com >> /etc/ssh/ssh_known_hosts && \ - sh /home/flux/verify_known_hosts.sh /etc/ssh/ssh_known_hosts && \ - rm /home/flux/verify_known_hosts.sh +ADD ./known_hosts.sh /home/flux/known_hosts.sh +RUN sh /home/flux/known_hosts.sh /etc/ssh/ssh_known_hosts && \ + rm /home/flux/verify_known_hosts.sh && \ + rm /home/flux/known_hosts.sh # Add default SSH config, which points at the private key we'll mount COPY ./ssh_config /etc/ssh/ssh_config diff --git a/docker/Dockerfile.helm-operator b/docker/Dockerfile.helm-operator index cf3dd0c2d..a8a9be70a 100644 --- a/docker/Dockerfile.helm-operator +++ b/docker/Dockerfile.helm-operator @@ -7,9 +7,10 @@ RUN apk add --no-cache openssh ca-certificates tini 'git>=2.3.0' # Add git hosts to known hosts file so we can use # StrickHostKeyChecking with git+ssh ADD ./verify_known_hosts.sh /home/flux/verify_known_hosts.sh -RUN ssh-keyscan github.com gitlab.com bitbucket.org ssh.dev.azure.com vs-ssh.visualstudio.com >> /etc/ssh/ssh_known_hosts && \ - sh /home/flux/verify_known_hosts.sh /etc/ssh/ssh_known_hosts && \ - rm /home/flux/verify_known_hosts.sh +ADD ./known_hosts.sh /home/flux/known_hosts.sh +RUN sh /home/flux/known_hosts.sh /etc/ssh/ssh_known_hosts && \ + rm /home/flux/verify_known_hosts.sh && \ + rm /home/flux/known_hosts.sh # Add default SSH config, which points at the private key we'll mount COPY ./ssh_config /etc/ssh/ssh_config diff --git a/docker/known_hosts.sh b/docker/known_hosts.sh new file mode 100755 index 000000000..cdcf2d132 --- /dev/null +++ b/docker/known_hosts.sh @@ -0,0 +1,20 @@ +#!/bin/sh + +set -eu + +known_hosts_file=${1} +known_hosts_file=${known_hosts_file:-/etc/ssh/ssh_known_hosts} + +retries=10 +count=0 +ok=false +until ${ok}; do + ssh-keyscan github.com gitlab.com bitbucket.org ssh.dev.azure.com vs-ssh.visualstudio.com >> ${known_hosts_file} && \ + sh /home/flux/verify_known_hosts.sh ${known_hosts_file} && ok=true || ok=false + sleep 2 + count=$(($count + 1)) + if [[ ${count} -eq ${retries} ]]; then + echo "No more retries left" + exit 1 + fi +done From 4b44ef9870eeccab000dd07fccab11b893a63710 Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Wed, 24 Apr 2019 14:34:01 +0300 Subject: [PATCH 2/5] Combine SSH generate and verify known hosts --- Makefile | 4 +-- docker/Dockerfile.flux | 2 -- docker/Dockerfile.helm-operator | 2 -- docker/known_hosts.sh | 44 ++++++++++++++++++++++++++++++--- docker/verify_known_hosts.sh | 41 ------------------------------ 5 files changed, 42 insertions(+), 51 deletions(-) delete mode 100755 docker/verify_known_hosts.sh diff --git a/Makefile b/Makefile index 788134e4e..2e6346c7f 100644 --- a/Makefile +++ b/Makefile @@ -69,8 +69,8 @@ build/.%.done: docker/Dockerfile.% -f build/docker/$*/Dockerfile.$* ./build/docker/$* touch $@ -build/.flux.done: build/fluxd build/kubectl docker/ssh_config docker/kubeconfig docker/verify_known_hosts.sh docker/known_hosts.sh -build/.helm-operator.done: build/helm-operator build/kubectl build/helm docker/ssh_config docker/verify_known_hosts.sh docker/known_hosts.sh docker/helm-repositories.yaml +build/.flux.done: build/fluxd build/kubectl docker/ssh_config docker/kubeconfig docker/known_hosts.sh +build/.helm-operator.done: build/helm-operator build/kubectl build/helm docker/ssh_config docker/known_hosts.sh docker/helm-repositories.yaml build/fluxd: $(FLUXD_DEPS) build/fluxd: cmd/fluxd/*.go diff --git a/docker/Dockerfile.flux b/docker/Dockerfile.flux index 6414b4919..4db8bfbce 100644 --- a/docker/Dockerfile.flux +++ b/docker/Dockerfile.flux @@ -6,10 +6,8 @@ RUN apk add --no-cache openssh ca-certificates tini 'git>=2.3.0' gnupg # Add git hosts to known hosts file so we can use # StrickHostKeyChecking with git+ssh -ADD ./verify_known_hosts.sh /home/flux/verify_known_hosts.sh ADD ./known_hosts.sh /home/flux/known_hosts.sh RUN sh /home/flux/known_hosts.sh /etc/ssh/ssh_known_hosts && \ - rm /home/flux/verify_known_hosts.sh && \ rm /home/flux/known_hosts.sh # Add default SSH config, which points at the private key we'll mount diff --git a/docker/Dockerfile.helm-operator b/docker/Dockerfile.helm-operator index a8a9be70a..fca75fc0a 100644 --- a/docker/Dockerfile.helm-operator +++ b/docker/Dockerfile.helm-operator @@ -6,10 +6,8 @@ RUN apk add --no-cache openssh ca-certificates tini 'git>=2.3.0' # Add git hosts to known hosts file so we can use # StrickHostKeyChecking with git+ssh -ADD ./verify_known_hosts.sh /home/flux/verify_known_hosts.sh ADD ./known_hosts.sh /home/flux/known_hosts.sh RUN sh /home/flux/known_hosts.sh /etc/ssh/ssh_known_hosts && \ - rm /home/flux/verify_known_hosts.sh && \ rm /home/flux/known_hosts.sh # Add default SSH config, which points at the private key we'll mount diff --git a/docker/known_hosts.sh b/docker/known_hosts.sh index cdcf2d132..7356a862e 100755 --- a/docker/known_hosts.sh +++ b/docker/known_hosts.sh @@ -4,17 +4,53 @@ set -eu known_hosts_file=${1} known_hosts_file=${known_hosts_file:-/etc/ssh/ssh_known_hosts} +hosts="github.com gitlab.com bitbucket.org ssh.dev.azure.com vs-ssh.visualstudio.com" retries=10 count=0 ok=false +wait=2 until ${ok}; do - ssh-keyscan github.com gitlab.com bitbucket.org ssh.dev.azure.com vs-ssh.visualstudio.com >> ${known_hosts_file} && \ - sh /home/flux/verify_known_hosts.sh ${known_hosts_file} && ok=true || ok=false - sleep 2 + ssh-keyscan ${hosts} > ${known_hosts_file} && ok=true || ok=false + sleep ${wait} count=$(($count + 1)) if [[ ${count} -eq ${retries} ]]; then - echo "No more retries left" + echo "ssh-keyscan failed, no more retries left" exit 1 fi done + +# The heredoc below was generated by constructing a known_hosts using +# +# ssh-keyscan github.com gitlab.com bitbucket.org ssh.dev.azure.com vs-ssh.visualstudio.com > ./known_hosts +# +# then generating the sorted fingerprints with +# +# ssh-keygen -l -f ./known_hosts | LC_ALL=C sort +# +# then checking against the published fingerprints from: +# - github.com: https://help.github.com/articles/github-s-ssh-key-fingerprints/ +# - gitlab.com: https://docs.gitlab.com/ee/user/gitlab_com/#ssh-host-keys-fingerprints +# - bitbucket.org: https://confluence.atlassian.com/bitbucket/ssh-keys-935365775.html +# - ssh.dev.azure.com & vs-ssh.visualstudio.com: sign in, then go to User settings -> SSH Public Keys +# (this is where the public key fingerprint is shown; it's not a setting) + +fingerprints=$(mktemp -t) +cleanup() { + rm -f "$fingerprints" +} +trap cleanup EXIT + +# make sure sorting is in the same locale as the heredoc +export LC_ALL=C +ssh-keygen -l -f ${known_hosts_file} | sort > "$fingerprints" + +diff - "$fingerprints" < ./known_hosts -# -# then generating the sorted fingerprints with -# -# ssh-keygen -l -f ./known_hosts | LC_ALL=C sort -# -# then checking against the published fingerprints from: -# - github.com: https://help.github.com/articles/github-s-ssh-key-fingerprints/ -# - gitlab.com: https://docs.gitlab.com/ee/user/gitlab_com/#ssh-host-keys-fingerprints -# - bitbucket.org: https://confluence.atlassian.com/bitbucket/ssh-keys-935365775.html -# - ssh.dev.azure.com & vs-ssh.visualstudio.com: sign in, then go to User settings -> SSH Public Keys -# (this is where the public key fingerprint is shown; it's not a setting) - -fingerprints=$(mktemp -t) -cleanup() { - rm -f "$fingerprints" -} -trap cleanup EXIT - -# make sure sorting is in the same locale as the heredoc -export LC_ALL=C -ssh-keygen -l -f ${known_hosts_file} | sort > "$fingerprints" - -diff - "$fingerprints" < Date: Wed, 24 Apr 2019 15:32:17 +0300 Subject: [PATCH 3/5] Retry known hosts validation --- docker/known_hosts.sh | 36 ++++++++++++++++++++++-------------- 1 file changed, 22 insertions(+), 14 deletions(-) diff --git a/docker/known_hosts.sh b/docker/known_hosts.sh index 7356a862e..33728c950 100755 --- a/docker/known_hosts.sh +++ b/docker/known_hosts.sh @@ -6,20 +6,6 @@ known_hosts_file=${1} known_hosts_file=${known_hosts_file:-/etc/ssh/ssh_known_hosts} hosts="github.com gitlab.com bitbucket.org ssh.dev.azure.com vs-ssh.visualstudio.com" -retries=10 -count=0 -ok=false -wait=2 -until ${ok}; do - ssh-keyscan ${hosts} > ${known_hosts_file} && ok=true || ok=false - sleep ${wait} - count=$(($count + 1)) - if [[ ${count} -eq ${retries} ]]; then - echo "ssh-keyscan failed, no more retries left" - exit 1 - fi -done - # The heredoc below was generated by constructing a known_hosts using # # ssh-keyscan github.com gitlab.com bitbucket.org ssh.dev.azure.com vs-ssh.visualstudio.com > ./known_hosts @@ -43,6 +29,12 @@ trap cleanup EXIT # make sure sorting is in the same locale as the heredoc export LC_ALL=C + +generate() { + ssh-keyscan ${hosts} > ${known_hosts_file} +} + +validate() { ssh-keygen -l -f ${known_hosts_file} | sort > "$fingerprints" diff - "$fingerprints" < Date: Wed, 24 Apr 2019 16:05:28 +0300 Subject: [PATCH 4/5] Exit without waiting on ssh-keyscan max retry Co-Authored-By: stefanprodan --- docker/known_hosts.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docker/known_hosts.sh b/docker/known_hosts.sh index 33728c950..a8421a005 100755 --- a/docker/known_hosts.sh +++ b/docker/known_hosts.sh @@ -55,6 +55,11 @@ ok=false wait=2 until ${ok}; do generate && validate && ok=true || ok=false + count=$(($count + 1)) + if [[ ${count} -eq ${retries} ]]; then + echo "ssh-keyscan failed, no more retries left" + exit 1 + fi sleep ${wait} count=$(($count + 1)) if [[ ${count} -eq ${retries} ]]; then From 63b8c9262a9b3322a05e98070991c86e0c0f6bdd Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Wed, 24 Apr 2019 16:08:37 +0300 Subject: [PATCH 5/5] Remove duplicate code --- docker/known_hosts.sh | 5 ----- 1 file changed, 5 deletions(-) diff --git a/docker/known_hosts.sh b/docker/known_hosts.sh index a8421a005..379f6a9d3 100755 --- a/docker/known_hosts.sh +++ b/docker/known_hosts.sh @@ -61,9 +61,4 @@ until ${ok}; do exit 1 fi sleep ${wait} - count=$(($count + 1)) - if [[ ${count} -eq ${retries} ]]; then - echo "ssh-keyscan failed, no more retries left" - exit 1 - fi done