Merge pull request #121 from open-quantum-safe/v_8_9_upgrade

V_8_9_P1 upgrade

.github/ci-status.md

@@ -0,0 +1,4 @@
+[![C/C++ CI](https://github.com/openssh/openssh-portable/actions/workflows/c-cpp.yml/badge.svg)](https://github.com/openssh/openssh-portable/actions/workflows/c-cpp.yml)
+[![C/C++ CI self-hosted](https://github.com/openssh/openssh-portable-selfhosted/actions/workflows/selfhosted.yml/badge.svg)](https://github.com/openssh/openssh-portable-selfhosted/actions/workflows/selfhosted.yml)
+[![Upstream self-hosted](https://github.com/openssh/openssh-portable-selfhosted/actions/workflows/upstream.yml/badge.svg)](https://github.com/openssh/openssh-portable-selfhosted/actions/workflows/upstream.yml)
+[![Fuzzing Status](https://oss-fuzz-build-logs.storage.googleapis.com/badges/openssh.svg)](https://bugs.chromium.org/p/oss-fuzz/issues/list?sort=-opened&can=1&q=proj:openssh)

.github/configs

@@ -15,13 +15,41 @@ LTESTS=""
 SKIP_LTESTS=""
 SUDO=sudo	# run with sudo by default
 TEST_SSH_UNSAFE_PERMISSIONS=1
+# Stop on first test failure to minimize logs
+TEST_SSH_FAIL_FATAL=yes
 
 CONFIGFLAGS=""
 LIBCRYPTOFLAGS=""
 
 case "$config" in
     default|sol64)
 	;;
+    c89)
+	CC="gcc"
+	CFLAGS="-Wall -std=c89 -pedantic -Werror=vla"
+	CONFIGFLAGS="--without-zlib"
+	LIBCRYPTOFLAGS="--without-openssl"
+	TEST_TARGET=t-exec
+	;;
+    cygwin-release)
+	CONFIGFLAGS="--with-libedit --with-xauth=/usr/bin/xauth --disable-strip --with-security-key-builtin"
+	;;
+   clang-12-Werror)
+	CC="clang-12"
+	# clang's implicit-fallthrough requires that the code be annotated with
+	# __attribute__((fallthrough)) and does not understand /* FALLTHROUGH */
+	CFLAGS="-Wall -Wextra -O2 -Wno-error=implicit-fallthrough"
+	CONFIGFLAGS="--with-pam --with-Werror"
+	;;
+    gcc-11-Werror)
+	CC="gcc"
+	# -Wnoformat-truncation in gcc 7.3.1 20180130 fails on fmt_scaled
+	CFLAGS="-Wall -Wextra -Wno-format-truncation -O2 -Wimplicit-fallthrough=4"
+	CONFIGFLAGS="--with-pam --with-Werror"
+	;;
+    clang*|gcc*)
+	CC="$config"
+	;;
     kitchensink)
 	CONFIGFLAGS="--with-kerberos5 --with-libedit --with-pam"
 	CONFIGFLAGS="${CONFIGFLAGS} --with-security-key-builtin --with-selinux"
@@ -30,21 +58,34 @@ case "$config" in
     hardenedmalloc)
 	CONFIGFLAGS="--with-ldflags=-lhardened_malloc"
 	;;
-    kerberos5)
+    tcmalloc)
+	CONFIGFLAGS="--with-ldflags=-ltcmalloc"
+	;;
+    krb5|heimdal)
 	CONFIGFLAGS="--with-kerberos5"
 	;;
     libedit)
 	CONFIGFLAGS="--with-libedit"
 	;;
+    musl)
+	CC="musl-gcc"
+	CONFIGFLAGS="--without-zlib"
+	LIBCRYPTOFLAGS="--without-openssl"
+	TEST_TARGET="t-exec"
+	;;
+    pam-krb5)
+	CONFIGFLAGS="--with-pam --with-kerberos5"
+	SSHD_CONFOPTS="UsePam yes"
+	;;
     *pam)
 	CONFIGFLAGS="--with-pam"
 	SSHD_CONFOPTS="UsePam yes"
 	;;
-    libressl-head)
-	LIBCRYPTOFLAGS="--with-ssl-dir=/opt/libressl/head --with-rpath=-Wl,-rpath,"
+    libressl-*)
+	LIBCRYPTOFLAGS="--with-ssl-dir=/opt/libressl --with-rpath=-Wl,-rpath,"
 	;;
-    openssl-head)
-	LIBCRYPTOFLAGS="--with-ssl-dir=/opt/openssl/head --with-rpath=-Wl,-rpath,"
+    openssl-*)
+	LIBCRYPTOFLAGS="--with-ssl-dir=/opt/openssl --with-rpath=-Wl,-rpath,"
 	;;
     selinux)
 	CONFIGFLAGS="--with-selinux"
@@ -66,9 +107,9 @@ case "$config" in
 	# Valgrind slows things down enough that the agent timeout test
 	# won't reliably pass, and the unit tests run longer than allowed
 	# by github so split into three separate tests.
-	tests2="rekey integrity"
-	tests3="krl forward-control sshsig"
-	tests4="cert-userkey cert-hostkey kextype sftp-perm keygen-comment"
+	tests2="rekey integrity try-ciphers sftp"
+	tests3="krl forward-control sshsig agent-restrict kextype"
+	tests4="cert-userkey cert-hostkey kextype sftp-perm keygen-comment percent"
 	case "$config" in
 	    valgrind-1)
 		# All tests except agent-timeout (which is flaky under valgrind)
@@ -104,11 +145,52 @@ case "$config" in
 esac
 
 case "${TARGET_HOST}" in
+    dfly58*|dfly60*)
+	# scp 3-way connection hangs on these so skip until sorted.
+	SKIP_LTESTS=scp3
+	;;
+    hurd)
+	SKIP_LTESTS="forwarding multiplex proxy-connect hostkey-agent agent-ptrace"
+	;;
+    minix3)
+	LIBCRYPTOFLAGS="--without-openssl --disable-security-key"
+	# Minix does not have a loopback interface so we have to skip any
+	# test that relies on one.
+	# Also, Minix seems to be very limited in the number of select()
+	# calls that can be operating concurrently, so prune additional tests for that.
+	T="addrmatch agent-restrict brokenkeys cfgmatch cfgmatchlisten cfgparse connect
+	    connect-uri exit-status forward-control forwarding hostkey-agent
+	    key-options keyscan knownhosts-command login-timeout multiplex
+	    reconfigure reexec rekey scp scp-uri scp3 sftp sftp-badcmds
+	    sftp-batch sftp-cmds sftp-glob sftp-perm sftp-uri stderr-data
+	    transfer"
+	SKIP_LTESTS="$(echo $T)"
+	TEST_TARGET=t-exec
+	SUDO=""
+	;;
+    nbsd4)
+	# System compiler will ICE on some files with fstack-protector
+	# SHA256 functions in sha2.h conflict with OpenSSL's breaking sk-dummy
+	CONFIGFLAGS="${CONFIGFLAGS} --without-hardening --disable-security-key"
+	;;
     sol10|sol11)
 	# sol10 VM is 32bit and the unit tests are slow.
 	# sol11 has 4 test configs so skip unit tests to speed up.
 	TEST_TARGET="tests SKIP_UNIT=1"
 	;;
+    win10)
+	# No sudo on Windows.
+	SUDO=""
+	;;
+esac
+
+# Unless specified otherwise, build without OpenSSL on Mac OS since
+# modern versions don't ship with libcrypto.
+case "`./config.guess`" in
+*-darwin*)
+	LIBCRYPTOFLAGS="--without-openssl"
+	TEST_TARGET=t-exec
+	;;
 esac
 
 # If we have a local openssl/libressl, use that.
@@ -123,4 +205,10 @@ fi
 
 CONFIGFLAGS="${CONFIGFLAGS} ${LIBCRYPTOFLAGS}"
 
-export LTESTS SUDO TEST_TARGET TEST_SSH_UNSAFE_PERMISSIONS
+if [ -x "$(which plink 2>/dev/null)" ]; then
+	REGRESS_INTEROP_PUTTY=yes
+	export REGRESS_INTEROP_PUTTY
+fi
+
+export CC CFLAGS LTESTS SUDO
+export TEST_TARGET TEST_SSH_UNSAFE_PERMISSIONS TEST_SSH_FAIL_FATAL

.github/configure.sh

@@ -2,5 +2,20 @@
 
 . .github/configs $1
 
-set -x
+printf "$ "
+
+if [ "x$CC" != "x" ]; then
+	printf "CC='$CC' "
+fi
+if [ "x$CFLAGS" != "x" ]; then
+	printf "CFLAGS='$CFLAGS' "
+fi
+if [ "x$CPPFLAGS" != "x" ]; then
+	printf "CPPFLAGS='$CPPFLAGS' "
+fi
+if [ "x$LDFLAGS" != "x" ]; then
+	printf "LDFLAGS='$LDFLAGS' "
+fi
+
+echo ./configure ${CONFIGFLAGS}
 ./configure ${CONFIGFLAGS}

.github/run_test.sh

@@ -1,33 +1,34 @@
-#!/usr/bin/env bash
+#!/bin/sh
 
-. .github/configs $1 $2
+. .github/configs $1
 
 [ -z "${SUDO}" ] || ${SUDO} mkdir -p /var/empty
 
 set -ex
 
+output_failed_logs() {
+    for i in regress/failed*; do
+        if [ -f "$i" ]; then
+            echo -------------------------------------------------------------------------
+            echo LOGFILE $i
+            cat $i
+            echo -------------------------------------------------------------------------
+        fi
+    done
+}
+trap output_failed_logs 0
+
 if [ -z "${LTESTS}" ]; then
     make ${TEST_TARGET} SKIP_LTESTS="${SKIP_LTESTS}"
-    result=$?
 else
     make ${TEST_TARGET} SKIP_LTESTS="${SKIP_LTESTS}" LTESTS="${LTESTS}"
-    result=$?
 fi
 
 if [ ! -z "${SSHD_CONFOPTS}" ]; then
-    echo "rerunning tests with TEST_SSH_SSHD_CONFOPTS='${SSHD_CONFOPTS}'"
-    make t-exec TEST_SSH_SSHD_CONFOPTS="${SSHD_CONFOPTS}"
-    result2=$?
-    if [ "${result2}" -ne 0 ]; then
-        result="${result2}"
+    echo "rerunning t-exec with TEST_SSH_SSHD_CONFOPTS='${SSHD_CONFOPTS}'"
+    if [ -z "${LTESTS}" ]; then
+        make t-exec SKIP_LTESTS="${SKIP_LTESTS}" TEST_SSH_SSHD_CONFOPTS="${SSHD_CONFOPTS}"
+    else
+        make t-exec SKIP_LTESTS="${SKIP_LTESTS}" LTESTS="${LTESTS}" TEST_SSH_SSHD_CONFOPTS="${SSHD_CONFOPTS}"
     fi
 fi
-
-if [ "$result" -ne "0" ]; then
-    for i in regress/failed*; do
-        echo -------------------------------------------------------------------------
-        echo LOGFILE $i
-        cat $i
-        echo -------------------------------------------------------------------------
-    done
-fi

.github/setup_ci.sh

@@ -1,4 +1,13 @@
-#!/usr/bin/env bash
+#!/bin/sh
+
+ . .github/configs $@
+
+case "`./config.guess`" in
+*-darwin*)
+	brew install automake
+	exit 0
+	;;
+esac
 
 case $(./config.guess) in
 *-darwin*)
@@ -11,6 +20,7 @@ TARGETS=$@
 
 PACKAGES=""
 INSTALL_FIDO_PPA="no"
+export DEBIAN_FRONTEND=noninteractive
 
 #echo "Setting up for '$TARGETS'"
 
@@ -19,23 +29,30 @@ set -ex
 lsb_release -a
 
 if [ "${TARGETS}" = "kitchensink" ]; then
-	TARGETS="kerberos5 libedit pam sk selinux"
+	TARGETS="krb5 libedit pam sk selinux"
 fi
 
+for flag in $CONFIGFLAGS; do
+    case "$flag" in
+    --with-pam)		PACKAGES="${PACKAGES} libpam0g-dev" ;;
+    --with-libedit)	PACKAGES="${PACKAGES} libedit-dev" ;;
+    esac
+done
+
 for TARGET in $TARGETS; do
     case $TARGET in
-    default|without-openssl|without-zlib)
+    default|without-openssl|without-zlib|c89|libedit|*pam)
         # nothing to do
         ;;
-    kerberos5)
-        PACKAGES="$PACKAGES heimdal-dev"
-        #PACKAGES="$PACKAGES libkrb5-dev"
-        ;;
-    libedit)
-        PACKAGES="$PACKAGES libedit-dev"
+    clang-*|gcc-*)
+        compiler=$(echo $TARGET | sed 's/-Werror//')
+        PACKAGES="$PACKAGES $compiler"
         ;;
-    *pam)
-        PACKAGES="$PACKAGES libpam0g-dev"
+    krb5)
+        PACKAGES="$PACKAGES libkrb5-dev"
+	;;
+    heimdal)
+        PACKAGES="$PACKAGES heimdal-dev"
         ;;
     sk)
         INSTALL_FIDO_PPA="yes"
@@ -46,12 +63,33 @@ for TARGET in $TARGETS; do
         ;;
     hardenedmalloc)
         INSTALL_HARDENED_MALLOC=yes
+        ;;
+    musl)
+	PACKAGES="$PACKAGES musl-tools"
+	;;
+    tcmalloc)
+        PACKAGES="$PACKAGES libgoogle-perftools-dev"
+        ;;
+    openssl-noec)
+	INSTALL_OPENSSL=OpenSSL_1_1_1k
+	SSLCONFOPTS="no-ec"
+	;;
+    openssl-*)
+        INSTALL_OPENSSL=$(echo ${TARGET} | cut -f2 -d-)
+        case ${INSTALL_OPENSSL} in
+          1.1.1_stable)	INSTALL_OPENSSL="OpenSSL_1_1_1-stable" ;;
+          1.*)	INSTALL_OPENSSL="OpenSSL_$(echo ${INSTALL_OPENSSL} | tr . _)" ;;
+          3.*)	INSTALL_OPENSSL="openssl-${INSTALL_OPENSSL}" ;;
+        esac
+        PACKAGES="${PACKAGES} putty-tools"
        ;;
-    openssl-head)
-        INSTALL_OPENSSL_HEAD=yes
-       ;;
-    libressl-head)
-        INSTALL_LIBRESSL_HEAD=yes
+    libressl-*)
+        INSTALL_LIBRESSL=$(echo ${TARGET} | cut -f2 -d-)
+        case ${INSTALL_LIBRESSL} in
+          master) ;;
+          *) INSTALL_LIBRESSL="v$(echo ${TARGET} | cut -f2 -d-)" ;;
+        esac
+        PACKAGES="${PACKAGES} putty-tools"
        ;;
     valgrind*)
        PACKAGES="$PACKAGES valgrind"
@@ -62,10 +100,10 @@ for TARGET in $TARGETS; do
     esac
 done
 
-if [ "yes" == "$INSTALL_FIDO_PPA" ]; then
+if [ "yes" = "$INSTALL_FIDO_PPA" ]; then
     sudo apt update -qq
-    sudo apt install software-properties-common
-    sudo apt-add-repository ppa:yubico/stable
+    sudo apt install -qy software-properties-common
+    sudo apt-add-repository -y ppa:yubico/stable
 fi
 
 if [ "x" != "x$PACKAGES" ]; then 
@@ -77,21 +115,25 @@ if [ "${INSTALL_HARDENED_MALLOC}" = "yes" ]; then
     (cd ${HOME} &&
      git clone https://github.com/GrapheneOS/hardened_malloc.git &&
      cd ${HOME}/hardened_malloc &&
-     make -j2 && sudo cp libhardened_malloc.so /usr/lib/)
+     make -j2 && sudo cp out/libhardened_malloc.so /usr/lib/)
 fi
 
-if [ "${INSTALL_OPENSSL_HEAD}" = "yes" ];then
+if [ ! -z "${INSTALL_OPENSSL}" ]; then
     (cd ${HOME} &&
      git clone https://github.com/openssl/openssl.git &&
      cd ${HOME}/openssl &&
-     ./config no-threads no-engine no-fips no-shared --prefix=/opt/openssl/head &&
-     make -j2 && sudo make install_sw)
+     git checkout ${INSTALL_OPENSSL} &&
+     ./config no-threads shared ${SSLCONFOPTS} \
+         --prefix=/opt/openssl &&
+     make && sudo make install_sw)
 fi
 
-if [ "${INSTALL_LIBRESSL_HEAD}" = "yes" ];then
+if [ ! -z "${INSTALL_LIBRESSL}" ]; then
     (mkdir -p ${HOME}/libressl && cd ${HOME}/libressl &&
      git clone https://github.com/libressl-portable/portable.git &&
-     cd ${HOME}/libressl/portable && sh update.sh && sh autogen.sh &&
-     ./configure --prefix=/opt/libressl/head &&
-     make -j2 && sudo make install_sw)
+     cd ${HOME}/libressl/portable &&
+     git checkout ${INSTALL_LIBRESSL} &&
+     sh update.sh && sh autogen.sh &&
+     ./configure --prefix=/opt/libressl &&
+     make -j2 && sudo make install)
 fi