diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 99bb82e59..4eefd1eed 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -17,6 +17,8 @@ jobs:
     runs-on: ${{ matrix.os }}
 
     name: Ruby ${{ matrix.ruby }} on ${{ matrix.os }}
+    env:
+      SKIP_SSH_TESTS: true
     steps:
     - uses: actions/checkout@master
     - name: update submodule
diff --git a/script/travisbuild b/script/travisbuild
index ed00d48da..dcdc64d41 100755
--- a/script/travisbuild
+++ b/script/travisbuild
@@ -2,12 +2,14 @@
 
 set -ex
 
-echo "Starting ssh daemon..."
-TMPDIR=${TMPDIR:-/tmp}
-HOME=`mktemp -d ${TMPDIR}/home.XXXXXXXX`
-SSHD_DIR=`mktemp -d ${TMPDIR}/sshd.XXXXXXXX`
+if [ -z "$SKIP_SSH_TESTS" ]; then
 
-cat >"${SSHD_DIR}/sshd_config" <<-EOF
+    echo "Starting ssh daemon..."
+    TMPDIR=${TMPDIR:-/tmp}
+    HOME=`mktemp -d ${TMPDIR}/home.XXXXXXXX`
+    SSHD_DIR=`mktemp -d ${TMPDIR}/sshd.XXXXXXXX`
+
+    cat >"${SSHD_DIR}/sshd_config" <<-EOF
 Port 2222
 ListenAddress 0.0.0.0
 Protocol 2
@@ -26,29 +28,30 @@ StrictModes no
 # Required here as sshd will simply close connection otherwise
 UsePAM no
 EOF
-ssh-keygen -t rsa -f "${SSHD_DIR}/id_rsa" -N "" -q
-/usr/sbin/sshd -f "${SSHD_DIR}/sshd_config" -E "${SSHD_DIR}/log"
+    ssh-keygen -t rsa -f "${SSHD_DIR}/id_rsa" -N "" -q
+    /usr/sbin/sshd -f "${SSHD_DIR}/sshd_config" -E "${SSHD_DIR}/log"
 
-# Set up keys
-mkdir "${HOME}/.ssh"
-ssh-keygen -t rsa -f "${HOME}/.ssh/id_rsa" -N "" -q
-cat "${HOME}/.ssh/id_rsa.pub" >>"${HOME}/.ssh/authorized_keys"
-while read algorithm key comment; do
-    echo "[localhost]:2222 $algorithm $key" >>"${HOME}/.ssh/known_hosts"
-done <"${SSHD_DIR}/id_rsa.pub"
+    # Set up keys
+    mkdir "${HOME}/.ssh"
+    ssh-keygen -t rsa -f "${HOME}/.ssh/id_rsa" -N "" -q
+    cat "${HOME}/.ssh/id_rsa.pub" >>"${HOME}/.ssh/authorized_keys"
+    while read algorithm key comment; do
+	echo "[localhost]:2222 $algorithm $key" >>"${HOME}/.ssh/known_hosts"
+    done <"${SSHD_DIR}/id_rsa.pub"
 
-# Append the github.com keys for the tests that don't override checks. Some
-# older libssh2 versions don't like it unless we have ssh-rsa in here. This also
-# tests that the automatic selection off of known_hosts is working.
-ssh-keyscan -t ssh-rsa github.com >>"${HOME}/.ssh/known_hosts"
+    # Append the github.com keys for the tests that don't override checks. Some
+    # older libssh2 versions don't like it unless we have ssh-rsa in here. This also
+    # tests that the automatic selection off of known_hosts is working.
+    ssh-keyscan -t ssh-rsa github.com >>"${HOME}/.ssh/known_hosts"
 
-# Get the fingerprint for localhost and remove the colons so we can
-# parse it as a hex number. Older versions have a different output
-# format.
-if [[ $(ssh -V 2>&1) == OpenSSH_6* ]]; then
-    SSH_FINGERPRINT=$(ssh-keygen -F '[localhost]:2222' -f "${HOME}/.ssh/known_hosts" -l | tail -n 1 | cut -d ' ' -f 2 | tr -d ':')
-else
-    SSH_FINGERPRINT=$(ssh-keygen -E md5 -F '[localhost]:2222' -f "${HOME}/.ssh/known_hosts" -l | tail -n 1 | cut -d ' ' -f 3 | cut -d : -f2- | tr -d :)
+    # Get the fingerprint for localhost and remove the colons so we can
+    # parse it as a hex number. Older versions have a different output
+    # format.
+    if [[ $(ssh -V 2>&1) == OpenSSH_6* ]]; then
+	SSH_FINGERPRINT=$(ssh-keygen -F '[localhost]:2222' -f "${HOME}/.ssh/known_hosts" -l | tail -n 1 | cut -d ' ' -f 2 | tr -d ':')
+    else
+	SSH_FINGERPRINT=$(ssh-keygen -E md5 -F '[localhost]:2222' -f "${HOME}/.ssh/known_hosts" -l | tail -n 1 | cut -d ' ' -f 3 | cut -d : -f2- | tr -d :)
+    fi
 fi
 
 # Create a test repo which we can use for the online tests
@@ -65,16 +68,18 @@ git daemon --listen=localhost --port=9419 --export-all --base-path=$HOME/_temp_r
 # On Actions we start with 777 which means sshd won't let us in
 chmod 750 $HOME
 
+export GITTEST_REMOTE_REPO_PATH="$HOME/_temp/test.git"
 export GITTEST_REMOTE_GIT_RO_URL="git://localhost:9419/testrepo.git"
 export GITTEST_REMOTE_GIT_URL="git://localhost/test.git"
-export GITTEST_REMOTE_SSH_URL="ssh://localhost:2222/$HOME/_temp/test.git"
-export GITTEST_REMOTE_SSH_USER=$USER
-export GITTEST_REMOTE_SSH_KEY="$HOME/.ssh/id_rsa"
-export GITTEST_REMOTE_SSH_PUBKEY="$HOME/.ssh/id_rsa.pub"
-export GITTEST_REMOTE_SSH_PASSPHRASE=""
-export GITTEST_REMOTE_REPO_PATH="$HOME/_temp/test.git"
+if [ -z "$SKIP_SSH_TESTS" ]; then
+    export GITTEST_REMOTE_SSH_URL="ssh://localhost:2222/$HOME/_temp/test.git"
+    export GITTEST_REMOTE_SSH_USER=$USER
+    export GITTEST_REMOTE_SSH_KEY="$HOME/.ssh/id_rsa"
+    export GITTEST_REMOTE_SSH_PUBKEY="$HOME/.ssh/id_rsa.pub"
+    export GITTEST_REMOTE_SSH_PASSPHRASE=""
 
-eval $(ssh-agent)
-ssh-add $GITTEST_REMOTE_SSH_KEY
+    eval $(ssh-agent)
+    ssh-add $GITTEST_REMOTE_SSH_KEY
+fi
 
 bundle exec rake -- --with-ssh || exit $?
diff --git a/test/online/push_test.rb b/test/online/push_test.rb
index e516f6e54..cbfbc2f94 100644
--- a/test/online/push_test.rb
+++ b/test/online/push_test.rb
@@ -29,7 +29,7 @@ def test_push_branches
 
 class OnlineSshPushTest < Rugged::OnlineTestCase
   def setup
-    skip unless Rugged.features.include?(:ssh) && git_creds?
+    skip unless Rugged.features.include?(:ssh) && ssh_creds?
 
     @repo = FixtureRepo.from_libgit2("push_src")
     @remote = @repo.remotes.create("test", ENV['GITTEST_REMOTE_SSH_URL'])