Merge pull request #193 from t0astbandit/add-ssh-loop-wait

Add ability to pass ssh socket heartbeat interval

README.md

@@ -401,9 +401,20 @@ You can configure it with a few options:
     set :ssh, true # enable ssh connections
     set :ssh_user, "myuser" # if you want to specify the user to connect as, otherwise your current user
     set :ssh_log_level, Logger::DEBUG # passed on to net/ssh, can be noisy; defaults to Logger::WARN
+    set :ssh_socket_heartbeat, 5 # passed on to net/ssh (Net::SSH::Connection::Session#loop); defaults to 30
   end
 ```
 
+#### Troubleshooting SSH connections
+
+In some cases you may notice your SSH commands are completing successfully,
+but the connection isn't aware they are done. This will manifest as a tunneled
+command taking much longer than anticipated - minutes depending on your
+server's configuration. You may need to set `:ssh_socket_heartbeat` to a
+smaller number. This will check more frequently if the command has completed
+which can alleviate this issue, though it will consume more CPU as it wakes up
+its threads more frequently.
+
 Deploying
 ---------
 

lib/centurion/deploy_dsl.rb

@@ -195,6 +195,7 @@ def build_server_params
       # nil is OK for both of these, defaults applied internally
       opts[:ssh_user] = fetch(:ssh_user)
       opts[:ssh_log_level] = fetch(:ssh_log_level)
+      opts[:ssh_socket_heartbeat] = fetch(:ssh_socket_heartbeat)
     end
 
     opts

lib/centurion/docker_via_api.rb

@@ -12,8 +12,7 @@ def initialize(hostname, port, connection_opts = {}, api_version = nil)
     if connection_opts[:ssh]
       @base_uri = hostname
       @ssh = true
-      @ssh_user = connection_opts[:ssh_user]
-      @ssh_log_level = connection_opts[:ssh_log_level]
+      @connection_opts = connection_opts
     else
       @base_uri = "http#{'s' if tls_enable?}://#{hostname}:#{port}"
     end
@@ -194,7 +193,7 @@ def with_excon(&block)
   end
 
   def with_excon_via_ssh
-    Centurion::SSH.with_docker_socket(@base_uri, @ssh_user, @ssh_log_level) do |socket|
+    Centurion::SSH.with_docker_socket(@base_uri, @connection_opts[:ssh_user], @connection_opts[:ssh_log_level], @connection_opts[:ssh_socket_heartbeat]) do |socket|
       conn = Excon.new('unix:///', socket: socket)
       yield conn
     end

lib/centurion/docker_via_cli.rb

@@ -97,7 +97,7 @@ def build_command(action, destination)
 
   def connect
     if @connection_opts[:ssh]
-      Centurion::SSH.with_docker_socket(@docker_host, @connection_opts[:ssh_user], @connection_opts[:ssh_log_level]) do |socket|
+      Centurion::SSH.with_docker_socket(@docker_host, @connection_opts[:ssh_user], @connection_opts[:ssh_log_level], @connection_opts[:ssh_socket_heartbeat]) do |socket|
         @socket = socket
         ret = yield
         @socket = nil

lib/centurion/ssh.rb

@@ -7,7 +7,7 @@ module Centurion; end
 module Centurion::SSH
   extend self
 
-  def with_docker_socket(hostname, user, log_level = nil)
+  def with_docker_socket(hostname, user, log_level = nil, ssh_socket_heartbeat = 30)
     log_level ||= Logger::WARN
 
     with_sshkit(hostname, user) do
@@ -24,7 +24,7 @@ def with_docker_socket(hostname, user, log_level = nil)
           yield local_socket_path
         end
 
-        ssh.loop { t.alive? }
+        ssh.loop(ssh_socket_heartbeat) { t.alive? }
         ssh.forward.cancel_local_socket local_socket_path
         local_socket_path_file.delete
         t.value

spec/docker_via_api_spec.rb

@@ -134,6 +134,7 @@
     let(:port) { nil }
     let(:ssh_user) { 'myuser' }
     let(:ssh_log_level) { nil }
+    let(:ssh_socket_heartbeat) { nil }
     let(:base_req) { {
       socket: '/tmp/socket/path'
     } }
@@ -142,12 +143,13 @@
       p = { ssh: true}
       p[:ssh_user] = ssh_user if ssh_user
       p[:ssh_log_level] = ssh_log_level if ssh_log_level
+      p[:ssh_socket_heartbeat] = ssh_socket_heartbeat if ssh_socket_heartbeat
       p
     end
 
     context 'with no log level' do
       before do
-        expect(Centurion::SSH).to receive(:with_docker_socket).with(hostname, ssh_user, nil).and_yield('/tmp/socket/path')
+        expect(Centurion::SSH).to receive(:with_docker_socket).with(hostname, ssh_user, nil, nil).and_yield('/tmp/socket/path')
       end
 
       it_behaves_like 'docker API'
@@ -157,7 +159,7 @@
       let(:ssh_user) { nil }
 
       before do
-        expect(Centurion::SSH).to receive(:with_docker_socket).with(hostname, nil, nil).and_yield('/tmp/socket/path')
+        expect(Centurion::SSH).to receive(:with_docker_socket).with(hostname, nil, nil, nil).and_yield('/tmp/socket/path')
       end
 
       it_behaves_like 'docker API'
@@ -167,7 +169,17 @@
       let(:ssh_log_level) { Logger::DEBUG }
 
       before do
-        expect(Centurion::SSH).to receive(:with_docker_socket).with(hostname, ssh_user, Logger::DEBUG).and_yield('/tmp/socket/path')
+        expect(Centurion::SSH).to receive(:with_docker_socket).with(hostname, ssh_user, Logger::DEBUG, nil).and_yield('/tmp/socket/path')
+      end
+
+      it_behaves_like 'docker API'
+    end
+
+    context 'with a socket heartbeat set' do
+      let(:ssh_socket_heartbeat) { 5 }
+
+      before do
+        expect(Centurion::SSH).to receive(:with_docker_socket).with(hostname, ssh_user, nil, 5).and_yield('/tmp/socket/path')
       end
 
       it_behaves_like 'docker API'

spec/docker_via_cli_spec.rb

@@ -69,18 +69,20 @@
     let(:hostname) { 'host1' }
     let(:ssh_user) { 'myuser' }
     let(:ssh_log_level) { nil }
+    let(:ssh_socket_heartbeat) { nil }
     let(:docker_via_cli) { Centurion::DockerViaCli.new(hostname, nil, docker_path, params) }
     let(:prefix) { "-H=unix:///tmp/socket/path" }
     let(:params) do
-      p = { ssh: true}
+      p = { ssh: true }
       p[:ssh_user] = ssh_user if ssh_user
       p[:ssh_log_level] = ssh_log_level if ssh_log_level
+      p[:ssh_socket_heartbeat] = ssh_socket_heartbeat if ssh_socket_heartbeat
       p
     end
 
     context 'with no log level' do
       before do
-        expect(Centurion::SSH).to receive(:with_docker_socket).with(hostname, ssh_user, nil).and_yield('/tmp/socket/path')
+        expect(Centurion::SSH).to receive(:with_docker_socket).with(hostname, ssh_user, nil, nil).and_yield('/tmp/socket/path')
       end
 
       it_behaves_like 'docker CLI'
@@ -90,7 +92,7 @@
       let(:ssh_user) { nil }
 
       before do
-        expect(Centurion::SSH).to receive(:with_docker_socket).with(hostname, nil, nil).and_yield('/tmp/socket/path')
+        expect(Centurion::SSH).to receive(:with_docker_socket).with(hostname, nil, nil, nil).and_yield('/tmp/socket/path')
       end
 
       it_behaves_like 'docker CLI'
@@ -100,7 +102,17 @@
       let(:ssh_log_level) { Logger::DEBUG }
 
       before do
-        expect(Centurion::SSH).to receive(:with_docker_socket).with(hostname, ssh_user, Logger::DEBUG).and_yield('/tmp/socket/path')
+        expect(Centurion::SSH).to receive(:with_docker_socket).with(hostname, ssh_user, Logger::DEBUG, nil).and_yield('/tmp/socket/path')
+      end
+
+      it_behaves_like 'docker CLI'
+    end
+
+    context 'with an ssh loop wait set' do
+      let(:ssh_socket_heartbeat) { 5 }
+
+      before do
+        expect(Centurion::SSH).to receive(:with_docker_socket).with(hostname, ssh_user, nil, 5).and_yield('/tmp/socket/path')
       end
 
       it_behaves_like 'docker CLI'