From c8a8442e74398c0cc45d0c57c9366b7e7742f3d7 Mon Sep 17 00:00:00 2001 From: Jerry Aldrich Date: Mon, 20 Nov 2017 22:29:43 -0600 Subject: [PATCH 01/26] Use named pipe to decrease local Windows runtime This uses PowerShell to spawn a process that listens on a named pipe for Base64 encoded commands and executes them. This drastically decreases the total runtime when running locally on Windows by using a single PowerShell session to execute commands instead of spawning a new session for each command. Huge thanks to @sdelano for the initial idea! Signed-off-by: Jerry Aldrich --- lib/train/extras/command_wrapper.rb | 41 ----------- lib/train/transports/local.rb | 110 ++++++++++++++++++++++++++-- 2 files changed, 105 insertions(+), 46 deletions(-) diff --git a/lib/train/extras/command_wrapper.rb b/lib/train/extras/command_wrapper.rb index 716d1796..0c8149be 100644 --- a/lib/train/extras/command_wrapper.rb +++ b/lib/train/extras/command_wrapper.rb @@ -121,43 +121,6 @@ def safe_string(str) end end - # this is required if you run locally on windows, - # winrm connections provide a PowerShell shell automatically - # TODO: only activate in local mode - class PowerShellCommand < CommandWrapperBase - Train::Options.attach(self) - - def initialize(backend, options) - @backend = backend - validate_options(options) - end - - def run(script) - # wrap the script to ensure we always run it via powershell - # especially in local mode, we cannot be sure that we get a Powershell - # we may just get a `cmd`. - # TODO: we may want to opt for powershell.exe -command instead of `encodeCommand` - "powershell -NoProfile -encodedCommand #{encoded(safe_script(script))}" - end - - # suppress the progress stream from leaking to stderr - def safe_script(script) - "$ProgressPreference='SilentlyContinue';" + script - end - - # Encodes the script so that it can be passed to the PowerShell - # --EncodedCommand argument. - # @return [String] The UTF-16LE base64 encoded script - def encoded(script) - encoded_script = safe_script(script).encode('UTF-16LE', 'UTF-8') - Base64.strict_encode64(encoded_script) - end - - def to_s - 'PowerShell CommandWrapper' - end - end - class CommandWrapper include_options LinuxCommand @@ -168,10 +131,6 @@ def self.load(transport, options) msg = res.verify fail Train::UserError, "Sudo failed: #{msg}" unless msg.nil? res - # only use powershell command for local transport. winrm transport - # uses powershell as default - elsif transport.os.windows? && transport.class == Train::Transports::Local::Connection - PowerShellCommand.new(transport, options) end end end diff --git a/lib/train/transports/local.rb b/lib/train/transports/local.rb index b052524c..9ba49bd4 100644 --- a/lib/train/transports/local.rb +++ b/lib/train/transports/local.rb @@ -16,7 +16,10 @@ def connection(_ = nil) @connection ||= Connection.new(@options) end - class Connection < BaseConnection + class Connection < BaseConnection # rubocop:disable Metrics/ClassLength + require 'json' + require 'base64' + def initialize(options) super(options) @cmd_wrapper = nil @@ -37,11 +40,108 @@ def local? private + def run_powershell_using_named_pipe(script) + pipe = nil + # Try to acquire pipe for 10 seconds with 0.1 second intervals. + # Removing this can result in instability due to the pipe being + # temporarily unavailable. + 100.times do + begin + pipe = open('//localhost/pipe/InSpec', 'r+') + break + rescue + sleep 0.1 + end + end + fail 'Could not open pipe `//localhost/pipe/InSpec`' if pipe.nil? + # Prevent progress stream from leaking to stderr + script = "$ProgressPreference='SilentlyContinue';" + script + encoded_script = Base64.strict_encode64(script) + pipe.puts(encoded_script) + pipe.flush + result = JSON.parse(Base64.decode64(pipe.readline)) + pipe.close + result + end + + def start_named_pipe_server # rubocop:disable Metrics/MethodLength + require 'win32/process' + + script = <<-EOF + $ProgressPreference = 'SilentlyContinue' + $ErrorActionPreference = 'Stop' + + Function Execute-UserCommand($userInput) { + $command = [System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String($userInput)) + + $scriptBlock = $ExecutionContext.InvokeCommand.NewScriptBlock($command) + try { + $stdout = & $scriptBlock | Out-String + $result = @{ 'stdout' = $stdout ; 'stderr' = ''; 'exitstatus' = 0 } + } catch { + $stderr = $_ | Out-String + $result = @{ 'stdout' = ''; 'stderr' = $_; 'exitstatus' = 1 } + } + return $result | ConvertTo-JSON + } + + Function Start-PipeServer { + while($true) { + # Attempt to acquire a pipe for 10 seconds, trying every 100 milliseconds + for($i=1; $i -le 100; $i++) { + try { + $pipeServer = New-Object System.IO.Pipes.NamedPipeServerStream('InSpec', [System.IO.Pipes.PipeDirection]::InOut) + break + } catch { + Start-Sleep -m 100 + if($i -eq 100) { throw } + } + } + $pipeReader = New-Object System.IO.StreamReader($pipeServer) + $pipeWriter = New-Object System.IO.StreamWriter($pipeServer) + + $pipeServer.WaitForConnection() + $pipeWriter.AutoFlush = $true + + $clientConnected = $true + while($clientConnected) { + $input = $pipeReader.ReadLine() + + if($input -eq $null) { + $clientConnected = $false + $pipeServer.Dispose() + } else { + $result = Execute-UserCommand($input) + $encodedResult = [System.Convert]::ToBase64String([System.Text.Encoding]::UTF8.GetBytes($result)) + $pipeWriter.WriteLine($encodedResult) + } + } + } + } + Start-PipeServer + EOF + + utf8_script = script.encode('UTF-16LE', 'UTF-8') + base64_script = Base64.strict_encode64(utf8_script) + cmd = "powershell -NoProfile -ExecutionPolicy bypass -NonInteractive -EncodedCommand #{base64_script}" + + server_pid = Process.create(command_line: cmd).process_id + + # Ensure process is killed when the Train process exits + at_exit { Process.kill('KILL', server_pid) } + end + def run_command_via_connection(cmd) - cmd = @cmd_wrapper.run(cmd) unless @cmd_wrapper.nil? - res = Mixlib::ShellOut.new(cmd) - res.run_command - CommandResult.new(res.stdout, res.stderr, res.exitstatus) + if defined?(@platform) && @platform.windows? + start_named_pipe_server unless File.exist?('//localhost/pipe/InSpec') + res = run_powershell_using_named_pipe(cmd) + CommandResult.new(res['stdout'], res['stderr'], res['exitstatus']) + else + cmd = @cmd_wrapper.run(cmd) unless @cmd_wrapper.nil? + res = Mixlib::ShellOut.new(cmd) + res.run_command + CommandResult.new(res.stdout, res.stderr, res.exitstatus) + end rescue Errno::ENOENT => _ CommandResult.new('', '', 1) end From 4540c4bbd9631e3f0ab0f19f8250a58ec5590a07 Mon Sep 17 00:00:00 2001 From: Jerry Aldrich Date: Wed, 22 Nov 2017 14:20:16 -0600 Subject: [PATCH 02/26] Modify named pipe creation to use unique name Signed-off-by: Jerry Aldrich --- lib/train/transports/local.rb | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/lib/train/transports/local.rb b/lib/train/transports/local.rb index 9ba49bd4..acab3c77 100644 --- a/lib/train/transports/local.rb +++ b/lib/train/transports/local.rb @@ -19,11 +19,13 @@ def connection(_ = nil) class Connection < BaseConnection # rubocop:disable Metrics/ClassLength require 'json' require 'base64' + require 'securerandom' def initialize(options) super(options) @cmd_wrapper = nil @cmd_wrapper = CommandWrapper.load(self, options) + @pipe_location = "//localhost/pipe/inspec_#{SecureRandom.hex}" end def login_command @@ -40,20 +42,20 @@ def local? private - def run_powershell_using_named_pipe(script) + def run_powershell_using_named_pipe(pipe_location, script) pipe = nil # Try to acquire pipe for 10 seconds with 0.1 second intervals. # Removing this can result in instability due to the pipe being # temporarily unavailable. 100.times do begin - pipe = open('//localhost/pipe/InSpec', 'r+') + pipe = open(pipe_location, 'r+') break rescue sleep 0.1 end end - fail 'Could not open pipe `//localhost/pipe/InSpec`' if pipe.nil? + fail "Could not open pipe `#{pipe_location}`" if pipe.nil? # Prevent progress stream from leaking to stderr script = "$ProgressPreference='SilentlyContinue';" + script encoded_script = Base64.strict_encode64(script) @@ -64,7 +66,7 @@ def run_powershell_using_named_pipe(script) result end - def start_named_pipe_server # rubocop:disable Metrics/MethodLength + def start_named_pipe_server(pipe_name) # rubocop:disable Metrics/MethodLength require 'win32/process' script = <<-EOF @@ -73,7 +75,6 @@ def start_named_pipe_server # rubocop:disable Metrics/MethodLength Function Execute-UserCommand($userInput) { $command = [System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String($userInput)) - $scriptBlock = $ExecutionContext.InvokeCommand.NewScriptBlock($command) try { $stdout = & $scriptBlock | Out-String @@ -90,7 +91,7 @@ def start_named_pipe_server # rubocop:disable Metrics/MethodLength # Attempt to acquire a pipe for 10 seconds, trying every 100 milliseconds for($i=1; $i -le 100; $i++) { try { - $pipeServer = New-Object System.IO.Pipes.NamedPipeServerStream('InSpec', [System.IO.Pipes.PipeDirection]::InOut) + $pipeServer = New-Object System.IO.Pipes.NamedPipeServerStream('#{pipe_name}', [System.IO.Pipes.PipeDirection]::InOut) break } catch { Start-Sleep -m 100 From 48fa1429c5b5de797709d7cb1963f26b358fe535 Mon Sep 17 00:00:00 2001 From: Jerry Aldrich Date: Wed, 22 Nov 2017 14:20:41 -0600 Subject: [PATCH 03/26] Remove `$ProgressPreference = 'SilentlyContinue'` Signed-off-by: Jerry Aldrich --- lib/train/transports/local.rb | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/train/transports/local.rb b/lib/train/transports/local.rb index acab3c77..c8e2c654 100644 --- a/lib/train/transports/local.rb +++ b/lib/train/transports/local.rb @@ -70,7 +70,6 @@ def start_named_pipe_server(pipe_name) # rubocop:disable Metrics/MethodLength require 'win32/process' script = <<-EOF - $ProgressPreference = 'SilentlyContinue' $ErrorActionPreference = 'Stop' Function Execute-UserCommand($userInput) { From bed61c562fd90256a29c5eada33274281e84cebf Mon Sep 17 00:00:00 2001 From: Jerry Aldrich Date: Wed, 22 Nov 2017 16:10:27 -0600 Subject: [PATCH 04/26] Modify code to persist pipe usage Signed-off-by: Jerry Aldrich --- lib/train/transports/local.rb | 106 +++++++++++++++------------------- 1 file changed, 46 insertions(+), 60 deletions(-) diff --git a/lib/train/transports/local.rb b/lib/train/transports/local.rb index c8e2c654..f6de7644 100644 --- a/lib/train/transports/local.rb +++ b/lib/train/transports/local.rb @@ -16,7 +16,7 @@ def connection(_ = nil) @connection ||= Connection.new(@options) end - class Connection < BaseConnection # rubocop:disable Metrics/ClassLength + class Connection < BaseConnection require 'json' require 'base64' require 'securerandom' @@ -25,7 +25,28 @@ def initialize(options) super(options) @cmd_wrapper = nil @cmd_wrapper = CommandWrapper.load(self, options) - @pipe_location = "//localhost/pipe/inspec_#{SecureRandom.hex}" + @pipe = acquire_named_pipe if @platform.windows? + end + + def acquire_named_pipe + pipe_name = "inspec_#{SecureRandom.hex}" + pipe_location = "//localhost/pipe/#{pipe_name}" + start_named_pipe_server(pipe_name) unless File.exist?(pipe_location) + + # Try to acquire pipe for 10 seconds with 0.1 second intervals. + # This allows time for PowerShell to start the pipe + pipe = nil + 100.times do + begin + pipe = open(pipe_location, 'r+') + break + rescue + sleep 0.1 + end + end + fail "Could not open named pipe #{pipe_location}" if pipe.nil? + + pipe end def login_command @@ -42,38 +63,33 @@ def local? private - def run_powershell_using_named_pipe(pipe_location, script) - pipe = nil - # Try to acquire pipe for 10 seconds with 0.1 second intervals. - # Removing this can result in instability due to the pipe being - # temporarily unavailable. - 100.times do - begin - pipe = open(pipe_location, 'r+') - break - rescue - sleep 0.1 - end - end - fail "Could not open pipe `#{pipe_location}`" if pipe.nil? - # Prevent progress stream from leaking to stderr + def run_powershell_using_named_pipe(script) script = "$ProgressPreference='SilentlyContinue';" + script encoded_script = Base64.strict_encode64(script) - pipe.puts(encoded_script) - pipe.flush - result = JSON.parse(Base64.decode64(pipe.readline)) - pipe.close - result + @pipe.puts(encoded_script) + @pipe.flush + JSON.parse(Base64.decode64(@pipe.readline)) end - def start_named_pipe_server(pipe_name) # rubocop:disable Metrics/MethodLength + def start_named_pipe_server(pipe_name) require 'win32/process' script = <<-EOF $ErrorActionPreference = 'Stop' - Function Execute-UserCommand($userInput) { - $command = [System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String($userInput)) + $pipeServer = New-Object System.IO.Pipes.NamedPipeServerStream('#{pipe_name}', [System.IO.Pipes.PipeDirection]::InOut) + $pipeReader = New-Object System.IO.StreamReader($pipeServer) + $pipeWriter = New-Object System.IO.StreamWriter($pipeServer) + + $pipeServer.WaitForConnection() + + # Create loop to receive and process user commands/scripts + $clientConnected = $true + while($clientConnected) { + $input = $pipeReader.ReadLine() + $command = [System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String($input)) + + # Execute user command/script and convert result to JSON $scriptBlock = $ExecutionContext.InvokeCommand.NewScriptBlock($command) try { $stdout = & $scriptBlock | Out-String @@ -82,43 +98,13 @@ def start_named_pipe_server(pipe_name) # rubocop:disable Metrics/MethodLength $stderr = $_ | Out-String $result = @{ 'stdout' = ''; 'stderr' = $_; 'exitstatus' = 1 } } - return $result | ConvertTo-JSON - } + $resultJSON = $result | ConvertTo-JSON - Function Start-PipeServer { - while($true) { - # Attempt to acquire a pipe for 10 seconds, trying every 100 milliseconds - for($i=1; $i -le 100; $i++) { - try { - $pipeServer = New-Object System.IO.Pipes.NamedPipeServerStream('#{pipe_name}', [System.IO.Pipes.PipeDirection]::InOut) - break - } catch { - Start-Sleep -m 100 - if($i -eq 100) { throw } - } - } - $pipeReader = New-Object System.IO.StreamReader($pipeServer) - $pipeWriter = New-Object System.IO.StreamWriter($pipeServer) - - $pipeServer.WaitForConnection() - $pipeWriter.AutoFlush = $true - - $clientConnected = $true - while($clientConnected) { - $input = $pipeReader.ReadLine() - - if($input -eq $null) { - $clientConnected = $false - $pipeServer.Dispose() - } else { - $result = Execute-UserCommand($input) - $encodedResult = [System.Convert]::ToBase64String([System.Text.Encoding]::UTF8.GetBytes($result)) - $pipeWriter.WriteLine($encodedResult) - } - } - } + # Encode JSON in Base64 and write to pipe + $encodedResult = [System.Convert]::ToBase64String([System.Text.Encoding]::UTF8.GetBytes($resultJSON)) + $pipeWriter.WriteLine($encodedResult) + $pipeWriter.Flush() } - Start-PipeServer EOF utf8_script = script.encode('UTF-16LE', 'UTF-8') From fe773e6f864d071462a60fbf46e1753ae36052b1 Mon Sep 17 00:00:00 2001 From: Jerry Aldrich Date: Wed, 22 Nov 2017 16:27:16 -0600 Subject: [PATCH 05/26] Removed pipe argument (default is In/Out) Signed-off-by: Jerry Aldrich --- lib/train/transports/local.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/train/transports/local.rb b/lib/train/transports/local.rb index f6de7644..4ddd205f 100644 --- a/lib/train/transports/local.rb +++ b/lib/train/transports/local.rb @@ -77,7 +77,7 @@ def start_named_pipe_server(pipe_name) script = <<-EOF $ErrorActionPreference = 'Stop' - $pipeServer = New-Object System.IO.Pipes.NamedPipeServerStream('#{pipe_name}', [System.IO.Pipes.PipeDirection]::InOut) + $pipeServer = New-Object System.IO.Pipes.NamedPipeServerStream('#{pipe_name}') $pipeReader = New-Object System.IO.StreamReader($pipeServer) $pipeWriter = New-Object System.IO.StreamWriter($pipeServer) From ccd1a8d7c2eedf9e2baeec1b3126921da77dc9d7 Mon Sep 17 00:00:00 2001 From: Jerry Aldrich Date: Thu, 23 Nov 2017 03:10:05 -0600 Subject: [PATCH 06/26] Privatize pipe related functions Signed-off-by: Jerry Aldrich --- lib/train/transports/local.rb | 51 +++++++++++++++++++++++++---------- 1 file changed, 37 insertions(+), 14 deletions(-) diff --git a/lib/train/transports/local.rb b/lib/train/transports/local.rb index 4ddd205f..84ee7675 100644 --- a/lib/train/transports/local.rb +++ b/lib/train/transports/local.rb @@ -28,6 +28,43 @@ def initialize(options) @pipe = acquire_named_pipe if @platform.windows? end + def run_command(cmd) + if defined?(@pipe) + res = run_powershell_using_named_pipe(cmd) + CommandResult.new(res['stdout'], res['stderr'], res['exitstatus']) + else + cmd = @cmd_wrapper.run(cmd) unless @cmd_wrapper.nil? + res = Mixlib::ShellOut.new(cmd) + res.run_command + CommandResult.new(res.stdout, res.stderr, res.exitstatus) + end + rescue Errno::ENOENT => _ + CommandResult.new('', '', 1) + end + + def local? + true + end + + def file(path) + @files[path] ||= \ + if os.windows? + Train::File::Local::Windows.new(self, path) + else + Train::File::Local::Unix.new(self, path) + end + end + + def login_command + nil # none, open your shell + end + + def uri + 'local://' + end + + private + def acquire_named_pipe pipe_name = "inspec_#{SecureRandom.hex}" pipe_location = "//localhost/pipe/#{pipe_name}" @@ -49,20 +86,6 @@ def acquire_named_pipe pipe end - def login_command - nil # none, open your shell - end - - def uri - 'local://' - end - - def local? - true - end - - private - def run_powershell_using_named_pipe(script) script = "$ProgressPreference='SilentlyContinue';" + script encoded_script = Base64.strict_encode64(script) From 8f2437bb039f741ba6ab2292e4b73b29660acedd Mon Sep 17 00:00:00 2001 From: Jerry Aldrich Date: Thu, 23 Nov 2017 03:26:55 -0600 Subject: [PATCH 07/26] Add fallback to not used pipe if creation fails Signed-off-by: Jerry Aldrich --- lib/train/extras/command_wrapper.rb | 41 +++++++++++++++++++++++++++++ lib/train/transports/local.rb | 12 ++++++--- 2 files changed, 50 insertions(+), 3 deletions(-) diff --git a/lib/train/extras/command_wrapper.rb b/lib/train/extras/command_wrapper.rb index 0c8149be..716d1796 100644 --- a/lib/train/extras/command_wrapper.rb +++ b/lib/train/extras/command_wrapper.rb @@ -121,6 +121,43 @@ def safe_string(str) end end + # this is required if you run locally on windows, + # winrm connections provide a PowerShell shell automatically + # TODO: only activate in local mode + class PowerShellCommand < CommandWrapperBase + Train::Options.attach(self) + + def initialize(backend, options) + @backend = backend + validate_options(options) + end + + def run(script) + # wrap the script to ensure we always run it via powershell + # especially in local mode, we cannot be sure that we get a Powershell + # we may just get a `cmd`. + # TODO: we may want to opt for powershell.exe -command instead of `encodeCommand` + "powershell -NoProfile -encodedCommand #{encoded(safe_script(script))}" + end + + # suppress the progress stream from leaking to stderr + def safe_script(script) + "$ProgressPreference='SilentlyContinue';" + script + end + + # Encodes the script so that it can be passed to the PowerShell + # --EncodedCommand argument. + # @return [String] The UTF-16LE base64 encoded script + def encoded(script) + encoded_script = safe_script(script).encode('UTF-16LE', 'UTF-8') + Base64.strict_encode64(encoded_script) + end + + def to_s + 'PowerShell CommandWrapper' + end + end + class CommandWrapper include_options LinuxCommand @@ -131,6 +168,10 @@ def self.load(transport, options) msg = res.verify fail Train::UserError, "Sudo failed: #{msg}" unless msg.nil? res + # only use powershell command for local transport. winrm transport + # uses powershell as default + elsif transport.os.windows? && transport.class == Train::Transports::Local::Connection + PowerShellCommand.new(transport, options) end end end diff --git a/lib/train/transports/local.rb b/lib/train/transports/local.rb index 84ee7675..ac3bb859 100644 --- a/lib/train/transports/local.rb +++ b/lib/train/transports/local.rb @@ -16,7 +16,7 @@ def connection(_ = nil) @connection ||= Connection.new(@options) end - class Connection < BaseConnection + class Connection < BaseConnection # rubocop:disable Metrics/ClassLength require 'json' require 'base64' require 'securerandom' @@ -25,11 +25,17 @@ def initialize(options) super(options) @cmd_wrapper = nil @cmd_wrapper = CommandWrapper.load(self, options) - @pipe = acquire_named_pipe if @platform.windows? + if @platform.windows? + begin + @pipe = acquire_named_pipe + rescue + @pipe = false + end + end end def run_command(cmd) - if defined?(@pipe) + if defined?(@pipe) && @pipe res = run_powershell_using_named_pipe(cmd) CommandResult.new(res['stdout'], res['stderr'], res['exitstatus']) else From 82ffa18f07ba1f7700979df2c68201cb5b73dddb Mon Sep 17 00:00:00 2001 From: Jerry Aldrich Date: Sun, 26 Nov 2017 20:08:21 -0600 Subject: [PATCH 08/26] Move Windows local command logic to separate class Signed-off-by: Jerry Aldrich --- lib/train/extras/command_wrapper.rb | 41 ------ lib/train/transports/local.rb | 195 +++++++++++++++++----------- test/windows/local_test.rb | 17 +++ 3 files changed, 134 insertions(+), 119 deletions(-) diff --git a/lib/train/extras/command_wrapper.rb b/lib/train/extras/command_wrapper.rb index 716d1796..0c8149be 100644 --- a/lib/train/extras/command_wrapper.rb +++ b/lib/train/extras/command_wrapper.rb @@ -121,43 +121,6 @@ def safe_string(str) end end - # this is required if you run locally on windows, - # winrm connections provide a PowerShell shell automatically - # TODO: only activate in local mode - class PowerShellCommand < CommandWrapperBase - Train::Options.attach(self) - - def initialize(backend, options) - @backend = backend - validate_options(options) - end - - def run(script) - # wrap the script to ensure we always run it via powershell - # especially in local mode, we cannot be sure that we get a Powershell - # we may just get a `cmd`. - # TODO: we may want to opt for powershell.exe -command instead of `encodeCommand` - "powershell -NoProfile -encodedCommand #{encoded(safe_script(script))}" - end - - # suppress the progress stream from leaking to stderr - def safe_script(script) - "$ProgressPreference='SilentlyContinue';" + script - end - - # Encodes the script so that it can be passed to the PowerShell - # --EncodedCommand argument. - # @return [String] The UTF-16LE base64 encoded script - def encoded(script) - encoded_script = safe_script(script).encode('UTF-16LE', 'UTF-8') - Base64.strict_encode64(encoded_script) - end - - def to_s - 'PowerShell CommandWrapper' - end - end - class CommandWrapper include_options LinuxCommand @@ -168,10 +131,6 @@ def self.load(transport, options) msg = res.verify fail Train::UserError, "Sudo failed: #{msg}" unless msg.nil? res - # only use powershell command for local transport. winrm transport - # uses powershell as default - elsif transport.os.windows? && transport.class == Train::Transports::Local::Connection - PowerShellCommand.new(transport, options) end end end diff --git a/lib/train/transports/local.rb b/lib/train/transports/local.rb index ac3bb859..42767242 100644 --- a/lib/train/transports/local.rb +++ b/lib/train/transports/local.rb @@ -16,28 +16,18 @@ def connection(_ = nil) @connection ||= Connection.new(@options) end - class Connection < BaseConnection # rubocop:disable Metrics/ClassLength - require 'json' - require 'base64' - require 'securerandom' - + class Connection < BaseConnection def initialize(options) super(options) @cmd_wrapper = nil @cmd_wrapper = CommandWrapper.load(self, options) - if @platform.windows? - begin - @pipe = acquire_named_pipe - rescue - @pipe = false - end - end + @platform = platform end def run_command(cmd) - if defined?(@pipe) && @pipe - res = run_powershell_using_named_pipe(cmd) - CommandResult.new(res['stdout'], res['stderr'], res['exitstatus']) + if defined?(@platform) && @platform.windows? + @windows_runner ||= WindowsRunner.new + @windows_runner.run_command(cmd) else cmd = @cmd_wrapper.run(cmd) unless @cmd_wrapper.nil? res = Mixlib::ShellOut.new(cmd) @@ -53,12 +43,11 @@ def local? end def file(path) - @files[path] ||= \ - if os.windows? - Train::File::Local::Windows.new(self, path) - else - Train::File::Local::Unix.new(self, path) - end + @files[path] ||= if os.windows? + Train::File::Local::Windows.new(self, path) + else + Train::File::Local::Unix.new(self, path) + end end def login_command @@ -69,81 +58,131 @@ def uri 'local://' end - private + class WindowsRunner + require 'json' + require 'base64' + require 'securerandom' + + def initialize + @pipe = acquire_pipe + end + + def run_command(cmd) + res = @pipe ? run_via_pipe(cmd) : run_via_shellout(cmd) + Local::CommandResult.new(res.stdout, res.stderr, res.exitstatus) + rescue Errno::ENOENT => _ + CommandResult.new('', '', 1) + end + + private + + def acquire_pipe + pipe_name = Dir.entries('//./pipe/').find { |f| f =~ /inspec_/ } - def acquire_named_pipe - pipe_name = "inspec_#{SecureRandom.hex}" - pipe_location = "//localhost/pipe/#{pipe_name}" - start_named_pipe_server(pipe_name) unless File.exist?(pipe_location) + return create_pipe("inspec_#{SecureRandom.hex}") if pipe_name.nil? - # Try to acquire pipe for 10 seconds with 0.1 second intervals. - # This allows time for PowerShell to start the pipe - pipe = nil - 100.times do begin - pipe = open(pipe_location, 'r+') - break + pipe = open("//./pipe/#{pipe_name}", 'r+') rescue - sleep 0.1 + # Pipes are closed when a Train connection ends. When running + # multiple independent scans (e.g. Unit tests) the pipe will be + # unavailable because the previous process is closing it. + # This creates a new pipe in that case + pipe = create_pipe("inspec_#{SecureRandom.hex}") end + + return false if pipe.nil? + + pipe end - fail "Could not open named pipe #{pipe_location}" if pipe.nil? - pipe - end + def create_pipe(pipe_name) + start_pipe_server(pipe_name) - def run_powershell_using_named_pipe(script) - script = "$ProgressPreference='SilentlyContinue';" + script - encoded_script = Base64.strict_encode64(script) - @pipe.puts(encoded_script) - @pipe.flush - JSON.parse(Base64.decode64(@pipe.readline)) - end + pipe = nil - def start_named_pipe_server(pipe_name) - require 'win32/process' + # PowerShell needs time to create pipe. + 100.times do + begin + pipe = open("//./pipe/#{pipe_name}", 'r+') + break + rescue + sleep 0.1 + end + end - script = <<-EOF - $ErrorActionPreference = 'Stop' + return false if pipe.nil? - $pipeServer = New-Object System.IO.Pipes.NamedPipeServerStream('#{pipe_name}') - $pipeReader = New-Object System.IO.StreamReader($pipeServer) - $pipeWriter = New-Object System.IO.StreamWriter($pipeServer) + pipe + end - $pipeServer.WaitForConnection() + def run_via_shellout(script) + # Prevent progress stream from leaking into stderr + script = "$ProgressPreference='SilentlyContinue';" + script - # Create loop to receive and process user commands/scripts - $clientConnected = $true - while($clientConnected) { - $input = $pipeReader.ReadLine() - $command = [System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String($input)) + # Encode script so PowerShell can use it + script = script.encode('UTF-16LE', 'UTF-8') + base64_script = Base64.strict_encode64(script) - # Execute user command/script and convert result to JSON - $scriptBlock = $ExecutionContext.InvokeCommand.NewScriptBlock($command) - try { - $stdout = & $scriptBlock | Out-String - $result = @{ 'stdout' = $stdout ; 'stderr' = ''; 'exitstatus' = 0 } - } catch { - $stderr = $_ | Out-String - $result = @{ 'stdout' = ''; 'stderr' = $_; 'exitstatus' = 1 } - } - $resultJSON = $result | ConvertTo-JSON + cmd = "powershell -NoProfile -EncodedCommand #{base64_script}" - # Encode JSON in Base64 and write to pipe - $encodedResult = [System.Convert]::ToBase64String([System.Text.Encoding]::UTF8.GetBytes($resultJSON)) - $pipeWriter.WriteLine($encodedResult) - $pipeWriter.Flush() - } - EOF + res = Mixlib::ShellOut.new(cmd) + res.run_command + end - utf8_script = script.encode('UTF-16LE', 'UTF-8') - base64_script = Base64.strict_encode64(utf8_script) - cmd = "powershell -NoProfile -ExecutionPolicy bypass -NonInteractive -EncodedCommand #{base64_script}" + def run_via_pipe(script) + script = "$ProgressPreference='SilentlyContinue';" + script + encoded_script = Base64.strict_encode64(script) + @pipe.puts(encoded_script) + @pipe.flush + OpenStruct.new(JSON.parse(Base64.decode64(@pipe.readline))) + end - server_pid = Process.create(command_line: cmd).process_id + def start_pipe_server(pipe_name) + require 'win32/process' + + script = <<-EOF + $ErrorActionPreference = 'Stop' + + $pipeServer = New-Object System.IO.Pipes.NamedPipeServerStream('#{pipe_name}') + $pipeReader = New-Object System.IO.StreamReader($pipeServer) + $pipeWriter = New-Object System.IO.StreamWriter($pipeServer) + + $pipeServer.WaitForConnection() + + # Create loop to receive and process user commands/scripts + $clientConnected = $true + while($clientConnected) { + $input = $pipeReader.ReadLine() + $command = [System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String($input)) + + # Execute user command/script and convert result to JSON + $scriptBlock = $ExecutionContext.InvokeCommand.NewScriptBlock($command) + try { + $stdout = & $scriptBlock | Out-String + $result = @{ 'stdout' = $stdout ; 'stderr' = ''; 'exitstatus' = 0 } + } catch { + $stderr = $_ | Out-String + $result = @{ 'stdout' = ''; 'stderr' = $_; 'exitstatus' = 1 } + } + $resultJSON = $result | ConvertTo-JSON + + # Encode JSON in Base64 and write to pipe + $encodedResult = [System.Convert]::ToBase64String([System.Text.Encoding]::UTF8.GetBytes($resultJSON)) + $pipeWriter.WriteLine($encodedResult) + $pipeWriter.Flush() + } + EOF - # Ensure process is killed when the Train process exits - at_exit { Process.kill('KILL', server_pid) } + utf8_script = script.encode('UTF-16LE', 'UTF-8') + base64_script = Base64.strict_encode64(utf8_script) + cmd = "powershell -NoProfile -ExecutionPolicy bypass -NonInteractive -EncodedCommand #{base64_script}" + + server_pid = Process.create(command_line: cmd).process_id + + # Ensure process is killed when the Train process exits + at_exit { Process.kill('KILL', server_pid) } + end end def run_command_via_connection(cmd) diff --git a/test/windows/local_test.rb b/test/windows/local_test.rb index a6517749..fc2e7c74 100644 --- a/test/windows/local_test.rb +++ b/test/windows/local_test.rb @@ -40,6 +40,23 @@ cmd.stderr.must_equal '' end + it 'when named pipe is not available it runs `Mixlib::Shellout`' do + # Must call `:conn` early so we can stub `:acquire_pipe` + connection = conn + + # Prevent named pipe from being created + Train::Transports::Local::Connection::WindowsRunner + .any_instance + .stubs(:acquire_pipe) + .returns(false) + + # Verify pipe was not created + SecureRandom.stubs(:hex).returns('minitest') + cmd = connection.run_command('Get-ChildItem //./pipe/ | Where-Object { $_.Name -Match "inspec_minitest" }') + cmd.stdout.must_equal '' + cmd.stderr.must_equal '' + end + describe 'file' do before do @temp = Tempfile.new('foo') From 776b8da195eee85e8646c4a66dddddd4077ae009 Mon Sep 17 00:00:00 2001 From: Jerry Aldrich Date: Mon, 27 Nov 2017 08:14:54 -0600 Subject: [PATCH 09/26] Revert file code format Signed-off-by: Jerry Aldrich --- lib/train/transports/local.rb | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/lib/train/transports/local.rb b/lib/train/transports/local.rb index 42767242..dc1e86ae 100644 --- a/lib/train/transports/local.rb +++ b/lib/train/transports/local.rb @@ -43,11 +43,12 @@ def local? end def file(path) - @files[path] ||= if os.windows? - Train::File::Local::Windows.new(self, path) - else - Train::File::Local::Unix.new(self, path) - end + @files[path] ||= \ + if os.windows? + Train::File::Local::Windows.new(self, path) + else + Train::File::Local::Unix.new(self, path) + end end def login_command From 44a7a6a0c605b0f9b2e2faffbfd4492801c50d43 Mon Sep 17 00:00:00 2001 From: Jerry Aldrich Date: Mon, 27 Nov 2017 08:16:07 -0600 Subject: [PATCH 10/26] Change `@platform` to `@os` for consistency Signed-off-by: Jerry Aldrich --- lib/train/transports/local.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/train/transports/local.rb b/lib/train/transports/local.rb index dc1e86ae..ae1eae97 100644 --- a/lib/train/transports/local.rb +++ b/lib/train/transports/local.rb @@ -21,11 +21,11 @@ def initialize(options) super(options) @cmd_wrapper = nil @cmd_wrapper = CommandWrapper.load(self, options) - @platform = platform + @os = os end def run_command(cmd) - if defined?(@platform) && @platform.windows? + if defined?(@os) && @os.windows? @windows_runner ||= WindowsRunner.new @windows_runner.run_command(cmd) else From 9681fe6cf7e707dce6f28c2ea85faab18a1096c8 Mon Sep 17 00:00:00 2001 From: Jerry Aldrich Date: Mon, 27 Nov 2017 08:50:09 -0600 Subject: [PATCH 11/26] Split Windows runners and create a `GenericRunner` Signed-off-by: Jerry Aldrich --- lib/train/transports/local.rb | 106 ++++++++++++++++++++-------------- test/windows/local_test.rb | 4 +- 2 files changed, 66 insertions(+), 44 deletions(-) diff --git a/lib/train/transports/local.rb b/lib/train/transports/local.rb index ae1eae97..b8154558 100644 --- a/lib/train/transports/local.rb +++ b/lib/train/transports/local.rb @@ -12,6 +12,8 @@ class Local < Train.plugin(1) include_options Train::Extras::CommandWrapper + class PipeError < ::StandardError; end + def connection(_ = nil) @connection ||= Connection.new(@options) end @@ -30,9 +32,8 @@ def run_command(cmd) @windows_runner.run_command(cmd) else cmd = @cmd_wrapper.run(cmd) unless @cmd_wrapper.nil? - res = Mixlib::ShellOut.new(cmd) - res.run_command - CommandResult.new(res.stdout, res.stderr, res.exitstatus) + @generic_runner ||= GenericRunner.new + @generic_runner.run_command(cmd) end rescue Errno::ENOENT => _ CommandResult.new('', '', 1) @@ -59,41 +60,86 @@ def uri 'local://' end + class GenericRunner + def run_command(cmd) + res = Mixlib::ShellOut.new(cmd) + res.run_command + Local::CommandResult.new(res.stdout, res.stderr, res.exitstatus) + end + end + class WindowsRunner + # Attempt to use a named pipe but fallback to ShellOut if that fails + def initialize + @runner = WindowsPipeRunner.new + rescue PipeError + @runner = WindowsShellRunner.new + end + + def run_command(cmd) + @runner.run_command(cmd) + end + end + + class WindowsShellRunner + require 'json' + require 'base64' + + def run_command(script) + # Prevent progress stream from leaking into stderr + script = "$ProgressPreference='SilentlyContinue';" + script + + # Encode script so PowerShell can use it + script = script.encode('UTF-16LE', 'UTF-8') + base64_script = Base64.strict_encode64(script) + + cmd = "powershell -NoProfile -EncodedCommand #{base64_script}" + + res = Mixlib::ShellOut.new(cmd) + res.run_command + Local::CommandResult.new(res.stdout, res.stderr, res.exitstatus) + end + end + + class WindowsPipeRunner require 'json' require 'base64' require 'securerandom' def initialize @pipe = acquire_pipe + fail PipeError if @pipe.nil? end def run_command(cmd) - res = @pipe ? run_via_pipe(cmd) : run_via_shellout(cmd) + script = "$ProgressPreference='SilentlyContinue';" + cmd + encoded_script = Base64.strict_encode64(script) + @pipe.puts(encoded_script) + @pipe.flush + res = OpenStruct.new(JSON.parse(Base64.decode64(@pipe.readline))) Local::CommandResult.new(res.stdout, res.stderr, res.exitstatus) - rescue Errno::ENOENT => _ - CommandResult.new('', '', 1) end private def acquire_pipe - pipe_name = Dir.entries('//./pipe/').find { |f| f =~ /inspec_/ } - - return create_pipe("inspec_#{SecureRandom.hex}") if pipe_name.nil? + current_pipe = Dir.entries('//./pipe/').find { |f| f =~ /inspec_/ } - begin - pipe = open("//./pipe/#{pipe_name}", 'r+') - rescue - # Pipes are closed when a Train connection ends. When running - # multiple independent scans (e.g. Unit tests) the pipe will be - # unavailable because the previous process is closing it. - # This creates a new pipe in that case + pipe = nil + if current_pipe.nil? pipe = create_pipe("inspec_#{SecureRandom.hex}") + else + begin + pipe = open("//./pipe/#{current_pipe}", 'r+') + rescue + # Pipes are closed when a Train connection ends. When running + # multiple independent scans (e.g. Unit tests) the pipe will be + # unavailable because the previous process is closing it. + # This creates a new pipe in that case + pipe = create_pipe("inspec_#{SecureRandom.hex}") + end end - return false if pipe.nil? - pipe end @@ -112,33 +158,9 @@ def create_pipe(pipe_name) end end - return false if pipe.nil? - pipe end - def run_via_shellout(script) - # Prevent progress stream from leaking into stderr - script = "$ProgressPreference='SilentlyContinue';" + script - - # Encode script so PowerShell can use it - script = script.encode('UTF-16LE', 'UTF-8') - base64_script = Base64.strict_encode64(script) - - cmd = "powershell -NoProfile -EncodedCommand #{base64_script}" - - res = Mixlib::ShellOut.new(cmd) - res.run_command - end - - def run_via_pipe(script) - script = "$ProgressPreference='SilentlyContinue';" + script - encoded_script = Base64.strict_encode64(script) - @pipe.puts(encoded_script) - @pipe.flush - OpenStruct.new(JSON.parse(Base64.decode64(@pipe.readline))) - end - def start_pipe_server(pipe_name) require 'win32/process' diff --git a/test/windows/local_test.rb b/test/windows/local_test.rb index fc2e7c74..c787e258 100644 --- a/test/windows/local_test.rb +++ b/test/windows/local_test.rb @@ -45,10 +45,10 @@ connection = conn # Prevent named pipe from being created - Train::Transports::Local::Connection::WindowsRunner + Train::Transports::Local::Connection::WindowsPipeRunner .any_instance .stubs(:acquire_pipe) - .returns(false) + .returns(nil) # Verify pipe was not created SecureRandom.stubs(:hex).returns('minitest') From d1ad9878979294a230f9d34b64986aacfd6aa8fa Mon Sep 17 00:00:00 2001 From: Jerry Aldrich Date: Mon, 27 Nov 2017 09:46:46 -0600 Subject: [PATCH 12/26] Add test for Windows named pipe usage Signed-off-by: Jerry Aldrich --- test/windows/local_test.rb | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/test/windows/local_test.rb b/test/windows/local_test.rb index c787e258..1cf1c196 100644 --- a/test/windows/local_test.rb +++ b/test/windows/local_test.rb @@ -40,6 +40,14 @@ cmd.stderr.must_equal '' end + it 'uses a named pipe if available' do + # Verify pipe is created by stubbing the random parts of the name + SecureRandom.stubs(:hex).returns('with_pipe') + cmd = conn.run_command('Get-ChildItem //./pipe/ | Where-Object { $_.Name -Match "inspec_with_pipe" }') + cmd.stdout.wont_be_nil + cmd.stderr.must_equal '' + end + it 'when named pipe is not available it runs `Mixlib::Shellout`' do # Must call `:conn` early so we can stub `:acquire_pipe` connection = conn From 18a42454014714ad4fe685fa77b033353ab6928e Mon Sep 17 00:00:00 2001 From: Jerry Aldrich Date: Mon, 27 Nov 2017 09:54:32 -0600 Subject: [PATCH 13/26] Tweak `GenericRunner` and default `run_command` Signed-off-by: Jerry Aldrich --- lib/train/transports/local.rb | 44 ++++++++++++++++------------------- 1 file changed, 20 insertions(+), 24 deletions(-) diff --git a/lib/train/transports/local.rb b/lib/train/transports/local.rb index b8154558..771a52fd 100644 --- a/lib/train/transports/local.rb +++ b/lib/train/transports/local.rb @@ -21,20 +21,23 @@ def connection(_ = nil) class Connection < BaseConnection def initialize(options) super(options) - @cmd_wrapper = nil - @cmd_wrapper = CommandWrapper.load(self, options) - @os = os + + # While OS is being discovered, use the GenericRunner + @runner = GenericRunner.new + @runner.cmd_wrapper = Local::CommandWrapper.load(self, options) + + if os.windows? + # Attempt to use a named pipe but fallback to ShellOut if that fails + begin + @runner = WindowsPipeRunner.new + rescue PipeError + @runner = WindowsShellRunner.new + end + end end def run_command(cmd) - if defined?(@os) && @os.windows? - @windows_runner ||= WindowsRunner.new - @windows_runner.run_command(cmd) - else - cmd = @cmd_wrapper.run(cmd) unless @cmd_wrapper.nil? - @generic_runner ||= GenericRunner.new - @generic_runner.run_command(cmd) - end + @runner.run_command(cmd) rescue Errno::ENOENT => _ CommandResult.new('', '', 1) end @@ -61,26 +64,19 @@ def uri end class GenericRunner + attr_writer :cmd_wrapper + def run_command(cmd) + if defined?(@cmd_wrapper) && !@cmd_wrapper.nil? + cmd = @cmd_wrapper.run(cmd) + end + res = Mixlib::ShellOut.new(cmd) res.run_command Local::CommandResult.new(res.stdout, res.stderr, res.exitstatus) end end - class WindowsRunner - # Attempt to use a named pipe but fallback to ShellOut if that fails - def initialize - @runner = WindowsPipeRunner.new - rescue PipeError - @runner = WindowsShellRunner.new - end - - def run_command(cmd) - @runner.run_command(cmd) - end - end - class WindowsShellRunner require 'json' require 'base64' From d3b6f860ee3a48659e330f0b2496e23a5dedfff7 Mon Sep 17 00:00:00 2001 From: Jerry Aldrich Date: Mon, 27 Nov 2017 10:27:40 -0600 Subject: [PATCH 14/26] Remove `Local::` from `CommandWrapper` Signed-off-by: Jerry Aldrich --- lib/train/transports/local.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/train/transports/local.rb b/lib/train/transports/local.rb index 771a52fd..9f169937 100644 --- a/lib/train/transports/local.rb +++ b/lib/train/transports/local.rb @@ -24,7 +24,7 @@ def initialize(options) # While OS is being discovered, use the GenericRunner @runner = GenericRunner.new - @runner.cmd_wrapper = Local::CommandWrapper.load(self, options) + @runner.cmd_wrapper = CommandWrapper.load(self, options) if os.windows? # Attempt to use a named pipe but fallback to ShellOut if that fails From bd2c7c38718119e3b7d90553748986011b45bba9 Mon Sep 17 00:00:00 2001 From: Jerry Aldrich Date: Mon, 27 Nov 2017 10:49:49 -0600 Subject: [PATCH 15/26] Fix test for Windows named pipe Signed-off-by: Jerry Aldrich --- test/windows/local_test.rb | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/test/windows/local_test.rb b/test/windows/local_test.rb index 1cf1c196..8ee07956 100644 --- a/test/windows/local_test.rb +++ b/test/windows/local_test.rb @@ -41,9 +41,11 @@ end it 'uses a named pipe if available' do - # Verify pipe is created by stubbing the random parts of the name + # Must call `:conn` early so we can stub `SecureRandom` + connection = conn + SecureRandom.stubs(:hex).returns('with_pipe') - cmd = conn.run_command('Get-ChildItem //./pipe/ | Where-Object { $_.Name -Match "inspec_with_pipe" }') + cmd = connection.run_command('Get-ChildItem //./pipe/ | Where-Object { $_.Name -Match "inspec_with_pipe" }') cmd.stdout.wont_be_nil cmd.stderr.must_equal '' end From e7d7d32f5389bd4ad0ebd9f1fcca69f579f0ccbd Mon Sep 17 00:00:00 2001 From: Jerry Aldrich Date: Mon, 27 Nov 2017 12:24:02 -0600 Subject: [PATCH 16/26] Make changes to integrate @jquick's cacheing magic Signed-off-by: Jerry Aldrich --- lib/train/transports/local.rb | 54 +++++++++++------------------------ 1 file changed, 16 insertions(+), 38 deletions(-) diff --git a/lib/train/transports/local.rb b/lib/train/transports/local.rb index 9f169937..bedb8be6 100644 --- a/lib/train/transports/local.rb +++ b/lib/train/transports/local.rb @@ -36,25 +36,10 @@ def initialize(options) end end - def run_command(cmd) - @runner.run_command(cmd) - rescue Errno::ENOENT => _ - CommandResult.new('', '', 1) - end - def local? true end - def file(path) - @files[path] ||= \ - if os.windows? - Train::File::Local::Windows.new(self, path) - else - Train::File::Local::Unix.new(self, path) - end - end - def login_command nil # none, open your shell end @@ -63,6 +48,22 @@ def uri 'local://' end + private + + def run_command_via_connection(cmd) + @runner.run_command(cmd) + rescue Errno::ENOENT => _ + CommandResult.new('', '', 1) + end + + def file_via_connection(path) + if os.windows? + Train::File::Local::Windows.new(self, path) + else + Train::File::Local::Unix.new(self, path) + end + end + class GenericRunner attr_writer :cmd_wrapper @@ -203,29 +204,6 @@ def start_pipe_server(pipe_name) at_exit { Process.kill('KILL', server_pid) } end end - - def run_command_via_connection(cmd) - if defined?(@platform) && @platform.windows? - start_named_pipe_server unless File.exist?('//localhost/pipe/InSpec') - res = run_powershell_using_named_pipe(cmd) - CommandResult.new(res['stdout'], res['stderr'], res['exitstatus']) - else - cmd = @cmd_wrapper.run(cmd) unless @cmd_wrapper.nil? - res = Mixlib::ShellOut.new(cmd) - res.run_command - CommandResult.new(res.stdout, res.stderr, res.exitstatus) - end - rescue Errno::ENOENT => _ - CommandResult.new('', '', 1) - end - - def file_via_connection(path) - if os.windows? - Train::File::Local::Windows.new(self, path) - else - Train::File::Local::Unix.new(self, path) - end - end end end end From ecad39f1e0f98335adadb76727ce0aad72393ef3 Mon Sep 17 00:00:00 2001 From: Jerry Aldrich Date: Mon, 27 Nov 2017 16:31:45 -0600 Subject: [PATCH 17/26] Remove unneeded logic for opening an existing pipe A pipe is opened only once per object, multiple processes shouldn't share the same pipe because it could be terminated by the other process. Signed-off-by: Jerry Aldrich --- lib/train/transports/local.rb | 21 +-------------------- 1 file changed, 1 insertion(+), 20 deletions(-) diff --git a/lib/train/transports/local.rb b/lib/train/transports/local.rb index bedb8be6..4f98b323 100644 --- a/lib/train/transports/local.rb +++ b/lib/train/transports/local.rb @@ -120,27 +120,8 @@ def run_command(cmd) private def acquire_pipe - current_pipe = Dir.entries('//./pipe/').find { |f| f =~ /inspec_/ } + pipe_name = "inspec_#{SecureRandom.hex}" - pipe = nil - if current_pipe.nil? - pipe = create_pipe("inspec_#{SecureRandom.hex}") - else - begin - pipe = open("//./pipe/#{current_pipe}", 'r+') - rescue - # Pipes are closed when a Train connection ends. When running - # multiple independent scans (e.g. Unit tests) the pipe will be - # unavailable because the previous process is closing it. - # This creates a new pipe in that case - pipe = create_pipe("inspec_#{SecureRandom.hex}") - end - end - - pipe - end - - def create_pipe(pipe_name) start_pipe_server(pipe_name) pipe = nil From 75170452f5e09fdd7c4ef25b38bc4fa191955f21 Mon Sep 17 00:00:00 2001 From: Jerry Aldrich Date: Thu, 30 Nov 2017 11:34:00 -0600 Subject: [PATCH 18/26] Add comments explaining pipe check tests Signed-off-by: Jerry Aldrich --- test/windows/local_test.rb | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/test/windows/local_test.rb b/test/windows/local_test.rb index 8ee07956..8ac89b9b 100644 --- a/test/windows/local_test.rb +++ b/test/windows/local_test.rb @@ -44,7 +44,8 @@ # Must call `:conn` early so we can stub `SecureRandom` connection = conn - SecureRandom.stubs(:hex).returns('with_pipe') + # Verify pipe is created by using PowerShell to check pipe location + SecureRandom.stubwass(:hex).returns('with_pipe') cmd = connection.run_command('Get-ChildItem //./pipe/ | Where-Object { $_.Name -Match "inspec_with_pipe" }') cmd.stdout.wont_be_nil cmd.stderr.must_equal '' @@ -60,7 +61,7 @@ .stubs(:acquire_pipe) .returns(nil) - # Verify pipe was not created + # Verify pipe was not created by using PowerShell to check pipe location SecureRandom.stubs(:hex).returns('minitest') cmd = connection.run_command('Get-ChildItem //./pipe/ | Where-Object { $_.Name -Match "inspec_minitest" }') cmd.stdout.must_equal '' From d494df245dfaced9bfb745a95b03076a298d0b91 Mon Sep 17 00:00:00 2001 From: Jerry Aldrich Date: Thu, 30 Nov 2017 11:38:55 -0600 Subject: [PATCH 19/26] Change `Shellout` to `ShellOut` Signed-off-by: Jerry Aldrich --- test/windows/local_test.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/windows/local_test.rb b/test/windows/local_test.rb index 8ac89b9b..a43877e5 100644 --- a/test/windows/local_test.rb +++ b/test/windows/local_test.rb @@ -51,7 +51,7 @@ cmd.stderr.must_equal '' end - it 'when named pipe is not available it runs `Mixlib::Shellout`' do + it 'when named pipe is not available it runs `Mixlib::ShellOut`' do # Must call `:conn` early so we can stub `:acquire_pipe` connection = conn From ab34ccc06983833984706dbc973f61fbb99f2a81 Mon Sep 17 00:00:00 2001 From: Jerry Aldrich Date: Thu, 30 Nov 2017 11:49:08 -0600 Subject: [PATCH 20/26] Change `stubs` to `expects` where appropriate Signed-off-by: Jerry Aldrich --- test/windows/local_test.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/windows/local_test.rb b/test/windows/local_test.rb index a43877e5..16c67e04 100644 --- a/test/windows/local_test.rb +++ b/test/windows/local_test.rb @@ -45,7 +45,7 @@ connection = conn # Verify pipe is created by using PowerShell to check pipe location - SecureRandom.stubwass(:hex).returns('with_pipe') + SecureRandom.expects(:hex).returns('with_pipe') cmd = connection.run_command('Get-ChildItem //./pipe/ | Where-Object { $_.Name -Match "inspec_with_pipe" }') cmd.stdout.wont_be_nil cmd.stderr.must_equal '' @@ -58,7 +58,7 @@ # Prevent named pipe from being created Train::Transports::Local::Connection::WindowsPipeRunner .any_instance - .stubs(:acquire_pipe) + .expects(:acquire_pipe) .returns(nil) # Verify pipe was not created by using PowerShell to check pipe location From 824f2e385bd015e5ef33d40b0f1d3ed13924b868 Mon Sep 17 00:00:00 2001 From: Jerry Aldrich Date: Thu, 30 Nov 2017 12:06:40 -0600 Subject: [PATCH 21/26] Modify pipe tests Signed-off-by: Jerry Aldrich --- test/windows/local_test.rb | 34 ++++++++++++++++++---------------- 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/test/windows/local_test.rb b/test/windows/local_test.rb index 16c67e04..70d0d032 100644 --- a/test/windows/local_test.rb +++ b/test/windows/local_test.rb @@ -8,6 +8,9 @@ require 'train' require 'tempfile' +# Loading here to ensure methods exist to be stubbed +require 'train/transports/local' + describe 'windows local command' do let(:conn) { # get final config @@ -40,32 +43,31 @@ cmd.stderr.must_equal '' end - it 'uses a named pipe if available' do - # Must call `:conn` early so we can stub `SecureRandom` - connection = conn - - # Verify pipe is created by using PowerShell to check pipe location + # Verify pipe is created by using PowerShell to check pipe location. This + # works by intercepting the `SecureRandom` call which controls the pipe + # name. If this command uses a pipe, then the `stdout` of this command will + # contain the pipe name. + it 'can execute a command using a named pipe' do SecureRandom.expects(:hex).returns('with_pipe') - cmd = connection.run_command('Get-ChildItem //./pipe/ | Where-Object { $_.Name -Match "inspec_with_pipe" }') - cmd.stdout.wont_be_nil + + cmd = conn.run_command('Get-ChildItem //./pipe/ | Where-Object { $_.Name -Match "inspec_with_pipe" }') + cmd.stdout.must_match /inspec_with_pipe/ cmd.stderr.must_equal '' end - it 'when named pipe is not available it runs `Mixlib::ShellOut`' do - # Must call `:conn` early so we can stub `:acquire_pipe` - connection = conn - + it 'when named pipe is not available it uses `WindowsShellRunner`' do # Prevent named pipe from being created Train::Transports::Local::Connection::WindowsPipeRunner .any_instance .expects(:acquire_pipe) .returns(nil) - # Verify pipe was not created by using PowerShell to check pipe location - SecureRandom.stubs(:hex).returns('minitest') - cmd = connection.run_command('Get-ChildItem //./pipe/ | Where-Object { $_.Name -Match "inspec_minitest" }') - cmd.stdout.must_equal '' - cmd.stderr.must_equal '' + Train::Transports::Local::Connection::WindowsShellRunner + .any_instance + .expects(:run_command) + + # Run command to trigger our stubs/expects + conn.run_command('Write-Host "using ShellOut"') end describe 'file' do From 9965ef894c139d2e31f57e21b695aaffe9664063 Mon Sep 17 00:00:00 2001 From: Jerry Aldrich Date: Fri, 1 Dec 2017 17:25:21 -0600 Subject: [PATCH 22/26] Move/Modify Unit tests and add Integration tests Signed-off-by: Jerry Aldrich --- test/unit/transports/local_test.rb | 35 ++++++++++++++++++++++++++++++ test/windows/local_test.rb | 33 ++++++++++++++++------------ 2 files changed, 54 insertions(+), 14 deletions(-) diff --git a/test/unit/transports/local_test.rb b/test/unit/transports/local_test.rb index 34aae961..52a57576 100644 --- a/test/unit/transports/local_test.rb +++ b/test/unit/transports/local_test.rb @@ -93,4 +93,39 @@ def mock_run_cmd(cmd, &block) end end end + + describe 'when running on Windows' do + let(:conn) do + Train::Platforms::Platform.any_instance.stubs(:windows?).returns(true) + Train::Transports::Local.new.connection + end + + let(:runner) { mock } + + it 'uses `WindowsPipeRunner` by default' do + Train::Transports::Local::Connection::WindowsPipeRunner + .expects(:new) + .returns(runner) + + Train::Transports::Local::Connection::WindowsShellRunner + .expects(:new) + .never + + runner.expects(:run_command).with('not actually executed') + conn.run_command('not actually executed') + end + + it 'uses `WindowsShellRunner` when a named pipe is not available' do + Train::Transports::Local::Connection::WindowsPipeRunner + .expects(:new) + .raises(Train::Transports::Local::PipeError) + + Train::Transports::Local::Connection::WindowsShellRunner + .expects(:new) + .returns(runner) + + runner.expects(:run_command).with('not actually executed') + conn.run_command('not actually executed') + end + end end diff --git a/test/windows/local_test.rb b/test/windows/local_test.rb index 70d0d032..c80c5e6e 100644 --- a/test/windows/local_test.rb +++ b/test/windows/local_test.rb @@ -43,31 +43,36 @@ cmd.stderr.must_equal '' end - # Verify pipe is created by using PowerShell to check pipe location. This - # works by intercepting the `SecureRandom` call which controls the pipe - # name. If this command uses a pipe, then the `stdout` of this command will - # contain the pipe name. it 'can execute a command using a named pipe' do - SecureRandom.expects(:hex).returns('with_pipe') + SecureRandom.expects(:hex).returns('via_pipe') - cmd = conn.run_command('Get-ChildItem //./pipe/ | Where-Object { $_.Name -Match "inspec_with_pipe" }') - cmd.stdout.must_match /inspec_with_pipe/ + Train::Transports::Local::Connection::WindowsShellRunner + .any_instance + .expects(:new) + .never + + cmd = conn.run_command('Write-Output "Create pipe"') + File.exist?('//./pipe/inspec_via_pipe').must_equal true + cmd.stdout.must_equal "Create pipe\r\n" cmd.stderr.must_equal '' end - it 'when named pipe is not available it uses `WindowsShellRunner`' do - # Prevent named pipe from being created + it 'can execute a command via ShellRunner if pipe creation fails' do + # By forcing `acquire_pipe` to fail to return a pipe, any attempts to create + # a `WindowsPipeRunner` object should fail. If we can still run a command, + # then we know that it was successfully executed by `Mixlib::ShellOut`. Train::Transports::Local::Connection::WindowsPipeRunner .any_instance .expects(:acquire_pipe) + .at_least_once .returns(nil) - Train::Transports::Local::Connection::WindowsShellRunner - .any_instance - .expects(:run_command) + proc { Train::Transports::Local::Connection::WindowsPipeRunner.new } + .must_raise(Train::Transports::Local::PipeError) - # Run command to trigger our stubs/expects - conn.run_command('Write-Host "using ShellOut"') + cmd = conn.run_command('Write-Output "test"') + cmd.stdout.must_equal "test\r\n" + cmd.stderr.must_equal '' end describe 'file' do From e63627f305c7f20b865901e5e66a053392821d41 Mon Sep 17 00:00:00 2001 From: Jared Quick Date: Sat, 2 Dec 2017 07:59:59 -0500 Subject: [PATCH 23/26] Fix local tests when running with others. Signed-off-by: Jared Quick --- test/unit/transports/local_test.rb | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/test/unit/transports/local_test.rb b/test/unit/transports/local_test.rb index 52a57576..b2c2032f 100644 --- a/test/unit/transports/local_test.rb +++ b/test/unit/transports/local_test.rb @@ -6,10 +6,11 @@ class TransportHelper attr_accessor :transport - def initialize + def initialize(type = :mock) Train::Platforms::Detect::Specifications::OS.load plat = Train::Platforms.name('mock').in_family('linux') plat.add_platform_methods + plat.stubs(:windows?).returns(true) if type == :windows Train::Platforms::Detect.stubs(:scan).returns(plat) @transport = Train::Transports::Local.new end @@ -96,8 +97,8 @@ def mock_run_cmd(cmd, &block) describe 'when running on Windows' do let(:conn) do - Train::Platforms::Platform.any_instance.stubs(:windows?).returns(true) - Train::Transports::Local.new.connection + TransportHelper.new(:windows).transport + transport.connection end let(:runner) { mock } From 17d9de7106e02b8a6b11af22ae78a425a0eb8739 Mon Sep 17 00:00:00 2001 From: Jerry Aldrich Date: Sat, 2 Dec 2017 19:28:14 -0600 Subject: [PATCH 24/26] Minor style changes Signed-off-by: Jerry Aldrich --- test/unit/transports/local_test.rb | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/test/unit/transports/local_test.rb b/test/unit/transports/local_test.rb index b2c2032f..91b528b3 100644 --- a/test/unit/transports/local_test.rb +++ b/test/unit/transports/local_test.rb @@ -8,7 +8,7 @@ class TransportHelper def initialize(type = :mock) Train::Platforms::Detect::Specifications::OS.load - plat = Train::Platforms.name('mock').in_family('linux') + plat = Train::Platforms.name('mock') plat.add_platform_methods plat.stubs(:windows?).returns(true) if type == :windows Train::Platforms::Detect.stubs(:scan).returns(plat) @@ -96,10 +96,7 @@ def mock_run_cmd(cmd, &block) end describe 'when running on Windows' do - let(:conn) do - TransportHelper.new(:windows).transport - transport.connection - end + let(:connection) { TransportHelper.new(:windows).transport.connection } let(:runner) { mock } @@ -113,7 +110,7 @@ def mock_run_cmd(cmd, &block) .never runner.expects(:run_command).with('not actually executed') - conn.run_command('not actually executed') + connection.run_command('not actually executed') end it 'uses `WindowsShellRunner` when a named pipe is not available' do @@ -126,7 +123,7 @@ def mock_run_cmd(cmd, &block) .returns(runner) runner.expects(:run_command).with('not actually executed') - conn.run_command('not actually executed') + connection.run_command('not actually executed') end end end From 6160bdeb296bcb682a5d59a9483bf7727f33896b Mon Sep 17 00:00:00 2001 From: Jerry Aldrich Date: Mon, 4 Dec 2017 12:18:50 -0600 Subject: [PATCH 25/26] Remove extra newline between `let`s Signed-off-by: Jerry Aldrich --- test/unit/transports/local_test.rb | 1 - 1 file changed, 1 deletion(-) diff --git a/test/unit/transports/local_test.rb b/test/unit/transports/local_test.rb index 91b528b3..cfc73d26 100644 --- a/test/unit/transports/local_test.rb +++ b/test/unit/transports/local_test.rb @@ -97,7 +97,6 @@ def mock_run_cmd(cmd, &block) describe 'when running on Windows' do let(:connection) { TransportHelper.new(:windows).transport.connection } - let(:runner) { mock } it 'uses `WindowsPipeRunner` by default' do From 0bfd1c1e011a0c937737ae62efacbcffd9d0ca83 Mon Sep 17 00:00:00 2001 From: Jerry Aldrich Date: Mon, 4 Dec 2017 15:48:14 -0600 Subject: [PATCH 26/26] Modify TransportHelper to accept platform options Signed-off-by: Jerry Aldrich --- test/unit/transports/local_test.rb | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/test/unit/transports/local_test.rb b/test/unit/transports/local_test.rb index cfc73d26..42a7ee8d 100644 --- a/test/unit/transports/local_test.rb +++ b/test/unit/transports/local_test.rb @@ -6,11 +6,12 @@ class TransportHelper attr_accessor :transport - def initialize(type = :mock) + def initialize(user_opts = {}) + opts = {platform_name: 'mock', family_hierarchy: ['mock']}.merge(user_opts) Train::Platforms::Detect::Specifications::OS.load - plat = Train::Platforms.name('mock') + plat = Train::Platforms.name(opts[:platform_name]) + plat.family_hierarchy = opts[:family_hierarchy] plat.add_platform_methods - plat.stubs(:windows?).returns(true) if type == :windows Train::Platforms::Detect.stubs(:scan).returns(plat) @transport = Train::Transports::Local.new end @@ -96,7 +97,9 @@ def mock_run_cmd(cmd, &block) end describe 'when running on Windows' do - let(:connection) { TransportHelper.new(:windows).transport.connection } + let(:connection) do + TransportHelper.new(family_hierarchy: ['windows']).transport.connection + end let(:runner) { mock } it 'uses `WindowsPipeRunner` by default' do