From 42aa10950f351fcb27d03d35eb57f2c1ef99ea71 Mon Sep 17 00:00:00 2001 From: Terence Lee Date: Fri, 8 Jul 2016 19:28:54 -0500 Subject: [PATCH 01/10] don't need these requires --- spec/support/buildpack_builder.rb | 2 -- 1 file changed, 2 deletions(-) diff --git a/spec/support/buildpack_builder.rb b/spec/support/buildpack_builder.rb index 2dee6160..62513fc3 100644 --- a/spec/support/buildpack_builder.rb +++ b/spec/support/buildpack_builder.rb @@ -1,5 +1,3 @@ -require "tmpdir" -require "fileutils" require "docker" require_relative "path_helper" From 375e679ccee4f8cfec4e9748969c30f014a008f9 Mon Sep 17 00:00:00 2001 From: Terence Lee Date: Tue, 12 Jul 2016 20:39:36 -0500 Subject: [PATCH 02/10] decrease container spin to once per test --- spec/simple_spec.rb | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/spec/simple_spec.rb b/spec/simple_spec.rb index 5f9ce087..40f12555 100644 --- a/spec/simple_spec.rb +++ b/spec/simple_spec.rb @@ -49,15 +49,17 @@ let(:name) { "clean_urls" } it "should drop the .html extension from URLs" do - response = app.get("/foo") - expect(response.code).to eq("200") - expect(response.body.chomp).to eq("foobar") + app.run do + response = app.get("/foo") + expect(response.code).to eq("200") + expect(response.body.chomp).to eq("foobar") - response = app.get("/bar") - expect(response.code).to eq("301") - response = app.get(response["Location"]) - expect(response.code).to eq("200") - expect(response.body.chomp).to eq("bar") + response = app.get("/bar") + expect(response.code).to eq("301") + response = app.get(response["Location"]) + expect(response.code).to eq("200") + expect(response.body.chomp).to eq("bar") + end end end From a60c0ad2899e8f4b787d4bc6889b184b2213b5a7 Mon Sep 17 00:00:00 2001 From: Terence Lee Date: Fri, 8 Jul 2016 19:29:59 -0500 Subject: [PATCH 03/10] move command to Dockerfile --- spec/support/app_runner.rb | 1 - spec/support/docker/Dockerfile | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/spec/support/app_runner.rb b/spec/support/app_runner.rb index 232689b7..0d811638 100644 --- a/spec/support/app_runner.rb +++ b/spec/support/app_runner.rb @@ -26,7 +26,6 @@ def initialize(fixture, env = {}, debug = false) @container = Docker::Container.create( "Image" => BuildpackBuilder::TAG, - "Cmd" => ["bash", "-c", "cp -rf /src/* /app/ && /app/bin/boot"], # Env format is [KEY1=VAL1 KEY2=VAL2] "Env" => env.to_a.map {|i| i.join("=") }, "HostConfig" => { diff --git a/spec/support/docker/Dockerfile b/spec/support/docker/Dockerfile index 6197f4a3..f3ed0b26 100644 --- a/spec/support/docker/Dockerfile +++ b/spec/support/docker/Dockerfile @@ -16,3 +16,4 @@ WORKDIR /app COPY ./spec/support/docker/init.sh /usr/bin/init.sh ENTRYPOINT ["/usr/bin/init.sh"] +CMD "/app/bin/boot" From ff012e3ae959f793f60e92901aa4a8fb26b69456 Mon Sep 17 00:00:00 2001 From: Terence Lee Date: Mon, 11 Jul 2016 18:16:56 -0500 Subject: [PATCH 04/10] rearchitect tests to support a real proxy API and routing frontend for SSL termination --- .../proxies/public_html/foo/bar/index.html | 1 - .../proxies/public_html/foo/baz/index.html | 1 - spec/simple_spec.rb | 52 ++++++++++--- spec/support/app_runner.rb | 77 +++++++++++++------ spec/support/buildpack_builder.rb | 25 +++--- spec/support/container_runner.rb | 29 +++++++ spec/support/docker/{ => app}/Dockerfile | 2 +- spec/support/docker/{ => app}/init.sh | 0 spec/support/docker/proxy/Dockerfile | 11 +++ spec/support/docker/proxy/Gemfile | 3 + spec/support/docker/proxy/Gemfile.lock | 20 +++++ spec/support/docker/proxy/config.ru | 7 ++ spec/support/docker/router/Dockerfile | 9 +++ .../docker/router/docker/conf/nginx.conf | 31 ++++++++ spec/support/docker/router/docker/hook/.keep | 0 spec/support/docker_builder.rb | 23 ++++++ spec/support/path_helper.rb | 4 + spec/support/proxy_builder.rb | 18 +++++ spec/support/proxy_runner.rb | 14 ++++ spec/support/router_builder.rb | 18 +++++ spec/support/router_runner.rb | 29 +++++++ 21 files changed, 320 insertions(+), 54 deletions(-) delete mode 100644 spec/fixtures/proxies/public_html/foo/bar/index.html delete mode 100644 spec/fixtures/proxies/public_html/foo/baz/index.html create mode 100644 spec/support/container_runner.rb rename spec/support/docker/{ => app}/Dockerfile (83%) rename spec/support/docker/{ => app}/init.sh (100%) create mode 100644 spec/support/docker/proxy/Dockerfile create mode 100644 spec/support/docker/proxy/Gemfile create mode 100644 spec/support/docker/proxy/Gemfile.lock create mode 100644 spec/support/docker/proxy/config.ru create mode 100644 spec/support/docker/router/Dockerfile create mode 100644 spec/support/docker/router/docker/conf/nginx.conf create mode 100644 spec/support/docker/router/docker/hook/.keep create mode 100644 spec/support/docker_builder.rb create mode 100644 spec/support/proxy_builder.rb create mode 100644 spec/support/proxy_runner.rb create mode 100644 spec/support/router_builder.rb create mode 100644 spec/support/router_runner.rb diff --git a/spec/fixtures/proxies/public_html/foo/bar/index.html b/spec/fixtures/proxies/public_html/foo/bar/index.html deleted file mode 100644 index eedd89b4..00000000 --- a/spec/fixtures/proxies/public_html/foo/bar/index.html +++ /dev/null @@ -1 +0,0 @@ -api diff --git a/spec/fixtures/proxies/public_html/foo/baz/index.html b/spec/fixtures/proxies/public_html/foo/baz/index.html deleted file mode 100644 index 76018072..00000000 --- a/spec/fixtures/proxies/public_html/foo/baz/index.html +++ /dev/null @@ -1 +0,0 @@ -baz diff --git a/spec/simple_spec.rb b/spec/simple_spec.rb index 40f12555..02cddb26 100644 --- a/spec/simple_spec.rb +++ b/spec/simple_spec.rb @@ -1,23 +1,30 @@ require "fileutils" require_relative "spec_helper" require_relative "support/app_runner" +require_relative "support/router_runner" require_relative "support/buildpack_builder" +require_relative "support/router_builder" +require_relative "support/proxy_builder" +require_relative "support/proxy_runner" require_relative "support/path_helper" RSpec.describe "Simple" do before(:all) do @debug = true BuildpackBuilder.new(@debug, ENV['CIRCLECI']) + RouterBuilder.new(@debug, ENV['CIRCLECI']) + ProxyBuilder.new(@debug, ENV["CIRCLE_CI"]) end after do app.destroy end - let(:app) { AppRunner.new(name, env, @debug) } + let(:proxy) { nil } + let(:app) { AppRunner.new(name, proxy, env, @debug) } - let(:name) { "hello_world" } - let(:env) { Hash.new } + let(:name) { "hello_world" } + let(:env) { Hash.new } it "should serve out of public_html by default" do response = app.get("/") @@ -89,7 +96,7 @@ it "should redirect and respect the http code & remove the port" do response = app.get("/old/gone") expect(response.code).to eq("302") - expect(response["location"]).to eq("http://#{AppRunner::HOST_IP}/") + expect(response["location"]).to eq("http://#{RouterRunner::HOST_IP}/") end context "interpolation" do @@ -102,7 +109,7 @@ it "should redirect using interpolated urls" do response = app.get("/old/interpolation") expect(response.code).to eq("302") - expect(response["location"]).to eq("http://#{AppRunner::HOST_IP}/interpolation.html") + expect(response["location"]).to eq("http://#{RouterRunner::HOST_IP}/interpolation.html") end end end @@ -133,6 +140,7 @@ include PathHelper let(:name) { "proxies" } + let(:proxy) { true } let(:static_json_path) { fixtures_path("proxies/static.json") } let(:setup_static_json) do Proc.new do |path| @@ -141,7 +149,7 @@ { "proxies": { "/api/": { - "origin": "http://#{AppRunner::HOST_IP}:#{AppRunner::HOST_PORT}#{path}" + "origin": "http://#{@proxy_ip_address}#{path}" } } } @@ -151,6 +159,10 @@ end end + before do + @proxy_ip_address = app.proxy.ip_address + end + after do FileUtils.rm(static_json_path) end @@ -186,10 +198,10 @@ { "proxies": { "/api/": { - "origin": "http://#{AppRunner::HOST_IP}:#{AppRunner::HOST_PORT}/foo" + "origin": "http://#{@proxy_ip_address}/foo" }, "/proxy/": { - "origin": "http://#{AppRunner::HOST_IP}:#{AppRunner::HOST_PORT}/foo" + "origin": "http://#{@proxy_ip_address}/foo" } }, "routes": { @@ -214,6 +226,14 @@ end context "env var substitution" do + let(:proxy) do + < "#{AppRunner::HOST_IP}:#{AppRunner::HOST_PORT}" + "PROXY_HOST" => "${PROXY_IP_ADDRESS}" } end @@ -347,6 +367,17 @@ let(:name) { "proxies" } let(:static_json_path) { fixtures_path("proxies/static.json") } + let(:proxy) do + < true) if @debug + @tmpdir = nil + @proxy = nil + env.merge!("STATIC_DEBUG" => "true") if @debug - @container = Docker::Container.create( + app_options = { + "name" => "app", "Image" => BuildpackBuilder::TAG, # Env format is [KEY1=VAL1 KEY2=VAL2] "Env" => env.to_a.map {|i| i.join("=") }, "HostConfig" => { - "Binds" => ["#{fixtures_path(fixture)}:/src"], - "PortBindings" => { - "#{CONTAINER_PORT}/tcp" => [{ - "HostIp" => HOST_IP, - "HostPort" => HOST_PORT, - }] - } + "Binds" => ["#{fixtures_path(fixture)}:/src"] } - ) + } + + if proxy + app_options["Links"] = ["proxy:proxy"] + if proxy.is_a?(String) + @tmpdir = Dir.mktmpdir + File.open("#{@tmpdir}/config.ru", "w") do |file| + file.puts %q{require "sinatra"} + file.puts proxy + file.puts "run Sinatra::Application" + end + end + + @proxy = ProxyRunner.new(@tmpdir) + @proxy.start + + # need to interpolate the PROXY_IP_ADDRESS since env is a parameter to this constructor and + # the proxy app needs to be started first to get the ip address docker provides. + # it's a bootstrapping problem to do env var substitution + env.select {|_, value| value.include?("${PROXY_IP_ADDRESS}") }.each do |key, value| + env[key] = NginxConfigUtil.interpolate(value, {"PROXY_IP_ADDRESS" => @proxy.ip_address}) + app_options["Env"] = env.to_a.map {|i| i.join("=") } + end + end + + @app = Docker::Container.create(app_options) + @router = RouterRunner.new end def run(capture_io = false) @@ -47,16 +65,17 @@ def run(capture_io = false) io_stream = StringIO.new run_thread = Thread.new { latch.wait(0.5) - yield(@container) + yield } container_thread = Thread.new { - @container.tap(&:start).attach do |stream, chunk| + @app.tap(&:start).attach do |stream, chunk| io_message = "#{stream}: #{chunk}" puts io_message if @debug io_stream << io_message if capture_io latch.count_down if chunk.include?("Starting nginx...") end } + @router.start retn = run_thread.value @@ -66,7 +85,8 @@ def run(capture_io = false) retn end ensure - @container.stop + @app.stop + @router.stop container_thread.join io_stream.close_write @run = false @@ -81,15 +101,22 @@ def get(path, capture_io = false, max_retries = 30) end def destroy - @container.delete(force: true) unless @debug + if @proxy + @proxy.stop + @proxy.destroy + end + @router.destroy + @app.delete(force: true) + ensure + FileUtils.rm_rf(@tmpdir) if @tmpdir end private def get_retry(path, max_retries) network_retry(max_retries) do uri = URI(path) - uri.host = HOST_IP if uri.host.nil? - uri.port = HOST_PORT if (uri.host == HOST_IP && uri.port != HOST_PORT) || uri.port.nil? + uri.host = RouterRunner::HOST_IP if uri.host.nil? + uri.port = RouterRunner::HOST_PORT if (uri.host == RouterRunner::HOST_IP && uri.port != RouterRunner::HOST_PORT) || uri.port.nil? uri.scheme = "http" if uri.scheme.nil? Net::HTTP.get_response(URI(uri.to_s)) diff --git a/spec/support/buildpack_builder.rb b/spec/support/buildpack_builder.rb index 62513fc3..2fc47f4e 100644 --- a/spec/support/buildpack_builder.rb +++ b/spec/support/buildpack_builder.rb @@ -1,28 +1,21 @@ -require "docker" require_relative "path_helper" +require_relative "docker_builder" class BuildpackBuilder include PathHelper + include DockerBuilder TAG = "hone/static:cedar-14" def initialize(debug = false, intermediates = false) @debug = debug @intermediates = intermediates - @image = build_image - end - - def build_image - print_output = - if @debug - -> (chunk) { - json = JSON.parse(chunk) - puts json["stream"] - } - else - -> (chunk) { nil } - end - - Docker::Image.build_from_dir(buildpack_path.to_s, 't' => TAG, 'rm' => !@intermediates, 'dockerfile' => "spec/support/docker/Dockerfile", &print_output) + @image = build( + context: buildpack_path.to_s, + dockerfile: docker_path("app/Dockerfile").relative_path_from(buildpack_path), + tag: TAG, + intermediates: @intermediates, + debug: @debug + ) end end diff --git a/spec/support/container_runner.rb b/spec/support/container_runner.rb new file mode 100644 index 00000000..bc80c963 --- /dev/null +++ b/spec/support/container_runner.rb @@ -0,0 +1,29 @@ +require "fiber" +require "docker" + +class ContainerRunner + attr_reader :ip_address + + def initialize(options) + @container = Docker::Container.create(options) + @ip_address = nil + @thread = nil + end + + def start + @thread = Fiber.new { + @container.start + Fiber.yield @container.json["NetworkSettings"]["IPAddress"] + } + @ip_address = @thread.resume + end + + def stop + @container.stop + @thread.resume if @thread.alive? + end + + def destroy + @container.delete(force: true) + end +end diff --git a/spec/support/docker/Dockerfile b/spec/support/docker/app/Dockerfile similarity index 83% rename from spec/support/docker/Dockerfile rename to spec/support/docker/app/Dockerfile index f3ed0b26..4f0df49a 100644 --- a/spec/support/docker/Dockerfile +++ b/spec/support/docker/app/Dockerfile @@ -14,6 +14,6 @@ EXPOSE 3000 WORKDIR /app -COPY ./spec/support/docker/init.sh /usr/bin/init.sh +COPY ./spec/support/docker/app/init.sh /usr/bin/init.sh ENTRYPOINT ["/usr/bin/init.sh"] CMD "/app/bin/boot" diff --git a/spec/support/docker/init.sh b/spec/support/docker/app/init.sh similarity index 100% rename from spec/support/docker/init.sh rename to spec/support/docker/app/init.sh diff --git a/spec/support/docker/proxy/Dockerfile b/spec/support/docker/proxy/Dockerfile new file mode 100644 index 00000000..a43860a6 --- /dev/null +++ b/spec/support/docker/proxy/Dockerfile @@ -0,0 +1,11 @@ +FROM ruby:2.3.1-alpine + +RUN mkdir -p /app +WORKDIR /app + +ADD Gemfile* /app/ +RUN bundle install --path /app/vendor/bundle +ADD config.ru /app/config/ + +EXPOSE 80 +CMD bundle exec rackup /app/config/config.ru --host 0.0.0.0 -p 80 diff --git a/spec/support/docker/proxy/Gemfile b/spec/support/docker/proxy/Gemfile new file mode 100644 index 00000000..94fc334d --- /dev/null +++ b/spec/support/docker/proxy/Gemfile @@ -0,0 +1,3 @@ +source "https://rubygems.org" + +gem "sinatra" diff --git a/spec/support/docker/proxy/Gemfile.lock b/spec/support/docker/proxy/Gemfile.lock new file mode 100644 index 00000000..70e41bec --- /dev/null +++ b/spec/support/docker/proxy/Gemfile.lock @@ -0,0 +1,20 @@ +GEM + remote: https://rubygems.org/ + specs: + rack (1.6.4) + rack-protection (1.5.3) + rack + sinatra (1.4.7) + rack (~> 1.5) + rack-protection (~> 1.4) + tilt (>= 1.3, < 3) + tilt (2.0.5) + +PLATFORMS + ruby + +DEPENDENCIES + sinatra + +BUNDLED WITH + 1.11.2 diff --git a/spec/support/docker/proxy/config.ru b/spec/support/docker/proxy/config.ru new file mode 100644 index 00000000..a1b63e8d --- /dev/null +++ b/spec/support/docker/proxy/config.ru @@ -0,0 +1,7 @@ +require "sinatra" + +get "/*" do + "api" +end + +run Sinatra::Application diff --git a/spec/support/docker/router/Dockerfile b/spec/support/docker/router/Dockerfile new file mode 100644 index 00000000..395b263b --- /dev/null +++ b/spec/support/docker/router/Dockerfile @@ -0,0 +1,9 @@ +FROM matsumotory/ngx-mruby:latest + +RUN echo $'\nUS\nTexas\nAustin\nHeroku\n\nexample.com\n\n' \ + | openssl req -x509 -nodes -days 365 -newkey rsa:1024 \ + -keyout /etc/ssl/private/myssl.key \ + -out /etc/ssl/certs/myssl.crt + +RUN mkdir -p /root/conf/ && \ + touch /root/conf/extend.conf diff --git a/spec/support/docker/router/docker/conf/nginx.conf b/spec/support/docker/router/docker/conf/nginx.conf new file mode 100644 index 00000000..d07915c7 --- /dev/null +++ b/spec/support/docker/router/docker/conf/nginx.conf @@ -0,0 +1,31 @@ +user daemon; +daemon off; +master_process off; +worker_processes 1; +error_log stderr; + +events { + worker_connections 1024; +} + +http { + upstream backend { + server app:3000; + } + + server { + listen 80; + listen 443 ssl; + + ssl_certificate /etc/ssl/certs/myssl.crt; + ssl_certificate_key /etc/ssl/private/myssl.key; + + location / { + proxy_pass http://backend; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + } + } +} diff --git a/spec/support/docker/router/docker/hook/.keep b/spec/support/docker/router/docker/hook/.keep new file mode 100644 index 00000000..e69de29b diff --git a/spec/support/docker_builder.rb b/spec/support/docker_builder.rb new file mode 100644 index 00000000..91c79ea2 --- /dev/null +++ b/spec/support/docker_builder.rb @@ -0,0 +1,23 @@ +require "docker" + +module DockerBuilder + def build(context:, tag:, intermediates:, debug:, dockerfile: nil) + print_output = + if debug + -> (chunk) { + json = JSON.parse(chunk) + puts json["stream"] + } + else + -> (chunk) { nil } + end + + options = { + 't' => tag, + 'rm' => !intermediates, + } + options["dockerfile"] = dockerfile + + Docker::Image.build_from_dir(context, options, &print_output) + end +end diff --git a/spec/support/path_helper.rb b/spec/support/path_helper.rb index 3495a027..7dbae218 100644 --- a/spec/support/path_helper.rb +++ b/spec/support/path_helper.rb @@ -6,6 +6,10 @@ def fixtures_path(*path) def buildpack_path(*path) __build_path("../../", *path) end + + def docker_path(*path) + __build_path("/docker", *path) + end private def __build_path(name, *path) diff --git a/spec/support/proxy_builder.rb b/spec/support/proxy_builder.rb new file mode 100644 index 00000000..5cc6cc17 --- /dev/null +++ b/spec/support/proxy_builder.rb @@ -0,0 +1,18 @@ +require_relative "docker_builder" +require_relative "path_helper" + +class ProxyBuilder + include DockerBuilder + include PathHelper + + TAG = "hone/static-proxy:latest" + + def initialize(debug = false, intermediates = false) + @build = build( + context: docker_path("proxy").to_s, + debug: debug, + tag: TAG, + intermediates: intermediates + ) + end +end diff --git a/spec/support/proxy_runner.rb b/spec/support/proxy_runner.rb new file mode 100644 index 00000000..e7bc0435 --- /dev/null +++ b/spec/support/proxy_runner.rb @@ -0,0 +1,14 @@ +require_relative "proxy_builder" +require_relative "container_runner" + +class ProxyRunner < ContainerRunner + def initialize(config_ru = nil) + options = { + "name" => "proxy", + "Image" => ProxyBuilder::TAG + } + options["HostConfig"] = { "Binds" => ["#{config_ru}:/app/config/"] } if config_ru + + super(options) + end +end diff --git a/spec/support/router_builder.rb b/spec/support/router_builder.rb new file mode 100644 index 00000000..7ae0708d --- /dev/null +++ b/spec/support/router_builder.rb @@ -0,0 +1,18 @@ +require_relative "path_helper" +require_relative "docker_builder" + +class RouterBuilder + include PathHelper + include DockerBuilder + + TAG = "hone/static-router:latest" + + def initialize(debug = false, intermediates = false) + @image = build( + context: docker_path("/router").to_s, + tag: TAG, + intermediates: intermediates, + debug: debug + ) + end +end diff --git a/spec/support/router_runner.rb b/spec/support/router_runner.rb new file mode 100644 index 00000000..57998b58 --- /dev/null +++ b/spec/support/router_runner.rb @@ -0,0 +1,29 @@ +require_relative "router_builder" +require_relative "container_runner" + +class RouterRunner < ContainerRunner + def self.boot2docker_ip + %x(boot2docker ip).match(/([0-9]{1,3}\.){3}[0-9]{1,3}/)[0] + rescue Errno::ENOENT + end + + CONTAINER_PORT = "80" + HOST_PORT = "80" + HOST_IP = boot2docker_ip || "127.0.0.1" + + def initialize + super({ + "name" => "router", + "Image" => RouterBuilder::TAG, + "HostConfig" => { + "Links" => ["app:app"], + "PortBindings" => { + "#{CONTAINER_PORT}/tcp" => [{ + "HostIp" => HOST_IP, + "HostPort" => HOST_PORT, + }] + } + } + }) + end +end From e9eeb11194c0ca846afe84c327c84f30a34da3f4 Mon Sep 17 00:00:00 2001 From: Terence Lee Date: Tue, 12 Jul 2016 20:41:09 -0500 Subject: [PATCH 05/10] actually test https with hitting https --- spec/simple_spec.rb | 17 ++++++++++++----- spec/support/app_runner.rb | 17 ++++++++++++----- spec/support/router_runner.rb | 14 +++++++++----- 3 files changed, 33 insertions(+), 15 deletions(-) diff --git a/spec/simple_spec.rb b/spec/simple_spec.rb index 02cddb26..434b87d5 100644 --- a/spec/simple_spec.rb +++ b/spec/simple_spec.rb @@ -118,11 +118,18 @@ let(:name) { "https_only" } it "should redirect http to https" do - response = app.get("/foo") - expect(response.code).to eq("301") - uri = URI(response['Location']) - expect(uri.scheme).to eq("https") - expect(uri.path).to eq("/foo") + app.run do + response = app.get("/foo.html") + expect(response.code).to eq("301") + uri = URI(response['Location']) + expect(uri.scheme).to eq("https") + expect(uri.path).to eq("/foo.html") + + response = app.get(uri) + expect(response.code).to eq("200") + expect(response.body.chomp).to eq("foobar") + end + end end end diff --git a/spec/support/app_runner.rb b/spec/support/app_runner.rb index e11b8341..3defcddd 100644 --- a/spec/support/app_runner.rb +++ b/spec/support/app_runner.rb @@ -115,11 +115,18 @@ def destroy def get_retry(path, max_retries) network_retry(max_retries) do uri = URI(path) - uri.host = RouterRunner::HOST_IP if uri.host.nil? - uri.port = RouterRunner::HOST_PORT if (uri.host == RouterRunner::HOST_IP && uri.port != RouterRunner::HOST_PORT) || uri.port.nil? - uri.scheme = "http" if uri.scheme.nil? - - Net::HTTP.get_response(URI(uri.to_s)) + uri.host = RouterRunner::HOST_IP if uri.host.nil? + uri.scheme = "http" if uri.scheme.nil? + + Net::HTTP.start( + uri.host, + uri.port, + use_ssl: uri.scheme == "https", + verify_mode: OpenSSL::SSL::VERIFY_NONE + ) do |http| + request = Net::HTTP::Get.new(uri.to_s) + http.request(request) + end end end diff --git a/spec/support/router_runner.rb b/spec/support/router_runner.rb index 57998b58..77b40802 100644 --- a/spec/support/router_runner.rb +++ b/spec/support/router_runner.rb @@ -7,9 +7,9 @@ def self.boot2docker_ip rescue Errno::ENOENT end - CONTAINER_PORT = "80" - HOST_PORT = "80" - HOST_IP = boot2docker_ip || "127.0.0.1" + HTTP_PORT = "80" + HTTPS_PORT = "443" + HOST_IP = boot2docker_ip || "127.0.0.1" def initialize super({ @@ -18,9 +18,13 @@ def initialize "HostConfig" => { "Links" => ["app:app"], "PortBindings" => { - "#{CONTAINER_PORT}/tcp" => [{ + "#{HTTP_PORT}/tcp" => [{ "HostIp" => HOST_IP, - "HostPort" => HOST_PORT, + "HostPort" => HTTP_PORT + }], + "#{HTTPS_PORT}/tcp" => [{ + "HostIp" => HOST_IP, + "HostPort" => HTTPS_PORT }] } } From ccc4e23109b3d7a699f9c04db48690aba9d1d80c Mon Sep 17 00:00:00 2001 From: Terence Lee Date: Tue, 12 Jul 2016 20:42:07 -0500 Subject: [PATCH 06/10] mitigate CRLF HTTP Header Injection Fixes the following: curl -i http://conway-hi-poc.herokuapp.com/%0d%0aset-cookie:%20test=123; HTTP/1.1 301 Moved Permanently Connection: keep-alive Server: nginx Date: Wed, 06 Jul 2016 23:44:00 GMT Content-Type: text/html Content-Length: 178 Location: https://conway-hi-poc.herokuapp.com/ Set-Cookie: test=123 Via: 1.1 vegur 301 Moved Permanently

301 Moved Permanently

nginx
--- scripts/config/templates/nginx.conf.erb | 2 +- spec/simple_spec.rb | 10 ++++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/scripts/config/templates/nginx.conf.erb b/scripts/config/templates/nginx.conf.erb index f529a6ba..72b0f344 100644 --- a/scripts/config/templates/nginx.conf.erb +++ b/scripts/config/templates/nginx.conf.erb @@ -60,7 +60,7 @@ http { <% if https_only %> if ($http_x_forwarded_proto != "https") { - return 301 https://$host$uri; + return 301 https://$host$request_uri; } <% end %> diff --git a/spec/simple_spec.rb b/spec/simple_spec.rb index 434b87d5..ce7838b4 100644 --- a/spec/simple_spec.rb +++ b/spec/simple_spec.rb @@ -130,6 +130,16 @@ expect(response.body.chomp).to eq("foobar") end end + + context "CRLF HTTP Header injection" do + let(:cookie) { "malicious=1" } + + it "should not expose cookie" do + app.run do + response = app.get("/foo.html#{URI.escape("\r\nSet-Cookie: #{cookie}")}") + expect(response['set-cookie']).not_to eq(cookie) + end + end end end From e3ce4e633731d53bc323a27ecaedc08a24f2ac74 Mon Sep 17 00:00:00 2001 From: Terence Lee Date: Wed, 13 Jul 2016 17:33:47 -0500 Subject: [PATCH 07/10] fix env vars for circleci --- spec/simple_spec.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/spec/simple_spec.rb b/spec/simple_spec.rb index ce7838b4..72439d16 100644 --- a/spec/simple_spec.rb +++ b/spec/simple_spec.rb @@ -13,7 +13,7 @@ @debug = true BuildpackBuilder.new(@debug, ENV['CIRCLECI']) RouterBuilder.new(@debug, ENV['CIRCLECI']) - ProxyBuilder.new(@debug, ENV["CIRCLE_CI"]) + ProxyBuilder.new(@debug, ENV["CIRCLECI"]) end after do @@ -21,7 +21,7 @@ end let(:proxy) { nil } - let(:app) { AppRunner.new(name, proxy, env, @debug) } + let(:app) { AppRunner.new(name, proxy, env, @debug, ENV['CIRCLECI']) } let(:name) { "hello_world" } let(:env) { Hash.new } From 4b1f2164393d6ff1c90e3f17c7320a5d94c85821 Mon Sep 17 00:00:00 2001 From: Terence Lee Date: Wed, 13 Jul 2016 17:48:57 -0500 Subject: [PATCH 08/10] don't delete containers in Circle CI Circle CI does not allow deletion of containers: Cannot destroy container 6f30ab3de4f445d4fe47059d68d9798d40f5f66b9a1120dcfd130c0cab378cb5: Driver btrfs failed to remove root filesystem 6f30ab3de4f445d4fe47059d68d9798d40f5f66b9a1120dcfd130c0cab378cb5: Failed to destroy btrfs snapshot: operation not permitted --- spec/simple_spec.rb | 2 +- spec/support/app_runner.rb | 20 ++++++++++---------- spec/support/container_runner.rb | 8 ++++++-- spec/support/proxy_runner.rb | 5 ++--- spec/support/router_runner.rb | 7 +++---- 5 files changed, 22 insertions(+), 20 deletions(-) diff --git a/spec/simple_spec.rb b/spec/simple_spec.rb index 72439d16..7650bf72 100644 --- a/spec/simple_spec.rb +++ b/spec/simple_spec.rb @@ -21,7 +21,7 @@ end let(:proxy) { nil } - let(:app) { AppRunner.new(name, proxy, env, @debug, ENV['CIRCLECI']) } + let(:app) { AppRunner.new(name, proxy, env, @debug, !ENV['CIRCLECI']) } let(:name) { "hello_world" } let(:env) { Hash.new } diff --git a/spec/support/app_runner.rb b/spec/support/app_runner.rb index 3defcddd..b61903fb 100644 --- a/spec/support/app_runner.rb +++ b/spec/support/app_runner.rb @@ -14,15 +14,15 @@ class AppRunner attr_reader :proxy - def initialize(fixture, proxy = nil, env = {}, debug = false) - @run = false - @debug = debug - @tmpdir = nil - @proxy = nil + def initialize(fixture, proxy = nil, env = {}, debug = false, delete = true) + @run = false + @debug = debug + @tmpdir = nil + @proxy = nil + @delete = delete env.merge!("STATIC_DEBUG" => "true") if @debug app_options = { - "name" => "app", "Image" => BuildpackBuilder::TAG, # Env format is [KEY1=VAL1 KEY2=VAL2] "Env" => env.to_a.map {|i| i.join("=") }, @@ -32,7 +32,6 @@ def initialize(fixture, proxy = nil, env = {}, debug = false) } if proxy - app_options["Links"] = ["proxy:proxy"] if proxy.is_a?(String) @tmpdir = Dir.mktmpdir File.open("#{@tmpdir}/config.ru", "w") do |file| @@ -42,7 +41,8 @@ def initialize(fixture, proxy = nil, env = {}, debug = false) end end - @proxy = ProxyRunner.new(@tmpdir) + @proxy = ProxyRunner.new(@tmpdir, @delete) + app_options["Links"] = ["#{@proxy.id}:proxy"] @proxy.start # need to interpolate the PROXY_IP_ADDRESS since env is a parameter to this constructor and @@ -55,7 +55,7 @@ def initialize(fixture, proxy = nil, env = {}, debug = false) end @app = Docker::Container.create(app_options) - @router = RouterRunner.new + @router = RouterRunner.new(@app.id, @delete) end def run(capture_io = false) @@ -106,7 +106,7 @@ def destroy @proxy.destroy end @router.destroy - @app.delete(force: true) + @app.delete(force: true) if @delete ensure FileUtils.rm_rf(@tmpdir) if @tmpdir end diff --git a/spec/support/container_runner.rb b/spec/support/container_runner.rb index bc80c963..8377e5f4 100644 --- a/spec/support/container_runner.rb +++ b/spec/support/container_runner.rb @@ -2,12 +2,16 @@ require "docker" class ContainerRunner + extend Forwardable + attr_reader :ip_address + def_delegators :@container, :id - def initialize(options) + def initialize(options, delete = true) @container = Docker::Container.create(options) @ip_address = nil @thread = nil + @delete = delete end def start @@ -24,6 +28,6 @@ def stop end def destroy - @container.delete(force: true) + @container.delete(force: true) if @delete end end diff --git a/spec/support/proxy_runner.rb b/spec/support/proxy_runner.rb index e7bc0435..819fee6c 100644 --- a/spec/support/proxy_runner.rb +++ b/spec/support/proxy_runner.rb @@ -2,13 +2,12 @@ require_relative "container_runner" class ProxyRunner < ContainerRunner - def initialize(config_ru = nil) + def initialize(config_ru = nil, delete = true) options = { - "name" => "proxy", "Image" => ProxyBuilder::TAG } options["HostConfig"] = { "Binds" => ["#{config_ru}:/app/config/"] } if config_ru - super(options) + super(options, delete) end end diff --git a/spec/support/router_runner.rb b/spec/support/router_runner.rb index 77b40802..c37613f4 100644 --- a/spec/support/router_runner.rb +++ b/spec/support/router_runner.rb @@ -11,12 +11,11 @@ def self.boot2docker_ip HTTPS_PORT = "443" HOST_IP = boot2docker_ip || "127.0.0.1" - def initialize + def initialize(app_id, delete = true) super({ - "name" => "router", "Image" => RouterBuilder::TAG, "HostConfig" => { - "Links" => ["app:app"], + "Links" => ["#{app_id}:app"], "PortBindings" => { "#{HTTP_PORT}/tcp" => [{ "HostIp" => HOST_IP, @@ -28,6 +27,6 @@ def initialize }] } } - }) + }, delete) end end From a9d6c9ed1c7d4c6782a1bcdcdf5bd31fc5e943cc Mon Sep 17 00:00:00 2001 From: Terence Lee Date: Wed, 13 Jul 2016 20:32:48 -0500 Subject: [PATCH 09/10] increase retries for CircleCI to 60 to compensate for multiple container orchestration --- spec/support/app_runner.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spec/support/app_runner.rb b/spec/support/app_runner.rb index b61903fb..7b054a82 100644 --- a/spec/support/app_runner.rb +++ b/spec/support/app_runner.rb @@ -92,7 +92,7 @@ def run(capture_io = false) @run = false end - def get(path, capture_io = false, max_retries = 30) + def get(path, capture_io = false, max_retries = 60) if @run get_retry(path, max_retries) else From c378c66528b60af72d4995be43f44dec5e3b02e9 Mon Sep 17 00:00:00 2001 From: Terence Lee Date: Wed, 13 Jul 2016 20:45:03 -0500 Subject: [PATCH 10/10] move tmpdir handling into ProxyRunner --- spec/support/app_runner.rb | 14 +------------- spec/support/proxy_runner.rb | 27 ++++++++++++++++++++++++++- 2 files changed, 27 insertions(+), 14 deletions(-) diff --git a/spec/support/app_runner.rb b/spec/support/app_runner.rb index 7b054a82..f7d8bdd3 100644 --- a/spec/support/app_runner.rb +++ b/spec/support/app_runner.rb @@ -17,7 +17,6 @@ class AppRunner def initialize(fixture, proxy = nil, env = {}, debug = false, delete = true) @run = false @debug = debug - @tmpdir = nil @proxy = nil @delete = delete env.merge!("STATIC_DEBUG" => "true") if @debug @@ -32,16 +31,7 @@ def initialize(fixture, proxy = nil, env = {}, debug = false, delete = true) } if proxy - if proxy.is_a?(String) - @tmpdir = Dir.mktmpdir - File.open("#{@tmpdir}/config.ru", "w") do |file| - file.puts %q{require "sinatra"} - file.puts proxy - file.puts "run Sinatra::Application" - end - end - - @proxy = ProxyRunner.new(@tmpdir, @delete) + @proxy = ProxyRunner.new(proxy, @delete) app_options["Links"] = ["#{@proxy.id}:proxy"] @proxy.start @@ -107,8 +97,6 @@ def destroy end @router.destroy @app.delete(force: true) if @delete - ensure - FileUtils.rm_rf(@tmpdir) if @tmpdir end private diff --git a/spec/support/proxy_runner.rb b/spec/support/proxy_runner.rb index 819fee6c..bc3edf05 100644 --- a/spec/support/proxy_runner.rb +++ b/spec/support/proxy_runner.rb @@ -1,13 +1,38 @@ +require "tmpdir" require_relative "proxy_builder" require_relative "container_runner" class ProxyRunner < ContainerRunner def initialize(config_ru = nil, delete = true) + @tmpdir = write_config_ru(config_ru) + options = { "Image" => ProxyBuilder::TAG } - options["HostConfig"] = { "Binds" => ["#{config_ru}:/app/config/"] } if config_ru + options["HostConfig"] = { "Binds" => ["#{@tmpdir}:/app/config/"] } if @tmpdir super(options, delete) end + + def destroy + super + ensure + FileUtils.rm_rf(@tmpdir) if @tmpdir + end + + private + def write_config_ru(config_ru) + tmpdir = nil + + if config_ru && config_ru.is_a?(String) + tmpdir = Dir.mktmpdir + File.open("#{tmpdir}/config.ru", "w") do |file| + file.puts %q{require "sinatra"} + file.puts config_ru + file.puts "run Sinatra::Application" + end + end + + tmpdir + end end