From 3c71d66cccaf220471dc020bba377ad52c850c6a Mon Sep 17 00:00:00 2001 From: Frank Schroeder Date: Wed, 31 Aug 2016 12:22:07 +0200 Subject: [PATCH] Issue #119: Transparent response body compression This patch adds an option 'proxy.gzip.contentype' which enables transparent response body compression if the client requests it, the content type of the response matches the proxy.gzip.contenttype regexp and the response is not already compressed. The gzip handler is mostly based on the code from @smanke from https://github.com/smanke/handler/gzip. --- assert/assert.go | 25 ++++++ config/config.go | 3 + config/load.go | 9 +++ config/load_test.go | 4 + fabio.properties | 20 +++++ proxy/gzip/gzip_handler.go | 104 +++++++++++++++++++++++++ proxy/gzip/gzip_handler_test.go | 134 ++++++++++++++++++++++++++++++++ proxy/proxy.go | 5 ++ 8 files changed, 304 insertions(+) create mode 100644 assert/assert.go create mode 100644 proxy/gzip/gzip_handler.go create mode 100644 proxy/gzip/gzip_handler_test.go diff --git a/assert/assert.go b/assert/assert.go new file mode 100644 index 000000000..ad574dc79 --- /dev/null +++ b/assert/assert.go @@ -0,0 +1,25 @@ +// Package assert provides a simple assert framework. +package assert + +import ( + "fmt" + "path/filepath" + "reflect" + "runtime" + "testing" +) + +// Equal provides an assertEqual function +func Equal(t *testing.T) func(got, want interface{}) { + return EqualDepth(t, 1, "") +} + +func EqualDepth(t *testing.T, calldepth int, desc string) func(got, want interface{}) { + return func(got, want interface{}) { + _, file, line, _ := runtime.Caller(calldepth) + if !reflect.DeepEqual(got, want) { + fmt.Printf("\t%s:%d: %s: got %v want %v\n", filepath.Base(file), line, desc, got, want) + t.Fail() + } + } +} diff --git a/config/config.go b/config/config.go index 03c14b3aa..0dd44ebd9 100644 --- a/config/config.go +++ b/config/config.go @@ -2,6 +2,7 @@ package config import ( "net/http" + "regexp" "time" ) @@ -60,6 +61,8 @@ type Proxy struct { ClientIPHeader string TLSHeader string TLSHeaderValue string + GZIPContentTypesValue string + GZIPContentTypes *regexp.Regexp } type Runtime struct { diff --git a/config/load.go b/config/load.go index f248c8124..9c2a64c51 100644 --- a/config/load.go +++ b/config/load.go @@ -7,6 +7,7 @@ import ( "log" "net/http" "os" + "regexp" "runtime" "strings" "time" @@ -100,6 +101,7 @@ func load(p *properties.Properties) (cfg *Config, err error) { f.StringVar(&cfg.Proxy.ClientIPHeader, "proxy.header.clientip", Default.Proxy.ClientIPHeader, "header for the request ip") f.StringVar(&cfg.Proxy.TLSHeader, "proxy.header.tls", Default.Proxy.TLSHeader, "header for TLS connections") f.StringVar(&cfg.Proxy.TLSHeaderValue, "proxy.header.tls.value", Default.Proxy.TLSHeaderValue, "value for TLS connection header") + f.StringVar(&cfg.Proxy.GZIPContentTypesValue, "proxy.gzip.contenttype", Default.Proxy.GZIPContentTypesValue, "regexp of content types to compress") f.StringSliceVar(&cfg.ListenerValue, "proxy.addr", Default.ListenerValue, "listener config") f.KVSliceVar(&cfg.CertSourcesValue, "proxy.cs", Default.CertSourcesValue, "certificate sources") f.DurationVar(&cfg.Proxy.ReadTimeout, "proxy.readtimeout", Default.Proxy.ReadTimeout, "read timeout for incoming requests") @@ -171,6 +173,13 @@ func load(p *properties.Properties) (cfg *Config, err error) { return nil, err } + if cfg.Proxy.GZIPContentTypesValue != "" { + cfg.Proxy.GZIPContentTypes, err = regexp.Compile(cfg.Proxy.GZIPContentTypesValue) + if err != nil { + return nil, fmt.Errorf("invalid expression for content types: %s", err) + } + } + // handle deprecations // deprecate := func(name, msg string) { // if f.IsSet(name) { diff --git a/config/load_test.go b/config/load_test.go index 3466ae972..782664e60 100644 --- a/config/load_test.go +++ b/config/load_test.go @@ -3,6 +3,7 @@ package config import ( "net/http" "reflect" + "regexp" "testing" "time" @@ -29,6 +30,7 @@ proxy.maxconn = 666 proxy.header.clientip = clientip proxy.header.tls = tls proxy.header.tls.value = tls-true +proxy.gzip.contenttype = ^(text/.*|application/(javascript|json|font-woff|xml)|.*\\+(json|xml))$ registry.backend = something registry.file.path = /foo/bar registry.static.routes = route add svc / http://127.0.0.1:6666/ @@ -91,6 +93,8 @@ aws.apigw.cert.cn = furb ClientIPHeader: "clientip", TLSHeader: "tls", TLSHeaderValue: "tls-true", + GZIPContentTypesValue: `^(text/.*|application/(javascript|json|font-woff|xml)|.*\+(json|xml))$`, + GZIPContentTypes: regexp.MustCompile(`^(text/.*|application/(javascript|json|font-woff|xml)|.*\+(json|xml))$`), }, Registry: Registry{ Backend: "something", diff --git a/fabio.properties b/fabio.properties index ce951a8a2..c97272277 100644 --- a/fabio.properties +++ b/fabio.properties @@ -337,6 +337,8 @@ # # The remoteIP is taken from http.Request.RemoteAddr. # +# The default is +# # proxy.header.clientip = @@ -351,6 +353,24 @@ # proxy.header.tls.value = +# proxy.gzip.contenttype configures which responses should be compressed. +# +# By default, responses sent to the client are not compressed even if the +# client accepts compressed responses by setting the 'Accept-Encoding: gzip' +# header. By setting this value responses are compressed if the Content-Type +# header of the response matches and the response is not already compressed. +# The list of compressable content types is defined as a regular expression. +# The regular expression must follow the rules outlined in golang.org/pkg/regexp. +# +# A typical example is +# +# proxy.gzip.contenttype = ^(text/.*|application/(javascript|json|font-woff|xml)|.*\\+(json|xml))$ +# +# The default is +# +# proxy.gzip.contenttype = + + # registry.backend configures which backend is used. # Supported backends are: consul, static, file # diff --git a/proxy/gzip/gzip_handler.go b/proxy/gzip/gzip_handler.go new file mode 100644 index 000000000..046eb5b14 --- /dev/null +++ b/proxy/gzip/gzip_handler.go @@ -0,0 +1,104 @@ +// Copyright (c) 2016 Sebastian Mancke and eBay, both MIT licensed + +// Package gzip provides an HTTP handler which compresses responses +// if the client supports this, the response is compressable and +// not already compressed. +// +// Based on https://github.com/smancke/handler/gzip +package gzip + +import ( + "compress/gzip" + "io" + "net/http" + "regexp" + "strings" + "sync" +) + +const ( + headerVary = "Vary" + headerAcceptEncoding = "Accept-Encoding" + headerContentEncoding = "Content-Encoding" + headerContentType = "Content-Type" + headerContentLength = "Content-Length" + encodingGzip = "gzip" +) + +var gzipWriterPool = sync.Pool{ + New: func() interface{} { return gzip.NewWriter(nil) }, +} + +// NewGzipHandler wraps an existing handler to transparently gzip the response +// body if the client supports it (via the Accept-Encoding header) and the +// response Content-Type matches the contentTypes expression. +func NewGzipHandler(h http.Handler, contentTypes *regexp.Regexp) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Add(headerVary, headerAcceptEncoding) + + if acceptsGzip(r) { + gzWriter := NewGzipResponseWriter(w, contentTypes) + defer gzWriter.Close() + h.ServeHTTP(gzWriter, r) + } else { + h.ServeHTTP(w, r) + } + }) +} + +type GzipResponseWriter struct { + writer io.Writer + gzipWriter *gzip.Writer + contentTypes *regexp.Regexp + http.ResponseWriter +} + +func NewGzipResponseWriter(w http.ResponseWriter, contentTypes *regexp.Regexp) *GzipResponseWriter { + return &GzipResponseWriter{ResponseWriter: w, contentTypes: contentTypes} +} + +func (grw *GzipResponseWriter) WriteHeader(code int) { + if grw.writer == nil { + if isCompressable(grw.Header(), grw.contentTypes) { + grw.Header().Del(headerContentLength) + grw.Header().Set(headerContentEncoding, encodingGzip) + grw.gzipWriter = gzipWriterPool.Get().(*gzip.Writer) + grw.gzipWriter.Reset(grw.ResponseWriter) + + grw.writer = grw.gzipWriter + } else { + grw.writer = grw.ResponseWriter + } + } + grw.ResponseWriter.WriteHeader(code) +} + +func (grw *GzipResponseWriter) Write(b []byte) (int, error) { + if grw.writer == nil { + if _, ok := grw.Header()[headerContentType]; !ok { + // Set content-type if not present. Otherwise golang would make application/gzip out of that. + grw.Header().Set(headerContentType, http.DetectContentType(b)) + } + grw.WriteHeader(http.StatusOK) + } + return grw.writer.Write(b) +} + +func (grw *GzipResponseWriter) Close() { + if grw.gzipWriter != nil { + grw.gzipWriter.Close() + gzipWriterPool.Put(grw.gzipWriter) + } +} + +func isCompressable(header http.Header, contentTypes *regexp.Regexp) bool { + // don't compress if it is already encoded + if header.Get(headerContentEncoding) != "" { + return false + } + return contentTypes.MatchString(header.Get(headerContentType)) +} + +func acceptsGzip(r *http.Request) bool { + return strings.Contains(r.Header.Get(headerAcceptEncoding), encodingGzip) +} diff --git a/proxy/gzip/gzip_handler_test.go b/proxy/gzip/gzip_handler_test.go new file mode 100644 index 000000000..a3d58e7f4 --- /dev/null +++ b/proxy/gzip/gzip_handler_test.go @@ -0,0 +1,134 @@ +// Copyright (c) 2016 Sebastian Mancke and eBay, both MIT licensed +package gzip + +import ( + "bytes" + "compress/gzip" + "io/ioutil" + "net/http" + "net/http/httptest" + "regexp" + "strconv" + "testing" + + "github.com/eBay/fabio/assert" +) + +var contentTypes = regexp.MustCompile(`^(text/.*|application/(javascript|json|font-woff|xml)|.*\+(json|xml))$`) + +func Test_GzipHandler_CompressableType(t *testing.T) { + server := httptest.NewServer(NewGzipHandler(test_text_handler(), contentTypes)) + + assertEqual := assert.Equal(t) + + r, err := http.NewRequest("GET", server.URL, nil) + assertEqual(err, nil) + r.Header.Set("Accept-Encoding", "gzip") + + resp, err := http.DefaultClient.Do(r) + assertEqual(err, nil) + + assertEqual(resp.Header.Get("Content-Type"), "text/plain; charset=utf-8") + assertEqual(resp.Header.Get("Content-Encoding"), "gzip") + + gzBytes, err := ioutil.ReadAll(resp.Body) + assertEqual(err, nil) + assertEqual(resp.Header.Get("Content-Length"), strconv.Itoa(len(gzBytes))) + + reader, err := gzip.NewReader(bytes.NewBuffer(gzBytes)) + assertEqual(err, nil) + defer reader.Close() + + bytes, err := ioutil.ReadAll(reader) + assertEqual(err, nil) + + assertEqual(string(bytes), "Hello World") +} + +func Test_GzipHandler_NotCompressingTwice(t *testing.T) { + server := httptest.NewServer(NewGzipHandler(test_already_compressed_handler(), contentTypes)) + + assertEqual := assert.Equal(t) + + r, err := http.NewRequest("GET", server.URL, nil) + assertEqual(err, nil) + r.Header.Set("Accept-Encoding", "gzip") + + resp, err := http.DefaultClient.Do(r) + assertEqual(err, nil) + + assertEqual(resp.Header.Get("Content-Encoding"), "gzip") + + reader, err := gzip.NewReader(resp.Body) + assertEqual(err, nil) + defer reader.Close() + + bytes, err := ioutil.ReadAll(reader) + assertEqual(err, nil) + + assertEqual(string(bytes), "Hello World") +} + +func Test_GzipHandler_CompressableType_NoAccept(t *testing.T) { + server := httptest.NewServer(NewGzipHandler(test_text_handler(), contentTypes)) + + assertEqual := assert.Equal(t) + + r, err := http.NewRequest("GET", server.URL, nil) + assertEqual(err, nil) + r.Header.Set("Accept-Encoding", "none") + + resp, err := http.DefaultClient.Do(r) + assertEqual(err, nil) + + assertEqual(resp.Header.Get("Content-Encoding"), "") + + bytes, err := ioutil.ReadAll(resp.Body) + assertEqual(err, nil) + + assertEqual(string(bytes), "Hello World") +} + +func Test_GzipHandler_NonCompressableType(t *testing.T) { + server := httptest.NewServer(NewGzipHandler(test_binary_handler(), contentTypes)) + + assertEqual := assert.Equal(t) + + r, err := http.NewRequest("GET", server.URL, nil) + assertEqual(err, nil) + r.Header.Set("Accept-Encoding", "gzip") + + resp, err := http.DefaultClient.Do(r) + assertEqual(err, nil) + + assertEqual(resp.Header.Get("Content-Encoding"), "") + + bytes, err := ioutil.ReadAll(resp.Body) + assertEqual(err, nil) + + assertEqual(bytes, []byte{42}) +} + +func test_text_handler() http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + b := []byte("Hello World") + w.Header().Set("Content-Length", strconv.Itoa(len(b))) + w.Write(b) + }) +} + +func test_binary_handler() http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "image/jpg") + w.Write([]byte{42}) + }) +} + +func test_already_compressed_handler() http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Encoding", "gzip") + gzWriter := gzip.NewWriter(w) + gzWriter.Write([]byte("Hello World")) + gzWriter.Close() + }) +} diff --git a/proxy/proxy.go b/proxy/proxy.go index 58a59c24a..cd199258e 100644 --- a/proxy/proxy.go +++ b/proxy/proxy.go @@ -6,6 +6,7 @@ import ( "github.com/eBay/fabio/config" "github.com/eBay/fabio/metrics" + "github.com/eBay/fabio/proxy/gzip" ) // httpProxy is a dynamic reverse proxy for HTTP and HTTPS protocols. @@ -60,6 +61,10 @@ func (p *httpProxy) ServeHTTP(w http.ResponseWriter, r *http.Request) { h = newHTTPProxy(t.URL, p.tr, time.Duration(0)) } + if p.cfg.GZIPContentTypes != nil { + h = gzip.NewGzipHandler(h, p.cfg.GZIPContentTypes) + } + start := time.Now() h.ServeHTTP(w, r) p.requests.UpdateSince(start)