Issue #119: Transparent response body compression

This patch adds an option 'proxy.gzip.contentype' which
enables transparent response body compression if the
client requests it, the content type of the response
matches the proxy.gzip.contenttype regexp and the
response is not already compressed.

The gzip handler is mostly based on the code from
@smanke from https://github.com/smanke/handler/gzip.

assert/assert.go

@@ -0,0 +1,25 @@
+// Package assert provides a simple assert framework.
+package assert
+
+import (
+	"fmt"
+	"path/filepath"
+	"reflect"
+	"runtime"
+	"testing"
+)
+
+// Equal provides an assertEqual function
+func Equal(t *testing.T) func(got, want interface{}) {
+	return EqualDepth(t, 1, "")
+}
+
+func EqualDepth(t *testing.T, calldepth int, desc string) func(got, want interface{}) {
+	return func(got, want interface{}) {
+		_, file, line, _ := runtime.Caller(calldepth)
+		if !reflect.DeepEqual(got, want) {
+			fmt.Printf("\t%s:%d: %s: got %v want %v\n", filepath.Base(file), line, desc, got, want)
+			t.Fail()
+		}
+	}
+}

config/config.go

@@ -2,6 +2,7 @@ package config
 
 import (
 	"net/http"
+	"regexp"
 	"time"
 )
 
@@ -60,6 +61,8 @@ type Proxy struct {
 	ClientIPHeader        string
 	TLSHeader             string
 	TLSHeaderValue        string
+	GZIPContentTypesValue string
+	GZIPContentTypes      *regexp.Regexp
 }
 
 type Runtime struct {

config/load.go

@@ -7,6 +7,7 @@ import (
 	"log"
 	"net/http"
 	"os"
+	"regexp"
 	"runtime"
 	"strings"
 	"time"
@@ -100,6 +101,7 @@ func load(p *properties.Properties) (cfg *Config, err error) {
 	f.StringVar(&cfg.Proxy.ClientIPHeader, "proxy.header.clientip", Default.Proxy.ClientIPHeader, "header for the request ip")
 	f.StringVar(&cfg.Proxy.TLSHeader, "proxy.header.tls", Default.Proxy.TLSHeader, "header for TLS connections")
 	f.StringVar(&cfg.Proxy.TLSHeaderValue, "proxy.header.tls.value", Default.Proxy.TLSHeaderValue, "value for TLS connection header")
+	f.StringVar(&cfg.Proxy.GZIPContentTypesValue, "proxy.gzip.contenttype", Default.Proxy.GZIPContentTypesValue, "regexp of content types to compress")
 	f.StringSliceVar(&cfg.ListenerValue, "proxy.addr", Default.ListenerValue, "listener config")
 	f.KVSliceVar(&cfg.CertSourcesValue, "proxy.cs", Default.CertSourcesValue, "certificate sources")
 	f.DurationVar(&cfg.Proxy.ReadTimeout, "proxy.readtimeout", Default.Proxy.ReadTimeout, "read timeout for incoming requests")
@@ -171,6 +173,13 @@ func load(p *properties.Properties) (cfg *Config, err error) {
 		return nil, err
 	}
 
+	if cfg.Proxy.GZIPContentTypesValue != "" {
+		cfg.Proxy.GZIPContentTypes, err = regexp.Compile(cfg.Proxy.GZIPContentTypesValue)
+		if err != nil {
+			return nil, fmt.Errorf("invalid expression for content types: %s", err)
+		}
+	}
+
 	// handle deprecations
 	// deprecate := func(name, msg string) {
 	// 	if f.IsSet(name) {

config/load_test.go

@@ -3,6 +3,7 @@ package config
 import (
 	"net/http"
 	"reflect"
+	"regexp"
 	"testing"
 	"time"
 
@@ -29,6 +30,7 @@ proxy.maxconn = 666
 proxy.header.clientip = clientip
 proxy.header.tls = tls
 proxy.header.tls.value = tls-true
+proxy.gzip.contenttype = ^(text/.*|application/(javascript|json|font-woff|xml)|.*\\+(json|xml))$
 registry.backend = something
 registry.file.path = /foo/bar
 registry.static.routes = route add svc / http://127.0.0.1:6666/
@@ -91,6 +93,8 @@ aws.apigw.cert.cn = furb
 			ClientIPHeader:        "clientip",
 			TLSHeader:             "tls",
 			TLSHeaderValue:        "tls-true",
+			GZIPContentTypesValue: `^(text/.*|application/(javascript|json|font-woff|xml)|.*\+(json|xml))$`,
+			GZIPContentTypes:      regexp.MustCompile(`^(text/.*|application/(javascript|json|font-woff|xml)|.*\+(json|xml))$`),
 		},
 		Registry: Registry{
 			Backend: "something",

fabio.properties

@@ -337,6 +337,8 @@
 #
 # The remoteIP is taken from http.Request.RemoteAddr.
 #
+# The default is
+#
 # proxy.header.clientip =
 
 
@@ -351,6 +353,24 @@
 # proxy.header.tls.value =
 
 
+# proxy.gzip.contenttype configures which responses should be compressed.
+#
+# By default, responses sent to the client are not compressed even if the
+# client accepts compressed responses by setting the 'Accept-Encoding: gzip'
+# header. By setting this value responses are compressed if the Content-Type
+# header of the response matches and the response is not already compressed.
+# The list of compressable content types is defined as a regular expression.
+# The regular expression must follow the rules outlined in golang.org/pkg/regexp.
+#
+# A typical example is
+#
+# proxy.gzip.contenttype = ^(text/.*|application/(javascript|json|font-woff|xml)|.*\\+(json|xml))$
+#
+# The default is
+#
+# proxy.gzip.contenttype =
+
+
 # registry.backend configures which backend is used.
 # Supported backends are: consul, static, file
 #

proxy/gzip/gzip_handler.go

@@ -0,0 +1,104 @@
+// Copyright (c) 2016 Sebastian Mancke and eBay, both MIT licensed
+
+// Package gzip provides an HTTP handler which compresses responses
+// if the client supports this, the response is compressable and
+// not already compressed.
+//
+// Based on https://github.com/smancke/handler/gzip
+package gzip
+
+import (
+	"compress/gzip"
+	"io"
+	"net/http"
+	"regexp"
+	"strings"
+	"sync"
+)
+
+const (
+	headerVary            = "Vary"
+	headerAcceptEncoding  = "Accept-Encoding"
+	headerContentEncoding = "Content-Encoding"
+	headerContentType     = "Content-Type"
+	headerContentLength   = "Content-Length"
+	encodingGzip          = "gzip"
+)
+
+var gzipWriterPool = sync.Pool{
+	New: func() interface{} { return gzip.NewWriter(nil) },
+}
+
+// NewGzipHandler wraps an existing handler to transparently gzip the response
+// body if the client supports it (via the Accept-Encoding header) and the
+// response Content-Type matches the contentTypes expression.
+func NewGzipHandler(h http.Handler, contentTypes *regexp.Regexp) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Add(headerVary, headerAcceptEncoding)
+
+		if acceptsGzip(r) {
+			gzWriter := NewGzipResponseWriter(w, contentTypes)
+			defer gzWriter.Close()
+			h.ServeHTTP(gzWriter, r)
+		} else {
+			h.ServeHTTP(w, r)
+		}
+	})
+}
+
+type GzipResponseWriter struct {
+	writer       io.Writer
+	gzipWriter   *gzip.Writer
+	contentTypes *regexp.Regexp
+	http.ResponseWriter
+}
+
+func NewGzipResponseWriter(w http.ResponseWriter, contentTypes *regexp.Regexp) *GzipResponseWriter {
+	return &GzipResponseWriter{ResponseWriter: w, contentTypes: contentTypes}
+}
+
+func (grw *GzipResponseWriter) WriteHeader(code int) {
+	if grw.writer == nil {
+		if isCompressable(grw.Header(), grw.contentTypes) {
+			grw.Header().Del(headerContentLength)
+			grw.Header().Set(headerContentEncoding, encodingGzip)
+			grw.gzipWriter = gzipWriterPool.Get().(*gzip.Writer)
+			grw.gzipWriter.Reset(grw.ResponseWriter)
+
+			grw.writer = grw.gzipWriter
+		} else {
+			grw.writer = grw.ResponseWriter
+		}
+	}
+	grw.ResponseWriter.WriteHeader(code)
+}
+
+func (grw *GzipResponseWriter) Write(b []byte) (int, error) {
+	if grw.writer == nil {
+		if _, ok := grw.Header()[headerContentType]; !ok {
+			// Set content-type if not present. Otherwise golang would make application/gzip out of that.
+			grw.Header().Set(headerContentType, http.DetectContentType(b))
+		}
+		grw.WriteHeader(http.StatusOK)
+	}
+	return grw.writer.Write(b)
+}
+
+func (grw *GzipResponseWriter) Close() {
+	if grw.gzipWriter != nil {
+		grw.gzipWriter.Close()
+		gzipWriterPool.Put(grw.gzipWriter)
+	}
+}
+
+func isCompressable(header http.Header, contentTypes *regexp.Regexp) bool {
+	// don't compress if it is already encoded
+	if header.Get(headerContentEncoding) != "" {
+		return false
+	}
+	return contentTypes.MatchString(header.Get(headerContentType))
+}
+
+func acceptsGzip(r *http.Request) bool {
+	return strings.Contains(r.Header.Get(headerAcceptEncoding), encodingGzip)
+}

proxy/gzip/gzip_handler_test.go

@@ -0,0 +1,134 @@
+// Copyright (c) 2016 Sebastian Mancke and eBay, both MIT licensed
+package gzip
+
+import (
+	"bytes"
+	"compress/gzip"
+	"io/ioutil"
+	"net/http"
+	"net/http/httptest"
+	"regexp"
+	"strconv"
+	"testing"
+
+	"github.com/eBay/fabio/assert"
+)
+
+var contentTypes = regexp.MustCompile(`^(text/.*|application/(javascript|json|font-woff|xml)|.*\+(json|xml))$`)
+
+func Test_GzipHandler_CompressableType(t *testing.T) {
+	server := httptest.NewServer(NewGzipHandler(test_text_handler(), contentTypes))
+
+	assertEqual := assert.Equal(t)
+
+	r, err := http.NewRequest("GET", server.URL, nil)
+	assertEqual(err, nil)
+	r.Header.Set("Accept-Encoding", "gzip")
+
+	resp, err := http.DefaultClient.Do(r)
+	assertEqual(err, nil)
+
+	assertEqual(resp.Header.Get("Content-Type"), "text/plain; charset=utf-8")
+	assertEqual(resp.Header.Get("Content-Encoding"), "gzip")
+
+	gzBytes, err := ioutil.ReadAll(resp.Body)
+	assertEqual(err, nil)
+	assertEqual(resp.Header.Get("Content-Length"), strconv.Itoa(len(gzBytes)))
+
+	reader, err := gzip.NewReader(bytes.NewBuffer(gzBytes))
+	assertEqual(err, nil)
+	defer reader.Close()
+
+	bytes, err := ioutil.ReadAll(reader)
+	assertEqual(err, nil)
+
+	assertEqual(string(bytes), "Hello World")
+}
+
+func Test_GzipHandler_NotCompressingTwice(t *testing.T) {
+	server := httptest.NewServer(NewGzipHandler(test_already_compressed_handler(), contentTypes))
+
+	assertEqual := assert.Equal(t)
+
+	r, err := http.NewRequest("GET", server.URL, nil)
+	assertEqual(err, nil)
+	r.Header.Set("Accept-Encoding", "gzip")
+
+	resp, err := http.DefaultClient.Do(r)
+	assertEqual(err, nil)
+
+	assertEqual(resp.Header.Get("Content-Encoding"), "gzip")
+
+	reader, err := gzip.NewReader(resp.Body)
+	assertEqual(err, nil)
+	defer reader.Close()
+
+	bytes, err := ioutil.ReadAll(reader)
+	assertEqual(err, nil)
+
+	assertEqual(string(bytes), "Hello World")
+}
+
+func Test_GzipHandler_CompressableType_NoAccept(t *testing.T) {
+	server := httptest.NewServer(NewGzipHandler(test_text_handler(), contentTypes))
+
+	assertEqual := assert.Equal(t)
+
+	r, err := http.NewRequest("GET", server.URL, nil)
+	assertEqual(err, nil)
+	r.Header.Set("Accept-Encoding", "none")
+
+	resp, err := http.DefaultClient.Do(r)
+	assertEqual(err, nil)
+
+	assertEqual(resp.Header.Get("Content-Encoding"), "")
+
+	bytes, err := ioutil.ReadAll(resp.Body)
+	assertEqual(err, nil)
+
+	assertEqual(string(bytes), "Hello World")
+}
+
+func Test_GzipHandler_NonCompressableType(t *testing.T) {
+	server := httptest.NewServer(NewGzipHandler(test_binary_handler(), contentTypes))
+
+	assertEqual := assert.Equal(t)
+
+	r, err := http.NewRequest("GET", server.URL, nil)
+	assertEqual(err, nil)
+	r.Header.Set("Accept-Encoding", "gzip")
+
+	resp, err := http.DefaultClient.Do(r)
+	assertEqual(err, nil)
+
+	assertEqual(resp.Header.Get("Content-Encoding"), "")
+
+	bytes, err := ioutil.ReadAll(resp.Body)
+	assertEqual(err, nil)
+
+	assertEqual(bytes, []byte{42})
+}
+
+func test_text_handler() http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		b := []byte("Hello World")
+		w.Header().Set("Content-Length", strconv.Itoa(len(b)))
+		w.Write(b)
+	})
+}
+
+func test_binary_handler() http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", "image/jpg")
+		w.Write([]byte{42})
+	})
+}
+
+func test_already_compressed_handler() http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Encoding", "gzip")
+		gzWriter := gzip.NewWriter(w)
+		gzWriter.Write([]byte("Hello World"))
+		gzWriter.Close()
+	})
+}

proxy/proxy.go

@@ -6,6 +6,7 @@ import (
 
 	"github.com/eBay/fabio/config"
 	"github.com/eBay/fabio/metrics"
+	"github.com/eBay/fabio/proxy/gzip"
 )
 
 // httpProxy is a dynamic reverse proxy for HTTP and HTTPS protocols.
@@ -60,6 +61,10 @@ func (p *httpProxy) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		h = newHTTPProxy(t.URL, p.tr, time.Duration(0))
 	}
 
+	if p.cfg.GZIPContentTypes != nil {
+		h = gzip.NewGzipHandler(h, p.cfg.GZIPContentTypes)
+	}
+
 	start := time.Now()
 	h.ServeHTTP(w, r)
 	p.requests.UpdateSince(start)