From 7fa5377de848d67b5b442dc74969b8a8428bc4f1 Mon Sep 17 00:00:00 2001 From: mirecl Date: Tue, 10 Sep 2024 14:17:21 +0300 Subject: [PATCH 1/9] [gzhttp] Add supported decompress request body --- gzhttp/compress.go | 14 ++++++++++++++ gzhttp/compress_test.go | 28 ++++++++++++++++++++++++++++ 2 files changed, 42 insertions(+) diff --git a/gzhttp/compress.go b/gzhttp/compress.go index a9be8e979..b8df6a2fa 100644 --- a/gzhttp/compress.go +++ b/gzhttp/compress.go @@ -464,6 +464,15 @@ func NewWrapper(opts ...option) (func(http.Handler) http.HandlerFunc, error) { return func(h http.Handler) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { w.Header().Add(vary, acceptEncoding) + if contentGzip(r) { + readerGzipBody, err := gzip.NewReader(r.Body) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + r.Body = io.NopCloser(readerGzipBody) + } + if acceptsGzip(r) { gw := grwPool.Get().(*GzipResponseWriter) *gw = GzipResponseWriter{ @@ -752,6 +761,11 @@ func RandomJitter(n, buffer int, paranoid bool) option { } } +func contentGzip(r *http.Request) bool { + // See more detail in `acceptsGzip` + return r.Method != http.MethodHead && parseEncodingGzip(r.Header.Get(contentEncoding)) > 0 +} + // acceptsGzip returns true if the given HTTP request indicates that it will // accept a gzipped response. func acceptsGzip(r *http.Request) bool { diff --git a/gzhttp/compress_test.go b/gzhttp/compress_test.go index c91de81b4..74ccebc9e 100644 --- a/gzhttp/compress_test.go +++ b/gzhttp/compress_test.go @@ -89,6 +89,34 @@ func TestMustNewGzipHandler(t *testing.T) { handler.ServeHTTP(res3, req3) assertEqual(t, http.DetectContentType([]byte(testBody)), res3.Header().Get("Content-Type")) + + // send not compress request body + + req4, _ := http.NewRequest("POST", "/whatever", bytes.NewBuffer(testBody)) + req4.Header.Set("Content-Encoding", "gzip") + resp4 := httptest.NewRecorder() + handler.ServeHTTP(resp4, req4) + res4 := resp4.Result() + + assertEqual(t, 400, res4.StatusCode) + + // send compress request body + + var b bytes.Buffer + writerGzip := gzip.NewWriter(&b) + writerGzip.Write(testBody) + writerGzip.Close() + + req5, _ := http.NewRequest("POST", "/whatever", &b) + req5.Header.Set("Content-Encoding", "gzip") + resp5 := httptest.NewRecorder() + handler.ServeHTTP(resp5, req5) + res5 := resp5.Result() + + assertEqual(t, 200, res5.StatusCode) + + body, _ := io.ReadAll(res5.Body) + assertEqual(t, len(testBody), len(body)) } func TestGzipHandlerSmallBodyNoCompression(t *testing.T) { From b29198d5a78987a13e0a8148de06bd8c16eb25ac Mon Sep 17 00:00:00 2001 From: mirecl Date: Tue, 10 Sep 2024 14:24:50 +0300 Subject: [PATCH 2/9] [gzhttp] Remove returns http-status bad request --- gzhttp/compress.go | 7 ++----- gzhttp/compress_test.go | 10 ---------- 2 files changed, 2 insertions(+), 15 deletions(-) diff --git a/gzhttp/compress.go b/gzhttp/compress.go index b8df6a2fa..0a40572be 100644 --- a/gzhttp/compress.go +++ b/gzhttp/compress.go @@ -465,12 +465,9 @@ func NewWrapper(opts ...option) (func(http.Handler) http.HandlerFunc, error) { return func(w http.ResponseWriter, r *http.Request) { w.Header().Add(vary, acceptEncoding) if contentGzip(r) { - readerGzipBody, err := gzip.NewReader(r.Body) - if err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return + if readerGzipBody, err := gzip.NewReader(r.Body); err == nil { + r.Body = io.NopCloser(readerGzipBody) } - r.Body = io.NopCloser(readerGzipBody) } if acceptsGzip(r) { diff --git a/gzhttp/compress_test.go b/gzhttp/compress_test.go index 74ccebc9e..c844b7653 100644 --- a/gzhttp/compress_test.go +++ b/gzhttp/compress_test.go @@ -90,16 +90,6 @@ func TestMustNewGzipHandler(t *testing.T) { assertEqual(t, http.DetectContentType([]byte(testBody)), res3.Header().Get("Content-Type")) - // send not compress request body - - req4, _ := http.NewRequest("POST", "/whatever", bytes.NewBuffer(testBody)) - req4.Header.Set("Content-Encoding", "gzip") - resp4 := httptest.NewRecorder() - handler.ServeHTTP(resp4, req4) - res4 := resp4.Result() - - assertEqual(t, 400, res4.StatusCode) - // send compress request body var b bytes.Buffer From db438c95bef532796cf09dc48a3b264938a72b8c Mon Sep 17 00:00:00 2001 From: mirecl Date: Tue, 10 Sep 2024 14:32:07 +0300 Subject: [PATCH 3/9] [gzhttp] Add docs --- gzhttp/compress.go | 1 + 1 file changed, 1 insertion(+) diff --git a/gzhttp/compress.go b/gzhttp/compress.go index 0a40572be..874f64649 100644 --- a/gzhttp/compress.go +++ b/gzhttp/compress.go @@ -758,6 +758,7 @@ func RandomJitter(n, buffer int, paranoid bool) option { } } +// contentGzip returns true if the given HTTP request indicates that it gzipped. func contentGzip(r *http.Request) bool { // See more detail in `acceptsGzip` return r.Method != http.MethodHead && parseEncodingGzip(r.Header.Get(contentEncoding)) > 0 From 96323270abfe59221fdf8ba1c9f4542dc7f614eb Mon Sep 17 00:00:00 2001 From: mirecl Date: Tue, 10 Sep 2024 15:36:42 +0300 Subject: [PATCH 4/9] [gzhttp] Remove Content-Encoding from request header --- gzhttp/compress.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/gzhttp/compress.go b/gzhttp/compress.go index 874f64649..12a6fb85e 100644 --- a/gzhttp/compress.go +++ b/gzhttp/compress.go @@ -465,6 +465,8 @@ func NewWrapper(opts ...option) (func(http.Handler) http.HandlerFunc, error) { return func(w http.ResponseWriter, r *http.Request) { w.Header().Add(vary, acceptEncoding) if contentGzip(r) { + r.Header.Del(contentEncoding) + if readerGzipBody, err := gzip.NewReader(r.Body); err == nil { r.Body = io.NopCloser(readerGzipBody) } From 858831fefa55c48b7c05b96e3490266334972d7e Mon Sep 17 00:00:00 2001 From: mirecl Date: Tue, 10 Sep 2024 15:37:50 +0300 Subject: [PATCH 5/9] [gzhttp] Add check exits request body --- gzhttp/compress.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gzhttp/compress.go b/gzhttp/compress.go index 12a6fb85e..8a591b83d 100644 --- a/gzhttp/compress.go +++ b/gzhttp/compress.go @@ -763,7 +763,7 @@ func RandomJitter(n, buffer int, paranoid bool) option { // contentGzip returns true if the given HTTP request indicates that it gzipped. func contentGzip(r *http.Request) bool { // See more detail in `acceptsGzip` - return r.Method != http.MethodHead && parseEncodingGzip(r.Header.Get(contentEncoding)) > 0 + return r.Method != http.MethodHead && r.Body != nil && parseEncodingGzip(r.Header.Get(contentEncoding)) > 0 } // acceptsGzip returns true if the given HTTP request indicates that it will From c33f0a3d4f9630d2cad4fe68bb10d3fefc153ceb Mon Sep 17 00:00:00 2001 From: mirecl Date: Tue, 10 Sep 2024 16:46:17 +0300 Subject: [PATCH 6/9] [gzhttp] Add gzReaderPool --- gzhttp/compress.go | 56 ++++++++++++++++++++++++++++++----------- gzhttp/compress_test.go | 3 ++- 2 files changed, 44 insertions(+), 15 deletions(-) diff --git a/gzhttp/compress.go b/gzhttp/compress.go index 8a591b83d..80774bb1e 100644 --- a/gzhttp/compress.go +++ b/gzhttp/compress.go @@ -464,11 +464,29 @@ func NewWrapper(opts ...option) (func(http.Handler) http.HandlerFunc, error) { return func(h http.Handler) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { w.Header().Add(vary, acceptEncoding) - if contentGzip(r) { + if c.allowCompressedRequests && contentGzip(r) { r.Header.Del(contentEncoding) - if readerGzipBody, err := gzip.NewReader(r.Body); err == nil { - r.Body = io.NopCloser(readerGzipBody) + var ( + err error + gz *gzip.Reader + ) + + zr, ok := gzReaderPool.Get().(*gzip.Reader) + if ok { + gz, err = zr, zr.Reset(r.Body) + } else { + gz, err = gzip.NewReader(r.Body) + } + + defer func() { + gzReaderPool.Put(gz) + gz = nil + }() + + if err == nil { + r.Body = io.NopCloser(gz) + defer r.Body.Close() } } @@ -544,17 +562,18 @@ func (pct parsedContentType) equals(mediaType string, params map[string]string) // Used for functional configuration. type config struct { - minSize int - level int - writer writer.GzipWriterFactory - contentTypes func(ct string) bool - keepAcceptRanges bool - setContentType bool - suffixETag string - dropETag bool - jitterBuffer int - randomJitter string - sha256Jitter bool + minSize int + level int + writer writer.GzipWriterFactory + contentTypes func(ct string) bool + keepAcceptRanges bool + setContentType bool + suffixETag string + dropETag bool + jitterBuffer int + randomJitter string + sha256Jitter bool + allowCompressedRequests bool } func (c *config) validate() error { @@ -587,6 +606,15 @@ func MinSize(size int) option { } } +// AllowCompressedRequests will enable or disable RFC 7694 compressed requests. +// By default this is Disabled. +// See https://datatracker.ietf.org/doc/html/rfc7694 +func AllowCompressedRequests(b bool) option { + return func(c *config) { + c.allowCompressedRequests = b + } +} + // CompressionLevel sets the compression level func CompressionLevel(level int) option { return func(c *config) { diff --git a/gzhttp/compress_test.go b/gzhttp/compress_test.go index c844b7653..41bedb231 100644 --- a/gzhttp/compress_test.go +++ b/gzhttp/compress_test.go @@ -91,6 +91,7 @@ func TestMustNewGzipHandler(t *testing.T) { assertEqual(t, http.DetectContentType([]byte(testBody)), res3.Header().Get("Content-Type")) // send compress request body + handlerCompressedRequests := newTestHandlerLevel(testBody, AllowCompressedRequests(true)) var b bytes.Buffer writerGzip := gzip.NewWriter(&b) @@ -100,7 +101,7 @@ func TestMustNewGzipHandler(t *testing.T) { req5, _ := http.NewRequest("POST", "/whatever", &b) req5.Header.Set("Content-Encoding", "gzip") resp5 := httptest.NewRecorder() - handler.ServeHTTP(resp5, req5) + handlerCompressedRequests.ServeHTTP(resp5, req5) res5 := resp5.Result() assertEqual(t, 200, res5.StatusCode) From 5a3488fd9ff8fe12e8d0c85f3bd1620bb2c46a50 Mon Sep 17 00:00:00 2001 From: mirecl Date: Tue, 10 Sep 2024 17:31:34 +0300 Subject: [PATCH 7/9] [gzhttp] Add gzipReader --- gzhttp/compress.go | 24 ++---------------------- 1 file changed, 2 insertions(+), 22 deletions(-) diff --git a/gzhttp/compress.go b/gzhttp/compress.go index 80774bb1e..3a5af9757 100644 --- a/gzhttp/compress.go +++ b/gzhttp/compress.go @@ -466,28 +466,8 @@ func NewWrapper(opts ...option) (func(http.Handler) http.HandlerFunc, error) { w.Header().Add(vary, acceptEncoding) if c.allowCompressedRequests && contentGzip(r) { r.Header.Del(contentEncoding) - - var ( - err error - gz *gzip.Reader - ) - - zr, ok := gzReaderPool.Get().(*gzip.Reader) - if ok { - gz, err = zr, zr.Reset(r.Body) - } else { - gz, err = gzip.NewReader(r.Body) - } - - defer func() { - gzReaderPool.Put(gz) - gz = nil - }() - - if err == nil { - r.Body = io.NopCloser(gz) - defer r.Body.Close() - } + r.Body = io.NopCloser(&gzipReader{body: r.Body}) + defer r.Body.Close() } if acceptsGzip(r) { From 5aac9f619925eea7fe09a5d64e86736c87062ab6 Mon Sep 17 00:00:00 2001 From: mirecl Date: Wed, 11 Sep 2024 15:59:29 +0300 Subject: [PATCH 8/9] [gzhttp] Fix decompress request bode + add unit-test --- gzhttp/compress.go | 3 +-- gzhttp/compress_test.go | 18 +++++++++++++++++- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/gzhttp/compress.go b/gzhttp/compress.go index 3a5af9757..28fe33195 100644 --- a/gzhttp/compress.go +++ b/gzhttp/compress.go @@ -466,8 +466,7 @@ func NewWrapper(opts ...option) (func(http.Handler) http.HandlerFunc, error) { w.Header().Add(vary, acceptEncoding) if c.allowCompressedRequests && contentGzip(r) { r.Header.Del(contentEncoding) - r.Body = io.NopCloser(&gzipReader{body: r.Body}) - defer r.Body.Close() + r.Body = &gzipReader{body: r.Body} } if acceptsGzip(r) { diff --git a/gzhttp/compress_test.go b/gzhttp/compress_test.go index 41bedb231..9007f8bbf 100644 --- a/gzhttp/compress_test.go +++ b/gzhttp/compress_test.go @@ -90,7 +90,7 @@ func TestMustNewGzipHandler(t *testing.T) { assertEqual(t, http.DetectContentType([]byte(testBody)), res3.Header().Get("Content-Type")) - // send compress request body + // send compress request body with `AllowCompressedRequests` handlerCompressedRequests := newTestHandlerLevel(testBody, AllowCompressedRequests(true)) var b bytes.Buffer @@ -108,6 +108,22 @@ func TestMustNewGzipHandler(t *testing.T) { body, _ := io.ReadAll(res5.Body) assertEqual(t, len(testBody), len(body)) + + // send compress request body without `AllowCompressedRequests` + writerGzip = gzip.NewWriter(&b) + writerGzip.Write(testBody) + writerGzip.Close() + + handlerCompressedRequests = newTestHandlerLevel(b.Bytes()) + + req6, _ := http.NewRequest("POST", "/whatever", &b) + resp6 := httptest.NewRecorder() + handlerCompressedRequests.ServeHTTP(resp6, req6) + res6 := resp6.Result() + + assertEqual(t, 200, res6.StatusCode) + body, _ = io.ReadAll(res6.Body) + assertEqual(t, b.Len(), len(body)) } func TestGzipHandlerSmallBodyNoCompression(t *testing.T) { From 5e01f31a85ef6375d269a09fc5d50aed587609d9 Mon Sep 17 00:00:00 2001 From: mirecl Date: Wed, 11 Sep 2024 16:05:52 +0300 Subject: [PATCH 9/9] [gzhttp] Fix unit-test compress request body --- gzhttp/compress_test.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/gzhttp/compress_test.go b/gzhttp/compress_test.go index 9007f8bbf..03c220d67 100644 --- a/gzhttp/compress_test.go +++ b/gzhttp/compress_test.go @@ -91,7 +91,7 @@ func TestMustNewGzipHandler(t *testing.T) { assertEqual(t, http.DetectContentType([]byte(testBody)), res3.Header().Get("Content-Type")) // send compress request body with `AllowCompressedRequests` - handlerCompressedRequests := newTestHandlerLevel(testBody, AllowCompressedRequests(true)) + handler = newTestHandlerLevel(testBody, AllowCompressedRequests(true)) var b bytes.Buffer writerGzip := gzip.NewWriter(&b) @@ -101,7 +101,7 @@ func TestMustNewGzipHandler(t *testing.T) { req5, _ := http.NewRequest("POST", "/whatever", &b) req5.Header.Set("Content-Encoding", "gzip") resp5 := httptest.NewRecorder() - handlerCompressedRequests.ServeHTTP(resp5, req5) + handler.ServeHTTP(resp5, req5) res5 := resp5.Result() assertEqual(t, 200, res5.StatusCode) @@ -114,11 +114,11 @@ func TestMustNewGzipHandler(t *testing.T) { writerGzip.Write(testBody) writerGzip.Close() - handlerCompressedRequests = newTestHandlerLevel(b.Bytes()) + handler = newTestHandlerLevel(b.Bytes()) req6, _ := http.NewRequest("POST", "/whatever", &b) resp6 := httptest.NewRecorder() - handlerCompressedRequests.ServeHTTP(resp6, req6) + handler.ServeHTTP(resp6, req6) res6 := resp6.Result() assertEqual(t, 200, res6.StatusCode)