From dce3b3962f94e59629b3de4d2b38932d04cc21f6 Mon Sep 17 00:00:00 2001 From: Jordan Keister Date: Thu, 27 Jun 2024 15:09:13 -0500 Subject: [PATCH] adding gzip content support Signed-off-by: Jordan Keister --- Makefile | 1 + go.mod | 1 + go.sum | 2 + pkg/storage/localdir.go | 4 +- pkg/storage/localdir_test.go | 181 +++++++++++++++++++++++++++++++++++ 5 files changed, 188 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index a22ef9f8..050a33ba 100644 --- a/Makefile +++ b/Makefile @@ -52,6 +52,7 @@ KIND_CLUSTER_IMAGE := kindest/node:v1.30.0@sha256:047357ac0cfea04663786a612ba1ea .PHONY: help help: ## Display this help. awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \033[36m\033[0m\n"} /^[a-zA-Z_0-9-]+:.*?##/ { printf " \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST) +.DEFAULT_GOAL := help ##@ Development diff --git a/go.mod b/go.mod index 2dc5eff6..e6612021 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module github.com/operator-framework/catalogd go 1.22.0 require ( + github.com/NYTimes/gziphandler v1.1.1 github.com/blang/semver/v4 v4.0.0 github.com/containerd/containerd v1.7.18 github.com/go-logr/logr v1.4.1 diff --git a/go.sum b/go.sum index 6176a561..98c0b228 100644 --- a/go.sum +++ b/go.sum @@ -33,6 +33,8 @@ github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERo github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= github.com/Microsoft/hcsshim v0.12.0-rc.3 h1:5GNGrobGs/sN/0nFO21W9k4lFn+iXXZAE8fCZbmdRak= github.com/Microsoft/hcsshim v0.12.0-rc.3/go.mod h1:WuNfcaYNaw+KpCEsZCIM6HCEmu0c5HfXpi+dDSmveP0= +github.com/NYTimes/gziphandler v1.1.1 h1:ZUDjpQae29j0ryrS0u/B8HZfJBtBQHjqw2rQ2cqUQ3I= +github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c= github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230305170008-8188dc5388df h1:7RFfzj4SSt6nnvCPbCqijJi1nWCd+TqAT3bYCStRC18= github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230305170008-8188dc5388df/go.mod h1:pSwJ0fSY5KhvocuWSx4fz3BA8OrA1bQn+K1Eli3BRwM= github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3dyBCFEj5IhUbnKptjxatkF07cF2ak3yi77so= diff --git a/pkg/storage/localdir.go b/pkg/storage/localdir.go index a34924c2..002655b7 100644 --- a/pkg/storage/localdir.go +++ b/pkg/storage/localdir.go @@ -9,6 +9,8 @@ import ( "os" "path/filepath" + "github.com/NYTimes/gziphandler" + "github.com/operator-framework/operator-registry/alpha/declcfg" ) @@ -56,7 +58,7 @@ func (s LocalDir) ContentURL(catalog string) string { func (s LocalDir) StorageServerHandler() http.Handler { mux := http.NewServeMux() - mux.Handle(s.BaseURL.Path, http.StripPrefix(s.BaseURL.Path, http.FileServer(http.FS(&filesOnlyFilesystem{os.DirFS(s.RootDir)})))) + mux.Handle(s.BaseURL.Path, gziphandler.GzipHandler(http.StripPrefix(s.BaseURL.Path, http.FileServer(http.FS(&filesOnlyFilesystem{os.DirFS(s.RootDir)}))))) return mux } diff --git a/pkg/storage/localdir_test.go b/pkg/storage/localdir_test.go index 2e15003d..83824ee9 100644 --- a/pkg/storage/localdir_test.go +++ b/pkg/storage/localdir_test.go @@ -1,6 +1,7 @@ package storage import ( + "bytes" "context" "fmt" "io" @@ -15,6 +16,8 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + "compress/gzip" + "github.com/google/go-cmp/cmp" "github.com/operator-framework/operator-registry/alpha/declcfg" @@ -132,6 +135,40 @@ var _ = Describe("LocalDir Server Handler tests", func() { Expect(os.WriteFile(filepath.Join(store.RootDir, "test-catalog", "foo.txt"), expectedContent, 0600)).To(Succeed()) expectFound(fmt.Sprintf("%s/%s", testServer.URL, "/catalogs/test-catalog/foo.txt"), expectedContent) }) + It("ignores accept-encoding for the path /catalogs/test-catalog/all.json with size < 1400 bytes", func() { + expectedContent := []byte("bar") + Expect(os.WriteFile(filepath.Join(store.RootDir, "test-catalog", "all.json"), expectedContent, 0600)).To(Succeed()) + req, err := http.NewRequest(http.MethodGet, fmt.Sprintf("%s/%s", testServer.URL, "/catalogs/test-catalog/all.json"), nil) + Expect(err).To(Not(HaveOccurred())) + req.Header.Set("Accept-Encoding", "gzip") + resp, err := http.DefaultClient.Do(req) + Expect(err).To(Not(HaveOccurred())) + Expect(resp.StatusCode).To(Equal(http.StatusOK)) + Expect(resp.Header.Get("Content-Encoding")).To(Not(Equal("gzip"))) + actualContent, err := io.ReadAll(resp.Body) + Expect(err).To(Not(HaveOccurred())) + Expect(actualContent).To(Equal(expectedContent)) + Expect(resp.Body.Close()).To(Succeed()) + }) + It("provides gzipped content for the path /catalogs/test-catalog/all.json with size > 1400 bytes", func() { + expectedContent := []byte(testCompressablePackage) + Expect(os.WriteFile(filepath.Join(store.RootDir, "test-catalog", "all.json"), expectedContent, 0600)).To(Succeed()) + req, err := http.NewRequest(http.MethodGet, fmt.Sprintf("%s/%s", testServer.URL, "/catalogs/test-catalog/all.json"), nil) + Expect(err).To(Not(HaveOccurred())) + req.Header.Set("Accept-Encoding", "gzip") + resp, err := http.DefaultClient.Do(req) + Expect(err).To(Not(HaveOccurred())) + Expect(resp.StatusCode).To(Equal(http.StatusOK)) + Expect(resp.Header.Get("Content-Encoding")).To(Equal("gzip")) + actualContent, err := io.ReadAll(resp.Body) + Expect(err).To(Not(HaveOccurred())) + gz, err := gzip.NewReader(io.NopCloser(io.MultiReader(bytes.NewReader(actualContent)))) + Expect(err).To(Not(HaveOccurred())) + decompBytes, err := io.ReadAll(gz) + Expect(err).To(Not(HaveOccurred())) + Expect(decompBytes).To(Equal(expectedContent)) + Expect(resp.Body.Close()).To(Succeed()) + }) AfterEach(func() { testServer.Close() }) @@ -184,3 +221,147 @@ name: %s entries: - name: %s ` + +// by default the compressor will only trigger for files larger than 1400 bytes +const testCompressablePackage = `{ + "schema": "olm.package", + "name": "cockroachdb", + "defaultChannel": "stable-v6.x", +} +{ + "schema": "olm.channel", + "name": "stable-5.x", + "package": "cockroachdb", + "entries": [ + { + "name": "cockroachdb.v5.0.3" + }, + { + "name": "cockroachdb.v5.0.4", + "replaces": "cockroachdb.v5.0.3" + } + ] +} +{ + "schema": "olm.channel", + "name": "stable-v6.x", + "package": "cockroachdb", + "entries": [ + { + "name": "cockroachdb.v6.0.0", + "skipRange": "<6.0.0" + } + ] +} +{ + "schema": "olm.bundle", + "name": "cockroachdb.v5.0.3", + "package": "cockroachdb", + "image": "quay.io/openshift-community-operators/cockroachdb@sha256:a5d4f4467250074216eb1ba1c36e06a3ab797d81c431427fc2aca97ecaf4e9d8", + "properties": [ + { + "type": "olm.gvk", + "value": { + "group": "charts.operatorhub.io", + "kind": "Cockroachdb", + "version": "v1alpha1" + } + }, + { + "type": "olm.package", + "value": { + "packageName": "cockroachdb", + "version": "5.0.3" + } + } + ], + "relatedImages": [ + { + "name": "", + "image": "gcr.io/kubebuilder/kube-rbac-proxy:v0.5.0" + }, + { + "name": "", + "image": "quay.io/helmoperators/cockroachdb:v5.0.3" + }, + { + "name": "", + "image": "quay.io/openshift-community-operators/cockroachdb@sha256:a5d4f4467250074216eb1ba1c36e06a3ab797d81c431427fc2aca97ecaf4e9d8" + } + ] +} +{ + "schema": "olm.bundle", + "name": "cockroachdb.v5.0.4", + "package": "cockroachdb", + "image": "quay.io/openshift-community-operators/cockroachdb@sha256:f42337e7b85a46d83c94694638e2312e10ca16a03542399a65ba783c94a32b63", + "properties": [ + { + "type": "olm.gvk", + "value": { + "group": "charts.operatorhub.io", + "kind": "Cockroachdb", + "version": "v1alpha1" + } + }, + { + "type": "olm.package", + "value": { + "packageName": "cockroachdb", + "version": "5.0.4" + } + }, + ], + "relatedImages": [ + { + "name": "", + "image": "gcr.io/kubebuilder/kube-rbac-proxy:v0.5.0" + }, + { + "name": "", + "image": "quay.io/helmoperators/cockroachdb:v5.0.4" + }, + { + "name": "", + "image": "quay.io/openshift-community-operators/cockroachdb@sha256:f42337e7b85a46d83c94694638e2312e10ca16a03542399a65ba783c94a32b63" + } + ] +} +{ + "schema": "olm.bundle", + "name": "cockroachdb.v6.0.0", + "package": "cockroachdb", + "image": "quay.io/openshift-community-operators/cockroachdb@sha256:d3016b1507515fc7712f9c47fd9082baf9ccb070aaab58ed0ef6e5abdedde8ba", + "properties": [ + { + "type": "olm.gvk", + "value": { + "group": "charts.operatorhub.io", + "kind": "Cockroachdb", + "version": "v1alpha1" + } + }, + { + "type": "olm.package", + "value": { + "packageName": "cockroachdb", + "version": "6.0.0" + } + }, + ], + "relatedImages": [ + { + "name": "", + "image": "gcr.io/kubebuilder/kube-rbac-proxy:v0.5.0" + }, + { + "name": "", + "image": "quay.io/cockroachdb/cockroach-helm-operator:6.0.0" + }, + { + "name": "", + "image": "quay.io/openshift-community-operators/cockroachdb@sha256:d3016b1507515fc7712f9c47fd9082baf9ccb070aaab58ed0ef6e5abdedde8ba" + } + ] +} +`