adding gzip content support (#293)

Signed-off-by: Jordan Keister <jordan@nimblewidget.com>

Makefile

@@ -1,3 +1,8 @@
+# Setting SHELL to bash allows bash commands to be executed by recipes.
+# Options are set to exit when a recipe line exits non-zero or a piped command fails.
+SHELL := /usr/bin/env bash -o pipefail
+.SHELLFLAGS := -ec
+
 ifeq ($(origin IMAGE_REPO), undefined)
 IMAGE_REPO := quay.io/operator-framework/catalogd
 endif
@@ -52,6 +57,7 @@ KIND_CLUSTER_IMAGE := kindest/node:v1.30.0@sha256:047357ac0cfea04663786a612ba1ea
 .PHONY: help
 help: ## Display this help.
 	awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n  make \033[36m<target>\033[0m\n"} /^[a-zA-Z_0-9-]+:.*?##/ { printf "  \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST)
+.DEFAULT_GOAL := help
 
 ##@ Development
 

go.mod

@@ -3,6 +3,7 @@ module github.com/operator-framework/catalogd
 go 1.22.0
 
 require (
+	github.com/NYTimes/gziphandler v1.1.1
 	github.com/blang/semver/v4 v4.0.0
 	github.com/containerd/containerd v1.7.18
 	github.com/go-logr/logr v1.4.1

go.sum

@@ -33,6 +33,8 @@ github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERo
 github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU=
 github.com/Microsoft/hcsshim v0.12.0-rc.3 h1:5GNGrobGs/sN/0nFO21W9k4lFn+iXXZAE8fCZbmdRak=
 github.com/Microsoft/hcsshim v0.12.0-rc.3/go.mod h1:WuNfcaYNaw+KpCEsZCIM6HCEmu0c5HfXpi+dDSmveP0=
+github.com/NYTimes/gziphandler v1.1.1 h1:ZUDjpQae29j0ryrS0u/B8HZfJBtBQHjqw2rQ2cqUQ3I=
+github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c=
 github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230305170008-8188dc5388df h1:7RFfzj4SSt6nnvCPbCqijJi1nWCd+TqAT3bYCStRC18=
 github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230305170008-8188dc5388df/go.mod h1:pSwJ0fSY5KhvocuWSx4fz3BA8OrA1bQn+K1Eli3BRwM=
 github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3dyBCFEj5IhUbnKptjxatkF07cF2ak3yi77so=

pkg/storage/localdir.go

@@ -9,6 +9,8 @@ import (
 	"os"
 	"path/filepath"
 
+	"github.com/NYTimes/gziphandler"
+
 	"github.com/operator-framework/operator-registry/alpha/declcfg"
 )
 
@@ -56,7 +58,7 @@ func (s LocalDir) ContentURL(catalog string) string {
 
 func (s LocalDir) StorageServerHandler() http.Handler {
 	mux := http.NewServeMux()
-	mux.Handle(s.BaseURL.Path, http.StripPrefix(s.BaseURL.Path, http.FileServer(http.FS(&filesOnlyFilesystem{os.DirFS(s.RootDir)}))))
+	mux.Handle(s.BaseURL.Path, gziphandler.GzipHandler(http.StripPrefix(s.BaseURL.Path, http.FileServer(http.FS(&filesOnlyFilesystem{os.DirFS(s.RootDir)})))))
 	return mux
 }
 

pkg/storage/localdir_test.go

@@ -1,6 +1,7 @@
 package storage
 
 import (
+	"compress/gzip"
 	"context"
 	"fmt"
 	"io"
@@ -132,6 +133,16 @@ var _ = Describe("LocalDir Server Handler tests", func() {
 		Expect(os.WriteFile(filepath.Join(store.RootDir, "test-catalog", "foo.txt"), expectedContent, 0600)).To(Succeed())
 		expectFound(fmt.Sprintf("%s/%s", testServer.URL, "/catalogs/test-catalog/foo.txt"), expectedContent)
 	})
+	It("ignores accept-encoding for the path /catalogs/test-catalog/all.json with size < 1400 bytes", func() {
+		expectedContent := []byte("bar")
+		Expect(os.WriteFile(filepath.Join(store.RootDir, "test-catalog", "all.json"), expectedContent, 0600)).To(Succeed())
+		expectFound(fmt.Sprintf("%s/%s", testServer.URL, "/catalogs/test-catalog/all.json"), expectedContent)
+	})
+	It("provides gzipped content for the path /catalogs/test-catalog/all.json with size > 1400 bytes", func() {
+		expectedContent := []byte(testCompressablePackage)
+		Expect(os.WriteFile(filepath.Join(store.RootDir, "test-catalog", "all.json"), expectedContent, 0600)).To(Succeed())
+		expectFound(fmt.Sprintf("%s/%s", testServer.URL, "/catalogs/test-catalog/all.json"), expectedContent)
+	})
 	AfterEach(func() {
 		testServer.Close()
 	})
@@ -145,11 +156,27 @@ func expectNotFound(url string) {
 }
 
 func expectFound(url string, expectedContent []byte) {
-	resp, err := http.Get(url) //nolint:gosec
+	req, err := http.NewRequest(http.MethodGet, url, nil)
 	Expect(err).To(Not(HaveOccurred()))
-	Expect(resp.StatusCode).To(Equal(http.StatusOK))
-	actualContent, err := io.ReadAll(resp.Body)
+	req.Header.Set("Accept-Encoding", "gzip")
+	resp, err := http.DefaultClient.Do(req)
 	Expect(err).To(Not(HaveOccurred()))
+	Expect(resp.StatusCode).To(Equal(http.StatusOK))
+
+	var actualContent []byte
+	switch resp.Header.Get("Content-Encoding") {
+	case "gzip":
+		Expect(len(expectedContent)).To(BeNumerically(">", 1400))
+		gz, err := gzip.NewReader(resp.Body)
+		Expect(err).To(Not(HaveOccurred()))
+		actualContent, err = io.ReadAll(gz)
+		Expect(err).To(Not(HaveOccurred()))
+	default:
+		Expect(len(expectedContent)).To(BeNumerically("<", 1400))
+		actualContent, err = io.ReadAll(resp.Body)
+		Expect(err).To(Not(HaveOccurred()))
+	}
+
 	Expect(actualContent).To(Equal(expectedContent))
 	Expect(resp.Body.Close()).To(Succeed())
 }
@@ -184,3 +211,147 @@ name: %s
 entries:
   - name: %s
 `
+
+// by default the compressor will only trigger for files larger than 1400 bytes
+const testCompressablePackage = `{
+  "schema": "olm.package",
+  "name": "cockroachdb",
+  "defaultChannel": "stable-v6.x",
+}
+{
+  "schema": "olm.channel",
+  "name": "stable-5.x",
+  "package": "cockroachdb",
+  "entries": [
+    {
+      "name": "cockroachdb.v5.0.3"
+    },
+    {
+      "name": "cockroachdb.v5.0.4",
+      "replaces": "cockroachdb.v5.0.3"
+    }
+  ]
+}
+{
+  "schema": "olm.channel",
+  "name": "stable-v6.x",
+  "package": "cockroachdb",
+  "entries": [
+    {
+      "name": "cockroachdb.v6.0.0",
+      "skipRange": "<6.0.0"
+    }
+  ]
+}
+{
+  "schema": "olm.bundle",
+  "name": "cockroachdb.v5.0.3",
+  "package": "cockroachdb",
+  "image": "quay.io/openshift-community-operators/cockroachdb@sha256:a5d4f4467250074216eb1ba1c36e06a3ab797d81c431427fc2aca97ecaf4e9d8",
+  "properties": [
+    {
+      "type": "olm.gvk",
+      "value": {
+        "group": "charts.operatorhub.io",
+        "kind": "Cockroachdb",
+        "version": "v1alpha1"
+      }
+    },
+    {
+      "type": "olm.package",
+      "value": {
+        "packageName": "cockroachdb",
+        "version": "5.0.3"
+      }
+    }
+  ],
+  "relatedImages": [
+    {
+      "name": "",
+      "image": "gcr.io/kubebuilder/kube-rbac-proxy:v0.5.0"
+    },
+    {
+      "name": "",
+      "image": "quay.io/helmoperators/cockroachdb:v5.0.3"
+    },
+    {
+      "name": "",
+      "image": "quay.io/openshift-community-operators/cockroachdb@sha256:a5d4f4467250074216eb1ba1c36e06a3ab797d81c431427fc2aca97ecaf4e9d8"
+    }
+  ]
+}
+{
+  "schema": "olm.bundle",
+  "name": "cockroachdb.v5.0.4",
+  "package": "cockroachdb",
+  "image": "quay.io/openshift-community-operators/cockroachdb@sha256:f42337e7b85a46d83c94694638e2312e10ca16a03542399a65ba783c94a32b63",
+  "properties": [
+    {
+      "type": "olm.gvk",
+      "value": {
+        "group": "charts.operatorhub.io",
+        "kind": "Cockroachdb",
+        "version": "v1alpha1"
+      }
+    },
+    {
+      "type": "olm.package",
+      "value": {
+        "packageName": "cockroachdb",
+        "version": "5.0.4"
+      }
+    },
+  ],
+  "relatedImages": [
+    {
+      "name": "",
+      "image": "gcr.io/kubebuilder/kube-rbac-proxy:v0.5.0"
+    },
+    {
+      "name": "",
+      "image": "quay.io/helmoperators/cockroachdb:v5.0.4"
+    },
+    {
+      "name": "",
+      "image": "quay.io/openshift-community-operators/cockroachdb@sha256:f42337e7b85a46d83c94694638e2312e10ca16a03542399a65ba783c94a32b63"
+    }
+  ]
+}
+{
+  "schema": "olm.bundle",
+  "name": "cockroachdb.v6.0.0",
+  "package": "cockroachdb",
+  "image": "quay.io/openshift-community-operators/cockroachdb@sha256:d3016b1507515fc7712f9c47fd9082baf9ccb070aaab58ed0ef6e5abdedde8ba",
+  "properties": [
+    {
+      "type": "olm.gvk",
+      "value": {
+        "group": "charts.operatorhub.io",
+        "kind": "Cockroachdb",
+        "version": "v1alpha1"
+      }
+    },
+    {
+      "type": "olm.package",
+      "value": {
+        "packageName": "cockroachdb",
+        "version": "6.0.0"
+      }
+    },
+  ],
+  "relatedImages": [
+    {
+      "name": "",
+      "image": "gcr.io/kubebuilder/kube-rbac-proxy:v0.5.0"
+    },
+    {
+      "name": "",
+      "image": "quay.io/cockroachdb/cockroach-helm-operator:6.0.0"
+    },
+    {
+      "name": "",
+      "image": "quay.io/openshift-community-operators/cockroachdb@sha256:d3016b1507515fc7712f9c47fd9082baf9ccb070aaab58ed0ef6e5abdedde8ba"
+    }
+  ]
+}
+`