fix(fetcher/fedora): determine mime type from the file extension

.github/workflows/fetch.yml

@@ -354,7 +354,7 @@ jobs:
           --health-timeout 5s
           --health-retries 5
     env:
-      Version: 32 33 34 35 36 37 38 39 40
+      Version: 32 33 34 35 36 37 38 39 40 41
     steps:
       - name: Check out code into the Go module directory
         uses: actions/checkout@v3

fetcher/fedora/fedora.go

@@ -7,6 +7,8 @@ import (
 	"errors"
 	"fmt"
 	"net/url"
+	"path"
+	"path/filepath"
 	"regexp"
 	"strconv"
 	"strings"
@@ -192,7 +194,7 @@ var errNoUpdateInfoField = xerrors.New("No updateinfo field in the repomd")
 
 func fetchUpdateInfosFedora(results []util.FetchResult) ([]util.FetchResult, error) {
 	log15.Info("start fetch updateinfo in repomd.xml")
-	updateInfoReqs, err := extractInfoFromRepoMd(results, "updateinfo", util.MIMETypeXz)
+	updateInfoReqs, err := extractInfoFromRepoMd(results, "updateinfo")
 	if err != nil {
 		return nil, xerrors.Errorf("Failed to extract updateinfo from xml, err: %w", err)
 	}
@@ -274,7 +276,7 @@ func fetchModuleFeedFilesFedora(reqs []util.FetchRequest) ([]util.FetchResult, e
 
 func fetchModulesYamlFedora(results []util.FetchResult) (moduleInfosPerVersion, error) {
 	log15.Info("start fetch modules.yaml in repomd.xml")
-	updateInfoReqs, err := extractInfoFromRepoMd(results, "modules", util.MIMETypeGzip)
+	updateInfoReqs, err := extractInfoFromRepoMd(results, "modules")
 	if err != nil {
 		return nil, xerrors.Errorf("Failed to extract modules from xml, err: %w", err)
 	}
@@ -377,7 +379,7 @@ func fetchCveIDsFromBugzilla(id string) ([]string, error) {
 	return ids, nil
 }
 
-func extractInfoFromRepoMd(results []util.FetchResult, rt string, mt util.MIMEType) ([]util.FetchRequest, error) {
+func extractInfoFromRepoMd(results []util.FetchResult, rt string) ([]util.FetchRequest, error) {
 	var updateInfoReqs []util.FetchRequest
 	for _, r := range results {
 		var repoMd repoMd
@@ -394,6 +396,23 @@ func extractInfoFromRepoMd(results []util.FetchResult, rt string, mt util.MIMETy
 				return nil, xerrors.Errorf("Failed to parse URL in XML. err: %w", err)
 			}
 			u.Path = strings.Replace(u.Path, "repodata/repomd.xml", repo.Location.Href, 1)
+
+			mt, err := func() (util.MIMEType, error) {
+				switch ext := filepath.Ext(path.Base(repo.Location.Href)); ext {
+				case ".gz":
+					return util.MIMETypeGzip, nil
+				case ".xz":
+					return util.MIMETypeXz, nil
+				case ".zst":
+					return util.MIMETypeZst, nil
+				default:
+					return util.MIMETypeUnknown, xerrors.Errorf("%q is not supported extension", ext)
+				}
+			}()
+			if err != nil {
+				return nil, xerrors.Errorf("Failed to get MIME Type. err: %w", err)
+			}
+
 			req := util.FetchRequest{
 				URL:      u.String(),
 				Target:   r.Target,

fetcher/util/fetcher.go

@@ -12,6 +12,7 @@ import (
 	"time"
 
 	"github.com/inconshreveable/log15"
+	"github.com/klauspost/compress/zstd"
 	"github.com/spf13/viper"
 	"github.com/ulikunitz/xz"
 	"golang.org/x/xerrors"
@@ -23,8 +24,10 @@ import (
 type MIMEType int
 
 const (
+	// MIMETypeUnknown :
+	MIMETypeUnknown MIMEType = iota
 	// MIMETypeXML :
-	MIMETypeXML MIMEType = iota
+	MIMETypeXML
 	// MIMETypeTxt :
 	MIMETypeTxt
 	// MIMETypeJSON :
@@ -39,6 +42,8 @@ const (
 	MIMETypeXz
 	// MIMETypeGzip :
 	MIMETypeGzip
+	// MIMETypeZst :
+	MIMETypeZst
 )
 
 func (m MIMEType) String() string {
@@ -59,6 +64,8 @@ func (m MIMEType) String() string {
 		return "xz"
 	case MIMETypeGzip:
 		return "gz"
+	case MIMETypeZst:
+		return "zst"
 	default:
 		return "Unknown"
 	}
@@ -195,6 +202,8 @@ func fetchFileWithUA(req FetchRequest) (body []byte, err error) {
 		return nil, err
 	}
 
+	resp.Header.Get("Content-Type")
+
 	var b bytes.Buffer
 	switch req.MIMEType {
 	case MIMETypeXML, MIMETypeTxt, MIMETypeJSON, MIMETypeYml, MIMETypeHTML:
@@ -219,6 +228,14 @@ func fetchFileWithUA(req FetchRequest) (body []byte, err error) {
 		if _, err = b.ReadFrom(r); err != nil {
 			return nil, xerrors.Errorf("Failed to read gzip file. err: %w", err)
 		}
+	case MIMETypeZst:
+		r, err := zstd.NewReader(bytes.NewReader(buf.Bytes()))
+		if err != nil {
+			return nil, xerrors.Errorf("Failed to open zstd file. err: %w", err)
+		}
+		if _, err = b.ReadFrom(r); err != nil {
+			return nil, xerrors.Errorf("Failed to read zstd file. err: %w", err)
+		}
 	}
 
 	return b.Bytes(), nil

go.mod

@@ -10,6 +10,7 @@ require (
 	github.com/hashicorp/go-version v1.7.0
 	github.com/inconshreveable/log15 v3.0.0-testing.5+incompatible
 	github.com/k0kubun/pp v3.0.1+incompatible
+	github.com/klauspost/compress v1.17.11
 	github.com/knqyf263/go-rpm-version v0.0.0-20220614171824-631e686d1075
 	github.com/labstack/echo/v4 v4.12.0
 	github.com/mitchellh/go-homedir v1.1.0

go.sum

@@ -62,6 +62,8 @@ github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88 h1:uC1QfSlInpQ
 github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88/go.mod h1:3w7q1U84EfirKl04SVQ/s7nPm1ZPhiXd34z40TNz36k=
 github.com/k0kubun/pp v3.0.1+incompatible h1:3tqvf7QgUnZ5tXO6pNAZlrvHgl6DvifjDrd9g2S9Z40=
 github.com/k0kubun/pp v3.0.1+incompatible/go.mod h1:GWse8YhT0p8pT4ir3ZgBbfZild3tgzSScAn6HmfYukg=
+github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc=
+github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0=
 github.com/knqyf263/go-rpm-version v0.0.0-20220614171824-631e686d1075 h1:aC6MEAs3PE3lWD7lqrJfDxHd6hcced9R4JTZu85cJwU=
 github.com/knqyf263/go-rpm-version v0.0.0-20220614171824-631e686d1075/go.mod h1:i4sF0l1fFnY1aiw08QQSwVAFxHEm311Me3WsU/X7nL0=
 github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=