Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

implement logic for getting image architecture from scratch, instead of relying on google/go-containerregistry/crane. #42

Merged
merged 1 commit into from
Feb 10, 2023
Merged

implement logic for getting image architecture from scratch, instead of relying on google/go-containerregistry/crane. #42

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 3 additions & 6 deletions go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,9 +11,8 @@ require (
github.com/gin-gonic/gin v1.8.2
github.com/go-git/go-git/v5 v5.5.1
github.com/google/go-cmp v0.5.9
github.com/google/go-containerregistry v0.12.1
github.com/labstack/echo/v4 v4.10.0
github.com/opencontainers/image-spec v1.1.0-rc2
github.com/mitchellh/mapstructure v1.5.0
github.com/robfig/cron/v3 v3.0.1
github.com/samber/lo v1.37.0
gopkg.in/yaml.v3 v3.0.1
Expand All @@ -38,7 +37,6 @@ require (
github.com/containerd/console v1.0.3 // indirect
github.com/containerd/containerd v1.6.14 // indirect
github.com/containerd/continuity v0.3.0 // indirect
github.com/containerd/stargz-snapshotter/estargz v0.12.1 // indirect
github.com/containerd/ttrpc v1.1.0 // indirect
github.com/containerd/typeurl v1.0.2 // indirect
github.com/coreos/go-systemd/v22 v22.5.0 // indirect
Expand All @@ -49,6 +47,7 @@ require (
github.com/docker/go v1.5.1-1.0.20160303222718-d30aec9fd63c // indirect
github.com/docker/go-metrics v0.0.1 // indirect
github.com/docker/go-units v0.5.0 // indirect
github.com/docker/libtrust v0.0.0-20160708172513-aabc10ec26b7 // indirect
github.com/dsnet/compress v0.0.2-0.20210315054119-f66993602bf5 // indirect
github.com/dustin/go-humanize v1.0.0 // indirect
github.com/emirpasic/gods v1.18.1 // indirect
Expand Down Expand Up @@ -100,8 +99,6 @@ require (
github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b // indirect
github.com/mholt/archiver/v3 v3.5.1 // indirect
github.com/miekg/pkcs11 v1.1.1 // indirect
github.com/mitchellh/go-homedir v1.1.0 // indirect
github.com/mitchellh/mapstructure v1.5.0 // indirect
github.com/moby/buildkit v0.10.4 // indirect
github.com/moby/locker v1.0.1 // indirect
github.com/moby/patternmatcher v0.5.0 // indirect
Expand All @@ -114,6 +111,7 @@ require (
github.com/morikuni/aec v1.0.0 // indirect
github.com/nwaples/rardecode v1.1.3 // indirect
github.com/opencontainers/go-digest v1.0.0 // indirect
github.com/opencontainers/image-spec v1.1.0-rc2 // indirect
github.com/opencontainers/runc v1.1.3 // indirect
github.com/pelletier/go-toml v1.9.4 // indirect
github.com/pierrec/lz4/v4 v4.1.17 // indirect
Expand Down Expand Up @@ -142,7 +140,6 @@ require (
github.com/ulikunitz/xz v0.5.11 // indirect
github.com/valyala/bytebufferpool v1.0.0 // indirect
github.com/valyala/fasttemplate v1.2.2 // indirect
github.com/vbatts/tar-split v0.11.2 // indirect
github.com/weppos/publicsuffix-go v0.20.0 // indirect
github.com/xanzy/ssh-agent v0.3.3 // indirect
github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect
Expand Down
10 changes: 0 additions & 10 deletions go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -133,8 +133,6 @@ github.com/containerd/containerd v1.6.14/go.mod h1:U2NnBPIhzJDm59xF7xB2MMHnKtggp
github.com/containerd/continuity v0.3.0 h1:nisirsYROK15TAMVukJOUyGJjz4BNQJBVsNvAXZJ/eg=
github.com/containerd/continuity v0.3.0/go.mod h1:wJEAIwKOm/pBZuBd0JmeTvnLquTB1Ag8espWhkykbPM=
github.com/containerd/fifo v1.0.0 h1:6PirWBr9/L7GDamKr+XM0IeUFXu5mf3M/BPpH9gaLBU=
github.com/containerd/stargz-snapshotter/estargz v0.12.1 h1:+7nYmHJb0tEkcRaAW+MHqoKaJYZmkikupxCqVtmPuY0=
github.com/containerd/stargz-snapshotter/estargz v0.12.1/go.mod h1:12VUuCq3qPq4y8yUW+l5w3+oXV3cx2Po3KSe/SmPGqw=
github.com/containerd/ttrpc v1.1.0 h1:GbtyLRxb0gOLR0TYQWt3O6B0NvT8tMdorEHqIQo/lWI=
github.com/containerd/ttrpc v1.1.0/go.mod h1:XX4ZTnoOId4HklF4edwc4DcqskFZuvXB1Evzy5KFQpQ=
github.com/containerd/typeurl v1.0.2 h1:Chlt8zIieDbzQFzXzAeBEF92KhExuE4p9p92/QmY7aY=
Expand Down Expand Up @@ -344,8 +342,6 @@ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE=
github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/google/go-containerregistry v0.12.1 h1:W1mzdNUTx4Zla4JaixCRLhORcR7G6KxE5hHl5fkPsp8=
github.com/google/go-containerregistry v0.12.1/go.mod h1:sdIK+oHQO7B93xI8UweYdl887YhuIwg9vz8BSLH3+8k=
github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ=
github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
Expand Down Expand Up @@ -443,7 +439,6 @@ github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
github.com/klauspost/compress v1.11.4/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
github.com/klauspost/compress v1.15.11/go.mod h1:QPwzmACJjUTFsnSHH934V6woptycfrDDJnH7hvFVbGM=
github.com/klauspost/compress v1.15.13 h1:NFn1Wr8cfnenSJSA46lLq4wHCcBzKTSjnBIexDMMOV0=
github.com/klauspost/compress v1.15.13/go.mod h1:QPwzmACJjUTFsnSHH934V6woptycfrDDJnH7hvFVbGM=
github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
Expand Down Expand Up @@ -504,8 +499,6 @@ github.com/mholt/archiver/v3 v3.5.1/go.mod h1:e3dqJ7H78uzsRSEACH1joayhuSyhnonssn
github.com/miekg/pkcs11 v1.0.2/go.mod h1:XsNlhZGX73bx86s2hdc/FuaLm2CPZJemRLMA+WTFxgs=
github.com/miekg/pkcs11 v1.1.1 h1:Ugu9pdy6vAYku5DEpVWVFPYnzV+bxB+iRdbuFSu7TvU=
github.com/miekg/pkcs11 v1.1.1/go.mod h1:XsNlhZGX73bx86s2hdc/FuaLm2CPZJemRLMA+WTFxgs=
github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
github.com/mitchellh/mapstructure v0.0.0-20150613213606-2caf8efc9366/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=
Expand Down Expand Up @@ -709,14 +702,11 @@ github.com/ulikunitz/xz v0.5.9/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oW
github.com/ulikunitz/xz v0.5.11 h1:kpFauv27b6ynzBNT/Xy+1k+fK4WswhN/6PN5WhFAGw8=
github.com/ulikunitz/xz v0.5.11/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14=
github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
github.com/urfave/cli v1.22.4/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
github.com/valyala/fasttemplate v1.2.1/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ=
github.com/valyala/fasttemplate v1.2.2 h1:lxLXG0uE3Qnshl9QyaK6XJxMXlQZELvChBOCmQD0Loo=
github.com/valyala/fasttemplate v1.2.2/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ=
github.com/vbatts/tar-split v0.11.2 h1:Via6XqJr0hceW4wff3QRzD5gAk/tatMw/4ZA7cTlIME=
github.com/vbatts/tar-split v0.11.2/go.mod h1:vV3ZuO2yWSVsz+pfFzDG/upWH1JhjOiEaWq6kXyQ3VI=
github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE=
github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17pCcGlemwknint6hfoeCVQrEMVwxRLRjXpq+BU=
github.com/weppos/publicsuffix-go v0.12.0/go.mod h1:z3LCPQ38eedDQSwmsSRW4Y7t2L8Ln16JPQ02lHAdn5k=
Expand Down
151 changes: 112 additions & 39 deletions pkg/docker/digest.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,15 +4,22 @@ credit: https://github.com/containrrr/watchtower
package docker

import (
"context"
"crypto/tls"
"encoding/base64"
"encoding/json"
"errors"
"fmt"
"io/ioutil"
"net"
"net/http"
"strings"
"time"

"github.com/docker/distribution/manifest"
"github.com/docker/distribution/manifest/manifestlist"
"github.com/docker/distribution/manifest/schema1"
"github.com/docker/distribution/manifest/schema2"
)

// RegistryCredentials is a credential pair used for basic auth
Expand All @@ -25,26 +32,15 @@ type RegistryCredentials struct {
const ContentDigestHeader = "Docker-Content-Digest"

// CompareDigest ...
func CompareDigest(imageName string, repoDigests []string, registryAuth string) (bool, error) {
func CompareDigest(imageName string, repoDigests []string) (bool, error) {
var digest string

registryAuth = TransformAuth(registryAuth)
challenge, err := GetChallenge(imageName)
if err != nil {
return false, err
}

token, err := GetToken(challenge, registryAuth, imageName)
if err != nil {
return false, err
}

digestURL, err := BuildManifestURL(imageName)
token, url, err := tokenAndURL(imageName)
if err != nil {
return false, err
}

if digest, err = GetDigest(digestURL, token); err != nil {
if digest, err = GetDigest(url, token); err != nil {
return false, err
}

Expand Down Expand Up @@ -75,36 +71,14 @@ func TransformAuth(registryAuth string) string {

// GetDigest from registry using a HEAD request to prevent rate limiting
func GetDigest(url string, token string) (string, error) {
tr := &http.Transport{
DialContext: (&net.Dialer{
Timeout: 30 * time.Second,
KeepAlive: 30 * time.Second,
}).DialContext,
DisableKeepAlives: true,
ExpectContinueTimeout: 1 * time.Second,
ForceAttemptHTTP2: true,
IdleConnTimeout: 90 * time.Second,
MaxIdleConns: 100,
Proxy: http.ProxyFromEnvironment,
TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, // nolint:gosec
TLSHandshakeTimeout: 10 * time.Second,
}
client := &http.Client{Transport: tr}

req, _ := http.NewRequest(http.MethodHead, url, nil)
// req.Header.Set("User-Agent", userAgent) - confirm if this is needed

if token == "" {
return "", errors.New("could not fetch token")
}

req.Header.Add("Authorization", token)
req.Header.Add("Accept", "application/vnd.docker.distribution.manifest.v2+json")
req.Header.Add("Accept", "application/vnd.docker.distribution.manifest.list.v2+json")
req.Header.Add("Accept", "application/vnd.docker.distribution.manifest.v1+json")
req.Header.Add("Accept", "application/vnd.oci.image.index.v1+json")
req, _ := http.NewRequest(http.MethodHead, url, nil)
addDefaultHeaders(&req.Header, token)

res, err := client.Do(req)
res, err := httpClient().Do(req)
if err != nil {
return "", err
}
Expand All @@ -119,3 +93,102 @@ func GetDigest(url string, token string) (string, error) {
}
return res.Header.Get(ContentDigestHeader), nil
}

func GetManifest(ctx context.Context, imageName string) (interface{}, string, error) {
token, url, err := tokenAndURL(imageName)
if err != nil {
return nil, "", err
}

req, _ := http.NewRequest(http.MethodGet, url, nil)
req = req.WithContext(ctx)
addDefaultHeaders(&req.Header, token)

res, err := httpClient().Do(req)
if err != nil {
return nil, "", err
}
defer res.Body.Close()

if res.StatusCode != http.StatusOK {
return nil, "", fmt.Errorf("registry responded to head request with %q", res.Status)
}

contentType := res.Header.Get("Content-Type")

buf, err := ioutil.ReadAll(res.Body)
if err != nil {
return nil, "", err
}

var baseManifest manifest.Versioned
if err := json.Unmarshal(buf, &baseManifest); err != nil {
return nil, contentType, fmt.Errorf("not a manifest content: %w", err)
}

manifest, ok := map[string]interface{}{
schema1.MediaTypeSignedManifest: schema1.SignedManifest{},
schema2.MediaTypeManifest: schema2.Manifest{},
manifestlist.MediaTypeManifestList: manifestlist.ManifestList{},
}[contentType]

if !ok {
return nil, contentType, fmt.Errorf("unknown content type: %s", contentType)
}

if err := json.Unmarshal(buf, &manifest); err != nil {
return nil, "", err
}

return manifest, contentType, nil
}

func tokenAndURL(imageName string) (string, string, error) {
opts, err := GetPullOptions(imageName)
if err != nil {
return "", "", err
}

registryAuth := TransformAuth(opts.RegistryAuth)
challenge, err := GetChallenge(imageName)
if err != nil {
return "", "", err
}

token, err := GetToken(challenge, registryAuth, imageName)
if err != nil {
return "", "", err
}

url, err := BuildManifestURL(imageName)
if err != nil {
return "", "", err
}

return token, url, nil
}

func httpClient() *http.Client {
return &http.Client{Transport: &http.Transport{
DialContext: (&net.Dialer{
Timeout: 30 * time.Second,
KeepAlive: 30 * time.Second,
}).DialContext,
DisableKeepAlives: true,
ExpectContinueTimeout: 1 * time.Second,
ForceAttemptHTTP2: true,
IdleConnTimeout: 90 * time.Second,
MaxIdleConns: 100,
Proxy: http.ProxyFromEnvironment,
TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, // nolint:gosec
TLSHandshakeTimeout: 10 * time.Second,
}}
}

func addDefaultHeaders(header *http.Header, token string) {
header.Add("Authorization", token)
// header.Add("Accept", schema2.MediaTypeManifest)
header.Add("Accept", manifestlist.MediaTypeManifestList)
// header.Add("Accept", schema1.MediaTypeManifest)
// header.Add("Accept", v1.MediaTypeImageIndex)
}
36 changes: 35 additions & 1 deletion pkg/docker/digest_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,11 @@ import (
"io"
"testing"

"github.com/docker/distribution/manifest/manifestlist"
"github.com/docker/distribution/manifest/schema1"
"github.com/docker/docker/api/types"
"github.com/docker/docker/client"
"github.com/mitchellh/mapstructure"
"go.uber.org/goleak"
"gotest.tools/v3/assert"
)
Expand Down Expand Up @@ -38,8 +41,39 @@ func TestCompareDigest(t *testing.T) {
imageInfo, _, err := cli.ImageInspectWithRaw(ctx, imageName)
assert.NilError(t, err)

match, err := CompareDigest(imageName, imageInfo.RepoDigests, "")
match, err := CompareDigest(imageName, imageInfo.RepoDigests)
assert.NilError(t, err)

assert.Assert(t, match)
}

func TestGetManifest1(t *testing.T) {
defer goleak.VerifyNone(t)

manifest, contentType, err := GetManifest(context.Background(), "hello-world:latest")
assert.NilError(t, err)
assert.Equal(t, contentType, manifestlist.MediaTypeManifestList)

var listManifest manifestlist.ManifestList
decoder, err := mapstructure.NewDecoder(&mapstructure.DecoderConfig{Result: &listManifest, Squash: true})
assert.NilError(t, err)

err = decoder.Decode(manifest)
assert.NilError(t, err)
assert.Assert(t, len(listManifest.Manifests) > 0)
}

func TestGetManifest2(t *testing.T) {
defer goleak.VerifyNone(t)
manifest, contentType, err := GetManifest(context.Background(), "wangxiaohu/brother-cups:latest")
assert.NilError(t, err)
assert.Equal(t, contentType, schema1.MediaTypeSignedManifest)

var signedManifest schema1.SignedManifest
decoder, err := mapstructure.NewDecoder(&mapstructure.DecoderConfig{Result: &signedManifest, Squash: true})
assert.NilError(t, err)

err = decoder.Decode(manifest)
assert.NilError(t, err)
assert.Assert(t, len(signedManifest.Architecture) > 0)
}
7 changes: 6 additions & 1 deletion pkg/docker/image.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,13 +26,18 @@ func Image(ctx context.Context, imageName string) (*types.ImageInspect, error) {
return &imageInfo, nil
}

func PullImage(ctx context.Context, imageName string, opts types.ImagePullOptions, handleOut func(io.ReadCloser)) error {
func PullImage(ctx context.Context, imageName string, handleOut func(io.ReadCloser)) error {
cli, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation())
if err != nil {
return err
}
defer cli.Close()

opts, err := GetPullOptions(imageName)
if err != nil {
return err
}

out, err := cli.ImagePull(ctx, imageName, opts)
if err != nil {
return err
Expand Down
Loading