diff --git a/.gitignore b/.gitignore index e69de29b..2d8e872b 100644 --- a/.gitignore +++ b/.gitignore @@ -0,0 +1,5 @@ +./certs +./parachute +./parachute.yaml +.idea/ +tests diff --git a/Makefile b/Makefile deleted file mode 100644 index d4f2faba..00000000 --- a/Makefile +++ /dev/null @@ -1,11 +0,0 @@ -push: rm - docker pull postgres:11 - docker tag postgres:11 0.0.0.0:5000/postgres:11 - docker push 0.0.0.0:5000/postgres:11 -push-only: - docker push 0.0.0.0:5000/postgres:11 -pull: rm - docker pull 0.0.0.0:5000/postgres:11 - -rm: - docker images -q | xargs -t docker rmi -f diff --git a/cache/store.go b/cache/store.go index ddf83292..722bd5f9 100644 --- a/cache/store.go +++ b/cache/store.go @@ -5,9 +5,7 @@ import ( "fmt" "net/http" - "github.com/NebulousLabs/go-skynet/v2" badger "github.com/dgraph-io/badger/v3" - "github.com/fatih/color" "github.com/jay-dee7/parachute/types" "github.com/labstack/echo/v4" ) @@ -19,14 +17,16 @@ type dataStore struct { type Store interface { Get(key []byte) ([]byte, error) Set(key, value []byte) error + GetDigest(digest string) (*types.LayerRef, error) + SetDigest(digest, skylink string) error Update(key, value []byte) error ListAll() ([]byte, error) ListWithPrefix(prefix []byte) ([]byte, error) Delete(key []byte) error GetSkynetURL(key string, ref string) (string, error) - GetSkynetURLWithHeaders(key string, ref string) (string, []skynet.Header, error) ResolveManifestRef(namespace, ref string) (string, error) Metadata(ctx echo.Context) error + LayerDigests(ctx echo.Context) error Close() error } @@ -43,6 +43,15 @@ func New(storeLocation string) (Store, error) { return &dataStore{db: db}, nil } +func (ds *dataStore) LayerDigests(ctx echo.Context) error { + bz, err := ds.ListWithPrefix([]byte(layerDigestNamespace)) + if err != nil { + return ctx.JSON(http.StatusBadRequest, echo.Map{"error": err.Error()}) + } + + return ctx.JSONBlob(http.StatusOK, bz) +} + func (ds *dataStore) Metadata(ctx echo.Context) error { key := ctx.QueryParam("namespace") if key == "" { @@ -53,7 +62,14 @@ func (ds *dataStore) Metadata(ctx echo.Context) error { }) } - return ctx.JSONBlob(http.StatusOK, bz) + var metadataList []types.Metadata + if err = json.Unmarshal(bz, &metadataList); err != nil { + return ctx.JSON(http.StatusInternalServerError, echo.Map{ + "error": err.Error(), + }) + } + + return ctx.JSON(http.StatusOK, metadataList) } val, err := ds.Get([]byte(key)) @@ -156,53 +172,36 @@ func (ds *dataStore) ResolveManifestRef(namespace, ref string) (string, error) { return "", fmt.Errorf("ref not found") } -func (ds *dataStore) GetSkynetURLWithHeaders(key, ref string) (string, []skynet.Header, error) { - - color.Yellow("key=%s ref=%s\n", key, ref) - var res []byte - err := ds.db.View(func(txn *badger.Txn) error { - item, err := txn.Get([]byte(key)) - if err != nil { - return err - } - - return item.Value(func(v []byte) error { - res = make([]byte, len(v)) - copy(res, v) - return nil - }) - }) - - if err != nil { - return "", nil, err - } +const layerDigestNamespace = "layers/digests" - var md types.Metadata - err = json.Unmarshal(res, &md) - if err != nil { - return "", nil, err +func (ds *dataStore) SetDigest(digest, skylink string) error { + key := fmt.Sprintf("%s/%s", layerDigestNamespace, digest) + value := types.LayerRef{ + Digest: digest, + Skylink: skylink, } - skylink, err := md.FindLinkForDigest(ref) - if err != nil { - return "", nil, err + if err := ds.Set([]byte(key), value.Bytes()); err != nil { + return err } - layer := md.FindLayer(ref) + return nil +} - headers := []skynet.Header{ - { - Key: "Range", - Value: fmt.Sprintf("bytes=%d-%d", layer.RangeStart, layer.RangeEnd), - }, +func (ds *dataStore) GetDigest(digest string) (*types.LayerRef, error) { + key := fmt.Sprintf("%s/%s", layerDigestNamespace, digest) + bz, err := ds.Get([]byte(key)) + if err != nil { + return nil, err } - return skylink, headers, nil + var layerRef types.LayerRef + err = json.Unmarshal(bz, &layerRef) + return &layerRef, err } func (ds *dataStore) GetSkynetURL(key, ref string) (string, error) { - color.Yellow("key=%s ref=%s\n", key, ref) var res []byte err := ds.db.View(func(txn *badger.Txn) error { item, err := txn.Get([]byte(key)) @@ -263,7 +262,7 @@ func (ds *dataStore) Get(key []byte) ([]byte, error) { } func (ds *dataStore) ListAll() ([]byte, error) { - var res []byte + var buf []types.Metadata err := ds.db.View(func(txn *badger.Txn) error { opts := badger.DefaultIteratorOptions @@ -273,10 +272,14 @@ func (ds *dataStore) ListAll() ([]byte, error) { for it.Rewind(); it.Valid(); it.Next() { item := it.Item() err := item.Value(func(v []byte) error { - res = make([]byte, len(v)) - copy(res, v) + var md types.Metadata + if err := json.Unmarshal(v, &md); err != nil { + return err + } + buf = append(buf, md) return nil }) + if err != nil { return err } @@ -284,11 +287,13 @@ func (ds *dataStore) ListAll() ([]byte, error) { return nil }) - return res, err + bz, _ := json.Marshal(buf) + + return bz, err } func (ds *dataStore) ListWithPrefix(prefix []byte) ([]byte, error) { - var res []byte + var buf []*types.LayerRef err := ds.db.View(func(txn *badger.Txn) error { it := txn.NewIterator(badger.DefaultIteratorOptions) @@ -296,8 +301,12 @@ func (ds *dataStore) ListWithPrefix(prefix []byte) ([]byte, error) { for it.Seek(prefix); it.ValidForPrefix(prefix); it.Next() { item := it.Item() err := item.Value(func(v []byte) error { - res = make([]byte, len(v)) - copy(res, v) + var layerRef types.LayerRef + if err := json.Unmarshal(v, &layerRef); err != nil { + return err + } + + buf = append(buf, &layerRef) return nil }) if err != nil { @@ -307,7 +316,8 @@ func (ds *dataStore) ListWithPrefix(prefix []byte) ([]byte, error) { return nil }) - return res, err + bz, _ := json.Marshal(buf) + return bz, err } func (ds *dataStore) Delete(key []byte) error { diff --git a/certs/alt_names.txt b/certs/alt_names.txt deleted file mode 100644 index 552f6ef5..00000000 --- a/certs/alt_names.txt +++ /dev/null @@ -1 +0,0 @@ -subjectAltName=DNS:jsdp.dev,DNS:parachute.jsdp.dev diff --git a/certs/certificate.txt b/certs/certificate.txt deleted file mode 100644 index 62520568..00000000 --- a/certs/certificate.txt +++ /dev/null @@ -1,32 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIFiDCCA3CgAwIBAgIJAP3+uAFdm2xaMA0GCSqGSIb3DQEBCwUAMG0xCzAJBgNV -BAYTAkZSMRYwFAYDVQQIDA1JbGUtZGUtRnJhbmNlMQ4wDAYDVQQHDAVQYXJpczEV -MBMGA1UECgwMUGFyYWNodXRlSW5jMR8wHQYDVQQDDBZwYXJhY2h1dGUuanNkcC5k -ZXYuY29tMB4XDTIxMDYwMzExMDgzNloXDTIyMDYwMzExMDgzNlowbTELMAkGA1UE -BhMCRlIxFjAUBgNVBAgMDUlsZS1kZS1GcmFuY2UxDjAMBgNVBAcMBVBhcmlzMRUw -EwYDVQQKDAxQYXJhY2h1dGVJbmMxHzAdBgNVBAMMFnBhcmFjaHV0ZS5qc2RwLmRl -di5jb20wggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQDBllrL0Zo1UZz3 -E41XnAadLpuLJl0vKd3pj5Q5uH+5zWraG6pTVleo/hK8mckT1VzQ+ar/HF3wMa9F -xusjbZerEsJUflwMPycWWPhKRtxHSVhB03BZM/TCpQ2EzG60GznObNn8t0yVkf/3 -fvntoNdROeIsj/mXgDNFJZE50PIL81kV2c8DqRuBhGMEItQjOol0jFfE8SY314y+ -nuTEBBpaRZYJWHBR2USW8jDKowe9Jm38BMCyy6SIVg5KTjcck3UERti9bLQPwjec -SaKND+c+y0+sEsT8/PgRS0Z07gvzzGjK2iMc/CA/q6pF56MqdMT6AdxMPODNfHWD -zky2kdz7EGfRxUK4waXWfR/ZwqL4hC0kwfKwkRjHlSMOt9NJBWTIgnEw/Lp9V2mL -ZxXcJTSQYBh4DN8ckiHSofyikf+i3FolzsIS6EPorvMlqzucKTlVCAcIjcOT34+N -VwzhWISc8CBM81LRiSDq8GZwEZiXcRFfmWJeM4xlxUBTHb6igmLz5AgKf0JCXNOb -OTEOv+PKQG1ae3QAYA94n/5dLwzsDUbxL+0N4/g6EdvsyZ95GbnZPsZOEzBiOaO2 -+sOM8zk4AAsk3zW9CGguruAem025mmNByS36ETkr/6Sh8zIkx37F3ZByE9GfWfGX -l+GMxSqs2MxM8yewlzA27fGaSrujswIDAQABoyswKTAnBgNVHREEIDAegghqc2Rw -LmRldoIScGFyYWNodXRlLmpzZHAuZGV2MA0GCSqGSIb3DQEBCwUAA4ICAQA7dswP -07Ju6csNczJIQHiet+zg/B8miRqnxHeETqw1TyCycbqwKVd8DKee8pdijLcDKria -MUxkvO4+uW+GAiB77ko8iuQ1Q9WKzHAjhNZsUYURrCV9nZQBGnjz7tH6BU6JPCVj -XUMJ0KlNqbwo5xLLnzHKOmNqxUEgqSAwjEKJh55DKjSR3oZM2jczYZeBmZWWjwl/ -+XIjBCHPUOYGGSyDqI8LMNrc/tl8IohHFI9qDpo4XirA8+rztdck7DMdVL5TkhZk -OR3doeCVODvcsOzhDlUUkA5fmObLnGrNIrCH7VUcI/fiAAqfz+lwvo/9TDIpj2xU -HKn9EDxS98kLwdM+nTNUTza7Uj1bPatUGKPkrMs8xsML+oR4b6F5/wIob+ceu+Mw -odOGUSkIpZ6Bo4d3n5/TfG0m92nWDC+CftmAbmOcGDi+ODd8IRVtLMNPVhjiZGOS -TPGMNqVvvoHqHyyghReY4ak6zdy98twlMVtOnWWsw79auOhCkWoawfcm9YWvcSKs -kyGjRhuE/uxxHANYLo7jQAI1rmhr00h19yeHTEUCBLNIQ6Enz+Aml+UX0CVdGGLm -iWsoSS43OMPwgSWrMzYPI+pXdj44oefRIEDbJF9zarFY9L9mG7vqjVBkdqJ9l7ez -Hf+uF7eFv+Tx8nj/uWOniw8Ia/SG31ACXEHg8Q== ------END CERTIFICATE----- diff --git a/certs/csr.txt b/certs/csr.txt deleted file mode 100644 index 9ff88ee8..00000000 --- a/certs/csr.txt +++ /dev/null @@ -1,28 +0,0 @@ ------BEGIN CERTIFICATE REQUEST----- -MIIEsjCCApoCAQAwbTELMAkGA1UEBhMCRlIxFjAUBgNVBAgMDUlsZS1kZS1GcmFu -Y2UxDjAMBgNVBAcMBVBhcmlzMRUwEwYDVQQKDAxQYXJhY2h1dGVJbmMxHzAdBgNV -BAMMFnBhcmFjaHV0ZS5qc2RwLmRldi5jb20wggIiMA0GCSqGSIb3DQEBAQUAA4IC -DwAwggIKAoICAQDBllrL0Zo1UZz3E41XnAadLpuLJl0vKd3pj5Q5uH+5zWraG6pT -Vleo/hK8mckT1VzQ+ar/HF3wMa9FxusjbZerEsJUflwMPycWWPhKRtxHSVhB03BZ -M/TCpQ2EzG60GznObNn8t0yVkf/3fvntoNdROeIsj/mXgDNFJZE50PIL81kV2c8D -qRuBhGMEItQjOol0jFfE8SY314y+nuTEBBpaRZYJWHBR2USW8jDKowe9Jm38BMCy -y6SIVg5KTjcck3UERti9bLQPwjecSaKND+c+y0+sEsT8/PgRS0Z07gvzzGjK2iMc -/CA/q6pF56MqdMT6AdxMPODNfHWDzky2kdz7EGfRxUK4waXWfR/ZwqL4hC0kwfKw -kRjHlSMOt9NJBWTIgnEw/Lp9V2mLZxXcJTSQYBh4DN8ckiHSofyikf+i3FolzsIS -6EPorvMlqzucKTlVCAcIjcOT34+NVwzhWISc8CBM81LRiSDq8GZwEZiXcRFfmWJe -M4xlxUBTHb6igmLz5AgKf0JCXNObOTEOv+PKQG1ae3QAYA94n/5dLwzsDUbxL+0N -4/g6EdvsyZ95GbnZPsZOEzBiOaO2+sOM8zk4AAsk3zW9CGguruAem025mmNByS36 -ETkr/6Sh8zIkx37F3ZByE9GfWfGXl+GMxSqs2MxM8yewlzA27fGaSrujswIDAQAB -oAAwDQYJKoZIhvcNAQELBQADggIBAJMpdPV58ENBpFmpojOYp9cdv+ZxCg3MGmJk -rfpdjEVO0PyTaG1dHvKCstkpCbW1AbPPqPfARZ39ExdgRMbW6jEvyBMxIeiUTHZ7 -kv7cv0yznHn7kwosD53rUhnLcx77TJbtwIg+YfkgkzPDs6CwGzNU7Ssk/L6rlumq -8bcxNGxunTr6/W2avcVtp5j8tdWJ6VcBmM3uHbDFg8beAuxEBPwgO8QcKnXVwGAN -1/X09bFD19NkaFfeSeDP5Zc4m8o4KDcOSwUQZxKLKftcVN1h3O71otQ72GjqAB48 -EstyOWybYjda9Ix6I1nOLsv6vDHhzvphC6Xig48iKKO1ME9EIpju72BAtgjW92/m -LvZsjxnW6TgmTLiBKkPQatirxqcvjlzQHZiOn2zag+iuudLQAtbqHte2B84UQ4kR -IhtzkPIPgndo5lpLr4g5a7Qg4e8VYNfbcmME1K3y1UefZRkl3BHb8mYHzCEb9TS9 -6xeOiuOyN0bnVHJtBk4ZQHz1Hbzf3nSmL1jhWIkrhBFi6niQlVbKft1mhKNHqNdV -1pewfsey+1D31igK3DxDSnQAVEvWdIY7IX2wOu9YKLqitkRfRVKG+8apAZKv2s/V -7saWKXfoQl1NGXupOpBavCBYm60S10QzmuOTn3j0Pd2754hFl4wQXPYN5gRDhByM -2C5hnmOR ------END CERTIFICATE REQUEST----- diff --git a/certs/private_key.txt b/certs/private_key.txt deleted file mode 100644 index 471c892e..00000000 --- a/certs/private_key.txt +++ /dev/null @@ -1,52 +0,0 @@ ------BEGIN PRIVATE KEY----- -MIIJQQIBADANBgkqhkiG9w0BAQEFAASCCSswggknAgEAAoICAQDBllrL0Zo1UZz3 -E41XnAadLpuLJl0vKd3pj5Q5uH+5zWraG6pTVleo/hK8mckT1VzQ+ar/HF3wMa9F -xusjbZerEsJUflwMPycWWPhKRtxHSVhB03BZM/TCpQ2EzG60GznObNn8t0yVkf/3 -fvntoNdROeIsj/mXgDNFJZE50PIL81kV2c8DqRuBhGMEItQjOol0jFfE8SY314y+ -nuTEBBpaRZYJWHBR2USW8jDKowe9Jm38BMCyy6SIVg5KTjcck3UERti9bLQPwjec -SaKND+c+y0+sEsT8/PgRS0Z07gvzzGjK2iMc/CA/q6pF56MqdMT6AdxMPODNfHWD -zky2kdz7EGfRxUK4waXWfR/ZwqL4hC0kwfKwkRjHlSMOt9NJBWTIgnEw/Lp9V2mL -ZxXcJTSQYBh4DN8ckiHSofyikf+i3FolzsIS6EPorvMlqzucKTlVCAcIjcOT34+N -VwzhWISc8CBM81LRiSDq8GZwEZiXcRFfmWJeM4xlxUBTHb6igmLz5AgKf0JCXNOb -OTEOv+PKQG1ae3QAYA94n/5dLwzsDUbxL+0N4/g6EdvsyZ95GbnZPsZOEzBiOaO2 -+sOM8zk4AAsk3zW9CGguruAem025mmNByS36ETkr/6Sh8zIkx37F3ZByE9GfWfGX -l+GMxSqs2MxM8yewlzA27fGaSrujswIDAQABAoICAE1tOXtzpd3VepZHt6xmty5+ -ya9p9LM8Z4lW583+QPerdSUvE3ypd/4bQJs2VvobyDe84+uNCPVcqeGaoclVt+ew -ZREuSNqNF7EUfJoYpVnNuAL78l97LFX9dd67dTuL+B7b4oP1PChGcL4ohwXH4taV -9CQmoYk6uUbas68QJv8ooWTf7LBmHp46uO+ajoZYj0mSKDiYytRi3eQP4SeVDIU2 -d7/dI29ZGtnUnDcv5ZR9nlfB/uCEGpZ89NI7XkeLoq3HOEDrDUmQQHW97cnFppOy -9Xa5OxiEhSNFnDSAUT99bhxUCt846B0PJ9c1wkjk0zO7MsU7RB5rvQSuqngK9vqx -jKgNwLBOWyaYbIaq5kLEHr+X82ppqNcf/mY+fV/DyVYeEM5tUAPNmIAZg+B2OFKB -2vUaEezgIDnqg0Ji3UQmrOh4jGoXHMRUNH+Wm0OsbGt1QfHT1UCBzWuybnTc70+p -G1Skln1Vlc006u7Qa5TXyOf1qf3chT1OMkmZOh1veOOKuExUeCPU5If5ZSL0I3Y3 -wQjiv7ecyB8Q8BeCS9X/0IXoDdCzbYdGeYtGAR4eYKa75bsAqBE1UP9l5AR426cH -wpmetlqy+ZRIbPSkXDegxAch6/XygvDblDUFBOyfr6waUYuP7NTFHsooTTxtYtX6 -MHRHDxXnWm1+ALWyOJJhAoIBAQDfyQr5xmptreLf5V2xD9d+KBXzRVNkYFVZ682S -XPYrH7Wi3zVvRSCt9/KN8h+pGK/Be4BQVOwrVAvEtpowYf6h7Dx4/xi1tCpr0L9m -jjWSu19JtzWqy/VpEbni6vIzUCBlzLMGwj+9vrhFd4Up/5lQVmCM7AoSzeyKjAJ8 -FNZvFIXyAOzVKN+apyHOBb/etRx2ZIRjJ+tXy5F9/ycKxeDTjWz8AumvFVe7/lsc -RLCKgamtsqAbQiK4gC5Jw5Jov4ba4Lqg4Cn7f33BVbU3tMNHJLR7S1utrDNWC0J5 -vnu1+K7H42yG+jJB2Edt5J0oI/MSeTlzt95mJFv8Z2j8UFHXAoIBAQDddHPLhHNf -OZLGBaMZ7QD3UO1NJM4QragEZCiaqgZtk8X1Qy55Wsk/9l24twj+qPobxD1Dmcln -QwiB9vRH1egCKfHgUgLz8yH9CrA6pgi3En2FTPsbdz6am9poqkgaz86xNQjVXjHO -8nERmPmPjvC8AENPowZcpCAMBj0QaxvzNd3VoNmklhTtxJbRCus/qNrDDIqVEX/B -hOkJZL1nekGv3/rNQG5XhB9bSaXW9MfcdxJDBLBeBPdf/T4CB1bOcvlEUxG++qdk -EAm0m/shuUUcoiHkfmpeJp/XN8ew1Bk4oEQ5wb9VhSVtimqe4TEkAbs/y9gpQ2S+ -YlbbXisG8fmFAoIBAC1tqoS5+rD5M60v6q0fAifiC1KvKBagV7NBYyHrLS3j6gN+ -PGKyLcgjJWIbXMH++O7WG14zXx4Mmbb9Op/HxOEqC7ykTGfWgyYP4kr3Y3i+AMH+ -Jy7c6JzDtHCUMUapZWpE+7LLjBUAunefo7C66mycnEz/PaxDkVkuwvJdaIp0KBzQ -g0NiYMIMW5hvIlwbeAOwtA9M80GHWamyOkK9mL5ZX15pWPbp5j2LjQNYDl6Po9pX -AJmH+3Ro0Py4buhEuSOFq10tJFihsO2mcHs2fs3abaj5y0qeQm2dK4l7WbG/IfwB -9u4PrFQhi56LYjvb0GRXp4oOMV+a0fGTLITL6VECggEAY80abkqXr/+DTOi3c3Si -7dIzz4NZtpx8TtZQkjszwGhLW1fWduxaLBA8doVOcW33oVHUBkYlRPspt2BzAHi/ -IVKD5gMvv37//0i02QmAffjARrXFqMAZn5XDXU34RtUGIkhzXa0gp1INcM/PEEpr -5V3898PRlyiEJMce9+o9Y2MQQTlHPGxCrFDfYwAXus2/IQ3i2oFTYQQj/Y6/6QVo -2vkGzz9NOvWZKAvNyEzyUUH8IeXYEdcNIN5A9qsWDN41xfvl0Hqnu2ma5iExkSYs -plXPDmmGSD4B5P0HaoJ7h1B6zsipEMJ7dp1dpZJ5EfAR0hNsrFPp7/7jg+MxYuTj -/QKCAQAiPHvnEtw4sRTbLFE/Bi8x7zbOGL8RPdgLMDcVi1fJJ9/kZ4j7l5JK4UZ2 -L6DzDsFu56SS1zvQh7ixBpN4rWQ1bRvKooGRYuybv1Z62ovpHQIdu6RF+/xX3x1o -B+I62VN7ricXFPwYphwpqTdLOKvEKFO9qShar/kqSbMsZyBtDdUnO0eIRi1jZBX/ -THUZzJlQ3JkQgNaaYFmek/O+jpBY3j9iVR54FlBnz9cjfiH0vXuFe8lDVhhgK8bq -nkIQfX5yrunTYSL+uPIPD/TAuvs/6zXZRbMjLs0S0s12CZGjFVrxGRjmPjDAZY6k -u3ndL7upCHegd+61fDoXpcplVZco ------END PRIVATE KEY----- diff --git a/config/config.go b/config/config.go index af9f0665..1ceb02d5 100644 --- a/config/config.go +++ b/config/config.go @@ -12,6 +12,7 @@ import ( type ( RegistryConfig struct { Debug bool `mapstructure:"debug"` + Environment string `mapstructure:"environment"` Host string `mapstructure:"host"` Port uint `mapstructure:"port"` SkynetPortalURL string `mapstructure:"skynet_portal_url"` @@ -73,6 +74,8 @@ func setDefaults() { os.MkdirAll(defaultLinkResolverPath, os.ModePerm) viper.SetDefault("debug", true) + viper.SetDefault("environment", "devel") + viper.SetDefault("domain", "alpha.openregistry.dev") viper.SetDefault("host", "0.0.0.0") viper.SetDefault("port", "5000") viper.SetDefault("tls_key_path", "./certs/key.pem") diff --git a/go.mod b/go.mod index 495660f7..3b2f502e 100644 --- a/go.mod +++ b/go.mod @@ -28,4 +28,4 @@ require ( google.golang.org/grpc v1.38.0 // indirect ) -replace github.com/NebulousLabs/go-skynet/v2 => /Users/gunjanvalecha/oss/go-skynet +replace github.com/NebulousLabs/go-skynet/v2 => /Users/jasdeep/oss/go-skynet diff --git a/main.go b/main.go index 73e18bdd..eaded988 100644 --- a/main.go +++ b/main.go @@ -1,6 +1,10 @@ package main import ( + "log" + "net/http" + "os" + "github.com/fatih/color" "github.com/jay-dee7/parachute/auth" "github.com/jay-dee7/parachute/cache" @@ -11,9 +15,6 @@ import ( "github.com/labstack/echo/v4" "github.com/labstack/echo/v4/middleware" "github.com/rs/zerolog" - "log" - "net/http" - "os" ) func main() { @@ -68,6 +69,7 @@ func main() { authRouter.Add(http.MethodPost, "/token", authSvc.SignIn) internal.Add(http.MethodGet, "/metadata", localCache.Metadata) + internal.Add(http.MethodGet, "/digests", localCache.LayerDigests) router := e.Group("/v2/:username/:imagename") router.Use(BasicAuth(authSvc.BasicAuth)) @@ -118,7 +120,7 @@ func main() { e.Add(http.MethodGet, "/v2/", reg.ApiVersion, BasicAuth(authSvc.BasicAuth)) - log.Fatalln(e.Start(cfg.Address())) + log.Println(e.Start(cfg.Address())) } func setupLogger() zerolog.Logger { diff --git a/parachute.yaml b/parachute.yaml index a5d21b35..6a9b4eb5 100644 --- a/parachute.yaml +++ b/parachute.yaml @@ -1,8 +1,9 @@ debug: true -host: 0.0.0.0 +domain: "beta.openregistry.dev" +host: 100.87.37.43 port: 5000 -tls_key_path: ./certs/private_key.txt -tls_cert_path: ./certs/certificate.txt +tls_key_path: ./certs/beta-openregistry.key +tls_cert_path: ./certs/beta-openregistry.crt skynet_store_path: /Users/jasdeep/.parachute skynet_link_resolvers: /Users/jasdeep/.parachute/links skynet_portal_url: https://siasky.net diff --git a/parachute.yaml.example b/parachute.yaml.example new file mode 100644 index 00000000..01d76492 --- /dev/null +++ b/parachute.yaml.example @@ -0,0 +1,13 @@ +debug: true +domain: "beta.openregistry.dev" +host: 100.87.37.43 +port: 5000 +tls_key_path: ./certs/beta-openregistry.key +tls_cert_path: ./certs/beta-openregistry.crt +skynet_store_path: /Users/jasdeep/.parachute +skynet_link_resolvers: /Users/jasdeep/.parachute/links +skynet_portal_url: https://siasky.net +skynet_config: + endpoint_path: "" + api_key: "" + custom_user_agent: "parachute-registry" diff --git a/server/registry/v2/blobs.go b/server/registry/v2/blobs.go index b42d422f..0fa525ba 100644 --- a/server/registry/v2/blobs.go +++ b/server/registry/v2/blobs.go @@ -9,7 +9,6 @@ import ( "strings" "github.com/fatih/color" - "github.com/jay-dee7/parachute/types" "github.com/labstack/echo/v4" ) @@ -63,17 +62,17 @@ func (b *blobs) remove(repo string) { func (b *blobs) HEAD(ctx echo.Context) error { - namespace := ctx.Param("username") + "/" + ctx.Param("imagename") + // namespace := ctx.Param("username") + "/" + ctx.Param("imagename") digest := ctx.Param("digest") // content is available if image is locally pushed - if c, ok := b.contents[digest]; ok { - ctx.Response().Header().Set("Content-Length", fmt.Sprint(len(c))) - ctx.Response().Header().Set("Docker-Content-Digest", digest) - return ctx.NoContent(http.StatusOK) - } + // if c, ok := b.contents[digest]; ok { + // ctx.Response().Header().Set("Content-Length", fmt.Sprint(len(c))) + // ctx.Response().Header().Set("Docker-Content-Digest", digest) + // return ctx.NoContent(http.StatusOK) + // } - skylink, err := b.registry.localCache.GetSkynetURL(namespace, digest) + layerRef, err := b.registry.localCache.GetDigest(digest) if err != nil { details := echo.Map{ "skynet": "skynet link not found", @@ -82,32 +81,27 @@ func (b *blobs) HEAD(ctx echo.Context) error { return ctx.JSON(http.StatusNotFound, errMsg) } - size, ok := b.registry.skynet.Metadata(skylink) + size, ok := b.registry.skynet.Metadata(layerRef.Skylink) if !ok { errMsg := b.errorResponse(RegistryErrorCodeManifestBlobUnknown, "Manifest does not exist", nil) return ctx.JSON(http.StatusNotFound, errMsg) } - bz, err := b.registry.localCache.Get([]byte(namespace)) - if err != nil { - errMsg := b.errorResponse(RegistryErrorCodeManifestInvalid, err.Error(), nil) - return ctx.JSONBlob(http.StatusNotFound, errMsg) - } - - var md types.Metadata - if err = json.Unmarshal(bz, &md); err != nil { - errMsg := b.errorResponse(RegistryErrorCodeManifestInvalid, err.Error(), nil) - return ctx.JSONBlob(http.StatusNotFound, errMsg) - } + // bz, err := b.registry.localCache.Get([]byte(namespace)) + // if err != nil { + // errMsg := b.errorResponse(RegistryErrorCodeManifestInvalid, err.Error(), nil) + // return ctx.JSONBlob(http.StatusNotFound, errMsg) + // } - if err != nil { - errMsg := b.errorResponse(RegistryErrorCodeManifestInvalid, err.Error(), nil) - return ctx.JSONBlob(http.StatusNotFound, errMsg) - } + // var md types.Metadata + // if err = json.Unmarshal(bz, &md); err != nil { + // errMsg := b.errorResponse(RegistryErrorCodeManifestInvalid, err.Error(), nil) + // return ctx.JSONBlob(http.StatusNotFound, errMsg) + // } ctx.Response().Header().Set("Content-Length", fmt.Sprintf("%d", size)) ctx.Response().Header().Set("Docker-Content-Digest", digest) - io.CopyN(ctx.Response(), bytes.NewReader(bz), int64(size)) + // io.CopyN(ctx.Response(), bytes.NewReader(bz), int64(size)) return ctx.String(http.StatusOK, "OK") } @@ -134,6 +128,7 @@ func (b *blobs) UploadBlob(ctx echo.Context) error { defer ctx.Request().Body.Close() b.uploads[uuid] = bz + locationHeader := fmt.Sprintf("/v2/%s/blobs/uploads/%s", namespace, uuid) ctx.Response().Header().Set("Location", locationHeader) ctx.Response().Header().Set("Range", fmt.Sprintf("0-%d", len(bz)-1)) diff --git a/server/registry/v2/types.go b/server/registry/v2/types.go index 37cdd9c8..a85b9184 100644 --- a/server/registry/v2/types.go +++ b/server/registry/v2/types.go @@ -83,7 +83,6 @@ func (r *registry) CompleteUpload(ctx echo.Context) error { dig := ctx.QueryParam("digest") namespace := ctx.Param("username") + "/" + ctx.Param("imagename") uuid := ctx.Param("uuid") - // contentRange := ctx.Request().Header.Get("Content-Range") bz, err := io.ReadAll(ctx.Request().Body) if err != nil { @@ -93,9 +92,6 @@ func (r *registry) CompleteUpload(ctx echo.Context) error { buf := bytes.NewBuffer(r.b.uploads[uuid]) buf.Write(bz) - // io.Copy(buf, ctx.Request().Body) - // io.CopyN(buf, ctx.Request().Body, ctx.Request().ContentLength-1) - ourHash := digest(buf.Bytes()) if ourHash != dig { @@ -115,7 +111,7 @@ func (r *registry) CompleteUpload(ctx echo.Context) error { }) } - r.b.contents[ourHash] = buf.Bytes() + // r.b.contents[ourHash] = buf.Bytes() blobNamespace := fmt.Sprintf("%s/blobs", namespace) skylink, err := r.skynet.Upload(blobNamespace, dig, buf.Bytes(), headers...) @@ -123,6 +119,10 @@ func (r *registry) CompleteUpload(ctx echo.Context) error { errMsg := r.errorResponse(RegistryErrorCodeBlobUploadInvalid, err.Error(), nil) return ctx.JSONBlob(http.StatusRequestedRangeNotSatisfiable, errMsg) } + if err := r.localCache.SetDigest(ourHash, skylink); err != nil { + errMsg := r.errorResponse(RegistryErrorCodeBlobUnknown, err.Error(), nil) + return ctx.JSONBlob(http.StatusInternalServerError, errMsg) + } // delete(r.b.uploads, ref) @@ -150,50 +150,6 @@ func (r *registry) CompleteUpload(ctx echo.Context) error { // Docker-Content-Digest: func (r *registry) LayerExists(ctx echo.Context) error { return r.b.HEAD(ctx) - - namespace := ctx.Param("username") + "/" + ctx.Param("imagename") - digest := ctx.Param("digest") // ref can be either tag or digest - - skylink, err := r.localCache.GetSkynetURL(namespace, digest) - if err != nil { - details := echo.Map{ - "skynet": "skynet link not found", - } - errMsg := r.errorResponse(RegistryErrorCodeManifestBlobUnknown, err.Error(), details) - return ctx.JSON(http.StatusNotFound, errMsg) - } - - size, ok := r.skynet.Metadata(skylink) - if !ok { - lm := logMsg{ - "warn": "metadata not found for skylink", - "skylink": skylink, - } - r.debugf(lm) - errMsg := r.errorResponse(RegistryErrorCodeManifestBlobUnknown, "Manifest does not exist", nil) - return ctx.JSON(http.StatusNotFound, errMsg) - } - - bz, err := r.localCache.Get([]byte(namespace)) - if err != nil { - errMsg := r.errorResponse(RegistryErrorCodeManifestInvalid, err.Error(), nil) - return ctx.JSONBlob(http.StatusNotFound, errMsg) - } - - var md types.Metadata - if err = json.Unmarshal(bz, &md); err != nil { - errMsg := r.errorResponse(RegistryErrorCodeManifestInvalid, err.Error(), nil) - return ctx.JSONBlob(http.StatusNotFound, errMsg) - } - - if err != nil { - errMsg := r.errorResponse(RegistryErrorCodeManifestInvalid, err.Error(), nil) - return ctx.JSONBlob(http.StatusNotFound, errMsg) - } - - ctx.Response().Header().Set("Content-Length", fmt.Sprintf("%d", size)) - ctx.Response().Header().Set("Docker-Content-Digest", digest) - return ctx.String(http.StatusOK, "OK") } // HEAD /v2//manifests/ @@ -270,71 +226,15 @@ func (r *registry) ManifestExists(ctx echo.Context) error { return ctx.JSONBlob(http.StatusBadRequest, errMsg) } + ctx.Response().Header().Set("Content-Type", "application/json") ctx.Response().Header().Set("Content-Length", fmt.Sprintf("%d", size)) ctx.Response().Header().Set("Docker-Content-Digest", manifest.Digest) - return ctx.String(http.StatusOK, "OK") + return ctx.NoContent(http.StatusOK) } // PATCH /v2//blobs/uploads/ func (r *registry) ChunkedUpload(ctx echo.Context) error { return r.b.UploadBlob(ctx) - - namespace := ctx.Param("username") + "/" + ctx.Param("imagename") - chunkID := ctx.Param("uuid") - - var headers []skynetsdk.Header - - for k, v := range ctx.Request().Header { - headers = append(headers, skynetsdk.Header{ - Key: k, Value: v[0], - }) - } - - bz, err := io.ReadAll(ctx.Request().Body) - if err != nil { - panic(err) - } - defer ctx.Request().Body.Close() - - dig := digest(bz) - skylink, err := r.skynet.Upload(namespace, dig, bz, headers...) - if err != nil { - errMsg := r.errorResponse(RegistryErrorCodeBlobUploadInvalid, err.Error(), nil) - lm := logMsg{ - "error": err.Error(), - "digest": dig, - "uuid": chunkID, - } - r.debugf(lm) - return ctx.JSONBlob(http.StatusRequestedRangeNotSatisfiable, errMsg) - } - - val := types.Metadata{ - Namespace: namespace, - Manifest: types.ImageManifest{ - SchemaVersion: 2, - MediaType: "", - Layers: []*types.Layer{ - { - SkynetLink: skylink, - Size: len(bz), - UUID: chunkID, - Digest: dig, - }, - }, - }, - } - - r.localCache.Update([]byte(namespace), val.Bytes()) - - // id := uuid.Generate() - - locationHeader := fmt.Sprintf("/v2/%s/blobs/uploads/%s", namespace, chunkID) - - ctx.Response().Header().Set("Location", locationHeader) - ctx.Response().Header().Set("Range", fmt.Sprintf("bytes=0-%d", len(bz)-1)) - ctx.Response().Header().Set("Docker-Upload-UUID", chunkID) - return ctx.NoContent(http.StatusAccepted) } func (r *registry) CancelUpload(ctx echo.Context) error { @@ -404,7 +304,6 @@ func (r *registry) PushManifest(ctx echo.Context) error { defer ctx.Request().Body.Close() dig := digest(bz) - color.Yellow("manifest digest: %s\n", dig) var manifest ManifestList if err = json.Unmarshal(bz, &manifest); err != nil { @@ -465,7 +364,6 @@ func (r *registry) PushManifest(ctx echo.Context) error { } locationHeader := r.getHttpUrlFromSkylink(skylink) - color.Red("location header for get manifest: %s\n", locationHeader) ctx.Response().Header().Set("Location", locationHeader) ctx.Response().Header().Set("Docker-Content-Digest", dig) ctx.Response().Header().Set("X-Docker-Content-ID", skylink) @@ -601,8 +499,6 @@ func (r *registry) StartUpload(ctx echo.Context) error { layer := &types.Layer{ MediaType: "application/vnd.docker.image.rootfs.diff.tar.gzip", - RangeStart: uint32(0), - RangeEnd: uint32(len(bz)-1), Size: len(bz), Digest: dig, SkynetLink: skylink, diff --git a/types/types.go b/types/types.go index 2adc0204..06f55176 100644 --- a/types/types.go +++ b/types/types.go @@ -3,8 +3,6 @@ package types import ( "encoding/json" "fmt" - - "github.com/fatih/color" ) type ( @@ -26,16 +24,28 @@ type ( Config []*Config `json:"config"` } - Layer struct { - MediaType string `json:"mediaType"` + Blob struct { RangeStart uint32 RangeEnd uint32 + Digest string + Skylink string + UUID string + } + + Layer struct { + MediaType string `json:"mediaType"` + Blobs []Blob `json:"blobs"` Size int `json:"size"` Digest string `json:"digest"` SkynetLink string `json:"skynetLink"` UUID string `json:"uuid"` } + LayerRef struct { + Digest string + Skylink string + } + Config struct { MediaType string `json:"mediaType"` Size int `json:"size"` @@ -96,17 +106,24 @@ func (md Metadata) FindLayer(ref string) *Layer { func (md Metadata) FindLinkForDigest(ref string) (string, error) { for _, c := range md.Manifest.Config { if c.Digest == ref || c.Reference == ref { - color.Red("found skylink from config: %s\n", ref) return c.SkynetLink, nil } } for _, l := range md.Manifest.Layers { if l.Digest == ref { - color.Red("found skylink from manifest: %s\n", ref) return l.SkynetLink, nil } } return "", fmt.Errorf("ref does not exists") } + +func (lr LayerRef) Bytes() []byte { + bz, err := json.Marshal(lr) + if err != nil { + return nil + } + + return bz +}