From 30f061699af303fd1a0a214fc60d520d83175b37 Mon Sep 17 00:00:00 2001 From: Matthew Sevey Date: Wed, 21 Jun 2023 11:08:56 -0400 Subject: [PATCH 1/4] Update ci_release.yml (#2384) ## Overview I missed adding the `merge_group` trigger when enabling the merge queue. oops. Adding it now :-) ## Checklist - [ ] New and updated code has appropriate documentation - [ ] New and updated code has new and/or updated testing - [ ] Required CI checks are passing - [ ] Visual proof for any user facing features like CLI or documentation updates - [ ] Linked issues closed with keywords --- .github/workflows/ci_release.yml | 1 + .github/workflows/docker-build-publish.yml | 1 + 2 files changed, 2 insertions(+) diff --git a/.github/workflows/ci_release.yml b/.github/workflows/ci_release.yml index 654c3c0a34..653add9256 100644 --- a/.github/workflows/ci_release.yml +++ b/.github/workflows/ci_release.yml @@ -1,5 +1,6 @@ name: CI and Release on: + merge_group: push: branches: - main diff --git a/.github/workflows/docker-build-publish.yml b/.github/workflows/docker-build-publish.yml index 0ddebbd750..c922e8b901 100644 --- a/.github/workflows/docker-build-publish.yml +++ b/.github/workflows/docker-build-publish.yml @@ -2,6 +2,7 @@ name: Docker Build & Publish # Trigger on all push events, new semantic version tags, and all PRs on: + merge_group: push: branches: - "**" From de88cffff715980177e9a2cfee4686ca440106b3 Mon Sep 17 00:00:00 2001 From: Ryan Date: Wed, 21 Jun 2023 17:24:24 +0200 Subject: [PATCH 2/4] feat(gateway): enabling CORS * on gateway (#2373) Closes #2263 Co-authored-by: Hlib Kanunnikov --- api/gateway/middleware.go | 8 ++++++++ api/gateway/server_test.go | 31 +++++++++++++++++++++++++++++-- 2 files changed, 37 insertions(+), 2 deletions(-) diff --git a/api/gateway/middleware.go b/api/gateway/middleware.go index 498b9c5d64..2c88b34185 100644 --- a/api/gateway/middleware.go +++ b/api/gateway/middleware.go @@ -18,9 +18,17 @@ func (h *Handler) RegisterMiddleware(srv *Server) { setContentType, checkPostDisabled(h.state), wrapRequestContext, + enableCors, ) } +func enableCors(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Access-Control-Allow-Origin", "*") + next.ServeHTTP(w, r) + }) +} + func setContentType(next http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Add("Content-Type", "application/json") diff --git a/api/gateway/server_test.go b/api/gateway/server_test.go index e98d7a6091..cb8e3d17ae 100644 --- a/api/gateway/server_test.go +++ b/api/gateway/server_test.go @@ -12,8 +12,12 @@ import ( "github.com/stretchr/testify/require" ) +const ( + address = "localhost" + port = "0" +) + func TestServer(t *testing.T) { - address, port := "localhost", "0" server := NewServer(address, port) ctx, cancel := context.WithCancel(context.Background()) @@ -42,10 +46,33 @@ func TestServer(t *testing.T) { require.NoError(t, err) } +func TestCorsEnabled(t *testing.T) { + server := NewServer(address, port) + server.RegisterMiddleware(enableCors) + + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + + err := server.Start(ctx) + require.NoError(t, err) + + // register ping handler + ping := new(ping) + server.RegisterHandlerFunc("/ping", ping.ServeHTTP, http.MethodGet) + + url := fmt.Sprintf("http://%s/ping", server.ListenAddr()) + + resp, err := http.Get(url) + require.NoError(t, err) + defer resp.Body.Close() + + require.NoError(t, err) + require.Equal(t, resp.Header.Get("Access-Control-Allow-Origin"), "*") +} + // TestServer_contextLeakProtection tests to ensure a context // deadline was added by the context wrapper middleware server-side. func TestServer_contextLeakProtection(t *testing.T) { - address, port := "localhost", "0" server := NewServer(address, port) server.RegisterMiddleware(wrapRequestContext) From 9adb61eb1d3dbe54e2c7e13cbd51e9de9205fc47 Mon Sep 17 00:00:00 2001 From: Hlib Kanunnikov Date: Wed, 21 Jun 2023 17:26:41 +0200 Subject: [PATCH 3/4] share: introduce Namespace type (#2367) Mostly copied from the app's Namespace type with multiple modifications. In preparation for https://github.com/celestiaorg/celestia-node/issues/2301. We want a more straightforward Namespace type from what the app provides: * Repsents 1:1 mapping of the serialized form of Namespace [version byte + namespace id] * So that it does not allocate on the hot path when getting data by namespace * and had simpler json serialization --- share/namespace.go | 151 ++++++++++++++++++++++++++++++++++++++++ share/namespace_test.go | 84 ++++++++++++++++++++++ share/nid.go | 1 + 3 files changed, 236 insertions(+) create mode 100644 share/namespace.go create mode 100644 share/namespace_test.go diff --git a/share/namespace.go b/share/namespace.go new file mode 100644 index 0000000000..0af1d1a60e --- /dev/null +++ b/share/namespace.go @@ -0,0 +1,151 @@ +package share + +import ( + "bytes" + "encoding/hex" + "fmt" + + appns "github.com/celestiaorg/celestia-app/pkg/namespace" + "github.com/celestiaorg/nmt/namespace" +) + +// Various reserved namespaces. +var ( + MaxReservedNamespace = Namespace(appns.MaxReservedNamespace.Bytes()) + ParitySharesNamespace = Namespace(appns.ParitySharesNamespace.Bytes()) + TailPaddingNamespace = Namespace(appns.TailPaddingNamespace.Bytes()) + ReservedPaddingNamespace = Namespace(appns.ReservedPaddingNamespace.Bytes()) + TxNamespace = Namespace(appns.TxNamespace.Bytes()) + PayForBlobNamespace = Namespace(appns.PayForBlobNamespace.Bytes()) +) + +// Namespace represents namespace of a Share. +// Consists of version byte and namespace ID. +type Namespace []byte + +// NamespaceFromBytes converts bytes into Namespace and validates it. +func NamespaceFromBytes(b []byte) (Namespace, error) { + n := Namespace(b) + return n, n.Validate() +} + +// Version reports version of the Namespace. +func (n Namespace) Version() byte { + return n[appns.NamespaceVersionSize-1] +} + +// ID reports ID of the Namespace. +func (n Namespace) ID() namespace.ID { + return namespace.ID(n[appns.NamespaceVersionSize:]) +} + +// ToNMT converts the whole Namespace(both Version and ID parts) into NMT's namespace.ID +// NOTE: Once https://github.com/celestiaorg/nmt/issues/206 is closed Namespace should become NNT's +// type. +func (n Namespace) ToNMT() namespace.ID { + return namespace.ID(n) +} + +// ToAppNamespace converts the Namespace to App's definition of Namespace. +// TODO: Unify types between node and app +func (n Namespace) ToAppNamespace() appns.Namespace { + return appns.Namespace{Version: n.Version(), ID: n.ID()} +} + +// Len reports the total length of the namespace. +func (n Namespace) Len() int { + return len(n) +} + +// String stringifies the Namespace. +func (n Namespace) String() string { + return hex.EncodeToString(n) +} + +// Equals compares two Namespaces. +func (n Namespace) Equals(target Namespace) bool { + return bytes.Equal(n, target) +} + +// Validate checks if the namespace is correct. +func (n Namespace) Validate() error { + if n.Len() != NamespaceSize { + return fmt.Errorf("invalid namespace length: expected %d, got %d", NamespaceSize, n.Len()) + } + if n.Version() != appns.NamespaceVersionZero && n.Version() != appns.NamespaceVersionMax { + return fmt.Errorf("invalid namespace version %v", n.Version()) + } + if len(n.ID()) != appns.NamespaceIDSize { + return fmt.Errorf("invalid namespace id length: expected %d, got %d", appns.NamespaceIDSize, n.ID().Size()) + } + if n.Version() == appns.NamespaceVersionZero && !bytes.HasPrefix(n.ID(), appns.NamespaceVersionZeroPrefix) { + return fmt.Errorf("invalid namespace id: expect %d leading zeroes", len(appns.NamespaceVersionZeroPrefix)) + } + return nil +} + +// ValidateDataNamespace checks if the Namespace contains real/useful data. +func (n Namespace) ValidateDataNamespace() error { + if err := n.Validate(); err != nil { + return err + } + if n.Equals(ParitySharesNamespace) || n.Equals(TailPaddingNamespace) { + return fmt.Errorf("invalid data namespace(%s): parity and tail padding namespace are forbidden", n) + } + return nil +} + +// ValidateBlobNamespace checks if the Namespace is valid blob namespace. +func (n Namespace) ValidateBlobNamespace() error { + if err := n.ValidateDataNamespace(); err != nil { + return err + } + if bytes.Compare(n, MaxReservedNamespace) < 1 { + return fmt.Errorf("invalid blob namespace(%s): reserved namespaces are forbidden", n) + } + return nil +} + +// IsAboveMax checks if the namespace is above the maximum namespace of the given hash. +func (n Namespace) IsAboveMax(nodeHash []byte) bool { + return !n.IsLessOrEqual(nodeHash[n.Len() : n.Len()*2]) +} + +// IsBelowMin checks if the target namespace is below the minimum namespace of the given hash. +func (n Namespace) IsBelowMin(nodeHash []byte) bool { + return n.IsLess(nodeHash[:n.Len()]) +} + +// IsOutsideRange checks if the namespace is outside the min-max range of the given hashes. +func (n Namespace) IsOutsideRange(leftNodeHash, rightNodeHash []byte) bool { + return n.IsBelowMin(leftNodeHash) || n.IsAboveMax(rightNodeHash) +} + +// Repeat copies the Namespace t times. +func (n Namespace) Repeat(t int) []Namespace { + ns := make([]Namespace, t) + for i := 0; i < t; i++ { + ns[i] = n + } + return ns +} + +// IsLess reports if the Namespace is less than the target. +func (n Namespace) IsLess(target Namespace) bool { + return bytes.Compare(n, target) == -1 +} + +// IsLessOrEqual reports if the Namespace is less than the target. +func (n Namespace) IsLessOrEqual(target Namespace) bool { + return bytes.Compare(n, target) < 1 +} + +// IsGreater reports if the Namespace is greater than the target. +func (n Namespace) IsGreater(target Namespace) bool { + return bytes.Compare(n, target) == 1 +} + +// IsGreaterOrEqualThan reports if the Namespace is greater or equal than the target. +func (n Namespace) IsGreaterOrEqualThan(target Namespace) bool { + return bytes.Compare(n, target) > -1 +} diff --git a/share/namespace_test.go b/share/namespace_test.go new file mode 100644 index 0000000000..8cc61b379b --- /dev/null +++ b/share/namespace_test.go @@ -0,0 +1,84 @@ +package share + +import ( + "bytes" + "testing" + + "github.com/stretchr/testify/assert" + + appns "github.com/celestiaorg/celestia-app/pkg/namespace" +) + +var ( + validID = append( + appns.NamespaceVersionZeroPrefix, + bytes.Repeat([]byte{1}, appns.NamespaceVersionZeroIDSize)..., + ) + tooShortID = append(appns.NamespaceVersionZeroPrefix, []byte{1}...) + tooLongID = append(appns.NamespaceVersionZeroPrefix, bytes.Repeat([]byte{1}, NamespaceSize)...) + invalidPrefixID = bytes.Repeat([]byte{1}, NamespaceSize) +) + +func TestFrom(t *testing.T) { + type testCase struct { + name string + bytes []byte + wantErr bool + want Namespace + } + validNamespace := []byte{} + validNamespace = append(validNamespace, appns.NamespaceVersionZero) + validNamespace = append(validNamespace, appns.NamespaceVersionZeroPrefix...) + validNamespace = append(validNamespace, bytes.Repeat([]byte{0x1}, appns.NamespaceVersionZeroIDSize)...) + parityNamespace := bytes.Repeat([]byte{0xFF}, NamespaceSize) + + testCases := []testCase{ + { + name: "valid namespace", + bytes: validNamespace, + wantErr: false, + want: append([]byte{appns.NamespaceVersionZero}, validID...), + }, + { + name: "parity namespace", + bytes: parityNamespace, + wantErr: false, + want: append([]byte{appns.NamespaceVersionMax}, bytes.Repeat([]byte{0xFF}, appns.NamespaceIDSize)...), + }, + { + name: "unsupported version", + bytes: append([]byte{1}, append( + appns.NamespaceVersionZeroPrefix, + bytes.Repeat([]byte{1}, NamespaceSize-len(appns.NamespaceVersionZeroPrefix))..., + )...), + wantErr: true, + }, + { + name: "unsupported id: too short", + bytes: append([]byte{appns.NamespaceVersionZero}, tooShortID...), + wantErr: true, + }, + { + name: "unsupported id: too long", + bytes: append([]byte{appns.NamespaceVersionZero}, tooLongID...), + wantErr: true, + }, + { + name: "unsupported id: invalid prefix", + bytes: append([]byte{appns.NamespaceVersionZero}, invalidPrefixID...), + wantErr: true, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + got, err := NamespaceFromBytes(tc.bytes) + if tc.wantErr { + assert.Error(t, err) + return + } + assert.NoError(t, err) + assert.Equal(t, tc.want, got) + }) + } +} diff --git a/share/nid.go b/share/nid.go index b7fd4e5836..7d960cc9e1 100644 --- a/share/nid.go +++ b/share/nid.go @@ -10,6 +10,7 @@ import ( // NewNamespaceV0 takes a variable size byte slice and creates a version 0 Namespace ID. // The byte slice must be <= 10 bytes. // If it is less than 10 bytes, it will be left padded to size 10 with 0s. +// TODO: Adapt for Namespace in the integration PR func NewNamespaceV0(subNId []byte) (namespace.ID, error) { if lnid := len(subNId); lnid > appns.NamespaceVersionZeroIDSize { return nil, fmt.Errorf("namespace id must be <= %v, but it was %v bytes", appns.NamespaceVersionZeroIDSize, lnid) From ce8ccb3677dd0498b5230ca5c3080a10003aa8d2 Mon Sep 17 00:00:00 2001 From: rene <41963722+renaynay@users.noreply.github.com> Date: Wed, 21 Jun 2023 17:28:02 +0200 Subject: [PATCH 4/4] feat(header): Check app version in eh validation and return err on mismatch (#2138) As recommended by @cmwaters, we should ensure (as a rudimentary check) that the headers being passed to us from core match in app versions as upgraded app versions can indicate breaking changes. We should error out if the app version does not match. This PR is still draft until celestia-app has a version out where they export the AppVersion as a const that we can depend on. Eventually we should support several versions via #2137 but for now, this is a good enough check. TODO: - [x] depend on const from app - [x] test --------- Co-authored-by: Hlib Kanunnikov --- go.mod | 2 +- go.sum | 4 ++-- header/header.go | 6 ++++++ header/headertest/verify_test.go | 8 ++++++++ 4 files changed, 17 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 5f5fe104a0..aef01d86a8 100644 --- a/go.mod +++ b/go.mod @@ -330,7 +330,7 @@ require ( ) replace ( - github.com/cosmos/cosmos-sdk => github.com/celestiaorg/cosmos-sdk v1.13.0-sdk-v0.46.11 + github.com/cosmos/cosmos-sdk => github.com/celestiaorg/cosmos-sdk v1.14.0-sdk-v0.46.11 github.com/filecoin-project/dagstore => github.com/celestiaorg/dagstore v0.0.0-20230413141458-735ab09a15d6 github.com/gogo/protobuf => github.com/regen-network/protobuf v1.3.3-alpha.regen.1 github.com/tendermint/tendermint => github.com/celestiaorg/celestia-core v1.21.2-tm-v0.34.27 diff --git a/go.sum b/go.sum index e548acff6a..065c88774f 100644 --- a/go.sum +++ b/go.sum @@ -346,8 +346,8 @@ github.com/celestiaorg/celestia-app v1.0.0-rc2 h1:/u7eespYtBpQtBSz3P8/rKfz9rW7QO github.com/celestiaorg/celestia-app v1.0.0-rc2/go.mod h1:uiTWKTtRpVwvSiFDl2zausrU1ZBHBWgk7z52pfzJqJU= github.com/celestiaorg/celestia-core v1.21.2-tm-v0.34.27 h1:nmr9O5BflgNR1aWehs1ZFw4obA//M/+g+SrSMK9sOBA= github.com/celestiaorg/celestia-core v1.21.2-tm-v0.34.27/go.mod h1:GVo91Wifg9KL/nFx9nPkpl0UIFdvvs4fhnly9GhGxZU= -github.com/celestiaorg/cosmos-sdk v1.13.0-sdk-v0.46.11 h1:Rd5EvJx1nG3KurBspVN51RVmvif0Lp2UVURbG2ad3Cs= -github.com/celestiaorg/cosmos-sdk v1.13.0-sdk-v0.46.11/go.mod h1:xCG6OUkJy5KUMEg20Zk010lra9XjkmKS3+bk0wp7bd8= +github.com/celestiaorg/cosmos-sdk v1.14.0-sdk-v0.46.11 h1:xDLC0ZvIwj9S2gTs9b8EespilL/u/vOGcqJuHz1r/eA= +github.com/celestiaorg/cosmos-sdk v1.14.0-sdk-v0.46.11/go.mod h1:IwvD2nN3vEMkjxhTw/SF5tyJ0+x3GB0EdyQJyteK06U= github.com/celestiaorg/dagstore v0.0.0-20230413141458-735ab09a15d6 h1:/yCwMCoOPcYCiG18u8/1pv5eXF04xczoQO3sR0bKsgM= github.com/celestiaorg/dagstore v0.0.0-20230413141458-735ab09a15d6/go.mod h1:ta/DlqIH10bvhwqJIw51Nq3QU4XVMp6pz3f0Deve9fM= github.com/celestiaorg/go-fraud v0.1.0 h1:v6mZvlmf2J5ELZfPnrtmmOvKbaYIUs/erDWPO8NbZyY= diff --git a/header/header.go b/header/header.go index 25c3b0e5cf..e59d3802c1 100644 --- a/header/header.go +++ b/header/header.go @@ -10,6 +10,7 @@ import ( tmjson "github.com/tendermint/tendermint/libs/json" core "github.com/tendermint/tendermint/types" + "github.com/celestiaorg/celestia-app/pkg/appconsts" "github.com/celestiaorg/celestia-app/pkg/da" libhead "github.com/celestiaorg/go-header" "github.com/celestiaorg/rsmt2d" @@ -115,6 +116,11 @@ func (eh *ExtendedHeader) Validate() error { return fmt.Errorf("ValidateBasic error on RawHeader at height %d: %w", eh.Height(), err) } + if eh.RawHeader.Version.App != appconsts.LatestVersion { + return fmt.Errorf("app version mismatch, expected: %d, got %d", appconsts.LatestVersion, + eh.RawHeader.Version.App) + } + err = eh.Commit.ValidateBasic() if err != nil { return fmt.Errorf("ValidateBasic error on Commit at height %d: %w", eh.Height(), err) diff --git a/header/headertest/verify_test.go b/header/headertest/verify_test.go index 78da779a54..177352a4cf 100644 --- a/header/headertest/verify_test.go +++ b/header/headertest/verify_test.go @@ -8,6 +8,7 @@ import ( "github.com/stretchr/testify/assert" tmrand "github.com/tendermint/tendermint/libs/rand" + "github.com/celestiaorg/celestia-app/pkg/appconsts" libhead "github.com/celestiaorg/go-header" ) @@ -80,6 +81,13 @@ func TestVerify(t *testing.T) { }, err: true, }, + { + prepare: func() libhead.Header { + untrustedAdj.RawHeader.Version.App = appconsts.LatestVersion + 1 + return untrustedAdj + }, + err: true, + }, } for i, test := range tests {