Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Content storage #14

Merged
merged 3 commits into from
Dec 13, 2023
Merged

Content storage #14

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
9818001
feat: content storage
fearlessfe Dec 11, 2023
7c7b9a5
faat: change the content_storage package
fearlessfe Dec 12, 2023
c5326cd
feat: add some commonts
fearlessfe Dec 13, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,7 @@ require (
github.com/kylelemons/godebug v1.1.0
github.com/mattn/go-colorable v0.1.13
github.com/mattn/go-isatty v0.0.17
github.com/mattn/go-sqlite3 v1.14.18
github.com/naoina/toml v0.1.2-0.20170918210437-9fafd6967416
github.com/olekukonko/tablewriter v0.0.5
github.com/optimism-java/utp-go v0.0.0-20231203033001-5a531e1e11a0
Expand Down
2 changes: 2 additions & 0 deletions go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -443,6 +443,8 @@ github.com/mattn/go-runewidth v0.0.3/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzp
github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
github.com/mattn/go-runewidth v0.0.13 h1:lTGmDsbAYt5DmK6OnoV7EuIF1wEIFAcxld6ypU4OSgU=
github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
github.com/mattn/go-sqlite3 v1.14.18 h1:JL0eqdCOq6DJVNPSvArO/bIV9/P7fbGrV00LZHc+5aI=
github.com/mattn/go-sqlite3 v1.14.18/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg=
github.com/mattn/goveralls v0.0.2/go.mod h1:8d1ZMHsd7fW6IRPKQh46F2WRpyib5/X4FOpevwGNQEw=
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 h1:I0XW9+e1XWDxdcEniV4rQAIOPUGDq67JSCiRCgGCZLI=
Expand Down
45 changes: 35 additions & 10 deletions p2p/discover/portal_protocol.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import (
"context"
"crypto/ecdsa"
crand "crypto/rand"
"crypto/sha256"
"encoding/binary"
"errors"
"fmt"
Expand All @@ -23,6 +24,7 @@ import (
"github.com/ethereum/go-ethereum/p2p/enode"
"github.com/ethereum/go-ethereum/p2p/enr"
"github.com/ethereum/go-ethereum/p2p/netutil"
"github.com/ethereum/go-ethereum/portalnetwork/storage"
"github.com/ethereum/go-ethereum/rlp"
ssz "github.com/ferranbt/fastssz"
"github.com/holiman/uint256"
Expand Down Expand Up @@ -62,6 +64,16 @@ const (
PersistOfferRequestKind byte = 0x02
)

var ErrNilContentKey = errors.New("content key cannot be nil")
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

shouldn't these all be moved to content_storage.go?

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this err is return from the PortalProtocol before call the content_storage, so this should be in PortalProtocol


var ContentNotFound = storage.ErrContentNotFound

type ContentStorage interface {
Get(contentId []byte) ([]byte, error)

Put(contentId []byte, content []byte) error
}

type ContentElement struct {
Node enode.ID
ContentKeys [][]byte
Expand All @@ -86,6 +98,8 @@ type OfferRequest struct {
Request interface{}
}

type PortalProtocolOption func(p *PortalProtocol)

type PortalProtocolConfig struct {
BootstrapNodes []*enode.Node

Expand Down Expand Up @@ -130,12 +144,18 @@ type PortalProtocol struct {
radiusCache *fastcache.Cache
closeCtx context.Context
cancelCloseCtx context.CancelFunc
storage Storage
storage ContentStorage
toContentId func(contentKey []byte) []byte

contentQueue chan *ContentElement
}

func NewPortalProtocol(config *PortalProtocolConfig, protocolId string, privateKey *ecdsa.PrivateKey, storage Storage, contentQueue chan *ContentElement) (*PortalProtocol, error) {
func defaultContentIdFunc(contentKey []byte) []byte {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

same as above?

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this should be in PortalProtocol, storage do not care about the contentKey to contentId

digest := sha256.Sum256(contentKey)
return digest[:]
}

func NewPortalProtocol(config *PortalProtocolConfig, protocolId string, privateKey *ecdsa.PrivateKey, storage ContentStorage, contentQueue chan *ContentElement, opts ...PortalProtocolOption) (*PortalProtocol, error) {
nodeDB, err := enode.OpenDB(config.NodeDBPath)
if err != nil {
return nil, err
Expand All @@ -159,9 +179,14 @@ func NewPortalProtocol(config *PortalProtocolConfig, protocolId string, privateK
localNode: localNode,
validSchemes: enode.ValidSchemes,
storage: storage,
toContentId: defaultContentIdFunc,
contentQueue: contentQueue,
}

for _, opt := range opts {
opt(protocol)
}

return protocol, nil
}

Expand Down Expand Up @@ -451,9 +476,9 @@ func (p *PortalProtocol) processOffer(target *enode.Node, resp []byte, request *
} else {
for _, index := range contentKeyBitlist.BitIndices() {
contentKey := request.Request.(*PersistOfferRequest).ContentKeys[index]
contentId := p.storage.ContentId(contentKey)
contentId := p.toContentId(contentKey)
if contentId != nil {
content, err = p.storage.Get(contentKey, contentId)
content, err = p.storage.Get(contentId)
if err != nil {
p.log.Error("failed to get content from storage", "err", err)
contents = append(contents, []byte{})
Expand Down Expand Up @@ -834,13 +859,13 @@ func (p *PortalProtocol) handleFindContent(id enode.ID, addr *net.UDPAddr, reque
enrOverhead := 4 //per added ENR, 4 bytes offset overhead
var err error

contentId := p.storage.ContentId(request.ContentKey)
contentId := p.toContentId(request.ContentKey)
if contentId == nil {
return nil, ContentNotFound
return nil, ErrNilContentKey
}

var content []byte
content, err = p.storage.Get(request.ContentKey, contentId)
content, err = p.storage.Get(contentId)
if err != nil && !errors.Is(err, ContentNotFound) {
return nil, err
}
Expand Down Expand Up @@ -1004,16 +1029,16 @@ func (p *PortalProtocol) handleOffer(id enode.ID, addr *net.UDPAddr, request *po

contentKeys := make([][]byte, 0)
for i, contentKey := range request.ContentKeys {
contentId := p.storage.ContentId(contentKey)
contentId := p.toContentId(contentKey)
if contentId != nil {
if inRange(p.Self().ID(), p.nodeRadius, contentId) {
if _, err = p.storage.Get(contentKey, contentId); err != nil {
if _, err = p.storage.Get(contentId); err != nil {
contentKeyBitlist.SetBitAt(uint64(i), true)
contentKeys = append(contentKeys, contentKey)
}
}
} else {
return nil, nil
return nil, ErrNilContentKey
}
}

Expand Down
15 changes: 5 additions & 10 deletions p2p/discover/portal_protocol_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,6 @@ import (
"github.com/optimism-java/utp-go"
"github.com/prysmaticlabs/go-bitfield"

"github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/internal/testlog"
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/p2p/discover/portalwire"
Expand All @@ -26,19 +25,15 @@ type MockStorage struct {
db map[string][]byte
}

func (m *MockStorage) ContentId(contentKey []byte) []byte {
return crypto.Keccak256(contentKey)
}

func (m *MockStorage) Get(contentKey []byte, contentId []byte) ([]byte, error) {
func (m *MockStorage) Get(contentId []byte) ([]byte, error) {
if content, ok := m.db[string(contentId)]; ok {
return content, nil
}
return nil, ContentNotFound
}

func (m *MockStorage) Put(contentKey []byte, content []byte) error {
m.db[string(m.ContentId(contentKey))] = content
func (m *MockStorage) Put(contentId []byte, content []byte) error {
m.db[string(contentId)] = content
return nil
}

Expand Down Expand Up @@ -243,7 +238,7 @@ func TestPortalWireProtocol(t *testing.T) {
return n.ID() == node2.localNode.Node().ID()
})

err = node1.storage.Put([]byte("test_key"), []byte("test_value"))
err = node1.storage.Put(node1.toContentId([]byte("test_key")), []byte("test_value"))
assert.NoError(t, err)

flag, content, err := node2.findContent(node1.localNode.Node(), []byte("test_key"))
Expand All @@ -263,7 +258,7 @@ func TestPortalWireProtocol(t *testing.T) {
_, err = rand.Read(largeTestContent)
assert.NoError(t, err)

err = node1.storage.Put([]byte("large_test_key"), largeTestContent)
err = node1.storage.Put(node1.toContentId([]byte("large_test_key")), largeTestContent)
assert.NoError(t, err)

flag, content, err = node2.findContent(node1.localNode.Node(), []byte("large_test_key"))
Expand Down
13 changes: 0 additions & 13 deletions p2p/discover/portal_storage.go
View file
Open in desktop

This file was deleted.

Loading