Merge branch 'main' into sync-puts-in-eds-store

.github/workflows/go-ci.yml

@@ -23,9 +23,9 @@ jobs:
           go-version: ${{ env.GO_VERSION }}
 
       - name: golangci-lint
-        uses: golangci/golangci-lint-action@v3.6.0
+        uses: golangci/golangci-lint-action@v3.7.0
         with:
-          version: v1.52.2
+          version: v1.54.2
 
   go_mod_tidy_check:
     name: Go Mod Tidy Check

.golangci.yml

@@ -4,7 +4,7 @@ run:
 linters:
   enable:
     - bodyclose
-    - depguard
+     # - depguard as of v1.54.2, the default config throws errors on our repo
     - dogsled
     - dupl
     - errcheck

Dockerfile

@@ -15,7 +15,7 @@ COPY . .
 
 RUN make build && make cel-key
 
-FROM docker.io/alpine:3.18.2
+FROM docker.io/alpine:3.18.3
 
 # Read here why UID 10001: https://github.com/hexops/dockerfile/blob/main/README.md#do-not-use-a-uid-below-10000
 ARG UID=10001

Makefile

@@ -1,5 +1,6 @@
 SHELL=/usr/bin/env bash
 PROJECTNAME=$(shell basename "$(PWD)")
+DIR_FULLPATH=$(shell pwd)
 versioningPath := "github.com/celestiaorg/celestia-node/nodebuilder/node"
 LDFLAGS=-ldflags="-X '$(versioningPath).buildTime=$(shell date)' -X '$(versioningPath).lastCommit=$(shell git rev-parse HEAD)' -X '$(versioningPath).semanticVersion=$(shell git describe --tags --dirty=-dev 2>/dev/null || git rev-parse --abbrev-ref HEAD)'"
 ifeq (${PREFIX},)
@@ -176,3 +177,14 @@ adr-gen:
 	@echo "--> Generating ADR"
 	@curl -sSL https://raw.githubusercontent.com/celestiaorg/.github/main/adr-template.md > docs/architecture/adr-$(NUM)-$(TITLE).md
 .PHONY: adr-gen
+
+## telemetry-infra-up: launches local telemetry infrastructure. This includes grafana, jaeger, loki, pyroscope, and an otel-collector.
+## you can access the grafana instance at localhost:3000 and login with admin:admin.
+telemetry-infra-up:
+	PWD="${DIR_FULLPATH}/docker/telemetry" docker-compose -f ./docker/telemetry/docker-compose.yml up
+.PHONY: telemetry-infra-up
+
+## telemetry-infra-down: tears the telemetry infrastructure down. The stores for grafana, prometheus, and loki will persist.
+telemetry-infra-down:
+	PWD="${DIR_FULLPATH}/docker/telemetry" docker-compose -f ./docker/telemetry/docker-compose.yml down
+.PHONY: telemetry-infra-down

README.md

@@ -24,7 +24,7 @@ Continue reading [here](https://blog.celestia.org/celestia-mvp-release-data-avai
   - [API docs](#api-docs)
   - [Node types](#node-types)
   - [Run a node](#run-a-node)
-    - [Environment variables](#environment-variables)
+  - [Environment variables](#environment-variables)
   - [Package-specific documentation](#package-specific-documentation)
   - [Code of Conduct](#code-of-conduct)
 
@@ -55,7 +55,7 @@ For more information on setting up a node and the hardware requirements needed,
 
 ## API docs
 
-Celestia-node public API is documented [here](https://docs.celestia.org/category/node-api/).
+The celestia-node public API is documented [here](https://docs.celestia.org/category/node-api/).
 
 ## Node types
 
@@ -67,7 +67,7 @@ More information can be found [here](https://github.com/celestiaorg/celestia-nod
 
 ## Run a node
 
-`<node_type>` can be `bridge`, `full` or `light`.
+`<node_type>` can be: `bridge`, `full` or `light`.
 
 ```sh
 celestia <node_type> init
@@ -77,7 +77,9 @@ celestia <node_type> init
 celestia <node_type> start
 ```
 
-### Environment variables
+Please refer to [this guide](https://docs.celestia.org/nodes/celestia-node/) for more information on running a node.
+
+## Environment variables
 
 | Variable                | Explanation                         | Default value | Required |
 | ----------------------- | ----------------------------------- | ------------- | -------- |

api/docgen/examples.go

@@ -56,7 +56,7 @@ var ExampleValues = map[reflect.Type]interface{}{
 	reflect.TypeOf(node.Full):                node.Full,
 	reflect.TypeOf(auth.Permission("admin")): auth.Permission("admin"),
 	reflect.TypeOf(byzantine.BadEncoding):    byzantine.BadEncoding,
-	reflect.TypeOf((*fraud.Proof)(nil)).Elem(): byzantine.CreateBadEncodingProof(
+	reflect.TypeOf((*fraud.Proof[*header.ExtendedHeader])(nil)).Elem(): byzantine.CreateBadEncodingProof(
 		[]byte("bad encoding proof"),
 		42,
 		&byzantine.ErrByzantine{

blob/blob.go

@@ -12,7 +12,6 @@ import (
 	"github.com/celestiaorg/nmt"
 
 	"github.com/celestiaorg/celestia-node/share"
-	"github.com/celestiaorg/celestia-node/share/ipld"
 )
 
 // Commitment is a Merkle Root of the subtree built from shares of the Blob.
@@ -62,42 +61,6 @@ func (p Proof) equal(input Proof) error {
 	return nil
 }
 
-type jsonProof struct {
-	Start int      `json:"start"`
-	End   int      `json:"end"`
-	Nodes [][]byte `json:"nodes"`
-}
-
-func (p *Proof) MarshalJSON() ([]byte, error) {
-	proofs := make([]jsonProof, 0, p.Len())
-	for _, pp := range *p {
-		proofs = append(proofs, jsonProof{
-			Start: pp.Start(),
-			End:   pp.End(),
-			Nodes: pp.Nodes(),
-		})
-	}
-
-	return json.Marshal(proofs)
-}
-
-func (p *Proof) UnmarshalJSON(data []byte) error {
-	var proofs []jsonProof
-	err := json.Unmarshal(data, &proofs)
-	if err != nil {
-		return err
-	}
-
-	nmtProofs := make([]*nmt.Proof, len(proofs))
-	for i, jProof := range proofs {
-		nmtProof := nmt.NewInclusionProof(jProof.Start, jProof.End, jProof.Nodes, ipld.NMTIgnoreMaxNamespace)
-		nmtProofs[i] = &nmtProof
-	}
-
-	*p = nmtProofs
-	return nil
-}
-
 // Blob represents any application-specific binary data that anyone can submit to Celestia.
 type Blob struct {
 	types.Blob `json:"blob"`

blob/service.go

@@ -32,7 +32,7 @@ type Submitter interface {
 
 type Service struct {
 	// accessor dials the given celestia-core endpoint to submit blobs.
-	blobSumitter Submitter
+	blobSubmitter Submitter
 	// shareGetter retrieves the EDS to fetch all shares from the requested header.
 	shareGetter share.Getter
 	// headerGetter fetches header by the provided height
@@ -45,9 +45,9 @@ func NewService(
 	headerGetter func(context.Context, uint64) (*header.ExtendedHeader, error),
 ) *Service {
 	return &Service{
-		blobSumitter: submitter,
-		shareGetter:  getter,
-		headerGetter: headerGetter,
+		blobSubmitter: submitter,
+		shareGetter:   getter,
+		headerGetter:  headerGetter,
 	}
 }
 
@@ -58,7 +58,7 @@ func NewService(
 func (s *Service) Submit(ctx context.Context, blobs []*Blob) (uint64, error) {
 	log.Debugw("submitting blobs", "amount", len(blobs))
 
-	resp, err := s.blobSumitter.SubmitPayForBlob(ctx, types.OneInt().Neg(), 0, blobs)
+	resp, err := s.blobSubmitter.SubmitPayForBlob(ctx, types.OneInt().Neg(), 0, blobs)
 	if err != nil {
 		return 0, err
 	}

blob/service_test.go

@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"context"
 	"crypto/sha256"
+	"encoding/json"
 	"testing"
 	"time"
 
@@ -272,14 +273,14 @@ func TestBlobService_Get(t *testing.T) {
 			doFn: func() (interface{}, error) {
 				proof, err := service.GetProof(ctx, 1, blobs0[1].Namespace(), blobs0[1].Commitment)
 				require.NoError(t, err)
-				return proof.MarshalJSON()
+				return json.Marshal(proof)
 			},
 			expectedResult: func(i interface{}, err error) {
 				require.NoError(t, err)
 				jsonData, ok := i.([]byte)
 				require.True(t, ok)
 				var proof Proof
-				require.NoError(t, proof.UnmarshalJSON(jsonData))
+				require.NoError(t, json.Unmarshal(jsonData, &proof))
 
 				newProof, err := service.GetProof(ctx, 1, blobs0[1].Namespace(), blobs0[1].Commitment)
 				require.NoError(t, err)

cmd/celestia/rpc.go

@@ -23,7 +23,7 @@ import (
 	"github.com/celestiaorg/celestia-node/state"
 )
 
-const authEnvKey = "CELESTIA_NODE_AUTH_TOKEN"
+const authEnvKey = "CELESTIA_NODE_AUTH_TOKEN" //nolint:gosec
 
 var requestURL string
 var authTokenFlag string

core/exchange.go

@@ -78,7 +78,7 @@ func (ce *Exchange) GetVerifiedRange(
 	from *header.ExtendedHeader,
 	amount uint64,
 ) ([]*header.ExtendedHeader, error) {
-	headers, err := ce.GetRangeByHeight(ctx, uint64(from.Height())+1, amount)
+	headers, err := ce.GetRangeByHeight(ctx, from.Height()+1, amount)
 	if err != nil {
 		return nil, err
 	}
@@ -115,7 +115,7 @@ func (ce *Exchange) Get(ctx context.Context, hash libhead.Hash) (*header.Extende
 		return nil, fmt.Errorf("extending block data for height %d: %w", &block.Height, err)
 	}
 	// construct extended header
-	eh, err := ce.construct(ctx, &block.Header, comm, vals, eds)
+	eh, err := ce.construct(&block.Header, comm, vals, eds)
 	if err != nil {
 		panic(fmt.Errorf("constructing extended header for height %d: %w", &block.Height, err))
 	}
@@ -133,7 +133,10 @@ func (ce *Exchange) Get(ctx context.Context, hash libhead.Hash) (*header.Extende
 	return eh, nil
 }
 
-func (ce *Exchange) Head(ctx context.Context) (*header.ExtendedHeader, error) {
+func (ce *Exchange) Head(
+	ctx context.Context,
+	_ ...libhead.HeadOption[*header.ExtendedHeader],
+) (*header.ExtendedHeader, error) {
 	log.Debug("requesting head")
 	return ce.getExtendedHeaderByHeight(ctx, nil)
 }
@@ -157,7 +160,7 @@ func (ce *Exchange) getExtendedHeaderByHeight(ctx context.Context, height *int64
 		return nil, fmt.Errorf("extending block data for height %d: %w", b.Header.Height, err)
 	}
 	// create extended header
-	eh, err := ce.construct(ctx, &b.Header, &b.Commit, &b.ValidatorSet, eds)
+	eh, err := ce.construct(&b.Header, &b.Commit, &b.ValidatorSet, eds)
 	if err != nil {
 		panic(fmt.Errorf("constructing extended header for height %d: %w", b.Header.Height, err))
 	}

core/header_test.go

@@ -33,7 +33,7 @@ func TestMakeExtendedHeaderForEmptyBlock(t *testing.T) {
 	eds, err := extendBlock(b.Data, b.Header.Version.App)
 	require.NoError(t, err)
 
-	headerExt, err := header.MakeExtendedHeader(ctx, &b.Header, comm, val, eds)
+	headerExt, err := header.MakeExtendedHeader(&b.Header, comm, val, eds)
 	require.NoError(t, err)
 
 	assert.Equal(t, header.EmptyDAH(), *headerExt.DAH)

core/listener.go

@@ -160,7 +160,7 @@ func (cl *Listener) handleNewSignedBlock(ctx context.Context, b types.EventDataS
 		return fmt.Errorf("extending block data: %w", err)
 	}
 	// generate extended header
-	eh, err := cl.construct(ctx, &b.Header, &b.Commit, &b.ValidatorSet, eds)
+	eh, err := cl.construct(&b.Header, &b.Commit, &b.ValidatorSet, eds)
 	if err != nil {
 		panic(fmt.Errorf("making extended header: %w", err))
 	}
@@ -181,7 +181,7 @@ func (cl *Listener) handleNewSignedBlock(ctx context.Context, b types.EventDataS
 	if !syncing {
 		err = cl.hashBroadcaster(ctx, shrexsub.Notification{
 			DataHash: eh.DataHash.Bytes(),
-			Height:   uint64(eh.Height()),
+			Height:   eh.Height(),
 		})
 		if err != nil && !errors.Is(err, context.Canceled) {
 			log.Errorw("listener: broadcasting data hash",

core/listener_test.go

@@ -31,8 +31,8 @@ func TestListener(t *testing.T) {
 	// create mocknet with two pubsub endpoints
 	ps0, ps1 := createMocknetWithTwoPubsubEndpoints(ctx, t)
 	subscriber := p2p.NewSubscriber[*header.ExtendedHeader](ps1, header.MsgID, networkID)
-	err := subscriber.AddValidator(func(context.Context, *header.ExtendedHeader) pubsub.ValidationResult {
-		return pubsub.ValidationAccept
+	err := subscriber.SetVerifier(func(context.Context, *header.ExtendedHeader) error {
+		return nil
 	})
 	require.NoError(t, err)
 	require.NoError(t, subscriber.Start(ctx))

das/coordinator_test.go

@@ -366,7 +366,7 @@ func (m *mockSampler) sample(ctx context.Context, h *header.ExtendedHeader) erro
 	m.lock.Lock()
 	defer m.lock.Unlock()
 
-	height := uint64(h.Height())
+	height := h.Height()
 	m.done[height]++
 
 	if len(m.done) > int(m.NetworkHead-m.SampleFrom) && !m.isFinished {
@@ -503,7 +503,7 @@ func (o *checkOrder) middleWare(out sampleFn) sampleFn {
 
 		if len(o.queue) > 0 {
 			// check last item in queue to be same as input
-			if o.queue[0] != uint64(h.Height()) {
+			if o.queue[0] != h.Height() {
 				defer o.lock.Unlock()
 				return fmt.Errorf("expected height: %v,got: %v", o.queue[0], h.Height())
 			}
@@ -573,7 +573,7 @@ func (l *lock) releaseAll(except ...uint64) {
 func (l *lock) middleWare(out sampleFn) sampleFn {
 	return func(ctx context.Context, h *header.ExtendedHeader) error {
 		l.m.Lock()
-		ch, blocked := l.blockList[uint64(h.Height())]
+		ch, blocked := l.blockList[h.Height()]
 		l.m.Unlock()
 		if !blocked {
 			return out(ctx, h)
@@ -589,7 +589,7 @@ func (l *lock) middleWare(out sampleFn) sampleFn {
 }
 
 func onceMiddleWare(out sampleFn) sampleFn {
-	db := make(map[int64]int)
+	db := make(map[uint64]int)
 	m := sync.Mutex{}
 	return func(ctx context.Context, h *header.ExtendedHeader) error {
 		m.Lock()

das/daser.go

@@ -25,7 +25,7 @@ type DASer struct {
 	params Parameters
 
 	da     share.Availability
-	bcast  fraud.Broadcaster
+	bcast  fraud.Broadcaster[*header.ExtendedHeader]
 	hsub   libhead.Subscriber[*header.ExtendedHeader] // listens for new headers in the network
 	getter libhead.Getter[*header.ExtendedHeader]     // retrieves past headers
 
@@ -47,7 +47,7 @@ func NewDASer(
 	hsub libhead.Subscriber[*header.ExtendedHeader],
 	getter libhead.Getter[*header.ExtendedHeader],
 	dstore datastore.Datastore,
-	bcast fraud.Broadcaster,
+	bcast fraud.Broadcaster[*header.ExtendedHeader],
 	shrexBroadcast shrexsub.BroadcastFn,
 	options ...Option,
 ) (*DASer, error) {
@@ -99,7 +99,7 @@ func (d *DASer) Start(ctx context.Context) error {
 		// attempt to get head info. No need to handle error, later DASer
 		// will be able to find new head from subscriber after it is started
 		if h, err := d.getter.Head(ctx); err == nil {
-			cp.NetworkHead = uint64(h.Height())
+			cp.NetworkHead = h.Height()
 		}
 	}
 	log.Info("starting DASer from checkpoint: ", cp.String())
@@ -152,7 +152,7 @@ func (d *DASer) sample(ctx context.Context, h *header.ExtendedHeader) error {
 		var byzantineErr *byzantine.ErrByzantine
 		if errors.As(err, &byzantineErr) {
 			log.Warn("Propagating proof...")
-			sendErr := d.bcast.Broadcast(ctx, byzantine.CreateBadEncodingProof(h.Hash(), uint64(h.Height()), byzantineErr))
+			sendErr := d.bcast.Broadcast(ctx, byzantine.CreateBadEncodingProof(h.Hash(), h.Height(), byzantineErr))
 			if sendErr != nil {
 				log.Errorw("fraud proof propagating failed", "err", sendErr)
 			}