Merge branch 'rollkit:main' into main

.github/workflows/ci_release.yml

@@ -24,24 +24,77 @@ on:
           - major
 
 jobs:
+  setup:
+    runs-on: ubuntu-latest
+    env:
+      # use consistent go version throughout pipeline here
+      GO_VERSION: "1.20"
+    outputs:
+      go-version: ${{ steps.set-vars.outputs.go-version }}
+    steps:
+      - name: Set go version
+        id: set-vars
+        run: echo "go-version=${{env.GO_VERSION}}" >> "$GITHUB_OUTPUT"
+
   lint:
+    needs: [setup]
     uses: ./.github/workflows/lint.yml
     with:
-      GO_VERSION: "1.20"
+      go-version: ${{ needs.setup.outputs.go-version }}
 
   test:
+    needs: [setup]
     uses: ./.github/workflows/test.yml
     with:
-      GO_VERSION: "1.20"
+      go-version: ${{ needs.setup.outputs.go-version }}
 
   proto:
     uses: ./.github/workflows/proto.yml
 
-  # Make a release if this is a manually trigger job, i.e. workflow_dispatch
+  # get_merged_pr_labels uses the listPullRequestsAssociatedWithCommit API
+  # endpoint to get the PR information for the commit during a push event. Once
+  # the PR information is received, we check to see if the create-release label
+  # was added to the pr.
+  get_merged_pr_labels:
+    runs-on: ubuntu-latest
+    outputs:
+      has_release_label: ${{ steps.set-outputs.outputs.has_release_label }}
+    steps:
+      # We only want to run this step on a push event, otherwise this will error
+      # out as the result is null. We have the if condition here as to not block
+      # steps that rely on this step and others if this step is skipped.
+      - name: Query listPullRequestsAssociatedWithCommit for the PR information
+        if: ${{ github.event_name == 'push' }}
+        uses: actions/github-script@v6
+        id: get_pr_data
+        with:
+          script: |
+            const prData = await github.rest.repos.listPullRequestsAssociatedWithCommit({
+              commit_sha: context.sha,
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+            });
+            const pr = prData.data[0];
+            const prLabels = pr ? pr.labels.map(label => label.name) : [];
+            const hasReleaseLabel = prLabels.includes('create-release');
+            return { hasReleaseLabel };
+      # Only run if the result is not null. We add this check so that the CI
+      # does not show a failure when the previous step is skipped.
+      - name: Set the outputs
+        if: steps.get_pr_data.outputs.result != null
+        id: set-outputs
+        run: echo "has_release_label=${{ fromJSON(steps.get_pr_data.outputs.result).hasReleaseLabel }}" >> "$GITHUB_OUTPUT"
+
+  # Make a release if this is a manually trigger job, i.e. workflow_dispatch, or
+  # there was a merged pr with the create-release label
   release:
-    needs: [lint, test, proto]
+    needs: [lint, test, proto, get_merged_pr_labels]
     runs-on: ubuntu-latest
-    if: ${{ github.event_name == 'workflow_dispatch' }}
+    if: |
+      github.event_name == 'workflow_dispatch' ||
+      (github.event_name == 'push' &&
+      contains(github.ref, 'refs/heads/main') &&
+      needs.get_merged_pr_labels.outputs.has_release_label)
     permissions: "write-all"
     steps:
       - uses: actions/checkout@v4

.github/workflows/lint.yml

@@ -4,7 +4,7 @@ name: lint
 on:
   workflow_call:
     inputs:
-      GO_VERSION:
+      go-version:
         description: "Go version to use"
         type: string
         required: true
@@ -17,7 +17,7 @@ jobs:
       - uses: actions/checkout@v4
       - uses: actions/setup-go@v4
         with:
-          go-version: ${{ inputs.GO_VERSION }}
+          go-version: ${{ inputs.go-version }}
         # This steps sets the GIT_DIFF environment variable to true
         # if files defined in PATTERS changed
       - uses: technote-space/get-diff-action@v6.1.2

.github/workflows/test.yml

@@ -4,7 +4,7 @@ name: Tests / Code Coverage
 on:
   workflow_call:
     inputs:
-      GO_VERSION:
+      go-version:
         description: "Go version to use"
         type: string
         required: true
@@ -17,7 +17,7 @@ jobs:
       - uses: actions/checkout@v4
       - uses: actions/setup-go@v4
         with:
-          go-version: ${{ inputs.GO_VERSION }}
+          go-version: ${{ inputs.go-version }}
       - run: go mod tidy
       - name: check for diff
         run: git diff --exit-code
@@ -30,7 +30,7 @@ jobs:
       - name: set up go
         uses: actions/setup-go@v4
         with:
-          go-version: ${{ inputs.GO_VERSION }}
+          go-version: ${{ inputs.go-version }}
       - name: Run unit test
         run: make test
       - name: upload coverage report
@@ -47,6 +47,6 @@ jobs:
       - name: set up go
         uses: actions/setup-go@v4
         with:
-          go-version: ${{ inputs.GO_VERSION }}
+          go-version: ${{ inputs.go-version }}
       - name: Integration Tests
         run: echo "No integration tests yet"

Makefile

@@ -1,9 +1,13 @@
 DOCKER := $(shell which docker)
 DOCKER_BUF := $(DOCKER) run --rm -v $(CURDIR):/workspace --workdir /workspace bufbuild/buf
 
-# Define pkgs, run, and cover vairables for test so that we can override them in
+# Define pkgs, run, and cover variables for test so that we can override them in
 # the terminal more easily.
-pkgs := $(shell go list ./...)
+
+# IGNORE_DIRS is a list of directories to ignore when running tests and linters.
+# This list is space separated.
+IGNORE_DIRS ?= third_party
+pkgs := $(shell go list ./... | grep -vE "$(IGNORE_DIRS)")
 run := .
 count := 1
 

block/block_cache.go

@@ -53,6 +53,12 @@ func (bc *BlockCache) setSeen(hash string) {
 	bc.hashes[hash] = true
 }
 
+func (bc *BlockCache) isHardConfirmed(hash string) bool {
+	bc.mtx.RLock()
+	defer bc.mtx.RUnlock()
+	return bc.hardConfirmations[hash]
+}
+
 func (bc *BlockCache) setHardConfirmed(hash string) {
 	bc.mtx.Lock()
 	defer bc.mtx.Unlock()

block/block_cache_test.go

@@ -0,0 +1,46 @@
+package block
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/rollkit/rollkit/types"
+)
+
+func TestBlockCache(t *testing.T) {
+	require := require.New(t)
+	// Create new BlockCache and verify not nil
+	bc := NewBlockCache()
+	require.NotNil(bc)
+
+	// Test setBlock and getBlock
+	height, nTxs := uint64(1), 2
+	block := types.GetRandomBlock(height, nTxs)
+	bc.setBlock(height, block)
+	gotBlock, ok := bc.getBlock(height)
+	require.True(ok, "getBlock should return true after setBlock")
+	require.Equal(block, gotBlock)
+
+	// Test overwriting a block
+	block1 := types.GetRandomBlock(height, nTxs)
+	bc.setBlock(height, block1)
+	gotBlock1, ok1 := bc.getBlock(height)
+	require.True(ok1, "getBlock should return true after overwriting a block")
+	require.Equal(block1, gotBlock1)
+
+	// Test deleteBlock
+	bc.deleteBlock(height)
+	_, ok = bc.getBlock(height)
+	require.False(ok, "getBlock should return false after deleteBlock")
+
+	// Test isSeen and setSeen
+	require.False(bc.isSeen("hash"), "isSeen should return false for unseen hash")
+	bc.setSeen("hash")
+	require.True(bc.isSeen("hash"), "isSeen should return true for seen hash")
+
+	// Test setHardConfirmed
+	require.False(bc.isHardConfirmed("hash"), "hardConfirmations should be false for unseen hash")
+	bc.setHardConfirmed("hash")
+	require.True(bc.isHardConfirmed("hash"), "hardConfirmations should be true for seen hash")
+}

block/header-sync.md

@@ -14,7 +14,7 @@ The nodes in the P2P network sync headers using the header sync service that imp
 
 ## Details
 
-All three types of nodes (sequencer, full, and light) run the header sync service to maintain the cannonical view of the rollup chain (with respect to the P2P network).
+All three types of nodes (sequencer, full, and light) run the header sync service to maintain the canonical view of the rollup chain (with respect to the P2P network).
 
 The header sync service inherits the `ConnectionGater` from the node's P2P client which enables blocking and allowing peers as needed by specifying the `P2PConfig.BlockedPeers` and `P2PConfig.AllowedPeers`.
 

block/manager.go

@@ -207,6 +207,11 @@ func (m *Manager) GetStoreHeight() uint64 {
 	return m.store.Height()
 }
 
+// GetHardConfirmation returns true if the block is hard confirmed
+func (m *Manager) GetHardConfirmation(hash types.Hash) bool {
+	return m.blockCache.isHardConfirmed(hash.String())
+}
+
 // AggregationLoop is responsible for aggregating transactions into rollup-blocks.
 func (m *Manager) AggregationLoop(ctx context.Context, lazy bool) {
 	initialHeight := uint64(m.genesis.InitialHeight)
@@ -459,18 +464,29 @@ func (m *Manager) getBlocksFromBlockStore(ctx context.Context, startHeight, endH
 
 // RetrieveLoop is responsible for interacting with DA layer.
 func (m *Manager) RetrieveLoop(ctx context.Context) {
+	// blockFoundCh is used to track when we successfully found a block so
+	// that we can continue to try and find blocks that are in the next DA height.
+	// This enables syncing faster than the DA block time.
+	blockFoundCh := make(chan struct{}, 1)
+	defer close(blockFoundCh)
 	for {
 		select {
 		case <-ctx.Done():
 			return
 		case <-m.retrieveCh:
+		case <-blockFoundCh:
 		}
 		daHeight := atomic.LoadUint64(&m.daHeight)
 		err := m.processNextDABlock(ctx)
 		if err != nil {
 			m.logger.Error("failed to retrieve block from DALC", "daHeight", daHeight, "errors", err.Error())
 			continue
 		}
+		// Signal the blockFoundCh to try and retrieve the next block
+		select {
+		case blockFoundCh <- struct{}{}:
+		default:
+		}
 		atomic.AddUint64(&m.daHeight, 1)
 	}
 }
@@ -699,6 +715,13 @@ func (m *Manager) publishBlock(ctx context.Context) error {
 		return err
 	}
 
+	// Check if the node has shutdown prior to publishing to channels
+	select {
+	case <-ctx.Done():
+		return nil
+	default:
+	}
+
 	// Publish header to channel so that header exchange service can broadcast
 	m.HeaderCh <- &block.SignedHeader
 

block/manager_test.go

@@ -102,3 +102,21 @@ func getMockDALC(logger log.Logger) da.DataAvailabilityLayerClient {
 	_ = dalc.Start()
 	return dalc
 }
+
+func TestGetHardConfirmation(t *testing.T) {
+	require := require.New(t)
+
+	// Create a minimalistic block manager
+	m := &Manager{
+		blockCache: NewBlockCache(),
+	}
+	hash := types.Hash([]byte("hash"))
+
+	// GetHardConfirmation should return false for unseen hash
+	require.False(m.GetHardConfirmation(hash))
+
+	// Set the hash as hard confirmed and verify GetHardConfirmation returns
+	// true
+	m.blockCache.setHardConfirmed(hash.String())
+	require.True(m.GetHardConfirmation(hash))
+}

buf.gen.yaml

@@ -4,7 +4,7 @@ version: v1beta1
 plugins:
   # The name of the plugin.
   - name: gogofaster
-     # The the relative output directory.
-    out: proto/pb
+     # The relative output directory.
+    out: types/pb
     # Any options to provide to the plugin.
     opt: plugins=grpc,paths=source_relative

da/grpc/grpc.go

@@ -5,9 +5,9 @@ import (
 	"encoding/json"
 	"strconv"
 
-	"google.golang.org/grpc"
-
 	ds "github.com/ipfs/go-datastore"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials/insecure"
 
 	"github.com/rollkit/rollkit/da"
 	"github.com/rollkit/rollkit/third_party/log"
@@ -58,7 +58,7 @@ func (d *DataAvailabilityLayerClient) Start() error {
 	var err error
 	var opts []grpc.DialOption
 	// TODO(tzdybal): add more options
-	opts = append(opts, grpc.WithInsecure())
+	opts = append(opts, grpc.WithTransportCredentials(insecure.NewCredentials()))
 	d.conn, err = grpc.Dial(d.config.Host+":"+strconv.Itoa(d.config.Port), opts...)
 	if err != nil {
 		return err

da/test/da_test.go

@@ -29,6 +29,8 @@ import (
 	"github.com/rollkit/rollkit/types"
 )
 
+const mockDaBlockTime = 100 * time.Millisecond
+
 var (
 	testNamespaceID = types.NamespaceID{0, 1, 2, 3, 4, 5, 6, 7}
 
@@ -163,7 +165,7 @@ func doTestRetrieve(t *testing.T, dalc da.DataAvailabilityLayerClient) {
 	for i := uint64(0); i < numBatches; i++ {
 		blocks := make([]*types.Block, blocksSubmittedPerBatch)
 		for j := 0; j < len(blocks); j++ {
-			blocks[j] = getRandomBlock(i*numBatches+uint64(j), rand.Int()%20) //nolint:gosec
+			blocks[j] = types.GetRandomBlock(i*numBatches+uint64(j), rand.Int()%20) //nolint:gosec
 		}
 		resp := dalc.SubmitBlocks(ctx, blocks)
 		assert.Equal(da.StatusSuccess, resp.Code, resp.Message)