Database History migration tool

.github/workflows/docker-build-scan.yaml

@@ -1,92 +1,50 @@
 name: Docker Build Scan
 on:
+  pull_request:
+    branches:
+      - "master"
+      - "celo*"
   workflow_dispatch:
 
 jobs:
-  Build-Scan-Container-op-ufm:
-    uses: celo-org/reusable-workflows/.github/workflows/container-cicd-local.yaml@v1.11.2
-    with:
-      dockerfile: op-ufm/Dockerfile
-
-  Build-Scan-Container-ops-bedrock-l1:
-    uses: celo-org/reusable-workflows/.github/workflows/container-cicd-local.yaml@v1.11.2
-    with:
-      dockerfile: ops-bedrock/Dockerfile.l1
-      context: ops-bedrock
-
-  Build-Scan-Container-ops-bedrock-l2:
-    uses: celo-org/reusable-workflows/.github/workflows/container-cicd-local.yaml@v1.11.2
-    with:
-      dockerfile: ops-bedrock/Dockerfile.l2
-      context: ops-bedrock
-
-  Build-Scan-Container-indexer:
-    uses: celo-org/reusable-workflows/.github/workflows/container-cicd-local.yaml@v1.11.2
-    with:
-      dockerfile: indexer/Dockerfile
-
-  Build-Scan-Container-op-heartbeat:
-    uses: celo-org/reusable-workflows/.github/workflows/container-cicd-local.yaml@v1.11.2
-    with:
-      dockerfile: op-heartbeat/Dockerfile
-
-  Build-Scan-Container-op-exporter:
-    uses: celo-org/reusable-workflows/.github/workflows/container-cicd-local.yaml@v1.11.2
-    with:
-      dockerfile: op-exporter/Dockerfile
-
-  Build-Scan-Container-op-program:
-    uses: celo-org/reusable-workflows/.github/workflows/container-cicd-local.yaml@v1.11.2
-    with:
-      dockerfile: op-program/Dockerfile
-
-  Build-Scan-Container-ops-bedrock:
-    uses: celo-org/reusable-workflows/.github/workflows/container-cicd-local.yaml@v1.11.2
-    with:
-      dockerfile: ops-bedrock/Dockerfile.stateviz
-
-  Build-Scan-Container-ci-builder:
-    uses: celo-org/reusable-workflows/.github/workflows/container-cicd-local.yaml@v1.11.2
-    with:
-      dockerfile: ops/docker/ci-builder/Dockerfile
-
-  Build-Scan-Container-proxyd:
-    uses: celo-org/reusable-workflows/.github/workflows/container-cicd-local.yaml@v1.11.2
-    with:
-      dockerfile: proxyd/Dockerfile
-
-  Build-Scan-Container-op-node:
-    uses: celo-org/reusable-workflows/.github/workflows/container-cicd-local.yaml@v1.11.2
-    with:
-      dockerfile: op-node/Dockerfile
-
-  Build-Scan-Container-op-batcher:
-    uses: celo-org/reusable-workflows/.github/workflows/container-cicd-local.yaml@v1.11.2
-    with:
-      dockerfile: op-batcher/Dockerfile
-
-  Build-Scan-Container-indexer-ui:
-    uses: celo-org/reusable-workflows/.github/workflows/container-cicd-local.yaml@v1.11.2
-    with:
-      dockerfile: indexer/ui/Dockerfile
-
-  Build-Scan-Container-op-proposer:
-    uses: celo-org/reusable-workflows/.github/workflows/container-cicd-local.yaml@v1.11.2
-    with:
-      dockerfile: op-proposer/Dockerfile
-
-  Build-Scan-Container-op-challenger:
-    uses: celo-org/reusable-workflows/.github/workflows/container-cicd-local.yaml@v1.11.2
-    with:
-      dockerfile: op-challenger/Dockerfile
-
-  Build-Scan-Container-endpoint-monitor:
-    uses: celo-org/reusable-workflows/.github/workflows/container-cicd-local.yaml@v1.11.2
-    with:
-      dockerfile: endpoint-monitor/Dockerfile
-
-  Build-Scan-Container-opwheel:
-    uses: celo-org/reusable-workflows/.github/workflows/container-cicd-local.yaml@v1.11.2
-    with:
-      dockerfile: op-wheel/Dockerfile
-
+  detect-files-changed:
+    runs-on: ubuntu-latest
+    outputs:
+      files-changed: ${{ steps.detect-files-changed.outputs.all_changed_files }}
+    steps:
+      - uses: actions/checkout@v4
+      - name: Detect files changed
+        id: detect-files-changed
+        uses: tj-actions/changed-files@v44
+        with:
+          separator: ','
+
+  build-cel2-migration-tool:
+    runs-on: ubuntu-latest
+    needs: detect-files-changed
+    if: |
+      contains(needs.detect-files-changed.outputs.files-changed, 'op-chain-ops/cmd/celo-dbmigrate') ||
+      contains(needs.detect-files-changed.outputs.files-changed, 'op-chain-ops/cmd/celo-migrate') ||
+      contains(needs.detect-files-changed.outputs.files-changed, 'op-chain-ops/Dockerfile')
+    permissions:
+      contents: read
+      id-token: write
+      security-events: write
+    steps:
+      - uses: actions/checkout@v4
+      - name: Login at GCP Artifact Registry
+        uses: celo-org/reusable-workflows/.github/actions/auth-gcp-artifact-registry@v2.0
+        with:
+          workload-id-provider: 'projects/1094498259535/locations/global/workloadIdentityPools/gh-optimism/providers/github-by-repos'
+          service-account: 'celo-optimism-gh@devopsre.iam.gserviceaccount.com'
+          docker-gcp-registries: us-west1-docker.pkg.dev
+      - name: Build and push container
+        uses: celo-org/reusable-workflows/.github/actions/build-container@v2.0
+        with:
+          platforms: linux/amd64
+          registry: us-west1-docker.pkg.dev/devopsre/dev-images/cel2-migration-tool
+          tags: ${{ github.sha }}
+          context: ./
+          dockerfile: ./op-chain-ops/Dockerfile
+          push: true
+          trivy: false

go.mod

@@ -219,7 +219,9 @@ require (
 	rsc.io/tmplfunc v0.0.3 // indirect
 )
 
-replace github.com/ethereum/go-ethereum => github.com/celo-org/op-geth v0.0.0-20240220150757-6c346a984571
+//replace github.com/ethereum/go-ethereum => github.com/celo-org/op-geth v0.0.0-20240220150757-6c346a984571
 
 //replace github.com/ethereum-optimism/superchain-registry/superchain => ../superchain-registry/superchain
-//replace github.com/ethereum/go-ethereum v1.13.5 => ../go-ethereum
+replace github.com/ethereum/go-ethereum => github.com/celo-org/op-geth v0.0.0-20240509214009-57109af77703
+
+//replace github.com/ethereum/go-ethereum => ../op-geth

go.sum

@@ -75,8 +75,8 @@ github.com/btcsuite/snappy-go v1.0.0/go.mod h1:8woku9dyThutzjeg+3xrA5iCpBRH8XEEg
 github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtEyQwv5/p4Mg4C0fgbePVuGr935/5ddU9Z3TmDRY=
 github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46fmI40EZs=
 github.com/buger/jsonparser v0.0.0-20181115193947-bf1c66bbce23/go.mod h1:bbYlZJ7hK1yFx9hf58LP0zeX7UjIGs20ufpu3evjr+s=
-github.com/celo-org/op-geth v0.0.0-20240220150757-6c346a984571 h1:U+6eOYz2hEun/zm+bGCF9ntKx6ZzxBW9lojOJ4ufybI=
-github.com/celo-org/op-geth v0.0.0-20240220150757-6c346a984571/go.mod h1:dQVCa+D5zi0oJJyrP3nwWciio0BJLZ2urH8OU7RJ2qs=
+github.com/celo-org/op-geth v0.0.0-20240509214009-57109af77703 h1:BU8PvKoZ4alcn9SjQVc3nkrF+sUGThQ1VfWkhZig5IM=
+github.com/celo-org/op-geth v0.0.0-20240509214009-57109af77703/go.mod h1:dQVCa+D5zi0oJJyrP3nwWciio0BJLZ2urH8OU7RJ2qs=
 github.com/cespare/cp v0.1.0 h1:SE+dxFebS7Iik5LK0tsi1k9ZCxEaFX4AjQmoyA+1dJk=
 github.com/cespare/cp v0.1.0/go.mod h1:SOGHArjBr4JWaSDEVpWpo/hNg6RoKrls6Oh40hiwW+s=
 github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=

op-chain-ops/Dockerfile

@@ -0,0 +1,30 @@
+FROM golang:1.21.1-alpine3.18 as builder
+
+RUN apk --no-cache add make
+
+COPY ./go.mod /app/go.mod
+COPY ./go.sum /app/go.sum
+
+WORKDIR /app
+
+RUN go mod download
+
+COPY ./op-bindings /app/op-bindings
+COPY ./op-service /app/op-service
+COPY ./op-node /app/op-node
+COPY ./op-chain-ops /app/op-chain-ops
+WORKDIR /app/op-chain-ops
+RUN make celo-dbmigrate celo-migrate
+
+FROM alpine:3.18
+RUN apk --no-cache add ca-certificates bash rsync
+
+# RUN addgroup -S app && adduser -S app -G app
+# USER app
+WORKDIR /app
+
+COPY --from=builder /app/op-chain-ops/bin/celo-dbmigrate /app
+COPY --from=builder /app/op-chain-ops/bin/celo-migrate /app
+ENV PATH="/app:${PATH}"
+
+ENTRYPOINT ["/app/celo-dbmigrate"]

op-chain-ops/Makefile

@@ -7,6 +7,9 @@ ecotone-scalar:
 receipt-reference-builder:
 	go build -o ./bin/receipt-reference-builder ./cmd/receipt-reference-builder/*.go
 
+celo-dbmigrate:
+	go build -o ./bin/celo-dbmigrate ./cmd/celo-dbmigrate/*.go
+
 celo-migrate:
 	go build -o ./bin/celo-migrate ./cmd/celo-migrate/*.go
 

op-chain-ops/cmd/celo-dbmigrate/ancients.go

@@ -0,0 +1,191 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"path/filepath"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/core/rawdb"
+	"github.com/ethereum/go-ethereum/ethdb"
+	"github.com/ethereum/go-ethereum/log"
+	"golang.org/x/sync/errgroup"
+)
+
+// RLPBlockRange is a range of blocks in RLP format
+type RLPBlockRange struct {
+	start    uint64
+	hashes   [][]byte
+	headers  [][]byte
+	bodies   [][]byte
+	receipts [][]byte
+	tds      [][]byte
+}
+
+func migrateAncientsDb(oldDBPath, newDBPath string, batchSize uint64) (uint64, error) {
+	oldFreezer, err := rawdb.NewChainFreezer(filepath.Join(oldDBPath, "ancient"), "", false) // TODO can't be readonly because we need the .meta files to be created
+	if err != nil {
+		return 0, fmt.Errorf("failed to open old freezer: %v", err)
+	}
+	defer oldFreezer.Close()
+
+	newFreezer, err := rawdb.NewChainFreezer(filepath.Join(newDBPath, "ancient"), "", false)
+	if err != nil {
+		return 0, fmt.Errorf("failed to open new freezer: %v", err)
+	}
+	defer newFreezer.Close()
+
+	numAncientsOld, err := oldFreezer.Ancients()
+	if err != nil {
+		return 0, fmt.Errorf("failed to get number of ancients in old freezer: %v", err)
+	}
+
+	numAncientsNew, err := newFreezer.Ancients()
+	if err != nil {
+		return 0, fmt.Errorf("failed to get number of ancients in new freezer: %v", err)
+	}
+
+	log.Info("Migration Started", "process", "ancients migration", "startBlock", numAncientsNew, "endBlock", numAncientsOld, "count", numAncientsOld-numAncientsNew+1)
+	g, ctx := errgroup.WithContext(context.Background())
+	readChan := make(chan RLPBlockRange, 10)
+	transformChan := make(chan RLPBlockRange, 10)
+
+	log.Info("Migrating data", "start", numAncientsNew, "end", numAncientsOld, "step", batchSize)
+
+	g.Go(func() error {
+		return readAncientBlocks(ctx, oldFreezer, numAncientsNew, numAncientsOld, batchSize, readChan)
+	})
+	g.Go(func() error { return transformBlocks(ctx, readChan, transformChan) })
+	g.Go(func() error { return writeAncientBlocks(ctx, newFreezer, transformChan) })
+
+	if err = g.Wait(); err != nil {
+		return 0, fmt.Errorf("failed to migrate ancients: %v", err)
+	}
+
+	numAncientsNew, err = newFreezer.Ancients()
+	if err != nil {
+		return 0, fmt.Errorf("failed to get number of ancients in new freezer: %v", err)
+	}
+
+	log.Info("Migration End", "process", "ancients migration", "totalBlocks", numAncientsNew)
+	return numAncientsNew, nil
+}
+
+func readAncientBlocks(ctx context.Context, freezer *rawdb.Freezer, startBlock, endBlock, batchSize uint64, out chan<- RLPBlockRange) error {
+	defer close(out)
+
+	for i := startBlock; i < endBlock; i += batchSize {
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		default:
+			count := min(batchSize, endBlock-i+1)
+			start := i
+
+			blockRange := RLPBlockRange{
+				start:    start,
+				hashes:   make([][]byte, count),
+				headers:  make([][]byte, count),
+				bodies:   make([][]byte, count),
+				receipts: make([][]byte, count),
+				tds:      make([][]byte, count),
+			}
+			var err error
+
+			blockRange.hashes, err = freezer.AncientRange(rawdb.ChainFreezerHashTable, start, count, 0)
+			if err != nil {
+				return fmt.Errorf("failed to read hashes from old freezer: %v", err)
+			}
+			blockRange.headers, err = freezer.AncientRange(rawdb.ChainFreezerHeaderTable, start, count, 0)
+			if err != nil {
+				return fmt.Errorf("failed to read headers from old freezer: %v", err)
+			}
+			blockRange.bodies, err = freezer.AncientRange(rawdb.ChainFreezerBodiesTable, start, count, 0)
+			if err != nil {
+				return fmt.Errorf("failed to read bodies from old freezer: %v", err)
+			}
+			blockRange.receipts, err = freezer.AncientRange(rawdb.ChainFreezerReceiptTable, start, count, 0)
+			if err != nil {
+				return fmt.Errorf("failed to read receipts from old freezer: %v", err)
+			}
+			blockRange.tds, err = freezer.AncientRange(rawdb.ChainFreezerDifficultyTable, start, count, 0)
+			if err != nil {
+				return fmt.Errorf("failed to read tds from old freezer: %v", err)
+			}
+
+			out <- blockRange
+		}
+	}
+	return nil
+}
+
+func transformBlocks(ctx context.Context, in <-chan RLPBlockRange, out chan<- RLPBlockRange) error {
+	// Transform blocks from the in channel and send them to the out channel
+	defer close(out)
+	for blockRange := range in {
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		default:
+			for i := range blockRange.hashes {
+				blockNumber := blockRange.start + uint64(i)
+
+				newHeader, err := transformHeader(blockRange.headers[i])
+				if err != nil {
+					return fmt.Errorf("can't transform header: %v", err)
+				}
+				newBody, err := transformBlockBody(blockRange.bodies[i])
+				if err != nil {
+					return fmt.Errorf("can't transform body: %v", err)
+				}
+
+				if yes, newHash := hasSameHash(newHeader, blockRange.hashes[i]); !yes {
+					log.Error("Hash mismatch", "block", blockNumber, "oldHash", common.BytesToHash(blockRange.hashes[i]), "newHash", newHash)
+					return fmt.Errorf("hash mismatch at block %d", blockNumber)
+				}
+
+				blockRange.headers[i] = newHeader
+				blockRange.bodies[i] = newBody
+			}
+			out <- blockRange
+		}
+	}
+	return nil
+}
+
+func writeAncientBlocks(ctx context.Context, freezer *rawdb.Freezer, in <-chan RLPBlockRange) error {
+	// Write blocks from the in channel to the newDb
+	for blockRange := range in {
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		default:
+			_, err := freezer.ModifyAncients(func(aWriter ethdb.AncientWriteOp) error {
+				for i := range blockRange.hashes {
+					blockNumber := blockRange.start + uint64(i)
+					if err := aWriter.AppendRaw(rawdb.ChainFreezerHashTable, blockNumber, blockRange.hashes[i]); err != nil {
+						return fmt.Errorf("can't write hash to Freezer: %v", err)
+					}
+					if err := aWriter.AppendRaw(rawdb.ChainFreezerHeaderTable, blockNumber, blockRange.headers[i]); err != nil {
+						return fmt.Errorf("can't write header to Freezer: %v", err)
+					}
+					if err := aWriter.AppendRaw(rawdb.ChainFreezerBodiesTable, blockNumber, blockRange.bodies[i]); err != nil {
+						return fmt.Errorf("can't write body to Freezer: %v", err)
+					}
+					if err := aWriter.AppendRaw(rawdb.ChainFreezerReceiptTable, blockNumber, blockRange.receipts[i]); err != nil {
+						return fmt.Errorf("can't write receipts to Freezer: %v", err)
+					}
+					if err := aWriter.AppendRaw(rawdb.ChainFreezerDifficultyTable, blockNumber, blockRange.tds[i]); err != nil {
+						return fmt.Errorf("can't write td to Freezer: %v", err)
+					}
+				}
+				return nil
+			})
+			if err != nil {
+				return fmt.Errorf("failed to write block range: %v", err)
+			}
+			log.Info("Wrote ancient blocks", "start", blockRange.start, "end", blockRange.start+uint64(len(blockRange.hashes)-1), "count", len(blockRange.hashes))
+		}
+	}
+	return nil
+}