feat: build: refactor actor bundling system

1. Include the builtin-actors in the lotus source tree.
2. Embed the bundle on build instead of downloading at runtime.
3. Avoid reading the bundle whenever possible by including bundle
   metadata (the bundle CID, the actor CIDs, etc.).
4. Remove everything related to dependency injection.
    1. We're no longer downloading the bundle, so doing anything ahead
       of time doesn't really help.
    2. We register the manifests on init because, unfortunately, they're
       global.
    3. We explicitly load the current actors bundle in the genesis
       state-tree method.
    4. For testing, we just change the in-use bundle with a bit of a
       hack. It's not great, but using dependency injection doesn't make
       any sense either because, again, the manifest information is
       global.

fixes #8701

Makefile

@@ -295,6 +295,12 @@ actors-gen:
 	$(GOCC) run ./chain/actors/agen
 	$(GOCC) fmt ./...
 
+bundle-gen:
+	$(GOCC) run ./gen/bundle
+	$(GOCC) fmt ./build/...
+.PHONY: bundle-gen
+
+
 api-gen:
 	$(GOCC) run ./gen/api
 	goimports -w api
@@ -343,7 +349,7 @@ docsgen-openrpc-gateway: docsgen-openrpc-bin
 
 .PHONY: docsgen docsgen-md-bin docsgen-openrpc-bin
 
-gen: actors-gen type-gen method-gen cfgdoc-gen docsgen api-gen circleci
+gen: actors-gen type-gen method-gen cfgdoc-gen docsgen api-gen circleci bundle-gen
 	@echo ">>> IF YOU'VE MODIFIED THE CLI OR CONFIG, REMEMBER TO ALSO MAKE docsgen-cli"
 .PHONY: gen
 

build/actors/README.md

@@ -0,0 +1,18 @@
+# Bundles
+
+This directory includes the actors bundles for each release. Each actor bundle is a zstd compressed
+tarfile containing one bundle per network type. These tarfiles are subsequently embedded in the
+lotus binary.
+
+## Updating
+
+To update, run the `./pack.sh` script. For example, the following will pack the builtin actors release `dev/20220602` into the `v8` tarfile.
+
+```bash
+./pack.sh v8 dev/20220602
+```
+
+This will:
+
+1. Download the actors bundles and pack them into the appropriate tarfile.
+2. Run `make bundle-gen` in the top-level directory to regenerate the bundle metadata file.

build/actors/pack.sh

@@ -0,0 +1,39 @@
+#!/bin/bash
+
+set -e
+
+if [[ $# -ne 2 ]]; then
+    echo "expected two arguments, an actors version (e.g., v8) and an actors release"
+    exit 1
+fi
+
+VERSION="$1" # actors version
+RELEASE="$2" # actors release name
+NETWORKS=(devnet mainnet caterpillarnet butterflynet testing testing-fake-proofs)
+
+echo "Downloading bundles for actors version ${VERSION}, release ${RELEASE}"
+
+TARGET_FILE="$(pwd)/${VERSION}.tar.zst"
+WORKDIR=$(mktemp --tmpdir -d "actor-bundles-${VERSION}.XXXXXXXXXX")
+trap 'rm -rf -- "$WORKDIR"' EXIT
+
+pushd "${WORKDIR}"
+encoded_release="$(jq -rn --arg release "$RELEASE" '$release | @uri')"
+for network in "${NETWORKS[@]}"; do
+    wget "https://github.com/filecoin-project/builtin-actors/releases/download/${encoded_release}/builtin-actors-${network}"{.car,.sha256}
+done
+
+echo "Checking the checksums..."
+
+sha256sum -c -- *.sha256
+
+
+echo "Packing..."
+
+rm -f -- "$TARGET_FILE"
+tar -cf "$TARGET_FILE" -I "zstd -19" -- *.car
+popd
+
+echo "Generating metadata..."
+
+make -C ../../ bundle-gen

build/builtin_actors.go

@@ -1,27 +1,261 @@
 package build
 
 import (
-	"bytes"
+	"archive/tar"
+	"context"
+	"embed"
+	"fmt"
+	"io"
+	"os"
+	"path"
+	"sort"
+	"strconv"
+	"strings"
 
+	"github.com/filecoin-project/lotus/blockstore"
 	"github.com/filecoin-project/lotus/chain/actors"
+	"github.com/filecoin-project/lotus/chain/actors/adt"
 
-	"github.com/BurntSushi/toml"
+	"github.com/DataDog/zstd"
+	"github.com/ipfs/go-cid"
+	cbor "github.com/ipfs/go-ipld-cbor"
+	"github.com/ipld/go-car"
+	"golang.org/x/xerrors"
 )
 
-var BuiltinActorReleases map[actors.Version]Bundle
+//go:embed actors/*.tar.zst
+var embeddedBuiltinActorReleases embed.FS
 
 func init() {
-	BuiltinActorReleases = make(map[actors.Version]Bundle)
+	if err := loadManifests(NetworkBundle); err != nil {
+		panic(err)
+	}
+}
+
+// UseNetworkBundle switches to a different network bundle, by name.
+func UseNetworkBundle(netw string) error {
+	if NetworkBundle == netw {
+		return nil
+	}
+	if err := loadManifests(netw); err != nil {
+		return err
+	}
+	NetworkBundle = netw
+	return nil
+}
+
+func loadManifests(netw string) error {
+	embedded := make(map[actors.Version]struct{})
+	newMetadata := make([]*BuiltinActorsMetadata, 0, len(BuiltinActorReleases))
+	// First, prefer external bundles and overrides.
+	for av, bd := range BuiltinActorReleases {
+		envvar := fmt.Sprintf("LOTUS_BUILTIN_ACTORS_V%d_BUNDLE", av)
+		var (
+			root      cid.Cid
+			actorCids map[string]cid.Cid
+			err       error
+		)
+		if path := os.Getenv(envvar); path != "" {
+			root, actorCids, err = readBundleManifestFromFile(path)
+		} else if path = bd.Path[netw]; path != "" {
+			root, actorCids, err = readBundleManifestFromFile(path)
+		} else {
+			embedded[av] = struct{}{}
+			continue
+		}
+		if err != nil {
+			return err
+		}
+		newMetadata = append(newMetadata, &BuiltinActorsMetadata{
+			Network:     netw,
+			Version:     av,
+			ManifestCid: root,
+			Actors:      actorCids,
+		})
+	}
+
+	// Then fallback on embedded bundles
+	for _, meta := range EmbeddedBuiltinActorsMetadata {
+		if meta.Network != netw {
+			continue
+		}
+		if _, ok := embedded[meta.Version]; !ok {
+			continue
+		}
+		delete(embedded, meta.Version)
+		newMetadata = append(newMetadata, meta)
+	}
+
+	if len(embedded) > 0 {
+		return xerrors.Errorf("failed to find some actor bundles: %v", embedded)
+	}
+
+	actors.ClearManifests()
 
-	spec := BundleSpec{}
+	for _, meta := range newMetadata {
+		actors.RegisterManifest(meta.Version, meta.ManifestCid, meta.Actors)
+	}
+
+	return nil
+}
 
-	r := bytes.NewReader(BuiltinActorBundles)
-	_, err := toml.DecodeReader(r, &spec)
+type BuiltinActorsMetadata struct {
+	Network     string
+	Version     actors.Version
+	ManifestCid cid.Cid
+	Actors      map[string]cid.Cid
+}
+
+// ReadEmbeddedBuiltinActorsMetadata reads the metadata from the embedded built-in actor bundles.
+// There should be no need to call this method as the result is cached in the
+// `EmbeddedBuiltinActorsMetadata` variable on `make gen`.
+func ReadEmbeddedBuiltinActorsMetadata() ([]*BuiltinActorsMetadata, error) {
+	files, err := embeddedBuiltinActorReleases.ReadDir("actors")
 	if err != nil {
-		panic(err)
+		return nil, xerrors.Errorf("failed to read embedded bundle directory: %s", err)
+	}
+	var bundles []*BuiltinActorsMetadata
+	for _, dirent := range files {
+		name := dirent.Name()
+		b, err := readEmbeddedBuiltinActorsMetadata(name)
+		if err != nil {
+			return nil, err
+		}
+		bundles = append(bundles, b...)
+	}
+	// Sort by network, then by bundle.
+	sort.Slice(bundles, func(i, j int) bool {
+		if bundles[i].Network == bundles[j].Network {
+			return bundles[i].Version < bundles[j].Version
+		}
+		return bundles[i].Network < bundles[j].Network
+	})
+	return bundles, nil
+}
+
+func readEmbeddedBuiltinActorsMetadata(bundle string) ([]*BuiltinActorsMetadata, error) {
+	const (
+		archiveExt   = ".tar.zst"
+		bundleExt    = ".car"
+		bundlePrefix = "builtin-actors-"
+	)
+
+	if !strings.HasPrefix(bundle, "v") {
+		return nil, xerrors.Errorf("bundle bundle '%q' doesn't start with a 'v'", bundle)
+	}
+	if !strings.HasSuffix(bundle, archiveExt) {
+		return nil, xerrors.Errorf("bundle bundle '%q' doesn't end with '%s'", bundle, archiveExt)
+	}
+	version, err := strconv.ParseInt(bundle[1:len(bundle)-len(archiveExt)], 10, 0)
+	if err != nil {
+		return nil, xerrors.Errorf("failed to parse actors version from bundle '%q': %s", bundle, err)
 	}
+	fi, err := embeddedBuiltinActorReleases.Open(fmt.Sprintf("actors/%s", bundle))
+	if err != nil {
+		return nil, err
+	}
+	defer fi.Close() //nolint
+
+	uncompressed := zstd.NewReader(fi)
+	defer uncompressed.Close() //nolint
+
+	var bundles []*BuiltinActorsMetadata
+
+	tarReader := tar.NewReader(uncompressed)
+	for {
+		header, err := tarReader.Next()
+		switch err {
+		case io.EOF:
+			return bundles, nil
+		case nil:
+		default:
+			return nil, err
+		}
+
+		// Read the network name from the bundle name.
+		name := path.Base(header.Name)
+		if !strings.HasSuffix(name, bundleExt) {
+			return nil, xerrors.Errorf("expected bundle to end with .car: %s", name)
+		}
+		if !strings.HasPrefix(name, bundlePrefix) {
+			return nil, xerrors.Errorf("expected bundle to end with .car: %s", name)
+		}
+		name = name[len(bundlePrefix) : len(name)-len(bundleExt)]
+
+		// Load the bundle.
+		root, actorCids, err := readBundleManifest(tarReader)
+		if err != nil {
+			return nil, xerrors.Errorf("error loading builtin actors bundle: %w", err)
+		}
+		bundles = append(bundles, &BuiltinActorsMetadata{
+			Network:     name,
+			Version:     actors.Version(version),
+			ManifestCid: root,
+			Actors:      actorCids,
+		})
+	}
+}
+
+func readBundleManifestFromFile(path string) (cid.Cid, map[string]cid.Cid, error) {
+	fi, err := os.Open(path)
+	if err != nil {
+		return cid.Undef, nil, err
+	}
+	defer fi.Close() //nolint
+
+	return readBundleManifest(fi)
+}
+
+func readBundleManifest(r io.Reader) (cid.Cid, map[string]cid.Cid, error) {
+	// Load the bundle.
+	bs := blockstore.NewMemory()
+	hdr, err := car.LoadCar(context.Background(), bs, r)
+	if err != nil {
+		return cid.Undef, nil, xerrors.Errorf("error loading builtin actors bundle: %w", err)
+	}
+
+	if len(hdr.Roots) != 1 {
+		return cid.Undef, nil, xerrors.Errorf("expected one root when loading actors bundle, got %d", len(hdr.Roots))
+	}
+	root := hdr.Roots[0]
+	actorCids, err := actors.ReadManifest(context.Background(), adt.WrapStore(context.Background(), cbor.NewCborStore(bs)), root)
+	if err != nil {
+		return cid.Undef, nil, err
+	}
+
+	return root, actorCids, nil
+}
+
+// GetEmbeddedBuiltinActorsBundle returns the builtin-actors bundle for the given actors version.
+func GetEmbeddedBuiltinActorsBundle(version actors.Version) ([]byte, bool) {
+	fi, err := embeddedBuiltinActorReleases.Open(fmt.Sprintf("actors/v%d.tar.zst", version))
+	if err != nil {
+		return nil, false
+	}
+	defer fi.Close() //nolint
+
+	uncompressed := zstd.NewReader(fi)
+	defer uncompressed.Close() //nolint
+
+	tarReader := tar.NewReader(uncompressed)
+	targetFileName := fmt.Sprintf("builtin-actors-%s.car", NetworkBundle)
+	for {
+		header, err := tarReader.Next()
+		switch err {
+		case io.EOF:
+			return nil, false
+		case nil:
+		default:
+			panic(err)
+		}
+		if header.Name != targetFileName {
+			continue
+		}
 
-	for _, b := range spec.Bundles {
-		BuiltinActorReleases[b.Version] = b
+		car, err := io.ReadAll(tarReader)
+		if err != nil {
+			panic(err)
+		}
+		return car, true
 	}
 }