Publish stable checkpoint periodically to Redis

cmd/rekor-server/app/root.go

@@ -80,6 +80,9 @@ func init() {
 	rootCmd.PersistentFlags().Uint("trillian_log_server.tlog_id", 0, "Trillian tree id")
 	rootCmd.PersistentFlags().String("trillian_log_server.sharding_config", "", "path to config file for inactive shards, in JSON or YAML")
 
+	rootCmd.PersistentFlags().Bool("enable_stable_checkpoint", true, "publish stable checkpoints to Redis. When disabled, gossiping may not be possible if the log checkpoint updates too frequently")
+	rootCmd.PersistentFlags().Uint("publish_frequency", 5, "how often to publish a new checkpoint, in minutes")
+
 	hostname, err := os.Hostname()
 	if err != nil {
 		hostname = "localhost"

docker-compose.yml

@@ -92,6 +92,7 @@ services:
       "--rekor_server.signer=memory",
       "--enable_attestation_storage",
       "--attestation_storage_bucket=file:///var/run/attestations",
+      "--enable_stable_checkpoint",
       # Uncomment this for production logging
       # "--log_type=prod",
       ]

go.mod

@@ -53,6 +53,7 @@ require (
 require (
 	github.com/AdamKorcz/go-fuzz-headers-1 v0.0.0-20230329111138-12e09aba5ebd
 	github.com/cyberphone/json-canonicalization v0.0.0-20220623050100-57a0ce2678a7
+	github.com/go-redis/redismock/v9 v9.0.3
 	github.com/golang/mock v1.6.0
 	github.com/hashicorp/go-cleanhttp v0.5.2
 	github.com/hashicorp/go-retryablehttp v0.7.2
@@ -64,6 +65,7 @@ require (
 	cloud.google.com/go/compute/metadata v0.2.3 // indirect
 	filippo.io/edwards25519 v1.0.0 // indirect
 	github.com/cyphar/filepath-securejoin v0.2.3 // indirect
+	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
 	github.com/go-logr/logr v1.2.3 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect

go.sum

@@ -1024,6 +1024,8 @@ github.com/go-playground/validator/v10 v10.2.0/go.mod h1:uOYAAleCW8F/7oMFd6aG0GO
 github.com/go-playground/validator/v10 v10.4.1/go.mod h1:nlOn6nFhuKACm19sB/8EGNn9GlaMV7XkbRSipzJ0Ii4=
 github.com/go-playground/validator/v10 v10.13.0 h1:cFRQdfaSMCOSfGCCLB20MHvuoHb/s5G8L5pu2ppK5AQ=
 github.com/go-playground/validator/v10 v10.13.0/go.mod h1:dwu7+CG8/CtBiJFZDz4e+5Upb6OLw04gtBYw0mcG/z4=
+github.com/go-redis/redismock/v9 v9.0.3 h1:mtHQi2l51lCmXIbTRTqb1EiHYe9tL5Yk5oorlSJJqR0=
+github.com/go-redis/redismock/v9 v9.0.3/go.mod h1:F6tJRfnU8R/NZ0E+Gjvoluk14MqMC5ueSZX6vVQypc0=
 github.com/go-resty/resty/v2 v2.1.1-0.20191201195748-d7b97669fe48/go.mod h1:dZGr0i9PLlaaTD4H/hoZIDjQ+r6xq8mgbRzHZf7f2J8=
 github.com/go-sql-driver/mysql v1.7.0/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI=
 github.com/go-sql-driver/mysql v1.7.1 h1:lUIinVbN1DY0xBg0eMOzmmtGoHwWBbvnWubQUrtU8EI=
@@ -1596,6 +1598,7 @@ github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OS
 github.com/ncw/swift v1.0.47/go.mod h1:23YIA4yWVnGwv2dQlN4bB7egfYX6YLn0Yo/S6zZO/ZM=
 github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
 github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
+github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE=
 github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU=
 github.com/oklog/run v1.1.0/go.mod h1:sVPdnTZT1zYwAJeCMu2Th4T21pA3FPOQRfWjQlk7DVU=
 github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4=
@@ -1615,6 +1618,7 @@ github.com/onsi/ginkgo v1.13.0/go.mod h1:+REjRxOmWfHCjfv9TTWB1jD1Frx4XydAD3zm1ls
 github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY=
 github.com/onsi/ginkgo v1.16.2/go.mod h1:CObGmKUOKaSC0RjmoAK7tKyn4Azo5P2IWuoMnvwxz1E=
 github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0=
+github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE=
 github.com/onsi/ginkgo/v2 v2.1.3/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c=
 github.com/onsi/ginkgo/v2 v2.1.4/go.mod h1:um6tUpWM/cxCK3/FK8BXqEiUMUwRgSM4JXG47RKZmLU=
 github.com/onsi/ginkgo/v2 v2.1.6/go.mod h1:MEH45j8TBi6u9BMogfbp0stKC5cdGjumZj5Y7AG4VIk=
@@ -1637,6 +1641,7 @@ github.com/onsi/gomega v1.20.1/go.mod h1:DtrZpjmvpn2mPm4YWQa0/ALMDj9v4YxLgojwPeR
 github.com/onsi/gomega v1.21.1/go.mod h1:iYAIXgPSaDHak0LCMA+AWBpIKBr8WZicMxnE8luStNc=
 github.com/onsi/gomega v1.22.1/go.mod h1:x6n7VNe4hw0vkyYUM4mjIXx3JbLiPaBPNgB7PRQ1tuM=
 github.com/onsi/gomega v1.23.0/go.mod h1:Z/NWtiqwBrwUt4/2loMmHL63EDLnYHmVbuBpDr2vQAg=
+github.com/onsi/gomega v1.25.0 h1:Vw7br2PCDYijJHSfBOWhov+8cAnUf8MfMaIOV323l6Y=
 github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk=
 github.com/opencontainers/go-digest v0.0.0-20170106003457-a6d0ee40d420/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s=
 github.com/opencontainers/go-digest v0.0.0-20180430190053-c9281466c8b2/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s=
@@ -2925,6 +2930,7 @@ gopkg.in/square/go-jose.v2 v2.6.0 h1:NGk74WTnPKBNUhNzQX7PYcTLUjoq7mzKk2OKbvwk2iI
 gopkg.in/square/go-jose.v2 v2.6.0/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI=
 gopkg.in/telebot.v3 v3.0.0/go.mod h1:7rExV8/0mDDNu9epSrDm/8j22KLaActH1Tbee6YjzWg=
 gopkg.in/telebot.v3 v3.1.2/go.mod h1:GJKwwWqp9nSkIVN51eRKU78aB5f5OnQuWdwiIZfPbko=
+gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
 gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI=
 gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74=

openapi.yaml

@@ -67,6 +67,12 @@ paths:
       operationId: getLogInfo
       tags:
         - tlog
+      parameters:
+        - in: query
+          name: stable
+          type: boolean
+          default: false
+          description: Whether to return a stable checkpoint for the active shard
       responses:
         200:
           description: A JSON object with the root hash and tree size as properties

pkg/api/api.go

@@ -35,6 +35,7 @@ import (
 	"github.com/sigstore/rekor/pkg/signer"
 	"github.com/sigstore/rekor/pkg/storage"
 	"github.com/sigstore/rekor/pkg/trillianclient"
+	"github.com/sigstore/rekor/pkg/witness"
 	"github.com/sigstore/sigstore/pkg/cryptoutils"
 	"github.com/sigstore/sigstore/pkg/signature"
 	"github.com/sigstore/sigstore/pkg/signature/options"
@@ -60,6 +61,8 @@ type API struct {
 	pubkey     string // PEM encoded public key
 	pubkeyHash string // SHA256 hash of DER-encoded public key
 	signer     signature.Signer
+	// stops checkpoint publishing
+	checkpointPublishCancel context.CancelFunc
 }
 
 func NewAPI(treeID uint) (*API, error) {
@@ -134,7 +137,8 @@ func ConfigureAPI(treeID uint) {
 	if err != nil {
 		log.Logger.Panic(err)
 	}
-	if viper.GetBool("enable_retrieve_api") || slices.Contains(viper.GetStringSlice("enabled_api_endpoints"), "searchIndex") {
+	if viper.GetBool("enable_retrieve_api") || viper.GetBool("enable_stable_checkpoint") ||
+		slices.Contains(viper.GetStringSlice("enabled_api_endpoints"), "searchIndex") {
 		redisClient = redis.NewClient(&redis.Options{
 			Addr:    fmt.Sprintf("%v:%v", viper.GetString("redis_server.address"), viper.GetUint64("redis_server.port")),
 			Network: "tcp",
@@ -148,4 +152,18 @@ func ConfigureAPI(treeID uint) {
 			log.Logger.Panic(err)
 		}
 	}
+
+	if viper.GetBool("enable_stable_checkpoint") {
+		checkpointPublisher := witness.NewCheckpointPublisher(context.Background(), api.logClient, api.logRanges.ActiveTreeID(),
+			viper.GetString("rekor_server.hostname"), api.signer, redisClient, viper.GetUint("publish_frequency"), CheckpointPublishCount)
+
+		// create context to cancel goroutine on server shutdown
+		ctx, cancel := context.WithCancel(context.Background())
+		api.checkpointPublishCancel = cancel
+		checkpointPublisher.StartPublisher(ctx)
+	}
+}
+
+func StopAPI() {
+	api.checkpointPublishCancel()
 }

pkg/api/metrics.go

@@ -53,6 +53,11 @@ var (
 		Help: "Api QPS by path, method, and response code",
 	}, []string{"path", "method", "code"})
 
+	CheckpointPublishCount = promauto.NewCounterVec(prometheus.CounterOpts{
+		Name: "rekor_checkpoint_publish",
+		Help: "Checkpoint publishing by shard and code",
+	}, []string{"shard", "code"})
+
 	_ = promauto.NewGaugeFunc(
 		prometheus.GaugeOpts{
 			Namespace: "rekor",

pkg/api/tlog.go

@@ -53,6 +53,38 @@ func GetLogInfoHandler(params tlog.GetLogInfoParams) middleware.Responder {
 		inactiveShards = append(inactiveShards, is)
 	}
 
+	if swag.BoolValue(params.Stable) && redisClient != nil {
+		// key is treeID/latest
+		key := fmt.Sprintf("%d/latest", api.logRanges.ActiveTreeID())
+		redisResult, err := redisClient.Get(params.HTTPRequest.Context(), key).Result()
+		if err != nil {
+			return handleRekorAPIError(params, http.StatusInternalServerError,
+				fmt.Errorf("error getting checkpoint from redis: %w", err), "error getting checkpoint from redis")
+		}
+		// should not occur, a checkpoint should always be present
+		if redisResult == "" {
+			return handleRekorAPIError(params, http.StatusInternalServerError,
+				fmt.Errorf("no checkpoint found in redis: %w", err), "no checkpoint found in redis")
+		}
+		decoded, err := hex.DecodeString(redisResult)
+		if err != nil {
+			return handleRekorAPIError(params, http.StatusInternalServerError,
+				fmt.Errorf("error decoding checkpoint from redis: %w", err), "error decoding checkpoint from redis")
+		}
+		checkpoint := util.SignedCheckpoint{}
+		if err := checkpoint.UnmarshalText(decoded); err != nil {
+			return handleRekorAPIError(params, http.StatusInternalServerError, fmt.Errorf("invalid checkpoint: %w", err), "invalid checkpoint")
+		}
+		logInfo := models.LogInfo{
+			RootHash:       stringPointer(hex.EncodeToString(checkpoint.Hash)),
+			TreeSize:       swag.Int64(int64(checkpoint.Size)),
+			SignedTreeHead: stringPointer(string(decoded)),
+			TreeID:         stringPointer(fmt.Sprintf("%d", api.logID)),
+			InactiveShards: inactiveShards,
+		}
+		return tlog.NewGetLogInfoOK().WithPayload(&logInfo)
+	}
+
 	resp := tc.GetLatest(0)
 	if resp.Status != codes.OK {
 		return handleRekorAPIError(params, http.StatusInternalServerError, fmt.Errorf("grpc error: %w", resp.Err), trillianCommunicationError)

pkg/generated/restapi/configure_rekor_server.go

@@ -156,7 +156,9 @@ func configureAPI(api *operations.RekorServerAPI) http.Handler {
 	api.RegisterFormat("signedCheckpoint", &util.SignedNote{}, util.SignedCheckpointValidator)
 
 	api.PreServerShutdown = func() {}
-	api.ServerShutdown = func() {}
+	api.ServerShutdown = func() {
+		pkgapi.StopAPI()
+	}
 
 	return setupGlobalMiddleware(api.Serve(setupMiddlewares))
 }