diff --git a/cmd/api.go b/cmd/api.go index bac28b7a..3848255e 100644 --- a/cmd/api.go +++ b/cmd/api.go @@ -32,6 +32,10 @@ var ( apiDefaultDataAPIEnabled = os.Getenv("DISABLE_DATA_API") != "1" apiDefaultProposerAPIEnabled = os.Getenv("DISABLE_PROPOSER_API") != "1" + localAuctionHost = os.Getenv("LOCAL_AUCTION_HOST") + remoteAuctionHost = os.Getenv("REMOTE_AUCTION_HOST") + auctionAuthToken = os.Getenv("PRIVATE_ROUTE_AUTH_TOKEN") + apiListenAddr string apiPprofEnabled bool apiSecretKey string @@ -142,7 +146,7 @@ var apiCmd = &cobra.Command{ } log.Info("Setting up datastore...") - ds, err := datastore.NewDatastore(redis, mem, db) + ds, err := datastore.NewDatastore(redis, mem, db, localAuctionHost, remoteAuctionHost, auctionAuthToken) if err != nil { log.WithError(err).Fatalf("Failed setting up prod datastore") } diff --git a/datastore/auction_api.go b/datastore/auction_api.go index be62d09f..9605b77e 100644 --- a/datastore/auction_api.go +++ b/datastore/auction_api.go @@ -6,6 +6,7 @@ import ( "io" "net/http" "net/url" + "time" builderApi "github.com/attestantio/go-builder-client/api" builderApiDeneb "github.com/attestantio/go-builder-client/api/deneb" @@ -15,38 +16,45 @@ import ( "github.com/pkg/errors" ) -const API_ROOT = "http://turbo-auction-api" - var ErrFailedToParsePayload = errors.New("failed to parse payload") -func GetPayloadContents(slot uint64, proposerPubkey, blockHash string) (*builderApi.VersionedSubmitBlindedBlockResponse, error) { +func getPayloadContents(slot uint64, proposerPubkey, blockHash, host, basePath, authToken string, timeout time.Duration) (*builderApi.VersionedSubmitBlindedBlockResponse, error) { + client := &http.Client{Timeout: timeout} + queryParams := url.Values{} queryParams.Add("slot", fmt.Sprintf("%d", slot)) queryParams.Add("proposer_pubkey", proposerPubkey) queryParams.Add("block_hash", blockHash) - fullUrl := fmt.Sprintf("%s/internal/payload_contents?%s", API_ROOT, queryParams.Encode()) + fullURL := fmt.Sprintf("%s/%s/payload_contents?%s", host, basePath, queryParams.Encode()) + req, err := http.NewRequest("GET", fullURL, nil) + if err != nil { + return nil, errors.Wrap(err, "failed to create payload contents request") + } - var err error + // Add auth token if provided + if authToken != "" { + req.Header.Add("x-auth-token", authToken) + } - resp, err := http.Get(fullUrl) + resp, err := client.Do(req) if err != nil { - return nil, err + return nil, errors.Wrap(err, "failed to fetch payload contents") } defer resp.Body.Close() - if resp.StatusCode == 404 { + if resp.StatusCode == http.StatusNotFound { return nil, ErrExecutionPayloadNotFound } body, err := io.ReadAll(resp.Body) if err != nil { - return nil, err + return nil, errors.Wrap(err, "failed to read payload contents response body") } // Try to parse deneb contents denebPayloadContents := new(builderApiDeneb.ExecutionPayloadAndBlobsBundle) - err = denebPayloadContents.UnmarshalSSZ([]byte(body)) + err = denebPayloadContents.UnmarshalSSZ(body) if err == nil { return &builderApi.VersionedSubmitBlindedBlockResponse{ @@ -57,7 +65,7 @@ func GetPayloadContents(slot uint64, proposerPubkey, blockHash string) (*builder // Try to parse capella payload capellaPayload := new(capella.ExecutionPayload) - err = capellaPayload.UnmarshalSSZ([]byte(body)) + err = capellaPayload.UnmarshalSSZ(body) if err == nil { return &builderApi.VersionedSubmitBlindedBlockResponse{ @@ -69,29 +77,45 @@ func GetPayloadContents(slot uint64, proposerPubkey, blockHash string) (*builder return nil, ErrFailedToParsePayload } -func GetBidTrace(slot uint64, proposerPubkey, blockHash string) (*common.BidTraceV2, error) { +func (ds *Datastore) LocalPayloadContents(slot uint64, proposerPubkey, blockHash string) (*builderApi.VersionedSubmitBlindedBlockResponse, error) { + return getPayloadContents(slot, proposerPubkey, blockHash, ds.localAuctionHost, "internal", "", 0) +} + +func (ds *Datastore) RemotePayloadContents(slot uint64, proposerPubkey, blockHash string) (*builderApi.VersionedSubmitBlindedBlockResponse, error) { + return getPayloadContents(slot, proposerPubkey, blockHash, ds.remoteAuctionHost, "private", ds.auctionAuthToken, 1*time.Second) +} + +func getBidTrace(slot uint64, proposerPubkey, blockHash, auctionHost, basePath, authToken string) (*common.BidTraceV2, error) { + client := &http.Client{} + queryParams := url.Values{} queryParams.Add("slot", fmt.Sprintf("%d", slot)) queryParams.Add("proposer_pubkey", proposerPubkey) queryParams.Add("block_hash", blockHash) - fullUrl := fmt.Sprintf("%s/internal/bid_trace?%s", API_ROOT, queryParams.Encode()) + fullURL := fmt.Sprintf("%s/%s/bid_trace?%s", auctionHost, basePath, queryParams.Encode()) + req, err := http.NewRequest("GET", fullURL, nil) + if err != nil { + return nil, errors.Wrap(err, "failed to create bid trace request") + } - var err error + if authToken != "" { + req.Header.Add("x-auth-token", authToken) + } - resp, err := http.Get(fullUrl) + resp, err := client.Do(req) if err != nil { return nil, err } defer resp.Body.Close() - if resp.StatusCode == 404 { + if resp.StatusCode == http.StatusNotFound { return nil, ErrBidTraceNotFound } body, err := io.ReadAll(resp.Body) if err != nil { - return nil, err + return nil, errors.Wrap(err, "failed to read bit trace response body") } bidtrace := new(common.BidTraceV2) @@ -102,3 +126,11 @@ func GetBidTrace(slot uint64, proposerPubkey, blockHash string) (*common.BidTrac return bidtrace, nil } + +func (ds *Datastore) LocalBidTrace(slot uint64, proposerPubkey, blockHash string) (*common.BidTraceV2, error) { + return getBidTrace(slot, proposerPubkey, blockHash, ds.localAuctionHost, "internal", "") +} + +func (ds *Datastore) RemoteBidTrace(slot uint64, proposerPubkey, blockHash string) (*common.BidTraceV2, error) { + return getBidTrace(slot, proposerPubkey, blockHash, ds.remoteAuctionHost, "private", ds.auctionAuthToken) +} diff --git a/datastore/datastore.go b/datastore/datastore.go index 19508b88..8e2bd0b6 100644 --- a/datastore/datastore.go +++ b/datastore/datastore.go @@ -2,15 +2,14 @@ package datastore import ( - "database/sql" "fmt" + "log" "strings" "sync" "time" builderApi "github.com/attestantio/go-builder-client/api" builderApiV1 "github.com/attestantio/go-builder-client/api/v1" - "github.com/bradfitz/gomemcache/memcache" "github.com/flashbots/mev-boost-relay/beaconclient" "github.com/flashbots/mev-boost-relay/common" "github.com/flashbots/mev-boost-relay/database" @@ -49,15 +48,33 @@ type Datastore struct { // Used for proposer-API readiness check KnownValidatorsWasUpdated uberatomic.Bool + + // Where we can find payloads for our local auction + // Should be protocol + hostname, e.g. http://turbo-auction-api, https://relay-builders-us.ultrasound.money + localAuctionHost string + remoteAuctionHost string + // Token used to remotely authenticate to auction API. + auctionAuthToken string } -func NewDatastore(redisCache *RedisCache, memcached *Memcached, db database.IDatabaseService) (ds *Datastore, err error) { +func NewDatastore(redisCache *RedisCache, memcached *Memcached, db database.IDatabaseService, localAuctionHost, remoteAuctionHost, auctionAuthToken string) (ds *Datastore, err error) { ds = &Datastore{ db: db, memcached: memcached, redis: redisCache, knownValidatorsByPubkey: make(map[common.PubkeyHex]uint64), knownValidatorsByIndex: make(map[uint64]common.PubkeyHex), + localAuctionHost: localAuctionHost, + remoteAuctionHost: remoteAuctionHost, + auctionAuthToken: auctionAuthToken, + } + + if localAuctionHost == "" { + log.Fatal("LOCAL_AUCTION_HOST is not set") + } + + if remoteAuctionHost == "" { + log.Fatal("REMOTE_AUCTION_HOST is not set") } return ds, err @@ -191,47 +208,27 @@ func (ds *Datastore) SaveValidatorRegistration(entry builderApiV1.SignedValidato return nil } -// GetGetPayloadResponse returns the getPayload response from memory or Redis or Database -func (ds *Datastore) GetGetPayloadResponse(log *logrus.Entry, slot uint64, proposerPubkey, blockHash string) (*builderApi.VersionedSubmitBlindedBlockResponse, error) { - log = log.WithField("datastoreMethod", "GetGetPayloadResponse") +// RedisPayload returns the getPayload response from Redis +func (ds *Datastore) RedisPayload(log *logrus.Entry, slot uint64, proposerPubkey, blockHash string) (*builderApi.VersionedSubmitBlindedBlockResponse, error) { + log = log.WithField("datastoreMethod", "RedisPayload") _proposerPubkey := strings.ToLower(proposerPubkey) _blockHash := strings.ToLower(blockHash) - // 1. try to get from Redis + // try to get from Redis resp, err := ds.redis.GetPayloadContents(slot, _proposerPubkey, _blockHash) + + // redis.Nil is a common error when the key is not found + // this may happen if we're asked for a payload we don't have. if errors.Is(err, redis.Nil) { log.WithError(err).Warn("execution payload not found in redis") - } else if err != nil { - log.WithError(err).Error("error getting execution payload from redis") - } else { - log.Debug("getPayload response from redis") - return resp, nil - } - - // 2. try to get from Memcached - if ds.memcached != nil { - resp, err = ds.memcached.GetExecutionPayload(slot, _proposerPubkey, _blockHash) - if errors.Is(err, memcache.ErrCacheMiss) { - log.WithError(err).Warn("execution payload not found in memcached") - } else if err != nil { - log.WithError(err).Error("error getting execution payload from memcached") - } else if resp != nil { - log.Debug("getPayload response from memcached") - return resp, nil - } + return nil, ErrExecutionPayloadNotFound } - // 3. try to get from database (should not happen, it's just a backup) - executionPayloadEntry, err := ds.db.GetExecutionPayloadEntryBySlotPkHash(slot, proposerPubkey, blockHash) - if errors.Is(err, sql.ErrNoRows) { - log.WithError(err).Warn("execution payload not found in database") - return nil, ErrExecutionPayloadNotFound - } else if err != nil { - log.WithError(err).Error("error getting execution payload from database") + if err != nil { + log.WithError(err).Error("error getting execution payload from redis") return nil, err } - // Got it from database, now deserialize execution payload and compile full response - log.Warn("getPayload response from database, primary storage failed") - return database.ExecutionPayloadEntryToExecutionPayload(executionPayloadEntry) + log.Debug("getPayload response from redis") + return resp, nil } diff --git a/datastore/datastore_test.go b/datastore/datastore_test.go index a0f99a64..de2762af 100644 --- a/datastore/datastore_test.go +++ b/datastore/datastore_test.go @@ -18,7 +18,7 @@ func setupTestDatastore(t *testing.T, mockDB *database.MockDB) *Datastore { redisDs, err := NewRedisCache("", redisTestServer.Addr(), "", "") require.NoError(t, err) - ds, err := NewDatastore(redisDs, nil, mockDB) + ds, err := NewDatastore(redisDs, nil, mockDB, "", "") require.NoError(t, err) return ds @@ -26,7 +26,7 @@ func setupTestDatastore(t *testing.T, mockDB *database.MockDB) *Datastore { func TestGetPayloadFailure(t *testing.T) { ds := setupTestDatastore(t, &database.MockDB{}) - _, err := ds.GetGetPayloadResponse(common.TestLog, 1, "a", "b") + _, err := ds.RedisPayload(common.TestLog, 1, "a", "b") require.Error(t, ErrExecutionPayloadNotFound, err) } @@ -65,7 +65,7 @@ func TestGetPayloadDatabaseFallback(t *testing.T) { }, } ds := setupTestDatastore(t, mockDB) - payload, err := ds.GetGetPayloadResponse(common.TestLog, 1, "a", "b") + payload, err := ds.RedisPayload(common.TestLog, 1, "a", "b") require.NoError(t, err) blockHash, err := payload.BlockHash() require.NoError(t, err) diff --git a/services/api/service.go b/services/api/service.go index 1633c6a6..8d1555a3 100644 --- a/services/api/service.go +++ b/services/api/service.go @@ -1270,6 +1270,60 @@ func (api *RelayAPI) checkProposerSignature(block *common.VersionedSignedBlinded } } +// getPayloadWithFallbacks tries to get the payload from local auction API, if not found, it tries to get it from remote auction API. +// If still not found, it retries with Redis. +// If still not found, it returns ErrExecutionPayloadNotFound if . +// We may get asked for payloads we never served a bid for, in which case we return ErrExecutionPayloadNotFound. This is fine. +// We may have failed to ask whether or not our stores have the payload, which is completely unaccepable. +// Improvement: we could ask all three in parallel. Any payload found will do. +func getPayloadWithFallbacks(log *logrus.Entry, datastoreRef *datastore.Datastore, slot uint64, proposerPubkey common.PubkeyHex, blockHash phase0.Hash32) (*builderApi.VersionedSubmitBlindedBlockResponse, error) { + getPayloadResp, err := datastoreRef.LocalPayloadContents(uint64(slot), proposerPubkey.String(), blockHash.String()) + if getPayloadResp != nil { + return getPayloadResp, nil + } + + if errors.Is(err, datastore.ErrExecutionPayloadNotFound) { + // Acceptable error as long as we didn't share the bid we're asked for, warn and continue. + log.WithError(err).Warn("payload not found in local auction API") + } else { + // We hit a bad error, this should never happen, but this is what our fallbacks are for. + log.WithError(err).Error("failed to get payload from local auction API") + } + + // In case the local auction API has crashed after serving a header somehow, it may still have succeeded in storing the payload in Redis. + // Check there. + getPayloadResp, err = datastoreRef.RedisPayload(log, uint64(slot), proposerPubkey.String(), blockHash.String()) + if getPayloadResp != nil { + return getPayloadResp, nil + } + + if errors.Is(err, datastore.ErrExecutionPayloadNotFound) { + // Acceptable error as long as we didn't share the bid we're asked for, warn and continue. + log.WithError(err).Warn("payload not found in Redis") + } else { + // We hit a bad error, this should never happen, but this is what our fallbacks are for. + log.WithError(err).Error("failed to get payload from Redis") + } + + // If we're failing to check for payloads entirely, checking whether the other geo has the payload probably won't help. + // If payload stores are responding but simply don't have the payload, we probably didn't offer the bid either. + // Regardless, its possible the other geo does. Check it, so we may help out in publishing, even if it wasn't "our bid". + getPayloadResp, err = datastoreRef.RemotePayloadContents(uint64(slot), proposerPubkey.String(), blockHash.String()) + if getPayloadResp != nil { + return getPayloadResp, nil + } + + if errors.Is(err, datastore.ErrExecutionPayloadNotFound) { + // Acceptable error as long as we didn't share the bid we're asked for, warn and continue. + log.WithError(err).Warn("payload not found in remote auction API") + } else { + // We hit a bad error, this should never happen, and we're out of fallbacks. + log.WithError(err).Error("failed to get payload from remote auction API") + } + + return nil, err +} + func (api *RelayAPI) handleGetPayload(w http.ResponseWriter, req *http.Request) { api.getPayloadCallsInFlight.Add(1) defer api.getPayloadCallsInFlight.Done() @@ -1406,9 +1460,12 @@ func (api *RelayAPI) handleGetPayload(w http.ResponseWriter, req *http.Request) // Save information about delivered payload defer func() { - bidTrace, err := datastore.GetBidTrace(uint64(slot), proposerPubkey.String(), blockHash.String()) + bidTrace, err := api.datastore.LocalBidTrace(uint64(slot), proposerPubkey.String(), blockHash.String()) + if errors.Is(err, datastore.ErrBidTraceNotFound) { + bidTrace, err = api.datastore.RemoteBidTrace(uint64(slot), proposerPubkey.String(), blockHash.String()) + } if err != nil { - log.WithError(err).Info("failed to get bidTrace for delivered payload from redis") + log.WithError(err).Info("failed to get bidTrace for delivered payload") return } @@ -1547,64 +1604,27 @@ func (api *RelayAPI) handleGetPayload(w http.ResponseWriter, req *http.Request) log.Debug(fmt.Sprintf("successfully archived payload request, block_hash: %s", blockHash.String())) }() - // Get the response - from Redis, Memcache or DB - // note that recent mev-boost versions only send getPayload to relays that provided the bid, older versions send getPayload to all relays. - // Additionally, proposers may feel it's safer to ask for a bid from all relays and fork. - getPayloadResp, err = datastore.GetPayloadContents(uint64(slot), proposerPubkey.String(), blockHash.String()) - if err != nil || getPayloadResp == nil { - log.WithError(err).Warn("failed first attempt to get execution payload") - - // Wait, then try again (this time from Redis) - time.Sleep(time.Duration(timeoutGetPayloadRetryMs) * time.Millisecond) - getPayloadResp, err = api.datastore.GetGetPayloadResponse(log, uint64(slot), proposerPubkey.String(), blockHash.String()) - - if err != nil || getPayloadResp == nil { - // Still not found! Error out now. - if errors.Is(err, datastore.ErrExecutionPayloadNotFound) { - // Couldn't find the execution payload, three options: - // 1. We're storing all payloads in postgres, we never received - // or served the bid, but someone still asked us for it. We can - // check this. - // 2. We do not store all payloads in postgres. The bid was - // never the top bid, so we didn't store it in Redis or - // Memcached either. We received, but never served the bid, but - // someone still asked us for it. - // 3. The bid was accepted but the payload was lost in all - // active stores. This is a critical error! If this ever - // happens, we have work to do. - // Annoyingly, we can't currently distinguish between 2 and 3! - - // Check for case 1 if possible. - if !api.ffDisablePayloadDBStorage { - _, err := api.db.GetBlockSubmissionEntry(uint64(slot), proposerPubkey.String(), blockHash.String()) - if errors.Is(err, sql.ErrNoRows) { - abortReason = "execution-payload-not-found" - log.Info("failed second attempt to get execution payload, discovered block was never submitted to this relay") - api.RespondError(w, http.StatusBadRequest, "no execution payload for this request, block was never seen by this relay") - return - } - if err != nil { - abortReason = "execution-payload-retrieval-error" - log.WithError(err).Error("failed second attempt to get execution payload, hit an error while checking if block was submitted to this relay") - api.RespondError(w, http.StatusInternalServerError, "no execution payload for this request, hit an error while checking if block was submitted to this relay") - return - } - } - - // Case 2 or 3, we don't know which. - abortReason = "execution-payload-not-found" - log.Warn("failed second attempt to get execution payload, not found case, block was never submitted to this relay or bid was accepted but payload was lost") - api.RespondError(w, http.StatusBadRequest, "no execution payload for this request, block was never seen by this relay or bid was accepted but payload was lost, if you got this bid from us, please contact the relay") - return - } else { - abortReason = "execution-payload-retrieval-error" - log.WithError(err).Error("failed second attempt to get execution payload, error case") - api.RespondError(w, http.StatusInternalServerError, "no execution payload for this request") - return - } - } - - // The second attempt succeeded. We may continue. + getPayloadResp, err = getPayloadWithFallbacks(log, api.datastore, uint64(slot), proposerPubkey, blockHash) + // This happens all the time and is acceptable. + if errors.Is(err, datastore.ErrExecutionPayloadNotFound) { + abortReason = "execution-payload-not-found" + log.Info("failed to get execution payload, discovered block was never submitted to this relay") + api.RespondError(w, http.StatusBadRequest, "no execution payload for this request, block was never seen by this relay") + return + } + // Should never happen. All methods of payload retrieval hit an unacceptable error. + if err != nil { + abortReason = "execution-payload-retrieval-error" + log.WithError(err).Error("failed second attempt to get execution payload, error case") + api.RespondError(w, http.StatusInternalServerError, "no execution payload for this request") + return + } + // Although getPayloadWithFallbacks should always return a payload or an err, this code is to critical to assume. We check it hasn't returned nil without err anyway. + if getPayloadResp == nil { + abortReason = "unexpected-nil-getpayloadresp" + log.Error("ended up with nil getPayloadResp, and no early return on error, this should be impossible") + api.RespondError(w, http.StatusInternalServerError, "no execution payload for this request") + return } // Now we know this relay also has the payload