794 improve pricing method (#916)

* wip

* use multiple numeraires

* cleanup

* fix tests

* wip

* use price relevance thresholds

* fix

* add docs

* fix test

* fix test

* review fixes

apps/extension/src/service-worker.ts

@@ -74,7 +74,6 @@ const startServices = async () => {
     grpcEndpoint,
     walletId: WalletId.fromJsonString(wallet0.id),
     fullViewingKey: FullViewingKey.fromJsonString(wallet0.fullViewingKey),
-    numeraireAssetId: USDC_ASSET_ID,
   });
   await services.initialize();
   return services;

packages/constants/src/assets.ts

@@ -6,6 +6,16 @@ export const localAssets: Metadata[] = LocalAssetRegistry.map(a =>
   Metadata.fromJson(a as JsonValue),
 );
 
+export const NUMERAIRE_DENOMS: string[] = ['test_usd', 'usdc'];
+export const NUMERAIRES: Metadata[] = localAssets.filter(m => NUMERAIRE_DENOMS.includes(m.display));
+
+// PRICE_RELEVANCE_THRESHOLDS defines how long prices for different asset types remain relevant (in blocks)
+// 1 block = 5 seconds, 200 blocks approximately equals 17 minutes
+export const PRICE_RELEVANCE_THRESHOLDS = {
+  delegationToken: 719,
+  default: 200,
+};
+
 export const STAKING_TOKEN = 'penumbra';
 export const STAKING_TOKEN_METADATA = localAssets.find(
   metadata => metadata.display === STAKING_TOKEN,

packages/query/src/block-processor.ts

@@ -33,7 +33,7 @@ import {
 } from '@buf/penumbra-zone_penumbra.bufbuild_es/penumbra/core/asset/v1/asset_pb';
 import { bech32IdentityKey } from '@penumbra-zone/bech32/src/identity-key';
 import { getAssetId } from '@penumbra-zone/getters/src/metadata';
-import { STAKING_TOKEN_METADATA } from '@penumbra-zone/constants/src/assets';
+import { NUMERAIRES, STAKING_TOKEN_METADATA } from '@penumbra-zone/constants/src/assets';
 import { toDecimalExchangeRate } from '@penumbra-zone/types/src/amount';
 import { ValidatorInfoResponse } from '@buf/penumbra-zone_penumbra.bufbuild_es/penumbra/core/component/stake/v1/stake_pb';
 import { uint8ArrayToHex } from '@penumbra-zone/types/src/hex';
@@ -48,7 +48,6 @@ interface QueryClientProps {
   querier: RootQuerier;
   indexedDb: IndexedDbInterface;
   viewServer: ViewServerInterface;
-  numeraireAssetId: string;
 }
 
 const blankTxSource = new CommitmentSource({
@@ -60,14 +59,12 @@ export class BlockProcessor implements BlockProcessorInterface {
   private readonly indexedDb: IndexedDbInterface;
   private readonly viewServer: ViewServerInterface;
   private readonly abortController: AbortController = new AbortController();
-  private readonly numeraireAssetId: string;
   private syncPromise: Promise<void> | undefined;
 
-  constructor({ indexedDb, viewServer, querier, numeraireAssetId }: QueryClientProps) {
+  constructor({ indexedDb, viewServer, querier }: QueryClientProps) {
     this.indexedDb = indexedDb;
     this.viewServer = viewServer;
     this.querier = querier;
-    this.numeraireAssetId = numeraireAssetId;
   }
 
   // If syncBlocks() is called multiple times concurrently, they'll all wait for
@@ -267,7 +264,7 @@ export class BlockProcessor implements BlockProcessorInterface {
       if (compactBlock.swapOutputs.length) {
         await updatePricesFromSwaps(
           this.indexedDb,
-          this.numeraireAssetId,
+          NUMERAIRES,
           compactBlock.swapOutputs,
           compactBlock.height,
         );

packages/query/src/price-indexer.test.ts

@@ -1,4 +1,4 @@
-import { updatePricesFromSwaps } from './price-indexer';
+import { deriveAndSavePriceFromBSOD } from './price-indexer';
 import { BatchSwapOutputData } from '@buf/penumbra-zone_penumbra.bufbuild_es/penumbra/core/component/dex/v1/dex_pb';
 import { beforeEach, describe, expect, it, Mock, vi } from 'vitest';
 import { IndexedDbInterface } from '@penumbra-zone/types/src/indexed-db';
@@ -9,10 +9,10 @@ describe('updatePricesFromSwaps()', () => {
   let indexedDbMock: IndexedDbInterface;
   const updatePriceMock: Mock = vi.fn();
   const height = 123n;
-  const numeraireAssetId = 'reum7wQmk/owgvGMWMZn/6RFPV24zIKq3W6In/WwZgg=';
-  const numeraireAsset: AssetId = new AssetId({
-    inner: base64ToUint8Array(numeraireAssetId),
+  const numeraireAssetId = new AssetId({
+    inner: base64ToUint8Array('reum7wQmk/owgvGMWMZn/6RFPV24zIKq3W6In/WwZgg='),
   });
+
   beforeEach(() => {
     vi.clearAllMocks();
 
@@ -27,25 +27,25 @@ describe('updatePricesFromSwaps()', () => {
       new BatchSwapOutputData({
         tradingPair: {
           asset1: asset1,
-          asset2: numeraireAsset,
+          asset2: numeraireAssetId,
         },
         delta1: { lo: 250n },
         lambda2: { lo: 1200n },
         unfilled1: { lo: 0n },
       }),
     ];
 
-    await updatePricesFromSwaps(indexedDbMock, numeraireAssetId, swapOutputs, height);
+    await deriveAndSavePriceFromBSOD(indexedDbMock, numeraireAssetId, swapOutputs, height);
     expect(updatePriceMock).toBeCalledTimes(1);
-    expect(updatePriceMock).toBeCalledWith(asset1, numeraireAsset, 4.8, height);
+    expect(updatePriceMock).toBeCalledWith(asset1, numeraireAssetId, 4.8, height);
   });
 
   it('should update prices correctly for a swapOutput with NUMERAIRE as swapAsset1', async () => {
     const asset1 = new AssetId({ inner: new Uint8Array(12) });
     const swapOutputs: BatchSwapOutputData[] = [
       new BatchSwapOutputData({
         tradingPair: {
-          asset1: numeraireAsset,
+          asset1: numeraireAssetId,
           asset2: asset1,
         },
         delta2: { lo: 40n },
@@ -54,17 +54,17 @@ describe('updatePricesFromSwaps()', () => {
       }),
     ];
 
-    await updatePricesFromSwaps(indexedDbMock, numeraireAssetId, swapOutputs, height);
+    await deriveAndSavePriceFromBSOD(indexedDbMock, numeraireAssetId, swapOutputs, height);
     expect(updatePriceMock).toBeCalledTimes(1);
-    expect(updatePriceMock).toBeCalledWith(asset1, numeraireAsset, 318.5, height);
+    expect(updatePriceMock).toBeCalledWith(asset1, numeraireAssetId, 318.5, height);
   });
 
   it('should not update prices if delta is zero', async () => {
     const asset1 = new AssetId({ inner: new Uint8Array(12) });
     const swapOutputs: BatchSwapOutputData[] = [
       new BatchSwapOutputData({
         tradingPair: {
-          asset1: numeraireAsset,
+          asset1: numeraireAssetId,
           asset2: asset1,
         },
         delta2: { lo: 0n },
@@ -73,7 +73,7 @@ describe('updatePricesFromSwaps()', () => {
       }),
     ];
 
-    await updatePricesFromSwaps(indexedDbMock, numeraireAssetId, swapOutputs, height);
+    await deriveAndSavePriceFromBSOD(indexedDbMock, numeraireAssetId, swapOutputs, height);
     expect(updatePriceMock).toBeCalledTimes(0);
   });
 
@@ -83,24 +83,24 @@ describe('updatePricesFromSwaps()', () => {
       new BatchSwapOutputData({
         tradingPair: {
           asset1: asset1,
-          asset2: numeraireAsset,
+          asset2: numeraireAssetId,
         },
         delta1: { lo: 250n },
         lambda2: { lo: 1200n },
         unfilled1: { lo: 100n },
       }),
     ];
-    await updatePricesFromSwaps(indexedDbMock, numeraireAssetId, swapOutputs, height);
+    await deriveAndSavePriceFromBSOD(indexedDbMock, numeraireAssetId, swapOutputs, height);
     expect(updatePriceMock).toBeCalledTimes(1);
-    expect(updatePriceMock).toBeCalledWith(asset1, numeraireAsset, 8, height);
+    expect(updatePriceMock).toBeCalledWith(asset1, numeraireAssetId, 8, height);
   });
 
   it('should not update prices if swap is fully unfilled', async () => {
     const asset1 = new AssetId({ inner: new Uint8Array(12) });
     const swapOutputs: BatchSwapOutputData[] = [
       new BatchSwapOutputData({
         tradingPair: {
-          asset1: numeraireAsset,
+          asset1: numeraireAssetId,
           asset2: asset1,
         },
         delta2: { lo: 100n },
@@ -109,7 +109,7 @@ describe('updatePricesFromSwaps()', () => {
       }),
     ];
 
-    await updatePricesFromSwaps(indexedDbMock, numeraireAssetId, swapOutputs, height);
+    await deriveAndSavePriceFromBSOD(indexedDbMock, numeraireAssetId, swapOutputs, height);
     expect(updatePriceMock).toBeCalledTimes(0);
   });
 });

packages/query/src/price-indexer.ts

@@ -1,7 +1,10 @@
 import { BatchSwapOutputData } from '@buf/penumbra-zone_penumbra.bufbuild_es/penumbra/core/component/dex/v1/dex_pb';
 import { IndexedDbInterface } from '@penumbra-zone/types/src/indexed-db';
 import { divideAmounts, isZero, subtractAmounts } from '@penumbra-zone/types/src/amount';
-import { AssetId } from '@buf/penumbra-zone_penumbra.bufbuild_es/penumbra/core/asset/v1/asset_pb';
+import {
+  AssetId,
+  Metadata,
+} from '@buf/penumbra-zone_penumbra.bufbuild_es/penumbra/core/asset/v1/asset_pb';
 import { Amount } from '@buf/penumbra-zone_penumbra.bufbuild_es/penumbra/core/num/v1/num_pb';
 import {
   getDelta1Amount,
@@ -13,7 +16,7 @@ import {
   getUnfilled1Amount,
   getUnfilled2Amount,
 } from '@penumbra-zone/getters/src/batch-swap-output-data';
-import { base64ToUint8Array } from '@penumbra-zone/types/src/base64';
+import { getAssetId } from '@penumbra-zone/getters/src/metadata';
 
 /**
  *
@@ -30,12 +33,24 @@ import { base64ToUint8Array } from '@penumbra-zone/types/src/base64';
  */
 export const calculatePrice = (delta: Amount, unfilled: Amount, lambda: Amount): number => {
   const filledAmount = subtractAmounts(delta, unfilled);
-  //
+
   return isZero(delta) || isZero(lambda) || isZero(filledAmount)
     ? 0
     : divideAmounts(lambda, filledAmount).toNumber();
 };
 
+export const updatePricesFromSwaps = async (
+  indexedDb: IndexedDbInterface,
+  numeraires: Metadata[],
+  swapOutputs: BatchSwapOutputData[],
+  height: bigint,
+) => {
+  for (const numeraireMetadata of numeraires) {
+    const numeraireAssetId = getAssetId(numeraireMetadata);
+    await deriveAndSavePriceFromBSOD(indexedDb, numeraireAssetId, swapOutputs, height);
+  }
+};
+
 /**
  * Each 'BatchSwapOutputData' (BSOD) can generate up to two prices
  * Each BSOD in block has a unique trading pair
@@ -46,17 +61,12 @@ export const calculatePrice = (delta: Amount, unfilled: Amount, lambda: Amount):
  * This function processes only (1) price and ignores (2) price
  * We can get a BSOD with zero deltas(inputs), and we shouldn't save the price in that case
  */
-export const updatePricesFromSwaps = async (
+export const deriveAndSavePriceFromBSOD = async (
   indexedDb: IndexedDbInterface,
-  /** base64-encoded asset ID of the numeraire */
-  numeraireAssetId: string,
+  numeraireAssetId: AssetId,
   swapOutputs: BatchSwapOutputData[],
   height: bigint,
 ) => {
-  const numeraireAsset: AssetId = new AssetId({
-    inner: base64ToUint8Array(numeraireAssetId),
-  });
-
   for (const swapOutput of swapOutputs) {
     const swapAsset1 = getSwapAsset1(swapOutput);
     const swapAsset2 = getSwapAsset2(swapOutput);
@@ -65,7 +75,7 @@ export const updatePricesFromSwaps = async (
     let pricedAsset: AssetId | undefined = undefined;
 
     // case for trading pair <pricedAsset,numéraire>
-    if (swapAsset2.equals(numeraireAsset)) {
+    if (swapAsset2.equals(numeraireAssetId)) {
       pricedAsset = swapAsset1;
       // numerairePerUnit = lambda2/(delta1-unfilled1)
       numerairePerUnit = calculatePrice(
@@ -75,7 +85,7 @@ export const updatePricesFromSwaps = async (
       );
     }
     // case for trading pair <numéraire,pricedAsset>
-    else if (swapAsset1.equals(numeraireAsset)) {
+    else if (swapAsset1.equals(numeraireAssetId)) {
       pricedAsset = swapAsset2;
       // numerairePerUnit = lambda1/(delta2-unfilled2)
       numerairePerUnit = calculatePrice(
@@ -87,6 +97,6 @@ export const updatePricesFromSwaps = async (
 
     if (pricedAsset === undefined || numerairePerUnit === 0) continue;
 
-    await indexedDb.updatePrice(pricedAsset, numeraireAsset, numerairePerUnit, height);
+    await indexedDb.updatePrice(pricedAsset, numeraireAssetId, numerairePerUnit, height);
   }
 };

packages/router/src/grpc/view-protocol-server/balances.test.ts

@@ -13,7 +13,7 @@ import {
 import { createContextValues, createHandlerContext, HandlerContext } from '@connectrpc/connect';
 import { beforeEach, describe, expect, test, vi } from 'vitest';
 import { Services } from '@penumbra-zone/services/src/index';
-import { IndexedDbMock, MockServices, testFullViewingKey } from '../test-utils';
+import { IndexedDbMock, MockServices, TendermintMock, testFullViewingKey } from '../test-utils';
 import {
   AssetId,
   EquivalentValue,
@@ -47,6 +47,7 @@ describe('Balances request handler', () => {
   let mockServices: MockServices;
   let mockCtx: HandlerContext;
   let mockIndexedDb: IndexedDbMock;
+  let mockTendermint: TendermintMock;
 
   beforeEach(() => {
     vi.resetAllMocks();
@@ -70,6 +71,10 @@ describe('Balances request handler', () => {
       fullViewingKey: testFullViewingKey,
     };
 
+    mockTendermint = {
+      latestBlockHeight: vi.fn(),
+    };
+
     mockServices = {
       // @ts-expect-error TODO: Improve mocking types
       getWalletServices: vi.fn(() =>
@@ -78,6 +83,7 @@ describe('Balances request handler', () => {
           viewServer: mockViewServer,
           querier: {
             shieldedPool: mockShieldedPool,
+            tendermint: mockTendermint,
           },
         }),
       ) as MockServices['getWalletServices'],

packages/router/src/grpc/view-protocol-server/balances.ts

@@ -28,16 +28,19 @@ import { Amount } from '@buf/penumbra-zone_penumbra.bufbuild_es/penumbra/core/nu
 import { Base64Str, uint8ArrayToBase64 } from '@penumbra-zone/types/src/base64';
 import { addLoHi } from '@penumbra-zone/types/src/lo-hi';
 import { IndexedDbInterface } from '@penumbra-zone/types/src/indexed-db';
-import { getAssetId } from '@penumbra-zone/getters/src/metadata';
 import { multiplyAmountByNumber } from '@penumbra-zone/types/src/amount';
 import { Stringified } from '@penumbra-zone/types/src/jsonified';
 
 // Handles aggregating amounts and filtering by account number/asset id
 export const balances: Impl['balances'] = async function* (req, ctx) {
   const services = ctx.values.get(servicesCtx);
-  const { indexedDb } = await services.getWalletServices();
+  const { indexedDb, querier } = await services.getWalletServices();
 
-  const aggregator = new BalancesAggregator(ctx, indexedDb);
+  // latestBlockHeight is needed to calculate the threshold of price relevance,
+  //it is better to use  rather than fullSyncHeight to avoid displaying old prices during the synchronization process
+  const latestBlockHeight = await querier.tendermint.latestBlockHeight();
+
+  const aggregator = new BalancesAggregator(ctx, indexedDb, latestBlockHeight);
 
   for await (const noteRecord of indexedDb.iterateSpendableNotes()) {
     if (noteRecord.heightSpent !== 0n) continue;
@@ -72,6 +75,7 @@ class BalancesAggregator {
   constructor(
     private readonly ctx: HandlerContext,
     private readonly indexedDb: IndexedDbInterface,
+    private readonly latestBlockHeight: bigint,
   ) {}
 
   async add(n: SpendableNoteRecord) {
@@ -180,9 +184,11 @@ class BalancesAggregator {
         valueView: { case: 'unknownAssetId', value: { assetId, amount: new Amount() } },
       });
     } else {
-      const assetId = getAssetId.optional()(new Metadata(denomMetadata));
-      if (assetId?.inner && !this.estimatedPriceByPricedAsset[uint8ArrayToBase64(assetId.inner)]) {
-        const prices = await this.indexedDb.getPricesForAsset(new AssetId(assetId));
+      if (!this.estimatedPriceByPricedAsset[uint8ArrayToBase64(assetId.inner)]) {
+        const prices = await this.indexedDb.getPricesForAsset(
+          denomMetadata as Metadata,
+          this.latestBlockHeight,
+        );
         this.estimatedPriceByPricedAsset[uint8ArrayToBase64(assetId.inner)] = prices;
       }
 

packages/services/src/index.ts

@@ -22,15 +22,10 @@ export interface ServicesConfig {
   readonly grpcEndpoint?: string;
   readonly walletId?: WalletId;
   readonly fullViewingKey?: FullViewingKey;
-  readonly numeraireAssetId: string;
 }
 
 const isCompleteServicesConfig = (c: Partial<ServicesConfig>): c is Required<ServicesConfig> =>
-  c.grpcEndpoint != null &&
-  c.idbVersion != null &&
-  c.walletId != null &&
-  c.fullViewingKey != null &&
-  c.numeraireAssetId != null;
+  c.grpcEndpoint != null && c.idbVersion != null && c.walletId != null && c.fullViewingKey != null;
 
 export class Services implements ServicesInterface {
   private walletServicesPromise: Promise<WalletServices> | undefined;
@@ -96,7 +91,7 @@ export class Services implements ServicesInterface {
   }
 
   private async initializeWalletServices(): Promise<WalletServices> {
-    const { walletId, fullViewingKey, idbVersion: dbVersion, numeraireAssetId } = await this.config;
+    const { walletId, fullViewingKey, idbVersion: dbVersion } = await this.config;
     const params = await this.querier.app.appParams();
     if (!params.sctParams?.epochDuration) throw new Error('Epoch duration unknown');
     const {
@@ -123,7 +118,6 @@ export class Services implements ServicesInterface {
       viewServer,
       querier: this.querier,
       indexedDb,
-      numeraireAssetId,
     });
 
     return { viewServer, blockProcessor, indexedDb, querier: this.querier };