feat: add fuzzy search on stake pool metadata

compose/common.yml

@@ -72,6 +72,7 @@ x-sdk-environment: &sdk-environment
   LOGGER_MIN_SEVERITY: ${LOGGER_MIN_SEVERITY:-info}
   NETWORK_INFO_PROVIDER_URL: http://provider-server:3000/
   OGMIOS_URL: ws://ogmios:1337
+  OVERRIDE_FUZZY_OPTIONS: true
   POSTGRES_DB_FILE_ASSET: /run/secrets/postgres_db_asset
   POSTGRES_DB_FILE_DB_SYNC: /run/secrets/postgres_db_db_sync
   POSTGRES_DB_FILE_HANDLE: /run/secrets/postgres_db_handle

nix/cardano-services/deployments/default.nix

@@ -167,7 +167,10 @@ in
           backend = {
             enabled = true;
           };
-          stake-pool-provider.enabled = true;
+          stake-pool-provider = {
+            enabled = true;
+            env.OVERRIDE_FUZZY_OPTIONS = "true";
+          };
           handle-provider.enabled = true;
           # asset-provider.enabled = true;
         };
@@ -213,7 +216,10 @@ in
             env.USE_BLOCKFROST = lib.mkForce "false";
             env.SUBMIT_API_URL = "http://${final.namespace}-cardano-stack.${final.namespace}.svc.cluster.local:8090";
           };
-          stake-pool-provider.enabled = true;
+          stake-pool-provider = {
+            enabled = true;
+            env.OVERRIDE_FUZZY_OPTIONS = "true";
+          };
         };
 
         projectors = {
@@ -259,7 +265,10 @@ in
             enabled = true;
             replicas = 3;
           };
-          stake-pool-provider.enabled = true;
+          stake-pool-provider = {
+            enabled = true;
+            env.OVERRIDE_FUZZY_OPTIONS = "true";
+          };
           handle-provider.enabled = true;
           # asset-provider.enabled = true;
         };
@@ -326,7 +335,10 @@ in
             enabled = true;
             env.USE_BLOCKFROST = lib.mkForce "false";
           };
-          stake-pool-provider.enabled = true;
+          stake-pool-provider = {
+            enabled = true;
+            env.OVERRIDE_FUZZY_OPTIONS = "true";
+          };
           handle-provider.enabled = true;
         };
 

packages/cardano-services/package.json

@@ -110,6 +110,7 @@
     "express-openapi-validator": "^4.13.8",
     "express-prom-bundle": "^6.4.1",
     "fraction.js": "^4.2.0",
+    "fuse.js": "^7.0.0",
     "json-bigint": "^1.0.0",
     "jsonschema": "^1.4.1",
     "lodash": "^4.17.21",

packages/cardano-services/src/Program/programs/providerServer.ts

@@ -134,12 +134,8 @@ const serviceMapFactory = (options: ServiceMapFactoryOptions) => {
 
   const getTypeormStakePoolProvider = withTypeOrmProvider('StakePool', (connectionConfig$) => {
     const entities = getEntities(['currentPoolMetrics', 'poolDelisted', 'poolMetadata', 'poolRewards']);
-    const { lastRosEpochs, paginationPageSizeLimit } = args;
 
-    return new TypeormStakePoolProvider(
-      { lastRosEpochs, paginationPageSizeLimit: paginationPageSizeLimit! },
-      { connectionConfig$, entities, logger }
-    );
+    return new TypeormStakePoolProvider(args, { cache: getDbCache(), connectionConfig$, entities, logger });
   });
 
   let networkInfoProvider: DbSyncNetworkInfoProvider | undefined;

packages/cardano-services/src/Program/programs/types.ts

@@ -6,6 +6,7 @@ import {
 } from '../options';
 import { HandlePolicyIdsProgramOptions } from '../options/policyIds';
 import { Milliseconds, Seconds } from '@cardano-sdk/core';
+import { TypeOrmStakePoolProviderProps } from '../../StakePool';
 import { defaultJobOptions } from '@cardano-sdk/projection-typeorm';
 
 /** cardano-services programs */
@@ -50,8 +51,10 @@ export enum ProviderServerOptionDescriptions {
   DisableDbCache = 'Disable DB cache',
   DisableStakePoolMetricApy = 'Omit this metric for improved query performance',
   EpochPollInterval = 'Epoch poll interval',
+  FuzzyOptions = 'Options for the fuzzy search on stake pool metadata',
   HandleProviderServerUrl = 'URL for the Handle provider server',
   HealthCheckCacheTtl = 'Health check cache TTL in seconds between 1 and 10',
+  OverrideFuzzyOptions = 'Allows the override of fuzzyOptions through queryStakePools call',
   PaginationPageSizeLimit = 'Pagination page size limit shared across all providers',
   SubmitApiUrl = 'cardano-submit-api URL',
   TokenMetadataCacheTtl = 'Token Metadata API cache TTL in seconds',
@@ -72,7 +75,8 @@ export type ProviderServerArgs = CommonProgramOptions &
   PosgresProgramOptions<'Asset'> &
   OgmiosProgramOptions &
   HandlePolicyIdsProgramOptions &
-  StakePoolMetadataProgramOptions & {
+  StakePoolMetadataProgramOptions &
+  TypeOrmStakePoolProviderProps & {
     allowedOrigins?: string[];
     assetCacheTTL?: Seconds;
     cardanoNodeConfigPath?: string;
@@ -82,7 +86,6 @@ export type ProviderServerArgs = CommonProgramOptions &
     epochPollInterval: number;
     handleProviderServerUrl?: string;
     healthCheckCacheTtl: Seconds;
-    paginationPageSizeLimit?: number;
     serviceNames: ServiceNames[];
     submitApiUrl?: URL;
     submitValidateHandles?: boolean;

packages/cardano-services/src/StakePool/DbSyncStakePoolProvider/DbSyncStakePoolProvider.ts

@@ -254,9 +254,8 @@ export class DbSyncStakePoolProvider extends DbSyncProvider(RunnableModule) impl
     return { poolMetrics, poolOwners, poolRegistrations, poolRelays, poolRetirements };
   }
 
-  public async queryStakePools(options: QueryStakePoolsArgs): Promise<Paginated<Cardano.StakePool>> {
-    const { filters, pagination, sort, apyEpochsBackLimit = APY_EPOCHS_BACK_LIMIT_DEFAULT } = options;
-    const useBlockfrost = this.#useBlockfrost;
+  public queryStakePoolsChecks(options: QueryStakePoolsArgs) {
+    const { filters, pagination, sort } = options;
 
     if (pagination.limit > this.#paginationPageSizeLimit) {
       throw new ProviderError(
@@ -266,6 +265,14 @@ export class DbSyncStakePoolProvider extends DbSyncProvider(RunnableModule) impl
       );
     }
 
+    if (filters?.text) {
+      throw new ProviderError(
+        ProviderFailure.NotImplemented,
+        undefined,
+        'DbSyncStakePoolProvider does not support text filter'
+      );
+    }
+
     if (filters?.identifier && filters.identifier.values.length > this.#paginationPageSizeLimit) {
       throw new ProviderError(
         ProviderFailure.BadRequest,
@@ -283,6 +290,13 @@ export class DbSyncStakePoolProvider extends DbSyncProvider(RunnableModule) impl
         `DbSyncStakePoolProvider doesn't support sort by ${sort?.field} `
       );
     }
+  }
+
+  public async queryStakePools(options: QueryStakePoolsArgs): Promise<Paginated<Cardano.StakePool>> {
+    const { filters, apyEpochsBackLimit = APY_EPOCHS_BACK_LIMIT_DEFAULT } = options;
+    const useBlockfrost = this.#useBlockfrost;
+
+    this.queryStakePoolsChecks(options);
 
     const { params, query } = useBlockfrost
       ? this.#builder.buildBlockfrostQuery(filters)

packages/cardano-services/src/StakePool/TypeormStakePoolProvider/TypeormStakePoolProvider.ts

@@ -1,5 +1,6 @@
 import {
   Cardano,
+  FuzzyOptions,
   Paginated,
   ProviderError,
   ProviderFailure,
@@ -8,38 +9,61 @@ import {
   StakePoolStats
 } from '@cardano-sdk/core';
 import { DataSource } from 'typeorm';
+import { InMemoryCache } from '../../InMemoryCache';
 import { MissingProgramOption } from '../../Program/errors';
 import { PoolDelistedEntity, StakePoolEntity } from '@cardano-sdk/projection-typeorm';
-import { PoolStatsModel, mapPoolStats, mapStakePoolsResult } from './mappers';
+import { PoolModel, PoolStatsModel, mapPoolStats, mapStakePoolsResult } from './mappers';
 import { ServiceNames } from '../../Program/programs/types';
 import { TypeormProvider, TypeormProviderDependencies } from '../../util';
 import {
   computeROS,
   getSortOptions,
   getWhereClauseAndArgs,
-  nullsInSort,
   stakePoolSearchSelection,
-  stakePoolSearchTotalCount
+  stakePoolSearchTotalCount,
+  withTextFilter
 } from './util';
+import Fuse from 'fuse.js';
 
-/** Properties that are need to create DbSyncStakePoolProvider */
+export const DEFAULT_FUZZY_SEARCH_OPTIONS: FuzzyOptions = {
+  threshold: 0.4,
+  weights: { description: 1, homepage: 2, name: 3, poolId: 4, ticker: 4 }
+};
+
+/** Properties that are need to create TypeormStakePoolProvider */
 export interface TypeOrmStakePoolProviderProps {
+  /** Options for the fuzzy search on stake pool metadata */
+  fuzzyOptions?: FuzzyOptions;
+
   /** Number of epochs over which lastRos is computed */
   lastRosEpochs?: number;
 
   /** Pagination page size limit used for provider methods constraint. */
   paginationPageSizeLimit: number;
+
+  /** If `true` allows the override of `fuzzyOptions` through `queryStakePools` call.*/
+  overrideFuzzyOptions?: boolean;
+}
+
+export interface TypeormStakePoolProviderDependencies extends TypeormProviderDependencies {
+  cache: InMemoryCache;
 }
 
 export class TypeormStakePoolProvider extends TypeormProvider implements StakePoolProvider {
+  #cache: InMemoryCache;
+  #fuzzyOptions: FuzzyOptions;
   #lastRosEpochs: number;
   #paginationPageSizeLimit: number;
+  #overrideFuzzyOptions: boolean;
 
-  constructor(config: TypeOrmStakePoolProviderProps, deps: TypeormProviderDependencies) {
+  constructor(config: TypeOrmStakePoolProviderProps, deps: TypeormStakePoolProviderDependencies) {
     const { lastRosEpochs, paginationPageSizeLimit } = config;
 
     super('TypeormStakePoolProvider', deps);
+    this.#cache = deps.cache;
+    this.#fuzzyOptions = DEFAULT_FUZZY_SEARCH_OPTIONS;
     this.#paginationPageSizeLimit = paginationPageSizeLimit;
+    this.#overrideFuzzyOptions = true;
 
     // Introduced following code repetition as the correct form is source of a circular-deps:check failure.
     // Solving it would require an invasive refactoring action, probably better to defer it.
@@ -50,9 +74,55 @@ export class TypeormStakePoolProvider extends TypeormProvider implements StakePo
     this.#lastRosEpochs = lastRosEpochs;
   }
 
+  async startImpl() {
+    await super.startImpl();
+
+    await this.withDataSource((dataSource) => this.getFuse(dataSource));
+  }
+
+  private async getFuse(dataSource: DataSource, fuzzyOptions?: FuzzyOptions) {
+    const {
+      threshold,
+      weights: { description, homepage, name, poolId, ticker }
+    } = this.#overrideFuzzyOptions ? { ...this.#fuzzyOptions, ...fuzzyOptions } : this.#fuzzyOptions;
+
+    const cacheKey = this.#overrideFuzzyOptions
+      ? `fuzzy-index-${JSON.stringify([description, homepage, name, threshold, ticker])}`
+      : 'fuzzy-index';
+
+    return this.#cache.get(cacheKey, async () => {
+      const metadata = await this.#cache.get('all-metadata', async () =>
+        dataSource
+          .createQueryBuilder()
+          .from(StakePoolEntity, 'pool')
+          .leftJoinAndSelect('pool.lastRegistration', 'params')
+          .leftJoinAndSelect('params.metadata', 'metadata')
+          .select(['description', 'homepage', 'name', 'pool.id AS pool_id', 'ticker'])
+          .getRawMany<{ description: string; homepage: string; name: string; pool_id: string; ticker: string }>()
+      );
+
+      const opts = {
+        ignoreFieldNorm: true,
+        ignoreLocation: true,
+        includeScore: true,
+        keys: [
+          { name: 'description', weight: description },
+          { name: 'homepage', weight: homepage },
+          { name: 'name', weight: name },
+          { name: 'pool_id', weight: poolId },
+          { name: 'ticker', weight: ticker }
+        ],
+        minMatchCharLength: 3,
+        threshold
+      };
+
+      return new Fuse(metadata, opts, Fuse.createIndex(opts.keys, metadata));
+    });
+  }
+
   // eslint-disable-next-line sonarjs/cognitive-complexity
   public async queryStakePools(options: QueryStakePoolsArgs): Promise<Paginated<Cardano.StakePool>> {
-    const { epochRewards, epochsLength, filters, pagination, sort } = options;
+    const { epochRewards, epochsLength, filters, fuzzyOptions, pagination, sort } = options;
 
     if (pagination.limit > this.#paginationPageSizeLimit) {
       throw new ProviderError(
@@ -72,28 +142,62 @@ export class TypeormStakePoolProvider extends TypeormProvider implements StakePo
       );
     }
 
-    this.logger.debug('About to query projected stake pools');
-
-    const { clause, args } = getWhereClauseAndArgs(filters);
-    const { field, order } = getSortOptions(sort);
+    // eslint-disable-next-line complexity
+    return this.withDataSource(async (dataSource: DataSource) => {
+      const queryRunner = dataSource.createQueryRunner();
+
+      let rawResult: PoolModel[];
+      let sortByScore = false;
+      let textFilter = false;
+
+      try {
+        if (withTextFilter(filters)) {
+          sortByScore = sort === undefined;
+          textFilter = true;
+
+          const values = (await this.getFuse(dataSource, fuzzyOptions))
+            .search(filters.text)
+            .map(({ item: { pool_id }, score }) => `('${pool_id}',${score})`)
+            .join(',');
+
+          await queryRunner.query(
+            [
+              'DROP TABLE IF EXISTS tmp_fuzzy',
+              'CREATE TEMPORARY TABLE tmp_fuzzy (pool_id VARCHAR, score NUMERIC) WITHOUT OIDS',
+              `INSERT INTO tmp_fuzzy VALUES ${values}`
+            ].join(';')
+          );
+        }
 
-    return this.withDataSource<Paginated<Cardano.StakePool>>(async (dataSource: DataSource) => {
-      const rawResult = await dataSource
-        .createQueryBuilder()
-        .from(StakePoolEntity, 'pool')
-        .leftJoinAndSelect('pool.metrics', 'metrics')
-        .leftJoinAndSelect('pool.lastRegistration', 'params')
-        .leftJoinAndSelect('params.metadata', 'metadata')
-        .leftJoin(PoolDelistedEntity, 'delist', 'delist.stakePoolId = pool.id')
-        .select(stakePoolSearchSelection)
-        .addSelect(stakePoolSearchTotalCount)
-        .where(clause, args)
-        .andWhere('delist.stakePoolId IS NULL')
-        .orderBy(field, order, !field.includes('cost') ? nullsInSort : undefined)
-        .addOrderBy('pool.id', 'ASC')
-        .offset(pagination.startAt)
-        .limit(pagination.limit)
-        .getRawMany();
+        this.logger.debug('About to query projected stake pools');
+
+        const { clause, args } = getWhereClauseAndArgs(filters, textFilter);
+        const { field, order } = getSortOptions(sortByScore, sort);
+
+        const queryBuilder1 = dataSource.createQueryBuilder(queryRunner).from(StakePoolEntity, 'pool');
+        const queryBuilder2 = textFilter ? queryBuilder1.innerJoin('tmp_fuzzy', 'tmp', 'id = pool_id') : queryBuilder1;
+
+        rawResult = await queryBuilder2
+          .leftJoinAndSelect('pool.metrics', 'metrics')
+          .leftJoinAndSelect('pool.lastRegistration', 'params')
+          .leftJoinAndSelect('params.metadata', 'metadata')
+          .leftJoin(PoolDelistedEntity, 'delist', 'delist.stakePoolId = pool.id')
+          .select(stakePoolSearchSelection)
+          .addSelect(stakePoolSearchTotalCount)
+          .where(clause, args)
+          .andWhere('delist.stakePoolId IS NULL')
+          .orderBy(field, order, 'NULLS LAST')
+          .addOrderBy('pool.id', 'ASC')
+          .offset(pagination.startAt)
+          .limit(pagination.limit)
+          .getRawMany<PoolModel>();
+      } finally {
+        try {
+          if (textFilter) await queryRunner.query('DROP TABLE IF EXISTS tmp_fuzzy');
+        } finally {
+          await queryRunner.release();
+        }
+      }
 
       const result = mapStakePoolsResult(rawResult);
       const requestedNotStdLength = epochsLength !== undefined && epochsLength !== this.#lastRosEpochs;

packages/cardano-services/src/StakePool/TypeormStakePoolProvider/index.ts

@@ -1 +1,2 @@
 export * from './TypeormStakePoolProvider';
+export { validateFuzzyOptions } from './util';