diff --git a/openlibrary/solr/update.py b/openlibrary/solr/update.py index 710da85f184..47c16e07c8d 100644 --- a/openlibrary/solr/update.py +++ b/openlibrary/solr/update.py @@ -1,3 +1,4 @@ +import functools import logging from pathlib import Path from typing import Literal, cast @@ -35,6 +36,23 @@ data_provider = cast(DataProvider, None) +@functools.cache +def get_solr_updaters() -> list[AbstractSolrUpdater]: + global data_provider + assert data_provider is not None + return [ + # ORDER MATTERS + EditionSolrUpdater(data_provider), + WorkSolrUpdater(data_provider), + AuthorSolrUpdater(data_provider), + ListSolrUpdater(data_provider), + ] + + +def can_update_key(key: str) -> bool: + return any(updater.key_test(key) for updater in get_solr_updaters()) + + async def update_keys( keys: list[str], commit=True, @@ -69,15 +87,7 @@ def _solr_update(update_state: SolrUpdateRequest): net_update = SolrUpdateRequest(commit=commit) - solr_updaters: list[AbstractSolrUpdater] = [ - # ORDER MATTERS - EditionSolrUpdater(data_provider), - WorkSolrUpdater(data_provider), - AuthorSolrUpdater(data_provider), - ListSolrUpdater(data_provider), - ] - - for updater in solr_updaters: + for updater in get_solr_updaters(): update_state = SolrUpdateRequest(commit=commit) updater_keys = uniq(k for k in keys if updater.key_test(k)) await updater.preload_keys(updater_keys) diff --git a/scripts/solr_updater.py b/scripts/solr_updater.py index a8ae54e4693..bebe7692f67 100644 --- a/scripts/solr_updater.py +++ b/scripts/solr_updater.py @@ -222,11 +222,7 @@ async def update_keys(keys): logger.debug("Args: %s" % str(args)) update.load_configs(args['ol_url'], args['ol_config'], 'default') - keys = [ - k - for k in keys - if k.count("/") == 2 and k.split("/")[1] in ("books", "authors", "works") - ] + keys = [k for k in keys if update.can_update_key(k)] count = 0 for chunk in web.group(keys, 100):