From a08c5d3b1823cdff6078eb71c37326d9ca8e46d6 Mon Sep 17 00:00:00 2001 From: jachamp <28732543+jimchamp@users.noreply.github.com> Date: Fri, 20 Dec 2024 16:46:24 -0800 Subject: [PATCH 1/2] Set up `requests` for partner imports --- scripts/partner_batch_imports.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/scripts/partner_batch_imports.py b/scripts/partner_batch_imports.py index 2b871f475c0..4782d03ddc8 100755 --- a/scripts/partner_batch_imports.py +++ b/scripts/partner_batch_imports.py @@ -22,6 +22,7 @@ from infogami import config # noqa: F401 side effects may be needed from openlibrary.config import load_config from openlibrary.core.imports import Batch +from openlibrary.plugins.openlibrary.code import setup_requests from scripts.solr_builder.solr_builder.fn_to_cli import FnToCLI logger = logging.getLogger("openlibrary.importer.bwb") @@ -82,6 +83,7 @@ "/openlibrary-client/master/olclient/schemata/import.schema.json" ) +required_fields = None class Biblio: ACTIVE_FIELDS = [ @@ -107,7 +109,6 @@ class Biblio: "width", "height", ] - REQUIRED_FIELDS = requests.get(SCHEMA_URL).json()['required'] NONBOOK = """A2 AA AB AJ AVI AZ BK BM C3 CD CE CF CR CRM CRW CX D3 DA DD DF DI DL DO DR DRM DRW DS DV EC FC FI FM FR FZ GB GC GM GR H3 H5 L3 L5 LP MAC MC MF MG MH ML @@ -116,6 +117,9 @@ class Biblio: VU VY VZ WA WC WI WL WM WP WT WX XL XZ ZF ZZ""".split() def __init__(self, data): + global required_fields + self.REQUIRED_FIELDS = required_fields + self.primary_format = data[6] self.product_type = data[121] assert ( @@ -312,7 +316,11 @@ def batch_import(path, batch, batch_size=5000): def main(ol_config: str, batch_path: str): + global required_fields + load_config(ol_config) + setup_requests() + required_fields = requests.get(SCHEMA_URL).json()['required'] # Partner data is offset ~15 days from start of month date = datetime.date.today() - datetime.timedelta(days=15) From c21761a5a6b6ce5bc5aca03894d7390b3527dfe0 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sat, 21 Dec 2024 00:49:35 +0000 Subject: [PATCH 2/2] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- scripts/partner_batch_imports.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/partner_batch_imports.py b/scripts/partner_batch_imports.py index 4782d03ddc8..2df1613b8a5 100755 --- a/scripts/partner_batch_imports.py +++ b/scripts/partner_batch_imports.py @@ -85,6 +85,7 @@ required_fields = None + class Biblio: ACTIVE_FIELDS = [ 'title',