Skip to content

Commit

Permalink
Make Standard Ebooks use streaming request
Browse files Browse the repository at this point in the history
  • Loading branch information
cdrini committed Jun 4, 2024
1 parent 85142a8 commit a248925
Showing 1 changed file with 7 additions and 5 deletions.
12 changes: 7 additions & 5 deletions scripts/import_standard_ebooks.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,8 +18,9 @@

def get_feed(auth: AuthBase):
"""Fetches and returns Standard Ebook's feed."""
r = requests.get(FEED_URL, auth=auth)
return feedparser.parse(r.text)
with requests.get(FEED_URL, auth=auth, stream=True) as r:
r.raise_for_status()
return feedparser.parse(r.raw, response_headers=r.headers)


def map_data(entry: dict) -> dict[str, Any]:
Expand Down Expand Up @@ -85,11 +86,12 @@ def import_job(
return

auth = HTTPBasicAuth(config.get('standard_ebooks_key'), '')
feed = [map_data(entry) for entry in get_feed(auth).entries]
feed = map(map_data, get_feed(auth).entries)

if not dry_run:
create_batch(feed)
print(f'{len(feed)} entries added to the batch import job.')
list_feed = list(feed)
create_batch(list_feed)
print(f'{len(list_feed)} entries added to the batch import job.')
else:
for record in feed:
print(json.dumps(record))
Expand Down

0 comments on commit a248925

Please sign in to comment.