Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

new-solr-updater.py: Replace urlopen with requests.get #4605

Closed
wants to merge 2 commits into from
Closed

new-solr-updater.py: Replace urlopen with requests.get #4605

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
952dc89
new-solr-updater.py: Replace urlopen with requests.get
cclauss Feb 15, 2021
e25903c
new-solr-updater.py: Replace urlopen with requests.get
cclauss Feb 15, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
42 changes: 22 additions & 20 deletions scripts/new-solr-updater.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,21 +8,22 @@
"""
import _init_path

from six.moves import urllib
import yaml
import logging
import json
import argparse
import datetime
import time
import web
import sys
import json
import logging
import re
import socket
import sys
import time

import requests
import web
import yaml

from openlibrary.solr import update_work
from openlibrary.config import load_config
from infogami import config
from openlibrary.config import load_config
from openlibrary.solr import update_work

logger = logging.getLogger("openlibrary.solr-updater")

Expand Down Expand Up @@ -77,19 +78,20 @@ def read_records(self, max_fetches=10):
url = "%s/%s?limit=100" % (self.base_url, self.offset)
logger.debug("Reading log from %s", url)
try:
jsontext = urllib.request.urlopen(url).read()
except urllib.error.URLError as e:
logger.error("Failed to open URL %s", url, exc_info=True)
if e.args and e.args[0].args == (111, 'Connection refused'):
logger.error('make sure infogami server is working, connection refused from %s', url)
response = requests.get(url)
response.raise_for_status()
d = response.json()
except requests.HTTPError:
logger.exception("Failed to open URL %s" % url)
if response.status_code == 111:
logger.error(
'make sure infogami server is working, connection refused '
'from %s' % url
)
sys.exit(1)
raise

try:
d = json.loads(jsontext)
except:
logger.error("Bad JSON: %s", jsontext)
raise
except json.JSONDecodeError:
logger.exception("Bad JSON: %s" % response.content)
data = d['data']
# no more data is available
if not data:
Expand Down