Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Remove duplicate version of archive_last_thousand_comments from comme… #41

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open

Remove duplicate version of archive_last_thousand_comments from comme… #41

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
52 changes: 0 additions & 52 deletions app/controllers/comment_controller.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -114,55 +114,3 @@ def archive_last_thousand_comments(self, subreddit_name):
subreddit_name = subreddit.name,
num_comments = len(db_comments)
))

def archive_last_thousand_comments(self, subreddit_name):
# fetch the subreddit ID
subreddit = self.db_session.query(Subreddit).filter(Subreddit.name == subreddit_name).first()

# fetch the last thousand comment IDs
comment_ids = [x['id'] for x in self.db_session.execute(text("select id from comments WHERE subreddit_id='{0}' ORDER BY created_utc DESC LIMIT 1000;".format(subreddit.id)))]

# fetch comments from reddit
comments = []
try:
limit_found = False
after_id = None
while(limit_found == False):
comment_result = self.r.get_comments(subreddit = subreddit_name, params={"after":after_id}, limit=100)
comments_returned = 0
for comment in comment_result:
comments_returned += 1
if(os.environ['CS_ENV'] !='test'):
comment = comment.json_dict
if(comment['id'] in comment_ids):
limit_found = True
else:
comments.append(comment)
after_id = "t1_" + comment['id']
if(comment_result is None or comments_returned == 0 ):
limit_found = True
except praw.errors.APIException:
self.log.error("Error querying latest {subreddit_name} comments from reddit API. Immediate attention needed.".format(subreddit_name=subreddit_name))
sys.exit(1)

db_comments = []
for comment in comments:
if((comment['id'] in comment_ids) != True):
db_comment = Comment(
id = comment['id'],
subreddit_id = subreddit.id,
created_utc = datetime.datetime.utcfromtimestamp(comment['created_utc']),
post_id = comment['link_id'].replace("t3_" ,""),
user_id = comment['author'],
comment_data = json.dumps(comment)
)
db_comments.append(db_comment)
try:
self.db_session.add_all(db_comments)
self.db_session.commit()
except sqlalchemy.exc.DBAPIError as e:
self.log.error("Error saving {0} comments to database. Immediate attention needed. Error: {1}".format(len(db_comments)),str(e))
self.log.info("Fetching up to the last thousand comments in {subreddit_name}. Total comments archived: {num_comments}".format(
subreddit_name = subreddit.name,
num_comments = len(db_comments)
))