Merge branch 'dev'

backend/app/cache/cache.py

@@ -203,6 +203,8 @@ def delete_comment_from_db(self, comment):
             db.session.flush()
 
     def filter_comment(self, comment):
+        if comment.author and self.reddit.is_user_special(comment.author.name):
+            return
         logging.info('Processing comment {}. Thread: {}.'.format(comment.id, comment.submission.url))
         if self.reddit.is_deleted(comment):
             self.delete_comment_from_db(comment)
@@ -278,6 +280,38 @@ def run_edited_stream(self):
                 logging.error(Errors.REDDIT_SERVER_ERROR)
                 sleep(30)
 
+    def scrap_not_scraped(self):
+        session = db.session
+        while True:
+            results = session.query(RedditComment)\
+                .join(SteamUser)\
+                .filter(SteamUser.not_scrapped == True)\
+                .all()
+            for comment in results:
+                submission = self.reddit.get_submission(comment.thread)
+                if 'error' in submission:
+                    continue
+                for scrapped_comment in self.reddit.get_comments(submission['success']):
+                    if comment.comment_id == scrapped_comment.id:
+                        self.filter_comment(scrapped_comment)
+                sleep(5000)
+
+    def run_edited_fallback_stream(self):
+        while True:
+            try:
+                for submission in self.reddit.get_subreddit()\
+                    .new(limit=current_app.config['REDDIT'].SUBMISSIONS_LIMIT):
+                    if not self.reddit.has_required_keywords(submission.title):
+                        continue
+                    logging.info('Processing thread {}.'.format(submission.url))
+                    for comment in submission.comments:
+                        self.filter_comment(comment)
+            except ServerError:
+                logging.error(Errors.REDDIT_SERVER_ERROR)
+                sleep(30)
+                continue
+            sleep(2 * 60 * 60)
+
     def __init__(self):
         self.steam = Steam(current_app.config['STEAM'])
         self.reddit = Reddit(self.steam, current_app.config['REDDIT'])

backend/app/models.py

@@ -75,4 +75,4 @@ def __init__(self, hash=None, eligible=None, winners=None, violators=None, not_e
         self.violators = violators
         self.not_entering = not_entering
         self.thread = thread
-        self.title = title
+        self.title = title

backend/app/reddit.py

@@ -111,7 +111,7 @@ def is_user_special(username):
         return username.find('_bot') != -1 or username == 'AutoModerator'
 
     def get_subreddit(self):
-        return self.subreddit.display_name
+        return self.subreddit
 
     def is_entering(self, comment):
         return self.not_entering not in comment.body.lower()
@@ -138,6 +138,9 @@ def get_submission_title(self, submission):
     def get_redditor_age(self, redditor):
         return datetime.fromtimestamp(self.get_redditor(redditor).created_utc)
 
+    def get_comment(self, id, url):
+        return praw.models.Comment(id, url)
+
     def __init__(self, steam, settings):
         self.steam_api = steam
         self.min_karma = settings.MIN_KARMA

backend/config.py

@@ -9,9 +9,10 @@ class RedditConfig:
     CLIENT_SECRET = os.environ['GOG_PICKER_REDDIT_CLIENT_SECRET']
     USERNAME = os.environ['GOG_PICKER_REDDIT_USERNAME']
     PASSWORD = os.environ['GOG_PICKER_REDDIT_PASSWORD']
-    USER_AGENT = 'python:gog-picker:v0.15.2 (by /u/izdwuut)'
+    USER_AGENT = 'python:gog-picker:v0.16.0 (by /u/izdwuut)'
     NOT_ENTERING = 'not entering'
     MIN_KARMA = 300
+    SUBMISSIONS_LIMIT = 30
 
 
 class SteamConfig:
@@ -42,7 +43,7 @@ class StagingConfig(Config):
 class DevelopmentConfig(Config):
     DEBUG = True
     REDDIT = RedditConfig()
-    REDDIT.SUBREDDIT = 'test'
+    REDDIT.SUBREDDIT = 'GiftofGames'
     REDDIT.MIN_KARMA = 0
     # JWT_ACCESS_TOKEN_EXPIRES = 5
     # JWT_REFRESH_TOKEN_EXPIRES = 30

backend/worker.py

@@ -1,19 +1,32 @@
 from app.cache.cache import GogCache
 from flask.cli import AppGroup
-import click
+import logging
 
 worker_cli = AppGroup('worker')
 
 
 @worker_cli.command("listen")
 def listen():
     gog_cache = GogCache()
-    print('Listens for non-edited comments...')
+    logging.info('Listens for non-edited comments...')
     gog_cache.run_stream()
 
 
 @worker_cli.command('listen-edited')
 def listen_edited():
     gog_cache = GogCache()
+    logging.info('Listens for non-edited comments...')
     gog_cache.run_edited_stream()
 
+@worker_cli.command('scrap-not-scraped')
+def scrap_not_scraped():
+    gog_cache = GogCache()
+    logging.info('Listens for non-scraped comments...')
+    gog_cache.scrap_not_scraped()
+
+@worker_cli.command('listen-edited-fallback')
+def listen_edited_fallback():
+    gog_cache = GogCache()
+    logging.info('Listens for non-edited comments (fallback)...')
+    gog_cache.run_edited_fallback_stream()
+

frontend/src/app/thread/thread.component.html

@@ -10,6 +10,12 @@ <h1>Scrapped comments</h1>
             Non entering comments and comments with errors are displayed at the bottom. They are automatically excluded
             as they can't participate in a drawing. Duplicate entries are discarded later on.
         </p>
+        Legend:
+        <ul class="legend">
+            <li><span class="yellow-highlight">warning</span></li>
+            <li><span class="gray-highlight">not entering</span></li>
+            <li><span class="red-highlight">violator</span></li>
+        </ul>
     </div>
     <div class="actions" *ngIf='comments && comments.length > 0'>
         <div>
@@ -84,7 +90,7 @@ <h1>Scrapped comments</h1>
             </mat-card>
         </div>
         <div *ngFor="let comment of comments">
-            <mat-card class="comment error" *ngIf="hasErrors(comment) && comment.entering">
+            <mat-card class="comment error" *ngIf="hasErrors(comment) && comment.entering && !hasWarnings(comment)">
                 <mat-card-title>
                     <a href="https://www.reddit.com/user/{{ comment.author.name }}"
                         target="_blank">{{ comment.author.name }}</a>

frontend/src/app/thread/thread.component.scss

@@ -13,8 +13,6 @@
 
   @media only screen and (max-width: $largeTabletWidth) {
     grid-template-columns: 0.25fr 2fr 0.25fr;
-
-
   }
 }
 
@@ -67,3 +65,22 @@
 .no-results {
   color: red;
 }
+
+.yellow-highlight {
+  background-color: #fafa0b;
+  padding: 5px;
+}
+
+.gray-highlight {
+  background-color: #b3b3b3;
+  padding: 5px;
+}
+
+.red-highlight {
+  background-color: #ff6464;
+  padding: 5px;
+}
+
+.legend li {
+  padding: 6px;
+}

frontend/src/app/thread/thread.component.ts

@@ -196,11 +196,11 @@ export class ThreadComponent implements OnInit, OnDestroy, AfterViewInit {
   getWarnings(comment: RedditComment): String[] {
     let warnings = Array<String>()
     if (!this.canScrapSteamProfile(comment)) {
-      warnings.push("couldn't scrap comment/inaccessible Steam profile")
+      warnings.push("couldn't scrap comment. Please check Steam profile manually.")
       return warnings
     }
     if (!comment.steamProfile.steamId) {
-      warnings.push('no Steam profile detected/inaccessible profile')
+      warnings.push('no Steam profile detected/inaccessible profile. Please check it manually.')
     }
     if (comment.steamProfile && comment.steamProfile.gamesCount >= environment.hoarderNumber) {
       warnings.push('potential hoarder (' + comment.steamProfile.gamesCount + ' games)')

frontend/src/index.html

@@ -12,7 +12,7 @@
 <body>
   <app-root></app-root>
   <footer class="footer">
-    Gift of Games Picker for Reddit <a href="https://github.com/izdwuut/gog-picker">v0.15.2</a>. 
+    Gift of Games Picker for Reddit <a href="https://github.com/izdwuut/gog-picker">v0.16.0</a>. 
     Scripted by <a href="https://reddit.com/u/izdwuut">u/izdwuut</a> and <a href="https://github.com/izdwuut/gog-picker/pulls">friends</a> in open source.
     Licensed under <a href="https://github.com/izdwuut/gog-picker/blob/dev/LICENSE">MIT Licence</a>
   </footer>