Merge pull request #23 from hammerhead226/emmercm/updates

Updates
hammerhead226 · Apr 20, 2018 · ea3bfdc · ea3bfdc
2 parents 2bef6d9 + 799bea8
commit ea3bfdc
Show file tree

Hide file tree

Showing 12 changed files with 154 additions and 51 deletions.
diff --git a/SharkScout.py b/SharkScout.py
@@ -33,8 +33,10 @@ def goodbye():
     parser.add_argument('-nb', '--no-browser', dest='browser', help='don\'t automatically open the web browser', action='store_false', default=True)
     parser.add_argument('-ut', '--update-teams', dest='update_teams', help='update TBA team list', action='store_true', default=False)
     parser.add_argument('-uti', '--update-teams-info', dest='update_teams_info', help='update TBA team info', action='store_true', default=False)
+    parser.add_argument('-utf', '--update-teams-favicon', dest='update_teams_favicon', help='update team website\'s favicon when updating team info', action='store_true', default=False)
     parser.add_argument('-ue', '--update-events', metavar='year', dest='update_events', help='update all TBA events in a year', type=pynumparser.NumberSequence(limits=(1992, date.today().year+1)))
     parser.add_argument('-uei', '--update-events-info', metavar='year', dest='update_events_info', help='update all TBA event info in a year', type=pynumparser.NumberSequence(limits=(1992, date.today().year+1)))
+    parser.add_argument('-uef', '--update-events-favicon', dest='update_events_favicon', help='update event website\'s favicon when updating event info', action='store_true', default=False)
     parser.add_argument('-d', '--dump', metavar='file', help='run mongodump after any update(s)', type=str)
     parser.add_argument('-r', '--restore', metavar='file', help='run mongorestore before any update(s)', type=argparse.FileType('r'))
     args = parser.parse_args()
@@ -78,9 +80,9 @@ def goodbye():
         mongo.teams_update()
         print()
     if args.update_teams_info:
-        print('Updating team info ...')
+        print('Updating teams ...')
         with concurrent.futures.ThreadPoolExecutor(max_workers=3) as pool:
-            futures = {pool.submit(mongo.team_update, team['key']): team for team in mongo.teams()}
+            futures = {pool.submit(mongo.team_update, team['key'], args.update_teams_favicon): team for team in mongo.teams()}
             for future in tqdm(concurrent.futures.as_completed(futures), total=len(futures), unit='team', leave=True):
                 future.result()
         print()
@@ -96,7 +98,7 @@ def goodbye():
     if args.update_events_info:
         for year in sorted(args.update_events_info):
             with concurrent.futures.ThreadPoolExecutor(max_workers=3) as pool:
-                futures = {pool.submit(mongo.event_update, event['key']): event for event in mongo.events(year)}
+                futures = {pool.submit(mongo.event_update, event['key'], args.update_events_favicon): event for event in mongo.events(year)}
                 if futures:
                     print('Updating ' + str(year) + ' events ...')
                     for future in tqdm(concurrent.futures.as_completed(futures), total=len(futures), unit='event', leave=True):

diff --git a/scripts/odroid-c.sh b/scripts/odroid-c.sh
@@ -23,9 +23,10 @@ cd "$(dirname "$0")"
 	sudo rm /etc/localtime
 	sudo ln -s /usr/share/zoneinfo/US/Eastern /etc/localtime
 
-	# Turn off startup fsck (potentially dangerous)
-	sudo sed -i 's/1$/0/g' /etc/fstab
-	sudo tune2fs -c 0 -i 0 -l /dev/mmcblk0p2
+	# Drastically shorten all service startups
+	for FILE in $(sudo grep --recursive --files-with-matches 5min /*/systemd/ 2>/dev/null); do
+		sudo sed --in-place 's/TimeoutStartSec=.\+/TimeoutStartSec=30sec/g' "${FILE}"
+	done
 
 	# ODROID C1/C2 LCD
 	if [ "$(systemctl | grep odroid-lcd35)" == "" ]; then
@@ -50,6 +51,7 @@ cd "$(dirname "$0")"
 
 	# panr + bt-pan
 	sudo apt-get -y install bluez bridge-utils ipcalc python python-dbus udhcpd
+	sudo update-rc.d udhcpd disable
 	wget -N https://raw.githubusercontent.com/mk-fg/fgtk/master/bt-pan
 	chmod +x bt-pan
 	wget -N https://raw.githubusercontent.com/emmercm/panr/master/panr
@@ -91,7 +93,9 @@ cd "$(dirname "$0")"
 (
 	sleep 10s
 	cd SharkScout
-	sed -i 's/\r\n/\n/g' *.py
+	for FILE in $(grep --recursive --files-with-matches "\r" *.py); do
+		sed --in-place 's/\r//g' "${FILE}"
+	done
 	while [ "" == "" ]; do
 		sudo ./SharkScout.py --port 80 --no-browser &> /dev/null
 	done

diff --git a/setup.py b/setup.py
@@ -1,7 +1,11 @@
 #!/usr/bin/env python3
 
 import argparse
-import pip
+
+try:
+    from pip import main as pipmain
+except:
+    from pip._internal import main as pipmain
 
 
 if __name__ == '__main__':
@@ -10,7 +14,7 @@
     args = parser.parse_args()
 
     if args.command == 'install':
-        pip.main([
+        pipmain([
             'install',
             'backoff',
             'cherrypy',

diff --git a/sharkscout/mongo.py b/sharkscout/mongo.py
@@ -205,9 +205,9 @@ def event(self, event_key):
                     for alliance in match['alliances']:
                         if match['key'] in scouting_matches and alliance in scouting_matches[match['key']]['alliances']:
                             if 'team_keys' in match['alliances'][alliance]:
-                                match['alliances'][alliance]['team_keys'] = list(set(match['alliances'][alliance]['team_keys'] + scouting_matches[match['key']]['alliances'][alliance]['teams']))
+                                match['alliances'][alliance]['team_keys'] += [t for t in scouting_matches[match['key']]['alliances'][alliance]['teams'] if t not in match['alliances'][alliance]['team_keys']]
                             if 'teams' in match['alliances'][alliance]:
-                                match['alliances'][alliance]['teams'] = list(set(match['alliances'][alliance]['teams'] + scouting_matches[match['key']]['alliances'][alliance]['teams']))
+                                match['alliances'][alliance]['teams'] += [t for t in scouting_matches[match['key']]['alliances'][alliance]['teams'] if t not in match['alliances'][alliance]['teams']]
                     # Attach scouting data to matches
                     if match['key'] in scouting_match_teams:
                         match['scouting'] = scouting_match_teams[match['key']]
@@ -248,13 +248,14 @@ def events_update(self, year):
             pass  # "No operations to execute"
 
     # TBA update an individual event
-    def event_update(self, event_key):
+    def event_update(self, event_key, update_favicon=False):
         event = self.tba_api.event(event_key, True)
         if event:
             # Info that can be known before an event starts
             event.update({k:v for k, v in {
                 'teams': sorted([t['key'] for t in self.tba_api.event_teams(event_key)]),
-                'matches': self.tba_api.event_matches(event_key)
+                'matches': self.tba_api.event_matches(event_key),
+                'favicon': sharkscout.Util.favicon(event['website'] if 'website' in event else '') if update_favicon else None
             }.items() if v})
             # Info that can't be known before an event starts
             if not event['start_date'] or datetime.strptime(event['start_date'],'%Y-%m-%d').date() <= date.today():
@@ -264,13 +265,13 @@ def event_update(self, event_key):
                     'awards': self.tba_api.event_awards(event_key),
                     'alliances': self.tba_api.event_alliances(event_key)
                 }.items() if v})
-        event['modified_timestamp'] = datetime.utcnow()
-        self.tba_events.update_one({
-            'key': event_key
-        }, {
-            '$set': event,
-            '$setOnInsert': {'created_timestamp': datetime.utcnow()}
-        }, upsert=True)
+            event['modified_timestamp'] = datetime.utcnow()
+            self.tba_events.update_one({
+                'key': event_key
+            }, {
+                '$set': event,
+                '$setOnInsert': {'created_timestamp': datetime.utcnow()}
+            }, upsert=True)
 
     # List of matches with scouting data
     def scouting_matches(self, event_key):
@@ -632,20 +633,22 @@ def team(self, team_key, year=None):
             return {}
 
     # TBA update an individual team
-    def team_update(self, team_key):
+    def team_update(self, team_key, update_favicon=False):
         team = self.tba_api.team(team_key, True)
         if team:
             team.update({k:v for k, v in {
                 'awards': self.tba_api.team_history_awards(team_key),
-                'districts': {str(d['year']): d for d in self.tba_api.team_districts(team_key)}
+                'districts': {str(d['year']): d for d in self.tba_api.team_districts(team_key)},
+                'favicon': sharkscout.Util.favicon(team['website'] if 'website' in team else '') if update_favicon else None,
+                'media': self.tba_api.team_media(team_key)
             }.items() if v})
-        team['modified_timestamp'] = datetime.utcnow()
-        self.tba_teams.update_one({
-            'key': team_key
-        }, {
-            '$set': team,
-            '$setOnInsert': {'created_timestamp': datetime.utcnow()}
-        }, upsert=True)
+            team['modified_timestamp'] = datetime.utcnow()
+            self.tba_teams.update_one({
+                'key': team_key
+            }, {
+                '$set': team,
+                '$setOnInsert': {'created_timestamp': datetime.utcnow()}
+            }, upsert=True)
 
     # Years that a team competed
     def team_stats(self, team_key):
@@ -680,5 +683,5 @@ def team_events(self, team_key, year):
 
     # TBA update all events a team is attending in a given year
     def team_update_events(self, team_key, year):
-        for event in self.tba_api.team_events(team_key, int(year)):
+        for event in self.tba_api.team_events(team_key, int(year), True):
             self.event_update(event['key'])
diff --git a/sharkscout/thebluealliance.py b/sharkscout/thebluealliance.py
@@ -179,7 +179,11 @@ def team_years_participated(self, team_key, ignore_cache=False):
         return self._get('team/' + team_key + '/years_participated', ignore_cache)
 
     def team_media(self, team_key, year=None, ignore_cache=False):
-        return self._get('team/' + team_key + '/media' + ('/' + str(year) if year else ''), ignore_cache)
+        if year is None:
+            year = date.today().year
+        media = self._get('team/' + team_key + '/media/' + str(year), ignore_cache)
+        media = {m['type']:m for m in media}
+        return media
 
     def team_robots(self, team_key, ignore_cache=False):
         return self._get('team/' + team_key + '/robots', ignore_cache)

diff --git a/sharkscout/util.py b/sharkscout/util.py
@@ -1,12 +1,30 @@
+import base64
 import collections
 import os
 import psutil
 import re
 import socket
 import string
+import urllib
+
+import requests
 
 
 class Util(object):
+    @staticmethod
+    def favicon(url):
+        if url:
+            try:
+                response = requests.get('https://www.google.com/s2/favicons', {'domain':url}, stream=True)
+                image = base64.b64encode(response.raw.read()).decode()
+                if image not in [
+                    'iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAAABHNCSVQICAgIfAhkiAAAABJJREFUOI1jYBgFo2AUjAIIAAAEEAABf014jgAAAABJRU5ErkJggg=='  # transparent 16x16 PNG
+                ]:
+                    return 'data:' + response.headers['Content-Type'] + ';base64,' + image
+            except Exception:
+                pass
+        return None
+
     @staticmethod
     def flatten(iterable):
         for item in iterable:

diff --git a/stats/2018.json b/stats/2018.json
@@ -285,7 +285,7 @@
                     "then": {"$concat": [
                         "$matches.number",
                         " ",
-                        "$matches.comments_offense"
+                        {"$substr": ["$matches.comments_offense", 0, -1]}
                     ]},
                     "else": ""
                 }
@@ -296,7 +296,7 @@
                     "then": {"$concat": [
                         "$matches.number",
                         " ",
-                        "$matches.comments_defense"
+                        {"$substr": ["$matches.comments_defense", 0, -1]}
                     ]},
                     "else": ""
                 }

diff --git a/www/event.html b/www/event.html
@@ -60,7 +60,7 @@
                     <span class="badge">${len(page['stats'])}</span>
                 </a>
             </li>
-            <li role="presentation" py:if="'scatter' in page and page['scatter']">
+            <li role="presentation" py:if="'scatter' in page and 'dataset' in page['scatter'] and page['scatter']['dataset']">
                 <a href="#comparison" role="tab" data-toggle="tab">
                     <span class="far fa-chart-bar"></span>&nbsp;&nbsp;Comparison
                     <span class="badge">${len(page['scatter']['dataset'])}</span>
@@ -149,6 +149,7 @@
                             <td py:for="team_key in alliance['picks']" py:choose="">
                                 <py:when test="'teams' in page['event'] and page['event']['teams'] and [t for t in page['event']['teams'] if t['key'] == team_key]">
                                     <py:with vars="team = [t for t in page['event']['teams'] if t['key'] == team_key][0]">
+                                        <img src="data:image/png;base64,${team['media']['avatar']['details']['base64Image']}" class="favicon" py:if="'media' in team and 'avatar' in team['media']"></img>
                                         <a href="/team/${team['key']}/${page['event']['year']}">${team['team_number']} - ${team['nickname']}</a>
                                         <span class="fas fa-home" title="Home Team" py:if="int(team['team_number']) == int(session['team_number'] or -1)"></span>
                                     </py:with>
@@ -196,7 +197,7 @@
                     </div>
                 </div>
             </div>
-            <div id="comparison" role="tabpanel" class="tab-page" py:if="'scatter' in page and page['scatter']">
+            <div id="comparison" role="tabpanel" class="tab-page" py:if="'scatter' in page and 'dataset' in page['scatter'] and page['scatter']['dataset']">
                 <div class="panel panel-default">
                     <div class="panel-heading clearfix">
                         <span class="far fa-chart-bar"></span>&nbsp;&nbsp;Scouting Comparison -
@@ -271,7 +272,10 @@
                                         <py:choose>
                                             <py:when test="recipient['team_number'] and 'teams' in page['event'] and page['event']['teams']">
                                                 <py:choose py:with="teams = [t for t in page['event']['teams'] if int(t['team_number']) == int(recipient_team_number_normalized)]; team = teams[0] if len(teams) > 0 else []">
-                                                    <a href="/team/${team['key']}/${page['event']['year']}" py:when="team">${recipient_team_number_normalized} - ${team['nickname']}</a>
+                                                    <py:when test="team">
+                                                        <img src="data:image/png;base64,${team['media']['avatar']['details']['base64Image']}" class="favicon" py:if="'media' in team and 'avatar' in team['media']"></img>&nbsp;
+                                                        <a href="/team/${team['key']}/${page['event']['year']}">${recipient_team_number_normalized} - ${team['nickname']}</a>
+                                                    </py:when>
                                                     <py:otherwise>${recipient_team_number_normalized}</py:otherwise>
                                                 </py:choose>
                                             </py:when>