Merge pull request #408 from Plant-Tracer/dev-database

backs up database

.github/workflows/continuous-integration-pip.yml

@@ -84,7 +84,7 @@ jobs:
       - name: Lint with pylint
         run: |
           make pylint
- 
+
       - name: JavaScript and HTML eslint
         run: |
           make eslint
@@ -93,18 +93,22 @@ jobs:
         if: startsWith(matrix.os, 'ubuntu')
         env:
           MYSQL_ROOT_PASSWORD: testrootpass
+          PLANTTRACER_CREDENTIALS: etc/credentials.ini
         run: |
           echo the following assumes that the root configuration is in etc/github_actions_mysql_rootconfig.ini
           make create_localdb
 
       - name: Validate app framework
+        env:
+          PLANTTRACER_CREDENTIALS: etc/credentials.ini
         run: |
           make pytest-app-framework
 
       - name: Run coverage test
         if: startsWith(matrix.os, 'ubuntu')
         env:
           SKIP_ENDPOINT_TEST:   ${{ vars.SKIP_ENDPOINT_TEST }}
+          PLANTTRACER_CREDENTIALS: etc/credentials.ini
         run: |
           CHROME_PATH=${{ steps.setup-chrome.outputs.chrome-path }} make coverage
 
@@ -117,10 +121,14 @@ jobs:
           files: ./coverage.xml
 
       - name: Show files in directory
+        env:
+          PLANTTRACER_CREDENTIALS: etc/credentials.ini
         run: |
           make coverage
           ls -l
 
       - name: Run JavaScript tests and coverage
+        env:
+          PLANTTRACER_CREDENTIALS: etc/credentials.ini
         run: |
           make jscoverage

Makefile

@@ -138,18 +138,22 @@ debug:
 	make debug-local
 
 DEBUG:=$(PYTHON) bottle_app.py --loglevel DEBUG
-debug-multi:
-	@echo run bottle locally in debug mode multi-threaded
-	$(DEBUG)  --dbcredentials etc/credentials.ini --multi
-
 debug-single:
 	@echo run bottle locally in debug mode single-threaded
 	$(DEBUG)  --dbcredentials etc/credentials.ini
 
+debug-multi:
+	@echo run bottle locally in debug mode multi-threaded
+	$(DEBUG)  --dbcredentials etc/credentials.ini --multi
+
 debug-local:
-	@echo run bottle locally in debug mode
+	@echo run bottle locally in debug mode, storing new data in database
 	$(DEBUG) --storelocal --dbcredentials etc/credentials-local.ini
 
+debug-dev:
+	@echo run bottle locally in debug mode, storing new data in S3, with the dev.planttracer.com database
+	$(DEBUG) --dbcredentials etc/credentials-dev.ini
+
 freeze:
 	$(PYTHON) -m pip freeze > requirements.txt
 

auth.py

@@ -5,16 +5,19 @@
 * Database authentication
 * Mailer authentication
 """
+import os
 import functools
 import configparser
-#import logging
+import logging
 
 import bottle
 from bottle import request
 
 import paths
-
 from lib.ctools import dbfile
+
+from constants import C
+
 API_KEY_COOKIE_BASE = 'api_key'
 COOKIE_MAXAGE = 60*60*24*180
 SMTP_ATTRIBS = ['SMTP_USERNAME','SMTP_PASSWORD','SMTP_PORT','SMTP_HOST']
@@ -25,10 +28,13 @@
 
 
 def credentials_file():
-    if paths.running_in_aws_lambda():
-        return paths.AWS_CREDENTIALS_FILE
-    else:
-        return paths.CREDENTIALS_FILE
+    try:
+        name = os.environ[ C.PLANTTRACER_CREDENTIALS ]
+    except KeyError as e:
+        raise RuntimeError(f"Environment variable {C.PLANTTRACER_CREDENTIALS} must be defined") from e
+    if not os.path.exists(name):
+        raise FileNotFoundError(name)
+    return name
 
 def smtp_config():
     """Get the smtp config from the [smtp] section of a credentials file.
@@ -59,6 +65,7 @@ def get_dbwriter():
     1 - the [dbwriter] section of the file specified by the DBCREDENTIALS_PATH environment variable if it exists.
     2 - the [dbwriter] section of the file etc/credentials.ini
     """
+    logging.debug("get_dbwriter. credentials_file=%s",credentials_file())
     return dbfile.DBMySQLAuth.FromConfigFile( credentials_file(), 'dbwriter')
 
 
@@ -110,11 +117,3 @@ def get_user_ipaddr():
 def get_param(k, default=None):
     """Get param v from the reqeust"""
     return request.query.get(k, request.forms.get(k, default))
-
-#def get_movie_id():
-#    movie_id = get_param('movie_id', None)
-#    if movie_id is not None:
-#        return movie_id
-#    raise bottle.HTTPResponse(body=json.dumps(INVALID_MOVIE_ID),
-#                              status=200,
-#                              headers={ 'Content-type': 'application/json'})

bottle_api.py

@@ -104,7 +104,7 @@ def get_user_id(allow_demo=True):
         logging.info("demo account blocks requeted action")
         raise bottle.HTTPResponse(
             body='{"Error":true,"message":"demo accounts not allowed to execute requested action."}',
-            status=503, headers={ 'Location': '/'})
+            status=404)
     return userdict['id']
 
 
@@ -117,6 +117,7 @@ def get_user_dict():
     """Returns the user_id of the currently logged in user, or throws a response"""
     logging.debug("get_user_dict(). request.url=%s",request.url)
     api_key = auth.get_user_api_key()
+    logging.debug("get_user_dict(). api_key=%s",api_key)
     if api_key is None:
         logging.info("api_key is none or invalid. request=%s",bottle.request.fullpath)
         if bottle.request.fullpath.startswith('/api/'):
@@ -127,15 +128,9 @@ def get_user_dict():
         raise bottle.HTTPResponse(body='', status=301, headers={ 'Location': '/'})
     userdict = db.validate_api_key(api_key)
     if not userdict:
-        logging.info("api_key %s is invalid  ipaddr=%s request.url=%s",
-                     api_key,request.environ.get('REMOTE_ADDR'),request.url)
+        logging.info("api_key %s is invalid  ipaddr=%s request.url=%s", api_key,request.environ.get('REMOTE_ADDR'),request.url)
         auth.clear_cookie()
-        # This will produce a "Session expired" message
-        if request.url.endswith("/error"):
-            logging.debug("/error so error 301 with /logout")
-            raise bottle.HTTPResponse(body='', status=301, headers={ 'Location': '/logout'})
-        logging.debug("no /error so error 301 with /logout")
-        raise bottle.HTTPResponse(body='', status=301, headers={ 'Location': '/error'})
+        raise bottle.HTTPResponse(body=f'Error 404: api_key {api_key} is invalid. ', status=404)
     return userdict
 
 ################################################################
@@ -369,23 +364,28 @@ def api_get_movie_data():
     :return:  IF MOVIE IS IN S3 - Redirect to a signed URL.
               IF MOVIE IS IN DB - The raw movie data as a movie.
     """
+    logging.debug("api_get_movie_data")
     try:
         movie_id = get_int('movie_id')
         movie = db.Movie(movie_id, user_id=get_user_id())
-    except db.UnauthorizedUser as e:
-        raise bottle.HTTPResponse(body=f'user={get_user_id()} movie_id={movie_id}', status=404) from e
+    except (db.UnauthorizedUser,bottle.HTTPResponse) as e:
+        logging.debug("user authentication error=%s",e)
+        return bottle.HTTPResponse(body=f'user={get_user_id()} movie_id={movie_id}', status=403)
 
     # If we have a movie, return it
     if movie.data is not None:
         bottle.response.set_header('Content-Type', movie.mime_type)
         return movie.data
 
     # Looks like we need a url
-    url = movie.url()
+    if movie.url is None:
+        logging.debug("no movie data for movie_id %s",movie_id)
+        return bottle.HTTPResponse(body=f'user={get_user_id()} movie_id={movie_id}', status=404)
+
     if get_bool('redirect_inline'):
         return "#REDIRECT " + url
-    logging.info("Redirecting movie_id=%s to %s",movie.movie_id, url)
-    return bottle.redirect(url)
+    logging.info("Redirecting movie_id=%s to %s",movie.movie_id, movie.url)
+    return bottle.redirect(movie.url)
 
 def set_movie_metadata(*,user_id, set_movie_id,movie_metadata):
     """Update the movie metadata."""
@@ -974,6 +974,7 @@ def api_ver():
 ## Demo and debug
 ##
 @api.route('/add', method=GET_POST)
+#@api.route('/get-movie-data', method=GET_POST)
 def api_add():
     a = get_float('a')
     b = get_float('b')

bottle_app.py

@@ -61,6 +61,7 @@
 import db_object
 import auth
 
+import paths
 from paths import view, STATIC_DIR
 from constants import C,__version__,GET,GET_POST
 
@@ -130,12 +131,12 @@ def page_dict(title='', *, require_auth=False, lookup=True, logout=False,debug=F
     :param: logout - if true, force the user to log out by issuing a clear-cookie command
     :param: lookup - if true, We weren't being called in an error condition, so we can lookup the api_key in the URL
     """
-    logging.debug("page_dict require_auth=%s logout=%s lookup=%s",require_auth,logout,lookup)
+    logging.debug("1. page_dict require_auth=%s logout=%s lookup=%s",require_auth,logout,lookup)
     o = urlparse(request.url)
     logging.debug("o=%s",o)
     if lookup:
         api_key = auth.get_user_api_key()
-        logging.info("auth.get_user_api_key=%s",api_key)
+        logging.debug("auth.get_user_api_key=%s",api_key)
         if api_key is None and require_auth is True:
             logging.debug("api_key is None and require_auth is True")
             raise bottle.HTTPResponse(body='', status=303, headers={ 'Location': '/'})
@@ -206,8 +207,8 @@ def page_dict(title='', *, require_auth=False, lookup=True, logout=False,debug=F
 @view('index.html')
 def func_root():
     """/ - serve the home page"""
+    logging.debug("func_root")
     ret = page_dict()
-    logging.info("func_root")
     if DEMO_MODE:
         demo_users = db.list_demo_users()
         demo_api_key = False
@@ -239,6 +240,7 @@ def func_audit():
 @view('list.html')
 def func_list():
     """/list - list movies and edit them and user info"""
+    logging.debug("/list")
     return page_dict('List Movies', require_auth=True)
 
 @bottle.route('/analyze', method=GET)
@@ -343,16 +345,30 @@ def demo_tracer1():
     parser = argparse.ArgumentParser(description="Run Bottle App with Bottle's built-in server unless a command is given",
                                      formatter_class=argparse.ArgumentDefaultsHelpFormatter)
 
-    parser.add_argument( '--dbcredentials', help='Specify .ini file with [dbreader] and [dbwriter] sections')
+    parser.add_argument('--dbcredentials', help='Specify .ini file with [dbreader] and [dbwriter] sections')
     parser.add_argument('--port', type=int, default=8080)
     parser.add_argument('--multi', help='Run multi-threaded server (no auto-reloader)', action='store_true')
     parser.add_argument('--storelocal', help='Store new objects locally, not in S3', action='store_true')
+    parser.add_argument("--info", help='print info about the runtime environment', action='store_true')
     clogging.add_argument(parser, loglevel_default='WARNING')
     args = parser.parse_args()
     clogging.setup(level=args.loglevel)
 
+    if args.info:
+        for name in logging.root.manager.loggerDict:
+            print("Logger: ",name)
+        exit(0)
+
+    if args.loglevel=='DEBUG':
+        # even though we've set the main loglevel to be debug, set the other loggers to a different log level
+        for name in logging.root.manager.loggerDict:
+            if name.startswith('boto'):
+                logging.getLogger(name).setLevel(logging.INFO)
+
+
     if args.dbcredentials:
-        os.environ[C.DBCREDENTIALS_PATH] = args.dbcredentials
+        paths.FORCE_CREDENTIALS_FILE = args.dbcredentials
+
 
     if args.storelocal:
         db_object.STORE_LOCAL=True

constants.py

@@ -12,8 +12,9 @@
 
 class C:
     """Constants"""
+    AWS_LAMBDA_ENVIRON = 'AWS_LAMBDA'
     TRACKING_COMPLETED='TRACKING COMPLETED' # keep case; it's used as a flag
-    DBCREDENTIALS_PATH = 'DBCREDENTIALS_PATH'
+    PLANTTRACER_CREDENTIALS = 'PLANTTRACER_CREDENTIALS'
     MAX_FILE_UPLOAD = 1024*1024*64
     MAX_FRAMES = 1e6            # max possible frames in a movie
     NOTIFY_UPDATE_INTERVAL = 5.0

db.py

@@ -175,7 +175,7 @@ def validate_api_key(api_key):
                               where api_key=%s and api_keys.enabled=1 and users.enabled=1 LIMIT 1""",
                               (api_key, ), asDicts=True)
 
-    logging.debug("validate_api_key(%s)=%s",api_key,ret)
+    logging.debug("validate_api_key(%s)=%s dbreader=%s",api_key,ret,get_dbreader())
     if ret:
         dbfile.DBMySQL.csfr(get_dbwriter(),
                             """UPDATE api_keys
@@ -555,6 +555,7 @@ def get_movie_metadata(*,user_id, movie_id, get_last_frame_tracked=False):
                     A.date_uploaded AS date_uploaded, A.mtime AS mtime, A.version AS version,
                     A.fps as fps, A.width as width, A.height as height,
                     A.total_frames AS total_frames, A.total_bytes as total_bytes, A.status AS status,
+                    A.movie_data_urn as movie_data_urn,
                     B.id AS tracked_movie_id
              FROM movies A
              LEFT JOIN movies B on A.id=B.orig_movie
@@ -706,6 +707,10 @@ def delete_movie(*,movie_id, delete=1):
 def course_id_for_movie_id(movie_id):
     return get_movie_metadata(user_id=0, movie_id=movie_id)[0]['course_id']
 
+@functools.lru_cache(maxsize=128)
+def movie_data_urn_for_movie_id(movie_id):
+    return get_movie_metadata(user_id=0, movie_id=movie_id)[0]['movie_data_urn']
+
 @log
 def movie_frames_info(*,movie_id):
     """Gets information about movie frames"""
@@ -1265,37 +1270,40 @@ class Movie():
     Not used for creating movies, but can be used for updating them
     More intelligence will move into this class over time.
     """
-    __slots__ = ['movie_id', 'urn', 'sha256', 'mime_type']
+    __slots__ = ['movie_id', 'sha256', 'mime_type']
     def __init__(self, movie_id, *, user_id=None):
         """
         :param movie_id: - the id of the movie
         :param user_id: - the user_id requesting access. If provided, the user must have access.
         """
         assert isinstance(movie_id,int)
         assert isinstance(user_id,(int,type(None)))
-        self.movie_id = movie_id
-        self.urn = None
+        self.movie_id  = movie_id
         self.mime_type = MIME.MP4
         self.sha256 = None
         if (user_id is not None) and not can_access_movie(user_id=user_id, movie_id=movie_id):
             raise UnauthorizedUser(f"user_id={user_id} movie_id={movie_id}")
 
     def __repr__(self):
-        return f"<Movie {self.movie_id} urn={self.urn} sha256={self.sha256}>"
+        return f"<Movie {self.movie_id} urn={self.movie_data_urn} sha256={self.sha256}>"
 
     @property
     def data(self):
         """Return the object data"""
         return get_movie_data(movie_id=self.movie_id)
 
+    @property
+    def movie_data_urn(self):
+        return movie_data_urn_for_movie_id(self.movie_id)
+
     @data.setter
     def data(self, movie_data):
         set_movie_data(movie_id=self.movie_id, movie_data=movie_data)
 
     @property
     def url(self):
         """Return a URL for accessing the data. This is a self-signed URL"""
-        return db_object.make_signed_url(urn=self.urn)
+        return db_object.make_signed_url(urn=self.movie_data_urn)
 
     @property
     def metadata(self):

db_object.py

@@ -20,6 +20,7 @@
 import hashlib
 import requests
 import boto3
+from botocore.exceptions import ClientError
 import bottle
 import uuid
 
@@ -165,7 +166,11 @@ def read_object(urn):
     logging.debug("urn=%s o=%s",urn,o)
     if o.scheme == C.SCHEME_S3 :
         # We are getting the object, so we do not need a presigned url
-        return s3_client().get_object(Bucket=o.netloc, Key=o.path[1:])["Body"].read()
+        try:
+            return s3_client().get_object(Bucket=o.netloc, Key=o.path[1:])["Body"].read()
+        except ClientError as ex:
+            logging.error("ClientError: %s  Bucket=%s  Key=%s",ex,o.netloc,o.path[1:])
+            return None
     elif o.scheme in ['http','https']:
         r = requests.get(urn, timeout=C.DEFAULT_GET_TIMEOUT)
         return r.content