Merge branch 'release/0.3'

.scrutinizer.yml

@@ -0,0 +1,9 @@
+checks:
+    python:
+        code_rating: true
+        duplicate_code: true
+    javascript: true
+filter:
+    excluded_paths:
+        - '*/test/*'
+        - '*/vendors/*'

README.md

@@ -8,7 +8,7 @@ and displays a list of experiments, their state, Sacred configuration and
 the standard output from the running program.  
 Python 3.5 and a modern web browser are  required for it to work properly.
 
-# Features in version 0.2
+# Features in version 0.3
 
 - Get an overview of running and finished experiments in a table,
  such as experiment name, machine on which it runs etc.
@@ -19,14 +19,15 @@ Python 3.5 and a modern web browser are  required for it to work properly.
     directly from the web console in order to see detailed information,
     charts and [Tensorflow](https://www.tensorflow.org) graph visualisations,
     provided that the experiment uses Sacred's 
-    [Integration with Tensorflow](https://github.com/IDSIA/sacred/blob/develop/docs/tensorflow.rst)
-     (currently in the development branch of Sacred).
-
+    [Integration with Tensorflow](http://sacred.readthedocs.io/en/latest/tensorflow.html)
+     (does not work with TensorFlow 1.2 now)
+- Visualise [Metrics](http://sacred.readthedocs.io/en/latest/collected_information.html#metrics-api) in a chart.
+- Use the MongoDB and newly also FileStorage backend (experimental, thanks to [Gideon Dresdner](https://github.com/gideonite))
 
 ## Roadmap
 
 ### Further Versions
-- Sacred Metrics API data viewer (currently in Sacred development branch)
+- Deleting experiments
 - TBD
 
 ## Screenshots

docs/jsdoc.json

@@ -0,0 +1,24 @@
+{
+  "tags": {
+    "allowUnknownTags": false
+  },
+  "source": {
+    "include": "sacredboard/static/scripts",
+    "includePattern": ".js$",
+    "excludePattern": "(node_modules/|docs)"
+  },
+  "plugins": [
+    "plugins/markdown"
+  ],
+  "opts": {
+    "template": "node_modules/docdash/",
+    "encoding": "utf8",
+    "destination": "docs/frontend",
+    "recurse": true,
+    "verbose": true
+  },
+  "templates": {
+    "cleverLinks": false,
+    "monospaceLinks": false
+  }
+}

docs/source/conf.py

@@ -61,9 +61,9 @@
 # built documents.
 #
 # The short X.Y version.
-version = '0.2'
+version = '0.3'
 # The full version, including alpha/beta/rc tags.
-release = '0.2'
+release = '0.3'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.

package.json

@@ -1,18 +1,20 @@
 {
   "name": "sacredboard",
-  "version": "0.2.0",
+  "version": "0.3.0",
   "description": "Sacredboard NPM package file for automatic tests.",
   "dependencies": {},
   "devDependencies": [
     "qunitjs",
     "requirejs",
     "eslint",
     "eslint-plugin-requirejs",
-    "eslint-plugin-jsdoc"
-
+    "eslint-plugin-jsdoc",
+    "jsdoc",
+    "docdash"
   ],
   "scripts": {
     "test": "cd sacredboard/static/scripts/tests && qunit node_tests.js",
-    "lint": "eslint \"sacredboard/static/scripts/**/*.js\""
+    "lint": "eslint \"sacredboard/static/scripts/**/*.js\"",
+    "generate-docs": "jsdoc -c docs/jsdoc.json"
   }
 }

sacredboard/app/data/__init__.py

@@ -1 +1,7 @@
 """Sacred(board) Data Access Layer."""
+from .datastorage import Cursor, DataStorage
+from sacredboard.app.data.errors import NotFoundError, DataSourceError
+from .metricsdao import MetricsDAO
+
+__all__ = ["Cursor", "DataStorage", "MetricsDAO", "NotFoundError",
+           "DataSourceError"]

sacredboard/app/data/datastorage.py

@@ -0,0 +1,64 @@
+"""Interfaces for data storage."""
+from .errors import NotFoundError
+from .metricsdao import MetricsDAO
+
+
+class Cursor:
+    """Interface that abstracts the cursor object returned from databases."""
+
+    def __init__(self):
+        """Declare a new cursor to iterate over runs."""
+        pass
+
+    def count(self):
+        """Return the number of items in this cursor."""
+        raise NotImplemented()
+
+    def __iter__(self):
+        """Iterate over elements."""
+        raise NotImplemented()
+
+
+class DataStorage:
+    """
+    Interface for data backends.
+
+    Defines the API for various data stores
+    databases, file stores, etc. --- that sacred supports.
+    """
+
+    def __init__(self):
+        """Initialize data accessor."""
+        pass
+
+    def get_run(self, run_id):
+        """Return the run associated with the id."""
+        raise NotImplemented()
+
+    def get_runs(self, sort_by=None, sort_direction=None,
+                 start=0, limit=None, query={"type": "and", "filters": []}):
+        """Return all runs that match the query."""
+        raise NotImplemented()
+
+    def get_metrics_dao(self):
+        """
+        Return a data access object for metrics.
+
+        By default, returns a dummy Data Access Object if not overridden.
+        Issue: https://github.com/chovanecm/sacredboard/issues/62
+
+        :return MetricsDAO
+        """
+        return DummyMetricsDAO()
+
+
+class DummyMetricsDAO(MetricsDAO):
+    """Dummy Metrics DAO that does not find any metric."""
+
+    def get_metric(self, run_id, metric_id):
+        """
+        Raise NotFoundError. Always.
+
+        :raise NotFoundError
+        """
+        raise NotFoundError("Metrics not supported by this backend.")

sacredboard/app/data/errors.py

@@ -0,0 +1,17 @@
+"""Errors that might occur during data access."""
+
+
+class NotFoundError(Exception):
+    """Record not found exception."""
+
+    def __init__(self, *args, **kwargs):
+        """Record not found exception."""
+        Exception.__init__(self, *args, **kwargs)
+
+
+class DataSourceError(Exception):
+    """Error when accessing the data source."""
+
+    def __init__(self, *args, **kwargs):
+        """Error when accessing the data source."""
+        Exception.__init__(self, *args, **kwargs)

sacredboard/app/data/filestorage.py

@@ -0,0 +1,120 @@
+"""Implements backend storage interface for sacred's file store."""
+
+import datetime
+import os
+import json
+
+from sacredboard.app.data.datastorage import Cursor, DataStorage
+
+CONFIG_JSON = "config.json"
+RUN_JSON = "run.json"
+INFO_JSON = "info.json"
+
+
+def _path_to_file(basepath, run_id, file_name):
+    return os.path.join(basepath, str(run_id), file_name)
+
+
+def _path_to_config(basepath, run_id):
+    return _path_to_file(basepath, str(run_id), CONFIG_JSON)
+
+
+def _path_to_info(basepath, run_id):
+    return _path_to_file(basepath, str(run_id), INFO_JSON)
+
+
+def _path_to_run(basepath, run_id):
+    return os.path.join(basepath, str(run_id), RUN_JSON)
+
+
+def _read_json(path_to_json):
+    with open(path_to_json) as f:
+        return json.load(f)
+
+
+def _create_run(run_id, runjson, configjson, infojson):
+    runjson["_id"] = run_id
+    runjson["config"] = configjson
+    runjson["info"] = infojson
+
+    # TODO probably want a smarter way of detecting
+    # which values have type "time."
+    for k in ["start_time", "stop_time", "heartbeat"]:
+        runjson[k] = datetime.datetime.strptime(runjson[k],
+                                                '%Y-%m-%dT%H:%M:%S.%f')
+    return runjson
+
+
+class FileStoreCursor(Cursor):
+    """Implements the cursor for file stores."""
+
+    def __init__(self, count, iterable):
+        """Initialize FileStoreCursor with a given iterable."""
+        self.iterable = iterable
+        self._count = count
+
+    def count(self):
+        """
+        Return the number of runs in this query.
+
+        :return: int
+        """
+        return self._count
+
+    def __iter__(self):
+        """Iterate over runs."""
+        return iter(self.iterable)
+
+
+class FileStorage(DataStorage):
+    """Object to interface with one of sacred's file stores."""
+
+    def __init__(self, path_to_dir):
+        """Initialize file storage run accessor."""
+        super().__init__()
+        self.path_to_dir = os.path.expanduser(path_to_dir)
+
+    def get_run(self, run_id):
+        """
+        Return the run associated with a particular `run_id`.
+
+        :param run_id:
+        :return: dict
+        :raises FileNotFoundError
+        """
+        config = _read_json(_path_to_config(self.path_to_dir, run_id))
+        run = _read_json(_path_to_run(self.path_to_dir, run_id))
+        info = _read_json(_path_to_info(self.path_to_dir, run_id))
+        return _create_run(run_id, run, config, info)
+
+    def get_runs(self, sort_by=None, sort_direction=None,
+                 start=0, limit=None, query={"type": "and", "filters": []}):
+        """
+        Return all runs in the file store.
+
+        If a run is corrupt, e.g. missing files, it is skipped.
+
+        :param sort_by: NotImplemented
+        :param sort_direction:  NotImplemented
+        :param start: NotImplemented
+        :param limit: NotImplemented
+        :param query: NotImplemented
+        :return: FileStoreCursor
+        """
+        all_run_ids = os.listdir(self.path_to_dir)
+
+        def run_iterator():
+            blacklist = set(["_sources"])
+            for id in all_run_ids:
+                if id in blacklist:
+                    continue
+                try:
+                    yield self.get_run(id)
+                except FileNotFoundError:
+                    # An incomplete experiment is a corrupt experiment.
+                    # Skip it for now.
+                    # TODO
+                    pass
+
+        count = len(all_run_ids)
+        return FileStoreCursor(count, run_iterator())

sacredboard/app/data/metricsdao.py

@@ -0,0 +1,37 @@
+"""
+Interface for accessing Sacred metrics.
+
+Issue: https://github.com/chovanecm/sacredboard/issues/60
+"""
+
+
+class MetricsDAO:
+    """
+    Interface for accessing Sacred metrics.
+
+    Issue: https://github.com/chovanecm/sacredboard/issues/58
+    """
+
+    def get_metric(self, run_id, metric_id):
+        """
+        Read a metric of the given id and run.
+
+        The returned object has the following format (timestamps are datetime
+         objects).
+
+        .. code::
+
+            {"steps": [0,1,20,40,...],
+            "timestamps": [timestamp1,timestamp2,timestamp3,...],
+            "values": [0,1 2,3,4,5,6,...],
+            "name": "name of the metric",
+            "metric_id": "metric_id",
+            "run_id": "run_id"}
+
+        :param run_id: ID of the Run that the metric belongs to.
+        :param metric_id: The ID fo the metric.
+        :return: The whole metric as specified.
+
+        :raise NotFoundError
+        """
+        raise NotImplementedError("The MetricsDAO method is abstract.")