Merge pull request #37 from dharmafly/feature/improved-cache

Add link cache timer. Add graph caching. #23

bin/node-socialgraph

@@ -1,12 +1,14 @@
 #!/usr/bin/env node
 
-var app      = require('http').createServer(handler),
-    fs       = require('fs'),
-    Grapher  = require('../lib/grapher.js'),
-    _        = require('underscore')._;
+var app     = require('http').createServer(handler),
+    fs      = require('fs'),
+    Grapher = require('../lib/grapher.js'),
+    cache   = require('../lib/cache.js'),
+    fn      = require('../lib/functions.js'),
+    _       = require('underscore')._;
 
 function handler(req, res) {
-  var rtn = [], grapher;
+  var rtn = [], grapher, cacheKey, queryUrl, options;
 
   // if the request if for a favicon, ignore it.
   if (req.url === '/favicon.ico') {
@@ -24,19 +26,65 @@ function handler(req, res) {
     return;
   }
 
-  // get the query object
+  // build the url that socialgraph will query against
   var query = require('url').parse(req.url, true).query;
 
   if (_.isEmpty(query)) {
-    grapher = Grapher.graph('http:/' + req.url, {strict:true});
+    queryUrl = normalizeUrl(req.url.substring(1));
   } else {
-    grapher = Grapher.graphFromQuery(query);
+    if (query.url) {
+      queryUrl = normalizeUrl(query.url);
+    } else {
+      res.writeHead(400, {'Content-Type': 'application/json; charset=utf-8'});
+      var error = JSON.stringify({
+        errors: "'url' is a required parameter"
+      });
+      error = query.callback ? query.callback + '(' + error + ');' : error;
+      res.end(error);
+      return;
+    }
   }
 
-  grapher.then(function(graph) {
-    res.writeHead(200, {'Content-Type': 'application/json; charset=utf-8'});
-    res.end(graph.toJSONP(query.callback));
-  });
+  // build the options object
+  options = {
+    strict:(query.strict !== undefined)
+  };
+
+  // build the object that the cache will cache against
+  cacheKey = 'graph;';
+  cacheKey += options.strict ? 'strict;' : '';
+  cacheKey += queryUrl;
+
+  if (cache.has(cacheKey)) {
+    cache.fetch(cacheKey, function (errors, jsonGraph) {
+      respond(res, jsonGraph, query.callback);
+      console.log('from cache   : ' + queryUrl);
+    });
+  } else {
+    grapher = Grapher.graph(queryUrl, options);
+
+    grapher.then(function(graph) {
+      var json = graph.toJSON();
+      cache.set(cacheKey, json);
+      respond(res, json, query.callback);
+    });
+  }
+}
+
+function respond (res, json, callback) {
+  res.writeHead(200, {'Content-Type': 'application/json; charset=utf-8'});
+  if (callback) {
+    res.end(callback + '(' + json + ');');
+  } else {
+    res.end(json);
+  }
+}
+
+function normalizeUrl (url) {
+  var urlBits = require('url').parse(url, true),
+      url     = fn.url.removeTrailingSlash(urlBits.href);
+
+  return urlBits.protocol ? url : "http://" + url;
 }
 
 app.listen(8888, 'localhost');

lib/cache.js

@@ -1,23 +1,44 @@
 var _     = require('underscore')._,
-    cache   = {};
+    limit = require('./options.js').cacheLimit,
+    cache = {};
 
 
 function get (url) {
-  return cache[url];
+  return cache[url].data;
 }
 
 function has(url) {
   return cache[url] !== undefined;
 }
 
 function fetch (url, callback) {
-  callback(null, cache[url]);
+  callback(null, cache[url].data);
 }
 
 function set (url, data) {
-  return cache[url] = data;
+  return cache[url] = {
+    time:new Date().getTime(),
+    data:data
+  };
 }
 
+function cleanUp () {
+  var time = new Date().getTime(),
+      i;
+
+  //console.log('cache size:', _.size(cache));
+
+  for (i in cache) {
+    if ((time - cache[i].time) > limit) {
+      delete cache[i];
+    }
+  }
+
+  setTimeout(cleanUp, 10000);
+}
+
+cleanUp();
+
 exports.get = get;
 exports.has = has;
 exports.set = set;

lib/grapher.js

@@ -355,26 +355,6 @@ function graph (url, options, callback) {
   return promise;
 }
 
-/**
- * Takes a query object like the kind generated by
- * require('url').parse(req.url, true).query.
- * It then calls graph() with that info.
- */
-function graphFromQuery(query, callback) {
-  var url     = query.q,
-      options = {strict:(query.strict !== undefined)},
-      urlBits = require('url').parse(url, true);
-
-  if (urlBits.protocol) {
-    url = urlBits.href;
-  } else {
-    url = "http://" + urlBits.href;
-  }
-
-  return graph(url, options, callback);
-}
-
 exports.Grapher = Grapher;
 exports.graph = graph;
-exports.graphFromQuery = graphFromQuery;
 exports.options = globalOptions;

lib/options.js

@@ -14,5 +14,10 @@ module.exports = {
   // If set to true then links at deeper path depths than
   // that of the shallowest on the same domain will be
   // discarded.
-  stripDeeperLinks: true
+  stripDeeperLinks: true,
+
+  // The amount of time graphs and links are kept in the
+  // cache for before they are discarded. Time in 
+  // milliseconds.
+  cacheLimit: 3600000
 }

static/index.html

@@ -37,7 +37,7 @@ <h1>Node Social Graph</h1>
       strict = document.getElementById('strict'),
       prog   = document.getElementById('prog'),
       strict = strict.checked ? '&strict' : '',
-      url    = location.href + '?q=' + encodeURIComponent(url.value) + strict,
+      url    = location.href + '?url=' + encodeURIComponent(url.value) + strict,
       xhr    = new XMLHttpRequest();
 
   prog.style.display = "block";