initialize

.gitignore

@@ -0,0 +1,66 @@
+# Compiled source #
+###################
+*.com
+*.class
+*.dll
+*.exe
+*.o
+*.so
+
+# Packages #
+############
+# it's better to unpack these files and commit the raw source
+# git has its own built in compression methods
+*.7z
+*.dmg
+*.gz
+*.iso
+*.jar
+*.rar
+*.tar
+*.zip
+
+# Logs and databases #
+######################
+*.log
+*.sql
+*.sqlite
+
+# OS generated files #
+######################
+.DS_Store*
+ehthumbs.db
+Icon?
+Thumbs.db
+
+# Node.js #
+###########
+lib-cov
+*.seed
+*.log
+*.csv
+*.dat
+*.out
+*.pid
+*.gz
+
+pids
+logs
+results
+
+node_modules
+npm-debug.log
+
+# Git #
+#######
+*.orig
+*.BASE.*
+*.BACKUP.*
+*.LOCAL.*
+*.REMOTE.*
+
+# Components #
+##############
+
+/build
+/components

Makefile

@@ -0,0 +1,10 @@
+BIN = node_modules/.bin/
+
+test:
+	@$(BIN)mocha \
+		--harmony-generators \
+		--require should \
+		--reporter spec \
+		--timeout 30s
+
+.PHONY: test

README.md

@@ -0,0 +1,27 @@
+# Component Crawler
+
+Crawl github users for components.
+
+## License
+
+The MIT License (MIT)
+
+Copyright (c) 2014 Jonathan Ong me@jongleberry.com
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

app/index.js

@@ -0,0 +1,98 @@
+
+var co = require('co');
+var koa = require('koa');
+var join = require('path').join;
+var finished = require('finished');
+var Through = require('stream').PassThrough;
+
+var crawler = require('../');
+var sse = require('./sse');
+
+var app = module.exports = koa();
+app.use(require('koa-logger')());
+app.use(require('koa-compress')({
+  flush: require('zlib').Z_SYNC_FLUSH,
+}));
+app.use(require('koa-etag')());
+app.use(require('koa-json')());
+app.use(require('koa-static')(join(__dirname, '..', 'public'), {
+  defer: true,
+}));
+
+// GET the entire .json file
+app.use(function* (next) {
+  if (this.request.path !== '/.json') return yield* next;
+
+  var METHODS = 'OPTIONS,HEAD,GET';
+  cors.call(this, METHODS);
+
+  switch (this.request.method) {
+    case 'HEAD':
+    case 'GET':
+      this.response.body = crawler.json;
+      return;
+    case 'OPTIONS':
+      this.response.set('Allow', METHODS);
+      this.response.status = 204;
+      return;
+    default:
+      this.response.set('Allow', METHODS);
+      this.response.status = 405;
+      return;
+  }
+})
+
+// SSE stream of logs
+app.use(function* (next) {
+  if (this.request.path !== '/log') return yield* next;
+
+  this.req.setTimeout(Infinity);
+  this.response.type = 'text/event-stream';
+  var through =
+  this.response.body = sse.pipe(new Through()).on('error', this.onerror);
+  finished(this, function () {
+    sse.unpipe(through);
+  });
+})
+
+// GET and PATCH a user
+app.use(function* (next) {
+  var match = /^\/(\w+)$/.exec(this.request.path);
+  if (!match) return yield* next;
+
+  var user = match[1].toLowerCase();
+  var METHODS = 'OPTIONS,HEAD,GET,PATCH';
+  cors.call(this, METHODS);
+
+  switch (this.request.method) {
+    case 'HEAD':
+    case 'GET':
+      this.response.body = crawler.json.components.filter(function (component) {
+        return component.repo.split('/')[0] === user;
+      })
+      return;
+    case 'PATCH':
+      co(crawler.patch(user))(this.onerror);
+      this.response.status = 202;
+      return;
+    case 'OPTIONS':
+      this.response.set('Allow', METHODS);
+      this.response.status = 204;
+      return;
+    default:
+      this.response.set('Allow', METHODS);
+      this.response.status = 405;
+      return;
+  }
+})
+
+function cors(METHODS) {
+  this.response.set('Access-Control-Allow-Origin', '*');
+  this.response.set('Access-Control-Allow-Methods', METHODS);
+}
+
+if (!module.parent) {
+  var port = process.env.PORT || 3000;
+  app.listen(port);
+  console.log('Component Crawler listening on port ' + port + '.');
+}

app/sse.js

@@ -0,0 +1,17 @@
+
+var Transform = require('stream').Transform;
+
+var transform = module.exports = new Transform({
+  maxListeners: Infinity
+});
+transform._writableState.objectMode = true;
+require('../lib').log.pipe(transform);
+
+transform._transform = function (obj, NULL, cb) {
+  this.push('data: ' + JSON.stringify(obj) + '\n\n');
+  setImmediate(cb);
+}
+
+transform.on('error', function (err) {
+  console.error(err.stack);
+})

lib/crawl.js

@@ -0,0 +1,117 @@
+
+var CONCURRENCY = parseInt(process.env.CONCURRENCY, 10) || 5;
+var GITHUB_USERNAME = process.env.GITHUB_USERNAME;
+var GITHUB_PASSWORD = process.env.GITHUB_PASSWORD;
+if (!GITHUB_USERNAME && !GITHUB_PASSWORD) throw new Error('github username and password is required.');
+
+var co = require('co');
+var channel = require('chanel');
+var request = require('cogent').extend({
+  auth: GITHUB_USERNAME + ':' + GITHUB_PASSWORD,
+});
+
+var store = require('./store');
+var log = require('./log');
+
+module.exports = function* (user) {
+  var start = Date.now();
+  log.write({
+    context: 'user',
+    user: user,
+    type: 'info',
+    message: 'Updating "' + user + '".',
+  })
+
+  // remove all components under this user
+  var components = store.json.components;
+  for (var i = 0; i < components.length; i++) {
+    var component = components[i];
+    if (component.repo.split('/')[0] === user) {
+      components.splice(i, 1);
+    }
+  }
+
+  // get all the user's repos
+  var res = yield* request('https://api.github.com/search/repositories?q=user:' + user, true);
+  if (res.statusCode === 404) {
+    res.resume();
+    log.write({
+      context: 'user',
+      user: user,
+      type: 'error',
+      message: 'User "' + user + '" not found.',
+    })
+    return;
+  }
+  if (res.statusCode !== 200) {
+    res.resume();
+    log.write({
+      context: 'user',
+      user: user,
+      type: 'error',
+      message: 'Error searching "' + user + '"\'s repositories.',
+    })
+    return;
+  }
+
+  // store the user info
+  var json = store.json.users[user] = res.body;
+
+  // iterate through each repo with concurrency
+  var ch = channel({
+    concurrency: CONCURRENCY,
+    discard: true,
+  });
+  var items = json.items;
+  for (var i = 0; i < items.length; i++) {
+    ch.push(co(crawlRepository(user, items[i])));
+  }
+  yield ch(true);
+  yield* store.put();
+
+  log.write({
+    context: 'user',
+    user: user,
+    type: 'info',
+    message: 'Updated "' + user + '" in ' + Math.round((Date.now() - start) / 1000) + ' seconds.',
+  })
+}
+
+function* crawlRepository(user, data) {
+  var repo = data.name.toLowerCase();
+  var master = data.default_branch;
+  var res = yield* request('https://raw.github.com/' + user + '/' + repo + '/' + master + '/component.json', true);
+  if (res.statusCode === 404) {
+    res.resume();
+    log.write({
+      context: 'repo',
+      user: user,
+      repo: repo,
+      type: 'ignore',
+      message: 'Repository "' + user + '/' + repo + '" does not have a component.json.',
+    });
+    return;
+  }
+  if (res.statusCode !== 200) {
+    res.resume();
+    log.write({
+      context: 'repo',
+      user: user,
+      repo: repo,
+      type: 'error',
+      message: 'Error GETing "' + user + '/' + repo + '"\'s component.json.',
+    });
+    return;
+  }
+  var json = res.body;
+  json.repo = user + '/' + repo;
+  json.github = data;
+  store.json.components.push(json);
+  log.write({
+    content: 'repo',
+    user: user,
+    repo: repo,
+    type: 'info',
+    message: 'Repository "' + user + '/' + repo + '" has been updated.',
+  });
+}