Merge pull request #244 from cnpm/missing-propertys

Add missing properies and sync missing star users. fixed #235

common/mysql.js

@@ -15,6 +15,7 @@
  * Module dependencies.
  */
 
+var thunkify = require('thunkify-wrap');
 var ready = require('ready');
 var mysql = require('mysql');
 var config = require('../config');
@@ -35,6 +36,10 @@ var pool = mysql.createPool({
 exports.pool = pool;
 
 exports.query = function (sql, values, cb) {
+  if (typeof values === 'function') {
+    cb = values;
+    values = null;
+  }
   pool.query(sql, values, function (err, rows) {
     cb(err, rows);
   });
@@ -59,6 +64,8 @@ exports.escape = function (val) {
 
 ready(exports);
 
+thunkify(exports);
+
 function init() {
   exports.query('show tables', function (err, rows) {
     if (err) {

controllers/registry/module.js

@@ -36,16 +36,27 @@ var DownloadTotal = require('../../proxy/download');
 var SyncModuleWorker = require('../../proxy/sync_module_worker');
 var logger = require('../../common/logger');
 var ModuleDeps = require('../../proxy/module_deps');
+var ModuleStar = require('../../proxy/module_star');
 
 /**
  * show all version of a module
  */
 exports.show = function *(next) {
   var name = this.params.name;
 
-  var r = yield [Module.listTags(name), Module.listByName(name)];
+  var r = yield [
+    Module.listTags(name),
+    Module.listByName(name),
+    ModuleStar.listUsers(name)
+  ];
   var tags = r[0];
   var rows = r[1];
+  var users = r[2];
+  var userMap = {};
+  for (var i = 0; i < users.length; i++) {
+    userMap[users[i]] = true;
+  }
+  users = userMap;
 
   debug('show module, user: %s, allowSync: %s, isAdmin: %s',
     this.session.name, this.session.allowSync, this.session.isAdmin);
@@ -81,6 +92,8 @@ exports.show = function *(next) {
   var versions = {};
   var times = {};
   var attachments = {};
+  var createdTime = null;
+  var modifiedTime = null;
   for (var i = 0; i < rows.length; i++) {
     var row = rows[i];
     if (row.version === 'next') {
@@ -91,12 +104,30 @@ exports.show = function *(next) {
     common.setDownloadURL(pkg, this);
     pkg._cnpm_publish_time = row.publish_time;
     versions[pkg.version] = pkg;
-    times[pkg.version] = row.publish_time ? new Date(row.publish_time) : row.gmt_modified;
+    var t = times[pkg.version] = row.publish_time ? new Date(row.publish_time) : row.gmt_modified;
     if ((!distTags.latest && !latestMod) || distTags.latest === row.version) {
       latestMod = row;
       readme = pkg.readme;
     }
     delete pkg.readme;
+
+    if (!createdTime || t < createdTime) {
+      createdTime = t;
+    }
+    if (!modifiedTime || t > modifiedTime) {
+      modifiedTime = t;
+    }
+  }
+
+  if (modifiedTime && createdTime) {
+    var ts = {
+      modified: modifiedTime,
+      created: createdTime,
+    };
+    for (var t in times) {
+      ts[t] = times[t];
+    }
+    times = ts;
   }
 
   if (!latestMod) {
@@ -112,21 +143,29 @@ exports.show = function *(next) {
     rev = String(nextMod.id);
   }
 
+  var pkg = latestMod.package;
+
   var info = {
     _id: name,
     _rev: rev,
     name: name,
-    description: latestMod.package.description,
+    description: pkg.description,
     "dist-tags": distTags,
-    maintainers: latestMod.package.maintainers,
+    maintainers: pkg.maintainers,
     time: times,
-    author: latestMod.package.author,
-    repository: latestMod.package.repository,
+    users: users,
+    author: pkg.author,
+    repository: pkg.repository,
     versions: versions,
     readme: readme,
     _attachments: attachments,
   };
 
+  info.readmeFilename = pkg.readmeFilename;
+  info.homepage = pkg.homepage;
+  info.bugs = pkg.bugs;
+  info.license = pkg.license;
+
   debug('show module %s: %s, latest: %s', name, rev, latestMod.version);
 
   this.body = info;

controllers/web/package.js

@@ -28,6 +28,7 @@ var sync = require('../sync');
 var Log = require('../../proxy/module_log');
 var ModuleDeps = require('../../proxy/module_deps');
 var setDownloadURL = require('../../lib/common').setDownloadURL;
+var ModuleStar = require('../../proxy/module_star');
 
 exports.display = function *(next) {
   var params = this.params;
@@ -47,20 +48,23 @@ exports.display = function *(next) {
   var r = yield [
     Module[getPackageMethod].apply(Module, getPackageArgs),
     down.total(name),
-    ModuleDeps.list(name)
+    ModuleDeps.list(name),
+    ModuleStar.listUsers(name),
   ];
   var pkg = r[0];
   var download = r[1];
   var dependents = (r[2] || []).map(function (item) {
     return item.deps;
   });
+  var users = r[3];
 
   if (!pkg || !pkg.package) {
     return yield next;
   }
 
   pkg.package.fromNow = moment(pkg.publish_time).fromNow();
   pkg = pkg.package;
+  pkg.users = users;
   pkg.readme = marked(pkg.readme || '');
   if (!pkg.readme) {
     pkg.readme = pkg.description || '';

docs/db.sql

@@ -25,6 +25,16 @@ CREATE TABLE `module_keyword` (
  KEY `name` (`name`)
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='module keyword';
 
+CREATE TABLE `module_star` (
+ `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'primary key',
+ `gmt_create` datetime NOT NULL COMMENT 'create time',
+ `user` varchar(100) CHARACTER SET utf8 COLLATE utf8_general_ci NOT NULL COMMENT 'user name',
+ `name` varchar(100) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT 'module name',
+ PRIMARY KEY (`id`),
+ UNIQUE KEY `user_module_name` (`user`,`name`),
+ KEY `name` (`name`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='module star';
+
 CREATE TABLE `module` (
  `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'primary key',
  `gmt_create` datetime NOT NULL COMMENT 'create time',

package.json

@@ -45,6 +45,7 @@
   },
   "devDependencies": {
     "autod": ">=0.0.13",
+    "co": "3.0.4",
     "contributors": "*",
     "cov": "*",
     "istanbul": "git://github.com/gotwarlost/istanbul.git#harmony",

proxy/module_star.js

@@ -0,0 +1,47 @@
+/**!
+ * cnpmjs.org - proxy/module_star.js
+ *
+ * Copyright(c) cnpmjs.org and other contributors.
+ * MIT Licensed
+ *
+ * Authors:
+ *  fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
+ */
+
+'use strict';
+
+/**
+ * Module dependencies.
+ */
+
+var mysql = require('../common/mysql');
+
+exports.add = function *add(name, user) {
+  var sql = 'INSERT INTO module_star(name, user) VALUES(?, ?);';
+  try {
+    yield mysql.query(sql, [name, user]);
+  } catch (err) {
+    if (err.code !== 'ER_DUP_ENTRY') {
+      throw err;
+    }
+  }
+};
+
+exports.remove = function *(name, user) {
+  var sql = 'DELETE FROM module_star WHERE name = ? AND user = ?;';
+  return yield mysql.query(sql, [name, user]);
+};
+
+exports.listUsers = function *(name) {
+  var sql = 'SELECT user FROM module_star WHERE name = ?;';
+  return (yield mysql.query(sql, [name])).map(function (r) {
+    return r.user;
+  });
+};
+
+exports.listUserModules = function *(user) {
+  var sql = 'SELECT name FROM module_star WHERE user = ?;';
+  return (yield mysql.query(sql, [user])).map(function (r) {
+    return r.name;
+  });
+};

proxy/sync_module_worker.js

@@ -15,6 +15,7 @@
  * Module dependencies.
  */
 
+var co = require('co');
 var thunkify = require('thunkify-wrap');
 var debug = require('debug')('cnpmjs.org:proxy:sync_module_worker');
 var EventEmitter = require('events').EventEmitter;
@@ -33,6 +34,7 @@ var Module = require('./module');
 var ModuleDeps = require('./module_deps');
 var Log = require('./module_log');
 var config = require('../config');
+var ModuleStar = require('./module_star');
 
 function SyncModuleWorker(options) {
   EventEmitter.call(this);
@@ -145,6 +147,36 @@ SyncModuleWorker.prototype.next = function (concurrencyId) {
   });
 };
 
+function _listStarUsers(modName, callback) {
+  co(function *() {
+    var users;
+    var err;
+    try {
+      users = yield ModuleStar.listUsers(modName);
+      var userMap = {};
+      for (var i = 0; i < users.length; i++) {
+        userMap[users[i]] = true;
+      }
+      users = userMap;
+    } catch (e) {
+      err = e;
+    }
+    callback(err, users);
+  })();
+}
+
+function _addStar(modName, username, callback) {
+  co(function *() {
+    var err;
+    try {
+      yield ModuleStar.add(modName, username);
+    } catch (e) {
+      err = e;
+    }
+    callback(err);
+  })();
+};
+
 SyncModuleWorker.prototype._sync = function (name, pkg, callback) {
   var username = this.username;
   var that = this;
@@ -194,12 +226,24 @@ SyncModuleWorker.prototype._sync = function (name, pkg, callback) {
     ep.emit('existsTags', tags);
   }));
 
+  _listStarUsers(name, ep.done('existsStarUsers'));
+
   var missingVersions = [];
   var missingTags = [];
   var missingDescriptions = [];
   var missingReadmes = [];
+  var missingStarUsers = [];
+
+  ep.all('existsMap', 'existsTags', 'existsStarUsers', function (map, tags, existsStarUsers) {
+    var starUsers = pkg.users || {};
+    for (var k in starUsers) {
+      if (!existsStarUsers[k]) {
+        missingStarUsers.push(k);
+      }
+    }
+
+    that.log('  [%s] found %d missing star users', name, missingStarUsers.length);
 
-  ep.all('existsMap', 'existsTags', function (map, tags) {
     var times = pkg.time || {};
     pkg.versions = pkg.versions || {};
     var versionNames = Object.keys(times);
@@ -300,7 +344,7 @@ SyncModuleWorker.prototype._sync = function (name, pkg, callback) {
       return a.publish_time - b.publish_time;
     });
     missingVersions = versions;
-    that.log('  [%s] %d versions', name, versions.length);
+    that.log('  [%s] %d versions need to sync', name, versions.length);
     ep.emit('syncModule', missingVersions.shift());
   });
 
@@ -318,13 +362,16 @@ SyncModuleWorker.prototype._sync = function (name, pkg, callback) {
 
       var nextVersion = missingVersions.shift();
       if (!nextVersion) {
+        ep.unbind('syncModule');
         return ep.emit('syncDone', result);
       }
+
+      // next
       ep.emit('syncModule', nextVersion);
     });
   });
 
-  ep.on('syncDone', function () {
+  ep.once('syncDone', function () {
     if (missingDescriptions.length === 0) {
       return ep.emit('descriptionDone');
     }
@@ -346,7 +393,7 @@ SyncModuleWorker.prototype._sync = function (name, pkg, callback) {
     });
   });
 
-  ep.on('syncDone', function () {
+  ep.once('syncDone', function () {
     if (missingTags.length === 0) {
       return ep.emit('tagDone');
     }
@@ -387,7 +434,28 @@ SyncModuleWorker.prototype._sync = function (name, pkg, callback) {
     });
   });
 
-  ep.all('tagDone', 'descriptionDone', 'readmeDone', function () {
+  ep.once('syncDone', function () {
+    if (missingStarUsers.length === 0) {
+      return ep.emit('starUserDone');
+    }
+
+    that.log('  [%s] saving %d star users', name, missingStarUsers.length);
+    missingStarUsers.forEach(function (username) {
+      _addStar(name, username, function (err) {
+        if (err) {
+          that.log('    add star user error, %s', err);
+        }
+        ep.emitLater('addStarUser');
+      });
+    });
+
+    ep.after('addStarUser', missingStarUsers.length, function () {
+      ep.emit('starUserDone');
+    });
+  });
+
+  ep.all('tagDone', 'descriptionDone', 'readmeDone', 'starUserDone',
+  function () {
     // TODO: set latest version
     callback(null, versionNames);
   });