diff --git a/client/src/index.js b/client/src/index.js index 370933c..3659d12 100644 --- a/client/src/index.js +++ b/client/src/index.js @@ -41,7 +41,8 @@ * - 'error': an error occured while connecting/syncing. The error * object is passed as the first arg to the event. * - * - 'connected': a connection was established with the sync server + * - 'connected': a connection was established with the sync server. This + * does not indicate that a sync has begun (use the 'syncing' event instead). * * - 'disconnected': the connection to the sync server was lost, either * due to the client or server. @@ -50,7 +51,14 @@ * or 'error' event should follow at some point, indicating whether * or not the sync was successful. * - * - 'completed': a sync has completed and was successful. + * - 'idle': a sync was requested but no sync was performed. This usually + * is triggered when no changes were made to the filesystem and hence, no + * changes were needed to be synced to the server. + * + * - 'completed': a file/directory/symlink has been synced successfully. + * + * - 'synced': MakeDrive has been synced and all paths are up-to-date with + * the server. * * * The `sync` property also exposes a number of methods, including: @@ -63,8 +71,8 @@ * * - disconnect(): disconnect from the sync server. * - * - request(path): request a sync with the server for the specified - * path. Such requests may or may not be processed right away. + * - request(): request a sync with the server. + * Such requests may or may not be processed right away. * * * Finally, the `sync` propery also exposes a `state`, which is the @@ -74,14 +82,12 @@ * sync.SYNC_CONNECTING = "SYNC CONNECTING" * sync.SYNC_CONNECTED = "SYNC CONNECTED" * sync.SYNC_SYNCING = "SYNC SYNCING" - * sync.SYNC_ERROR = "SYNC ERROR" */ var SyncManager = require('./sync-manager.js'); var SyncFileSystem = require('./sync-filesystem.js'); var Filer = require('../../lib/filer.js'); var EventEmitter = require('events').EventEmitter; -var resolvePath = require('../../lib/sync-path-resolver.js').resolveFromArray; var log = require('./logger.js'); var MakeDrive = {}; @@ -125,22 +131,18 @@ function createFS(options) { var fs = new SyncFileSystem(_fs); var sync = fs.sync = new EventEmitter(); var manager; + Object.defineProperty(sync, 'downstreamQueue', { + get: function() { return manager && manager.downstreams; } + }); // Auto-sync handles var autoSync; - var pathCache; - - // Path that needs to be used for an upstream sync - // to sync files that were determined to be more - // up-to-date on the client during a downstream sync - var upstreamPath; // State of the sync connection sync.SYNC_DISCONNECTED = "SYNC DISCONNECTED"; sync.SYNC_CONNECTING = "SYNC CONNECTING"; sync.SYNC_CONNECTED = "SYNC CONNECTED"; sync.SYNC_SYNCING = "SYNC SYNCING"; - sync.SYNC_ERROR = "SYNC ERROR"; // Intitially we are not connected sync.state = sync.SYNC_DISCONNECTED; @@ -170,34 +172,6 @@ function createFS(options) { manager = null; } - function requestSync(path) { - // If we're not connected (or are already syncing), ignore this request - if(sync.state === sync.SYNC_DISCONNECTED || sync.state === sync.SYNC_ERROR) { - sync.emit('error', new Error('Invalid state. Expected ' + sync.SYNC_CONNECTED + ', got ' + sync.state)); - log.warn('Tried to sync in invalid state: ' + sync.state); - return; - } - - // If there were no changes to the filesystem and - // no path was passed to sync, ignore this request - if(!fs.pathToSync && !path) { - log.debug('Skipping sync request, no changes to sync'); - return; - } - - // If a path was passed sync using it - if(path) { - log.info('Requesting sync for ' + path); - return manager.syncPath(path); - } - - // Cache the path that needs to be synced for error recovery - pathCache = fs.pathToSync; - fs.pathToSync = null; - log.info('Requesting sync for ' + pathCache); - manager.syncPath(pathCache); - } - // Turn on auto-syncing if its not already on sync.auto = function(interval) { var syncInterval = interval|0 > 0 ? interval|0 : 15 * 1000; @@ -219,23 +193,22 @@ function createFS(options) { } }; - // The server stopped our upstream sync mid-way through. - sync.onInterrupted = function() { - fs.pathToSync = pathCache; - sync.state = sync.SYNC_CONNECTED; - sync.emit('error', new Error('Sync interrupted by server.')); - log.warn('Sync interrupted by server, caching current path: ' + pathCache); + // The sync was stopped mid-way through. + sync.onInterrupted = function(path) { + sync.emit('error', new Error('Sync interrupted for path ' + path)); + log.warn('Sync interrupted by server for ' + path); }; sync.onError = function(err) { - // Regress to the path that needed to be synced but failed - // (likely because of a sync LOCK) - fs.pathToSync = upstreamPath || pathCache; - sync.state = sync.SYNC_ERROR; sync.emit('error', err); log.error('Sync error', err); }; + sync.onIdle = function(reason) { + sync.emit('idle', reason); + log.info('No sync took place: ' + reason); + }; + sync.onDisconnected = function() { // Remove listeners so we don't leak instance variables if("onbeforeunload" in global) { @@ -249,26 +222,28 @@ function createFS(options) { sync.state = sync.SYNC_DISCONNECTED; sync.emit('disconnected'); - log.info('Disconnected'); + log.info('Disconnected from MakeDrive server'); }; // Request that a sync begin. + // sync.request does not take any parameters + // as the path to sync is determined internally sync.request = function() { - // sync.request does not take any parameters - // as the path to sync is determined internally - // requestSync on the other hand optionally takes - // a path to sync which can be specified for - // internal use - requestSync(); + if(sync.state === sync.SYNC_CONNECTING || sync.state === sync.SYNC_DISCONNECTED) { + sync.emit('error', new Error('MakeDrive error: MakeDrive cannot sync as it is either disconnected or trying to connect')); + log.warn('Tried to sync in invalid state: ' + sync.state); + return; + } + + log.info('Requesting sync'); + manager.syncUpstream(); }; // Try to connect to the server. sync.connect = function(url, token) { // Bail if we're already connected - if(sync.state !== sync.SYNC_DISCONNECTED && - sync.state !== sync.ERROR) { + if(sync.state !== sync.SYNC_DISCONNECTED) { log.warn('Tried to connect, but already connected'); - sync.emit('error', new Error("MakeDrive: Attempted to connect to \"" + url + "\", but a connection already exists!")); return; } @@ -281,116 +256,101 @@ function createFS(options) { log.info('Connecting to MakeDrive server'); sync.state = sync.SYNC_CONNECTING; - function downstreamSyncCompleted(paths, needUpstream) { - var startTime; - - // Re-wire message handler functions for regular syncing - // now that initial downstream sync is completed. - sync.onSyncing = function() { - sync.state = sync.SYNC_SYNCING; - sync.emit('syncing'); - log.info('Started syncing'); - startTime = Date.now(); - }; - - sync.onCompleted = function(paths, needUpstream) { - // If changes happened to the files that needed to be synced - // during the sync itself, they will be overwritten - // https://github.com/mozilla/makedrive/issues/129 - - function complete() { - sync.state = sync.SYNC_CONNECTED; - sync.emit('completed'); - var duration = (Date.now() - startTime) + 'ms'; - log.info('Completed syncing in ' + duration); - } - - if(!paths && !needUpstream) { - return complete(); - } - - // Changes in the client are newer (determined during - // the sync) and need to be upstreamed - if(needUpstream) { - upstreamPath = resolvePath(needUpstream); - complete(); - log.debug('Client changes are newer for ' + upstreamPath); - return requestSync(upstreamPath); - } - - // If changes happened during a downstream sync - // Change the path that needs to be synced - manager.resetUnsynced(paths, function(err) { - if(err) { - log.error('Error resetting unsynced paths for ' + paths, err); - return sync.onError(err); - } - - upstreamPath = null; - complete(); - }); - }; - - // Upgrade connection state to 'connected' - sync.state = sync.SYNC_CONNECTED; - - // If we're in manual mode, bail before starting auto-sync - if(options.manual) { - sync.manual(); - } else { - sync.auto(options.interval); - } - - // In a browser, try to clean-up after ourselves when window goes away - if("onbeforeunload" in global) { - log.debug('Adding window.beforeunload handler'); - global.addEventListener('beforeunload', windowCloseHandler); - } - if("onunload" in global){ - log.debug('Adding window.unload handler'); - global.addEventListener('unload', cleanupManager); - } - - log.info('Connected'); - sync.emit('connected'); - - // If the downstream was completed and some - // versions of files were not synced as they were - // newer on the client, upstream them - if(needUpstream) { - upstreamPath = resolvePath(needUpstream); - log.debug('Client changes are newer for ' + upstreamPath); - requestSync(upstreamPath); - } else { - upstreamPath = null; - } - } - function connect(token) { // Try to connect to provided server URL. Use the raw Filer fs // instance for all rsync operations on the filesystem, so that we // can untangle changes done by user vs. sync code. - manager = new SyncManager(sync, _fs); + manager = new SyncManager(sync, fs, _fs); + manager.init(url, token, options, function(err) { if(err) { log.error('Error connecting to ' + url, err); sync.onError(err); return; } + // If we're in manual mode, bail before starting auto-sync + if(options.manual) { + sync.manual(); + } else { + sync.auto(options.interval); + } + + // In a browser, try to clean-up after ourselves when window goes away + if("onbeforeunload" in global) { + global.addEventListener('beforeunload', windowCloseHandler); + } + if("onunload" in global){ + global.addEventListener('unload', cleanupManager); + } + + // Deals with race conditions if a sync was + // started immediately after connecting before + // this callback could be triggered + if(sync.state !== sync.SYNC_SYNCING) { + sync.state = sync.SYNC_CONNECTED; + sync.emit('connected', url); + log.info('MakeDrive connected to server at ' + url); + } - var startTime; + sync.onSyncing = function(path) { + // A downstream sync might have just started, + // update the queue + sync.state = sync.SYNC_SYNCING; + sync.emit('syncing', 'Sync started for ' + path); + log.info('Sync started for ' + path); + }; - // Wait on initial downstream sync events to complete - sync.onSyncing = function() { - // do nothing, wait for onCompleted() - log.info('Starting initial downstream sync'); - startTime = Date.now(); + // A sync (either upstream or downstream) has completed for a single + // file/directory/symlink. The paths left to sync upstream needs to be + // updated and an event should be emitted. + sync.onCompleted = function(path, needsUpstream) { + var downstreamQueue = manager.downstreams; + needsUpstream = needsUpstream || []; + + // If during a downstream sync was performed and it was found that + // the path is more up-to-date on the client and hence needs to be + // upstreamed to the server, add it to the upstream queue. + fs.appendPathsToSync(needsUpstream, function(err) { + if(err) { + sync.emit('error', err); + log.error('Error appending paths to upstream after sync completed for ' + path + ' with error', err); + return; + } + + fs.getPathsToSync(function(err, pathsToSync) { + var syncsLeft; + + if(err) { + sync.emit('error', err); + log.error('Error retrieving paths to sync after sync completed for ' + path + ' with error', err); + return; + } + + // Determine if there are any more syncs remaining (both upstream and downstream) + syncsLeft = pathsToSync ? pathsToSync.concat(downstreamQueue) : downstreamQueue; + + if(path) { + sync.emit('completed', path); + log.info('Sync completed for ' + path); + } + + if(!syncsLeft.length) { + sync.allCompleted(); + } + }); + }); }; - sync.onCompleted = function(paths, needUpstream) { - // Downstream sync is done, finish connect() setup - var duration = (Date.now() - startTime) + 'ms'; - log.info('Completed initial downstream sync in ' + duration); - downstreamSyncCompleted(paths, needUpstream); + + // This is called when all nodes have been synced + // upstream and all downstream syncs have completed + sync.allCompleted = function() { + if(sync.state !== sync.SYNC_DISCONNECTED) { + // Reset the state + sync.state = sync.SYNC_CONNECTED; + } + + sync.emit('synced', 'MakeDrive has been synced'); + log.info('All syncs completed'); }; }); } @@ -400,10 +360,8 @@ function createFS(options) { // Disconnect from the server sync.disconnect = function() { // Bail if we're not already connected - if(sync.state === sync.SYNC_DISCONNECTED || - sync.state === sync.ERROR) { + if(sync.state === sync.SYNC_DISCONNECTED) { log.warn('Tried to disconnect while not connected'); - sync.emit('error', new Error("MakeDrive: Attempted to disconnect, but no server connection exists!")); return; } @@ -411,7 +369,6 @@ function createFS(options) { if(autoSync) { clearInterval(autoSync); autoSync = null; - fs.pathToSync = null; } // Do a proper network shutdown @@ -440,4 +397,3 @@ MakeDrive.fs = function(options) { } return sharedFS; }; - diff --git a/client/src/message-handler.js b/client/src/message-handler.js index f2e3616..a976100 100644 --- a/client/src/message-handler.js +++ b/client/src/message-handler.js @@ -2,280 +2,523 @@ var SyncMessage = require('../../lib/syncmessage'); var rsync = require('../../lib/rsync'); var rsyncUtils = rsync.utils; var rsyncOptions = require('../../lib/constants').rsyncDefaults; +var syncTypes = require('../../lib/constants').syncTypes; var serializeDiff = require('../../lib/diff').serialize; var deserializeDiff = require('../../lib/diff').deserialize; -var states = require('./sync-states'); -var steps = require('./sync-steps'); -var dirname = require('../../lib/filer').Path.dirname; -var async = require('../../lib/async-lite'); var fsUtils = require('../../lib/fs-utils'); +var log = require('./logger.js'); +var findPathIndexInArray = require('../../lib/util.js').findPathIndexInArray; function onError(syncManager, err) { - syncManager.session.step = steps.FAILED; syncManager.sync.onError(err); } -// Checks if path is in masterPath -function hasCommonPath(masterPath, path) { - if(masterPath === path) { - return true; +function sendChecksums(syncManager, path, type, sourceList) { + var fs = syncManager.fs; + var rawFs = syncManager.rawFs; + var sync = syncManager.sync; + var message; + + // If the server requests to downstream a path that is not in the + // root, ignore the downstream. + if(path.indexOf(fs.root) !== 0) { + message = SyncMessage.response.root; + message.content = {path: path, type: type}; + log.info('Ignoring ' + type + ' downstream sync for ' + path); + return syncManager.send(message.stringify()); } - if(path === '/') { - return false; - } + syncManager.downstreams.push(path); + sync.onSyncing(path); - return hasCommonPath(masterPath, dirname(path)); + rsync.checksums(rawFs, path, sourceList, rsyncOptions, function(err, checksums) { + if(err) { + log.error('Failed to generate checksums for ' + path + ' during downstream sync', err); + message = SyncMessage.request.delay; + message.content = {path: path, type: type}; + syncManager.send(message.stringify()); + return onError(syncManager, err); + } + + fs.trackChanges(path, sourceList); + message = SyncMessage.request.diffs; + message.content = {path: path, type: type, checksums: checksums}; + syncManager.send(message.stringify()); + }); } function handleRequest(syncManager, data) { var fs = syncManager.fs; + var rawFs = syncManager.rawFs; var sync = syncManager.sync; - var session = syncManager.session; function handleChecksumRequest() { - var srcList = session.srcList = data.content.srcList; - session.path = data.content.path; - fs.modifiedPath = null; - sync.onSyncing(); + if(data.invalidContent(['type', 'sourceList'])) { + log.error('Path, type or source list not sent by server in handleChecksumRequest.', data); + return onError(syncManager, new Error('Server sent insufficient content')); + } + + sendChecksums(syncManager, data.content.path, data.content.type, data.content.sourceList); + } + + function handleDiffRequest() { + if(data.invalidContent(['type', 'checksums'])) { + log.warn(data, 'Upstream sync message received from the server without sufficient information in handleDiffRequest'); + fs.delaySync(function(err, path) { + if(err) { + log.error(err, 'An error occured while updating paths to sync in handleDiffRequest'); + return onError(syncManager, err); + } + + log.info('Sync delayed for ' + path + ' in handleDiffRequest'); + syncManager.currentSync = false; + syncManager.syncUpstream(); + }); + return; + } - rsync.checksums(fs, session.path, srcList, rsyncOptions, function(err, checksums) { - if (err) { + var path = data.content.path; + var type = data.content.type; + var checksums = data.content.checksums; + var message; + + rsync.diff(rawFs, path, checksums, rsyncOptions, function(err, diffs) { + if(err){ + log.error(err, 'Error generating diffs in handleDiffRequest for ' + path); + + fs.delaySync(function(delayErr, delayedPath) { + if(delayErr) { + log.error(err, 'Error updating paths to sync in handleDiffRequest after failing to generate diffs for ' + path); + return onError(syncManager, delayErr); + } + + log.info('Sync delayed for ' + delayedPath + ' in handleDiffRequest'); + syncManager.currentSync = false; + syncManager.syncUpstream(); + }); + } else { + message = SyncMessage.response.diffs; + message.content = {path: path, type: type, diffs: serializeDiff(diffs)}; + syncManager.send(message.stringify()); + } + }); + } + + function handleRenameRequest() { + if(data.invalidContent(['type', 'oldPath'])) { + log.error('Path, type or old path not sent by server in handleRenameRequest.', data); + return onError(syncManager, new Error('Server sent insufficient content')); + } + + var path = data.content.path; + var oldPath = data.content.oldPath; + var type = data.content.type; + var message; + + // If the server requests to downstream a path that is not in the + // root, ignore the downstream. + if(path.indexOf(fs.root) !== 0) { + message = SyncMessage.response.root; + message.content = {path: path}; + log.info('Ignoring downstream sync for ' + path); + return syncManager.send(message.stringify()); + } + + syncManager.downstreams.push(path); + sync.onSyncing(oldPath); + + rsyncUtils.rename(rawFs, oldPath, path, function(err) { + if(err) { + log.error('Failed to rename ' + oldPath + ' to ' + path + ' during downstream sync', err); + message = SyncMessage.request.delay; + message.content = {path: path, type: type}; + syncManager.send(message.stringify()); return onError(syncManager, err); } - session.step = steps.PATCH; + rsyncUtils.generateChecksums(rawFs, [path], true, function(err, checksum) { + if(err) { + log.error('Failed to generate checksums for ' + path + ' during downstream rename', err); + message = SyncMessage.request.delay; + message.content = {path: path, type: type}; + syncManager.send(message.stringify()); + return onError(syncManager, err); + } - var message = SyncMessage.request.diffs; - message.content = {checksums: checksums}; - syncManager.send(message.stringify()); + message = SyncMessage.response.patch; + message.content = {path: path, type: type, checksum: checksum}; + syncManager.send(message.stringify()); + }); }); } - function handleDiffRequest() { - rsync.diff(fs, session.path, data.content.checksums, rsyncOptions, function(err, diffs) { - if(err){ + function handleDeleteRequest() { + if(data.invalidContent(['type'])) { + log.error('Path or type not sent by server in handleRenameRequest.', data); + return onError(syncManager, new Error('Server sent insufficient content')); + } + + var path = data.content.path; + var type = data.content.type; + var message; + + // If the server requests to downstream a path that is not in the + // root, ignore the downstream. + if(path.indexOf(fs.root) !== 0) { + message = SyncMessage.response.root; + message.content = {path: path, type: type}; + log.info('Ignoring downstream sync for ' + path); + return syncManager.send(message.stringify()); + } + + syncManager.downstreams.push(path); + sync.onSyncing(path); + + rsyncUtils.del(rawFs, path, function(err) { + if(err) { + log.error('Failed to delete ' + path + ' during downstream sync', err); + message = SyncMessage.request.delay; + message.content = {path: path, type: type}; + syncManager.send(message.stringify()); return onError(syncManager, err); } - session.step = steps.PATCH; + rsyncUtils.generateChecksums(rawFs, [path], false, function(err, checksum) { + if(err) { + log.error('Failed to generate checksums for ' + path + ' during downstream delete', err); + message = SyncMessage.request.delay; + message.content = {path: path, type: type}; + syncManager.send(message.stringify()); + return onError(syncManager, err); + } - var message = SyncMessage.response.diffs; - message.content = {diffs: serializeDiff(diffs)}; - syncManager.send(message.stringify()); + message = SyncMessage.response.patch; + message.content = {path: path, type: type, checksum: checksum}; + syncManager.send(message.stringify()); + }); }); } - - if(data.is.chksum && session.is.ready && - (session.is.synced || session.is.failed)) { + if(data.is.checksums) { // DOWNSTREAM - CHKSUM handleChecksumRequest(); - } else if(data.is.diffs && session.is.syncing && session.is.diffs) { + } else if(data.is.diffs) { // UPSTREAM - DIFFS handleDiffRequest(); + } else if(data.is.rename) { + // DOWNSTREAM - RENAME + handleRenameRequest(); + } else if(data.is.del) { + // DOWNSTREAM - DELETE + handleDeleteRequest(); } else { - onError(syncManager, new Error('Failed to sync with the server. Current step is: ' + - session.step + '. Current state is: ' + session.state)); } + onError(syncManager, new Error('Failed to sync with the server.')); + } } function handleResponse(syncManager, data) { var fs = syncManager.fs; + var rawFs = syncManager.rawFs; var sync = syncManager.sync; - var session = syncManager.session; - - function resendChecksums() { - if(!session.srcList) { - // Sourcelist was somehow reset, the entire downstream sync - // needs to be restarted - session.step = steps.FAILED; - syncManager.send(SyncMessage.response.reset.stringify()); - return onError(syncManager, new Error('Fatal Error: Could not sync filesystem from server...trying again!')); + + function handleSourceListResponse() { + if(data.invalidContent(['type'])) { + log.warn(data, 'Upstream sync message received from the server without sufficient information in handleSourceListResponse'); + return fs.delaySync(function(err, path) { + if(err) { + log.error(err, 'An error occured while updating paths to sync in handleSourceListResponse'); + return onError(syncManager, err); + } + + log.info('Sync delayed for ' + path + ' in handleSourceListResponse'); + syncManager.currentSync = false; + syncManager.syncUpstream(); + }); } - rsync.checksums(fs, session.path, session.srcList, rsyncOptions, function(err, checksums) { - if(err) { - syncManager.send(SyncMessage.response.reset.stringify()); - return onError(syncManager, err); - } + var message; + var path = data.content.path; + var type = data.content.type; - var message = SyncMessage.request.diffs; - message.content = {checksums: checksums}; - syncManager.send(message.stringify()); - }); - } + sync.onSyncing(path); - function handleSrcListResponse() { - session.state = states.SYNCING; - session.step = steps.INIT; - session.path = data.content.path; - sync.onSyncing(); + if(type === syncTypes.RENAME) { + message = SyncMessage.request.rename; + message.content = {path: path, oldPath: data.content.oldPath, type: type}; + return syncManager.send(message.stringify()); + } - rsync.sourceList(fs, session.path, rsyncOptions, function(err, srcList) { + if(type === syncTypes.DELETE) { + message = SyncMessage.request.del; + message.content = {path: path, type: type}; + return syncManager.send(message.stringify()); + } + + rsync.sourceList(rawFs, path, rsyncOptions, function(err, sourceList) { if(err){ - syncManager.send(SyncMessage.request.reset.stringify()); - return onError(syncManager, err); - } + log.error(err, 'Error generating source list in handleSourceListResponse for ' + path); + return fs.delaySync(function(delayErr, delayedPath) { + if(delayErr) { + log.error(err, 'Error updating paths to sync in handleSourceListResponse after failing to generate source list for ' + path); + return onError(syncManager, delayErr); + } - session.step = steps.DIFFS; + log.info('Sync delayed for ' + delayedPath + ' in handleSourceListResponse'); + syncManager.currentSync = false; + syncManager.syncUpstream(); + }); + } - var message = SyncMessage.request.chksum; - message.content = {srcList: srcList}; + message = SyncMessage.request.checksums; + message.content = {path: path, type: type, sourceList: sourceList}; syncManager.send(message.stringify()); }); } + // As soon as an upstream sync happens, the file synced + // becomes the last synced version and must be stamped + // with its checksum to version it and the unsynced attribute + // must be removed function handlePatchAckResponse() { - var syncedPaths = data.content.syncedPaths; - session.state = states.READY; - session.step = steps.SYNCED; + var syncedPath = data.content.path; - function stampChecksum(path, callback) { - fs.lstat(path, function(err, stats) { - if(err) { - if(err.code !== 'ENOENT') { - return callback(err); - } + function complete() { + fsUtils.removeUnsynced(fs, syncedPath, function(err) { + if(err && err.code !== 'ENOENT') { + log.error('Failed to remove unsynced attribute for ' + syncedPath + ' in handlePatchAckResponse, complete()'); + } + + syncManager.syncNext(syncedPath); + }); + } - // Non-existent paths (usually due to renames or - // deletes that are included in the syncedPaths) - // cannot be stamped with a checksum - return callback(); + fs.lstat(syncedPath, function(err, stats) { + if(err) { + if(err.code !== 'ENOENT') { + log.error('Failed to access ' + syncedPath + ' in handlePatchAckResponse'); + return fs.delaySync(function(delayErr, delayedPath) { + if(delayErr) { + log.error('Failed to delay upstream sync for ' + delayedPath + ' in handlePatchAckResponse'); + } + onError(syncManager, err); + }); } - if(!stats.isFile()) { - return callback(); + // Non-existent paths usually due to renames or + // deletes cannot be stamped with a checksum + return complete(); + } + + if(!stats.isFile()) { + return complete(); + } + + rsyncUtils.getChecksum(rawFs, syncedPath, function(err, checksum) { + if(err) { + log.error('Failed to get the checksum for ' + syncedPath + ' in handlePatchAckResponse'); + return fs.delaySync(function(delayErr, delayedPath) { + if(delayErr) { + log.error('Failed to delay upstream sync for ' + delayedPath + ' in handlePatchAckResponse while getting checksum'); + } + onError(syncManager, err); + }); } - rsyncUtils.getChecksum(fs, path, function(err, checksum) { + fsUtils.setChecksum(rawFs, syncedPath, checksum, function(err) { if(err) { - return callback(err); + log.error('Failed to stamp the checksum for ' + syncedPath + ' in handlePatchAckResponse'); + return fs.delaySync(function(delayErr, delayedPath) { + if(delayErr) { + log.error('Failed to delay upstream sync for ' + delayedPath + ' in handlePatchAckResponse while setting checksum'); + } + onError(syncManager, err); + }); } - fsUtils.setChecksum(fs, path, checksum, callback); + complete(); }); }); - } - - // As soon as an upstream sync happens, the files synced - // become the last synced versions and must be stamped - // with their checksums to version them - async.eachSeries(syncedPaths, stampChecksum, function(err) { - if(err) { - return onError(syncManager, err); - } - - sync.onCompleted(data.content.syncedPaths); }); } - function handlePatchResponse() { - var modifiedPath = fs.modifiedPath; - fs.modifiedPath = null; + function handleDiffResponse() { + var message; - // If there was a change to the filesystem that shares a common path with - // the path being synced, regenerate the checksums and send them - // (even if it is the initial one) - if(modifiedPath && hasCommonPath(session.path, modifiedPath)) { - return resendChecksums(); + if(data.invalidContent(['type', 'diffs'])) { + log.error('Path, type or diffs not sent by server in handleDiffResponse.', data); + return onError(syncManager, new Error('Server sent insufficient content')); } - var diffs = data.content.diffs; - diffs = deserializeDiff(diffs); + var path = data.content.path; + var type = data.content.type; + var diffs = deserializeDiff(data.content.diffs); + var changedDuringDownstream = fs.changesDuringDownstream.indexOf(path); + var cachedSourceList = fs.untrackChanges(path); + + if(changedDuringDownstream !== -1) { + // Resend the checksums for that path + return sendChecksums(syncManager, path, type, cachedSourceList); + } - rsync.patch(fs, session.path, diffs, rsyncOptions, function(err, paths) { - if (err) { - var message = SyncMessage.response.reset; + rsync.patch(rawFs, path, diffs, rsyncOptions, function(err, paths) { + if(err) { + log.error('Failed to patch ' + path + ' during downstream sync', err); + message = SyncMessage.request.delay; + message.content = {path: path, type: type}; syncManager.send(message.stringify()); return onError(syncManager, err); } - if(paths.needsUpstream.length) { - session.needsUpstream = paths.needsUpstream; - } + var needsUpstream = paths.needsUpstream; + syncManager.needsUpstream = syncManager.needsUpstream || []; + syncManager.needsUpstream.forEach(function(upstreamPath) { + if(needsUpstream.indexOf(upstreamPath) === -1) { + syncManager.needsUpstream.push(upstreamPath); + } + }); - rsyncUtils.generateChecksums(fs, paths.synced, true, function(err, checksums) { + fsUtils.getPathsToSync(rawFs, fs.root, function(err, pathsToSync) { if(err) { - var message = SyncMessage.response.reset; + log.error('Failed to update paths to sync during downstream sync', err); + message = SyncMessage.request.delay; + message.content = {path: path, type: type}; syncManager.send(message.stringify()); return onError(syncManager, err); } - var message = SyncMessage.response.patch; - message.content = {checksums: checksums}; - syncManager.send(message.stringify()); + var indexInPathsToSync; + + if(pathsToSync && pathsToSync.toSync && needsUpstream.indexOf(path) === -1) { + indexInPathsToSync = findPathIndexInArray(pathsToSync.toSync, path); + if(indexInPathsToSync !== -1) { + pathsToSync.toSync.splice(indexInPathsToSync, 1); + } + } + + fsUtils.setPathsToSync(rawFs, fs.root, pathsToSync, function(err) { + if(err) { + log.error('Failed to update paths to sync during downstream sync', err); + message = SyncMessage.request.delay; + message.content = {path: path, type: type}; + syncManager.send(message.stringify()); + return onError(syncManager, err); + } + + rsyncUtils.generateChecksums(rawFs, paths.synced, true, function(err, checksum) { + if(err) { + log.error('Failed to generate checksums for ' + paths.synced + ' during downstream patch', err); + message = SyncMessage.request.delay; + message.content = {path: path, type: type}; + syncManager.send(message.stringify()); + return onError(syncManager, err); + } + + message = SyncMessage.response.patch; + message.content = {path: path, type: type, checksum: checksum}; + syncManager.send(message.stringify()); + }); + }); }); }); } function handleVerificationResponse() { - session.srcList = null; - session.step = steps.SYNCED; - var needsUpstream = session.needsUpstream; - delete session.needsUpstream; - sync.onCompleted(null, needsUpstream); - } - - function handleUpstreamResetResponse() { - var message = SyncMessage.request.sync; - message.content = {path: session.path}; - syncManager.send(message.stringify()); + var path = data.content && data.content.path; + syncManager.downstreams.splice(syncManager.downstreams.indexOf(path), 1); + sync.onCompleted(path, syncManager.needsUpstream); } if(data.is.sync) { // UPSTREAM - INIT - handleSrcListResponse(); - } else if(data.is.patch && session.is.syncing && session.is.patch) { + handleSourceListResponse(); + } else if(data.is.patch) { // UPSTREAM - PATCH handlePatchAckResponse(); - } else if(data.is.diffs && session.is.ready && session.is.patch) { + } else if(data.is.diffs) { // DOWNSTREAM - PATCH - handlePatchResponse(); - } else if(data.is.verification && session.is.ready && session.is.patch) { + handleDiffResponse(); + } else if(data.is.verification) { // DOWNSTREAM - PATCH VERIFICATION handleVerificationResponse(); - } else if (data.is.reset && session.is.failed) { - handleUpstreamResetResponse(); - } else { - onError(syncManager, new Error('Failed to sync with the server. Current step is: ' + - session.step + '. Current state is: ' + session.state)); } + } else { + onError(syncManager, new Error('Failed to sync with the server.')); + } } function handleError(syncManager, data) { var sync = syncManager.sync; - var session = syncManager.session; - var message = SyncMessage.response.reset; + var fs = syncManager.fs; + var path = data.content && data.content.path; + + function handleForcedDownstream() { + fs.dequeueSync(function(err, syncsLeft, removedPath) { + if(err) { + log.fatal('Fatal error trying to dequeue sync in handleForcedDownstream'); + return; + } - // DOWNSTREAM - ERROR - if((((data.is.srclist && session.is.synced)) || - (data.is.diffs && session.is.patch) && (session.is.ready || session.is.syncing))) { - session.state = states.READY; - session.step = steps.SYNCED; + syncManager.currentSync = false; + sync.onInterrupted(removedPath); + }); + } - syncManager.send(message.stringify()); - onError(syncManager, new Error('Could not sync filesystem from server... trying again')); - } else if(data.is.verification && session.is.patch && session.is.ready) { - syncManager.send(message.stringify()); - onError(syncManager, new Error('Could not sync filesystem from server... trying again')); - } else if(data.is.locked && session.is.ready && session.is.synced) { - // UPSTREAM - LOCK - onError(syncManager, new Error('Current sync in progress! Try again later!')); - } else if(((data.is.chksum && session.is.diffs) || - (data.is.patch && session.is.patch)) && - session.is.syncing) { - // UPSTREAM - ERROR - var message = SyncMessage.request.reset; - syncManager.send(message.stringify()); - onError(syncManager, new Error('Could not sync filesystem from server... trying again')); - } else if(data.is.maxsizeExceeded) { - // We are only emitting the error since this is can be sync again from the client - syncManager.sync.emit('error', new Error('Maximum file size exceeded')); - } else if(data.is.interrupted && session.is.syncing) { - // SERVER INTERRUPTED SYNC (LOCK RELEASED EARLY) - sync.onInterrupted(); + function handleUpstreamError() { + fs.delaySync(function(err, delayedPath) { + if(err) { + log.fatal('Fatal error trying to delay sync in handleUpstreamError'); + return; + } + + syncManager.currentSync = false; + sync.onInterrupted(delayedPath); + }); + } + + function handleDownstreamError() { + if(syncManager.downstreams && syncManager.downstreams.length) { + syncManager.downstreams.splice(syncManager.downstreams.indexOf(path), 1); + } + + fs.untrackChanges(path); + sync.onInterrupted(path); + } + + if(data.is.content) { + log.error('Invalid content was sent to the server'); + } else if(data.is.needsDownstream) { + log.warn('Cancelling upstream for ' + path + ', downstreaming instead'); + handleForcedDownstream(); + } else if(data.is.impl) { + log.error('Server could not initialize upstream sync for ' + path); + handleUpstreamError(); + } else if(data.is.interrupted) { + log.error('Server interrupted upstream sync due to incoming downstream for ' + path); + handleUpstreamError(); + } else if(data.is.locked) { + log.error('Server cannot process upstream request due to ' + path + ' being locked'); + handleUpstreamError(); + } else if(data.is.fileSizeError) { + log.error('Maximum file size for upstream syncs exceeded for ' + path); + handleUpstreamError(); + } else if(data.is.checksums) { + log.error('Error generating checksums on the server for ' + path); + handleUpstreamError(); + } else if(data.is.patch) { + log.error('Error patching ' + path + ' on the server'); + handleUpstreamError(); + } else if(data.is.sourceList) { + log.fatal('Fatal error, server could not generate source list'); + } else if(data.is.diffs) { + log.error('Error generating diffs on the server for ' + path); + handleDownstreamError(); + } else if(data.is.downstreamLocked) { + log.error('Cannot downstream due to lock on ' + path + ' on the server'); + handleDownstreamError(); + } else if(data.is.verification) { + log.fatal('Patch could not be verified due to incorrect patching on downstreaming ' + path + '. Possible file corruption.'); + handleDownstreamError(); } else { - onError(syncManager, new Error('Failed to sync with the server. Current step is: ' + - session.step + '. Current state is: ' + session.state)); + log.fatal(data, 'Unknown error sent by the server'); } } diff --git a/client/src/sync-filesystem.js b/client/src/sync-filesystem.js index 554e300..7110f10 100644 --- a/client/src/sync-filesystem.js +++ b/client/src/sync-filesystem.js @@ -6,39 +6,245 @@ var Filer = require('../../lib/filer.js'); var Shell = require('../../lib/filer-shell.js'); -var Path = Filer.Path; var fsUtils = require('../../lib/fs-utils.js'); var conflict = require('../../lib/conflict.js'); -var resolvePath = require('../../lib/sync-path-resolver.js').resolve; +var syncTypes = require('../../lib/constants.js').syncTypes; +var findPathIndexInArray = require('../../lib/util.js').findPathIndexInArray; +var log = require('./logger.js'); function SyncFileSystem(fs) { var self = this; - var pathToSync; - var modifiedPath; - - // Manage path resolution for sync path - Object.defineProperty(self, 'pathToSync', { - get: function() { return pathToSync; }, - set: function(path) { - if(path) { - pathToSync = resolvePath(pathToSync, path); - } else { - pathToSync = null; - } + var root = '/'; + // Record changes during a downstream sync + // Is a parallel array with sourceListCache + var trackedPaths = []; + var sourceListCache = []; + var changesDuringDownstream = []; + + // Expose the root used to sync for the filesystem + // Defaults to '/' + Object.defineProperties(self, { + 'root': { + get: function() { return root; } + }, + 'changesDuringDownstream': { + get: function() { return changesDuringDownstream; } } }); - // Record modifications to the filesystem during a sync - Object.defineProperty(fs, 'modifiedPath', { - get: function() { return modifiedPath; }, - set: function(path) { - if(path) { - modifiedPath = resolvePath(modifiedPath, path); - } else { - modifiedPath = null; + // Watch the given path for any changes made to it and cache + // the source list for that path + self.trackChanges = function(path, sourceList) { + trackedPaths.push(path); + sourceListCache.push(sourceList); + }; + + // Stop watching the given paths for changes and return the + // cached source list + self.untrackChanges = function(path) { + var indexInTrackedPaths = trackedPaths.indexOf(path); + var indexInChangesDuringDownstream = changesDuringDownstream.indexOf(path); + + if(indexInTrackedPaths === -1) { + log.error('Path ' + path + ' not found in tracked paths list'); + return null; + } + + trackedPaths.splice(indexInTrackedPaths, 1); + + if(indexInChangesDuringDownstream !== -1) { + changesDuringDownstream.splice(changesDuringDownstream.indexOf(path), 1); + } + + return sourceListCache.splice(indexInTrackedPaths, 1)[0]; + }; + + // Get the paths queued up to sync + self.getPathsToSync = function(callback) { + fsUtils.getPathsToSync(fs, root, function(err, pathsToSync) { + if(err) { + return callback(err); } + + callback(null, pathsToSync && pathsToSync.toSync); + }); + }; + + // Add paths to the sync queue where paths is an array + self.appendPathsToSync = function(paths, callback) { + if(!paths || !paths.length) { + return callback(); } - }); + + var syncPaths = []; + + paths.forEach(function(pathObj) { + var syncObj = pathObj.path ? pathObj : {path: pathObj, type: syncTypes.CREATE}; + if(syncObj.path.indexOf(root) === 0) { + syncPaths.push(syncObj); + } + }); + + fsUtils.getPathsToSync(fs, root, function(err, pathsToSync) { + if(err) { + return callback(err); + } + + pathsToSync = pathsToSync || {}; + pathsToSync.toSync = pathsToSync.toSync || []; + var toSync = pathsToSync.toSync; + + syncPaths.forEach(function(syncObj) { + // Ignore redundancies + var exists = !(toSync.every(function(objToSync) { + return objToSync.path !== syncObj.path; + })); + + if(!exists) { + pathsToSync.toSync.push(syncObj); + } + }); + + fsUtils.setPathsToSync(fs, root, pathsToSync, callback); + }); + }; + + // Get the path that was modified during a sync + self.getModifiedPath = function(callback) { + fsUtils.getPathsToSync(fs, root, function(err, pathsToSync) { + if(err) { + return callback(err); + } + + callback(null, pathsToSync && pathsToSync.modified); + }); + }; + + // Indicate that the path at the front of the queue has + // begun syncing + self.setSyncing = function(callback) { + fsUtils.getPathsToSync(fs, root, function(err, pathsToSync) { + if(err) { + return callback(err); + } + + if(!pathsToSync || !pathsToSync.toSync || !pathsToSync.toSync[0]) { + log.warn('setSyncing() called when no paths to sync present'); + return callback(); + } + + pathsToSync.toSync[0].syncing = true; + + callback(); + }); + }; + + // Delay the sync of the currently syncing path + // by moving it to the end of the sync queue + self.delaySync = function(callback) { + fsUtils.getPathsToSync(fs, root, function(err, pathsToSync) { + if(err) { + return callback(err); + } + + if(!pathsToSync || !pathsToSync.toSync || !pathsToSync.toSync[0]) { + log.warn('delaySync() called when no paths to sync present'); + return callback(); + } + + var delayedSync = pathsToSync.toSync.shift(); + pathsToSync.toSync.push(delayedSync); + delete pathsToSync.modified; + + fsUtils.setPathsToSync(fs, root, pathsToSync, function(err) { + if(err) { + return callback(err); + } + + callback(null, delayedSync.path); + }); + }); + }; + + // Remove the path that was just synced + self.dequeueSync = function(callback) { + fsUtils.getPathsToSync(fs, root, function(err, pathsToSync) { + if(err) { + return callback(err); + } + + if(!pathsToSync || !pathsToSync.toSync || !pathsToSync.toSync[0]) { + log.warn('dequeueSync() called when no paths to sync present'); + return callback(); + } + + var removedSync = pathsToSync.toSync.shift(); + if(!pathsToSync.toSync.length) { + delete pathsToSync.toSync; + } + delete pathsToSync.modified; + + fsUtils.setPathsToSync(fs, root, pathsToSync, function(err) { + if(err) { + return callback(err); + } + + callback(null, pathsToSync.toSync, removedSync.path); + }); + }); + }; + + // Set the sync root for the filesystem. + // The path provided must name an existing directory + // or the setter will fail. + // Once the new root is set, the paths remaining to + // sync and the path that was modified during a sync + // are filtered out if they are not under the new root. + self.setRoot = function(newRoot, callback) { + function containsRoot(pathOrObj) { + if(typeof pathOrObj === 'object') { + pathOrObj = pathOrObj.path || ''; + } + + return pathOrObj.indexOf(newRoot) === 0; + } + + fs.lstat(newRoot, function(err, stats) { + if(err) { + return callback(err); + } + + if(!stats.isDirectory()) { + return callback(new Filer.Errors.ENOTDIR('the given root is not a directory', newRoot)); + } + + fsUtils.getPathsToSync(fs, root, function(err, pathsToSync) { + if(err) { + return callback(err); + } + + root = newRoot; + + if(!pathsToSync) { + return callback(); + } + + if(pathsToSync.toSync) { + pathsToSync.toSync = pathsToSync.toSync.filter(containsRoot); + + if(!pathsToSync.toSync.length) { + delete pathsToSync.toSync; + } + } + + if(pathsToSync.modified && !containsRoot(pathsToSync.modified)) { + delete pathsToSync.modified; + } + + callback(); + }); + }); + }; // The following non-modifying fs operations can be run as normal, // and are simply forwarded to the fs instance. NOTE: we have @@ -64,7 +270,7 @@ function SyncFileSystem(fs) { // for syncing (i.e., changes we need to sync back to the server), such that we // can track things. Different fs methods need to do this in slighly different ways, // but the overall logic is the same. The wrapMethod() fn defines this logic. - function wrapMethod(method, pathArgPos, setUnsyncedFn, useParentPath) { + function wrapMethod(method, pathArgPos, setUnsyncedFn, type) { return function() { var args = Array.prototype.slice.call(arguments, 0); var lastIdx = args.length - 1; @@ -75,6 +281,28 @@ function SyncFileSystem(fs) { // second for some. var pathOrFD = args[pathArgPos]; + function wrappedCallback() { + var args = Array.prototype.slice.call(arguments, 0); + if(args[0] || type === syncTypes.DELETE) { + return callback.apply(null, args); + } + + setUnsyncedFn(pathOrFD, function(err) { + if(err) { + return callback(err); + } + callback.apply(null, args); + }); + } + + args[lastIdx] = wrappedCallback; + + if(type === syncTypes.DELETE && pathOrFD === root) { + // Deal with deletion of the sync root + // https://github.com/mozilla/makedrive/issues/465 + log.warn('Tried to delete the sync root ' + root); + } + // Don't record extra sync-level details about modifications to an // existing conflicted copy, since we don't sync them. conflict.isConflictedCopy(fs, pathOrFD, function(err, conflicted) { @@ -84,38 +312,59 @@ function SyncFileSystem(fs) { return callback.apply(null, [err]); } - // In most cases we want to use the path itself, but in the case - // that a node is being removed, we want the parent dir. - pathOrFD = useParentPath ? Path.dirname(pathOrFD) : pathOrFD; - conflicted = !!conflicted; - // Record base sync path if this is a regular, non-conflicted path. - if(!conflicted && !fs.openFiles[pathOrFD]) { - // Check to see if it is a path or an open file descriptor - // TODO: Deal with a case of fs.open for a path with a write flag - // https://github.com/mozilla/makedrive/issues/210. - self.pathToSync = pathOrFD; - // Record the path that was modified on the fs - fs.modifiedPath = pathOrFD; + // Check to see if it is a path or an open file descriptor + // and do not record the path if it is not contained + // in the specified syncing root of the filesystem, or if it is conflicted. + // TODO: Deal with a case of fs.open for a path with a write flag + // https://github.com/mozilla/makedrive/issues/210. + if(fs.openFiles[pathOrFD] || pathOrFD.indexOf(root) !== 0 || conflicted) { + fs[method].apply(fs, args); + return; } - args[lastIdx] = function wrappedCallback() { - var args = Array.prototype.slice.call(arguments, 0); - // Error object on callback, bail now - if(args[0]) { - return callback.apply(null, args); + if(trackedPaths.indexOf(pathOrFD) !== -1 && self.changesDuringDownstream.indexOf(pathOrFD) === -1) { + self.changesDuringDownstream.push(pathOrFD); + } + + // Queue the path for syncing in the pathsToSync + // xattr on the sync root + fsUtils.getPathsToSync(fs, root, function(err, pathsToSync) { + if(err) { + return callback(err); + } + + var syncPath = { + path: pathOrFD, + type: type + }; + if(type === syncTypes.RENAME) { + syncPath.oldPath = args[pathArgPos - 1]; + } + var indexInPathsToSync; + + + pathsToSync = pathsToSync || {}; + pathsToSync.toSync = pathsToSync.toSync || []; + indexInPathsToSync = findPathIndexInArray(pathsToSync.toSync, pathOrFD); + + if(indexInPathsToSync === 0 && pathsToSync.toSync[0].syncing) { + // If at the top of pathsToSync, the path is + // currently syncing so change the modified path + pathsToSync.modified = pathOrFD; + } else if(indexInPathsToSync === -1) { + pathsToSync.toSync.push(syncPath); } - setUnsyncedFn(pathOrFD, function(err) { + fsUtils.setPathsToSync(fs, root, pathsToSync, function(err) { if(err) { return callback(err); } - callback.apply(null, args); - }); - }; - fs[method].apply(fs, args); + fs[method].apply(fs, args); + }); + }); }); }; } @@ -123,28 +372,27 @@ function SyncFileSystem(fs) { // Wrapped fs methods that have path at first arg position and use paths ['truncate', 'mknod', 'mkdir', 'utimes', 'writeFile', 'appendFile'].forEach(function(method) { - self[method] = wrapMethod(method, 0, setUnsynced); + self[method] = wrapMethod(method, 0, setUnsynced, syncTypes.CREATE); }); // Wrapped fs methods that have path at second arg position ['link', 'symlink'].forEach(function(method) { - self[method] = wrapMethod(method, 1, setUnsynced); + self[method] = wrapMethod(method, 1, setUnsynced, syncTypes.CREATE); }); - // Wrapped fs methods that have path at second arg position, and need to use the parent path. + // Wrapped fs methods that have path at second arg position ['rename'].forEach(function(method) { - self[method] = wrapMethod(method, 1, setUnsynced, true); + self[method] = wrapMethod(method, 1, setUnsynced, syncTypes.RENAME); }); // Wrapped fs methods that use file descriptors ['ftruncate', 'futimes', 'write'].forEach(function(method) { - self[method] = wrapMethod(method, 0, fsetUnsynced); + self[method] = wrapMethod(method, 0, fsetUnsynced, syncTypes.CREATE); }); - // Wrapped fs methods that have path at first arg position and use parent - // path for writing unsynced metadata (i.e., removes node) + // Wrapped fs methods that have path at first arg position ['rmdir', 'unlink'].forEach(function(method) { - self[method] = wrapMethod(method, 0, setUnsynced, true); + self[method] = wrapMethod(method, 0, setUnsynced, syncTypes.DELETE); }); // We also want to do extra work in the case of a rename. @@ -162,11 +410,37 @@ function SyncFileSystem(fs) { return callback(err); } - if(conflicted) { - conflict.removeFileConflict(fs, newPath, callback); - } else { - callback(); + if(!conflicted) { + return callback(); } + + conflict.removeFileConflict(fs, newPath, function(err) { + if(err) { + return callback(err); + } + + fsUtils.getPathsToSync(fs, root, function(err, pathsToSync) { + var indexInPathsToSync; + var syncInfo; + + if(err) { + return callback(err); + } + + indexInPathsToSync = findPathIndexInArray(pathsToSync.toSync, newPath); + + if(indexInPathsToSync === -1) { + return; + } + + syncInfo = pathsToSync.toSync[indexInPathsToSync]; + syncInfo.type = syncTypes.CREATE; + delete syncInfo.oldPath; + pathsToSync.toSync[indexInPathsToSync] = syncInfo; + + fsUtils.setPathsToSync(fs, root, pathsToSync, callback); + }); + }); }); }); }; @@ -177,9 +451,7 @@ function SyncFileSystem(fs) { // ourselves. The other down side of this is that we're now including // the Shell code twice (once in filer.js, once here). We need to // optimize this when we look at making MakeDrive smaller. - self.Shell = function(options) { - return new Shell(self, options); - }; + self.Shell = Shell.bind(undefined, self); // Expose extra operations for checking whether path/fd is unsynced self.getUnsynced = function(path, callback) { diff --git a/client/src/sync-manager.js b/client/src/sync-manager.js index 487b5e9..1a525d3 100644 --- a/client/src/sync-manager.js +++ b/client/src/sync-manager.js @@ -1,64 +1,22 @@ var SyncMessage = require( '../../lib/syncmessage' ), messageHandler = require('./message-handler'), - states = require('./sync-states'), - steps = require('./sync-steps'), WS = require('ws'), - fsUtils = require('../../lib/fs-utils'), - async = require('../../lib/async-lite.js'), request = require('request'), - url = require('url'); + url = require('url'), + log = require('./logger.js'); -function SyncManager(sync, fs) { +function SyncManager(sync, fs, _fs) { var manager = this; manager.sync = sync; manager.fs = fs; - manager.session = { - state: states.CLOSED, - step: steps.SYNCED, - path: '/', - - is: Object.create(Object.prototype, { - // States - syncing: { - get: function() { return manager.session.state === states.SYNCING; } - }, - ready: { - get: function() { return manager.session.state === states.READY; } - }, - error: { - get: function() { return manager.session.state === states.ERROR; } - }, - closed: { - get: function() { return manager.session.state === states.CLOSED; } - }, - - // Steps - init: { - get: function() { return manager.session.step === steps.INIT; } - }, - chksum: { - get: function() { return manager.session.step === steps.CHKSUM; } - }, - diffs: { - get: function() { return manager.session.step === steps.DIFFS; } - }, - patch: { - get: function() { return manager.session.step === steps.PATCH; } - }, - synced: { - get: function() { return manager.session.step === steps.SYNCED; } - }, - failed: { - get: function() { return manager.session.step === steps.FAILED; } - } - }) - }; + manager.rawFs = _fs; + manager.downstreams = []; + manager.needsUpstream = []; } SyncManager.prototype.init = function(wsUrl, token, options, callback) { var manager = this; - var session = manager.session; var sync = manager.sync; var reconnectCounter = 0; var socket; @@ -74,9 +32,6 @@ SyncManager.prototype.init = function(wsUrl, token, options, callback) { } if(data.is.response && data.is.authz) { - session.state = states.READY; - session.step = steps.SYNCED; - socket.onmessage = function(event) { var data = event.data || event; messageHandler(manager, data); @@ -195,39 +150,66 @@ SyncManager.prototype.init = function(wsUrl, token, options, callback) { connect(); }; -SyncManager.prototype.syncPath = function(path) { +SyncManager.prototype.syncUpstream = function() { var manager = this; + var fs = manager.fs; + var sync = manager.sync; var syncRequest; + var syncInfo; if(!manager.socket) { throw new Error('sync called before init'); } - syncRequest = SyncMessage.request.sync; - syncRequest.content = {path: path}; - manager.send(syncRequest.stringify()); -}; + if(manager.currentSync) { + sync.onError(new Error('Sync currently underway')); + return; + } -// Remove the unsynced attribute for a list of paths -SyncManager.prototype.resetUnsynced = function(paths, callback) { - var fs = this.fs; + fs.getPathsToSync(function(err, pathsToSync) { + if(err) { + sync.onError(err); + return; + } + + if(!pathsToSync || !pathsToSync.length) { + log.warn('Nothing to sync'); + sync.onIdle('No changes made to the filesystem'); + return; + } - function removeUnsyncedAttr(path, callback) { - fsUtils.removeUnsynced(fs, path, function(err) { - if(err && err.code !== 'ENOENT') { - return callback(err); + syncInfo = pathsToSync[0]; + + fs.setSyncing(function(err) { + if(err) { + sync.onError(err); + return; } - callback(); + manager.currentSync = syncInfo; + syncRequest = SyncMessage.request.sync; + syncRequest.content = {path: syncInfo.path, type: syncInfo.type}; + if(syncInfo.oldPath) { + syncRequest.content.oldPath = syncInfo.oldPath; + } + manager.send(syncRequest.stringify()); }); - } + }); +}; + +SyncManager.prototype.syncNext = function(syncedPath) { + var manager = this; + var fs = manager.fs; + var sync = manager.sync; - async.eachSeries(paths, removeUnsyncedAttr, function(err) { + fs.dequeueSync(function(err, syncsLeft, dequeuedSync) { if(err) { - return callback(err); + log.error('Failed to dequeue sync for ' + syncedPath + ' in SyncManager.syncNext()'); } - callback(); + sync.onCompleted(dequeuedSync || syncedPath); + manager.currentSync = false; + manager.syncUpstream(); }); }; diff --git a/client/src/sync-states.js b/client/src/sync-states.js deleted file mode 100644 index dc8c2a7..0000000 --- a/client/src/sync-states.js +++ /dev/null @@ -1,6 +0,0 @@ -module.exports = { - SYNCING: "SYNC IN PROGRESS", - READY: "READY", - ERROR: "ERROR", - CLOSED: "CLOSED" -}; \ No newline at end of file diff --git a/client/src/sync-steps.js b/client/src/sync-steps.js index de8a819..fe55cde 100644 --- a/client/src/sync-steps.js +++ b/client/src/sync-steps.js @@ -5,4 +5,4 @@ module.exports = { PATCH: "PATCH", SYNCED: "SYNCED", FAILED: "FAILED" -}; \ No newline at end of file +}; diff --git a/lib/constants.js b/lib/constants.js index b44087b..fe66ad7 100644 --- a/lib/constants.js +++ b/lib/constants.js @@ -2,13 +2,23 @@ module.exports = { rsyncDefaults: { size: 5, time: true, - recursive: true + recursive: false, + superficial: true }, attributes: { unsynced: 'makedrive-unsynced', conflict: 'makedrive-conflict', - checksum: 'makedrive-checksum' + checksum: 'makedrive-checksum', + partial: 'makedrive-partial', + pathsToSync: 'makedrive-pathsToSync' + }, + + // Sync Type constants + syncTypes: { + CREATE: 'create', + RENAME: 'rename', + DELETE: 'delete' }, server: { @@ -23,6 +33,7 @@ module.exports = { LISTENING: 'LISTENING', INIT: 'INIT', OUT_OF_DATE: 'OUT_OF_DATE', + SYNCING: 'SYNCING', CHKSUM: 'CHKSUM', PATCH: 'PATCH', ERROR: 'ERROR' diff --git a/lib/fs-utils.js b/lib/fs-utils.js index af53cf5..9ec3236 100644 --- a/lib/fs-utils.js +++ b/lib/fs-utils.js @@ -3,26 +3,9 @@ */ var constants = require('./constants.js'); -// copy oldPath to newPath, deleting newPath if it exists -function forceCopy(fs, oldPath, newPath, callback) { - fs.unlink(newPath, function(err) { - if(err && err.code !== 'ENOENT') { - return callback(err); - } - - fs.readFile(oldPath, function(err, buf) { - if(err) { - return callback(err); - } - - fs.writeFile(newPath, buf, callback); - }); - }); -} - -// See if a given path a) exists, and whether it is marked unsynced. -function isPathUnsynced(fs, path, callback) { - fs.getxattr(path, constants.attributes.unsynced, function(err, unsynced) { +// See if a given path a) exists, and whether it is marked with an xattr. +function hasAttr(fs, path, attr, callback) { + fs.getxattr(path, attr, function(err, attrVal) { // File doesn't exist locally at all if(err && err.code === 'ENOENT') { return callback(null, false); @@ -33,13 +16,20 @@ function isPathUnsynced(fs, path, callback) { return callback(err); } - callback(null, !!unsynced); + callback(null, !!attrVal); }); } -// Remove the unsynced metadata from a path -function removeUnsynced(fs, path, callback) { - fs.removexattr(path, constants.attributes.unsynced, function(err) { +// Remove the metadata from a path or file descriptor +function removeAttr(fs, pathOrFd, attr, isFd, callback) { + var removeFn = 'fremovexattr'; + + if(isFd !== true) { + callback = isFd; + removeFn = 'removexattr'; + } + + fs[removeFn](pathOrFd, attr, function(err) { if(err && err.code !== 'ENOATTR') { return callback(err); } @@ -47,16 +37,55 @@ function removeUnsynced(fs, path, callback) { callback(); }); } -function fremoveUnsynced(fs, fd, callback) { - fs.fremovexattr(fd, constants.attributes.unsynced, function(err) { + +// Get the metadata for a path or file descriptor +function getAttr(fs, pathOrFd, attr, isFd, callback) { + var getFn = 'fgetxattr'; + + if(isFd !== true) { + callback = isFd; + getFn = 'getxattr'; + } + + fs[getFn](pathOrFd, attr, function(err, value) { if(err && err.code !== 'ENOATTR') { return callback(err); } - callback(); + callback(null, value); }); } +// copy oldPath to newPath, deleting newPath if it exists +function forceCopy(fs, oldPath, newPath, callback) { + fs.unlink(newPath, function(err) { + if(err && err.code !== 'ENOENT') { + return callback(err); + } + + fs.readFile(oldPath, function(err, buf) { + if(err) { + return callback(err); + } + + fs.writeFile(newPath, buf, callback); + }); + }); +} + +// See if a given path a) exists, and whether it is marked unsynced. +function isPathUnsynced(fs, path, callback) { + hasAttr(fs, path, constants.attributes.unsynced, callback); +} + +// Remove the unsynced metadata from a path +function removeUnsynced(fs, path, callback) { + removeAttr(fs, path, constants.attributes.unsynced, callback); +} +function fremoveUnsynced(fs, fd, callback) { + removeAttr(fs, fd, constants.attributes.unsynced, true, callback); +} + // Set the unsynced metadata for a path function setUnsynced(fs, path, callback) { fs.setxattr(path, constants.attributes.unsynced, Date.now(), callback); @@ -67,42 +96,18 @@ function fsetUnsynced(fs, fd, callback) { // Get the unsynced metadata for a path function getUnsynced(fs, path, callback) { - fs.getxattr(path, constants.attributes.unsynced, function(err, value) { - if(err && err.code !== 'ENOATTR') { - return callback(err); - } - - callback(null, value); - }); + getAttr(fs, path, constants.attributes.unsynced, callback); } function fgetUnsynced(fs, fd, callback) { - fs.fgetxattr(fd, constants.attributes.unsynced, function(err, value) { - if(err && err.code !== 'ENOATTR') { - return callback(err); - } - - callback(null, value); - }); + getAttr(fs, fd, constants.attributes.unsynced, true, callback); } // Remove the Checksum metadata from a path function removeChecksum(fs, path, callback) { - fs.removexattr(path, constants.attributes.checksum, function(err) { - if(err && err.code !== 'ENOATTR') { - return callback(err); - } - - callback(); - }); + removeAttr(fs, path, constants.attributes.checksum, callback); } function fremoveChecksum(fs, fd, callback) { - fs.fremovexattr(fd, constants.attributes.checksum, function(err) { - if(err && err.code !== 'ENOATTR') { - return callback(err); - } - - callback(); - }); + removeAttr(fs, fd, constants.attributes.checksum, true, callback); } // Set the Checksum metadata for a path @@ -115,22 +120,55 @@ function fsetChecksum(fs, fd, checksum, callback) { // Get the Checksum metadata for a path function getChecksum(fs, path, callback) { - fs.getxattr(path, constants.attributes.checksum, function(err, value) { - if(err && err.code !== 'ENOATTR') { - return callback(err); - } - - callback(null, value); - }); + getAttr(fs, path, constants.attributes.checksum, callback); } function fgetChecksum(fs, fd, callback) { - fs.fgetxattr(fd, constants.attributes.checksum, function(err, value) { - if(err && err.code !== 'ENOATTR') { - return callback(err); - } + getAttr(fs, fd, constants.attributes.checksum, true, callback); +} - callback(null, value); - }); +// See if a given path a) exists, and whether it is marked partial. +function isPathPartial(fs, path, callback) { + hasAttr(fs, path, constants.attributes.partial, callback); +} + +// Remove the partial metadata from a path +function removePartial(fs, path, callback) { + removeAttr(fs, path, constants.attributes.partial, callback); +} +function fremovePartial(fs, fd, callback) { + removeAttr(fs, fd, constants.attributes.partial, true, callback); +} + +// Set the partial metadata for a path +function setPartial(fs, path, nodeCount, callback) { + fs.setxattr(path, constants.attributes.partial, nodeCount, callback); +} +function fsetPartial(fs, fd, nodeCount, callback) { + fs.fsetxattr(fd, constants.attributes.partial, nodeCount, callback); +} + +// Get the partial metadata for a path +function getPartial(fs, path, callback) { + getAttr(fs, path, constants.attributes.partial, callback); +} +function fgetPartial(fs, fd, callback) { + getAttr(fs, fd, constants.attributes.partial, true, callback); +} + +// Set the pathsToSync metadata for a path +function setPathsToSync(fs, path, pathsToSync, callback) { + fs.setxattr(path, constants.attributes.pathsToSync, pathsToSync, callback); +} +function fsetPathsToSync(fs, fd, pathsToSync, callback) { + fs.fsetxattr(fd, constants.attributes.pathsToSync, pathsToSync, callback); +} + +// Get the pathsToSync metadata for a path +function getPathsToSync(fs, path, callback) { + getAttr(fs, path, constants.attributes.pathsToSync, callback); +} +function fgetPathsToSync(fs, fd, callback) { + getAttr(fs, fd, constants.attributes.pathsToSync, true, callback); } module.exports = { @@ -151,5 +189,20 @@ module.exports = { setChecksum: setChecksum, fsetChecksum: fsetChecksum, getChecksum: getChecksum, - fgetChecksum: fgetChecksum + fgetChecksum: fgetChecksum, + + // Partial attr utils + isPathPartial: isPathPartial, + removePartial: removePartial, + fremovePartial: fremovePartial, + setPartial: setPartial, + fsetPartial: fsetPartial, + getPartial: getPartial, + fgetPartial: fgetPartial, + + // Paths to sync utils + setPathsToSync: setPathsToSync, + fsetPathsToSync: fsetPathsToSync, + getPathsToSync: getPathsToSync, + fgetPathsToSync: fgetPathsToSync }; diff --git a/lib/rsync/diff.js b/lib/rsync/diff.js index c164bef..918ae46 100644 --- a/lib/rsync/diff.js +++ b/lib/rsync/diff.js @@ -105,7 +105,6 @@ module.exports = function diff(fs, path, checksumList, options, callback) { if(options.versions) { return appendChecksum(diffNode, checksumNodePath, callback); } - diffList.push(diffNode); callback(null, diffList); @@ -121,9 +120,25 @@ module.exports = function diff(fs, path, checksumList, options, callback) { // Directory if(checksumNode.type === 'DIRECTORY') { diffNode.diffs = []; - diffList.push(diffNode); - return callback(); + if(options.recursive) { + diffList.push(diffNode); + return callback(); + } + + // If syncing is not done recursively, determine + // the number of nodes in the directory to indicate + // that that many nodes still need to be synced + return fs.readdir(checksumNodePath, function(err, entries) { + if(err) { + return callback(err); + } + + diffNode.nodeList = entries || []; + diffList.push(diffNode); + + return callback(); + }); } // Link @@ -160,7 +175,7 @@ module.exports = function diff(fs, path, checksumList, options, callback) { } // If the path was a file, clearly there was only one checksum - // entry i.e. the length of checksumList will be 1 which will + // entry i.e. the length of checksumList will be 1 which will // be stored in checksumList[0] var checksumNode = checksumList[0]; diff --git a/lib/rsync/patch.js b/lib/rsync/patch.js index 967260f..cc312b0 100644 --- a/lib/rsync/patch.js +++ b/lib/rsync/patch.js @@ -23,14 +23,15 @@ function difference(arr, farr) { }); } -// Path the destination filesystem by applying diffs +// Patch the destination filesystem by applying diffs module.exports = function patch(fs, path, diffList, options, callback) { callback = rsyncUtils.findCallback(callback, options); var paths = { synced: [], failed: [], - needsUpstream: [] + needsUpstream: [], + partial: [] }; var pathsToSync = extractPathsFromDiffs(diffList); @@ -41,8 +42,6 @@ module.exports = function patch(fs, path, diffList, options, callback) { options = rsyncUtils.configureOptions(options); - // Taken from - function handleError(err, callback) { // Determine the node paths for those that were not synced // by getting the difference between the paths that needed to @@ -60,7 +59,7 @@ module.exports = function patch(fs, path, diffList, options, callback) { // don't exist upstream yet, since an upstream sync will add them). function removeDeletedNodes(path, callback) { - function maybeUnlink(pathToDelete, callback) { + function maybeUnlink(pathToDelete, stats, callback) { if(pathsToSync.indexOf(pathToDelete) !== -1) { return callback(); } @@ -89,7 +88,12 @@ module.exports = function patch(fs, path, diffList, options, callback) { } paths.synced.push(pathToDelete); - fs.unlink(pathToDelete, callback); + + if(stats.isDirectory()) { + fs.rmdir(pathToDelete, callback); + } else { + fs.unlink(pathToDelete, callback); + } }); }); } @@ -103,25 +107,25 @@ module.exports = function patch(fs, path, diffList, options, callback) { } if(!stats.isDirectory()) { - return maybeUnlink(nodePath, callback); + return maybeUnlink(nodePath, stats, callback); } removeDeletedNodes(nodePath, callback); }); } - function removeDeletedNodesInDir(dirContents) { + function removeDeletedNodesInDir(dirContents, stats) { async.eachSeries(dirContents, processRemoval, function(err) { if(err) { return handleError(err, callback); } - maybeUnlink(path, function(err) { + maybeUnlink(path, stats, function(err) { if(err) { return handleError(err, callback); } - callback(null, paths); + callback(null, paths); }); }); } @@ -131,7 +135,7 @@ module.exports = function patch(fs, path, diffList, options, callback) { return callback(err); } - // Bail if the path is a file/link or + // Bail if the path is a file/link or // the path does not exist, i.e. nothing was patched if((err && err.code === 'ENOENT') || !stats.isDirectory()) { return callback(null, paths); @@ -142,7 +146,46 @@ module.exports = function patch(fs, path, diffList, options, callback) { return handleError(err, callback); } - removeDeletedNodesInDir(dirContents); + removeDeletedNodesInDir(dirContents, stats); + }); + }); + } + + function updatePartialListInParent(nodePath, parent, callback) { + // Now that the node has been synced, + // update the parent directory's list of nodes + // that still need to be synced + fsUtils.getPartial(fs, parent, function(err, nodeList) { + if(err) { + return callback(err); + } + + if(!nodeList) { + return callback(null, paths); + } + + nodeList.splice(nodeList.indexOf(nodePath), 1); + + // No more nodes left to be synced in the parent + // remove the partial attribute + if(!nodeList.length) { + return fsUtils.removePartial(fs, parent, function(err) { + if(err) { + return callback(err); + } + + callback(null, paths); + }); + } + + // Update the parent's partial attribute with the remaining + // nodes that need to be synced + fsUtils.setPartial(fs, parent, nodeList, function(err) { + if(err) { + return callback(err); + } + + callback(null, paths); }); }); } @@ -212,7 +255,7 @@ module.exports = function patch(fs, path, diffList, options, callback) { // the server if(checksum === diffNode.checksum) { paths.needsUpstream.push(filePath); - return callback(null, paths); + return updatePartialListInParent(filePath, Path.dirname(filePath), callback); } applyPatch(getPatchedData(data)); @@ -227,7 +270,7 @@ module.exports = function patch(fs, path, diffList, options, callback) { } paths.synced.push(filePath); - callback(null, paths); + updatePartialListInParent(filePath, Path.dirname(filePath), callback); }); } @@ -249,7 +292,7 @@ module.exports = function patch(fs, path, diffList, options, callback) { } paths.synced.push(filePath); - callback(null, paths); + updatePartialListInParent(filePath, Path.dirname(filePath), callback); }); }); } @@ -327,7 +370,8 @@ module.exports = function patch(fs, path, diffList, options, callback) { } paths.synced.push(linkPath); - callback(null, paths); + + updatePartialListInParent(linkPath, Path.dirname(linkPath), callback); }); } @@ -335,12 +379,36 @@ module.exports = function patch(fs, path, diffList, options, callback) { var dirPath = diffNode.path; fs.mkdir(dirPath, function(err) { - if(err && err.code !== 'EEXIST') { - return handleError(err, callback); + if(err) { + if(err.code !== 'EEXIST') { + return handleError(err, callback); + } + + paths.synced.push(dirPath); + return callback(null, paths); + } + + // Syncing is recursive so directory contents will + // be subsequently synced and thus the directory is + // not partial + if(options.recursive) { + paths.synced.push(dirPath); + return callback(null, paths); } - paths.synced.push(dirPath); - callback(null, paths); + // Newly created directory will be marked as + // partial to indicate that its contents still + // need to be synced + fsUtils.setPartial(fs, dirPath, diffNode.nodeList, function(err) { + if(err) { + return handleError(err, callback); + } + + paths.synced.push(dirPath); + paths.partial.push(dirPath); + + updatePartialListInParent(dirPath, Path.dirname(dirPath), callback); + }); }); } @@ -375,13 +443,44 @@ module.exports = function patch(fs, path, diffList, options, callback) { return handleError(err, callback); } - removeDeletedNodes(path, callback); + fsUtils.getPartial(fs, path, function(err, nodeList) { + if(err) { + return callback(err); + } + + if(!nodeList || nodeList.length !== 0) { + if(options.superficial) { + return callback(null, paths); + } + + return removeDeletedNodes(path, callback); + } + + // Now that the immediate nodes in the directory + // have been created or the directory has no nodes + // in it, remove the partial attr if it exists + fsUtils.removePartial(fs, path, function(err) { + if(err) { + return handleError(err, callback); + } + + // Remove the directory entry whose contents have just + // been synced, from the partial array + paths.partial.splice(paths.partial.indexOf(path), 1); + + if(options.superficial) { + callback(null, paths); + } else { + removeDeletedNodes(path, callback); + } + }); + }); }); } // Create any parent directories that do not exist function createParentDirectories(path, callback) { - fs.Shell().mkdirp(Path.dirname(path), function(err) { + (new fs.Shell()).mkdirp(Path.dirname(path), function(err) { if(err && err.code !== 'EEXIST') { return callback(err); } diff --git a/lib/rsync/rsync-utils.js b/lib/rsync/rsync-utils.js index e3147f9..3a49e8e 100644 --- a/lib/rsync/rsync-utils.js +++ b/lib/rsync/rsync-utils.js @@ -13,7 +13,9 @@ */ var MD5 = require('MD5'); -var Errors = require('../filer').Errors; +var Filer = require('filer'); +var Errors = Filer.Errors; +var Path = Filer.Path; var async = require('../async-lite'); var fsUtils = require('../fs-utils'); @@ -29,6 +31,8 @@ var fsUtils = require('../fs-utils'); // false: sync symbolic links as the files they link to in destination [default] // versions - true: do not sync a node if the last synced version matches the version it needs to be synced to [default] // false: sync nodes irrespective of the last synced version +// superficial- true: if a directory path is provided, only sync the directory and not it's contents +// false: if a directory path is provided, sync it's contents [default] function configureOptions(options) { if(!options || typeof options === 'function') { options = {}; @@ -40,6 +44,7 @@ function configureOptions(options) { options.time = options.time || false; options.links = options.links || false; options.versions = options.versions !== false; + options.superficial = options.superficial || false; return options; } @@ -203,7 +208,7 @@ function roll(data, checksums, blockSize) { return results; } -// Rsync function to calculate checksums for +// Rsync function to calculate checksums for // a file by dividing it into blocks of data // whose size is passed in and checksuming each // block of data @@ -368,9 +373,9 @@ function generateChecksums(fs, paths, stampNode, callback) { }); } -// Compare two file systems. This is done by comparing the -// checksums for a collection of paths in one file system -// against the checksums for the same those paths in +// Compare two file systems. This is done by comparing the +// checksums for a collection of paths in one file system +// against the checksums for the same those paths in // another file system function compareContents(fs, checksumList, callback) { var ECHKSUM = "Checksums do not match"; @@ -427,6 +432,38 @@ function compareContents(fs, checksumList, callback) { }); } +function del(fs, path, callback) { + var paramError = validateParams(fs, path); + if(paramError) { + return callback(paramError); + } + + fs.lstat(path, function(err, stats) { + if(err) { + return callback(err); + } + + if(stats.isDirectory()) { + fs.rmdir(path, callback); + } else { + fs.unlink(path, callback); + } + }); +} + +function rename(fs, oldPath, newPath, callback) { + var paramError = validateParams(fs, oldPath) && (newPath ? null : new Errors.EINVAL('New name not specified')); + if(paramError) { + return callback(paramError); + } + + if(Path.dirname(oldPath) !== Path.dirname(newPath)) { + return callback(new Errors.EINVAL('New path name does not have the same parent as the old path')); + } + + fs.rename(oldPath, newPath, callback); +} + module.exports = { blockChecksums: blockChecksums, getChecksum: getChecksum, @@ -435,5 +472,7 @@ module.exports = { compareContents: compareContents, configureOptions: configureOptions, findCallback: findCallback, - validateParams: validateParams + validateParams: validateParams, + del: del, + rename: rename }; diff --git a/lib/rsync/source-list.js b/lib/rsync/source-list.js index f1fe55c..1a43ae0 100644 --- a/lib/rsync/source-list.js +++ b/lib/rsync/source-list.js @@ -20,7 +20,7 @@ module.exports = function sourceList(fs, path, options, callback) { this.path = path; this.modified = stats.mtime; this.size = stats.size; - this.type = stats.type; + this.type = stats.type; } // Make sure this isn't a conflicted copy before adding @@ -40,6 +40,11 @@ module.exports = function sourceList(fs, path, options, callback) { } function getSrcListForDir(stats) { + if(options.superficial) { + sources.push(new SourceNode(path, stats)); + return callback(null, sources); + } + fs.readdir(path, function(err, entries) { if(err) { return callback(err); @@ -67,7 +72,7 @@ module.exports = function sourceList(fs, path, options, callback) { } sources = sources.concat(items); - + callback(); }); }); diff --git a/lib/sync-path-resolver.js b/lib/sync-path-resolver.js deleted file mode 100644 index 772013a..0000000 --- a/lib/sync-path-resolver.js +++ /dev/null @@ -1,77 +0,0 @@ -/** - * Sync path resolver is a library that provides - * functionality to determine 'syncable' paths - * It exposes the following methods: - * - * resolve - This method takes two paths as arguments. - * The goal is to find the most common ancestor - * between them. For e.g. the most common ancestor - * between '/dir' and '/dir/file.txt' is '/dir' while - * between '/dir' and '/file.txt' would be '/'. - * - * resolveFromArray - This method works exactly like resolve but works for arrays of paths instead -*/ - -var pathResolver = {}; -var dirname = require('./filer').Path.dirname; - -function getDepth(path) { - if(path === '/') { - return 0; - } - - return 1 + getDepth(dirname(path)); -} - -function commonAncestor(path1, depth1, path2, depth2) { - if(path1 === path2) { - return path1; - } - - // Regress the appropriate path - if(depth1 === depth2) { - path1 = dirname(path1); - depth1--; - path2 = dirname(path2); - depth2--; - } else if(depth1 > depth2) { - path1 = dirname(path1); - depth1--; - } else { - path2 = dirname(path2); - depth2--; - } - - return commonAncestor(path1, depth1, path2, depth2); -} - -pathResolver.resolve = function(path1, path2) { - if(!path1 && !path2) { - return '/'; - } - - if(!path1 || !path2) { - return path1 || path2; - } - - var path1Depth = getDepth(path1); - var path2Depth = getDepth(path2); - - return commonAncestor(path1, path1Depth, path2, path2Depth); -}; - -pathResolver.resolveFromArray = function(paths) { - if(!paths) { - return '/'; - } - - var resolvedPath, i; - - for(i = 1, resolvedPath = paths[0]; i < paths.length; i++) { - resolvedPath = pathResolver.resolve(resolvedPath, paths[i]); - } - - return resolvedPath; -}; - -module.exports = pathResolver; diff --git a/lib/syncmessage.js b/lib/syncmessage.js index c89fb92..272ed67 100644 --- a/lib/syncmessage.js +++ b/lib/syncmessage.js @@ -26,14 +26,14 @@ function SyncMessage(type, name, content) { }, // Names - get srclist() { - return that.name === SyncMessage.SRCLIST; + get sourceList() { + return that.name === SyncMessage.SOURCELIST; }, get sync() { return that.name === SyncMessage.SYNC; }, - get chksum() { - return that.name === SyncMessage.CHKSUM; + get checksums() { + return that.name === SyncMessage.CHECKSUMS; }, get diffs() { return that.name === SyncMessage.DIFFS; @@ -56,6 +56,9 @@ function SyncMessage(type, name, content) { get impl() { return that.name === SyncMessage.IMPL; }, + get content() { + return that.name === SyncMessage.INCONT; + }, get serverReset() { return that.name === SyncMessage.SERVER_RESET; }, @@ -64,26 +67,50 @@ function SyncMessage(type, name, content) { }, get fileSizeError() { return that.type === SyncMessage.ERROR && that.name === SyncMessage.MAXSIZE; + }, + get root() { + return that.name === SyncMessage.ROOT; + }, + get needsDownstream() { + return that.type === SyncMessage.ERROR && that.name === SyncMessage.NEEDS_DOWNSTREAM; + }, + get interrupted() { + return that.name === SyncMessage.INTERRUPTED; + }, + get delay() { + return that.type === SyncMessage.REQUEST && that.name === SyncMessage.DELAY; + }, + get rename() { + return that.name === SyncMessage.RENAME; + }, + get del() { + return that.name === SyncMessage.DEL; } }; } SyncMessage.isValidName = function(name) { - return name === SyncMessage.SRCLIST || - name === SyncMessage.CHKSUM || - name === SyncMessage.DIFFS || - name === SyncMessage.LOCKED || - name === SyncMessage.PATCH || - name === SyncMessage.VERIFICATION || - name === SyncMessage.SYNC || - name === SyncMessage.RESET || - name === SyncMessage.AUTHZ || - name === SyncMessage.IMPL || - name === SyncMessage.INFRMT || - name === SyncMessage.INCONT || - name === SyncMessage.SERVER_RESET || - name === SyncMessage.DOWNSTREAM_LOCKED || - name === SyncMessage.MAXSIZE; + return name === SyncMessage.SOURCELIST || + name === SyncMessage.CHECKSUMS || + name === SyncMessage.DIFFS || + name === SyncMessage.LOCKED || + name === SyncMessage.PATCH || + name === SyncMessage.VERIFICATION || + name === SyncMessage.SYNC || + name === SyncMessage.RESET || + name === SyncMessage.AUTHZ || + name === SyncMessage.IMPL || + name === SyncMessage.INFRMT || + name === SyncMessage.INCONT || + name === SyncMessage.SERVER_RESET || + name === SyncMessage.DOWNSTREAM_LOCKED || + name === SyncMessage.MAXSIZE || + name === SyncMessage.ROOT || + name === SyncMessage.NEEDS_DOWNSTREAM || + name === SyncMessage.INTERRUPTED || + name === SyncMessage.DELAY || + name === SyncMessage.RENAME || + name === SyncMessage.DEL; }; SyncMessage.isValidType = function(type) { @@ -118,9 +145,9 @@ SyncMessage.RESPONSE = "RESPONSE"; SyncMessage.ERROR = "ERROR"; // SyncMessage Name constants -SyncMessage.SRCLIST = "SRCLIST"; +SyncMessage.SOURCELIST = "SOURCELIST"; SyncMessage.SYNC = "SYNC"; -SyncMessage.CHKSUM = "CHKSUM"; +SyncMessage.CHECKSUMS = "CHECKSUMS"; SyncMessage.DIFFS = "DIFFS"; SyncMessage.PATCH = "PATCH"; SyncMessage.VERIFICATION = "VERIFICATION"; @@ -132,6 +159,11 @@ SyncMessage.SERVER_RESET = "SERVER_RESET"; SyncMessage.DOWNSTREAM_LOCKED = "DOWNSTREAM_LOCKED"; SyncMessage.MAXSIZE = "MAXSIZE"; SyncMessage.INTERRUPTED = "INTERRUPTED"; +SyncMessage.ROOT = "ROOT"; +SyncMessage.NEEDS_DOWNSTREAM = "NEEDS DOWNSTREAM"; +SyncMessage.DELAY = "DELAY DOWNSTREAM"; +SyncMessage.RENAME = "RENAME"; +SyncMessage.DEL = "DELETE"; // SyncMessage Error constants SyncMessage.INFRMT = "INVALID FORMAT"; @@ -142,14 +174,23 @@ SyncMessage.request = { get diffs() { return new SyncMessage(SyncMessage.REQUEST, SyncMessage.DIFFS); }, - get chksum() { - return new SyncMessage(SyncMessage.REQUEST, SyncMessage.CHKSUM); + get checksums() { + return new SyncMessage(SyncMessage.REQUEST, SyncMessage.CHECKSUMS); }, get sync() { return new SyncMessage(SyncMessage.REQUEST, SyncMessage.SYNC); }, get reset() { return new SyncMessage(SyncMessage.REQUEST, SyncMessage.RESET); + }, + get delay() { + return new SyncMessage(SyncMessage.REQUEST, SyncMessage.DELAY); + }, + get rename() { + return new SyncMessage(SyncMessage.REQUEST, SyncMessage.RENAME); + }, + get del() { + return new SyncMessage(SyncMessage.REQUEST, SyncMessage.DEL); } }; SyncMessage.response = { @@ -170,11 +211,14 @@ SyncMessage.response = { }, get reset() { return new SyncMessage(SyncMessage.RESPONSE, SyncMessage.RESET); + }, + get root() { + return new SyncMessage(SyncMessage.RESPONSE, SyncMessage.ROOT); } }; SyncMessage.error = { - get srclist() { - return new SyncMessage(SyncMessage.ERROR, SyncMessage.SRCLIST); + get sourceList() { + return new SyncMessage(SyncMessage.ERROR, SyncMessage.SOURCELIST); }, get diffs() { return new SyncMessage(SyncMessage.ERROR, SyncMessage.DIFFS); @@ -182,8 +226,8 @@ SyncMessage.error = { get locked() { return new SyncMessage(SyncMessage.ERROR, SyncMessage.LOCKED); }, - get chksum() { - return new SyncMessage(SyncMessage.ERROR, SyncMessage.CHKSUM); + get checksums() { + return new SyncMessage(SyncMessage.ERROR, SyncMessage.CHECKSUMS); }, get patch() { return new SyncMessage(SyncMessage.ERROR, SyncMessage.PATCH); @@ -217,7 +261,31 @@ SyncMessage.error = { }, get interrupted() { return new SyncMessage(SyncMessage.ERROR, SyncMessage.INTERRUPTED); + }, + get needsDownstream() { + return new SyncMessage(SyncMessage.ERROR, SyncMessage.NEEDS_DOWNSTREAM); + }, + get rename() { + return new SyncMessage(SyncMessage.ERROR, SyncMessage.RENAME); + }, + get del() { + return new SyncMessage(SyncMessage.ERROR, SyncMessage.DEL); + } +}; + +SyncMessage.prototype.invalidContent = function(keys) { + var content = this.content; + keys = keys || []; + + if(!content || !content.path) { + return true; } + + for(var i = 0; i < keys.length; i++) { + if(!content[keys[i]]) return true; + } + + return false; }; module.exports = SyncMessage; diff --git a/lib/util.js b/lib/util.js new file mode 100644 index 0000000..87e1ae0 --- /dev/null +++ b/lib/util.js @@ -0,0 +1,15 @@ +// General utility methods + +function findPathIndexInArray(array, path) { + for(var i = 0; i < array.length; i++) { + if(array[i].path === path) { + return i; + } + } + + return -1; +} + +module.exports = { + findPathIndexInArray: findPathIndexInArray +}; diff --git a/package.json b/package.json index 1055fc2..4e26236 100644 --- a/package.json +++ b/package.json @@ -57,8 +57,8 @@ "browser-request": "^0.3.1", "bunyan": "^1.1.3", "events": "^1.0.1", - "express": "3.4.5", - "filer": "git://github.com/alicoding/filer.git#mozfest", + "express": "^3.19.1", + "filer": "^0.0.41", "filer-fs": "0.1.2", "filer-s3": "0.1.1", "filer-sql": "0.0.3", @@ -76,9 +76,10 @@ "passport-strategy": "^1.0.0", "recluster": "^0.3.7", "redis": "^0.12.1", + "server-destroy": "^1.0.0", "useragent": "^2.1.0", "webmaker-auth": "0.0.14", - "ws": "^0.4.31" + "ws": "^0.6.5" }, "optionalDependencies": { "hiredis": "^0.1.17" diff --git a/server/index.js b/server/index.js index 37542d2..e84cbb9 100644 --- a/server/index.js +++ b/server/index.js @@ -4,7 +4,7 @@ */ var recluster = require('recluster'); var env = require('./lib/environment'); -var serverPath = require('path').join(__dirname, 'server.js'); +var serverPath = require('path').join(__dirname, 'server-cluster.js'); var cluster = recluster(serverPath, { workers: env.get('FORKS') || 1, diff --git a/server/lib/client-manager.js b/server/lib/client-manager.js index 57aaa5f..efc159a 100644 --- a/server/lib/client-manager.js +++ b/server/lib/client-manager.js @@ -33,13 +33,13 @@ function runClient(client) { data = JSON.parse(msg.data); message = SyncMessage.parse(data); - - // Delegate ws messages to the sync protocol handler at this point - client.handler.handleMessage(message); } catch(error) { log.error({client: client, err: error}, 'Unable to parse/handle client message. Data was `%s`', msg.data); - invalidMessage(); + return invalidMessage(); } + + // Delegate ws messages to the sync protocol handler at this point + client.handler.handleMessage(message); } else { log.warn({client: client}, 'Expected string but got binary data over web socket.'); invalidMessage(); @@ -47,7 +47,7 @@ function runClient(client) { }; // Send an AUTHZ response to let client know normal sync'ing can begin. - client.state = States.INIT; + client.state = States.LISTENING; client.sendMessage(SyncMessage.response.authz); log.debug({client: client}, 'Starting authorized client session'); } @@ -120,6 +120,10 @@ var clients = []; * or life-cycle. */ function remove(client) { + if(!clients) { + return; + } + var idx = clients.indexOf(client); if(idx > -1) { clients.splice(idx, 1); @@ -135,6 +139,7 @@ function add(client) { remove(client); }); + clients = clients || []; clients.push(client); initClient(client); } @@ -144,25 +149,37 @@ function add(client) { */ function shutdown(callback) { var closed = 0; - var connected = clients.length; + var connected = clients ? clients.length : 0; function maybeFinished() { if(++closed >= connected) { clients = null; log.info('[Shutdown] All client connections safely closed.'); - callback(); + return callback(); } + log.info('[Shutdown] Closed client %s of %s.', closed, connected); } - if(connected === 0) { + if(!connected) { return maybeFinished(); } - clients.forEach(function(client) { - client.once('closed', maybeFinished); - client.close(); - }); + var client; + + for(var i = 0; i < connected; i++) { + client = clients[i] || null; + + if(!client) { + maybeFinished(); + } else { + client.once('closed', maybeFinished); + + if(client.state !== States.CLOSING && client.state !== States.CLOSED) { + client.close(); + } + } + } } module.exports = { diff --git a/server/lib/client.js b/server/lib/client.js index 015d561..e90f2b6 100644 --- a/server/lib/client.js +++ b/server/lib/client.js @@ -6,14 +6,15 @@ * protocol as normal. */ var SyncProtocolHandler = require('./sync-protocol-handler.js'); -var SyncMessage = require('../../lib/syncmessage.js'); var EventEmitter = require('events').EventEmitter; var ClientInfo = require('./client-info.js'); var Constants = require('../../lib/constants.js'); var States = Constants.server.states; +var syncTypes = Constants.syncTypes; var redis = require('../redis-clients.js'); var util = require('util'); var log = require('./logger.js'); +var findPathIndexInArray = require('../../lib/util.js').findPathIndexInArray; var noop = function(){}; @@ -42,18 +43,15 @@ function handleBroadcastMessage(msg, client) { return; } - // Re-hydrate a full SyncMessage object from partial data sent via msg - var response = new SyncMessage.parse(msg.syncMessage); - - // If this client was in the process of a downstream sync, we - // want to reactivate it with a path that is the common ancestor - // of the path originally being synced, and the path that was just - // updated in this upstream sync. - if(client.downstreamInterrupted) { - client.handler.restartDownstream(response.content.path); - } else { - client.handler.sendOutOfDate(response); + client.outOfDate = client.outOfDate || []; + client.currentDownstream = client.currentDownstream || []; + var outOfDateSync = {path: msg.path, type: msg.type}; + if(msg.type === syncTypes.RENAME) { + outOfDateSync.oldPath = msg.oldPath; } + client.outOfDate.push(outOfDateSync); + + client.handler.syncDownstream(); } function Client(ws) { @@ -79,9 +77,9 @@ function Client(ws) { // whether or not we are in a sync step (e.g., patching) that will leave the // server's filesystem corrupt if not completed. self.closable = true; - self.path = '/'; // We start using this in client-manager.js when a client is fully authenticated. self.handler = new SyncProtocolHandler(self); + self.outOfDate = []; ws.onerror = function(err) { log.error({err: err, client: self}, 'Web Socket error'); @@ -160,10 +158,11 @@ Client.prototype.close = function(error) { // If we're passed error info, try to close with that first if(self.ws) { error = error || {}; + self.ws.onclose = noop; + if(error.code && error.message) { // Ignore onerror, oncall with this call self.ws.onerror = noop; - self.ws.onclose = noop; self.ws.close(error.code, error.message); } @@ -204,7 +203,7 @@ Client.prototype.sendMessage = function(syncMessage) { if(info) { info.bytesSent += Buffer.byteLength(data, 'utf8'); } - + ws.send(syncMessage.stringify()); log.debug({syncMessage: syncMessage, client: self}, 'Sending Sync Protocol Message'); } catch(err) { @@ -215,4 +214,42 @@ Client.prototype.sendMessage = function(syncMessage) { } }; +Client.prototype.delaySync = function(path) { + var self = this; + var indexInCurrent = findPathIndexInArray(self.currentDownstream, path); + var delayedSync = indexInCurrent === -1 ? null : self.currentDownstream.splice(indexInCurrent, 1); + var syncTime; + + if(delayedSync) { + syncTime = Date.now() - (delayedSync._syncStarted || 0); + log.info({client: self}, 'Downstream sync delayed for ' + path + ' after ' + syncTime + ' ms'); + } else { + log.warn({client: self}, 'Sync entry not found in current downstreams when attempting to delay sync for ' + path); + } +}; + +Client.prototype.endDownstream = function(path) { + var self = this; + var indexInCurrent = findPathIndexInArray(self.currentDownstream, path); + var indexInOutOfDate = findPathIndexInArray(self.outOfDate, path); + var syncEnded; + var syncTime; + + if(indexInCurrent === -1) { + log.warn({client: self}, 'Sync entry not found in current downstreams when attempting to end sync for ' + path); + } else { + syncEnded = self.currentDownstream.splice(indexInCurrent, 1); + } + + syncTime = syncEnded ? Date.now() - syncEnded._syncStarted : 0; + + if(indexInOutOfDate === -1) { + log.warn({client: self}, 'Sync entry not found in out of date list when attempting to end sync for ' + path); + return; + } + + self.outOfDate.splice(indexInOutOfDate, 1); + log.info({client: self}, 'Downstream sync completed for ' + path + ' in ' + syncTime + ' ms'); +}; + module.exports = Client; diff --git a/server/lib/filer-www/default-handler.js b/server/lib/filer-www/default-handler.js index a50440b..88337bc 100644 --- a/server/lib/filer-www/default-handler.js +++ b/server/lib/filer-www/default-handler.js @@ -43,7 +43,7 @@ function handleFile(fs, path, res) { * Send an Apache-style directory listing */ function handleDir(fs, path, res) { - var sh = fs.Shell(); + var sh = new fs.Shell(); var parent = Path.dirname(path); var header = '' + diff --git a/server/lib/filer-www/json-handler.js b/server/lib/filer-www/json-handler.js index f0e125f..a472943 100644 --- a/server/lib/filer-www/json-handler.js +++ b/server/lib/filer-www/json-handler.js @@ -49,7 +49,7 @@ function handleFile(fs, path, res) { * Send recursive dir listing */ function handleDir(fs, path, res) { - var sh = fs.Shell(); + var sh = new fs.Shell(); sh.ls(path, {recursive: true}, function(err, listing) { if(err) { diff --git a/server/lib/logger.js b/server/lib/logger.js index 8b6b53b..71c111d 100644 --- a/server/lib/logger.js +++ b/server/lib/logger.js @@ -1,5 +1,5 @@ var env = require('./environment.js'); -var messina = require('messina'); +var messina = require('messina'); var PrettyStream = require('bunyan-prettystream'); var NODE_ENV = env.get('NODE_ENV') || 'development'; @@ -21,7 +21,8 @@ var logger = messina({ syncMessage: function syncMessageSerializer(msg) { return { type: msg.type, - name: msg.name + name: msg.name, + path: msg.content && msg.content.path }; }, // See server/lib/client.js @@ -64,6 +65,7 @@ var logger = messina({ // "synclock:" username: lock.key.split(':')[1], id: lock.value, + path: lock.path, allowLockRequest: lock.allowLockRequest, isUnlocked: !!lock.unlocked, ageInMS: lock.age diff --git a/server/lib/sync-lock.js b/server/lib/sync-lock.js index eaeec98..f9a7656 100644 --- a/server/lib/sync-lock.js +++ b/server/lib/sync-lock.js @@ -27,20 +27,20 @@ function handleLockRequest(message, lock) { } // Not meant for this lock, skip - if(lock.key !== message.key) { + if(lock.key !== message.key || lock.path !== message.path) { return; } // If the owner thinks this lock is not yet unlockable, respond as such if(!lock.allowLockRequest) { log.debug({syncLock: lock}, 'Denying lock override request for client id=%s.', message.id); - redis.publish(Constants.server.lockResponseChannel, JSON.stringify({key: lock.key, unlocked: false})); + redis.publish(Constants.server.lockResponseChannel, JSON.stringify({key: lock.key, path: lock.path, unlocked: false})); return; } // Otherwise, give up the lock by overwriting the value with the // requesting client's ID (replacing ours), and respond that we've released it. - redis.set(lock.key, message.id, function(err, reply) { + redis.hset(lock.key, lock.path, message.id, function(err, reply) { if(err) { log.error({err: err, syncLock: lock}, 'Error setting redis lock key.'); return; @@ -54,15 +54,16 @@ function handleLockRequest(message, lock) { log.debug({syncLock: lock}, 'Allowing lock override request for id=%s.', message.id); lock.unlocked = true; lock.emit('unlocked'); - redis.publish(Constants.server.lockResponseChannel, JSON.stringify({key: lock.key, unlocked: true})); + redis.publish(Constants.server.lockResponseChannel, JSON.stringify({key: lock.key, path: lock.path, unlocked: true})); }); } -function SyncLock(key, id) { +function SyncLock(key, id, path) { EventEmitter.call(this); this.key = key; this.value = id; + this.path = path; // Listen for requests to release this lock early. var lock = this; @@ -97,13 +98,14 @@ SyncLock.generateKey = function(username) { SyncLock.prototype.release = function(callback) { var lock = this; var key = lock.key; + var path = lock.path; // Stop listening for requests to release this lock redis.removeListener('lock-request', lock._handleLockRequestFn); lock._handleLockRequestFn = null; // Try to delete the lock in redis - redis.del(key, function(err, reply) { + redis.hdel(key, path, function(err, reply) { // NOTE: we don't emit the unlocked event here, but use the callback instead. // The unlocked event indicates that the lock was released without calling release(). lock.unlocked = true; @@ -118,7 +120,7 @@ SyncLock.prototype.release = function(callback) { }); }; -function handleLockResponse(message, key, client, waitTimer, callback) { +function handleLockResponse(message, key, path, client, waitTimer, callback) { var id = client.id; try { @@ -129,7 +131,7 @@ function handleLockResponse(message, key, client, waitTimer, callback) { } // Not meant for this lock, skip - if(key !== message.key) { + if(key !== message.key || path !== message.path) { return; } @@ -142,11 +144,11 @@ function handleLockResponse(message, key, client, waitTimer, callback) { // The result of the request is defined in the `unlocked` param, // which is true if we now hold the lock, false if not. if(message.unlocked) { - var lock = new SyncLock(key, id); + var lock = new SyncLock(key, id, path); log.debug({syncLock: lock}, 'Lock override acquired.'); callback(null, lock); } else { - log.debug('Lock override denied for key %s.', key); + log.debug('Lock override denied for %s in key %s.', path, key); callback(); } } @@ -154,25 +156,25 @@ function handleLockResponse(message, key, client, waitTimer, callback) { /** * Request a lock for the current client. */ -function request(client, callback) { +function request(client, path, callback) { var key = SyncLock.generateKey(client.username); var id = client.id; - // Try to set this key/value pair, but fail if the key already exists. - redis.setnx(key, id, function(err, reply) { + // Try to set this key/value pair, but fail if the path for the key already exists. + redis.hsetnx(key, path, id, function(err, reply) { if(err) { - log.error({err: err, client: client}, 'Error trying to set redis key with setnx'); + log.error({err: err, client: client}, 'Error trying to set redis key with hsetnx'); return callback(err); } if(reply === 1) { - // Success, we have the lock (key was set). Return a new SyncLock instance - var lock = new SyncLock(key, id); + // Success, we have the lock (path for the key was set). Return a new SyncLock instance + var lock = new SyncLock(key, id, path); log.debug({client: client, syncLock: lock}, 'Lock acquired.'); return callback(null, lock); } - // Key was not set (held by another client). See if the lock owner would be + // Path for key was not set (held by another client). See if the lock owner would be // willing to let us take it. We'll wait a bit for a reply, and if // we don't get one, assume the client holding the lock, or its server, // has crashed, and the lock is OK to take. @@ -183,18 +185,13 @@ function request(client, callback) { redis.removeListener('lock-response', client._handleLockResponseFn); client._handleLockResponseFn = null; - redis.set(key, id, function(err, reply) { + redis.hset(key, path, id, function(err) { if(err) { log.error({err: err, client: client}, 'Error setting redis lock key.'); return callback(err); } - if(reply !== 'OK') { - log.error({err: err, client: client}, 'Error setting redis lock key, expected OK reply, got %s.', reply); - return callback(new Error('Unexepcted redis response: ' + reply)); - } - - var lock = new SyncLock(key, id); + var lock = new SyncLock(key, id, path); log.debug({client: client, syncLock: lock}, 'Lock request timeout, setting lock manually.'); callback(null, lock); }); @@ -203,22 +200,22 @@ function request(client, callback) { // Listen for a response from the client holding the lock client._handleLockResponseFn = function(message) { - handleLockResponse(message, key, client, waitTimer, callback); + handleLockResponse(message, key, path, client, waitTimer, callback); }; redis.on('lock-response', client._handleLockResponseFn); // Ask the client holding the lock to give it to us - log.debug({client: client}, 'Requesting lock override.'); - redis.publish(Constants.server.lockRequestChannel, JSON.stringify({key: key, id: id})); + log.debug({client: client}, 'Requesting lock override for ' + path); + redis.publish(Constants.server.lockRequestChannel, JSON.stringify({key: key, id: id, path: path})); }); } /** * Check to see if a lock is held for the given username. */ -function isUserLocked(username, callback) { +function isUserLocked(username, path, callback) { var key = SyncLock.generateKey(username); - redis.get(key, function(err, value) { + redis.hget(key, path, function(err, value) { if(err) { log.error(err, 'Error getting redis lock key %s.', key); return callback(err); diff --git a/server/lib/sync-protocol-handler.js b/server/lib/sync-protocol-handler.js index bfcba7c..5eb06e2 100644 --- a/server/lib/sync-protocol-handler.js +++ b/server/lib/sync-protocol-handler.js @@ -3,19 +3,19 @@ var rsync = require('../../lib/rsync'); var diffHelper = require('../../lib/diff'); var EventEmitter = require('events').EventEmitter; var util = require('util'); -var getCommonPath = require('../../lib/sync-path-resolver').resolve; var SyncLock = require('./sync-lock.js'); var redis = require('../redis-clients.js'); var log = require('./logger.js'); var Constants = require('../../lib/constants.js'); +var findPathIndexInArray = require('../../lib/util.js').findPathIndexInArray; var ServerStates = Constants.server.states; var rsyncOptions = Constants.rsyncDefaults; var States = Constants.server.states; +var syncTypes = Constants.syncTypes; var env = require('./environment.js'); var MAX_SYNC_SIZE_BYTES = env.get('MAX_SYNC_SIZE_BYTES') || Math.Infinity; - function SyncProtocolHandler(client) { EventEmitter.call(this); this.client = client; @@ -45,7 +45,7 @@ function ensureClient(client) { // If we're in an unexpected state for syncing (i.e., not a sync step), log that switch(client.state) { case ServerStates.CREATED: - case ServerStates.CLOSESD: + case ServerStates.CLOSED: case ServerStates.CLOSING: case ServerStates.CONNECTING: case ServerStates.ERROR: @@ -56,20 +56,162 @@ function ensureClient(client) { return client; } -SyncProtocolHandler.prototype.handleMessage = function(message) { - var client = ensureClient(this.client); +function getListOfSyncs(fs, callback) { + var syncList = []; + var sh = new fs.Shell(); - if(message.is.request) { - log.debug({syncMessage: message, client: client}, 'Received sync protocol Request message'); - this.handleRequest(message); - } else if(message.is.response) { - log.debug({syncMessage: message, client: client}, 'Received sync protocol Response message'); - this.handleResponse(message); - } else { - log.warn({client: client, syncMessage: message}, 'Invalid sync message type'); - client.sendMessage(SyncProtocolHandler.error.type); + function trimTrailingSlash(path, next) { + if(path === '/') { + return next(); + } + + if(path.charAt(path.length - 1) === '/') { + path = path.substring(0, path.length - 1); + } + + syncList.push({path: path, type: syncTypes.CREATE}); + next(); } -}; + + sh.find('/', {exec: trimTrailingSlash}, function(err) { + if(err) { + return callback(err); + } + // Needs to be optimized as this array contains redundant syncs + // For e.g. the array will contain [{path: '/dir'}, {path: '/dir/file'}] + // In this case, the '/dir' entry can be removed because the sync for + // '/dir/file' will create '/dir' + callback(null, syncList); + }); +} + +// Most upstream sync steps require a lock to be held. +// It's a bug if we get into one of these steps without the lock. +function ensureLock(client, path) { + var lock = client.lock; + if(!(lock && !('unlocked' in lock))) { + // Create an error so we get a stack, too. + var err = new Error('Attempted sync step without lock.'); + log.error({client: client, err: err}, 'Client should own lock but does not for ' + path); + return false; + } + return true; +} + +function releaseLock(client) { + client.lock.removeAllListeners(); + client.lock = null; + + // Figure out how long this sync was active + var startTime = client.upstreamSync.started; + return Date.now() - startTime; +} + +// Returns true if file sizes are all within limit, false if not. +// The client's lock is released, and an error sent to client in +// the false case. +function checkFileSizeLimit(client, srcList) { + function maxSizeExceeded(obj) { + var errorMsg; + + client.lock.release(function(err) { + if(err) { + log.error({err: err, client: client}, 'Error releasing sync lock'); + } + + releaseLock(client); + + errorMsg = SyncMessage.error.maxsizeExceeded; + errorMsg.content = {path: obj.path}; + client.sendMessage(errorMsg); + }); + } + + for (var key in srcList) { + if(srcList.hasOwnProperty(key)) { + var obj = srcList[key]; + for (var prop in obj) { + if(obj.hasOwnProperty(prop) && prop === 'size') { + if(obj.size > MAX_SYNC_SIZE_BYTES) { + // Fail this sync, contains a file that is too large. + log.warn({client: client}, + 'Client tried to exceed file sync size limit: file was %s bytes, limit is %s', + obj.size, MAX_SYNC_SIZE_BYTES); + maxSizeExceeded(obj); + return false; + } + } + } + } + } + + return true; +} + +function generateSourceListResponse(client, path, type, callback) { + rsync.sourceList(client.fs, path, rsyncOptions, function(err, sourceList) { + var response; + + if(err) { + log.error({err: err, client: client}, 'rsync.sourceList() error for ' + path); + response = SyncMessage.error.sourceList; + response.content = {path: path, type: type}; + } else { + response = SyncMessage.request.checksums; + response.content = {path: path, type: type, sourceList: sourceList}; + } + + callback(err, response); + }); +} + +function sendSourceList(client, syncInfo) { + var syncInfoCopy = {}; + var response; + + if(syncInfo.type === syncTypes.RENAME) { + // Make a copy so that we don't store the sync times in the + // list of paths that need to be eventually synced + Object.keys(syncInfo).forEach(function(key) { + syncInfoCopy[key] = syncInfo[key]; + }); + + syncInfoCopy._syncStarted = Date.now(); + client.currentDownstream.push(syncInfoCopy); + response = SyncMessage.request.rename; + response.content = {path: syncInfo.path, type: syncInfo.type, oldPath: syncInfo.oldPath}; + return client.sendMessage(response); + } + + if(syncInfo.type === syncTypes.DELETE) { + // Make a copy so that we don't store the sync times in the + // list of paths that need to be eventually synced + Object.keys(syncInfo).forEach(function(key) { + syncInfoCopy[key] = syncInfo[key]; + }); + + syncInfoCopy._syncStarted = Date.now(); + client.currentDownstream.push(syncInfoCopy); + response = SyncMessage.request.del; + response.content = {path: syncInfo.path, type: syncInfo.type}; + return client.sendMessage(response); + } + + generateSourceListResponse(client, syncInfo.path, syncInfo.type, function(err, response) { + if(!err) { + // Make a copy so that we don't store the sync times in the + // list of paths that need to be eventually synced + Object.keys(syncInfo).forEach(function(key) { + syncInfoCopy[key] = syncInfo[key]; + }); + + syncInfoCopy._syncStarted = Date.now(); + client.currentDownstream.push(syncInfoCopy); + } + + client.sendMessage(response); + }); +} // Close and finalize the sync session SyncProtocolHandler.prototype.close = function(callback) { @@ -116,167 +258,127 @@ SyncProtocolHandler.prototype.close = function(callback) { } }; -// If this client was in the process of a downstream sync, we -// want to reactivate it with a path that is the common ancestor -// of the path originally being synced, and the path that was just -// updated in the upstream sync. -SyncProtocolHandler.prototype.restartDownstream = function(path) { - var client = ensureClient(this.client); - - if(!client.downstreamInterrupted) { - log.warn({client: client}, 'Unexpected call to restartDownstream()'); - return; - } - - delete client.downstreamInterrupted; - client.state = States.OUT_OF_DATE; - client.path = getCommonPath(path, client.path); - - rsync.sourceList(client.fs, client.path, rsyncOptions, function(err, srcList) { - var response; - - if(err) { - log.error({err: err, client: client}, 'rsync.sourceList error'); - response = SyncMessage.error.srclist; - } else { - response = SyncMessage.request.chksum; - response.content = {srcList: srcList, path: client.path}; - } - - client.sendMessage(response); - }); -}; - -// When this client goes out of sync (i.e., another client for the same -// user has finished an upstream sync). -SyncProtocolHandler.prototype.sendOutOfDate = function(syncMessage) { +SyncProtocolHandler.prototype.handleMessage = function(message) { var client = ensureClient(this.client); - client.state = States.OUT_OF_DATE; - client.path = syncMessage.content.path; - client.sendMessage(syncMessage); -}; - -SyncProtocolHandler.error = { - get type() { - var message = SyncMessage.error.impl; - message.content = {error: 'The Sync message cannot be handled by the server'}; - return message; - }, - get request() { - var message = SyncMessage.error.impl; - message.content = {error: 'Request cannot be processed'}; - return message; - }, - get response() { - var message = SyncMessage.error.impl; - message.content = {error: 'The resource sent as a response cannot be processed'}; - return message; + if(message.is.request) { + log.debug({syncMessage: message, client: client}, 'Received sync protocol Request message'); + this.handleRequest(message); + } else if(message.is.response) { + log.debug({syncMessage: message, client: client}, 'Received sync protocol Response message'); + this.handleResponse(message); + } else { + log.warn({client: client, syncMessage: message}, 'Invalid sync message sent by client, relaying it back'); + client.sendMessage(message); } }; SyncProtocolHandler.prototype.handleRequest = function(message) { var client = ensureClient(this.client); - if(message.is.reset && !client.is.downstreaming) { - this.handleUpstreamReset(message); - } else if(message.is.diffs && client.is.downstreaming) { + if(message.is.diffs) { this.handleDiffRequest(message); - } else if(message.is.sync && !client.is.downstreaming) { + } else if(message.is.sync) { this.handleSyncInitRequest(message); - } else if(message.is.chksum && client.is.chksum) { + } else if(message.is.checksums) { this.handleChecksumRequest(message); + } else if(message.is.delay) { + this.handleDelayRequest(message); + } else if(message.is.rename) { + this.handleRenameRequest(message); + } else if(message.is.del) { + this.handleDeleteRequest(message); } else { - log.warn({syncMessage: message, client: client}, 'Unable to handle request at this time.'); - client.sendMessage(SyncProtocolHandler.error.request); + log.warn({syncMessage: message, client: client}, 'Client sent unknown request'); } }; SyncProtocolHandler.prototype.handleResponse = function(message) { var client = ensureClient(this.client); - if (message.is.reset || message.is.authz) { - this.handleDownstreamReset(message); - } else if(message.is.diffs && client.is.patch) { + if(message.is.authz) { + this.handleFullDownstream(message); + } else if(message.is.diffs) { this.handleDiffResponse(message); - } else if(message.is.patch && client.is.downstreaming) { + } else if(message.is.patch) { this.handlePatchResponse(message); + } else if(message.is.root) { + this.handleRootResponse(message); } else { - log.warn({syncMessage: message, client: client}, 'Unable to handle response at this time.'); - client.sendMessage(SyncProtocolHandler.error.response); + log.warn({syncMessage: message, client: client}, 'Client sent unknown response'); } }; - /** * Upstream Sync Steps */ -// Most upstream sync steps require a lock to be held. -// It's a bug if we get into one of these steps without the lock. -function ensureLock(client) { - var lock = client.lock; - if(!(lock && !('unlocked' in lock))) { - // Create an error so we get a stack, too. - var err = new Error('Attempted sync step without lock.'); - log.error({client: client, err: err}, 'Client should own lock but does not.'); - return false; - } - return true; -} - -function releaseLock(client) { - client.lock.removeAllListeners(); - client.lock = null; - client.state = States.LISTENING; - - // Figure out how long this sync was active - var startTime = client._syncStarted; - delete client._syncStarted; - return Date.now() - startTime; -} - SyncProtocolHandler.prototype.handleSyncInitRequest = function(message) { var client = ensureClient(this.client); + var response; + var path; + var type; + var comparePath; + if(!client) { return; } - if(!message.content || !message.content.path) { - log.warn({client: client, syncMessage: message}, 'Missing content.path expected by handleSyncInitRequest()'); - return client.sendMessage(SyncMessage.error.content, true); + if(message.invalidContent(['type'])) { + log.warn({client: client, syncMessage: message}, 'Missing content.path or content.type expected by handleSyncInitRequest()'); + response = SyncMessage.error.content; + return client.sendMessage(response); } - SyncLock.request(client, function(err, lock) { - var response; + path = message.content.path; + type = message.content.type; + comparePath = type === syncTypes.RENAME ? message.content.oldPath : path; + // Check if the client has not downstreamed the file for which an upstream + // sync was requested. + if(findPathIndexInArray(client.outOfDate, comparePath) !== -1) { + log.debug({client: client}, 'Upstream sync declined as downstream is required for ' + comparePath); + response = SyncMessage.error.needsDownstream; + response.content = {path: comparePath, type: type}; + client.sendMessage(response); + // Force the client to downstream that path + return sendSourceList(client, {path: comparePath}); + } + + SyncLock.request(client, comparePath, function(err, lock) { if(err) { log.error({err: err, client: client}, 'SyncLock.request() error'); response = SyncMessage.error.impl; + response.content = {path: path, type: type}; } else { if(lock) { - log.debug({client: client, syncLock: lock}, 'Lock request successful, lock acquired.'); + log.debug({client: client, syncLock: lock}, 'Lock request successful, lock acquired for ' + comparePath); lock.once('unlocked', function() { - log.debug({client: client, syncLock: lock}, 'Lock unlocked'); + log.debug({client: client, syncLock: lock}, 'Lock unlocked for ' + lock.path); releaseLock(client); - client.sendMessage(SyncMessage.error.interrupted); + var interruptedResponse = SyncMessage.error.interrupted; + interruptedResponse.content = {path: lock.path}; + client.sendMessage(interruptedResponse); }); client.lock = lock; - client.state = States.CHKSUM; - client.path = message.content.path; // Track the length of time this sync takes - client._syncStarted = Date.now(); + client.upstreamSync = {path: path, type: type, started: Date.now()}; response = SyncMessage.response.sync; - response.content = {path: message.content.path}; + response.content = {path: path, type: type}; + + if(type === syncTypes.RENAME) { + client.upstreamSync.oldPath = comparePath; + response.content.oldPath = comparePath; + } } else { - log.debug({client: client}, 'Lock request unsuccessful, lock denied.'); + log.debug({client: client}, 'Lock request unsuccessful, lock denied for ' + comparePath); response = SyncMessage.error.locked; - response.content = {error: 'Sync already in progress.'}; + response.content = {error: 'Sync already in progress', path: path, type: type}; } } @@ -284,69 +386,43 @@ SyncProtocolHandler.prototype.handleSyncInitRequest = function(message) { }); }; -// Returns true if file sizes are all within limit, false if not. -// The client's lock is released, and an error sent to client in -// the false case. -function checkFileSizeLimit(client, srcList) { - function maxSizeExceeded() { - client.lock.release(function(err) { - if(err) { - log.error({err: err, client: client}, 'Error releasing sync lock'); - } - - releaseLock(client); - - client.sendMessage(SyncMessage.error.maxsizeExceeded); - }); - } - - for (var key in srcList) { - if(srcList.hasOwnProperty(key)) { - var obj = srcList[key]; - for (var prop in obj) { - if(obj.hasOwnProperty(prop) && prop === 'size') { - if(obj.size > MAX_SYNC_SIZE_BYTES) { - // Fail this sync, contains a file that is too large. - log.warn({client: client}, - 'Client tried to exceed file sync size limit: file was %s bytes, limit is %s', - obj.size, MAX_SYNC_SIZE_BYTES); - maxSizeExceeded(); - return false; - } - } - } - } - } - - return true; -} - -SyncProtocolHandler.prototype.handleChecksumRequest = function(message) { +SyncProtocolHandler.prototype.handleRenameRequest = function(message) { var client = ensureClient(this.client); + var response; + var path; + var oldPath; + var type; + var sync = this; + if(!client) { return; } - if(!ensureLock(client)) { - return; - } - if(!message.content || !message.content.srcList) { - log.warn({client: client, syncMessage: message}, 'Missing content.srcList expected by handleChecksumRequest'); - return client.sendMessage(SyncMessage.error.content); + if(message.invalidContent(['oldPath', 'type'])) { + log.warn({client: client, syncMessage: message}, 'Missing path or old path expected by handleRenameRequest()'); + response = SyncMessage.error.content; + return client.sendMessage(response); } - var srcList = message.content.srcList; + path = message.content.path; + oldPath = message.content.oldPath; + type = message.content.type; - // Enforce sync file size limits (if set in .env) - if(!checkFileSizeLimit(client, srcList)) { + if(!ensureLock(client, oldPath)) { return; } - rsync.checksums(client.fs, client.path, srcList, rsyncOptions, function(err, checksums) { - var response; + // Called when the client is closable again + function closable() { + client.closable = true; + sync.emit('closable'); + } + + client.closable = false; + rsync.utils.rename(client.fs, oldPath, path, function(err) { if(err) { - log.error({err: err, client: client}, 'rsync.checksums() error'); + log.error({err: err, client: client}, 'rsync.utils.rename() error when renaming ' + oldPath + ' to ' + path); client.lock.release(function(err) { if(err) { log.error({err: err, client: client}, 'Error releasing sync lock'); @@ -354,128 +430,153 @@ SyncProtocolHandler.prototype.handleChecksumRequest = function(message) { releaseLock(client); - response = SyncMessage.error.chksum; + response = SyncMessage.error.rename; + response.content = {path: path, oldPath: oldPath}; client.sendMessage(response); + closable(); }); } else { - response = SyncMessage.request.diffs; - response.content = {checksums: checksums}; - client.state = States.PATCH; - - client.sendMessage(response); + response = SyncMessage.response.patch; + response.content = {path: path, oldPath: oldPath, type: type}; + sync.end(response); + closable(); } }); }; -// Broadcast an out-of-date message to the all clients for a given user -// other than the active sync client after an upstream sync process has completed. -// Also, if any downstream syncs were interrupted during this upstream sync, -// they will be retriggered when the message is received. -SyncProtocolHandler.prototype.broadcastUpdate = function(response) { +SyncProtocolHandler.prototype.handleDeleteRequest = function(message) { var client = ensureClient(this.client); + var response; + var path; + var type; + var sync = this; + if(!client) { return; } - // Send a message indicating the username and client that just updated, - // as well as the default SyncMessage to broadcast. All other connected - // clients for that username will need to sync (downstream) to get the - // new updates. - var msg = { - username: client.username, - id: client.id, - syncMessage: { - type: response.type, - name: response.name, - content: response.content - } - }; + if(message.invalidContent(['type'])) { + log.warn({client: client, syncMessage: message}, 'Missing path or type expected by handleRenameRequest()'); + response = SyncMessage.error.content; + return client.sendMessage(response); + } - log.debug({client: client, syncMessage: msg.syncMessage}, 'Broadcasting out-of-date'); - redis.publish(Constants.server.syncChannel, JSON.stringify(msg)); -}; + path = message.content.path; + type = message.content.type; -// End a completed sync for a client -SyncProtocolHandler.prototype.end = function(patchResponse) { - var self = this; - var client = ensureClient(this.client); - if(!client) { + if(!ensureLock(client, path)) { return; } - if(!ensureLock(client)) { - return; + + // Called when the client is closable again + function closable() { + client.closable = true; + sync.emit('closable'); } - // Broadcast to (any) other clients for this username that there are changes - rsync.sourceList(client.fs, client.path, rsyncOptions, function(err, srcList) { - var response; + client.closable = false; + rsync.utils.del(client.fs, path, function(err) { if(err) { - log.error({err: err, client: client}, 'rsync.sourceList error'); - response = SyncMessage.error.srclist; - } else { - response = SyncMessage.request.chksum; - response.content = {srcList: srcList, path: client.path}; - } - - client.lock.release(function(err) { - if(err) { - log.error({err: err, client: client}, 'Error releasing lock'); - } - - var duration = releaseLock(client); - var info = client.info(); - if(info) { - info.upstreamSyncs++; - } - log.info({client: client}, 'Completed upstream sync to server in %s ms.', duration); + log.error({err: err, client: client}, 'rsync.utils.del() error when deleting ' + path); + client.lock.release(function(err) { + if(err) { + log.error({err: err, client: client}, 'Error releasing sync lock'); + } - client.sendMessage(patchResponse); + releaseLock(client); - // Also let all other connected clients for this uesr know that - // they are now out of date, and need to do a downstream sync. - self.broadcastUpdate(response); - }); + response = SyncMessage.error.del; + response.content = {path: path}; + client.sendMessage(response); + closable(); + }); + } else { + response = SyncMessage.response.patch; + response.content = {path: path, type: type}; + sync.end(response); + closable(); + } }); }; -SyncProtocolHandler.prototype.handleUpstreamReset = function() { +SyncProtocolHandler.prototype.handleChecksumRequest = function(message) { var client = ensureClient(this.client); + var response; + var path; + var type; + var sourceList; + if(!client) { return; } - if(!ensureLock(client)) { + + if(message.invalidContent(['type', 'sourceList'])) { + log.warn({client: client, syncMessage: message}, 'Missing path, type or sourceList expected by handleChecksumRequest()'); + response = SyncMessage.error.content; + return client.sendMessage(response); + } + + path = message.content.path; + type = message.content.type; + sourceList = message.content.sourceList; + + if(!ensureLock(client, path)) { return; } - client.lock.release(function(err) { + // Enforce sync file size limits (if set in .env) + if(!checkFileSizeLimit(client, sourceList)) { + return; + } + + rsync.checksums(client.fs, path, sourceList, rsyncOptions, function(err, checksums) { if(err) { - log.error({err: err, client: client}, 'Error releasing lock'); - } + log.error({err: err, client: client}, 'rsync.checksums() error'); + client.lock.release(function(err) { + if(err) { + log.error({err: err, client: client}, 'Error releasing sync lock'); + } - releaseLock(client); + releaseLock(client); - client.sendMessage(SyncMessage.response.reset); + response = SyncMessage.error.checksums; + response.content = {path: path, type: type}; + client.sendMessage(response); + }); + } else { + response = SyncMessage.request.diffs; + response.content = {path: path, type: type, checksums: checksums}; + client.sendMessage(response); + } }); }; SyncProtocolHandler.prototype.handleDiffResponse = function(message) { - var sync = this; var client = ensureClient(this.client); + var sync = this; + var response; + var path; + var type; + var diffs; + if(!client) { return; } - if(!ensureLock(client)) { - return; - } - if(!message.content || !message.content.diffs) { - log.warn({client: client, syncMessage: message}, 'Missing content.diffs expected by handleDiffResponse'); - return client.sendMessage(SyncMessage.error.content); + if(message.invalidContent(['type', 'diffs'])) { + log.warn({client: client, syncMessage: message}, 'Missing path, type or diffs expected by handleDiffResponse()'); + response = SyncMessage.error.content; + return client.sendMessage(response); } - var diffs = diffHelper.deserialize(message.content.diffs); - client.state = States.LISTENING; + path = message.content.path; + type = message.content.type; + diffs = diffHelper.deserialize(message.content.diffs); + + if(!ensureLock(client, path)) { + return; + } // Called when the client is closable again function closable() { @@ -489,28 +590,24 @@ SyncProtocolHandler.prototype.handleDiffResponse = function(message) { try { // Block attempts to stop this sync until the patch completes. client.closable = false; - - rsync.patch(client.fs, client.path, diffs, rsyncOptions, function(err, paths) { - var response; - + rsync.patch(client.fs, path, diffs, rsyncOptions, function(err) { if(err) { log.error({err: err, client: client}, 'rsync.patch() error'); client.lock.release(function(err) { if(err) { - log.error({err: err, client: client}, 'Error releasing sync lock'); + log.error({err: err, client: client}, 'Error releasing sync lock for ' + path); } releaseLock(client); response = SyncMessage.error.patch; - response.content = paths; + response.content = {path: path, type: type}; client.sendMessage(response); - closable(); }); } else { response = SyncMessage.response.patch; - response.content = {syncedPaths: paths.synced}; + response.content = {path: path, type: type}; sync.end(response); closable(); } @@ -518,101 +615,206 @@ SyncProtocolHandler.prototype.handleDiffResponse = function(message) { } catch(e) { // Handle rsync failing badly on a patch step // TODO: https://github.com/mozilla/makedrive/issues/31 - log.error({err: e, client: client}, 'rsync.patch() error'); + log.error({err: e, client: client}, 'rsync.patch() error on ' + path); + } +}; + +// End a completed sync for a client +SyncProtocolHandler.prototype.end = function(patchResponse) { + var self = this; + var client = ensureClient(this.client); + var path = patchResponse.content.path; + var type = patchResponse.content.type; + + if(!client) { + return; + } + if(!ensureLock(client, path)) { + return; } + + client.lock.release(function(err) { + if(err) { + log.error({err: err, client: client}, 'Error releasing lock for ' + path); + } + + var duration = releaseLock(client); + client.upstreamSync = null; + var info = client.info(); + if(info) { + info.upstreamSyncs++; + } + log.info({client: client}, 'Completed upstream sync to server for ' + path + ' in %s ms.', duration); + + client.sendMessage(patchResponse); + + // Also let all other connected clients for this user know that + // they are now out of date, and need to do a downstream sync. + self.broadcastUpdate(path, type, patchResponse.content.oldPath); + }); }; +// Broadcast an out-of-date message to the all clients for a given user +// other than the active sync client after an upstream sync process has completed. +// Also, if any downstream syncs were interrupted during this upstream sync, +// they will be retriggered when the message is received. +SyncProtocolHandler.prototype.broadcastUpdate = function(path, type, oldPath) { + var client = ensureClient(this.client); + if(!client) { + return; + } + + // Send a message indicating the username and client that just updated, + // as well as the default SyncMessage to broadcast. All other connected + // clients for that username will need to sync (downstream) to get the + // new updates. + var msg = { + username: client.username, + id: client.id, + path: path, + type: type, + oldPath: oldPath + }; + + log.debug({client: client, syncMessage: msg}, 'Broadcasting out-of-date'); + redis.publish(Constants.server.syncChannel, JSON.stringify(msg)); +}; /** * Downstream Sync Steps */ -SyncProtocolHandler.prototype.handleDiffRequest = function(message) { - var client = ensureClient(this.client); - var response; - if(!message.content || !message.content.checksums) { - log.warn({client: client, syncMessage: message}, 'Missing content.checksums in handleDiffRequest()'); - return client.sendMessage(SyncMessage.error.content); +SyncProtocolHandler.prototype.syncDownstream = function() { + var client = ensureClient(this.client); + if(!client) { + return; } - // We reject downstream sync SyncMessages unless the sync - // is part of an initial downstream sync for a connection - // or no upstream sync is in progress. - SyncLock.isUserLocked(client.username, function(err, locked) { + var syncs = client.outOfDate; + + // For each path in the filesystem, generate the source list and + // trigger a downstream sync + syncs.forEach(function(syncInfo) { + sendSourceList(client, syncInfo); + }); +}; + +SyncProtocolHandler.prototype.handleFullDownstream = function() { + var client = ensureClient(this.client); + + getListOfSyncs(client.fs, function(err, syncs) { if(err) { - log.error({err: err, client: client}, 'Error trying to look-up lock for user with redis'); - delete client._syncStarted; - response = SyncMessage.error.srclist; - client.sendMessage(response); + log.error({err: err, client: client}, 'fatal error generating list of syncs to occur in handleFullDownstream'); return; } - if(locked && !client.is.initiating) { - response = SyncMessage.error.downstreamLocked; - client.downstreamInterrupted = true; - delete client._syncStarted; - client.sendMessage(response); - return; + // Nothing in the filesystem, so nothing to sync + if(!syncs || !syncs.length) { + return client.sendMessage(SyncMessage.response.verification); } - var checksums = message.content.checksums; - - rsync.diff(client.fs, client.path, checksums, rsyncOptions, function(err, diffs) { - if(err) { - log.error({err: err, client: client}, 'rsync.diff() error'); - delete client._syncStarted; - response = SyncMessage.error.diffs; - } else { - response = SyncMessage.response.diffs; - response.content = { - diffs: diffHelper.serialize(diffs), - path: client.path - }; - } + client.outOfDate = syncs; + client.currentDownstream = []; - client.sendMessage(response); - }); + client.handler.syncDownstream(); }); }; -SyncProtocolHandler.prototype.handleDownstreamReset = function(message) { +SyncProtocolHandler.prototype.handleRootResponse = function(message) { + var client = ensureClient(this.client); + if(!client) { + return; + } + + if(message.invalidContent()) { + log.warn({client: client, syncMessage: message}, 'Missing content.path expected by handleRootResponse'); + return client.sendMessage(SyncMessage.error.content); + } + + var path = message.content.path; + var currentDownstreams = client.currentDownstream; + var remainingDownstreams = client.outOfDate; + var indexInCurrent = findPathIndexInArray(currentDownstreams, path); + var indexInOutOfDate = findPathIndexInArray(remainingDownstreams, path); + + if(indexInCurrent === -1) { + log.warn({client: client, syncMessage: message}, 'Client sent a path in handleRootResponse that is not currently downstreaming'); + return; + } + + client.currentDownstream.splice(indexInCurrent, 1); + client.outOfDate.splice(indexInOutOfDate, 1); + log.info({client: client}, 'Ignored downstream sync due to ' + path + ' being out of client\'s root'); +}; + +SyncProtocolHandler.prototype.handleDelayRequest = function(message) { + var client = ensureClient(this.client); + if(!client) { + return; + } + + if(message.invalidContent()) { + log.warn({client: client, syncMessage: message}, 'Missing content.path expected by handleDelayRequest'); + return client.sendMessage(SyncMessage.error.content); + } + + client.delaySync(message.content.path); +}; + +SyncProtocolHandler.prototype.handleDiffRequest = function(message) { var client = ensureClient(this.client); var response; + var path; + var type; + var checksums; + + if(!client) { + return; + } + + if(message.invalidContent(['type', 'checksums'])) { + log.warn({client: client, syncMessage: message}, 'Missing content.checksums in handleDiffRequest()'); + return client.sendMessage(SyncMessage.error.content); + } - // We reject downstream sync SyncMessages unless the sync - // is part of an initial downstream sync for a connection - // or no upstream sync is in progress. - SyncLock.isUserLocked(client.username, function(err, locked) { + path = message.content.path; + type = message.content.type; + + // We reject downstream sync SyncMessages unless + // no upstream sync for that path is in progress. + SyncLock.isUserLocked(client.username, path, function(err, locked) { if(err) { - log.error({err: err, client: client}, 'Error trying to look up lock for user with redis'); - response = SyncMessage.error.srclist; + log.error({err: err, client: client}, 'Error trying to look-up lock for user with redis'); + client.delaySync(path); + response = SyncMessage.error.diffs; + response.content = {path: path, type: type}; client.sendMessage(response); return; } - if(locked && !client.is.initiating) { + if(locked) { response = SyncMessage.error.downstreamLocked; - client.downstreamInterrupted = true; + response.content = {path: path, type: type}; + client.delaySync(path); client.sendMessage(response); return; } - // Track the length of time this sync takes - client._syncStarted = Date.now(); + checksums = message.content.checksums; - rsync.sourceList(client.fs, '/', rsyncOptions, function(err, srcList) { + rsync.diff(client.fs, path, checksums, rsyncOptions, function(err, diffs) { if(err) { - log.error({err: err, client: client}, 'rsync.sourceList() error'); - delete client._syncStarted; - response = SyncMessage.error.srclist; + log.error({err: err, client: client}, 'rsync.diff() error for ' + path); + client.delaySync(path); + response = SyncMessage.error.diffs; + response.content = {path: path, type: type}; } else { - response = SyncMessage.request.chksum; - response.content = {srcList: srcList, path: '/'}; - - // `handleDownstreamReset` can be called for a client's initial downstream - // filesystem update, or as a trigger for a new one. The state of the `sync` - // object must be different in each case. - client.state = message.is.authz ? States.INIT : States.OUT_OF_DATE; + response = SyncMessage.response.diffs; + response.content = { + diffs: diffHelper.serialize(diffs), + path: path, + type: type + }; } client.sendMessage(response); @@ -623,33 +825,38 @@ SyncProtocolHandler.prototype.handleDownstreamReset = function(message) { SyncProtocolHandler.prototype.handlePatchResponse = function(message) { var client = ensureClient(this.client); - if(!message.content || !message.content.checksums) { - log.warn({client: client, syncMessage: message}, 'Missing content.checksums expected by handlePatchResponse'); + if(!client) { + return; + } + + if(message.invalidContent(['type', 'checksum'])) { + log.warn({client: client, syncMessage: message}, 'Missing content.type or content.checksum in handlePatchResponse()'); return client.sendMessage(SyncMessage.error.content); } - var checksums = message.content.checksums; + var checksum = message.content.checksum; + var type = message.content.type; + var path = message.content.path; - rsync.utils.compareContents(client.fs, checksums, function(err, equal) { + rsync.utils.compareContents(client.fs, checksum, function(err, equal) { var response; // We need to check if equal is true because equal can have three possible // return value. 1. equal = true, 2. equal = false, 3. equal = undefined // we want to send error verification in case of err return or equal is false. if(equal) { - client.state = States.LISTENING; response = SyncMessage.response.verification; + client.endDownstream(path); } else { response = SyncMessage.error.verification; } - var duration = Date.now() - client._syncStarted; - delete client._syncStarted; + response.content = {path: path, type: type}; + var info = client.info(); if(info) { info.downstreamSyncs++; } - log.info({client: client}, 'Completed downstream sync to client in %s ms', duration); client.sendMessage(response); }); diff --git a/server/redis-clients.js b/server/redis-clients.js index bee45da..7d2db6b 100644 --- a/server/redis-clients.js +++ b/server/redis-clients.js @@ -142,6 +142,8 @@ module.exports.start = function(callback) { }; module.exports.close = function(callback) { + var channelCount = 0; + // While we're closing, don't worry about hang-ups, errors from server closing = true; @@ -153,23 +155,43 @@ module.exports.close = function(callback) { // XXX: due to https://github.com/mranney/node_redis/issues/439 we // can't (currently) rely on our client.quit(callback) callback to - // fire. As such, we fire and forget. - store.quit(); - store = null; - log.info('Redis connection 1/3 closed.'); - - pub.quit(); - pub = null; - log.info('Redis connection 2/3 closed.'); - - sub.unsubscribe(ChannelConstants.syncChannel, - ChannelConstants.lockRequestChannel, - ChannelConstants.lockResponseChannel); - sub.quit(); - sub = null; - log.info('Redis connection 3/3 closed.'); - - callback(); + // fire. However, for now we will use it until a better solution + // comes up. + store.quit(function(err) { + if(err) { + log.error(err, 'Could not shutdown redis store client'); + return callback(err); + } + + store = null; + log.info('Redis connection 1/3 closed.'); + + pub.quit(function(err) { + if(err) { + log.error(err, 'Could not shutdown redis publish client'); + return callback(err); + } + + pub = null; + log.info('Redis connection 2/3 closed.'); + + sub.on('unsubscribe', function unsubscribe() { + if(++channelCount !== 3) { + return; + } + + sub.removeListener('unsubscribe', unsubscribe); + sub.end(); + sub = null; + log.info('Redis connection 3/3 closed.'); + closing = false; + + callback(); + }); + + sub.unsubscribe(); + }); + }); }; // NOTE: start() must be called before the following methods will be available. @@ -182,38 +204,38 @@ module.exports.publish = function(channel, message) { pub.publish(channel, message); }; -module.exports.del = function(key, callback) { +module.exports.hdel = function(key, field, callback) { if(!store) { log.error('Called redis.del() before start()'); return callback(new Error('Not connected to Redis.')); } - store.del(key, callback); + store.hdel(key, field, callback); }; -module.exports.set = function(key, value, callback) { +module.exports.hset = function(key, field, value, callback) { if(!store) { log.error('Called redis.set() before start()'); return callback(new Error('Not connected to Redis.')); } - store.set(key, value, callback); + store.hset(key, field, value, callback); }; -module.exports.setnx = function(key, value, callback) { +module.exports.hsetnx = function(key, field, value, callback) { if(!store) { log.error('Called redis.setnx() before start()'); return callback(new Error('Not connected to Redis.')); } - store.setnx(key, value, callback); + store.hsetnx(key, field, value, callback); }; -module.exports.get = function(key, callback) { +module.exports.hget = function(key, field, callback) { if(!store) { log.error('Called redis.get() before start()'); return callback(new Error('Not connected to Redis.')); } - store.get(key, callback); + store.hget(key, field, callback); }; diff --git a/server/server-cluster.js b/server/server-cluster.js new file mode 100644 index 0000000..905d4f5 --- /dev/null +++ b/server/server-cluster.js @@ -0,0 +1,21 @@ +var server = require('./server.js'); +var cluster = require('cluster'); +var log = require('./lib/logger.js'); + +server.once('shutdown', function(err) { + if(err) { + log.error(err, 'Unable to complete clean shutdown process'); + } + + if (cluster.worker) { + cluster.worker.disconnect(); + } + + log.fatal('Killing server process'); + process.exit(1); +}); + +server.start(function() { + process.send({cmd: 'ready'}); + log.info('Started Server Worker.'); +}); diff --git a/server/server.js b/server/server.js index e69c3c8..c61220c 100644 --- a/server/server.js +++ b/server/server.js @@ -3,7 +3,6 @@ if(process.env.NEW_RELIC_ENABLED) { } var EventEmitter = require('events').EventEmitter; -var cluster = require('cluster'); var log = require('./lib/logger.js'); var WebServer = require('./web-server.js'); @@ -20,19 +19,6 @@ module.exports = new EventEmitter(); * module's ready property. */ var isReady = false; -function ready() { - if(process.send) { - process.send({cmd: 'ready'}); - } - - // Signal (to recluster master if we're a child process, - // and any event listeners like tests, and console) that - // server is running - isReady = true; - module.exports.emit('ready'); - - log.info('Started Server Worker.'); -} function shutdown(err) { // Deal with multiple things dying at once @@ -41,18 +27,10 @@ function shutdown(err) { return; } + isReady = false; shutdown.inProcess = true; - log.fatal(err, 'Starting shutdown process'); - function kill() { - if (cluster.worker) { - cluster.worker.disconnect(); - } - log.fatal('Killing server process'); - process.exit(1); - } - try { log.info('Attempting to shut down Socket Server [1/3]...'); SocketServer.close(function() { @@ -61,13 +39,14 @@ function shutdown(err) { log.info('Attempting to shut down Redis Clients [3/3]...'); RedisClients.close(function() { log.info('Finished clean shutdown.'); - kill(); + shutdown.inProcess = false; + module.exports.emit('shutdown'); }); }); }); } catch(err2) { - log.error(err2, 'Unable to complete clean shutdown process'); - kill(); + shutdown.inProcess = false; + module.exports.emit('shutdown', err2); } } @@ -89,30 +68,45 @@ RedisClients.on('error', shutdownAndLog('Redis Clients')); process.on('SIGINT', shutdownAndLog('SIGINT')); process.on('error', shutdownAndLog('process.error')); -RedisClients.start(function(err) { - if(err) { - log.fatal(err, 'Redis Clients Startup Error'); - return shutdown(err); - } - - WebServer.start(function(err, server) { +module.exports.start = function(callback) { + RedisClients.start(function(err) { if(err) { - log.fatal(err, 'Web Server Startup Error'); + log.fatal(err, 'Redis Clients Startup Error'); return shutdown(err); } - SocketServer.start(server, function(err) { + WebServer.start(function(err, server) { if(err) { - log.fatal(err, 'Socket Server Startup Error'); + log.fatal(err, 'Web Server Startup Error'); return shutdown(err); } - ready(); + SocketServer.start(server, function(err) { + if(err) { + log.fatal(err, 'Socket Server Startup Error'); + return shutdown(err); + } + + isReady = true; + module.exports.emit('ready'); + + log.info('Started Server Worker.'); + callback(); + }); }); }); -}); +}; -module.exports.app = WebServer.app; +module.exports.shutdown = function(callback) { + module.exports.once('shutdown', callback); + shutdown('Requested shutdown'); +}; + +Object.defineProperty(module.exports, 'app', { + get: function() { + return WebServer.app; + } +}); Object.defineProperty(module.exports, 'ready', { get: function() { return isReady; diff --git a/server/web-server.js b/server/web-server.js index 510c6ca..10bb5a8 100644 --- a/server/web-server.js +++ b/server/web-server.js @@ -15,6 +15,7 @@ var middleware = require('./middleware'); var routes = require('./routes'); var log = require('./lib/logger.js'); var nunjucks = require('nunjucks'); +var enableDestroy = require('server-destroy'); var app = express(); @@ -77,6 +78,7 @@ module.exports.start = function(callback) { return callback(err); } log.info('Started web server on port %s', port); + enableDestroy(server); callback(null, server); }); }; @@ -85,5 +87,8 @@ module.exports.close = function(callback) { return callback(); } - server.close(callback); + server.destroy(function() { + server = null; + callback.apply(null, arguments); + }); }; diff --git a/tests/integration/conflicted-copy.js b/tests/integration/conflicted-copy.js index d62015c..2c97fb5 100644 --- a/tests/integration/conflicted-copy.js +++ b/tests/integration/conflicted-copy.js @@ -1,5 +1,6 @@ var expect = require('chai').expect; var util = require('../lib/util.js'); +var server = require('../lib/server-utils.js'); var Filer = require('../../lib/filer.js'); var conflict = require('../../lib/conflict.js'); var fsUtils = require('../../lib/fs-utils.js'); @@ -40,25 +41,37 @@ describe('MakeDrive Client - conflicted copy integration', function(){ return Filer.Path.join('/dir1', entries[0]); } + before(function(done) { + server.start(done); + }); + after(function(done) { + server.shutdown(done); + }); + // Create 2 sync clients, do downstream syncs, then dirty both filesystems beforeEach(function(done) { - util.ready(function() { + server.run(function() { var username = util.username(); - util.setupSyncClient({username: username, layout: layout, manual: true}, function(err, client) { + server.setupSyncClient({username: username, layout: layout, manual: true}, function(err, client) { if(err) throw err; client1 = client; - util.setupSyncClient({username: username, manual: true}, function(err, client) { - if(err) throw err; - client2 = client; + server.ensureRemoteFilesystem(layout, client.jar, function(err) { + if(err) throw err; - // Make sure the initial downstream sync produced the same layout as client1 - util.ensureFilesystem(client2.fs, layout, function(err) { + server.setupSyncClient({username: username, manual: true}, function(err, client) { if(err) throw err; - modifyClients(done); + client2 = client; + + // Make sure the initial downstream sync produced the same layout as client1 + util.ensureFilesystem(client2.fs, layout, function(err) { + if(err) throw err; + + modifyClients(done); + }); }); }); }); @@ -67,18 +80,19 @@ describe('MakeDrive Client - conflicted copy integration', function(){ // Cleanly shut down both clients afterEach(function(done) { - client1.sync.once('disconnected', function() { + util.disconnectClient(client1.sync, function(err) { + if(err) throw err; + client1 = null; - client2.sync.once('disconnected', function() { + util.disconnectClient(client2.sync, function(err) { + if(err) throw err; + client2 = null; + done(); }); - - client2.sync.disconnect(); }); - - client1.sync.disconnect(); }); /** @@ -87,7 +101,7 @@ describe('MakeDrive Client - conflicted copy integration', function(){ * this conflicted copy is created, and that it is not synced back to the server. */ it('should handle conflicted copy in downstream and upstream syncs', function(done) { - client2.sync.once('completed', function() { + client2.sync.once('synced', function() { // Make sure we have a confliced copy now + the new file. client2.fs.readdir('/dir1', function(err, entries) { if(err) throw err; @@ -114,7 +128,7 @@ describe('MakeDrive Client - conflicted copy integration', function(){ client2.fs.writeFile('/dir1/file2', 'contents of file2', function(err) { if(err) throw err; - client2.sync.once('completed', function() { + client2.sync.once('synced', function() { // Our server's filesystem should now look like this: var newLayout = { // NOTE: /dir1/file1 should have client1's changes, not client2's, @@ -124,7 +138,7 @@ describe('MakeDrive Client - conflicted copy integration', function(){ '/dir1/file2': 'contents of file2' }; - util.ensureRemoteFilesystem(newLayout, client2.jar, function(err) { + server.ensureRemoteFilesystem(newLayout, client2.jar, function(err) { expect(err).not.to.exist; done(); }); @@ -149,7 +163,7 @@ describe('MakeDrive Client - conflicted copy integration', function(){ * also doesn't sync. */ it('should not sync conflicted copy when modified (i.e., not rename)', function(done) { - client2.sync.once('completed', function() { + client2.sync.once('synced', function() { // Make sure we have a confliced copy now + the new file. client2.fs.readdir('/dir1', function(err, entries) { if(err) throw err; @@ -180,7 +194,7 @@ describe('MakeDrive Client - conflicted copy integration', function(){ client2.fs.writeFile('/dir1/file2', 'contents of file2', function(err) { if(err) throw err; - client2.sync.once('completed', function() { + client2.sync.once('synced', function() { // Our server's filesystem should now look like this: var newLayout = { // NOTE: /dir1/file1 should have client1's changes, not client2's, @@ -190,7 +204,7 @@ describe('MakeDrive Client - conflicted copy integration', function(){ '/dir1/file2': 'contents of file2' }; - util.ensureRemoteFilesystem(newLayout, client2.jar, function(err) { + server.ensureRemoteFilesystem(newLayout, client2.jar, function(err) { expect(err).not.to.exist; done(); }); @@ -213,57 +227,53 @@ describe('MakeDrive Client - conflicted copy integration', function(){ * clear the conflict, then does a sync back to the server, checking that it synced. */ it('should handle a rename to a conflicted copy in downstream and upstream syncs', function(done) { - // Wait for client1 changes to sync to server - client1.sync.once('completed', function() { + client2.sync.once('synced', function() { + // Make sure we have a confliced copy now + the new file. + client2.fs.readdir('/dir1', function(err, entries) { + if(err) throw err; + expect(entries.length).to.equal(2); + expect(entries).to.include('file1'); - client2.sync.once('completed', function() { - // Make sure we have a confliced copy now + the new file. - client2.fs.readdir('/dir1', function(err, entries) { + // Make sure this is a real conflicted copy, both in name + // and also in terms of attributes on the file. + var conflictedCopyFilename = findConflictedFilename(entries); + expect(conflict.filenameContainsConflicted(conflictedCopyFilename)).to.be.true; + conflict.isConflictedCopy(client2.fs, conflictedCopyFilename, function(err, conflicted) { if(err) throw err; - expect(entries.length).to.equal(2); - expect(entries).to.include('file1'); - - // Make sure this is a real conflicted copy, both in name - // and also in terms of attributes on the file. - var conflictedCopyFilename = findConflictedFilename(entries); - expect(conflict.filenameContainsConflicted(conflictedCopyFilename)).to.be.true; - conflict.isConflictedCopy(client2.fs, conflictedCopyFilename, function(err, conflicted) { + expect(conflicted).to.be.true; + + // Make sure the conflicted copy has the changes we expect + client2.fs.readFile(conflictedCopyFilename, 'utf8', function(err, data) { if(err) throw err; - expect(conflicted).to.be.true; - // Make sure the conflicted copy has the changes we expect - client2.fs.readFile(conflictedCopyFilename, 'utf8', function(err, data) { - if(err) throw err; + // Should have client2's modifications + expect(data).to.equal('data+2'); - // Should have client2's modifications - expect(data).to.equal('data+2'); + // Rename the conflicted file and re-sync with server, making + // sure that the file gets sent this time. + client2.fs.rename(conflictedCopyFilename, '/dir1/resolved', function(err) { + if(err) throw err; - // Rename the conflicted file and re-sync with server, making - // sure that the file gets sent this time. - client2.fs.rename(conflictedCopyFilename, '/dir1/resolved', function(err) { + // Make sure the rename removed the conflict + conflict.isConflictedCopy(client2.fs, '/dir1/resolved', function(err, conflicted) { if(err) throw err; + expect(conflicted).to.be.false; - // Make sure the rename removed the conflict - conflict.isConflictedCopy(client2.fs, '/dir1/resolved', function(err, conflicted) { - if(err) throw err; - expect(conflicted).to.be.false; - - client2.sync.once('completed', function() { - // Our server's filesystem should now look like this: - var newLayout = { - // NOTE: /dir1/resolved should have client2's changes, not client1's. - '/dir1/file1': 'data+1', - '/dir1/resolved': 'data+2' - }; - - util.ensureRemoteFilesystem(newLayout, client2.jar, function(err) { - expect(err).not.to.exist; - done(); - }); - }); + client2.sync.once('synced', function() { + // Our server's filesystem should now look like this: + var newLayout = { + // NOTE: /dir1/resolved should have client2's changes, not client1's. + '/dir1/file1': 'data+1', + '/dir1/resolved': 'data+2' + }; - client2.sync.request(); + server.ensureRemoteFilesystem(newLayout, client2.jar, function(err) { + expect(err).not.to.exist; + done(); + }); }); + + client2.sync.request(); }); }); }); @@ -279,39 +289,35 @@ describe('MakeDrive Client - conflicted copy integration', function(){ var layout1 = {'/dir1/file1': layout['/dir1/file1'] + '+1'}; var layout2 = {'/dir1/file1': layout1['/dir1/file1']}; - client1.sync.once('completed', function() { - client2.sync.once('completed', function() { - client2.fs.readdir('/dir1', function(err, entries) { - if(err) throw err; - expect(entries.length).to.equal(2); - expect(entries).to.include('file1'); + client2.sync.once('synced', function() { + client2.fs.readdir('/dir1', function(err, entries) { + if(err) throw err; + expect(entries.length).to.equal(2); + expect(entries).to.include('file1'); - // Make sure this is a real conflicted copy, both in name - // and also in terms of attributes on the file. - var conflictedCopyFilename = findConflictedFilename(entries); - expect(conflict.filenameContainsConflicted(conflictedCopyFilename)).to.be.true; - // Add the conflicted copy to the layout - layout2[conflictedCopyFilename] = 'Changed Data'; + // Make sure this is a real conflicted copy, both in name + // and also in terms of attributes on the file. + var conflictedCopyFilename = findConflictedFilename(entries); + expect(conflict.filenameContainsConflicted(conflictedCopyFilename)).to.be.true; + // Add the conflicted copy to the layout + layout2[conflictedCopyFilename] = 'Changed Data'; - client2.fs.writeFile(conflictedCopyFilename, layout2[conflictedCopyFilename], function(err) { - if(err) throw err; + client2.fs.writeFile(conflictedCopyFilename, layout2[conflictedCopyFilename], function(err) { + if(err) throw err; - client2.sync.once('completed', function() { - util.ensureFilesystem(client2.fs, layout2, function(err) { - expect(err).not.to.exist; - }); - }); + client2.sync.once('idle', function() { + util.ensureFilesystem(client2.fs, layout2, function(err) { + expect(err).not.to.exist; - client1.sync.once('completed', function() { util.ensureFilesystem(client1.fs, layout1, function(err) { expect(err).not.to.exist; done(); }); }); - - client2.sync.request(); }); + + client2.sync.request(); }); }); }); diff --git a/tests/integration/delete.js b/tests/integration/delete.js new file mode 100644 index 0000000..c6b56d4 --- /dev/null +++ b/tests/integration/delete.js @@ -0,0 +1,112 @@ +var expect = require('chai').expect; +var util = require('../lib/util.js'); +var server = require('../lib/server-utils.js'); + +describe('MakeDrive Client - file delete integration', function(){ + var client1; + var client2; + var layout = {'/dir1/file1': 'data'}; + + before(function(done) { + server.start(done); + }); + after(function(done) { + server.shutdown(done); + }); + + // Create 2 sync clients, do downstream syncs + beforeEach(function(done) { + server.run(function() { + var username = util.username(); + + server.setupSyncClient({username: username, layout: layout, manual: true}, function(err, client) { + if(err) throw err; + + client1 = client; + server.setupSyncClient({username: username, manual: true}, function(err, client) { + if(err) throw err; + + client2 = client; + + // Make sure the initial downstream sync produced the same layout as client1 + util.ensureFilesystem(client2.fs, layout, function(err) { + if(err) throw err; + + done(); + }); + }); + }); + }); + }); + + // Cleanly shut down both clients + afterEach(function(done) { + util.disconnectClient(client1.sync, function(err) { + if(err) throw err; + + client1 = null; + + util.disconnectClient(client2.sync, function(err) { + if(err) throw err; + + client2 = null; + + done(); + }); + }); + }); + + /* + * This test creates 2 simultaneous clients for the same user, and simulates + * a situation where a file is deleted by one client. It then makes sure that + * this deleted file is synced to the other client. + */ + it('should handle file deletes in downstream and upstream syncs', function(done) { + var finalLayout = {'/dir1': null}; + + // 2: Client2 gets the delete command from the server + client2.sync.once('synced', function() { + util.ensureFilesystem(client2.fs, finalLayout, function(err) { + expect(err).not.to.exist; + + client1.sync.once('disconnected', function() { + server.getWebsocketToken(client1, function(err, result) { + if(err) throw err; + + // 4: Client1 connects back + client1.sync.connect(server.socketURL, result.token); + }); + }); + + // 3: Client1 disconnects + client1.sync.disconnect(); + }); + }); + + // 1: Sync client1's delete to the server + client1.sync.once('synced', function() { + util.ensureFilesystem(client1.fs, finalLayout, function(err) { + expect(err).not.to.exist; + + server.ensureRemoteFilesystem(finalLayout, client1.jar, function(err) { + expect(err).not.to.exist; + + // 5: Expect that the file is not downstreamed back from the server + client1.sync.once('synced', function() { + util.ensureFilesystem(client1.fs, finalLayout, function(err) { + expect(err).not.to.exist; + done(); + }); + }); + }); + }); + }); + + // Delete the file on client1 + client1.fs.unlink('/dir1/file1', function(err) { + if(err) throw err; + + client1.sync.request(); + }); + }); +}); diff --git a/tests/integration/rename.js b/tests/integration/rename.js index 0cec5d4..0a08dbc 100644 --- a/tests/integration/rename.js +++ b/tests/integration/rename.js @@ -1,21 +1,29 @@ var expect = require('chai').expect; var util = require('../lib/util.js'); +var server = require('../lib/server-utils.js'); describe('MakeDrive Client - file rename integration', function(){ var client1; var client2; var layout = {'/dir1/file1': 'data'}; + before(function(done) { + server.start(done); + }); + after(function(done) { + server.shutdown(done); + }); + // Create 2 sync clients, do downstream syncs beforeEach(function(done) { - util.ready(function() { + server.run(function() { var username = util.username(); - util.setupSyncClient({username: username, layout: layout, manual: true}, function(err, client) { + server.setupSyncClient({username: username, layout: layout, manual: true}, function(err, client) { if(err) throw err; client1 = client; - util.setupSyncClient({username: username, manual: true}, function(err, client) { + server.setupSyncClient({username: username, manual: true}, function(err, client) { if(err) throw err; client2 = client; @@ -33,27 +41,28 @@ describe('MakeDrive Client - file rename integration', function(){ // Cleanly shut down both clients afterEach(function(done) { - client1.sync.once('disconnected', function() { + util.disconnectClient(client1.sync, function(err) { + if(err) throw err; + client1 = null; - client2.sync.once('disconnected', function() { + util.disconnectClient(client2.sync, function(err) { + if(err) throw err; + client2 = null; + done(); }); - - client2.sync.disconnect(); }); - - client1.sync.disconnect(); }); /** * This test creates 2 simultaneous clients for the same user, and simulates * a situation where a file is renamed by one client. It then makes sure that - * this renamed file is sync'ed to the other client. + * this renamed file is synced to the other client. */ it('should handle file renames in downstream and upstream syncs', function(done) { - client2.sync.once('completed', function() { + client2.sync.once('synced', function() { // Make sure we have a confliced copy now + the new file. client2.fs.readdir('/dir1', function(err, entries) { if(err) throw err; @@ -73,4 +82,29 @@ describe('MakeDrive Client - file rename integration', function(){ }); }); + it('should be able to rename and end up with single file after renamed', function(done) { + var originalLayout = {'/dir/file.txt': 'This is file 1'}; + var newLayout = {'/dir/newFile.txt': 'This is file 1'}; + + server.setupSyncClient({layout: originalLayout, manual: true}, function(err, client) { + expect(err).not.to.exist; + + var fs = client.fs; + + fs.rename('/dir/file.txt', '/dir/newFile.txt', function(err) { + expect(err).not.to.exist; + + client.sync.once('completed', function after() { + server.ensureRemoteFilesystem(newLayout, client.jar, function(err) { + expect(err).not.to.exist; + + client.sync.on('disconnected', done); + client.sync.disconnect(); + }); + }); + + client.sync.request(); + }); + }); + }); }); diff --git a/tests/integration/sync-tests.js b/tests/integration/sync-tests.js index cb29a7a..b96949b 100644 --- a/tests/integration/sync-tests.js +++ b/tests/integration/sync-tests.js @@ -1,13 +1,21 @@ var expect = require('chai').expect; var util = require('../lib/util.js'); +var server = require('../lib/server-utils.js'); var MakeDrive = require('../../client/src'); var Filer = require('../../lib/filer.js'); describe('Two clients', function(){ var provider1, provider2; + before(function(done) { + server.start(done); + }); + after(function(done) { + server.shutdown(done); + }); + beforeEach(function(done) { - util.ready(function() { + server.run(function() { var username = util.username(); provider1 = new Filer.FileSystem.providers.Memory(username + '_1'); provider2 = new Filer.FileSystem.providers.Memory(username + '_2'); @@ -25,7 +33,7 @@ describe('Two clients', function(){ var finalLayout = { '/dir/file1.txt': 'This is file 1', '/dir/file2.txt': 'This is file 2' }; - util.authenticatedConnection(function(err, result1) { + server.authenticatedConnection(function(err, result1) { expect(err).not.to.exist; // Filesystem and sync object of first client @@ -34,7 +42,7 @@ describe('Two clients', function(){ // Step 1: First client has connected sync1.once('connected', function onClient1Connected() { - util.authenticatedConnection({username: result1.username}, function(err, result2) { + server.authenticatedConnection({username: result1.username}, function(err, result2) { expect(err).not.to.exist; // Filesystem and sync object of second client @@ -44,30 +52,42 @@ describe('Two clients', function(){ // Step 2: Second client has connected sync2.once('connected', function onClient2Connected() { // Step 3: First client has completed upstream sync #1 - sync1.once('completed', function onClient1Upstream1() { - util.ensureRemoteFilesystem(file1, result1.jar, function(err) { + sync1.once('synced', function onClient1Upstream1() { + server.ensureRemoteFilesystem(file1, result1.jar, function(err) { expect(err).not.to.exist; }); }); // Step 4: Second client has pulled down first client's upstream patch #1 - sync2.once('completed', function onClient2Downstream1() { + sync2.on('completed', function onClient2Downstream1(path) { + // Only continue if the sync was completed for + // /dir/file1.txt, i.e. not /dir which will occur first + if(path !== '/dir/file1.txt') { + return; + } + + sync2.removeListener('completed', onClient2Downstream1); + util.ensureFilesystem(fs2, file1, function(err) { expect(err).not.to.exist; // Step 5: First client has completed upstream sync #2 - sync1.once('completed', function onClient1Upstream2() { - util.ensureRemoteFilesystem(finalLayout, result1.jar, function(err) { + sync1.once('synced', function onClient1Upstream2() { + server.ensureRemoteFilesystem(finalLayout, result1.jar, function(err) { expect(err).not.to.exist; }); }); // Step 6: Second client has pulled down first client's upstream patch #2 - sync2.once('completed', function onClient2Downstream2() { + sync2.once('synced', function onClient2Downstream2() { util.ensureFilesystem(fs2, finalLayout, function(err) { expect(err).not.to.exist; - done(); + util.disconnectClient(sync1, function(err) { + if(err) throw err; + + util.disconnectClient(sync2, done); + }); }); }); @@ -86,11 +106,11 @@ describe('Two clients', function(){ }); }); - sync2.connect(util.socketURL, result2.token); + sync2.connect(server.socketURL, result2.token); }); }); - sync1.connect(util.socketURL, result1.token); + sync1.connect(server.socketURL, result1.token); }); }); }); diff --git a/tests/lib/client-utils.js b/tests/lib/client-utils.js new file mode 100644 index 0000000..381a2d2 --- /dev/null +++ b/tests/lib/client-utils.js @@ -0,0 +1,73 @@ +var WebServer = require('../../server/web-server.js'); +var WebSocketServer = require('ws').Server; +var env = require('../../server/lib/environment'); +var expect = require('chai').expect; +var SyncMessage = require('../../lib/syncmessage.js'); + +var SocketServer; +var serverURL = 'http://127.0.0.1:' + env.get('PORT'); +var socketURL = serverURL.replace( 'http', 'ws' ); +var TOKEN = 'TOKEN'; + +function run(callback) { + if(SocketServer) { + return callback(SocketServer); + } + + WebServer.start(function(err, server) { + if(err) { + return callback(err); + } + + SocketServer = new WebSocketServer({server: server}); + callback(SocketServer); + }); +} + +function decodeSocketMessage(message) { + expect(message).to.exist; + + try { + message = JSON.parse(message); + } catch(err) { + expect(err, 'Could not parse ' + message).not.to.exist; + } + + return message; +} + +function authenticateAndRun(sync, callback) { + SocketServer.once('connection', function(client) { + client.once('message', function() { + client.once('message', function(message) { + callback(client, message); + }); + + client.send(SyncMessage.response.authz.stringify()); + }); + }); + + sync.connect(socketURL, TOKEN); +} + +function close(callback) { + if(!SocketServer) { + return callback(); + } + + WebServer.close(function() { + SocketServer.close(); + SocketServer = null; + callback.apply(null, arguments); + }); +} + +module.exports = { + serverURL: serverURL, + socketURL: socketURL, + TOKEN: TOKEN, + run: run, + decodeSocketMessage: decodeSocketMessage, + authenticateAndRun: authenticateAndRun, + close: close +}; diff --git a/tests/lib/server-utils.js b/tests/lib/server-utils.js new file mode 100644 index 0000000..1a7e81b --- /dev/null +++ b/tests/lib/server-utils.js @@ -0,0 +1,417 @@ +var request = require('request'); +var expect = require('chai').expect; +var ws = require('ws'); +var filesystem = require('../../server/lib/filesystem.js'); +var SyncMessage = require('../../lib/syncmessage'); +var Filer = require('../../lib/filer.js'); +var Buffer = Filer.Buffer; +var async = require('../../lib/async-lite.js'); +var deepEqual = require('deep-equal'); +var MakeDrive = require('../../client/src/index.js'); +var util = require('./util.js'); + +// Ensure the client timeout restricts tests to a reasonable length +var env = require('../../server/lib/environment'); +env.set('CLIENT_TIMEOUT_MS', 1000); +// Set maximum file size limit to 2000000 bytes +env.set('MAX_SYNC_SIZE_BYTES', 2000000); + +// Enable a username:password for BASIC_AUTH_USERS to enable /api/get route +env.set('BASIC_AUTH_USERS', 'testusername:testpassword'); +env.set('AUTHENTICATION_PROVIDER', 'passport-webmaker'); + +var server = require('../../server/server.js'); +var app = server.app; + +var serverURL = 'http://127.0.0.1:' + env.get('PORT'), + socketURL = serverURL.replace( 'http', 'ws' ); + +// Mock Webmaker auth +app.post('/mocklogin/:username', function(req, res) { + var username = req.params.username; + if(!username){ + // Expected username. + res.send(500); + } else if(req.session && req.session.user && req.session.user.username === username) { + // Already logged in. + res.send(401); + } else{ + // Login worked. + req.session.user = {username: username}; + res.send(200); + } +}); + +// Mock File Upload into Filer FileSystem. URLs will look like this: +// /upload/:username/:path (where :path will contain /s) +app.post('/upload/*', function(req, res) { + var parts = req.path.split('/'); + var username = parts[2]; + var path = '/' + parts.slice(3).join('/'); + + var fileData = []; + req.on('data', function(chunk) { + fileData.push(new Buffer(chunk)); + }); + req.on('end', function() { + fileData = Buffer.concat(fileData); + + var fs = filesystem.create(username); + fs.writeFile(path, fileData, function(err) { + if(err) { + res.send(500, {error: err}); + return; + } + + res.send(200); + }); + + }); +}); + +/** + * Misc Helpers + */ + function run(callback) { + if(server.ready) { + callback(); + } else { + server.once('ready', callback); + } +} + +function upload(username, path, contents, callback) { + run(function() { + request.post({ + url: serverURL + '/upload/' + username + path, + headers: { + 'Content-Type': 'application/octet-stream' + }, + body: contents + }, function(err, res) { + expect(err).not.to.exist; + expect(res.statusCode).to.equal(200); + callback(); + }); + }); +} + +/** + * Connection Helpers + */ +function getWebsocketToken(options, callback){ + // Fail early and noisily when missing options.jar + if(!(options && options.jar)) { + throw('Expected options.jar'); + } + + run(function() { + request({ + url: serverURL + '/api/sync', + jar: options.jar, + json: true + }, function(err, response, body) { + expect(err, "[Error getting a token: " + err + "]").to.not.exist; + expect(response.statusCode, "[Error getting a token: " + response.body.message + "]").to.equal(200); + + options.token = body; + callback(null, options); + }); + }); +} + +function jar() { + return request.jar(); +} + +function authenticate(options, callback){ + // If no options passed, generate a unique username and jar + if(typeof options === 'function') { + callback = options; + options = {}; + } + options.jar = options.jar || jar(); + options.username = options.username || util.username(); + options.logoutUser = function (cb) { + // Reset the jar to kill existing auth cookie + options.jar = jar(); + cb(); + }; + + run(function() { + request.post({ + url: serverURL + '/mocklogin/' + options.username, + jar: options.jar + }, function(err, res) { + if(err) { + return callback(err); + } + + expect(res.statusCode).to.equal(200); + callback(null, options); + }); + }); +} + +function authenticateAndConnect(options, callback) { + if(typeof options === 'function') { + callback = options; + options = {}; + } + + authenticate(options, function(err, result) { + if(err) { + return callback(err); + } + + getWebsocketToken(result, function(err, result1) { + if(err){ + return callback(err); + } + + var testDone = result1.done; + result1.done = function() { + options.logoutUser(function() { + testDone && testDone(); + }); + }; + + callback(null, result1); + }); + }); +} + +function authenticatedSocket(options, callback) { + var socket; + + if(typeof options === 'function') { + callback = options; + options = {}; + } + + authenticateAndConnect(options, function(err, result) { + if(err) { + return callback(err); + } + + socket = new ws(socketURL); + socket.onopen = function() { + socket.send(JSON.stringify({token: result.token})); + }; + socket.onmessage = function(message) { + expect(message).to.exist; + expect(message.data).to.exist; + + var data = JSON.parse(message.data); + + expect(data.type).to.equal(SyncMessage.RESPONSE); + expect(data.name).to.equal(SyncMessage.AUTHZ); + callback(null, result, socket); + }; + }); +} + +/** + * Socket Helpers + */ +function decodeSocketMessage(message) { + expect(message).to.exist; + expect(message.data).to.exist; + + try { + message = JSON.parse(message.data); + } catch(err) { + expect(err, 'Could not parse ' + message.data).not.to.exist; + } + + return message; +} + +/** + * Makes sure that the layout given matches what's actually + * in the remote fs. Use ensureFilesystemContents if you + * want to ensure file/dir contents vs. paths. + */ +function ensureRemoteFilesystemLayout(layout, jar, callback) { + // Start by creating the layout, then compare a deep ls() + var layoutFS = new Filer.FileSystem({provider: new Filer.FileSystem.providers.Memory(util.username())}); + util.createFilesystemLayout(layoutFS, layout, function(err) { + if(err) { + return callback(err); + } + + var sh = new layoutFS.Shell(); + sh.ls('/', {recursive: true}, function(err, layoutFSListing) { + if(err) { + return callback(err); + } + + run(function() { + // Now grab the remote server listing using the /j/* route + request.get({ + url: serverURL + '/j/', + jar: jar, + json: true + }, function(err, res, remoteFSListing) { + expect(err).not.to.exist; + expect(res.statusCode).to.equal(200); + + // Remove modified + layoutFSListing = util.stripModified(layoutFSListing); + remoteFSListing = util.stripModified(remoteFSListing); + + expect(deepEqual(remoteFSListing, + layoutFSListing, + {ignoreArrayOrder: true, compareFn: util.comparePaths})).to.be.true; + callback(err); + }); + }); + }); + }); +} + +/** + * Ensure that the remote files and dirs match the layout's contents. + * Use ensureRemoteFilesystemLayout if you want to ensure file/dir paths vs. contents. + */ +function ensureRemoteFilesystemContents(layout, jar, callback) { + function ensureRemoteFileContents(filename, expectedContents, callback) { + request.get({ + url: serverURL + '/j' + filename, + jar: jar, + json: true + }, function(err, res, actualContents) { + expect(err).not.to.exist; + expect(res.statusCode).to.equal(200); + + if(!Buffer.isBuffer(expectedContents)) { + expectedContents = new Buffer(expectedContents); + } + + if(!Buffer.isBuffer(actualContents)) { + actualContents = new Buffer(actualContents); + } + + expect(actualContents).to.deep.equal(expectedContents); + callback(err); + }); + } + + function ensureRemoteEmptyDir(dirname, callback) { + request.get({ + url: serverURL + '/j' + dirname, + jar: jar, + json: true + }, function(err, res, listing) { + expect(err).not.to.exist; + expect(res.statusCode).to.equal(200); + + expect(Array.isArray(listing)).to.be.true; + expect(listing.length).to.equal(0); + + callback(err); + }); + } + + function processPath(path, callback) { + run(function() { + var contents = layout[path]; + if(contents) { + ensureRemoteFileContents(path, contents, callback); + } else { + ensureRemoteEmptyDir(path, callback); + } + }); + } + + async.eachSeries(Object.keys(layout), processPath, callback); +} + +/** + * Runs ensureRemoteFilesystemLayout and ensureRemoteFilesystemContents + * for given layout, making sure all paths and files/dirs match expected. + */ +function ensureRemoteFilesystem(layout, jar, callback) { + ensureRemoteFilesystemLayout(layout, jar, function(err) { + if(err) { + return callback(err); + } + ensureRemoteFilesystemContents(layout, jar, callback); + }); +} + +/** + * Setup a new client connection and do a downstream sync, leaving the + * connection open. If a layout is given, we also sync that up to the server. + * Callers should disconnect the client when done. Callers can pass Filer + * FileSystem options on the options object. + */ +function setupSyncClient(options, callback) { + authenticateAndConnect(options, function(err, result) { + if(err) { + return callback(err); + } + + // Make sure we have sane defaults on the options object for a filesystem + options.provider = options.provider || + new Filer.FileSystem.providers.Memory(result.username + Date.now()); + options.manual = options.manual !== false; + options.forceCreate = true; + + var fs = MakeDrive.fs(options); + var sync = fs.sync; + var client = { + jar: result.jar, + username: result.username, + fs: fs, + sync: sync + }; + + sync.once('connected', function onConnected() { + sync.once('synced', function onUpstreamCompleted(message) { + if(message === 'MakeDrive has been synced') { + callback(null, client); + } + }); + + if(!options.layout) { + return; + } + + util.createFilesystemLayout(fs, options.layout, function(err) { + if(err) { + return callback(err); + } + sync.request(); + }); + }); + + sync.once('error', function(err) { + // This should never happen, and if it does, we need to fail loudly. + console.error('Unexepcted sync `error` event', err.stack); + throw err; + }); + + sync.connect(socketURL, result.token); + }); +} + +module.exports = { + start: server.start, + shutdown: server.shutdown, + app: app, + serverURL: serverURL, + socketURL: socketURL, + run: run, + upload: upload, + getWebsocketToken: getWebsocketToken, + jar: jar, + authenticate: authenticate, + authenticatedConnection: authenticateAndConnect, + authenticatedSocket: authenticatedSocket, + decodeSocketMessage: decodeSocketMessage, + ensureRemoteFilesystemLayout: ensureRemoteFilesystemLayout, + ensureRemoteFilesystemContents: ensureRemoteFilesystemContents, + ensureRemoteFilesystem: ensureRemoteFilesystem, + setupSyncClient: setupSyncClient +}; diff --git a/tests/lib/unzip.js b/tests/lib/unzip.js index 0a7d143..3a7912e 100644 --- a/tests/lib/unzip.js +++ b/tests/lib/unzip.js @@ -3,7 +3,7 @@ module.exports = function(fs, zipfile, options, callback) { var Path = Filer.Path; var JSZip = require('jszip'); var async = require('async'); - var sh = fs.Shell(); + var sh = new fs.Shell(); if(typeof options === 'function') { callback = options; options = {}; diff --git a/tests/lib/util.js b/tests/lib/util.js index e75caa7..48c7b65 100644 --- a/tests/lib/util.js +++ b/tests/lib/util.js @@ -1,110 +1,16 @@ -var request = require('request'); var expect = require('chai').expect; -var ws = require('ws'); -var filesystem = require('../../server/lib/filesystem.js'); -var SyncMessage = require('../../lib/syncmessage'); -var rsync = require('../../lib/rsync'); -var rsyncUtils = rsync.utils; -var rsyncOptions = require('../../lib/constants').rsyncDefaults; var Filer = require('../../lib/filer.js'); var Buffer = Filer.Buffer; var Path = Filer.Path; var uuid = require( "node-uuid" ); -var async = require('async'); -var diffHelper = require("../../lib/diff"); +var async = require('../../lib/async-lite.js'); var deepEqual = require('deep-equal'); -var MakeDrive = require('../../client/src/index.js'); - -// Ensure the client timeout restricts tests to a reasonable length -var env = require('../../server/lib/environment'); -env.set('CLIENT_TIMEOUT_MS', 1000); -// Set maximum file size limit to 2000000 bytes -env.set('MAX_SYNC_SIZE_BYTES', 2000000); - -// Enable a username:password for BASIC_AUTH_USERS to enable /api/get route -env.set('BASIC_AUTH_USERS', 'testusername:testpassword'); -env.set('AUTHENTICATION_PROVIDER', 'passport-webmaker'); - -var server = require('../../server/server.js'); -var app = server.app; - -var serverURL = 'http://127.0.0.1:' + env.get('PORT'), - socketURL = serverURL.replace( 'http', 'ws' ); - -// Mock Webmaker auth -app.post('/mocklogin/:username', function(req, res) { - var username = req.param('username'); - if(!username){ - // Expected username. - res.send(500); - } else if( req.session && req.session.user && req.session.user.username === username) { - // Already logged in. - res.send(401); - } else{ - // Login worked. - req.session.user = {username: username}; - res.send(200); - } -}); - -// Mock File Upload into Filer FileSystem. URLs will look like this: -// /upload/:username/:path (where :path will contain /s) -app.post('/upload/*', function(req, res) { - var parts = req.path.split('/'); - var username = parts[2]; - var path = '/' + parts.slice(3).join('/'); - - var fileData = []; - req.on('data', function(chunk) { - fileData.push(new Buffer(chunk)); - }); - req.on('end', function() { - fileData = Buffer.concat(fileData); - - var fs = filesystem.create(username); - fs.writeFile(path, fileData, function(err) { - if(err) { - res.send(500, {error: err}); - return; - } - - res.send(200); - }); - - }); -}); - -/** - * Misc Helpers - */ -function ready(callback) { - if(server.ready) { - callback(); - } else { - server.once('ready', callback); - } -} +var MD5 = require('MD5'); function uniqueUsername() { return 'user' + uuid.v4(); } -function upload(username, path, contents, callback) { - ready(function() { - request.post({ - url: serverURL + '/upload/' + username + path, - headers: { - 'Content-Type': 'application/octet-stream' - }, - body: contents - }, function(err, res) { - expect(err).not.to.exist; - expect(res.statusCode).to.equal(200); - callback(); - }); - }); -} - function comparePaths(a, b) { // If objects have a .path property, use it. if(a.path && b.path) { @@ -116,545 +22,53 @@ function comparePaths(a, b) { return 0; } - -function toSyncMessage(string) { - try { - string = JSON.parse(string); - string = SyncMessage.parse(string); - } catch(e) { - expect(e, "[Error parsing a SyncMessage to JSON]").to.not.exist; - } - return string; -} - /** - * Connection Helpers + * Sync Helpers */ -function getWebsocketToken(options, callback){ - // Fail early and noisily when missing options.jar - if(!(options && options.jar)) { - throw('Expected options.jar'); - } - - ready(function() { - request({ - url: serverURL + '/api/sync', - jar: options.jar, - json: true - }, function(err, response, body) { - expect(err, "[Error getting a token: " + err + "]").to.not.exist; - expect(response.statusCode, "[Error getting a token: " + response.body.message + "]").to.equal(200); - - options.token = body; - callback(null, options); - }); +function generateSourceList(files) { + return files.map(function(file) { + return { + path: file.path, + modified: Math.floor(Math.random() * 10000000000), + size: file.content.length, + type: 'FILE' + }; }); } -function jar() { - return request.jar(); -} +function generateChecksums(files) { + var sourceList = generateSourceList(files); -function authenticate(options, callback){ - // If no options passed, generate a unique username and jar - if(typeof options === 'function') { - callback = options; - options = {}; - } - options.jar = options.jar || jar(); - options.username = options.username || uniqueUsername(); - options.logoutUser = function (cb) { - // Reset the jar to kill existing auth cookie - options.jar = jar(); - cb(); - }; - - ready(function() { - request.post({ - url: serverURL + '/mocklogin/' + options.username, - jar: options.jar - }, function(err, res) { - if(err) { - return callback(err); - } - - expect(res.statusCode).to.equal(200); - callback(null, options); - }); + return sourceList.map(function(file) { + delete file.size; + file.checksums = []; + return file; }); } -function authenticateAndConnect(options, callback) { - if(typeof options === 'function') { - callback = options; - options = {}; - } - - authenticate(options, function(err, result) { - if(err) { - return callback(err); - } +function generateDiffs(files) { + var checksums = generateChecksums(files); - getWebsocketToken(result, function(err, result1) { - if(err){ - return callback(err); - } - - var testDone = result1.done; - result1.done = function() { - options.logoutUser(function() { - testDone && testDone(); - }); - }; - - callback(null, result1); - }); + return checksums.map(function(file, index) { + delete file.checksums; + file.diffs = [{data: new Buffer(files[index].content)}]; + return file; }); } -/** - * Socket Helpers - */ -function openSocket(socketData, options) { - if (typeof options !== "object") { - if (socketData && !socketData.token) { - options = socketData; - socketData = null; - } - } - options = options || {}; - - var socket = new ws(socketURL); - - function defaultHandler(msg, failout) { - failout = failout || true; - - return function(code, data) { - var details = ""; - - if (code) { - details += ": " + code.toString(); - } - if (data) { - details += " " + data.toString(); - } - - expect(failout, "[Unexpected socket on" + msg + " event]" + details).to.be.false; - }; - } - - if (socketData) { - var customMessageHandler = options.onMessage; - options.onOpen = function() { - socket.send(JSON.stringify({ - token: socketData.token - })); - }; - options.onMessage = function(message) { - expect(message).to.equal(SyncMessage.response.authz.stringify()); - if (customMessageHandler) { - socket.once("message", customMessageHandler); - } - socket.send(SyncMessage.response.authz.stringify()); - }; - } - - var onOpen = options.onOpen || defaultHandler("open"); - var onMessage = options.onMessage || defaultHandler("message"); - var onError = options.onError || defaultHandler("error"); - var onClose = options.onClose || defaultHandler("close"); - - socket.once("message", onMessage); - socket.once("error", onError); - socket.once("open", onOpen); - socket.once("close", onClose); - - return { - socket: socket, - onClose: onClose, - onMessage: onMessage, - onError: onError, - onOpen: onOpen, - setClose: function(func){ - socket.removeListener("close", onClose); - socket.once("close", func); - - this.onClose = socket._events.close.listener; - }, - setMessage: function(func){ - socket.removeListener("message", onMessage); - socket.once("message", func); - - this.onMessage = socket._events.message.listener; - }, - setError: function(func){ - socket.removeListener("error", onError); - socket.once("error", func); - - this.onError = socket._events.error.listener; - }, - setOpen: function(func){ - socket.removeListener("open", onOpen); - socket.once("open", func); - - this.onOpen = socket._events.open.listener; - } - }; -} - -// Expects 1 parameter, with each subsequent one being an object -// containing a socket and an onClose callback to be deregistered -function cleanupSockets(done) { - var sockets = Array.prototype.slice.call(arguments, 1); - sockets.forEach(function(socketPackage) { - var socket = socketPackage.socket; - - socket.removeListener('close', socketPackage.onClose); - socket.close(); - }); - done(); -} - -function sendSyncMessage(socketPackage, syncMessage, callback) { - socketPackage.setMessage(callback); - socketPackage.socket.send(syncMessage.stringify()); -} - -/** - * Sync Helpers - */ -var downstreamSyncSteps = { - requestSync: function(socketPackage, data, fs, customAssertions, cb) { - if (!cb) { - cb = customAssertions; - customAssertions = null; - } - - socketPackage.socket.removeListener("message", socketPackage.onMessage); - socketPackage.socket.once("message", function(message) { - // Reattach original listener - socketPackage.socket.once("message", socketPackage.onMessage); - - if (!customAssertions) { - message = toSyncMessage(message); - - expect(message.type, "[Diffs error: \"" + (message.content && message.content.error) + "\"]").to.equal(SyncMessage.REQUEST); - expect(message.name).to.equal(SyncMessage.CHKSUM); - expect(message.content).to.exist; - expect(message.content.srcList).to.exist; - expect(message.content.path).to.exist; - - return cb(null, data); - } - - customAssertions(message, cb); - }); - - // send response reset - var resetDownstream = SyncMessage.response.reset; - socketPackage.socket.send(resetDownstream.stringify()); - }, - generateDiffs: function(socketPackage, data, fs, customAssertions, cb) { - if (!cb) { - cb = customAssertions; - customAssertions = null; - } - - var path = data.path; - var srcList = data.srcList; - - socketPackage.socket.removeListener("message", socketPackage.onMessage); - socketPackage.socket.once("message", function(message) { - // Reattach original listener - socketPackage.socket.once("message", socketPackage.onMessage); - - if (!customAssertions) { - message = toSyncMessage(message); - - expect(message.type, "[Diffs error: \"" + (message.content && message.content.error) + "\"]").to.equal(SyncMessage.RESPONSE); - expect(message.name).to.equal(SyncMessage.DIFFS); - expect(message.content).to.exist; - expect(message.content.diffs).to.exist; - expect(message.content.path).to.exist; - data.diffs = diffHelper.deserialize(message.content.diffs); - - return cb(null, data); - } - - customAssertions(message, cb); - }); - - rsync.checksums(fs, path, srcList, rsyncOptions, function( err, checksums ) { - if(err){ - cb(err); - } - - var diffRequest = SyncMessage.request.diffs; - diffRequest.content = { - checksums: checksums - }; - - socketPackage.socket.send(diffRequest.stringify()); - }); - }, - patchClientFilesystem: function(socketPackage, data, fs, customAssertions, cb) { - if (!cb) { - cb = customAssertions; - customAssertions = null; - } - - socketPackage.socket.removeListener("message", socketPackage.onMessage); - socketPackage.socket.once("message", function(message) { - // Reattach the original listener - socketPackage.socket.once("message", socketPackage.onMessage); - - if(!customAssertions) { - message = toSyncMessage(message); - expect(message).to.exist; - expect(message).to.deep.equal(SyncMessage.response.verification); - return cb(null, data); - } - customAssertions(message, cb); - }); - - rsync.patch(fs, data.path, data.diffs, rsyncOptions, function(err, paths) { - expect(err, "[Rsync patch error: \"" + err + "\"]").not.to.exist; - - rsyncUtils.generateChecksums(fs, paths.synced, function(err, checksums) { - expect(err, "[Rsync path checksum error: \"" + err + "\"]").not.to.exist; - expect(checksums).to.exist; - - var patchResponse = SyncMessage.response.patch; - patchResponse.content = {checksums: checksums}; - - socketPackage.socket.send(patchResponse.stringify()); - }); - }); - } -}; - -var upstreamSyncSteps = { - requestSync: function(socketPackage, data, customAssertions, cb) { - if (!cb) { - cb = customAssertions; - customAssertions = null; - } - - socketPackage.socket.removeListener("message", socketPackage.onMessage); - socketPackage.socket.once("message", function(message) { - // Reattach original listener - socketPackage.socket.once("message", socketPackage.onMessage); - - if (!customAssertions) { - message = toSyncMessage(message); - - expect(message).to.exist; - expect(message.type).to.equal(SyncMessage.RESPONSE); - expect(message.name, "[SyncMessage Type error. SyncMessage.content was: " + message.content + "]").to.equal(SyncMessage.SYNC); - - return cb(data); - } - - customAssertions(message, cb); - }); - - var requestSyncMessage = SyncMessage.request.sync; - requestSyncMessage.content = {path: data.path}; - socketPackage.socket.send(requestSyncMessage.stringify()); - }, - generateChecksums: function(socketPackage, data, customAssertions, cb) { - if (!cb) { - cb = customAssertions; - customAssertions = null; - } - - socketPackage.socket.removeListener("message", socketPackage.onMessage); - socketPackage.socket.once("message", function(message) { - // Reattach original listener - socketPackage.socket.once("message", socketPackage.onMessage); - - message = toSyncMessage(message); - if (!customAssertions) { - expect(message).to.exist; - expect(message.type).to.equal(SyncMessage.REQUEST); - expect(message.name, "[SyncMessage Type error. SyncMessage.content was: " + message.content + "]").to.equal(SyncMessage.DIFFS); - expect(message.content).to.exist; - expect(message.content.checksums).to.exist; - expect(message.content.path).to.exist; - - return cb(); - } - - customAssertions(message, cb); - }); - - var requestChksumMsg = SyncMessage.request.chksum; - requestChksumMsg.content = { - srcList: data.srcList +function generateValidationChecksums(files) { + return files.map(function(file) { + return { + path: file.path, + type: 'FILE', + checksum: MD5(new Buffer(file.content)).toString() }; - socketPackage.socket.send(requestChksumMsg.stringify()); - }, - patchServerFilesystem: function(socketPackage, data, fs, customAssertions, cb) { - if (!cb) { - cb = customAssertions; - customAssertions = null; - } - - var path = data.path; - var checksums = data.checksums; - - socketPackage.socket.removeListener("message", socketPackage.onMessage); - socketPackage.socket.once("message", function(message) { - // Reattach original listener - socketPackage.socket.once("message", socketPackage.onMessage); - - if (!customAssertions) { - message = toSyncMessage(message); - - expect(message.type, "[Diffs error: \"" + (message.content && message.content.error) + "\"]").to.equal(SyncMessage.RESPONSE); - expect(message.name).to.equal(SyncMessage.PATCH); - - return cb(); - } - - customAssertions(message, cb); - }); - - rsync.diff(fs, path, checksums, rsyncOptions, function( err, diffs ) { - if(err){ - return cb(err); - } - - var patchResponse = SyncMessage.response.patch; - patchResponse.content = { - diffs: diffHelper.serialize(diffs) - }; - - socketPackage.socket.send(patchResponse.stringify()); - }); - } -}; - -function prepareDownstreamSync(finalStep, username, token, cb){ - if (typeof cb !== "function") { - cb = token; - token = username; - username = finalStep; - finalStep = null; - } - - var testFile = { - name: "test.txt", - content: "Hello world!" - }; - - // Set up server filesystem - upload(username, '/' + testFile.name, testFile.content, function() { - // Set up client filesystem - var fs = filesystem.create(username + 'client'); - - var socketPackage = openSocket({ - onMessage: function(message) { - message = toSyncMessage(message); - - expect(message).to.exist; - expect(message.type).to.equal(SyncMessage.RESPONSE); - expect(message.name).to.equal(SyncMessage.AUTHZ); - expect(message.content).to.be.null; - - socketPackage.socket.once("message", function(message) { - message = toSyncMessage(message); - expect(message).to.exist; - expect(message.type).to.equal(SyncMessage.REQUEST); - expect(message.name).to.equal(SyncMessage.CHKSUM); - expect(message.content).to.exist; - expect(message.content.srcList).to.exist; - expect(message.content.path).to.exist; - - var downstreamData = { - srcList: message.content.srcList, - path: message.content.path - }; - - // Complete required sync steps - if (!finalStep) { - return cb(null, downstreamData, fs, socketPackage); - } - downstreamSyncSteps.generateDiffs(socketPackage, downstreamData, fs, function(err, data1) { - if(err){ - return cb(err); - } - if (finalStep === "generateDiffs") { - return cb(null, data1, fs, socketPackage); - } - downstreamSyncSteps.patchClientFilesystem(socketPackage, data1, fs, function(err, data2) { - if(err){ - return cb(err); - } - cb(null, data2, fs, socketPackage); - }); - }); - }); - - socketPackage.socket.send(SyncMessage.response.authz.stringify()); - }, - onOpen: function() { - socketPackage.socket.send(JSON.stringify({ - token: token - })); - } - }); - }); -} - -function completeDownstreamSync(username, token, cb) { - prepareDownstreamSync("patch", username, token, function(err, data, fs, socketPackage) { - if(err){ - cb(err); - } - cb(null, data, fs, socketPackage); - }); -} - -function prepareUpstreamSync(finalStep, username, token, cb){ - if (typeof cb !== "function") { - cb = token; - token = username; - username = finalStep; - finalStep = null; - } - - completeDownstreamSync(username, token, function(err, data, fs, socketPackage) { - if(err){ - return cb(err); - } - // Complete required sync steps - if (!finalStep) { - return cb(null, data, fs, socketPackage); - } - - upstreamSyncSteps.requestSync(socketPackage, data, function(data1) { - if (finalStep === "requestSync") { - return cb(data1, fs, socketPackage); - } - upstreamSyncSteps.generateChecksums(socketPackage, data1, fs, function(data2) { - if (finalStep === "generateChecksums") { - return cb(data2, fs, socketPackage); - } - upstreamSyncSteps.patchServerFilesystem(socketPackage, data2, fs, function(err, data3) { - if(err){ - return cb(err); - } - cb(null, data3, fs, socketPackage); - }); - }); - }); }); } function createFilesystemLayout(fs, layout, callback) { var paths = Object.keys(layout); - var sh = fs.Shell(); + var sh = new fs.Shell(); function createPath(path, callback) { var contents = layout[path]; @@ -693,7 +107,7 @@ function deleteFilesystemLayout(fs, paths, callback) { deleteFilesystemLayout(fs, entries, callback); }); } else { - var sh = fs.Shell(); + var sh = new fs.Shell(); var rm = function(path, callback) { sh.rm(path, {recursive: true}, callback); }; @@ -731,13 +145,13 @@ function ensureFilesystemLayout(fs, layout, callback) { return callback(err); } - var sh = fs.Shell(); + var sh = new fs.Shell(); sh.ls('/', {recursive: true}, function(err, fsListing) { if(err) { return callback(err); } - var sh2 = fs2.Shell(); + var sh2 = new fs2.Shell(); sh2.ls('/', {recursive: true}, function(err, fs2Listing) { if(err) { return callback(err); @@ -754,49 +168,6 @@ function ensureFilesystemLayout(fs, layout, callback) { }); } -/** - * Makes sure that the layout given matches what's actually - * in the remote fs. Use ensureFilesystemContents if you - * want to ensure file/dir contents vs. paths. - */ -function ensureRemoteFilesystemLayout(layout, jar, callback) { - // Start by creating the layout, then compare a deep ls() - var layoutFS = new Filer.FileSystem({provider: new Filer.FileSystem.providers.Memory(uniqueUsername())}); - createFilesystemLayout(layoutFS, layout, function(err) { - if(err) { - return callback(err); - } - - var sh = layoutFS.Shell(); - sh.ls('/', {recursive: true}, function(err, layoutFSListing) { - if(err) { - return callback(err); - } - - ready(function() { - // Now grab the remote server listing using the /j/* route - request.get({ - url: serverURL + '/j/', - jar: jar, - json: true - }, function(err, res, remoteFSListing) { - expect(err).not.to.exist; - expect(res.statusCode).to.equal(200); - - // Remove modified - layoutFSListing = stripModified(layoutFSListing); - remoteFSListing = stripModified(remoteFSListing); - - expect(deepEqual(remoteFSListing, - layoutFSListing, - {ignoreArrayOrder: true, compareFn: comparePaths})).to.be.true; - callback(err); - }); - }); - }); - }); -} - /** * Ensure that the files and dirs match the layout's contents. * Use ensureFilesystemLayout if you want to ensure file/dir paths vs. contents. @@ -846,63 +217,6 @@ function ensureFilesystemContents(fs, layout, callback) { async.eachSeries(Object.keys(layout), processPath, callback); } -/** - * Ensure that the remote files and dirs match the layout's contents. - * Use ensureRemoteFilesystemLayout if you want to ensure file/dir paths vs. contents. - */ -function ensureRemoteFilesystemContents(layout, jar, callback) { - function ensureRemoteFileContents(filename, expectedContents, callback) { - request.get({ - url: serverURL + '/j' + filename, - jar: jar, - json: true - }, function(err, res, actualContents) { - expect(err).not.to.exist; - expect(res.statusCode).to.equal(200); - - if(!Buffer.isBuffer(expectedContents)) { - expectedContents = new Buffer(expectedContents); - } - - if(!Buffer.isBuffer(actualContents)) { - actualContents = new Buffer(actualContents); - } - - expect(actualContents).to.deep.equal(expectedContents); - callback(err); - }); - } - - function ensureRemoteEmptyDir(dirname, callback) { - request.get({ - url: serverURL + '/j' + dirname, - jar: jar, - json: true - }, function(err, res, listing) { - expect(err).not.to.exist; - expect(res.statusCode).to.equal(200); - - expect(Array.isArray(listing)).to.be.true; - expect(listing.length).to.equal(0); - - callback(err); - }); - } - - function processPath(path, callback) { - ready(function() { - var contents = layout[path]; - if(contents) { - ensureRemoteFileContents(path, contents, callback); - } else { - ensureRemoteEmptyDir(path, callback); - } - }); - } - - async.eachSeries(Object.keys(layout), processPath, callback); -} - /** * Runs ensureFilesystemLayout and ensureFilesystemContents on fs * for given layout, making sure all paths and files/dirs match expected. @@ -917,109 +231,29 @@ function ensureFilesystem(fs, layout, callback) { }); } -/** - * Runs ensureRemoteFilesystemLayout and ensureRemoteFilesystemContents - * for given layout, making sure all paths and files/dirs match expected. - */ -function ensureRemoteFilesystem(layout, jar, callback) { - ensureRemoteFilesystemLayout(layout, jar, function(err) { - if(err) { - return callback(err); - } - ensureRemoteFilesystemContents(layout, jar, callback); - }); -} - -/** - * Setup a new client connection and do a downstream sync, leaving the - * connection open. If a layout is given, we also sync that up to the server. - * Callers should disconnect the client when done. Callers can pass Filer - * FileSystem options on the options object. - */ -function setupSyncClient(options, callback) { - authenticateAndConnect(options, function(err, result) { - if(err) { - return callback(err); - } - - // Make sure we have sane defaults on the options object for a filesystem - options.provider = options.provider || - new Filer.FileSystem.providers.Memory(result.username + Date.now()); - options.manual = options.manual !== false; - options.forceCreate = true; - - var fs = MakeDrive.fs(options); - var sync = fs.sync; - var client = { - jar: result.jar, - username: result.username, - fs: fs, - sync: sync - }; - - sync.once('connected', function onConnected() { - if(options.layout) { - sync.once('completed', function onUpstreamCompleted() { - callback(null, client); - }); - - createFilesystemLayout(fs, options.layout, function(err) { - if(err) { - return callback(err); - } - sync.request(); - }); - } else { - callback(null, client); - } - }); +function disconnect(sync, callback) { + sync.removeAllListeners(); - sync.once('error', function(err) { - // This should never happen, and if it does, we need to fail loudly. - console.error('Unexepcted sync `error` event', err.stack); - throw err; - }); + if(sync.state === sync.SYNC_DISCONNECTED) { + return callback(); + } - sync.connect(socketURL, result.token); - }); + sync.once('disconnected', callback); + sync.disconnect(); } module.exports = { - // Misc helpers - ready: ready, - app: app, - serverURL: serverURL, - socketURL: socketURL, username: uniqueUsername, - createJar: jar, - toSyncMessage: toSyncMessage, - - // Connection helpers - authenticate: authenticate, - authenticatedConnection: authenticateAndConnect, - getWebsocketToken: getWebsocketToken, - - // Socket helpers - openSocket: openSocket, - upload: upload, - cleanupSockets: cleanupSockets, - - // Filesystem helpers + comparePaths: comparePaths, + stripModified: stripModified, createFilesystemLayout: createFilesystemLayout, deleteFilesystemLayout: deleteFilesystemLayout, ensureFilesystemContents: ensureFilesystemContents, ensureFilesystemLayout: ensureFilesystemLayout, - ensureRemoteFilesystemContents: ensureRemoteFilesystemContents, - ensureRemoteFilesystemLayout: ensureRemoteFilesystemLayout, ensureFilesystem: ensureFilesystem, - ensureRemoteFilesystem: ensureRemoteFilesystem, - - // Sync helpers - prepareDownstreamSync: prepareDownstreamSync, - prepareUpstreamSync: prepareUpstreamSync, - downstreamSyncSteps: downstreamSyncSteps, - upstreamSyncSteps: upstreamSyncSteps, - sendSyncMessage: sendSyncMessage, - completeDownstreamSync: completeDownstreamSync, - setupSyncClient: setupSyncClient + generateSourceList: generateSourceList, + generateChecksums: generateChecksums, + generateDiffs: generateDiffs, + generateValidationChecksums: generateValidationChecksums, + disconnectClient: disconnect }; diff --git a/tests/unit/bugs.js b/tests/unit/bugs.js index e34a5e5..5594054 100644 --- a/tests/unit/bugs.js +++ b/tests/unit/bugs.js @@ -1,50 +1,23 @@ var expect = require('chai').expect; var util = require('../lib/util.js'); -var SyncMessage = require('../../lib/syncmessage'); +var server = require('../lib/server-utils.js'); var MakeDrive = require('../../client/src'); var Filer = require('../../lib/filer.js'); var fsUtils = require('../../lib/fs-utils.js'); describe("Server bugs", function() { - describe("[Issue 169]", function() { - it("The server shouldn't crash when two clients connect on the same session.", function(done){ - util.authenticatedConnection(function( err, connectionData ) { - expect(err).not.to.exist; - var socketData = { - token: connectionData.token - }; - - var socketPackage = util.openSocket(socketData, { - onMessage: function(message) { - message = util.toSyncMessage(message); - expect(message).to.exist; - expect(message.type).to.equal(SyncMessage.REQUEST); - expect(message.name).to.equal(SyncMessage.CHKSUM); - expect(message.content).to.be.an('object'); - - util.getWebsocketToken(connectionData, function(err, socketData2) { - expect(err).to.not.exist; - - var socketPackage2 = util.openSocket(socketData2, { - onMessage: function(message) { - util.cleanupSockets(function() { - connectionData.done(); - done(); - }, socketPackage, socketPackage2); - }, - }); - }); - } - }); - }); - }); + before(function(done) { + server.start(done); + }); + after(function(done) { + server.shutdown(done); }); describe('[Issue 287]', function(){ it('should fix timing issue with server holding onto active sync for user after completed', function(done) { var layout = {'/dir/file.txt': 'This is file 1'}; - util.setupSyncClient({manual: true, layout: layout}, function(err, client) { + server.setupSyncClient({manual: true, layout: layout}, function(err, client) { expect(err).not.to.exist; var fs = client.fs; @@ -66,13 +39,28 @@ describe("Server bugs", function() { }); describe('Client bugs', function() { - var provider; + var fs; + var sync; + + before(function(done) { + server.start(done); + }); + after(function(done) { + server.shutdown(done); + }); beforeEach(function() { - provider = new Filer.FileSystem.providers.Memory(util.username()); + fs = MakeDrive.fs({provider: new Filer.FileSystem.providers.Memory(util.username()), manual: true, forceCreate: true}); + sync = fs.sync; }); - afterEach(function() { - provider = null; + afterEach(function(done) { + util.disconnectClient(sync, function(err) { + if(err) throw err; + + sync = null; + fs = null; + done(); + }); }); describe('[Issue 372]', function(){ @@ -81,11 +69,9 @@ describe('Client bugs', function() { * and change the file's content then try to connect and sync again. */ it('should upstream newer changes made when disconnected and not create a conflicted copy', function(done) { - var fs = MakeDrive.fs({provider: provider, manual: true, forceCreate: true}); - var sync = fs.sync; var jar; - util.authenticatedConnection(function(err, result) { + server.authenticatedConnection(function(err, result) { if(err) throw err; var layout = {'/hello': 'hello', @@ -93,39 +79,50 @@ describe('Client bugs', function() { }; jar = result.jar; - sync.once('connected', function onConnected() { + sync.once('synced', function onConnected() { util.createFilesystemLayout(fs, layout, function(err) { if(err) throw err; + sync.once('synced', function onUpstreamCompleted() { + sync.disconnect(); + }); + sync.request(); }); }); - sync.once('completed', function onUpstreamCompleted() { - sync.disconnect(); - }); - sync.once('disconnected', function onDisconnected() { - // Re-sync with server and make sure we get our empty dir back - sync.once('connected', function onSecondDownstreamSync() { + var syncsCompleted = []; + + sync.on('completed', function reconnectedDownstream(path) { + syncsCompleted.push(path); + + if(syncsCompleted.length !== 3) { + return; + } + + sync.removeListener('completed', reconnectedDownstream); layout['/hello'] = 'hello world'; util.ensureFilesystem(fs, layout, function(err) { expect(err).not.to.exist; - }); - }); - sync.once('completed', function reconnectedUpstream() { - util.ensureRemoteFilesystem(layout, jar, function(err) { - expect(err).not.to.exist; - done(); + sync.once('synced', function reconnectedUpstream() { + server.ensureRemoteFilesystem(layout, jar, function(err) { + expect(err).not.to.exist; + + done(); + }); + }); + + sync.request(); }); }); - util.ensureRemoteFilesystem(layout, jar, function(err) { + server.ensureRemoteFilesystem(layout, jar, function(err) { if(err) throw err; - fs.writeFile('/hello', 'hello world', function (err) { + fs.writeFile('/hello', 'hello world', function(err) { if(err) throw err; fsUtils.isPathUnsynced(fs, '/hello', function(err, unsynced) { @@ -134,18 +131,19 @@ describe('Client bugs', function() { expect(unsynced).to.be.true; // Get a new token for this second connection - util.getWebsocketToken(result, function(err, result) { + server.getWebsocketToken(result, function(err, result) { if(err) throw err; jar = result.jar; - sync.connect(util.socketURL, result.token); + + sync.connect(server.socketURL, result.token); }); }); }); }); }); - sync.connect(util.socketURL, result.token); + sync.connect(server.socketURL, result.token); }); }); }); diff --git a/tests/unit/client-socket-tests.js b/tests/unit/client-socket-tests.js new file mode 100644 index 0000000..54e8e02 --- /dev/null +++ b/tests/unit/client-socket-tests.js @@ -0,0 +1,459 @@ +var expect = require('chai').expect; +var util = require('../lib/client-utils.js'); +var testUtils = require('../lib/util.js'); +var SyncMessage = require('../../lib/syncmessage.js'); +var MakeDrive = require('../../client/src'); +var Filer = require('../../lib/filer.js'); +var syncTypes = require('../../lib/constants.js').syncTypes; +var diffHelper = require('../../lib/diff.js'); +var FAKE_DATA = 'FAKE DATA'; + +function validateSocketMessage(message, expectedMessage, checkExists) { + message = util.decodeSocketMessage(message); + checkExists = checkExists || []; + + expect(message.type).to.equal(expectedMessage.type); + expect(message.name).to.equal(expectedMessage.name); + + if(!expectedMessage.content) { + expect(message.content).not.to.exist; + return; + } + + expect(message.content).to.exist; + + if(typeof message.content !== 'object') { + expect(message.content).to.deep.equal(expectedMessage.content); + return; + } + + Object.keys(expectedMessage.content).forEach(function(key) { + if(checkExists.indexOf(key) !== -1) { + expect(message.content[key]).to.exist; + } else { + expect(message.content[key]).to.deep.equal(expectedMessage.content[key]); + } + }); +} + +function incorrectEvent() { + expect(true, '[Incorrect sync event emitted]').to.be.false; +} + +describe('The Client', function() { + var SocketServer; + + after(function(done) { + util.close(done); + }); + + describe('Socket protocol', function() { + var fs; + var sync; + + beforeEach(function(done) { + util.run(function(server) { + SocketServer = server; + fs = MakeDrive.fs({forceCreate: true, manual: true, provider: new Filer.FileSystem.providers.Memory(testUtils.username())}); + sync = fs.sync; + done(); + }); + }); + + afterEach(function(done){ + testUtils.disconnectClient(sync, function(err) { + if(err) throw err; + + sync = null; + fs = null; + done(); + }); + }); + + it('should emit a sync error if authentication fails', function(done) { + SocketServer.once('connection', function(client) { + client.once('message', function(message) { + var message = SyncMessage.error.format; + message.content = {error: 'Unable to parse/handle message, invalid message format.'}; + client.send(message.stringify()); + }); + }); + + sync.once('connected', incorrectEvent); + sync.once('error', function(err) { + expect(err).to.exist; + expect(err.message).to.equal('Cannot handle message'); + done(); + }); + + sync.connect(util.socketURL, 'This is not a token'); + }); + + it('should send emit a connected event on successfully authenticating with the server', function(done) { + SocketServer.once('connection', function(client) { + client.once('message', function() { + client.send(SyncMessage.response.authz.stringify()); + }); + }); + + sync.once('connected', function(url) { + expect(url).to.equal(util.socketURL); + done(); + }); + sync.once('disconnected', incorrectEvent); + sync.once('error', incorrectEvent); + + sync.connect(util.socketURL, 'This is a valid token'); + }); + }); + + describe('Downstream syncs', function() { + var fs; + var sync; + + beforeEach(function(done) { + util.run(function(server) { + SocketServer = server; + fs = MakeDrive.fs({forceCreate: true, manual: true, provider: new Filer.FileSystem.providers.Memory(testUtils.username())}); + sync = fs.sync; + done(); + }); + }); + + afterEach(function(done){ + testUtils.disconnectClient(sync, function(err) { + if(err) throw err; + + sync = null; + fs = null; + done(); + }); + }); + + it('should send a "RESPONSE" of "AUTHORIZED" which triggers an initial downstream sync', function(done) { + util.authenticateAndRun(sync, function(client, message) { + validateSocketMessage(message, SyncMessage.response.authz); + done(); + }); + }); + + it('should send a "REQUEST" for "DIFFS" containing checksums when requested for checksums for a path under the sync root', function(done) { + var file = {path: '/file', content: 'This is a file'}; + var checksumRequest = SyncMessage.request.checksums; + checksumRequest.content = {path: file.path, type: syncTypes.CREATE, sourceList: testUtils.generateSourceList([file])}; + + util.authenticateAndRun(sync, function(client) { + client.once('message', function(message) { + var expectedMessage = SyncMessage.request.diffs; + expectedMessage.content = {path: file.path, type: syncTypes.CREATE, checksums: FAKE_DATA}; + validateSocketMessage(message, expectedMessage, ['checksums']); + done(); + }); + + client.send(checksumRequest.stringify()); + }); + }); + + it('should send a "RESPONSE" of "ROOT" when requested for checksums for a path not under the sync root', function(done) { + var file = {path: '/file', content: 'This is a file'}; + var checksumRequest = SyncMessage.request.checksums; + checksumRequest.content = {path: file.path, type: syncTypes.CREATE, sourceList: testUtils.generateSourceList([file])}; + + fs.mkdir('/dir', function(err) { + if(err) throw err; + + fs.setRoot('/dir', function(err) { + if(err) throw err; + + util.authenticateAndRun(sync, function(client) { + client.once('message', function(message) { + var expectedMessage = SyncMessage.response.root; + expectedMessage.content = {path: file.path, type: syncTypes.CREATE}; + validateSocketMessage(message, expectedMessage); + done(); + }); + + client.send(checksumRequest.stringify()); + }); + }); + }); + }); + + it('should patch the file being synced and send a "RESPONSE" of "PATCH" if the file was not changed during the sync', function(done) { + var file = {path: '/file', content: 'This is a file'}; + var diffResponse = SyncMessage.response.diffs; + diffResponse.content = {path: file.path, type: syncTypes.CREATE, diffs: diffHelper.serialize(testUtils.generateDiffs([file]))}; + var layout = {}; + layout[file.path] = file.content; + + util.authenticateAndRun(sync, function(client) { + client.once('message', function(message) { + var expectedMessage = SyncMessage.response.patch; + expectedMessage.content = {path: file.path, type: syncTypes.CREATE, checksum: testUtils.generateValidationChecksums([file])}; + validateSocketMessage(message, expectedMessage); + testUtils.ensureFilesystem(fs, layout, function(err) { + expect(err).not.to.exist; + done(); + }); + }); + + client.send(diffResponse.stringify()); + }); + }); + + it('should not patch the file being synced and send a "REQUEST" for "DIFFS" with checksums if the file was changed during the sync', function(done) { + var file = {path: '/file', content: 'This is a file'}; + var checksumRequest = SyncMessage.request.checksums; + checksumRequest.content = {path: file.path, type: syncTypes.CREATE, sourceList: testUtils.generateSourceList([file])}; + var diffResponse = SyncMessage.response.diffs; + diffResponse.content = {path: file.path, type: syncTypes.CREATE, diffs: diffHelper.serialize(testUtils.generateDiffs([file]))}; + var layout = {}; + layout[file.path] = 'This file was changed'; + + util.authenticateAndRun(sync, function(client) { + client.once('message', function() { + fs.writeFile(file.path, layout[file.path], function(err) { + if(err) throw err; + + client.once('message', function(message) { + var expectedMessage = SyncMessage.request.diffs; + expectedMessage.content = {path: file.path, type: syncTypes.CREATE, checksums: FAKE_DATA}; + validateSocketMessage(message, expectedMessage, ['checksums']); + testUtils.ensureFilesystem(fs, layout, function(err) { + expect(err).not.to.exist; + done(); + }); + }); + + client.send(diffResponse.stringify()); + }); + }); + + client.send(checksumRequest.stringify()); + }); + }); + + it('should emit a completed event on completing a downstream sync', function(done) { + var file = {path: '/file', content: 'This is a file'}; + var verificationResponse = SyncMessage.response.verification; + verificationResponse.content = {path: file.path}; + + util.authenticateAndRun(sync, function(client) { + sync.once('completed', function(path) { + expect(path).to.equal(file.path); + done(); + }); + + client.send(verificationResponse.stringify()); + }); + }); + }); + + describe('Upstream syncs', function() { + var fs; + var sync; + + beforeEach(function(done) { + util.run(function(server) { + SocketServer = server; + fs = MakeDrive.fs({forceCreate: true, manual: true, provider: new Filer.FileSystem.providers.Memory(testUtils.username())}); + sync = fs.sync; + done(); + }); + }); + + afterEach(function(done){ + testUtils.disconnectClient(sync, function(err) { + if(err) throw err; + + sync = null; + fs = null; + done(); + }); + }); + + it('should send a "REQUEST" for "SYNC" if a sync is requested and there are changes to the filesystem', function(done) { + var file = {path: '/file', content: 'This is a file'}; + + util.authenticateAndRun(sync, function(client) { + fs.writeFile(file.path, file.content, function(err) { + if(err) throw err; + + client.once('message', function(message) { + var expectedMessage = SyncMessage.request.sync; + expectedMessage.content = {path: file.path, type: syncTypes.CREATE}; + + validateSocketMessage(message, expectedMessage); + done(); + }); + + sync.request(); + }); + }); + }); + + it('should emit an interrupted and syncing event when an upstream sync is requested for a file that has not been downstreamed', function(done) { + var file = {path: '/file', content: 'This is a file'}; + var downstreamError = SyncMessage.error.needsDownstream; + downstreamError.content = {path: file.path, type: syncTypes.CREATE}; + var checksumRequest = SyncMessage.request.checksums; + checksumRequest.content = {path: file.path, type: syncTypes.CREATE, sourceList: testUtils.generateSourceList([file])}; + var errorEventEmitted = false; + var assertionsCompleted = false; + + function endTest() { + if(assertionsCompleted) { + done(); + } else { + assertionsCompleted = true; + } + } + + util.authenticateAndRun(sync, function(client) { + fs.writeFile(file.path, file.content, function(err) { + if(err) throw err; + + client.once('message', function() { + sync.once('error', function(err) { + errorEventEmitted = true; + + client.once('message', endTest); + + expect(err).to.exist; + expect(err.message).to.equal('Sync interrupted for path ' + file.path); + client.send(checksumRequest.stringify()); + }); + sync.once('syncing', function(message) { + expect(message).to.equal('Sync started for ' + file.path); + expect(errorEventEmitted).to.be.true; + endTest(); + }); + + client.send(downstreamError.stringify()); + }); + + sync.request(); + }); + }); + }); + + it('should trigger a syncing event and send a "REQUEST" for "CHECKSUMS" when the request to sync has been approved', function(done) { + var file = {path: '/file', content: 'This is a file'}; + var syncResponse = SyncMessage.response.sync; + syncResponse.content = {path: file.path, type: syncTypes.CREATE}; + + util.authenticateAndRun(sync, function(client) { + fs.writeFile(file.path, file.content, function(err) { + if(err) throw err; + + client.once('message', function(message) { + var expectedMessage = SyncMessage.request.checksums; + expectedMessage.content = {path: file.path, type: syncTypes.CREATE, sourceList: FAKE_DATA}; + validateSocketMessage(message, expectedMessage, ['sourceList']); + done(); + }); + + sync.once('syncing', function(message) { + expect(message).to.equal('Sync started for ' + file.path); + }); + + client.send(syncResponse.stringify()); + }); + }); + }); + + it('should send a "RESPONSE" of "DIFFS" when requested for diffs', function(done) { + var file = {path: '/file', content: 'This is a file'}; + var diffRequest = SyncMessage.request.diffs; + diffRequest.content = {path: file.path, type: syncTypes.CREATE, checksums: testUtils.generateChecksums([file])}; + + util.authenticateAndRun(sync, function(client) { + fs.writeFile(file.path, file.content, function(err) { + if(err) throw err; + + client.once('message', function(message) { + var expectedMessage = SyncMessage.response.diffs; + expectedMessage.content = {path: file.path, type: syncTypes.CREATE, diffs: FAKE_DATA}; + validateSocketMessage(message, expectedMessage, ['diffs']); + done(); + }); + + client.send(diffRequest.stringify()); + }); + }); + }); + + it('should emit a completed and synced event when all upstream syncs are completed', function(done) { + var file = {path: '/file', content: 'This is a file'}; + var patchResponse = SyncMessage.response.patch; + patchResponse.content = {path: file.path, type: syncTypes.CREATE}; + var completedEventEmitted = false; + + util.authenticateAndRun(sync, function(client) { + fs.writeFile(file.path, file.content, function(err) { + if(err) throw err; + + sync.once('synced', function() { + expect(completedEventEmitted).to.be.true; + done(); + }); + sync.once('error', incorrectEvent); + sync.once('completed', function(path) { + expect(path).to.equal(file.path); + completedEventEmitted = true; + }); + + client.send(patchResponse.stringify()); + }); + }); + }); + + it('should automatically trigger the next upstream sync in the queue once an upstream sync finishes', function(done) { + var file = {path: '/file', content: 'This is a file'}; + var file2 = {path: '/file2', content: 'This is another file'}; + var patchResponse = SyncMessage.response.patch; + patchResponse.content = {path: file.path, type: syncTypes.CREATE}; + + util.authenticateAndRun(sync, function(client) { + fs.writeFile(file.path, file.content, function(err) { + if(err) throw err; + + fs.writeFile(file2.path, file2.content, function(err) { + if(err) throw err; + + client.once('message', function(message) { + var expectedMessage = SyncMessage.request.sync; + expectedMessage.content = {path: file2.path, type: syncTypes.CREATE}; + validateSocketMessage(message, expectedMessage); + done(); + }); + + client.send(patchResponse.stringify()); + }); + }); + }); + }); + + it('should emit an error event when a sync is requested while another upstream sync is occurring', function(done) { + var file = {path: '/file', content: 'This is a file'}; + + util.authenticateAndRun(sync, function(client) { + fs.writeFile(file.path, file.content, function(err) { + if(err) throw err; + + sync.once('error', function(err) { + expect(err).to.exist; + expect(err.message).to.equal('Sync currently underway'); + done(); + }); + + client.once('message', function() { + sync.request(); + }); + + sync.request(); + }); + }); + }); + }); +}); diff --git a/tests/unit/client/auto-syncing.js b/tests/unit/client/auto-syncing.js index 971260e..23f4d6b 100644 --- a/tests/unit/client/auto-syncing.js +++ b/tests/unit/client/auto-syncing.js @@ -1,22 +1,35 @@ var expect = require('chai').expect; var util = require('../../lib/util.js'); +var server = require('../../lib/server-utils.js'); var MakeDrive = require('../../../client/src'); var Filer = require('../../../lib/filer.js'); describe('MakeDrive Client - Automatic syncing', function(){ - var provider; var syncingEventFired; + var fs; + var sync; - beforeEach(function(done) { - util.ready(function() { - provider = new Filer.FileSystem.providers.Memory(util.username()); - syncingEventFired = false; + before(function(done) { + server.start(done); + }); + after(function(done) { + server.shutdown(done); + }); + + beforeEach(function() { + fs = MakeDrive.fs({provider: new Filer.FileSystem.providers.Memory(util.username()), forceCreate: true}); + sync = fs.sync; + syncingEventFired = false; + }); + afterEach(function(done) { + util.disconnectClient(sync, function(err) { + if(err) throw err; + + sync = null; + fs = null; done(); }); }); - afterEach(function() { - provider = null; - }); /** * This integration test runs through a normal sync process with options @@ -24,12 +37,9 @@ describe('MakeDrive Client - Automatic syncing', function(){ * non-developer */ it('should complete a sync process with the default time interval', function(done) { - util.authenticatedConnection(function( err, result ) { + server.authenticatedConnection(function(err, result) { expect(err).not.to.exist; - var fs = MakeDrive.fs({provider: provider, forceCreate: true}); - var sync = fs.sync; - var layout = { '/file1': 'contents of file1' }; @@ -46,7 +56,7 @@ describe('MakeDrive Client - Automatic syncing', function(){ sync.once('completed', function onUpstreamCompleted() { // Make sure the file made it to the server - util.ensureRemoteFilesystem(layout, result.jar, function() { + server.ensureRemoteFilesystem(layout, result.jar, function() { sync.disconnect(); }); }); @@ -56,7 +66,7 @@ describe('MakeDrive Client - Automatic syncing', function(){ expect(err).not.to.exist; // Re-sync with server and make sure we get our file back - sync.once('connected', function onSecondDownstreamSync() { + sync.once('completed', function onSecondDownstreamSync() { sync.once('disconnected', function onSecondDisconnected() { util.ensureFilesystem(fs, layout, function(err) { @@ -71,30 +81,29 @@ describe('MakeDrive Client - Automatic syncing', function(){ }); // Get a new token for this second connection - util.getWebsocketToken(result, function(err, result) { + server.getWebsocketToken(result, function(err, result) { expect(err).not.to.exist; - sync.connect(util.socketURL, result.token); + sync.connect(server.socketURL, result.token); }); }); }); - sync.connect(util.socketURL, result.token); + sync.connect(server.socketURL, result.token); }); }); it('should complete a sync process with a custom time interval', function(done) { - util.authenticatedConnection(function( err, result ) { + server.authenticatedConnection(function(err, result) { expect(err).not.to.exist; - var fs = MakeDrive.fs({provider: provider, interval: 10000, forceCreate: true}); - var sync = fs.sync; - var layout = { '/file1': 'contents of file1' }; sync.once('connected', function onConnected() { + sync.auto(10000); + util.createFilesystemLayout(fs, layout, function(err) { expect(err).not.to.exist; }); @@ -106,7 +115,7 @@ describe('MakeDrive Client - Automatic syncing', function(){ sync.once('completed', function onUpstreamCompleted() { // Make sure the file made it to the server - util.ensureRemoteFilesystem(layout, result.jar, function() { + server.ensureRemoteFilesystem(layout, result.jar, function() { sync.disconnect(); }); }); @@ -116,7 +125,7 @@ describe('MakeDrive Client - Automatic syncing', function(){ expect(err).not.to.exist; // Re-sync with server and make sure we get our file back - sync.once('connected', function onSecondDownstreamSync() { + sync.once('completed', function onSecondDownstreamSync() { sync.once('disconnected', function onSecondDisconnected() { util.ensureFilesystem(fs, layout, function(err) { @@ -131,16 +140,15 @@ describe('MakeDrive Client - Automatic syncing', function(){ }); // Get a new token for this second connection - util.getWebsocketToken(result, function(err, result) { + server.getWebsocketToken(result, function(err, result) { expect(err).not.to.exist; - sync.connect(util.socketURL, result.token); + sync.connect(server.socketURL, result.token); }); }); }); - sync.connect(util.socketURL, result.token); + sync.connect(server.socketURL, result.token); }); }); - }); diff --git a/tests/unit/client/deep-and-wide.js b/tests/unit/client/deep-and-wide.js index 74679e1..4496021 100644 --- a/tests/unit/client/deep-and-wide.js +++ b/tests/unit/client/deep-and-wide.js @@ -1,20 +1,33 @@ var expect = require('chai').expect; var util = require('../../lib/util.js'); +var server = require('../../lib/server-utils.js'); var MakeDrive = require('../../../client/src'); var Filer = require('../../../lib/filer.js'); describe('MakeDrive Client - sync many dirs, many files', function(){ - var provider; + var fs; + var sync; - beforeEach(function(done) { - util.ready(function() { - provider = new Filer.FileSystem.providers.Memory(util.username()); + before(function(done) { + server.start(done); + }); + after(function(done) { + server.shutdown(done); + }); + + beforeEach(function() { + fs = MakeDrive.fs({provider: new Filer.FileSystem.providers.Memory(util.username()), manual: true, forceCreate: true}); + sync = fs.sync; + }); + afterEach(function(done) { + util.disconnectClient(sync, function(err) { + if(err) throw err; + + sync = null; + fs = null; done(); }); }); - afterEach(function() { - provider = null; - }); function smallFile(number) { return ' '+ @@ -34,12 +47,9 @@ describe('MakeDrive Client - sync many dirs, many files', function(){ * downstream sync brings them back. */ it('should sync many dirs, many files', function(done) { - util.authenticatedConnection(function( err, result ) { + server.authenticatedConnection(function(err, result) { expect(err).not.to.exist; - var fs = MakeDrive.fs({provider: provider, manual: true, forceCreate: true}); - var sync = fs.sync; - // Make a layout with 25 dirs, each with sub-dirs, and files var layout = {}; for(var i=0; i<25; i++) { @@ -50,18 +60,18 @@ describe('MakeDrive Client - sync many dirs, many files', function(){ } } - sync.once('connected', function onConnected() { + sync.once('synced', function onDownstreamCompleted() { util.createFilesystemLayout(fs, layout, function(err) { expect(err).not.to.exist; - sync.request(); - }); - }); + sync.once('synced', function onUpstreamCompleted() { + server.ensureRemoteFilesystem(layout, result.jar, function(err) { + expect(err).not.to.exist; + sync.disconnect(); + }); + }); - sync.once('completed', function onUpstreamCompleted() { - util.ensureRemoteFilesystem(layout, result.jar, function(err) { - expect(err).not.to.exist; - sync.disconnect(); + sync.request(); }); }); @@ -70,7 +80,7 @@ describe('MakeDrive Client - sync many dirs, many files', function(){ expect(err).not.to.exist; // Re-sync with server and make sure we get our deep dir back - sync.once('connected', function onSecondDownstreamSync() { + sync.once('synced', function onSecondDownstreamSync() { sync.once('disconnected', function onSecondDisconnected() { util.ensureFilesystem(fs, layout, function(err) { @@ -84,15 +94,15 @@ describe('MakeDrive Client - sync many dirs, many files', function(){ }); // Get a new token for this second connection - util.getWebsocketToken(result, function(err, result) { + server.getWebsocketToken(result, function(err, result) { expect(err).not.to.exist; - sync.connect(util.socketURL, result.token); + sync.connect(server.socketURL, result.token); }); }); }); - sync.connect(util.socketURL, result.token); + sync.connect(server.socketURL, result.token); }); }); diff --git a/tests/unit/client/deep-tree.js b/tests/unit/client/deep-tree.js index a0aa1ce..5dde72a 100644 --- a/tests/unit/client/deep-tree.js +++ b/tests/unit/client/deep-tree.js @@ -1,20 +1,33 @@ var expect = require('chai').expect; var util = require('../../lib/util.js'); +var server = require('../../lib/server-utils.js'); var MakeDrive = require('../../../client/src'); var Filer = require('../../../lib/filer.js'); describe('MakeDrive Client - sync deep tree structure', function(){ - var provider; + var fs; + var sync; - beforeEach(function(done) { - util.ready(function() { - provider = new Filer.FileSystem.providers.Memory(util.username()); + before(function(done) { + server.start(done); + }); + after(function(done) { + server.shutdown(done); + }); + + beforeEach(function() { + fs = MakeDrive.fs({provider: new Filer.FileSystem.providers.Memory(util.username()), manual: true, forceCreate: true}); + sync = fs.sync; + }); + afterEach(function(done) { + util.disconnectClient(sync, function(err) { + if(err) throw err; + + sync = null; + fs = null; done(); }); }); - afterEach(function() { - provider = null; - }); /** * This test creates series of deep dir trees, syncs, and checks that @@ -22,18 +35,22 @@ describe('MakeDrive Client - sync deep tree structure', function(){ * downstream sync brings them back. */ it('should sync an deep dir structure', function(done) { - util.authenticatedConnection(function( err, result ) { + server.authenticatedConnection(function( err, result ) { expect(err).not.to.exist; - var fs = MakeDrive.fs({provider: provider, manual: true, forceCreate: true}); - var sync = fs.sync; - // Make a directory 20 levels deep with one file inside. var layout = { '/1/2/3/4/5/6/7/8/9/10/11/12/13/14/15/16/17/18/19/20/file': 'This is a file' }; - sync.once('connected', function onConnected() { + sync.once('synced', function onDownstreamCompleted() { + sync.once('synced', function onUpstreamCompleted() { + server.ensureRemoteFilesystem(layout, result.jar, function(err) { + expect(err).not.to.exist; + sync.disconnect(); + }); + }); + util.createFilesystemLayout(fs, layout, function(err) { expect(err).not.to.exist; @@ -41,19 +58,12 @@ describe('MakeDrive Client - sync deep tree structure', function(){ }); }); - sync.once('completed', function onUpstreamCompleted() { - util.ensureRemoteFilesystem(layout, result.jar, function(err) { - expect(err).not.to.exist; - sync.disconnect(); - }); - }); - sync.once('disconnected', function onDisconnected() { util.deleteFilesystemLayout(fs, null, function(err) { expect(err).not.to.exist; // Re-sync with server and make sure we get our deep dir back - sync.once('connected', function onSecondDownstreamSync() { + sync.once('synced', function onSecondDownstreamSync() { sync.once('disconnected', function onSecondDisconnected() { util.ensureFilesystem(fs, layout, function(err) { @@ -67,15 +77,15 @@ describe('MakeDrive Client - sync deep tree structure', function(){ }); // Get a new token for this second connection - util.getWebsocketToken(result, function(err, result) { + server.getWebsocketToken(result, function(err, result) { expect(err).not.to.exist; - sync.connect(util.socketURL, result.token); + sync.connect(server.socketURL, result.token); }); }); }); - sync.connect(util.socketURL, result.token); + sync.connect(server.socketURL, result.token); }); }); diff --git a/tests/unit/client/empty-dir.js b/tests/unit/client/empty-dir.js index b041a73..6c5f2bb 100644 --- a/tests/unit/client/empty-dir.js +++ b/tests/unit/client/empty-dir.js @@ -1,20 +1,33 @@ var expect = require('chai').expect; var util = require('../../lib/util.js'); +var server = require('../../lib/server-utils.js'); var MakeDrive = require('../../../client/src'); var Filer = require('../../../lib/filer.js'); describe('MakeDrive Client - sync empty dir', function(){ - var provider; + var fs; + var sync; - beforeEach(function(done) { - util.ready(function() { - provider = new Filer.FileSystem.providers.Memory(util.username()); + before(function(done) { + server.start(done); + }); + after(function(done) { + server.shutdown(done); + }); + + beforeEach(function() { + fs = MakeDrive.fs({provider: new Filer.FileSystem.providers.Memory(util.username()), manual: true, forceCreate: true}); + sync = fs.sync; + }); + afterEach(function(done) { + util.disconnectClient(sync, function(err) { + if(err) throw err; + + sync = null; + fs = null; done(); }); }); - afterEach(function() { - provider = null; - }); /** * This test creates an empty dir, syncs, and checks that it exists @@ -22,15 +35,12 @@ describe('MakeDrive Client - sync empty dir', function(){ * brings it back. */ it('should sync an empty dir', function(done) { - util.authenticatedConnection(function( err, result ) { + server.authenticatedConnection(function(err, result) { expect(err).not.to.exist; - var fs = MakeDrive.fs({provider: provider, manual: true, forceCreate: true}); - var sync = fs.sync; - var layout = {'/empty': null}; - sync.once('connected', function onConnected() { + sync.once('synced', function onDownstreamCompleted() { util.createFilesystemLayout(fs, layout, function(err) { expect(err).not.to.exist; @@ -39,7 +49,7 @@ describe('MakeDrive Client - sync empty dir', function(){ }); sync.once('completed', function onUpstreamCompleted() { - util.ensureRemoteFilesystem(layout, result.jar, function() { + server.ensureRemoteFilesystem(layout, result.jar, function() { sync.disconnect(); }); }); @@ -49,7 +59,7 @@ describe('MakeDrive Client - sync empty dir', function(){ expect(err).not.to.exist; // Re-sync with server and make sure we get our empty dir back - sync.once('connected', function onSecondDownstreamSync() { + sync.once('synced', function onSecondDownstreamSync() { sync.once('disconnected', function onSecondDisconnected() { util.ensureFilesystem(fs, layout, function(err) { @@ -63,15 +73,15 @@ describe('MakeDrive Client - sync empty dir', function(){ }); // Get a new token for this second connection - util.getWebsocketToken(result, function(err, result) { + server.getWebsocketToken(result, function(err, result) { expect(err).not.to.exist; - sync.connect(util.socketURL, result.token); + sync.connect(server.socketURL, result.token); }); }); }); - sync.connect(util.socketURL, result.token); + sync.connect(server.socketURL, result.token); }); }); diff --git a/tests/unit/client/existing-file-sync.js b/tests/unit/client/existing-file-sync.js index e35e6c6..dfb9570 100644 --- a/tests/unit/client/existing-file-sync.js +++ b/tests/unit/client/existing-file-sync.js @@ -1,44 +1,68 @@ var expect = require('chai').expect; var util = require('../../lib/util.js'); +var server = require('../../lib/server-utils.js'); var MakeDrive = require('../../../client/src'); var Filer = require('../../../lib/filer.js'); describe('Syncing when a file already exists on the client', function(){ - var provider1; + var fs; + var sync; + var username; + + before(function(done) { + server.start(done); + }); + after(function(done) { + server.shutdown(done); + }); beforeEach(function() { - var username = util.username(); - provider1 = new Filer.FileSystem.providers.Memory(username); + username = util.username(); + fs = MakeDrive.fs({provider: new Filer.FileSystem.providers.Memory(username), manual: true, forceCreate: true}); + sync = fs.sync; }); - afterEach(function() { - provider1 = null; + afterEach(function(done) { + util.disconnectClient(sync, function(err) { + if(err) throw err; + + sync = null; + fs = null; + done(); + }); }); it('should be able to sync when the client already has a file and is performing an initial downstream sync', function(done) { - var fs1 = MakeDrive.fs({provider: provider1, manual: true, forceCreate: true}); var everError = false; // 1. Write some file on local filesystem. - fs1.writeFile('/abc.txt', 'this is a simple file', function(err) { + fs.writeFile('/abc.txt', 'this is a simple file', function(err) { if(err) throw err; - // 2. try to connect after successfully changing the local filesystem - util.authenticatedConnection(function(err, result1) { + server.upload(username, '/file', 'This is a file that should be downstreamed', function(err){ if(err) throw err; - var sync1 = fs1.sync; - // 4. should not have any error after trying to connect to the server. - sync1.once('error', function error(err) { - everError = err; - }); + // 2. try to connect after successfully changing the local filesystem + server.authenticatedConnection({username: username}, function(err, result) { + if(err) throw err; - sync1.once('connected', function connected() { - expect(everError).to.be.false; - done(); - }); + // 4. should not have any error after trying to connect to the server. + sync.once('error', function error(err) { + everError = err; + }); + + sync.once('completed', function completed(path) { + expect(path).to.equal('/file'); + expect(everError).to.be.false; + done(); + }); - // 3. try and conect to the server - sync1.connect(util.socketURL, result1.token); + sync.once('synced', function synced() { + expect(true, 'Makedrive should not be completely synced').to.be.false; + }); + + // 3. try and conect to the server + sync.connect(server.socketURL, result.token); + }); }); }); }); diff --git a/tests/unit/client/file-size-limit.js b/tests/unit/client/file-size-limit.js index d458ce6..9580f5f 100644 --- a/tests/unit/client/file-size-limit.js +++ b/tests/unit/client/file-size-limit.js @@ -1,74 +1,86 @@ var expect = require('chai').expect; var util = require('../../lib/util.js'); +var server = require('../../lib/server-utils.js'); var MakeDrive = require('../../../client/src'); var Filer = require('../../../lib/filer.js'); var MAX_SIZE_BYTES = 2000000; describe('Syncing file larger than size limit', function(){ - var provider; + var fs; + var sync; + var username; + + before(function(done) { + server.start(done); + }); + after(function(done) { + server.shutdown(done); + }); beforeEach(function() { - var username = util.username(); - provider = new Filer.FileSystem.providers.Memory(username); + username = util.username(); + fs = MakeDrive.fs({provider: new Filer.FileSystem.providers.Memory(username), manual: true, forceCreate: true}); + sync = fs.sync; }); - afterEach(function() { - provider = null; + afterEach(function(done) { + util.disconnectClient(sync, function(err) { + if(err) throw err; + + sync = null; + fs = null; + done(); + }); }); it('should return an error if file exceeded the size limit', function(done) { - util.authenticatedConnection(function(err, result) { + server.authenticatedConnection(function(err, result) { expect(err).not.to.exist; - var fs = MakeDrive.fs({provider: provider, manual: true, forceCreate: true}); - var sync = fs.sync; var layout = {'/hello.txt': new Filer.Buffer(MAX_SIZE_BYTES+1) }; - sync.once('connected', function onClientConnected() { - + sync.once('synced', function onClientConnected() { util.createFilesystemLayout(fs, layout, function(err) { expect(err).not.to.exist; sync.request(); }); - }); + sync.once('error', function onClientError(error) { - expect(error).to.eql(new Error('Maximum file size exceeded')); + expect(error).to.eql(new Error('Sync interrupted for path /hello.txt')); done(); }); - sync.connect(util.socketURL, result.token); + sync.connect(server.socketURL, result.token); }); }); it('should not return an error if file did not exceed the size limit', function(done) { var everError = false; - util.authenticatedConnection(function(err, result) { + server.authenticatedConnection(function(err, result) { expect(err).not.to.exist; - var fs = MakeDrive.fs({provider: provider, manual: true, forceCreate: true}); - var sync = fs.sync; var layout = {'/hello.txt': new Filer.Buffer(MAX_SIZE_BYTES) }; sync.once('connected', function onClientConnected() { - util.createFilesystemLayout(fs, layout, function(err) { expect(err).not.to.exist; sync.request(); }); - }); + sync.once('completed', function() { expect(everError).to.be.false; done(); }); + sync.once('error', function onClientError() { everError = true; }); - sync.connect(util.socketURL, result.token); + sync.connect(server.socketURL, result.token); }); }); }); diff --git a/tests/unit/client/large-files.js b/tests/unit/client/large-files.js index 1f096bf..22681ed 100644 --- a/tests/unit/client/large-files.js +++ b/tests/unit/client/large-files.js @@ -1,20 +1,33 @@ var expect = require('chai').expect; var util = require('../../lib/util.js'); +var server = require('../../lib/server-utils.js'); var MakeDrive = require('../../../client/src'); var Filer = require('../../../lib/filer.js'); describe('MakeDrive Client - sync large files', function(){ - var provider; + var fs; + var sync; - beforeEach(function(done) { - util.ready(function() { - provider = new Filer.FileSystem.providers.Memory(util.username()); + before(function(done) { + server.start(done); + }); + after(function(done) { + server.shutdown(done); + }); + + beforeEach(function() { + fs = MakeDrive.fs({provider: new Filer.FileSystem.providers.Memory(util.username()), manual: true, forceCreate: true}); + sync = fs.sync; + }); + afterEach(function(done) { + util.disconnectClient(sync, function(err) { + if(err) throw err; + + sync = null; + fs = null; done(); }); }); - afterEach(function() { - provider = null; - }); function makeBuffer(n) { // Make a Buffer of size n and fill it with 7s (who doesn't like 7?) @@ -30,12 +43,9 @@ describe('MakeDrive Client - sync large files', function(){ * downstream sync brings them back. */ it('should sync large files', function(done) { - util.authenticatedConnection(function( err, result ) { + server.authenticatedConnection(function( err, result ) { expect(err).not.to.exist; - var fs = MakeDrive.fs({provider: provider, manual: true, forceCreate: true}); - var sync = fs.sync; - // Make a layout with /project and some large files var layout = { '/project/1': makeBuffer(50), @@ -44,7 +54,14 @@ describe('MakeDrive Client - sync large files', function(){ '/project/4': makeBuffer(1024) }; - sync.once('connected', function onConnected() { + sync.once('synced', function onDownstreamCompleted() { + sync.once('synced', function onUpstreamCompleted() { + server.ensureRemoteFilesystem(layout, result.jar, function(err) { + expect(err).not.to.exist; + sync.disconnect(); + }); + }); + util.createFilesystemLayout(fs, layout, function(err) { expect(err).not.to.exist; @@ -52,19 +69,12 @@ describe('MakeDrive Client - sync large files', function(){ }); }); - sync.once('completed', function onUpstreamCompleted() { - util.ensureRemoteFilesystem(layout, result.jar, function(err) { - expect(err).not.to.exist; - sync.disconnect(); - }); - }); - sync.once('disconnected', function onDisconnected() { util.deleteFilesystemLayout(fs, null, function(err) { expect(err).not.to.exist; // Re-sync with server and make sure we get our deep dir back - sync.once('connected', function onSecondDownstreamSync() { + sync.once('synced', function onSecondDownstreamSync() { sync.once('disconnected', function onSecondDisconnected() { util.ensureFilesystem(fs, layout, function(err) { @@ -78,15 +88,15 @@ describe('MakeDrive Client - sync large files', function(){ }); // Get a new token for this second connection - util.getWebsocketToken(result, function(err, result) { + server.getWebsocketToken(result, function(err, result) { expect(err).not.to.exist; - sync.connect(util.socketURL, result.token); + sync.connect(server.socketURL, result.token); }); }); }); - sync.connect(util.socketURL, result.token); + sync.connect(server.socketURL, result.token); }); }); diff --git a/tests/unit/client/makedrive-api.js b/tests/unit/client/makedrive-api.js index 9a533f8..67d42db 100644 --- a/tests/unit/client/makedrive-api.js +++ b/tests/unit/client/makedrive-api.js @@ -1,21 +1,22 @@ var expect = require('chai').expect; var util = require('../../lib/util.js'); +var server = require('../../lib/server-utils.js'); var MakeDrive = require('../../../client/src'); var Filer = require('../../../lib/filer.js'); -var SyncMessage = require('../../../lib/syncmessage'); -var WebSocketServer = require('ws').Server; -var rsync = require("../../../lib/rsync"); -var rsyncOptions = require('../../../lib/constants').rsyncDefaults; describe('MakeDrive Client API', function(){ + before(function(done) { + server.start(done); + }); + after(function(done) { + server.shutdown(done); + }); + describe('Core API', function() { var provider; - beforeEach(function(done) { - util.ready(function() { - provider = new Filer.FileSystem.providers.Memory(util.username()); - done(); - }); + beforeEach(function() { + provider = new Filer.FileSystem.providers.Memory(util.username()); }); afterEach(function() { provider = null; @@ -23,31 +24,31 @@ describe('MakeDrive Client API', function(){ it('should have expected methods and properites', function() { // Bits copied from Filer - expect(MakeDrive.Buffer).to.be.a.function; + expect(MakeDrive.Buffer).to.be.a('function'); expect(MakeDrive.Path).to.exist; - expect(MakeDrive.Path.normalize).to.be.a.function; + expect(MakeDrive.Path.normalize).to.be.a('function'); expect(MakeDrive.Errors).to.exist; // MakeDrive.fs() - expect(MakeDrive.fs).to.be.a.function; + expect(MakeDrive.fs).to.be.a('function'); var fs = MakeDrive.fs({memory: true, manual: true}); var fs2 = MakeDrive.fs({memory: true, manual: true}); expect(fs).to.equal(fs2); // MakeDrive.fs().sync property expect(fs.sync).to.exist; - expect(fs.sync.on).to.be.a.function; - expect(fs.sync.off).to.be.a.function; - expect(fs.sync.connect).to.be.a.function; - expect(fs.sync.disconnect).to.be.a.function; - expect(fs.sync.sync).to.be.a.function; + expect(fs.sync.on).to.be.a('function'); + expect(fs.sync.connect).to.be.a('function'); + expect(fs.sync.disconnect).to.be.a('function'); + expect(fs.sync.request).to.be.a('function'); + expect(fs.sync.manual).to.be.a('function'); + expect(fs.sync.auto).to.be.a('function'); // Sync States expect(fs.sync.SYNC_DISCONNECTED).to.equal("SYNC DISCONNECTED"); expect(fs.sync.SYNC_CONNECTING).to.equal("SYNC CONNECTING"); expect(fs.sync.SYNC_CONNECTED).to.equal("SYNC CONNECTED"); expect(fs.sync.SYNC_SYNCING).to.equal("SYNC SYNCING"); - expect(fs.sync.SYNC_ERROR).to.equal("SYNC ERROR"); expect(fs.sync.state).to.equal(fs.sync.SYNC_DISCONNECTED); }); @@ -89,7 +90,7 @@ describe('MakeDrive Client API', function(){ * checks that the file was uploaded. */ it('should go through proper steps with connect(), request(), disconnect()', function(done) { - util.authenticatedConnection(function( err, result ) { + server.authenticatedConnection(function(err, result) { expect(err).not.to.exist; var token = result.token; @@ -104,7 +105,9 @@ describe('MakeDrive Client API', function(){ var everSeenError = false; sync.once('connected', function onConnected() { - expect(sync.state).to.equal(sync.SYNC_CONNECTED); + expect(sync.state).to.satisfy(function(state) { + return state === sync.SYNC_CONNECTED || state === sync.SYNC_SYNCING; + }); // Write a file and try to sync util.createFilesystemLayout(fs, layout, function(err) { @@ -114,14 +117,18 @@ describe('MakeDrive Client API', function(){ }); sync.once('syncing', function onUpstreamSyncing() { - everSeenSyncing = sync.state; + expect(sync.state).to.satisfy(function(state) { + return state === sync.SYNC_CONNECTED || state === sync.SYNC_SYNCING; + }); + + everSeenSyncing = true; }); - sync.once('completed', function onUpstreamCompleted() { + sync.once('synced', function onUpstreamCompleted() { everSeenCompleted = sync.state; // Confirm file was really uploaded and remote fs matches what we expect - util.ensureRemoteFilesystem(layout, result.jar, function() { + server.ensureRemoteFilesystem(layout, result.jar, function() { sync.disconnect(); }); }); @@ -135,7 +142,7 @@ describe('MakeDrive Client API', function(){ expect(everSeenError).to.be.false; expect(sync.state).to.equal(sync.SYNC_DISCONNECTED); - expect(everSeenSyncing).to.equal(sync.SYNC_SYNCING); + expect(everSeenSyncing).to.be.true; expect(everSeenCompleted).to.equal(sync.SYNC_CONNECTED); // Make sure client fs is in the same state we left it @@ -143,7 +150,7 @@ describe('MakeDrive Client API', function(){ }); expect(sync.state).to.equal(sync.SYNC_DISCONNECTED); - sync.connect(util.socketURL, token); + sync.connect(server.socketURL, token); expect(sync.state).to.equal(sync.SYNC_CONNECTING); }); }); @@ -156,327 +163,35 @@ describe('MakeDrive Client API', function(){ * sent to the client. */ describe('Protocol & Error tests', function() { - var provider; - var socket; - var port = 1212; - var testServer; - - beforeEach(function(done) { - util.ready(function() { - provider = new Filer.FileSystem.providers.Memory(util.username()); - - testServer = new WebSocketServer({port: port}); - testServer.once('error', function(err){ - expect(err, "[Error creating socket server]").to.not.exist; - }); - testServer.once('listening', function() { - done(); - }); - }); - }); - afterEach(function() { - provider = null; + var fs; + var sync; - if (socket) { - socket.close(); - } - testServer.close(); - testServer = null; + beforeEach(function() { + fs = MakeDrive.fs({provider: new Filer.FileSystem.providers.Memory(util.username()), manual: true, forceCreate: true}); + sync = fs.sync; }); + afterEach(function(done) { + util.disconnectClient(sync, function(err) { + if(err) throw err; - function endTestSession(sync, done) { - sync.once('disconnected', function() { sync = null; + fs = null; done(); }); - - sync.disconnect(); - } - - function parseMessage(msg) { - msg = msg.data || msg; - - msg = JSON.parse(msg); - msg = SyncMessage.parse(msg); - - return msg; - } - - it('should restart a downstream sync on receiving a CHKSUM ERROR SyncMessage instead of a sourceList.', function(done){ - function clientLogic() { - var fs = MakeDrive.fs({provider: provider, manual: true, forceCreate: true, autoReconnect: false}); - var sync = fs.sync; - sync.on('error', function(err) { - // Confirm our client-side error is emitted as expected - expect(err).to.deep.equal(new Error('Could not sync filesystem from server')); - }); - - sync.connect("ws://127.0.0.1:" + port, "this-is-not-relevant"); - } - - - // First, prepare the stub of the server. - testServer.on('connection', function(ws){ - socket = ws; - - // Stub WS auth - ws.once('message', function(msg) { - msg = msg.data || msg; - - try { - msg = JSON.parse(msg); - } catch(e) { - expect(e, "[Error parsing fake token]").to.not.exist; - } - - ws.once('message', function(msg){ - msg = parseMessage(msg); - expect(msg).to.deep.equal(SyncMessage.response.authz); - - ws.once('message', function(msg) { - // The next message from the client should be a RESPONSE RESET - msg = parseMessage(msg); - - expect(msg).to.deep.equal(SyncMessage.response.reset); - done(); - }); - - ws.send(SyncMessage.error.srclist.stringify()); - }); - - ws.send(SyncMessage.response.authz.stringify()); - }); - }); - - clientLogic(); - }); - - it('should restart a downstream sync on receiving a DIFFS ERROR SyncMessage instead of a sourceList.', function(done){ - var fs; - var sync; - - function clientLogic() { - fs = MakeDrive.fs({provider: provider, manual: true, forceCreate: true, autoReconnect: false}); - sync = fs.sync; - sync.on('error', function(err) { - // Confirm our client-side error is emitted as expected - expect(err).to.deep.equal(new Error('Could not sync filesystem from server')); - }); - - sync.connect("ws://127.0.0.1:" + port, "this-is-not-relevant"); - } - - // First, prepare the stub of the server. - testServer.on('connection', function(ws){ - socket = ws; - - // Stub WS auth - ws.once('message', function(msg) { - msg = msg.data || msg; - msg = parseMessage(msg); - - ws.once('message', function(msg){ - msg = parseMessage(msg); - expect(msg).to.deep.equal(SyncMessage.response.authz); - - ws.once('message', function(msg) { - // The second message from the client should be a REQUEST DIFFS - msg = parseMessage(msg); - expect(msg.type).to.equal(SyncMessage.REQUEST); - expect(msg.name).to.equal(SyncMessage.DIFFS); - - ws.once('message', function(msg) { - // The third message should be a RESPONSE RESET - msg = parseMessage(msg); - - expect(msg).to.deep.equal(SyncMessage.response.reset); - done(); - }); - - var diffsErrorMessage = SyncMessage.error.diffs; - ws.send(diffsErrorMessage.stringify()); - }); - - rsync.sourceList(fs, '/', rsyncOptions, function(err, srcList) { - expect(err, "[SourceList generation error]").to.not.exist; - var chksumRequest = SyncMessage.request.chksum; - chksumRequest.content = { - srcList: srcList, - path: '/' - }; - - ws.send(chksumRequest.stringify()); - }); - }); - - ws.send(SyncMessage.response.authz.stringify()); - }); - }); - - clientLogic(); - }); - - it('should reset a downstream sync on failing to patch', function(done){ - var fs; - var sync; - - function clientLogic() { - fs = MakeDrive.fs({provider: provider, manual: true, forceCreate: true, autoReconnect: false}); - sync = fs.sync; - sync.on('error', function(err) { - // Confirm our client-side error is emitted as expected - expect(err).to.exist; - }); - - sync.connect("ws://127.0.0.1:" + port); - } - - // First, prepare the stub of the server. - testServer.on('connection', function(ws){ - socket = ws; - - // Stub WS auth - ws.once('message', function(msg) { - msg = msg.data || msg; - msg = parseMessage(msg); - - // after auth - ws.once('message', function(msg){ - msg = parseMessage(msg); - expect(msg).to.deep.equal(SyncMessage.response.authz); - - ws.once('message', function(msg) { - // The second message from the client should be a REQUEST DIFFS - msg = parseMessage(msg); - expect(msg.type).to.equal(SyncMessage.REQUEST); - expect(msg.name).to.equal(SyncMessage.DIFFS); - var message = SyncMessage.response.diffs; - message.content = {}; - message.content.diffs = []; - message.content.diffs[0] = {}; - message.content.diffs[0].diffs = [ { data: [ 102, 117, 110 ] } ]; - - ws.once('message', function(msg) { - // The third message should be a RESPONSE RESET - msg = parseMessage(msg); - expect(msg).to.deep.equal(SyncMessage.response.reset); - - endTestSession(sync, done); - }); - - ws.send(message.stringify()); - }); - rsync.sourceList(fs, '/', rsyncOptions, function(err, srcList) { - expect(err, "[SourceList generation error]").to.not.exist; - var chksumRequest = SyncMessage.request.chksum; - chksumRequest.content = { - srcList: srcList, - path: '/' - }; - - ws.send(chksumRequest.stringify()); - }); - }); - - ws.send(SyncMessage.response.authz.stringify()); - }); - }); - - clientLogic(); - }); - - it('should restart a downstream sync on receiving a verification error', function(done){ - var fs; - var sync; - - function clientLogic() { - fs = MakeDrive.fs({provider: provider, manual: true, forceCreate: true, autoReconnect: false}); - sync = fs.sync; - sync.on('error', function(err) { - // Confirm our client-side error is emitted as expected - expect(err).to.exist; - }); - - sync.connect("ws://127.0.0.1:" + port); - } - - // First, prepare the stub of the server. - testServer.on('connection', function(ws){ - socket = ws; - - // Stub WS auth - ws.once('message', function(msg) { - msg = msg.data || msg; - msg = parseMessage(msg); - - // after auth - ws.once('message', function(msg){ - msg = parseMessage(msg); - expect(msg).to.deep.equal(SyncMessage.response.authz); - - ws.once('message', function(msg) { - // The second message from the client should be a REQUEST DIFFS - msg = parseMessage(msg); - expect(msg.type).to.equal(SyncMessage.REQUEST); - expect(msg.name).to.equal(SyncMessage.DIFFS); - var message = SyncMessage.response.diffs; - message.content = {}; - message.content.diffs = []; - message.content.diffs[0] = {path: '/'}; - message.content.diffs[0].diffs = []; - message.content.diffs[1] = {path: '/file.txt'}; - message.content.diffs[1].diffs = [ { data: [ 102, 117, 110 ] } ]; - - ws.once('message', function(msg) { - // The third message should be a RESPONSE RESET - ws.once('message', function(msg) { - msg = parseMessage(msg); - expect(msg).to.deep.equal(SyncMessage.response.reset); - endTestSession(sync, done); - }); - msg = parseMessage(msg); - - ws.send(SyncMessage.error.verification.stringify()); - }); - ws.send(message.stringify()); - - }); - rsync.sourceList(fs, '/', rsyncOptions, function(err, srcList) { - expect(err, "[SourceList generation error]").to.not.exist; - var chksumRequest = SyncMessage.request.chksum; - chksumRequest.content = { - srcList: srcList, - path: '/' - }; - - ws.send(chksumRequest.stringify()); - }); - }); - - ws.send(SyncMessage.response.authz.stringify()); - }); - }); - - clientLogic(); }); it('should emit an error describing an incorrect SYNC_STATE in the sync.request step', function(done){ - util.authenticatedConnection(function( err, result ) { + server.authenticatedConnection(function( err, result ) { expect(err).not.to.exist; var token = result.token; var layout = {'/file': 'data'}; - var fs; - var sync; - fs = MakeDrive.fs({provider: provider, manual: true, forceCreate: true, autoReconnect: false}); - sync = fs.sync; - - sync.once('connected', function onConnected() { + sync.once('synced', function onDownstreamCompleted() { sync.once('disconnected', function onDisconnected() { sync.once('error', function(err){ expect(err).to.exist; - expect(err).to.deep.equal(new Error('Invalid state. Expected ' + sync.SYNC_CONNECTED + ', got ' + sync.SYNC_DISCONNECTED)); + expect(err).to.deep.equal(new Error('MakeDrive error: MakeDrive cannot sync as it is either disconnected or trying to connect')); done(); }); @@ -492,27 +207,22 @@ describe('MakeDrive Client API', function(){ }); }); - sync.connect(util.socketURL, token); + sync.connect(server.socketURL, token); }); }); it('should emit an error warning about an unexpected sync.state when calling the sync.auto step', function(done){ - util.authenticatedConnection(function( err, result ) { + server.authenticatedConnection(function(err, result) { expect(err).not.to.exist; var token = result.token; var layout = {'/file': 'data'}; - var fs; - var sync; - fs = MakeDrive.fs({provider: provider, manual: true, forceCreate: true, autoReconnect: false}); - sync = fs.sync; - - sync.once('connected', function onConnected() { + sync.once('synced', function onDownstreamCompleted() { sync.once('disconnected', function onDisconnected() { sync.once('error', function(err){ expect(err).to.exist; - expect(err).to.deep.equal(new Error('Invalid state. ' + sync.state + ' is unexpected.')); + expect(err).to.deep.equal(new Error('MakeDrive error: MakeDrive cannot sync as it is either disconnected or trying to connect')); done(); }); @@ -528,63 +238,7 @@ describe('MakeDrive Client API', function(){ }); }); - sync.connect(util.socketURL, token); - }); - }); - - it('should emit an error describing an already CONNECTED sync.state in the sync.connect step', function(done){ - util.authenticatedConnection(function( err, result ) { - expect(err).not.to.exist; - - var token = result.token; - var fs; - var sync; - - fs = MakeDrive.fs({provider: provider, manual: true, forceCreate: true, autoReconnect: false}); - sync = fs.sync; - - sync.once('connected', function onConnected() { - sync.once('error', function(err){ - expect(err).to.exist; - expect(err).to.deep.equal(new Error("MakeDrive: Attempted to connect to the websocket URL, but a connection already exists!")); - - done(); - }); - - sync.connect(util.socketURL, token); - }); - - sync.connect(util.socketURL, token); - }); - }); - - it('should emit an error on an attempted sync.disconnect() call when already DISCONNECTED', function(done){ - util.authenticatedConnection(function( err, result ) { - expect(err).not.to.exist; - - var token = result.token; - var fs; - var sync; - - fs = MakeDrive.fs({provider: provider, manual: true, forceCreate: true, autoReconnect: false}); - sync = fs.sync; - - sync.once('connected', function onConnected() { - sync.once('disconnected', function onDisconnected() { - sync.once('error', function(err){ - expect(err).to.exist; - expect(err).to.deep.equal(new Error("MakeDrive: Attempted to disconnect, but no server connection exists!")); - - done(); - }); - - sync.disconnect(); - }); - - sync.disconnect(); - }); - - sync.connect(util.socketURL, token); + sync.connect(server.socketURL, token); }); }); }); diff --git a/tests/unit/client/many-small-files.js b/tests/unit/client/many-small-files.js index d920608..813522e 100644 --- a/tests/unit/client/many-small-files.js +++ b/tests/unit/client/many-small-files.js @@ -1,20 +1,33 @@ var expect = require('chai').expect; var util = require('../../lib/util.js'); +var server = require('../../lib/server-utils.js'); var MakeDrive = require('../../../client/src'); var Filer = require('../../../lib/filer.js'); describe('MakeDrive Client - sync many small files', function(){ - var provider; + var fs; + var sync; - beforeEach(function(done) { - util.ready(function() { - provider = new Filer.FileSystem.providers.Memory(util.username()); + before(function(done) { + server.start(done); + }); + after(function(done) { + server.shutdown(done); + }); + + beforeEach(function() { + fs = MakeDrive.fs({provider: new Filer.FileSystem.providers.Memory(util.username()), manual: true, forceCreate: true}); + sync = fs.sync; + }); + afterEach(function(done) { + util.disconnectClient(sync, function(err) { + if(err) throw err; + + sync = null; + fs = null; done(); }); }); - afterEach(function() { - provider = null; - }); function smallFile(number) { return ' '+ @@ -34,19 +47,23 @@ describe('MakeDrive Client - sync many small files', function(){ * downstream sync brings them back. */ it('should sync many small files', function(done) { - util.authenticatedConnection(function( err, result ) { + server.authenticatedConnection(function(err, result) { expect(err).not.to.exist; - var fs = MakeDrive.fs({provider: provider, manual: true, forceCreate: true}); - var sync = fs.sync; - // Make a layout with /project and 100 small html files inside var layout = {}; for(var i=0; i<100; i++) { layout['/project/small-file' + i + '.html'] = smallFile(i); } - sync.once('connected', function onConnected() { + sync.once('synced', function onDownstreamCompleted() { + sync.once('synced', function onUpstreamCompleted() { + server.ensureRemoteFilesystem(layout, result.jar, function(err) { + expect(err).not.to.exist; + sync.disconnect(); + }); + }); + util.createFilesystemLayout(fs, layout, function(err) { expect(err).not.to.exist; expect(sync.state).to.equal(sync.SYNC_CONNECTED); @@ -58,19 +75,12 @@ describe('MakeDrive Client - sync many small files', function(){ expect(err).not.to.exist; }); - sync.once('completed', function onUpstreamCompleted() { - util.ensureRemoteFilesystem(layout, result.jar, function(err) { - expect(err).not.to.exist; - sync.disconnect(); - }); - }); - sync.once('disconnected', function onDisconnected() { util.deleteFilesystemLayout(fs, null, function(err) { expect(err).not.to.exist; // Re-sync with server and make sure we get our deep dir back - sync.once('connected', function onSecondDownstreamSync() { + sync.once('synced', function onSecondDownstreamSync() { sync.once('disconnected', function onSecondDisconnected() { util.ensureFilesystem(fs, layout, function(err) { @@ -84,15 +94,15 @@ describe('MakeDrive Client - sync many small files', function(){ }); // Get a new token for this second connection - util.getWebsocketToken(result, function(err, result) { + server.getWebsocketToken(result, function(err, result) { expect(err).not.to.exist; - sync.connect(util.socketURL, result.token); + sync.connect(server.socketURL, result.token); }); }); }); - sync.connect(util.socketURL, result.token); + sync.connect(server.socketURL, result.token); }); }); diff --git a/tests/unit/client/multiple-file-sync.js b/tests/unit/client/multiple-file-sync.js index 825c795..5828253 100644 --- a/tests/unit/client/multiple-file-sync.js +++ b/tests/unit/client/multiple-file-sync.js @@ -1,20 +1,33 @@ var expect = require('chai').expect; var util = require('../../lib/util.js'); +var server = require('../../lib/server-utils.js'); var MakeDrive = require('../../../client/src'); var Filer = require('../../../lib/filer.js'); describe('MakeDrive Client - sync multiple files', function(){ - var provider; + var fs; + var sync; - beforeEach(function(done) { - util.ready(function() { - provider = new Filer.FileSystem.providers.Memory(util.username()); + before(function(done) { + server.start(done); + }); + after(function(done) { + server.shutdown(done); + }); + + beforeEach(function() { + fs = MakeDrive.fs({provider: new Filer.FileSystem.providers.Memory(util.username()), manual: true, forceCreate: true}); + sync = fs.sync; + }); + afterEach(function(done) { + util.disconnectClient(sync, function(err) { + if(err) throw err; + + sync = null; + fs = null; done(); }); }); - afterEach(function() { - provider = null; - }); /** * This test creates multiple files, syncs, and checks that they exist @@ -22,19 +35,23 @@ describe('MakeDrive Client - sync multiple files', function(){ * brings them back. */ it('should sync multiple files', function(done) { - util.authenticatedConnection(function( err, result ) { + server.authenticatedConnection(function( err, result ) { expect(err).not.to.exist; - var fs = MakeDrive.fs({provider: provider, manual: true, forceCreate: true}); - var sync = fs.sync; - var layout = { '/file1': 'contents of file1', '/file2': 'contents of file2', '/file3': 'contents of file3' }; - sync.once('connected', function onConnected() { + sync.once('synced', function onDownstreamCompleted() { + sync.once('synced', function onUpstreamCompleted() { + // Make sure all 3 files made it to the server + server.ensureRemoteFilesystem(layout, result.jar, function() { + sync.disconnect(); + }); + }); + util.createFilesystemLayout(fs, layout, function(err) { expect(err).not.to.exist; @@ -42,19 +59,12 @@ describe('MakeDrive Client - sync multiple files', function(){ }); }); - sync.once('completed', function onUpstreamCompleted() { - // Make sure all 3 files made it to the server - util.ensureRemoteFilesystem(layout, result.jar, function() { - sync.disconnect(); - }); - }); - sync.once('disconnected', function onDisconnected() { util.deleteFilesystemLayout(fs, null, function(err) { expect(err).not.to.exist; // Re-sync with server and make sure we get our files back - sync.once('connected', function onSecondDownstreamSync() { + sync.once('synced', function onSecondDownstreamSync() { sync.once('disconnected', function onSecondDisconnected() { util.ensureFilesystem(fs, layout, function(err) { @@ -68,15 +78,15 @@ describe('MakeDrive Client - sync multiple files', function(){ }); // Get a new token for this second connection - util.getWebsocketToken(result, function(err, result) { + server.getWebsocketToken(result, function(err, result) { expect(err).not.to.exist; - sync.connect(util.socketURL, result.token); + sync.connect(server.socketURL, result.token); }); }); }); - sync.connect(util.socketURL, result.token); + sync.connect(server.socketURL, result.token); }); }); diff --git a/tests/unit/client/same-name-dirs.js b/tests/unit/client/same-name-dirs.js index 1f1f00d..56b3792 100644 --- a/tests/unit/client/same-name-dirs.js +++ b/tests/unit/client/same-name-dirs.js @@ -1,80 +1,91 @@ var expect = require('chai').expect; var util = require('../../lib/util.js'); +var server = require('../../lib/server-utils.js'); var MakeDrive = require('../../../client/src'); var Filer = require('../../../lib/filer.js'); describe('Syncing dirs with entries of the same name', function(){ - var provider1; + var fs; + var sync; + + before(function(done) { + server.start(done); + }); + after(function(done) { + server.shutdown(done); + }); beforeEach(function() { - var username = util.username(); - provider1 = new Filer.FileSystem.providers.Memory(username); + fs = MakeDrive.fs({provider: new Filer.FileSystem.providers.Memory(util.username()), manual: true, forceCreate: true}); + sync = fs.sync; }); - afterEach(function() { - provider1 = null; + afterEach(function(done) { + util.disconnectClient(sync, function(err) { + if(err) throw err; + + sync = null; + fs = null; + done(); + }); }); it('should be able to sync a file contained within a directory of the same name', function(done) { - util.authenticatedConnection(function(err, result1) { + server.authenticatedConnection(function(err, result1) { expect(err).not.to.exist; - var fs1 = MakeDrive.fs({provider: provider1, manual: true, forceCreate: true}); - var sync1 = fs1.sync; var file1 = {'/path/path': 'This is file 1'}; - sync1.once('connected', function onClient1Connected() { + sync.once('synced', function onClient1Connected() { expect(err).not.to.exist; - util.createFilesystemLayout(fs1, file1, function(err) { - expect(err).not.to.exist; - - util.ensureFilesystem(fs1, file1, function(err) { + sync.once('synced', function onClient1Upstream1() { + server.ensureRemoteFilesystem(file1, result1.jar, function(err) { expect(err).not.to.exist; - sync1.request(); + done(); }); }); - sync1.once('completed', function onClient1Upstream1() { - util.ensureRemoteFilesystem(file1, result1.jar, function(err) { + util.createFilesystemLayout(fs, file1, function(err) { + expect(err).not.to.exist; + + util.ensureFilesystem(fs, file1, function(err) { expect(err).not.to.exist; - done(); + sync.request(); }); }); }); - sync1.connect(util.socketURL, result1.token); + sync.connect(server.socketURL, result1.token); }); }); it('should be able to sync directories contained in a direcotry with the same name if it contains a file', function(done) { - util.authenticatedConnection(function(err, result1) { + server.authenticatedConnection(function(err, result1) { expect(err).not.to.exist; - var fs1 = MakeDrive.fs({provider: provider1, manual: true, forceCreate: true}); - var sync1 = fs1.sync; var file1 = {'/dir/dir/file1.txt': 'This is file 1'}; - sync1.once('connected', function onClient1Connected() { + sync.once('synced', function onClient1Connected() { expect(err).not.to.exist; - util.createFilesystemLayout(fs1, file1, function(err) { - expect(err).not.to.exist; - - util.ensureFilesystem(fs1, file1, function(err) { + sync.once('synced', function onClient1Upstream1() { + server.ensureRemoteFilesystem(file1, result1.jar, function(err) { expect(err).not.to.exist; - sync1.request(); + done(); }); }); - sync1.once('completed', function onClient1Upstream1() { - util.ensureRemoteFilesystem(file1, result1.jar, function(err) { + util.createFilesystemLayout(fs, file1, function(err) { + expect(err).not.to.exist; + + util.ensureFilesystem(fs, file1, function(err) { expect(err).not.to.exist; - done(); + sync.request(); }); }); }); - sync1.connect(util.socketURL, result1.token); + sync.connect(server.socketURL, result1.token); }); }); }); diff --git a/tests/unit/client/symlink-test.js b/tests/unit/client/symlink-test.js index 083a5a0..c216e4b 100644 --- a/tests/unit/client/symlink-test.js +++ b/tests/unit/client/symlink-test.js @@ -1,16 +1,32 @@ var expect = require('chai').expect; var util = require('../../lib/util.js'); +var server = require('../../lib/server-utils.js'); var MakeDrive = require('../../../client/src'); var Filer = require('../../../lib/filer.js'); describe('MakeDrive Client - sync symlink', function() { - var provider; + var fs; + var sync; + + before(function(done) { + server.start(done); + }); + after(function(done) { + server.shutdown(done); + }); beforeEach(function() { - provider = new Filer.FileSystem.providers.Memory(util.username()); + fs = MakeDrive.fs({provider: new Filer.FileSystem.providers.Memory(util.username()), manual: true, forceCreate: true}); + sync = fs.sync; }); - afterEach(function() { - provider = null; + afterEach(function(done) { + util.disconnectClient(sync, function(err) { + if(err) throw err; + + sync = null; + fs = null; + done(); + }); }); /** @@ -18,16 +34,9 @@ describe('MakeDrive Client - sync symlink', function() { * and check that they both exists. */ it('should sync symlink', function(done) { - util.authenticatedConnection(function(err, result) { + server.authenticatedConnection(function(err, result) { expect(err).not.to.exist; - var fs = MakeDrive.fs({ - provider: provider, - manual: true, - forceCreate: true - }); - var sync = fs.sync; - var layout = { '/file1': 'contents of file1' }; @@ -36,31 +45,28 @@ describe('MakeDrive Client - sync symlink', function() { '/file2': 'contents of file1' }; - sync.once('connected', function onConnected() { - util.createFilesystemLayout(fs, layout, function(err) { - expect(err).not.to.exist; + sync.once('synced', function onDownstreamCompleted() { + sync.once('synced', function onUpstreamCompleted() { + server.ensureRemoteFilesystem(layout, result.jar, function() { + fs.symlink('/file1', '/file2', function(err) { + if (err) throw err; + sync.once('completed', function onWriteSymlink() { + server.ensureRemoteFilesystem(finalLayout, result.jar, done); + }); - sync.request(); + sync.request(); + }); + }); }); - }); - sync.once('completed', function onUpstreamCompleted() { - util.ensureRemoteFilesystem(layout, result.jar, function() { - fs.symlink('/file1', '/file2', function(err) { - if (err) throw err; - sync.once('completed', function onWriteSymlink() { - util.ensureRemoteFilesystem(finalLayout, result.jar, function() { - done(); - }); - }); + util.createFilesystemLayout(fs, layout, function(err) { + expect(err).not.to.exist; - sync.request(); - }); + sync.request(); }); }); - sync.connect(util.socketURL, result.token); + sync.connect(server.socketURL, result.token); }); }); - }); diff --git a/tests/unit/client/sync-filesystem.js b/tests/unit/client/sync-filesystem.js index cca20e2..c6b935f 100644 --- a/tests/unit/client/sync-filesystem.js +++ b/tests/unit/client/sync-filesystem.js @@ -30,7 +30,7 @@ describe('MakeDrive Client SyncFileSystem', function(){ function expectMakeDriveUnsyncedAttribForPath(path, callback) { fs.getUnsynced(path, function(err, unsynced) { expect(err).not.to.exist; - expect(unsynced).to.be.a.number; + expect(unsynced).to.be.a('number'); callback(); }); @@ -39,48 +39,48 @@ describe('MakeDrive Client SyncFileSystem', function(){ function expectMakeDriveUnsyncedAttribForFD(fd, callback) { fs.fgetUnsynced(fd, function(err, unsynced) { expect(err).not.to.exist; - expect(unsynced).to.be.a.number; + expect(unsynced).to.be.a('number'); callback(); }); } it('should have all the usual properties of a regular fs', function() { - expect(fs.rename).to.be.a.function; - expect(fs.ftruncate).to.be.a.function; - expect(fs.truncate).to.be.a.function; - expect(fs.stat).to.be.a.function; - expect(fs.fstat).to.be.a.function; - expect(fs.exists).to.be.a.function; - expect(fs.link).to.be.a.function; - expect(fs.symlink).to.be.a.function; - expect(fs.readlink).to.be.a.function; - expect(fs.realpath).to.be.a.function; - expect(fs.unlink).to.be.a.function; - expect(fs.mknod).to.be.a.function; - expect(fs.mkdir).to.be.a.function; - expect(fs.readdir).to.be.a.function; - expect(fs.close).to.be.a.function; - expect(fs.open).to.be.a.function; - expect(fs.utimes).to.be.a.function; - expect(fs.futimes).to.be.a.function; - expect(fs.fsync).to.be.a.function; - expect(fs.write).to.be.a.function; - expect(fs.read).to.be.a.function; - expect(fs.readFile).to.be.a.function; - expect(fs.writeFile).to.be.a.function; - expect(fs.appendFile).to.be.a.function; - expect(fs.setxattr).to.be.a.function; - expect(fs.fsetxattr).to.be.a.function; - expect(fs.getxattr).to.be.a.function; - expect(fs.removexattr).to.be.a.function; - expect(fs.fremovexattr).to.be.a.function; - expect(fs.watch).to.be.a.function; - expect(fs.Shell).to.be.a.function; + expect(fs.rename).to.be.a('function'); + expect(fs.ftruncate).to.be.a('function'); + expect(fs.truncate).to.be.a('function'); + expect(fs.stat).to.be.a('function'); + expect(fs.fstat).to.be.a('function'); + expect(fs.exists).to.be.a('function'); + expect(fs.link).to.be.a('function'); + expect(fs.symlink).to.be.a('function'); + expect(fs.readlink).to.be.a('function'); + expect(fs.realpath).to.be.a('function'); + expect(fs.unlink).to.be.a('function'); + expect(fs.mknod).to.be.a('function'); + expect(fs.mkdir).to.be.a('function'); + expect(fs.readdir).to.be.a('function'); + expect(fs.close).to.be.a('function'); + expect(fs.open).to.be.a('function'); + expect(fs.utimes).to.be.a('function'); + expect(fs.futimes).to.be.a('function'); + expect(fs.fsync).to.be.a('function'); + expect(fs.write).to.be.a('function'); + expect(fs.read).to.be.a('function'); + expect(fs.readFile).to.be.a('function'); + expect(fs.writeFile).to.be.a('function'); + expect(fs.appendFile).to.be.a('function'); + expect(fs.setxattr).to.be.a('function'); + expect(fs.fsetxattr).to.be.a('function'); + expect(fs.getxattr).to.be.a('function'); + expect(fs.removexattr).to.be.a('function'); + expect(fs.fremovexattr).to.be.a('function'); + expect(fs.watch).to.be.a('function'); + expect(fs.Shell).to.be.a('function'); // Extra SyncFileSystem specific things - expect(fs.getUnsynced).to.be.a.function; - expect(fs.fgetUnsynced).to.be.a.function; + expect(fs.getUnsynced).to.be.a('function'); + expect(fs.fgetUnsynced).to.be.a('function'); }); it('should allow fs.rename and mark unsynced', function(done) { @@ -155,26 +155,26 @@ describe('MakeDrive Client SyncFileSystem', function(){ }); }); - it('should allow fs.rmdir and mark parent dir unsynced', function(done) { + it('should allow fs.rmdir', function(done) { fs.mkdir('/newdir', function(err) { if(err) throw err; fs.rmdir('/newdir', function(err) { if(err) throw err; - expectMakeDriveUnsyncedAttribForPath('/', done); + done(); }); }); }); - it('should allow fs.unlink and mark parent dir unsynced', function(done) { + it('should allow fs.unlink', function(done) { fs.writeFile('/file', 'data', function(err) { if(err) throw err; fs.unlink('/file', function(err) { if(err) throw err; - expectMakeDriveUnsyncedAttribForPath('/', done); + done(); }); }); }); @@ -244,7 +244,7 @@ describe('MakeDrive Client SyncFileSystem', function(){ }); it('should allow fs.Shell()', function(done) { - var sh = fs.Shell(); + var sh = new fs.Shell(); // Run a shell command to make sure it's working. sh.rm('/dir', {recursive: true}, function(err) { diff --git a/tests/unit/client/sync-interruptions.js b/tests/unit/client/sync-interruptions.js deleted file mode 100644 index 8d9457f..0000000 --- a/tests/unit/client/sync-interruptions.js +++ /dev/null @@ -1,131 +0,0 @@ -var expect = require('chai').expect; -var util = require('../../lib/util.js'); -var MakeDrive = require('../../../client/src'); -var Filer = require('../../../lib/filer.js'); -var SyncMessage = require('../../../lib/syncmessage.js'); -var rsync = require('../../../lib/rsync'); -var WebSocketServer = require('ws').Server; -var rsyncOptions = require('../../../lib/constants').rsyncDefaults; - -describe('[Interruptions during a sync]', function() { - describe('Filesystem changes', function() { - var provider; - var socket; - var port = 1212; - var testServer; - - beforeEach(function(done) { - util.ready(function() { - provider = new Filer.FileSystem.providers.Memory(util.username()); - testServer = new WebSocketServer({port: port}); - - testServer.once('error', function(err){ - expect(err, "[Error creating socket server]").to.not.exist; - }); - testServer.once('listening', function() { - done(); - }); - }); - }); - afterEach(function() { - provider = null; - - if (socket) { - socket.close(); - } - - testServer.close(); - testServer = null; - }); - - function parseMessage(msg) { - msg = msg.data || msg; - - msg = JSON.parse(msg); - msg = SyncMessage.parse(msg); - - return msg; - } - - function endTestSession(sync, done) { - sync.once('disconnected', function() { - sync = null; - done(); - }); - - sync.disconnect(); - } - - it('should trigger a downstream reset if they occur during a downstream sync', function(done) { - var fs; - var sync; - - function clientLogic() { - fs = MakeDrive.fs({provider: provider, manual: true, forceCreate: true}); - sync = fs.sync; - - sync.connect("ws://127.0.0.1:" + port); - } - - // First, prepare the stub of the server. - testServer.on('connection', function(ws){ - socket = ws; - - // Stub WS auth - ws.once('message', function(msg) { - msg = msg.data || msg; - msg = parseMessage(msg); - - // after auth - ws.once('message', function(msg){ - msg = parseMessage(msg); - expect(msg).to.deep.equal(SyncMessage.response.authz); - - ws.once('message', function(msg) { - // The second message from the client should be a REQUEST DIFFS - msg = parseMessage(msg); - expect(msg.type).to.equal(SyncMessage.REQUEST); - expect(msg.name).to.equal(SyncMessage.DIFFS); - - fs.writeFile('/newfile.txt', 'This changes the file system', function(err) { - if(err) throw err; - - rsync.diff(fs, '/', msg.content.checksums, rsyncOptions, function(err, diffs) { - if(err) throw err; - - var message = SyncMessage.response.diffs; - - ws.once('message', function(msg) { - // The client should resend checksums and request diffs - msg = parseMessage(msg); - expect(msg.type).to.equal(SyncMessage.REQUEST); - expect(msg.name).to.equal(SyncMessage.DIFFS); - endTestSession(sync, done); - }); - - message.content = {diffs: diffs}; - ws.send(message.stringify()); - }); - }); - }); - - rsync.sourceList(fs, '/', rsyncOptions, function(err, srcList) { - expect(err, "[SourceList generation error]").to.not.exist; - var chksumRequest = SyncMessage.request.chksum; - chksumRequest.content = { - srcList: srcList, - path: '/' - }; - - ws.send(chksumRequest.stringify()); - }); - }); - - ws.send(SyncMessage.response.authz.stringify()); - }); - }); - - clientLogic(); - }); - }); -}); diff --git a/tests/unit/client/unsynced-attr-tests.js b/tests/unit/client/unsynced-attr-tests.js index d6ff514..d530078 100644 --- a/tests/unit/client/unsynced-attr-tests.js +++ b/tests/unit/client/unsynced-attr-tests.js @@ -1,24 +1,36 @@ var expect = require('chai').expect; var util = require('../../lib/util.js'); +var server = require('../../lib/server-utils.js'); var MakeDrive = require('../../../client/src'); var Filer = require('../../../lib/filer.js'); var fsUtils = require('../../../lib/fs-utils.js'); var FILE_CONTENT = 'This is a file'; -var syncManager = require('../../../client/src/sync-manager.js'); var async = require('async'); describe('MakeDrive Client FileSystem Unsynced Attribute', function() { - var provider; + var fs; + var sync; - beforeEach(function(done) { - util.ready(function() { - provider = new Filer.FileSystem.providers.Memory(util.username()); + before(function(done) { + server.start(done); + }); + after(function(done) { + server.shutdown(done); + }); + + beforeEach(function() { + fs = MakeDrive.fs({provider: new Filer.FileSystem.providers.Memory(util.username()), manual: true, forceCreate: true}); + sync = fs.sync; + }); + afterEach(function(done) { + util.disconnectClient(sync, function(err) { + if(err) throw err; + + sync = null; + fs = null; done(); }); }); - afterEach(function() { - provider = null; - }); // Check whether a list of paths have the unsynced attribute attached // The 'unsynced' flag indicates what to check for. true makes sure that @@ -48,136 +60,117 @@ describe('MakeDrive Client FileSystem Unsynced Attribute', function() { } it('should remove unsynced attribute from all nodes after an upstream sync', function(done) { - util.authenticatedConnection(function(err, result) { + server.authenticatedConnection(function(err, result) { expect(err).not.to.exist; - var fs = MakeDrive.fs({provider: provider, manual: true, forceCreate: true}); - var sync = fs.sync; - var layout = { '/dir/myfile1.txt': FILE_CONTENT, '/myfile2.txt': FILE_CONTENT, '/dir/subdir/myfile3.txt': FILE_CONTENT }; - sync.once('connected', function onConnected() { - util.createFilesystemLayout(fs, layout, function(err) { - expect(err).not.to.exist; + sync.once('synced', function onDownstreamCompleted() { + sync.once('synced', function onUpstreamCompleted() { + server.ensureRemoteFilesystem(layout, result.jar, function(err) { + expect(err).not.to.exist; - // Check that the unsynced attribute is attached to the new nodes - checkUnsyncedAttr(fs, layout, true, function(err, unsynced) { - expect(err).to.not.exist; - expect(unsynced).to.be.true; + // Check that the unsynced attribute is absent in the synced nodes + checkUnsyncedAttr(fs, layout, false, function(err, synced) { + expect(err).to.not.exist; + expect(synced).to.be.true; - sync.request(); + done(); + }); }); }); - }); - sync.once('completed', function onUpstreamCompleted() { - util.ensureRemoteFilesystem(layout, result.jar, function(err) { + util.createFilesystemLayout(fs, layout, function(err) { expect(err).not.to.exist; - // Check that the unsynced attribute is absent in the synced nodes - checkUnsyncedAttr(fs, layout, false, function(err, synced) { + // Check that the unsynced attribute is attached to the new nodes + checkUnsyncedAttr(fs, layout, true, function(err, unsynced) { expect(err).to.not.exist; - expect(synced).to.be.true; + expect(unsynced).to.be.true; - done(); + sync.request(); }); }); }); - sync.connect(util.socketURL, result.token); + sync.connect(server.socketURL, result.token); }); }); it('should remove unsynced attributes for unsynced nodes only after an upstream sync', function (done) { - util.authenticatedConnection(function(err, result) { + server.authenticatedConnection(function(err, result) { expect(err).not.to.exist; - var fs = MakeDrive.fs({provider: provider, manual: true, forceCreate: true}); - var sync = fs.sync; - var layout = { '/dir/myfile1.txt': FILE_CONTENT, '/myfile2.txt': FILE_CONTENT, '/dir/subdir/myfile3.txt': FILE_CONTENT }; var newLayout = { '/dir/subdir/myfile3.txt': 'New content' }; - sync.once('connected', function onConnected() { - util.createFilesystemLayout(fs, layout, function(err) { - expect(err).not.to.exist; - - sync.request(); - }); - }); - - sync.once('completed', function onUpstreamCompleted() { - util.ensureRemoteFilesystem(layout, result.jar, function(err) { - expect(err).not.to.exist; + sync.once('synced', function onDownstreamCompleted() { + sync.once('synced', function onUpstreamCompleted() { + server.ensureRemoteFilesystem(layout, result.jar, function(err) { + expect(err).not.to.exist; - sync.once('completed', function onSecondUpstreamCompleted() { - // Add the synced path back to the 'synced' layout - for(var k in newLayout) { - if(newLayout.hasOwnProperty(k)) { - layout[k] = newLayout[k]; + sync.once('completed', function onSecondUpstreamCompleted() { + // Add the synced path back to the 'synced' layout + for(var k in newLayout) { + if(newLayout.hasOwnProperty(k)) { + layout[k] = newLayout[k]; + } } - } - util.ensureRemoteFilesystem(layout, result.jar, function(err) { - expect(err).not.to.exist; - - // Check that no file has the unsynced attribute - checkUnsyncedAttr(fs, layout, false, function(err, synced) { + server.ensureRemoteFilesystem(layout, result.jar, function(err) { expect(err).not.to.exist; - expect(synced).to.be.true; - done(); + // Check that no file has the unsynced attribute + checkUnsyncedAttr(fs, layout, false, function(err, synced) { + expect(err).not.to.exist; + expect(synced).to.be.true; + + done(); + }); }); }); - }); - fs.writeFile('/dir/subdir/myfile3.txt', 'New content', function(err) { - if(err) throw err; + fs.writeFile('/dir/subdir/myfile3.txt', 'New content', function(err) { + if(err) throw err; - // Check that the changed unsynced file has the unsynced attribute - checkUnsyncedAttr(fs, newLayout, true, function(err, unsynced) { - expect(err).not.to.exist; - expect(unsynced).to.be.true; + // Check that the changed unsynced file has the unsynced attribute + checkUnsyncedAttr(fs, newLayout, true, function(err, unsynced) { + expect(err).not.to.exist; + expect(unsynced).to.be.true; - // Remove the unsynced path from the 'synced' layout - for(var k in newLayout) { - if(newLayout.hasOwnProperty(k)) { - delete layout[k]; + // Remove the unsynced path from the 'synced' layout + for(var k in newLayout) { + if(newLayout.hasOwnProperty(k)) { + delete layout[k]; + } } - } - // Check that the synced files do not have the unsynced attribute - checkUnsyncedAttr(fs, layout, false, function(err, synced) { - expect(err).not.to.exist; - expect(synced).to.be.true; + // Check that the synced files do not have the unsynced attribute + checkUnsyncedAttr(fs, layout, false, function(err, synced) { + expect(err).not.to.exist; + expect(synced).to.be.true; - sync.request(); + sync.request(); + }); }); }); }); }); - }); - sync.connect(util.socketURL, result.token); - }); - }); - - it('should remove the \'unsynced\' attribute on non-existent files', function (done) { - var fs = MakeDrive.fs({provider: provider, manual: true, forceCreate: true}); - var sync = fs.sync; - var manager = new syncManager(sync, fs); + util.createFilesystemLayout(fs, layout, function(err) { + expect(err).not.to.exist; - var paths = ['/random/garbage/', '/woah/daddy/']; + sync.request(); + }); + }); - manager.resetUnsynced(paths, function(err){ - expect(err).not.to.exist; - done(); + sync.connect(server.socketURL, result.token); }); }); }); diff --git a/tests/unit/fs-utils.js b/tests/unit/fs-utils.js index 7859017..3a22119 100644 --- a/tests/unit/fs-utils.js +++ b/tests/unit/fs-utils.js @@ -27,20 +27,27 @@ describe('MakeDrive fs-utils.js', function(){ }); it('should have all the expected properties', function() { - expect(fsUtils.forceCopy).to.be.a.function; - expect(fsUtils.isPathUnsynced).to.be.a.function; - expect(fsUtils.removeUnsynced).to.be.a.function; - expect(fsUtils.fremoveUnsynced).to.be.a.function; - expect(fsUtils.setUnsynced).to.be.a.function; - expect(fsUtils.fsetUnsynced).to.be.a.function; - expect(fsUtils.getUnsynced).to.be.a.function; - expect(fsUtils.fgetUnsynced).to.be.a.function; - expect(fsUtils.removeChecksum).to.be.a.function; - expect(fsUtils.fremoveChecksum).to.be.a.function; - expect(fsUtils.setChecksum).to.be.a.function; - expect(fsUtils.fsetChecksum).to.be.a.function; - expect(fsUtils.getChecksum).to.be.a.function; - expect(fsUtils.fgetChecksum).to.be.a.function; + expect(fsUtils.forceCopy).to.be.a('function'); + expect(fsUtils.isPathUnsynced).to.be.a('function'); + expect(fsUtils.removeUnsynced).to.be.a('function'); + expect(fsUtils.fremoveUnsynced).to.be.a('function'); + expect(fsUtils.setUnsynced).to.be.a('function'); + expect(fsUtils.fsetUnsynced).to.be.a('function'); + expect(fsUtils.getUnsynced).to.be.a('function'); + expect(fsUtils.fgetUnsynced).to.be.a('function'); + expect(fsUtils.removeChecksum).to.be.a('function'); + expect(fsUtils.fremoveChecksum).to.be.a('function'); + expect(fsUtils.setChecksum).to.be.a('function'); + expect(fsUtils.fsetChecksum).to.be.a('function'); + expect(fsUtils.getChecksum).to.be.a('function'); + expect(fsUtils.fgetChecksum).to.be.a('function'); + expect(fsUtils.isPathPartial).to.be.a('function'); + expect(fsUtils.removePartial).to.be.a('function'); + expect(fsUtils.fremovePartial).to.be.a('function'); + expect(fsUtils.setPartial).to.be.a('function'); + expect(fsUtils.fsetPartial).to.be.a('function'); + expect(fsUtils.getPartial).to.be.a('function'); + expect(fsUtils.fgetPartial).to.be.a('function'); }); it('should copy an existing file on forceCopy()', function(done) { @@ -220,4 +227,92 @@ describe('MakeDrive fs-utils.js', function(){ }); }); }); + + it('should report false for isPathPartial() if path does not exist', function(done) { + fsUtils.isPathPartial(fs, '/no/such/file', function(err, partial) { + expect(err).not.to.exist; + expect(partial).to.be.false; + done(); + }); + }); + + it('should report false for isPathPartial() if path has no metadata', function(done) { + fsUtils.isPathPartial(fs, '/dir/file', function(err, partial) { + expect(err).not.to.exist; + expect(partial).to.be.false; + done(); + }); + }); + + it('should report true for isPathPartial() if path has partial metadata', function(done) { + fsUtils.setPartial(fs, '/dir/file', 15, function(err) { + expect(err).not.to.exist; + + fsUtils.isPathPartial(fs, '/dir/file', function(err, partial) { + expect(err).not.to.exist; + expect(partial).to.be.true; + done(); + }); + }); + }); + + it('should give node count for getPartial() if path has partial metadata', function(done) { + fsUtils.setPartial(fs, '/dir/file', 10, function(err) { + expect(err).not.to.exist; + + fsUtils.getPartial(fs, '/dir/file', function(err, partial) { + expect(err).not.to.exist; + expect(partial).to.equal(10); + done(); + }); + }); + }); + + it('should remove metadata when calling removePartial()', function(done) { + fsUtils.setPartial(fs, '/dir/file', 10, function(err) { + expect(err).not.to.exist; + + fsUtils.getPartial(fs, '/dir/file', function(err, partial) { + expect(err).not.to.exist; + expect(partial).to.equal(10); + + fsUtils.removePartial(fs, '/dir/file', function(err) { + expect(err).not.to.exist; + + fsUtils.getPartial(fs, '/dir/file', function(err, partial) { + expect(err).not.to.exist; + expect(partial).not.to.exist; + done(); + }); + }); + }); + }); + }); + + it('should work with fd vs. path for partial metadata', function(done) { + fs.open('/dir/file', 'w', function(err, fd) { + if(err) throw err; + + fsUtils.fsetPartial(fs, fd, 10, function(err) { + expect(err).not.to.exist; + + fsUtils.fgetPartial(fs, fd, function(err, partial) { + expect(err).not.to.exist; + expect(partial).to.equal(10); + + fsUtils.fremovePartial(fs, fd, function(err) { + expect(err).not.to.exist; + + fsUtils.fgetPartial(fs, fd, function(err, unsynced) { + expect(err).not.to.exist; + expect(unsynced).not.to.exist; + + fs.close(fd); + done(); + }); + }); + }); + }); + }); + }); }); diff --git a/tests/unit/http-routes-tests.js b/tests/unit/http-routes-tests.js index 11ef6dd..9085b6b 100644 --- a/tests/unit/http-routes-tests.js +++ b/tests/unit/http-routes-tests.js @@ -1,31 +1,40 @@ var expect = require('chai').expect; var request = require('request'); var util = require('../lib/util'); +var server = require('../lib/server-utils.js'); var Filer = require('../../lib/filer.js'); var FileSystem = Filer.FileSystem; var Path = Filer.Path; var env = require('../../server/lib/environment'); -env.set('ALLOWED_CORS_DOMAINS', util.serverURL); +env.set('ALLOWED_CORS_DOMAINS', server.serverURL); var ALLOW_DOMAINS = process.env.ALLOWED_CORS_DOMAINS; var unzip = require("../lib/unzip.js"); describe('[HTTP route tests]', function() { + before(function(done) { + server.start(done); + }); + after(function(done) { + server.shutdown(done); + }); it('should allow CORS access to /api/sync route', function(done) { - request.get(util.serverURL + '/api/sync', { headers: {origin: ALLOW_DOMAINS }}, function(req, res) { - expect(ALLOW_DOMAINS).to.contain(res.headers['access-control-allow-origin']); - done(); + server.run(function() { + request.get(server.serverURL + '/api/sync', { headers: {origin: ALLOW_DOMAINS }}, function(req, res) { + expect(ALLOW_DOMAINS).to.contain(res.headers['access-control-allow-origin']); + done(); + }); }); }); describe('/p/ route tests', function() { it('should return a 404 error page if the path is not recognized', function(done) { - util.authenticate(function(err, result) { + server.authenticate(function(err, result) { expect(err).not.to.exist; expect(result.jar).to.exist; request.get({ - url: util.serverURL + '/p/no/file/here.html', + url: server.serverURL + '/p/no/file/here.html', jar: result.jar }, function(err, res, body) { expect(err).not.to.exist; @@ -41,15 +50,15 @@ describe('[HTTP route tests]', function() { var username = util.username(); var content = "This is the content of the file."; - util.upload(username, '/index.html', content, function(err) { + server.upload(username, '/index.html', content, function(err) { if(err) throw err; - util.authenticate({username: username}, function(err, result) { + server.authenticate({username: username}, function(err, result) { if(err) throw err; // /p/index.html should come back as uploaded request.get({ - url: util.serverURL + '/p/index.html', + url: server.serverURL + '/p/index.html', jar: result.jar }, function(err, res, body) { expect(err).not.to.exist; @@ -58,7 +67,7 @@ describe('[HTTP route tests]', function() { // /p/ should come back with dir listing request.get({ - url: util.serverURL + '/p/', + url: server.serverURL + '/p/', jar: result.jar }, function(err, res, body) { expect(err).not.to.exist; @@ -77,12 +86,12 @@ describe('[HTTP route tests]', function() { describe('/j/ route tests', function() { it('should return a 404 error page if the path is not recognized', function(done) { - util.authenticate(function(err, result) { + server.authenticate(function(err, result) { expect(err).not.to.exist; expect(result.jar).to.exist; request.get({ - url: util.serverURL + '/j/no/file/here.html', + url: server.serverURL + '/j/no/file/here.html', jar: result.jar, json: true }, function(err, res, body) { @@ -99,15 +108,15 @@ describe('[HTTP route tests]', function() { var username = util.username(); var content = "This is the content of the file."; - util.upload(username, '/index.html', content, function(err) { + server.upload(username, '/index.html', content, function(err) { if(err) throw err; - util.authenticate({username: username}, function(err, result) { + server.authenticate({username: username}, function(err, result) { if(err) throw err; // /j/index.html should come back as JSON request.get({ - url: util.serverURL + '/j/index.html', + url: server.serverURL + '/j/index.html', jar: result.jar, json: true }, function(err, res, body) { @@ -117,7 +126,7 @@ describe('[HTTP route tests]', function() { // /j/ should come back with dir listing request.get({ - url: util.serverURL + '/j/', + url: server.serverURL + '/j/', jar: result.jar, json: true }, function(err, res, body) { @@ -149,12 +158,12 @@ describe('[HTTP route tests]', function() { describe('/z/ route tests', function() { it('should return a 404 error page if the path is not recognized', function(done) { - util.authenticate(function(err, result) { + server.authenticate(function(err, result) { expect(err).not.to.exist; expect(result.jar).to.exist; request.get({ - url: util.serverURL + '/z/no/file/here.html', + url: server.serverURL + '/z/no/file/here.html', jar: result.jar }, function(err, res, body) { expect(err).not.to.exist; @@ -170,16 +179,16 @@ describe('[HTTP route tests]', function() { var username = util.username(); var content = "This is the content of the file."; - util.upload(username, '/index.html', content, function(err) { + server.upload(username, '/index.html', content, function(err) { if(err) throw err; - util.authenticate({username: username}, function(err, result) { + server.authenticate({username: username}, function(err, result) { if(err) throw err; // /z/ should come back as export.zip with one dir and file // in the archive. request.get({ - url: util.serverURL + '/z/', + url: server.serverURL + '/z/', jar: result.jar, encoding: null }, function(err, res, body) { @@ -190,7 +199,7 @@ describe('[HTTP route tests]', function() { // Write the zip file to filer, unzip, and compare file to original var fs = new FileSystem({provider: new FileSystem.providers.Memory(username)}); - var sh = fs.Shell(); + var sh = new fs.Shell(); sh.tempDir(function(err, tmp) { if(err) throw err; @@ -221,11 +230,11 @@ describe('[HTTP route tests]', function() { var username = util.username(); var content = "This is the content of the file."; - util.upload(username, '/index.html', content, function(err) { + server.upload(username, '/index.html', content, function(err) { if(err) throw err; request.get({ - url: util.serverURL + '/s/' + username + '/no/file/here.html', + url: server.serverURL + '/s/' + username + '/no/file/here.html', auth: { user: 'testusername', pass: 'testpassword' @@ -243,11 +252,11 @@ describe('[HTTP route tests]', function() { var username = util.username(); var content = "This is the content of the file."; - util.upload(username, '/index.html', content, function(err) { + server.upload(username, '/index.html', content, function(err) { if(err) throw err; request.get({ - url: util.serverURL + '/s/' + username + '/no/file/here.html', + url: server.serverURL + '/s/' + username + '/no/file/here.html', auth: { user: 'wrong-testusername', pass: 'wrong-testpassword' @@ -265,11 +274,11 @@ describe('[HTTP route tests]', function() { var username = util.username(); var content = "This is the content of the file."; - util.upload(username, '/index.html', content, function(err) { + server.upload(username, '/index.html', content, function(err) { if(err) throw err; request.get({ - url: util.serverURL + '/s/' + username + '/index.html', + url: server.serverURL + '/s/' + username + '/index.html', auth: { user: 'testusername', pass: 'testpassword' @@ -287,11 +296,11 @@ describe('[HTTP route tests]', function() { var username = util.username(); var content = new Buffer([1, 2, 3, 4]); - util.upload(username, '/binary', content, function(err) { + server.upload(username, '/binary', content, function(err) { if(err) throw err; request.get({ - url: util.serverURL + '/s/' + username + '/binary', + url: server.serverURL + '/s/' + username + '/binary', auth: { user: 'testusername', pass: 'testpassword' @@ -310,11 +319,11 @@ describe('[HTTP route tests]', function() { var username = util.username(); var content = "This is the content of the file."; - util.upload(username, '/index.html', content, function(err) { + server.upload(username, '/index.html', content, function(err) { if(err) throw err; request.get({ - url: util.serverURL + '/s/' + username + '/', + url: server.serverURL + '/s/' + username + '/', auth: { user: 'testusername', pass: 'testpassword' diff --git a/tests/unit/rename.js b/tests/unit/rename.js deleted file mode 100644 index 2535b9f..0000000 --- a/tests/unit/rename.js +++ /dev/null @@ -1,30 +0,0 @@ -var expect = require('chai').expect; -var util = require('../lib/util.js'); - -describe('Renaming a file', function(){ - it('should be able to rename and end up with single file after renamed', function(done) { - var originalLayout = {'/dir/file.txt': 'This is file 1'}; - var newLayout = {'/dir/newFile.txt': 'This is file 1'}; - - util.setupSyncClient({layout: originalLayout, manual: true}, function(err, client) { - expect(err).not.to.exist; - - var fs = client.fs; - - fs.rename('/dir/file.txt', '/dir/newFile.txt', function(err) { - expect(err).not.to.exist; - - client.sync.once('completed', function after() { - util.ensureRemoteFilesystem(newLayout, client.jar, function(err) { - expect(err).not.to.exist; - - client.sync.on('disconnected', done); - client.sync.disconnect(); - }); - }); - - client.sync.request(); - }); - }); - }); -}); diff --git a/tests/unit/rsync-tests.js b/tests/unit/rsync-tests.js index 21d9974..70b0afc 100644 --- a/tests/unit/rsync-tests.js +++ b/tests/unit/rsync-tests.js @@ -454,7 +454,7 @@ describe('[Rsync Functional tests]', function() { var layout = {'/test/folder/1.txt': 'This is my 1st file. It does not have any typos.', '/test/folder/2.txt': 'This is my 2nd file. It is longer than the destination file.'}; var patchedPaths = {synced: ['/test', '/test/folder', '/test/folder/1.txt', '/test/folder/2.txt']}; - + testUtils.createFilesystemLayout(fs, layout, function (err) { if(err) throw err; rsyncAssertions('/test', OPTION_REC_SIZE, patchedPaths, function () { @@ -476,7 +476,7 @@ describe('[Rsync Functional tests]', function() { testUtils.createFilesystemLayout(fs, layout, function (err) { if(err) throw err; - fs2.Shell().mkdirp('/test/sync', function (err) { + (new fs2.Shell()).mkdirp('/test/sync', function (err) { if(err) throw err; fs2.writeFile('/test/sync/3.txt', 'This shouldn\'t sync.', function (err) { if(err) throw err; diff --git a/tests/unit/server-socket-tests.js b/tests/unit/server-socket-tests.js new file mode 100644 index 0000000..7274edc --- /dev/null +++ b/tests/unit/server-socket-tests.js @@ -0,0 +1,561 @@ +var expect = require('chai').expect; +var util = require('../lib/server-utils.js'); +var testUtils = require('../lib/util.js'); +var SyncMessage = require('../../lib/syncmessage'); +var WS = require('ws'); +var syncTypes = require('../../lib/constants').syncTypes; +var diffHelper = require('../../lib/diff'); +var FAKE_DATA = 'FAKE DATA'; + +function validateSocketMessage(message, expectedMessage, checkExists) { + message = util.decodeSocketMessage(message); + checkExists = checkExists || []; + + expect(message.type).to.equal(expectedMessage.type); + expect(message.name).to.equal(expectedMessage.name); + + if(!expectedMessage.content) { + expect(message.content).not.to.exist; + return; + } + + expect(message.content).to.exist; + + if(typeof message.content !== 'object') { + expect(message.content).to.deep.equal(expectedMessage.content); + return; + } + + Object.keys(expectedMessage.content).forEach(function(key) { + if(checkExists.indexOf(key) !== -1) { + expect(message.content[key]).to.exist; + } else { + expect(message.content[key]).to.deep.equal(expectedMessage.content[key]); + } + }); +} + +describe('The Server', function(){ + before(function(done) { + util.start(done); + }); + after(function(done) { + util.shutdown(done); + }); + + describe('[Socket protocol] -', function() { + var socket, socket2; + + afterEach(function() { + if(socket) { + socket.close(); + } + if(socket2) { + socket2.close(); + } + }); + + it('should close a socket if bad data is sent in place of websocket-auth token', function(done) { + util.run(function() { + socket = new WS(util.socketURL); + + socket.onmessage = function() { + expect(true).to.be.false; + }; + socket.onopen = function() { + socket.send('This is not a token'); + }; + socket.onclose = function(closeMessage) { + expect(closeMessage).to.exist; + expect(closeMessage.code).to.equal(1011); + done(); + }; + }); + }); + + it('shouldn\'t allow the same token to be used twice', function(done) { + util.authenticatedConnection({done: done}, function(err, result) { + expect(err).not.to.exist; + + socket = new WS(util.socketURL); + var authMsg = {token: result.token}; + + socket.onmessage = function() { + socket2 = new WS(util.socketURL); + + socket2.onmessage = function() { + expect(true).to.be.false; + }; + socket2.onopen = function() { + socket2.send(JSON.stringify(authMsg)); + }; + socket2.onclose = function(closeMessage) { + expect(closeMessage).to.exist; + expect(closeMessage.code).to.equal(1008); + done(); + }; + }; + socket.onopen = function() { + socket.send(JSON.stringify(authMsg)); + }; + }); + }); + + it('should send a "RESPONSE" of "AUTHZ" after receiving a valid token and syncId', function(done) { + util.authenticatedConnection({done: done}, function(err, result) { + expect(err).not.to.exist; + + socket = new WS(util.socketURL); + socket.onmessage = function(message) { + validateSocketMessage(message, SyncMessage.response.authz); + done(); + }; + socket.onopen = function() { + socket.send(JSON.stringify({token: result.token})); + }; + }); + }); + + it('should allow two socket connections for the same username from different clients', function(done) { + util.authenticatedConnection(function(err, result) { + expect(err).not.to.exist; + + socket = new WS(util.socketURL); + socket.onmessage = function() { + util.authenticatedConnection({username: result.username}, function(err, result2) { + expect(err).not.to.exist; + expect(result2).to.exist; + expect(result2.username).to.equal(result.username); + expect(result2.token).not.to.equal(result.token); + + socket2 = new WS(util.socketURL); + socket2.onmessage = function(message) { + validateSocketMessage(message, SyncMessage.response.authz); + done(); + }; + socket2.onopen = function() { + socket2.send(JSON.stringify({token: result2.token})); + }; + }); + }; + socket.onopen = function() { + socket.send(JSON.stringify({token: result.token})); + }; + }); + }); + + it('should send a format SyncMessage error if a non-SyncMessage is sent', function(done) { + util.authenticatedSocket(function(err, result, socket) { + if(err) throw err; + + socket.onmessage = function(message) { + var expectedMessage = SyncMessage.error.format; + expectedMessage.content = 'Message must be formatted as a sync message'; + validateSocketMessage(message, expectedMessage); + socket.close(); + done(); + }; + + socket.send(JSON.stringify({message: 'This is not a sync message'})); + }); + }); + }); + + describe('[Downstream syncs] -', function(){ + var authResponse = SyncMessage.response.authz.stringify(); + + it('should send a "REQUEST" for "CHECKSUMS" to trigger a downstream when a client connects and the server has a non-empty filesystem', function(done) { + var username = testUtils.username(); + var file = {path: '/file', content: 'This is a file'}; + + util.upload(username, file.path, file.content, function(err) { + if(err) throw err; + + util.authenticatedSocket({username: username}, function(err, result, socket) { + if(err) throw err; + + socket.onmessage = function(message) { + var expectedMessage = SyncMessage.request.checksums; + expectedMessage.content = {path: file.path, type: syncTypes.CREATE, sourceList: FAKE_DATA}; + validateSocketMessage(message, expectedMessage, ['sourceList']); + socket.close(); + done(); + }; + + socket.send(authResponse); + }); + }); + }); + + it('should send a "RESPONSE" of "DIFFS" when requested for diffs', function(done) { + var username = testUtils.username(); + var file = {path: '/file', content: 'This is a file'}; + var checksums = testUtils.generateChecksums([file]); + var diffRequest = SyncMessage.request.diffs; + diffRequest.content = {path: file.path, type: syncTypes.CREATE, checksums: checksums[0]}; + + util.upload(username, file.path, file.content, function(err) { + if(err) throw err; + + util.authenticatedSocket({username: username}, function(err, result, socket) { + if(err) throw err; + + socket.onmessage = function(message) { + var expectedMessage = SyncMessage.response.diffs; + expectedMessage.content = {path: file.path, type: syncTypes.CREATE, diffs: FAKE_DATA}; + validateSocketMessage(message, expectedMessage, ['diffs']); + socket.close(); + done(); + }; + + socket.send(diffRequest.stringify()); + }); + }); + }); + + it('should send a "RESPONSE" of "VERIFICATION" on receiving a patch response', function(done) { + var username = testUtils.username(); + var initializedDownstream = false; + var file = {path: '/file', content: 'This is a file'}; + var checksums = testUtils.generateValidationChecksums([file]); + var patchResponse = SyncMessage.response.patch; + patchResponse.content = {path: file.path, type: syncTypes.CREATE, checksum: checksums}; + + util.upload(username, file.path, file.content, function(err) { + if(err) throw err; + + util.authenticatedSocket({username: username}, function(err, result, socket) { + if(err) throw err; + + socket.onmessage = function(message) { + var expectedMessage = SyncMessage.response.verification; + expectedMessage.content = {path: file.path, type: syncTypes.CREATE}; + + if(!initializedDownstream) { + initializedDownstream = true; + return socket.send(patchResponse.stringify()); + } + + validateSocketMessage(message, expectedMessage); + socket.close(); + done(); + }; + + socket.send(authResponse); + }); + }); + }); + + it('should allow an upstream sync request for a file if that file has been downstreamed', function(done) { + var username = testUtils.username(); + var file = {path: '/file', content: 'This is a file'}; + var currentStep = 'AUTH'; + var checksums = testUtils.generateValidationChecksums([file]); + var patchResponse = SyncMessage.response.patch; + patchResponse.content = {path: file.path, type: syncTypes.CREATE, checksum: checksums}; + var syncRequest = SyncMessage.request.sync; + syncRequest.content = {path: file.path, type: syncTypes.CREATE}; + + util.upload(username, file.path, file.content, function(err) { + if(err) throw err; + + util.authenticatedSocket({username: username}, function(err, result, socket) { + if(err) throw err; + + socket.onmessage = function(message) { + if(currentStep === 'AUTH') { + currentStep = 'PATCH'; + socket.send(patchResponse.stringify()); + } else if(currentStep === 'PATCH') { + currentStep = null; + socket.send(syncRequest.stringify()); + } else { + var expectedMessage = SyncMessage.response.sync; + expectedMessage.content = {path: file.path, type: syncTypes.CREATE}; + validateSocketMessage(message, expectedMessage); + socket.close(); + done(); + } + }; + + socket.send(authResponse); + }); + }); + }); + + it('should handle root responses from the client by removing the file from the downstream queue for that client', function(done) { + // Since we do not have access to the internals of the server, + // we test this case by sending a root response to the server + // and requesting an upstream sync for the same file, which + // should succeed. + var username = testUtils.username(); + var file = {path: '/file', content: 'This is a file'}; + var rootMessage = SyncMessage.response.root; + rootMessage.content = {path: file.path, type: syncTypes.CREATE}; + var syncRequest = SyncMessage.request.sync; + syncRequest.content = {path: file.path, type: syncTypes.CREATE}; + var rootMessageSent = false; + + util.upload(username, file.path, file.content, function(err) { + if(err) throw err; + + util.authenticatedSocket({username: username}, function(err, result, socket) { + if(err) throw err; + + socket.onmessage = function(message) { + if(!rootMessageSent) { + // NOTE: Under normal circumstances, a sync request + // message would not be sent to the server, however + // since that is the only way to test server internals + // (indirectly), this has an important implication. + // This test may fail as two socket messages are sent + // one after the other and an ASSUMPTION has been made + // that the first socket message executes completely + // before the second socket message executes. If this + // test fails, the most likely cause would be the below + // three lines of code that introduces a timing issue. + socket.send(rootMessage.stringify()); + rootMessageSent = true; + return socket.send(syncRequest.stringify()); + } + + var expectedMessage = SyncMessage.response.sync; + expectedMessage.content = {path: file.path, type: syncTypes.CREATE}; + validateSocketMessage(message, expectedMessage); + socket.close(); + done(); + }; + + socket.send(authResponse); + }); + }); + }); + + it('should send a "DOWNSTREAM_LOCKED" "ERROR" if a "REQUEST" for "DIFFS" is sent while an upstream sync is triggered for the same file by another client', function(done) { + var username = testUtils.username(); + var file = {path: '/file', content: 'This is a file'}; + var checksums = testUtils.generateChecksums([file]); + var diffRequest = SyncMessage.request.diffs; + diffRequest.content = {path: file.path, type: syncTypes.CREATE, checksums: checksums[0]}; + var authorized = false; + var syncRequest = SyncMessage.request.sync; + syncRequest.content = {path: file.path, type: syncTypes.CREATE}; + + util.upload(username, file.path, file.content, function(err) { + if(err) throw err; + + util.authenticatedSocket({username: username}, function(err, result, socket) { + if(err) throw err; + + util.authenticatedSocket({username: username}, function(err, result, socket2) { + if(err) throw err; + + socket.onmessage = function(message) { + if(!authorized) { + authorized = true; + return socket2.send(syncRequest.stringify()); + } + + var expectedMessage = SyncMessage.error.downstreamLocked; + expectedMessage.content = {path: file.path, type: syncTypes.CREATE}; + validateSocketMessage(message, expectedMessage); + socket.close(); + socket2.close(); + done(); + }; + + socket2.onmessage = function() { + socket.send(diffRequest.stringify()); + }; + + socket.send(authResponse); + }); + }); + }); + }); + + it('should send a "VERIFICATION" "ERROR" on receiving a patch response that incorrectly patched a file on the client', function(done) { + var username = testUtils.username(); + var initializedDownstream = false; + var file = {path: '/file', content: 'This is a file'}; + var patchResponse = SyncMessage.response.patch; + patchResponse.content = {path: file.path, type: syncTypes.CREATE}; + + util.upload(username, file.path, file.content, function(err) { + if(err) throw err; + + file.content = 'Modified content'; + patchResponse.content.checksum = testUtils.generateValidationChecksums([file]); + + util.authenticatedSocket({username: username}, function(err, result, socket) { + if(err) throw err; + + socket.onmessage = function(message) { + var expectedMessage = SyncMessage.error.verification; + expectedMessage.content = {path: file.path, type: syncTypes.CREATE}; + + if(!initializedDownstream) { + initializedDownstream = true; + return socket.send(patchResponse.stringify()); + } + + validateSocketMessage(message, expectedMessage); + socket.close(); + done(); + }; + + socket.send(authResponse); + }); + }); + }); + }); + + describe('[Upstream syncs] -', function() { + var file = {path: '/file', content: 'This is a file'}; + + it('should send a "RESPONSE" of "SYNC" if a sync is requested on a file without a lock', function(done) { + var syncRequest = SyncMessage.request.sync; + syncRequest.content = {path: file.path, type: syncTypes.CREATE}; + + util.authenticatedSocket(function(err, result, socket) { + if(err) throw err; + + socket.onmessage = function(message) { + var expectedMessage = SyncMessage.response.sync; + expectedMessage.content = {path: file.path, type: syncTypes.CREATE}; + + validateSocketMessage(message, expectedMessage); + socket.close(); + done(); + }; + + socket.send(syncRequest.stringify()); + }); + }); + + it('should send a "LOCKED" "ERROR" if a sync is requested on a file that is locked', function(done) { + var syncRequest = SyncMessage.request.sync; + syncRequest.content = {path: file.path, type: syncTypes.CREATE}; + + util.authenticatedSocket(function(err, result, socket) { + if(err) throw err; + + util.authenticatedSocket({username: result.username}, function(err, result, socket2) { + if(err) throw err; + + socket.onmessage = function() { + socket2.send(syncRequest.stringify()); + }; + + socket2.onmessage = function(message) { + var expectedMessage = SyncMessage.error.locked; + expectedMessage.content = {error: 'Sync already in progress', path: file.path, type: syncTypes.CREATE}; + + validateSocketMessage(message, expectedMessage); + socket.close(); + socket2.close(); + done(); + }; + + socket.send(syncRequest.stringify()); + }); + }); + }); + + it('should send a "REQUEST" for "DIFFS" containing checksums when requested for checksums', function(done) { + var syncRequested = false; + var syncRequest = SyncMessage.request.sync; + syncRequest.content = {path: file.path, type: syncTypes.CREATE}; + var checksumRequest = SyncMessage.request.checksums; + checksumRequest.content = {path: file.path, type: syncTypes.CREATE, sourceList: testUtils.generateSourceList([file])}; + + util.authenticatedSocket(function(err, result, socket) { + if(err) throw err; + + socket.onmessage = function(message) { + var expectedMessage = SyncMessage.request.diffs; + expectedMessage.content = {path: file.path, type: syncTypes.CREATE, checksums: FAKE_DATA}; + + if(!syncRequested) { + syncRequested = true; + return socket.send(checksumRequest.stringify()); + } + + validateSocketMessage(message, expectedMessage, ['checksums']); + socket.close(); + done(); + }; + + socket.send(syncRequest.stringify()); + }); + }); + + it('should patch the file being synced and send a "RESPONSE" of "PATCH" on receiving a diff response', function(done) { + var syncRequested = false; + var syncRequest = SyncMessage.request.sync; + syncRequest.content = {path: file.path, type: syncTypes.CREATE}; + var diffResponse = SyncMessage.response.diffs; + diffResponse.content = {path: file.path, type: syncTypes.CREATE, diffs: diffHelper.serialize(testUtils.generateDiffs([file]))}; + var layout = {}; + layout[file.path] = file.content; + + util.authenticatedSocket(function(err, result, socket) { + if(err) throw err; + + socket.onmessage = function(message) { + var expectedMessage = SyncMessage.response.patch; + expectedMessage.content = {path: file.path, type: syncTypes.CREATE}; + + if(!syncRequested) { + syncRequested = true; + return socket.send(diffResponse.stringify()); + } + + validateSocketMessage(message, expectedMessage); + util.ensureRemoteFilesystem(layout, result.jar, function(err) { + expect(err).not.to.exist; + socket.close(); + done(); + }); + }; + + socket.send(syncRequest.stringify()); + }); + }); + + it('should trigger a downstream sync on other clients on completing an upstream sync', function(done) { + var syncRequest = SyncMessage.request.sync; + var syncRequested = false; + syncRequest.content = {path: file.path, type: syncTypes.CREATE}; + var diffResponse = SyncMessage.response.diffs; + diffResponse.content = {path: file.path, type: syncTypes.CREATE, diffs: diffHelper.serialize(testUtils.generateDiffs([file]))}; + + util.authenticatedSocket(function(err, result, socket) { + if(err) throw err; + + util.authenticatedSocket({username: result.username}, function(err, result, socket2) { + if(err) throw err; + + socket2.onmessage = function(message) { + var expectedMessage = SyncMessage.request.checksums; + expectedMessage.content = {path: file.path, type: syncTypes.CREATE, sourceList: FAKE_DATA}; + + validateSocketMessage(message, expectedMessage, ['sourceList']); + socket.close(); + socket2.close(); + done(); + }; + + socket.onmessage = function() { + if(!syncRequested) { + syncRequested = true; + socket.send(diffResponse.stringify()); + } + }; + + socket.send(syncRequest.stringify()); + }); + }); + }); + }); +}); diff --git a/tests/unit/socket-tests.js b/tests/unit/socket-tests.js deleted file mode 100644 index 72a1b6d..0000000 --- a/tests/unit/socket-tests.js +++ /dev/null @@ -1,443 +0,0 @@ -var expect = require('chai').expect; -var util = require('../lib/util.js'); -var SyncMessage = require('../../lib/syncmessage'); - -describe('[Downstream Syncing with Websockets]', function(){ - describe('The server', function(){ - it('should close a socket if bad data is sent in place of websocket-auth token', function(done) { - util.authenticatedConnection({ done: done }, function( err, result ) { - expect(err).not.to.exist; - - var gotMessage = false; - - var socketPackage = util.openSocket({ - onMessage: function() { - gotMessage = true; - }, - onClose: function() { - expect(gotMessage).to.be.false; - util.cleanupSockets(result.done, socketPackage); - }, - onOpen: function() { - socketPackage.socket.send("this-is-garbage"); - } - }); - }); - }); - - it('shouldn\'t allow the same token to be used twice', function(done) { - util.authenticatedConnection({ done: done }, function( err, result ) { - expect(err).not.to.exist; - var socketData = { - token: result.token - }; - - var socketPackage = util.openSocket(socketData, { - onMessage: function(message) { - message = util.toSyncMessage(message); - - expect(message).to.exist; - expect(message.type).to.equal(SyncMessage.REQUEST); - expect(message.name).to.equal(SyncMessage.CHKSUM); - expect(message.content).to.be.an('object'); - - var socketPackage2 = util.openSocket(socketData, { - onClose: function(code) { - expect(code).to.equal(1008); - util.cleanupSockets(result.done, socketPackage, socketPackage2); - } - }); - } - }); - }); - }); - - it(', after receiving a valid token and syncId, should send a RESPONSE named "AUTHZ"', function(done) { - util.authenticatedConnection({ done: done }, function( err, result ) { - expect(err).not.to.exist; - var socketData = { - token: result.token - }; - - var socketPackage = util.openSocket({ - onMessage: function(message) { - expect(message).to.equal(SyncMessage.response.authz.stringify()); - util.cleanupSockets(result.done, socketPackage); - }, - onOpen: function() { - socketPackage.socket.send(JSON.stringify(socketData)); - } - }); - }); - }); - - it('should allow two socket connections for the same username from different clients', function(done) { - util.authenticatedConnection(function( err, result ) { - expect(err).not.to.exist; - var socketData = { - token: result.token - }; - - var socketPackage = util.openSocket(socketData, { - onMessage: function(message) { - message = util.toSyncMessage(message); - expect(message).to.exist; - expect(message.type).to.equal(SyncMessage.REQUEST); - expect(message.name).to.equal(SyncMessage.CHKSUM); - expect(message.content).to.be.an('object'); - - util.authenticatedConnection({username: result.username}, function(err, result2) { - expect(err).not.to.exist; - socketData = { - syncId: result2.syncId, - token: result2.token - }; - - var socketPackage2 = util.openSocket(socketData, { - onMessage: function() { - util.cleanupSockets(function() { - result.done(); - result2.done(); - done(); - }, socketPackage, socketPackage2); - }, - }); - }); - } - }); - }); - }); - - it('should send an "implementation" SyncMessage error object when a non-syncmessage object is sent', function(done) { - util.authenticatedConnection({ done: done }, function(err, result) { - expect(err).not.to.exist; - - var socketData = { - token: result.token - }; - - var socketPackage = util.openSocket(socketData, { - onMessage: function(message) { - // First, confirm server acknowledgment - message = util.toSyncMessage(message); - expect(message).to.exist; - expect(message.type).to.equal(SyncMessage.REQUEST); - expect(message.name).to.equal(SyncMessage.CHKSUM); - expect(message.content).to.be.an('object'); - - // Listen for SyncMessage error - socketPackage.socket.on("message", function(message) { - var implMsg = SyncMessage.error.impl; - implMsg.content = { error: "The Sync message cannot be handled by the server" }; - - expect(message).to.equal(implMsg.stringify()); - util.cleanupSockets(result.done, socketPackage); - }); - - var invalidMessage = { - anything: "else" - }; - - socketPackage.socket.send(JSON.stringify(invalidMessage)); - } - }); - }); - }); - - it('should allow an initial downstream sync for a new client after an upstream sync has been started', function(done) { - // First client connects - util.authenticatedConnection({done: done}, function(err, result1) { - expect(err).not.to.exist; - - // Second client connects - util.authenticatedConnection({username: result1.username}, function(err, result2) { - expect(err).not.to.exist; - - // First client completes the initial downstream sync & begins an upstream sync - util.prepareUpstreamSync('requestSync', result1.username, result1.token, function(data1, fs1, socketPackage1){ - - // Second client attempts an initial downstream sync - util.completeDownstreamSync(result1.username, result2.token, function(err, data2, fs2, socketPackage2) { - expect(err).not.to.exist; - - util.cleanupSockets(function() { - result1.done(); - result2.done(); - }, socketPackage1, socketPackage2); - }); - }); - }); - }); - }); - - it('should block a downstream sync reset request from the client after an upstream sync has been started', function(done) { - // First client connects - util.authenticatedConnection({done: done}, function(err, result1) { - expect(err).not.to.exist; - - // Second client connects - util.authenticatedConnection({username: result1.username}, function(err, result2) { - expect(err).not.to.exist; - - // First client completes the initial downstream sync - util.prepareUpstreamSync(result1.username, result1.token, function(err, data1, fs1, socketPackage1){ - expect(err).not.to.exist; - - // Second client begins an upstream sync - util.prepareUpstreamSync('requestSync', result1.username, result2.token, function(data2, fs2, socketPackage2) { - // First client sends RESPONSE RESET to start a downstream sync on the first client and expect an error - util.downstreamSyncSteps.requestSync(socketPackage1, data1, fs1, function(msg, cb) { - msg = util.toSyncMessage(msg); - - expect(msg).to.exist; - expect(msg.type).to.equal(SyncMessage.ERROR); - expect(msg.name).to.equal(SyncMessage.DOWNSTREAM_LOCKED); - - cb(); - }, function() { - util.cleanupSockets(function() { - result1.done(); - result2.done(); - }, socketPackage1, socketPackage2); - }); - }); - }); - }); - }); - }); - - it('should block a downstream sync diffs request from the client after an upstream sync has been started', function(done) { - // First client connects - util.authenticatedConnection({done: done}, function(err, result1) { - expect(err).not.to.exist; - - // Second client connects - util.authenticatedConnection({username: result1.username}, function(err, result2) { - expect(err).not.to.exist; - - // First client completes the initial downstream sync - util.prepareUpstreamSync(result1.username, result1.token, function(err, data1, fs1, socketPackage1){ - expect(err).not.to.exist; - - // First client completes the first step of a downstream sync - util.downstreamSyncSteps.requestSync(socketPackage1, data1, fs1, function(err, downstreamData) { - expect(err).to.not.exist; - - // Second client begins an upstream sync - util.prepareUpstreamSync('requestSync', result1.username, result2.token, function(data2, fs2, socketPackage2) { - // First client attempts the second step of an downstream sync, expecting an error - util.downstreamSyncSteps.generateDiffs(socketPackage1, downstreamData, fs1, function(msg, cb) { - msg = util.toSyncMessage(msg); - - expect(msg).to.exist; - expect(msg.type).to.equal(SyncMessage.ERROR); - expect(msg.name).to.equal(SyncMessage.DOWNSTREAM_LOCKED); - - cb(); - }, function() { - util.cleanupSockets(function(){ - result1.done(); - result2.done(); - }, socketPackage1, socketPackage2); - }); - }); - }); - }); - }); - }); - }); - - it('should allow the patch verification from the client after an upstream sync has been started', function(done) { - // First client connects - util.authenticatedConnection({done: done}, function(err, result1) { - expect(err).not.to.exist; - - // Second client connects - util.authenticatedConnection({username: result1.username}, function(err, result2) { - expect(err).not.to.exist; - - // First client completes the initial downstream sync - util.prepareUpstreamSync(result1.username, result1.token, function(err, data1, fs1, socketPackage1){ - expect(err).not.to.exist; - - // First client completes the first step of a downstream sync - util.downstreamSyncSteps.requestSync(socketPackage1, data1, fs1, function(err, downstreamData) { - expect(err).to.not.exist; - - // First client completes the second step of a downstream sync - util.downstreamSyncSteps.generateDiffs(socketPackage1, downstreamData, fs1, function(err, downstreamData2) { - expect(err).to.not.exist; - - // Second client begins an upstream sync - util.prepareUpstreamSync('requestSync', result1.username, result2.token, function(data2, fs2, socketPackage2) { - - // First client attempts the final step of an downstream sync, expect all to be well. - util.downstreamSyncSteps.patchClientFilesystem(socketPackage1, downstreamData2, fs1, function(err) { - expect(err).to.not.exist; - - util.cleanupSockets(function(){ - result1.done(); - result2.done(); - }, socketPackage1, socketPackage2); - }); - }); - }); - }); - }); - }); - }); - }); - }); - - describe('Generate Diffs', function() { - it('should return an RESPONSE message with the diffs', function(done) { - util.authenticatedConnection({ done: done }, function( err, result ) { - expect(err).not.to.exist; - - util.prepareDownstreamSync(result.username, result.token, function(err, syncData, fs, socketPackage) { - util.downstreamSyncSteps.generateDiffs(socketPackage, syncData, fs, function(msg, cb) { - msg = util.toSyncMessage(msg); - - expect(msg.type, "[Message type error: \"" + (msg.content && msg.content.error) +"\"]" ).to.equal(SyncMessage.RESPONSE); - expect(msg.name).to.equal(SyncMessage.DIFFS); - expect(msg.content).to.exist; - expect(msg.content.diffs).to.exist; - cb(); - }, function() { - util.cleanupSockets(result.done, socketPackage); - }); - }); - }); - }); - - it('should return an ERROR type message named DIFFS when faulty checksums are sent', function(done) { - util.authenticatedConnection({ done: done }, function( err, result ) { - expect(err).not.to.exist; - - util.prepareDownstreamSync(result.username, result.token, function(err, syncData, fs, socketPackage) { - var diffRequest = SyncMessage.request.diffs; - diffRequest.content = { - checksums: "jargon" - }; - util.sendSyncMessage(socketPackage, diffRequest, function(msg) { - msg = util.toSyncMessage(msg); - - expect(msg.type, "[Message type error: \"" + (msg.content && msg.content.error) +"\"]" ).to.equal(SyncMessage.ERROR); - expect(msg.name).to.equal(SyncMessage.DIFFS); - util.cleanupSockets(result.done, socketPackage); - }); - }); - }); - }); - - it('should return an SyncMessage with error content when no checksums are sent', function(done) { - util.authenticatedConnection({ done: done }, function( err, result ) { - expect(err).not.to.exist; - - util.prepareDownstreamSync(result.username, result.token, function(err, syncData, fs, socketPackage) { - var diffRequest = SyncMessage.request.diffs; - util.sendSyncMessage(socketPackage, diffRequest, function(msg) { - msg = util.toSyncMessage(msg); - - expect(msg).to.eql(SyncMessage.error.content); - util.cleanupSockets(result.done, socketPackage); - }); - }); - }); - }); - }); - - describe('Patch the client filesystem', function() { - it('should make the server respond with a RESPONSE SYNC SyncMessage after ending a downstream sync, and initiating an upstream sync', function(done) { - util.authenticatedConnection({ done: done }, function( err, result ) { - expect(err).not.to.exist; - - util.prepareDownstreamSync('generateDiffs', result.username, result.token, function(err, syncData, fs, socketPackage) { - util.downstreamSyncSteps.patchClientFilesystem(socketPackage, syncData, fs, function(msg, cb) { - msg = util.toSyncMessage(msg); - var startSyncMsg = SyncMessage.request.sync; - startSyncMsg.content = {path: '/'}; - util.sendSyncMessage(socketPackage, startSyncMsg, function(message){ - message = util.toSyncMessage(message); - - expect(message).to.exist; - expect(message.type).to.equal(SyncMessage.RESPONSE); - expect(message.name).to.equal(SyncMessage.SYNC); - - cb(); - }); - }, function() { - util.cleanupSockets(result.done, socketPackage); - }); - }); - }); - }); - - it('should return an IMPLEMENTATION ERROR SyncMessage when sent out of turn', function(done) { - util.authenticatedConnection({ done: done }, function( err, result ) { - expect(err).not.to.exist; - - util.prepareDownstreamSync(result.username, result.token, function(err, data, fs, socketPackage) { - var startSyncMsg = SyncMessage.request.sync; - startSyncMsg.content = {path: '/'}; - util.sendSyncMessage(socketPackage, startSyncMsg, function(msg){ - msg = util.toSyncMessage(msg); - - expect(msg).to.exist; - expect(msg.type).to.equal(SyncMessage.ERROR); - expect(msg.name).to.equal(SyncMessage.IMPL); - expect(msg.content).to.exist; - expect(msg.content.error).to.exist; - - util.cleanupSockets(result.done, socketPackage); - }); - }); - }); - }); - }); - - describe('Request checksums', function() { - it('should return a CONTENT error SyncMessage if srcList isn\'t passed', function(done) { - // Authorize a user, open a socket, authorize and complete a downstream sync - util.authenticatedConnection({ done: done }, function( err, result ) { - expect(err).not.to.exist; - - util.prepareUpstreamSync('requestSync', result.username, result.token, function(syncData, fs, socketPackage) { - var requestChksumMsg = SyncMessage.request.chksum; - requestChksumMsg.content = { - path: syncData.path - }; - socketPackage.socket.send(requestChksumMsg.stringify()); - - util.sendSyncMessage(socketPackage, requestChksumMsg, function(msg) { - msg = util.toSyncMessage(msg); - - expect(msg).to.deep.equal(SyncMessage.error.content); - - util.cleanupSockets(result.done, socketPackage); - }); - }); - }); - }); - it('should return a CONTENT error SyncMessage if no data is passed', function(done) { - // Authorize a user, open a socket, authorize and complete a downstream sync - util.authenticatedConnection({ done: done }, function( err, result ) { - expect(err).not.to.exist; - - util.prepareUpstreamSync('requestSync', result.username, result.token, function(syncData, fs, socketPackage) { - var requestChksumMsg = SyncMessage.request.chksum; - requestChksumMsg.content = {}; - socketPackage.socket.send(requestChksumMsg.stringify()); - - util.sendSyncMessage(socketPackage, requestChksumMsg, function(msg) { - msg = util.toSyncMessage(msg); - - expect(msg).to.deep.equal(SyncMessage.error.content); - - util.cleanupSockets(result.done, socketPackage); - }); - }); - }); - }); - }); -}); diff --git a/tests/unit/sync-path-resolver-tests.js b/tests/unit/sync-path-resolver-tests.js deleted file mode 100644 index 2fd19d6..0000000 --- a/tests/unit/sync-path-resolver-tests.js +++ /dev/null @@ -1,56 +0,0 @@ -var expect = require('chai').expect; -var pathResolver = require('../../lib/sync-path-resolver.js'); - -describe('Resolution path tests', function () { - it('should have resolve as a function', function () { - expect(pathResolver.resolve).to.be.a('function'); - }); - - it('should have resolveFromArray as a function', function() { - expect(pathResolver.resolveFromArray).to.be.a('function'); - }); - - it('should return / as the common path', function () { - expect(pathResolver.resolve(null, null)).to.equal('/'); - }); - - it('should return /dir as the common path', function () { - expect(pathResolver.resolve('/dir')).to.equal('/dir'); - }); - - it('should return /dir as the common path', function () { - expect(pathResolver.resolve(null, '/dir')).to.equal('/dir'); - }); - - it('should return /dir as the common path', function () { - expect(pathResolver.resolve('/dir/myfile.txt', '/dir/myfile2.txt')).to.equal('/dir'); - }); - - it('should return /dir as the common path', function () { - expect(pathResolver.resolve('/dir/myfile.txt', '/dir')).to.equal('/dir'); - }); - - it('should return / as the common path', function () { - expect(pathResolver.resolve('/dir/myfile.txt', '/dir2/myfile.txt')).to.equal('/'); - }); - - it('should return / as the common path', function () { - expect(pathResolver.resolve('/', '/dir/subdir/subsubdir')).to.equal('/'); - }); - - it('should return / as the common path', function () { - expect(pathResolver.resolveFromArray([null, null])).to.equal('/'); - }); - - it('should return /dir as the common path', function () { - expect(pathResolver.resolveFromArray(['/dir'])).to.equal('/dir'); - }); - - it('should return /dir as the common path', function () { - expect(pathResolver.resolveFromArray([null, '/dir', null, '/dir/file'])).to.equal('/dir'); - }); - - it('should return /dir as the common path', function () { - expect(pathResolver.resolveFromArray(['/dir/myfile1', '/dir', '/dir/myfile2', '/dir/dir2/myfile3'])).to.equal('/dir'); - }); -}); diff --git a/tests/unit/util-tests.js b/tests/unit/util-tests.js deleted file mode 100644 index c407e73..0000000 --- a/tests/unit/util-tests.js +++ /dev/null @@ -1,410 +0,0 @@ -var expect = require('chai').expect; -var util = require('../lib/util.js'); -var request = require('request'); -var SyncMessage = require('../../lib/syncmessage'); -var Filer = require('../../lib/filer.js'); -var FileSystem = Filer.FileSystem; - -describe('Test util.js', function(){ - describe('[Connection Helpers]', function() { - it('util.authenticate should signin the given user and set session.user.username', function(done) { - var username = util.username(); - util.authenticate({username: username}, function(err, result) { - expect(err).not.to.exist; - expect(result.username).to.equal(username); - expect(result.jar).to.exist; - - // Trying to login a second time as this user will 401 if session info is set - request.post({ - url: util.serverURL + '/mocklogin/' + username, - jar: result.jar - }, function(err, res) { - expect(err).not.to.exist; - expect(res.statusCode).to.equal(401); - done(); - }); - }); - }); - - it('util.authenticate should work with no options object passed', function(done) { - util.authenticate(function(err, result) { - expect(err).not.to.exist; - expect(result.username).to.be.a.string; - expect(result.jar).to.exist; - done(); - }); - }); - - it('util.getWebsocketToken should return a token on callback', function(done) { - var username = util.username(); - util.authenticate({username: username}, function(err, authResult) { - util.getWebsocketToken(authResult, function(err, tokenResult) { - expect(tokenResult.token).to.be.a('string'); - done(); - }); - }); - }); - - it('util.authenticatedConnection should signin and get a username, and ws token', function(done) { - util.authenticatedConnection(function(err, result) { - expect(err, "[err]").not.to.exist; - expect(result, "[result]").to.exist; - expect(result.jar, "[result.jar]").to.exist; - expect(result.username, "[result.username]").to.be.a("string"); - expect(result.token, "[result.token]").to.be.a("string"); - expect(result.done, "[result.done]").to.be.a("function"); - - request.get({ - url: util.serverURL + '/', - jar: result.jar - }, function(err, res) { - expect(err).not.to.exist; - expect(res.statusCode).to.equal(200); - result.done(); - done(); - }); - }); - }); - - it('util.authenticatedConnection should accept done function', function(done) { - util.authenticatedConnection({done: done}, function(err, result) { - expect(err).not.to.exist; - expect(result).to.exist; - expect(result.jar).to.exist; - expect(result.syncId).to.be.a.string; - expect(result.username).to.be.a.string; - expect(result.done).to.be.a.function; - - result.done(); - }); - }); - }); - - describe('[Misc Helpers]', function(){ - it('util.app should return the Express app instance', function () { - expect(util.app).to.exist; - }); - - it('util.username should generate a unique username with each call', function() { - var username1 = util.username(); - var username2 = util.username(); - expect(username1).to.be.a.string; - expect(username2).to.be.a.string; - expect(username1).not.to.equal(username2); - }); - - it('util.upload should allow a file to be uploaded and served', function(done) { - var fs = require('fs'); - var Path = require('path'); - var content = fs.readFileSync(Path.resolve(__dirname, '../test-files/index.html'), {encoding: null}); - var username = util.username(); - - util.upload(username, '/index.html', content, function(err) { - expect(err).not.to.exist; - - util.authenticate({username: username}, function(err, result) { - expect(err).not.to.exist; - expect(result.jar).to.exist; - - // /p/index.html should come back as uploaded - request.get({ - url: util.serverURL + '/p/index.html', - jar: result.jar - }, function(err, res, body) { - expect(err).not.to.exist; - expect(res.statusCode).to.equal(200); - expect(body).to.equal(content.toString('utf8')); - - // /p/ should come back with dir listing - request.get({ - url: util.serverURL + '/p/', - jar: result.jar - }, function(err, res, body) { - expect(err).not.to.exist; - expect(res.statusCode).to.equal(200); - // Look for artifacts we'd expect in the directory listing - expect(body).to.match(/Index of \/<\/title>/); - expect(body).to.match(/<a href="\/p\/index.html">index.html<\/a>/); - done(); - }); - }); - }); - }); - }); - }); - - describe('[Filesystem Helpers]', function() { - var provider; - - beforeEach(function() { - provider = new FileSystem.providers.Memory(util.username()); - }); - afterEach(function() { - provider = null; - }); - - it('should createFilesystemLayout and ensureFilesystem afterward', function(done) { - var fs = new FileSystem({provider: provider}); - var layout = { - "/file1": "contents file1", - "/dir1/file1": new Buffer([1,2,3]), - "/dir1/file2": "contents file2", - "/dir2": null - }; - - util.createFilesystemLayout(fs, layout, function(err) { - expect(err).not.to.exist; - - util.ensureFilesystem(fs, layout, done); - }); - }); - }); - - describe('[Socket Helpers]', function(){ - it('util.openSocket should open a socket connection with default handlers if none are provided', function(done){ - util.authenticatedConnection({ done: done }, function(err, result) { - var socketPackage = util.openSocket(); - expect(socketPackage.socket).to.exist; - expect(socketPackage.socket.readyState).to.exist; - expect(socketPackage.socket.url).to.exist; - expect(socketPackage.socket._events).to.exist; - expect(typeof socketPackage.onOpen).to.deep.equal("function"); - expect(typeof socketPackage.onClose).to.deep.equal("function"); - expect(typeof socketPackage.onError).to.deep.equal("function"); - expect(typeof socketPackage.onMessage).to.deep.equal("function"); - - socketPackage.setClose(function() { - result.done(); - }); - socketPackage.socket.close(); - }); - }); - - it('util.openSocket should open a socket with custom handlers when passed', function(done){ - util.authenticatedConnection({ done: done }, function(err, result) { - function onClose() {} - function onError() {} - function onOpen() {} - function onMessage() {} - - var socketPackage = util.openSocket({ - onClose: onClose, - onError: onError, - onOpen: onOpen, - onMessage: onMessage - }); - - expect(socketPackage.socket).to.exist; - expect(socketPackage.socket.readyState).to.exist; - expect(socketPackage.socket.url).to.exist; - expect(socketPackage.socket._events).to.exist; - expect(socketPackage.onOpen).to.deep.equal(onOpen); - expect(socketPackage.onClose).to.deep.equal(onClose); - expect(socketPackage.onError).to.deep.equal(onError); - expect(socketPackage.onMessage).to.deep.equal(onMessage); - - socketPackage.setClose(function() { - result.done(); - }); - socketPackage.socket.close(); - }); - }); - - it('util.openSocket should automatically generate an onOpen handler to send syncId to the server when passed syncId', function(done) { - util.authenticatedConnection({ done: done }, function(err, result) { - var socketData = { - token: result.token - }; - - var socketPackage = util.openSocket(socketData, { - onMessage: function(message){ - // First, confirm server acknowledgment - message = util.toSyncMessage(message); - expect(message).to.exist; - expect(message.type).to.equal(SyncMessage.REQUEST); - expect(message.name).to.equal(SyncMessage.CHKSUM); - expect(message.content).to.be.an('object'); - - socketPackage.setClose(function() { - result.done(); - }); - socketPackage.socket.close(); - } - }); - }); - }); - - it('util.openSocket\'s returned socketPackage.setXXX functions should change the socket\'s event handlers', function(done) { - util.authenticatedConnection({ done: done }, function(err, result) { - var socketPackage = util.openSocket(); - - var newClose = function (){}; - var newOpen = function () {}; - var newError = function () {}; - var newMessage = function () {}; - - socketPackage.setClose(newClose); - socketPackage.setError(newError); - socketPackage.setOpen(newOpen); - socketPackage.setMessage(newMessage); - - expect(socketPackage.onClose).to.deep.equal(newClose); - expect(socketPackage.onError).to.deep.equal(newError); - expect(socketPackage.onOpen).to.deep.equal(newOpen); - expect(socketPackage.onMessage).to.deep.equal(newMessage); - - socketPackage.setClose(function() { - result.done(); - }); - socketPackage.socket.close(); - }); - }); - - it('util.cleanupSockets should close a single socket and end tests', function(done){ - util.authenticatedConnection({ done: done }, function(err, result) { - var socketPackage = util.openSocket(); - util.cleanupSockets(function() { - expect(socketPackage.socket.readyState).to.equal(3); - result.done(); - }, socketPackage); - }); - }); - - it('util.cleanupSockets should close multiple sockets and end tests', function(done){ - util.authenticatedConnection({ done: done }, function(err, result) { - var socketPackage1 = util.openSocket(); - var socketPackage2 = util.openSocket(); - var socketPackage3 = util.openSocket(); - - util.cleanupSockets(function() { - expect(socketPackage1.socket.readyState).to.equal(3); - expect(socketPackage2.socket.readyState).to.equal(3); - expect(socketPackage3.socket.readyState).to.equal(3); - result.done(); - }, socketPackage1, socketPackage2, socketPackage3); - }); - }); - }); - - describe('[Downstream Sync Helpers]', function() { - it('util.prepareDownstreamSync should prepare a filesystem for the passed user when finalStep isn\'t specified', function(done) { - util.authenticatedConnection({done: done}, function(err, result) { - var username = util.username(); - - util.prepareDownstreamSync(username, result.token, function(err, syncData, fs, socketPackage) { - expect(err).to.not.exist; - expect(fs).to.exist; - expect(fs.provider).to.exist; - util.cleanupSockets(result.done, socketPackage); - }); - }); - }); - - it('util.downstreamSyncSteps.generateDiffs should return the diffs to the client', function(done) { - util.authenticatedConnection({ done: done }, function(err, result) { - var username = util.username(); - - util.prepareDownstreamSync(username, result.token, function(err, syncData, fs, socketPackage) { - util.downstreamSyncSteps.generateDiffs(socketPackage, syncData, fs, function(message, cb) { - message = util.toSyncMessage(message); - - expect(message.type).to.equal(SyncMessage.RESPONSE); - expect(message.name).to.equal(SyncMessage.DIFFS); - expect(message.content).to.exist; - expect(message.content.diffs).to.exist; - expect(message.content.path).to.exist; - - cb(); - }, function() { - util.cleanupSockets(result.done, socketPackage); - }); - }); - }); - }); - - it('util.prepareDownstreamSync should complete the generateDiffs step automatically when passed \'generateDiffs\' as the finalStep', function(done) { - util.authenticatedConnection({ done: done }, function(err, result) { - var username = util.username(); - util.prepareDownstreamSync("generateDiffs", username, result.token, function(err, syncData, fs, socketPackage) { - expect(syncData.diffs).to.exist; - - util.cleanupSockets(result.done, socketPackage); - }); - }); - }); - - it('util.syncSteps.patchClientFilesystem should execute properly, allowing a client to begin an upstream sync', function(done) { - util.authenticatedConnection({ done: done }, function(err, result) { - var username = util.username(); - - util.prepareDownstreamSync("generateDiffs", username, result.token, function(err, syncData, fs, socketPackage) { - util.downstreamSyncSteps.patchClientFilesystem(socketPackage, syncData, fs, function(err, data) { - expect(data).to.exist; - expect(data.path).to.exist; - - util.upstreamSyncSteps.requestSync(socketPackage, data, function() { - util.cleanupSockets(result.done, socketPackage); - }); - }); - }); - }); - }); - - it('util.prepareDownstreamSync should complete the patchClientFilesystem step automatically when passed \'patchClientFilesystem\' as the finalStep, allowing a client to begin an upstream sync', function(done) { - util.authenticatedConnection({ done: done }, function(err, result) { - var username = util.username(); - - util.prepareDownstreamSync("patchClientFilesystem", username, result.token, function(err, syncData, fs, socketPackage) { - expect(syncData).to.exist; - expect(syncData.path).to.exist; - - util.upstreamSyncSteps.requestSync(socketPackage, syncData, function() { - util.cleanupSockets(result.done, socketPackage); - }); - }); - }); - }); - }); - - describe('[Upstream Sync Helpers]', function() { - it('util.prepareUpstreamSync.prepareSync should return confirmation of a sync to the client', function(done) { - util.authenticatedConnection({done: done}, function(err, result) { - var username = util.username(); - - util.prepareUpstreamSync(username, result.token, function(err, syncData, fs, socketPackage) { - expect(fs).to.exist; - expect(fs.provider).to.exist; - util.upstreamSyncSteps.requestSync(socketPackage, syncData, function(message, cb) { - message = util.toSyncMessage(message); - - expect(message).to.exist; - expect(message.type).to.equal(SyncMessage.RESPONSE); - expect(message.name, "[SyncMessage Type error. SyncMessage.content was: " + message.content + "]").to.equal(SyncMessage.SYNC); - - cb(); - }, function() { - util.cleanupSockets(result.done, socketPackage); - }); - }); - }); - }); - - it('util.prepareUpstreamSync should complete a downstream sync, allowing the client to initiate an upstream sync, and prepare a filesystem for the client when finalStep isn\'t specified', function(done) { - util.authenticatedConnection({done: done}, function(err, result) { - var username = util.username(); - - util.prepareUpstreamSync(username, result.token, function(err, syncData, fs, socketPackage) { - expect(err).to.not.exist; - expect(fs).to.exist; - expect(fs.provider).to.exist; - expect(syncData).to.exist; - expect(syncData.path).to.exist; - - util.upstreamSyncSteps.requestSync(socketPackage, syncData, function() { - util.cleanupSockets(result.done, socketPackage); - }); - }); - }); - }); - }); -});