From 2797c2fc5b9913cc1682063f57f9605b213bb4c4 Mon Sep 17 00:00:00 2001 From: Chad Weider Date: Sat, 11 Feb 2012 20:53:24 -0800 Subject: [PATCH 1/9] Use the correct expression. --- node/utils/Minify.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/node/utils/Minify.js b/node/utils/Minify.js index 859ee07ae461..1830b0817371 100644 --- a/node/utils/Minify.js +++ b/node/utils/Minify.js @@ -232,7 +232,8 @@ function getAceFile(callback) { async.forEach(founds, function (item, callback) { var filename = item.match(/"([^"]*)"/)[1]; var type = item.match(/INCLUDE_([A-Z]+)/)[1]; - var shortFilename = (filename.match(/^..\/static\/js\/(.*)$/, '')||[])[1]; + var shortFilename = + (filename.match(/^\.\.\/static\/js\/(.*)$/, '') || [])[1]; //read the included files if (shortFilename) { From d6d4178dbc90d14623c94065f664e44babe3998c Mon Sep 17 00:00:00 2001 From: Chad Weider Date: Sun, 22 Jan 2012 17:19:46 -0800 Subject: [PATCH 2/9] Extract lastmodified from handler. --- node/utils/Minify.js | 82 ++++++++++++++++++++++++-------------------- 1 file changed, 45 insertions(+), 37 deletions(-) diff --git a/node/utils/Minify.js b/node/utils/Minify.js index 1830b0817371..8032c586f343 100644 --- a/node/utils/Minify.js +++ b/node/utils/Minify.js @@ -78,43 +78,11 @@ function _handle(req, res, jsFilename, jsFiles) { async.series([ //find out the highest modification date - function(callback) - { - var folders2check = [CSS_DIR, JS_DIR]; - - //go trough this two folders - async.forEach(folders2check, function(path, callback) - { - //read the files in the folder - fs.readdir(path, function(err, files) - { - if(ERR(err, callback)) return; - - //we wanna check the directory itself for changes too - files.push("."); - - //go trough all files in this folder - async.forEach(files, function(filename, callback) - { - //get the stat data of this file - fs.stat(path + "/" + filename, function(err, stats) - { - if(ERR(err, callback)) return; - - //get the modification time - var modificationTime = stats.mtime.getTime(); - - //compare the modification time to the highest found - if(modificationTime > latestModification) - { - latestModification = modificationTime; - } - - callback(); - }); - }, callback); - }); - }, callback); + function (callback) { + lastModifiedDate(function (date) { + latestModification = date; + callback() + }); }, function(callback) { @@ -280,6 +248,46 @@ function getAceFile(callback) { }); } +function lastModifiedDate(callback) { + var folders2check = [CSS_DIR, JS_DIR]; + var latestModification = 0; + //go trough this two folders + async.forEach(folders2check, function(path, callback) + { + //read the files in the folder + fs.readdir(path, function(err, files) + { + if(ERR(err, callback)) return; + + //we wanna check the directory itself for changes too + files.push("."); + + //go trough all files in this folder + async.forEach(files, function(filename, callback) + { + //get the stat data of this file + fs.stat(path + "/" + filename, function(err, stats) + { + if(ERR(err, callback)) return; + + //get the modification time + var modificationTime = stats.mtime.getTime(); + + //compare the modification time to the highest found + if(modificationTime > latestModification) + { + latestModification = modificationTime; + } + + callback(); + }); + }, callback); + }); + }, function () { + callback(latestModification); + }); +} + exports.requireDefinition = requireDefinition; function requireDefinition() { return 'var require = ' + RequireKernel.kernelSource + ';\n'; From c2669360d1f6333066af95bc5c8420d190cf239a Mon Sep 17 00:00:00 2001 From: Chad Weider Date: Sun, 29 Jan 2012 00:00:11 -0800 Subject: [PATCH 3/9] Catch 304's early. --- node/utils/Minify.js | 31 +++++++++++++++++++------------ 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/node/utils/Minify.js b/node/utils/Minify.js index 8032c586f343..18a6609b2ee4 100644 --- a/node/utils/Minify.js +++ b/node/utils/Minify.js @@ -69,21 +69,28 @@ exports.minifyJS = function(req, res, next) function _handle(req, res, jsFilename, jsFiles) { res.header("Content-Type","text/javascript"); - - //minifying is enabled - if(settings.minify) + + lastModifiedDate(function (date) { + date = new Date(date); + res.setHeader('last-modified', date.toUTCString()); + res.setHeader('date', (new Date()).toUTCString()); + + if (new Date(req.headers['if-modified-since']) >= date) { + res.writeHead(304, {}); + res.end(); + } else if (settings.minify) { + respondMinified(); + } else { + respondRaw(); + } + }); + + function respondMinified() { var result = undefined; - var latestModification = 0; + var latestModification = new Date(res.getHeader('last-modified')); async.series([ - //find out the highest modification date - function (callback) { - lastModifiedDate(function (date) { - latestModification = date; - callback() - }); - }, function(callback) { //check the modification time of the minified js @@ -169,7 +176,7 @@ function _handle(req, res, jsFilename, jsFiles) { }) } //minifying is disabled, so put the files together in one file - else + function respondRaw() { tarCode( jsFiles From a09e208b0a5683c923ab164d2418ea96beede29d Mon Sep 17 00:00:00 2001 From: Chad Weider Date: Sun, 29 Jan 2012 12:54:12 -0800 Subject: [PATCH 4/9] Return 404's. --- node/utils/Minify.js | 39 ++++++++++++++++++++++++--------------- 1 file changed, 24 insertions(+), 15 deletions(-) diff --git a/node/utils/Minify.js b/node/utils/Minify.js index 18a6609b2ee4..93bb263f80ba 100644 --- a/node/utils/Minify.js +++ b/node/utils/Minify.js @@ -57,13 +57,7 @@ exports.minifyJS = function(req, res, next) } else { // Not in tar list, but try anyways, if it fails, pass to `next`. jsFiles = [jsFilename]; - fs.stat(JS_DIR + jsFilename, function (error, stats) { - if (error || !stats.isFile()) { - next(); - } else { - _handle(req, res, jsFilename, jsFiles); - } - }); + _handle(req, res, jsFilename, jsFiles); } } @@ -75,14 +69,29 @@ function _handle(req, res, jsFilename, jsFiles) { res.setHeader('last-modified', date.toUTCString()); res.setHeader('date', (new Date()).toUTCString()); - if (new Date(req.headers['if-modified-since']) >= date) { - res.writeHead(304, {}); - res.end(); - } else if (settings.minify) { - respondMinified(); - } else { - respondRaw(); - } + fs.stat(JS_DIR + jsFiles[0], function (error, stats) { + if (error) { + if (error.code == "ENOENT") { + res.writeHead(404, {}); + res.end(); + } else { + res.writeHead(500, {}); + res.end(); + } + } else if (!stats.isFile()) { + res.writeHead(404, {}); + res.end(); + } else if (new Date(req.headers['if-modified-since']) >= date) { + res.writeHead(304, {}); + res.end(); + } else { + if (settings.minify) { + respondMinified(); + } else { + respondRaw(); + } + } + }); }); function respondMinified() From a5365f254748b37eaa1f646a47112dd57d29047c Mon Sep 17 00:00:00 2001 From: Chad Weider Date: Sun, 29 Jan 2012 12:57:49 -0800 Subject: [PATCH 5/9] Support GET and HEAD. --- node/server.js | 2 +- node/utils/Minify.js | 14 +++++++++++--- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/node/server.js b/node/server.js index a6a57497aee9..e25127f49a2d 100644 --- a/node/server.js +++ b/node/server.js @@ -155,7 +155,7 @@ async.waterfall([ }); //serve minified files - app.get('/minified/:filename', minify.minifyJS); + app.all('/minified/:filename', minify.minifyJS); //checks for padAccess function hasPadAccess(req, res, callback) diff --git a/node/utils/Minify.js b/node/utils/Minify.js index 93bb263f80ba..23d3d87991ab 100644 --- a/node/utils/Minify.js +++ b/node/utils/Minify.js @@ -85,10 +85,18 @@ function _handle(req, res, jsFilename, jsFiles) { res.writeHead(304, {}); res.end(); } else { - if (settings.minify) { - respondMinified(); + if (req.method == 'HEAD') { + res.writeHead(200, {}); + res.end(); + } else if (req.method == 'GET') { + if (settings.minify) { + respondMinified(); + } else { + respondRaw(); + } } else { - respondRaw(); + res.writeHead(405, {'allow': 'HEAD, GET'}); + res.end(); } } }); From 1fe9d2a21eb860a3c78701084e85327b4a91c75e Mon Sep 17 00:00:00 2001 From: Chad Weider Date: Mon, 6 Feb 2012 23:04:02 -0800 Subject: [PATCH 6/9] Allow maxAge of server to be specified in a setting. --- node/server.js | 3 +-- node/utils/Settings.js | 5 +++++ settings.json.template | 4 ++++ 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/node/server.js b/node/server.js index e25127f49a2d..9bd97547158e 100644 --- a/node/server.js +++ b/node/server.js @@ -61,8 +61,7 @@ console.log("Report bugs at https://github.com/Pita/etherpad-lite/issues") var serverName = "Etherpad-Lite " + version + " (http://j.mp/ep-lite)"; -//cache 6 hours -exports.maxAge = 1000*60*60*6; +exports.maxAge = settings.maxAge; //set loglevel log4js.setGlobalLogLevel(settings.loglevel); diff --git a/node/utils/Settings.js b/node/utils/Settings.js index e95702112d17..dcab13cc0795 100644 --- a/node/utils/Settings.js +++ b/node/utils/Settings.js @@ -55,6 +55,11 @@ exports.requireSession = false; */ exports.editOnly = false; +/** + * Max age that responses will have (affects caching layer). + */ +exports.maxAge = 1000*60*60*6; // 6 hours + /** * A flag that shows if minification is enabled or not */ diff --git a/settings.json.template b/settings.json.template index 7cd0e8195cf4..57bc5711b5c8 100644 --- a/settings.json.template +++ b/settings.json.template @@ -38,6 +38,10 @@ /* if true, all css & js will be minified before sending to the client. This will improve the loading performance massivly, but makes it impossible to debug the javascript/css */ "minify" : true, + + /* How long may clients use served javascript code? Without versioning this + is may cause problems during deployment. */ + "maxAge" : 1000*60*60*6, // 6 hours /* This is the path to the Abiword executable. Setting it to null, disables abiword. Abiword is needed to enable the import/export of pads*/ From 43f4184e8d5839c3c33643dae0af679f6a7285be Mon Sep 17 00:00:00 2001 From: Chad Weider Date: Sat, 11 Feb 2012 16:03:44 -0800 Subject: [PATCH 7/9] Minify specifies maxAge. --- node/utils/Minify.js | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/node/utils/Minify.js b/node/utils/Minify.js index 23d3d87991ab..fd3517326ba1 100644 --- a/node/utils/Minify.js +++ b/node/utils/Minify.js @@ -68,6 +68,11 @@ function _handle(req, res, jsFilename, jsFiles) { date = new Date(date); res.setHeader('last-modified', date.toUTCString()); res.setHeader('date', (new Date()).toUTCString()); + if (server.maxAge) { + var expiresDate = new Date((new Date()).getTime() + server.maxAge*1000); + res.setHeader('expires', expiresDate.toUTCString()); + res.setHeader('cache-control', 'max-age=' + server.maxAge); + } fs.stat(JS_DIR + jsFiles[0], function (error, stats) { if (error) { From bb10f026ca5eb2907d1693b4b79e5a314d1f6b4d Mon Sep 17 00:00:00 2001 From: Chad Weider Date: Sun, 29 Jan 2012 00:42:54 -0800 Subject: [PATCH 8/9] Create CachingMiddleware for zipping and caching. --- node/server.js | 4 +- node/utils/Minify.js | 100 +++--------------- node/utils/caching_middleware.js | 175 +++++++++++++++++++++++++++++++ 3 files changed, 190 insertions(+), 89 deletions(-) create mode 100644 node/utils/caching_middleware.js diff --git a/node/server.js b/node/server.js index 9bd97547158e..0e61916c45a8 100644 --- a/node/server.js +++ b/node/server.js @@ -31,6 +31,7 @@ var async = require('async'); var express = require('express'); var path = require('path'); var minify = require('./utils/Minify'); +var CachingMiddleware = require('./utils/caching_middleware'); var formidable = require('formidable'); var apiHandler; var exportHandler; @@ -154,7 +155,8 @@ async.waterfall([ }); //serve minified files - app.all('/minified/:filename', minify.minifyJS); + var assetCache = new CachingMiddleware; + app.all('/minified/:filename', assetCache.handle, minify.minifyJS); //checks for padAccess function hasPadAccess(req, res, callback) diff --git a/node/utils/Minify.js b/node/utils/Minify.js index fd3517326ba1..52f92513686f 100644 --- a/node/utils/Minify.js +++ b/node/utils/Minify.js @@ -27,16 +27,12 @@ var cleanCSS = require('clean-css'); var jsp = require("uglify-js").parser; var pro = require("uglify-js").uglify; var path = require('path'); -var Buffer = require('buffer').Buffer; -var zlib = require('zlib'); var RequireKernel = require('require-kernel'); var server = require('../server'); -var os = require('os'); var ROOT_DIR = path.normalize(__dirname + "/../" ); var JS_DIR = ROOT_DIR + '../static/js/'; var CSS_DIR = ROOT_DIR + '../static/css/'; -var CACHE_DIR = ROOT_DIR + '../var/'; var TAR_PATH = path.join(__dirname, 'tar.json'); var tar = JSON.parse(fs.readFileSync(TAR_PATH, 'utf8')); @@ -110,96 +106,24 @@ function _handle(req, res, jsFilename, jsFiles) { function respondMinified() { var result = undefined; - var latestModification = new Date(res.getHeader('last-modified')); - - async.series([ - function(callback) - { - //check the modification time of the minified js - fs.stat(CACHE_DIR + "/minified_" + jsFilename, function(err, stats) - { - if(err && err.code != "ENOENT") - { - ERR(err, callback); - return; - } - - //there is no minfied file or there new changes since this file was generated, so continue generating this file - if((err && err.code == "ENOENT") || stats.mtime.getTime() < latestModification) - { - callback(); - } - //the minified file is still up to date, stop minifying - else - { - callback("stop"); - } - }); - }, - //load all js files - function (callback) - { - var values = []; - tarCode( - jsFiles - , function (content) {values.push(content)} - , function (err) { - if(ERR(err)) return; - - result = values.join(''); - callback(); - }); - }, - //put all together and write it into a file - function(callback) - { - async.parallel([ - //write the results plain in a file - function(callback) - { - fs.writeFile(CACHE_DIR + "minified_" + jsFilename, result, "utf8", callback); - }, - //write the results compressed in a file - function(callback) - { - zlib.gzip(result, function(err, compressedResult){ - //weird gzip bug that returns 0 instead of null if everything is ok - err = err === 0 ? null : err; - - if(ERR(err, callback)) return; - - fs.writeFile(CACHE_DIR + "minified_" + jsFilename + ".gz", compressedResult, callback); - }); - } - ],callback); - } - ], function(err) - { - if(err && err != "stop") - { + var values = []; + res.writeHead(200, {}); + tarCode( + jsFiles + , function (content) {values.push(content)} + , function (err) { if(ERR(err)) return; + + result = values.join(''); + res.write(result); + res.end(); } - - //check if gzip is supported by this browser - var gzipSupport = req.header('Accept-Encoding', '').indexOf('gzip') != -1; - - var pathStr; - if(gzipSupport && os.type().indexOf("Windows") == -1) - { - pathStr = path.normalize(CACHE_DIR + "minified_" + jsFilename + ".gz"); - res.header('Content-Encoding', 'gzip'); - } - else - { - pathStr = path.normalize(CACHE_DIR + "minified_" + jsFilename ); - } - - res.sendfile(pathStr, { maxAge: server.maxAge }); - }) + ); } //minifying is disabled, so put the files together in one file function respondRaw() { + res.writeHead(200, {}); tarCode( jsFiles , function (content) {res.write(content)} diff --git a/node/utils/caching_middleware.js b/node/utils/caching_middleware.js new file mode 100644 index 000000000000..e1d8677eaf82 --- /dev/null +++ b/node/utils/caching_middleware.js @@ -0,0 +1,175 @@ +/* + * 2011 Peter 'Pita' Martischka (Primary Technology Ltd) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS-IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +var async = require('async'); +var Buffer = require('buffer').Buffer; +var fs = require('fs'); +var path = require('path'); +var server = require('../server'); +var zlib = require('zlib'); +var util = require('util'); + +var ROOT_DIR = path.normalize(__dirname + "/../"); +var CACHE_DIR = ROOT_DIR + '../var/'; + +var responseCache = {}; + +/* + This caches and compresses 200 and 404 responses to GET and HEAD requests. + TODO: Caching and compressing are solved problems, a middleware configuration + should replace this. +*/ + +function CachingMiddleware() { +} +CachingMiddleware.prototype = new function () { + function handle(req, res, next) { + if (!(req.method == "GET" || req.method == "HEAD")) { + return next(undefined, req, res); + } + + var old_req = {}; + var old_res = {}; + + var supportsGzip = + req.header('Accept-Encoding', '').indexOf('gzip') != -1; + + var path = require('url').parse(req.url).path; + var cacheKey = (new Buffer(path)).toString('base64').replace(/[\/\+=]/g, ''); + + fs.stat(CACHE_DIR + 'minified_' + cacheKey, function (error, stats) { + var modifiedSince = (req.headers['if-modified-since'] + && new Date(req.headers['if-modified-since'])); + var lastModifiedCache = stats && stats.mtime; + if (lastModifiedCache) { + req.headers['if-modified-since'] = lastModifiedCache.toUTCString(); + } else { + delete req.headers['if-modified-since']; + } + + // Always issue get to downstream. + old_req.method = req.method; + req.method = 'GET'; + + var expirationDate = new Date(((responseCache[cacheKey] || {}).headers || {})['expires']); + if (expirationDate > new Date()) { + // Our cached version is still valid. + return respond(); + } + + var _headers = {}; + old_res.setHeader = res.setHeader; + res.setHeader = function (key, value) { + _headers[key.toLowerCase()] = value; + old_res.setHeader.call(res, key, value); + }; + + old_res.writeHead = res.writeHead; + res.writeHead = function (status, headers) { + var lastModified = (res.getHeader('last-modified') + && new Date(res.getHeader('last-modified'))); + + res.writeHead = old_res.writeHead; + if (status == 200 || status == 404) { + // Update cache + var buffer = ''; + + Object.keys(headers).forEach(function (key) { + res.setHeader(key, headers[key]); + }); + headers = _headers; + responseCache[cacheKey] = {statusCode: status, headers: headers}; + + old_res.write = res.write; + old_res.end = res.end; + res.write = function(data, encoding) { + buffer += data.toString(encoding); + }; + res.end = function(data, encoding) { + async.parallel([ + function (callback) { + var path = CACHE_DIR + 'minified_' + cacheKey; + fs.writeFile(path, buffer, function (error, stats) { + callback(); + }); + } + , function (callback) { + var path = CACHE_DIR + 'minified_' + cacheKey + '.gz'; + zlib.gzip(buffer, function(error, content) { + if (error) { + callback(); + } else { + fs.writeFile(path, content, function (error, stats) { + callback(); + }); + } + }); + } + ], respond); + }; + } else if (status == 304) { + // Nothing new changed from the cached version. + old_res.write = res.write; + old_res.end = res.end; + res.write = function(data, encoding) {}; + res.end = function(data, encoding) { respond() }; + } else { + res.writeHead(status, headers); + } + }; + + next(undefined, req, res); + + // This handles read/write synchronization as well as its predecessor, + // which is to say, not at all. + // TODO: Implement locking on write or ditch caching of gzip and use + // existing middlewares. + function respond() { + req.method = old_req.method || req.method; + res.write = old_res.write || res.write; + res.end = old_res.end || res.end; + + var headers = responseCache[cacheKey].headers; + var statusCode = responseCache[cacheKey].statusCode; + + var pathStr = CACHE_DIR + 'minified_' + cacheKey; + if (supportsGzip) { + pathStr = pathStr + '.gz'; + headers['content-encoding'] = 'gzip'; + } + + var lastModified = (headers['last-modified'] + && new Date(headers['last-modified'])); + + if (statusCode == 200 && lastModified <= modifiedSince) { + res.writeHead(304, headers); + res.end(); + } else if (req.method == 'GET') { + var readStream = fs.createReadStream(pathStr); + res.writeHead(statusCode, headers); + util.pump(readStream, res); + } else { + res.writeHead(200, headers); + res.end(); + } + } + }); + } + + this.handle = handle; +}(); + +module.exports = CachingMiddleware; From b661ef5960fd0d22a069b72d83892418c34e569d Mon Sep 17 00:00:00 2001 From: Chad Weider Date: Tue, 31 Jan 2012 00:57:50 -0800 Subject: [PATCH 9/9] Inline response generation. --- node/utils/Minify.js | 46 +++++++++++--------------------------------- 1 file changed, 11 insertions(+), 35 deletions(-) diff --git a/node/utils/Minify.js b/node/utils/Minify.js index 52f92513686f..f54dc4d3a045 100644 --- a/node/utils/Minify.js +++ b/node/utils/Minify.js @@ -90,11 +90,17 @@ function _handle(req, res, jsFilename, jsFiles) { res.writeHead(200, {}); res.end(); } else if (req.method == 'GET') { - if (settings.minify) { - respondMinified(); - } else { - respondRaw(); - } + res.writeHead(200, {}); + tarCode( + jsFiles + , function (content) { + res.write(content); + } + , function (err) { + if(ERR(err)) return; + res.end(); + } + ); } else { res.writeHead(405, {'allow': 'HEAD, GET'}); res.end(); @@ -102,36 +108,6 @@ function _handle(req, res, jsFilename, jsFiles) { } }); }); - - function respondMinified() - { - var result = undefined; - var values = []; - res.writeHead(200, {}); - tarCode( - jsFiles - , function (content) {values.push(content)} - , function (err) { - if(ERR(err)) return; - - result = values.join(''); - res.write(result); - res.end(); - } - ); - } - //minifying is disabled, so put the files together in one file - function respondRaw() - { - res.writeHead(200, {}); - tarCode( - jsFiles - , function (content) {res.write(content)} - , function (err) { - if(ERR(err)) return; - res.end(); - }); - } } // find all includes in ace.js and embed them.