Merge pull request #174 from SergioCrisostomo/restore-hash

Restore builder hashes.

.gitignore

@@ -2,3 +2,4 @@ node_modules/
 *.DS_Store
 *.tern-port
 config/api_keys.json
+config/databases.json

README.md

@@ -8,6 +8,7 @@ To install everything you need to clone the repository and submodules (for the d
 	cd website
 	npm install
 	cp config/api_keys.sample.json config/api_keys.json
+	cp config/databases.sample.json config/databases.json
 	node build/all
 	node index
 
@@ -28,7 +29,7 @@ To run it, you can either use node directly.
 	node index
 
 ```
-Now the website is running on `http://localhost:3000`. 
+Now the website is running on `http://localhost:3000`.
 
 Alternatively use `forever` or use `supervisor`. `supervisor` watches all files, while wrapup/stylus write to the `public` folder. It is advised to use `supervisor --ignore public index`.
 

builder/index.js

@@ -1,66 +1,93 @@
 "use strict";
 
+var async = require('async');
 var path = require('path');
 var express = require('express');
 var UglifyJS = require('uglify-js');
 var packager = require('mootools-packager');
 var getFiles = require('../lib/getFiles');
 var projectPath = require('../lib/projectPath');
 var bodyParser = require('body-parser');
-
+var pkgProjects = require('../package.json')._projects;
+var builderHash = require('../lib/BuilderHash')(require('../config/databases.json'));
 var copyright = '/* MooTools: the javascript framework. license: MIT-style license. copyright: Copyright (c) 2006-' + new Date().getFullYear() + ' [Valerio Proietti](http://mad4milk.net/).*/ ';
-var allVersions = require('../package.json');
 
 function uglify(source){
 	var uglifyed = UglifyJS.minify(source, {
 		fromString : true,
 		mangle: ['sort'] // to assign shorter names to most frequently used variables.
 	});
-	return copyright + uglifyed.code;
+	return uglifyed.code;
 }
 
-function processPost(req, res){
+function processPost(req, res, next){
 
 	var postData = req.body;
 	var minified = postData.minified;
 	var project = postData.project;
-	var version = allVersions._projects[project.toLowerCase()].versions[0];
+	var project_ = project.toLowerCase();
 
-	var corePath = projectPath('core', version);
-	var morePath = projectPath('more', version);
+	if (!pkgProjects[project_]) {
+		var e = new Error('This project does not exist');
+		e.status = 404;
+		next(e);
+		return;
+	}
 
+	var version = pkgProjects[project_].versions[0];
 	var modules = !postData.modules ? [project + '/*'] : postData.modules;
-	var packagerOptions = {
-		name: {
-			Core: corePath,
-			More: morePath
-		},
-		noOutput: true,
-		callback: stream,
-		removeCoreDependencies: postData.removeCoreDependencies
-	};
-
-	if (modules.length) packagerOptions.only = modules;
-	if (!postData.compat) packagerOptions.strip = ['.*compat'];
-
-	// get all files and send to packager
-	var sourceFiles = [corePath, morePath].reduce(function(files, folder){
-		var folderPath = path.join(__dirname, '/../', folder);
-		return getFiles(folderPath, files, '.js');
-	}, []);
-
-	// compile files
-	packager(sourceFiles, packagerOptions);
-
-	// callback from packager
-	function stream(data){
+
+	function packageFile(cb) {
+
+		var corePath = projectPath('core', version);
+		var morePath = projectPath('more', version);
+
+		var packagerOptions = {
+			name: {
+				Core: corePath,
+				More: morePath
+			},
+			noOutput: true,
+			callback: function(data){
+				cb(null, data);
+			},
+			removeCoreDependencies: postData.removeCoreDependencies
+		};
+
+		if (modules.length) packagerOptions.only = modules;
+		if (!postData.compat) packagerOptions.strip = ['.*compat'];
+
+		// get all files and send to packager
+		var sourceFiles = [corePath, morePath].reduce(function(files, folder){
+			var folderPath = path.join(__dirname, '/../', folder);
+			return getFiles(folderPath, files, '.js');
+		}, []);
+
+		// compile files
+		packager(sourceFiles, packagerOptions);
+	}
+
+	function hash(cb){
+		if (!modules || minified) return cb();
+		builderHash.save(project_, modules, cb);
+	}
+
+	async.parallel([packageFile, hash], function(err, results){
+		if (err) return next(err);
+
 		var filename = ['MooTools-', project, '-', version, (postData.compat ? '-compat' : '') + (minified ? '-compressed' : ''), '.js'].join('');
+		var data = results[0];
 		if (minified) data = uglify(data);
+
+		data = copyright + '\n' +
+			(results[1] ? '/*\nWeb Build: http://mootools.net/' + project_ + '/builder/' + results[1].hash + '\n*/\n' : '') +
+			data;
+
 		res.setHeader('Content-Type', 'application/javascript');
 		res.setHeader('Content-Disposition', 'attachment; filename=' + filename);
 		res.write(data);
 		res.end();
-	}
+	});
 }
 
 module.exports = function(app){

config/databases.sample.json

@@ -0,0 +1,4 @@
+{
+  "core": "tests/database/core.db",
+  "more": "tests/database/more.db"
+}

core/index.js

@@ -1,15 +1,20 @@
 "use strict";
 
-var docs = require('../middleware/docs')('core', {
+var project = 'core';
+
+var docs = require('../middleware/docs')(project, {
 	title: "MooTools Core Documentation"
 });
 
-var guides = require('../middleware/guides')('core', {
+var guides = require('../middleware/guides')(project, {
 	title: "MooTools Core Guides"
 });
 
-var project = 'core';
-var versions = require('../package.json')._projects[project].versions;
+var hash = require('../middleware/buildHash')(project);
+
+var pkgProject = require('../package.json')._projects[project];
+var versions = pkgProject.versions;
+
 var links = versions.slice(1).map(function(version){
 	return {
 		version: version,
@@ -30,7 +35,6 @@ module.exports = function(app){
 	};
 
 	app.get('/core', core, function(req, res){
-
 		res.render('core/index', {
 			page: "/core",
 			title: "MooTools Core",
@@ -41,13 +45,14 @@ module.exports = function(app){
 		});
 	});
 
-	app.get('/core/builder', function(req, res){
+	app.get('/core/builder/:hash?', hash, function(req, res){
 		res.render('builder/index', {
 			title: 'MooTools Core Builder',
 			navigation: 'core',
 			page: 'builder',
 			project: 'Core',
 			site: 'core',
+			hashDependencies: res.locals.hash || [],
 			version: versions[0],
 			versions: links,
 			dependencies: require('../builder/dependencies.js')(project, versions[0])
@@ -61,4 +66,9 @@ module.exports = function(app){
 	app.get('/core/guides', core, guides.index);
 	app.get('/core/guides/:guide', core, guides.article);
 
+	// hash build redirect
+	app.get('/core/:hash', function(req, res){
+		res.redirect('/core/builder/' + req.params.hash);
+	});
+
 };

index.js

@@ -154,17 +154,14 @@ app.use(function(err, req, res, next){
 	});
 });
 
-app.use(function(err, req, res, next){
-	res.status(500);
-	res.render('errors/500', {
-		site: 'mootools'
+if (app.get('env') != 'development'){
+	app.use(function(err, req, res, next){
+		res.status(500);
+		res.render('errors/500', {
+			site: 'mootools'
+		});
 	});
-});
-
-// general error handler
-app.use(function(err){
-	console.error(err);
-});
+}
 
 // starting server
 app.listen(app.get('port'), function(){

lib/BuilderHash.js

@@ -0,0 +1,83 @@
+"use strict";
+
+var path = require('path');
+var async = require('async');
+var sqlite3 = require('sqlite3');
+var md5 = require('md5');
+var waitForIt = require('../lib/waitForIt');
+
+function BuilderDatabase(paths){
+	this.paths = paths;
+	this.databases = {};
+}
+
+BuilderDatabase.prototype.getDatabase = function(project, callback){
+	var getDB = this.databases[project];
+	if (getDB){
+		return getDB.get(callback);
+	}
+	var dbPath = path.join(__dirname, '..', this.paths[project]);
+	if (!path){
+		return callback(Error('No database found for "' + project + '".'));
+	}
+	getDB = waitForIt(function(cb){
+		var db = new sqlite3.Database(dbPath, sqlite3.OPEN_READWRITE, function(error){
+			if (error) cb(error);
+			else cb(null, db);
+		});
+	});
+	getDB.get(callback);
+};
+
+BuilderDatabase.prototype.loadHash = function(project, hash, callback){
+	function getHash(db, cb){
+		db.get('SELECT * FROM hashes WHERE md5 = ?', {1: hash}, function(err, row){
+			cb(err, row ? {hash: row.md5, packages: row.packages.split(';')} : null);
+		});
+	}
+	async.compose(
+		getHash,
+		this.getDatabase.bind(this, project)
+	)(callback);
+};
+
+BuilderDatabase.prototype.saveHash = function(project, packages, callback){
+	if (!packages || !packages.length){
+		if (callback) callback(null);
+		return;
+	}
+
+	var packageString = typeof packages == 'string' ? packages : packages.join(';');
+	var hash = md5.digest_s(packageString);
+
+	function hashCount(db, cb){
+		db.get('SELECT COUNT(*) AS count FROM hashes WHERE md5 = ?', {1: hash}, function(err, row){
+			cb(err, row && {count: row.count, db: db});
+		});
+	}
+	function insertIfNotExisting(res, cb){
+		if (res.hash){
+			cb(null, {hash: hash, packages: packages});
+		} else {
+			var values = {1: hash, 2: packageString, 3: Math.round(Date.now() / 1000)};
+			res.db.run('INSERT INTO hashes (md5, packages, date) VALUES (?, ?, ?)', values, function(error){
+				if (error) cb(error);
+				else cb(null, {hash: hash, packages: packages});
+			});
+		}
+	}
+
+	async.compose(
+		insertIfNotExisting,
+		hashCount,
+		this.getDatabase.bind(this, project)
+	)(callback);
+};
+
+module.exports = function(paths){
+	var db = new BuilderDatabase(paths);
+	return {
+		load: db.loadHash.bind(db),
+		save: db.saveHash.bind(db)
+	};
+};

middleware/buildHash.js

@@ -0,0 +1,20 @@
+"use strict";
+
+var builderHash = require('../lib/BuilderHash')(require('../config/databases.json'));
+
+module.exports = function(project){
+
+	return function(req, res, next){
+		var hash = req.params.hash;
+		if (hash){
+			builderHash.load(project, hash, function(err, data) {
+				if (data){
+					res.locals.hash = data.packages;
+				}
+				next(err);
+			});
+		} else {
+			return next();
+		}
+	};
+};

more/index.js

@@ -1,15 +1,19 @@
 "use strict";
 
-var docs = require('../middleware/docs')('more', {
+var project = 'more';
+
+var docs = require('../middleware/docs')(project, {
 	title: "MooTools More Documentation"
 });
 
-var guides = require('../middleware/guides')('more', {
+var guides = require('../middleware/guides')(project, {
 	title: "MooTools More Guides"
 });
 
-var project = 'more';
-var lastVersion = require('../package.json')._projects[project].versions[0];
+var hash = require('../middleware/buildHash')(project);
+
+var pkgProject = require('../package.json')._projects[project];
+var lastVersion = pkgProject.versions[0];
 
 module.exports = function(app){
 
@@ -28,13 +32,14 @@ module.exports = function(app){
 		});
 	});
 
-	app.get('/more/builder', function(req, res){
+	app.get('/more/builder/:hash?', hash, function(req, res){
 		res.render('builder/index', {
 			title: 'MooTools More Builder',
 			navigation: 'more',
 			page: 'builder',
 			project: 'More',
 			site: 'more',
+			hashDependencies: res.locals.hash || [],
 			version: lastVersion,
 			dependencies: require('../builder/dependencies.js')(project, lastVersion)
 		});
@@ -47,4 +52,9 @@ module.exports = function(app){
 	app.get('/more/guides', more, guides.index);
 	app.get('/more/guides/:guide', more, guides.article);
 
+	// hash build redirect
+	app.get('/more/:hash', function(req, res){
+		res.redirect('/more/builder/' + req.params.hash);
+	});
+
 };