From 58defba3301e8c889b545fbebd1e9a0ace5c73c2 Mon Sep 17 00:00:00 2001 From: Stefan Penner Date: Sat, 11 Apr 2015 12:40:48 -0700 Subject: [PATCH 1/7] initial commit --- .gitignore | 1 + index.js | 40 ++++++++++++++++++++++++++++++++++++++++ package.json | 27 +++++++++++++++++++++++++++ 3 files changed, 68 insertions(+) create mode 100644 .gitignore create mode 100644 index.js create mode 100644 package.json diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..3c3629e --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +node_modules diff --git a/index.js b/index.js new file mode 100644 index 0000000..5281cfe --- /dev/null +++ b/index.js @@ -0,0 +1,40 @@ +var Filter = require('broccoli-filter'); +var Cache = require('async-disk-cache'); +var crypto = require('crypto'); +var fs = require('fs'); + +module.exports = PersistentFilter; + +function PersistentFilter(inputTree, options) { + Filter.call(this, inputTree, options); + this.cache = new Cache(this.cacheKey()); +} + +PersistentFilter.prototype = Object.create(Filter.prototype); + +PersistentFilter.prototype.cacheKey = function() { + // this will be have to be derived from the checksum of the dependencies + return 'persistent-filter-3'; +}; + +PersistentFilter.prototype.cacheKeyProcessString = function(string, relativePath) { + return crypto.createHash('md5').update(string).digest('hex'); +}; + +PersistentFilter.prototype.processFile = function(srcDir, destDir, relativePath) { + var filter = this; + var inputEncoding = (this.inputEncoding === undefined) ? 'utf8' : this.inputEncoding; + var outputEncoding = (this.outputEncoding === undefined) ? 'utf8' : this.outputEncoding; + var string = fs.readFileSync(srcDir + '/' + relativePath, { encoding: inputEncoding }); + var cache = this.cache; + var key = this.cacheKeyProcessString(string, relativePath); + + return cache.get(key).then(function(entry) { + return entry.isCached ? entry.value : filter.processString(string, relativePath); + }).then(function(outputString) { + var outputPath = filter.getDestFilePath(relativePath); + fs.writeFileSync(destDir + '/' + outputPath, outputString, { encoding: outputEncoding }); + + return cache.set(key, outputString); + }); +}; diff --git a/package.json b/package.json new file mode 100644 index 0000000..c3b4cd3 --- /dev/null +++ b/package.json @@ -0,0 +1,27 @@ +{ + "name": "broccoli-persistent-filter", + "version": "0.0.1", + "description": "broccoli filter but with a persistent cache", + "main": "index.js", + "scripts": { + "test": "npm test" + }, + "repository": { + "type": "git", + "url": "git+https://github.com/stefanpenner/broccoli-persistent-filter.git" + }, + "keywords": [ + "broccoli", + "broccoli-plugin" + ], + "author": "Stefan Penner ", + "license": "ISC", + "bugs": { + "url": "https://github.com/stefanpenner/broccoli-persistent-filter/issues" + }, + "homepage": "https://github.com/stefanpenner/broccoli-persistent-filter#readme", + "dependencies": { + "async-disk-cache": "0.0.1", + "broccoli-filter": "^0.1.12" + } +} From 4e164d485f7da8f9ad4a0c3075977304ffcf89a1 Mon Sep 17 00:00:00 2001 From: Stefan Penner Date: Sat, 11 Apr 2015 21:58:15 -0700 Subject: [PATCH 2/7] some inline docs --- index.js | 86 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 86 insertions(+) diff --git a/index.js b/index.js index 5281cfe..371b83f 100644 --- a/index.js +++ b/index.js @@ -5,6 +5,68 @@ var fs = require('fs'); module.exports = PersistentFilter; +/* + * @public + * + * `broccoli-persistent-filter` is broccoli-filter but it is able to persit + * state across restarts. This exists to mitigate the upfront cost of some more + * expensive transforms on warm boot. + * + * Why isn't this the default behaviour? + * + * Deriving the correct cache key for a + * given filter can be tricky. In addition, this should be seen as a last + * resort, if a given filter is too slow often times it should be improved + * rather then opting for caching. + * + * What does this do? + * + * * This does not aim to improve incremental build performance, if it does, it + * should indicate something is wrong with the filter ir input filter in + * question. + * + * * This does not improve cold boot times. + * + * How does it work? + * + * It does so but establishing a 2 layer file cache. + * The first layer, is the entire bucket. The second, `cacheKeyProcessString` + * is a per file cache key. + * + * Together, these two layers should provide the right balance of speed and + * sensibility. + * + * The bucket level cacheKey must be stable but also never become stale. If the + * key is not stable, state between restarts will be lost and performance will + * suffer. On the flip-side, if the cacheKey becomes stale changes may not be + * correctly reflected. + * + * It is configured by subclassing and refining `cacheKey` method. A good key + * here, is likely the name of the plugin, its version and the actual versions + * of its dependencies + * + * ```js + * Subclass.prototype.cacheKey = function() { + * return md5(Filter.prototype.call(this) + inputOptionsChecksum + dependencyVersionChecksum); + * } + * ``` + * + * The second key, represents the contents of the file. Typically the + * base-class's functionality is sufficient, as it merely generates a checksum + * of the file contents. If for some reason this is not sufficient, it can be + * re-configured via subclassing. + * + * ```js + * Subbclass.prototype.cacheKeyProcessString = function(string, relativePath) { + * return superAwsomeDigest(string); + * } + * ``` + * + * @class PersistentFilter + * @param {Tree} inputTree + * @param {Object} options + * + * */ function PersistentFilter(inputTree, options) { Filter.call(this, inputTree, options); this.cache = new Cache(this.cacheKey()); @@ -12,15 +74,39 @@ function PersistentFilter(inputTree, options) { PersistentFilter.prototype = Object.create(Filter.prototype); +/* + * @public + * + * + * @method cachKey + * @return {String} this filters top-level cache key + */ PersistentFilter.prototype.cacheKey = function() { // this will be have to be derived from the checksum of the dependencies return 'persistent-filter-3'; }; + +/* + * @public + * + * @method cacheKeyProcessString + * @return {String} this filters top-level cache key + */ PersistentFilter.prototype.cacheKeyProcessString = function(string, relativePath) { return crypto.createHash('md5').update(string).digest('hex'); }; + +/* + * @private + * + * @method processFile + * @param {String} srcDir + * @param {String} destDir + * @param {String} relativePath + * @return {Promise} + */ PersistentFilter.prototype.processFile = function(srcDir, destDir, relativePath) { var filter = this; var inputEncoding = (this.inputEncoding === undefined) ? 'utf8' : this.inputEncoding; From e33f0bcc1f5db67cc07479dadaa37c638d903493 Mon Sep 17 00:00:00 2001 From: Stefan Penner Date: Mon, 13 Apr 2015 14:47:48 -0700 Subject: [PATCH 3/7] first usage of hash-for-deps --- index.js | 3 ++- package.json | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/index.js b/index.js index 371b83f..330a0e8 100644 --- a/index.js +++ b/index.js @@ -2,6 +2,7 @@ var Filter = require('broccoli-filter'); var Cache = require('async-disk-cache'); var crypto = require('crypto'); var fs = require('fs'); +var hashForDep = require('hash-for-dep'); module.exports = PersistentFilter; @@ -83,7 +84,7 @@ PersistentFilter.prototype = Object.create(Filter.prototype); */ PersistentFilter.prototype.cacheKey = function() { // this will be have to be derived from the checksum of the dependencies - return 'persistent-filter-3'; + return 'persistent-filter-3' + hashForDep('./'); }; diff --git a/package.json b/package.json index c3b4cd3..8a46ade 100644 --- a/package.json +++ b/package.json @@ -22,6 +22,7 @@ "homepage": "https://github.com/stefanpenner/broccoli-persistent-filter#readme", "dependencies": { "async-disk-cache": "0.0.1", - "broccoli-filter": "^0.1.12" + "broccoli-filter": "^0.1.12", + "hash-for-dep": "0.0.1" } } From 63c06f6da6fdd432349f61cc8eac413ddcc03d29 Mon Sep 17 00:00:00 2001 From: Stefan Penner Date: Mon, 13 Apr 2015 15:19:20 -0700 Subject: [PATCH 4/7] bump version --- index.js | 15 +++++++++++---- package.json | 4 ++-- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/index.js b/index.js index 330a0e8..c9bf0af 100644 --- a/index.js +++ b/index.js @@ -2,7 +2,7 @@ var Filter = require('broccoli-filter'); var Cache = require('async-disk-cache'); var crypto = require('crypto'); var fs = require('fs'); -var hashForDep = require('hash-for-dep'); +var hashForDep = require('hash-for-dep'); module.exports = PersistentFilter; @@ -76,17 +76,24 @@ function PersistentFilter(inputTree, options) { PersistentFilter.prototype = Object.create(Filter.prototype); /* - * @public + * @private * * * @method cachKey * @return {String} this filters top-level cache key */ PersistentFilter.prototype.cacheKey = function() { - // this will be have to be derived from the checksum of the dependencies - return 'persistent-filter-3' + hashForDep('./'); + return hashForDep(this.baseDir()); }; +/* @public + * + * @method baseDir + * @returns {String} absolute path to the root of the filter... + */ +PersistentFilter.prototype.baseDir = function() { + throw Error('Filter must implement prototype.baseDir'); +}; /* * @public diff --git a/package.json b/package.json index 8a46ade..3d333b5 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "broccoli-persistent-filter", - "version": "0.0.1", + "version": "0.0.2", "description": "broccoli filter but with a persistent cache", "main": "index.js", "scripts": { @@ -23,6 +23,6 @@ "dependencies": { "async-disk-cache": "0.0.1", "broccoli-filter": "^0.1.12", - "hash-for-dep": "0.0.1" + "hash-for-dep": "0.0.3" } } From 8e86ab793a2fe6074579e96e1e7e3ef6c38e547c Mon Sep 17 00:00:00 2001 From: Igor Minar Date: Thu, 16 Apr 2015 10:01:57 -0700 Subject: [PATCH 5/7] docs: fix typos --- index.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/index.js b/index.js index c9bf0af..8fb6231 100644 --- a/index.js +++ b/index.js @@ -23,7 +23,7 @@ module.exports = PersistentFilter; * What does this do? * * * This does not aim to improve incremental build performance, if it does, it - * should indicate something is wrong with the filter ir input filter in + * should indicate something is wrong with the filter or input filter in * question. * * * This does not improve cold boot times. @@ -59,7 +59,7 @@ module.exports = PersistentFilter; * * ```js * Subbclass.prototype.cacheKeyProcessString = function(string, relativePath) { - * return superAwsomeDigest(string); + * return superAwesomeDigest(string); * } * ``` * From 7e4d5b9349699c0afb739b33455dceade8eeeec2 Mon Sep 17 00:00:00 2001 From: Alex Navasardyan Date: Thu, 4 Jun 2015 10:24:15 -0700 Subject: [PATCH 6/7] updates `async-disk-cache` to the latest version --- index.js | 4 +++- package.json | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/index.js b/index.js index 8fb6231..b11b9f7 100644 --- a/index.js +++ b/index.js @@ -70,7 +70,9 @@ module.exports = PersistentFilter; * */ function PersistentFilter(inputTree, options) { Filter.call(this, inputTree, options); - this.cache = new Cache(this.cacheKey()); + this.cache = new Cache(this.cacheKey(), { + compression: 'deflate' + }); } PersistentFilter.prototype = Object.create(Filter.prototype); diff --git a/package.json b/package.json index 3d333b5..4c21593 100644 --- a/package.json +++ b/package.json @@ -21,7 +21,7 @@ }, "homepage": "https://github.com/stefanpenner/broccoli-persistent-filter#readme", "dependencies": { - "async-disk-cache": "0.0.1", + "async-disk-cache": "1.0.0", "broccoli-filter": "^0.1.12", "hash-for-dep": "0.0.3" } From a29ef28c7a4d456dd057556da8ded8dd06559ca5 Mon Sep 17 00:00:00 2001 From: Alex Navasardyan Date: Thu, 4 Jun 2015 14:41:24 -0700 Subject: [PATCH 7/7] Introduces md5-hex --- index.js | 6 +++--- package.json | 3 ++- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/index.js b/index.js index 8fb6231..4e64b0d 100644 --- a/index.js +++ b/index.js @@ -1,6 +1,6 @@ var Filter = require('broccoli-filter'); var Cache = require('async-disk-cache'); -var crypto = require('crypto'); +var md5Hex = require('md5-hex'); var fs = require('fs'); var hashForDep = require('hash-for-dep'); @@ -77,7 +77,7 @@ PersistentFilter.prototype = Object.create(Filter.prototype); /* * @private - * + * * * @method cachKey * @return {String} this filters top-level cache key @@ -102,7 +102,7 @@ PersistentFilter.prototype.baseDir = function() { * @return {String} this filters top-level cache key */ PersistentFilter.prototype.cacheKeyProcessString = function(string, relativePath) { - return crypto.createHash('md5').update(string).digest('hex'); + return md5Hex(string); }; diff --git a/package.json b/package.json index 3d333b5..9f4af31 100644 --- a/package.json +++ b/package.json @@ -23,6 +23,7 @@ "dependencies": { "async-disk-cache": "0.0.1", "broccoli-filter": "^0.1.12", - "hash-for-dep": "0.0.3" + "hash-for-dep": "0.0.3", + "md5-hex": "^1.0.2" } }