diff --git a/README.md b/README.md index 786ffd4..0ef19f3 100644 --- a/README.md +++ b/README.md @@ -92,6 +92,48 @@ var node = new Awk('docs', 'ES6', 'ECMAScript 2015'); module.exports = node; ``` +## Persistent Cache + +__Note: This feature is experimental and is only available on Unix based systems.__ + +Adding persist flag allows a subclass to persist state across restarts. This exists to mitigate the upfront cost of some more expensive transforms on warm boot. __It does not aim to improve incremental build performance, if it does, it should indicate something is wrong with the filter or input filter in question.__ + +### How does it work? + +It does so but establishing a 2 layer file cache. The first layer, is the entire bucket. +The second, `cacheKeyProcessString` is a per file cache key. + +Together, these two layers should provide the right balance of speed and sensibility. + +The bucket level cacheKey must be stable but also never become stale. If the key is not +stable, state between restarts will be lost and performance will suffer. On the flip-side, +if the cacheKey becomes stale changes may not be correctly reflected. + +It is configured by subclassing and refining `cacheKey` method. A good key here, is +likely the name of the plugin, its version and the actual versions of its dependencies. + +```js +Subclass.prototype.cacheKey = function() { + return md5(Filter.prototype.call(this) + inputOptionsChecksum + dependencyVersionChecksum); +} +``` + +The second key, represents the contents of the file. Typically the base-class's functionality +is sufficient, as it merely generates a checksum of the file contents. If for some reason this +is not sufficient, it can be re-configured via subclassing. + +```js +Subclass.prototype.cacheKeyProcessString = function(string, relativePath) { + return superAwesomeDigest(string); +} +``` + +It is recommended that persistent re-builds is opt-in by the consumer as it does not currently work on all systems. + +```js +var myTree = new SomePlugin('lib', { persist: true }); +``` + ## FAQ ### Upgrading from 0.1.x to 1.x diff --git a/index.js b/index.js index c92a36f..9272032 100644 --- a/index.js +++ b/index.js @@ -13,9 +13,14 @@ var copyDereferenceSync = require('copy-dereference').sync; var Cache = require('./lib/cache'); var debugGenerator = require('debug'); var keyForFile = require('./lib/key-for-file'); +var PersistentCache = require('async-disk-cache'); +var md5Hex = require('md5-hex'); +var Processor = require('./lib/processor'); +var defaultProccessor = require('./lib/strategies/default'); module.exports = Filter; + Filter.prototype = Object.create(Plugin.prototype); Filter.prototype.constructor = Filter; function Filter(inputTree, options) { @@ -33,6 +38,9 @@ function Filter(inputTree, options) { Plugin.call(this, [inputTree]); + this.processor = new Processor(options); + this.processor.setStrategy(defaultProccessor); + /* Destructuring assignment in node 0.12.2 would be really handy for this! */ if (options) { if (options.extensions != null) @@ -43,8 +51,13 @@ function Filter(inputTree, options) { this.inputEncoding = options.inputEncoding; if (options.outputEncoding != null) this.outputEncoding = options.outputEncoding; + if (options.persist) { + this.processor.setStrategy(require('./lib/strategies/persistent')); + } } + this.processor.init(this); + this._cache = new Cache(); this._canProcessCache = Object.create(null); this._destFilePathCache = Object.create(null); @@ -75,6 +88,26 @@ Filter.prototype.build = function build() { }); }; +/* @public + * + * @method baseDir + * @returns {String} absolute path to the root of the filter... + */ +Filter.prototype.baseDir = function() { + throw Error('Filter must implement prototype.baseDir'); +}; + +/** + * @public + * + * optionally override this to build a more rhobust cache key + * @param {String} string The contents of a file that is being processed + * @return {String} A cache key + */ +Filter.prototype.cacheKeyProcessString = function(string /*, relativePath*/) { + return md5Hex(string); +}; + Filter.prototype.canProcessFile = function canProcessFile(relativePath) { return !!this.getDestFilePath(relativePath); @@ -94,7 +127,7 @@ Filter.prototype.getDestFilePath = function getDestFilePath(relativePath) { } } return null; -} +}; Filter.prototype.processAndCacheFile = function processAndCacheFile(srcDir, destDir, relativePath) { @@ -129,7 +162,7 @@ Filter.prototype.processAndCacheFile = e.file = relativePath; e.treeDir = srcDir; throw e; - }) + }); function copyToCache() { var entry = { @@ -161,18 +194,20 @@ Filter.prototype.processFile = var contents = fs.readFileSync( srcDir + '/' + relativePath, { encoding: inputEncoding }); - return Promise.resolve(this.processString(contents, relativePath)). - then(function asyncOutputFilteredFile(outputString) { - var outputPath = self.getDestFilePath(relativePath); - if (outputPath == null) { - throw new Error('canProcessFile("' + relativePath + '") is true, but getDestFilePath("' + relativePath + '") is null'); - } - outputPath = destDir + '/' + outputPath; - mkdirp.sync(path.dirname(outputPath)); - fs.writeFileSync(outputPath, outputString, { - encoding: outputEncoding - }); - }); + return this.processor.processString(this, contents, relativePath).then(function asyncOutputFilteredFile(result) { + var outputString = result.string; + var outputPath = self.getDestFilePath(relativePath); + if (outputPath == null) { + throw new Error('canProcessFile("' + relativePath + '") is true, but getDestFilePath("' + relativePath + '") is null'); + } + outputPath = destDir + '/' + outputPath; + mkdirp.sync(path.dirname(outputPath)); + fs.writeFileSync(outputPath, outputString, { + encoding: outputEncoding + }); + + return self.processor.done(self, result); + }); }; Filter.prototype.processString = diff --git a/lib/processor.js b/lib/processor.js new file mode 100644 index 0000000..7a12792 --- /dev/null +++ b/lib/processor.js @@ -0,0 +1,27 @@ +function Processor(options) { + options = options || {}; + this.processor = {}; + this.persistent = options.persist; +} + +Processor.prototype.setStrategy = function(stringProcessor) { + if (this.persistent && /^win/.test(process.platform)) { + console.log('Unfortunately persistent cache is currently not available on windows based systems. Please see https://github.com/stefanpenner/hash-for-dep/issues/8.'); + return; + } + this.processor = stringProcessor; +}; + +Processor.prototype.init = function(ctx) { + this.processor.init(ctx); +}; + +Processor.prototype.processString = function(ctx, contents, relativePath) { + return this.processor.processString(ctx, contents, relativePath); +}; + +Processor.prototype.done = function(ctx, result) { + return this.processor.done(ctx, result); +}; + +module.exports = Processor; diff --git a/lib/strategies/default.js b/lib/strategies/default.js new file mode 100644 index 0000000..9c26a45 --- /dev/null +++ b/lib/strategies/default.js @@ -0,0 +1,13 @@ +var Promise = require('rsvp').Promise; + +module.exports = { + init: function() {}, + + processString: function(ctx, contents, relativePath) { + return Promise.resolve({ string: ctx.processString(contents, relativePath) }); + }, + + done: function(ctx) { + return Promise.resolve(ctx.outputPath); + } +}; diff --git a/lib/strategies/persistent.js b/lib/strategies/persistent.js new file mode 100644 index 0000000..8b2f84f --- /dev/null +++ b/lib/strategies/persistent.js @@ -0,0 +1,49 @@ +'use strict'; + +var md5Hex = require('md5-hex'); +var PersistentCache = require('async-disk-cache'); +var hashForDep = require('hash-for-dep'); + +module.exports = { + + _peristentCache: {}, + + init: function(ctx) { + if (!ctx.constructor.cacheKey) { + ctx.constructor.cacheKey = this.cacheKey(ctx); + } + + this._peristentCache = new PersistentCache(ctx.constructor.cacheKey, { + compression: 'deflate' + }); + }, + + cacheKey: function(ctx) { + return hashForDep(ctx.baseDir()); + }, + + processString: function(ctx, contents, relativePath) { + var key = ctx.cacheKeyProcessString(contents, relativePath); + return this._peristentCache.get(key).then(function(entry) { + var result; + + if (entry.isCached) { + result = { + string: entry.value, + key: key + }; + } else { + result = { + string: ctx.processString(contents, relativePath), + key: key + }; + } + + return result; + }); + }, + + done: function(ctx, result) { + return this._peristentCache.set(result.key, result.string); + } +}; diff --git a/package.json b/package.json index 2bbedf0..114049c 100644 --- a/package.json +++ b/package.json @@ -25,10 +25,13 @@ "cache" ], "dependencies": { + "async-disk-cache": "^1.0.0", "broccoli-kitchen-sink-helpers": "^0.2.7", "broccoli-plugin": "^1.0.0", "copy-dereference": "^1.0.0", "debug": "^2.2.0", + "hash-for-dep": "0.0.3", + "md5-hex": "^1.0.2", "mkdirp": "^0.5.1", "promise-map-series": "^0.2.1", "rsvp": "^3.0.18", diff --git a/test/fixtures/directory/.gitkeep b/test/fixtures/directory/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/test/key-for-file-tests.js b/test/key-for-file-tests.js index 4b644fd..b73933d 100644 --- a/test/key-for-file-tests.js +++ b/test/key-for-file-tests.js @@ -8,7 +8,7 @@ describe('keyForFile', function () { describe('when given a path to a directory', function () { it('throws an error', function () { expect(function () { - keyForFile('./test/fixtures/directory'); + keyForFile('./test/fixtures/dir'); }).to.throw(/cannot diff directory/i); }); }); diff --git a/test/test.js b/test/test.js index 61217ec..ac8df74 100644 --- a/test/test.js +++ b/test/test.js @@ -45,6 +45,10 @@ ReplaceFilter.prototype.processString = function(contents, relativePath) { return result; }; +ReplaceFilter.prototype.baseDir = function() { + return '../'; +}; + function IncompleteFilter(inputTree, options) { if (!this) return new IncompleteFilter(inputTree, options); Filter.call(this, inputTree, options); @@ -290,6 +294,58 @@ describe('Filter', function() { to.equal('utf8'); }); + if (!/^win/.test(process.platform)) { + describe('persistent cache', function() { + var f; + function F(inputTree, options) { Filter.call(this, inputTree, options); } + inherits(F, Filter); + F.prototype.baseDir = function() { + return '../'; + }; + + beforeEach(function() { + f = new F(fixturePath, { persist: true }); + }); + + it('cache is initialized', function() { + expect(f.processor.processor._peristentCache).to.be.ok; + }); + + it('default `baseDir` implementation throws an Unimplemented Exception', function() { + function F(inputTree, options) { Filter.call(this, inputTree, options); } + inherits(F, Filter); + expect(function() { + new F(fixturePath, { persist: true }); + }).to.throw(/Filter must implement prototype.baseDir/); + }); + + it('`cacheKeyProcessString` return correct first level file cache', function() { + expect(f.cacheKeyProcessString('foo-bar-baz', 'relative-path')).to.eql('4c43793687f9a7170a9149ad391cbf70'); + }); + + it('filter properly reads file tree', function() { + var builder = makeBuilder(ReplaceFilter, fixturePath, function(awk) { + return awk; + }); + + return builder('dir', { + persist: true, + glob: '**/*.md', + search: 'dogs', + replace: 'cats' + }).then(function(results) { + expect(results.files).to.deep.eql([ + 'a/', + 'a/README.md', + 'a/bar/', + 'a/bar/bar.js', + 'a/foo.js' + ]); + }); + }); + }); + } + describe('processFile', function() { beforeEach(function() { sinon.spy(fs, 'mkdirSync');