diff --git a/.npmrc b/.npmrc new file mode 100644 index 0000000..43c97e7 --- /dev/null +++ b/.npmrc @@ -0,0 +1 @@ +package-lock=false diff --git a/README.md b/README.md new file mode 100644 index 0000000..ea6fb1b --- /dev/null +++ b/README.md @@ -0,0 +1,94 @@ +# action-walk + +Minimal utility to walk directory trees performing actions on each directory +entry. `action-walk` has no production dependencies other than +node core modules and has only one strong opinion - don't presume anything +about why the directory tree is being walked. + +No presumptions means that this does little more than walk the tree. There +are two options to facilitate implementing your code on top of `action-walk`. +If the boolean option `stat` is truthy `action-walk` will execute `fs.stat` +on the entry and pass that to you action handler. If the option `own` is +present `action-walk` will pass that to the action functions in a context +object. + +### usage + +`npm install action-walk` + +### examples + +``` +const walk = require('action-walk'); + +function dirAction (path, context) { + const {dirent, stat, own} = context; + if (own.skipDirs && own.skipDirs.indexOf(dirent.name) >= 0) { + return 'skip'; + } + own.total += stat.size; +} +function fileAction (path, context) { + const {stat, own} = context; + own.total += stat.size; +} + +const own = {total: 0, skipDirs: ['node_modules']}; +const options = { + dirAction, + fileAction, + own, + stat: true +}; + +await walk('.', options); + +console.log('total bytes in "."', ctx.total); + +// executed in the await-walk package root it will print something like +// total bytes in "." 14778 +``` + +see `test/basics.test.js` for another example. + +### api + +`await walk(directory, options = {})` + +options +- `dirAction` - called for each directory. +- `fileAction` - called for each file. +- `otherAction` - called for non-file, non-directory entries. +- `stat` - call `fs.stat` on the entry and add it to the action context. +- `own` - add this to the action context. + +It's possible to call `walk()` with no options but probably not useful unless +all you're wanting to do is seed the disk cache with directory entries. The +action functions are where task-specific work is done. + +Each of the action function (`dirAction`, `fileAction`, `otherAction`) is +called with two arguments: +- `filepath` for the entry starting with the `directory`, e.g., if +`directory` is `test` and the entry is `basics.test.js` then `filepath` +will be `test/basics.test.js`. (It is created using node's `path.join` so +note that if `directory` is `.` it will *not* be present in `filepath`.) +- `context` is an object as follows. +``` +{ + dirent, // the fs.Dirent object for the directory entry + stat, // if `options.stat` the object returned by `fs.stat` + own // `options.own` if provided. +} +``` + +`dirAction` is the only function with return value that matters. If +`dirAction` returns the string `'skip'` (either directly or via a +Promise) then `walk()` will not walk that branch of the directory tree. + +All the action functions can return a promise if they need to perform +asynchronous work but only the value of `dirAction` is meaningful. + +### todo + +- add error handling +- let otherAction return indicator that a symbolic link should be followed. diff --git a/action-walk.js b/action-walk.js new file mode 100644 index 0000000..63412ec --- /dev/null +++ b/action-walk.js @@ -0,0 +1,47 @@ +const fsp = require('fs').promises; +const path = require('path'); + +async function walk (dir, options = {}) { + const noop = async () => undefined; + let fileAction = noop; + let dirAction = noop; + let otherAction = noop; + + if (options.fileAction) { + fileAction = async (filepath, ctx) => options.fileAction(filepath, ctx); + } + if (options.dirAction) { + dirAction = async (filepath, ctx) => options.dirAction(filepath, ctx); + } + if (options.otherAction) { + otherAction = async (filepath, ctx) => options.otherAction(filepath, ctx); + } + + // + // walk through a directory tree calling user functions for each entry. + // + async function walker (dir) { + for await (const dirent of await fsp.opendir(dir)) { + const entry = path.join(dir, dirent.name); + const ctx = {dirent}; + if (options.own) { + ctx.own = options.own; + } + if (options.stat) { + ctx.stat = await fsp.stat(entry); + } + if (dirent.isDirectory() && await dirAction(entry, ctx) !== 'skip') { + await walker(entry); + } else if (dirent.isFile()) { + await fileAction(entry, ctx); + } else { + await otherAction(entry, ctx); + } + } + return undefined; + } + + return walker(dir); +} + +module.exports = walk; diff --git a/package.json b/package.json new file mode 100644 index 0000000..31a6efa --- /dev/null +++ b/package.json @@ -0,0 +1,31 @@ +{ + "name": "action-walk", + "version": "1.0.0", + "description": "walk a directory tree performing actions", + "main": "action-walk.js", + "directories": { + "test": "test" + }, + "scripts": { + "test": "mocha test/*.test.js" + }, + "keywords": [ + "directory", + "tree", + "action", + "iterate", + "general", + "utility", + "recursive", + "walk", + "flexible" + ], + "author": "Bruce A. MacNaughton", + "license": "ISC", + "dependencies": { + }, + "devDependencies": { + "mocha": "^8.1.3", + "chai": "^4.2.0" + } +} diff --git a/test/basics.test.js b/test/basics.test.js new file mode 100644 index 0000000..98c06c0 --- /dev/null +++ b/test/basics.test.js @@ -0,0 +1,126 @@ +const fsp = require('fs').promises; +const {execCommandLine} = require('./utilities/exec'); +const walk = require('../action-walk'); +const {expect} = require('chai'); + +const testdir = '.'; +let testdirStat; +const duOutput = { + wo_node: {}, + w_node: {}, +} + +describe('verify that action-walk works as expected', function () { + // tests need to account for du counting the target directory itself + // while walk treats that as a starting point and only counts the + // contents of the directory. + before(function getTestDirSize () { + return fsp.stat(testdir) + .then(s => { + testdirStat = s; + }); + }) + before(function getDuOutput () { + // output is size-in-bytes path-starting-with-dir-name + const p = [ + execCommandLine(`du -ab --exclude=node_modules ${testdir}`), + execCommandLine(`du -ab ${testdir}`), + ]; + return Promise.all(p) + .then(r => { + expect(r[0].stderr).equal(''); + expect(r[1].stderr).equal(''); + duOutput.wo_node = parseDuOutput(r[0].stdout); + duOutput.w_node = parseDuOutput(r[1].stdout); + }); + }); + + it('should match du -ab output', function () { + const own = {total: 0}; + const options = {dirAction, fileAction, own, stat: true}; + return walk(testdir, options) + .then(() => { + expect(own.total + testdirStat.size).equal(duOutput.w_node[testdir]); + }) + }); + + it('should match du -ab --exclude=node_modules', function () { + const own = {total: 0, skipDirs: ['node_modules']}; + const options = {dirAction, fileAction, own, stat: true}; + return walk(testdir, options) + .then(() => { + expect(own.total + testdirStat.size).equal(duOutput.wo_node[testdir]); + }) + }); + + it('should execute recursively matching du -b --exclude=node_modules', function () { + const own = {total: 0, dirTotals: {}, skipDirs: ['node_modules']}; + const options = {dirAction: daDirOnly, fileAction, own, stat: true}; + return walk(testdir, options) + .then(() => { + expect(own.total + testdirStat.size).equal(duOutput.wo_node[testdir]); + for (const dir in own.dirTotals) { + expect(own.dirTotals[dir]).equal(duOutput.w_node[`./${dir}`]); + } + }); + }); + + it('should execute recursively matching du -b', function () { + const own = {total: 0, dirTotals: {}, skipDirs: []}; + const options = {dirAction: daDirOnly, fileAction, own, stat: true}; + return walk(testdir, options) + .then(() => { + expect(own.total + testdirStat.size).equal(duOutput.w_node[testdir]); + for (const dir in own.dirTotals) { + expect(own.dirTotals[dir]).equal(duOutput.w_node[`./${dir}`]); + } + }); + }); + + +}); + + +// +// utilities +// +function dirAction (path, ctx) { + const {dirent, stat, own} = ctx; + if (own.skipDirs && own.skipDirs.indexOf(dirent.name) >= 0) { + return 'skip'; + } + own.total += stat.size; +} +function fileAction (path, ctx) { + const {stat, own} = ctx; + own.total += stat.size; +} + +async function daDirOnly (path, ctx) { + const {dirent, stat, own} = ctx; + if (own.skipDirs && own.skipDirs.indexOf(dirent.name) >= 0) { + return 'skip'; + } + own.dirTotals[path] = 0; + const newown = {total: 0, dirTotals: own.dirTotals}; + const options = { + dirAction: daDirOnly, + fileAction, + own: newown, + stat: true, + }; + await walk(path, options); + own.dirTotals[path] = newown.total + stat.size; + own.total += newown.total + stat.size; + + // skip it because the recursive call counted the subtree. + return 'skip'; +} + +function parseDuOutput (text) { + const o = {}; + for (const m of text.matchAll(/(?\d+)\s+(?.+)/g)) { + o[m.groups.path] = +m.groups.size; + } + return o; +} diff --git a/test/utilities/exec.js b/test/utilities/exec.js new file mode 100644 index 0000000..a44c22c --- /dev/null +++ b/test/utilities/exec.js @@ -0,0 +1,18 @@ +const exec = require('child_process').exec; + +async function execCommandLine (cmdline, options = {}) { + return new Promise((resolve, reject) => { + // eslint-disable-next-line no-unused-vars + const cp = exec(cmdline, options, function (error, stdout, stderr) { + if (error) { + reject({error, stdout, stderr}); + } else { + resolve({stdout, stderr}); + } + }); + }); +} + +module.exports = { + execCommandLine, +}