search

paulmillr / readdirp

Recursive version of fs.readdir with streaming api.
https://paulmillr.com
MIT License
378 stars 51 forks source link

Excluding Filter not working for recursive directory #1

Closed nick-jonas closed 11 years ago

nick-jonas commented 11 years ago

With the following filter:

!.assemble_config, !.svn, !.git, !.DS_Store, !thumbs.db, !.sass-cache

The files within the .sass-cache folder are still getting read in:

sass/.sass-cache/f83244f75401c5c54214ac1ea0576281d46e48b6/_images.scssc sass/.sass-cache/f83244f75401c5c54214ac1ea0576281d46e48b6/_shared.scssc sass/.sass-cache/f83244f75401c5c54214ac1ea0576281d46e48b6/_opacity.scssc sass/.sass-cache/f83244f75401c5c54214ac1ea0576281d46e48b6/_text-shadow.scssc sass/.sass-cache/f83244f75401c5c54214ac1ea0576281d46e48b6/_transform.scssc sass/.sass-cache/f83244f75401c5c54214ac1ea0576281d46e48b6/_transition.scssc sass/.sass-cache/ecabf3aa2225c27adc86b9ecf19429105699b9f8/_sprites.scssc sass/.sass-cache/ff92cd3b06900c7c8e7ac5f21950030e881d2f60/_support.scssc

nick-jonas commented 11 years ago

Would also be cool if a file (like .gitignore or .npmignore) can be provided in place of the glob/array...

thlorenz commented 11 years ago

Could you please provide some sample code or ideally a failing test? Thanks.

BTW, I assume you are passing in an array as in: [ '!.assemble_config', '!.svn', '!.git', '!.DS_Store', '!thumbs.db', '!.sass-cache' ]

Otherwise it is not a valid filter.

thlorenz commented 11 years ago

In regards to a .gitignore like feature, I'm happy to take a pull request if you get it in. Also if you get lots of people to up vote your request, I'll try to find some time to do it myself.

nick-jonas commented 11 years ago

Here is a test case I made for this:

https://dl.dropbox.com/u/13037196/github/readdirp-testcase.zip

Just run node main.js, and you'll notice via console.log that all the files within .sass-cache are read.

Sidenote: Is there a way to attach files (other than images) to GitHub issues yet? Is this the best way of going about this?

And if I have the time I'll sit down and do the .gitignore-like feature as soon as I can...

thlorenz commented 11 years ago

You were setting fileFilter for files and directories instead of using directoryFilter for directories.

Change main to:

#!/usr/bin/env node

var readdirp = require('readdirp'),
    path = require('path');

readdirp({
    root: path.join(__dirname, '/files'),
    fileFilter: [ '!.assembleconfig', '!.DS_Store', '!thumbs.db' ],
    directoryFilter: [ '!.svn', '!.git', '!.sass-cache' ]
}).on('data', function(entry){
    console.log(entry.path);
});

and it works as planned.

Also next time please post an example on github instead of a zip file and try to make it as simple as possible, i.e. include the least amount of files/code needed to expose the problem.

When implementing the .gitignore feature please ensure it follows the readdirp coding style and is well tested, otherwise I won't be able to accept the pull request.

In the meantime, closing this, since it is not a bug, but rather misunderstanding of the API.

Cheers.

Roms1383 commented 5 years ago

I'm trying to use the library with version 3.0.1 and I'm getting the same inconsistency :

test case

Trying to get all the JS files excluding .git and node_modules folders.

code

const chalk = require('chalk')
const readdirp = require('readdirp')
const { promise } = readdirp
const options = { directoryFilter: ['!.git', '!node_modules'] }
const streamed = async () => new Promise((resolve) => {
  const entries = []
  readdirp('.', options)
  .on('data', entry => { entries.push(entry) })
  .on('end', () => resolve(entries))
})
const promised = async () => promise('.', options)
const map = ({ path }) => path
const test = async () => {
  console.log(chalk.blue('from stream'))
  const s = await streamed()
  console.log(chalk.blue(JSON.stringify(s.map(map), null, 2)))
  console.log('---')
  console.log(chalk.cyan('from promise'))
  const p = await promised()
  console.log(chalk.cyan(JSON.stringify(p.map(map), null, 2)))
}
test()

result

(I removed parts of the output as it's too big)

from stream
[
  ".commitlintrc.yml",
  ".eslintrc",
  ".gitignore",
  ".huskyrc",
  ".git/COMMIT_EDITMSG",
  ".lintstagedrc",
  ".git/FETCH_HEAD",
  ".nvmrc",
  ".git/HEAD",
  ".releaserc",
  ".git/ORIG_HEAD",
  ".remarkrc",
  ".renovaterc",
  ".git/config",
  ".travis.yml",
  ".git/description",
  ".yo-rc.json",
  "LICENSE",
  ".git/index",
  "README.md",
  ".git/hooks/applypatch-msg",
  ".git/hooks/applypatch-msg.sample",
  ".git/info/exclude",
  ".git/hooks/commit-msg",
  "coverage/clover.xml",
  "jest.config.js",
  ".git/packed-refs",
  ".git/logs/HEAD",
  ".git/hooks/commit-msg.sample",
  "coverage/coverage-final.json",
  ".git/hooks/post-applypatch",
  ".git/sourcetreeconfig",
  ".git/hooks/post-checkout",
  "coverage/lcov.info",
  "my-tests/introspect.js",
  "package.json",
  "generators/app/index.js",
  ".git/hooks/post-commit",
  "coverage/lcov-report/base.css",
  ".git/objects/00/66aa68dde5b4afba8618c752cf9e255b598f14",
  "generators/codacy/README.md",
  ".git/hooks/post-merge",
  "coverage/lcov-report/block-navigation.js",
  ".git/objects/01/2b627794e72a17bfcd7846621974055ba510fb",
  ".git/objects/00/d2d36b0a01ce570ca90c385d31dcd85878c6c2",
  "yarn.lock",
  "generators/commitlint/index.js",
  "generators/codacy/index.js",
  ".git/refs/heads/develop",
  ".git/logs/refs/heads/develop",
  ".git/hooks/post-receive",
  ".git/objects/02/6f3bd2fd2a9a51c65a14c8b6033c7631d8afff",
  ".git/objects/01/7d7f8ad70d382d59015592a330d14042c4aea1",
  "generators/eslint/index.js",
  ".git/refs/heads/master",
  ".git/logs/refs/heads/master",
  ".git/hooks/post-rewrite",
  "coverage/lcov-report/index.html",
  ".git/objects/03/9d1e4a288216f5b80e404837d23cb5a8fcf0f7",
  "node_modules/.yarn-integrity",
  ".git/objects/02/e6cf2442e8ed35916d0d2bd1dfcf7d4d304354",
  "generators/husky/index.js",
  ".git/refs/tags/v1.0.0",
  ".git/hooks/post-update",
  "coverage/lcov-report/format/index.html",
  ".git/objects/05/ac3f99caed0f9a3579dfd3f6d9562eb954360d",
  ".git/objects/03/b88cc88a646ae899603dfb6f9dc71d8a3cf6d9",
  "node_modules/.bin/JSONStream",
  "utils/arrayify/index.js",
  "generators/commitlint/templates/conf",
  "generators/jest/index.js",
  ".git/refs/tags/v1.0.1",
  ".git/refs/remotes/origin/HEAD",
  ".git/logs/refs/remotes/origin/HEAD",
  ".git/hooks/post-update.sample",
  "coverage/lcov-report/format/index.js.html",
  ".git/objects/05/beea9df2ebd91585aeb191035d3cba65f115b4",
  ".git/objects/06/9a6c91cf1bf6cd4405e8d6ae27a8797a7872f9",
  "node_modules/.bin/_mocha",
  "utils/format/index.js",
  "utils/arrayify/index.test.js",
  "generators/license/index.js",
  ".git/refs/tags/v1.1.0",
  "generators/eslint/templates/conf",
  ".git/refs/remotes/origin/develop",
  ".git/logs/refs/remotes/origin/develop",
  ".git/hooks/pre-applypatch",
  ".git/objects/07/028d30f964e7a3d35237496ac6b6e2853aae9d",
  ".git/objects/06/ccff9400c3224b5629e8151f9e149201e3a1a2",
  "node_modules/.bin/acorn",
  "utils/generator/index.js",
  "utils/format/index.test.js",
  "generators/lint/index.js",
  "generators/husky/templates/conf",
  ".git/refs/remotes/origin/master",
  ".git/logs/refs/remotes/origin/master",
  ".git/hooks/pre-applypatch.sample",
  "coverage/lcov-report/prettify.css",
  ".git/objects/08/4dcf9e5026d36113deecf917611b46654efb55",
  ".git/objects/07/ba1f9cc0e0d81a16047abf25f636be6c0ce744",
  ".git/objects/06/f14045e841fe5b40f23c239af21d0bcaad042d",
  "node_modules/.bin/atob",
  "generators/lint-staged/index.js",
  "generators/jest/templates/conf",
  ".git/hooks/pre-auto-gc",
  "coverage/lcov-report/prettify.js",
  "coverage/lcov-report/inspect/contains/index.html",
  ".git/objects/09/b765609b7ceb0cf005748ed49f9cc5e3c8eba7",
  ".git/objects/08/d2d7a1df6d8c52bc48705f872127ac71c30bf7",
  ".git/objects/07/ba2b355fad6b749d4620e8149909818c8d3085",
  "node_modules/@babel/code-frame/LICENSE",
  "node_modules/.bin/cdl",
  "utils/introspect/index.js",
  "generators/nvm/index.js",
  ".git/hooks/pre-commit",
  "coverage/lcov-report/sort-arrow-sprite.png",
  "coverage/lcov-report/npm/latest/index.html",
  "coverage/lcov-report/inspect/files/index.html",
  "coverage/lcov-report/inspect/contains/index.js.html",
  ".git/objects/0a/4cb35212ae3afd50fce9e23de788249263ae9d",
  "node_modules/@cnakazawa/watch/LICENSE",
  "node_modules/@babel/core/LICENSE",
  "node_modules/@babel/code-frame/README.md",
  "node_modules/.bin/codacy-coverage",
  "utils/npm/index.js",
  "utils/inspect/contains/index.js",
  "generators/remark-lint/index.js",
  ".git/hooks/pre-commit.sample",
  "coverage/lcov-report/sorter.js",
  "coverage/lcov-report/package/author/index.html",
  "coverage/lcov-report/npm/peers/fetch.js.html",
  "coverage/lcov-report/npm/latest/index.js.html",
  "coverage/lcov-report/inspect/files/index.js.html",
  "coverage/lcov-report/inspect/folders/index.html",
  ".git/objects/0c/15c4e8b901d67e68c17462becc631bd85af382",
  ".git/objects/0a/73dc4be458b56cf80a489c050a23354a148490",
  "node_modules/@commitlint/cli/CHANGELOG.md",
  "node_modules/@babel/generator/LICENSE",
  "node_modules/@cnakazawa/watch/README.md",
  "node_modules/@babel/core/README.md",
  "node_modules/.bin/commitlint",
  "utils/inspect/file/dummy.txt",
  "utils/inspect/contains/index.test.js",
  "generators/renovate/index.js",
  "generators/lint-staged/templates/conf",
  ".git/hooks/pre-push",
  "coverage/lcov-report/package/author/parse.js.html",
  "coverage/lcov-report/npm/peers/index.html",
  "coverage/lcov-report/inspect/folders/index.js.html",
  ".git/objects/0d/996f008aeeca2b5ee66599ab252b0d7c95217d",
  ".git/objects/0c/2c6eaab290d9af4ae3d4ab9730ced08eb4930e",
  "node_modules/@commitlint/ensure/CHANGELOG.md",
  "node_modules/@commitlint/cli/README.md",
  "node_modules/@jest/console/LICENSE",
  "node_modules/@babel/helper-function-name/LICENSE",
  "node_modules/@babel/generator/README.md",
  "node_modules/@cnakazawa/watch/cli.js",
  "node_modules/@babel/code-frame/package.json",
  "node_modules/.bin/conventional-changelog-writer",
  "utils/package/description.js",
  "utils/inspect/files/index.js",
  "utils/inspect/index.js",
  "utils/inspect/file/index.js",
  "generators/semantic-release/index.js",
  "generators/nvm/templates/conf",
  ".git/hooks/pre-push.sample",
  "coverage/lcov-report/npm/peers/parse.js.html",
  ".git/objects/0e/1f7d734edfaf791ad6d31e237f59a7570a6a2f",
  ".git/objects/0c/80334d8ad3c3704919d82b5c5079d72afa938d",
  "node_modules/@marionebl/sander/CHANGELOG.md",
  "node_modules/@commitlint/execute-rule/CHANGELOG.md",
  "node_modules/@commitlint/cli/index.js",
  "node_modules/@jest/core/LICENSE",
  "node_modules/@babel/helper-function-name/README.md",
  "node_modules/@babel/helper-get-function-arity/LICENSE",
  "node_modules/@cnakazawa/watch/main.js",
  "node_modules/@babel/code-frame/lib/index.js",
  "node_modules/.bin/conventional-commits-parser",
  ...
]
---
from promise
[
  ".commitlintrc.yml",
  ".eslintrc",
  ".gitignore",
  ".huskyrc",
  ".git/COMMIT_EDITMSG",
  ".lintstagedrc",
  ".git/FETCH_HEAD",
  ".nvmrc",
  ".git/HEAD",
  ".releaserc",
  ".git/ORIG_HEAD",
  ".remarkrc",
  ".renovaterc",
  ".git/config",
  ".travis.yml",
  ".git/description",
  ".yo-rc.json",
  "LICENSE",
  ".git/index",
  "README.md",
  ".git/hooks/applypatch-msg",
  ".git/hooks/applypatch-msg.sample",
  ".git/info/exclude",
  ".git/hooks/commit-msg",
  "jest.config.js",
  "coverage/clover.xml",
  ".git/packed-refs",
  ".git/logs/HEAD",
  ".git/hooks/commit-msg.sample",
  "coverage/coverage-final.json",
  ".git/hooks/post-applypatch",
  ".git/sourcetreeconfig",
  ".git/hooks/post-checkout",
  "my-tests/introspect.js",
  "package.json",
  "generators/app/index.js",
  "coverage/lcov.info",
  ".git/hooks/post-commit",
  "generators/codacy/README.md",
  "coverage/lcov-report/base.css",
  ".git/objects/00/66aa68dde5b4afba8618c752cf9e255b598f14",
  ".git/hooks/post-merge",
  "yarn.lock",
  "generators/commitlint/index.js",
  "generators/codacy/index.js",
  "coverage/lcov-report/block-navigation.js",
  ".git/objects/01/2b627794e72a17bfcd7846621974055ba510fb",
  ".git/objects/00/d2d36b0a01ce570ca90c385d31dcd85878c6c2",
  ".git/refs/heads/develop",
  ".git/logs/refs/heads/develop",
  ".git/hooks/post-receive",
  "generators/eslint/index.js",
  ".git/objects/02/6f3bd2fd2a9a51c65a14c8b6033c7631d8afff",
  ".git/objects/01/7d7f8ad70d382d59015592a330d14042c4aea1",
  ".git/refs/heads/master",
  ".git/logs/refs/heads/master",
  ".git/hooks/post-rewrite",
  "node_modules/.yarn-integrity",
  "generators/husky/index.js",
  "coverage/lcov-report/index.html",
  ".git/objects/03/9d1e4a288216f5b80e404837d23cb5a8fcf0f7",
  ".git/objects/02/e6cf2442e8ed35916d0d2bd1dfcf7d4d304354",
  ".git/refs/tags/v1.0.0",
  ".git/hooks/post-update",
  "node_modules/.bin/JSONStream",
  "utils/arrayify/index.js",
  "generators/jest/index.js",
  "generators/commitlint/templates/conf",
  "coverage/lcov-report/format/index.html",
  ".git/objects/05/ac3f99caed0f9a3579dfd3f6d9562eb954360d",
  ".git/objects/03/b88cc88a646ae899603dfb6f9dc71d8a3cf6d9",
  ".git/refs/tags/v1.0.1",
  ".git/refs/remotes/origin/HEAD",
  ".git/logs/refs/remotes/origin/HEAD",
  ".git/hooks/post-update.sample",
  "node_modules/.bin/_mocha",
  "utils/format/index.js",
  "utils/arrayify/index.test.js",
  "generators/license/index.js",
  "generators/eslint/templates/conf",
  "coverage/lcov-report/format/index.js.html",
  ".git/objects/06/9a6c91cf1bf6cd4405e8d6ae27a8797a7872f9",
  ".git/objects/05/beea9df2ebd91585aeb191035d3cba65f115b4",
  ".git/refs/tags/v1.1.0",
  ".git/refs/remotes/origin/develop",
  ".git/logs/refs/remotes/origin/develop",
  ".git/hooks/pre-applypatch",
  "node_modules/.bin/acorn",
  "utils/generator/index.js",
  "utils/format/index.test.js",
  "generators/lint/index.js",
  "generators/husky/templates/conf",
  ".git/objects/07/028d30f964e7a3d35237496ac6b6e2853aae9d",
  ".git/objects/06/ccff9400c3224b5629e8151f9e149201e3a1a2",
  ".git/refs/remotes/origin/master",
  ".git/logs/refs/remotes/origin/master",
  ".git/hooks/pre-applypatch.sample",
  "node_modules/.bin/atob",
  "generators/lint-staged/index.js",
  "generators/jest/templates/conf",
  "coverage/lcov-report/prettify.css",
  ".git/objects/08/4dcf9e5026d36113deecf917611b46654efb55",
  ".git/objects/07/ba1f9cc0e0d81a16047abf25f636be6c0ce744",
  ".git/objects/06/f14045e841fe5b40f23c239af21d0bcaad042d",
  ".git/hooks/pre-auto-gc",
  "node_modules/@babel/code-frame/LICENSE",
  "node_modules/.bin/cdl",
  "utils/introspect/index.js",
  "generators/nvm/index.js",
  "coverage/lcov-report/prettify.js",
  "coverage/lcov-report/inspect/contains/index.html",
  ".git/objects/09/b765609b7ceb0cf005748ed49f9cc5e3c8eba7",
  ".git/objects/08/d2d7a1df6d8c52bc48705f872127ac71c30bf7",
  ".git/objects/07/ba2b355fad6b749d4620e8149909818c8d3085",
  ".git/hooks/pre-commit",
  "node_modules/@cnakazawa/watch/LICENSE",
  "node_modules/@babel/core/LICENSE",
  "node_modules/@babel/code-frame/README.md",
  "node_modules/.bin/codacy-coverage",
  "utils/npm/index.js",
  "utils/inspect/contains/index.js",
  "generators/remark-lint/index.js",
  "coverage/lcov-report/sort-arrow-sprite.png",
  "coverage/lcov-report/npm/latest/index.html",
  "coverage/lcov-report/inspect/files/index.html",
  "coverage/lcov-report/inspect/contains/index.js.html",
  ".git/objects/0a/4cb35212ae3afd50fce9e23de788249263ae9d",
  ".git/hooks/pre-commit.sample",
  "node_modules/@commitlint/cli/CHANGELOG.md",
  "node_modules/@cnakazawa/watch/README.md",
  "node_modules/@babel/generator/LICENSE",
  "node_modules/@babel/core/README.md",
  "node_modules/.bin/commitlint",
  "utils/inspect/file/dummy.txt",
  "utils/inspect/contains/index.test.js",
  "generators/renovate/index.js",
  "generators/lint-staged/templates/conf",
  "coverage/lcov-report/sorter.js",
  "coverage/lcov-report/package/author/index.html",
  "coverage/lcov-report/npm/latest/index.js.html",
  "coverage/lcov-report/npm/peers/fetch.js.html",
  "coverage/lcov-report/inspect/files/index.js.html",
  "coverage/lcov-report/inspect/folders/index.html",
  ".git/objects/0c/15c4e8b901d67e68c17462becc631bd85af382",
  ".git/objects/0a/73dc4be458b56cf80a489c050a23354a148490",
  ".git/hooks/pre-push",
  "node_modules/@jest/console/LICENSE",
  "node_modules/@commitlint/ensure/CHANGELOG.md",
  "node_modules/@commitlint/cli/README.md",
  "node_modules/@cnakazawa/watch/cli.js",
  "node_modules/@babel/helper-function-name/LICENSE",
  "node_modules/@babel/generator/README.md",
  "node_modules/@babel/code-frame/package.json",
  "node_modules/.bin/conventional-changelog-writer",
  "utils/package/description.js",
  "utils/inspect/index.js",
  "utils/inspect/files/index.js",
  "utils/inspect/file/index.js",
  "generators/semantic-release/index.js",
  "generators/nvm/templates/conf",
  "coverage/lcov-report/package/author/parse.js.html",
  "coverage/lcov-report/npm/peers/index.html",
  "coverage/lcov-report/inspect/folders/index.js.html",
  ".git/objects/0d/996f008aeeca2b5ee66599ab252b0d7c95217d",
  ".git/objects/0c/2c6eaab290d9af4ae3d4ab9730ced08eb4930e",
  ".git/hooks/pre-push.sample",
  "node_modules/@marionebl/sander/CHANGELOG.md",
  "node_modules/@jest/core/LICENSE",
  "node_modules/@commitlint/execute-rule/CHANGELOG.md",
  "node_modules/@commitlint/cli/index.js",
  "node_modules/@cnakazawa/watch/main.js",
  "node_modules/@babel/helper-function-name/README.md",
  "node_modules/@babel/helper-get-function-arity/LICENSE",
  "node_modules/@babel/code-frame/lib/index.js",
  "node_modules/.bin/conventional-commits-parser",
  "utils/npm/latest/index.js",
  "utils/package/author/index.js",
  "utils/package/files.js",
  "utils/inspect/files/index.test.js",
  "utils/inspect/folders/index.js",
  "utils/inspect/file/index.test.js",
  "generators/travis/index.js",
  "generators/remark-lint/templates/conf",
  "coverage/lcov-report/npm/peers/parse.js.html",
  ".git/objects/0e/1f7d734edfaf791ad6d31e237f59a7570a6a2f",
  ".git/objects/0c/80334d8ad3c3704919d82b5c5079d72afa938d",
  ".git/hooks/pre-rebase",
  "node_modules/@marionebl/sander/README.md",
  "node_modules/@mrmlnc/readdir-enhanced/CHANGELOG.md",
  "node_modules/@jest/core/README.md",
  "node_modules/@jest/environment/LICENSE",
  "node_modules/@jest/console/package.json",
  "node_modules/@commitlint/format/CHANGELOG.md",
  "node_modules/@commitlint/ensure/license.md",
  "node_modules/@cnakazawa/watch/package.json",
  "node_modules/@babel/helper-plugin-utils/LICENSE",
  "node_modules/@babel/helper-get-function-arity/README.md",
  "node_modules/@babel/core/package.json",
  "node_modules/.bin/detect-libc",
  "utils/npm/latest/index.test.js",
  "utils/package/author/parse.js",
  "utils/npm/peers/fetch.js",
  "utils/package/fs.js",
  "utils/inspect/folders/index.test.js",
  "generators/renovate/templates/conf",
  ".git/objects/0f/faff16056ed859b7494d1c873053af8938fa2d",
  ".git/objects/0e/ed6d75a4dec6e9443a25aa38e11f7f8aed4cd9",
  ".git/objects/0c/af0a03dda7f408bc667fdfacf91d2c6fc68d4b",
  ".git/hooks/pre-rebase.sample",
  "node_modules/@nodelib/fs.stat/README.md",
  "node_modules/@mrmlnc/readdir-enhanced/LICENSE",
  "node_modules/@jest/fake-timers/LICENSE",
  "node_modules/@jest/console/tsconfig.json",
  "node_modules/@jest/console/build/BufferedConsole.d.ts",
  "node_modules/@commitlint/is-ignored/CHANGELOG.md",
  "node_modules/@commitlint/format/README.md",
  "node_modules/@commitlint/execute-rule/license.md",
  "node_modules/@commitlint/ensure/package.json",
  "node_modules/@commitlint/ensure/lib/case.js",
  "node_modules/@commitlint/cli/license.md",
  "node_modules/@babel/helper-split-export-declaration/LICENSE",
  "node_modules/@babel/helper-plugin-utils/README.md",
  "node_modules/@babel/helper-function-name/package.json",
  "node_modules/@babel/generator/package.json",
  "node_modules/@babel/generator/lib/buffer.js",
  "node_modules/@babel/core/lib/index.js",
  "node_modules/.bin/errno",
  ...
]

Could you point out my mistake ?