Nested Dependancies?

[reggi]

I ran this on a file's source and I'm only getting the deps for that file, not the ones it's loading. It doesn't seem to be walking the AST, isn't that the whole point of acorn? Is there a setting I'm missing? Thanks!

[reggi]

Here's my code:

var Promise = require("bluebird")
var fs = Promise.promisifyAll(require('fs'))
var acorn = require("acorn")
var umd = require("acorn-umd")

fs.readFileAsync(paths.mainFile)
.then(function(src){

  var ast = acorn.parse(src, {
    ecmaVersion: 6,
    sourceType: "module"
  });
  return umd(ast, {
      es6: true, amd: false, cjs: true
  });

})
.then(function(modules){
  console.log(JSON.stringify(modules, null, 2))
})

[megawac]

Hey, the project doesn't load nested dependencies, it only processes the given ast for imports as it was designed for just identifying the imports in a single file -- though it wouldn't be hard to make it process recursive-like, as below (untested and won't work for node_module requires)

var fs = reqruire('fs');
var path = require('path');

function _umd(ast) {
    return umd(ast, {
        es6: true, amd: false, cjs: true
    });
}

function recursiveUmd(curPath, code) {
    var ast = acorn.parse(code, {ecmaVersion: 6});
    var imports = _umd(ast);

    imports.forEach(imp => {
        var newPath = path.relative(curPath, import.source.value);
        var code = fs.readFileSync(newPath)
        var ast = acorn.parse(code, {ecmaVersion: 6});
        imports.push(..._umd(ast));
    });

    return imports;
});

[reggi]

Getting some crazy error here

Unhandled rejection SyntaxError: Unexpected token (1:8)
    at Parser.pp.raise (/Users/thomas/Desktop/wiskers/node_modules/acorn/dist/acorn.js:975:13)
    at Parser.pp.unexpected (/Users/thomas/Desktop/wiskers/node_modules/acorn/dist/acorn.js:1524:8)
    at Parser.pp.expect (/Users/thomas/Desktop/wiskers/node_modules/acorn/dist/acorn.js:1518:26)
    at Parser.pp.parseExprList (/Users/thomas/Desktop/wiskers/node_modules/acorn/dist/acorn.js:635:12)
    at Parser.pp.parseExprAtom (/Users/thomas/Desktop/wiskers/node_modules/acorn/dist/acorn.js:310:28)
    at Parser.pp.parseExprSubscripts (/Users/thomas/Desktop/wiskers/node_modules/acorn/dist/acorn.js:221:19)
    at Parser.pp.parseMaybeUnary (/Users/thomas/Desktop/wiskers/node_modules/acorn/dist/acorn.js:202:19)
    at Parser.pp.parseExprOps (/Users/thomas/Desktop/wiskers/node_modules/acorn/dist/acorn.js:156:19)
    at Parser.pp.parseMaybeConditional (/Users/thomas/Desktop/wiskers/node_modules/acorn/dist/acorn.js:138:19)
    at Parser.pp.parseMaybeAssign (/Users/thomas/Desktop/wiskers/node_modules/acorn/dist/acorn.js:116:19)
    at Parser.pp.parseExpression (/Users/thomas/Desktop/wiskers/node_modules/acorn/dist/acorn.js:90:19)
    at Parser.pp.parseStatement (/Users/thomas/Desktop/wiskers/node_modules/acorn/dist/acorn.js:1737:23)
    at Parser.pp.parseTopLevel (/Users/thomas/Desktop/wiskers/node_modules/acorn/dist/acorn.js:1653:21)
    at Parser.parse (/Users/thomas/Desktop/wiskers/node_modules/acorn/dist/acorn.js:1623:17)
    at Object.parse (/Users/thomas/Desktop/wiskers/node_modules/acorn/dist/acorn.js:937:44)
    at /Users/thomas/Desktop/wiskers/crawl-deps.js:22:21

I had to turn you code into es5.

var acorn = require("acorn")
var umd = require("acorn-umd")

var acornSettings = {
  ecmaVersion: 6,
  sourceType: "module"
}

function _umd(ast) {
  return umd(ast, {
      es6: true, amd: false, cjs: true
  });
}

function recursiveUmd(code) {
  var imp = []
  var ast = acorn.parse(code, acornSettings);
  var rootImports = _umd(ast);
  imp.push(rootImports)

  rootImports.forEach(function(individualImport) {
    var ast = acorn.parse(individualImport, acornSettings);
    var childImports = _umd(ast)
    imp.push(childImports)
  });

  return imp;
}

module.exports = recursiveUmd

[reggi]

I got it running with this.

var acorn = require("acorn")
var umd = require("acorn-umd")

var acornSettings = {
  ecmaVersion: 6,
  sourceType: "module"
}

function _umd(ast) {
  return umd(ast, {
      es6: true, amd: false, cjs: true
  });
}

function _toConsumableArray(arr) { if (Array.isArray(arr)) { for (var i = 0, arr2 = Array(arr.length); i < arr.length; i++) arr2[i] = arr[i]; return arr2; } else { return Array.from(arr); } }

function recursiveUmd(code) {
    var ast = acorn.parse(code, acornSettings);
    var imports = _umd(ast);

    imports.forEach(function (imp) {
        var ast = acorn.parse(code, acornSettings);
        imports.push.apply(imports, _toConsumableArray(_umd(ast)));
    });

    return imports;
};

module.exports = recursiveUmd

[reggi]

Doesn't seem to be working though.

file ./test-1.js

var _ = require("underscore")
var express = require("express")
// var routes = require("./test-2")
import routes from "test-2"
var app = express()
module.exports = app

file ./test-2.js

var crypto = require("crypto")
var Promise = require("bluebird")
module.exports = {}

The code.

var Promise = require("bluebird")
var fs = Promise.promisifyAll(require('fs'))
var crawlDeps = require("./crawl-deps")
// var acorn = require("acorn")
// var umd = require("acorn-umd")
// var requiredAsync = Promise.promisify(require('required'))

fs.readFileAsync(paths.mainFile)
  .then(function(src){
    return crawlDeps(src)
  })
  .then(function(items){
    return _.map(items, function(item){
      return item.source.value
    })
  }).then(console.log)

This is what's being logged

[ 'underscore',
  'express',
  'test-2',
  'underscore',
  'express',
  'test-2',
  'underscore',
  'express',
  'test-2',
  'underscore',
  'express',
  'test-2' ]

[reggi]

It's taking "code" from the first root run and applying it three times. What string am I supposed to feed into that loop?

[megawac]

I made a mistake in the original code, the inner loop should have been something like

imports.forEach(imp => {
        var newPath = path.relative(curPath, import.source.value);
        var code = fs.readFileSync(newPath)
        imports.push(...recursiveUmd(newPath, code));
});

I had originally made it not recursive by mistake.

In your modified code you made it process the same file for each dependency instead of loading that file over fs

[reggi]

Thanks for all your help! Source doesn't have extension. It's also not gonna check if dep is in node_module or local. I don't think I can easily find recursive deps using your lib. Thoughts?

[megawac]

Hmm, could try require.resolve after the path.relative

Something like var newPath = require.resolve(path.relative(curPath, import.source.value));

I'm on a tablet so I can't verify any of this

[megawac]

I think you may also need to do a path.basename on curPath

[reggi]

I'm gonna keep trying to build this out. Thing is I don't wanna recurse npm modules, only local files, so I have to check for ./ and ../.

[reggi]

Or if contains / at all.

[reggi]

no wait can't do that.

[reggi]

require("express/lib/view")

[reggi]

Thoughts:

var acorn = require("acorn")
var umd = require("acorn-umd")

function crawlDep(src){
  var ast = acorn.parse(src, {
    sourceType: "module",
    ecmaVersion: 6
  })
  return umd(ast, {
      es6: true, amd: false, cjs: true
  })
}

fs.readFileAsync(paths.mainFile)
.then(crawlDep)
.then(function(deps){
  var recursiveDep = _.chain(deps)
    .map(function(dep){
      return dep.source.value
    })
    .filter(function(dep){
      return dep.match(/^.\.\/|^.\//)
    })
    .value()
  return Promise.map(recursiveDep, function(dep){
    var ext = path.extname(paths.mainFile)
    var filePath = path.join(paths.mainFile, "..", dep+ext)
    return fs.readFileAsync(filePath)
    .then(crawlDep)
  }).then(function(restDeps){
    return _.flatten([deps, restDeps])
  })
}).then(function(deps){
  console.log(deps)
})

[megawac]

That may run into issues for multiple directories (e.g. x/y.js requiring x/w/z.js by way of require('./w/z')). You may also want to do a uniq to prevent duplicate deps.

Another issue you may run into is if a file is required from multiple directories as the path will be slightly different.

[megawac]

There are may be a couple other gotchas you may run into here

[reggi]

My version isn't recursive either, just goes 2 deep. GRRR

[reggi]

Think I might have something:

var path = require("path")
var _ = require("underscore")
var Promise = require("bluebird")
var fs = Promise.promisifyAll(require('fs'))
var acorn = require("acorn")
var umd = require("acorn-umd")

function crawlDep(src){
  var ast = acorn.parse(src, {
    sourceType: "module",
    ecmaVersion: 6
  })
  return umd(ast, {
      es6: true, amd: false, cjs: true
  })
}

function getRecursiveDeps(deps, localRegex){
  return _.chain(deps)
    .map(function(dep){
      return dep.source.value
    })
    .filter(function(dep){
      return dep.match(localRegex)
    })
    .value()
}

function recursiveDeps(filePath, deps, localRegex){
  var allDeps = []
  function _recursiveDeps(deps){
    return Promise.map(deps, function(dep){
      var ext = path.extname(filePath)
      var depFilePath = path.join(filePath, "..", dep+ext)
      return fs.readFileAsync(depFilePath).then(crawlDep)
    }).then(function(newDeps){
      newDeps = _.flatten(newDeps)
      allDeps.push(newDeps)
      if(newDeps.length == 0 ) return allDeps
      var getDeps = getRecursiveDeps(newDeps, localRegex)
      return _recursiveDeps(getDeps)
    })
  }
  return _recursiveDeps(deps)
}

function crawlDeps(filePath, localRegex){
  localRegex = (localRegex || /^.\.\/|^.\/|^\//)
  return fs.readFileAsync(filePath)
  .then(crawlDep)
  .then(function(deps){
    return Promise.method(getRecursiveDeps)(deps, localRegex)
    .then(function(deps){
      return recursiveDeps(filePath, deps, localRegex)
    })
    .then(function(restDeps){
      return _.chain([deps, restDeps])
      .flatten()
      .value()
    })
  })
}

module.exports = crawlDeps

[reggi]

@megawac any red flags pop out?

[reggi]

Is this a reliable way to sort them?

var natives = Object.keys(process.binding('natives'))
var crawlDeps = require("./crawl-deps")

crawlDeps(paths.mainFile).then(function(ogDeps){
  var depValues = _.map(ogDeps, function(dep){
    return dep.source.value
  })
  var deps = {}
  deps.local = _.filter(depValues, function(dep){
    return dep.match(crawlDeps.localRegex)
  })
  deps.native = _.filter(depValues, function(dep){
    return _.contains(natives, dep)
  })
  deps.npm = _.filter(depValues, function(dep){
    return !dep.match(crawlDeps.localRegex) && !_.contains(natives, dep)
  })
  return deps
}).then(console.log)

// { local: [ './test-2' ],
//   native: [ 'crypto' ],
//   npm: [ 'underscore', 'express', 'bluebird' ] }

[megawac]

No obvious flags, looks like youre on the right track! I'd be interested in seeing the solution when you iron out any other kinks