Tool to re-root, de-root existing project.

[coolaj86]

I discovered this because I've inherited a project with (literally) hundreds of files, many deeply nested.

(I was searching around a bit after having realized Windows symlinks wouldn't work - not yet knowing about junctions - and stumbled upon a blog that mentioned this while hoping to find an updated 'official' solution)

Anyway, I'm writing a tool that will convert a project's requires over to "the basetag way". It may provide a nice starting point to build something that could be put in this project as a bin script and be able to do something like this:

npx basetag rebase-requires

[coolaj86]

My first pass:

Usage

Must be run from the package root.

node reroot-requires.js

Example output:

# [ ./src/middleware no-cache.js ]
# $/src/api/util/error.js <= ../api/util/error.js
# $/src/models/user.js <= ../models/user.js
git add src/middleware/no-cache.js ;

# [ ./src/middleware/errors api-error-handler.js ]
# $/src/api/util/error.js <= ../../api/util/error.js
git add src/middleware/errors/api-error-handler.js ;

Source

'use strict';

var path = require('path');
var fs = require('fs').promises;

// assume that the command is run from the package root
var pkglen = process.cwd().length; // no trailing '/'

// matches requires that start with '../' (leaves child-relative requires alone)
var parentRequires = /(require\(['"])(\.\..*)(['"]\))/g;
var parentImports = /(import\s*\(?[\w\s{}]*['"])(\.\..*)(['"]\)?)/g;
// matches requires that start with './' (includes child-relative requires)
var allRequires = /(require\(['"])(\..*)(['"]\))/g;
var allImports = /(import\s*\(?[\w\s{}]*['"])(\..*)(['"]\)?)/g;

// add flag parsing
var opts = {};
[['all', '-a', '--all']].forEach(function (flags) {
  flags.slice(1).some(function (alias) {
    if (process.argv.slice(2).includes(alias)) {
      opts[flags[0]] = true;
    }
  });
});

async function rootify(pathname, filename) {
  // TODO not sure if this load order is exactly correct
  var loadable = ['.js', '.cjs', '.mjs', '.json'];
  if (!loadable.includes(path.extname(filename))) {
    //console.warn("# warn: skipping non-js file '%s'", filename);
    return;
  }

  var dirname = path.dirname(pathname);
  pathname = path.resolve(pathname);

  var requiresRe;
  var importsRe;
  if (opts.all) {
    requiresRe = allRequires;
    importsRe = allImports;
  } else {
    requiresRe = parentRequires;
    importsRe = parentImports;
  }

  var oldTxt = await fs.readFile(pathname, 'utf8');
  var changes = [];
  var txt = oldTxt.replace(requiresRe, replaceImports).replace(importsRe, replaceImports);

  function replaceImports(_, a, b, c) {
    //console.log(a, b, c);
    // a = 'require("' OR 'import("' OR 'import "'
    // b = '../../foo.js'
    // c = '")' OR ''

    // /User/me/project/lib/foo/bar + ../foo.js
    // becomes $/lib/foo/foo.js
    var pkgpath = '$' + path.resolve(dirname + '/', b).slice(pkglen);

    var result = a + pkgpath + c;
    changes.push([pkgpath, b]);
    return result;
  }

  if (oldTxt != txt) {
    console.info('\n# [', dirname, filename, ']');
    changes.forEach(function ([pkgpath, b]) {
      console.log('#', pkgpath, '<=', b);
    });
    await fs.writeFile(pathname, txt);
    console.info('git add', path.join(dirname, filename), ';');
  }
}

walk('.', async function (err, pathname, dirent) {
  if (['node_modules', '.git'].includes(dirent.name)) {
    return false;
  }

  if (!dirent.isFile()) {
    return;
  }

  return rootify(pathname, dirent.name).catch(function (e) {
    console.error(e);
  });
});

@root/walk:

async function walk(pathname, walkFunc, _dirent) {
  const fs = require('fs').promises;
  const path = require('path');
  const _pass = (err) => err;
  let dirent = _dirent;

  let err;

  // special case: walk the very first file or folder
  if (!dirent) {
    let filename = path.basename(path.resolve(pathname));
    dirent = await fs.lstat(pathname).catch(_pass);
    if (dirent instanceof Error) {
      err = dirent;
    } else {
      dirent.name = filename;
    }
  }

  // run the user-supplied function and either skip, bail, or continue
  err = await walkFunc(err, pathname, dirent).catch(_pass);
  if (false === err) {
    // walkFunc can return false to skip
    return;
  }
  if (err instanceof Error) {
    // if walkFunc throws, we throw
    throw err;
  }

  // "walk does not follow symbolic links"
  // (doing so could cause infinite loops)
  if (!dirent.isDirectory()) {
    return;
  }
  let result = await fs.readdir(pathname, { withFileTypes: true }).catch(_pass);
  if (result instanceof Error) {
    // notify on directory read error
    return walkFunc(result, pathname, dirent);
  }
  for (let entity of result) {
    await walk(path.join(pathname, entity.name), walkFunc, entity);
  }
}

[janniks]

Wow! Great idea 💡 would be happy to have that in basetag's bin. Would even use it frequently, IDEs and editors mostly prefer the ../s on auto-complete...

[janniks]

Script looks great, I'll test it out over the weekend 👍🏻

[janniks]

Works pretty well so far — I can take over if you wish...

Leaving myself some notes for the script for later:

• [ ] add Windows Support (replace with path.sep etc.), check if pkglen still works as previously
• [ ] add (un-hardcode) ignore/filter support, read filter list from .gitignore (trim and ignore #/empty lines)
• [ ] add import support (in addition to require)
• [ ] add mode for asking before each edit
• [ ] add npm prompt package for configuring these options (import/require, continue (i.e. overwrite files))
• [ ] add npm Commander.js package for command line options (e.g. force -y)
• [ ] think about publishing as separate package

[coolaj86]

For v1 I'd say make it simple, in repo, no dependencies - I could update this to use fs.readdir({withFileTypes: true}) rather than walk. If you need options, just process.argv.slice(2).includes("-y").

• [ ] Windows support should not need any changes, as require and import should use windows paths and path.resolve will do "the right thing" with backslashes (needs testing).
• [ ] support import (I made the change above, but I need to test it)

add mode for asking before each edit

Rather than this, I'd say add the reverse operation. Aside from that, git already handles the problem here.

[coolaj86]

I updated the script above:

• updated and tested the import support
• removed walk dependency
• added simple flag parsing
• support replacing parent-only or ALL paths

There's a LOT of ways to use imports, but most of them aren't useful in node (unless it's transpiled from some other language). I only support the basic usages:

import { x } as x from "whatever"
import x from "whatever"
await import("whatever").default

[janniks]

Thanks for the input, I like your proposal. 👍🏻

Can you create a branch (on a fork) and commit your script? I will then create a feature branch and merge in your branch, so I can base off your work (and keep your contributions/commits) 😉

In that feature branch I will switch basetag to a more script oriented approach. I'm thinking about a simple CLI that breaks down into a few scripts:

• npx basetag link would be used instead of the postinstall.
• npx basetag rebase <--dry-run> would be used to rebase to the $ aka your script
• npx basetag restore to undo rebasing