Problem writing a resolver that resolves Http Urls

[llacroix]

Here's what I have so far:

const path = require('path');
const fetch = require('node-fetch');
const url = require('url')
const fs = require('promise-fs');
const sha1 = require('sha1')

class HttpResolver {

  async download_save(request, resolveContext) {
      console.log(request, resolveContext)

      var target = url.parse(request.request)
      var response = await fetch(request.request)
      var content = await response.text()

      try {
        await fs.stat('_remote')
      } catch(exc) {
        await fs.mkdir('_remote')
      }

      var filename = `${sha1(request.request)}.js`
      var file_path = `_remote/${filename}`

      await fs.writeFile(file_path, content)

      var abs_path = path.resolve(file_path)
      var url_path = `${target.protocol}://${target.hostname}/`
      var obj = {
        path: abs_path,
        request: request.request,
        query: '',
      }

      console.log(`${request.request} saved to ${abs_path}`)

      return obj
  }

  apply(resolver) {
    var self = this
    const target = resolver.ensureHook("resolved")

    resolver.getHook("module")
      .tapAsync("FetchResolverPlugin", (request, resolveContext, callback) => {
        self.download_save(request, resolveContext)
          .then((obj) => resolver.doResolve(target, obj, `${obj.request} -> ${obj.path}`, resolveContext, callback))
          .catch((err) => {
            console.log(err)
            callback()
          })
      })
  }
}

It does get a few urls that starts with http or https. But when it fetch and resolve a file containing an import as such:

import {something} from '/something/relative/to/url';

It doesn't even trigger an new request. In theory, I could track the actual parent path and compute the actual path to download but for some reasons, it simply fail to resolve with something like this:

ERROR in ./_remote/eb24e298beb533b66a611bba9a85f148cb23b848.js
Module not found: Error: Can't resolve '/-/@pika/polyfill@v0.0.3/dist=es2017/polyfill.js' in '/local/path/to/sale_static/_remote'
 @ ./_remote/eb24e298beb533b66a611bba9a85f148cb23b848.js 25:0-58
 @ ./_remote/f80b922b2dd42bdfaaba4e9f4fc3c84b9cc04fca.js
 @ ./src/index.js

But '/-/@pika/polyfill@v0.0.3...' is relative to an url and not the path. How do I tell the resolver to not check for the path I resolved for the previous file?

[alexander-akait]

@llacroix Sorry for delay, do you have problems?

[sokra]

enhanced-resolve only handles resolving on fs. URL resolving would be handled on webpack side.

Maybe you want to specify /^\// as external? In future we will have a module external type, but import external type should also work for now.

[llacroix]

@evilebottnawi I did work around a few bugs and it's working much better than when I wrote that. I'll post an updated version tomorrow.

Back story for this extension if it wasn't clear: It's designed to package CDN hosted resources or local resources. A framework I have to use doesn't properly handle scripts of type module and it tries to bundle everything instead of keeping them as is. It's a bit counter productive to package CDN hosted files but in order to make JS of type module work I don't have much choice than to package them.

As far as I remember, the only I had was related to the production release. I'm not sure I fixed that and somehow assets would work in development builds but not in production builds.

Thanks anyway for answering.

[llacroix]

Here:

class HttpResolver {
  constructor() {
    this.origins = {}
  }

  async handle_relative(request, resolveContext) {
    try {
      // file exist so we don't go further
      await fs.stat(request.path)
      return request
    } catch(err) {
    }

    var origin = this.get_origin(request.context.issuer)

    try {
      var target =  url.parse(origin)
    }
    catch (err) {
      // don't try to use if you can't parse the url
      return request
    }

    target.pathname = path.resolve(target.pathname, request.path)

    var obj = {
      ...request,
      path: '',
      request: url.format(target),
      prev_request: request.request,
    }

    return await this.download_save(obj, resolveContext)
  }

  async download_save(request, resolveContext) {
      var target = url.parse(request.request)
      var response = await fetch(request.request)
      var content = await response.text()

      try {
        await fs.stat('_remote')
      } catch(exc) {
        await fs.mkdir('_remote')
      }

      var filename = `${sha1(request.request)}.js`
      var file_path = `_remote/${filename}`

      await fs.writeFile(file_path, content)

      var abs_path = path.resolve(file_path)
      var url_path = `${target.protocol}//${target.hostname}/`

      var obj = {
        ...request,
        path: abs_path,
        relativePath: url_path
      }

      this.set_origin(abs_path, request.request) 

      return obj
  }

  get_origin(path) {
    return this.origins[path]
  }

  set_origin(path, origin) {
    this.origins[path] = origin
  }

  apply(resolver) {
    var self = this
    const target = resolver.ensureHook("resolved")
    const target_relative = resolver.ensureHook("described-relative")

    resolver
      .getHook("relative")
      .tapAsync("FetchResolverRelativePlugin", (request, resolveContext, callback) => {
        //resolver.doResolve(target_relative, request, '', resolveContext, callback)
        self.handle_relative(request, resolveContext)
          .then((obj) => {
            var message = `${obj.request} -> ${obj.path}`
            resolver.doResolve(target_relative, obj, message, resolveContext, callback)
          })
          .catch((err) => {
            return callback()
          })
      })

    //resolver.getHook("module")
    resolver
      .getHook("module")
      .tapAsync("FetchResolverPlugin", (request, resolveContext, callback) => {
        self.download_save(request, resolveContext)
          .then((obj) => {
            var message = `Resolve ${obj.request} in ${obj.path}`
            resolver.doResolve(target, obj, message, resolveContext, callback)
          })
          .catch((err) => {
            return callback()
          })
      })
  }
}