Optimize performance of crawler

[stereobooster]

Something like minimalcss doing. This, theoretically, should decrease memory usage.

await page.setRequestInterception(true)
page.on('request', req => {
  if (
    req.resourceType === 'stylesheet' ||
    req.resourceType === 'font' ||
    req.resourceType === 'image'
  ) {
    req.abort()
  } else {
    req.continue()
  }
})

Use something else instead of networkidle, because networkidle waits 0,5 second after last network activity finished. Instead we can use page.waitFor. This, theoretically, should speed up things.

See: https://github.com/peterbe/minimalcss/pull/118

[stereobooster]

this is not the best way to do this:

  page.on("response", async response => {
   ....
        } else {
          await page.evaluate(route => {
            const linkTag = document.createElement("link");
            linkTag.setAttribute("rel", "preload");
            linkTag.setAttribute("as", "image");
            linkTag.setAttribute("href", route);
            document.body.appendChild(linkTag);
          }, route);
        }

[stereobooster]

https://github.com/stereobooster/react-snap/issues/139

} else if (preconnectThirdParty) {
  const urlObj = url.parse(responseUrl);
  const domain = `${urlObj.protocol}//${urlObj.host}`;
  if (uniqueResources.has(domain)) return;
  await page.evaluate(route => {
    const linkTag = document.createElement("link");
    linkTag.setAttribute("rel", "preconnect");
    linkTag.setAttribute("href", route);
    document.head.appendChild(linkTag);
  }, domain);
  uniqueResources.add(domain);
}