Crawler or Sitemap is too slow

[edwhiskey]

The Crawler or Sitemap is too slow? Any Chance to improve it to allow multisite crawler which can use more CPU and GPU power?

[edwhiskey]

I edited the TS as below, and now it get much better performance on Crawler

import axios from "axios"; import { CheerioAPI, load } from "cheerio"; import puppeteerFetch from "./puppeteer-fetch"; import { Worker, isMainThread, parentPort, workerData } from "worker_threads";

type CrawlResult = { links: Set; errors: Set; };

const visitedLinks: Set = new Set(); const errorLinks: Set = new Set(); const queuedLinks: Set = new Set();

const crawlPage = async (url: string) => { // Your crawling logic here const html = await axios.get(url); const $ = load(html.data); const links = new Set(); // Add link to set return { links }; };

import axios from "axios"; import { CheerioAPI, load } from "cheerio"; import puppeteerFetch from "./puppeteer-fetch"; import { Worker, isMainThread, parentPort, workerData } from "worker_threads";

type CrawlResult = { links: Set; };

const crawlPage = async (url: string) => { // Your crawling logic here const html = await axios.get(url); const $ = load(html.data); const links = new Set(); // Add link to set return { links }; };

const crawlUrls = async (urls: string[]) => { const results: CrawlResult[] = []; for (const url of urls) { const result = await crawlPage(url); results.push(result); } return results; };

if (!isMainThread) { // In worker thread, get the data from parent const { crawlUrl } = workerData;

// Do some work here... crawlUrls([crawlUrl]) .then((results) => { console.log(results); }) .catch((err) => { console.error(err); }); } else { // In main thread, spawn multiple workers const urlsToCrawl = ["url1", "url2", ...]; const numWorkers = process.cpuCount(); const chunkSize = Math.ceil(urlsToCrawl.length / numWorkers);

for (let i = 0; i < numWorkers; i++) { const start = i * chunkSize; const end = Math.min(start + chunkSize, urlsToCrawl.length); const chunkUrls = urlsToCrawl.slice(start, end); const worker = new Worker(__filename, { workerData: { crawlUrl: chunkUrls }, }); worker.on("message", (result) => { console.log(result); }); } }

export const crawl = async ( startUrl: string, maxDepth = 2, maxLinks = 20, usePuppeteerFetch = false ): Promise => { const queue: { url: string; depth: number }[] = [{ url: startUrl, depth: 0 }]; const fetchedLinks: Set = new Set();

while (queue.length > 0 && visitedLinks.size < maxLinks) { const batch = queue.splice(0, Math.min(queue.length, maxLinks - visitedLinks.size));

await Promise.all(
  batch.map(async ({ url, depth }) => {
    if (visitedLinks.has(url) || depth > maxDepth) {
      return;
    }

    try {

      let $: CheerioAPI;

      if (usePuppeteerFetch) {
        const response = await puppeteerFetch(url);
        $ = load(response);
      } else {
        const response = await axios.get(url, {
          headers: {
            Accept: "text/html",
            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3",
          },
        });

        const contentType = response.headers['content-type'];
        if (!contentType || !contentType.includes("text/html")) {
          return;
        }

        $ = load(response.data);
      }
      visitedLinks.add(url);
      fetchedLinks.add(url);

      $("a").each((_, element) => {
        const href = $(element).attr("href");
        if (!href) {
          return;
        }

        const absoluteUrl = normalizeUrl(new URL(href, url).href);
        if (isSameDomain(absoluteUrl, startUrl) && !visitedLinks.has(absoluteUrl) && !queuedLinks.has(absoluteUrl)) {
          queue.push({ url: absoluteUrl, depth: depth + 1 });
          queuedLinks.add(absoluteUrl);
        }
      });
    } catch (error: any) {
      console.error(`Failed to fetch ${url}:`, error?.message || error);
      errorLinks.add(url);
    }
  })
);

}

return { links: fetchedLinks, errors: errorLinks }; };

const isSameDomain = (url1: string, url2: string): boolean => { const { hostname: host1 } = new URL(url1); const { hostname: host2 } = new URL(url2); return host1 === host2; };

const normalizeUrl = (url: string): string => { try { const urlObj = new URL(url); urlObj.hash = ''; return urlObj.href; } catch (error) { return url; } };