BuilderIO / gpt-crawler

Crawl a site to generate knowledge files to create your own custom GPT from a URL
https://www.builder.io/blog/custom-gpt
ISC License
18.14k stars 1.88k forks source link

cookie example #158

Open cmbcbe opened 3 months ago

cmbcbe commented 3 months ago

hello,

i cannot find a way to set cookie for authenticated crawling. I have export cookie from authenticated session with chrome extension "export this cookie" https://chromewebstore.google.com/detail/exportthiscookie/dannllckdimllhkiplchkcaoheibealk Now i try to put in my config.ts without success. Here is my code

import { Config } from "./src/config";

export const defaultConfig: Config = {
  url: "https://doku.mydomain.com",
  match: "https://doku.mydomain.com/**",
  maxPagesToCrawl: 5000,
  outputFileName: "doku.json",
  maxTokens: 2000000,
  cookie: {"DokuWiki":"os2svsfsplem3e6f5npkag","FCK_NmSp":"main_page","plugin_acmenu_open_items":"[]","DW7fa065a06cb74b536c1e56ac6d3":"Y2VkcmljLm1hcmNvdXg%3D%7C1%7CT7Ljmh7OC%2Fx5f%2FzqpVGSYS%2FkZzW%2BTI%2FbREg29quTC8ySrWo2R9SOcvOeu","FCK_NmSp_acl":"os2svsfsplem3e6f590nbnpkag","FCK_SCAYT_AUTO":"on","FCK_SCAYT_LANG":"fr_FR"}
};

this give the error The expected type comes from property 'cookie' which is declared here on type '{ url: string; match: (string | string[]) & (string | string[] | undefined); maxPagesToCrawl: number; outputFileName: string; exclude?: string | string[] | undefined; selector?: string | undefined; ... 5 more ...; maxTokens?: number | undefined; }'

Is it possible to get a syntax example of cookie usage? Thank you