Closed abgit closed 11 years ago
Hi, Here goes an additional crawlers list with 330 more referrer signatures. Feel free to add it in util.c .
{"SeoCheckBot", "Crawlers"}, {"MJ12bot", "Crawlers"}, {"Genieo", "Crawlers"}, {"proximic", "Crawlers"}, {"80legs", "Crawlers"}, {"ExB Language Crawler", "Crawlers"}, {"NaverBot", "Crawlers"}, {"Qirina Hurdler", "Crawlers"}, {"ChangeDetection", "Crawlers"}, {"IntegromeDB", "Crawlers"}, {"GrapeshotCrawler", "Crawlers"}, {"IstellaBot", "Crawlers"}, {"Yahoo!", "Crawlers"}, {"sogou spider", "Crawlers"}, {"AddThis.com", "Crawlers"}, {"Zookabot", "Crawlers"}, {"ShopWiki", "Crawlers"}, {"iCjobs", "Crawlers"}, {"search.KumKie.com", "Crawlers"}, {"ZumBot", "Crawlers"}, {"Netseer", "Crawlers"}, {"magpie-crawler", "Crawlers"}, {"rogerbot", "Crawlers"}, {"ShowyouBot", "Crawlers"}, {"bitlybot", "Crawlers"}, {"Spinn3r", "Crawlers"}, {"Jyxobot", "Crawlers"}, {"Woko", "Crawlers"}, {"coccoc", "Crawlers"}, {"Wotbox", "Crawlers"}, {"Mail.Ru bot", "Crawlers"}, {"Vagabondo", "Crawlers"}, {"uMBot-FC", "Crawlers"}, {"MetaJobBot", "Crawlers"}, {"SearchmetricsBot", "Crawlers"}, {"Infohelfer", "Crawlers"}, {"Peeplo Screenshot Bot", "Crawlers"}, {"Qualidator.com Bot", "Crawlers"}, {"Daumoa", "Crawlers"}, {"Ezooms", "Crawlers"}, {"TurnitinBot", "Crawlers"}, {"NetcraftSurveyAgent", "Crawlers"}, {"Crawler4j", "Crawlers"}, {"Aboundexbot", "Crawlers"}, {"aiHitBot", "Crawlers"}, {"VoilaBot", "Crawlers"}, {"AraBot", "Crawlers"}, {"YioopBot", "Crawlers"}, {"Nuhk", "Crawlers"}, {"yacybot", "Crawlers"}, {"SEOENGBot", "Crawlers"}, {"Najdi.si", "Crawlers"}, {"linkdex.com", "Crawlers"}, {"Yahoo! JAPAN", "Crawlers"}, {"psbot", "Crawlers"}, {"trendictionbot", "Crawlers"}, {"BLEXBot", "Crawlers"}, {"spbot", "Crawlers"}, {"webmastercoffee", "Crawlers"}, {"PaperLiBot", "Crawlers"}, {"QuerySeekerSpider", "Crawlers"}, {"FacebookExternalHit", "Crawlers"}, {"UASlinkChecker", "Crawlers"}, {"Browsershots", "Crawlers"}, {"MetaURI API", "Crawlers"}, {"woriobot", "Crawlers"}, {"oBot", "Crawlers"}, {"FyberSpider", "Crawlers"}, {"Steeler", "Crawlers"}, {"Blekkobot", "Crawlers"}, {"LinkAider", "Crawlers"}, {"SEOkicks-Robot", "Crawlers"}, {"firmilybot", "Crawlers"}, {"netEstate Crawler", "Crawlers"}, {"AMZNKAssocBot", "Crawlers"}, {"OpenindexSpider", "Crawlers"}, {"BacklinkCrawler", "Crawlers"}, {"HubSpot Connect", "Crawlers"}, {"WBSearchBot", "Crawlers"}, {"MetaHeadersBot", "Crawlers"}, {"UnisterBot", "Crawlers"}, {"Strokebot", "Crawlers"}, {"GeliyooBot", "Crawlers"}, {"bot-pge.chlooe.com", "Crawlers"}, {"ownCloud Server Crawler", "Crawlers"}, {"CirrusExplorer", "Crawlers"}, {"bixocrawler", "Crawlers"}, {"ProCogSEOBot", "Crawlers"}, {"Falconsbot", "Crawlers"}, {"Dlvr.it/1.0", "Crawlers"}, {"thumbshots-de-Bot", "Crawlers"}, {"200PleaseBot", "Crawlers"}, {"discoverybot", "Crawlers"}, {"MojeekBot", "Crawlers"}, {"Panscient", "Crawlers"}, {"Plukkie", "Crawlers"}, {"R6 bot", "Crawlers"}, {"bl.uk_lddc_bot", "Crawlers"}, {"SolomonoBot", "Crawlers"}, {"Grahambot", "Crawlers"}, {"Automattic", "Crawlers"}, {"emefgebot", "Crawlers"}, {"SBSearch", "Crawlers"}, {"PiplBot", "Crawlers"}, {"TinEye", "Crawlers"}, {"FlightDeckReportsBot", "Crawlers"}, {"fastbot crawler", "Crawlers"}, {"4seohuntBot", "Crawlers"}, {"Updownerbot", "Crawlers"}, {"JikeSpider", "Crawlers"}, {"NLNZ_IAHarvester2013", "Crawlers"}, {"Nigma.ru", "Crawlers"}, {"wsAnalyzer", "Crawlers"}, {"OpenWebSpider", "Crawlers"}, {"YodaoBot", "Crawlers"}, {"SpiderLing", "Crawlers"}, {"Esribot", "Crawlers"}, {"Thumbshots.ru", "Crawlers"}, {"BlogPulse", "Crawlers"}, {"NextGenSearchBot", "Crawlers"}, {"bot.wsowner.com", "Crawlers"}, {"wscheck.com", "Crawlers"}, {"Qseero", "Crawlers"}, {"drupact", "Crawlers"}, {"HuaweiSymantecSpider", "Crawlers"}, {"PagePeeker", "Crawlers"}, {"HomeTags", "Crawlers"}, {"facebookplatform", "Crawlers"}, {"Pixray-Seeker", "Crawlers"}, {"BDFetch", "Crawlers"}, {"MeMoNewsBot", "Crawlers"}, {"ProCogBot", "Crawlers"}, {"WillyBot", "Crawlers"}, {"peerindex", "Crawlers"}, {"Job Roboter Spider", "Crawlers"}, {"MLBot", "Crawlers"}, {"WebNL", "Crawlers"}, {"Peepowbot", "Crawlers"}, {"Semager", "Crawlers"}, {"MIA Bot", "Crawlers"}, {"Eurobot", "Crawlers"}, {"DripfeedBot", "Crawlers"}, {"webinatorbot", "Crawlers"}, {"Whoismindbot", "Crawlers"}, {"Bad-Neighborhood", "Crawlers"}, {"Hailoobot", "Crawlers"}, {"akula", "Crawlers"}, {"MetamojiCrawler", "Crawlers"}, {"Page2RSS", "Crawlers"}, {"EasyBib AutoCite", "Crawlers"}, {"suggybot", "Crawlers"}, {"NerdByNature.Bot", "Crawlers"}, {"EventGuruBot", "Crawlers"}, {"quickobot", "Crawlers"}, {"gonzo", "Crawlers"}, {"CCBot", "Crawlers"}, {"bnf.fr_bot", "Crawlers"}, {"UptimeRobot", "Crawlers"}, {"Influencebot", "Crawlers"}, {"MSRBOT", "Crawlers"}, {"KeywordDensityRobot", "Crawlers"}, {"heritrix", "Crawlers"}, {"Ronzoobot", "Crawlers"}, {"RyzeCrawler", "Crawlers"}, {"ScoutJet", "Crawlers"}, {"Twikle", "Crawlers"}, {"SWEBot", "Crawlers"}, {"RADaR-Bot", "Crawlers"}, {"DCPbot", "Crawlers"}, {"Castabot", "Crawlers"}, {"percbotspider", "Crawlers"}, {"WeSEE:Search", "Crawlers"}, {"CatchBot", "Crawlers"}, {"imbot", "Crawlers"}, {"EdisterBot", "Crawlers"}, {"WASALive-Bot", "Crawlers"}, {"Accelobot", "Crawlers"}, {"PostPost", "Crawlers"}, {"factbot", "Crawlers"}, {"Setoozbot", "Crawlers"}, {"biwec", "Crawlers"}, {"GarlikCrawler", "Crawlers"}, {"Search17Bot", "Crawlers"}, {"Lijit", "Crawlers"}, {"MetaGeneratorCrawler", "Crawlers"}, {"Robots_Tester", "Crawlers"}, {"JUST-CRAWLER", "Crawlers"}, {"Apercite", "Crawlers"}, {"pmoz.info", "Crawlers"}, {"LemurWebCrawler", "Crawlers"}, {"Covario-IDS", "Crawlers"}, {"Holmes", "Crawlers"}, {"RankurBot", "Crawlers"}, {"DotBot", "Crawlers"}, {"envolk", "Crawlers"}, {"LexxeBot", "Crawlers"}, {"adressendeutschland", "Crawlers"}, {"StackRambler", "Crawlers"}, {"Abrave Spider", "Crawlers"}, {"EvriNid", "Crawlers"}, {"arachnode.net", "Crawlers"}, {"CamontSpider", "Crawlers"}, {"wikiwix-bot", "Crawlers"}, {"Nymesis", "Crawlers"}, {"trendictionbot", "Crawlers"}, {"Sitedomain-Bot", "Crawlers"}, {"SEODat", "Crawlers"}, {"SygolBot", "Crawlers"}, {"Snapbot", "Crawlers"}, {"OpenCalaisSemanticProxy", "Crawlers"}, {"ZookaBot", "Crawlers"}, {"CligooRobot", "Crawlers"}, {"cityreview", "Crawlers"}, {"nworm", "Crawlers"}, {"AboutUsBot", "Crawlers"}, {"ICC-Crawler", "Crawlers"}, {"SBIder", "Crawlers"}, {"TwengaBot", "Crawlers"}, {"Dot TK - spider", "Crawlers"}, {"EuripBot", "Crawlers"}, {"ParchBot", "Crawlers"}, {"Peew", "Crawlers"}, {"AntBot", "Crawlers"}, {"YRSpider", "Crawlers"}, {"Urlfilebot", "Crawlers"}, {"Gaisbot", "Crawlers"}, {"WatchMouse", "Crawlers"}, {"Tagoobot", "Crawlers"}, {"Motoricerca", "Crawlers"}, {"WebWatch", "Crawlers"}, {"urlfan-bot", "Crawlers"}, {"StatoolsBot", "Crawlers"}, {"page_verifier", "Crawlers"}, {"SSLBot", "Crawlers"}, {"SAI Crawler", "Crawlers"}, {"DomainDB", "Crawlers"}, {"LinkWalker", "Crawlers"}, {"WMCAI_robot", "Crawlers"}, {"voyager", "Crawlers"}, {"copyright sheriff", "Crawlers"}, {"Ocelli", "Crawlers"}, {"amibot", "Crawlers"}, {"abby", "Crawlers"}, {"NetResearchServer", "Crawlers"}, {"VideoSurf_bot", "Crawlers"}, {"XML Sitemaps Generator", "Crawlers"}, {"BlinkaCrawler", "Crawlers"}, {"nodestackbot", "Crawlers"}, {"Pompos", "Crawlers"}, {"taptubot", "Crawlers"}, {"BabalooSpider", "Crawlers"}, {"Yaanb", "Crawlers"}, {"Girafabot", "Crawlers"}, {"livedoor ScreenShot", "Crawlers"}, {"eCairn-Grabber", "Crawlers"}, {"FauBot", "Crawlers"}, {"Toread-Crawler", "Crawlers"}, {"Setoozbot", "Crawlers"}, {"MetaURI", "Crawlers"}, {"L.webis", "Crawlers"}, {"Web-sniffer", "Crawlers"}, {"FairShare", "Crawlers"}, {"Ruky-Roboter", "Crawlers"}, {"ThumbShots-Bot", "Crawlers"}, {"BotOnParade", "Crawlers"}, {"Amagit.COM", "Crawlers"}, {"HatenaScreenshot", "Crawlers"}, {"HolmesBot", "Crawlers"}, {"dotSemantic", "Crawlers"}, {"Karneval-Bot", "Crawlers"}, {"HostTracker.com", "Crawlers"}, {"AportWorm", "Crawlers"}, {"XmarksFetch", "Crawlers"}, {"FeedFinder", "Crawlers"}, {"CorpusCrawler", "Crawlers"}, {"Willow Internet Crawler", "Crawlers"}, {"OrgbyBot", "Crawlers"}, {"GingerCrawler", "Crawlers"}, {"pingdom.com_bot", "Crawlers"}, {"baypup", "Crawlers"}, {"Linguee Bot", "Crawlers"}, {"Mp3Bot", "Crawlers"}, {"192.comAgent", "Crawlers"}, {"Surphace Scout", "Crawlers"}, {"WikioFeedBot", "Crawlers"}, {"Szukacz", "Crawlers"}, {"DBLBot", "Crawlers"}, {"Thumbnail.CZ", "Crawlers"}, {"LinguaBot", "Crawlers"}, {"GurujiBot", "Crawlers"}, {"Charlotte", "Crawlers"}, {"50.nu", "Crawlers"}, {"SanszBot", "Crawlers"}, {"moba-crawler", "Crawlers"}, {"HeartRails_Capture", "Crawlers"}, {"SurveyBot", "Crawlers"}, {"MnoGoSearch", "Crawlers"}, {"smart.apnoti.com Robot", "Crawlers"}, {"Topicbot", "Crawlers"}, {"JadynAveBot", "Crawlers"}, {"OsObot", "Crawlers"}, {"WebImages", "Crawlers"}, {"WinWebBot", "Crawlers"}, {"Scooter", "Crawlers"}, {"Scarlett", "Crawlers"}, {"GOFORITBOT", "Crawlers"}, {"DKIMRepBot", "Crawlers"}, {"Yanga", "Crawlers"}, {"DNS-Digger-Explorer", "Crawlers"}, {"Robozilla", "Crawlers"}, {"adidxbot", "Crawlers"}, {"YowedoBot", "Crawlers"}, {"botmobi", "Crawlers"}, {"Fooooo_Web_Video_Crawl", "Crawlers"}, {"UptimeDog", "Crawlers"}, {"^Nail", "Crawlers"}, {"Metaspinner", "Crawlers"}, {"Touche", "Crawlers"}, {"RSSMicro", "Crawlers"}, {"SniffRSS", "Crawlers"}, {"FeedCatBot", "Crawlers"}, {"WebRankSpider", "Crawlers"}, {"Flatland Industries Web Spider", "Crawlers"}, {"DealGates Bot", "Crawlers"}, {"Link Valet Online", "Crawlers"}, {"Shelob", "Crawlers"}, {"Technoratibot", "Crawlers"}, {"Flocke bot", "Crawlers"}, {"FollowSite Bot", "Crawlers"}, {"Visbot", "Crawlers"},
btw, there's currently a duplicate crawler entry: KaloogaBot
best,
Merged. Thanks!
Hi, Here goes an additional crawlers list with 330 more referrer signatures. Feel free to add it in util.c .
btw, there's currently a duplicate crawler entry: KaloogaBot
best,