<?php
require_once('wfUtils.php');
class wfCrawl {
	public static function isCrawler($UA){
		foreach(self::$patterns as $pat){
			if(preg_match($pat . 'i', $UA)){
				return true;
			}
		}
		return false;
	}
	public static function verifyCrawlerPTR($hostPattern, $IP){
		global $wpdb; $table = $wpdb->base_prefix . 'wfCrawlers';
		$db = new wfDB();
		$IPn = wfUtils::inet_aton($IP);
		$status = $db->querySingle("select status from $table where IP=%s and patternSig=UNHEX(MD5('%s')) and lastUpdate > unix_timestamp() - %d", $IPn, $hostPattern, WORDFENCE_CRAWLER_VERIFY_CACHE_TIME);
		if($status){
			if($status == 'verified'){
				return true;
			} else {
				return false;
			}
		}
		$wfLog = new wfLog(wfConfig::get('apiKey'), wfUtils::getWPVersion());
		$host = wfUtils::reverseLookup($IP);
		if(! $host){ 
			$db->queryWrite("insert into $table (IP, patternSig, status, lastUpdate, PTR) values (%s, UNHEX(MD5('%s')), '%s', unix_timestamp(), '%s') ON DUPLICATE KEY UPDATE status='%s', lastUpdate=unix_timestamp(), PTR='%s'", $IPn, $hostPattern, 'noPTR', '', 'noPTR', '');
			return false; 
		}
		if(preg_match($hostPattern, $host)){
			$resultIPs = gethostbynamel($host);
			$addrsMatch = false;
			foreach($resultIPs as $resultIP){
				if($resultIP == $IP){
					$addrsMatch = true;
					break;
				}
			}
			if($addrsMatch){
				$db->queryWrite("insert into $table (IP, patternSig, status, lastUpdate, PTR) values (%s, UNHEX(MD5('%s')), '%s', unix_timestamp(), '%s') ON DUPLICATE KEY UPDATE status='%s', lastUpdate=unix_timestamp(), PTR='%s'", $IPn, $hostPattern, 'verified', $host, 'verified', $host);
				return true;
			} else {
				$db->queryWrite("insert into $table (IP, patternSig, status, lastUpdate, PTR) values (%s, UNHEX(MD5('%s')), '%s', unix_timestamp(), '%s') ON DUPLICATE KEY UPDATE status='%s', lastUpdate=unix_timestamp(), PTR='%s'", $IPn, $hostPattern, 'fwdFail', $host, 'fwdFail', $host);
				return false;
			}
		} else {
			$db->queryWrite("insert into $table (IP, patternSig, status, lastUpdate, PTR) values (%s, UNHEX(MD5('%s')), '%s', unix_timestamp(), '%s') ON DUPLICATE KEY UPDATE status='%s', lastUpdate=unix_timestamp(), PTR='%s'", $IPn, $hostPattern, 'badPTR', $host, 'badPTR', $host);
			return false;
		}
	}
	public static function isBingCrawler(){
		$UA = (isset($_SERVER['HTTP_USER_AGENT']) ? $_SERVER['HTTP_USER_AGENT'] : '');
		foreach(self::$bingPat as $pat){
			if(preg_match($pat . 'i', $UA)){
				return true;
			}
		}
		return false;
	}
	private static $bingPat = array(
'@^Mozilla/5\\.0 \\(compatible; bingbot/2\\..*http\\://www\\.bing\\.com/bingbot\\.htm\\)$@',
'@^msnbot\\-webmaster/1\\.0 \\(.*http\\://search\\.msn\\.com/msnbot\\.htm\\)$@',
'@^msnbot\\-NewsBlogs/2\\..* \\(\\+http\\://search\\.msn\\.com/msnbot\\.htm\\)$@',
'@^renlifangbot/1\\.0 \\(.http\\://search\\.msn\\.com/msnbot\\.htm\\)$@',
'@^adidxbot/1\\.1 \\(.http\\://search\\.msn\\.com/msnbot\\.htm\\)$@',
'@^msnbot\\-media/1\\.1.*$@',
'@^msnbot\\-media/1\\.0.*$@',
'@^msnbot\\-products$@',
'@^msnbot/2\\.0b.*$@',
'@^msnbot/1\\.0.*$@',
'@^msnbot/1\\.1.*$@'
	);
	public static function isGooglebot(){
		$UA = (isset($_SERVER['HTTP_USER_AGENT']) ? $_SERVER['HTTP_USER_AGENT'] : '');
		if(preg_match('/Googlebot\/\d\.\d/', $UA)){ // UA: Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html) or (rarely used): Googlebot/2.1 (+http://www.google.com/bot.html)
			return true;
		}
		return false;
	}
	public static function isGoogleCrawler(){
		$UA = (isset($_SERVER['HTTP_USER_AGENT']) ? $_SERVER['HTTP_USER_AGENT'] : '');
		foreach(self::$googPat as $pat){
			if(preg_match($pat . 'i', $UA)){
				return true;
			}
		}
		return false;
	}
	private static $googPat = array(
'@^Mozilla/5\\.0 \\(.*Google Keyword Tool.*\\)$@',
'@^Mozilla/5\\.0 \\(.*Feedfetcher\\-Google.*\\)$@',
'@^Feedfetcher\\-Google\\-iGoogleGadgets.*$@',
'@^searchbot admin\\@google\\.com$@',
'@^Google\\-Site\\-Verification.*$@',
'@^Google OpenSocial agent.*$@',
'@^.*Googlebot\\-Mobile/2\\..*$@',
'@^AdsBot\\-Google\\-Mobile.*$@',
'@^google \\(.*Enterprise.*\\)$@',
'@^Mediapartners\\-Google.*$@',
'@^GoogleFriendConnect.*$@',
'@^googlebot\\-urlconsole$@',
'@^.*Google Web Preview.*$@',
'@^Feedfetcher\\-Google.*$@',
'@^AppEngine\\-Google.*$@',
'@^Googlebot\\-Video.*$@',
'@^Googlebot\\-Image.*$@',
'@^Google\\-Sitemaps.*$@',
'@^Googlebot/Test.*$@',
'@^Googlebot\\-News.*$@',
'@^.*Googlebot/2\\.1.*$@',
'@^AdsBot\\-Google.*$@',
'@^Google$@'
	);
	private static $patterns = array(
'@^Mozilla/5\\.0 \\(Windows; U; Windows NT 5\\.1; en\\-US\\) Speedy Spider \\(http\\://www\\.entireweb\\.com/about/search_tech/speedy_spider/\\)$@',
'@^Mozilla/4\\.0 \\(compatible; MSIE .*; Windows NT; Girafabot; girafabot at girafa dot com; http\\://www\\.girafa\\.com\\)$@',
'@^Mozilla/5\\.0 \\(compatible; MSIE 7\\.0; MSIE 6\\.0; ScanAlert; \\+http\\://www\\.scanalert\\.com/bot\\.jsp\\) Firefox/.*$@',
'@^Mozilla/5\\.0 \\(compatible; Speedy Spider; http\\://www\\.entireweb\\.com/about/search_tech/speedy_spider/\\)$@',
'@^Runnk online rss reader \\: http\\://www\\.runnk\\.com/ \\: RSS favorites \\: RSS ranking \\: RSS aggregator.*$@',
'@^SaladSpoon/ShopSalad 1\\..* \\(Search Engine crawler for ShopSalad\\.com; .*; crawler\\@shopsalad\\.com\\)$@',
'@^Mozilla/5\\.0 \\(compatible; Kyluka crawl; http\\://www\\.kyluka\\.com/crawl\\.html; crawl\\@kyluka\\.com\\)$@',
'@^Mozilla/5\\.0 \\(X11; U; Linux.*; .*\\) AppleWebKit/.*\\(KHTML, like Gecko\\) Safari/.* Epiphany/2\\.30\\..*$@',
'@^DomainCrawler/1\\.0 \\(info\\@domaincrawler\\.com; http\\://www\\.domaincrawler\\.com/domains/view/.*\\)$@',
'@^Mozilla/5\\.0 \\(compatible; YoudaoBot/1\\..*; http\\://www\\.youdao\\.com/help/webmaster/spider/.*\\)$@',
'@^Mozilla/5\\.0 \\(compatible; Yahoo\\! DE Slurp; http\\://help\\.yahoo\\.com/help/us/ysearch/slurp\\)$@',
'@^Metaspinner/0\\.01 \\(Metaspinner; http\\://www\\.meta\\-spinner\\.de/; support\\@meta\\-spinner\\.de/\\)$@',
'@^browscap updater; interval\\:weekly; server\\:rohan\\.doppy\\.nl; questions\\:support\\@doppy\\.nl;$@',
'@^Mozilla/5\\.0 \\(compatible; Scrubby/.*; \\+http\\://www\\.scrubtheweb\\.com/abs/meta\\-check\\.html\\)$@',
'@^Mozilla/5\\.0 \\(compatible; Yahoo\\! Slurp.*;.*http\\://help\\.yahoo\\.com/help/us/ysearch/slurp\\)$@',
'@^Automated Browscap\\.ini Updater\\. To report issues contact us at\\+http\\://www\\.skycomp\\.ca$@',
'@^Mozilla/4\\.0 \\(compatible; http\\://search\\.thunderstone\\.com/texis/websearch/about\\.html\\)$@',
'@^Mozilla/5\\.0 \\(compatible; Baiduspider/2\\.0; \\+http\\://www\\.baidu\\.com/search/spider\\.html\\)$@',
'@^Mozilla/5\\.0 \\(compatible; KaloogaBot; http\\://www\\.kalooga\\.com/info\\.html.page\\=crawler\\)$@',
'@^compatible; Mozilla 4\\.0; MSIE 5\\.5; \\(SqwidgeBot v1\\.01 \\- http\\://www\\.sqwidge\\.com/bot/\\)$@',
'@^Automated Browscap\\.ini Updater\\. Gary contact me at serge\\@skycomp\\.ca with problems\\.$@',
'@^Mozilla/5\\.0 \\(compatible; Yahoo\\! Slurp China.*;.*http\\://misc\\.yahoo\\.com\\.cn/help\\.html\\)$@',
'@^Mozilla/5\\.0 \\(compatible; Yahoo\\!\\-AdCrawler;.*http\\://help\\.yahoo\\.com/yahoo_adcrawler\\)$@',
'@^OOZBOT/0\\.20 \\( http\\://www\\.setooz\\.com/oozbot\\.html ; agentname at setooz dot_com \\)$@',
'@^Mozilla/5\\.0 \\(compatible; MSIE 6\\.0; Podtech Network; crawler_admin\\@podtech\\.net\\)$@',
'@^Mozilla/5\\.0 \\(compatible; FriendFeedBot/0\\..*; \\+Http\\://friendfeed\\.com/about/bot\\)$@',
'@^Mozilla/5\\.0 \\(compatible; SuchbaerBot/0\\..*; \\+http\\://bot\\.suchbaer\\.de/info\\.html\\)$@',
'@^Mozilla/5\\.0 \\(Yahoo\\-MMCrawler/.*; mailto\\:vertical\\-crawl\\-support\\@yahoo\\-inc\\.com\\)$@',
'@^Mozilla/5\\.0 \\(compatible; Chirp/1\\.0; \\+http\\://www\\.binarycanary\\.com/chirp\\.cfm\\)$@',
'@^Mozilla/5\\.0 \\(Macintosh; .; PPC Mac OS X;.*\\) AppleWebKit/.* \\(.*\\) HistoryHound/.*$@',
'@^Mozilla/5\\.0 \\(X11; U; Linux.*; .*; rv\\:1\\..*; aggregator\\:NewsMonster; .*\\) Gecko/.*$@',
'@^Mozilla/5\\.0 \\(.*Mac OS X.*\\) AppleWebKit/.* \\(KHTML, like Gecko, Safari/.*\\) ADM/.*$@',
'@^Mozilla/5\\.0 \\(compatible; del\\.icio\\.us\\-thumbnails/.*; .*\\) KHTML/.* \\(like Gecko\\)$@',
'@^Cocoal\\.icio\\.us/1\\.0 \\(v43\\) \\(Mac OS X; http\\://www\\.scifihifi\\.com/cocoalicious\\)$@',
'@^Simpy/.* \\(Simpy; http\\://www\\.simpy\\.com/.ref\\=bot; feedback at simpy dot com\\)$@',
'@^Mozilla/5\\.0 \\(compatible; ActiveTouristBot.*; http\\://www\\.activetourist\\.com\\)$@',
'@^Speedy Spider \\(http\\://www\\.entireweb\\.com/about/search_tech/speedy_spider/\\)$@',
'@^Mozilla/5\\.0 \\(.*\\) Gecko/.* Firefox/2\\.0 OneRiot/1\\.0 \\(http\\://www\\.oneriot\\.com\\) $@',
'@^Mozilla/4\\.0 \\(compatible; .*Vagabondo/.*; webcrawler at wise\\-guys dot nl; .*\\)$@',
'@^Mozilla/5\\.0 \\(X11; U; Linux.*; .*; rv\\:1\\..*; aggregator\\:FeedParser; .*\\) Gecko/.*$@',
'@^facebookexternalhit/.* \\(.http\\://www\\.facebook\\.com/externalhit_uatext\\.php\\).*$@',
'@^Mozilla/5\\.0 \\(Windows; U; Windows .*; .*; rv\\:.*\\) Gecko/.* Firefox/.* webaroo/.*$@',
'@^Mozilla/5\\.0 \\(compatible; LegalAnalysisAgent/1\\..*; http\\://www\\.legalx\\.net\\)$@',
'@^Mozilla/5\\.0 \\(compatible; .*; http\\://www\\.80legs\\.com/spider\\.html;\\) Gecko/.*$@',
'@^Tumblr/.* RSS syndication \\( http\\://www\\.tumblr\\.com/\\) \\(support\\@tumblr\\.com\\)$@',
'@^Mozilla/5\\.0 \\(compatible; Plukkie/1\\..; http\\://www\\.botje\\.com/plukkie\\.htm\\)$@',
'@^Mozilla/5\\.0 \\(compatible; Webscan v0\\..*; \\+http\\://otc\\.dyndns\\.org/webscan/\\)$@',
'@^Mozilla/5\\.0 \\(compatible; Newz Crawler .*; http\\://www\\.newzcrawler\\.com/.\\)$@',
'@^Mozilla/5\\.0 \\(Macintosh; U; .*Mac OS X; .*\\) AppleWebKit/.* \\(.*\\) Pandora/2\\..*$@',
'@^Microsoft Data Access Internet Publishing Provider Protocol Discovery$@',
'@^Mozilla/5\\.0 \\(compatible; JadynAveBot; .*http\\://www\\.jadynave\\.com/robot.*$@',
'@^Mozilla/5\\.0 \\(compatible; Lipperhey Spider; http\\://www\\.lipperhey\\.com/\\)$@',
'@^Mozilla/5\\.0 \\(compatible; NetcraftSurveyAgent/1\\.0; .*info\\@netcraft\\.com\\)$@',
'@^Mozilla/5\\.0 \\(compatible; MojeekBot/.*; http\\://www\\.mojeek\\.com/bot\\.html\\)$@',
'@^Mozilla/5\\.0 \\(compatible; spbot/.*; \\+http\\://www\\.seoprofiler\\.com/bot/ \\)$@',
'@^Mozilla/5\\.0 \\(compatible; bingbot/2\\..*http\\://www\\.bing\\.com/bingbot\\.htm\\)$@',
'@^Atrax Solutions atraxbot/0\\..*; http\\://www\\.atraxsolutions\\.com/atraxbot$@',
'@^Mozilla/5\\.0 \\(.*; Rojo .*; http\\://www\\.rojo\\.com/corporate/help/agg; .*\\).*$@',
'@^Mozilla/5\\.0 \\(X11; U; Linux.*; .*; rv\\:1\\..*; aggregator\\:Rojo; .*\\) Gecko/.*$@',
'@^Mozilla/5\\.0 \\(compatible; DBLBot/1\\.0; .http\\://www\\.dontbuylists\\.com/\\)$@',
'@^CC Metadata Scaper http\\://wiki\\.creativecommons\\.org/Metadata_Scraper$@',
'@^Mozilla/4\\.7 \\(compatible; WhizBang; http\\://www\\.whizbang\\.com/crawler\\)$@',
'@^Mozilla/4\\.0 \\(compatible; MSIE 6\\.0; Windows 98\\) Web Link Validator.*$@',
'@^rssImagesBot/0\\.1 \\(.*http\\://herbert\\.groot\\.jebbink\\.nl/.app\\=rssImages\\)$@',
'@^Mozilla/5\\.0 \\(compatible; DotBot/.*; http\\://www\\.dotnetdotcom\\.org/.*\\)$@',
'@^Mozilla/5\\.0 \\(compatible; Twingly Recon; http\\://www\\.twingly\\.com/\\)$@',
'@^Microsoft Data Access Internet Publishing Provider Cache Manager$@',
'@^Knight/0\\.. \\(Zook Knight; http\\://knight\\.zook\\.in/; knight\\@zook\\.in\\)$@',
'@^Mozilla/5\\.0 \\(compatible; MSIE 7\\.0 .http\\://www\\.europarchive\\.org\\)$@',
'@^Mozilla/.* \\(compatible; linktiger/.*; .*http\\://www\\.linktiger\\.com.*\\)$@',
'@^Mozilla/4\\.0 \\(compatible; Link Utility; http\\://net\\-promoter\\.com\\)$@',
'@^Mozilla/5\\.0 \\(compatible; unwrapbot/2\\..*; \\+http\\://www\\.unwrap\\.jp.*\\)$@',
'@^roboobot/1\\..* \\(roboo; http\\://wap\\.roboo\\.com; winter\\.pi\\@roboo\\.com\\)$@',
'@^Mozilla/4\\.0 \\(compatible; MSIE .\\.0; GomezAgent .\\.0; Windows NT\\)$@',
'@^Mozilla/5\\.0 \\(compatible; ClixSense; http\\://www\\.clixsense\\.com/\\)$@',
'@^Mozilla/5\\.0 \\(compatible; Diffbot/0\\.1; \\+http\\://www\\.diffbot\\.com\\)$@',
'@^mozilla/5\\.0 \\(compatible; genevabot \\+http\\://www\\.healthdash\\.com\\)$@',
'@^Mozilla/5\\.0 \\(.*aggregator\\:TailRank; http\\://tailrank\\.com/robot\\).*$@',
'@^Mozilla/5\\.0 \\(compatible; SEODat/0\\..* http\\://crawler\\.seodat\\.com\\)$@',
'@^WebImages .* \\(.http\\://herbert\\.groot\\.jebbink\\.nl/.app\\=WebImages.\\)$@',
'@^Mozilla/5\\.0 \\(compatible; ScoutJet; \\+http\\://www\\.scoutjet\\.com/\\)$@',
'@^Mozilla/5\\.0 \\(compatible; aiHitBot.*/.*; \\+http\\://www\\.aihit\\.com/\\)$@',
'@^e\\-SocietyRobot\\(http\\://www\\.yama\\.info\\.waseda\\.ac\\.jp/~yamana/es/\\)$@',
'@^Mozilla/4\\.0 \\(compatible; MSIE 7\\.0; Windows NT 6\\.0; Webaroo/.*\\)$@',
'@^Mozilla/4\\.0 \\(compatible; BOTW Feed Grabber; .*http\\://botw\\.org\\)$@',
'@^\\(HTML Validator http\\://www\\.searchengineworld\\.com/validator/\\)$@',
'@^FeedOnFeeds/0\\.1\\..* \\( http\\://minutillo\\.com/steve/feedonfeeds/\\)$@',
'@^Mozilla/4\\.0 \\(compatible; MSIE 7\\.0; Win32\\) Link Commander 3\\.0$@',
'@^Mozilla/5\\.0 \\(compatible; Crawly/1\\..*; \\+http\\://.*/crawler\\.html\\)$@',
'@^TwengaBot\\-Discover \\(http\\://www\\.twenga\\.fr/bot\\-discover\\.html\\)$@',
'@^Mozilla/5\\.0 \\(compatible; Seznam screenshot\\-generator 2\\.0;.*\\)$@',
'@^OpenTaggerBot \\(http\\://www\\.opentagger\\.com/opentaggerbot\\.htm\\)$@',
'@^Mozilla/.* \\(compatible; OffByOne; Windows.*\\) Webster Pro V3\\..*$@',
'@^Mozilla/5\\.0 \\(compatible; Twitturls; \\+http\\://twitturls\\.com\\)$@',
'@^InfociousBot \\(.http\\://corp\\.infocious\\.com/tech_crawler\\.php\\)$@',
'@^Mozilla/5\\.0 \\(compatible; RSSMicro\\.com RSS/Atom Feed Robot\\)$@',
'@^onCHECK Linkchecker von www\\.scientec\\.de fuer www\\.onsinn\\.de$@',
'@^Mozilla/5\\.0 \\(compatible; Ezooms/1\\.0; ezooms\\.bot\\@gmail\\.com\\)$@',
'@^Mozilla/4\\.0 \\(compatible; smartBot/1\\..*; checking links; .*\\)$@',
'@^Healthbot/Health_and_Longevity_Project_\\(HealthHaven\\.com\\) $@',
'@^Linguee Bot \\(http\\://www\\.linguee\\.com/bot; bot\\@linguee\\.com\\)$@',
'@^livedoor FeedFetcher/0\\.0.* \\(http\\://reader\\.livedoor\\.com/;.*\\)$@',
'@^msnbot\\-webmaster/1\\.0 \\(.*http\\://search\\.msn\\.com/msnbot\\.htm\\)$@',
'@^msnbot\\-NewsBlogs/2\\..* \\(\\+http\\://search\\.msn\\.com/msnbot\\.htm\\)$@',
'@^DuckDuckBot/.*; \\(.http\\://duckduckgo\\.com/duckduckbot\\.html\\)$@',
'@^Mobitype .* \\(compatible; Mozilla/.*; MSIE .*\\..*; Windows .*\\)$@',
'@^Mozilla/4\\.0 \\(compatible; BOTW Spider; .*http\\://botw\\.org\\)$@',
'@^Microsoft Data Access Internet Publishing Provider DAV.*$@',
'@^Mozilla/.*\\(.*redditbot/.*http\\://www\\.reddit\\.com/feedback.*\\)$@',
'@^FAST MetaWeb Crawler \\(helpdesk at fastsearch dot com\\)$@',
'@^voyager/2\\.0 \\(http\\://www\\.kosmix\\.com/html/crawler\\.html\\)$@',
'@^eStyleSearch .* \\(compatible; MSIE 6\\.0; Windows NT 5\\.0\\)$@',
'@^1st ZipCommander \\(Net\\) \\- http\\://www\\.zipcommander\\.com/$@',
'@^Mozilla/5\\.0 \\(compatible; YandexBot/.*; MirrorDetector\\)$@',
'@^Busiversebot/v1\\.0 \\(http\\://www\\.busiverse\\.com/bot\\.php\\)$@',
'@^Mozilla/5\\.0 \\(Danger hiptop 3\\..*; U; rv\\:1\\.7\\..*\\) Gecko/.*$@',
'@^Strategic Board Bot \\(.http\\://www\\.strategicboard\\.com\\)$@',
'@^renlifangbot/1\\.0 \\(.http\\://search\\.msn\\.com/msnbot\\.htm\\)$@',
'@^envolk.ITS.spider/.* \\(.http\\://www\\.envolk\\.com/envolk.*\\)$@',
'@^Mozilla/5\\.0 http\\://www\\.inclue\\.com; graeme\\@inclue\\.com$@',
'@^Mozilla/4\\.0 \\(compatible; MSIE 4\\.01; Vonna\\.com b o t\\)$@',
'@^Mozilla/5\\.0 \\(.*http\\://gnomit\\.com/\\) Gecko/.* Gnomit/1\\.0$@',
'@^Browscap Mirror System/1\\..* \\(browscap\\.giantrealm\\.com\\)$@',
'@^Mozilla/5\\.0 \\(compatible; AdHitz; http\\://adhitz\\.com/\\)$@',
'@^neTVision AG andreas\\.heidoetting\\@thomson\\-webcast\\.net$@',
'@^FeedHub MetaDataFetcher/1\\.0 \\(http\\://www\\.feedhub\\.com\\)$@',
'@^Pingdom\\.com_bot_version_.*_\\(http\\://www\\.pingdom\\.com/\\)$@',
'@^MapoftheInternet\\.com.\\(.http\\://MapoftheInternet\\.com\\)$@',
'@^T\\-Mobile Dash Mozilla/4\\.0 \\(.*\\) MSNBOT\\-MOBILE/1\\.1 \\(.*\\)$@',
'@^Mozilla/4\\.0 \\(compatible; Advanced Email Extractor.*\\)$@',
'@^Windows\\-RSS\\-Platform/2\\.0 \\(MSIE .\\.0; Windows NT .*\\..*\\)$@',
'@^online link validator \\(http\\://www\\.dead\\-links\\.com/\\)$@',
'@^Godzilla/.* \\(Basic.*; .*; Commodore C\\=64; .*; rv\\:1\\..*\\).*$@',
'@^FeedHub FeedDiscovery/1\\.0 \\(http\\://www\\.feedhub\\.com\\)$@',
'@^Mozilla/5\\.0 \\(Macintosh; Intel Mac OS X\\) Excel/12\\..*$@',
'@^Mozilla/4\\.0 \\(compatible; MSIE 6\\.0; Bluecoat DRTR\\)$@',
'@^Patwebbot \\(http\\://www\\.herz\\-power\\.de/technik\\.html\\)$@',
'@^findfiles\\.net/.* \\(Robot;test_robot\\@gmx\\-topmail\\.de\\)$@',
'@^Mozilla/5\\.0 \\(compatible; IPCheck Server Monitor.*\\)$@',
'@^DataFountains/DMOZ Feature Vector Corpus Creator.*$@',
'@^WCC Browscap Updater/0\\..* \\(PHP\\: file_get_contents\\)$@',
'@^Inne\\: Mozilla/4\\.0 \\(compatible; Cerberian Drtrs.*\\)$@',
'@^wwwster/.* \\(Beta, mailto\\:gue\\@cis\\.uni\\-muenchen\\.de\\)$@',
'@^FeedHub FeedFetcher/1\\.0 \\(http\\://www\\.feedhub\\.com\\)$@',
'@^WebmasterWorld StickyMail Server Header Checker.*$@',
'@^SafariBookmarkChecker.*\\(.http\\://www\\.coriolis\\.ch/\\)$@',
'@^Sosospider.\\(\\+http\\://help\\.soso\\.com/webspider\\.htm\\)$@',
'@^adidxbot/1\\.1 \\(.http\\://search\\.msn\\.com/msnbot\\.htm\\)$@',
'@^gosospider Mozilla/5\\.0 \\(compatible; GosoSpider.*\\)$@',
'@^Mozilla/5\\.0 \\(compatible; YahooSeeker/M1A1\\-R2D2.*\\)$@',
'@^Mozilla/5\\.0 \\(compatible; BMI/.* \\(Y\\!J\\-AGENT; 1\\.0\\)\\)$@',
'@^UniversalFeedParser/4\\..* \\+http\\://feedparser\\.org/$@',
'@^Mozilla/5\\.0 \\(compatible; FirstSearchBot/1\\.0; .*\\)$@',
'@^Mozilla/4\\.0 \\(compatible; Y\\!J; for robot study.*\\)$@',
'@^Windows\\-Live\\-Social\\-Object\\-Extractor\\-Engine/1\\.0$@',
'@^Mozilla/5\\.0 \\(compatible; YandexImageResizer/.*\\).*$@',
'@^Rome Client \\(http\\://tinyurl\\.com/64t5n\\) Ver\\: 0\\..*$@',
'@^Made by ZmEu \\@ WhiteHat v0\\..* \\(www\\.WhiteHat\\.ro\\)$@',
'@^Mozilla/4\\.0 \\(compatible; MSIE 4\\.01; Windows95\\)$@',
'@^Mozilla/5\\.0 \\(SnapPreviewBot\\) Gecko/.* Firefox/.*$@',
'@^exactseek\\-pagereaper\\-.* \\(crawler\\@exactseek\\.com\\)$@',
'@^Liferea/1\\..* \\(Linux; .*; http\\://liferea\\.sf\\.net/\\)$@',
'@^Mozilla/5\\.0 \\(compatible; YandexBlogs/.*; robot\\)$@',
'@^Mozilla/5\\.0 \\(compatible; Yahoo\\! SearchMonkey.*\\)$@',
'@^mp3Spider cn\\-search\\-devel at yahoo\\-inc dot com$@',
'@^Mozilla/2\\.0 \\(compatible; NEWT ActiveX; Win32\\)$@',
'@^GOFORITBOT \\(.http\\://www\\.goforit\\.com/about/.\\)$@',
'@^psbot/.* \\(.http\\://www\\.picsearch\\.com/bot\\.html\\)$@',
'@^inetbot/.* \\(.http\\://www\\.inetbot\\.com/bot\\.html\\)$@',
'@^Mozilla/4\\.0 \\(compatible; Scumbot/.*; Linux/.*\\)$@',
'@^Mozilla/5\\.0 \\(compatible; Exabot\\-Images/3\\.0.*\\)$@',
'@^CipinetBot \\(http\\://www\\.cipinet\\.com/bot\\.html\\)$@',
'@^Tasap\\-image\\-robot/0\\..* \\(http\\://www\\.tasap\\.com\\)$@',
'@^Browser Capabilities Project \\- PHP Browscap.*$@',
'@^Mozilla/5\\.0 \\(compatible; BecomeJPBot/2\\.3; .*\\)$@',
'@^UofTDB_experiment.* \\(leehyun\\@cs\\.toronto\\.edu\\)$@',
'@^ddetailsbot \\(http\\://www\\.displaydetails\\.com\\)$@',
'@^Mozilla/4\\.0\\(.compatible; MSIE 6\\.0; Qihoo .*\\)$@',
'@^CMS crawler \\(.http\\://buytaert\\.net/crawler/\\)$@',
'@^Mozilla/5\\.0 \\(compatible;YodaoBot\\-Image/1\\..*\\)$@',
'@^Mozilla/5\\.0 \\(compatible; BMF/.* \\(Y\\!J\\-AGENT\\)\\)$@',
'@^Mozilla/5\\.0 \\(compatible; BMC/.* \\(Y\\!J\\-AGENT\\)\\)$@',
'@^Mozilla/5\\.0 \\(compatible; Yahoo\\! Verifier/.*\\)$@',
'@^Mozilla/5\\.0 \\(compatible; YandexFavicons/.*\\).*$@',
'@^Susie \\(http\\://www\\.sync2it\\.com/bms/susie\\.php$@',
'@^Mozilla/.\\.0 \\(compatible; Ask Jeeves/Teoma.*\\)$@',
'@^Mozilla/5\\.0 \\(compatible; Butterfly/1\\.0; .*\\).*$@',
'@^Mozilla/5\\.0 \\(compatible; archive\\.org_bot.*\\)$@',
'@^Mozilla/4\\.0 \\(compatible; Cerberian Drtrs.*\\)$@',
'@^Mozilla/5\\.0 \\(compatible; Viralheat Bot/.*\\) $@',
'@^WAP_Browser/5\\.0 \\(compatible; YodaoBot/1\\..*\\)$@',
'@^Tecomi Bot \\(http\\://www\\.tecomi\\.com/bot\\.htm\\)$@',
'@^HTML2JPG Blackbox, http\\://www\\.html2jpg\\.com$@',
'@^NextGenSearchBot.*\\(for information visit .*\\)$@',
'@^Mozilla/5\\.0 \\(compatible; Quantcastbot/1\\..*\\)$@',
'@^Mozilla/4\\.0 \\(compatible; ChangeDetection/.*$@',
'@^Mozilla/5\\.0 \\(compatible; BuzzRankingBot/.*\\)$@',
'@^Mozilla/5\\.0 \\(compatible; YandexMetrika/.*\\).*$@',
'@^ZIBB Crawler \\(email address / WWW address\\)$@',
'@^Mozilla/5\\.0 \\(compatible; YandexCatalog/.*\\).*$@',
'@^Search Engine World Robots\\.txt Validator.*$@',
'@^Mozilla/5\\.0 \\(compatible; EARTHCOM\\.info/.*\\)$@',
'@^Mozilla/4\\.0 \\(compatible; SuperCleaner.*;.*\\)$@',
'@^Html Link Validator \\(www\\.lithopssoft\\.com\\)$@',
'@^Mozilla/5\\.0 \\(compatible; YandexImages/.*\\).*$@',
'@^Mozilla/5\\.0 \\(compatible; YandexDirect/.*\\).*$@',
'@^Mozilla/5\\.0 \\(compatible; YandexAddurl/.*\\).*$@',
'@^envolk/.* \\(.http\\://www\\.envolk\\.com/envolk.*\\)$@',
'@^Mozilla/5\\.0 \\(compatible; nextthing\\.org/.*\\)$@',
'@^Mozilla/5\\.0 \\(compatible; CXL\\-FatAssANT.*\\)$@',
'@^Mozilla/5\\.0 \\(compatible; Charlotte/.*; .*\\)$@',
'@^Mozilla/4\\.0 \\(compatible; BorderManager.*\\)$@',
'@^Mozilla/5\\.0 \\(compatible; Theophrastus/.*\\)$@',
'@^Mozilla/5\\.0 \\(compatible; ParchBot/1\\.0;.*\\)$@',
'@^BookmarkBase\\(2/;http\\://bookmarkbase\\.com\\)$@',
'@^AideRSS/1\\.0 \\(aiderss\\.com\\); .* subscribers$@',
'@^DotBot/.* \\(http\\://www\\.dotnetdotcom\\.org/.*\\)$@',
'@^JRTwine Software Check Favorites Utility$@',
'@^DocWeb Link Crawler \\(http\\://doc\\.php\\.net\\)$@',
'@^Mozilla/5\\.0 \\(compatible; YandexMedia/.*\\).*$@',
'@^FAST Data Search Document Retriever/4\\.0.*$@',
'@^Mozilla/5\\.0 \\(compatible; YandexVideo/.*\\).*$@',
'@^Mozilla/4\\.0 \\(compatible; Spider; Linux\\)$@',
'@^Bloglines/.* \\(http\\://www\\.bloglines\\.com.*\\)$@',
'@^Mozilla/5\\.0 \\(compatible; YandexNews/.*\\).*$@',
'@^Mozilla/5\\.0 \\(compatible; ayna\\-crawler.*\\)$@',
'@^Mozilla/4\\.0 \\(compatible; MyFamilyBot/.*\\)$@',
'@^Microsoft Bing Mobile SocialStreams Bot$@',
'@^http\\://hilfe\\.acont\\.de/bot\\.html ACONTBOT$@',
'@^Mozilla/5\\.0 \\(compatible;.*newstin\\.com;.*\\)$@',
'@^JS\\-Kit URL Resolver, http\\://js\\-kit\\.com/$@',
'@^Mahalobot/1\\.0 \\(.http\\://www\\.mahalo\\.com/\\)$@',
'@^WIRE/.* \\(Linux.*Bot,Robot,Spider,Crawler\\)$@',
'@^Microsoft Office/.* \\(.*Picture Manager.*\\)$@',
'@^Willow Internet Crawler by Twotrees V.*$@',
'@^Visited by http\\://tools\\.geek\\-tools\\.org$@',
'@^Mozilla/5\\.0 \\(compatible; YodaoBot/1\\..*\\)$@',
'@^Mozilla/5\\.0 \\(compatible; YandexBot/.*\\).*$@',
'@^OpenDNS Domain Crawler noc\\@opendns\\.com$@',
'@^annotate_google; http\\://ponderer\\.org/.*$@',
'@^Mozilla/5\\.0 \\(compatible; DNS\\-Digger/.*\\)$@',
'@^Mozilla/5\\.0 \\(compatible; DKIMRepBot/.*\\)$@',
'@^www\\.fi crawler, contact crawler\\@www\\.fi$@',
'@^Mozilla/3\\.0 \\(compatible; Indy Library\\)$@',
'@^Amico Alpha .* \\(.*\\) Gecko/.* AmicoAlpha/.*$@',
'@^Kolinka Forum Search \\(www\\.kolinka\\.com\\)$@',
'@^Mozilla/5\\.0 \\(compatible; MJ12bot/v1\\..*\\)$@',
'@^Mozilla/5\\.0 \\(compatible; AboutUsBot/.*\\)$@',
'@^Best Whois \\(http\\://www\\.bestwhois\\.net/\\)$@',
'@^BlogPulseLive \\(support\\@blogpulse\\.com\\)$@',
'@^ah\\-ha\\.com crawler \\(crawler\\@ah\\-ha\\.com\\)$@',
'@^Mozilla/4\\.0 \\(compatible; ibisBrowser\\)$@',
'@^Decode Framework 0\\..* browscap library$@',
'@^AnswerBus \\(http\\://www\\.answerbus\\.com/\\)$@',
'@^\\!Susie \\(http\\://www\\.sync2it\\.com/susie\\)$@',
'@^Mozilla/5\\.0 \\(compatible\\) GM RSS Panel$@',
'@^Mozilla/5\\.0 \\(compatible; Exabot/3\\.0.*\\)$@',
'@^Microsoft Windows Network Diagnostics$@',
'@^Mozilla/5\\.0 \\(compatible; NLCrawler/.*$@',
'@^Mozilla/2\\.0 \\(compatible; Ask Jeeves\\)$@',
'@^CatchBot/.*; \\+http\\://www\\.catchbot\\.com$@',
'@^Microsoft Office Existence Discovery$@',
'@^Feedster Crawler/.\\.0; Feedster, Inc\\.$@',
'@^A1 Website Download/1\\..* \\(.*\\) miggibot$@',
'@^Mozilla/4\\.0 \\(compatible; N\\-Stealth\\)$@',
'@^Mozilla/.* \\(compatible; WebCapture.*\\)$@',
'@^Mozilla/4\\.0 \\(compatible; DepSpid/.*\\)$@',
'@^RixBot \\(http\\://babelserver\\.org/rix\\)$@',
'@^Feedreader .* \\(Powered by Newsbrain\\)$@',
'@^Microsoft Office Protocol Discovery$@',
'@^Mozilla/5\\.0 \\(.*Google Keyword Tool.*\\)$@',
'@^Mozilla/5\\.0 \\(compatible; Pogodak\\..*\\)$@',
'@^Internet Content Rating Association$@',
'@^check_http/.* \\(nagios\\-plugins 1\\.4\\..*\\)$@',
'@^check_http/.* \\(nagios\\-plugins 1\\.3\\..*\\)$@',
'@^check_http/.* \\(nagios\\-plugins 1\\.2\\..*\\)$@',
'@^check_http/.* \\(nagios\\-plugins 1\\.1\\..*\\)$@',
'@^PHP Browser Capabilities Project/.*$@',
'@^Mono Browser Capabilities Updater.*$@',
'@^.*Spinn3r.*http\\://spinn3r\\.com/robot.*$@',
'@^Superpages URL Verification Engine$@',
'@^kinjabot \\(http\\://www\\.kinja\\.com; .*\\)$@',
'@^BlijbolReallySimpleAggregator/2\\.0.*$@',
'@^Mozilla/4\\.0 \\(Compatible\\); URLBase.*$@',
'@^Mozilla/4\\.0 \\(compatible; Getleft.*\\)$@',
'@^Mozilla/5\\.0 \\(compatible; Webbot/.*\\)$@',
'@^Mozilla/5\\.0 \\(.*Feedfetcher\\-Google.*\\)$@',
'@^Feedfetcher\\-Google\\-iGoogleGadgets.*$@',
'@^Mozilla/5\\.0 \\(compatible; Vermut.*\\)$@',
'@^Pagebull http\\://www\\.pagebull\\.com/$@',
'@^check_http/.* \\(nagios\\-plugins 1\\..*\\)$@',
'@^SCEJ PSP BROWSER 0102pspNavigator$@',
'@^Mozilla/4\\.0 \\(compatible; Arachmo\\)$@',
'@^Mozilla/5\\.0 \\(compatible; NGBot/.*\\)$@',
'@^Nozilla/P\\.N \\(Just for IDS woring\\)$@',
'@^Fooky\\.com/ScorpionBot/ScoutOut;.*$@',
'@^NewsGator FetchLinks extension/.*$@',
'@^Mozilla/4\\.0 \\(compatible; Win32\\)$@',
'@^AcadiaUniversityWebCensusClient$@',
'@^MetaProducts Download Express/.*$@',
'@^Mozilla/0\\.9.* no dos \\:\\) \\(Linux.*\\)$@',
'@^Lorkyll .*\\..* \\-\\- lorkyll\\@444\\.net$@',
'@^FAST\\-WebCrawler/.*/PartnerSite.*$@',
'@^Mozilla/.* \\(Win32;.*Escape..*; .\\)$@',
'@^Mozilla/5\\.0 CostaCider Search.*$@',
'@^DataFountains/DMOZ Downloader.*$@',
'@^Mozilla/5\\.0 \\(compatible; OsO;.*$@',
'@^Mozilla/5\\.0 \\(RSS Reader Panel\\)$@',
'@^Kapere \\(http\\://www\\.kapere\\.com\\)$@',
'@^Mozilla/5\\.0 \\(.*Google Desktop.*\\)$@',
'@^MFC Foundation Class Library.*$@',
'@^Bookmark Renewal Check Agent.*$@',
'@^Jigsaw/.* W3C_CSS_Validator.*/.*$@',
'@^West Wind Internet Protocols.*$@',
'@^YooW\\!/.* \\(.http\\://www\\.yoow\\.eu\\)$@',
'@^WebIndexer/.* \\(Web Indexer; .*\\)$@',
'@^SynapticSearch/AI Crawler 1\\..$@',
'@^FAST\\-WebCrawler/.*.Multimedia.*$@',
'@^PigBlock \\(Windows NT 5\\.1; U\\).*$@',
'@^ConveraMultiMediaCrawler/0\\.1.*$@',
'@^Mozilla/5\\.0 GurujiBot/1\\.0 \\(.*\\)$@',
'@^Microsoft\\-WebDAV\\-MiniRedir/.*$@',
'@^FAST\\-WebCrawler/.*/FirstPage.*$@',
'@^Irish Blogs Aggregator/.*1\\.0.*$@',
'@^ogeb browser , Version 1\\.1\\.0$@',
'@^KakleBot \\- www\\.kakle\\.com/0\\.1$@',
'@^.*Netcraft Web Server Survey.*$@',
'@^HolmesBot \\(http\\://holmes\\.ge\\)$@',
'@^Norbert the Spider\\(Burf\\.com\\)$@',
'@^Epsilon SoftWorks\' MailMunky$@',
'@^Miva \\(AlgoFeedback\\@miva\\.com\\)$@',
'@^sitecheck\\.internetseer\\.com.*$@',
'@^PDFBot \\(crawler\\@pdfind\\.com\\)$@',
'@^FastSearch Web Crawler for.*$@',
'@^Mozilla/.* \\(.*\\) \\- BrowseX \\(.*\\)$@',
'@^POE\\-Component\\-Client\\-HTTP/.*$@',
'@^Live \\(http\\://www\\.live\\.com/\\)$@',
'@^.*Netcraft Webserver Survey.*$@',
'@^http\\://www\\.yellowpages\\.com.*$@',
'@^Microsoft Internet Explorer$@',
'@^yp\\-crawl\\@attinteractive\\.com$@',
'@^Newsgroupreporter LinkCheck$@',
'@^Yahoo\\! Slurp/Site Explorer$@',
'@^Der gro\\xdfe BilderSauger.*$@',
'@^Mozilla/5\\.0 \\(Yahoo\\-Test/.*\\)$@',
'@^searchbot admin\\@google\\.com$@',
'@^.*E\\-Mail Address Extractor.*$@',
'@^DomainsDB\\.net MetaCrawler.*$@',
'@^AideRSS 2\\..* \\(postrank\\.com\\)$@',
'@^LucidMedia ClickSense/4\\..$@',
'@^Mozilla/5\\.0 gURLChecker/.*$@',
'@^Mozilla/5\\.0 \\(YahooYSMcm.*\\)$@',
'@^AideRSS/2\\.0 \\(aiderss\\.com\\)$@',
'@^Windows\\-RSS\\-Platform/1\\.0.*$@',
'@^Fast PartnerSite Crawler.*$@',
'@^.*FAST Enterprise Crawler.*$@',
'@^Mozilla/5\\.0 \\(.*\\) VoilaBot.*$@',
'@^bot/.* \\(bot; .*bot\\@bot\\.bot\\)$@',
'@^Google\\-Site\\-Verification.*$@',
'@^Lincoln State Web Browser$@',
'@^ICRA_Semantic_spider/0\\..$@',
'@^Google OpenSocial agent.*$@',
'@^Lycos\\-Spider_\\(modspider\\)$@',
'@^ICRA_label_generator/1\\..$@',
'@^ShowXML/1\\.0 libwww/5\\.4\\.0$@',
'@^Nutch/0\\.. \\(OpenX Spider\\)$@',
'@^FAST\\-WebCrawler/.*/Fresh.*$@',
'@^Oracle Enterprise Search$@',
'@^Tcl http client package.*$@',
'@^appie.*\\(www\\.walhello\\.com\\)$@',
'@^Kopiczek/.* \\(WyderOS.*; .*\\)$@',
'@^IPiumBot laurion\\(dot\\)com$@',
'@^BlogPulse \\(ISSpider\\-3\\..*\\)$@',
'@^spider \\(tspyyp\\@tom\\.com\\)$@',
'@^PubSub\\-RSS\\-Reader/.* \\(.*\\)$@',
'@^gazz/.*\\(gazz\\@nttr\\.co\\.jp\\)$@',
'@^blogsearchbot\\-pumpkin\\-2$@',
'@^WebTrends Link Analyzer$@',
'@^Domain Dossier utility.*$@',
'@^Favorites Sweeper v\\.3\\..*$@',
'@^ATA\\-Translation\\-Service$@',
'@^CyberSpyder Link Test/.*$@',
'@^XML Sitemaps Generator.*$@',
'@^Bloglines Title Fetch/.*$@',
'@^Shelob \\(shelob\\@gmx\\.net\\)$@',
'@^SocialSpider\\-Finder/0\\..*$@',
'@^sitemonitor\\@dnsvr\\.com/.*$@',
'@^Myst Monitor Service v.*$@',
'@^Open Web Analytics Bot.*$@',
'@^Mozilla/5\\.0 \\(Twiceler.*\\)$@',
'@^Extreme Picture Finder$@',
'@^TargetYourNews\\.com bot$@',
'@^Iltrovatore\\-.etaccio/.*$@',
'@^JetBrains Omea Reader.*$@',
'@^Microsoft URL Control.*$@',
'@^Miscellaneous Browsers$@',
'@^YahooSeeker/CafeKelsa.*$@',
'@^AndroidDownloadManager$@',
'@^Sogou develop spider/.*$@',
'@^WinScripter iNet Tools$@',
'@^websitepulse checker/.*$@',
'@^HuaweiSymantecSpider/.*$@',
'@^.*maxamine\\.com\\-\\-robot.*$@',
'@^Mozilla/3\\.0 \\(INGRID/.*$@',
'@^Netchart Adv Crawler.*$@',
'@^.*BecomeBot\\@exava\\.com.*$@',
'@^Browscap Mirror v1\\.30$@',
'@^MSNBot\\-NewsBlogs/1\\.0.*$@',
'@^Bookmark search tool.*$@',
'@^.*Googlebot\\-Mobile/2\\..*$@',
'@^Infoseek SideWinder/.*$@',
'@^Mozilla/5\\.0 NewsFox/.*$@',
'@^Cocoal\\.icio\\.us/.* \\(.*\\).*$@',
'@^ezic\\.com http agent .*$@',
'@^http\\://arachnode\\.net.*$@',
'@^AdsBot\\-Google\\-Mobile.*$@',
'@^Atomic_Email_Hunter/.*$@',
'@^copyright sheriff \\(.*\\)$@',
'@^iCopyright Conductor.*$@',
'@^google \\(.*Enterprise.*\\)$@',
'@^Mediapartners\\-Google.*$@',
'@^NSO_Debugger_User/2\\.0$@',
'@^Copyright/Plagiarism$@',
'@^MSNBot\\-Academic/1\\.0.*$@',
'@^GoogleFriendConnect.*$@',
'@^Lycos\\-Spider_\\(T\\-Rex\\)$@',
'@^googlebot\\-urlconsole$@',
'@^8484 Boston Project.*$@',
'@^Sogou Orion spider/.*$@',
'@^.*Baiduspider\\-mobile.*$@',
'@^TeragramWebcrawler/.*$@',
'@^NetCarta_WebMapper/.*$@',
'@^LinkLint\\-checkonly/.*$@',
'@^.*Google Web Preview.*$@',
'@^.*Web Link Validator.*$@',
'@^Internet Exploiter/.*$@',
'@^HenryTheMiragoRobot.*$@',
'@^SOFTWING_TEAR_AGENT.*$@',
'@^Funnel Web Profiler.*$@',
'@^Radiation Retriever.*$@',
'@^Microsoft Visio MSIE$@',
'@^BackStreet Browser .*$@',
'@^Wizz RSS News Reader$@',
'@^Poodle_predictor_1\\.0$@',
'@^Web Image Collector.*$@',
'@^GigabotSiteSearch/.*$@',
'@^semanticdiscovery/.*$@',
'@^Sogou Push Spider/.*$@',
'@^wadaino\\.jp\\-crawler.*$@',
'@^Feedfetcher\\-Google.*$@',
'@^Dumbot \\(version .*\\).*$@',
'@^.*Baiduspider\\-image.*$@',
'@^Seeker\\.lookseek\\.com$@',
'@^Z\\-Add Link Checker.*$@',
'@^Oracle Ultra Search$@',
'@^Internet Explorer .*$@',
//'@^WordPress\\-Do\\-P\\-/2\\..*$@',
'@^Sogou\\-Test\\-Spider/.*$@',
'@^PlantyNet_WebRobot.*$@',
'@^ExactSeek Crawler/.*$@',
'@^.*Baiduspider\\-video.*$@',
'@^MetaGer\\-LinkChecker$@',
'@^.*Fluffy the spider.*$@',
'@^QuickFinder Crawler$@',
'@^Website Downloader.*$@',
'@^LightningDownload/.*$@',
'@^Offline Downloader.*$@',
'@^Sogou Pic Spider/.*$@',
'@^Sogou head spider.*$@',
'@^Site Valet Online.*$@',
'@^.*LinksManager\\.com.*$@',
'@^Twisted PageGetter$@',
'@^Feed43 Proxy/.* \\(.*\\)$@',
'@^AppleSyndication/.*$@',
'@^BitTorrent Clients$@',
'@^Link Valet Online.*$@',
'@^Xenu.* Link Sleuth.*$@',
'@^Flaming AttackBot.*$@',
'@^Internet Explore .*$@',
'@^Artera \\(Version .*\\)$@',
'@^Forschungsportal/.*$@',
'@^masidani_bot_v0\\.6.*$@',
'@^magpie\\-crawler/1\\..*$@',
'@^BrowscapUpdater1\\.0$@',
'@^Searchmee\\! Spider.*$@',
'@^W3C\\-mobileOK/DDC\\-.*$@',
'@^DomainWatcher Bot.*$@',
'@^PEAR HTTP_Request.*$@',
'@^NetZip Downloader.*$@',
'@^NexTools WebAgent.*$@',
'@^Links4US\\-Crawler,.*$@',
'@^LetsCrawl\\.com/1\\.0.*$@',
'@^Website eXtractor.*$@',
'@^FollowSite\\.com \\(.*\\)$@',
'@^A \\.NET Web Crawler$@',
'@^.*Exabot\\@exava\\.com.*$@',
'@^.*Baiduspider\\-cpro.*$@',
'@^.*Baiduspider\\-favo.*$@',
'@^YahooExternalCache$@',
'@^boitho\\.com\\-robot/.*$@',
'@^.*Baiduspider\\-news.*$@',
'@^NewsGatorOnline/.*$@',
'@^Xerka WebBot v1\\..*$@',
'@^msnbot\\-media/1\\.1.*$@',
'@^Keyword Density/.*$@',
'@^YahooVideoSearch.*$@',
'@^Research Projects$@',
'@^WebsiteExtractor.*$@',
'@^msnbot\\-media/1\\.0.*$@',
'@^Scooter/.*Y\\!CrawlX$@',
'@^HatenaScreenshot.*$@',
'@^AppEngine\\-Google.*$@',
'@^Inet \\- Eureka App$@',
'@^niXXieBot.Foster.*$@',
'@^Hatena\\:\\:Crawler/.*$@',
'@^sogou js robot\\(.*\\)$@',
'@^sogou web spider.*$@',
'@^Y\\!J SearchMonkey.*$@',
'@^Talkro Web\\-Shot/.*$@',
'@^Crawl_Application$@',
'@^Tutorial Crawler.*$@',
'@^REL Link Checker.*$@',
'@^DomainsBotBot/1\\..*$@',
'@^Star.*Downloader/.*$@',
'@^FAST\\-WebCrawler/.*$@',
'@^Xaldon WebSpider.*$@',
'@^Offline Explorer.*$@',
'@^BabalooSpider/1\\..*$@',
'@^WWWeasel Robot v.*$@',
'@^Exabot\\-Images/1\\.0$@',
'@^Download Managers$@',
'@^.*Baiduspider\\-ads.*$@',
'@^Download Express.*$@',
'@^WebAlta Crawler/.*$@',
'@^E\\-Mail Harvesters$@',
'@^Blue Coat Systems$@',
'@^BaiduImageSpider.*$@',
'@^Feeds Syndicators$@',
'@^Hatena Bookmark/.*$@',
'@^Social Networking$@',
'@^LinkextractorPro.*$@',
'@^Franklin Locator.*$@',
'@^Missigua Locator.*$@',
'@^InternetArchive/.*$@',
'@^librabot/1\\.0 \\(.*\\)$@',
'@^MSNBot\\-News/1\\.0.*$@',
'@^Googlebot\\-Video.*$@',
'@^Zend_Http_Client$@',
'@^Version Checkers$@',
'@^YahooFeedSeeker.*$@',
'@^Yahoo\\-MMCrawler.*$@',
'@^.*W3C\\-checklink/.*$@',
'@^Web Downloader/.*$@',
'@^UtilMind HTTPGet$@',
'@^Desktop Sidebar.*$@',
'@^Browscap updater$@',
'@^Hatena Antenna/.*$@',
'@^USyd\\-NLP\\-Spider.*$@',
//'@^WordPress\\-B\\-/2\\..*$@',
'@^STEROID Download$@',
'@^NuSearch Spider.*$@',
'@^Net\\:\\:Trackback/.*$@',
'@^.*PhotoStickies/.*$@',
'@^FeedDigest/.* \\(.*\\)$@',
'@^BlueCoat ProxySG$@',
'@^CerberianDrtrs/.*$@',
'@^Download Wonder.*$@',
'@^Download Master.*$@',
'@^DownloadSession.*$@',
'@^URL Spider Pro/.*$@',
'@^ZeBot_www\\.ze\\.bz.*$@',
'@^SquigglebotBot/.*$@',
'@^ActiveBookmark .*$@',
'@^Bookmark\\-Manager$@',
'@^.*WinHttpRequest.*$@',
'@^Internet Archive$@',
'@^Toata dragostea.*$@',
'@^TencentTraveler.*$@',
'@^Googlebot\\-Image.*$@',
'@^Google\\-Sitemaps.*$@',
'@^Taiga web spider$@',
'@^WebFilter Robot.*$@',
'@^Microsoft BITS/.*$@',
'@^RPT\\-HTTPClient/.*$@',
'@^MSN Feed Manager$@',
'@^Offline Browsers$@',
'@^Go\\-Ahead\\-Got\\-It.*$@',
'@^PluckItCrawler/.*$@',
'@^fetch libfetch/.*$@',
'@^General Crawlers$@',
'@^YodaoBot/1\\..* \\(.*\\)$@',
'@^.*altervista\\.org.*$@',
'@^ConveraCrawler/.*$@',
'@^Sogou Pic Agent$@',
'@^Web Downloader.*$@',
'@^Lsearch/sondeur$@',
'@^FeedDemon/.* \\(.*\\)$@',
'@^NewsAlloy/.* \\(.*\\)$@',
'@^boitho\\.com\\-dc/.*$@',
'@^MagpieRSS/.* \\(.*\\)$@',
'@^Technoratibot/.*$@',
'@^Googlebot/Test.*$@',
'@^Website Quester$@',
'@^SpeedDownload/.*$@',
'@^UpTime Checker.*$@',
'@^SmartDownload/.*$@',
'@^MSR\\-ISRCCrawler$@',
'@^OpenISearch/1\\..*$@',
'@^ObjectsSearch/.*$@',
'@^EmailCollector.*$@',
'@^MFC_Tear_Sample$@',
'@^msnbot\\-products$@',
'@^WWW\\-Mechanize/.*$@',
'@^CherryPicker.*/.*$@',
'@^LinkedInBot/1\\..*$@',
'@^Googlebot\\-News.*$@',
'@^HyperEstraier/.*$@',
'@^Webclipping\\.com$@',
'@^Visicom Toolbar$@',
'@^JUST\\-CRAWLER\\(.*\\)$@',
'@^BitTorrentMac/.*$@',
'@^Enigma Browser.*$@',
'@^Tagyu Agent/1\\.0$@',
'@^FeedValidator/.*$@',
'@^Best of the Web$@',
'@^HTML Validators$@',
'@^libcurl\\-agent/.*$@',
'@^libWeb/clsHTTP.*$@',
'@^YRL_ODP_CRAWLER$@',
'@^ScollSpider/2\\..*$@',
'@^Bookmark Buddy.*$@',
'@^.*Googlebot/2\\.1.*$@',
'@^FLATARTS_FAVICO$@',
'@^Pete\\-Spider/1\\..*$@',
'@^Protopage/.* \\(.*\\)$@',
'@^W3C_Validator/.*$@',
'@^FreshDownload/.*$@',
'@^UpdateBrowscap.*$@',
'@^Internet Ninja.*$@',
'@^URL Spider SQL.*$@',
'@^Yahoo\\-MMAudVid.*$@',
'@^CopyRightCheck.*$@',
'@^Download Ninja.*$@',
'@^Kontiki Client.*$@',
'@^Download Demon.*$@',
'@^ActiveRefresh.*$@',
'@^.*avantbrowser.*$@',
'@^Jayde Crawler.*$@',
'@^Feedshow/.* \\(.*\\)$@',
'@^ALeadSoftbot/.*$@',
'@^VCI WebViewer.*$@',
'@^Twitterbot/0\\..*$@',
'@^W3C\\-mobileOK/.*$@',
'@^.*NetNewsWire/.*$@',
'@^.*NetVisualize.*$@',
'@^Cogentbot/1\\...*$@',
'@^Apple\\-PubSub/.*$@',
'@^favorstarbot/.*$@',
'@^Yahoo\\! Mindset$@',
'@^FairAd Client.*$@',
'@^Transmission/.*$@',
'@^BigCliqueBOT/.*$@',
'@^NetID\\.com Bot.*$@',
'@^shaboyi spider$@',
'@^WebDownloader.*$@',
'@^StackRambler/.*$@',
'@^HiddenMarket\\-.*$@',
'@^naoFavicon4IE.*$@',
'@^WebCrawler_1\\..*$@',
'@^Kenjin Spider.*$@',
'@^Kevin http\\://.*$@',
'@^Y\\!J\\-BRP/YFSBJ.*$@',
'@^Fast/AllTheWeb$@',
'@^AdsBot\\-Google.*$@',
'@^Search Engines$@',
'@^hitcrawler_0\\..*$@',
'@^RealDownload/.*$@',
'@^Netprospector.*$@',
'@^YadowsCrawler.*$@',
'@^NetNewsWire.*/.*$@',
'@^CydralSpider/.*$@',
'@^Image Crawlers$@',
'@^CrawlConvera.*$@',
'@^ShopWiki/1\\.0.*$@',
'@^.*Powermarks/.*$@',
'@^GameSpyHTTP/.*$@',
'@^AC\\-BaiduBot/.*$@',
'@^www\\.zatka\\.com$@',
'@^TheInformant.*$@',
'@^Theme Spider.*$@',
'@^.*Baiduspider.*$@',
'@^Y\\!J\\-BRO/YFSJ.*$@',
'@^GurujiBot/1\\..*$@',
'@^Shim.Crawler.*$@',
'@^GoGuidesBot/.*$@',
'@^LinkSweeper/.*$@',
'@^TurnitinBot/.*$@',
'@^BlockNote\\.Net$@',
'@^JCheckLinks/.*$@',
'@^TutorGigBot/.*$@',
'@^i1searchbot/.*$@',
'@^exactseek\\.com$@',
'@^LinkChecker/.*$@',
'@^Link Checkers$@',
'@^CazoodleBot/.*$@',
'@^GetRightPro/.*$@',
'@^P3P Validator$@',
'@^ProWebWalker.*$@',
'@^LocalcomBot/.*$@',
'@^Y\\!J\\-VSC/ViSe.*$@',
'@^Yahoo Mindset$@',
'@^FeedGhost/1\\..*$@',
'@^Twingly Recon$@',
'@^TerrawizBot/.*$@',
'@^URLy Warning.*$@',
'@^.*TweakMASTER.*$@',
'@^Y\\!J\\-BRQ/DLCK.*$@',
'@^RedCell/.* \\(.*\\)$@',
'@^Site Monitors$@',
'@^.*MSIECrawler.*$@',
'@^ExtractorPro.*$@',
'@^EMAILsearcher$@',
'@^GreatNews/1\\.0$@',
'@^metatagsdir/.*$@',
'@^Exabot\\-Test/.*$@',
'@^IrssiUrlLog/.*$@',
'@^Net Vampire/.*$@',
'@^Covario\\-IDS/.*$@',
'@^MonkeyCrawl/.*$@',
'@^uTorrentMac/.*$@',
'@^SharpReader/.*$@',
'@^W3CLineMode/.*$@',
'@^flatlandbot/.*$@',
'@^WWW_Browser/.*$@',
'@^MovableType/.*$@',
'@^panscient\\.com$@',
'@^FyberSpider.*$@',
'@^Ace Explorer$@',
'@^GroschoBot/.*$@',
'@^NetRecorder.*$@',
'@^GreenBrowser$@',
'@^Jayde Online$@',
'@^IconSurf/2\\..*$@',
'@^FeedBurner/.*$@',
'@^intraVnews/.*$@',
'@^Netvibes \\(.*\\)$@',
'@^Omnipelagos.*$@',
'@^Exalead NG/.*$@',
'@^WebEnhancer.*$@',
'@^Atomic_Email$@',
'@^LMQueueBot/.*$@',
'@^ContactBot/.*$@',
'@^EmailSiphon.*$@',
'@^Mnogosearch.*$@',
'@^BitTornado/.*$@',
'@^BitTorrent/.*$@',
'@^RLinkCheker.*$@',
'@^CheckLinks/.*$@',
'@^iVia Project$@',
'@^Net_Vampire.*$@',
'@^webcrawl\\.net$@',
'@^.*AgentName/.*$@',
'@^.*FDSE robot.*$@',
'@^URL Control.*$@',
'@^http generic$@',
'@^URL_Access/.*$@',
'@^HTMLParser/.*$@',
'@^msnbot/2\\.0b.*$@',
'@^NetMechanic.*$@',
'@^IP.*Works\\!.*/.*$@',
'@^iexplore\\.exe$@',
'@^libtorrent/.*$@',
'@^.*WebGrabber.*$@',
'@^.*SqwidgeBot.*$@',
'@^Seram Server$@',
'@^DeepIndexer.*$@',
'@^Y\\!J\\-BRG/GSC.*$@',
'@^Hatena RSS/.*$@',
'@^sogou spider$@',
'@^WebSnatcher.*$@',
'@^Webster Pro.*$@',
'@^Feed\\:\\:Find/.*$@',
'@^MSMOBOT/1\\.1.*$@',
'@^WebGatherer.*$@',
'@^WebStripper.*$@',
'@^SiteSucker/.*$@',
'@^NextopiaBOT.*$@',
'@^gsa\\-crawler.*$@',
'@^Nudelsalat/.*$@',
'@^ia_archiver.*$@',
'@^SiteSnagger.*$@',
'@^SiteParser/.*$@',
'@^W3C\\-WebCon/.*$@',
'@^DISCo Pump .*$@',
'@^ProxyTester.*$@',
'@^YahooSeeker.*$@',
'@^bitlybot/2\\..*$@',
'@^Gulper Web .*$@',
'@^CyberPatrol.*$@',
'@^UbiCrawler/.*$@',
'@^.*BecomeBot/.*$@',
'@^User.*Agent\\:.*$@',
'@^Yahoo\\-Blogs.*$@',
'@^URI\\:\\:Fetch/.*$@',
'@^Yahoo Pipes.*$@',
'@^.*www4mail/.*$@',
'@^NewsGator/.*$@',
'@^Feeds Blogs$@',
'@^msnbot/1\\.0.*$@',
'@^DoCoMo/2\\.0.*$@',
'@^KBeeBot/0\\..*$@',
'@^WebWhacker.*$@',
'@^SlySearch/.*$@',
'@^DoCoMo/1\\.0.*$@',
'@^.*EasyRider.*$@',
'@^INGRID/.\\.0.*$@',
'@^BitBeamer/.*$@',
'@^Sqeobot/0\\..*$@',
'@^Mail\\.Ru/1\\.0$@',
'@^Translators$@',
'@^Tarantula/.*$@',
'@^Unicorn/1\\..*$@',
'@^FANGCrawl/.*$@',
'@^NG\\-Search/.*$@',
'@^SITECHECKER$@',
'@^NetReality.*$@',
'@^Popdexter/.*$@',
'@^msnbot/1\\.1.*$@',
'@^Cynthia 1\\.0$@',
'@^Search\\-10/.*$@',
'@^WebPatrol/.*$@',
'@^PicaLoader.*$@',
'@^NameProtect$@',
'@^.*WebMon .\\..*$@',
'@^BasicHTTP/.*$@',
'@^CFNetwork/.*$@',
'@^CFSCHEDULE.*$@',
'@^LinkAlarm/.*$@',
'@^LinkWalker.*$@',
'@^Ad Muncher.*$@',
'@^FindLinks/.*$@',
'@^ColdFusion.*$@',
'@^Sunrise/0\\..*$@',
'@^DataCha0s/.*$@',
'@^IIITBOT/1\\..*$@',
'@^YahooYSMcm.*$@',
'@^Surf Knight$@',
'@^SurveyBot/.*$@',
'@^Robozilla/.*$@',
'@^Zao\\-Crawler$@',
'@^Amfibibot/.*$@',
'@^Blaiz\\-Bee/.*$@',
'@^SlimBrowser$@',
'@^Eule.Robot.*$@',
'@^.*Check&Get.*$@',
'@^WebTrends/.*$@',
'@^webcollage.*$@',
'@^OutfoxBot/.*$@',
'@^FOTOCHECKER$@',
'@^Gregarius/.*$@',
'@^SiteWinder.*$@',
'@^ScoutAbout.*$@',
'@^llssbot/1\\.0$@',
'@^RepoMonkey.*$@',
'@^Mozdex/0\\.7.*$@',
'@^SuperHTTP/.*$@',
'@^webbandit/.*$@',
//'@^WordPress/.*$@',
'@^Marvin v0\\.3$@',
'@^Akregator/.*$@',
'@^Web Magnet.*$@',
'@^gnome\\-vfs/.*$@',
'@^RssBandit/.*$@',
'@^httpclient.*$@',
'@^Lycos\\-Proxy$@',
'@^HTTPFetch/.*$@',
'@^HttpSession$@',
'@^Mister PIX.*$@',
'@^shelob v1\\..*$@',
'@^SeznamBot/.*$@',
'@^pixfinder/.*$@',
'@^RssBar/1\\.2.*$@',
'@^Camcrawler.*$@',
'@^RedCarpet/.*$@',
'@^SimplePie/.*$@',
'@^General RSS$@',
'@^LapozzBot/.*$@',
'@^.*CFNetwork.*$@',
'@^ICE_GetFile$@',
'@^ZipppBot/.*$@',
'@^BitComet/.*$@',
'@^KTorrent/.*$@',
'@^EuripBot/.*$@',
'@^Search Fst$@',
'@^Pajaczek/.*$@',
'@^USER_AGENT$@',
'@^VengaBot/.*$@',
'@^DiamondBot$@',
'@^antibot\\-V.*$@',
'@^JetBrains.*$@',
'@^Drupal \\(.*\\)$@',
'@^Exabot/2\\.0$@',
'@^My Browser$@',
'@^Exabot/3\\.0$@',
'@^DNSGroup/.*$@',
'@^BilgiBot/.*$@',
'@^TouchStone$@',
'@^iSEEKbot/.*$@',
'@^KRetrieve/$@',
'@^search\\.ch.*$@',
'@^Tkensaku/.*$@',
'@^SiteSpider$@',
'@^NavissoBot$@',
'@^Linknzbot.*$@',
'@^MaSagool/.*$@',
'@^miniRank/.*$@',
'@^YodaoBot/.*$@',
'@^HooWWWer/.*$@',
'@^SuperBot/.*$@',
'@^InetURL\\:/.*$@',
'@^PageNest/.*$@',
'@^httpunit/.*$@',
'@^Webcheck .*$@',
'@^NetPumper.*$@',
'@^P3P Client$@',
'@^NetSucker.*$@',
'@^sherlock/.*$@',
'@^EmailWolf.*$@',
'@^WebSauger.*$@',
'@^NewsFire/.*$@',
'@^MFHttpScan$@',
'@^W3CRobot/.*$@',
'@^Twitturly.*$@',
'@^LinkScan/.*$@',
'@^WorQmada/.*$@',
'@^.*LinkLint.*$@',
'@^WebCopier.*$@',
'@^URL2File/.*$@',
'@^webfetch/.*$@',
'@^RSS Menu/.*$@',
'@^WebReaper.*$@',
'@^lftp/3\\.2\\.1$@',
'@^Word Press$@',
'@^uTorrent/.*$@',
'@^rtorrent/.*$@',
'@^.*heritrix.*$@',
'@^AutoHotkey$@',
'@^GetRight/.*$@',
'@^GetSmart/.*$@',
'@^.*research.*$@',
'@^.*Pingdom.*$@',
'@^Harvest/.*$@',
'@^Net Probe$@',
'@^httperf/.*$@',
'@^SpankBot.*$@',
'@^.*ickHTTP.*$@',
'@^WebRipper$@',
'@^LeechGet.*$@',
'@^OCN\\-SOC/.*$@',
'@^Pageload.*$@',
'@^OSSProxy.*$@',
'@^MediaGet.*$@',
'@^woriobot.*$@',
'@^Steeler/.*$@',
'@^INetURL/.*$@',
'@^Entireweb$@',
'@^IlseBot/.*$@',
'@^MarsEdit.*$@',
'@^NewsGator$@',
'@^eventax/.*$@',
'@^cisco\\-IOS$@',
'@^polybot..*$@',
'@^Snapbot/.*$@',
'@^WinkBot/.*$@',
'@^WebAuto/.*$@',
'@^Checkbot.*$@',
'@^SiteBar/.*$@',
'@^URLBase/.*$@',
'@^Bookdog/.*$@',
'@^.*Linkman.*$@',
'@^sna\\-0\\.0\\..*$@',
'@^WebMiner.*$@',
'@^Weblide/.*$@',
'@^Hotzonu/.*$@',
'@^Microsoft$@',
'@^.*HTTrack.*$@',
'@^iGetter/.*$@',
'@^Deepindex$@',
'@^Faxobot/.*$@',
'@^Filangy/.*$@',
'@^MSProxy/.*$@',
'@^geniebot.*$@',
'@^Gaisbot/.*$@',
'@^Chilkat/.*$@',
'@^Scrubby/.*$@',
'@^WISEbot/.*$@',
'@^WebCorp/.*$@',
'@^updated/.*$@',
'@^TMCrawler$@',
'@^AutoMate5$@',
'@^DNS Tools$@',
'@^LOOQ/0\\.1.*$@',
'@^zibber\\-v.*$@',
'@^IWAgent/.*$@',
'@^Go\\!Zilla.*$@',
'@^Gozilla/.*$@',
'@^Prozilla.*$@',
'@^sEasyDL/.*$@',
'@^shareaza.*$@',
'@^PageDown.*$@',
'@^Seekbot/.*$@',
'@^Szukacz/.*$@',
'@^SygolBot.*$@',
'@^sproose/.*$@',
'@^Teleport.*$@',
'@^Vegas95/.*$@',
'@^ASPSeek/.*$@',
'@^Y\\!J\\-PSC.*$@',
'@^Netcraft$@',
'@^Y\\!J\\-NSC.*$@',
'@^Y\\!J\\-SRD.*$@',
'@^Y\\!OASIS.*$@',
'@^JPluck/.*$@',
'@^Gigabot.*$@',
'@^Y\\!J\\-DSC.*$@',
'@^Y\\!J\\-BSC.*$@',
'@^Scooter.*$@',
'@^Linkbot.*$@',
'@^WebTrans$@',
'@^Y\\!J\\-BRE.*$@',
'@^VadixBot$@',
'@^Y\\!J\\-BRI.*$@',
'@^Particls$@',
'@^YodaoBot$@',
'@^.*libwww.*$@',
'@^PEERbot.*$@',
'@^Spinne/.*$@',
'@^3D\\-FTP/.*$@',
'@^Pompos/.*$@',
'@^XSpider.*$@',
'@^CobWeb/.*$@',
'@^FlashGet$@',
'@^NetAnts.*$@',
'@^.*Larbin.*$@',
'@^WinTools$@',
'@^Facebook$@',
'@^IRLbot/.*$@',
'@^SBIder/.*$@',
'@^Lachesis$@',
'@^PycURL/.*$@',
'@^SBL\\-BOT.*$@',
'@^Wotbox/.*$@',
'@^Ocelli/.*$@',
'@^LeechFTP$@',
'@^PhpDig/.*$@',
'@^WhizBang$@',
'@^Azureus.*$@',
'@^Moozilla$@',
'@^HTTPGrab$@',
'@^n4p_bot.*$@',
'@^Holmes/.*$@',
'@^WebFetch$@',
'@^EasyDL/.*$@',
'@^WinHttp.*$@',
'@^NetSurf.*$@',
'@^Gaisbot.*$@',
'@^URLCHECK$@',
'@^FDM 1\\.x$@',
'@^Exalead$@',
'@^3wGet/.*$@',
'@^Deluge.*$@',
'@^.*Nutch.*$@',
'@^Gigabot$@',
'@^Rippers$@',
'@^AIBOT/.*$@',
'@^VMBot/.*$@',
'@^SynoBot$@',
'@^Myzilla$@',
'@^Beamer.*$@',
'@^Abacho.*$@',
'@^BDFetch$@',
'@^WeBoX/.*$@',
'@^Convera$@',
'@^MSRBOT.*$@',
'@^Webaroo$@',
'@^WebZIP.*$@',
'@^pavuk/.*$@',
'@^eCatch.*$@',
'@^Dumbot.*$@',
'@^moget/.*$@',
'@^SyncMgr$@',
'@^Reblog.*$@',
'@^.*naver.*$@',
'@^nicebot$@',
'@^Pockey.*$@',
'@^WebPix.*$@',
'@^Python.*$@',
'@^CTerm/.*$@',
'@^HLoader$@',
'@^IECheck$@',
'@^FeedHub$@',
'@^htdig/.*$@',
'@^.*Amiga.*$@',
'@^Foobot.*$@',
'@^cosmos.*$@',
'@^PMAFind$@',
'@^SMBot/.*$@',
'@^Qweery.*$@',
'@^btbot/.*$@',
'@^Snoopy.*$@',
'@^NPBot.*$@',
'@^WebGet$@',
'@^Custo.*$@',
'@^.*grub.*$@',
'@^hcat/.*$@',
'@^FavOrg$@',
'@^abot/.*$@',
'@^.*Zeus.*$@',
'@^Become$@',
'@^Nagios$@',
'@^DoCoMo$@',
'@^cz32ts$@',
'@^nabot.*$@',
'@^NV32ts$@',
'@^Lycos.*$@',
'@^Google$@',
'@^Hatena$@',
'@^BTSP/.*$@',
'@^Envolk$@',
'@^WWWC/.*$@',
'@^MQbot.*$@',
'@^.*java.*$@',
'@^DotBot$@',
'@^Yandex$@',
'@^Boitho$@',
'@^UCmore$@',
'@^Baidu$@',
'@^Zao/.*$@',
'@^Nutch$@',
'@^curl.*$@',
'@^Lycos$@',
'@^Sogou$@',
'@^Wget.*$@',
'@^FGet.*$@',
'@^DLC/.*$@',
'@^Yahoo$@',
'@^VSE/.*$@',
'@^sohu.*$@',
'@^slurp$@',
'@^b2w/.*$@',
'@^Ilse$@',
'@^FDM.*$@',
'@^lwp.*$@',
'@^ng/.*$@',
'@^CAST$@',
'@^NP/.*$@',
'@^Snap$@',
'@^DA .*$@',
'@^oBot$@',
'@^PHP.*$@',
'@^Ask$@',
'@^MSN$@',
'@^W3C$@',
'@^0$@'
	);
}
?>
