User-agent: !Susie (http://www.sync2it.com/susie)
Disallow: /
User-agent: ( Robots.txt Validator http://www.searchengineworld.com/cgi-bin/robotcheck.cgi )
Disallow: /
User-agent: (DreamPassport/3.0; isao/MyDiGiRabi)
Disallow: /
User-agent: (Privoxy/1.0)
Disallow: /
User-agent: */Nutch-0.9-dev
Disallow: /
User-agent: +SitiDi.net/SitiDiBot/1.0 (+Have Good Day)
Disallow: /
User-agent: -DIE-KRAEHE- META-SEARCH-ENGINE/1.1 http://www.die-kraehe.de
Disallow: /
User-agent: 123spider-Bot (Version: 1.02, powered by www.123spider.de
Disallow: /
User-agent: 192.comAgent
Disallow: /
User-agent: 1st ZipCommander (Net) - http://www.zipcommander.com/
Disallow: /
User-agent: 2Bone_LinkChecker/1.0 libwww-perl/5.64
Disallow: /
User-agent: 4anything.com LinkChecker v2.0
Disallow: /
User-agent: 8484 Boston Project v 1.0
Disallow: /
User-agent: :robot/1.0 (linux) ( admin e-mail: undefined http://www.neofonie.de/loesungen/search/robot.html )
Disallow: /
User-agent: <a href='http://www.unchaos.com/'> UnChaos </a> From Chaos To Order Hybrid Web Search Engine.(vadim_gonchar@unchaos.com)
Disallow: /
User-agent: <a href='http://www.unchaos.com/'> UnChaos Bot Hybrid Web Search Engine. </a> (vadim_gonchar@unchaos.com)
Disallow: /
User-agent: <b> UnChaosBot From Chaos To Order UnChaos Hybrid Web Search Engine at www.unchaos.com </b> (info@unchaos.com)
Disallow: /
User-agent: <http://www.sygol.com/> http://www.sygol.com
Disallow: /
User-agent: A-Online Search
Disallow: /
User-agent: A1 Keyword Research/1.0.2 (+http://www.micro-sys.dk/products/keyword-research/) miggibot/2007.03.27
Disallow: /
User-agent: A1 Sitemap Generator/1.0 (+http://www.micro-sys.dk/products/sitemap-generator/) miggibot/2006.01.24
Disallow: /
User-agent: ABCdatos BotLink/5.xx.xxx#BBL
Disallow: /
User-agent: AESOP_com_SpiderMan
Disallow: /
User-agent: AIBOT/2.1 By +(www.21seek.com A Real artificial intelligence search engine China)
Disallow: /
User-agent: ANTFresco/x.xx
Disallow: /
User-agent: ASAHA Search Engine Turkey V.001 (http://www.asaha.com/)
Disallow: /
User-agent: ASPSeek/1.2.5
Disallow: /
User-agent: ASPSeek/1.2.x
Disallow: /
User-agent: ASPSeek/1.2.xa
Disallow: /
User-agent: ASPSeek/1.2.xxpre
Disallow: /
User-agent: ASPseek/1.2.9d
Disallow: /
User-agent: ASPseek/1.2.xx
Disallow: /
User-agent: ASSORT/0.10
Disallow: /
User-agent: AU-MIC/2.0 MMP/2.0
Disallow: /
User-agent: AUDIOVOX-SMT5600
Disallow: /
User-agent: AV Fetch 1.0
Disallow: /
User-agent: AVSearch-1.0(peter.turney@nrc.ca)
Disallow: /
User-agent: AVSearch-2.0-fusionIdx-14-CompetitorWebSites
Disallow: /
User-agent: AVSearch-3.0(AltaVista/AVC)
Disallow: /
User-agent: AWeb
Disallow: /
User-agent: AbachoBOT
Disallow: /
User-agent: AbachoBOT (Mozilla compatible)
Disallow: /
User-agent: Aberja Checkomat
Disallow: /
User-agent: About/0.1libwww-perl/5.47
Disallow: /
User-agent: Accelatech RSSCrawler/0.4
Disallow: /
User-agent: Accoona-AI-Agent/1.1.1 (crawler at accoona dot com)
Disallow: /
User-agent: Accoona-AI-Agent/1.1.2 (aicrawler at accoonabot dot com)
Disallow: /
User-agent: Ace Explorer
Disallow: /
User-agent: Ack (http://www.ackerm.com/)
Disallow: /
User-agent: AcoiRobot
Disallow: /
User-agent: Acoon Robot v1.50.001
Disallow: /
User-agent: Acoon Robot v1.52 (http://www.acoon.de)
Disallow: /
User-agent: Acoon-Robot 4.0.x.[xx] (http://www.acoon.de)
Disallow: /
User-agent: Acoon-Robot v3.xx (http://www.acoon.de and http://www.acoon.com)
Disallow: /
User-agent: Acorn/Nutch-0.9 (Non-Profit Search Engine; acorn.isara.org; acorn at isara dot org)
Disallow: /
User-agent: ActiveBookmark 1.x
Disallow: /
User-agent: ActiveWorlds/3.xx (xxx)
Disallow: /
User-agent: Activeworlds
Disallow: /
User-agent: Ad Muncher v4.xx.x
Disallow: /
User-agent: Ad Muncher v4x Build xxxxx
Disallow: /
User-agent: Adaxas Spider (http://www.adaxas.net/)
Disallow: /
User-agent: Advanced Browser (http://www.avantbrowser.com)
Disallow: /
User-agent: Agent-SharewarePlazaFileCheckBot/2.0+(+http://www.SharewarePlaza.com)
Disallow: /
User-agent: AgentName/0.1 libwww-perl/5.48
Disallow: /
User-agent: AideRSS/1.0 (aiderss.com)
Disallow: /
User-agent: Aladin/3.324
Disallow: /
User-agent: Alcatel-BG3/1.0 UP.Browser/5.0.3.1.2
Disallow: /
User-agent: Aleksika Spider/1.0 (+http://www.aleksika.com/)
Disallow: /
User-agent: AlkalineBOT/1.3
Disallow: /
User-agent: AlkalineBOT/1.4 (1.4.0326.0 RTM)
Disallow: /
User-agent: Allesklar/0.1 libwww-perl/5.46
Disallow: /
User-agent: Alligator 1.31 (www.nearsoftware.com)
Disallow: /
User-agent: AltaVista Intranet V2.0 AVS EVAL search@freeit.com
Disallow: /
User-agent: AltaVista Intranet V2.0 Compaq Altavista Eval sveand@altavista.net
Disallow: /
User-agent: AltaVista Intranet V2.0 evreka.com crawler@evreka.com
Disallow: /
User-agent: AltaVista V2.0B crawler@evreka.com
Disallow: /
User-agent: AmfibiBOT
Disallow: /
User-agent: Amfibibot/0.06 (Amfibi Web Search; http://www.amfibi.com; agent@amfibi.com)
Disallow: /
User-agent: Amfibibot/0.07 (Amfibi Robot; http://www.amfibi.com; agent@amfibi.com)
Disallow: /
User-agent: AmiTCP Miami (AmigaOS 2.04)
Disallow: /
User-agent: Amiga-AWeb/3.4.167SE
Disallow: /
User-agent: AmigaVoyager/3.4.4 (MorphOS/PPC native)
Disallow: /
User-agent: Amoi 8512/R21.0 NF-Browser/3.3
Disallow: /
User-agent: AnnoMille spider 0.1 alpha - http://www.annomille.it
Disallow: /
User-agent: Anonymized by ProxyOS: http://www.megaproxy.com
Disallow: /
User-agent: Anonymizer/1.1
Disallow: /
User-agent: AnswerBus (http://www.answerbus.com/)
Disallow: /
User-agent: AnswerChase PROve x.0
Disallow: /
User-agent: AnswerChase x.0
Disallow: /
User-agent: AnzwersCrawl/2.0 (anzwerscrawl@anzwers.com.au;Engine)
Disallow: /
User-agent: Apexoo Spider 1.x
Disallow: /
User-agent: Aplix HTTP/1.0.1
Disallow: /
User-agent: Aplix_SANYO_browser/1.x (Japanese)
Disallow: /
User-agent: Aplix_SEGASATURN_browser/1.x (Japanese)
Disallow: /
User-agent: Aport
Disallow: /
User-agent: Apple iPhone v1.1.4 CoreMedia v1.0.0.4A102
Disallow: /
User-agent: ArabyBot (compatible; Mozilla/5.0; GoogleBot; FAST Crawler 6.4; http://www.araby.com;)
Disallow: /
User-agent: ArachBot
Disallow: /
User-agent: Arachnoidea (arachnoidea@euroseek.com)
Disallow: /
User-agent: ArchitextSpider
Disallow: /
User-agent: Argus/1.1 (Nutch; http://www.simpy.com/bot.html; feedback at simpy dot com)
Disallow: /
User-agent: Arikus_Spider
Disallow: /
User-agent: Arquivo-web-crawler  (compatible; heritrix/1.12.1 +http://arquivo-web.fccn.pt)
Disallow: /
User-agent: Asahina-Antenna/1.x
Disallow: /
User-agent: Asahina-Antenna/1.x (libhina.pl/x.x ; libtime.pl/x.x)
Disallow: /
User-agent: AskAboutOil/0.06-rcp (Nutch; http://www.nutch.org/docs/en/bot.html; nutch-agent@askaboutoil.com)
Disallow: /
User-agent: AtlocalBot/1.1 +(http://www.atlocal.com/local-web-site-owner.html)
Disallow: /
User-agent: Atomic_Email_Hunter/4.0
Disallow: /
User-agent: Atomz/1.0
Disallow: /
User-agent: Attentio/Nutch-0.9-dev (Attentio's beta blog crawler; www.attentio.com; info@attentio.com)
Disallow: /
User-agent: Avant Browser (http://www.avantbrowser.com)
Disallow: /
User-agent: AxmoRobot - Crawling your site for better indexing on www.axmo.com search engine.
Disallow: /
User-agent: Azureus 2.x.x.x
Disallow: /
User-agent: BDFetch
Disallow: /
User-agent: BDNcentral Crawler v2.3 [en] (http://www.bdncentral.com/robot.html) (X11; I; Linux 2.0.44 i686)
Disallow: /
User-agent: BIGLOTRON (Beta 2;GNU/Linux)
Disallow: /
User-agent: BMCLIENT
Disallow: /
User-agent: BMLAUNCHER
Disallow: /
User-agent: BPImageWalker/2.0 (www.bdbrandprotect.com)
Disallow: /
User-agent: BSDSeek/1.0
Disallow: /
User-agent: BStop.BravoBrian.it Agent Detector
Disallow: /
User-agent: BTWebClient/180B(9704)
Disallow: /
User-agent: BTbot/0.x (+http://www.btbot.com/btbot.html)
Disallow: /
User-agent: BW-C-2.0
Disallow: /
User-agent: BaboomBot/1.x.x (+http://www.baboom.us)
Disallow: /
User-agent: BackStreet Browser 3.x
Disallow: /
User-agent: BaiDuSpider
Disallow: /
User-agent: BaiduImagespider+(+http://www.baidu.jp/search/s308.html)
Disallow: /
User-agent: Baiduspider+(+http://help.baidu.jp/system/05.html)
Disallow: /
User-agent: Baiduspider+(+http://www.baidu.com/search/spider.htm)
Disallow: /
User-agent: Baiduspider+(+http://www.baidu.com/search/spider_jp.html)
Disallow: /
User-agent: Balihoo/Nutch-1.0-dev (Crawler for Balihoo.com search engine - obeys robots.txt and robots meta tags ; http://balihoo.com/index.aspx; robot at balihoo dot com)
Disallow: /
User-agent: BanBots/1.2 (spider@banbots.com)
Disallow: /
User-agent: Barca/2.0.xxxx
Disallow: /
User-agent: BarcaPro/1.4.xxxx
Disallow: /
User-agent: BarraHomeCrawler (albertof@barrahome.org)
Disallow: /
User-agent: BeamMachine/0.5 (dead link remover of www.beammachine.net)
Disallow: /
User-agent: BebopBot/2.5.1 ( crawler http://www.apassion4jazz.net/bebopbot.html )
Disallow: /
User-agent: BeebwareDirectory/v0.01
Disallow: /
User-agent: Big Brother (http://pauillac.inria.fr/~fpottier/)
Disallow: /
User-agent: Big Fish v1.0
Disallow: /
User-agent: BigBrother/1.6e
Disallow: /
User-agent: BigCliqueBOT/1.03-dev (bigclicbot; http://www.bigclique.com; bot@bigclique.com)
Disallow: /
User-agent: Bigsearch.ca/Nutch-x.x-dev (Bigsearch.ca Internet Spider; http://www.bigsearch.ca/; info@enhancededge.com)
Disallow: /
User-agent: Bilbo/2.3b-UNIX
Disallow: /
User-agent: BilgiBetaBot/0.8-dev (bilgi.com (Beta) ; http://lucene.apache.org/nutch/bot.html; nutch-agent@lucene.apache.org)
Disallow: /
User-agent: BilgiBot/1.0(beta) (http://www.bilgi.com/; bilgi at bilgi dot com)
Disallow: /
User-agent: Bitacle Robot (V:1.0;) (http://www.bitacle.com)
Disallow: /
User-agent: Bitacle bot/1.1
Disallow: /
User-agent: Biyubi/x.x (Sistema Fenix; G11; Familia Toledo; es-mx)
Disallow: /
User-agent: BlackBerry7520/4.0.0 Profile/MIDP-2.0 Configuration/CLDC-1.1 UP.Browser/5.0.3.3 UP.Link/5.1.2.12 (Google WAP Proxy/1.0)
Disallow: /
User-agent: BlackWidow
Disallow: /
User-agent: Blaiz-Bee/1.0 (+http://www.blaiz.net)
Disallow: /
User-agent: Blaiz-Bee/2.00.8222 (BE Internet Search Engine http://www.rawgrunt.com)
Disallow: /
User-agent: Blaiz-Bee/2.00.xxxx (+http://www.blaiz.net)
Disallow: /
User-agent: BlitzBOT@tricus.net
Disallow: /
User-agent: BlitzBOT@tricus.net (Mozilla compatible)
Disallow: /
User-agent: BlockNote.Net
Disallow: /
User-agent: BlogBot/1.x
Disallow: /
User-agent: BlogBridge 2.13 (http://www.blogbridge.com/)
Disallow: /
User-agent: BlogMap (http://www.feedmap.net)
Disallow: /
User-agent: BlogPulseLive (support@blogpulse.com)
Disallow: /
User-agent: BlogSearch/1.x +http://www.icerocket.com/
Disallow: /
User-agent: BlogVibeBot-v1.1 (spider@blogvibe.nl)
Disallow: /
User-agent: Bloglines Title Fetch/1.0 (http://www.bloglines.com)
Disallow: /
User-agent: Bloglines-Images/0.1 (http://www.bloglines.com)
Disallow: /
User-agent: Blogpulse (info@blogpulse.com)
Disallow: /
User-agent: BlogsNowBot, V 2.01 (+http://www.blogsnow.com/)
Disallow: /
User-agent: BlogzIce/1.0 (+http://icerocket.com; rhodes@icerocket.com)
Disallow: /
User-agent: BlogzIce/1.0 +http://www.icerocket.com/
Disallow: /
User-agent: BloobyBot
Disallow: /
User-agent: Bloodhound/Nutch-0.9 (Testing Crawler for Research - obeys robots.txt and robots meta tags ; http://balihoo.com/index.aspx; robot at balihoo dot com)
Disallow: /
User-agent: Bobby/4.0.x RPT-HTTPClient/0.3-3E
Disallow: /
User-agent: Bookdog/x.x
Disallow: /
User-agent: Bookmark Buddy bookmark checker (http://www.bookmarkbuddy.net/)
Disallow: /
User-agent: Bookmark Renewal Check Agent [http://www.bookmark.ne.jp/]
Disallow: /
User-agent: Bookmark Renewal Check Agent [http://www.bookmark.ne.jp/] (Version 2.0beta)
Disallow: /
User-agent: BookmarkBase(2/;http://bookmarkbase.com)
Disallow: /
User-agent: Bot mailto:craftbot@yahoo.com
Disallow: /
User-agent: BravoBrian SpiderEngine MarcoPolo
Disallow: /
User-agent: BravoBrian bstop.bravobrian.it
Disallow: /
User-agent: BrightCrawler (http://www.brightcloud.com/brightcrawler.asp)
Disallow: /
User-agent: BruinBot (+http://webarchive.cs.ucla.edu/bruinbot.html)
Disallow: /
User-agent: BuildCMS crawler (http://www.buildcms.com/crawler)
Disallow: /
User-agent: Bulkfeeds/r1752 (http://bulkfeeds.net/)
Disallow: /
User-agent: BullsEye
Disallow: /
User-agent: BunnySlippers
Disallow: /
User-agent: BurstFindCrawler/1.1 (crawler.burstfind.com; http://crawler.burstfind.com; crawler@burstfind.com)
Disallow: /
User-agent: Buscaplus Robi/1.0 (http://www.buscaplus.com/robi/)
Disallow: /
User-agent: CCBot/1.0 (+http://www.commoncrawl.org/bot.html)
Disallow: /
User-agent: CDR/1.7.1 Simulator/0.7(+http://timewe.net) Profile/MIDP-1.0 Configuration/CLDC-1.0
Disallow: /
User-agent: CE-Preload
Disallow: /
User-agent: CFNetwork/x.x
Disallow: /
User-agent: CHttpClient by Open Text Corporation
Disallow: /
User-agent: CJ Spider/
Disallow: /
User-agent: CJB.NET Proxy
Disallow: /
User-agent: COAST WebMaster Pro/4.x.x.xx (Windows NT)
Disallow: /
User-agent: CSE HTML Validator Professional (http://www.htmlvalidator.com/)
Disallow: /
User-agent: Cabot/Nutch-0.9 (Amfibi's web-crawling robot; http://www.amfibi.com/cabot/; agent@amfibi.com)
Disallow: /
User-agent: Cabot/Nutch-1.0-dev (Amfibi's web-crawling robot; http://www.amfibi.com/cabot/; agent@amfibi.com)
Disallow: /
User-agent: CamelHttpStream/1.0
Disallow: /
User-agent: Cancer Information and Support International;
Disallow: /
User-agent: Carnegie_Mellon_University_Research_WebBOT-->PLEASE READ-->http://www.andrew.cmu.edu/~brgordon/webbot/index.html http://www.andrew.cmu.edu/~brgordon/webbot/index.html
Disallow: /
User-agent: Carnegie_Mellon_University_WebCrawler http://www.andrew.cmu.edu/~brgordon/webbot/index.html
Disallow: /
User-agent: Catall Spider
Disallow: /
User-agent: CazoodleBot/CazoodleBot-0.1 (CazoodleBot Crawler; http://www.cazoodle.com/cazoodlebot; cazoodlebot@cazoodle.com)
Disallow: /
User-agent: CentiverseBot
Disallow: /
User-agent: CentiverseBot - investigator
Disallow: /
User-agent: CentiverseBot/3.0 (http://www.centiverse-project.net)
Disallow: /
User-agent: Ceramic Tile Installation Guide (http://www.floorstransformed.com)
Disallow: /
User-agent: Charon/1.x (Amiga)
Disallow: /
User-agent: CheckLinks/1.x.x
Disallow: /
User-agent: CheckUrl
Disallow: /
User-agent: CheckWeb
Disallow: /
User-agent: Checkbot/1.xx LWP/5.xx
Disallow: /
User-agent: Chilkat/1.0.0 (+http://www.chilkatsoft.com/ChilkatHttpUA.asp)
Disallow: /
User-agent: China Local Browse 2.6
Disallow: /
User-agent: Chitika ContentHit 1.0
Disallow: /
User-agent: ChristCRAWLER 2.0
Disallow: /
User-agent: CipinetBot (http://www.cipinet.com/bot.html)
Disallow: /
User-agent: Cityreview Robot (+http://www.cityreview.org/crawler/)
Disallow: /
User-agent: ClariaBot/1.0
Disallow: /
User-agent: Claymont.com
Disallow: /
User-agent: CloakDetect/0.9 (+http://fulltext.seznam.cz/)
Disallow: /
User-agent: Clushbot/2.x (+http://www.clush.com/bot.html)
Disallow: /
User-agent: Clushbot/3.x-BinaryFury (+http://www.clush.com/bot.html)
Disallow: /
User-agent: Clushbot/3.xx-Ajax (+http://www.clush.com/bot.html)
Disallow: /
User-agent: Clushbot/3.xx-Hector (+http://www.clush.com/bot.html)
Disallow: /
User-agent: Clushbot/3.xx-Peleus (+http://www.clush.com/bot.html)
Disallow: /
User-agent: CoBITSProbe
Disallow: /
User-agent: Cocoal.icio.us/1.0 (v36) (Mac OS X; http://www.scifihifi.com/cocoalicious)
Disallow: /
User-agent: ColdFusion
Disallow: /
User-agent: ColdFusion (BookmarkTracker.com)
Disallow: /
User-agent: Combine/2.0 http://combine.it.lth.se/
Disallow: /
User-agent: Combine/3 http://combine.it.lth.se/
Disallow: /
User-agent: Combine/x.0
Disallow: /
User-agent: Commerce Browser Center
Disallow: /
User-agent: Computer_and_Automation_Research_Institute_Crawler crawler@ilab.sztaki.hu
Disallow: /
User-agent: Comrite/0.7.1 (Nutch; http://lucene.apache.org/nutch/bot.html; nutch-agent@lucene.apache.org)
Disallow: /
User-agent: Contact
Disallow: /
User-agent: ContactBot/0.2
Disallow: /
User-agent: ContentSmartz
Disallow: /
User-agent: Convera Internet Spider V6.x
Disallow: /
User-agent: ConveraCrawler/0.2
Disallow: /
User-agent: ConveraCrawler/0.9d (+http://www.authoritativeweb.com/crawl)
Disallow: /
User-agent: ConveraMultiMediaCrawler/0.1 (+http://www.authoritativeweb.com/crawl)
Disallow: /
User-agent: CoolBot
Disallow: /
User-agent: CoralWebPrx/0.1.1x (See http://coralcdn.org/)
Disallow: /
User-agent: CoteoNutchCrawler/Nutch-0.9 (info [at] coteo [dot] com)
Disallow: /
User-agent: CougarSearch/0.x (+http://www.cougarsearch.com/faq.shtml)
Disallow: /
User-agent: Covac TexAs Arachbot
Disallow: /
User-agent: Cowbot-0.1 (NHN Corp. / +82-2-3011-1954 / nhnbot@naver.com)
Disallow: /
User-agent: Cowbot-0.1.x (NHN Corp. / +82-2-3011-1954 / nhnbot@naver.com)
Disallow: /
User-agent: CrawlConvera0.1 (CrawlConvera@yahoo.com)
Disallow: /
User-agent: Crawler
Disallow: /
User-agent: Crawler (cometsearch@cometsystems.com)
Disallow: /
User-agent: Crawler V 0.2.x admin@crawler.de
Disallow: /
User-agent: Crawler admin@crawler.de
Disallow: /
User-agent: CrawlerBoy Pinpoint.com
Disallow: /
User-agent: Crawllybot/0.1 (Crawllybot; +http://www.crawlly.com; crawler@crawlly.com)
Disallow: /
User-agent: CreativeCommons/0.06-dev (Nutch; http://www.nutch.org/docs/en/bot.html; nutch-agent@lists.sourceforge.net)
Disallow: /
User-agent: CrocCrawler vx.3 [en] (http://www.croccrawler.com) (X11; I; Linux 2.0.44 i686)
Disallow: /
User-agent: Cuam Ver0.050bx
Disallow: /
User-agent: Cuasarbot/0.9b http://www.cuasar.com/spider_beta/
Disallow: /
User-agent: CurryGuide SiteScan 1.1
Disallow: /
User-agent: Custo x.x (www.netwu.com)
Disallow: /
User-agent: Custom Spider www.bisnisseek.com /1.0
Disallow: /
User-agent: CyberPatrol SiteCat Webbot (http://www.cyberpatrol.com/cyberpatrolcrawler.asp)
Disallow: /
User-agent: CyberSpyder Link Test/2.1.12 (admin@mspennyworth.com)
Disallow: /
User-agent: Cyberdog/2.0 (Macintosh; 68k)
Disallow: /
User-agent: CydralSpider/1.x (Cydral Web Image Search; http://www.cydral.com)
Disallow: /
User-agent: CydralSpider/3.0 (Cydral Image Search; http://www.cydral.com)
Disallow: /
User-agent: DA 3.5 (www.lidan.com)
Disallow: /
User-agent: DA 4.0
Disallow: /
User-agent: DA 4.0 (www.downloadaccelerator.com)
Disallow: /
User-agent: DA 5.0
Disallow: /
User-agent: DA 7.0
Disallow: /
User-agent: DBrowse 1.4b
Disallow: /
User-agent: DBrowse 1.4d
Disallow: /
User-agent: DC-Sakura/x.xx
Disallow: /
User-agent: DDD
Disallow: /
User-agent: DIIbot/1.2
Disallow: /
User-agent: DISCo Pump x.x
Disallow: /
User-agent: DNSRight.com WebBot Link Ckeck Tool. Report abuse to: dnsr@dnsright.com
Disallow: /
User-agent: DSurf15a 01
Disallow: /
User-agent: DSurf15a 71
Disallow: /
User-agent: DSurf15a 81
Disallow: /
User-agent: DSurf15a VA
Disallow: /
User-agent: DTAAgent
Disallow: /
User-agent: Dart Communications PowerTCP
Disallow: /
User-agent: DataCha0s/2.0
Disallow: /
User-agent: DataFountains/DMOZ Downloader
Disallow: /
User-agent: DataFountains/DMOZ Feature Vector Corpus Creator (http://ivia.ucr.edu/useragents.shtml)
Disallow: /
User-agent: DataFountains/Dmoz Downloader (http://ivia.ucr.edu/useragents.shtml)
Disallow: /
User-agent: DataSpear/1.0 (Spider; http://www.dataspear.com/spider.html; spider@dataspear.com)
Disallow: /
User-agent: DataSpearSpiderBot/0.2 (DataSpear Spider Bot; http://dssb.dataspear.com/bot.html; dssb@dataspear.com)
Disallow: /
User-agent: DataparkSearch/4.47 (+http://dataparksearch.org/bot)
Disallow: /
User-agent: DataparkSearch/4.xx (http://www.dataparksearch.org/)
Disallow: /
User-agent: DatenBot( http://www.sicher-durchs-netz.de/bot.html)
Disallow: /
User-agent: DaviesBot/1.7 (www.wholeweb.net)
Disallow: /
User-agent: DeadLinkCheck/0.4.0 libwww-perl/5.xx
Disallow: /
User-agent: Deep Link Calculator v1.0
Disallow: /
User-agent: DeepIndex
Disallow: /
User-agent: DeepIndex ( http://www.zetbot.com )
Disallow: /
User-agent: DeepIndex (www.en.deepindex.com)
Disallow: /
User-agent: DeepIndexer.ca
Disallow: /
User-agent: DeleGate/9.0.5-fix1
Disallow: /
User-agent: Demo Bot DOT 16b
Disallow: /
User-agent: Demo Bot Z 16b
Disallow: /
User-agent: Denmex websearch (http://search.denmex.com)
Disallow: /
User-agent: Der große BilderSauger 2.00u
Disallow: /
User-agent: DevComponents.com HtmlDocument Object
Disallow: /
User-agent: DiaGem/1.1 (http://www.skyrocket.gr.jp/diagem.html)
Disallow: /
User-agent: Diamond/x.0
Disallow: /
User-agent: DiamondBot
Disallow: /
User-agent: DigOut4U
Disallow: /
User-agent: Digger/1.0 JDK/1.3.0rc3
Disallow: /
User-agent: Dillo/0.8.5-i18n-misc
Disallow: /
User-agent: Dillo/0.x.x
Disallow: /
User-agent: DittoSpyder
Disallow: /
User-agent: DoCoMo/1.0/Nxxxi/c10
Disallow: /
User-agent: DoCoMo/1.0/Nxxxi/c10/TB
Disallow: /
User-agent: DoCoMo/1.0/P502i/c10 (Google CHTML Proxy/1.0)
Disallow: /
User-agent: DoCoMo/2.0 P900iV(c100;TB;W24H11)
Disallow: /
User-agent: DoCoMo/2.0 SH901iS(c100;TB;W24H12),gzip(gfe) (via translate.google.com)
Disallow: /
User-agent: DoCoMo/2.0 SH902i (compatible; Y!J-SRD/1.0; http://help.yahoo.co.jp/help/jp/search/indexing/indexing-27.html)
Disallow: /
User-agent: DoCoMo/2.0/SO502i (compatible; Y!J-SRD/1.0; http://help.yahoo.co.jp/help/jp/search/indexing/indexing-27.html)
Disallow: /
User-agent: DocZilla/1.0 (Windows; U; WinNT4.0; en-US; rv:1.0.0) Gecko/20020804
Disallow: /
User-agent: DonutP; Windows98SE
Disallow: /
User-agent: Doubanbot/1.0 (bot@douban.com http://www.douban.com)
Disallow: /
User-agent: Download Demon/3.x.x.x
Disallow: /
User-agent: Download Druid 2.x
Disallow: /
User-agent: Download Express 1.0
Disallow: /
User-agent: Download Master
Disallow: /
User-agent: Download Ninja 3.0
Disallow: /
User-agent: Download Wonder
Disallow: /
User-agent: Download-Tipp Linkcheck (http://download-tipp.de/)
Disallow: /
User-agent: Download.exe(1.1) (+http://www.sql-und-xml.de/freeware-tools/)
Disallow: /
User-agent: DownloadDirect.1.0
Disallow: /
User-agent: Dr.Web (R) online scanner: http://online.drweb.com/
Disallow: /
User-agent: Dragonfly File Reader
Disallow: /
User-agent: Drecombot/1.0 (http://career.drecom.jp/bot.html)
Disallow: /
User-agent: Drupal (+http://drupal.org/)
Disallow: /
User-agent: Dual Proxy
Disallow: /
User-agent: DuckDuckBot/1.0; (+http://duckduckgo.com/duckduckbot.html)
Disallow: /
User-agent: Dumbot(version 0.1 beta - dumbfind.com)
Disallow: /
User-agent: Dumbot(version 0.1 beta - http://www.dumbfind.com/dumbot.html)
Disallow: /
User-agent: Dumbot(version 0.1 beta)
Disallow: /
User-agent: EARTHCOM.info/1.x [www.earthcom.info]
Disallow: /
User-agent: EARTHCOM.info/1.xbeta [www.earthcom.info]
Disallow: /
User-agent: EBrowse 1.4b
Disallow: /
User-agent: ELI/20070402:2.0 (DAUM RSS Robot, Daum Communications Corp.; +http://ws.daum.net/aboutkr.html)
Disallow: /
User-agent: ELinks (0.x.x; Linux 2.4.20 i586; 132x60)
Disallow: /
User-agent: ELinks/0.x.x (textmode; NetBSD 1.6.2 sparc; 132x43)
Disallow: /
User-agent: EMPAS_ROBOT
Disallow: /
User-agent: ES.NET_Crawler/2.0 (http://search.innerprise.net/)
Disallow: /
User-agent: ESISmartSpider
Disallow: /
User-agent: ESurf15a 15
Disallow: /
User-agent: EasyDL/3.xx
Disallow: /
User-agent: EasyDL/3.xx http://keywen.com/Encyclopedia/Bot
Disallow: /
User-agent: EchO!/2.0
Disallow: /
User-agent: Educate Search VxB
Disallow: /
User-agent: EgotoBot/4.8 (+http://www.egoto.com/about.htm)
Disallow: /
User-agent: EldoS TimelyWeb/3.x
Disallow: /
User-agent: EmailSiphon
Disallow: /
User-agent: EmailSpider
Disallow: /
User-agent: EmailWolf 1.00
Disallow: /
User-agent: EmeraldShield.com WebBot
Disallow: /
User-agent: EmeraldShield.com WebBot (http://www.emeraldshield.com/webbot.aspx)
Disallow: /
User-agent: EnaBot/1.x (http://www.enaball.com/crawler.html)
Disallow: /
User-agent: Enfish Tracker
Disallow: /
User-agent: Enterprise_Search/1.0
Disallow: /
User-agent: Enterprise_Search/1.0.xxx
Disallow: /
User-agent: Enterprise_Search/1.00.xxx;MSSQL (http://www.innerprise.net/es-spider.asp)
Disallow: /
User-agent: EroCrawler
Disallow: /
User-agent: EuripBot/0.x (+http://www.eurip.com) GetFile
Disallow: /
User-agent: EuripBot/0.x (+http://www.eurip.com) GetRobots
Disallow: /
User-agent: EuripBot/0.x (+http://www.eurip.com) PreCheck
Disallow: /
User-agent: Eurobot/1.0 (http://www.ayell.eu)
Disallow: /
User-agent: EvaalSE - bot@evaal.com
Disallow: /
User-agent: Everest-Vulcan Inc./0.1 (R&D project; host=e-1-24; http://everest.vulcan.com/crawlerhelp)
Disallow: /
User-agent: Everest-Vulcan Inc./0.1 (R&D project; http://everest.vulcan.com/crawlerhelp)
Disallow: /
User-agent: Exabot-Images/1.0
Disallow: /
User-agent: Exabot-Test/1.0
Disallow: /
User-agent: Exabot/2.0
Disallow: /
User-agent: Exabot/3.0
Disallow: /
User-agent: ExactSearch
Disallow: /
User-agent: ExactSeek Crawler/0.1
Disallow: /
User-agent: Exalead NG/MimeLive Client (convert/http/0.120)
Disallow: /
User-agent: Excalibur Internet Spider V6.5.4
Disallow: /
User-agent: Execrawl/1.0 (Execrawl; http://www.execrawl.com/; bot@execrawl.com)
Disallow: /
User-agent: ExperimentalHenrytheMiragoRobot
Disallow: /
User-agent: Expired Domain Sleuth
Disallow: /
User-agent: Express WebPictures (www.express-soft.com)
Disallow: /
User-agent: ExtractorPro
Disallow: /
User-agent: Extreme Picture Finder
Disallow: /
User-agent: EyeCatcher (Download-tipp.de)/1.0
Disallow: /
User-agent: FANGCrawl/0.01
Disallow: /
User-agent: FARK.com link verifier
Disallow: /
User-agent: FAST Enterprise Crawler 6 (Experimental)
Disallow: /
User-agent: FAST Enterprise Crawler 6 / Scirus scirus-crawler@fast.no; http://www.scirus.com/srsapp/contactus/
Disallow: /
User-agent: FAST Enterprise Crawler 6 used by Cobra Development (admin@fastsearch.com)
Disallow: /
User-agent: FAST Enterprise Crawler 6 used by Comperio AS (sts@comperio.no)
Disallow: /
User-agent: FAST Enterprise Crawler 6 used by FAST (FAST)
Disallow: /
User-agent: FAST Enterprise Crawler 6 used by Pages Jaunes (pvincent@pagesjaunes.fr)
Disallow: /
User-agent: FAST Enterprise Crawler 6 used by Sensis.com.au Web Crawler (search_comments\at\sensis\dot\com\dot\au)
Disallow: /
User-agent: FAST Enterprise Crawler 6 used by Singapore Press Holdings (crawler@sphsearch.sg)
Disallow: /
User-agent: FAST Enterprise Crawler 6 used by WWU (wardi@uni-muenster.de)
Disallow: /
User-agent: FAST Enterprise Crawler/6 (www.fastsearch.com)
Disallow: /
User-agent: FAST Enterprise Crawler/6.4 (helpdesk at fast.no)
Disallow: /
User-agent: FAST FirstPage retriever (compatible; MSIE 5.5; Mozilla/4.0)
Disallow: /
User-agent: FAST MetaWeb Crawler (helpdesk at fastsearch dot com)
Disallow: /
User-agent: FAST-WebCrawler/2.2.10 (Multimedia Search) (crawler@fast.no; http://www.fast.no/faq/faqfastwebsearch/faqfastwebcrawler.html)
Disallow: /
User-agent: FAST-WebCrawler/2.2.6 (crawler@fast.no; http://www.fast.no/faq/faqfastwebsearch/faqfastwebcrawler.html)
Disallow: /
User-agent: FAST-WebCrawler/2.2.7 (crawler@fast.no; http://www.fast.no/faq/faqfastwebsearch/faqfastwebcrawler.html)http://www.fast.no
Disallow: /
User-agent: FAST-WebCrawler/2.2.8 (crawler@fast.no; http://www.fast.no/faq/faqfastwebsearch/faqfastwebcrawler.html)http://www.fast.no
Disallow: /
User-agent: FAST-WebCrawler/3.2 test
Disallow: /
User-agent: FAST-WebCrawler/3.3 (crawler@fast.no; http://fast.no/support.php?c=faqs/crawler)
Disallow: /
User-agent: FAST-WebCrawler/3.4/Nirvana (crawler@fast.no; http://fast.no/support.php?c=faqs/crawler)
Disallow: /
User-agent: FAST-WebCrawler/3.4/PartnerSite (crawler@fast.no; http://fast.no/support.php?c=faqs/crawler)
Disallow: /
User-agent: FAST-WebCrawler/3.5 (atw-crawler at fast dot no; http://fast.no/support.php?c=faqs/crawler)
Disallow: /
User-agent: FAST-WebCrawler/3.6 (atw-crawler at fast dot no; http://fast.no/support/crawler.asp)
Disallow: /
User-agent: FAST-WebCrawler/3.6/FirstPage (crawler@fast.no; http://fast.no/support.php?c=faqs/crawler)
Disallow: /
User-agent: FAST-WebCrawler/3.7 (atw-crawler at fast dot no; http://fast.no/support/crawler.asp)
Disallow: /
User-agent: FAST-WebCrawler/3.7/FirstPage (atw-crawler at fast dot no;http://fast.no/support/crawler.asp)
Disallow: /
User-agent: FAST-WebCrawler/3.8 (atw-crawler at fast dot no; http://fast.no/support/crawler.asp)
Disallow: /
User-agent: FAST-WebCrawler/3.8/Fresh (atw-crawler at fast dot no; http://fast.no/support/crawler.asp)
Disallow: /
User-agent: FAST-WebCrawler/3.x Multimedia
Disallow: /
User-agent: FAST-WebCrawler/3.x Multimedia (mm dash crawler at fast dot no)
Disallow: /
User-agent: FDM 1.x
Disallow: /
User-agent: FDM 2.x
Disallow: /
User-agent: FFC Trap Door Spider
Disallow: /
User-agent: FLATARTS_FAVICO
Disallow: /
User-agent: FSurf15a 01
Disallow: /
User-agent: FaEdit/2.0.x
Disallow: /
User-agent: Factbot 1.09 (see http://www.factbites.com/webmasters.php)
Disallow: /
User-agent: FairAd Client
Disallow: /
User-agent: Fast Crawler Gold Edition
Disallow: /
User-agent: Fast PartnerSite Crawler
Disallow: /
User-agent: FastBug http://www.ay-up.com
Disallow: /
User-agent: FastCrawler 3.0.1 (crawler@1klik.dk)
Disallow: /
User-agent: FastSearch Web Crawler for Verizon SuperPages (kevin.watters@fastsearch.com)
Disallow: /
User-agent: FavIconizer
Disallow: /
User-agent: FavOrg
Disallow: /
User-agent: Favcollector/2.0 (info@favcollector.com http://www.favcollector.com/)
Disallow: /
User-agent: Favorites Checking (http://campulka.net)
Disallow: /
User-agent: Favorites Sweeper v.2.03
Disallow: /
User-agent: Faxobot/1.0
Disallow: /
User-agent: Feed Seeker Bot (RSS Feed Seeker http://www.MyNewFavoriteThing.com/fsb.php)
Disallow: /
User-agent: Feed24.com
Disallow: /
User-agent: Feed::Find/0.0x
Disallow: /
User-agent: FeedChecker/0.01
Disallow: /
User-agent: FeedDemon/2.7 (http://www.newsgator.com/; Microsoft Windows XP)
Disallow: /
User-agent: FeedForAll rss2html.php v2
Disallow: /
User-agent: FeedHub FeedDiscovery/1.0 (http://www.feedhub.com)
Disallow: /
User-agent: FeedHub MetaDataFetcher/1.0 (http://www.feedhub.com)
Disallow: /
User-agent: FeedZcollector v1.x (Platinum) http://www.feeds4all.com/feedzcollector
Disallow: /
User-agent: Feedable/0.1 (compatible; MSIE 6.0; Windows NT 5.1)
Disallow: /
User-agent: Feedfetcher-Google-iGoogleGadgets; (+http://www.google.com/feedfetcher.html)
Disallow: /
User-agent: Feedfetcher-Google; (+http://www.google.com/feedfetcher.html)
Disallow: /
User-agent: Feedjit Favicon Crawler 1.0
Disallow: /
User-agent: Feedreader 3.xx (Powered by Newsbrain)
Disallow: /
User-agent: Feedshow/x.0 (http://www.feedshow.com; 1 subscriber)
Disallow: /
User-agent: FeedshowOnline (http://www.feedshow.com)
Disallow: /
User-agent: Feedster Crawler/3.0; Feedster, Inc.
Disallow: /
User-agent: Felix - Mixcat Crawler (+http://mixcat.com)
Disallow: /
User-agent: Filangy/0.01-beta (Filangy; http://www.nutch.org/docs/en/bot.html; filangy-agent@filangy.com)
Disallow: /
User-agent: Filangy/1.0x (Filangy; http://www.filangy.com/filangyinfo.jsp?inc=robots.jsp; filangy-agent@filangy.com)
Disallow: /
User-agent: Filangy/1.0x (Filangy; http://www.nutch.org/docs/en/bot.html; filangy-agent@filangy.com)
Disallow: /
User-agent: FileHound x.x
Disallow: /
User-agent: Filtrbox/1.0
Disallow: /
User-agent: FindAnISP.com_ISP_Finder_v99a
Disallow: /
User-agent: Findexa Crawler (http://www.findexa.no/gulesider/article26548.ece)
Disallow: /
User-agent: FineBot
Disallow: /
User-agent: Finjan-prefetch
Disallow: /
User-agent: Firefly/1.0
Disallow: /
User-agent: Firefly/1.0 (compatible; Mozilla 4.0; MSIE 5.5)
Disallow: /
User-agent: Firefox (kastaneta03@hotmail.com)
Disallow: /
User-agent: Firefox_1.0.6 (kasparek@naparek.cz)
Disallow: /
User-agent: FirstGov.gov Search - POC:firstgov.webmasters@gsa.gov
Disallow: /
User-agent: Flapbot/0.7.2 (Flaptor Crawler; http://www.flaptor.com; crawler at flaptor period com)
Disallow: /
User-agent: FlashGet
Disallow: /
User-agent: Flexum spider
Disallow: /
User-agent: Flexum/2.0
Disallow: /
User-agent: FlickBot 2.0 RPT-HTTPClient/0.3-3
Disallow: /
User-agent: FocusedSampler/1.0
Disallow: /
User-agent: Folkd.com Spider/0.1 beta 1 (www.folkd.com)
Disallow: /
User-agent: Fooky.com/ScorpionBot/ScoutOut; http://www.fooky.com/scorpionbots
Disallow: /
User-agent: Francis/1.0 (francis@neomo.de http://www.neomo.de/)
Disallow: /
User-agent: Franklin Locator 1.8
Disallow: /
User-agent: FreeFind.com-SiteSearchEngine/1.0 (http://freefind.com; spiderinfo@freefind.com)
Disallow: /
User-agent: FreshDownload/x.xx
Disallow: /
User-agent: FreshNotes crawler< report problems to crawler-at-freshnotes-dot-com
Disallow: /
User-agent: Full Web Bot 0416B
Disallow: /
User-agent: Full Web Bot 0516B
Disallow: /
User-agent: Full Web Bot 2816B
Disallow: /
User-agent: FuseBulb.Com
Disallow: /
User-agent: FyberSpider (+http://www.fybersearch.com/fyberspider.php)
Disallow: /
User-agent: GAIS Robot/1.0B2
Disallow: /
User-agent: GNODSPIDER (www.gnod.net)
Disallow: /
User-agent: GOFORITBOT ( http://www.goforit.com/about/ )
Disallow: /
User-agent: GSiteCrawler/v1.xx rev. xxx (http://gsitecrawler.com/)
Disallow: /
User-agent: Gagglebot
Disallow: /
User-agent: Gaisbot/3.0 (indexer@gais.cs.ccu.edu.tw; http://gais.cs.ccu.edu.tw/robot.php)
Disallow: /
User-agent: Gaisbot/3.0+(robot06@gais.cs.ccu.edu.tw;+http://gais.cs.ccu.edu.tw/robot.php)
Disallow: /
User-agent: GalaxyBot/1.0 (http://www.galaxy.com/galaxybot.html)
Disallow: /
User-agent: Gallent Search Spider v1.4 Robot 2 (http://robot.GallentSearch.com)
Disallow: /
User-agent: Gamespy_Arcade
Disallow: /
User-agent: GammaSpider/1.0
Disallow: /
User-agent: Generic Mobile Phone (compatible; Googlebot-Mobile/2.1; +http://www.google.com/bot.html)
Disallow: /
User-agent: GeoBot/1.0
Disallow: /
User-agent: GeoURLBot 1.0 (http://geourl.org)
Disallow: /
User-agent: GeonaBot 1.x; http://www.geona.com/
Disallow: /
User-agent: GetBot
Disallow: /
User-agent: GetRight/3.x.x
Disallow: /
User-agent: GetRight/4.5xx
Disallow: /
User-agent: GetRight/4.x
Disallow: /
User-agent: GetRight/4.x[a-e]
Disallow: /
User-agent: GetRight/6.1 (Pro)
Disallow: /
User-agent: GetRightPro/6.0beta2
Disallow: /
User-agent: GetWeb/0.1 libwww-perl/5.16
Disallow: /
User-agent: GhostRouteHunter/20021130 (https://www.sixxs.net/tools/grh/; info@sixxs.net)
Disallow: /
User-agent: Gigabot/2.0 (gigablast.com)
Disallow: /
User-agent: Gigabot/2.0/gigablast.com/spider.html
Disallow: /
User-agent: Gigabot/2.0; http://www.gigablast.com/spider.html
Disallow: /
User-agent: Gigabot/2.0att
Disallow: /
User-agent: Gigabot/3.0 (http://www.gigablast.com/spider.html)
Disallow: /
User-agent: Gigabot/x.0
Disallow: /
User-agent: GigabotSiteSearch/2.0 (sitesearch.gigablast.com)
Disallow: /
User-agent: Go!Zilla 3.x (www.gozilla.com)
Disallow: /
User-agent: Go!Zilla/4.x.x.xx
Disallow: /
User-agent: Go-Ahead-Got-It/1.1
Disallow: /
User-agent: GoForIt.com
Disallow: /
User-agent: GoGuides.Org Link Check
Disallow: /
User-agent: Goblin/0.9 (http://www.goguides.org/)
Disallow: /
User-agent: Goblin/0.9.x (http://www.goguides.org/goblin-info.html)
Disallow: /
User-agent: GoldenFeed Spider 1.0 (http://www.goldenfeed.com)
Disallow: /
User-agent: Goldfire Server
Disallow: /
User-agent: Goofer/0.2
Disallow: /
User-agent: Google Talk
Disallow: /
User-agent: Googlebot-Image/1.0
Disallow: /
User-agent: Googlebot-Image/1.0 ( http://www.googlebot.com/bot.html)
Disallow: /
User-agent: Googlebot/2.1 ( http://www.google.com/bot.html)
Disallow: /
User-agent: Googlebot/2.1 ( http://www.googlebot.com/bot.html)
Disallow: /
User-agent: Googlebot/Test ( http://www.googlebot.com/bot.html)
Disallow: /
User-agent: GrapeFX/0.3 libwww/5.4.0
Disallow: /
User-agent: GreatNews/1.0
Disallow: /
User-agent: GreenBrowser
Disallow: /
User-agent: GrigorBot 0.8 (http://www.grigor.biz/bot.html)
Disallow: /
User-agent: Gromit/1.0
Disallow: /
User-agent: Guestbook Auto Submitter
Disallow: /
User-agent: Gulliver/1.3
Disallow: /
User-agent: Gulper Web Bot 0.2.4 (www.ecsl.cs.sunysb.edu/~maxim/cgi-bin/Link/GulperBot)
Disallow: /
User-agent: Gungho/0.08004 (http://code.google.com/p/gungho-crawler/wiki/Index)
Disallow: /
User-agent: GurujiBot/1.0 (+http://www.guruji.com/WebmasterFAQ.html)
Disallow: /
User-agent: GurujiImageBot/1.0 (+http://www.guruji.com/en/WebmasterFAQ.html)
Disallow: /
User-agent: HLoader
Disallow: /
User-agent: HPL/Nutch-0.9 -
Disallow: /
User-agent: HTML2JPG Blackbox, http://www.html2jpg.com
Disallow: /
User-agent: HTML2JPG Enterprise
Disallow: /
User-agent: HTMLParser/1.x
Disallow: /
User-agent: HTTP Retriever
Disallow: /
User-agent: HTTP::Lite/2.x.x
Disallow: /
User-agent: HTTPEyes
Disallow: /
User-agent: HTTPResume v. 1.x
Disallow: /
User-agent: HappyFunBot/1.1
Disallow: /
User-agent: Harvest-NG/1.0.2
Disallow: /
User-agent: Haste/0.12 (HOME: http://haste.kytoon.com/)
Disallow: /
User-agent: Hatena Antenna/0.4 (http://a.hatena.ne.jp/help#robot)
Disallow: /
User-agent: Hatena Mobile Gateway/1.0
Disallow: /
User-agent: Hatena Pagetitle Agent/1.0
Disallow: /
User-agent: Hatena RSS/0.3 (http://r.hatena.ne.jp)
Disallow: /
User-agent: HatenaScreenshot/1.0 (checker)
Disallow: /
User-agent: HeinrichderMiragoRobot
Disallow: /
User-agent: HeinrichderMiragoRobot (http://www.miragorobot.com/scripts/deinfo.asp)
Disallow: /
User-agent: Helix/1.x ( http://www.sitesearch.ca/helix/)
Disallow: /
User-agent: HenriLeRobotMirago (http://www.miragorobot.com/scripts/frinfo.asp)
Disallow: /
User-agent: HenryTheMiragoRobot (http://www.miragorobot.com/scripts/mrinfo.asp)
Disallow: /
User-agent: HenrytheMiragoRobot
Disallow: /
User-agent: Hi! I'm CsCrawler my homepage: http://www.kde.cs.uni-kassel.de/lehre/ss2005/googlespam/crawler.html RPT-HTTPClient/0.3-3
Disallow: /
User-agent: HiDownload
Disallow: /
User-agent: Hippias/0.9 Beta
Disallow: /
User-agent: HitList
Disallow: /
User-agent: Hitwise Spider v1.0 http://www.hitwise.com
Disallow: /
User-agent: HomePageSearch(hpsearch.uni-trier.de)
Disallow: /
User-agent: Homerbot: www.homerweb.com
Disallow: /
User-agent: Honda-Search/0.7.2 (Nutch; http://lucene.apache.org/nutch/bot.html; search@honda-search.com)
Disallow: /
User-agent: HooWWWer/2.1.3 (debugging run) (+http://cosco.hiit.fi/search/hoowwwer/ | mailto:crawler-info<at>hiit.fi)
Disallow: /
User-agent: HooWWWer/2.1.x ( http://cosco.hiit.fi/search/hoowwwer/ | mailto:crawler-info<at>hiit.fi)
Disallow: /
User-agent: HotJava/1.0.1/JRE1.1.x
Disallow: /
User-agent: Hotzonu/x.0
Disallow: /
User-agent: Html Link Validator (www.lithopssoft.com)
Disallow: /
User-agent: Hybrid/1.2 [en] (OS Independent)
Disallow: /
User-agent: HyperEstraier/1.x.xx
Disallow: /
User-agent: IAArchiver-1.0
Disallow: /
User-agent: IBrowse/2.2 (AmigaOS 3.5)
Disallow: /
User-agent: IBrowse/2.2 (Windows 3.1)
Disallow: /
User-agent: ICC-Crawler(Mozilla-compatible; http://kc.nict.go.jp/icc/crawl.html; icc-crawl(at)ml(dot)nict(dot)go(dot)jp)
Disallow: /
User-agent: ICC-Crawler(Mozilla-compatible;http://kc.nict.go.jp/icc/crawl.html;icc-crawl-contact(at)ml(dot)nict(dot)go(dot)jp)
Disallow: /
User-agent: ICCrawler - ICjobs (http://www.icjobs.de/bot.htm)
Disallow: /
User-agent: ICE Browser/5.05 (Java 1.4.0; Windows 2000 5.0 x86)
Disallow: /
User-agent: ICOO Loader v.x.x.x
Disallow: /
User-agent: ICRA_label_spider/x.0
Disallow: /
User-agent: IDA
Disallow: /
User-agent: IEFav172Free
Disallow: /
User-agent: IIITBOT/1.1 (Indian Language Web Search Engine; http://webkhoj.iiit.net; pvvpr at iiit dot ac dot in)
Disallow: /
User-agent: INFOMINE/8.0 Adders
Disallow: /
User-agent: INFOMINE/8.0 RemoteServices
Disallow: /
User-agent: INFOMINE/8.0 VLCrawler (http://infomine.ucr.edu/useragents)
Disallow: /
User-agent: INGRID/3.0 MT (webcrawler@NOSPAMexperimental.net; http://webmaster.ilse.nl/jsp/webmaster.jsp)
Disallow: /
User-agent: IP*Works! V5 HTTP/S Component - by /n software - www.nsoftware.com
Disallow: /
User-agent: IP2MapBot/1.1 <a href=http://www.ip2map.com>http://www.ip2map.com</a>
Disallow: /
User-agent: IPiumBot laurion(dot)com
Disallow: /
User-agent: IRLbot/1.0 ( http://irl.cs.tamu.edu/crawler)
Disallow: /
User-agent: IRLbot/3.0 (compatible; MSIE 6.0; http://irl.cs.tamu.edu/crawler/)
Disallow: /
User-agent: ISC Systems iRc Search 2.1
Disallow: /
User-agent: IUPUI Research Bot v 1.9a
Disallow: /
User-agent: IWAgent/ 1.0 - www.brandprotect.com
Disallow: /
User-agent: IconSurf/2.0 favicon finder (see http://iconsurf.com/robot.html)
Disallow: /
User-agent: IconSurf/2.0 favicon monitor (see http://iconsurf.com/robot.html)
Disallow: /
User-agent: IlTrovatore-Setaccio ( http://www.iltrovatore.it)
Disallow: /
User-agent: IlTrovatore-Setaccio/1.2 ( http://www.iltrovatore.it/aiuto/faq.html)
Disallow: /
User-agent: IlTrovatore/1.2 (IlTrovatore; http://www.iltrovatore.it/bot.html; bot@iltrovatore.it)
Disallow: /
User-agent: IlseBot/1.x
Disallow: /
User-agent: Iltrovatore-Setaccio/0.3-dev (Indexing; http://www.iltrovatore.it/bot.html; info@iltrovatore.it)
Disallow: /
User-agent: Iltrovatore-Setaccio/1.2 (It-bot; http://www.iltrovatore.it/bot.html; info@iltrovatore.it)
Disallow: /
User-agent: ImageVisu/v4.x.x
Disallow: /
User-agent: ImageWalker/2.0 (www.bdbrandprotect.com)
Disallow: /
User-agent: Incutio HttpClient v0.x
Disallow: /
User-agent: IncyWincy data gatherer(webmaster@loopimprovements.com
Disallow: /
User-agent: IncyWincy page crawler(webmaster@loopimprovements.com
Disallow: /
User-agent: IncyWincy(http://www.look.com)
Disallow: /
User-agent: IncyWincy(http://www.loopimprovements.com/robot.html)
Disallow: /
User-agent: IncyWincy/2.1(loopimprovements.com/robot.html)
Disallow: /
User-agent: IndexTheWeb.com Crawler7
Disallow: /
User-agent: Industry Program 1.0.x
Disallow: /
User-agent: Inet library
Disallow: /
User-agent: InetURL/1.0
Disallow: /
User-agent: InfoFly/1.0 (http://www.versions-project.org/)
Disallow: /
User-agent: InfoLink/1.x
Disallow: /
User-agent: InfoNaviRobot(F107)
Disallow: /
User-agent: InfoSeek Sidewinder/0.9
Disallow: /
User-agent: InfoSeek Sidewinder/1.0A
Disallow: /
User-agent: InfoSeek Sidewinder/1.1A
Disallow: /
User-agent: Infoseek SideWinder/1.45 (Compatible; MSIE 10.0; UNIX)
Disallow: /
User-agent: Infoseek SideWinder/2.0B (Linux 2.4 i686)
Disallow: /
User-agent: Inktomi Search
Disallow: /
User-agent: InnerpriseBot/1.0 (http://www.innerprise.com/)
Disallow: /
User-agent: Insitor.com search and find world wide!
Disallow: /
User-agent: Insitornaut
Disallow: /
User-agent: InstallShield DigitalWizard
Disallow: /
User-agent: Intelix/0.x (cs; http://www.microton.cz/intelix/; microton@@microton.cz)
Disallow: /
User-agent: Interarchy/x.x.x (InterarchyCrawler)
Disallow: /
User-agent: Internet Ninja x.0
Disallow: /
User-agent: InternetArchive/0.8-dev(Nutch;http://lucene.apache.org/nutch/bot.html;nutch-agent@lucene.apache
Disallow: /
User-agent: InternetLinkAgent/3.1
Disallow: /
User-agent: InternetSeer.com
Disallow: /
User-agent: IpselonBot/0.xx-beta (Ipselon; http://www.ipselon.com; ipselonbot@ipselon.com)
Disallow: /
User-agent: Iria/1.xxa
Disallow: /
User-agent: IrssiUrlLog/0.2
Disallow: /
User-agent: Irvine/1.x.x
Disallow: /
User-agent: J-PHONE/3.0/J-SH07
Disallow: /
User-agent: JBH Agent 2.0
Disallow: /
User-agent: JCheckLinks/0.1 RPT-HTTPClient/0.3-1
Disallow: /
User-agent: JDK/1.1
Disallow: /
User-agent: JOC Web Spider
Disallow: /
User-agent: JRTS Check Favorites Utility
Disallow: /
User-agent: JRTwine Software Check Favorites Utility
Disallow: /
User-agent: Jabot/6.x (http://odin.ingrid.org/)
Disallow: /
User-agent: Jabot/7.x.x (http://odin.ingrid.org/)
Disallow: /
User-agent: Jack
Disallow: /
User-agent: Jakarta Commons-HttpClient/2.0xxx
Disallow: /
User-agent: Jakarta Commons-HttpClient/3.0-rcx
Disallow: /
User-agent: Jambot/0.1.x (Jambot; http://www.jambot.com/blog; crawler@jambot.com)
Disallow: /
User-agent: Jambot/0.2.1 (Jambot; http://www.jambot.com/blog/static.php?page=webmaster-robot; crawler@jambot.com)
Disallow: /
User-agent: Java 1.1
Disallow: /
User-agent: Java/1.4.1_01
Disallow: /
User-agent: Java1.0.21.0
Disallow: /
User-agent: Java1.1.xx.x
Disallow: /
User-agent: Java1.3.0rc1
Disallow: /
User-agent: Java1.3.x
Disallow: /
User-agent: Java1.4.0
Disallow: /
User-agent: Jayde Crawler. http://www.jayde.com
Disallow: /
User-agent: Jeeves
Disallow: /
User-agent: Jeode/1.x.x
Disallow: /
User-agent: JetBrains Omea Reader 1.0.x (http://www.jetbrains.com/omea_reader/)
Disallow: /
User-agent: JetBrains Omea Reader 2.0 Release Candidate 1 (http://www.jetbrains.com/omea_reader/)
Disallow: /
User-agent: JetCar
Disallow: /
User-agent: Jetbot/1.0
Disallow: /
User-agent: Jigsaw/2.2.0 W3C_CSS_Validator_JFouffa/2.0
Disallow: /
User-agent: JoBo/1.x (http://www.matuschek.net/jobo.html)
Disallow: /
User-agent: JoBo/@JOBO_VERSION@(http://www.matuschek.net/jobo.html)
Disallow: /
User-agent: JobSpider_BA/1.1
Disallow: /
User-agent: JordoMedia/1.0 RSS File Reader (http://www.jordomedia.com)
Disallow: /
User-agent: Journster [alpha] (http://journster.com/)
Disallow: /
User-agent: Journster.com RSS/Atom aggregator 0.5 (http://www.journster.com/bot.phtml)
Disallow: /
User-agent: Jyxobot/x
Disallow: /
User-agent: K-Meleon/0.6 (Windows; U; Windows NT 5.1; en-US; rv:0.9.5) Gecko/20011011
Disallow: /
User-agent: KAIST AITrc Crawler
Disallow: /
User-agent: KDDI-SN22 UP.Browser/6.0.7 (GUI) MMP/1.1 (Google WAP Proxy/1.0)
Disallow: /
User-agent: KE_1.0/2.0 libwww/5.2.8
Disallow: /
User-agent: KFSW-Bot (Version: 1.01 powered by KFSW www.kfsw.de)
Disallow: /
User-agent: KIT-Fireball/2.0
Disallow: /
User-agent: KIT-Fireball/2.0 (compatible; Mozilla 4.0; MSIE 5.5)
Disallow: /
User-agent: KSbot/1.0 (KnowledgeStorm crawler; http://www.knowledgestorm.com/resources/content/crawler/index.html; crawleradmin@knowledgestorm.com)
Disallow: /
User-agent: KakleBot - www.kakle.com/0.1 (KakleBot - www.kakle.com; http:// www.kakle.com/bot.html; support@kakle.com)
Disallow: /
User-agent: Kapere (http://www.kapere.com)
Disallow: /
User-agent: Kazehakase/0.x.x.[x]
Disallow: /
User-agent: Kenjin Spider
Disallow: /
User-agent: Kevin http://dznet.com/kevin/
Disallow: /
User-agent: Kevin http://websitealert.net/kevin/
Disallow: /
User-agent: Klondike/1.50 (WSP Win32) (Google WAP Proxy/1.0)
Disallow: /
User-agent: KnowItAll(knowitall@cs.washington.edu)
Disallow: /
User-agent: Knowledge.com/0.x
Disallow: /
User-agent: Kontiki Client x.xx
Disallow: /
User-agent: Krugle/Krugle,Nutch/0.8+ (Krugle web crawler; http://www.krugle.com/crawler/info.html; webcrawler@krugle.com)
Disallow: /
User-agent: KummHttp/1.1 (compatible; KummClient; Linux rulez)
Disallow: /
User-agent: LARBIN-EXPERIMENTAL (efp@gmx.net)
Disallow: /
User-agent: LECodeChecker/3.0 libgetdoc/1.0
Disallow: /
User-agent: LEIA/2.90
Disallow: /
User-agent: LEIA/3.01pr (LEIAcrawler; [SNIP])
Disallow: /
User-agent: LG/U8138/v1.0
Disallow: /
User-agent: LMQueueBot/0.2
Disallow: /
User-agent: LNSpiderguy
Disallow: /
User-agent: LTI/LemurProject Nutch Spider/Nutch-1.0-dev (Research spider using Nutch; http://www.lemurproject.org; mhoy@cs.cmu.edu)
Disallow: /
User-agent: LTI/LemurProject Nutch Spider/Nutch-1.0-dev (lti crawler for CMU; http://www.lti.cs.cmu.edu; changkuk at cmu dot edu)
Disallow: /
User-agent: LWP::Simple/5.22
Disallow: /
User-agent: LWP::Simple/5.36
Disallow: /
User-agent: LWP::Simple/5.48
Disallow: /
User-agent: LWP::Simple/5.50
Disallow: /
User-agent: LWP::Simple/5.51
Disallow: /
User-agent: LWP::Simple/5.53
Disallow: /
User-agent: LWP::Simple/5.63
Disallow: /
User-agent: LWP::Simple/5.803
Disallow: /
User-agent: Lachesis
Disallow: /
User-agent: LapozzBot/1.4 ( http://robot.lapozz.com)
Disallow: /
User-agent: LapozzBot/1.5 (+http://robot.lapozz.hu)
Disallow: /
User-agent: LeapTag/0.8.1.beta081.r3750 (compatible; Mozilla 4.0; MSIE 5.5; robot@yoriwa.com)
Disallow: /
User-agent: LeechGet 200x (www.leechget.de)
Disallow: /
User-agent: LetsCrawl.com/1.0 +http://letscrawl.com/
Disallow: /
User-agent: LexiBot/1.00
Disallow: /
User-agent: Libby_1.1/libwww-perl/5.47
Disallow: /
User-agent: LibertyW (+http://www.lw01.com)
Disallow: /
User-agent: Liferea/0.x.x (Linux; en_US.UTF-8; http://liferea.sf.net/)
Disallow: /
User-agent: Liferea/1.x.x (Linux; es_ES.UTF-8; http://liferea.sf.net/)
Disallow: /
User-agent: LightningDownload/1.0beta2
Disallow: /
User-agent: LightningDownload/1.x.x
Disallow: /
User-agent: LightningDownload/1.x.x [Accelerated x]
Disallow: /
User-agent: LijitSpider/Nutch-0.9 (Reports crawler; http://www.lijit.com/; info(a)lijit(d)com)
Disallow: /
User-agent: Lincoln State Web Browser
Disallow: /
User-agent: Link Valet Online 1.x
Disallow: /
User-agent: LinkAlarm/2.x
Disallow: /
User-agent: LinkCheck (linkcheck@inter7.com http://www.inter7.com/linkcheck)
Disallow: /
User-agent: LinkLint-checkonly/2.x.x
Disallow: /
User-agent: LinkLint-spider/2.x.x
Disallow: /
User-agent: LinkPimpin v1.0
Disallow: /
User-agent: LinkProver 2.1
Disallow: /
User-agent: LinkScan/11.0beta2 UnixShareware robot from Elsop.com (used by Indiafocus/Indiainfo)
Disallow: /
User-agent: LinkScan/9.0g Unix
Disallow: /
User-agent: LinkScan/x.x Unix
Disallow: /
User-agent: LinkSonar/1.35
Disallow: /
User-agent: LinkSweeper/1.x
Disallow: /
User-agent: LinkWalker
Disallow: /
User-agent: Linkbot
Disallow: /
User-agent: Linkbot x.0
Disallow: /
User-agent: Links (0.9x; Linux 2.4.7-10 i686)
Disallow: /
User-agent: Links (0.9xpre12; Linux 2.2.14-5.0 i686; 80x24)
Disallow: /
User-agent: Links (2.xpre7; Linux 2.4.18 i586; x)
Disallow: /
User-agent: Links - http://gossamer-threads.com/scripts/links/
Disallow: /
User-agent: Links 2.0 (http://gossamer-threads.com/scripts/links/)
Disallow: /
User-agent: Links SQL (http://gossamer-threads.com/scripts/links-sql/)
Disallow: /
User-agent: Links4US-Crawler, (+http://links4us.com/)
Disallow: /
User-agent: LinksManager.com (http://linksmanager.com/linkchecker.html)
Disallow: /
User-agent: ListBidBot (freelance job spider http://listbid.com)<a href=http://listbid.com>Freelance</a>
Disallow: /
User-agent: LiveTrans/Nutch-0.9 (maintainer: cobain at iis dot sinica dot edu dot tw; http://wkd.iis.sinica.edu.tw/LiveTrans/)
Disallow: /
User-agent: Llaut/1.0 (http://mnm.uib.es/~gallir/llaut/bot.html)
Disallow: /
User-agent: LocalBot/1.0 ( http://www.localbot.co.uk/)
Disallow: /
User-agent: LocalcomBot/1.2.x ( http://www.local.com/bot.htm)
Disallow: /
User-agent: Lockstep Spider/1.0
Disallow: /
User-agent: Look.com
Disallow: /
User-agent: Lotus-Notes/4.5 ( Windows-NT )
Disallow: /
User-agent: LotusDiscovery/x.0 (compatible; Mozilla 4.0; MSIE 4.01; Windows NT)
Disallow: /
User-agent: Lovel as 1.0 ( +http://www.everatom.com)
Disallow: /
User-agent: Lunascape
Disallow: /
User-agent: Lycos_Spider_(T-Rex)
Disallow: /
User-agent: Lycos_Spider_(modspider)
Disallow: /
User-agent: Lynx/2-4-2 (Bobcat/0.5 [DOS] Jp Beta04)
Disallow: /
User-agent: Lynx/2.6 libwww-FM/2.14
Disallow: /
User-agent: Lynx/2.8 (;http://seebot.org)
Disallow: /
User-agent: Lynx/2.8.3dev.9 libwww-FM/2.14 SSL-MM/1.4.1 OpenSSL/0.9.6
Disallow: /
User-agent: Lynx/2.8.4rel.1 libwww-FM/2.14 SSL-MM/1.4.1 OpenSSL/0.9.6c (human-guided@lerly.net)
Disallow: /
User-agent: MARTINI
Disallow: /
User-agent: MDbot/1.0 (+http://www.megadownload.net/bot.html)
Disallow: /
User-agent: MFC Foundation Class Library 4.0
Disallow: /
User-agent: MFC_Tear_Sample
Disallow: /
User-agent: MFHttpScan
Disallow: /
User-agent: MIIxpc/4.2
Disallow: /
User-agent: MJ12bot/vx.x.x (http://majestic12.co.uk/bot.php?+)
Disallow: /
User-agent: MJ12bot/vx.x.x (http://www.majestic12.co.uk/projects/dsearch/mj12bot.php)
Disallow: /
User-agent: MJBot (SEO assessment)
Disallow: /
User-agent: MLBot (www.metadatalabs.com)
Disallow: /
User-agent: MQBOT/Nutch-0.9-dev (MQBOT Nutch Crawler; http://falcon.cs.uiuc.edu; mqbot@cs.uiuc.edu)
Disallow: /
User-agent: MQbot metaquerier.cs.uiuc.edu/crawler
Disallow: /
User-agent: MSFrontPage/4.0
Disallow: /
User-agent: MSIE 4.0 (Win95)
Disallow: /
User-agent: MSIE-5.13 (larbin@unspecified.mail)
Disallow: /
User-agent: MSNBOT/0.xx (http://search.msn.com/msnbot.htm)
Disallow: /
User-agent: MSNBOT_Mobile MSMOBOT Mozilla/2.0 (compatible; MSIE 4.02; Windows CE; Default)
Disallow: /
User-agent: MSNPTC/1.0
Disallow: /
User-agent: MSProxy/2.0
Disallow: /
User-agent: MSRBOT
Disallow: /
User-agent: MSRBOT (http://research.microsoft.com/research/sv/msrbot)
Disallow: /
User-agent: MVAClient
Disallow: /
User-agent: MaSagool/1.0 (MaSagool; http://sagool.jp/; info@sagool.jp)
Disallow: /
User-agent: Mac Finder 1.0.xx
Disallow: /
User-agent: Mackster( http://www.ukwizz.com )
Disallow: /
User-agent: Mag-Net
Disallow: /
User-agent: MagicWML/1.0 (forcewml)
Disallow: /
User-agent: MagpieRSS/0.7x (+http://magpierss.sf.net)
Disallow: /
User-agent: Mahiti.Com/Mahiti Crawler-1.0 (Mahiti.Com; http://mahiti.com ; mahiti.com)
Disallow: /
User-agent: Mail.Ru/1.0
Disallow: /
User-agent: MantraAgent
Disallow: /
User-agent: MapoftheInternet.com ( http://MapoftheInternet.com)
Disallow: /
User-agent: Mariner/5.1b [de] (Win95; I ;Kolibri gncwebbot)
Disallow: /
User-agent: Marketwave Hit List
Disallow: /
User-agent: Martini
Disallow: /
User-agent: Marvin v0.3
Disallow: /
User-agent: Mass Downloader 2.x
Disallow: /
User-agent: MasterSeek
Disallow: /
User-agent: Mata Hari/2.00
Disallow: /
User-agent: Matrix S.p.A. - FAST Enterprise Crawler 6 (Unknown admin e-mail address)
Disallow: /
User-agent: McBot/5.001 (windows; U; NT4.0; en-us)
Disallow: /
User-agent: Media Player Classic
Disallow: /
User-agent: MediaCrawler-1.0 (Experimental)
Disallow: /
User-agent: MediaSearch/0.1
Disallow: /
User-agent: Mediapartners-Google/2.1 ( http://www.googlebot.com/bot.html)
Disallow: /
User-agent: MegaSheep v1.0 (www.searchuk.com internet sheep)
Disallow: /
User-agent: Megite2.0 (http://www.megite.com)
Disallow: /
User-agent: Mercator-1.x
Disallow: /
User-agent: Mercator-2.0
Disallow: /
User-agent: Mercator-Scrub-1.1
Disallow: /
User-agent: MetaGer-LinkChecker
Disallow: /
User-agent: MetaGer_PreChecker0.1
Disallow: /
User-agent: MetaProducts Download Express/1.x
Disallow: /
User-agent: Metaeuro Web Crawler/0.2 (MetaEuro Web Search Clustering Engine; http://www.metaeuro.com; crawler at metaeuro dot com)
Disallow: /
User-agent: MetagerBot/0.8-dev (MetagerBot; http://metager.de;  )
Disallow: /
User-agent: Metaspinner/0.01 (Metaspinner; http://www.meta-spinner.de/; support@meta-spinner.de/)
Disallow: /
User-agent: MicroBaz
Disallow: /
User-agent: Microsoft Data Access Internet Publishing Provider Cache Manager
Disallow: /
User-agent: Microsoft Data Access Internet Publishing Provider DAV
Disallow: /
User-agent: Microsoft Data Access Internet Publishing Provider Protocol Discovery
Disallow: /
User-agent: Microsoft Log Parser 2.2
Disallow: /
User-agent: Microsoft Small Business Indexer
Disallow: /
User-agent: Microsoft URL Control - 6.00.8xxx
Disallow: /
User-agent: MicrosoftPrototypeCrawler (How's my crawling? mailto:newbiecrawler@hotmail.com)
Disallow: /
User-agent: Microsoft_Internet_Explorer_5.00.438 (fjones@isd.net)
Disallow: /
User-agent: Mindjet MindManager
Disallow: /
User-agent: MiracleAlphaTest
Disallow: /
User-agent: Missauga Locate 1.0.0
Disallow: /
User-agent: Missigua Locator 1.9
Disallow: /
User-agent: Missouri College Browse
Disallow: /
User-agent: Mister PiX version.dll
Disallow: /
User-agent: Mister Pix II 2.02a
Disallow: /
User-agent: Misterbot-Nutch/0.7.1 (Misterbot-Nutch; http://www.misterbot.fr; admin@misterbot.fr)
Disallow: /
User-agent: Miva (AlgoFeedback@miva.com)
Disallow: /
User-agent: Mizzu Labs 2.2
Disallow: /
User-agent: MnogoSearch/3.2.xx
Disallow: /
User-agent: Mo College 1.9
Disallow: /
User-agent: MojeekBot/0.x (archi; http://www.mojeek.com/bot.html)
Disallow: /
User-agent: MoonBrowser (version 0.41 Beta4)
Disallow: /
User-agent: Moreoverbot/x.00 (+http://www.moreover.com)
Disallow: /
User-agent: Morris - Mixcat Crawler ( http://mixcat.com)
Disallow: /
User-agent: Motoricerca-Robots.txt-Checker/1.0 (http://tool.motoricerca.info/robots-checker.phtml)
Disallow: /
User-agent: Motorola-V3m Obigo
Disallow: /
User-agent: Mouse-House/7.4 (spider_monkey spider info at www.mobrien.com/sm.shtml)
Disallow: /
User-agent: MovableType/x.x
Disallow: /
User-agent: Mozi!
Disallow: /
User-agent: Mozilla
Disallow: /
User-agent: Mozilla (Mozilla@somewhere.com)
Disallow: /
User-agent: Mozilla 4.0(compatible; BotSeer/1.0; +http://botseer.ist.psu.edu)
Disallow: /
User-agent: Mozilla/1.1 (compatible; MSPIE 2.0; Windows CE)
Disallow: /
User-agent: Mozilla/1.10 [en] (Compatible; RISC OS 3.70; Oregano 1.10)
Disallow: /
User-agent: Mozilla/1.22 (compatible; MSIE 2.0d; Windows NT)
Disallow: /
User-agent: Mozilla/1.22 (compatible; MSIE 5.01; PalmOS 3.0) EudoraWeb 2
Disallow: /
User-agent: Mozilla/2.0
Disallow: /
User-agent: Mozilla/2.0 (Compatible; AOL-IWENG 3.0; Win16)
Disallow: /
User-agent: Mozilla/2.0 (compatible; AOL 3.0; Mac_PowerPC)
Disallow: /
User-agent: Mozilla/2.0 (compatible; Ask Jeeves)
Disallow: /
User-agent: Mozilla/2.0 (compatible; Ask Jeeves/Teoma)
Disallow: /
User-agent: Mozilla/2.0 (compatible; Ask Jeeves/Teoma; http://about.ask.com/en/docs/about/webmasters.shtml)
Disallow: /
User-agent: Mozilla/2.0 (compatible; Ask Jeeves/Teoma; http://sp.ask.com/docs/about/tech_crawling.html)
Disallow: /
User-agent: Mozilla/2.0 (compatible; EZResult -- Internet Search Engine)
Disallow: /
User-agent: Mozilla/2.0 (compatible; MS FrontPage x.0)
Disallow: /
User-agent: Mozilla/2.0 (compatible; MSIE 2.1; Mac_PowerPC)
Disallow: /
User-agent: Mozilla/2.0 (compatible; MSIE 3.02; Update a; AK; Windows NT)
Disallow: /
User-agent: Mozilla/2.0 (compatible; MSIE 3.02; Update a; AOL 3.0; Windows 95)
Disallow: /
User-agent: Mozilla/2.0 (compatible; MSIE 3.0; AK; Windows 95)
Disallow: /
User-agent: Mozilla/2.0 (compatible; MSIE 3.0; Windows 3.1)
Disallow: /
User-agent: Mozilla/2.0 (compatible; MSIE 3.0B; Win32)
Disallow: /
User-agent: Mozilla/2.0 (compatible; NEWT ActiveX; Win32)
Disallow: /
User-agent: Mozilla/2.0 (compatible; T-H-U-N-D-E-R-S-T-O-N-E)
Disallow: /
User-agent: Mozilla/2.0 compatible; Check&Get 1.1x (Windows 98)
Disallow: /
User-agent: Mozilla/2.01 (Win16; I)
Disallow: /
User-agent: Mozilla/2.02Gold (Win95; I)
Disallow: /
User-agent: Mozilla/3.0 (Compatible;Viking/1.8)
Disallow: /
User-agent: Mozilla/3.0 (DreamPassport/3.0)
Disallow: /
User-agent: Mozilla/3.0 (INGRID/3.0 MT; webcrawler@NOSPAMexperimental.net; http://aanmelden.ilse.nl/?aanmeld_mode=webhints)
Disallow: /
User-agent: Mozilla/3.0 (Liberate DTV 1.1)
Disallow: /
User-agent: Mozilla/3.0 (Planetweb/2.100 JS SSL US; Dreamcast US)
Disallow: /
User-agent: Mozilla/3.0 (Slurp.so/Goo; slurp@inktomi.com; http://www.inktomi.com/slurp.html)
Disallow: /
User-agent: Mozilla/3.0 (Slurp/cat; slurp@inktomi.com; http://www.inktomi.com/slurp.html)
Disallow: /
User-agent: Mozilla/3.0 (Slurp/si; slurp@inktomi.com; http://www.inktomi.com/slurp.html)
Disallow: /
User-agent: Mozilla/3.0 (Vagabondo/1.1 MT; webcrawler@NOSPAMwise-guys.nl; http://webagent.wise-guys.nl/)
Disallow: /
User-agent: Mozilla/3.0 (Vagabondo/1.x MT; webagent@wise-guys.nl; http://webagent.wise-guys.nl/)
Disallow: /
User-agent: Mozilla/3.0 (Vagabondo/2.0 MT; webcrawler@NOSPAMexperimental.net; http://aanmelden.ilse.nl/?aanmeld_mode=webhints)
Disallow: /
User-agent: Mozilla/3.0 (Vagabondo/2.0 MT; webcrawler@NOSPAMwise-guys.nl; http://webagent.wise-guys.nl/)
Disallow: /
User-agent: Mozilla/3.0 (Win16; I)
Disallow: /
User-agent: Mozilla/3.0 (Win95; I)
Disallow: /
User-agent: Mozilla/3.0 (WinNT; I)
Disallow: /
User-agent: Mozilla/3.0 (WorldGate Gazelle 3.5.1 build 11; FreeBSD2.2.8-STABLE)
Disallow: /
User-agent: Mozilla/3.0 (X11; I; OSF1 V4.0 alpha)
Disallow: /
User-agent: Mozilla/3.0 (compatible)
Disallow: /
User-agent: Mozilla/3.0 (compatible; AvantGo 3.2)
Disallow: /
User-agent: Mozilla/3.0 (compatible; Fluffy the spider; http://www.searchhippo.com/; info@searchhippo.com)
Disallow: /
User-agent: Mozilla/3.0 (compatible; HP Web PrintSmart 04b0 1.0.1.34)
Disallow: /
User-agent: Mozilla/3.0 (compatible; Indy Library)
Disallow: /
User-agent: Mozilla/3.0 (compatible; Linkman)
Disallow: /
User-agent: Mozilla/3.0 (compatible; MuscatFerret/1.5.4; claude@euroferret.com)
Disallow: /
User-agent: Mozilla/3.0 (compatible; MuscatFerret/1.5; olly@muscat.co.uk)
Disallow: /
User-agent: Mozilla/3.0 (compatible; MuscatFerret/1.6.x; claude@euroferret.com)
Disallow: /
User-agent: Mozilla/3.0 (compatible; NetPositive/2.2)
Disallow: /
User-agent: Mozilla/3.0 (compatible; Opera/3.0; Windows 3.1) v3.1
Disallow: /
User-agent: Mozilla/3.0 (compatible; Opera/3.0; Windows 95/NT4) 3.2
Disallow: /
User-agent: Mozilla/3.0 (compatible; PerMan Surfer 3.0; Win95)
Disallow: /
User-agent: Mozilla/3.0 (compatible; REL Software Web Link Validator 2.x)
Disallow: /
User-agent: Mozilla/3.0 (compatible; ScollSpider; http://www.webwobot.com)
Disallow: /
User-agent: Mozilla/3.0 (compatible; Web Link Validator 2.x)Web Link Validator http://www.relsoftware.com/ link validation software
Disallow: /
User-agent: Mozilla/3.0 (compatible; WebCapture x.x; Auto; Windows)
Disallow: /
User-agent: Mozilla/3.0 (compatible; Webinator-DEV01.home.iprospect.com/2.56)
Disallow: /
User-agent: Mozilla/3.0 (compatible; Webinator-indexer.cyberalert.com/2.56)
Disallow: /
User-agent: Mozilla/3.0 (compatible; netart generator/1.0; libwww-perl/5.64)
Disallow: /
User-agent: Mozilla/3.0 (compatible; scan4mail (advanced version) http://www.peterspages.net/?scan4mail)
Disallow: /
User-agent: Mozilla/3.0 NAVIO_AOLTV (11; 13; Philips; PH200; 1; R2.0C36_AOL.0110OPTIK; R2.0.0139d_OPTIK)
Disallow: /
User-agent: Mozilla/3.0 WebTV/1.2 (compatible; MSIE 2.0)
Disallow: /
User-agent: Mozilla/3.01 (Compatible; Links2Go Similarity Engine)
Disallow: /
User-agent: Mozilla/3.01 (compatible; AmigaVoyager/2.95; AmigaOS/MC680x0)
Disallow: /
User-agent: Mozilla/3.01 (compatible; Netbox/3.5 R92; Linux 2.2)
Disallow: /
User-agent: Mozilla/3.01-C-MACOS8 (Macintosh; I; PPC)
Disallow: /
User-agent: Mozilla/3.01Gold (X11; I; Linux 2.0.32 i486)
Disallow: /
User-agent: Mozilla/3.01Gold (X11; I; SunOS 5.5.1 sun4m)
Disallow: /
User-agent: Mozilla/3.01SGoldC-SGI (X11; I; IRIX 6.3 IP32)
Disallow: /
User-agent: Mozilla/3.04 (compatible; ANTFresco/2.13; RISC OS 4.02)
Disallow: /
User-agent: Mozilla/3.04 (compatible; NCBrowser/2.35; ANTFresco/2.17; RISC OS-NC 5.13 Laz1UK1309)
Disallow: /
User-agent: Mozilla/3.04 (compatible;QNX Voyager 2.03B ;Photon)
Disallow: /
User-agent: Mozilla/3.x (I-Opener 1.1; Netpliance)
Disallow: /
User-agent: Mozilla/4.0
Disallow: /
User-agent: Mozilla/4.0 (Compatible); URLBase 6
Disallow: /
User-agent: Mozilla/4.0 (JemmaTheTourist;http://www.activtourist.com)
Disallow: /
User-agent: Mozilla/4.0 (MobilePhone PM-8200/US/1.0) NetFront/3.x MMP/2.0
Disallow: /
User-agent: Mozilla/4.0 (MobilePhone SCP-5500/US/1.0) NetFront/3.0 MMP/2.0 (compatible; Googlebot/2.1; http://www.google.com/bot.html)
Disallow: /
User-agent: Mozilla/4.0 (MobilePhone SCP-5500/US/1.0) NetFront/3.0 MMP/2.0 FAKE (compatible; Googlebot/2.1; http://www.google.com/bot.html)
Disallow: /
User-agent: Mozilla/4.0 (Mozilla; http://www.mozilla.org/docs/en/bot.html; master@mozilla.com)
Disallow: /
User-agent: Mozilla/4.0 (Sleek Spider/1.2)
Disallow: /
User-agent: Mozilla/4.0 (agadine3.0) www.agada.de
Disallow: /
User-agent: Mozilla/4.0 (compatible: AstraSpider V.2.1 : astrafind.com)
Disallow: /
User-agent: Mozilla/4.0 (compatible;  Vagabondo/2.2; webcrawler at wise-guys dot nl; http://webagent.wise-guys.nl/)
Disallow: /
User-agent: Mozilla/4.0 (compatible;  Vagabondo/4.0Beta; webcrawler at wise-guys dot nl; http://webagent.wise-guys.nl/)
Disallow: /
User-agent: Mozilla/4.0 (compatible; <a href=http://www.reget.com>ReGet Deluxe 5.1</a>; Windows NT 5.1)
Disallow: /
User-agent: Mozilla/4.0 (compatible; Advanced Email Extractor v2.xx)
Disallow: /
User-agent: Mozilla/4.0 (compatible; Arachmo)
Disallow: /
User-agent: Mozilla/4.0 (compatible; BOTW Spider; +http://botw.org)
Disallow: /
User-agent: Mozilla/4.0 (compatible; B_L_I_T_Z_B_O_T)
Disallow: /
User-agent: Mozilla/4.0 (compatible; BorderManager 3.0)
Disallow: /
User-agent: Mozilla/4.0 (compatible; Cerberian Drtrs Version-3.2-Build-0)
Disallow: /
User-agent: Mozilla/4.0 (compatible; Check&Get 3.0; Windows NT)
Disallow: /
User-agent: Mozilla/4.0 (compatible; ChristCrawler.com ChristCrawler@ChristCENTRAL.com)
Disallow: /
User-agent: Mozilla/4.0 (compatible; DAUMOA-video; +http://ws.daum.net/aboutkr.html)
Disallow: /
User-agent: Mozilla/4.0 (compatible; DepSpid/5.0x; +http://about.depspid.net)
Disallow: /
User-agent: Mozilla/4.0 (compatible; DnloadMage 1.0)
Disallow: /
User-agent: Mozilla/4.0 (compatible; FDSE robot)
Disallow: /
User-agent: Mozilla/4.0 (compatible; FastCrawler3 support-fastcrawler3@fast.no)
Disallow: /
User-agent: Mozilla/4.0 (compatible; GPU p2p crawler http://gpu.sourceforge.net/search_engine.php)
Disallow: /
User-agent: Mozilla/4.0 (compatible; ICS 1.2.xxx)
Disallow: /
User-agent: Mozilla/4.0 (compatible; IE-Favorites-Check-0.5)
Disallow: /
User-agent: Mozilla/4.0 (compatible; Iplexx Spider/1.0 http://www.iplexx.at)
Disallow: /
User-agent: Mozilla/4.0 (compatible; KeepNI web site monitor)
Disallow: /
User-agent: Mozilla/4.0 (compatible; Link Utility; http://net-promoter.com)
Disallow: /
User-agent: Mozilla/4.0 (compatible; Lotus-Notes/5.0; Windows-NT)
Disallow: /
User-agent: Mozilla/4.0 (compatible; MSIE 4.01; AOL 4.0; Windows 98)
Disallow: /
User-agent: Mozilla/4.0 (compatible; MSIE 4.01; MSIECrawler; Windows 95)
Disallow: /
User-agent: Mozilla/4.0 (compatible; MSIE 4.01; Mac_PowerPC)
Disallow: /
User-agent: Mozilla/4.0 (compatible; MSIE 4.01; Vonna.com b o t)
Disallow: /
User-agent: Mozilla/4.0 (compatible; MSIE 4.01; Windows 95)
Disallow: /
User-agent: Mozilla/4.0 (compatible; MSIE 4.01; Windows CE; MSN Companion 2.0; 800x600; Compaq)
Disallow: /
User-agent: Mozilla/4.0 (compatible; MSIE 4.01; Windows CE; PPS; 240x320)
Disallow: /
User-agent: Mozilla/4.0 (compatible; MSIE 4.01; Windows NT Windows CE)
Disallow: /
User-agent: Mozilla/4.0 (compatible; MSIE 4.01; Windows NT)
Disallow: /
User-agent: Mozilla/4.0 (compatible; MSIE 4.01; Windows NT; MS Search 4.0 Robot) Microsoft
Disallow: /
User-agent: Mozilla/4.0 (compatible; MSIE 4.0; Windows NT; Site Server 3.0 Robot) ACR
Disallow: /
User-agent: Mozilla/4.0 (compatible; MSIE 4.0; Windows NT; Site Server 3.0 Robot) Indonesia Interactive
Disallow: /
User-agent: Mozilla/4.0 (compatible; MSIE 4.0; Windows NT; Site Server 3.0 Robot) WebQuest Designs
Disallow: /
User-agent: Mozilla/4.0 (compatible; MSIE 5.01; Windows 95) via <B>Avirt Gateway Server</B> v4.0
Disallow: /
User-agent: Mozilla/4.0 (compatible; MSIE 5.01; Windows NT 5.0) (samualt9@bigfoot.com)
Disallow: /
User-agent: Mozilla/4.0 (compatible; MSIE 5.01; Windows NT 5.0; NetCaptor 6.5.0RC1)
Disallow: /
User-agent: Mozilla/4.0 (compatible; MSIE 5.0; AOL 5.0; Windows 95; DigExt; Gateway2000; sureseeker.com)
Disallow: /
User-agent: Mozilla/4.0 (compatible; MSIE 5.0; Mac_PowerPC; AtHome021)
Disallow: /
User-agent: Mozilla/4.0 (compatible; MSIE 5.0; NetNose-Crawler 2.0; A New Search Experience: http://www.netnose.com)
Disallow: /
User-agent: Mozilla/4.0 (compatible; MSIE 5.0; Win32) via proxy gateway CERN-HTTPD/3.0 libwww/2.17
Disallow: /
User-agent: Mozilla/4.0 (compatible; MSIE 5.0; Windows 95) TrueRobot; 1.5
Disallow: /
User-agent: Mozilla/4.0 (compatible; MSIE 5.0; Windows 95) VoilaBot BETA 1.2 (http://www.voila.com/)
Disallow: /
User-agent: Mozilla/4.0 (compatible; MSIE 5.0; Windows 95) VoilaBot; 1.6
Disallow: /
User-agent: Mozilla/4.0 (compatible; MSIE 5.0; Windows ME) Opera 5.11 [en]
Disallow: /
User-agent: Mozilla/4.0 (compatible; MSIE 5.0; Windows ME; Link Checker 2.x.xx http://www.kyosoft.com)
Disallow: /
User-agent: Mozilla/4.0 (compatible; MSIE 5.0; Windows NT; DigExt; DTS Agent
Disallow: /
User-agent: Mozilla/4.0 (compatible; MSIE 5.0; Windows NT; Girafabot; girafabot at girafa dot com; http://www.girafa.com)
Disallow: /
User-agent: Mozilla/4.0 (compatible; MSIE 5.0; YANDEX)
Disallow: /
User-agent: Mozilla/4.0 (compatible; MSIE 5.0; www.galaxy.com; www.psychedelix.com)
Disallow: /
User-agent: Mozilla/4.0 (compatible; MSIE 5.0; www.galaxy.com; www.psychedelix.com/; http://www.galaxy.com/info/crawler.html)
Disallow: /
User-agent: Mozilla/4.0 (compatible; MSIE 5.5; AOL 4.0; Windows 98; GoBeez (www.gobeez.com))
Disallow: /
User-agent: Mozilla/4.0 (compatible; MSIE 5.5; Windows 95; Transmission Segment; Hotbar 2.0)
Disallow: /
User-agent: Mozilla/4.0 (compatible; MSIE 5.5; Windows 98; Crazy Browser 1.x.x)
Disallow: /
User-agent: Mozilla/4.0 (compatible; MSIE 5.5; Windows 98; KITV4.7 Wanadoo)
Disallow: /
User-agent: Mozilla/4.0 (compatible; MSIE 5.5; Windows 98; SAFEXPLORER TL)
Disallow: /
User-agent: Mozilla/4.0 (compatible; MSIE 5.5; Windows 98; SYMPA; Katiesoft 7; SimulBrowse 3.0)
Disallow: /
User-agent: Mozilla/4.0 (compatible; MSIE 5.5; Windows 98; Win 9x 4.90; BTinternet V8.1)
Disallow: /
User-agent: Mozilla/4.0 (compatible; MSIE 5.5; Windows 98; Win 9x 4.90; MSIECrawler)
Disallow: /
User-agent: Mozilla/4.0 (compatible; MSIE 5.5; Windows NT 4.0; QXW03018)
Disallow: /
User-agent: Mozilla/4.0 (compatible; MSIE 5.5; Windows NT 4.0; obot)
Disallow: /
User-agent: Mozilla/4.0 (compatible; MSIE 5.5; Windows NT 5.0) Active Cache Request
Disallow: /
User-agent: Mozilla/4.0 (compatible; MSIE 5.5; Windows NT 5.0) Fetch API Request
Disallow: /
User-agent: Mozilla/4.0 (compatible; MSIE 5.5; Windows NT 5.0; .NET CLR 1.0.3705)
Disallow: /
User-agent: Mozilla/4.0 (compatible; MSIE 5.5; Windows NT 5.0; AIRF)
Disallow: /
User-agent: Mozilla/4.0 (compatible; MSIE 5.5; Windows NT 5.0; AspTear 1.5)
Disallow: /
User-agent: Mozilla/4.0 (compatible; MSIE 5.5; Windows NT 5.0; N_o_k_i_a)
Disallow: /
User-agent: Mozilla/4.0 (compatible; MSIE 5.5; Windows NT 5.0; T312461) RPT-HTTPClient/0.3-3E
Disallow: /
User-agent: Mozilla/4.0 (compatible; MSIE 6.0 compatible; Asterias Crawler v4; +http://www.singingfish.com/help/spider.html; webmaster@singingfish.com); SpiderThread  Revision: 3.10
Disallow: /
User-agent: Mozilla/4.0 (compatible; MSIE 6.0; AOL 9.0; Windows 98; .NET CLR 1.1.4322; MEGAUPLOAD 2.0)
Disallow: /
User-agent: Mozilla/4.0 (compatible; MSIE 6.0; AOL 9.0; Windows NT 5.1; SV1; HbTools 4.7.2)
Disallow: /
User-agent: Mozilla/4.0 (compatible; MSIE 6.0; MSIE 5.5; Windows NT 5.1) Skampy/0.9.x [en]
Disallow: /
User-agent: Mozilla/4.0 (compatible; MSIE 6.0; TargetSeek/1.0; +http://www.targetgroups.net/TargetSeek.html)
Disallow: /
User-agent: Mozilla/4.0 (compatible; MSIE 6.0; Win32) WebWasher 3.0
Disallow: /
User-agent: Mozilla/4.0 (compatible; MSIE 6.0; Windows 98) REL Software Web Link Validator 2.x)
Disallow: /
User-agent: Mozilla/4.0 (compatible; MSIE 6.0; Windows 98) Web Link Validator 2.x)
Disallow: /
User-agent: Mozilla/4.0 (compatible; MSIE 6.0; Windows 98; Net M@nager V3.02 - www.vinn.com.au)
Disallow: /
User-agent: Mozilla/4.0 (compatible; MSIE 6.0; Windows 98; Win 9x 4.90; http://www.Abolimba.de)
Disallow: /
User-agent: Mozilla/4.0 (compatible; MSIE 6.0; Windows 98; support@illumit.com; http://www.illumit.com/Products/weblight/)
Disallow: /
User-agent: Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0; .NET CLR 1.1.4322; Lunascape 2.1.3)
Disallow: /
User-agent: Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0; Google Wireless Transcoder;)
Disallow: /
User-agent: Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0; ODP entries t_st; http://tuezilla.de/t_st-odp-entries-agent.html)
Disallow: /
User-agent: Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0; ODP links test; http://tuezilla.de/test-odp-links-agent.html)
Disallow: /
User-agent: Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0; ZoomSpider.net bot; .NET CLR 1.1.4322)
Disallow: /
User-agent: Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1) (dns_admin@c-a-s-h.com)
Disallow: /
User-agent: Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; Covac UPPS Cathan 1.2.5;)
Disallow: /
User-agent: Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; Crayon Crawler; snprtz|T04056566514940; (R1 1.5))
Disallow: /
User-agent: Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; Deepnet Explorer)
Disallow: /
User-agent: Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; Hotbar 3.0)
Disallow: /
User-agent: Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; KKman3.0)
Disallow: /
User-agent: Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; MathPlayer2.0)
Disallow: /
User-agent: Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; Maxthon)
Disallow: /
User-agent: Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; PeoplePal 3.0; MSIECrawler)
Disallow: /
User-agent: Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; Q312461; IOpener Release 1.1.04)
Disallow: /
User-agent: Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; QihooBot 1.0  qihoobot@qihoo.net)
Disallow: /
User-agent: Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SIMBAR Enabled; InfoPath.1)
Disallow: /
User-agent: Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1;  Embedded Web Browser from: http://bsalsa.com/; MSIECrawler)
Disallow: /
User-agent: Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1;  http://www.changedetection.com/bot.html )
Disallow: /
User-agent: Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 1.1.4322)
Disallow: /
User-agent: Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; DX-Browser 5.0.0.0)
Disallow: /
User-agent: Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; FunWebProducts; ezPeer+ v1.0 Beta (0.4.1.98); ezPeer+ v1.0 (0.5.0.00); .NET CLR 1.1.4322; MSIECrawler)
Disallow: /
User-agent: Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; IBP; .NET CLR 1.1.4322)
Disallow: /
User-agent: Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; MRA 4.3 (build 01218))
Disallow: /
User-agent: Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; StumbleUpon.com 1.760; .NET CLR 1.1.4322)
Disallow: /
User-agent: Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; heritrix/1.3.0 http://www.cs.washington.edu/research/networking/websys/)
Disallow: /
User-agent: Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; iOpus-I-M)
Disallow: /
User-agent: Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; iRider 2.21.1108; FDM)
Disallow: /
User-agent: Mozilla/4.0 (compatible; MSIE 6.0; Windows NT; MS Search 4.0 Robot)
Disallow: /
User-agent: Mozilla/4.0 (compatible; MSIE 7.0; Win32) Link Commander 4.0
Disallow: /
User-agent: Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; bgft)
Disallow: /
User-agent: Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.0)
Disallow: /
User-agent: Mozilla/4.0 (compatible; MSIE enviable; DAUMOA 2.0; DAUM Web Robot; Daum Communications Corp., Korea; +http://ws.daum.net/aboutkr.html)
Disallow: /
User-agent: Mozilla/4.0 (compatible; MSIE is not me; DAUMOA/1.0.1; DAUM Web Robot; Daum Communications Corp., Korea)
Disallow: /
User-agent: Mozilla/4.0 (compatible; NaverBot/1.0; http://help.naver.com/delete_main.asp)
Disallow: /
User-agent: Mozilla/4.0 (compatible; NetPromoter Spider;http://www.net-promoter.com/)
Disallow: /
User-agent: Mozilla/4.0 (compatible; Netcraft Web Server Survey)
Disallow: /
User-agent: Mozilla/4.0 (compatible; Opera/3.0; Windows 4.10) 3.51 [en]
Disallow: /
User-agent: Mozilla/4.0 (compatible; Powermarks/3.5; Windows 95/98/2000/NT)
Disallow: /
User-agent: Mozilla/4.0 (compatible; RSS Popper)
Disallow: /
User-agent: Mozilla/4.0 (compatible; SPENG)
Disallow: /
User-agent: Mozilla/4.0 (compatible; SiteKiosk 4.0; MSIE 5.0; Windows 98; SiteCoach 1.0)
Disallow: /
User-agent: Mozilla/4.0 (compatible; SpeedySpider; www.entireweb.com)
Disallow: /
User-agent: Mozilla/4.0 (compatible; SuperCleaner 2.xx; Windows 98)
Disallow: /
User-agent: Mozilla/4.0 (compatible; Synapse)
Disallow: /
User-agent: Mozilla/4.0 (compatible; WSN Links)
Disallow: /
User-agent: Mozilla/4.0 (compatible; WebCapture 3.0; Windows)
Disallow: /
User-agent: Mozilla/4.0 (compatible; Win32; WinHttp.WinHttpRequest.5)
Disallow: /
User-agent: Mozilla/4.0 (compatible; Y!J; for robot study; keyoshid)
Disallow: /
User-agent: Mozilla/4.0 (compatible; Yahoo Japan; for robot study; kasugiya)
Disallow: /
User-agent: Mozilla/4.0 (compatible; crawlx, crawler@trd.overture.com)
Disallow: /
User-agent: Mozilla/4.0 (compatible; grub-client-0.2.x; Crawl your stuff with http://grub.org)
Disallow: /
User-agent: Mozilla/4.0 (compatible; grub-client-0.3.x; Crawl your own stuff with http://grub.org)
Disallow: /
User-agent: Mozilla/4.0 (compatible; grub-client-2.x)
Disallow: /
User-agent: Mozilla/4.0 (compatible; ibisBrowser)
Disallow: /
User-agent: Mozilla/4.0 (compatible; www.euro-directory.com; urlchecker1.0)
Disallow: /
User-agent: Mozilla/4.0 (compatible; www.galaxy.com)
Disallow: /
User-agent: Mozilla/4.0 (compatible; www.linkguard.com Linkguard Online 1.0; Windows NT)
Disallow: /
User-agent: Mozilla/4.0 (compatible;MSIE 6.0; Windows NT 5.0; H010818)
Disallow: /
User-agent: Mozilla/4.0 (fantomBrowser)
Disallow: /
User-agent: Mozilla/4.0 (fantomCrew Browser)
Disallow: /
User-agent: Mozilla/4.0 (hhjhj@yahoo.com)
Disallow: /
User-agent: Mozilla/4.0 WebTV/2.6 (compatible; MSIE 4.0)
Disallow: /
User-agent: Mozilla/4.0 [en] (Ask Jeeves Corporate Spider)
Disallow: /
User-agent: Mozilla/4.0 compatible FurlBot/Furl Search 2.0 (FurlBot; http://www.furl.net; wn.furlbot@looksmart.net)
Disallow: /
User-agent: Mozilla/4.0 compatible ZyBorg/1.0 (ZyBorg@WISEnutbot.com; http://www.WISEnutbot.com)
Disallow: /
User-agent: Mozilla/4.0 compatible ZyBorg/1.0 (wn.zyborg@looksmart.net; http://www.WISEnutbot.com)
Disallow: /
User-agent: Mozilla/4.0 compatible ZyBorg/1.0 Dead Link Checker (wn.zyborg@looksmart.net; http://www.WISEnutbot.com)
Disallow: /
User-agent: Mozilla/4.0 compatible ZyBorg/1.0 for Homepage (ZyBorg@WISEnutbot.com; http://www.WISEnutbot.com)
Disallow: /
User-agent: Mozilla/4.0 efp@gmx.net
Disallow: /
User-agent: Mozilla/4.0(compatible; Zealbot 1.0)
Disallow: /
User-agent: Mozilla/4.01 (compatible; NORAD National Defence Network)
Disallow: /
User-agent: Mozilla/4.01 [en](Win95;I)
Disallow: /
User-agent: Mozilla/4.02 [en] (X11; I; SunOS 5.6 sun4u)
Disallow: /
User-agent: Mozilla/4.04 (compatible; Dulance bot; +http://www.dulance.com/bot.jsp)
Disallow: /
User-agent: Mozilla/4.04 [en] (X11; I; HP-UX B.10.20 9000/712)
Disallow: /
User-agent: Mozilla/4.04 [en] (X11; I; IRIX 5.3 IP22)
Disallow: /
User-agent: Mozilla/4.05 (Macintosh; I; 68K Nav)
Disallow: /
User-agent: Mozilla/4.05 (Macintosh; I; PPC Nav)
Disallow: /
User-agent: Mozilla/4.05 [en] (X11; I; SunOS 4.1.4 sun4m)
Disallow: /
User-agent: Mozilla/4.08 [en] (Win98; U ;Nav)
Disallow: /
User-agent: Mozilla/4.08 [en] (WinNT; U)
Disallow: /
User-agent: Mozilla/4.0_(compatible;_MSIE_5.0;_Windows_95)_TrueRobot/1.4 libwww/5.2.8
Disallow: /
User-agent: Mozilla/4.0_(compatible;_MSIE_5.0;_Windows_95)_VoilaBot/1.6 libwww/5.3.2
Disallow: /
User-agent: Mozilla/4.5 (compatible; HTTrack 3.0x; Windows 98)
Disallow: /
User-agent: Mozilla/4.5 (compatible; OmniWeb/4.0.5; Mac_PowerPC)
Disallow: /
User-agent: Mozilla/4.5 (compatible; OmniWeb/4.1-beta-1; Mac_PowerPC)
Disallow: /
User-agent: Mozilla/4.5 (compatible; iCab 2.5.3; Macintosh; I; PPC)
Disallow: /
User-agent: Mozilla/4.5 RPT-HTTPClient/0.3-2
Disallow: /
User-agent: Mozilla/4.5 [en]C-CCK-MCD {RuralNet} (Win98; I)
Disallow: /
User-agent: Mozilla/4.5b1 [en] (X11; I; Linux 2.0.35 i586)
Disallow: /
User-agent: Mozilla/4.6 [en] (http://www.cnet.com/)
Disallow: /
User-agent: Mozilla/4.61 [de] (OS/2; I)
Disallow: /
User-agent: Mozilla/4.61 [en] (X11; U; ) - BrowseX (2.0.0 Windows)
Disallow: /
User-agent: Mozilla/4.7
Disallow: /
User-agent: Mozilla/4.7 (compatible; Intelliseek; http://www.intelliseek.com)
Disallow: /
User-agent: Mozilla/4.7 (compatible; OffByOne; Windows 98) Webster Pro V3.2
Disallow: /
User-agent: Mozilla/4.7 (compatible; WhizBang; http://www.whizbang.com/crawler)
Disallow: /
User-agent: Mozilla/4.7 (compatible; Whizbang)
Disallow: /
User-agent: Mozilla/4.7 (compatible; http://eidetica.com/spider)
Disallow: /
User-agent: Mozilla/4.7 [en](BecomeBot@exava.com)
Disallow: /
User-agent: Mozilla/4.7 [en](Exabot@exava.com)
Disallow: /
User-agent: Mozilla/4.7 [en]C-CCK-MCD {Yahoo;YIP052400} (Win95; I)
Disallow: /
User-agent: Mozilla/4.72 [en] (BACS http://www.ba.be)
Disallow: /
User-agent: Mozilla/4.72C-CCK-MCD Caldera Systems OpenLinux [en] (X11; U; Linux 2.2.14 i686)
Disallow: /
User-agent: Mozilla/4.75C-ja [ja] (X11; U; OSF1 V5.1 alpha)
Disallow: /
User-agent: Mozilla/4.76 (Windows 98; U) Opera 5.12 [en]
Disallow: /
User-agent: Mozilla/4.76 [en] (X11; U; FreeBSD 4.4-STABLE i386)
Disallow: /
User-agent: Mozilla/4.76 [en] (X11; U; SunOS 5.7 sun4u)
Disallow: /
User-agent: Mozilla/4.77C-SGI [en] (X11; U; IRIX 6.5 IP32)
Disallow: /
User-agent: Mozilla/5.0
Disallow: /
User-agent: Mozilla/5.0 (+http://www.eurekster.com/mammoth) Mammoth/0.1
Disallow: /
User-agent: Mozilla/5.0 (+http://www.sli-systems.com/) Mammoth/0.1
Disallow: /
User-agent: Mozilla/5.0 (Clustered-Search-Bot/1.0; support@clush.com; http://www.clush.com/)
Disallow: /
User-agent: Mozilla/5.0 (Gecko/20070310 Mozshot/0.0.20070628; http://mozshot.nemui.org/)
Disallow: /
User-agent: Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.4; en-US; rv:1.9b5) Gecko/2008032619 Firefox/3.0b5
Disallow: /
User-agent: Mozilla/5.0 (Macintosh; U; PPC Mac OS X Mach-O; en-US; rv:1.0.1) Gecko/20021219 Chimera/0.6
Disallow: /
User-agent: Mozilla/5.0 (Macintosh; U; PPC Mac OS X Mach-O; en-US; rv:1.0.1) Gecko/20030306 Camino/0.7
Disallow: /
User-agent: Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en-US) AppleWebKit/xx (KHTML like Gecko) OmniWeb/v5xx.xx
Disallow: /
User-agent: Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en-us) AppleWebKit/xxx.x (KHTML like Gecko) Safari/12x.x
Disallow: /
User-agent: Mozilla/5.0 (Macintosh; U; PPC; en-US; rv:0.9.2) Gecko/20010726 Netscape6/6.1
Disallow: /
User-agent: Mozilla/5.0 (Sage)
Disallow: /
User-agent: Mozilla/5.0 (Slurp/cat; slurp@inktomi.com; http://www.inktomi.com/slurp.html)
Disallow: /
User-agent: Mozilla/5.0 (Slurp/si; slurp@inktomi.com; http://www.inktomi.com/slurp.html)
Disallow: /
User-agent: Mozilla/5.0 (SunOS 5.8 sun4u; U) Opera 5.0 [en]
Disallow: /
User-agent: Mozilla/5.0 (Twiceler-0.9 http://www.cuill.com/twiceler/robot.html)
Disallow: /
User-agent: Mozilla/5.0 (Version: xxxx Type:xx)
Disallow: /
User-agent: Mozilla/5.0 (Windows; U; Win98; en-US; rv:0.9.2) Gecko/20010726 Netscape6/6.1
Disallow: /
User-agent: Mozilla/5.0 (Windows; U; Win98; en-US; rv:x.xx) Gecko/20030423 Firebird Browser/0.6
Disallow: /
User-agent: Mozilla/5.0 (Windows; U; Win9x; en; Stable) Gecko/20020911 Beonex/0.8.1-stable
Disallow: /
User-agent: Mozilla/5.0 (Windows; U; Windows NT 5.0; en-US; rv:1.7.7) NimbleCrawler 1.11 obeys UserAgent NimbleCrawler For problems contact: crawler_at_dataalchemy.com
Disallow: /
User-agent: Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8.0.5) Gecko/20060731 Firefox/1.5.0.5 Flock/0.7.4.1
Disallow: /
User-agent: Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8.1.11) Gecko/20071127 Firefox/2.0.0.4/Megaupload x.0
Disallow: /
User-agent: Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:x.x.x) Gecko/20041107 Firefox/x.x
Disallow: /
User-agent: Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:x.xx) Gecko/20030504 Mozilla Firebird/0.6
Disallow: /
User-agent: Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:x.xxx) Gecko/20041027 Mnenhy/0.6.0.104
Disallow: /
User-agent: Mozilla/5.0 (Windows; U; Windows NT 5.1; fr; rv:1.8.1) VoilaBot BETA 1.2 (support.voilabot@orange-ftgroup.com)
Disallow: /
User-agent: Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US; rv:1.9b5) Gecko/2008032620 Firefox/3.0b5
Disallow: /
User-agent: Mozilla/5.0 (Windows; U;XMPP Tiscali Communicator v.10.0.1; Windows NT 5.1; it; rv:1.8.1.3) Gecko/20070309 Firefox/2.0.0.3
Disallow: /
User-agent: Mozilla/5.0 (Windows;) NimbleCrawler 1.12 obeys UserAgent NimbleCrawler For problems contact: crawler@health
Disallow: /
User-agent: Mozilla/5.0 (Windows;) NimbleCrawler 1.12 obeys UserAgent NimbleCrawler For problems contact: crawler@healthline.com
Disallow: /
User-agent: Mozilla/5.0 (X11; Linux i686; U;rv: 1.7.13) Gecko/20070322 Kazehakase/0.4.4.1
Disallow: /
User-agent: Mozilla/5.0 (X11; U; Linux 2.4.2-2 i586; en-US; m18) Gecko/20010131 Netscape6/6.01
Disallow: /
User-agent: Mozilla/5.0 (X11; U; Linux i686; de-AT; rv:1.8.0.2) Gecko/20060309 SeaMonkey/1.0
Disallow: /
User-agent: Mozilla/5.0 (X11; U; Linux i686; en-GB; rv:1.7.6) Gecko/20050405 Epiphany/1.6.1 (Ubuntu) (Ubuntu package 1.0.2)
Disallow: /
User-agent: Mozilla/5.0 (X11; U; Linux i686; en-US; Nautilus/1.0Final) Gecko/20020408
Disallow: /
User-agent: Mozilla/5.0 (X11; U; Linux i686; en-US; rv:0.9.3) Gecko/20010801
Disallow: /
User-agent: Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.2b) Gecko/20021007 Phoenix/0.3
Disallow: /
User-agent: Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.6) Gecko/20040413 Epiphany/1.2.1
Disallow: /
User-agent: Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.8.0.7) Gecko/20060909 Firefox/1.5.0.7 SnapPreviewBot
Disallow: /
User-agent: Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.8.1) Gecko/20061129 BonEcho/2.0
Disallow: /
User-agent: Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.8.1.1) Gecko/20061205 Iceweasel/2.0.0.1 (Debian-2.0.0.1+dfsg-2)
Disallow: /
User-agent: Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9a8) Gecko/2007100619 GranParadiso/3.0a8
Disallow: /
User-agent: Mozilla/5.0 (compatible) GM RSS Panel X
Disallow: /
User-agent: Mozilla/5.0 (compatible; AnsearchBot/1.x; +http://www.ansearch.com.au/)
Disallow: /
User-agent: Mozilla/5.0 (compatible; Ask Jeeves/Teoma;  http://about.ask.com/en/docs/about/webmasters.shtml)
Disallow: /
User-agent: Mozilla/5.0 (compatible; BanBots/2.0b; Fetch; +http://www.banbots.com)
Disallow: /
User-agent: Mozilla/5.0 (compatible; BecomeBot/1.23; http://www.become.com/webmasters.html)
Disallow: /
User-agent: Mozilla/5.0 (compatible; BecomeBot/1.xx; MSIE 6.0 compatible; http://www.become.com/webmasters.html)
Disallow: /
User-agent: Mozilla/5.0 (compatible; BecomeBot/2.0beta; http://www.become.com/webmasters.html)
Disallow: /
User-agent: Mozilla/5.0 (compatible; BecomeBot/2.x; MSIE 6.0 compatible; http://www.become.com/site_owners.html)
Disallow: /
User-agent: Mozilla/5.0 (compatible; BecomeJPBot/2.3; MSIE 6.0 compatible; +http://www.become.co.jp/site_owners.html)
Disallow: /
User-agent: Mozilla/5.0 (compatible; BlogRefsBot/0.1; http://www.blogrefs.com/about/bloggers)
Disallow: /
User-agent: Mozilla/5.0 (compatible; Bot; +http://pressemitteilung.ws/spamfilter
Disallow: /
User-agent: Mozilla/5.0 (compatible; BuzzRankingBot/1.0; +http://www.buzzrankingbot.com/)
Disallow: /
User-agent: Mozilla/5.0 (compatible; Charlotte/1.0b; charlotte@betaspider.com)
Disallow: /
User-agent: Mozilla/5.0 (compatible; Charlotte/1.0b; http://www.searchme.com/support/)
Disallow: /
User-agent: Mozilla/5.0 (compatible; Crawling jpeg; http://www.yama.info.waseda.ac.jp)
Disallow: /
User-agent: Mozilla/5.0 (compatible; Custo 3 (Netwu.com); Windows NT 5.1)
Disallow: /
User-agent: Mozilla/5.0 (compatible; DNS-Digger-Explorer/1.0; +http://www.dnsdigger.com)
Disallow: /
User-agent: Mozilla/5.0 (compatible; DNS-Digger/1.0; +http://www.dnsdigger.com)
Disallow: /
User-agent: Mozilla/5.0 (compatible; EARTHCOM.info/2.01; http://www.earthcom.info)
Disallow: /
User-agent: Mozilla/5.0 (compatible; EARTHCOM/2.2; +http://enter4u.eu)
Disallow: /
User-agent: Mozilla/5.0 (compatible; Exabot Test/3.0; +http://www.exabot.com/go/robot)
Disallow: /
User-agent: Mozilla/5.0 (compatible; FatBot 2.0; http://www.thefind.com/main/CrawlerFAQs.fhtml)
Disallow: /
User-agent: Mozilla/5.0 (compatible; Google Desktop) Paros/3.2.12
Disallow: /
User-agent: Mozilla/5.0 (compatible; Googlebot/2.1; http://www.google.com/bot.html)
Disallow: /
User-agent: Mozilla/5.0 (compatible; Heritrix/1.8.0 http://www.hanzoarchives.com)
Disallow: /
User-agent: Mozilla/5.0 (compatible; Hermit Search. Com; +http://www.hermitsearch.com)
Disallow: /
User-agent: Mozilla/5.0 (compatible; HyperixScoop/1.3; +http://www.hyperix.com)
Disallow: /
User-agent: Mozilla/5.0 (compatible; IDBot/1.0; +http://www.id-search.org/bot.html)
Disallow: /
User-agent: Mozilla/5.0 (compatible; InterseekWeb/3.x)
Disallow: /
User-agent: Mozilla/5.0 (compatible; Jim +http://www.hanzoarchives.com)
Disallow: /
User-agent: Mozilla/5.0 (compatible; Konqueror/2.0.1; X11); Supports MD5-Digest; Supports gzip encoding
Disallow: /
User-agent: Mozilla/5.0 (compatible; Konqueror/2.1.1; X11)
Disallow: /
User-agent: Mozilla/5.0 (compatible; Konqueror/2.2.2)
Disallow: /
User-agent: Mozilla/5.0 (compatible; Konqueror/2.2.2; Linux 2.4.14-xfs; X11; i686)
Disallow: /
User-agent: Mozilla/5.0 (compatible; Konqueror/3.5; Linux) KHTML/3.5.5 (like Gecko) (Exabot-Thumbnails)
Disallow: /
User-agent: Mozilla/5.0 (compatible; LemSpider 0.1)
Disallow: /
User-agent: Mozilla/5.0 (compatible; LinkStash Bookmark Manager; http://www.xrayz.co.uk/)
Disallow: /
User-agent: Mozilla/5.0 (compatible; LinksManager.com_bot http://linksmanager.com/linkchecker.html)
Disallow: /
User-agent: Mozilla/5.0 (compatible; MOSBookmarks/v2.6-Plus; Link Checker)
Disallow: /
User-agent: Mozilla/5.0 (compatible; MSIE 6.0; Podtech Network; crawler_admin@podtech.net)
Disallow: /
User-agent: Mozilla/5.0 (compatible; MojeekBot/2.0; http://www.mojeek.com/bot.html)
Disallow: /
User-agent: Mozilla/5.0 (compatible; OnetSzukaj/5.0; http://szukaj.onet.pl)
Disallow: /
User-agent: Mozilla/5.0 (compatible; PEAR HTTP_Request class; http://feed.moo.jp/)
Disallow: /
User-agent: Mozilla/5.0 (compatible; PWeBot/3.1; http://www.programacionweb.net/robot.php)
Disallow: /
User-agent: Mozilla/5.0 (compatible; PagestackerBot; http://www.pagestacker.com)
Disallow: /
User-agent: Mozilla/5.0 (compatible; PalmeraBot; http://www.links24h.com/help/palmera) Version 0.001
Disallow: /
User-agent: Mozilla/5.0 (compatible; Phonifier; +http://www.phonifier.com)
Disallow: /
User-agent: Mozilla/5.0 (compatible; Pogodak.hr/3.1)
Disallow: /
User-agent: Mozilla/5.0 (compatible; Proximic crawler; +http://www.proximic.com/en/about-us/contact-us.html)
Disallow: /
User-agent: Mozilla/5.0 (compatible; Quantcastbot/1.0; www.quantcast.com)
Disallow: /
User-agent: Mozilla/5.0 (compatible; SYCLIKControl/LinkChecker;)
Disallow: /
User-agent: Mozilla/5.0 (compatible; ScoutJet; +http://www.scoutjet.com/)
Disallow: /
User-agent: Mozilla/5.0 (compatible; Scrubby/2.2;  http://www.scrubtheweb.com/)
Disallow: /
User-agent: Mozilla/5.0 (compatible; ShunixBot/1.x.x +http://www.shunix.com/robot.htm)
Disallow: /
User-agent: Mozilla/5.0 (compatible; ShunixBot/1.x; http://www.shunix.com/bot.htm)
Disallow: /
User-agent: Mozilla/5.0 (compatible; SkreemRBot +http://skreemr.com)
Disallow: /
User-agent: Mozilla/5.0 (compatible; SnapPreviewBot; en-US; rv:1.8.0.9) Gecko/20061206 Firefox/1.5.0.9
Disallow: /
User-agent: Mozilla/5.0 (compatible; SpurlBot/0.2)
Disallow: /
User-agent: Mozilla/5.0 (compatible; SummizeBot +http://www.summize.com)
Disallow: /
User-agent: Mozilla/5.0 (compatible; Synoobot/0.9; http://www.synoo.com/search/bot.html)
Disallow: /
User-agent: Mozilla/5.0 (compatible; Theophrastus/x.x; http://users.cs.cf.ac.uk/N.A.Smith/theophrastus.php)
Disallow: /
User-agent: Mozilla/5.0 (compatible; TridentSpider/3.1)
Disallow: /
User-agent: Mozilla/5.0 (compatible; Vagabondo/2.1; webcrawler at wise-guys dot nl; http://webagent.wise-guys.nl/)
Disallow: /
User-agent: Mozilla/5.0 (compatible; Windows NT 5.0; phpwebbrainBot/0.1 - http://www.monsterli.ch/phpwebbrain/)
Disallow: /
User-agent: Mozilla/5.0 (compatible; XTbot/1.0v; +http://www.externaltest.com)
Disallow: /
User-agent: Mozilla/5.0 (compatible; Yahoo! DE Slurp; http://help.yahoo.com/help/us/ysearch/slurp)
Disallow: /
User-agent: Mozilla/5.0 (compatible; Yahoo! Slurp China; http://misc.yahoo.com.cn/help.html)
Disallow: /
User-agent: Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)
Disallow: /
User-agent: Mozilla/5.0 (compatible; Yoono; http://www.yoono.com/)
Disallow: /
User-agent: Mozilla/5.0 (compatible; Zenbot/1.3; +http://zen.co.za/webmasters/)
Disallow: /
User-agent: Mozilla/5.0 (compatible; archive.org_bot/1.10.0 +http://www.loc.gov/minerva/crawl.html)
Disallow: /
User-agent: Mozilla/5.0 (compatible; archive.org_bot/1.13.1x http://crawler.archive.org)
Disallow: /
User-agent: Mozilla/5.0 (compatible; archive.org_bot/1.5.0-200506132127 http://crawler.archive.org) Hurricane Katrina
Disallow: /
User-agent: Mozilla/5.0 (compatible; de/1.13.2 +http://www.de.com)
Disallow: /
User-agent: Mozilla/5.0 (compatible; egothor/8.0g; +http://ego.ms.mff.cuni.cz/)
Disallow: /
User-agent: Mozilla/5.0 (compatible; heritrix/1.10.2 +http://i.stanford.edu/)
Disallow: /
User-agent: Mozilla/5.0 (compatible; heritrix/1.12.1 +http://newstin.com/)
Disallow: /
User-agent: Mozilla/5.0 (compatible; heritrix/1.12.1 +http://www.page-
Disallow: /
User-agent: Mozilla/5.0 (compatible; heritrix/1.12.1 +http://www.page-store.com)
Disallow: /
User-agent: Slurp
Disallow: /
User-agent: Yahoo-MMCrawler
Disallow: /
User-agent: msnb
Disallow: /
User-agent: *
Disallow: /*.gif$
Disallow: /*.Gif$
Disallow: /*.GIF$
Disallow: /*.jpg$
Disallow: /*.Jpg$
Disallow: /*.JPG$
Disallow: /*.jpeg$
Disallow: /*.Jpeg$
Disallow: /*.JPEG$
Disallow: /*.pdf$
Disallow: /*.Pdf$
Disallow: /*.PDF$
Disallow: /*.zip$
Disallow: /*.Zip$
Disallow: /*.ZIP$