# # Dato: 28-04-2005 # # © HOSTMASTER # www.sydweb.dk # hostmaster@sydweb.dk # # Denne robots.txt fil må frit disktribueret. # # Filen skal placeres i root (www.dinhjemmeside.dk/robots.txt) for at kunne virke, en anden placering vil IKKE virke. # # I denne fil er AL adgang for søgemasker banlyst, du skal selv ændre indstillingen ved at fjerne / efter Disallow: # # HUSK: Har du tilmeldt dig en eller flere søgemaskiner, skal du tilpasse din robots.txt fil ved at fjerne / efter Disallow: # HUSK: Du skal tilpasse dine META TAGS før din robots.txt virker efter hensigten # # Finder du en robot som ikke er på denne liste, venligst email til ovennævnte adresse med komplet user-agent: # # Nedenstående skal tilpasses din root. User-agent: * Disallow: /_borders/ Disallow: /_derived/ Disallow: /_fpclass/ Disallow: /_overlay/ Disallow: /_private/ Disallow: /_themes/ Disallow: /_vti_bin/ Disallow: /_vti_cnf/ Disallow: /_vti_log/ Disallow: /_vti_map/ Disallow: /_vti_pvt/ Disallow: /_vti_txt/ Disallow: http://yamaha-rd.dk/forum1/phpBB2/admin/ Disallow: http://yamaha-rd.dk/forum1/phpBB2/db/ Disallow: http://yamaha-rd.dk/forum1/phpBB2/images/ Disallow: http://yamaha-rd.dk/forum1/phpBB2/includes/ Disallow: http://yamaha-rd.dk/forum1/phpBB2/language/ Disallow: http://yamaha-rd.dk/forum1/phpBB2/templates/ Disallow: http://yamaha-rd.dk/forum1/phpBB2/common.php Disallow: http://yamaha-rd.dk/forum1/phpBB2/config.php Disallow: http://yamaha-rd.dk/forum1/phpBB2/faq.php Disallow: http://yamaha-rd.dk/forum1/phpBB2/groupcp.php Disallow: http://yamaha-rd.dk/forum1/phpBB2/login.php Disallow: http://yamaha-rd.dk/forum1/phpBB2/memberlist.php Disallow: http://yamaha-rd.dk/forum1/phpBB2/modcp.php Disallow: http://yamaha-rd.dk/forum1/phpBB2/posting.php Disallow: http://yamaha-rd.dk/forum1/phpBB2/privmsg.php Disallow: http://yamaha-rd.dk/forum1/phpBB2/profile.php Disallow: http://yamaha-rd.dk/forum1/phpBB2/search.php Disallow: http://yamaha-rd.dk/forum1/phpBB2/viewonline.php Disallow: http://yamaha-rd.dk/forum1/phpBB2/cgi-bin/ Disallow: http://yamaha-rd.dk/forum1/phpBB2/tmp/ Disallow: http://yamaha-rd.dk/forum1/phpBB2/temp/ Disallow: http://yamaha-rd.dk/forum1/phpBB2/private/ Disallow: http://yamaha-rd.dk/forum1/phpBB2/chat/ Disallow: http://yamaha-rd.dk/forum1/phpBB2/intergration/ Disallow: http://yamaha-rd.dk/forum1/phpBB2/medals/ Disallow: http://yamaha-rd.dk/forum1/phpBB2/stat_modules/ Disallow: http://yamaha-rd.dk/forum1/phpBB2/hl/ Disallow: http://yamaha-rd.dk/forum1/phpBB2/xs_mod/ Disallow: http://yamaha-rd.dk/forum1/phpBB2/sounds/ Disallow: http://yamaha-rd.dk/forum1/phpBB2/smile_creator/ Disallow: http://yamaha-rd.dk/forum1/phpBB2/mods/ Disallow: http://yamaha-rd.dk/forum1/phpBB2/ftp/ Disallow: http://yamaha-rd.dk/forum1/phpBB2/mysql/ Disallow: http://yamaha-rd.dk/forum1/phpBB2/dseitz_mods/ Disallow: http://yamaha-rd.dk/forum1/phpBB2/auction/ Disallow: http://yamaha-rd.dk/forum1/phpBB2/usage/ Disallow: http://yamaha-rd.dk/forum1/phpBB2/catalog/ Disallow: http://yamaha-rd.dk/forum1/phpBB2/catalog/admin/ Disallow: http://yamaha-rd.dk/forum1/phpBB2/catalog/download/ Disallow: http://yamaha-rd.dk/forum1/phpBB2/catalog/elmar/ Disallow: http://yamaha-rd.dk/forum1/phpBB2/catalog/pub/ Disallow: http://yamaha-rd.dk/forum1/phpBB2/catalog/account.php Disallow: http://yamaha-rd.dk/forum1/phpBB2/catalog/advanced_search.php Disallow: http://yamaha-rd.dk/forum1/phpBB2/catalog/checkout_shipping.php Disallow: http://yamaha-rd.dk/forum1/phpBB2/catalog/create_account.php Disallow: http://yamaha-rd.dk/forum1/phpBB2/catalog/login.php Disallow: http://yamaha-rd.dk/forum1/phpBB2/catalog/password_forgotten.php Disallow: http://yamaha-rd.dk/forum1/phpBB2/catalog/popup_image.php Disallow: http://yamaha-rd.dk/forum1/phpBB2/catalog/shopping_cart.php Disallow: http://yamaha-rd.dk/forum1/phpBB2/googleactivity/ # MSN User-agent: MSNBot Disallow: / User-agent: MSNBot* Disallow: / User-agent: MSNBot/0.1 Disallow: / User-agent: MSNBot/0.11 Disallow: / User-agent: MSNBot/0.3+ Disallow: / User-agent: MSNBot/1.0+ Disallow: / User-agent: MSNBOT/0.xx (http://search.msn.com/msnbot.htm) Disallow: / User-agent: msnbot/x.xx (+http://search.msn.com/msnbot.htm) Disallow: / # Øvrige Internationale robots, spider, crawler, email-grapper og spam-bots. # A User-agent: arks Disallow: / User-agent: ABCdatos BotLink Disallow: / User-agent: ABCDee Disallow: / User-agent: Accessdiver Disallow: / User-agent: Acme.Spider Disallow: / User-agent: Ahoy! The Homepage Finder Disallow: / User-agent: Alexa Disallow: / User-agent: Alexibot Disallow: / User-agent: Alkaline Disallow: / User-agent: ananzi Disallow: / User-agent: Anthill Disallow: / User-agent: Aqua_Products Disallow: / User-agent: Arachnophilia Disallow: / User-agent: Arale Disallow: / User-agent: Araneo Disallow: / User-agent: AraybOt Disallow: / User-agent: Archive Disallow: / User-agent: Aretha Disallow: / User-agent: ARIADNE Disallow: / User-agent: arks Disallow: / User-agent: AskJeeves Disallow: / User-agent: ASpider (Associative Spider) Disallow: / User-agent: asterias Disallow: / User-agent: ATN Worldwide Disallow: / User-agent: Atomz Disallow: / User-agent: Atomz.com Search Robot Disallow: / User-agent: ATV Disallow: / User-agent: AURESYS Disallow: / User-agent: A-Online Search Disallow: / User-agent: AbachoBOT Disallow: / User-agent: AbachoBOT (Mozilla compatible) Disallow: / User-agent: ABCdatos BotLink/5.xx.xxx Disallow: / User-agent: Aberja Checkomat Disallow: / User-agent: abot/0.1 (abot; http://www.abot.com; abot@abot.com) Disallow: / User-agent: About/0.1libwww-perl/5.47 Disallow: / User-agent: Ace Explorer Disallow: / User-agent: AcoiRobot Disallow: / User-agent: Acoon Robot v1.50.001 Disallow: / User-agent: Acoon Robot v1.52 (http://www.acoon.de) Disallow: / User-agent: accoona Disallow: / User-agent: ActiveBookmark 1.0 Disallow: / User-agent: ActiveWorlds/3.xx (xxx) Disallow: / User-agent: Ad Muncher v4.xx.x Disallow: / User-agent: AESOP_com_SpiderMan Disallow: / User-agent: agadine/1.x.x (+http://www.agada.de) Disallow: / User-agent: Agent-SharewarePlazaFileCheckBot/2.0+(+http://www.SharewarePlaza.com) Disallow: / User-agent: AgentName/0.1 libwww-perl/5.48 Disallow: / User-agent: Aladin/3.324 Disallow: / User-agent: AlkalineBOT/1.3 Disallow: / User-agent: AlkalineBOT/1.4 (1.4.0326.0 RTM) Disallow: / User-agent: Aleksika Spider/1.0 (+http://www.aleksika.com/) Disallow: / User-agent: Allesklar/0.1 libwww-perl/5.46 Disallow: / User-agent: Alligator 1.31 (www.nearsoftware.com) Disallow: / User-agent: AltaVista Intranet V2.0 AVS EVAL search@freeit.com Disallow: / User-agent: AltaVista Intranet V2.0 Compaq Altavista Eval sveand@altavista.net Disallow: / User-agent: AltaVista Intranet V2.0 evreka.com crawler@evreka.com Disallow: / User-agent: AltaVista V2.0B crawler@evreka.com Disallow: / User-agent: AmfibiBOT Disallow: / User-agent: Amfibibot/0.06 (Amfibi Web Search; http://www.amfibi.com; agent@amfibi.com) Disallow: / User-agent: amibot Disallow: / User-agent: Amiga-AWeb/3.4.167SE Disallow: / User-agent: AmigaVoyager/3.4.4 (MorphOS/PPC native) Disallow: / User-agent: AnnoMille spider 0.1 alpha - http://www.annomille.it Disallow: / User-agent: Anonymized by ProxyOS: http://www.megaproxy.com Disallow: / User-agent: Anonymizer/1.1 Disallow: / User-agent: AnswerChase x.0 Disallow: / User-agent: AnswerChase PROve x.0 Disallow: / User-agent: ANTFresco/x.xx Disallow: / User-agent: AnzwersCrawl/2.0 (anzwerscrawl@anzwers.com.au;Engine) Disallow: / User-agent: antibot-V1.1.5/i586-linux-2.2 Disallow: / User-agent: Aplix HTTP/1.0.1 Disallow: / User-agent: Aplix_SANYO_browser/1.x (Japanese) Disallow: / User-agent: Aplix_SEGASATURN_browser/1.x (Japanese) Disallow: / User-agent: appie 1.1 (www.walhello.com) Disallow: / User-agent: Aport Disallow: / User-agent: Arachnoidea (arachnoidea@euroseek.com) Disallow: / User-agent: ArchitextSpider Disallow: / User-agent: archive.org_bot Disallow: / User-agent: Argus/1.1 (Nutch; http://www.simpy.com/bot.html; feedback at simpy dot com) Disallow: / User-agent: Asahina-Antenna/1.x Disallow: / User-agent: Asahina-Antenna/1.x (libhina.pl/x.x ; libtime.pl/x.x) Disallow: / User-agent: ask.24x.info Disallow: / User-agent: ASPSeek/1.2.x Disallow: / User-agent: ASPseek/1.2.xx Disallow: / User-agent: ASPSeek/1.2.xxpre Disallow: / User-agent: ASPSeek/1.2.xa Disallow: / User-agent: ASPSeek/1.2.5 Disallow: / User-agent: ASPseek/1.2.9d Disallow: / User-agent: ASSORT/0.10 Disallow: / User-agent: asterias/2.0 Disallow: / User-agent: AtlocalBot/1.1 +(http://www.atlocal.com/local-web-site-owner.html) Disallow: / User-agent: Atomz/1.0 Disallow: / User-agent: atSpider/1.0 Disallow: / User-agent: augurfind Disallow: / User-agent: augurnfind V-1.x Disallow: / User-agent: autoemailspider Disallow: / User-agent: autohttp Disallow: / User-agent: AV Fetch 1.0 Disallow: / User-agent: AVSearch-1.0(peter.turney@nrc.ca) Disallow: / User-agent: AVSearch-2.0-fusionIdx-14-CompetitorWebSites Disallow: / User-agent: AVSearch-3.0(AltaVista/AVC) Disallow: / User-agent: AxmoRobot - Crawling your site for better indexing on www.axmo.com search engine. Disallow: / # B User-agent: b2w/0.1 Disallow: / User-agent: BackDoorBot/1.0 Disallow: / User-agent: BackRub Disallow: / User-agent: Baldgorilla Disallow: / User-agent: BBot Disallow: / User-agent: BecomeBot Disallow: / User-agent: Big Brother Disallow: / User-agent: Bjaaland Disallow: / User-agent: Black Hole Disallow: / User-agent: Bloodhound Disallow: / User-agent: BlowFish/1.0 Disallow: / User-agent: BoardViewer Disallow: / User-agent: Bookmark search tool Disallow: / User-agent: Borg-Bot Disallow: / User-agent: BotALot Disallow: / User-agent: BotRightHere Disallow: / User-agent: BoxSeaBot Disallow: / User-agent: bright.net caching robot Disallow: / User-agent: Browse Disallow: / User-agent: BSpider Disallow: / User-agent: BuiltBotTough Disallow: / User-agent: Bullseye/1.0 Disallow: / User-agent: Burner Disallow: / User-agent: BaboomBot/1.x.x (+http://www.baboom.us) Disallow: / User-agent: BaiDuSpider Disallow: / User-agent: Baiduspider+ (+http://www.baidu.com/search/spider.htm) Disallow: / User-agent: BanBots/1.2 (spider@banbots.com) Disallow: / User-agent: BarraHomeCrawler (albertof@barrahome.org) Disallow: / User-agent: bdcindexer_2.6.2 (research@bdc) Disallow: / User-agent: BDFetch Disallow: / User-agent: BDNcentral Crawler v2.3 [en] (http://www.bdncentral.com/robot.html) (X11; I; Linux 2.0.44 i686) Disallow: / User-agent: beautybot/1.0 (+http://www.uchoose.de/crawler/beautybot/) Disallow: / User-agent: BeebwareDirectory/v0.01 - http://directory.beebware.co.uk Linkchecker Disallow: / User-agent: Big Brother (http://pauillac.inria.fr/~fpottier/) Disallow: / User-agent: BigBrother/1.6e Disallow: / User-agent: BigCliqueBOT/1.03-dev (bigclicbot; http://www.bigclique.com; bot@bigclique.com) Disallow: / User-agent: Bilbo/2.3b-UNIX Disallow: / User-agent: billbot wjj@cs.cmu.edu Disallow: / User-agent: Biyubi/x.x (Sistema Fenix; G11; Familia Toledo; es-mx) Disallow: / User-agent: BlackWidow Disallow: / User-agent: Blaiz-Bee/1.0 (+http://www.blaiz.net) Disallow: / User-agent: BlitzBOT@tricus.net Disallow: / User-agent: BlitzBOT@tricus.net (Mozilla compatible) Disallow: / User-agent: BlogBot/1.x Disallow: / User-agent: Bloglines Title Fetch/1.0 (http://www.bloglines.com) Disallow: / User-agent: blogWatcher_Spider/0.1 (http://www.lr.pi.titech.ac.jp/blogWatcher/) Disallow: / User-agent: bluefish 0.6 HTML editor Disallow: / User-agent: BMCLIENT Disallow: / User-agent: Bobby/4.0.x RPT-HTTPClient/0.3-3E Disallow: / User-agent: boitho.com-dc/0.xx (http://www.boitho.com/dcbot.html) Disallow: / User-agent: boitho.com-robot/1.x Disallow: / User-agent: boitho.com-robot/1.x (http://www.boitho.com/bot.html) Disallow: / User-agent: Bookmark Buddy bookmark checker (http://www.bookmarkbuddy.net/) Disallow: / User-agent: Bookmark Renewal Check Agent [http://www.bookmark.ne.jp/] Disallow: / User-agent: Bot mailto:craftbot@yahoo.com Disallow: / User-agent: BravoBrian bstop.bravobrian.it Disallow: / User-agent: BravoBrian SpiderEngine MarcoPolo Disallow: / User-agent: BSDSeek/1.0 Disallow: / User-agent: BStop.BravoBrian.it Agent Detector Disallow: / User-agent: BullsEye Disallow: / User-agent: bumblebee@relevare.com Disallow: / User-agent: BunnySlippers Disallow: / User-agent: Buscaplus Robi/1.0 (http://www.buscaplus.com/robi/) Disallow: / # C User-agent: CACTVS Chemistry Spider Disallow: / User-agent: Calif Disallow: / User-agent: Cassandra Disallow: / User-agent: Catalog Disallow: / User-agent: Cegbfeieh Disallow: / User-agent: Cerb Disallow: / User-agent: Checkbot Disallow: / User-agent: CheeseBot Disallow: / User-agent: CherryPicker Disallow: / User-agent: CherryPicker /1.0 Disallow: / User-agent: CherryPickerElite/1.0 Disallow: / User-agent: CherryPickerSE/1.0 Disallow: / User-agent: ChinaClaw Disallow: / User-agent: ChristCrawler.com Disallow: / User-agent: churl Disallow: / User-agent: cIeNcIaFiCcIoN.nEt Disallow: / User-agent: Clickgarden Disallow: / User-agent: cmc Disallow: / User-agent: CMC/0.01 Disallow: / User-agent: Collective Disallow: / User-agent: Collector Disallow: / User-agent: Combine System Disallow: / User-agent: ComputingSite Robi/1.0 Disallow: / User-agent: ConfuzzledBot Disallow: / User-agent: CoolBot Disallow: / User-agent: Copernic Disallow: / User-agent: CopyRightCheck Disallow: / User-agent: cosmos Disallow: / User-agent: Crawl Disallow: / User-agent: Crescent Disallow: / User-agent: Crescent Internet ToolPak HTTP OLE Control v.1.0 Disallow: / User-agent: CtPhotoBook Disallow: / User-agent: Cusco Disallow: / User-agent: CyberSpyder Link Test Disallow: / User-agent: CydralSpider Disallow: / User-agent: Cancer Information and Support International; Disallow: / User-agent: Carnegie_Mellon_University_WebCrawler,http://www.andrew.cmu.edu/~brgordon/webbot/index.html Disallow: / User-agent: Carnegie_Mellon_University_Research_WebBOT-->PLEASE READ-->http://www.andrew.cmu.edu/~brgordon/webbot/index.html Disallow: / User-agent: Ceramic Tile Installation Guide (http://www.floorstransformed.com) Disallow: / User-agent: cg-eye interactive Disallow: / User-agent: Charon/1.x (Amiga) Disallow: / User-agent: Checkbot/1.xx LWP/5.xx Disallow: / User-agent: CheckLinks/1.x.x Disallow: / User-agent: CheckUrl Disallow: / User-agent: CheckWeb Disallow: / User-agent: ChristCRAWLER 2.0 Disallow: / User-agent: CipinetBot (http://www.cipinet.com/bot.html) Disallow: / User-agent: CJ Spider/ Disallow: / User-agent: ClariaBot/1.0 Disallow: / User-agent: Claymont.com Disallow: / User-agent: Clushbot/2.1 (+http://www.clush.com/bot.html) Disallow: / User-agent: Clushbot/3.3-BinaryFury (+http://www.clush.com/bot.html) Disallow: / User-agent: ColdFusion Disallow: / User-agent: ColdFusion (BookmarkTracker.com) Disallow: / User-agent: collage.cgi/1.82 Disallow: / User-agent: combine/0.0 Disallow: / User-agent: Combine/x.0 Disallow: / User-agent: Commerce Browser Center Disallow: / User-agent: Contact Disallow: / User-agent: contype Disallow: / User-agent: ConveraCrawler/0.2 Disallow: / User-agent: Convera Internet Spider V6.x Disallow: / User-agent: ConveraMultiMediaCrawler/0.1 (+http://www.authoritativeweb.com/crawl) Disallow: / User-agent: cosmos/0.8_(robot@xyleme.com) Disallow: / User-agent: cosmos/0.9_(robot@xyleme.com) Disallow: / User-agent: Cowbot-0.1 (NHN Corp. / +82-2-3011-1954 / nhnbot@naver.com) Disallow: / User-agent: Cowbot-0.1.x (NHN Corp. / +82-2-3011-1954 / nhnbot@naver.com) Disallow: / User-agent: CrawlConvera0.1 (CrawlConvera@yahoo.com) Disallow: / User-agent: Crawler Disallow: / User-agent: Crawler (cometsearch@cometsystems.com) Disallow: / User-agent: Crawler admin@crawler.de Disallow: / User-agent: Crawler V 0.2.x admin@crawler.de Disallow: / User-agent: crawler@alexa.com Disallow: / User-agent: CrawlerBoy Pinpoint.com Disallow: / User-agent: CreativeCommons/0.06-dev (Nutch; http://www.nutch.org/docs/en/bot.html; nutch-agent@lists.sourceforge.net) Disallow: / User-agent: CrocCrawler vx.3 [en] (http://www.croccrawler.com) (X11; I; Linux 2.0.44 i686) Disallow: / User-agent: Cuam Ver0.050bx Disallow: / User-agent: Cuasarbot/0.9b http://www.cuasar.com/spider_beta/ Disallow: / User-agent: curl/7.7.1 (i386--freebsd4.3) libcurl 7.7.1 (SSL 0.9.6) (ipv6 enabled) Disallow: / User-agent: curl/7.8 (i686-pc-linux-gnu) libcurl 7.8 (OpenSSL 0.9.6) Disallow: / User-agent: curl/7.9.x (win32) libcurl 7.9.x Disallow: / User-agent: CurryGuide SiteScan 1.1 Disallow: / User-agent: Custo x.x (www.netwu.com) Disallow: / User-agent: Custom Spider www.bisnisseek.com /1.0 Disallow: / User-agent: Cyberdog/2.0 (Macintosh; 68k) Disallow: / User-agent: CyberSpyder Link Test/2.1.12 (admin@mspennyworth.com) Disallow: / User-agent: CydralSpider/1.x (Cydral Web Image Search; http://www.cydral.com) Disallow: / # D User-agent: DA Disallow: / User-agent: Desert Realm Spider Disallow: / User-agent: DeWeb(c) Katalog/Index Disallow: / User-agent: Die Blinde Kuh Disallow: / User-agent: DienstSpider Disallow: / User-agent: Digger Disallow: / User-agent: Digimarc MarcSpider Disallow: / User-agent: Digimarc Marcspider/CGI Disallow: / User-agent: Digital Integrity Robot Disallow: / User-agent: DIIbot Disallow: / User-agent: Direct Hit Grabber Disallow: / User-agent: DISCoFinder Disallow: / User-agent: DNAbot Disallow: / User-agent: DownLoad Express Disallow: / User-agent: Downloader Disallow: / User-agent: Downloadit Disallow: / User-agent: DragonBot Disallow: / User-agent: DWCP (Dridus' Web Cataloging Project) Disallow: / User-agent: DA 3.5 (www.lidan.com) Disallow: / User-agent: DA 4.0 Disallow: / User-agent: DA 4.0 (www.downloadaccelerator.com) Disallow: / User-agent: DA 5.0 Disallow: / User-agent: DA 7.0 Disallow: / User-agent: Dart Communications PowerTCP Disallow: / User-agent: DataparkSearch/4.xx (http://www.dataparksearch.org/) Disallow: / User-agent: DaviesBot/1.7 (www.wholeweb.net) Disallow: / User-agent: dbDig(http://www.prairielandconsulting.com) Disallow: / User-agent: DBrowse 1.4b Disallow: / User-agent: DBrowse 1.4d Disallow: / User-agent: DC-Sakura/x.xx Disallow: / User-agent: dCSbot/1.1 Disallow: / User-agent: DDD Disallow: / User-agent: dds explorer v1.0 beta Disallow: / User-agent: DeadLinkCheck/0.4.0 libwww-perl/5.xx Disallow: / User-agent: deepak-USC/ISI Disallow: / User-agent: DeepIndex Disallow: / User-agent: DeepIndex ( http://www.zetbot.com ) Disallow: / User-agent: DeepIndex (www.en.deepindex.com) Disallow: / User-agent: DeepIndexer.ca Disallow: / User-agent: Demo Bot DOT 16b Disallow: / User-agent: Demo Bot Z 16b Disallow: / User-agent: Denmex websearch (http://search.denmex.com) Disallow: / User-agent: DevComponents.com HtmlDocument Object Disallow: / User-agent: DiaGem/1.1 (http://www.skyrocket.gr.jp/diagem.html) Disallow: / User-agent: Diamond/1.0 Disallow: / User-agent: DiamondBot Disallow: / User-agent: Digger/1.0 JDK/1.3.0rc3 Disallow: / User-agent: DigOut4U Disallow: / User-agent: Dillo/0.x.x Disallow: / User-agent: DittoSpyder Disallow: / User-agent: DIIbot/1.2 Disallow: / User-agent: dlman Disallow: / User-agent: dloader(NaverRobot)/1.0 Disallow: / User-agent: DoCoMo/1.0/Nxxxi/c10 Disallow: / User-agent: DoCoMo/1.0/Nxxxi/c10/TB Disallow: / User-agent: DoCoMo/1.0/P502i/c10 (Google CHTML Proxy/1.0) Disallow: / User-agent: DoCoMo/2.0 P900iV(c100;TB;W24H11) Disallow: / User-agent: DocZilla/1.0 (Windows; U; WinNT4.0; en-US; rv:1.0.0) Gecko/20020804 Disallow: / User-agent: Download Demon/3.x.x.x Disallow: / User-agent: Download Express 1.0 Disallow: / User-agent: Download Master Disallow: / User-agent: Download Wonder Disallow: / User-agent: Download-Tipp Linkcheck (http://download-tipp.de/) Disallow: / User-agent: Download.exe(1.1) (+http://www.sql-und-xml.de/freeware-tools/) Disallow: / User-agent: Drecombot/1.0 (http://career.drecom.jp/bot.html) Disallow: / User-agent: DSurf15a 01 Disallow: / User-agent: DSurf15a 71 Disallow: / User-agent: DSurf15a 81 Disallow: / User-agent: DSurf15a VA Disallow: / User-agent: dtSearchSpider Disallow: / User-agent: Dual Proxy Disallow: / User-agent: Dumbot(version 0.1 beta) Disallow: / User-agent: Dumbot(version 0.1 beta - dumbfind.com) Disallow: / # E User-agent: EbiNess Disallow: / User-agent: eCatch Disallow: / User-agent: e-collector Disallow: / User-agent: Educate Disallow: / User-agent: efp@gmx\.net Disallow: / User-agent: EirGrabber Disallow: / User-agent: EIT Link Verifier Robot Disallow: / User-agent: ELFINBOT Disallow: / User-agent: Emacs-w3 Search Engine Disallow: / User-agent: EmailCollector Disallow: / User-agent: EmailWolf Disallow: / User-agent: esculapio Disallow: / User-agent: Esther Disallow: / User-agent: Evliya Celebi Disallow: / User-agent: EyeNetIE Disallow: / User-agent: e-sense 1.0 ea(www.vigiltech.com/esensedisclaim.html) Disallow: / User-agent: EARTHCOM.info/1.x [www.earthcom.info] Disallow: / User-agent: EARTHCOM.info/1.xbeta [www.earthcom.info] Disallow: / User-agent: EasyDL/3.xx Disallow: / User-agent: EasyDL/3.xx http://keywen.com/Encyclopedia/Bot Disallow: / User-agent: EBrowse 1.4b Disallow: / User-agent: eCatch/3.0 Disallow: / User-agent: Educate Search VxB Disallow: / User-agent: EchO!/2.0 Disallow: / User-agent: egothor/3.0a (+http://www.xdefine.org/robot.html) Disallow: / User-agent: EgotoBot/4.8 (+http://www.egoto.com/about.htm) Disallow: / User-agent: EldoS TimelyWeb/3.x Disallow: / User-agent: elfbot/1.0 (+http://www.uchoose.de/crawler/elfbot/) Disallow: / User-agent: ELinks (0.x.x; Linux 2.4.20 i586; 132x60) Disallow: / User-agent: ELinks/0.x.x (textmode; NetBSD 1.6.2 sparc; 132x43) Disallow: / User-agent: EmailSiphon Disallow: / User-agent: EmailWolf 1.00 Disallow: / User-agent: EmeraldShield.com WebBot Disallow: / User-agent: EmeraldShield.com WebBot (http://www.emeraldshield.com/webbot.aspx) Disallow: / User-agent: Enfish Tracker Disallow: / User-agent: Enterprise_Search/1.0 Disallow: / User-agent: Enterprise_Search/1.0.xxx Disallow: / User-agent: Enterprise_Search/1.00.xxx;MSSQL (http://www.innerprise.net/es-spider.asp) Disallow: / User-agent: EroCrawler Disallow: / User-agent: ES.NET_Crawler/2.0 (http://search.innerprise.net/) Disallow: / User-agent: eseek-larbin_2.6.2 (crawler@exactseek.com) Disallow: / User-agent: ESISmartSpider Disallow: / User-agent: eStyleSearch 4 (compatible; MSIE 6.0; Windows NT 5.0) Disallow: / User-agent: ESurf15a 15 Disallow: / User-agent: Exalead NG/MimeLive Client (convert/http/0.120) Disallow: / User-agent: ExactSeek Crawler/0.1 Disallow: / User-agent: exactseek-crawler-2.63 (crawler@exactseek.com) Disallow: / User-agent: Excalibur Internet Spider V6.5.4 Disallow: / User-agent: ExperimentalHenrytheMiragoRobot Disallow: / User-agent: Expired Domain Sleuth Disallow: / User-agent: Express WebPictures (www.express-soft.com) Disallow: / User-agent: ExtractorPro Disallow: / User-agent: EyeCatcher (Download-tipp.de)/1.0 Disallow: / # F User-agent: FastCrawler Disallow: / User-agent: Felix IDE Disallow: / User-agent: FetchRover Disallow: / User-agent: fido Disallow: / User-agent: Fish search Disallow: / User-agent: Flaming AttackBot Disallow: / User-agent: Flashget Disallow: / User-agent: FlickBot Disallow: / User-agent: Fluid Dynamics Search Engine robot Disallow: / User-agent: Foobot Disallow: / User-agent: Fouineur Disallow: / User-agent: Franklin Disallow: / User-agent: Freecrawl Disallow: / User-agent: Full Disallow: / User-agent: FunnelWeb Disallow: / User-agent: FairAd Client Disallow: / User-agent: FastBug http://www.ay-up.com Disallow: / User-agent: Fast Crawler Gold Edition Disallow: / User-agent: FAST Enterprise Crawler 6 (Experimental) Disallow: / User-agent: FAST Enterprise Crawler 6 used by Cobra Development (admin@fastsearch.com) Disallow: / User-agent: FAST Enterprise Crawler 6 used by FAST (FAST) Disallow: / User-agent: FAST Enterprise Crawler/6 (www.fastsearch.com) Disallow: / User-agent: FAST FirstPage retriever (compatible; MSIE 5.5; Mozilla/4.0) Disallow: / User-agent: Fast PartnerSite Crawler Disallow: / User-agent: FastCrawler 3.0.1 (crawler@1klik.dk) Disallow: / User-agent: FAST-WebCrawler/2.2.6 (crawler@fast.no; http://www.fast.no/faq/faqfastwebsearch/faqfastwebcrawler.html) Disallow: / User-agent: FAST-WebCrawler/2.2.7 (crawler@fast.no; http://www.fast.no/faq/faqfastwebsearch/faqfastwebcrawler.html) Disallow: / User-agent: FAST-WebCrawler/2.2.8 (crawler@fast.no; http://www.fast.no/faq/faqfastwebsearch/faqfastwebcrawler.html) Disallow: / User-agent: FAST-WebCrawler/2.2.10 (Multimedia Search) (crawler@fast.no; http://www.fast.no/faq/faqfastwebsearch/faqfastwebcrawler.html) Disallow: / User-agent: FAST-WebCrawler/3.2 test Disallow: / User-agent: FAST-WebCrawler/3.3 (crawler@fast.no; http://fast.no/support.php?c=faqs/crawler) Disallow: / User-agent: FAST-WebCrawler/3.4/Nirvana (crawler@fast.no; http://fast.no/support.php?c=faqs/crawler) Disallow: / User-agent: FAST-WebCrawler/3.4/PartnerSite (crawler@fast.no; http://fast.no/support.php?c=faqs/crawler) Disallow: / User-agent: FAST-WebCrawler/3.5 (atw-crawler at fast dot no; http://fast.no/support.php?c=faqs/crawler) Disallow: / User-agent: FAST-WebCrawler/3.6 (atw-crawler at fast dot no; http://fast.no/support/crawler.asp) Disallow: / User-agent: FAST-WebCrawler/3.7 (atw-crawler at fast dot no; http://fast.no/support/crawler.asp) Disallow: / User-agent: FAST-WebCrawler/3.8 (atw-crawler at fast dot no; http://fast.no/support/crawler.asp) Disallow: / User-agent: FAST-WebCrawler/3.8/Fresh (atw-crawler at fast dot no; http://fast.no/support/crawler.asp) Disallow: / User-agent: FAST-WebCrawler/3.6/FirstPage (crawler@fast.no; http://fast.no/support.php?c=faqs/crawler) Disallow: / User-agent: FAST-WebCrawler/3.7/FirstPage (atw-crawler at fast dot no;http://fast.no/support/crawler.asp) Disallow: / User-agent: FAST-WebCrawler/3.x Multimedia Disallow: / User-agent: FAST-WebCrawler/3.x Multimedia (mm dash crawler at fast dot no) Disallow: / User-agent: FastSearch Web Crawler for Verizon SuperPages (kevin.watters@fastsearch.com) Disallow: / User-agent: Favorites Sweeper v.2.03 Disallow: / User-agent: Faxobot/1.0 Disallow: / User-agent: Felix - Mixcat Crawler (+http://mixcat.com) Disallow: / User-agent: FFC Trap Door Spider Disallow: / User-agent: Filangy/0.01-beta (Filangy; http://www.nutch.org/docs/en/bot.html; filangy-agent@filangy.com) Disallow: / User-agent: Filangy/1.0x (Filangy; http://www.nutch.org/docs/en/bot.html; filangy-agent@filangy.com) Disallow: / User-agent: FineBot Disallow: / User-agent: FindAnISP.com_ISP_Finder_v99a Disallow: / User-agent: findlinks/0.xxx (+http://wortschatz.uni-leipzig.de/findlinks/) Disallow: / User-agent: Firefly/1.0 Disallow: / User-agent: Firefly/1.0 (compatible; Mozilla 4.0; MSIE 5.5) Disallow: / User-agent: FirstGov.gov Search - POC:firstgov.webmasters@gsa.gov Disallow: / User-agent: firstsbot Disallow: / User-agent: FlickBot 2.0 RPT-HTTPClient/0.3-3 Disallow: / User-agent: flunky Disallow: / User-agent: fly/6.01 libwww/4.0D Disallow: / User-agent: FocusedSampler/1.0 Disallow: / User-agent: Francis/1.0 (francis@neomo.de http://www.neomo.de/) Disallow: / User-agent: Franklin Locator 1.8 Disallow: / User-agent: FreeFind.com-SiteSearchEngine/1.0 (http://freefind.com; spiderinfo@freefind.com) Disallow: / User-agent: FSurf15a 01 Disallow: / User-agent: Full Web Bot 0416B Disallow: / User-agent: Full Web Bot 0516B Disallow: / User-agent: Full Web Bot 2816B Disallow: / User-agent: FyberSpider (+http://www.fybersearch.com/fyberspider.php) Disallow: / # G User-agent: Gaisbot Disallow: / User-agent: gammaSpider, FocusedCrawler Disallow: / User-agent: gazz Disallow: / User-agent: GCreep Disallow: / User-agent: GetRight Disallow: / User-agent: GetRight/4.2 Disallow: / User-agent: GetterroboPlus Puu Disallow: / User-agent: GetURL Disallow: / User-agent: GetWebPage Disallow: / User-agent: Gigabot Disallow: / User-agent: Golem Disallow: / User-agent: Gozilla Disallow: / User-agent: grab Disallow: / User-agent: Grafula Disallow: / User-agent: Grapnel/0.01 Experiment Disallow: / User-agent: Green Disallow: / User-agent: Griffon Disallow: / User-agent: Gromit Disallow: / User-agent: grub Disallow: / User-agent: Grub.org Disallow: / User-agent: Gulper Bot Disallow: / User-agent: Gagglebot Disallow: / User-agent: GAIS Robot/1.0B2 Disallow: / User-agent: Gaisbot/3.0+(indexer@gais.cs.ccu.edu.tw;+http://gais.cs.ccu.edu.tw/robot.php) Disallow: / User-agent: GalaxyBot/1.0 (http://www.galaxy.com/galaxybot.html) Disallow: / User-agent: GammaSpider/1.0 Disallow: / User-agent: Gamespy_Arcade, GameSpyHTTP/1.0 Disallow: / User-agent: gazz/x.x (gazz@nttrd.com) Disallow: / User-agent: geckobot Disallow: / User-agent: generic_crawler/01.0217/ Disallow: / User-agent: GeonaBot 1.x; http://www.geona.com/ Disallow: / User-agent: GetBot Disallow: / User-agent: GetRight/3.3.3 Disallow: / User-agent: GetRight/4.x Disallow: / User-agent: GetRight/4.x[a-e] Disallow: / User-agent: GetRight/4.5xx Disallow: / User-agent: GhostRouteHunter/20021130 (https://www.sixxs.net/tools/grh/; info@sixxs.net) Disallow: / User-agent: gigabaz/3.14 (baz@gigabaz.com; http://gigabaz.com/gigabaz/) Disallow: / User-agent: Gigabot/x.0 Disallow: / User-agent: Gigabot/2.0 (gigablast.com) Disallow: / User-agent: GigabotSiteSearch/2.0 (sitesearch.gigablast.com) Disallow: / User-agent: GNODSPIDER (www.gnod.net) Disallow: / User-agent: Go!Zilla 3.x (www.gozilla.com) Disallow: / User-agent: Go!Zilla/4.x.x.xx Disallow: / User-agent: Go-Ahead-Got-It/1.1 Disallow: / User-agent: Goblin/0.9 (http://www.goguides.org/) Disallow: / User-agent: Goblin/0.9.x (http://www.goguides.org/goblin-info.html) Disallow: / User-agent: GoForIt.com Disallow: / User-agent: Goofer/0.2 Disallow: / User-agent: GrigorBot 0.8 (http://www.grigor.biz/bot.html) Disallow: / User-agent: grub-client Disallow: / User-agent: grub crawler(http://www.grub.org) Disallow: / User-agent: gsa-crawler (Enterprise; GID-01422; jplastiras@google.com) Disallow: / User-agent: gsa-crawler (Enterprise; GID-01742;gsatesting@rediffmail.com) Disallow: / User-agent: Gulliver/1.3 Disallow: / User-agent: Gulper Web Bot 0.2.4 (www.ecsl.cs.sunysb.edu/~maxim/cgi-bin/Link/GulperBot) Disallow: / # H User-agent: HamBot Disallow: / User-agent: Harvest Disallow: / User-agent: Harvest/1.5 Disallow: / User-agent: havIndex Disallow: / User-agent: HI (HTML Index) Search Disallow: / User-agent: HKU WWW Octopus Disallow: / User-agent: HLoad Disallow: / User-agent: hloader Disallow: / User-agent: HMView Disallow: / User-agent: Hometown Spider Pro Disallow: / User-agent: ht://Dig Disallow: / User-agent: html_analyzer Disallow: / User-agent: HTMLgobble Disallow: / User-agent: HttpDownload/2.0 Disallow: / User-agent: httplib Disallow: / User-agent: HTTrack Disallow: / User-agent: humanlinks Disallow: / User-agent: Hyper-Decontextualizer Disallow: / User-agent: Hämähäkki Disallow: / User-agent: Harvest-NG/1.0.2 Disallow: / User-agent: Haste/0.12 (HOME: http://haste.kytoon.com/) Disallow: / User-agent: Hatena Antenna/0.4 (http://a.hatena.ne.jp/help/robot) Disallow: / User-agent: HeinrichderMiragoRobot Disallow: / User-agent: HeinrichderMiragoRobot (http://www.miragorobot.com/scripts/deinfo.asp) Disallow: / User-agent: HenrytheMiragoRobot Disallow: / User-agent: Hippias/0.9 Beta Disallow: / User-agent: HitList Disallow: / User-agent: Hitwise Spider v1.0 http://www.hitwise.com Disallow: / User-agent: HLoader Disallow: / User-agent: holmes/x.x Disallow: / User-agent: HomePageSearch(hpsearch.uni-trier.de) Disallow: / User-agent: Homerbot: www.homerweb.com Disallow: / User-agent: HotJava/1.0.1/JRE1.1.x Disallow: / User-agent: Hotzonu/x.0 Disallow: / User-agent: HooWWWer/2.1.0 (+http://cosco.hiit.fi/search/hoowwwer/ | mailto:crawler-infohiit.fi) Disallow: / User-agent: htdig/3.1.x (root@localhost) Disallow: / User-agent: htdig/3.1.6 (http://computerorgs.com) Disallow: / User-agent: Html Link Validator (www.lithopssoft.com) Disallow: / User-agent: http://Ask.24x.Info/ (http://narres.it/) Disallow: / User-agent: http://www.almaden.ibm.com/cs/crawler Disallow: / User-agent: http://www.almaden.ibm.com/cs/crawler   [wf216] Disallow: / User-agent: HTTPResume v. 1.x Disallow: / # I User-agent: I, Robot Disallow: / User-agent: iajaBot Disallow: / User-agent: IBM_Planetwide Disallow: / User-agent: ICB Disallow: / User-agent: image.kapsi.net Disallow: / User-agent: imagefetch Disallow: / User-agent: Imagelock Disallow: / User-agent: IncyWincy Disallow: / User-agent: Industry Disallow: / User-agent: Indy Disallow: / User-agent: InfoNaviRobot Disallow: / User-agent: Informant Disallow: / User-agent: InfoSeek Robot 1.0 Disallow: / User-agent: Infoseek Sidewinder Disallow: / User-agent: InfoSpiders Disallow: / User-agent: Ingrid Disallow: / User-agent: Inktomi Slurp Disallow: / User-agent: Inspector Web Disallow: / User-agent: IntelliAgent Disallow: / User-agent: InterGET Disallow: / User-agent: Internet Cruiser Robot Disallow: / User-agent: Internet Shinchakubin Disallow: / User-agent: Iron33 Disallow: / User-agent: Iron33/1.0.2 Disallow: / User-agent: iseeker Disallow: / User-agent: Israeli-search Disallow: / User-agent: IU Disallow: / User-agent: ia_archiver Disallow: / User-agent: ia_archiver-web.archive.org Disallow: / User-agent: ia_archiver/1.6 Disallow: / User-agent: IAArchiver-1.0 Disallow: / User-agent: IBrowse/2.2 (AmigaOS 3.5) Disallow: / User-agent: IBrowse/2.2 (Windows 3.1) Disallow: / User-agent: iCab/2.5.2 (Macintosh; I; PPC) Disallow: / User-agent: ICE Browser/5.05 (Java 1.4.0; Windows 2000 5.0 x86) Disallow: / User-agent: ichiro/1.0 (ichiro@nttr.co.jp) Disallow: / User-agent: IconSurf/2.0 favicon finder (see http://iconsurf.com/robot.html) Disallow: / User-agent: IconSurf/2.0 favicon monitor (see http://iconsurf.com/robot.html) Disallow: / User-agent: ICOO Loader v.x.x.x Disallow: / User-agent: ICRA_label_spider/x.0 Disallow: / User-agent: icsbot-0.1 Disallow: / User-agent: ideare - SignSite/1.x Disallow: / User-agent: IEFav172Free Disallow: / User-agent: iGetter/1.x (Macintosh;G;PPC) Disallow: / User-agent: IlTrovatore-Setaccio (+http://www.iltrovatore.it) Disallow: / User-agent: Iltrovatore-Setaccio/0.3-dev (Indexing; http://www.iltrovatore.it/bot.html; info@iltrovatore.it) Disallow: / User-agent: IlTrovatore-Setaccio/1.2 (+http://www.iltrovatore.it/aiuto/faq.html) Disallow: / User-agent: Iltrovatore-Setaccio/1.2 (It-bot; http://www.iltrovatore.it/bot.html; info@iltrovatore.it) Disallow: / User-agent: iltrovatore-setaccio/1.2-dev (spidering; http://www.iltrovatore.it/aiuto/.....) Disallow: / User-agent: IncyWincy(http://www.look.com) Disallow: / User-agent: IncyWincy(http://www.loopimprovements.com/robot.html) Disallow: / User-agent: IncyWincy/2.1(loopimprovements.com/robot.html) Disallow: / User-agent: IncyWincy data gatherer(webmaster@loopimprovements.com,http://www.loopimprovements.com/robot.html) Disallow: / User-agent: IncyWincy page crawler(webmaster@loopimprovements.com,http://www.loopimprovements.com/robot.html) Disallow: / User-agent: IndexTheWeb.com Crawler7 Disallow: / User-agent: Industry Program 1.0.x Disallow: / User-agent: InelaBot/0.2 (+http://inelegant.org/bot) Disallow: / User-agent: Inet library Disallow: / User-agent: InfoFly/1.0 (http://www.versions-project.org/) Disallow: / User-agent: InfoLink/1.x Disallow: / User-agent: INFOMINE/8.0 Adders Disallow: / User-agent: INFOMINE/8.0 RemoteServices Disallow: / User-agent: INFOMINE/8.0 VLCrawler (http://infomine.ucr.edu/useragents) Disallow: / User-agent: InfoNaviRobot(F107) Disallow: / User-agent: InfoSeek Sidewinder/0.9 Disallow: / User-agent: InfoSeek Sidewinder/1.0A Disallow: / User-agent: InfoSeek Sidewinder/1.1A Disallow: / User-agent: Infoseek SideWinder/1.45 (Compatible; MSIE 10.0; UNIX) Disallow: / User-agent: Infoseek SideWinder/2.0B (Linux 2.4 i686) Disallow: / User-agent: Inktomi Search Disallow: / User-agent: InnerpriseBot/1.0 (http://www.innerprise.com/) Disallow: / User-agent: InstallShield DigitalWizard Disallow: / User-agent: InternetLinkAgent/3.1 Disallow: / User-agent: InternetSeer.com Disallow: / User-agent: Internet Ninja 6.0 Disallow: / User-agent: intraVnews/1.x Disallow: / User-agent: IP*Works! V5 HTTP/S Component - by /n software - www.nsoftware.com Disallow: / User-agent: IPiumBot laurion(dot)com Disallow: / User-agent: Iria/1.xxa Disallow: / User-agent: IRLbot/1.0 (+http://irl.cs.tamu.edu/crawler) Disallow: / User-agent: Irvine/1.x.x Disallow: / User-agent: isurf (tszhu@canada.com) Disallow: / User-agent: IUPUI Research Bot v 1.9a Disallow: / # J User-agent: Java Disallow: / User-agent: JavaBee Disallow: / User-agent: JBot Java Web Robot Disallow: / User-agent: JCrawler Disallow: / User-agent: JennyBot Disallow: / User-agent: JoBo Java Web Robot Disallow: / User-agent: Jobot Disallow: / User-agent: JoeBot Disallow: / User-agent: JumpStation Disallow: / User-agent: J-PHONE/3.0/J-SH07 Disallow: / User-agent: Jabot/7.x.x (http://odin.ingrid.org/) Disallow: / User-agent: Jabot/6.x (http://odin.ingrid.org/) Disallow: / User-agent: Jack Disallow: / User-agent: Jakarta Commons-HttpClient/2.0xxx Disallow: / User-agent: Java/1.4.1_01 Disallow: / User-agent: Java1.0.21.0 Disallow: / User-agent: Java 1.1 Disallow: / User-agent: Java1.1.xx.x Disallow: / User-agent: Java1.3.x Disallow: / User-agent: Java1.3.0rc1 Disallow: / User-agent: Java1.4.0 Disallow: / User-agent: JBH Agent 2.0 Disallow: / User-agent: jBrowser/J2ME Profile/MIDP-1.0 Configuration/CLDC-1.0 (Google WAP Proxy/1.0) Disallow: / User-agent: JDK/1.1 Disallow: / User-agent: JCheckLinks/0.1 RPT-HTTPClient/0.3-1 Disallow: / User-agent: Jeode/1.x.x Disallow: / User-agent: JetBrains Omea Reader 1.0.x (http://www.jetbrains.com/omea_reader/) Disallow: / User-agent: JetCar Disallow: / User-agent: Jetbot/1.0 Disallow: / User-agent: Jigsaw/2.2.0 W3C_CSS_Validator_JFouffa/2.0 Disallow: / User-agent: JoBo/@JOBO_VERSION@(http://www.matuschek.net/jobo.html) Disallow: / User-agent: JoBo/1.x (http://www.matuschek.net/jobo.html) Disallow: / User-agent: JOC Web Spider Disallow: / User-agent: JRTS Check Favorites Utility Disallow: / User-agent: JRTwine Software Check Favorites Utility Disallow: / User-agent: Jyxobot/x Disallow: / # K User-agent: Katipo Disallow: / User-agent: KDD-Explorer Disallow: / User-agent: Keyword Density/0.9 Disallow: / User-agent: Kilroy Disallow: / User-agent: KIT-Fireball Disallow: / User-agent: KO_Yappo_Robot Disallow: / User-agent: K-Meleon/0.6 (Windows; U; Windows NT 5.1; en-US; rv:0.9.5) Gecko/20011011 Disallow: / User-agent: k2spider Disallow: / User-agent: Kapere (http://www.kapere.com) Disallow: / User-agent: KDDI-SN22 UP.Browser/6.0.7 (GUI) MMP/1.1 (Google WAP Proxy/1.0) Disallow: / User-agent: KE_1.0/2.0 libwww/5.2.8 Disallow: / User-agent: Kenjin Spider Disallow: / User-agent: Kevin http://dznet.com/kevin/ Disallow: / User-agent: Kevin http://websitealert.net/kevin/ Disallow: / User-agent: KIT-Fireball/2.0 Disallow: / User-agent: KIT-Fireball/2.0 (compatible; Mozilla 4.0; MSIE 5.5) Disallow: / User-agent: Klondike/1.50 (WSP Win32) (Google WAP Proxy/1.0) Disallow: / User-agent: Knowledge.com/0.x Disallow: / User-agent: Kontiki Client x.xx Disallow: / User-agent: kulokobot www.kuloko.com kuloko@backweave.com Disallow: / User-agent: kuloko-bot/0.x Disallow: / User-agent: kulturarw3/0.1 Disallow: / # L User-agent: LabelGrabber Disallow: / User-agent: larbin Disallow: / User-agent: LeechGet Disallow: / User-agent: legs Disallow: / User-agent: LexiBot Disallow: / User-agent: libfetch Disallow: / User-agent: libWeb/clsHTTP Disallow: / User-agent: Lincoln Disallow: / User-agent: Link Validator Disallow: / User-agent: LinkextractorPro Disallow: / User-agent: LINKS ARoMATIZED Disallow: / User-agent: LinkScan Disallow: / User-agent: LinkScan/8.1a Unix Disallow: / User-agent: Lockon Disallow: / User-agent: logo.gif Crawler Disallow: / User-agent: lwp Disallow: / User-agent: LWP::Simple Disallow: / User-agent: lwp-trivial Disallow: / User-agent: Lachesis Disallow: / # NOTE larbin er googlebots User-agent: LARBIN-EXPERIMENTAL (efp@gmx.net) Disallow: / User-agent: larbin_2.1.1 larbin2.1.1@somewhere.com Disallow: / User-agent: larbin_2.2.0 (crawl@compete.com) Disallow: / User-agent: larbin_2.2.1_de_Viennot (Laurent.Viennot@inria.fr) Disallow: / User-agent: larbin_2.2.2_guillaume (guillaume@liafa.jussieu.fr) Disallow: / User-agent: larbin_2.2.2 (sugayama@lab7.kuis.kyoto-u.ac.jp) Disallow: / User-agent: larbin_2.6_basileocaml (basile.starynkevitch@cea.fr) Disallow: / User-agent: larbin_2.6.0 (larbin2.6.0@unspecified.mail) Disallow: / User-agent: larbin_2.6.1 (larbin2.6.1@unspecified.mail) Disallow: / User-agent: larbin_2.6.2 (hamasaki@grad.nii.ac.jp) Disallow: / User-agent: larbin_2.6.2 (larbin2.6.2@unspecified.mail) Disallow: / User-agent: larbin_2.6.2 (listonATccDOTgatechDOTedu) Disallow: / User-agent: larbin_2.6.2 (pimenas@systems.tuc.gr) Disallow: / User-agent: larbin_2.6.2 (tom@lemurconsulting.com) Disallow: / User-agent: larbin_2.6.2 (vitalbox1@hotmail.com) Disallow: / User-agent: larbin_2.6.3 (wgao@genieknows.com) Disallow: / User-agent: larbin_2.6.3_for_(http://cosco.hiit.fi/search/) tsilande@hiit.fi Disallow: / User-agent: larbin_devel (http://pauillac.inria.fr/~ailleret/prog/larbin/) Disallow: / User-agent: larbin (samualt9@bigfoot.com) Disallow: / User-agent: lcabotAccept: */* Disallow: / User-agent: LECodeChecker/3.0 libgetdoc/1.0 Disallow: / User-agent: LeechGet 200x (www.leechget.de) Disallow: / User-agent: LEIA/2.90 Disallow: / User-agent: LEIA/3.01pr (LEIAcrawler; [SNIP]) Disallow: / User-agent: LexiBot/1.00 Disallow: / User-agent: LG/U8138/v1.0 Disallow: / User-agent: Libby_1.1/libwww-perl/5.47 Disallow: / User-agent: libWeb/clsHTTP -- hiongun@kt.co.kr Disallow: / User-agent: libwww/5.3.2 Disallow: / User-agent: libwww-perl/5.41 Disallow: / User-agent: libwww-perl/5.45 Disallow: / User-agent: libwww-perl/5.48 Disallow: / User-agent: libwww-perl/5.50 Disallow: / User-agent: libwww-perl/5.52 FP/2.1 Disallow: / User-agent: libwww-perl/5.52 FP/4.0 Disallow: / User-agent: libwww-perl/5.53 Disallow: / User-agent: libwww-perl/5.63 Disallow: / User-agent: libwww-perl/5.64 Disallow: / User-agent: libwww-perl/5.65 Disallow: / User-agent: LightningDownload/1.x.x Disallow: / User-agent: LightningDownload/1.0beta2 Disallow: / User-agent: LightningDownload/1.x.x [Accelerated x] Disallow: / User-agent: Lincoln State Web Browser Disallow: / User-agent: LinkAlarm/2.x Disallow: / User-agent: Link Valet Online 1.x Disallow: / User-agent: linkbot Disallow: / User-agent: Linkbot Disallow: / User-agent: Linkbot x.0 Disallow: / User-agent: LinkCheck (linkcheck@inter7.com http://www.inter7.com/linkcheck) Disallow: / User-agent: LinkLint-checkonly/2.x.x Disallow: / User-agent: LinkLint-spider/2.x.x Disallow: / User-agent: LinkProver 2.1 Disallow: / User-agent: Links (0.9x; Linux 2.4.7-10 i686) Disallow: / User-agent: Links (0.9xpre12; Linux 2.2.14-5.0 i686; 80x24) Disallow: / User-agent: Links (2.xpre7; Linux 2.4.18 i586; x) Disallow: / User-agent: Links - http://gossamer-threads.com/scripts/links/ Disallow: / User-agent: Links 2.0 (http://gossamer-threads.com/scripts/links/) Disallow: / User-agent: Links SQL (http://gossamer-threads.com/scripts/links-sql/) Disallow: / User-agent: LinkScan/x.x Unix Disallow: / User-agent: LinkScan/9.0g Unix Disallow: / User-agent: LinkScan/11.0beta2 Unix Disallow: / User-agent: LinksManager.com (http://linksmanager.com/linkchecker.html) Disallow: / User-agent: LinkSweeper/1.x Disallow: / User-agent: LinkWalker Disallow: / User-agent: linknzbot Disallow: / User-agent: ListBidBot (freelance job spider http://listbid.com)Freelance Disallow: / User-agent: Llaut/1.0 (http://mnm.uib.es/~gallir/llaut/bot.html) Disallow: / User-agent: lmspider (lmspider@scansoft.com) Disallow: / User-agent: LNSpiderguy Disallow: / User-agent: Lockstep Spider/1.0 Disallow: / User-agent: Look.com Disallow: / User-agent: Lotus-Notes/4.5 ( Windows-NT ) Disallow: / User-agent: LotusDiscovery/x.0 (compatible; Mozilla 4.0; MSIE 4.01; Windows NT) Disallow: / User-agent: LWP::Simple/5.22 Disallow: / User-agent: LWP::Simple/5.36 Disallow: / User-agent: LWP::Simple/5.48 Disallow: / User-agent: LWP::Simple/5.50 Disallow: / User-agent: LWP::Simple/5.51 Disallow: / User-agent: LWP::Simple/5.53 Disallow: / User-agent: LWP::Simple/5.63 Disallow: / User-agent: lwp-trivial/1.32 Disallow: / User-agent: lwp-trivial/1.34 Disallow: / User-agent: lwp-trivial/1.35 Disallow: / User-agent: Lycos Disallow: / User-agent: T-Rex (lycos) Disallow: / User-agent: Lycos_Spider_(T-Rex) Disallow: / User-agent: Lycos_Spider_(modspider) Disallow: / User-agent: Lynx/2.6 libwww-FM/2.14 Disallow: / User-agent: Lynx/2.8.3dev.9 libwww-FM/2.14 SSL-MM/1.4.1 OpenSSL/0.9.6 Disallow: / User-agent: Lynx/2.8.4rel.1 libwww-FM/2.14 SSL-MM/1.4.1 OpenSSL/0.9.6c (human-guided@lerly.net) Disallow: / # M User-agent: MMCrawler Disallow: / User-agent: Mac Finder Disallow: / User-agent: Mac WWWWorm Disallow: / User-agent: Magpie Disallow: / User-agent: marvin/infoseek Disallow: / User-agent: Mass Disallow: / User-agent: Mass Downloader Disallow: / User-agent: Mata Hari Disallow: / User-agent: Mattie Disallow: / User-agent: MD Disallow: / User-agent: MediaFox Disallow: / User-agent: Mega Moose Disallow: / User-agent: MemoWeb Disallow: / User-agent: MerzScope Disallow: / User-agent: MFC_Tear Disallow: / User-agent: Microsoft URL Control Disallow: / User-agent: Microsoft URL Control - 5.01.4511 Disallow: / User-agent: Microsoft URL Control - 6.00.8169 Disallow: / User-agent: Microsoft.URL Disallow: / User-agent: MIIxpc Disallow: / User-agent: MindCrawler Disallow: / User-agent: Mirago Disallow: / User-agent: mirror Disallow: / User-agent: Missauga Disallow: / User-agent: Missigua Disallow: / User-agent: Mister PiX Disallow: / User-agent: Miz Disallow: / User-agent: MM3 Disallow: / User-agent: MnoGo Disallow: / User-agent: mnoGoSearch search engine software Disallow: / User-agent: moget Disallow: / User-agent: moget/2.1 Disallow: / User-agent: MOMspider Disallow: / User-agent: Monster Disallow: / User-agent: Mooz Disallow: / User-agent: Motor Disallow: / User-agent: Mozilla/[0-9]\.[0-9]+ \(compatible; Win32; WinHttp\.WinHttpRequest\.[0-9]+\) Disallow: / User-agent: Mozilla/3\.0 \(compatible\) Disallow: / User-agent: mozilla/4 Disallow: / User-agent: Mozilla/4.0 (compatible; BullsEye; Windows 95) Disallow: / User-agent: Mozilla/4.0 (compatible; MSIE 4.0; Windows 2000) Disallow: / User-agent: Mozilla/4.0 (compatible; MSIE 4.0; Windows 95) Disallow: / User-agent: Mozilla/4.0 (compatible; MSIE 4.0; Windows 98) Disallow: / User-agent: Mozilla/4.0 (compatible; MSIE 4.0; Windows ME) Disallow: / User-agent: Mozilla/4.0 (compatible; MSIE 4.0; Windows NT) Disallow: / User-agent: Mozilla/4.0 (compatible; MSIE 4.0; Windows XP) Disallow: / User-agent: mozilla/5 Disallow: / User-agent: MSIECrawler Disallow: / User-agent: Muncher Disallow: / User-agent: Muninn Disallow: / User-agent: Muscat Ferret Disallow: / User-agent: Mwd.Search Disallow: / User-agent: MyGetRight Disallow: / User-agent: Mac Finder 1.0.xx Disallow: / User-agent: Mackster( http://www.ukwizz.com ) Disallow: / User-agent: Mag-Net Disallow: / User-agent: mailto:webcraft@bea.com Disallow: / User-agent: mammoth/1.0 (+http://www.sli-systems.com/) Disallow: / User-agent: MantraAgent Disallow: / User-agent: Mariner/5.1b [de] (Win95; I ;Kolibri gncwebbot) Disallow: / User-agent: Marketwave Hit List Disallow: / User-agent: Martini Disallow: / User-agent: Marvin v0.3 Disallow: / User-agent: Mass Downloader 2.x Disallow: / User-agent: Mata Hari/2.00 Disallow: / User-agent: maxomobot/dev-20051201 (maxomo; http://67.102.134.34:4047/MAXOMO/MAXOMObot.html; maxomobot@maxomo.com) Disallow: / User-agent: MediaCrawler-1.0 (Experimental) Disallow: / User-agent: MediaSearch/0.1 Disallow: / User-agent: MegaSheep v1.0 (www.searchuk.com internet sheep) Disallow: / User-agent: Mercator-1.x Disallow: / User-agent: Mercator-2.0 Disallow: / User-agent: Mercator-Scrub-1.1 Disallow: / User-agent: MetaGer-LinkChecker Disallow: / User-agent: MetaProducts Download Express/1.x Disallow: / User-agent: Metaspinner/0.01 (Metaspinner; http://www.meta-spinner.de/; support@meta-spinner.de/) Disallow: / User-agent: MFC Foundation Class Library 4.0 Disallow: / User-agent: MFC_Tear_Sample Disallow: / User-agent: MFHttpScan Disallow: / User-agent: Microsoft Data Access Internet Publishing Provider Protocol Discovery Disallow: / User-agent: MicrosoftPrototypeCrawler (How's my crawling? mailto:newbiecrawler@hotmail.com) Disallow: / User-agent: Microsoft URL Control - 6.00.8xxx Disallow: / User-agent: MIIxpc/4.2 Disallow: / User-agent: minibot Disallow: / User-agent: MiracleAlphaTest Disallow: / User-agent: Missauga Locate 1.0.0 Disallow: / User-agent: Missigua Locator 1.9 Disallow: / User-agent: Missouri College Browse Disallow: / User-agent: Mister PiX version.dll Disallow: / User-agent: Mister Pix II 2.02a Disallow: / User-agent: MJ12bot/vx.x.x (http://www.majestic12.co.uk/projects/dsearch/mj12bot.php) Disallow: / User-agent: MnogoSearch/3.2.xx Disallow: / User-agent: Mouse-House/7.4 (spider_monkey spider info at www.mobrien.com/sm.shtml) Disallow: / User-agent: moget/x.x (moget@goo.ne.jp) Disallow: / User-agent: mogimogi/1.0 Disallow: / User-agent: moiNAG 0.02 Disallow: / User-agent: MojeekBot/0.x (archi; http://www.mojeek.com/bot.html) Disallow: / User-agent: Morris - Mixcat Crawler (+http://mixcat.com) Disallow: / User-agent: MovableType/x.x Disallow: / User-agent: mozDex/0.04-dev (mozDex; http://www.mozdex.com/en/bot.html; spider@mozdex.com) Disallow: / User-agent: Mozi! Disallow: / User-agent: Mozilla Disallow: / User-agent: Mozilla (Mozilla@somewhere.com) Disallow: / User-agent: Mozilla/1.1 (compatible; MSPIE 2.0; Windows CE) Disallow: / User-agent: Mozilla/1.10 [en] (Compatible; RISC OS 3.70; Oregano 1.10) Disallow: / User-agent: Mozilla/1.22 (compatible; MSIE 2.0d; Windows NT) Disallow: / User-agent: Mozilla/1.22 (compatible; MSIE 5.01; PalmOS 3.0) EudoraWeb 2 Disallow: / User-agent: Mozilla/2.0 (compatible; AOL 3.0; Mac_PowerPC) Disallow: / User-agent: Mozilla/2.0 (Compatible; AOL-IWENG 3.0; Win16) Disallow: / User-agent: Mozilla/2.0 (compatible; Ask Jeeves) Disallow: / User-agent: Mozilla/2.0 (compatible; Ask Jeeves/Teoma) Disallow: / User-agent: Mozilla/2.0 (compatible; Ask Jeeves/Teoma; +http://sp.ask.com/docs/about/tech_crawling.html) Disallow: / User-agent: Mozilla/2.0 (compatible; EZResult -- Internet Search Engine) Disallow: / User-agent: Mozilla/2.0 (compatible; MS FrontPage x.0) Disallow: / User-agent: Mozilla/2.0 (compatible; MSIE 2.1; Mac_PowerPC) Disallow: / User-agent: Mozilla/2.0 (compatible; MSIE 3.0; AK; Windows 95) Disallow: / User-agent: Mozilla/2.0 (compatible; MSIE 3.0; Windows 3.1) Disallow: / User-agent: Mozilla/2.0 (compatible; MSIE 3.02; Update a; AK; Windows NT) Disallow: / User-agent: Mozilla/2.0 (compatible; MSIE 3.02; Update a; AOL 3.0; Windows 95) Disallow: / User-agent: Mozilla/2.0 (compatible; MSIE 3.0B; Win32) Disallow: / User-agent: Mozilla/2.0 (compatible; NEWT ActiveX; Win32) Disallow: / User-agent: Mozilla/2.0 (compatible; T-H-U-N-D-E-R-S-T-O-N-E) Disallow: / User-agent: Mozilla/2.0 compatible; Check&Get 1.1x (Windows 98) Disallow: / User-agent: Mozilla/2.01 (Win16; I) Disallow: / User-agent: Mozilla/2.02Gold (Win95; I) Disallow: / User-agent: Mozilla/3.0 (compatible) Disallow: / User-agent: Mozilla/3.0 (compatible; AvantGo 3.2) Disallow: / User-agent: Mozilla/3.0 (compatible; Fluffy the spider; http://www.searchhippo.com/; info@searchhippo.com) Disallow: / User-agent: Mozilla/3.0 (compatible; HP Web PrintSmart 04b0 1.0.1.34) Disallow: / User-agent: Mozilla/3.0 (compatible; Indy Library) Disallow: / User-agent: Mozilla/3.0 (compatible; Linkman) Disallow: / User-agent: Mozilla/3.0 (compatible; MuscatFerret/1.5.4; claude@euroferret.com) Disallow: / User-agent: Mozilla/3.0 (compatible; MuscatFerret/1.5; olly@muscat.co.uk) Disallow: / User-agent: Mozilla/3.0 (compatible; MuscatFerret/1.6.x; claude@euroferret.com) Disallow: / User-agent: Mozilla/3.0 (compatible; netart generator/1.0; libwww-perl/5.64) Disallow: / User-agent: Mozilla/3.0 (compatible; NetPositive/2.2) Disallow: / User-agent: Mozilla/3.0 (compatible; Opera/3.0; Windows 3.1) v3.1 Disallow: / User-agent: Mozilla/3.0 (compatible; Opera/3.0; Windows 95/NT4) 3.2 Disallow: / User-agent: Mozilla/3.0 (compatible; PerMan Surfer 3.0; Win95) Disallow: / User-agent: Mozilla/3.0 (compatible; scan4mail (advanced version), http://www.peterspages.net/?scan4mail) Disallow: / User-agent: Mozilla/3.0 (compatible; REL Software, Web Link Validator 2.x) Disallow: / User-agent: Mozilla/3.0 (Compatible;Viking/1.8) Disallow: / User-agent: Mozilla/3.0 (compatible; WebCapture x.x; Auto; Windows) Disallow: / User-agent: Mozilla/3.0 (compatible; Webinator-DEV01.home.iprospect.com/2.56) Disallow: / User-agent: Mozilla/3.0 (compatible; Webinator-indexer.cyberalert.com/2.56) Disallow: / User-agent: Mozilla/3.0 (compatible; Web Link Validator 2.x) Disallow: / User-agent: Mozilla/3.0 (DreamPassport/3.0) Disallow: / User-agent: Mozilla/3.0 (INGRID/3.0 MT; webcrawler@NOSPAMexperimental.net; http://aanmelden.ilse.nl/?aanmeld_mode=webhints) Disallow: / User-agent: Mozilla/3.0 (Liberate DTV 1.1) Disallow: / User-agent: Mozilla/3.0 (Planetweb/2.100 JS SSL US; Dreamcast US) Disallow: / User-agent: Mozilla/3.0 (Slurp/cat; slurp@inktomi.com; http://www.inktomi.com/slurp.html) Disallow: / User-agent: Mozilla/3.0 (Slurp/si; slurp@inktomi.com; http://www.inktomi.com/slurp.html) Disallow: / User-agent: Mozilla/3.0 (Slurp.so/Goo; slurp@inktomi.com; http://www.inktomi.com/slurp.html) Disallow: / User-agent: Mozilla/3.0 (Vagabondo/1.x MT; webagent@wise-guys.nl; http://webagent.wise-guys.nl/) Disallow: / User-agent: Mozilla/3.0 (Vagabondo/1.1 MT; webcrawler@NOSPAMwise-guys.nl; http://webagent.wise-guys.nl/) Disallow: / User-agent: Mozilla/3.0 (Vagabondo/2.0 MT; webcrawler@NOSPAMexperimental.net; http://aanmelden.ilse.nl/?aanmeld_mode=webhints) Disallow: / User-agent: Mozilla/3.0 (Vagabondo/2.0 MT; webcrawler@NOSPAMwise-guys.nl; http://webagent.wise-guys.nl/) Disallow: / User-agent: Mozilla/3.0 (Win16; I) Disallow: / User-agent: Mozilla/3.0 (Win95; I) Disallow: / User-agent: Mozilla/3.0 (WinNT; I) Disallow: / User-agent: Mozilla/3.0 (WorldGate Gazelle 3.5.1 build 11; FreeBSD2.2.8-STABLE) Disallow: / User-agent: Mozilla/3.0 (X11; I; OSF1 V4.0 alpha) Disallow: / User-agent: Mozilla/3.0 NAVIO_AOLTV (11; 13; Philips; PH200; 1; R2.0C36_AOL.0110OPTIK; R2.0.0139d_OPTIK) Disallow: / User-agent: Mozilla/3.0 WebTV/1.2 (compatible; MSIE 2.0) Disallow: / User-agent: Mozilla/3.01 (compatible; AmigaVoyager/2.95; AmigaOS/MC680x0) Disallow: / User-agent: Mozilla/3.01 (Compatible; Links2Go Similarity Engine) Disallow: / User-agent: Mozilla/3.01 (compatible; Netbox/3.5 R92; Linux 2.2) Disallow: / User-agent: Mozilla/3.01-C-MACOS8 (Macintosh; I; PPC) Disallow: / User-agent: Mozilla/3.01Gold (X11; I; Linux 2.0.32 i486) Disallow: / User-agent: Mozilla/3.01Gold (X11; I; SunOS 5.5.1 sun4m) Disallow: / User-agent: Mozilla/3.01SGoldC-SGI (X11; I; IRIX 6.3 IP32) Disallow: / User-agent: Mozilla/3.04 (compatible; ANTFresco/2.13; RISC OS 4.02) Disallow: / User-agent: Mozilla/3.04 (compatible; NCBrowser/2.35; ANTFresco/2.17; RISC OS-NC 5.13 Laz1UK1309) Disallow: / User-agent: Mozilla/3.x (I-Opener 1.1; Netpliance) Disallow: / User-agent: Mozilla/4.0 (agadine3.0) www.agada.de Disallow: / User-agent: Mozilla/4.0 (compatible; Advanced Email Extractor v2.xx) Disallow: / User-agent: Mozilla/4.0 (compatible; Arachmo) Disallow: / User-agent: Mozilla/4.0 (compatible: AstraSpider V.2.1 : astrafind.com) Disallow: / User-agent: Mozilla/4.0 (compatible; B_L_I_T_Z_B_O_T) Disallow: / User-agent: Mozilla/4.0 (compatible; BorderManager 3.0) Disallow: / User-agent: Mozilla/4.0 (compatible; ChristCrawler.com, ChristCrawler@ChristCENTRAL.com) Disallow: / User-agent: Mozilla/4.0 (compatible; DnloadMage 1.0) Disallow: / User-agent: Mozilla/4.0 (compatible; FastCrawler3, support-fastcrawler3@fast.no) Disallow: / User-agent: Mozilla/4.0 (compatible; FDSE robot) Disallow: / User-agent: Mozilla/4.0 (compatible; www.galaxy.com) Disallow: / User-agent: Mozilla/4.0 (compatible; grub-client-0.3.x; Crawl your own stuff with http://grub.org) Disallow: / User-agent: Mozilla/4.0 (compatible; grub-client-0.2.x; Crawl your stuff with http://grub.org) Disallow: / User-agent: Mozilla/4.0 (compatible; grub-client-2.x) Disallow: / User-agent: Mozilla/4.0 (compatible; ICS 1.2.xxx) Disallow: / User-agent: Mozilla/4.0 (compatible; Iplexx Spider/1.0, http://www.iplexx.at) Disallow: / User-agent: Mozilla/4.0 (compatible; Lotus-Notes/5.0; Windows-NT) Disallow: / User-agent: Mozilla/4.0 (compatible; MSIE 4.0; Windows NT; Site Server 3.0 Robot) Indonesia Interactive Disallow: / User-agent: Mozilla/4.0 (compatible; MSIE 4.0; Windows NT; Site Server 3.0 Robot) WebQuest Designs Disallow: / User-agent: Mozilla/4.0 (compatible; MSIE 4.01; AOL 4.0; Windows 98) Disallow: / User-agent: Mozilla/4.0 (compatible; MSIE 4.01; Mac_PowerPC) Disallow: / User-agent: Mozilla/4.0 (compatible; MSIE 4.01; MSIECrawler; Windows 95) Disallow: / User-agent: Mozilla/4.0 (compatible; MSIE 4.01; Windows 95) Disallow: / User-agent: Mozilla/4.0 (compatible; MSIE 4.01; Windows CE; MSN Companion 2.0; 800x600; Compaq) Disallow: / User-agent: Mozilla/4.0 (compatible; MSIE 4.01; Windows NT) Disallow: / User-agent: Mozilla/4.0 (compatible; MSIE 4.01; Windows NT Windows CE) Disallow: / User-agent: Mozilla/4.0 (compatible; MSIE 4.01; Windows NT; MS Search 4.0 Robot) Microsoft Disallow: / User-agent: Mozilla/4.0 (compatible; MSIE 5.0; AOL 5.0; Windows 95; DigExt; Gateway2000; sureseeker.com) Disallow: / User-agent: Mozilla/4.0 (compatible; MSIE 5.0; Mac_PowerPC; AtHome021) Disallow: / User-agent: Mozilla/4.0 (compatible; MSIE 5.0; NetNose-Crawler 2.0; A New Search Experience: http://www.netnose.com) Disallow: / User-agent: Mozilla/4.0 (compatible; MSIE 5.0; Win32) via proxy gateway CERN-HTTPD/3.0 libwww/2.17 Disallow: / User-agent: Mozilla/4.0 (compatible; MSIE 5.0; Windows 95) TrueRobot; 1.5 Disallow: / User-agent: Mozilla/4.0 (compatible; MSIE 5.0; Windows 95) VoilaBot BETA 1.2 (http://www.voila.com/) Disallow: / User-agent: Mozilla/4.0 (compatible; MSIE 5.0; Windows 95) VoilaBot; 1.6 Disallow: / User-agent: Mozilla/4.0 (compatible; MSIE 5.0; Windows ME) Opera 5.11 [en] Disallow: / User-agent: Mozilla/4.0 (compatible; MSIE 5.0; Windows ME; Link Checker 2.x.xx, http://www.kyosoft.com) Disallow: / User-agent: Mozilla/4.0 (compatible; MSIE 5.0; Windows NT; DigExt; DTS Agent Disallow: / User-agent: Mozilla/4.0 (compatible; MSIE 5.0; Windows NT; Girafabot; girafabot at girafa dot com; http://www.girafa.com) Disallow: / User-agent: Mozilla/4.0 (compatible; MSIE 5.0; www.galaxy.com; www.psychedelix.com) Disallow: / User-agent: Mozilla/4.0 (compatible; MSIE 5.0; www.galaxy.com; www.psychedelix.com/; +http://www.galaxy.com/info/crawler.html) Disallow: / User-agent: Mozilla/4.0 (compatible; MSIE 5.0; YANDEX) Disallow: / User-agent: Mozilla/4.0_(compatible;_MSIE_5.0;_Windows_95)_TrueRobot/1.4 libwww/5.2.8 Disallow: / User-agent: Mozilla/4.0_(compatible;_MSIE_5.0;_Windows_95)_VoilaBot/1.6 libwww/5.3.2 Disallow: / User-agent: Mozilla/4.0 (compatible; MSIE 5.01; Windows 95) via Avirt Gateway Server v4.0 Disallow: / User-agent: Mozilla/4.0 (compatible; MSIE 5.01; Windows NT 5.0; NetCaptor 6.5.0RC1) Disallow: / User-agent: Mozilla/4.0 (compatible; MSIE 5.01; Windows NT 5.0) (samualt9@bigfoot.com) Disallow: / User-agent: Mozilla/4.0 (compatible; MSIE 5.5; AOL 4.0; Windows 98; GoBeez (www.gobeez.com)) Disallow: / User-agent: Mozilla/4.0 (compatible; MSIE 5.5; Windows 95; Transmission Segment; Hotbar 2.0) Disallow: / User-agent: Mozilla/4.0 (compatible; MSIE 5.5; Windows 98; Crazy Browser 1.x.x) Disallow: / User-agent: Mozilla/4.0 (compatible; MSIE 5.5; Windows 98; KITV4.7 Wanadoo) Disallow: / User-agent: Mozilla/4.0 (compatible; MSIE 5.5; Windows 98; SAFEXPLORER TL) Disallow: / User-agent: Mozilla/4.0 (compatible; MSIE 5.5; Windows 98; SYMPA; Katiesoft 7; SimulBrowse 3.0) Disallow: / User-agent: Mozilla/4.0 (compatible; MSIE 5.5; Windows 98; Win 9x 4.90; BTinternet V8.1) Disallow: / User-agent: Mozilla/4.0 (compatible; MSIE 5.5; Windows 98; Win 9x 4.90; MSIECrawler) Disallow: / User-agent: Mozilla/4.0 (compatible; MSIE 5.5; Windows NT 4.0; obot) Disallow: / User-agent: Mozilla/4.0 (compatible; MSIE 5.5; Windows NT 4.0; QXW03018) Disallow: / User-agent: Mozilla/4.0 (compatible; MSIE 5.5; Windows NT 5.0; .NET CLR 1.0.3705) Disallow: / User-agent: Mozilla/4.0 (compatible; MSIE 5.5; Windows NT 5.0) Active Cache Request Disallow: / User-agent: Mozilla/4.0 (compatible; MSIE 5.5; Windows NT 5.0; AIRF) Disallow: / User-agent: Mozilla/4.0 (compatible; MSIE 5.5; Windows NT 5.0; AspTear 1.5) Disallow: / User-agent: Mozilla/4.0 (compatible; MSIE 5.5; Windows NT 5.0) Fetch API Request Disallow: / User-agent: Mozilla/4.0 (compatible; MSIE 5.5; Windows NT 5.0; N_o_k_i_a) Disallow: / User-agent: Mozilla/4.0 (compatible; MSIE 5.5; Windows NT 5.0; T312461) RPT-HTTPClient/0.3-3E Disallow: / User-agent: Mozilla/4.0 (compatible; MSIE 6.0; MSIE 5.5; Windows NT 5.1) Skampy/0.9.x [en] Disallow: / User-agent: Mozilla/4.0 (compatible; MSIE 6.0; Win32) WebWasher 3.0 Disallow: / User-agent: Mozilla/4.0 (compatible; MSIE 6.0; Windows 98; Net M@nager V3.02 - www.vinn.com.au) Disallow: / User-agent: Mozilla/4.0 (compatible; MSIE 6.0; Windows 98) Web Link Validator 2.x) Disallow: / User-agent: Mozilla/4.0 (compatible; MSIE 6.0; Windows 98) REL Software, Web Link Validator 2.x) Disallow: / User-agent: Mozilla/4.0 (compatible; MSIE 6.0; Windows 98; Win 9x 4.90; http://www.Abolimba.de) Disallow: / User-agent: Mozilla/4.0 (compatible; MSIE 6.0; Windows NT; MS Search 4.0 Robot) Disallow: / User-agent: Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1) (dns_admin@c-a-s-h.com) Disallow: / User-agent: Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; Deepnet Explorer) Disallow: / User-agent: Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; heritrix/1.3.0 +http://www.cs.washington.edu/research/networking/websys/) Disallow: / User-agent: Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; Hotbar 3.0) Disallow: / User-agent: Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; iOpus-I-M) Disallow: / User-agent: Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; iRider 2.21.1108; FDM) Disallow: / User-agent: Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; KKman3.0) Disallow: / User-agent: Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; MathPlayer2.0) Disallow: / User-agent: Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; Maxthon) Disallow: / User-agent: Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; PeoplePal 3.0; MSIECrawler) Disallow: / User-agent: Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; Q312461; IOpener Release 1.1.04) Disallow: / User-agent: Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; StumbleUpon.com 1.760; .NET CLR 1.1.4322) Disallow: / User-agent: Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; DX-Browser 5.0.0.0) Disallow: / User-agent: Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET  CLR 1.1.4322) Disallow: / User-agent: Mozilla/4.0 (compatible; Netcraft Web Server Survey) Disallow: / User-agent: Mozilla/4.0 (compatible; Opera/3.0; Windows 4.10) 3.51 [en] Disallow: / User-agent: Mozilla/4.0 (compatible; Powermarks/3.5; Windows 95/98/2000/NT) Disallow: / User-agent: Mozilla/4.0 (compatible; SiteKiosk 4.0; MSIE 5.0; Windows 98; SiteCoach 1.0) Disallow: / User-agent: Mozilla/4.0 (compatible; SpeedySpider; www.entireweb.com) Disallow: / User-agent: Mozilla/4.0 (compatible; SPENG) Disallow: / User-agent: Mozilla/4.0 (compatible; SuperCleaner 2.xx; Windows 98) Disallow: / User-agent: Mozilla/4.0 (compatible; Win32; WinHttp.WinHttpRequest.5) Disallow: / User-agent: Mozilla/4.0 (compatible; www.linkguard.com Linkguard Online 1.0; Windows NT) Disallow: / User-agent: Mozilla/4.0(compatible; Zealbot 1.0) Disallow: / User-agent: Mozilla/4.0 (fantomBrowser) Disallow: / User-agent: Mozilla/4.0 (fantomCrew Browser) Disallow: / User-agent: Mozilla/4.0 (hhjhj@yahoo.com) Disallow: / User-agent: Mozilla/4.0 (JemmaTheTourist;http://www.activtourist.com) Disallow: / User-agent: Mozilla/4.0 (MobilePhone PM-8200/US/1.0) NetFront/3.x MMP/2.0 Disallow: / User-agent: Mozilla/4.0 (MobilePhone SCP-5500/US/1.0) NetFront/3.0 MMP/2.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html) Disallow: / User-agent: Mozilla/4.0 (MobilePhone SCP-5500/US/1.0) NetFront/3.0 MMP/2.0 FAKE (compatible; Googlebot/2.1; +http://www.google.com/bot.html) Disallow: / User-agent: Mozilla/4.0 (Sleek Spider/1.2) Disallow: / User-agent: Mozilla/4.0 [en] (Ask Jeeves Corporate Spider) Disallow: / User-agent: Mozilla/4.0 compatible ZyBorg/1.0 (wn.zyborg@looksmart.net; http://www.WISEnutbot.com) Disallow: / User-agent: Mozilla/4.0 compatible ZyBorg/1.0 (ZyBorg@WISEnutbot.com; http://www.WISEnutbot.com) Disallow: / User-agent: Mozilla/4.0 compatible ZyBorg/1.0 Dead Link Checker (wn.zyborg@looksmart.net; http://www.WISEnutbot.com) Disallow: / User-agent: Mozilla/4.0 compatible ZyBorg/1.0 for Homepage (ZyBorg@WISEnutbot.com; http://www.WISEnutbot.com) Disallow: / User-agent: Mozilla/4.0 efp@gmx.net Disallow: / User-agent: Mozilla/4.0 WebTV/2.6 (compatible; MSIE 4.0) Disallow: / User-agent: Mozilla/4.01 [en](Win95;I) Disallow: / User-agent: Mozilla/4.02 [en] (X11; I; SunOS 5.6 sun4u) Disallow: / User-agent: Mozilla/4.04 [en] (X11; I; HP-UX B.10.20 9000/712) Disallow: / User-agent: Mozilla/4.04 [en] (X11; I; IRIX 5.3 IP22) Disallow: / User-agent: Mozilla/4.05 (Macintosh; I; 68K, Nav) Disallow: / User-agent: Mozilla/4.05 (Macintosh; I; PPC, Nav) Disallow: / User-agent: Mozilla/4.05 [en] (X11; I; SunOS 4.1.4 sun4m) Disallow: / User-agent: Mozilla/4.08 [en] (Win98; U ;Nav)" [Spaces are not mistakes]Help->About: "Version 4.08 [en]-98306" Disallow: / User-agent: Mozilla/4.08 [en] (WinNT; U) Disallow: / User-agent: Mozilla/4.5 (compatible; HTTrack 3.0x; Windows 98) Disallow: / User-agent: Mozilla/4.5 (compatible; iCab 2.5.3; Macintosh; I; PPC) Disallow: / User-agent: Mozilla/4.5 (compatible; OmniWeb/4.0.5; Mac_PowerPC) Disallow: / User-agent: Mozilla/4.5 (compatible; OmniWeb/4.1-beta-1; Mac_PowerPC) Disallow: / User-agent: Mozilla/4.5 [en]C-CCK-MCD {RuralNet} (Win98; I) Disallow: / User-agent: Mozilla/4.5 RPT-HTTPClient/0.3-2 Disallow: / User-agent: Mozilla/4.5b1 [en] (X11; I; Linux 2.0.35 i586) Disallow: / User-agent: Mozilla/4.61 [de] (OS/2; I) Disallow: / User-agent: Mozilla/4.61 [en] (X11; U; ) - BrowseX (2.0.0 Windows) Disallow: / User-agent: Mozilla/4.7 Disallow: / User-agent: Mozilla/4.7 (compatible; http://eidetica.com/spider) Disallow: / User-agent: Mozilla/4.7 (compatible; Intelliseek; http://www.intelliseek.com) Disallow: / User-agent: Mozilla/4.7 (compatible; OffByOne; Windows 98) Webster Pro V3.2 Disallow: / User-agent: Mozilla/4.7 (compatible; Whizbang) Disallow: / User-agent: Mozilla/4.7 (compatible; WhizBang; http://www.whizbang.com/crawler) Disallow: / User-agent: Mozilla/4.7 [en](BecomeBot@exava.com) Disallow: / User-agent: Mozilla/4.7 [en](Exabot@exava.com) Disallow: / User-agent: Mozilla/4.7 [en]C-CCK-MCD {Yahoo;YIP052400} (Win95; I) Disallow: / User-agent: Mozilla/4.72 [en] (BACS http://www.ba.be) Disallow: / User-agent: Mozilla/4.72C-CCK-MCD Caldera Systems OpenLinux [en] (X11; U; Linux 2.2.14 i686) Disallow: / User-agent: Mozilla/4.75C-ja  [ja] (X11; U; OSF1 V5.1 alpha) Disallow: / User-agent: Mozilla/4.76 [en] (X11; U; FreeBSD 4.4-STABLE i386) Disallow: / User-agent: Mozilla/4.76 [en] (X11; U; SunOS 5.7 sun4u) Disallow: / User-agent: Mozilla/4.76 (Windows 98; U) Opera 5.12 [en] Disallow: / User-agent: Mozilla/4.77C-SGI [en] (X11; U; IRIX 6.5 IP32) Disallow: / User-agent: Mozilla/5.0 Disallow: / User-agent: Mozilla/5.0 [en] (compatible; Gulper Web Bot 0.2.4 www.ecsl.cs.sunysb.edu/~maxim/cgi-bin/Link/GulperBot) Disallow: / User-agent: Mozilla/5.0 (compatible; BecomeBot/1.23; +http://www.become.com/webmasters.html) Disallow: / User-agent: Mozilla/5.0 (compatible; BecomeBot/1.68; MSIE 6.0 compatible; +http://www.become.com/webmasters.html) Disallow: / User-agent: Mozilla/5.0 (compatible; BecomeBot/2.0beta; +http://www.become.com/webmasters.html) Disallow: / User-agent: Mozilla/5.0 (Clustered-Search-Bot/1.0; support@clush.com; http://www.clush.com/) Disallow: / User-agent: Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html) Disallow: / User-agent: mozilla/5.0 (compatible; heritrix/1.3.0 +http://archive.crawler.org) Disallow: / User-agent: Mozilla/5.0 (compatible; Konqueror/2.0.1; X11); Supports MD5-Digest; Supports gzip encoding Disallow: / User-agent: Mozilla/5.0 (compatible; Konqueror/2.1.1; X11) Disallow: / User-agent: Mozilla/5.0 (compatible; Konqueror/2.2.2) Disallow: / User-agent: Mozilla/5.0 (compatible; Konqueror/2.2.2; Linux 2.4.14-xfs; X11; i686) Disallow: / User-agent: Mozilla/5.0 (compatible; LemSpider 0.1) Disallow: / User-agent: Mozilla/5.0 (compatible; SpurlBot/0.2) Disallow: / User-agent: Mozilla/5.0 (compatible; SYCLIKControl/LinkChecker;) Disallow: / User-agent: Mozilla/5.0 (compatible; Vagabondo/2.1; webcrawler at wise-guys dot nl; http://webagent.wise-guys.nl/) Disallow: / User-agent: Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp) Disallow: / User-agent: Mozilla/5.0 (Macintosh; U; PPC Mac OS X Mach-O; en-US; rv:1.0.1) Gecko/20030306 Camino/0.7 Disallow: / User-agent: Mozilla/5.0 (Macintosh; U; PPC; en-US; rv:0.9.2) Gecko/20010726 Netscape6/6.1 Disallow: / User-agent: Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en-us) AppleWebKit/xxx.x (KHTML, like Gecko) Safari/12x.x Disallow: / User-agent: Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en-US) AppleWebKit/xx (KHTML, like Gecko) OmniWeb/v5xx.xx Disallow: / User-agent: Mozilla/5.0 (Macintosh; U; PPC Mac OS X Mach-O; en-US; rv:1.0.1) Gecko/20021219 Chimera/0.6+ Disallow: / User-agent: Mozilla/5.0 (research@mediatrec.com) Disallow: / User-agent: Mozilla/5.0 (Slurp/cat; slurp@inktomi.com; http://www.inktomi.com/slurp.html) Disallow: / User-agent: Mozilla/5.0 (Slurp/si; slurp@inktomi.com; http://www.inktomi.com/slurp.html) Disallow: / User-agent: Mozilla/5.0 (SunOS 5.8 sun4u; U) Opera 5.0 [en] Disallow: / User-agent: Mozilla/5.0 (Windows; U; Win98; en-US; rv:0.9.2) Gecko/20010726 Netscape6/6.1 Disallow: / User-agent: Mozilla/5.0 (Windows; U; Win98; en-US; rv:x.xx) Gecko/20030423 Firebird Browser/0.6 Disallow: / User-agent: Mozilla/5.0 (Windows; U; Win9x; en; Stable) Gecko/20020911 Beonex/0.8.1-stable Disallow: / User-agent: Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:x.xx) Gecko/20030504 Mozilla Firebird/0.6 Disallow: / User-agent: Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:x.xxx) Gecko/20041027 Mnenhy/0.6.0.104 Disallow: / User-agent: Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:x.x.x) Gecko/20041107 Firefox/x.x Disallow: / User-agent: Mozilla/5.0 (X11; U; Linux 2.4.2-2 i586; en-US; m18) Gecko/20010131 Netscape6/6.01 Disallow: / User-agent: Mozilla/5.0 (X11; U; Linux i686; en-US; Nautilus/1.0Final) Gecko/20020408 Disallow: / User-agent: Mozilla/5.0 (X11; U; Linux i686; en-US; rv:0.9.3) Gecko/20010801 Disallow: / User-agent: Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.2b) Gecko/20021007 Phoenix/0.3 Disallow: / User-agent: Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.6) Gecko/20040413 Epiphany/1.2.1 Disallow: / User-agent: Mozilla/5.0 Galeon/1.0.2 (X11; Linux i686; U;) Gecko/20011224 Disallow: / User-agent: Mozilla/5.0 gURLChecker/0.x.x (Linux) Disallow: / User-agent: Mozilla/5.0 URL-Spider Disallow: / User-agent: Mozilla/5.0 usww.com-Spider-for-w8.net Disallow: / User-agent: Mozilla/5.0 wgao@genieknows.com Disallow: / User-agent: MSFrontPage/4.0 Disallow: / User-agent: MSProxy/2.0 Disallow: / User-agent: MSIE 4.0 (Win95) Disallow: / User-agent: MSIE-5.13 (larbin@unspecified.mail) Disallow: / User-agent: MSNPTC/1.0 Disallow: / User-agent: multiBlocker browser Disallow: / User-agent: MultiText/0.1 Disallow: / User-agent: MusicWalker2.0 (+http://www.somusical.com) Disallow: / User-agent: myDaemon Disallow: / User-agent: MyGetRight/1.0b Disallow: / User-agent: MyGetRight/1.0.0 Disallow: / User-agent: Mylinea.com Crawler 2.0 Disallow: / # N User-agent: NaverBot Disallow: / User-agent: NavRoad Disallow: / User-agent: Navroad Disallow: / User-agent: NDSpider Disallow: / User-agent: NearSite Disallow: / User-agent: NEC-MeshExplorer Disallow: / User-agent: Nederland.zoek Disallow: / User-agent: Net M@nager Disallow: / User-agent: Net\ Vampire Disallow: / User-agent: NetAnts Disallow: / User-agent: NetCarta WebMap Engine Disallow: / User-agent: NetMechanic Disallow: / User-agent: NetScoop Disallow: / User-agent: NetSpider Disallow: / User-agent: NetstreamRecorder Disallow: / User-agent: newscan-online Disallow: / User-agent: NHSE Web Forager Disallow: / User-agent: NICErsPRO Disallow: / User-agent: Nomad Disallow: / User-agent: Northern Light Gulliver Disallow: / User-agent: NP Disallow: / User-agent: nzexplorer Disallow: / User-agent: nabot_1.0 Disallow: / User-agent: NABOT/5.0 Disallow: / User-agent: NationalDirectoryAddURL/1.0 Disallow: / User-agent: NationalDirectory-WebSpider/1.3 Disallow: / User-agent: NaverBot-1.0 (NHN Corp. / +82-2-3011-1954 / nhnbot@naver.com) Disallow: / User-agent: NaverBot_dloader/1.5 Disallow: / User-agent: NCSA Beta 1 (http://vias.ncsa.uiuc.edu/viasarchivinginformation.html) Disallow: / User-agent: NEC-Hayek/1.0 Disallow: / User-agent: NEC Research Agent -- compuman at research.nj.nec.com Disallow: / User-agent: NetAnts/1.2x Disallow: / User-agent: NETCOMplete/x.xx Disallow: / User-agent: NetLookout/2.24 Disallow: / User-agent: NetMechanic Vx.0 Disallow: / User-agent: NetNoseCrawler/v1.0 Disallow: / User-agent: Netprospector JavaCrawler Disallow: / User-agent: NetPumper/x.xx Disallow: / User-agent: NetResearchServer(http://www.look.com) Disallow: / User-agent: NetResearchServer/x.x(loopimprovements.com/robot.html) Disallow: / User-agent: NetZippy Disallow: / User-agent: NeuralBot/0.2 Disallow: / User-agent: NextGenSearchBot 1 (for information visit http://www.eliyon.com/NextGenSearchBot) Disallow: / User-agent: NextopiaBOT (+http://www.nextopia.com) distributed crawler client beta v0.x Disallow: / User-agent: newsearchengine (ThisUser@unspecified.mail) Disallow: / User-agent: NG/1.0 Disallow: / User-agent: NICO/1.0 Disallow: / User-agent: NITLE Blog Spider/0.01 Disallow: / User-agent: Nitro Downloader 1.x (www.klsofttools.com) Disallow: / User-agent: Noago Spider Disallow: / User-agent: Nocilla/1.0 Disallow: / User-agent: Nokia-WAPToolkit/1.2 googlebot(at)googlebot.com Disallow: / User-agent: Nokia7110/1.0 (05.01) (Google WAP Proxy/1.0) Disallow: / User-agent: NokodoBot/1.x (+http://nokodo.com/bot.htm) Disallow: / User-agent: NP/0.1 (NP; http://www.nameprotect.com; npbot@nameprotect.com) Disallow: / User-agent: NPBot (http://www.nameprotect.com/botinfo.html) Disallow: / User-agent: NPBot-1/2.0 Disallow: / User-agent: nttdirectory_robot/0.9 (super-robot@super.navi.ocn.ne.jp) Disallow: / User-agent: nuSearch Spider www.nusearch.com (compatible; MSIE 4.01) Disallow: / User-agent: NuSearch Spider www.nusearch.com Disallow: / User-agent: Nutch Disallow: / User-agent: NutchCVS/0.0x-dev (Nutch; http://www.nutch.org/docs/bot.html; nutch-agent@lists.sourceforge.net) Disallow: / User-agent: NutchOrg/0.0x-dev (Nutch; http://www.nutch.org/docs/bot.html; nutch-agent@lists.sourceforge.net) Disallow: / # O User-agent: ObjectsSearch Disallow: / User-agent: oBot Disallow: / User-agent: Occam Disallow: / User-agent: offline Disallow: / User-agent: Offline Explorer Disallow: / User-agent: Offline\ Navigator Disallow: / User-agent: OntoSpider Disallow: / User-agent: Openbot Disallow: / User-agent: Openfind Disallow: / User-agent: Openfind data gathere Disallow: / User-agent: Openfind data gatherer Disallow: / User-agent: Orb Search Disallow: / User-agent: obidos-bot (just looking for books.) Disallow: / User-agent: ObjectsSearch/0.01-dev (ObjectsSearch;http://www.ObjectsSearch.com/bot.html; support@thesoftwareobjects.com) Disallow: / User-agent: oBot ((compatible;Win32)) Disallow: / User-agent: Ocelli/1.x (http://www.globalspec.com/Ocelli) Disallow: / User-agent: Octopus Disallow: / User-agent: Offline Explorer 1.* Disallow: / User-agent: OliverPerry Disallow: / User-agent: OmniExplorer_Bot/1.07 (+http://www.omni-explorer.com) Internet Categorizer Disallow: / User-agent: OmniExplorer_Bot/1.07 (+http://www.omni-explorer.com) Job Crawler Disallow: / User-agent: OmniExplorer_Bot/1.18 (+http://www.omni-explorer.com) Torrent Crawler Disallow: / User-agent: online link validator (http://www.dead-links.com/) Disallow: / User-agent: OntoSpider/1.0 libwww-perl/5.65 Disallow: / User-agent: Openbot/3.0+(robot-response@openfind.com.tw;+http://www.openfind.com.tw/robot.html) Disallow: / User-agent: Openfind data gatherer, Openbot/3.0+(robot-response@openfind.com.tw;+http://www.openfind.com.tw/robot.html) Disallow: / User-agent: Openfind Robot/1.1A2 Disallow: / User-agent: OpenTextSiteCrawler/2.9.2 Disallow: / User-agent: OpenWebSpider/x Disallow: / User-agent: Opera/5.0 (Linux 2.0.38 i386; U) [en] Disallow: / User-agent: Opera/5.11 (Windows ME; U) [ru] Disallow: / User-agent: Opera/5.12 (Windows 98; U) [en] Disallow: / User-agent: Opera/6.x (Linux 2.4.8-26mdk i686; U)  [en] Disallow: / User-agent: Opera/6.x (Windows NT 4.0; U) [de] Disallow: / User-agent: Opera/6.01 (larbin@unspecified.mail) Disallow: / User-agent: Opera/7.x (Windows NT 5.1; U) [en] Disallow: / User-agent: Opera/8.xx (Windows NT 5.1; U; en) Disallow: / User-agent: OpidooBOT (larbin2.6.3@unspecified.mail) Disallow: / User-agent: Oracle iMTCrawler Disallow: / User-agent: Oracle Ultra Search Disallow: / User-agent: Overture-WebCrawler/3.8/Fresh (atw-crawler at fast dot no; http://fast.no/support/crawler.asp) Disallow: / User-agent: OWR_Crawler 0.1 Disallow: / # P User-agent: Pack Rat User-agent: PageBoy Disallow: / User-agent: PageDown Disallow: / User-agent: PageRipper Disallow: / User-agent: PageSucker Disallow: / User-agent: ParaSite Disallow: / User-agent: Patric Disallow: / User-agent: Patwebbot Disallow: / User-agent: pcBrowser Disallow: / User-agent: pegasus Disallow: / User-agent: PerlCrawler 1.0 Disallow: / User-agent: Personal Internet Engine Disallow: / User-agent: PersonaPilot Disallow: / User-agent: PGP Key Agent Disallow: / User-agent: Phantom Disallow: / User-agent: PhpDig Disallow: / User-agent: PiltdownMan Disallow: / User-agent: Pimptrain.com's robot Disallow: / User-agent: Pioneer Disallow: / User-agent: PlumtreeWebAccessor Disallow: / User-agent: Poppi Disallow: / User-agent: Popular Iconoclast Disallow: / User-agent: Portal Juice Spider Disallow: / User-agent: PortalB Spider Disallow: / User-agent: Program Disallow: / User-agent: Project Disallow: / User-agent: ProPowerBot/2.14 Disallow: / User-agent: ProWebWalker Disallow: / User-agent: psbot Disallow: / User-agent: Python-urllib Disallow: / User-agent: PagmIEDownload Disallow: / User-agent: parallelContextFocusCrawler1.1parallelContextFocusCrawler1.1 Disallow: / User-agent: ParaSite/1.0b (http://www.ianett.com/parasite/) Disallow: / User-agent: Patwebbot (http://www.herz-power.de/technik.html) Disallow: / User-agent: pavuk/0.9pl29b i686-pc-linux-gnu Disallow: / User-agent: PBrowse 1.4b Disallow: / User-agent: pd02_1.0.0 pd02_1.0.0@dzimi@post.sk Disallow: / User-agent: PEAR HTTP_Request class ( http://pear.php.net/ ) Disallow: / User-agent: PEERbot www.peerbot.com Disallow: / User-agent: PeopleChat/Search_Engine Disallow: / User-agent: PEval 1.4b Disallow: / User-agent: PHP/3.x.xx Disallow: / User-agent: PHP/4.0.4pl1 Disallow: / User-agent: PHP/4.0.6 Disallow: / User-agent: PHP/4.1.1 Disallow: / User-agent: PHP/4.1.2 Disallow: / User-agent: PicoSearch/1.0 Disallow: / User-agent: PingALink Monitoring Services 1.0 Disallow: / User-agent: PingALink Monitoring Services 1.0 (http://www.pingalink.com) Disallow: / User-agent: pipeLiner/0.10 (PipeLine Spider; http://www.pipeline-search.com/webmaster.html) Disallow: / User-agent: pipeLiner/0.3a (PipeLine Spider;http://www.pipeline-search.com/webmaster.html; webmaster'at'pipeline-search.com) Disallow: / User-agent: Pita Disallow: / User-agent: Pizilla++ ver 2.45 Disallow: / User-agent: PJspider/3.0 (pjspider@portaljuice.com; http://www.portaljuice.com) Disallow: / User-agent: PlagiarBot/1.0 Disallow: / User-agent: PlantyNet_WebRobot_V1.9 dhkang@plantynet.com Disallow: / User-agent: Pockey/x.x.x Disallow: / User-agent: Pockey7.x.x(WIN32GUI) Disallow: / User-agent: Pockey-GetHTML/x.xx Disallow: / User-agent: Pockey-GetHTML/4.12.0 (Win32; GUI; ix86) Disallow: / User-agent: POE-Component-Client-HTTP/0.64 (perl; N; POE; en; rv:0.640000) Disallow: / User-agent: polybot 1.0 (http://cis.poly.edu/polybot/) Disallow: / User-agent: Pompos/1.x pompos@iliad.fr Disallow: / User-agent: Pompos/1.x http://dir.com/pompos.html Disallow: / User-agent: Popdexter/1.0 Disallow: / User-agent: PortalBSpider/2.0 (spider@portalb.com) Disallow: / User-agent: potbot 1.0 Disallow: / User-agent: Privoxy/3.0 (Anonymous) Disallow: / User-agent: Production Bot 0116B Disallow: / User-agent: Production Bot 2016B Disallow: / User-agent: Production Bot DOT 3016B Disallow: / User-agent: Program Shareware 1.0.2 Disallow: / User-agent: Progressive Download Disallow: / User-agent: Progressive Download HTTP check Disallow: / User-agent: Project XP5 [2.03.07-111203] Disallow: / User-agent: PROve AnswerBot 4.0 Disallow: / User-agent: ProWebGuide Link Checker (http://www.prowebguide.com) Disallow: / User-agent: psbot/0.1 (+http://www.picsearch.com/bot.html) Disallow: / User-agent: PSurf15a 11 Disallow: / User-agent: PSurf15a 51 Disallow: / User-agent: PSurf15a VA Disallow: / User-agent: PubCrawl (pubcrawl.stanford.edu) Disallow: / User-agent: puf/0.91beta6a (Linux 2.2.18; i686) Disallow: / User-agent: PureSight Disallow: / User-agent: PuxaRapido v1.0 Disallow: / User-agent: PycURL Disallow: / User-agent: Python-urllib/1.15 Disallow: / User-agent: Python-urllib/2.0a1 Disallow: / # Q User-agent: Que Disallow: / User-agent: Qango.com Web Directory (http://www.qango.com/) Disallow: / User-agent: QPCreep Test Rig ( We are not indexing, just testing ) Disallow: / User-agent: QuepasaCreep ( crawler@quepasacorp.com ) Disallow: / User-agent: QuepasaCreep v0.9.1x Disallow: / User-agent: QueryN Metasearch Disallow: / # R User-agent: Radiation Retriever 1.1 Disallow: / User-agent: RafaBot Disallow: / User-agent: Raven Search Disallow: / User-agent: RBSE Spider Disallow: / User-agent: RepoMonkey Disallow: / User-agent: Resume Robot Disallow: / User-agent: River Disallow: / User-agent: RixBot Disallow: / User-agent: RMA Disallow: / User-agent: Road Runner: The ImageScape Robot Disallow: / User-agent: RoadHouse Crawling System Disallow: / User-agent: roadrunner Disallow: / User-agent: Robbie the Robot Disallow: / User-agent: RoboCrawl Spider Disallow: / User-agent: RoboFox Disallow: / User-agent: Robot Francoroute Disallow: / User-agent: Robozilla Disallow: / User-agent: Roverbot Disallow: / User-agent: RPT Disallow: / User-agent: RuLeS Disallow: / User-agent: rabaz (rabaz at gigabaz dot com) Disallow: / User-agent: RaBot/1.0 Agent-admin/phortse@hanmail.net Disallow: / User-agent: Rainbot1.1 Disallow: / User-agent: ramBot xtreme x.x Disallow: / User-agent: Rational SiteCheck (Windows NT) Disallow: / User-agent: RealDownload/4.0.0.4x Disallow: / User-agent: Reaper [2.03.10-031204] (http://www.sitesearch.ca/reaper/) Disallow: / User-agent: Reaper/2.0x (+http://www.sitesearch.ca/reaper) Disallow: / User-agent: REBOL Core 2.x.x.x.x Disallow: / User-agent: REL Link Checker Lite x.x Disallow: / User-agent: RepoMonkey Bait & Tackle/v1.01 Disallow: / User-agent: Rewebber/1.2 libwww-perl/5.41 Disallow: / User-agent: rico/0.1 Disallow: / User-agent: RMA/1.0 (compatible; RealMedia) Disallow: / User-agent: RoboCrawl (www.canadiancontent.net) Disallow: / User-agent: RoboPal (http://www.findpal.com/) Disallow: / User-agent: Robot@SuperSnooper.Com Disallow: / User-agent: Robot/www.pj-search.com Disallow: / User-agent: Robot: NutchCrawler, Owner: wdavies@acm.org Disallow: / User-agent: Robozilla/1.0 Disallow: / User-agent: Rotondo/3.1 libwww/5.3.1 Disallow: / User-agent: RPT-HTTPClient/0.3-x Disallow: / User-agent: RRC (crawler_admin@bigfoot.com) Disallow: / User-agent: RSurf15a 41 Disallow: / User-agent: RSurf15a 51 Disallow: / User-agent: RSurf15a 81 Disallow: / User-agent: Rumours-Agent Disallow: / # S User-agent: SafetyNet Robot Disallow: / User-agent: Scooter Disallow: / User-agent: Scrubby Disallow: / User-agent: Search.Aus-AU.COM Disallow: / User-agent: SearchProcess Disallow: / User-agent: Secret Disallow: / User-agent: Selfseek Disallow: / User-agent: Senrigan Disallow: / User-agent: SG-Scout Disallow: / User-agent: ShagSeeker Disallow: / User-agent: Shai'Hulud Disallow: / User-agent: Sift Disallow: / User-agent: Simmany Robot Ver1.0 Disallow: / User-agent: Simple/ Disallow: / User-agent: Site Searcher Disallow: / User-agent: Site Valet Disallow: / User-agent: SiteMapper Disallow: / User-agent: SiteTech-Rover Disallow: / User-agent: Skymob.com Disallow: / User-agent: SLCrawler Disallow: / User-agent: Sleek Disallow: / User-agent: Slurp Disallow: / User-agent: Smart Spider Disallow: / User-agent: SnapItUp Disallow: / User-agent: Snoop Disallow: / User-agent: Snooper Disallow: / User-agent: Solbot Disallow: / User-agent: SpankBot Disallow: / User-agent: spanner Disallow: / User-agent: Speedy Spider Disallow: / User-agent: spider_monkey Disallow: / User-agent: SpiderBot Disallow: / User-agent: Spiderline Crawler Disallow: / User-agent: SpiderView(tm) Disallow: / User-agent: Spry Wizard Robot Disallow: / User-agent: sqworm Disallow: / User-agent: Sqworm Disallow: / User-agent: StackRambler Disallow: / User-agent: Star Disallow: / User-agent: Suke Disallow: / User-agent: Sundoh Disallow: / User-agent: suntek search engine Disallow: / User-agent: SuperBot Disallow: / User-agent: SuperHTTP Disallow: / User-agent: Surfbot Disallow: / User-agent: SurfSaver Disallow: / User-agent: SurveyBot Disallow: / User-agent: Sven Disallow: / User-agent: Sygol Disallow: / User-agent: Szukacz/1.4 Disallow: / User-agent: S.T.A.L.K.E.R. (http://www.seo-tools.net/en/bot.aspx) Disallow: / User-agent: SafariBookmarkChecker (+http://www.coriolis.ch/) Disallow: / User-agent: SBL-BOT (http://sbl.net) Disallow: / User-agent: ScanWeb Disallow: / User-agent: schwarzmann.biz-Spider_for_paddel.org+(http://www.innerprise.net/usp-spider.asp) Disallow: / User-agent: Science Traveller International 1X/1.0 Disallow: / User-agent: Scooter/1.0 Disallow: / User-agent: Scooter/1.0 scooter@pa.dec.com Disallow: / User-agent: Scooter/1.1 (custom) Disallow: / User-agent: Scooter/2.0 G.R.A.B. V1.1.0 Disallow: / User-agent: Scooter/2.0 G.R.A.B. X2.0 Disallow: / User-agent: Scooter-3.0.EU Disallow: / User-agent: Scooter-3.0.FS Disallow: / User-agent: Scooter-3.0.HD Disallow: / User-agent: Scooter-3.0.VNS Disallow: / User-agent: Scooter-3.0QI Disallow: / User-agent: Scooter-3.2 Disallow: / User-agent: Scooter-3.2.BT Disallow: / User-agent: Scooter-3.2.DIL Disallow: / User-agent: Scooter-3.2.EX Disallow: / User-agent: Scooter-3.2.JT Disallow: / User-agent: Scooter-3.2.NIV Disallow: / User-agent: Scooter-3.2.SF0 Disallow: / User-agent: Scooter-3.2.snippet Disallow: / User-agent: Scooter/3.3 Disallow: / User-agent: Scooter/3.3.QA.pczukor Disallow: / User-agent: Scooter/3.3_SF Disallow: / User-agent: Scooter/3.3.vscooter Disallow: / User-agent: Scooter-3.3dev Disallow: / User-agent: Scooter-ARS-1.1 Disallow: / User-agent: Scooter-ARS-1.1-ih Disallow: / User-agent: Scooter_bh0-3.0.3 Disallow: / User-agent: Scooter_trk3-3.0.3 Disallow: / User-agent: scooter-venus-3.0.vns Disallow: / User-agent: Scooter-W3-1.0 Disallow: / User-agent: Scooter-W3.1.2 Disallow: / User-agent: Scooter2_Mercator_x-x.0 Disallow: / User-agent: ScoutAbout Disallow: / User-agent: Scrubby/2.x (http://www.scrubtheweb.com/) Disallow: / User-agent: Search+ Disallow: / User-agent: Search/1.0 (http://www.innerprise.net/es-spider.asp) Disallow: / User-agent: search.ch V1.4 Disallow: / User-agent: search.ch V1.4.2 (spiderman@search.ch; http://www.search.ch) Disallow: / User-agent: SearchByUsa/2 (SearchByUsa; http://www.SearchByUsa.com/bot.html; info@SearchByUsa.com) Disallow: / User-agent: SearchExpress Spider0.99 Disallow: / User-agent: SearchGuild_DMOZ_Experiment (chris@searchguild.com) Disallow: / User-agent: Searchit-Now Robot/2.2 (+http://www.searchit-now.co.uk) Disallow: / User-agent: SearchSpider.com/1.1 Disallow: / User-agent: Searchspider/1.2 (SearchSpider; http://www.searchspider.com; webmaster@searchspider.com) Disallow: / User-agent: SearchTone2.0 - IDEARE Disallow: / User-agent: Seeker.lookseek.com Disallow: / User-agent: Seekbot/1.0 (http://www.seekbot.net/bot.html) HTTPFetcher/0.3 Disallow: / User-agent: Seekbot/1.0 (http://www.seekbot.net/bot.html) RobotsTxtFetcher/1.0 (XDF) Disallow: / User-agent: semanticdiscovery/0.x Disallow: / User-agent: Sensis.com.au Web Crawler (search_comments\at\sensis\dot\com\dot\au) Disallow: / User-agent: SeznamBot/1.0 Disallow: / User-agent: SeznamBot/1.0 (+http://fulltext.seznam.cz/) Disallow: / User-agent: Shareaza v1.x.x.xx Disallow: / User-agent: SharewarePlazaFileCheckBot/1.0+(+http://www.SharewarePlaza.com) Disallow: / User-agent: Simpy 1.x; http://www.simpy.com/ Disallow: / User-agent: Simpy/1.x (Simpy; http://www.simpy.com/?ref=bot; feedback at simpy dot com) Disallow: / User-agent: SiteBar/x.x Disallow: / User-agent: sitecheck.internetseer.com Disallow: / User-agent: sitecheck.internetseer.com (For more info see: http://sitecheck.internetseer.com) Disallow: / User-agent: SiteRecon+(xx) Disallow: / User-agent: SiteSnagger Disallow: / User-agent: SiteSpider +(http://www.SiteSpider.com/) Disallow: / User-agent: SiteWinder Disallow: / User-agent: SiteXpert Disallow: / User-agent: Skampy/0.9.x (http://www.skaffe.com/skampy-info.html) Disallow: / User-agent: Skimpy/0.x (http://www.skaffe.com/skampy-info.html) Disallow: / User-agent: Slarp/0.1 Disallow: / User-agent: Sleipnir Version 1.xx Disallow: / User-agent: Slider_Search_v1-de Disallow: / User-agent: SlimBrowser Disallow: / User-agent: Slurp/2.0 (slurp@inktomi.com; http://www.inktomi.com/slurp.html) Disallow: / User-agent: Slurp/2.0-KiteWeekly (slurp@inktomi.com; http://www.inktomi.com/slurp.html) Disallow: / User-agent: Slurp/si (slurp@inktomi.com; http://www.inktomi.com/slurp.html) Disallow: / User-agent: SlySearch (slysearch@slysearch.com) Disallow: / User-agent: SlySearch/1.0 http://www.plagiarism.org/crawler/robotinfo.html Disallow: / User-agent: SlySearch/1.x http://www.slysearch.com Disallow: / User-agent: SmartDownload/1.2.67 (Win32; Jan 12 1999) Disallow: / User-agent: SmartDownload/1.2.77 (Win32; Feb 1 2000) Disallow: / User-agent: SmartDownload/1.2.77 (Win32; Jun 19 2001) Disallow: / User-agent: smartwit.com Disallow: / User-agent: sna-0.0.1 mikeelliott@hotmail.com Disallow: / User-agent: SnoopRob/x.x Disallow: / User-agent: Snoopy_v0.xx Disallow: / User-agent: Snoopy v1.xx Disallow: / User-agent: Snoopy v1.xx, : User-Agent: Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; MyIE2) Disallow: / User-agent: SnykeBot/0.6 (http://www.snyke.com) Disallow: / User-agent: SoftHypermarketFileCheckBot/1.0+(+http://www.softhypermaket.com) Disallow: / User-agent: sohu-search Disallow: / User-agent: SpeedDownload/1.x Disallow: / User-agent: speedfind ramBot xtreme 8.1 Disallow: / User-agent: Speedy Spider (Beta/x.x; speedy@entireweb.com) Disallow: / User-agent: Speedy_Spider (http://www.entireweb.com) Disallow: / User-agent: Spida/0.1 Disallow: / User-agent: Spider/maxbot.com admin@maxbot.com Disallow: / User-agent: spider.batsch.com Disallow: / User-agent: spider.yellopet.com - www.yellopet.com Disallow: / User-agent: Spider-Sleek/2.0 (+http://search-info.com/linktous.html) Disallow: / User-agent: SpiderKU/0.x Disallow: / User-agent: SpiderMan Disallow: / User-agent: SpiderMonkey/7.0x (SpiderMonkey.ca info at http://spidermonkey.ca/sm.shtml) Disallow: / User-agent: Spinne/2.0 Disallow: / User-agent: Spinne/2.0 med Disallow: / User-agent: Spinne/2.0 med_AH Disallow: / User-agent: SQ Webscanner Disallow: / User-agent: Sqworm/2.9.81-BETA (beta_release; 20011102-760; i686-pc-linux-gnu) Disallow: / User-agent: Sqworm/2.9.85-BETA (beta_release; 20011115-775; i686-pc-linux-gnu) Disallow: / User-agent: Sqworm/2.9.89-BETA (beta_release; 20020130-839; i686-pc-linux-gnu) Disallow: / User-agent: SSurf15a 11 Disallow: / User-agent: StackRambler/x.x Disallow: / User-agent: Stamina/1.4 Disallow: / User-agent: Star Downloader Disallow: / User-agent: stat statcrawler@gmail.com Disallow: / User-agent: Steeler/1.x (http://www.tkl.iis.u-tokyo.ac.jp/~crawler/) Disallow: / User-agent: Submission Spider at surfsafely.com Disallow: / User-agent: suchbot Disallow: / User-agent: Suchknecht.at-Robot Disallow: / User-agent: SuperBot/x.x (Win32) Disallow: / User-agent: SURF Disallow: / User-agent: SurferF3 1/0 Disallow: / User-agent: SurfMaster Disallow: / User-agent: SurveyBot/2.2 Whois Source Disallow: / User-agent: SurveyBot/2.3 (Whois Source) Disallow: / User-agent: suzuran Disallow: / User-agent: SWB/V1.4 (HP) Disallow: / User-agent: swbot/0.9c libwww/5.3.1 Disallow: / User-agent: http://www.sygol.com Disallow: / User-agent: Sylera/1.2.x Disallow: / User-agent: SyncBot Disallow: / User-agent: SyncIT/x.x Disallow: / User-agent: SynoBot Disallow: / User-agent: Szukacz/1.x Disallow: / User-agent: Szukacz/1.x (robot; www.szukacz.pl/jakdzialarobot.html; szukacz@proszynski.pl) Disallow: / # T User-agent: TACH Black Widow Disallow: / User-agent: tAkeOut Disallow: / User-agent: Tarantula Disallow: / User-agent: tarspider Disallow: / User-agent: Tcl W3 Robot Disallow: / User-agent: TechBOT Disallow: / User-agent: Teleport Disallow: / User-agent: Teleport Pro Disallow: / User-agent: Teleport\ Pro Disallow: / User-agent: TeleportPro Disallow: / User-agent: Telesoft Disallow: / User-agent: Templeton Disallow: / User-agent: Teoma Disallow: / User-agent: Terrapin Disallow: / User-agent: The Intraformant Disallow: / User-agent: The NorthStar Robot Disallow: / User-agent: The NWI Robot Disallow: / User-agent: The Peregrinator Disallow: / User-agent: The Python Robot Disallow: / User-agent: The TkWWW Robot Disallow: / User-agent: The Web Moose Disallow: / User-agent: The Web Wombat Disallow: / User-agent: The Webfoot Robot Disallow: / User-agent: the World Wide Web Wanderer Disallow: / User-agent: The World Wide Web Worm Disallow: / User-agent: TheNomad Disallow: / User-agent: TightTwatBot Disallow: / User-agent: TITAN Disallow: / User-agent: Titan Disallow: / User-agent: TitIn Disallow: / User-agent: TLSpider Disallow: / User-agent: True_Robot Disallow: / User-agent: True_Robot/1.0 Disallow: / User-agent: turingos Disallow: / User-agent: Turn It In Disallow: / User-agent: Turnitin Disallow: / User-agent: Turnitinbot Disallow: / User-agent: Tagword (http://tagword.com/dmoz_survey.php) Disallow: / User-agent: Talkro Web-Shot/1.0 (E-mail: webshot@daumsoft.com, Home: http://222.122.15.190/webshot) Disallow: / User-agent: TAMU_CS_IRL_CRAWLER/1.0 Disallow: / User-agent: targetblaster.com/0.9k Disallow: / User-agent: TE Disallow: / User-agent: TECOMAC-Crawler/0.x Disallow: / User-agent: TeamSoft WinInet Component Disallow: / User-agent: Teleport Pro/1.2x(.1xxx) Disallow: / User-agent: Teoma MP Disallow: / User-agent: teomaagent crawler-admin@teoma.com Disallow: / User-agent: teomaagent1 [crawler-admin@teoma.com] Disallow: / User-agent: teoma_agent1 Disallow: / User-agent: Teradex Mapper; mapper@teradex.com; http://www.teradex.com Disallow: / User-agent: Test spider Disallow: / User-agent: The Expert HTML Source Viewer (http://www.expert-html.com) Disallow: / User-agent: TheSuBot/0.1 (www.thesubot.de) Disallow: / User-agent: tivraSpider/1.0 (crawler@tivra.com) Disallow: / User-agent: TJG/Spider Disallow: / User-agent: TJvMultiHttpGrabber Component Disallow: / User-agent: Tkensaku/x.x(http://www.tkensaku.com/q.html) Disallow: / User-agent: toCrawl/UrlDispatcher Disallow: / User-agent: Toutatis x.x-x Disallow: / User-agent: Toutatis x.x (hoppa.com) Disallow: / User-agent: Toutatis x-xx.x (hoppa.com) Disallow: / User-agent: Trampelpfad-Spider Disallow: / User-agent: Trampelpfad-Spider-v0.1 Disallow: / User-agent: tricosMetaCheck 1.2216-08-1999 (http://www.tricos.com/metacheck) Disallow: / User-agent: TSurf15a 11 Disallow: / User-agent: TulipChain/5.x (http://ostermiller.org/tulipchain/) Java/1.x.1_0x (http://java.sun.com/) Linux/2.4.17 Disallow: / User-agent: TulipChain/5.xx (http://ostermiller.org/tulipchain/) Java/1.x.1_0x (http://apple.com/) Mac_OS_X/10.2.8 Disallow: / User-agent: TurnitinBot/x.x (http://www.turnitin.com/robot/crawlerinfo.html) Disallow: / User-agent: Turnpike Emporium LinkChecker/0.1 Disallow: / User-agent: TutorGig/1.5 (+http://www.tutorgig.com/crawler) Disallow: / User-agent: Tutorial Crawler 1.4 (http://www.tutorgig.com/crawler) Disallow: / User-agent: Twotrees Reactive Filter V2.0 Disallow: / User-agent: TygoBot Disallow: / User-agent: TygoProwler Disallow: / # U User-agent: UnChaos (From Chaos To Order, Hybrid Web Search Engine. (vadim_gonchar@unchaos.com)) Disallow: / User-agent: UnChaos Bot (Hybrid Web Search Engine.(vadim_gonchar@unchaos.com)) Disallow: / User-agent: UnChaosBot (From Chaos To Order, UnChaos Hybrid Web Search Engine at www.unchaos.com (info@unchaos.com)) Disallow: / User-agent: UCSD Crawl Disallow: / User-agent: UdmSearch Disallow: / User-agent: unnamed Disallow: / User-agent: UptimeBot Disallow: / User-agent: URL Check Disallow: / User-agent: URL Control Disallow: / User-agent: URL Spider Pro Disallow: / User-agent: URL_Spider_Pro Disallow: / User-agent: URLy Warning Disallow: / User-agent: UCmore Disallow: / User-agent: UCMore Crawler App Disallow: / User-agent: UDM Disallow: / User-agent: UdmSearch/3.1.x Disallow: / User-agent: UIowaCrawler/1.0 Disallow: / User-agent: Ultraseek Disallow: / User-agent: Under the Rainbow 2.2 Disallow: / User-agent: UP.Browser/3.01-IG01 UP.Link/3.2.3.4 Disallow: / User-agent: updated/0.1beta (updated.com; http://www.updated.com; crawler@updated.om) Disallow: / User-agent: UPG1 UP/4.0 (compatible; Blazer 1.0) Disallow: / User-agent: Uptimebot Disallow: / User-agent: UptimeBot(www.uptimebot.com) Disallow: / User-agent: URLBlaze Disallow: / User-agent: URLGetFile Disallow: / User-agent: URL Spider Pro/x.xx (innerprise.net) Disallow: / User-agent: URL_Spider_Pro/x.x Disallow: / User-agent: URL_Spider_Pro/x.x+(http://www.innerprise.net/usp-spider.asp) Disallow: / User-agent: user-agent=Mozilla/3.01Gold Disallow: / User-agent: User-Agent: FileHeap! file downloader (http://www.fileheap.com) Disallow: / User-agent: User-Agent: Mozilla/4.0 (SKIZZLE! Distributed Internet Spider v1.0 - www.SKIZZLE.com) Disallow: / User-agent: USyd-NLP-Spider (http://www.it.usyd.edu.au/~vinci/bot.html) Disallow: / User-agent: UtilMind HTTPGet Disallow: / User-agent: Utopia WebWasher 3.0 Disallow: / # V User-agent: Valkyrie Disallow: / User-agent: vayala Disallow: / User-agent: VCI Disallow: / User-agent: VCI WebViewer VCI WebViewer Win32 Disallow: / User-agent: Verticrawl Disallow: / User-agent: Victoria Disallow: / User-agent: vision-search Disallow: / User-agent: Visual Web Task Disallow: / User-agent: void-bot Disallow: / User-agent: VoidEYE Disallow: / User-agent: Voyager Disallow: / User-agent: VWbot Disallow: / User-agent: Vagabondo/1.x MT (webagent@wise-guys.nl) Disallow: / User-agent: Vagabondo/2.0 MT Disallow: / User-agent: Vagabondo/2.0 MT (webagent at wise-guys dot nl) Disallow: / User-agent: Vagabondo/2.0 MT (webagent@NOSPAMwise-guys.nl) Disallow: / User-agent: Vakes/0.01 (Vakes; http://www.vakes.com/; search@vakes.com) Disallow: / User-agent: Vayala|Creep-v0.0.1 (codepoet@wildties.com) Disallow: / User-agent: VayalaCreep-v0.0.1 (haploid@haploid.com) Disallow: / User-agent: vb wininet Disallow: / User-agent: versus 0.2 (+http://versus.integis.ch) Disallow: / User-agent: VeryGoodSearch.com.DaddyLongLegs Disallow: / User-agent: verzamelgids.nl - Networking4all Bot/x.x Disallow: / User-agent: vspider Disallow: / User-agent: vspider/3.x Disallow: / # W User-agent: w@pSpider by wap4.com Disallow: / User-agent: W3M2 Disallow: / User-agent: w3mir Disallow: / User-agent: Walhello appie Disallow: / User-agent: WallPaper (alias crawlpaper) Disallow: / User-agent: WEB Disallow: / User-agent: Web Core / Roots Disallow: / User-agent: Web Downloader Disallow: / User-agent: Web Image Disallow: / User-agent: Web Page Finder Disallow: / User-agent: Web Sucker Disallow: / User-agent: Web2Map Disallow: / User-agent: WebAuto Disallow: / User-agent: WebBandit Disallow: / User-agent: WebBandit Web Spider Disallow: / User-agent: WebBandit/3.50 Disallow: / User-agent: webcapture Disallow: / User-agent: WebCatcher Disallow: / User-agent: WebCopier Disallow: / User-agent: WebCopy Disallow: / User-agent: Webdupe Disallow: / User-agent: WebEMailExtrac.* Disallow: / User-agent: WebEnhancer Disallow: / User-agent: webfetcher Disallow: / User-agent: WebFountain Disallow: / User-agent: WebHook Disallow: / User-agent: Webinator Disallow: / User-agent: weblayers Disallow: / User-agent: WebLinker Disallow: / User-agent: Weblog Monitor Disallow: / User-agent: WebmasterWorldForumBot Disallow: / User-agent: WebMiner Disallow: / User-agent: WebMirror Disallow: / User-agent: webpictureboss Disallow: / User-agent: WebPictures Disallow: / User-agent: WebQuest Disallow: / User-agent: WebReaper Disallow: / User-agent: webs Disallow: / User-agent: WebSauger Disallow: / User-agent: Website Quester Disallow: / User-agent: WebSlinky Disallow: / User-agent: Websnarf Disallow: / User-agent: WebSpider Disallow: / User-agent: Webster Disallow: / User-agent: Webster Pro Disallow: / User-agent: WebStolperer Disallow: / User-agent: WebStripper Disallow: / User-agent: WebVac Disallow: / User-agent: WebVCR Disallow: / User-agent: webwalk Disallow: / User-agent: WebWalker Disallow: / User-agent: WebWatch Disallow: / User-agent: WebWhacker Disallow: / User-agent: WebZinger Disallow: / User-agent: WebZip Disallow: / User-agent: WebZip/4.0 Disallow: / User-agent: WEP Disallow: / User-agent: Wget Disallow: / User-agent: Wget/1.5.3 Disallow: / User-agent: Wget/1.6 Disallow: / User-agent: whatUseek Winona Disallow: / User-agent: Whizg Disallow: / User-agent: WhoWhere Robot Disallow: / User-agent: Widow Disallow: / User-agent: Wild Ferret Web Hopper Disallow: / User-agent: Wired Digital Disallow: / User-agent: Wisenutbot Disallow: / User-agent: worm Disallow: / User-agent: WWWC Ver 0.2.5 Disallow: / User-agent: WWW-Collector-E Disallow: / User-agent: [Ww]eb[Bb]andit Disallow: / User-agent: W3C-checklink/3.x.x.x libwww-perl/5.xx Disallow: / User-agent: W3C-checklink/4.x [4.xx] libwww-perl/5.xxx Disallow: / User-agent: W3C_Validator/1.xxx libwww-perl/5.xx Disallow: / User-agent: W3CLineMode/5.4.0 libwww/5.x.x Disallow: / User-agent: W3CRobot/5.4.0 libwww/5.4.0 Disallow: / User-agent: w3m/0.x.xx Disallow: / User-agent: W3SiteSearch Crawler_v1.1 http://www.w3sitesearch.de Disallow: / User-agent: WannaBe (Macintosh; PPC) Disallow: / User-agent: WDG_Validator/1.1 Disallow: / User-agent: Web Image Collector Disallow: / User-agent: Web Link Validator 1.5 Disallow: / User-agent: Web Snooper Disallow: / User-agent: Web-Bot V1.03 Disallow: / User-agent: Web-Robot/5.0 (en-US; web-robot.com/policy.html) Web-Robot Crawler/2.0.3 Disallow: / User-agent: WebAuto/3.4xxx (WinNT; I) Disallow: / User-agent: webbandit/4.xx.0 Disallow: / User-agent: Webclipping.com Disallow: / User-agent: webcollage/1.xx Disallow: / User-agent: WebCompass 2.0 Disallow: / User-agent: WebCopier vx.x Disallow: / User-agent: WebCopier vx.xa Disallow: / User-agent: webcrawl.net Disallow: / User-agent: WebDownloader for X 1.30 Disallow: / User-agent: Webdup/0.9 Disallow: / User-agent: WebFetch Disallow: / User-agent: webfetch/5.x.x Disallow: / User-agent: WebFilter Robot 1.x Disallow: / User-agent: WebFindBot(http://www.web-find.com) Disallow: / User-agent: Webglimpse 2.xx.x (http://webglimpse.net) Disallow: / User-agent: webhack Disallow: / User-agent: Weblink's checker/ Disallow: / User-agent: WebMiner/x.x [en] (Win98; I) Disallow: / User-agent: WeBoX/0.xx Disallow: / User-agent: WebPix 1.0 (www.netwu.com) Disallow: / User-agent: WebQL Disallow: / User-agent: WebRACE/1.1 (University of Cyprus, Distributed Crawler) Disallow: / User-agent: WebReaper [info@webreaper.net] Disallow: / User-agent: WebReaper [webreaper@webreaper.net] Disallow: / User-agent: WebReaper vx.x - www.webreaper.net Disallow: / User-agent: WebSearch.COM.AU/3.0.1 (The Australian Search Engine; http://WebSearch.COM.AU; Search@WebSearch.COM.AU) Disallow: / User-agent: WebSearchBench WebCrawler v0.1(Experimental) Disallow: / User-agent: WebSearchBench WebCrawler V1.0 (Beta), Prof. Dr.-Ing. Christoph Lindemann, Universität Dortmund, cl@cs.uni-dortmund.de, http://websearchbench.cs.uni-dortmund.de/ Disallow: / User-agent: Website eXtractor Disallow: / User-agent: Webspinne/1.0 webmaster@webspinne.de Disallow: / User-agent: WebStat/1.0 (Unix; beta; 20040314) Disallow: / User-agent: WebStripper/2.xx Disallow: / User-agent: WebTrafficExpress/x.0 Disallow: / User-agent: WebTrends/3.0 (WinNT) Disallow: / User-agent: WebVac (webmaster@pita.stanford.edu) Disallow: / User-agent: WebVal/1.0 Disallow: / User-agent: WebWatcherMonitor/2.01 Disallow: / User-agent: WebZIP/x.x (http://www.spidersoft.com) Disallow: / User-agent: WEP Search 00 Disallow: / User-agent: WFARC Disallow: / User-agent: Wget/1.x(.x) Disallow: / User-agent: Wget/1.x.x+cvs Disallow: / User-agent: Wget/1.x+cvs-stable (Red Hat modified) Disallow: / User-agent: Whatsup/x.x Disallow: / User-agent: whatUseek_winona/3.0 Disallow: / User-agent: WhizBang! Lab Disallow: / User-agent: Wildsoft Surfer Disallow: / User-agent: Willow Internet Crawler by Twotrees V2.1 Disallow: / User-agent: WinampMPEG/2.00 (larbin@unspecified.mail) Disallow: / User-agent: WincerSong Agent v1.0 Disallow: / User-agent: WIRE/0.1 (Linux; i686; Bot,Robot,Spider,Crawler) Disallow: / User-agent: WISEbot/1.0 (WISEbot@koreawisenut.com; http://wisebot.koreawisenut.com) Disallow: / User-agent: WiseWire-Spider2 Disallow: / User-agent: WordChampBot Disallow: / User-agent: WorldLight Disallow: / User-agent: WorQmada/1.0 Disallow: / User-agent: Wotbox/alpha0.6 (bot@wotbox.com; http://www.wotbox.com) Disallow: / User-agent: Wotbox/alpha0.x.x (bot@wotbox.com; http://www.wotbox.com)   Java/1.4.1_02 Disallow: / User-agent: WSB WebCrawler V1.0 (Beta), cl@cs.uni-dortmund.de Disallow: / User-agent: WSB, http://websearchbench.cs.uni-dortmund.de Disallow: / User-agent: Wusage/x.0@boutell.com Disallow: / User-agent: WWSBOT 1.x [--- http://www.analyzer.nu ---] Disallow: / User-agent: www.arianna.it Disallow: / User-agent: www.business-socket.com registry verify/1.x Disallow: / User-agent: www4mail/2.x libwww-FM/2.14 (Unix; I) Disallow: / User-agent: WWWC/1.0x Disallow: / User-agent: WWWeasel Robot v1.00 (http://wwweasel.de) Disallow: / User-agent: WWWOFFLE/2.x Disallow: / User-agent: wwwster/1.2 (Beta, mailto:gue@cis.uni-muenchen.de) Disallow: / # X User-agent: Xaldon\ WebSpider Disallow: / User-agent: Xenu's Disallow: / User-agent: Xenu's Link Sleuth 1.1c Disallow: / User-agent: XGET Disallow: / User-agent: XYLEME Robot Disallow: / User-agent: X-Crawler Disallow: / User-agent: Xaldon WebSpider Disallow: / User-agent: Xenu's Link Sleuth 1.x[a-z] Disallow: / User-agent: Xenu Link Sleuth 1.xx Disallow: / User-agent: xyro_(xcrawler@cosmos.inria.fr) Disallow: / # Y User-agent: Y!TunnelPro Disallow: / User-agent: Yahoo-MMCrawler/3.x (mm dash crawler at trd dot overture dot com) Disallow: / User-agent: YahooSeeker/1.0 (compatible; Mozilla 4.0; MSIE 5.5; http://help.yahoo.com/help/us/shop/merchant/) Disallow: / User-agent: YahooSeeker/1.0 (compatible; Mozilla 4.0; MSIE 5.5; http://search.yahoo.com/yahooseeker.html) Disallow: / User-agent: YahooSeeker/1.1 (compatible; Mozilla 4.0; MSIE 5.5; http://help.yahoo.com/help/us/shop/merchant/) Disallow: / User-agent: YahooSeeker/CafeKelsa-dev (compatible; Konqueror/3.2; FreeBSD ;cafekelsa-dev-webmaster@yahoo-inc.com ) Disallow: / User-agent: Yahoo-VerticalCrawler-FormerWebCrawler/3.9 crawler at trd dot overture dot com; http://www.alltheweb.com/help/webmaster/crawler Disallow: / User-agent: yarienavoir.net/0.2 Disallow: / User-agent: YottaCars_Bot/4.12 (+http://www.yottacars.com) Car Search Engine Disallow: / User-agent: YottaShopping_Bot/4.12 (+http://www.yottashopping.com) Shopping Search Engine Disallow: / User-agent: YTunnelPro Disallow: / # Z User-agent: Zeus Disallow: / User-agent: Zeus 32297 Webster Pro V2.9 Win32 Disallow: / User-agent: Zeus Link Scout Disallow: / User-agent: Zeus.*Webster Disallow: / User-agent: Zleecher Disallow: / User-agent: Z-Add Link Checker (http://w3.z-add.co.uk/linkcheck/) Disallow: / User-agent: Zao/0.1 (http://www.kototoi.org/zao/) Disallow: / User-agent: Zao-Crawler Disallow: / User-agent: Zao-Crawler 0.2b Disallow: / User-agent: ZBot/1.00 (icaulfield@zeus.com) Disallow: / User-agent: Zeus xxxxx Webster Pro V2.9 Win32 Disallow: / User-agent: Zeus ThemeSite Viewer Webster Pro V2.9 Win32 Disallow: / User-agent: Ziggy -- The Clown From Hell!! Disallow: / User-agent: ZipppBot/0.xx (ZipppBot; http://www.zippp.net; webmaster@zippp.net) Disallow: / User-agent: ZIPPPCVS/0.xx (ZipppBot/.xx;http://www.zippp.net; webmaster@zippp.net) Disallow: / User-agent: Zippy v2.0 - Zippyfinder.com Disallow: / User-agent: ZyBorg/1.0 (ZyBorg@WISEnut.com; http://www.WISEnut.com) Disallow: / # DIVERSE User-agent: -?$ Disallow: / User-agent: :robot/1.0 (linux) ( admin e-mail: undefined http://www.neofonie.de/loesungen/search/robot.html ) Disallow: / User-agent: 2Bone_LinkChecker/1.0 libwww-perl/5.64 Disallow: / User-agent: 4anything.com LinkChecker v2.0 Disallow: / http://www.WISEnut.com) Disallow: / # DIVERSE User-agent: -?$ Disallow: / User-agent: :robot/1.0 (linux) ( admin e-mail: undefined http://www.neofonie.de/loesungen/search/robot.html ) Disallow: / User-agent: 2Bone_LinkChecker/1.0 libwww-perl/5.64 Disallow: / User-agent: 4anything.com LinkChecker v2.0 Disallow: /