User-Agent: SlySearch Disallow: / # various lame bots User-Agent: arianna.iol.it Linux/2.2.17-14smp (linux) Disallow: / User-Agent: Webclipping.com Disallow: / User-Agent: larbin_2.6.2 larbin2.6.2@unspecified.mail Disallow: / User-Agent: larbin_2.6.2 Disallow: / User-Agent: larbin_2.6.1 larbin2.6.2@unspecified.mail Disallow: / User-Agent: larbin_2.6.1 Disallow: / User-Agent: libwww-perl/5.5.3 Disallow: / User-Agent: libwww-perl/5.53 Disallow: / User-Agent: libwww-perl Disallow: / User-Agent: Python-urllib/1.10 Disallow: / User-Agent: Python-urllib Disallow: / User-Agent: Java1.3.0 Disallow: / User-Agent: Java1.4.0 Disallow: / User-Agent: Java Disallow: / User-Agent: WinampMPEG/2.00 larbin@unspecified.mail Disallow: / User-Agent: WinampMPEG Disallow: / User-Agent: MSIE-5.13 larbin@unspecified.mail Disallow: / User-Agent: Opera/6.01 larbin2.6.2@unspecified.mail Disallow: / User-Agent: Opera/6.01 larbin@unspecified.mail Disallow: / User-Agent: Mozilla/5.0 larbin2.6.2@unspecified.mail Disallow: / User-Agent: bumblebee@relevare.com Disallow: / User-Agent: Zeus 64087 Webster Pro V2.9 Win32 Disallow: / User-Agent: NetResearchServer/2.3(loopimprovements.com/robot.html) Disallow: / User-Agent: NetResearchServer Disallow: / User-Agent: Openfind data gatherer, Openbot/3.0+(robot-response@openfind.com.tw;+http://www.openfind.com.tw/robot.html) Disallow: / User-Agent: Openbot Disallow: / User-Agent: http://www.almaden.ibm.com/cs/crawler Disallow: / User-Agent: NetMechanic V3.0 Disallow: / User-Agent: NetMechanic Disallow: / # No spaces. First the ID, then a space, then a comment. User-Agent: Ariadne RPT-HTTPClient/0.3-3 Disallow: / User-Agent: Ariadne Disallow: / User-Agent: Mozilla/4.0 compatible ZyBorg/1.0 (ZyBorg@WISEnutbot.com; http://www.WISEnutbot.com) Disallow: / User-Agent: ZyBorg Disallow: / User-Agent: appie 1.1 (www.walhello.com) Disallow: / User-Agent: appie Disallow: / User-Agent: search.ch V1.4.2 (spiderman@search.ch; http://www.search.ch) Disallow: / User-Agent: search.ch Disallow: / # No spaces. User-Agent: PingALink Monitoring Services 1.0 (http://www.pingalink.com) Disallow: / User-Agent: PingALink Monitoring Services Disallow: / User-Agent: PingALink Disallow: / User-Agent: AWeb-II v3.4SE/AMIGAOS-3.9 Disallow: / User-Agent: AWeb-II Disallow: / # Other misc blocks User-Agent: Enterprise_Search/1.0 (http://www.innerprise.net/es-spider.asp) Disallow: / User-Agent: BlitzBot Disallow: / # spoofing the href to sites that have annoying javascript and other stuff User-Agent: Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; .WONKZ) Disallow: / User-Agent: EmailSiphon Disallow: / User-Agent: EmailWolf Disallow: / User-Agent: ExtractorPro Disallow: / User-Agent: CherryPicker Disallow: / User-Agent: NICErsPRO Disallow: / User-Agent: Teleport Disallow: / User-Agent: EmailCollector Disallow: / User-Agent: DirBuster-0.9.7 www.sittinglittleduck.com Disallow: / # lame CGI scanners User-Agent: crimscanner Disallow: /