# Domo Arigato Mr. Roboto # # Robots.txt for http://www.mayhemnetworks.net/ # # Last edited 05/31/2007 10:55 PST # # Block spiders from all kinds of places # they shouldn't be # User-agent: * Disallow: /inc/ Disallow: /tools/ Disallow: /blog/category/ # # Nasty spiders that I don't want # visiting if I can control it # User-agent: Wget Disallow: / # User-agent: Lynx Disallow: / # User-agent: Curl Disallow: / # User-agent: libwww-perl Disallow: / # User-agent: EmailSiphon Disallow: / # User-agent: EmailWolf Disallow: / # User-agent: ExtractorPro Disallow: / # User-agent: Crescent Disallow: / # User-agent: CherryPicker Disallow: / # User-agent: [Ww]eb[Bb]andit Disallow: / # User-agent: WebEMailExtrac.* Disallow: / # User-agent: NICErsPRO Disallow: / # User-agent: Telesoft Disallow: / # User-agent: Zeus.*Webster Disallow: / # User-agent: EmailCollector Disallow: / # User-agent: internetseer.com Disallow: / # User-agent: InternetSeer.com Disallow: / # User-agent: sitecheck.internetseer.com Disallow: /