# robots.txt file for http://piru.dyndns.org/ # version 1.5 # No archive.org User-agent: ia_archiver Disallow: / # no m$ search here User-agent: msnbot Disallow: / User-agent: msnbot/2.0b Disallow: / # fasterfox is cool, but not with my upstream User-agent: Fasterfox Disallow: / # No recursive wget leeching, thank you User-agent: Wget Disallow: / # everyone, please stay out of these dirs User-agent: * Crawl-delay: 30 Disallow: /cgi-bin Disallow: /~p/tmp/ Disallow: /~p/temp/ Disallow: /~p/towerscan/ # most of these robots ignore this, but ask anyway User-agent: CherryPicker Disallow: / User-agent: Crescent Internet ToolPak Disallow: / User-agent: EmailCollector Disallow: / User-agent: EmailSiphon Disallow: / User-agent: EmailWolf Disallow: / User-agent: Explorer www Disallow: / User-agent: ExtractorPro Disallow: / User-agent: Mozilla/2.0 (compatible; NEWT ActiveX; Win32) Disallow: / User-agent: WebBandit Disallow: / User-agent: Webbandit Disallow: /