# robots.txt for http://Domains/ # http://http://info.webcrawler.com/mak/projects/robots/robots.html # User-agents: # AltaVista: Scooter # Infoseek: Sidewinder # WebCrawler User-agent: * Disallow: /cgi/ Disallow: /cgi-bin/ Disallow: /cgibin/ Disallow: /mc-icons/ Disallow: /ns-icons/ Disallow: /images/ User-Agent: sitecheck.internetseer.com Disallow: /file-to-disallow