# # Bulk downloading of these pages is not permitted for the # purposes of making archival copies. This includes browsers # such as IE which normally permit "off-line viewing" (i.e., # bulk downloading). These browsers, those masquerading # as them, and bulk downloading programs like Wget are therefore # explicitly disallowed using this robots.txt file. # User-agent: * Disallow: / # www.google.com User-agent: googlebot Disallow: # ask.com User-agent: Teoma Disallow: # Local Google Search Appliance User-agent: gsa-crawler Disallow: # Local Google Search Appliance User-agent: google-crawler-external Disallow: # Local Google Search Appliance User-agent: google-crawler-internal Disallow: # www.altavista.com User-Agent: scooter Disallow: # www.inktomi.com User-Agent: slurp Disallow: # MS live search User-agent: msnbot Disallow: # Local lucene indexer User-agent: nutch-crawler Disallow: