User-agent: * Disallow: /admin/ Disallow: /js/ Disallow: /css/ Disallow: /img/ Disallow: /join/ Disallow: /signin? Disallow: /messages? Disallow: /restaurant/join.php? Disallow: /restaurant/signup Disallow: /redirect.php? Disallow: /users/flag_content.php? # Site scrapers that are completely disallowed User-agent: HttpTool* Disallow: / User-agent: http://www.almaden.ibm.com/cs/crawler Disallow: / User-agent: ia_archiver Disallow: / User-agent: Bilbo/2.3b-UNIX Disallow: / User-agent: FAST-WebCrawler/3.6/FirstPage Disallow: / User-agent: TurnitinBot Disallow: / User-agent: Offline Explorer/1.9 Disallow: / User-agent: Missigua Locator 1.9 Disallow: / User-agent: grub-client Disallow: /