User-agent: * Disallow: /bblog/ Disallow: /cache/ Disallow: /cgi-bin/ # these two are subdomains, do don't allow them to be scanned as if they # weren't Disallow: /xtalk/ Disallow: /frosh/