# Default robots.txt file # Last updated: 01.26.04 # Generated by: me User-agent: * Disallow: /cgi-bin/ Disallow: /images/ Disallow: /media/ Disallow: /wp-content/ Disallow: /wp-admin/ Disallow: /wp-includes/ User-agent: crawl User-agent: larbin User-agent: bot- User-agent: bot/ User-agent: Java User-agent: TencentTraveler User-agent: Nutch Disallow: /