# /robots.txt file for http://webcrawler.com/
# mail webmaster@webcrawler.com for constructive criticism

User-agent: webcrawler
Disallow: /admin
Disallow: /database

User-agent: lycra
Disallow: /admin
Disallow: /database

User-agent: *
Disallow: /admin
Disallow: /database

User-agent: *
Allow: /searchhistory/
Disallow: /news?output=xhtml&
Allow: /news?output=xhtml

Disallow: /admin
Disallow: /includes