# /robots.txt file for http://webcrawler.com/
# mail webmaster@webcrawler.com for constructive criticism



User-agent: *
Disallow: /bin
Disallow: /aspnet_client
Disallow: /component
Disallow: /app_data
Disallow: /data
Disallow: /css
Disallow: /mfcnetxml
Disallow: /xmlpagecontext