# All robots will spider the domain
User-agent: *
Disallow: /cgi-bin
Disallow: /css
Disallow: /devel
Disallow: /flash
Disallow: /images
Disallow: /large
Disallow: /oldpress
Disallow: /pdf
Disallow: /scripts
Disallow: /video