User-agent: *
Disallow: /search/
Disallow: /admin/
Disallow: /js/
Disallow: /styles/
Disallow: /recommend/
Disallow: /skins/
Disallow: /go/
Disallow: /syndicate/
Disallow: /goto.php
Disallow: /go.php

User-agent: penthesilea
Disallow: /
User-agent: Heritrix
Disallow: /
User-agent: thesubot
Disallow: /
User-Agent: ia_archiver
Disallow: /
User-Agent: dotbot
Disallow: /
User-Agent: sistrix
Disallow: /
User-Agent: yacy
Disallow: /
User-Agent: DotBot/1.0.1
Disallow: /
User-Agent: Mozilla/5.0 (compatible; DotBot/1.1;
Disallow: /
User-agent: grub-client
Disallow: /
User-agent: grub
Disallow: /
User-agent: wget
Disallow: /
User-Agent: BoeserSeoSpider
Disallow: /

Sitemap: /sitemap.xml