Sitemap: http://www.renetta.nl/sitemap.xml.gz User-agent: * # disallow all files in these directories Disallow: /contact #Disallow: /contact?lang=en Disallow: /cgi-bin Disallow: /gastenboek #Disallow: /gastenboek?lang=en Disallow: /wp-admin Disallow: /wp-includes #Disallow: /wp-content Disallow: /themes Disallow: /trackback Disallow: /awstats Disallow: /webmail # allow google image bot to search all images User-agent: Googlebot-Image Disallow: /upload # disallow archiving site User-agent: ia_archiver Disallow: / # disallow majestic User-agent: MJ12bot Disallow: / # disable duggmirror User-agent: duggmirror Disallow: / # Disallow specific bots from indexing or crawling the site at all # Most recent additions first. User-agent: GigaBot Disallow: / User-agent: Voyager Disallow: / User-agent: BaiDuSpider Disallow: / User-agent: BackRub/*.* Disallow: / User-agent: Grub.org Disallow: / User-agent: BotRightHere Disallow: / User-agent: larbin Disallow: / User-agent: psbot Disallow: / User-agent: Walhello appie Disallow: / User-agent: Python-urllib Disallow: / User-agent: CherryPicker Disallow: / User-agent: EmailCollector Disallow: / User-agent: WebBandit Disallow: / User-agent: EmailWolf Disallow: / User-agent: CopyRightCheck Disallow: / User-agent: Crescent Disallow: /