Part 9

robots.txt

@@ -0,0 +1,67 @@
+# Google
+
+User-agent: Googlebot-Image
+User-agent: AdsBot-Google
+User-agent: Googlebot
+Disallow: /
+
+# Bing
+
+User-agent: Bingbot
+Disallow: /
+
+# Marginalia.nu
+
+User-Agent: search.marginalia.nu
+Disallow:
+
+# Qwant
+
+User-agent: Qwantify
+Disallow: /
+
+# WayBack Machine
+
+User-agent: ia_archiver
+Disallow: /
+
+# Misc
+
+User-agent: webproxy-mozz
+Disallow:
+
+User-agent: serpstatbot
+Disallow: /
+
+User-agent: ltx71
+Disallow: /
+
+User-agent: oBot
+Disallow: /
+
+User-agent: DuckDuckBot
+Disallow:
+
+User-agent: ltx71 - (http://ltx71.com/)
+Disallow: /
+
+User-agent: BLEXBot
+Disallow: /
+
+User-agent: Twingly Recon
+Disallow: /
+
+User-agent: PageThing
+Disallow: /
+
+User-agent: adsbot
+Disallow: /
+
+User-agent: SurdotlyBot
+Disallow: /
+
+User-agent: DataForSeoBot
+Disallow: /
+
+User-agent: SpiderLing
+Disallow: /