feat: add Dotbot

2026-02-03 12:15:29 +00:00 · 2024-08-09 11:57:28 +02:00
parent a41290c297
commit 0437240919
248 changed files with 1205 additions and 44 deletions
--- a/8
+++ b/8
@@ -0,0 +1,8 @@
+# Disguise as Googlebot
+# user-agent = "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)"
+
+# When following a redirect, automatically set the previous URL as referer.
+referer = ";auto"
+
+# Wait 60 seconds before timing out.
+connect-timeout = 60