DI-443 updating robot.txt to block AI scrapers

cu-library · Sep 15, 2023 · a9c2bbd · a9c2bbd
1 parent b8c6de7
commit a9c2bbd
Showing 1 changed file with 9 additions and 0 deletions.
diff --git a/public/robots.txt b/public/robots.txt
@@ -1,3 +1,12 @@
 # See http://www.robotstxt.org/robotstxt.html for documentation on how to use the robots.txt file
 User-agent: SemrushBot
 Disallow: /
+
+User-agent: CCBot
+Disallow: /
+
+User-agent: ChatGPT-User
+Disallow: /
+
+User-agent: GPTBot
+Disallow: /