fix(robots.txt): add config serving bots

freeCodeCamp · Jan 3, 2025 · 07118dd · 07118dd
1 parent 8801daa
commit 07118dd
Show file tree

Hide file tree

Showing 9 changed files with 51 additions and 14 deletions.
diff --git a/sites-enabled/10-www.freecodecamp.org.conf b/sites-enabled/10-www.freecodecamp.org.conf
@@ -497,6 +497,9 @@ server {
   # additional config
   include snippets/common/general.conf;
 
+  # robots config
+  include snippets/common/robots-org.conf;
+
   # error page
   error_page 503 @maintenance;
   location @maintenance {

diff --git a/sites-enabled/11-api.freecodecamp.org.conf b/sites-enabled/11-api.freecodecamp.org.conf
@@ -25,6 +25,10 @@ server {
 
   # additional config
   include snippets/common/general.conf;
+
+  # robots config
+  include snippets/common/robots-org.conf;
+
 }
 
 # HTTP redirect

diff --git a/sites-enabled/15-chinese.freecodecamp.org.conf b/sites-enabled/15-chinese.freecodecamp.org.conf
@@ -50,6 +50,9 @@ server {
   # additional config
   include snippets/common/general.conf;
 
+  # robots config
+  include snippets/common/robots-org.conf;
+
   # error page
   error_page 503 @maintenance;
   location @maintenance {

diff --git a/sites-enabled/20-www.freecodecamp.dev.conf b/sites-enabled/20-www.freecodecamp.dev.conf
@@ -73,11 +73,6 @@ server {
     add_header Content-Type text/plain;
   }
 
-  # Do not index on search engines
-  location /robots.txt {
-    return 200 "User-agent: YandexDisallow: /\nUser-agent: *\nDisallow: /";
-  }
-
   # redirect /forum/ and /forum to subdomain
   rewrite ^/forum(?:|/(.*))$ https://forum.freecodecamp.dev/$1 permanent;
 
@@ -234,6 +229,9 @@ server {
 
   # additional config
   include snippets/common/general.conf;
+
+  # robots config
+  include snippets/common/robots-dev.conf;
 
   # error page
   error_page 503 @maintenance;

diff --git a/sites-enabled/21-api.freecodecamp.dev.conf b/sites-enabled/21-api.freecodecamp.dev.conf
@@ -14,11 +14,6 @@ server {
   # security
   include snippets/common/security.conf;
 
-  # Do not index on search engines
-  location /robots.txt {
-    return 200 "User-agent: YandexDisallow: /\nUser-agent: *\nDisallow: /";
-  }
-
   # reverse proxy api
   location / {
     # Do not index on search engines
@@ -33,6 +28,10 @@ server {
 
   # additional config
   include snippets/common/general.conf;
+
+  # robots config
+  include snippets/common/robots-dev.conf;
+
 }
 
 # HTTP redirect

diff --git a/sites-enabled/30-chinese.freecodecamp.dev.conf b/sites-enabled/30-chinese.freecodecamp.dev.conf
@@ -23,10 +23,6 @@ server {
     add_header Content-Type text/plain;
   }
 
-  location /robots.txt {
-    return 200 "User-agent: YandexDisallow: /\nUser-agent: *\nDisallow: /";
-  }
-
   # reverse proxy news - Ghost
   location ^~ /news/ghost/api/v3/admin/snippets/ {
     return 500;
@@ -51,6 +47,9 @@ server {
   # additional config
   include snippets/common/general.conf;
 
+  # robots config
+  include snippets/common/robots-dev.conf;
+
   # error page
   error_page 503 @maintenance;
   location @maintenance {

diff --git a/snippets/common/general.conf b/snippets/common/general.conf
@@ -4,8 +4,17 @@ location = /favicon.ico {
   access_log off;
 }
 
+map $host $robots_txt {
+  www.freecodecamp.org "etc/nginx/snippets/files/10-www.freecodecamp.org.robots.txt";
+  default "etc/nginx/snippets/files/00-default.robots.txt";
+}
+
 # robots.txt
 location = /robots.txt {
+  alias $robots_txt;
+  expires 15d;
+  add_header Cache-Control "public";
+
   log_not_found off;
   access_log off;
 }

diff --git a/snippets/files/00-default.robots.txt b/snippets/files/00-default.robots.txt
@@ -0,0 +1,2 @@
+User-Agent: *
+Disallow: /
diff --git a/snippets/files/10-www.freecodecamp.org.robots.txt b/snippets/files/10-www.freecodecamp.org.robots.txt
@@ -0,0 +1,20 @@
+User-Agent: *
+Allow: /
+Disallow: /learn
+
+User-agent: GPTBot
+Disallow: /
+
+User-agent: ChatGPT-User
+Disallow: /
+
+User-agent: anthropic-ai
+Disallow: /
+
+User-agent: Google-Extended
+Disallow: /
+
+User-agent: Bytespider
+Disallow: /
+
+Sitemap: https://www.freecodecamp.org/news/sitemap.xml