fix(robots.txt): add config serving bots

freeCodeCamp · Jan 6, 2025 · e63035f · e63035f
1 parent d4cc80e
commit e63035f
Show file tree

Hide file tree

Showing 10 changed files with 82 additions and 102 deletions.
diff --git a/sites-enabled/10-www.freecodecamp.org.conf b/sites-enabled/10-www.freecodecamp.org.conf
@@ -483,6 +483,16 @@ server {
     return 302 https://freecodecamp.typeform.com/to/X3VXzZ;
   }
 
+  # robots.txt
+  location = /robots.txt {
+    alias /etc/nginx/snippets/files/10-www.freecodecamp.org.robots.txt;
+    expires 15d;
+    add_header Cache-Control "public";
+
+    log_not_found off;
+    access_log off;
+  }
+
   #legacy redirects
   include snippets/common/legacy-redirects.conf;
 

diff --git a/sites-enabled/11-api.freecodecamp.org.conf b/sites-enabled/11-api.freecodecamp.org.conf
@@ -24,6 +24,16 @@ server {
     include snippets/common/proxy-params.conf;
   }
 
+  # robots.txt
+  location = /robots.txt {
+    alias /etc/nginx/snippets/files/00-default.robots.txt;
+    expires 15d;
+    add_header Cache-Control "public";
+
+    log_not_found off;
+    access_log off;
+  }
+
   # additional config
   include snippets/common/general.conf;
 }

diff --git a/sites-enabled/15-chinese.freecodecamp.org.conf b/sites-enabled/15-chinese.freecodecamp.org.conf
@@ -42,6 +42,16 @@ server {
     include snippets/common/proxy-params.conf;
   }
 
+  # robots.txt
+  location = /robots.txt {
+    alias /etc/nginx/snippets/files/00-default.robots.txt;
+    expires 15d;
+    add_header Cache-Control "public";
+
+    log_not_found off;
+    access_log off;
+  }
+
   # catch everything else
   location ~ ^/(.*)$ {
     return 302 https://www.freecodecamp.org/chinese/$1;

diff --git a/sites-enabled/20-www.freecodecamp.dev.conf b/sites-enabled/20-www.freecodecamp.dev.conf
@@ -73,19 +73,11 @@ server {
     add_header Content-Type text/plain;
   }
 
-  # Do not index on search engines
-  location /robots.txt {
-    return 200 "User-agent: YandexDisallow: /\nUser-agent: *\nDisallow: /";
-  }
-
   # redirect /forum/ and /forum to subdomain
   rewrite ^/forum(?:|/(.*))$ https://forum.freecodecamp.dev/$1 permanent;
 
   # reverse proxy client
   location / {
-    # Do not index on search engines
-    include snippets/common/add-noindex-headers.conf;
-
     # app specific configs
     include snippets/app/learn.dev.conf;
 
@@ -95,9 +87,6 @@ server {
 
   # reverse proxy api
   location /api {
-    # Do not index on search engines
-    include snippets/common/add-noindex-headers.conf;
-
     if (-f /etc/nginx/maintenance/API.txt) {
       return 503;
     }
@@ -107,9 +96,6 @@ server {
 
   # reverse proxy news
   location /news {
-    # Do not index on search engines
-    include snippets/common/add-noindex-headers.conf;
-
     if (-f /etc/nginx/maintenance/NEWS-ENG.txt) {
       return 503;
     }
@@ -125,109 +111,86 @@ server {
 
   # reverse proxy client (chinese)
   location /chinese/ {
-    # Do not index on search engines
-    include snippets/common/add-noindex-headers.conf;
-
     # the trailing hash is needed here.
     proxy_pass http://client-chn/;
     include snippets/common/proxy-params.conf;
   }
 
   # catch common subpath for langs that we have special configs for
   location ~ ^/spanish(?:|/(.*))$ {
-    # Do not index on search engines
-    include snippets/common/add-noindex-headers.conf;
-
     return 302 https://www.freecodecamp.dev/espanol/$1;
   }
 
   # reverse proxy client (espanol)
   location /espanol/ {
-    # Do not index on search engines
-    include snippets/common/add-noindex-headers.conf;
-
     # the trailing hash is needed here.
     proxy_pass http://client-esp/;
     include snippets/common/proxy-params.conf;
   }
 
   # reverse proxy client (chinese-traditional)
   location /chinese-traditional/ {
-    # Do not index on search engines
-    include snippets/common/add-noindex-headers.conf;
-
     # the trailing hash is needed here.
     proxy_pass http://client-cnt/;
     include snippets/common/proxy-params.conf;
   }
 
   # reverse proxy client (italian)
   location /italian/ {
-    # Do not index on search engines
-    include snippets/common/add-noindex-headers.conf;
-
     # the trailing hash is needed here.
     proxy_pass http://client-ita/;
     include snippets/common/proxy-params.conf;
   }
 
   # reverse proxy client (portuguese)
   location /portuguese/ {
-    # Do not index on search engines
-    include snippets/common/add-noindex-headers.conf;
-
     # the trailing hash is needed here.
     proxy_pass http://client-por/;
     include snippets/common/proxy-params.conf;
   }
 
   # reverse proxy client (japanese)
   location /japanese/ {
-    # Do not index on search engines
-    include snippets/common/add-noindex-headers.conf;
-
     # the trailing hash is needed here.
     proxy_pass http://client-jpn/;
     include snippets/common/proxy-params.conf;
   }
 
   # reverse proxy client (ukrainian)
   location /ukrainian/ {
-    # Do not index on search engines
-    include snippets/common/add-noindex-headers.conf;
-
     # the trailing hash is needed here.
     proxy_pass http://client-ukr/;
     include snippets/common/proxy-params.conf;
   }
 
   # reverse proxy client (german)
   location /german/ {
-    # Do not index on search engines
-    include snippets/common/add-noindex-headers.conf;
-
     # the trailing hash is needed here.
     proxy_pass http://client-ger/;
     include snippets/common/proxy-params.conf;
   }
 
   # reverse proxy client (swahili)
   location /swahili/ {
-    # Do not index on search engines
-    include snippets/common/add-noindex-headers.conf;
-
     # the trailing hash is needed here.
     proxy_pass http://client-swa/;
     include snippets/common/proxy-params.conf;
   }
 
   location /partners {
-    # Do not index on search engines
-    include snippets/common/add-noindex-headers.conf;
-
     return 302 https://freecodecamp.typeform.com/to/X3VXzZ;
   }
 
+  # robots.txt
+  location = /robots.txt {
+    alias /etc/nginx/snippets/files/00-default.robots.txt;
+    expires 15d;
+    add_header Cache-Control "public";
+
+    log_not_found off;
+    access_log off;
+  }
+
   #legacy redirects
   include snippets/common/legacy-redirects.conf;
 
@@ -237,9 +200,6 @@ server {
   # error page
   error_page 503 @maintenance;
   location @maintenance {
-    # Do not index on search engines
-    include snippets/common/add-noindex-headers.conf;
-
     root /var/www/html/error-pages;
     rewrite ^(.*)$ /maintenance.html break;
   }
@@ -255,8 +215,6 @@ server {
     if (-f /etc/nginx/maintenance/NEWS-ENG.txt) {
       return 503;
     }
-    # Do not index on search engines
-    include snippets/common/add-noindex-headers.conf;
     access_log /var/log/nginx/access.log combined if=$log_4xx_5xx;
     proxy_pass http://news-eng;
     include snippets/common/proxy-params.conf;
@@ -271,8 +229,6 @@ server {
   #   if (-f /etc/nginx/maintenance/NEWS-CHN.txt) {
   #     return 503;
   #   }
-  #   # Do not index on search engines
-  #   include snippets/common/add-noindex-headers.conf;
   #   access_log /var/log/nginx/access.log combined if=$log_4xx_5xx;
   #   proxy_pass http://news-chn;
   #   include snippets/common/proxy-params.conf;
@@ -282,8 +238,6 @@ server {
     if (-f /etc/nginx/maintenance/NEWS-CHN.txt) {
       return 503;
     }
-    # Do not index on search engines
-    include snippets/common/add-noindex-headers.conf;
     access_log /var/log/nginx/access.log combined if=$log_4xx_5xx;
 
     proxy_cache NEWS_CACHE_DEV_CHN;
@@ -303,8 +257,6 @@ server {
     if (-f /etc/nginx/maintenance/NEWS-ESP.txt) {
       return 503;
     }
-    # Do not index on search engines
-    include snippets/common/add-noindex-headers.conf;
     access_log /var/log/nginx/access.log combined if=$log_4xx_5xx;
     proxy_pass http://news-esp;
     include snippets/common/proxy-params.conf;
@@ -315,8 +267,6 @@ server {
     if (-f /etc/nginx/maintenance/NEWS-ESP.txt) {
       return 503;
     }
-    # Do not index on search engines
-    include snippets/common/add-noindex-headers.conf;
     access_log /var/log/nginx/access.log combined if=$log_4xx_5xx;
 
     proxy_cache NEWS_CACHE_DEV_ESP;
@@ -336,8 +286,6 @@ server {
     if (-f /etc/nginx/maintenance/NEWS-ITA.txt) {
       return 503;
     }
-    # Do not index on search engines
-    include snippets/common/add-noindex-headers.conf;
     access_log /var/log/nginx/access.log combined if=$log_4xx_5xx;
     proxy_pass http://news-ita;
     include snippets/common/proxy-params.conf;
@@ -348,8 +296,6 @@ server {
     if (-f /etc/nginx/maintenance/NEWS-ITA.txt) {
       return 503;
     }
-    # Do not index on search engines
-    include snippets/common/add-noindex-headers.conf;
     access_log /var/log/nginx/access.log combined if=$log_4xx_5xx;
 
     proxy_cache NEWS_CACHE_DEV_ITA;
@@ -369,8 +315,6 @@ server {
     if (-f /etc/nginx/maintenance/NEWS-POR.txt) {
       return 503;
     }
-    # Do not index on search engines
-    include snippets/common/add-noindex-headers.conf;
     access_log /var/log/nginx/access.log combined if=$log_4xx_5xx;
     proxy_pass http://news-por;
     include snippets/common/proxy-params.conf;
@@ -381,8 +325,6 @@ server {
     if (-f /etc/nginx/maintenance/NEWS-POR.txt) {
       return 503;
     }
-    # Do not index on search engines
-    include snippets/common/add-noindex-headers.conf;
     access_log /var/log/nginx/access.log combined if=$log_4xx_5xx;
 
     proxy_cache NEWS_CACHE_DEV_POR;
@@ -402,8 +344,6 @@ server {
     if (-f /etc/nginx/maintenance/NEWS-JPN.txt) {
       return 503;
     }
-    # Do not index on search engines
-    include snippets/common/add-noindex-headers.conf;
     access_log /var/log/nginx/access.log combined if=$log_4xx_5xx;
     proxy_pass http://news-jpn;
     include snippets/common/proxy-params.conf;
@@ -414,8 +354,6 @@ server {
     if (-f /etc/nginx/maintenance/NEWS-JPN.txt) {
       return 503;
     }
-    # Do not index on search engines
-    include snippets/common/add-noindex-headers.conf;
     access_log /var/log/nginx/access.log combined if=$log_4xx_5xx;
 
     proxy_cache NEWS_CACHE_DEV_JPN;
@@ -435,8 +373,6 @@ server {
     if (-f /etc/nginx/maintenance/NEWS-UKR.txt) {
       return 503;
     }
-    # Do not index on search engines
-    include snippets/common/add-noindex-headers.conf;
     access_log /var/log/nginx/access.log combined if=$log_4xx_5xx;
     proxy_pass http://news-ukr;
     include snippets/common/proxy-params.conf;
@@ -447,8 +383,6 @@ server {
     if (-f /etc/nginx/maintenance/NEWS-UKR.txt) {
       return 503;
     }
-    # Do not index on search engines
-    include snippets/common/add-noindex-headers.conf;
     access_log /var/log/nginx/access.log combined if=$log_4xx_5xx;
 
     proxy_cache NEWS_CACHE_DEV_UKR;
@@ -468,8 +402,6 @@ server {
     if (-f /etc/nginx/maintenance/NEWS-KOR.txt) {
       return 503;
     }
-    # Do not index on search engines
-    include snippets/common/add-noindex-headers.conf;
     access_log /var/log/nginx/access.log combined if=$log_4xx_5xx;
     proxy_pass http://news-kor;
     include snippets/common/proxy-params.conf;
@@ -480,8 +412,6 @@ server {
     if (-f /etc/nginx/maintenance/NEWS-KOR.txt) {
       return 503;
     }
-    # Do not index on search engines
-    include snippets/common/add-noindex-headers.conf;
     access_log /var/log/nginx/access.log combined if=$log_4xx_5xx;
 
     proxy_cache NEWS_CACHE_DEV_KOR;

diff --git a/sites-enabled/21-api.freecodecamp.dev.conf b/sites-enabled/21-api.freecodecamp.dev.conf
@@ -14,15 +14,8 @@ server {
   # security
   include snippets/common/security.conf;
 
-  # Do not index on search engines
-  location /robots.txt {
-    return 200 "User-agent: YandexDisallow: /\nUser-agent: *\nDisallow: /";
-  }
-
   # reverse proxy api
   location / {
-    # Do not index on search engines
-    include snippets/common/add-noindex-headers.conf;
 
     if (-f /etc/nginx/maintenance/API.txt) {
       return 503;
@@ -32,6 +25,16 @@ server {
     include snippets/common/proxy-params.conf;
   }
 
+  # robots.txt
+  location = /robots.txt {
+    alias /etc/nginx/snippets/files/00-default.robots.txt;
+    expires 15d;
+    add_header Cache-Control "public";
+
+    log_not_found off;
+    access_log off;
+  }
+
   # additional config
   include snippets/common/general.conf;
 }