Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix(robots,security): add config serving bots and security #106

Merged
merged 2 commits into from
Jan 6, 2025
Merged

fix(robots,security): add config serving bots and security #106

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
d4cc80e
fix(security.txt): add configs
raisedadead Jan 3, 2025
e63035f
fix(robots.txt): add config serving bots
ojeytonwilliams Feb 20, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions sites-enabled/10-www.freecodecamp.org.conf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -483,6 +483,16 @@ server {
return 302 https://freecodecamp.typeform.com/to/X3VXzZ;
}

# robots.txt
location = /robots.txt {
alias /etc/nginx/snippets/files/10-www.freecodecamp.org.robots.txt;
expires 15d;
add_header Cache-Control "public";

log_not_found off;
access_log off;
}

#legacy redirects
include snippets/common/legacy-redirects.conf;

Expand Down
10 changes: 10 additions & 0 deletions sites-enabled/11-api.freecodecamp.org.conf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,16 @@ server {
include snippets/common/proxy-params.conf;
}

# robots.txt
location = /robots.txt {
alias /etc/nginx/snippets/files/00-default.robots.txt;
expires 15d;
add_header Cache-Control "public";

log_not_found off;
access_log off;
}

# additional config
include snippets/common/general.conf;
}
Expand Down
10 changes: 10 additions & 0 deletions sites-enabled/15-chinese.freecodecamp.org.conf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,16 @@ server {
include snippets/common/proxy-params.conf;
}

# robots.txt
location = /robots.txt {
alias /etc/nginx/snippets/files/00-default.robots.txt;
expires 15d;
add_header Cache-Control "public";

log_not_found off;
access_log off;
}

# catch everything else
location ~ ^/(.*)$ {
return 302 https://www.freecodecamp.org/chinese/$1;
Expand Down
90 changes: 10 additions & 80 deletions sites-enabled/20-www.freecodecamp.dev.conf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -73,19 +73,11 @@ server {
add_header Content-Type text/plain;
}

# Do not index on search engines
location /robots.txt {
return 200 "User-agent: YandexDisallow: /\nUser-agent: *\nDisallow: /";
}

# redirect /forum/ and /forum to subdomain
rewrite ^/forum(?:|/(.*))$ https://forum.freecodecamp.dev/$1 permanent;

# reverse proxy client
location / {
# Do not index on search engines
include snippets/common/add-noindex-headers.conf;

# app specific configs
include snippets/app/learn.dev.conf;

Expand All @@ -95,9 +87,6 @@ server {

# reverse proxy api
location /api {
# Do not index on search engines
include snippets/common/add-noindex-headers.conf;

if (-f /etc/nginx/maintenance/API.txt) {
return 503;
}
Expand All @@ -107,9 +96,6 @@ server {

# reverse proxy news
location /news {
# Do not index on search engines
include snippets/common/add-noindex-headers.conf;

if (-f /etc/nginx/maintenance/NEWS-ENG.txt) {
return 503;
}
Expand All @@ -125,109 +111,86 @@ server {

# reverse proxy client (chinese)
location /chinese/ {
# Do not index on search engines
include snippets/common/add-noindex-headers.conf;

# the trailing hash is needed here.
proxy_pass http://client-chn/;
include snippets/common/proxy-params.conf;
}

# catch common subpath for langs that we have special configs for
location ~ ^/spanish(?:|/(.*))$ {
# Do not index on search engines
include snippets/common/add-noindex-headers.conf;

return 302 https://www.freecodecamp.dev/espanol/$1;
}

# reverse proxy client (espanol)
location /espanol/ {
# Do not index on search engines
include snippets/common/add-noindex-headers.conf;

# the trailing hash is needed here.
proxy_pass http://client-esp/;
include snippets/common/proxy-params.conf;
}

# reverse proxy client (chinese-traditional)
location /chinese-traditional/ {
# Do not index on search engines
include snippets/common/add-noindex-headers.conf;

# the trailing hash is needed here.
proxy_pass http://client-cnt/;
include snippets/common/proxy-params.conf;
}

# reverse proxy client (italian)
location /italian/ {
# Do not index on search engines
include snippets/common/add-noindex-headers.conf;

# the trailing hash is needed here.
proxy_pass http://client-ita/;
include snippets/common/proxy-params.conf;
}

# reverse proxy client (portuguese)
location /portuguese/ {
# Do not index on search engines
include snippets/common/add-noindex-headers.conf;

# the trailing hash is needed here.
proxy_pass http://client-por/;
include snippets/common/proxy-params.conf;
}

# reverse proxy client (japanese)
location /japanese/ {
# Do not index on search engines
include snippets/common/add-noindex-headers.conf;

# the trailing hash is needed here.
proxy_pass http://client-jpn/;
include snippets/common/proxy-params.conf;
}

# reverse proxy client (ukrainian)
location /ukrainian/ {
# Do not index on search engines
include snippets/common/add-noindex-headers.conf;

# the trailing hash is needed here.
proxy_pass http://client-ukr/;
include snippets/common/proxy-params.conf;
}

# reverse proxy client (german)
location /german/ {
# Do not index on search engines
include snippets/common/add-noindex-headers.conf;

# the trailing hash is needed here.
proxy_pass http://client-ger/;
include snippets/common/proxy-params.conf;
}

# reverse proxy client (swahili)
location /swahili/ {
# Do not index on search engines
include snippets/common/add-noindex-headers.conf;

# the trailing hash is needed here.
proxy_pass http://client-swa/;
include snippets/common/proxy-params.conf;
}

location /partners {
# Do not index on search engines
include snippets/common/add-noindex-headers.conf;

return 302 https://freecodecamp.typeform.com/to/X3VXzZ;
}

# robots.txt
location = /robots.txt {
alias /etc/nginx/snippets/files/00-default.robots.txt;
expires 15d;
add_header Cache-Control "public";

log_not_found off;
access_log off;
}

#legacy redirects
include snippets/common/legacy-redirects.conf;

Expand All @@ -237,9 +200,6 @@ server {
# error page
error_page 503 @maintenance;
location @maintenance {
# Do not index on search engines
include snippets/common/add-noindex-headers.conf;

root /var/www/html/error-pages;
rewrite ^(.*)$ /maintenance.html break;
}
Expand All @@ -255,8 +215,6 @@ server {
if (-f /etc/nginx/maintenance/NEWS-ENG.txt) {
return 503;
}
# Do not index on search engines
include snippets/common/add-noindex-headers.conf;
access_log /var/log/nginx/access.log combined if=$log_4xx_5xx;
proxy_pass http://news-eng;
include snippets/common/proxy-params.conf;
Expand All @@ -271,8 +229,6 @@ server {
# if (-f /etc/nginx/maintenance/NEWS-CHN.txt) {
# return 503;
# }
# # Do not index on search engines
# include snippets/common/add-noindex-headers.conf;
# access_log /var/log/nginx/access.log combined if=$log_4xx_5xx;
# proxy_pass http://news-chn;
# include snippets/common/proxy-params.conf;
Expand All @@ -282,8 +238,6 @@ server {
if (-f /etc/nginx/maintenance/NEWS-CHN.txt) {
return 503;
}
# Do not index on search engines
include snippets/common/add-noindex-headers.conf;
access_log /var/log/nginx/access.log combined if=$log_4xx_5xx;

proxy_cache NEWS_CACHE_DEV_CHN;
Expand All @@ -303,8 +257,6 @@ server {
if (-f /etc/nginx/maintenance/NEWS-ESP.txt) {
return 503;
}
# Do not index on search engines
include snippets/common/add-noindex-headers.conf;
access_log /var/log/nginx/access.log combined if=$log_4xx_5xx;
proxy_pass http://news-esp;
include snippets/common/proxy-params.conf;
Expand All @@ -315,8 +267,6 @@ server {
if (-f /etc/nginx/maintenance/NEWS-ESP.txt) {
return 503;
}
# Do not index on search engines
include snippets/common/add-noindex-headers.conf;
access_log /var/log/nginx/access.log combined if=$log_4xx_5xx;

proxy_cache NEWS_CACHE_DEV_ESP;
Expand All @@ -336,8 +286,6 @@ server {
if (-f /etc/nginx/maintenance/NEWS-ITA.txt) {
return 503;
}
# Do not index on search engines
include snippets/common/add-noindex-headers.conf;
access_log /var/log/nginx/access.log combined if=$log_4xx_5xx;
proxy_pass http://news-ita;
include snippets/common/proxy-params.conf;
Expand All @@ -348,8 +296,6 @@ server {
if (-f /etc/nginx/maintenance/NEWS-ITA.txt) {
return 503;
}
# Do not index on search engines
include snippets/common/add-noindex-headers.conf;
access_log /var/log/nginx/access.log combined if=$log_4xx_5xx;

proxy_cache NEWS_CACHE_DEV_ITA;
Expand All @@ -369,8 +315,6 @@ server {
if (-f /etc/nginx/maintenance/NEWS-POR.txt) {
return 503;
}
# Do not index on search engines
include snippets/common/add-noindex-headers.conf;
access_log /var/log/nginx/access.log combined if=$log_4xx_5xx;
proxy_pass http://news-por;
include snippets/common/proxy-params.conf;
Expand All @@ -381,8 +325,6 @@ server {
if (-f /etc/nginx/maintenance/NEWS-POR.txt) {
return 503;
}
# Do not index on search engines
include snippets/common/add-noindex-headers.conf;
access_log /var/log/nginx/access.log combined if=$log_4xx_5xx;

proxy_cache NEWS_CACHE_DEV_POR;
Expand All @@ -402,8 +344,6 @@ server {
if (-f /etc/nginx/maintenance/NEWS-JPN.txt) {
return 503;
}
# Do not index on search engines
include snippets/common/add-noindex-headers.conf;
access_log /var/log/nginx/access.log combined if=$log_4xx_5xx;
proxy_pass http://news-jpn;
include snippets/common/proxy-params.conf;
Expand All @@ -414,8 +354,6 @@ server {
if (-f /etc/nginx/maintenance/NEWS-JPN.txt) {
return 503;
}
# Do not index on search engines
include snippets/common/add-noindex-headers.conf;
access_log /var/log/nginx/access.log combined if=$log_4xx_5xx;

proxy_cache NEWS_CACHE_DEV_JPN;
Expand All @@ -435,8 +373,6 @@ server {
if (-f /etc/nginx/maintenance/NEWS-UKR.txt) {
return 503;
}
# Do not index on search engines
include snippets/common/add-noindex-headers.conf;
access_log /var/log/nginx/access.log combined if=$log_4xx_5xx;
proxy_pass http://news-ukr;
include snippets/common/proxy-params.conf;
Expand All @@ -447,8 +383,6 @@ server {
if (-f /etc/nginx/maintenance/NEWS-UKR.txt) {
return 503;
}
# Do not index on search engines
include snippets/common/add-noindex-headers.conf;
access_log /var/log/nginx/access.log combined if=$log_4xx_5xx;

proxy_cache NEWS_CACHE_DEV_UKR;
Expand All @@ -468,8 +402,6 @@ server {
if (-f /etc/nginx/maintenance/NEWS-KOR.txt) {
return 503;
}
# Do not index on search engines
include snippets/common/add-noindex-headers.conf;
access_log /var/log/nginx/access.log combined if=$log_4xx_5xx;
proxy_pass http://news-kor;
include snippets/common/proxy-params.conf;
Expand All @@ -480,8 +412,6 @@ server {
if (-f /etc/nginx/maintenance/NEWS-KOR.txt) {
return 503;
}
# Do not index on search engines
include snippets/common/add-noindex-headers.conf;
access_log /var/log/nginx/access.log combined if=$log_4xx_5xx;

proxy_cache NEWS_CACHE_DEV_KOR;
Expand Down
17 changes: 10 additions & 7 deletions sites-enabled/21-api.freecodecamp.dev.conf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,15 +14,8 @@ server {
# security
include snippets/common/security.conf;

# Do not index on search engines
location /robots.txt {
return 200 "User-agent: YandexDisallow: /\nUser-agent: *\nDisallow: /";
}

# reverse proxy api
location / {
# Do not index on search engines
include snippets/common/add-noindex-headers.conf;

if (-f /etc/nginx/maintenance/API.txt) {
return 503;
Expand All @@ -32,6 +25,16 @@ server {
include snippets/common/proxy-params.conf;
}

# robots.txt
location = /robots.txt {
alias /etc/nginx/snippets/files/00-default.robots.txt;
expires 15d;
add_header Cache-Control "public";

log_not_found off;
access_log off;
}

# additional config
include snippets/common/general.conf;
}
Expand Down
Loading