From 450a925c5c0d334b72a6edee2f666a33a6535fc9 Mon Sep 17 00:00:00 2001 From: github-actions <41898282+github-actions[bot]@users.noreply.github.com> Date: Sat, 2 Nov 2024 01:04:23 +0000 Subject: [PATCH] chore: generate robots.txt --- base/var/www/localhost/htdocs/robots.txt | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/base/var/www/localhost/htdocs/robots.txt b/base/var/www/localhost/htdocs/robots.txt index 58e5acb9..c70d2d4b 100644 --- a/base/var/www/localhost/htdocs/robots.txt +++ b/base/var/www/localhost/htdocs/robots.txt @@ -1,3 +1,9 @@ +# AI Data Scraper +# https://darkvisitors.com/agents/ai2bot + +User-agent: AI2Bot +Disallow: / + # AI Search Crawler # https://darkvisitors.com/agents/amazonbot @@ -64,6 +70,12 @@ Disallow: / User-agent: Diffbot Disallow: / +# AI Assistant +# https://darkvisitors.com/agents/duckassistbot + +User-agent: DuckAssistBot +Disallow: / + # AI Data Scraper # https://darkvisitors.com/agents/facebookbot