From 5d112aa565bb88d64b95c8ec687e0fc41de7dbae Mon Sep 17 00:00:00 2001 From: mitchellkrogza Date: Wed, 10 Apr 2024 12:23:49 +0000 Subject: [PATCH] V4.2024.04.4424 [ci skip] --- AUTO-CONFIGURATION.md | 4 +- MANUAL-CONFIGURATION.md | 4 +- README.md | 4 +- _generator_lists/bad-user-agents.list | 4 +- _google_analytics_ghost_spam/README.md | 4 +- .../etc/nginx/conf.d/globalblacklist.conf | 8 +- conf.d/globalblacklist-testing-version.conf | 8 +- conf.d/globalblacklist.conf | 8 +- dev-tools/_robots_input/robots-input.txt | 2 + dev-tools/buildnumber | 2 +- dev-tools/globalblacklist-testing.template | 8 +- dev-tools/globalblacklist.template | 8 +- .../bots.d/whitelist-ips.conf | 2 +- .../conf.d/globalblacklist.conf | 8 +- .../bots.d/blacklist-ips.conf | 2 +- .../bots.d/whitelist-ips.conf | 2 +- .../conf.d/globalblacklist.conf | 8 +- .../bots.d/blacklist-ips.conf | 2 +- .../bots.d/whitelist-ips.conf | 2 +- .../conf.d/globalblacklist.conf | 8 +- .../bots.d/blacklist-ips.conf | 2 +- .../bots.d/blacklist-user-agents.conf | 2 + .../bots.d/whitelist-ips.conf | 2 +- .../conf.d/globalblacklist.conf | 8 +- .../bots.d/whitelist-ips.conf | 2 +- .../conf.d/globalblacklist.conf | 8 +- .../bots.d/whitelist-ips.conf | 2 +- .../conf.d/globalblacklist.conf | 8 +- dev-tools/test_units/blacklist-ips.conf | 2 +- .../test_units/blacklist-user-agents.conf | 2 + .../random-bots-for-test-quick.list | 20 +- .../random-bots-for-whitelist-test.list | 164 +++++++-------- .../random-referrers-for-test-quick.list | 20 +- .../random-referrers-for-whitelist-test.list | 194 +++++++++--------- dev-tools/test_units/whitelist-ips.conf | 2 +- dev-tools/tmp/robots.txt | 10 +- robots.txt/robots.txt | 10 +- 37 files changed, 296 insertions(+), 260 deletions(-) diff --git a/AUTO-CONFIGURATION.md b/AUTO-CONFIGURATION.md index 78c7319df..2f8cf75ee 100755 --- a/AUTO-CONFIGURATION.md +++ b/AUTO-CONFIGURATION.md @@ -4,9 +4,9 @@ ### PLEASE READ CONFIGURATION INSTRUCTIONS BELOW THOROUGHLY :exclamation: _______________ -#### Version: V4.2024.04.4423 +#### Version: V4.2024.04.4424 #### Bad Referrers Blocked: [7104](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-referrers.list) -#### Bad User-Agents (Bots) Blocked: [655](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) +#### Bad User-Agents (Bots) Blocked: [657](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) #### Fake Googlebots Blocked: [217](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/fake-googlebots.list) ____________________ diff --git a/MANUAL-CONFIGURATION.md b/MANUAL-CONFIGURATION.md index 73928fbee..2f9f540ce 100755 --- a/MANUAL-CONFIGURATION.md +++ b/MANUAL-CONFIGURATION.md @@ -4,9 +4,9 @@ ### PLEASE READ CONFIGURATION INSTRUCTIONS BELOW THOROUGHLY :exclamation: _______________ -#### Version: V4.2024.04.4423 +#### Version: V4.2024.04.4424 #### Bad Referrers Blocked: [7104](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-referrers.list) -#### Bad User-Agents (Bots) Blocked: [655](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) +#### Bad User-Agents (Bots) Blocked: [657](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) #### Fake Googlebots Blocked: [217](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/fake-googlebots.list) ____________________ diff --git a/README.md b/README.md index faeb550f3..6204aac25 100755 --- a/README.md +++ b/README.md @@ -14,9 +14,9 @@ ##### The Ultimate Nginx Bad Bot, User-Agent, Spam Referrer Blocker, Adware, Malware and Ransomware Blocker, Clickjacking Blocker, Click Re-Directing Blocker, SEO Companies and Bad IP Blocker with Anti DDOS System, Nginx Rate Limiting and Wordpress Theme Detector Blocking. Stop and Block all kinds of bad internet traffic even Fake Googlebots from ever reaching your web sites. [PLEASE SEE: Definition of Bad Bots](#define-bad-bots) _______________ -#### Version: V4.2024.04.4423 +#### Version: V4.2024.04.4424 #### Bad Referrers Blocked: [7104](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-referrers.list) -#### Bad User-Agents (Bots) Blocked: [655](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) +#### Bad User-Agents (Bots) Blocked: [657](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) #### Fake Googlebots Blocked: [217](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/fake-googlebots.list) ____________________ diff --git a/_generator_lists/bad-user-agents.list b/_generator_lists/bad-user-agents.list index d75891dad..544e05d14 100755 --- a/_generator_lists/bad-user-agents.list +++ b/_generator_lists/bad-user-agents.list @@ -1,5 +1,3 @@ -AwarioBot -awario.com 01h4x.com 360Spider 404checker @@ -32,6 +30,7 @@ Aspiegel Asterias Atomseobot Attach +AwarioBot AwarioRssBot AwarioSmartBot BBBike @@ -581,6 +580,7 @@ archive.org_bot arquivo-web-crawler arquivo.pt autoemailspider +awario.com backlink-check cah.io.community check1.exe diff --git a/_google_analytics_ghost_spam/README.md b/_google_analytics_ghost_spam/README.md index 0dd919245..ea511bc21 100644 --- a/_google_analytics_ghost_spam/README.md +++ b/_google_analytics_ghost_spam/README.md @@ -3,9 +3,9 @@ # EASY CONFIGURATION INSTRUCTIONS FOR STOPPING GOOGLE ANALYTICS "GHOST" SPAM _______________ -#### Version: V4.2024.04.4423 +#### Version: V4.2024.04.4424 #### Bad Referrers Blocked: [7104](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-referrers.list) -#### Bad User-Agents (Bots) Blocked: [655](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) +#### Bad User-Agents (Bots) Blocked: [657](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) #### Fake Googlebots Blocked: [217](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/fake-googlebots.list) ____________________ diff --git a/_sample_config_files/Engintron_for_cPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf b/_sample_config_files/Engintron_for_cPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf index 8b2f8fa34..0801f8313 100755 --- a/_sample_config_files/Engintron_for_cPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf +++ b/_sample_config_files/Engintron_for_cPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2024.04.4423 -### Updated: Wed Apr 10 10:03:14 UTC 2024 +### Version: V4.2024.04.4424 +### Updated: Wed Apr 10 12:20:08 UTC 2024 ### Bad Referrer Count: 7104 -### Bad Bot Count: 655 +### Bad Bot Count: 657 ################################################### ### VERSION INFORMATION ## @@ -179,6 +179,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)Asterias(?:\b)" 3; "~*(?:\b)Atomseobot(?:\b)" 3; "~*(?:\b)Attach(?:\b)" 3; + "~*(?:\b)AwarioBot(?:\b)" 3; "~*(?:\b)AwarioRssBot(?:\b)" 3; "~*(?:\b)AwarioSmartBot(?:\b)" 3; "~*(?:\b)BBBike(?:\b)" 3; @@ -728,6 +729,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)arquivo-web-crawler(?:\b)" 3; "~*(?:\b)arquivo.pt(?:\b)" 3; "~*(?:\b)autoemailspider(?:\b)" 3; + "~*(?:\b)awario.com(?:\b)" 3; "~*(?:\b)backlink-check(?:\b)" 3; "~*(?:\b)cah.io.community(?:\b)" 3; "~*(?:\b)check1.exe(?:\b)" 3; diff --git a/conf.d/globalblacklist-testing-version.conf b/conf.d/globalblacklist-testing-version.conf index bfec8dd08..3f9375644 100644 --- a/conf.d/globalblacklist-testing-version.conf +++ b/conf.d/globalblacklist-testing-version.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2024.04.4423 -### Updated: Wed Apr 10 10:03:14 UTC 2024 +### Version: V3.2024.04.4424 +### Updated: Wed Apr 10 12:20:09 UTC 2024 ### Bad Referrer Count: 7104 -### Bad Bot Count: 655 +### Bad Bot Count: 657 ################################################### ### VERSION INFORMATION ## @@ -179,6 +179,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)Asterias(?:\b)" 3; "~*(?:\b)Atomseobot(?:\b)" 3; "~*(?:\b)Attach(?:\b)" 3; + "~*(?:\b)AwarioBot(?:\b)" 3; "~*(?:\b)AwarioRssBot(?:\b)" 3; "~*(?:\b)AwarioSmartBot(?:\b)" 3; "~*(?:\b)BBBike(?:\b)" 3; @@ -728,6 +729,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)arquivo-web-crawler(?:\b)" 3; "~*(?:\b)arquivo.pt(?:\b)" 3; "~*(?:\b)autoemailspider(?:\b)" 3; + "~*(?:\b)awario.com(?:\b)" 3; "~*(?:\b)backlink-check(?:\b)" 3; "~*(?:\b)cah.io.community(?:\b)" 3; "~*(?:\b)check1.exe(?:\b)" 3; diff --git a/conf.d/globalblacklist.conf b/conf.d/globalblacklist.conf index 8b2f8fa34..0801f8313 100644 --- a/conf.d/globalblacklist.conf +++ b/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2024.04.4423 -### Updated: Wed Apr 10 10:03:14 UTC 2024 +### Version: V4.2024.04.4424 +### Updated: Wed Apr 10 12:20:08 UTC 2024 ### Bad Referrer Count: 7104 -### Bad Bot Count: 655 +### Bad Bot Count: 657 ################################################### ### VERSION INFORMATION ## @@ -179,6 +179,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)Asterias(?:\b)" 3; "~*(?:\b)Atomseobot(?:\b)" 3; "~*(?:\b)Attach(?:\b)" 3; + "~*(?:\b)AwarioBot(?:\b)" 3; "~*(?:\b)AwarioRssBot(?:\b)" 3; "~*(?:\b)AwarioSmartBot(?:\b)" 3; "~*(?:\b)BBBike(?:\b)" 3; @@ -728,6 +729,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)arquivo-web-crawler(?:\b)" 3; "~*(?:\b)arquivo.pt(?:\b)" 3; "~*(?:\b)autoemailspider(?:\b)" 3; + "~*(?:\b)awario.com(?:\b)" 3; "~*(?:\b)backlink-check(?:\b)" 3; "~*(?:\b)cah.io.community(?:\b)" 3; "~*(?:\b)check1.exe(?:\b)" 3; diff --git a/dev-tools/_robots_input/robots-input.txt b/dev-tools/_robots_input/robots-input.txt index 2d8f85c20..5ec331ab2 100644 --- a/dev-tools/_robots_input/robots-input.txt +++ b/dev-tools/_robots_input/robots-input.txt @@ -30,6 +30,7 @@ Aspiegel Asterias Atomseobot Attach +AwarioBot AwarioRssBot AwarioSmartBot BBBike @@ -579,6 +580,7 @@ archive.org_bot arquivo-web-crawler arquivo.pt autoemailspider +awario.com backlink-check cah.io.community check1.exe diff --git a/dev-tools/buildnumber b/dev-tools/buildnumber index e2ba7ca0d..7def58190 100644 --- a/dev-tools/buildnumber +++ b/dev-tools/buildnumber @@ -1 +1 @@ -4423 +4424 diff --git a/dev-tools/globalblacklist-testing.template b/dev-tools/globalblacklist-testing.template index bfec8dd08..3f9375644 100644 --- a/dev-tools/globalblacklist-testing.template +++ b/dev-tools/globalblacklist-testing.template @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2024.04.4423 -### Updated: Wed Apr 10 10:03:14 UTC 2024 +### Version: V3.2024.04.4424 +### Updated: Wed Apr 10 12:20:09 UTC 2024 ### Bad Referrer Count: 7104 -### Bad Bot Count: 655 +### Bad Bot Count: 657 ################################################### ### VERSION INFORMATION ## @@ -179,6 +179,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)Asterias(?:\b)" 3; "~*(?:\b)Atomseobot(?:\b)" 3; "~*(?:\b)Attach(?:\b)" 3; + "~*(?:\b)AwarioBot(?:\b)" 3; "~*(?:\b)AwarioRssBot(?:\b)" 3; "~*(?:\b)AwarioSmartBot(?:\b)" 3; "~*(?:\b)BBBike(?:\b)" 3; @@ -728,6 +729,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)arquivo-web-crawler(?:\b)" 3; "~*(?:\b)arquivo.pt(?:\b)" 3; "~*(?:\b)autoemailspider(?:\b)" 3; + "~*(?:\b)awario.com(?:\b)" 3; "~*(?:\b)backlink-check(?:\b)" 3; "~*(?:\b)cah.io.community(?:\b)" 3; "~*(?:\b)check1.exe(?:\b)" 3; diff --git a/dev-tools/globalblacklist.template b/dev-tools/globalblacklist.template index 8b2f8fa34..0801f8313 100755 --- a/dev-tools/globalblacklist.template +++ b/dev-tools/globalblacklist.template @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2024.04.4423 -### Updated: Wed Apr 10 10:03:14 UTC 2024 +### Version: V4.2024.04.4424 +### Updated: Wed Apr 10 12:20:08 UTC 2024 ### Bad Referrer Count: 7104 -### Bad Bot Count: 655 +### Bad Bot Count: 657 ################################################### ### VERSION INFORMATION ## @@ -179,6 +179,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)Asterias(?:\b)" 3; "~*(?:\b)Atomseobot(?:\b)" 3; "~*(?:\b)Attach(?:\b)" 3; + "~*(?:\b)AwarioBot(?:\b)" 3; "~*(?:\b)AwarioRssBot(?:\b)" 3; "~*(?:\b)AwarioSmartBot(?:\b)" 3; "~*(?:\b)BBBike(?:\b)" 3; @@ -728,6 +729,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)arquivo-web-crawler(?:\b)" 3; "~*(?:\b)arquivo.pt(?:\b)" 3; "~*(?:\b)autoemailspider(?:\b)" 3; + "~*(?:\b)awario.com(?:\b)" 3; "~*(?:\b)backlink-check(?:\b)" 3; "~*(?:\b)cah.io.community(?:\b)" 3; "~*(?:\b)check1.exe(?:\b)" 3; diff --git a/dev-tools/test1_conf_files/bots.d/whitelist-ips.conf b/dev-tools/test1_conf_files/bots.d/whitelist-ips.conf index 1bae753aa..7f9356973 100644 --- a/dev-tools/test1_conf_files/bots.d/whitelist-ips.conf +++ b/dev-tools/test1_conf_files/bots.d/whitelist-ips.conf @@ -45,4 +45,4 @@ # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST -20.172.7.68 0; +20.55.46.193 0; diff --git a/dev-tools/test1_conf_files/conf.d/globalblacklist.conf b/dev-tools/test1_conf_files/conf.d/globalblacklist.conf index 8b2f8fa34..0801f8313 100644 --- a/dev-tools/test1_conf_files/conf.d/globalblacklist.conf +++ b/dev-tools/test1_conf_files/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2024.04.4423 -### Updated: Wed Apr 10 10:03:14 UTC 2024 +### Version: V4.2024.04.4424 +### Updated: Wed Apr 10 12:20:08 UTC 2024 ### Bad Referrer Count: 7104 -### Bad Bot Count: 655 +### Bad Bot Count: 657 ################################################### ### VERSION INFORMATION ## @@ -179,6 +179,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)Asterias(?:\b)" 3; "~*(?:\b)Atomseobot(?:\b)" 3; "~*(?:\b)Attach(?:\b)" 3; + "~*(?:\b)AwarioBot(?:\b)" 3; "~*(?:\b)AwarioRssBot(?:\b)" 3; "~*(?:\b)AwarioSmartBot(?:\b)" 3; "~*(?:\b)BBBike(?:\b)" 3; @@ -728,6 +729,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)arquivo-web-crawler(?:\b)" 3; "~*(?:\b)arquivo.pt(?:\b)" 3; "~*(?:\b)autoemailspider(?:\b)" 3; + "~*(?:\b)awario.com(?:\b)" 3; "~*(?:\b)backlink-check(?:\b)" 3; "~*(?:\b)cah.io.community(?:\b)" 3; "~*(?:\b)check1.exe(?:\b)" 3; diff --git a/dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf b/dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf index 016b0a3fa..06f3c6af3 100644 --- a/dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf +++ b/dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.120 1; +10.1.0.42 1; 127.0.0.1 1; diff --git a/dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf b/dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf index f53e1ef9c..054495e31 100644 --- a/dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf +++ b/dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.120 0; +10.1.0.42 0; 127.0.0.1 0; diff --git a/dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf b/dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf index 8b2f8fa34..0801f8313 100644 --- a/dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf +++ b/dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2024.04.4423 -### Updated: Wed Apr 10 10:03:14 UTC 2024 +### Version: V4.2024.04.4424 +### Updated: Wed Apr 10 12:20:08 UTC 2024 ### Bad Referrer Count: 7104 -### Bad Bot Count: 655 +### Bad Bot Count: 657 ################################################### ### VERSION INFORMATION ## @@ -179,6 +179,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)Asterias(?:\b)" 3; "~*(?:\b)Atomseobot(?:\b)" 3; "~*(?:\b)Attach(?:\b)" 3; + "~*(?:\b)AwarioBot(?:\b)" 3; "~*(?:\b)AwarioRssBot(?:\b)" 3; "~*(?:\b)AwarioSmartBot(?:\b)" 3; "~*(?:\b)BBBike(?:\b)" 3; @@ -728,6 +729,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)arquivo-web-crawler(?:\b)" 3; "~*(?:\b)arquivo.pt(?:\b)" 3; "~*(?:\b)autoemailspider(?:\b)" 3; + "~*(?:\b)awario.com(?:\b)" 3; "~*(?:\b)backlink-check(?:\b)" 3; "~*(?:\b)cah.io.community(?:\b)" 3; "~*(?:\b)check1.exe(?:\b)" 3; diff --git a/dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf b/dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf index 016b0a3fa..06f3c6af3 100644 --- a/dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf +++ b/dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.120 1; +10.1.0.42 1; 127.0.0.1 1; diff --git a/dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf b/dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf index f53e1ef9c..054495e31 100644 --- a/dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf +++ b/dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.120 0; +10.1.0.42 0; 127.0.0.1 0; diff --git a/dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf b/dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf index 8b2f8fa34..0801f8313 100644 --- a/dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf +++ b/dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2024.04.4423 -### Updated: Wed Apr 10 10:03:14 UTC 2024 +### Version: V4.2024.04.4424 +### Updated: Wed Apr 10 12:20:08 UTC 2024 ### Bad Referrer Count: 7104 -### Bad Bot Count: 655 +### Bad Bot Count: 657 ################################################### ### VERSION INFORMATION ## @@ -179,6 +179,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)Asterias(?:\b)" 3; "~*(?:\b)Atomseobot(?:\b)" 3; "~*(?:\b)Attach(?:\b)" 3; + "~*(?:\b)AwarioBot(?:\b)" 3; "~*(?:\b)AwarioRssBot(?:\b)" 3; "~*(?:\b)AwarioSmartBot(?:\b)" 3; "~*(?:\b)BBBike(?:\b)" 3; @@ -728,6 +729,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)arquivo-web-crawler(?:\b)" 3; "~*(?:\b)arquivo.pt(?:\b)" 3; "~*(?:\b)autoemailspider(?:\b)" 3; + "~*(?:\b)awario.com(?:\b)" 3; "~*(?:\b)backlink-check(?:\b)" 3; "~*(?:\b)cah.io.community(?:\b)" 3; "~*(?:\b)check1.exe(?:\b)" 3; diff --git a/dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf b/dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf index 016b0a3fa..06f3c6af3 100644 --- a/dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf +++ b/dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.120 1; +10.1.0.42 1; 127.0.0.1 1; diff --git a/dev-tools/test1_conf_files_whitelist/bots.d/blacklist-user-agents.conf b/dev-tools/test1_conf_files_whitelist/bots.d/blacklist-user-agents.conf index 8a4467e3b..6bf67272e 100644 --- a/dev-tools/test1_conf_files_whitelist/bots.d/blacklist-user-agents.conf +++ b/dev-tools/test1_conf_files_whitelist/bots.d/blacklist-user-agents.conf @@ -91,6 +91,7 @@ "~*(?:\b)Asterias(?:\b|)" 0; "~*(?:\b)Atomseobot(?:\b|)" 0; "~*(?:\b)Attach(?:\b|)" 0; + "~*(?:\b)AwarioBot(?:\b|)" 0; "~*(?:\b)AwarioRssBot(?:\b|)" 0; "~*(?:\b)AwarioSmartBot(?:\b|)" 0; "~*(?:\b)BBBike(?:\b|)" 0; @@ -640,6 +641,7 @@ "~*(?:\b)arquivo-web-crawler(?:\b|)" 0; "~*(?:\b)arquivo.pt(?:\b|)" 0; "~*(?:\b)autoemailspider(?:\b|)" 0; + "~*(?:\b)awario.com(?:\b|)" 0; "~*(?:\b)backlink-check(?:\b|)" 0; "~*(?:\b)cah.io.community(?:\b|)" 0; "~*(?:\b)check1.exe(?:\b|)" 0; diff --git a/dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf b/dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf index f53e1ef9c..054495e31 100644 --- a/dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf +++ b/dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.120 0; +10.1.0.42 0; 127.0.0.1 0; diff --git a/dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf b/dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf index 8b2f8fa34..0801f8313 100644 --- a/dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf +++ b/dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2024.04.4423 -### Updated: Wed Apr 10 10:03:14 UTC 2024 +### Version: V4.2024.04.4424 +### Updated: Wed Apr 10 12:20:08 UTC 2024 ### Bad Referrer Count: 7104 -### Bad Bot Count: 655 +### Bad Bot Count: 657 ################################################### ### VERSION INFORMATION ## @@ -179,6 +179,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)Asterias(?:\b)" 3; "~*(?:\b)Atomseobot(?:\b)" 3; "~*(?:\b)Attach(?:\b)" 3; + "~*(?:\b)AwarioBot(?:\b)" 3; "~*(?:\b)AwarioRssBot(?:\b)" 3; "~*(?:\b)AwarioSmartBot(?:\b)" 3; "~*(?:\b)BBBike(?:\b)" 3; @@ -728,6 +729,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)arquivo-web-crawler(?:\b)" 3; "~*(?:\b)arquivo.pt(?:\b)" 3; "~*(?:\b)autoemailspider(?:\b)" 3; + "~*(?:\b)awario.com(?:\b)" 3; "~*(?:\b)backlink-check(?:\b)" 3; "~*(?:\b)cah.io.community(?:\b)" 3; "~*(?:\b)check1.exe(?:\b)" 3; diff --git a/dev-tools/test2_conf_files/bots.d/whitelist-ips.conf b/dev-tools/test2_conf_files/bots.d/whitelist-ips.conf index 1bae753aa..7f9356973 100644 --- a/dev-tools/test2_conf_files/bots.d/whitelist-ips.conf +++ b/dev-tools/test2_conf_files/bots.d/whitelist-ips.conf @@ -45,4 +45,4 @@ # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST -20.172.7.68 0; +20.55.46.193 0; diff --git a/dev-tools/test2_conf_files/conf.d/globalblacklist.conf b/dev-tools/test2_conf_files/conf.d/globalblacklist.conf index 9339dd530..ccc75e29c 100644 --- a/dev-tools/test2_conf_files/conf.d/globalblacklist.conf +++ b/dev-tools/test2_conf_files/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2024.04.4423 -### Updated: Wed Apr 10 10:03:14 UTC 2024 +### Version: V4.2024.04.4424 +### Updated: Wed Apr 10 12:20:08 UTC 2024 ### Bad Referrer Count: 7104 -### Bad Bot Count: 655 +### Bad Bot Count: 657 ################################################### ### VERSION INFORMATION ## @@ -179,6 +179,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)Asterias(?:\b)" 3; "~*(?:\b)Atomseobot(?:\b)" 3; "~*(?:\b)Attach(?:\b)" 3; + "~*(?:\b)AwarioBot(?:\b)" 3; "~*(?:\b)AwarioRssBot(?:\b)" 3; "~*(?:\b)AwarioSmartBot(?:\b)" 3; "~*(?:\b)BBBike(?:\b)" 3; @@ -728,6 +729,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)arquivo-web-crawler(?:\b)" 3; "~*(?:\b)arquivo.pt(?:\b)" 3; "~*(?:\b)autoemailspider(?:\b)" 3; + "~*(?:\b)awario.com(?:\b)" 3; "~*(?:\b)backlink-check(?:\b)" 3; "~*(?:\b)cah.io.community(?:\b)" 3; "~*(?:\b)check1.exe(?:\b)" 3; diff --git a/dev-tools/test3_conf_files/bots.d/whitelist-ips.conf b/dev-tools/test3_conf_files/bots.d/whitelist-ips.conf index 1bae753aa..7f9356973 100644 --- a/dev-tools/test3_conf_files/bots.d/whitelist-ips.conf +++ b/dev-tools/test3_conf_files/bots.d/whitelist-ips.conf @@ -45,4 +45,4 @@ # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST -20.172.7.68 0; +20.55.46.193 0; diff --git a/dev-tools/test3_conf_files/conf.d/globalblacklist.conf b/dev-tools/test3_conf_files/conf.d/globalblacklist.conf index 76e70bbe6..2d81e8c0d 100644 --- a/dev-tools/test3_conf_files/conf.d/globalblacklist.conf +++ b/dev-tools/test3_conf_files/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2024.04.4423 -### Updated: Wed Apr 10 10:03:14 UTC 2024 +### Version: V4.2024.04.4424 +### Updated: Wed Apr 10 12:20:08 UTC 2024 ### Bad Referrer Count: 7104 -### Bad Bot Count: 655 +### Bad Bot Count: 657 ################################################### ### VERSION INFORMATION ## @@ -179,6 +179,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)Asterias(?:\b)" 3; "~*(?:\b)Atomseobot(?:\b)" 3; "~*(?:\b)Attach(?:\b)" 3; + "~*(?:\b)AwarioBot(?:\b)" 3; "~*(?:\b)AwarioRssBot(?:\b)" 3; "~*(?:\b)AwarioSmartBot(?:\b)" 3; "~*(?:\b)BBBike(?:\b)" 3; @@ -728,6 +729,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)arquivo-web-crawler(?:\b)" 3; "~*(?:\b)arquivo.pt(?:\b)" 3; "~*(?:\b)autoemailspider(?:\b)" 3; + "~*(?:\b)awario.com(?:\b)" 3; "~*(?:\b)backlink-check(?:\b)" 3; "~*(?:\b)cah.io.community(?:\b)" 3; "~*(?:\b)check1.exe(?:\b)" 3; diff --git a/dev-tools/test_units/blacklist-ips.conf b/dev-tools/test_units/blacklist-ips.conf index 016b0a3fa..06f3c6af3 100644 --- a/dev-tools/test_units/blacklist-ips.conf +++ b/dev-tools/test_units/blacklist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.120 1; +10.1.0.42 1; 127.0.0.1 1; diff --git a/dev-tools/test_units/blacklist-user-agents.conf b/dev-tools/test_units/blacklist-user-agents.conf index 8a4467e3b..6bf67272e 100644 --- a/dev-tools/test_units/blacklist-user-agents.conf +++ b/dev-tools/test_units/blacklist-user-agents.conf @@ -91,6 +91,7 @@ "~*(?:\b)Asterias(?:\b|)" 0; "~*(?:\b)Atomseobot(?:\b|)" 0; "~*(?:\b)Attach(?:\b|)" 0; + "~*(?:\b)AwarioBot(?:\b|)" 0; "~*(?:\b)AwarioRssBot(?:\b|)" 0; "~*(?:\b)AwarioSmartBot(?:\b|)" 0; "~*(?:\b)BBBike(?:\b|)" 0; @@ -640,6 +641,7 @@ "~*(?:\b)arquivo-web-crawler(?:\b|)" 0; "~*(?:\b)arquivo.pt(?:\b|)" 0; "~*(?:\b)autoemailspider(?:\b|)" 0; + "~*(?:\b)awario.com(?:\b|)" 0; "~*(?:\b)backlink-check(?:\b|)" 0; "~*(?:\b)cah.io.community(?:\b|)" 0; "~*(?:\b)check1.exe(?:\b|)" 0; diff --git a/dev-tools/test_units/random-bots-for-test-quick.list b/dev-tools/test_units/random-bots-for-test-quick.list index ba74eb3d4..0161d10b7 100644 --- a/dev-tools/test_units/random-bots-for-test-quick.list +++ b/dev-tools/test_units/random-bots-for-test-quick.list @@ -1,10 +1,10 @@ -IDBTE4M -NimbleCrawler -ProPowerBot -Teleport -Thumbor -Twice -VeriCiteCrawler -iubenda-radar -mediawords -oBot +BDCbot +BLEXBot +NearSite +Nmap +ProWebWalker +Quick-Crawler +Sucker +Suzuran +TwengaBot +x09Mozilla diff --git a/dev-tools/test_units/random-bots-for-whitelist-test.list b/dev-tools/test_units/random-bots-for-whitelist-test.list index 9ace40753..f71e42eec 100644 --- a/dev-tools/test_units/random-bots-for-whitelist-test.list +++ b/dev-tools/test_units/random-bots-for-whitelist-test.list @@ -1,100 +1,100 @@ -AdsTxtCrawlerTP -AfD-Verbotsverfahren -AwarioRssBot -BackDoorBot -Battleztar Bazinga -Blow -Bullseye -ChatGPT-User -Chlooe -Cloud mapping -Collector -CopyRightCheck +Acunetix +AhrefsBot +AwarioBot +BackWeb +Backlink-Ceck +BlowFish +BuiltBotTough +Bytespider +CheeseBot +Citoid +Copier Cosmos -CrazyWebCrawler +Craftbot Crescent -CrunchBot -DBLBot +DSearch +DatabaseDriverMysqli +Demon Disco -DomainStatsBot -EirGrabber -EroCrawler -Exabot -Fimap -Firefox/7.0 -Flunky -Fyrebot -GalaxyBot -GetRight -Gigabot -GrabNet +DomainCrawler +DomainSigmaCrawler +Domains Project +EasyDL Grabber -GridBot +HTTP::Lite +HTTrack Harvest -Image Fetch -InterGET -InternetSeer -JOC Web Spider -LeechGet -Likse +HonoluluBot +Id-search +IndeedBot +IstellaBot +JamesBOT +JennyBot +Jorgee +JustView +Lightspeedsystems LinksManager -Mag-Net -Majestic-SEO -Mata Hari -MeanPath Bot -MegaIndex.ru +Lmspider Minefield -Nameprotect -Ninja -Nuclei -Octopus -OutfoxBot +Mojolicious +Musobot +NearSite +NetMechanic +NextGenSearchBot +NimbleCrawler +Nimbostratus +Nutch +OrangeBot +OutclicksBot +POE-Component-Client-HTTP +PageAnalyzer PageGrabber -Pavuk -Petalbot -Pi-Monster +PageThing.com Proximic Psbot -Quick-Crawler -Re-re -RealDownload -RocketCrawler -SEOstats -Siphon -Sogou web spider -Sottopop -Spinn3r -Sqworm -Sucker -Suzuran -Swiftbot -Teleport -Turnitin +RankFlex +Rankivabot +SemanticJuice +SenutoBot +Seomoz +SiteSucker +Sitevigil +SmartDownload +Sosospider +Szukacz +T0PHackTeam +Thumbor +Titan +Toata +Trendictionbot Twice -Typhoeus -Virusdie -Web Collage -Web Fuck -WebSucker -WebWhacker -WebmasterWorldForumBot -Webster -Woobot -Xaldon WebSpider -Xenu -Zeus -ZoominfoBot +VidibleScraper +WPScan +Wallpapers/3.0 +WallpapersHD +WebCollage +WebEnhancer +WebFuck +WebZIP +Website Quester +Whack +Wprecon +YoudaoBot +Zitebot +archive.org_bot arquivo-web-crawler +autoemailspider +awario.com +check1.exe +clark-crawler cohere-ai -crawler.feedback +domainsproject.org +heritrix ips-agent iubenda-radar -muhstik-scan +lwp-trivial openai.com -page scorer -seocompany.store +polaris version spyfu -sysscan -trendiction.de -ubermetrics-technologies.com webmeup-crawler +x22Mozilla diff --git a/dev-tools/test_units/random-referrers-for-test-quick.list b/dev-tools/test_units/random-referrers-for-test-quick.list index 1f721c2e6..1389d01ac 100644 --- a/dev-tools/test_units/random-referrers-for-test-quick.list +++ b/dev-tools/test_units/random-referrers-for-test-quick.list @@ -1,10 +1,10 @@ -bestcoin.cc -blockworld.ru -domaincrawler.com -kazan.zrus.org -lottospring.com -mackeeper-land-672695126.us-east-1.elb.amazonaws.com -mydirtyhobby.com -tsan.net -xn--e1afanlbnfckd7c3d.xn--p1ai -zojirushi-products.ru +76brighton.co.uk +cinemaflix.website +druzhininevgeniy63.blogspot.com +greendream.com.ua +lion.bolegapakistan.com +luxup.ru +millionare.com +morpicert.pw +producm.ru +referencemoi.com diff --git a/dev-tools/test_units/random-referrers-for-whitelist-test.list b/dev-tools/test_units/random-referrers-for-whitelist-test.list index d50b201a7..a85a390c1 100644 --- a/dev-tools/test_units/random-referrers-for-whitelist-test.list +++ b/dev-tools/test_units/random-referrers-for-whitelist-test.list @@ -1,100 +1,100 @@ -1adult.com -24-ak.ru -888.com -add-add.men -amazon-adsystem.com -animalia-life.club -apelsinnik.website -arendovalka.xyz -auspolice.com -aversis.net -avirasecureserver.com -avtointeres.ru -azarttoto.com -bestdooz.com -bonvillan.com -bukleteg.xyz -cdn.walkme.com -cenokos.ru -clicksotrk.com -colobit.biz -compliance-george.top -compliance-josh.top -compucelunlock.net -de.zapmeta.com -degerlund.net -digital-video-processing.com -dmmspy.com -domainroam.win -dosug-lux.ru -dvrlists.com -dwomlink.info -educatemetv.com -essayservicewriting.org -falcon-images.blogspot.com -favornews.com -ge0ip.net -girlgamerdaily.com -hamilton.ca -herokuapp.com -houtings.xyz -hoverboard360.de -igrovyeavtomaty777.ru -imgarit.pw -iqoption.com -itbc.kiev.ua -jose.mulinohouse.co -kakablog.net -kambasoft.com -kevblog.top -keywordhut.com -lemon-ade.site -magnetpress.sk -matpre.top -maximpartnerspr.com +3xforum.ro +70casino.online +8jn.dating-app.ru +adktrailmap.com +adpremium.org +adverster.com +allpdfmags.net +amazingpic.net +appearance-cool.com +autobrennero.it +automatic-seo.com +back.dog +brainzpod.win +buyfriend.ru +buzzonclick.com +bytimedance.ru +chelyabinsk.xrus.org +cinemaflix.website +clickso.com +clipartnew.com +compliance-margo.top +crystalslot.com +customerguru.in +daretodonate.co +dbmkfhqk.bloger.index.hr +defenderxtactical.com +diminishedvalueoforegon.com +domaincrawler.com +downloadmefiranaratb1972.xpg.com.br +elvenmachine.com +eshop4u.jp +ewebarticle.info +failingmarriege.blogspot.com +fatmaelgarny.com +feel-planet.com +fetroshok.ru +figuringmoneyout.com +filkhbr.com +floridamhca.org +forum-engineering.ru +freejabs.com +freemags.cc +funponsel.com +futbolkisales.ru +fyl.com.ru +glicol.kz +hdxnxxtube.mobi +highstairs-a.akamaihd.net +hptwaakw.blog.fc2.com +ibb.com.ua +iboss.com +interior-stickers.ru +investzalog.ru +jolic2.com +katushka.net +kinosed.net +knowsitall.info +koptims.tiu.ru +maranbrinfo.com.br +middlerush-a.akamaihd.net mobilemedia.md -mobilierland.com -mostantikor.ru -myghillie.info -mymobilemoneypages.com -neobux-bg.info -nevansk.ru -nhl09.ru -omsk.xrus.org -palocco.it -pl-top.pl -pl-vouchers.com -pochemychka.net -potolokelekor.ru -predmety.in.ua -profitfx.online -pskcijdc.bloger.index.hr -ragecash.com -rankia.com -rusenvironmental.net -sardinie.us -saveindex.xyz -scrapy.org -secretscook.ru -seksotur.ru -shitting.pro -shopvilleroyboch.com.ua -slavia.info -snyatie-lomki-v-stacionare.ru -softxaker.ru -space4update.pw -ssconstruction.co -svnuppsalaorebro.se -takethatad.com +mompussy.net +partnerads.men +peugeot-club.org +photochki.com +privatov-zapisi.ru +pufip.com +pussysaga.com +revistaindustria.com +santechnik.jimdo.com +searchwebknow-a.akamaihd.net +seeingmeerkat.com +sendearnings.com +seo18.su +seoprofiler.com +sexpartygirls.net +sibtest.ru +sideeffectsoftizanidine.blogspot.com +siteimprove.com +sloopyjoes.com +socialbuttons.xyz +telemetryverification.net +thin.me.pn +titan-cloud.life +tizanidinewithdrawal.blogspot.com +toloka.hurtom.com +top-deal.com.pl +torrentdownloadhub.com +trafficmania.com trahvid.com -urldelivery.com -veerotech.com -vladimir.zrus.org -voloomoney.com -whatzmyip.net -whiteproduct.com +uggsale.online +utrolive.ru +viel.su +welcomeauto.ru +winterclassichockeyjerseys.com +www.ehscloud.cn xn----7sbaaabaei0cc8aj5bj0bncejx.xn--p1ai -yourmovies.pl -zeroredirect6.com -zhuravlev.info -zreloeporno.tv +xn--80ab4aa2g.xn--p1ai +xnxx-n.com +xxx-treker.ru diff --git a/dev-tools/test_units/whitelist-ips.conf b/dev-tools/test_units/whitelist-ips.conf index f53e1ef9c..054495e31 100644 --- a/dev-tools/test_units/whitelist-ips.conf +++ b/dev-tools/test_units/whitelist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.120 0; +10.1.0.42 0; 127.0.0.1 0; diff --git a/dev-tools/tmp/robots.txt b/dev-tools/tmp/robots.txt index 8eb4a7bc5..e7d67a64b 100644 --- a/dev-tools/tmp/robots.txt +++ b/dev-tools/tmp/robots.txt @@ -6,9 +6,9 @@ ### Version Information # ################################################### -### Version: V4.2024.04.4423 -### Updated: Wed Apr 10 10:03:15 UTC 2024 -### Bad Bot Count: 655 +### Version: V4.2024.04.4424 +### Updated: Wed Apr 10 12:20:09 UTC 2024 +### Bad Bot Count: 657 ################################################### ### Version Information ## @@ -79,6 +79,8 @@ User-agent: Atomseobot Disallow:/ User-agent: Attach Disallow:/ +User-agent: AwarioBot +Disallow:/ User-agent: AwarioRssBot Disallow:/ User-agent: AwarioSmartBot @@ -1177,6 +1179,8 @@ User-agent: arquivo.pt Disallow:/ User-agent: autoemailspider Disallow:/ +User-agent: awario.com +Disallow:/ User-agent: backlink-check Disallow:/ User-agent: cah.io.community diff --git a/robots.txt/robots.txt b/robots.txt/robots.txt index 8eb4a7bc5..e7d67a64b 100644 --- a/robots.txt/robots.txt +++ b/robots.txt/robots.txt @@ -6,9 +6,9 @@ ### Version Information # ################################################### -### Version: V4.2024.04.4423 -### Updated: Wed Apr 10 10:03:15 UTC 2024 -### Bad Bot Count: 655 +### Version: V4.2024.04.4424 +### Updated: Wed Apr 10 12:20:09 UTC 2024 +### Bad Bot Count: 657 ################################################### ### Version Information ## @@ -79,6 +79,8 @@ User-agent: Atomseobot Disallow:/ User-agent: Attach Disallow:/ +User-agent: AwarioBot +Disallow:/ User-agent: AwarioRssBot Disallow:/ User-agent: AwarioSmartBot @@ -1177,6 +1179,8 @@ User-agent: arquivo.pt Disallow:/ User-agent: autoemailspider Disallow:/ +User-agent: awario.com +Disallow:/ User-agent: backlink-check Disallow:/ User-agent: cah.io.community