diff --git a/AUTO-CONFIGURATION.md b/AUTO-CONFIGURATION.md index 23c9e9b25..d5714cfb6 100755 --- a/AUTO-CONFIGURATION.md +++ b/AUTO-CONFIGURATION.md @@ -4,9 +4,9 @@ ### PLEASE READ CONFIGURATION INSTRUCTIONS BELOW THOROUGHLY :exclamation: _______________ -#### Version: V4.2024.10.4835 +#### Version: V4.2024.10.4836 #### Bad Referrers Blocked: [7104](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-referrers.list) -#### Bad User-Agents (Bots) Blocked: [667](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) +#### Bad User-Agents (Bots) Blocked: [668](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) #### Fake Googlebots Blocked: [217](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/fake-googlebots.list) ____________________ diff --git a/MANUAL-CONFIGURATION.md b/MANUAL-CONFIGURATION.md index f3c63076b..7b79c4155 100755 --- a/MANUAL-CONFIGURATION.md +++ b/MANUAL-CONFIGURATION.md @@ -4,9 +4,9 @@ ### PLEASE READ CONFIGURATION INSTRUCTIONS BELOW THOROUGHLY :exclamation: _______________ -#### Version: V4.2024.10.4835 +#### Version: V4.2024.10.4836 #### Bad Referrers Blocked: [7104](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-referrers.list) -#### Bad User-Agents (Bots) Blocked: [667](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) +#### Bad User-Agents (Bots) Blocked: [668](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) #### Fake Googlebots Blocked: [217](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/fake-googlebots.list) ____________________ diff --git a/README.md b/README.md index e64b9cadb..3e1f2541a 100755 --- a/README.md +++ b/README.md @@ -14,9 +14,9 @@ ##### The Ultimate Nginx Bad Bot, User-Agent, Spam Referrer Blocker, Adware, Malware and Ransomware Blocker, Clickjacking Blocker, Click Re-Directing Blocker, SEO Companies and Bad IP Blocker with Anti DDOS System, Nginx Rate Limiting and Wordpress Theme Detector Blocking. Stop and Block all kinds of bad internet traffic even Fake Googlebots from ever reaching your web sites. [PLEASE SEE: Definition of Bad Bots](#define-bad-bots) _______________ -#### Version: V4.2024.10.4835 +#### Version: V4.2024.10.4836 #### Bad Referrers Blocked: [7104](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-referrers.list) -#### Bad User-Agents (Bots) Blocked: [667](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) +#### Bad User-Agents (Bots) Blocked: [668](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) #### Fake Googlebots Blocked: [217](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/fake-googlebots.list) ____________________ diff --git a/_generator_lists/bad-user-agents.list b/_generator_lists/bad-user-agents.list index d677d0ba5..40001c1d6 100755 --- a/_generator_lists/bad-user-agents.list +++ b/_generator_lists/bad-user-agents.list @@ -1,4 +1,3 @@ -everyfeed-spider 01h4x.com 360Spider 404checker @@ -608,6 +607,7 @@ demandbase-bot domainsproject.org eCatch evc-batch +everyfeed-spider facebookscraper gopher heritrix diff --git a/_google_analytics_ghost_spam/README.md b/_google_analytics_ghost_spam/README.md index e3d7c82d9..ece9f70a3 100644 --- a/_google_analytics_ghost_spam/README.md +++ b/_google_analytics_ghost_spam/README.md @@ -3,9 +3,9 @@ # EASY CONFIGURATION INSTRUCTIONS FOR STOPPING GOOGLE ANALYTICS "GHOST" SPAM _______________ -#### Version: V4.2024.10.4835 +#### Version: V4.2024.10.4836 #### Bad Referrers Blocked: [7104](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-referrers.list) -#### Bad User-Agents (Bots) Blocked: [667](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) +#### Bad User-Agents (Bots) Blocked: [668](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) #### Fake Googlebots Blocked: [217](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/fake-googlebots.list) ____________________ diff --git a/_sample_config_files/Engintron_for_cPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf b/_sample_config_files/Engintron_for_cPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf index a3aaacfcd..b70423ac7 100755 --- a/_sample_config_files/Engintron_for_cPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf +++ b/_sample_config_files/Engintron_for_cPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2024.10.4835 -### Updated: Fri Oct 25 22:01:17 UTC 2024 +### Version: V4.2024.10.4836 +### Updated: Sat Oct 26 05:47:29 UTC 2024 ### Bad Referrer Count: 7104 -### Bad Bot Count: 667 +### Bad Bot Count: 668 ################################################### ### VERSION INFORMATION ## @@ -760,6 +760,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)domainsproject.org(?:\b)" 3; "~*(?:\b)eCatch(?:\b)" 3; "~*(?:\b)evc-batch(?:\b)" 3; + "~*(?:\b)everyfeed-spider(?:\b)" 3; "~*(?:\b)facebookscraper(?:\b)" 3; "~*(?:\b)gopher(?:\b)" 3; "~*(?:\b)heritrix(?:\b)" 3; diff --git a/conf.d/globalblacklist-testing-version.conf b/conf.d/globalblacklist-testing-version.conf index 7db0472c0..9fc805cb3 100644 --- a/conf.d/globalblacklist-testing-version.conf +++ b/conf.d/globalblacklist-testing-version.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2024.10.4835 -### Updated: Fri Oct 25 22:01:18 UTC 2024 +### Version: V3.2024.10.4836 +### Updated: Sat Oct 26 05:47:29 UTC 2024 ### Bad Referrer Count: 7104 -### Bad Bot Count: 667 +### Bad Bot Count: 668 ################################################### ### VERSION INFORMATION ## @@ -757,6 +757,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)domainsproject.org(?:\b)" 3; "~*(?:\b)eCatch(?:\b)" 3; "~*(?:\b)evc-batch(?:\b)" 3; + "~*(?:\b)everyfeed-spider(?:\b)" 3; "~*(?:\b)facebookscraper(?:\b)" 3; "~*(?:\b)gopher(?:\b)" 3; "~*(?:\b)heritrix(?:\b)" 3; diff --git a/conf.d/globalblacklist.conf b/conf.d/globalblacklist.conf index a3aaacfcd..b70423ac7 100644 --- a/conf.d/globalblacklist.conf +++ b/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2024.10.4835 -### Updated: Fri Oct 25 22:01:17 UTC 2024 +### Version: V4.2024.10.4836 +### Updated: Sat Oct 26 05:47:29 UTC 2024 ### Bad Referrer Count: 7104 -### Bad Bot Count: 667 +### Bad Bot Count: 668 ################################################### ### VERSION INFORMATION ## @@ -760,6 +760,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)domainsproject.org(?:\b)" 3; "~*(?:\b)eCatch(?:\b)" 3; "~*(?:\b)evc-batch(?:\b)" 3; + "~*(?:\b)everyfeed-spider(?:\b)" 3; "~*(?:\b)facebookscraper(?:\b)" 3; "~*(?:\b)gopher(?:\b)" 3; "~*(?:\b)heritrix(?:\b)" 3; diff --git a/dev-tools/_robots_input/robots-input.txt b/dev-tools/_robots_input/robots-input.txt index 568cbc36c..5bc63a048 100644 --- a/dev-tools/_robots_input/robots-input.txt +++ b/dev-tools/_robots_input/robots-input.txt @@ -607,6 +607,7 @@ demandbase-bot domainsproject.org eCatch evc-batch +everyfeed-spider facebookscraper gopher heritrix diff --git a/dev-tools/buildnumber b/dev-tools/buildnumber index 113368211..1bb196823 100644 --- a/dev-tools/buildnumber +++ b/dev-tools/buildnumber @@ -1 +1 @@ -4835 +4836 diff --git a/dev-tools/globalblacklist-testing.template b/dev-tools/globalblacklist-testing.template index 7db0472c0..9fc805cb3 100644 --- a/dev-tools/globalblacklist-testing.template +++ b/dev-tools/globalblacklist-testing.template @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2024.10.4835 -### Updated: Fri Oct 25 22:01:18 UTC 2024 +### Version: V3.2024.10.4836 +### Updated: Sat Oct 26 05:47:29 UTC 2024 ### Bad Referrer Count: 7104 -### Bad Bot Count: 667 +### Bad Bot Count: 668 ################################################### ### VERSION INFORMATION ## @@ -757,6 +757,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)domainsproject.org(?:\b)" 3; "~*(?:\b)eCatch(?:\b)" 3; "~*(?:\b)evc-batch(?:\b)" 3; + "~*(?:\b)everyfeed-spider(?:\b)" 3; "~*(?:\b)facebookscraper(?:\b)" 3; "~*(?:\b)gopher(?:\b)" 3; "~*(?:\b)heritrix(?:\b)" 3; diff --git a/dev-tools/globalblacklist.template b/dev-tools/globalblacklist.template index a3aaacfcd..b70423ac7 100755 --- a/dev-tools/globalblacklist.template +++ b/dev-tools/globalblacklist.template @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2024.10.4835 -### Updated: Fri Oct 25 22:01:17 UTC 2024 +### Version: V4.2024.10.4836 +### Updated: Sat Oct 26 05:47:29 UTC 2024 ### Bad Referrer Count: 7104 -### Bad Bot Count: 667 +### Bad Bot Count: 668 ################################################### ### VERSION INFORMATION ## @@ -760,6 +760,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)domainsproject.org(?:\b)" 3; "~*(?:\b)eCatch(?:\b)" 3; "~*(?:\b)evc-batch(?:\b)" 3; + "~*(?:\b)everyfeed-spider(?:\b)" 3; "~*(?:\b)facebookscraper(?:\b)" 3; "~*(?:\b)gopher(?:\b)" 3; "~*(?:\b)heritrix(?:\b)" 3; diff --git a/dev-tools/test1_conf_files/bots.d/whitelist-ips.conf b/dev-tools/test1_conf_files/bots.d/whitelist-ips.conf index 1242e9bb0..1dea6b79e 100644 --- a/dev-tools/test1_conf_files/bots.d/whitelist-ips.conf +++ b/dev-tools/test1_conf_files/bots.d/whitelist-ips.conf @@ -45,4 +45,4 @@ # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST -172.183.122.254 0; +172.171.255.193 0; diff --git a/dev-tools/test1_conf_files/conf.d/globalblacklist.conf b/dev-tools/test1_conf_files/conf.d/globalblacklist.conf index a3aaacfcd..b70423ac7 100644 --- a/dev-tools/test1_conf_files/conf.d/globalblacklist.conf +++ b/dev-tools/test1_conf_files/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2024.10.4835 -### Updated: Fri Oct 25 22:01:17 UTC 2024 +### Version: V4.2024.10.4836 +### Updated: Sat Oct 26 05:47:29 UTC 2024 ### Bad Referrer Count: 7104 -### Bad Bot Count: 667 +### Bad Bot Count: 668 ################################################### ### VERSION INFORMATION ## @@ -760,6 +760,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)domainsproject.org(?:\b)" 3; "~*(?:\b)eCatch(?:\b)" 3; "~*(?:\b)evc-batch(?:\b)" 3; + "~*(?:\b)everyfeed-spider(?:\b)" 3; "~*(?:\b)facebookscraper(?:\b)" 3; "~*(?:\b)gopher(?:\b)" 3; "~*(?:\b)heritrix(?:\b)" 3; diff --git a/dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf b/dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf index 0d8dca8c3..99575d41d 100644 --- a/dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf +++ b/dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.48 1; +10.1.0.64 1; 127.0.0.1 1; diff --git a/dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf b/dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf index 5d111b732..0ddde4728 100644 --- a/dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf +++ b/dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.48 0; +10.1.0.64 0; 127.0.0.1 0; diff --git a/dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf b/dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf index a3aaacfcd..b70423ac7 100644 --- a/dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf +++ b/dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2024.10.4835 -### Updated: Fri Oct 25 22:01:17 UTC 2024 +### Version: V4.2024.10.4836 +### Updated: Sat Oct 26 05:47:29 UTC 2024 ### Bad Referrer Count: 7104 -### Bad Bot Count: 667 +### Bad Bot Count: 668 ################################################### ### VERSION INFORMATION ## @@ -760,6 +760,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)domainsproject.org(?:\b)" 3; "~*(?:\b)eCatch(?:\b)" 3; "~*(?:\b)evc-batch(?:\b)" 3; + "~*(?:\b)everyfeed-spider(?:\b)" 3; "~*(?:\b)facebookscraper(?:\b)" 3; "~*(?:\b)gopher(?:\b)" 3; "~*(?:\b)heritrix(?:\b)" 3; diff --git a/dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf b/dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf index 0d8dca8c3..99575d41d 100644 --- a/dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf +++ b/dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.48 1; +10.1.0.64 1; 127.0.0.1 1; diff --git a/dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf b/dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf index 5d111b732..0ddde4728 100644 --- a/dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf +++ b/dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.48 0; +10.1.0.64 0; 127.0.0.1 0; diff --git a/dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf b/dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf index a3aaacfcd..b70423ac7 100644 --- a/dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf +++ b/dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2024.10.4835 -### Updated: Fri Oct 25 22:01:17 UTC 2024 +### Version: V4.2024.10.4836 +### Updated: Sat Oct 26 05:47:29 UTC 2024 ### Bad Referrer Count: 7104 -### Bad Bot Count: 667 +### Bad Bot Count: 668 ################################################### ### VERSION INFORMATION ## @@ -760,6 +760,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)domainsproject.org(?:\b)" 3; "~*(?:\b)eCatch(?:\b)" 3; "~*(?:\b)evc-batch(?:\b)" 3; + "~*(?:\b)everyfeed-spider(?:\b)" 3; "~*(?:\b)facebookscraper(?:\b)" 3; "~*(?:\b)gopher(?:\b)" 3; "~*(?:\b)heritrix(?:\b)" 3; diff --git a/dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf b/dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf index 0d8dca8c3..99575d41d 100644 --- a/dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf +++ b/dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.48 1; +10.1.0.64 1; 127.0.0.1 1; diff --git a/dev-tools/test1_conf_files_whitelist/bots.d/blacklist-user-agents.conf b/dev-tools/test1_conf_files_whitelist/bots.d/blacklist-user-agents.conf index 8a5c58f47..ced3f3f35 100644 --- a/dev-tools/test1_conf_files_whitelist/bots.d/blacklist-user-agents.conf +++ b/dev-tools/test1_conf_files_whitelist/bots.d/blacklist-user-agents.conf @@ -668,6 +668,7 @@ "~*(?:\b)domainsproject.org(?:\b|)" 0; "~*(?:\b)eCatch(?:\b|)" 0; "~*(?:\b)evc-batch(?:\b|)" 0; + "~*(?:\b)everyfeed-spider(?:\b|)" 0; "~*(?:\b)facebookscraper(?:\b|)" 0; "~*(?:\b)gopher(?:\b|)" 0; "~*(?:\b)heritrix(?:\b|)" 0; diff --git a/dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf b/dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf index 5d111b732..0ddde4728 100644 --- a/dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf +++ b/dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.48 0; +10.1.0.64 0; 127.0.0.1 0; diff --git a/dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf b/dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf index a3aaacfcd..b70423ac7 100644 --- a/dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf +++ b/dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2024.10.4835 -### Updated: Fri Oct 25 22:01:17 UTC 2024 +### Version: V4.2024.10.4836 +### Updated: Sat Oct 26 05:47:29 UTC 2024 ### Bad Referrer Count: 7104 -### Bad Bot Count: 667 +### Bad Bot Count: 668 ################################################### ### VERSION INFORMATION ## @@ -760,6 +760,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)domainsproject.org(?:\b)" 3; "~*(?:\b)eCatch(?:\b)" 3; "~*(?:\b)evc-batch(?:\b)" 3; + "~*(?:\b)everyfeed-spider(?:\b)" 3; "~*(?:\b)facebookscraper(?:\b)" 3; "~*(?:\b)gopher(?:\b)" 3; "~*(?:\b)heritrix(?:\b)" 3; diff --git a/dev-tools/test2_conf_files/bots.d/whitelist-ips.conf b/dev-tools/test2_conf_files/bots.d/whitelist-ips.conf index 1242e9bb0..1dea6b79e 100644 --- a/dev-tools/test2_conf_files/bots.d/whitelist-ips.conf +++ b/dev-tools/test2_conf_files/bots.d/whitelist-ips.conf @@ -45,4 +45,4 @@ # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST -172.183.122.254 0; +172.171.255.193 0; diff --git a/dev-tools/test2_conf_files/conf.d/globalblacklist.conf b/dev-tools/test2_conf_files/conf.d/globalblacklist.conf index 216c0b627..7be4c1fe4 100644 --- a/dev-tools/test2_conf_files/conf.d/globalblacklist.conf +++ b/dev-tools/test2_conf_files/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2024.10.4835 -### Updated: Fri Oct 25 22:01:17 UTC 2024 +### Version: V4.2024.10.4836 +### Updated: Sat Oct 26 05:47:29 UTC 2024 ### Bad Referrer Count: 7104 -### Bad Bot Count: 667 +### Bad Bot Count: 668 ################################################### ### VERSION INFORMATION ## @@ -760,6 +760,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)domainsproject.org(?:\b)" 3; "~*(?:\b)eCatch(?:\b)" 3; "~*(?:\b)evc-batch(?:\b)" 3; + "~*(?:\b)everyfeed-spider(?:\b)" 3; "~*(?:\b)facebookscraper(?:\b)" 3; "~*(?:\b)gopher(?:\b)" 3; "~*(?:\b)heritrix(?:\b)" 3; diff --git a/dev-tools/test3_conf_files/bots.d/whitelist-ips.conf b/dev-tools/test3_conf_files/bots.d/whitelist-ips.conf index 1242e9bb0..1dea6b79e 100644 --- a/dev-tools/test3_conf_files/bots.d/whitelist-ips.conf +++ b/dev-tools/test3_conf_files/bots.d/whitelist-ips.conf @@ -45,4 +45,4 @@ # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST -172.183.122.254 0; +172.171.255.193 0; diff --git a/dev-tools/test3_conf_files/conf.d/globalblacklist.conf b/dev-tools/test3_conf_files/conf.d/globalblacklist.conf index 59a27b7ae..72359bfa1 100644 --- a/dev-tools/test3_conf_files/conf.d/globalblacklist.conf +++ b/dev-tools/test3_conf_files/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2024.10.4835 -### Updated: Fri Oct 25 22:01:17 UTC 2024 +### Version: V4.2024.10.4836 +### Updated: Sat Oct 26 05:47:29 UTC 2024 ### Bad Referrer Count: 7104 -### Bad Bot Count: 667 +### Bad Bot Count: 668 ################################################### ### VERSION INFORMATION ## @@ -760,6 +760,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)domainsproject.org(?:\b)" 3; "~*(?:\b)eCatch(?:\b)" 3; "~*(?:\b)evc-batch(?:\b)" 3; + "~*(?:\b)everyfeed-spider(?:\b)" 3; "~*(?:\b)facebookscraper(?:\b)" 3; "~*(?:\b)gopher(?:\b)" 3; "~*(?:\b)heritrix(?:\b)" 3; diff --git a/dev-tools/test_units/blacklist-ips.conf b/dev-tools/test_units/blacklist-ips.conf index 0d8dca8c3..99575d41d 100644 --- a/dev-tools/test_units/blacklist-ips.conf +++ b/dev-tools/test_units/blacklist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.48 1; +10.1.0.64 1; 127.0.0.1 1; diff --git a/dev-tools/test_units/blacklist-user-agents.conf b/dev-tools/test_units/blacklist-user-agents.conf index 8a5c58f47..ced3f3f35 100644 --- a/dev-tools/test_units/blacklist-user-agents.conf +++ b/dev-tools/test_units/blacklist-user-agents.conf @@ -668,6 +668,7 @@ "~*(?:\b)domainsproject.org(?:\b|)" 0; "~*(?:\b)eCatch(?:\b|)" 0; "~*(?:\b)evc-batch(?:\b|)" 0; + "~*(?:\b)everyfeed-spider(?:\b|)" 0; "~*(?:\b)facebookscraper(?:\b|)" 0; "~*(?:\b)gopher(?:\b|)" 0; "~*(?:\b)heritrix(?:\b|)" 0; diff --git a/dev-tools/test_units/random-bots-for-test-quick.list b/dev-tools/test_units/random-bots-for-test-quick.list index cb1e5be41..f58d24082 100644 --- a/dev-tools/test_units/random-bots-for-test-quick.list +++ b/dev-tools/test_units/random-bots-for-test-quick.list @@ -1,10 +1,10 @@ -DigitalPebble -Lftp -LinkWalker -Nutch -Sucker -TeleportPro -TheNomad -Turnitin -URLy Warning -clark-crawler +BDFetch +Hloader +MIDown tool +Musobot +Navroad +RankActive +RebelMouse +Sottopop +adscanner +satoristudio.net diff --git a/dev-tools/test_units/random-bots-for-whitelist-test.list b/dev-tools/test_units/random-bots-for-whitelist-test.list index e621b282a..4b08d7a7b 100644 --- a/dev-tools/test_units/random-bots-for-whitelist-test.list +++ b/dev-tools/test_units/random-bots-for-whitelist-test.list @@ -1,100 +1,100 @@ -360Spider -ALittle Client -ASPSeek -Aspiegel -BDCbot +404enemy +AIBOT +Abonti +AfD-Verbotsverfahren +Atomseobot +AwarioSmartBot +BDFetch BLEXBot +BackDoorBot Backlink-Ceck -Bigfoot +Badass +Blackboard Blow -Bullseye -BunnySlippers -ChinaClaw -Citoid -Claritybot -CyotekWebCopy -DIIbot -DSearch +BuiltWith +CATExplorador +CensysInspect +CheTeam +CheeseBot +Cocolyzebot +Copyscape +Crawling at Home Project +Crescent +Curious +Deusu Digincore -Download Wonder -FDM -Freeuploader -GrapeFX -HTMLparser -Heritrix +DomainCrawler +FHscan +FemtosearchBot +FrontPage +FyberSpider +G-i-g-a-b-o-t +GPTBot +GetWeb +GridBot +HTTrack Humanlinks -HybridBot +IRLbot Iblog -Intelliseek -Internet Ninja -Jetty -Lanshanbot -LieBaoFast -Lightspeedsystems -MJ12bot -MSIECrawler -Mass Downloader -Mata Hari -Meanpathbot -Mediatoolkitbot -Mojeek -MolokaiBot -Nessus -NetAnts -Nettrack -NextGenSearchBot -Nibbler -Ninja -OpenLinkProfiler +JOC Web Spider +JustView +Jyxobot +LNSpiderguy +LWP::Simple +Ltx71 +Masscan +Morfeus Fucking Scanner +Mr.4x3 +Net Vampire +Nikto +NimbleCrawler +Offline Explorer OrangeSpider -Page Analyzer -PeoplePal -Petalbot -Picscout -ProPowerBot -Reaper -SalesIntelligent -ScanAlert -SearchmetricsBot -SentiBot -Seomoz -SiteCheckerBotCrawler +OutclicksBot +Pandalytics +Pi-Monster +ProWebWalker +Pu_iN +PxBroker +PyCurl +Rankivabot +RankurBot +SMTBot +ScoutJet +Scrapy +SeobilityBot SiteExplorer -SiteLockSpider -SmartDownload -SocialRankIOBot -Spammen -Spider_Bot -Sqworm -Steeler -Swiftbot -T0PHackTeam -Telesoft -Telesphoreo -Toweyabot -Turingos -VeriCiteCrawler -Virusdie -WBSearchBot -WWW-Collector-E -WWWOFFLE -Web Auto -WebStripper -WebSucker -Whacker +Sosospider +SpaceBison +Spinn3r +SuperBot +T8Abot +Thumbor +TinyTestBot +Web Fuck +WebCopier +WebLeacher +WebSauger +WebZIP +Whack +Whatweb +Widow +Wotbox +Xaldon WebSpider Xenu -Zermelo +YoudaoBot Zeus -cognitiveseo -dataforseobot -evc-batch -imagesift.com -iubenda-radar -linkfluence -magpie-crawler -mediawords -page scorer -probe-image-size -ripz -sexsearcher -zgrab +anthropic-ai +archive.org_bot +instabid +ips-agent +isitwp.com +lwp-request +omgili +s1z.ru +satoristudio.net +seocompany.store +sogouspider +trendiction.de +webgains-bot +webpros.com diff --git a/dev-tools/test_units/random-referrers-for-test-quick.list b/dev-tools/test_units/random-referrers-for-test-quick.list index 5a4c8df64..c77f48b59 100644 --- a/dev-tools/test_units/random-referrers-for-test-quick.list +++ b/dev-tools/test_units/random-referrers-for-test-quick.list @@ -1,10 +1,10 @@ -carina-sy.de -compliance-george.top -divci-hry.info -eloconcream.blogspot.com -keywordsking.com -kupit-adenu.ru -manuscript.su -onlywoman.org -szamponrevita.pl -urccvfmc.bloger.index.hr +bookmark4you.com.biz +cashonads.com +compliance-barak.xyz +gg-arena.ru +mindbox.co.za +ok-ua.info +slowmac.tech +trafficjunky.net +uhdtv.website +urengoy.pro diff --git a/dev-tools/test_units/random-referrers-for-whitelist-test.list b/dev-tools/test_units/random-referrers-for-whitelist-test.list index 7d11e914e..504682cc2 100644 --- a/dev-tools/test_units/random-referrers-for-whitelist-test.list +++ b/dev-tools/test_units/random-referrers-for-whitelist-test.list @@ -1,100 +1,100 @@ -1millionusd.xyz -aa81bf391151884adfa3dd677e41f94be1.com -actulite.com -afftrack001.com -alibestsale.com -arcteryxsale.online -autoua.top -ayakino.net -babieca.com -beachpics.com -bear.gotcher.us -belstaffstore.online -bestmouthwash.club -bitcoinpile.com -bocoarchives.org -brainboostingsupplements.org -brandbucket.com -buigas.00it.com -chcu.net -cheapmoncler.win -chinese-amezon.com -dealitright.click -diegolopezcastan.com -drunkmoms.net -elektrozigaretten2.yn.lt -ezigarettekaufen1.hpage.com -f07.de -farmprofi.net -fickluder69.com -formseo.com -freejabs.com -fugarif.ga -gallery.rennlist.com -girls-ufa.ru -handicapbathtubarea.com -highland-homes.com -histock.info -hostcritique.com -hostingclub.lk -humanorightswatch.org -ikearugs.xyz -ikritikimou.gr -intimshop-fantasy.ru -jav-fetish.site -keenoutlet.online -kustanay.kz -law-check-nine.xyz -lion.playtap.us -liveinternet.ro -localmatchbook.com -lonely-mature.com -marketingvici.com +4replicawatch.net +66cpwgln.space +aaa38852e886ac4af1a3cff9b47cab6272.com +academiacsmendoza.org +ads-cool.pro +adviceforum.info +androids-store.com +arkkivoltti.net +asociatia-tipografilor-transilvania.ro +azazu.ru +bausparen.bz.it +bonkers.name +bonzbuddy.com +buzz-porno.info +centrumcoachingu.com +cialischsrx.com +clean-start.net +compliance-george.top +compliance-irvin.xyz +compliance-willy.top +custom-product-labels.com +data1.scopich.com +dealighted.com +domaincdn.xyz +drpornogratisx.xxx +ekto.ee +eluxer.net +epngo.bz +essayservicewriting.org +eyessurgery.ru +fefo.gdn +filmania-x.ru +freefoto.ca +gazporno.com +ge0ip.net +globalscam.ga +growboxbl.ru +gunsvicceadadebt.tk +hdseriale.pl +housediz.com +housemilan.ru +humbmosquina.tk +ilmexico.com +images-free.net +imagez.co +imediadesk.com +intim-uslugi.info +japan-bearings.ru +jerseywholesalechinabiz.com +katadhin.co +kchaxton.com +keywordglobal.co.za +keywordsuggests.com +lflash.ru maximilitary.ru -mixed-wrestling.ru -mlf.hordo.win -mobplayer.ru -moncleronline.xyz -muabancantho.info -mypornfree.ru -na-telefon.biz -notfastfood.ru -novodigs.com -nuit-artisanale.com -nuker.com -oxford-book.com.ua -parajumpersstore.online -pathwhelp.org -physfunc.ru -pojdelo.weebly.com -produkto.net -prohoster.info -redirectme.net -russia-tao.ru -samolet.fr -sexspornotub.com -sexuria.net -sharpchallenge.com -soft-terminal.ru -special-porn.com -studworks.org -sugarkun.com -summerlinhomes411.info -teenforporn.com -top-l2.com -trahic.ru -url-extractor.xyz +mesbuta.info +moroccosurfadventures.com +mrcsa.com.au +mrlmedia.net +myfreemp3.eu +myfreetutorials.com +najaden.no +no-rx.info +ownshop.cf +pandarastore.top +parlament.biz +phormchina.com +piccshare.com +portadd.men +privatefx-in.ru +pussysaga.com +remont-ustanovka-tehniki.ru +robot-forex.biz +rus-pornuha.com +ruscoininvest.company +safe-app.net +seozoom.it +shareyards.com +shemalegalls.blogporn.in +sim-service.net +snip.to +spacash.com +staff.prairiesouth.ca +strag-invest.ru +tako3.com +thebluffs.com +tizanidinehcl4mgisitanarcotic.blogspot.com +tpu.ru +treasuretrack-a.akamaihd.net +tubeoffline.com +v24s.net vbikse.com -viandpet.com -vikistars.com -vpnhowto.info -waterpurifier.club -wineitudes.wordpress.com -wormix-cheats.ru -www.ehscloud.cn -xxx-treker.ru -xxxguitars.com -xxxkaz.org -zed21.net -zeroredirect8.com -zryydi.com +villacoloniale.com +vote-up.ru +wholesalecheapjerseysfree.com +womensplay.net +www.timer4web.com +x-true.info +xn--80aafb2a.xn--p1ai +zmoda.hostreo.com diff --git a/dev-tools/test_units/whitelist-ips.conf b/dev-tools/test_units/whitelist-ips.conf index 5d111b732..0ddde4728 100644 --- a/dev-tools/test_units/whitelist-ips.conf +++ b/dev-tools/test_units/whitelist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.48 0; +10.1.0.64 0; 127.0.0.1 0; diff --git a/dev-tools/tmp/robots.txt b/dev-tools/tmp/robots.txt index 34e1babf0..a0b049d89 100644 --- a/dev-tools/tmp/robots.txt +++ b/dev-tools/tmp/robots.txt @@ -6,9 +6,9 @@ ### Version Information # ################################################### -### Version: V4.2024.10.4835 -### Updated: Fri Oct 25 22:01:18 UTC 2024 -### Bad Bot Count: 667 +### Version: V4.2024.10.4836 +### Updated: Sat Oct 26 05:47:30 UTC 2024 +### Bad Bot Count: 668 ################################################### ### Version Information ## @@ -1233,6 +1233,8 @@ User-agent: eCatch Disallow:/ User-agent: evc-batch Disallow:/ +User-agent: everyfeed-spider +Disallow:/ User-agent: facebookscraper Disallow:/ User-agent: gopher diff --git a/robots.txt/robots.txt b/robots.txt/robots.txt index 34e1babf0..a0b049d89 100644 --- a/robots.txt/robots.txt +++ b/robots.txt/robots.txt @@ -6,9 +6,9 @@ ### Version Information # ################################################### -### Version: V4.2024.10.4835 -### Updated: Fri Oct 25 22:01:18 UTC 2024 -### Bad Bot Count: 667 +### Version: V4.2024.10.4836 +### Updated: Sat Oct 26 05:47:30 UTC 2024 +### Bad Bot Count: 668 ################################################### ### Version Information ## @@ -1233,6 +1233,8 @@ User-agent: eCatch Disallow:/ User-agent: evc-batch Disallow:/ +User-agent: everyfeed-spider +Disallow:/ User-agent: facebookscraper Disallow:/ User-agent: gopher