From c86f03fd4b4050600c37e4e857d96ddf13a79217 Mon Sep 17 00:00:00 2001 From: mitchellkrogza Date: Mon, 1 May 2023 05:53:49 +0000 Subject: [PATCH] V4.2023.05.3718 [ci skip] --- AUTO-CONFIGURATION.md | 4 +- MANUAL-CONFIGURATION.md | 4 +- README.md | 4 +- _google_analytics_ghost_spam/README.md | 4 +- .../etc/nginx/conf.d/globalblacklist.conf | 7 +- conf.d/globalblacklist-testing-version.conf | 7 +- conf.d/globalblacklist.conf | 7 +- dev-tools/_robots_input/robots-input.txt | 1 - dev-tools/buildnumber | 2 +- dev-tools/globalblacklist-testing.template | 7 +- dev-tools/globalblacklist.template | 7 +- .../bots.d/whitelist-ips.conf | 2 +- .../conf.d/globalblacklist.conf | 7 +- .../bots.d/blacklist-ips.conf | 2 +- .../bots.d/whitelist-ips.conf | 2 +- .../conf.d/globalblacklist.conf | 7 +- .../bots.d/blacklist-ips.conf | 2 +- .../bots.d/whitelist-ips.conf | 2 +- .../conf.d/globalblacklist.conf | 7 +- .../bots.d/blacklist-ips.conf | 2 +- .../bots.d/blacklist-user-agents.conf | 1 - .../bots.d/whitelist-ips.conf | 2 +- .../conf.d/globalblacklist.conf | 7 +- .../bots.d/whitelist-ips.conf | 2 +- .../conf.d/globalblacklist.conf | 7 +- .../bots.d/whitelist-ips.conf | 2 +- .../conf.d/globalblacklist.conf | 7 +- dev-tools/test_units/blacklist-ips.conf | 2 +- .../test_units/blacklist-user-agents.conf | 1 - .../random-bots-for-test-quick.list | 20 +- .../random-bots-for-whitelist-test.list | 172 +++++++-------- .../random-referrers-for-test-quick.list | 20 +- .../random-referrers-for-whitelist-test.list | 200 +++++++++--------- dev-tools/test_units/whitelist-ips.conf | 2 +- dev-tools/tmp/robots.txt | 8 +- robots.txt/robots.txt | 8 +- 36 files changed, 265 insertions(+), 283 deletions(-) diff --git a/AUTO-CONFIGURATION.md b/AUTO-CONFIGURATION.md index 7073e2b69..72accd693 100755 --- a/AUTO-CONFIGURATION.md +++ b/AUTO-CONFIGURATION.md @@ -4,9 +4,9 @@ ### PLEASE READ CONFIGURATION INSTRUCTIONS BELOW THOROUGHLY :exclamation: _______________ -#### Version: V4.2023.05.3717 +#### Version: V4.2023.05.3718 #### Bad Referrers Blocked: [7104](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-referrers.list) -#### Bad User-Agents (Bots) Blocked: [640](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) +#### Bad User-Agents (Bots) Blocked: [639](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) #### Fake Googlebots Blocked: [217](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/fake-googlebots.list) ____________________ diff --git a/MANUAL-CONFIGURATION.md b/MANUAL-CONFIGURATION.md index fdbd51ec1..c26cdb3ed 100755 --- a/MANUAL-CONFIGURATION.md +++ b/MANUAL-CONFIGURATION.md @@ -4,9 +4,9 @@ ### PLEASE READ CONFIGURATION INSTRUCTIONS BELOW THOROUGHLY :exclamation: _______________ -#### Version: V4.2023.05.3717 +#### Version: V4.2023.05.3718 #### Bad Referrers Blocked: [7104](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-referrers.list) -#### Bad User-Agents (Bots) Blocked: [640](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) +#### Bad User-Agents (Bots) Blocked: [639](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) #### Fake Googlebots Blocked: [217](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/fake-googlebots.list) ____________________ diff --git a/README.md b/README.md index 1214196b4..0c3b5a4fc 100755 --- a/README.md +++ b/README.md @@ -14,9 +14,9 @@ ##### The Ultimate Nginx Bad Bot, User-Agent, Spam Referrer Blocker, Adware, Malware and Ransomware Blocker, Clickjacking Blocker, Click Re-Directing Blocker, SEO Companies and Bad IP Blocker with Anti DDOS System, Nginx Rate Limiting and Wordpress Theme Detector Blocking. Stop and Block all kinds of bad internet traffic even Fake Googlebots from ever reaching your web sites. [PLEASE SEE: Definition of Bad Bots](#define-bad-bots) _______________ -#### Version: V4.2023.05.3717 +#### Version: V4.2023.05.3718 #### Bad Referrers Blocked: [7104](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-referrers.list) -#### Bad User-Agents (Bots) Blocked: [640](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) +#### Bad User-Agents (Bots) Blocked: [639](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) #### Fake Googlebots Blocked: [217](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/fake-googlebots.list) ____________________ diff --git a/_google_analytics_ghost_spam/README.md b/_google_analytics_ghost_spam/README.md index 800948f76..2d8de192c 100644 --- a/_google_analytics_ghost_spam/README.md +++ b/_google_analytics_ghost_spam/README.md @@ -3,9 +3,9 @@ # EASY CONFIGURATION INSTRUCTIONS FOR STOPPING GOOGLE ANALYTICS "GHOST" SPAM _______________ -#### Version: V4.2023.05.3717 +#### Version: V4.2023.05.3718 #### Bad Referrers Blocked: [7104](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-referrers.list) -#### Bad User-Agents (Bots) Blocked: [640](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) +#### Bad User-Agents (Bots) Blocked: [639](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) #### Fake Googlebots Blocked: [217](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/fake-googlebots.list) ____________________ diff --git a/_sample_config_files/Engintron_for_cPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf b/_sample_config_files/Engintron_for_cPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf index 21a18ec40..0924ed25a 100755 --- a/_sample_config_files/Engintron_for_cPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf +++ b/_sample_config_files/Engintron_for_cPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2023.05.3717 -### Updated: Mon May 1 04:37:16 UTC 2023 +### Version: V4.2023.05.3718 +### Updated: Mon May 1 05:50:03 UTC 2023 ### Bad Referrer Count: 7104 -### Bad Bot Count: 640 +### Bad Bot Count: 639 ################################################### ### VERSION INFORMATION ## @@ -466,7 +466,6 @@ map $http_user_agent $bad_bot { "~*(?:\b)Nimbostratus(?:\b)" 3; "~*(?:\b)Ninja(?:\b)" 3; "~*(?:\b)Nmap(?:\b)" 3; - "~*(?:\b)Not(?:\b)" 3; "~*(?:\b)Nuclei(?:\b)" 3; "~*(?:\b)Nutch(?:\b)" 3; "~*(?:\b)Octopus(?:\b)" 3; diff --git a/conf.d/globalblacklist-testing-version.conf b/conf.d/globalblacklist-testing-version.conf index 1d2eb1b24..df006d64f 100644 --- a/conf.d/globalblacklist-testing-version.conf +++ b/conf.d/globalblacklist-testing-version.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2023.05.3717 -### Updated: Mon May 1 04:37:17 UTC 2023 +### Version: V3.2023.05.3718 +### Updated: Mon May 1 05:50:04 UTC 2023 ### Bad Referrer Count: 7104 -### Bad Bot Count: 640 +### Bad Bot Count: 639 ################################################### ### VERSION INFORMATION ## @@ -466,7 +466,6 @@ map $http_user_agent $bad_bot { "~*(?:\b)Nimbostratus(?:\b)" 3; "~*(?:\b)Ninja(?:\b)" 3; "~*(?:\b)Nmap(?:\b)" 3; - "~*(?:\b)Not(?:\b)" 3; "~*(?:\b)Nuclei(?:\b)" 3; "~*(?:\b)Nutch(?:\b)" 3; "~*(?:\b)Octopus(?:\b)" 3; diff --git a/conf.d/globalblacklist.conf b/conf.d/globalblacklist.conf index 21a18ec40..0924ed25a 100644 --- a/conf.d/globalblacklist.conf +++ b/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2023.05.3717 -### Updated: Mon May 1 04:37:16 UTC 2023 +### Version: V4.2023.05.3718 +### Updated: Mon May 1 05:50:03 UTC 2023 ### Bad Referrer Count: 7104 -### Bad Bot Count: 640 +### Bad Bot Count: 639 ################################################### ### VERSION INFORMATION ## @@ -466,7 +466,6 @@ map $http_user_agent $bad_bot { "~*(?:\b)Nimbostratus(?:\b)" 3; "~*(?:\b)Ninja(?:\b)" 3; "~*(?:\b)Nmap(?:\b)" 3; - "~*(?:\b)Not(?:\b)" 3; "~*(?:\b)Nuclei(?:\b)" 3; "~*(?:\b)Nutch(?:\b)" 3; "~*(?:\b)Octopus(?:\b)" 3; diff --git a/dev-tools/_robots_input/robots-input.txt b/dev-tools/_robots_input/robots-input.txt index 6b38a1b5f..5e0c4116f 100644 --- a/dev-tools/_robots_input/robots-input.txt +++ b/dev-tools/_robots_input/robots-input.txt @@ -317,7 +317,6 @@ NimbleCrawler Nimbostratus Ninja Nmap -Not Nuclei Nutch Octopus diff --git a/dev-tools/buildnumber b/dev-tools/buildnumber index 69e041273..03d011a14 100644 --- a/dev-tools/buildnumber +++ b/dev-tools/buildnumber @@ -1 +1 @@ -3717 +3718 diff --git a/dev-tools/globalblacklist-testing.template b/dev-tools/globalblacklist-testing.template index 1d2eb1b24..df006d64f 100644 --- a/dev-tools/globalblacklist-testing.template +++ b/dev-tools/globalblacklist-testing.template @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2023.05.3717 -### Updated: Mon May 1 04:37:17 UTC 2023 +### Version: V3.2023.05.3718 +### Updated: Mon May 1 05:50:04 UTC 2023 ### Bad Referrer Count: 7104 -### Bad Bot Count: 640 +### Bad Bot Count: 639 ################################################### ### VERSION INFORMATION ## @@ -466,7 +466,6 @@ map $http_user_agent $bad_bot { "~*(?:\b)Nimbostratus(?:\b)" 3; "~*(?:\b)Ninja(?:\b)" 3; "~*(?:\b)Nmap(?:\b)" 3; - "~*(?:\b)Not(?:\b)" 3; "~*(?:\b)Nuclei(?:\b)" 3; "~*(?:\b)Nutch(?:\b)" 3; "~*(?:\b)Octopus(?:\b)" 3; diff --git a/dev-tools/globalblacklist.template b/dev-tools/globalblacklist.template index 21a18ec40..0924ed25a 100755 --- a/dev-tools/globalblacklist.template +++ b/dev-tools/globalblacklist.template @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2023.05.3717 -### Updated: Mon May 1 04:37:16 UTC 2023 +### Version: V4.2023.05.3718 +### Updated: Mon May 1 05:50:03 UTC 2023 ### Bad Referrer Count: 7104 -### Bad Bot Count: 640 +### Bad Bot Count: 639 ################################################### ### VERSION INFORMATION ## @@ -466,7 +466,6 @@ map $http_user_agent $bad_bot { "~*(?:\b)Nimbostratus(?:\b)" 3; "~*(?:\b)Ninja(?:\b)" 3; "~*(?:\b)Nmap(?:\b)" 3; - "~*(?:\b)Not(?:\b)" 3; "~*(?:\b)Nuclei(?:\b)" 3; "~*(?:\b)Nutch(?:\b)" 3; "~*(?:\b)Octopus(?:\b)" 3; diff --git a/dev-tools/test1_conf_files/bots.d/whitelist-ips.conf b/dev-tools/test1_conf_files/bots.d/whitelist-ips.conf index 40955ca94..c3f591b74 100644 --- a/dev-tools/test1_conf_files/bots.d/whitelist-ips.conf +++ b/dev-tools/test1_conf_files/bots.d/whitelist-ips.conf @@ -45,4 +45,4 @@ # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST -40.86.28.128 0; +104.45.200.55 0; diff --git a/dev-tools/test1_conf_files/conf.d/globalblacklist.conf b/dev-tools/test1_conf_files/conf.d/globalblacklist.conf index 21a18ec40..0924ed25a 100644 --- a/dev-tools/test1_conf_files/conf.d/globalblacklist.conf +++ b/dev-tools/test1_conf_files/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2023.05.3717 -### Updated: Mon May 1 04:37:16 UTC 2023 +### Version: V4.2023.05.3718 +### Updated: Mon May 1 05:50:03 UTC 2023 ### Bad Referrer Count: 7104 -### Bad Bot Count: 640 +### Bad Bot Count: 639 ################################################### ### VERSION INFORMATION ## @@ -466,7 +466,6 @@ map $http_user_agent $bad_bot { "~*(?:\b)Nimbostratus(?:\b)" 3; "~*(?:\b)Ninja(?:\b)" 3; "~*(?:\b)Nmap(?:\b)" 3; - "~*(?:\b)Not(?:\b)" 3; "~*(?:\b)Nuclei(?:\b)" 3; "~*(?:\b)Nutch(?:\b)" 3; "~*(?:\b)Octopus(?:\b)" 3; diff --git a/dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf b/dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf index ee9141611..53409d87c 100644 --- a/dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf +++ b/dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.44 1; +10.1.0.16 1; 127.0.0.1 1; diff --git a/dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf b/dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf index 9dcba65e6..529aeb017 100644 --- a/dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf +++ b/dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.44 0; +10.1.0.16 0; 127.0.0.1 0; diff --git a/dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf b/dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf index 21a18ec40..0924ed25a 100644 --- a/dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf +++ b/dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2023.05.3717 -### Updated: Mon May 1 04:37:16 UTC 2023 +### Version: V4.2023.05.3718 +### Updated: Mon May 1 05:50:03 UTC 2023 ### Bad Referrer Count: 7104 -### Bad Bot Count: 640 +### Bad Bot Count: 639 ################################################### ### VERSION INFORMATION ## @@ -466,7 +466,6 @@ map $http_user_agent $bad_bot { "~*(?:\b)Nimbostratus(?:\b)" 3; "~*(?:\b)Ninja(?:\b)" 3; "~*(?:\b)Nmap(?:\b)" 3; - "~*(?:\b)Not(?:\b)" 3; "~*(?:\b)Nuclei(?:\b)" 3; "~*(?:\b)Nutch(?:\b)" 3; "~*(?:\b)Octopus(?:\b)" 3; diff --git a/dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf b/dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf index ee9141611..53409d87c 100644 --- a/dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf +++ b/dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.44 1; +10.1.0.16 1; 127.0.0.1 1; diff --git a/dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf b/dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf index 9dcba65e6..529aeb017 100644 --- a/dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf +++ b/dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.44 0; +10.1.0.16 0; 127.0.0.1 0; diff --git a/dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf b/dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf index 21a18ec40..0924ed25a 100644 --- a/dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf +++ b/dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2023.05.3717 -### Updated: Mon May 1 04:37:16 UTC 2023 +### Version: V4.2023.05.3718 +### Updated: Mon May 1 05:50:03 UTC 2023 ### Bad Referrer Count: 7104 -### Bad Bot Count: 640 +### Bad Bot Count: 639 ################################################### ### VERSION INFORMATION ## @@ -466,7 +466,6 @@ map $http_user_agent $bad_bot { "~*(?:\b)Nimbostratus(?:\b)" 3; "~*(?:\b)Ninja(?:\b)" 3; "~*(?:\b)Nmap(?:\b)" 3; - "~*(?:\b)Not(?:\b)" 3; "~*(?:\b)Nuclei(?:\b)" 3; "~*(?:\b)Nutch(?:\b)" 3; "~*(?:\b)Octopus(?:\b)" 3; diff --git a/dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf b/dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf index ee9141611..53409d87c 100644 --- a/dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf +++ b/dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.44 1; +10.1.0.16 1; 127.0.0.1 1; diff --git a/dev-tools/test1_conf_files_whitelist/bots.d/blacklist-user-agents.conf b/dev-tools/test1_conf_files_whitelist/bots.d/blacklist-user-agents.conf index dde25a039..8af9f5370 100644 --- a/dev-tools/test1_conf_files_whitelist/bots.d/blacklist-user-agents.conf +++ b/dev-tools/test1_conf_files_whitelist/bots.d/blacklist-user-agents.conf @@ -378,7 +378,6 @@ "~*(?:\b)Nimbostratus(?:\b|)" 0; "~*(?:\b)Ninja(?:\b|)" 0; "~*(?:\b)Nmap(?:\b|)" 0; - "~*(?:\b)Not(?:\b|)" 0; "~*(?:\b)Nuclei(?:\b|)" 0; "~*(?:\b)Nutch(?:\b|)" 0; "~*(?:\b)Octopus(?:\b|)" 0; diff --git a/dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf b/dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf index 9dcba65e6..529aeb017 100644 --- a/dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf +++ b/dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.44 0; +10.1.0.16 0; 127.0.0.1 0; diff --git a/dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf b/dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf index 21a18ec40..0924ed25a 100644 --- a/dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf +++ b/dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2023.05.3717 -### Updated: Mon May 1 04:37:16 UTC 2023 +### Version: V4.2023.05.3718 +### Updated: Mon May 1 05:50:03 UTC 2023 ### Bad Referrer Count: 7104 -### Bad Bot Count: 640 +### Bad Bot Count: 639 ################################################### ### VERSION INFORMATION ## @@ -466,7 +466,6 @@ map $http_user_agent $bad_bot { "~*(?:\b)Nimbostratus(?:\b)" 3; "~*(?:\b)Ninja(?:\b)" 3; "~*(?:\b)Nmap(?:\b)" 3; - "~*(?:\b)Not(?:\b)" 3; "~*(?:\b)Nuclei(?:\b)" 3; "~*(?:\b)Nutch(?:\b)" 3; "~*(?:\b)Octopus(?:\b)" 3; diff --git a/dev-tools/test2_conf_files/bots.d/whitelist-ips.conf b/dev-tools/test2_conf_files/bots.d/whitelist-ips.conf index 40955ca94..c3f591b74 100644 --- a/dev-tools/test2_conf_files/bots.d/whitelist-ips.conf +++ b/dev-tools/test2_conf_files/bots.d/whitelist-ips.conf @@ -45,4 +45,4 @@ # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST -40.86.28.128 0; +104.45.200.55 0; diff --git a/dev-tools/test2_conf_files/conf.d/globalblacklist.conf b/dev-tools/test2_conf_files/conf.d/globalblacklist.conf index fc8fe99c4..6f16a8d71 100644 --- a/dev-tools/test2_conf_files/conf.d/globalblacklist.conf +++ b/dev-tools/test2_conf_files/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2023.05.3717 -### Updated: Mon May 1 04:37:16 UTC 2023 +### Version: V4.2023.05.3718 +### Updated: Mon May 1 05:50:03 UTC 2023 ### Bad Referrer Count: 7104 -### Bad Bot Count: 640 +### Bad Bot Count: 639 ################################################### ### VERSION INFORMATION ## @@ -466,7 +466,6 @@ map $http_user_agent $bad_bot { "~*(?:\b)Nimbostratus(?:\b)" 3; "~*(?:\b)Ninja(?:\b)" 3; "~*(?:\b)Nmap(?:\b)" 3; - "~*(?:\b)Not(?:\b)" 3; "~*(?:\b)Nuclei(?:\b)" 3; "~*(?:\b)Nutch(?:\b)" 3; "~*(?:\b)Octopus(?:\b)" 3; diff --git a/dev-tools/test3_conf_files/bots.d/whitelist-ips.conf b/dev-tools/test3_conf_files/bots.d/whitelist-ips.conf index 40955ca94..c3f591b74 100644 --- a/dev-tools/test3_conf_files/bots.d/whitelist-ips.conf +++ b/dev-tools/test3_conf_files/bots.d/whitelist-ips.conf @@ -45,4 +45,4 @@ # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST -40.86.28.128 0; +104.45.200.55 0; diff --git a/dev-tools/test3_conf_files/conf.d/globalblacklist.conf b/dev-tools/test3_conf_files/conf.d/globalblacklist.conf index 723fd89bc..c8eb2647b 100644 --- a/dev-tools/test3_conf_files/conf.d/globalblacklist.conf +++ b/dev-tools/test3_conf_files/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2023.05.3717 -### Updated: Mon May 1 04:37:16 UTC 2023 +### Version: V4.2023.05.3718 +### Updated: Mon May 1 05:50:03 UTC 2023 ### Bad Referrer Count: 7104 -### Bad Bot Count: 640 +### Bad Bot Count: 639 ################################################### ### VERSION INFORMATION ## @@ -466,7 +466,6 @@ map $http_user_agent $bad_bot { "~*(?:\b)Nimbostratus(?:\b)" 3; "~*(?:\b)Ninja(?:\b)" 3; "~*(?:\b)Nmap(?:\b)" 3; - "~*(?:\b)Not(?:\b)" 3; "~*(?:\b)Nuclei(?:\b)" 3; "~*(?:\b)Nutch(?:\b)" 3; "~*(?:\b)Octopus(?:\b)" 3; diff --git a/dev-tools/test_units/blacklist-ips.conf b/dev-tools/test_units/blacklist-ips.conf index ee9141611..53409d87c 100644 --- a/dev-tools/test_units/blacklist-ips.conf +++ b/dev-tools/test_units/blacklist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.44 1; +10.1.0.16 1; 127.0.0.1 1; diff --git a/dev-tools/test_units/blacklist-user-agents.conf b/dev-tools/test_units/blacklist-user-agents.conf index dde25a039..8af9f5370 100644 --- a/dev-tools/test_units/blacklist-user-agents.conf +++ b/dev-tools/test_units/blacklist-user-agents.conf @@ -378,7 +378,6 @@ "~*(?:\b)Nimbostratus(?:\b|)" 0; "~*(?:\b)Ninja(?:\b|)" 0; "~*(?:\b)Nmap(?:\b|)" 0; - "~*(?:\b)Not(?:\b|)" 0; "~*(?:\b)Nuclei(?:\b|)" 0; "~*(?:\b)Nutch(?:\b|)" 0; "~*(?:\b)Octopus(?:\b|)" 0; diff --git a/dev-tools/test_units/random-bots-for-test-quick.list b/dev-tools/test_units/random-bots-for-test-quick.list index 2df82da04..576ffd200 100644 --- a/dev-tools/test_units/random-bots-for-test-quick.list +++ b/dev-tools/test_units/random-bots-for-test-quick.list @@ -1,10 +1,10 @@ -CCBot -EyeNetIE -IDBTE4M -LinkWalker -MegaIndex.ru -Metauri -NetZIP -Titan -trendiction.com -voyagerx.com +Black Hole +Go!Zilla +Heritrix +JetCar +Jorgee +LinkScan +PxBroker +SEOkicks +Thumbor +Whacker diff --git a/dev-tools/test_units/random-bots-for-whitelist-test.list b/dev-tools/test_units/random-bots-for-whitelist-test.list index 461510c42..2191aa92d 100644 --- a/dev-tools/test_units/random-bots-for-whitelist-test.list +++ b/dev-tools/test_units/random-bots-for-whitelist-test.list @@ -1,100 +1,100 @@ -AIBOT -Alligator -Anarchy -AwarioRssBot -BlackWidow -Boardreader -CazoodleBot -CherryPicker -DBLBot -DatabaseDriverMysqli -Deusu -Dirbuster -DnBCrawler-Analytics -DomainAppender -EMail Wolf -EasyDL -EirGrabber -FDM -Firefox/7.0 -GT::WWW -Getintent -Go!Zilla -Go-Ahead-Got-It -GoZilla -GrabNet -HEADMasterSEO +404checker +Acunetix +AiHitBot +Aipbot +AwarioSmartBot +BBBike +BLEXBot +BackDoorBot +Battleztar Bazinga +Blackboard +BlowFish +BunnySlippers +CensysInspect +Citoid +Copier +Digincore +Dispatch +DittoSpyder +DomCopBot +DomainSigmaCrawler +Domains Project +Dragonfly +ECCP/1.0 +Ebingbong +Exabot +ExtLinksBot +ExtractorPro +EyeNetIE +Fimap +Fuzz +GalaxyBot +Gotit HTMLparser Humanlinks -Internet Ninja -Iskanie -Kinza -Larbin -LibWeb -LinkScan -Linkbot -LinkextractorPro -LinkpadBot +HybridBot +IlseBot +Image Fetch +IndeedBot +Indy Library +Likse Lipperhey Spider -MQQBrowser -Moblie Safari -Mojeek -Nameprotect +MSIECrawler +Mail.RU_Bot +Masscan +Minefield +MolokaiBot +Morfeus Fucking Scanner Nessus -NetSpider -Nettrack -Octopus -Offline Navigator -OpenLinkProfiler -OpenVAS +Nibbler +Nikto +Nimbostratus +Ninja +Nutch OutclicksBot -Page Analyzer -Papa Foto -Pi-Monster -PleaseCrawl -Pockey -Probethenet -Pu_iN -Recorder -RepoMonkey -Ripper -SEOkicks -ScanAlert -Scanbot +PageScorer +Panscient +PxBroker +Quick-Crawler +Rankivabot +Rogerbot +SEOstats +SMTBot ScrepyBot -Semrush Seomoz Sitebeam -Sociscraper -Steeler -Sucker -T8Abot +Sogou web spider +SpaceBison +SputnikBot +SuperBot +Suzuran +Telesphoreo Telesphorep -Titan -Trendictionbot -VCI -Vacuum -Virusdie -WEBDAV -WISENutbot -Web Enhancer -WebAuto +TightTwatBot +Toweyabot +Voltron +WASALive-Bot +WWW::Mechanize +WallpapersHD +Web Sauger +Web Sucker +WebCollage WebCopier -WebImageCollector -WebWhacker -Website Quester -WebsiteExtractor -Xaldon WebSpider -Zauba -ZoominfoBot +WebStripper +Whacker +Wotbox +Zitebot +ZmEu +ZoomBot archive.org_bot -arquivo-web-crawler -gopher +backlink-check +crawler.feedback +eCatch heritrix -netEstate NE Crawler -oBot +plumanalytics +probe-image-size +satoristudio.net scan.lol -seobility -sexsearcher +seostar spyfu -webgains-bot +xpymep1.exe diff --git a/dev-tools/test_units/random-referrers-for-test-quick.list b/dev-tools/test_units/random-referrers-for-test-quick.list index 41c74c983..7d0984b7b 100644 --- a/dev-tools/test_units/random-referrers-for-test-quick.list +++ b/dev-tools/test_units/random-referrers-for-test-quick.list @@ -1,10 +1,10 @@ -board.f00d.de -dnepr-avtospar.com.ua -favoritemoney.ru -kskjco.club -massage-info.nl -repeatlogo.co.uk -shoesonlinebuy.cn -waycash.net -wheelchairliftsarea.com -womensterritory.ru +3w24.com +focalink.com +landscapebackgrounds.blogspot.com +microstatic.pl +opinionreelle.com +petedrummond.com +petitions.whitehouse.gov +picturesmania.com +sexyali.com +tizanidinecost.blogspot.com diff --git a/dev-tools/test_units/random-referrers-for-whitelist-test.list b/dev-tools/test_units/random-referrers-for-whitelist-test.list index 56e7e3970..b4e263b2a 100644 --- a/dev-tools/test_units/random-referrers-for-whitelist-test.list +++ b/dev-tools/test_units/random-referrers-for-whitelist-test.list @@ -1,100 +1,100 @@ -007angels.com -1kinobig.ru -7search.com -aa625d84f1587749c1ab011d6f269f7d64.com -aaronabel.com -accessoires-mode.in -acunetix-referrer.com -adman.gr -adprotect.net -agadelha.com.br -anniemation.com -apibrowsesmartne-a.akamaihd.net -app5.letmacworkfaster.world -appiq.mobi -atmovs.com -auto-moto-elektronika.cz -autoua.top -aviav.ru -babyguns.ru -blue-square.biz -bonzibuddi.com -buchananshardware.com -candycrushshop.com -cashonads.com -chastnoeporno.com -cheapestjerseysonwholesale.com -chelnytruck.ru -customcatchcan.com -detskie-konstruktory.ru -downloadme.life -e-c.al -edudocs.net -elektronischezigarettekaufen2.tumblr.com -eloconcream.blogspot.com -eropho.net -esopini.com -everflora.ru -eywords-monitoring-your-success.com -fapgon.com -floridamhca.org -gk170.ru -graphid.com -gsasearchengineranker.site -gsasearchengineranker.top -hamptonoaks.ca -hd720kino.ru -howmuchdoestizanidinecost.blogspot.com -iamsport.org -insider.pro -intellekt21.ru -iopeninghours.co.uk -kakadu-interior.com.ua -kinogonew.ru -konkursov.net -kosmetyki.tm.pl -kursy-ege.ru -leftofcentrist.com -limtu.ifmo.ru -linkrr.com -michaelkorsoutletstore.net -mp3ringtone.info -openmultipleurl.com -pihl.se -pizdeishn.com -porngalleries.top -prombudpostach.com.ua -promoheads.com -prostitutki-kharkova.org -pukaporn.com -qwesa.ru -ritlweb.com -running-line.ru -s-forum.biz -scanner-jessica.top -sex.hotblog.top -sexystrippe.info -sisi-go.ru -siteimprove.com -snomer1.ru -socialbuttons.xyz -soft-program.com -softwaretrend.net -solution4u.com -sport7777.net -strana-krasoty.ru -testbotprocessor44.com -tizanidinecapsules.blogspot.com -top10registrycleaners.com -trafficmp.com -undergroundcityphoto.com -valoresito.com -vapomnoncri.tk -versaut.xxx-cam.webcam -wait3sec.org -whatsappdownload10.com -xjrul.com -xlovecam.com -youtubologia.it -zeroredirect7.com -znakomstva-piter78.ru +7makemoneyonline.com +aanapa.ru +airlandsea.info +all-streaming-media.com +allkrim.com +apimossnetinfo-a.akamaihd.net +apiwoodensealcom-a.akamaihd.net +bad-stars.net +balans.shahterworld.org +bestmarriages.com +bget.ru +bigpenisguide.com +braintobe.win +buckspar.gq +candlehandmade.com +caulderwoodkennels.com +cercacamion.it +ci.ua +cookingmeat.ru +credit-online.ws +dealighted.com +digital-sale.su +dolohen.com +domcran.net +domini.cat +dominterior.org +dumpsmania24.com +eloxal.ru +eu-cookie-law.blogspot.com +failingmarriege.blogspot.com +fortunejack.com +foto-sisek.porngalleries.top +frcls.fr +fsakhalin.ru +gsasearchengineranker.xyz +hulfingtonpost.com +igru-xbox.net +inspiring-desperate.tk +interesnie-faktu.ru +irkutsk.online-podarki.com +james13prix.info +jestr.org +konpax.com +krasula.pp.ua +linerdrilling.com +lordzfilmz.me +love-baby.cz +makenahartlin.com +moviezbonkerssk.cf +nbsproject.ru +nicovideo.jp +niroo.info +offf.info +okonich.com.ua +parvezmia.xyz +penisvergrotendepillennl.ovh +percin.biz.ly +pochemychka.net +polimga.pw +polska-poezja.com +popander.mobi +pyrodesigns.com.au +regdefense.com +ricorsogiustizia.org +seo-smm.kz +seocheckupx.com +sextracker.com +shop.xz618.com +sicfor.bcu.cc +social-s-iii.xyz +socialbookmarksubmission.org +solnplast.ru +stefanbakosab.se +thegolfclub.info +topappspro.com +topvidos.ru +trahvid.com +trumpetedextremes.com +tuberkuleznik.ru +twitlinks.com +tyumen.xrus.org +ukrobstep.com +unblocksit.es +uptime.com +usiad.net +video-chat.cn +videochat.tv.br +vip.51.la +vostoktrade.info +warning.or.kr +web-betting.ru +wnhjavlhezp.gq +woman-orgasm.ru +wttavern.com +xaylapdiendanang.com +xn----itbeirbjbi7bc6bh2d.xn--p1ai +zeg-distribution.com +zeroredirect9.com +zigarettenonlinekaufen2.bloog.pl +zytpirwai.net diff --git a/dev-tools/test_units/whitelist-ips.conf b/dev-tools/test_units/whitelist-ips.conf index 9dcba65e6..529aeb017 100644 --- a/dev-tools/test_units/whitelist-ips.conf +++ b/dev-tools/test_units/whitelist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.44 0; +10.1.0.16 0; 127.0.0.1 0; diff --git a/dev-tools/tmp/robots.txt b/dev-tools/tmp/robots.txt index 0982b2372..b8923f14a 100644 --- a/dev-tools/tmp/robots.txt +++ b/dev-tools/tmp/robots.txt @@ -6,9 +6,9 @@ ### Version Information # ################################################### -### Version: V4.2023.05.3717 -### Updated: Mon May 1 04:37:18 UTC 2023 -### Bad Bot Count: 640 +### Version: V4.2023.05.3718 +### Updated: Mon May 1 05:50:05 UTC 2023 +### Bad Bot Count: 639 ################################################### ### Version Information ## @@ -653,8 +653,6 @@ User-agent: Ninja Disallow:/ User-agent: Nmap Disallow:/ -User-agent: Not -Disallow:/ User-agent: Nuclei Disallow:/ User-agent: Nutch diff --git a/robots.txt/robots.txt b/robots.txt/robots.txt index 0982b2372..b8923f14a 100644 --- a/robots.txt/robots.txt +++ b/robots.txt/robots.txt @@ -6,9 +6,9 @@ ### Version Information # ################################################### -### Version: V4.2023.05.3717 -### Updated: Mon May 1 04:37:18 UTC 2023 -### Bad Bot Count: 640 +### Version: V4.2023.05.3718 +### Updated: Mon May 1 05:50:05 UTC 2023 +### Bad Bot Count: 639 ################################################### ### Version Information ## @@ -653,8 +653,6 @@ User-agent: Ninja Disallow:/ User-agent: Nmap Disallow:/ -User-agent: Not -Disallow:/ User-agent: Nuclei Disallow:/ User-agent: Nutch