diff --git a/AUTO-CONFIGURATION.md b/AUTO-CONFIGURATION.md index cf5f46b68..9367ff28a 100755 --- a/AUTO-CONFIGURATION.md +++ b/AUTO-CONFIGURATION.md @@ -4,9 +4,9 @@ ### PLEASE READ CONFIGURATION INSTRUCTIONS BELOW THOROUGHLY :exclamation: _______________ -#### Version: V4.2023.08.3918 +#### Version: V4.2023.08.3919 #### Bad Referrers Blocked: [7104](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-referrers.list) -#### Bad User-Agents (Bots) Blocked: [642](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) +#### Bad User-Agents (Bots) Blocked: [645](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) #### Fake Googlebots Blocked: [217](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/fake-googlebots.list) ____________________ diff --git a/MANUAL-CONFIGURATION.md b/MANUAL-CONFIGURATION.md index d2417acb7..04bebb7d8 100755 --- a/MANUAL-CONFIGURATION.md +++ b/MANUAL-CONFIGURATION.md @@ -4,9 +4,9 @@ ### PLEASE READ CONFIGURATION INSTRUCTIONS BELOW THOROUGHLY :exclamation: _______________ -#### Version: V4.2023.08.3918 +#### Version: V4.2023.08.3919 #### Bad Referrers Blocked: [7104](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-referrers.list) -#### Bad User-Agents (Bots) Blocked: [642](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) +#### Bad User-Agents (Bots) Blocked: [645](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) #### Fake Googlebots Blocked: [217](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/fake-googlebots.list) ____________________ diff --git a/README.md b/README.md index ad75d27ee..aeb3eced9 100755 --- a/README.md +++ b/README.md @@ -14,9 +14,9 @@ ##### The Ultimate Nginx Bad Bot, User-Agent, Spam Referrer Blocker, Adware, Malware and Ransomware Blocker, Clickjacking Blocker, Click Re-Directing Blocker, SEO Companies and Bad IP Blocker with Anti DDOS System, Nginx Rate Limiting and Wordpress Theme Detector Blocking. Stop and Block all kinds of bad internet traffic even Fake Googlebots from ever reaching your web sites. [PLEASE SEE: Definition of Bad Bots](#define-bad-bots) _______________ -#### Version: V4.2023.08.3918 +#### Version: V4.2023.08.3919 #### Bad Referrers Blocked: [7104](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-referrers.list) -#### Bad User-Agents (Bots) Blocked: [642](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) +#### Bad User-Agents (Bots) Blocked: [645](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) #### Fake Googlebots Blocked: [217](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/fake-googlebots.list) ____________________ diff --git a/_generator_lists/bad-user-agents.list b/_generator_lists/bad-user-agents.list index 298c6522f..8f3904dc1 100755 --- a/_generator_lists/bad-user-agents.list +++ b/_generator_lists/bad-user-agents.list @@ -1,6 +1,3 @@ -GPTBot -openai.com -openai 01h4x.com 360Spider 404checker @@ -162,6 +159,7 @@ Fuzz FyberSpider Fyrebot G-i-g-a-b-o-t +GPTBot GT::WWW GalaxyBot Genieo @@ -607,6 +605,8 @@ mediawords muhstik-scan netEstate\ NE\ Crawler oBot +openai +openai.com page\ scorer pcBrowser plumanalytics diff --git a/_google_analytics_ghost_spam/README.md b/_google_analytics_ghost_spam/README.md index de06ce832..0ba1b8bff 100644 --- a/_google_analytics_ghost_spam/README.md +++ b/_google_analytics_ghost_spam/README.md @@ -3,9 +3,9 @@ # EASY CONFIGURATION INSTRUCTIONS FOR STOPPING GOOGLE ANALYTICS "GHOST" SPAM _______________ -#### Version: V4.2023.08.3918 +#### Version: V4.2023.08.3919 #### Bad Referrers Blocked: [7104](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-referrers.list) -#### Bad User-Agents (Bots) Blocked: [642](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) +#### Bad User-Agents (Bots) Blocked: [645](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) #### Fake Googlebots Blocked: [217](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/fake-googlebots.list) ____________________ diff --git a/_sample_config_files/Engintron_for_cPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf b/_sample_config_files/Engintron_for_cPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf index eaad9db32..7a307e699 100755 --- a/_sample_config_files/Engintron_for_cPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf +++ b/_sample_config_files/Engintron_for_cPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2023.08.3918 -### Updated: Mon Aug 7 22:01:24 UTC 2023 +### Version: V4.2023.08.3919 +### Updated: Tue Aug 8 08:59:23 UTC 2023 ### Bad Referrer Count: 7104 -### Bad Bot Count: 642 +### Bad Bot Count: 645 ################################################### ### VERSION INFORMATION ## @@ -308,6 +308,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)FyberSpider(?:\b)" 3; "~*(?:\b)Fyrebot(?:\b)" 3; "~*(?:\b)G-i-g-a-b-o-t(?:\b)" 3; + "~*(?:\b)GPTBot(?:\b)" 3; "~*(?:\b)GT::WWW(?:\b)" 3; "~*(?:\b)GalaxyBot(?:\b)" 3; "~*(?:\b)Genieo(?:\b)" 3; @@ -753,6 +754,8 @@ map $http_user_agent $bad_bot { "~*(?:\b)muhstik-scan(?:\b)" 3; "~*(?:\b)netEstate\ NE\ Crawler(?:\b)" 3; "~*(?:\b)oBot(?:\b)" 3; + "~*(?:\b)openai(?:\b)" 3; + "~*(?:\b)openai.com(?:\b)" 3; "~*(?:\b)page\ scorer(?:\b)" 3; "~*(?:\b)pcBrowser(?:\b)" 3; "~*(?:\b)plumanalytics(?:\b)" 3; diff --git a/conf.d/globalblacklist-testing-version.conf b/conf.d/globalblacklist-testing-version.conf index cddbe3c80..edde84efb 100644 --- a/conf.d/globalblacklist-testing-version.conf +++ b/conf.d/globalblacklist-testing-version.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2023.08.3918 -### Updated: Mon Aug 7 22:01:25 UTC 2023 +### Version: V3.2023.08.3919 +### Updated: Tue Aug 8 08:59:23 UTC 2023 ### Bad Referrer Count: 7104 -### Bad Bot Count: 642 +### Bad Bot Count: 645 ################################################### ### VERSION INFORMATION ## @@ -308,6 +308,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)FyberSpider(?:\b)" 3; "~*(?:\b)Fyrebot(?:\b)" 3; "~*(?:\b)G-i-g-a-b-o-t(?:\b)" 3; + "~*(?:\b)GPTBot(?:\b)" 3; "~*(?:\b)GT::WWW(?:\b)" 3; "~*(?:\b)GalaxyBot(?:\b)" 3; "~*(?:\b)Genieo(?:\b)" 3; @@ -753,6 +754,8 @@ map $http_user_agent $bad_bot { "~*(?:\b)muhstik-scan(?:\b)" 3; "~*(?:\b)netEstate\ NE\ Crawler(?:\b)" 3; "~*(?:\b)oBot(?:\b)" 3; + "~*(?:\b)openai(?:\b)" 3; + "~*(?:\b)openai.com(?:\b)" 3; "~*(?:\b)page\ scorer(?:\b)" 3; "~*(?:\b)pcBrowser(?:\b)" 3; "~*(?:\b)plumanalytics(?:\b)" 3; diff --git a/conf.d/globalblacklist.conf b/conf.d/globalblacklist.conf index eaad9db32..7a307e699 100644 --- a/conf.d/globalblacklist.conf +++ b/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2023.08.3918 -### Updated: Mon Aug 7 22:01:24 UTC 2023 +### Version: V4.2023.08.3919 +### Updated: Tue Aug 8 08:59:23 UTC 2023 ### Bad Referrer Count: 7104 -### Bad Bot Count: 642 +### Bad Bot Count: 645 ################################################### ### VERSION INFORMATION ## @@ -308,6 +308,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)FyberSpider(?:\b)" 3; "~*(?:\b)Fyrebot(?:\b)" 3; "~*(?:\b)G-i-g-a-b-o-t(?:\b)" 3; + "~*(?:\b)GPTBot(?:\b)" 3; "~*(?:\b)GT::WWW(?:\b)" 3; "~*(?:\b)GalaxyBot(?:\b)" 3; "~*(?:\b)Genieo(?:\b)" 3; @@ -753,6 +754,8 @@ map $http_user_agent $bad_bot { "~*(?:\b)muhstik-scan(?:\b)" 3; "~*(?:\b)netEstate\ NE\ Crawler(?:\b)" 3; "~*(?:\b)oBot(?:\b)" 3; + "~*(?:\b)openai(?:\b)" 3; + "~*(?:\b)openai.com(?:\b)" 3; "~*(?:\b)page\ scorer(?:\b)" 3; "~*(?:\b)pcBrowser(?:\b)" 3; "~*(?:\b)plumanalytics(?:\b)" 3; diff --git a/dev-tools/_robots_input/robots-input.txt b/dev-tools/_robots_input/robots-input.txt index 21ee78c30..a10a3d1dc 100644 --- a/dev-tools/_robots_input/robots-input.txt +++ b/dev-tools/_robots_input/robots-input.txt @@ -159,6 +159,7 @@ Fuzz FyberSpider Fyrebot G-i-g-a-b-o-t +GPTBot GT::WWW GalaxyBot Genieo @@ -604,6 +605,8 @@ mediawords muhstik-scan netEstate NE Crawler oBot +openai +openai.com page scorer pcBrowser plumanalytics diff --git a/dev-tools/buildnumber b/dev-tools/buildnumber index f854f4208..8b42dbe14 100644 --- a/dev-tools/buildnumber +++ b/dev-tools/buildnumber @@ -1 +1 @@ -3918 +3919 diff --git a/dev-tools/globalblacklist-testing.template b/dev-tools/globalblacklist-testing.template index cddbe3c80..edde84efb 100644 --- a/dev-tools/globalblacklist-testing.template +++ b/dev-tools/globalblacklist-testing.template @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2023.08.3918 -### Updated: Mon Aug 7 22:01:25 UTC 2023 +### Version: V3.2023.08.3919 +### Updated: Tue Aug 8 08:59:23 UTC 2023 ### Bad Referrer Count: 7104 -### Bad Bot Count: 642 +### Bad Bot Count: 645 ################################################### ### VERSION INFORMATION ## @@ -308,6 +308,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)FyberSpider(?:\b)" 3; "~*(?:\b)Fyrebot(?:\b)" 3; "~*(?:\b)G-i-g-a-b-o-t(?:\b)" 3; + "~*(?:\b)GPTBot(?:\b)" 3; "~*(?:\b)GT::WWW(?:\b)" 3; "~*(?:\b)GalaxyBot(?:\b)" 3; "~*(?:\b)Genieo(?:\b)" 3; @@ -753,6 +754,8 @@ map $http_user_agent $bad_bot { "~*(?:\b)muhstik-scan(?:\b)" 3; "~*(?:\b)netEstate\ NE\ Crawler(?:\b)" 3; "~*(?:\b)oBot(?:\b)" 3; + "~*(?:\b)openai(?:\b)" 3; + "~*(?:\b)openai.com(?:\b)" 3; "~*(?:\b)page\ scorer(?:\b)" 3; "~*(?:\b)pcBrowser(?:\b)" 3; "~*(?:\b)plumanalytics(?:\b)" 3; diff --git a/dev-tools/globalblacklist.template b/dev-tools/globalblacklist.template index eaad9db32..7a307e699 100755 --- a/dev-tools/globalblacklist.template +++ b/dev-tools/globalblacklist.template @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2023.08.3918 -### Updated: Mon Aug 7 22:01:24 UTC 2023 +### Version: V4.2023.08.3919 +### Updated: Tue Aug 8 08:59:23 UTC 2023 ### Bad Referrer Count: 7104 -### Bad Bot Count: 642 +### Bad Bot Count: 645 ################################################### ### VERSION INFORMATION ## @@ -308,6 +308,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)FyberSpider(?:\b)" 3; "~*(?:\b)Fyrebot(?:\b)" 3; "~*(?:\b)G-i-g-a-b-o-t(?:\b)" 3; + "~*(?:\b)GPTBot(?:\b)" 3; "~*(?:\b)GT::WWW(?:\b)" 3; "~*(?:\b)GalaxyBot(?:\b)" 3; "~*(?:\b)Genieo(?:\b)" 3; @@ -753,6 +754,8 @@ map $http_user_agent $bad_bot { "~*(?:\b)muhstik-scan(?:\b)" 3; "~*(?:\b)netEstate\ NE\ Crawler(?:\b)" 3; "~*(?:\b)oBot(?:\b)" 3; + "~*(?:\b)openai(?:\b)" 3; + "~*(?:\b)openai.com(?:\b)" 3; "~*(?:\b)page\ scorer(?:\b)" 3; "~*(?:\b)pcBrowser(?:\b)" 3; "~*(?:\b)plumanalytics(?:\b)" 3; diff --git a/dev-tools/test1_conf_files/bots.d/whitelist-ips.conf b/dev-tools/test1_conf_files/bots.d/whitelist-ips.conf index 97d79a509..3cbe6f240 100644 --- a/dev-tools/test1_conf_files/bots.d/whitelist-ips.conf +++ b/dev-tools/test1_conf_files/bots.d/whitelist-ips.conf @@ -45,4 +45,4 @@ # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST -138.91.87.24 0; +104.210.132.191 0; diff --git a/dev-tools/test1_conf_files/conf.d/globalblacklist.conf b/dev-tools/test1_conf_files/conf.d/globalblacklist.conf index eaad9db32..7a307e699 100644 --- a/dev-tools/test1_conf_files/conf.d/globalblacklist.conf +++ b/dev-tools/test1_conf_files/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2023.08.3918 -### Updated: Mon Aug 7 22:01:24 UTC 2023 +### Version: V4.2023.08.3919 +### Updated: Tue Aug 8 08:59:23 UTC 2023 ### Bad Referrer Count: 7104 -### Bad Bot Count: 642 +### Bad Bot Count: 645 ################################################### ### VERSION INFORMATION ## @@ -308,6 +308,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)FyberSpider(?:\b)" 3; "~*(?:\b)Fyrebot(?:\b)" 3; "~*(?:\b)G-i-g-a-b-o-t(?:\b)" 3; + "~*(?:\b)GPTBot(?:\b)" 3; "~*(?:\b)GT::WWW(?:\b)" 3; "~*(?:\b)GalaxyBot(?:\b)" 3; "~*(?:\b)Genieo(?:\b)" 3; @@ -753,6 +754,8 @@ map $http_user_agent $bad_bot { "~*(?:\b)muhstik-scan(?:\b)" 3; "~*(?:\b)netEstate\ NE\ Crawler(?:\b)" 3; "~*(?:\b)oBot(?:\b)" 3; + "~*(?:\b)openai(?:\b)" 3; + "~*(?:\b)openai.com(?:\b)" 3; "~*(?:\b)page\ scorer(?:\b)" 3; "~*(?:\b)pcBrowser(?:\b)" 3; "~*(?:\b)plumanalytics(?:\b)" 3; diff --git a/dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf b/dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf index 56a00ac24..af7aacb1b 100644 --- a/dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf +++ b/dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.121 1; +10.1.0.65 1; 127.0.0.1 1; diff --git a/dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf b/dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf index 3e6546af9..96ba8d603 100644 --- a/dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf +++ b/dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.121 0; +10.1.0.65 0; 127.0.0.1 0; diff --git a/dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf b/dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf index eaad9db32..7a307e699 100644 --- a/dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf +++ b/dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2023.08.3918 -### Updated: Mon Aug 7 22:01:24 UTC 2023 +### Version: V4.2023.08.3919 +### Updated: Tue Aug 8 08:59:23 UTC 2023 ### Bad Referrer Count: 7104 -### Bad Bot Count: 642 +### Bad Bot Count: 645 ################################################### ### VERSION INFORMATION ## @@ -308,6 +308,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)FyberSpider(?:\b)" 3; "~*(?:\b)Fyrebot(?:\b)" 3; "~*(?:\b)G-i-g-a-b-o-t(?:\b)" 3; + "~*(?:\b)GPTBot(?:\b)" 3; "~*(?:\b)GT::WWW(?:\b)" 3; "~*(?:\b)GalaxyBot(?:\b)" 3; "~*(?:\b)Genieo(?:\b)" 3; @@ -753,6 +754,8 @@ map $http_user_agent $bad_bot { "~*(?:\b)muhstik-scan(?:\b)" 3; "~*(?:\b)netEstate\ NE\ Crawler(?:\b)" 3; "~*(?:\b)oBot(?:\b)" 3; + "~*(?:\b)openai(?:\b)" 3; + "~*(?:\b)openai.com(?:\b)" 3; "~*(?:\b)page\ scorer(?:\b)" 3; "~*(?:\b)pcBrowser(?:\b)" 3; "~*(?:\b)plumanalytics(?:\b)" 3; diff --git a/dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf b/dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf index 56a00ac24..af7aacb1b 100644 --- a/dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf +++ b/dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.121 1; +10.1.0.65 1; 127.0.0.1 1; diff --git a/dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf b/dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf index 3e6546af9..96ba8d603 100644 --- a/dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf +++ b/dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.121 0; +10.1.0.65 0; 127.0.0.1 0; diff --git a/dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf b/dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf index eaad9db32..7a307e699 100644 --- a/dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf +++ b/dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2023.08.3918 -### Updated: Mon Aug 7 22:01:24 UTC 2023 +### Version: V4.2023.08.3919 +### Updated: Tue Aug 8 08:59:23 UTC 2023 ### Bad Referrer Count: 7104 -### Bad Bot Count: 642 +### Bad Bot Count: 645 ################################################### ### VERSION INFORMATION ## @@ -308,6 +308,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)FyberSpider(?:\b)" 3; "~*(?:\b)Fyrebot(?:\b)" 3; "~*(?:\b)G-i-g-a-b-o-t(?:\b)" 3; + "~*(?:\b)GPTBot(?:\b)" 3; "~*(?:\b)GT::WWW(?:\b)" 3; "~*(?:\b)GalaxyBot(?:\b)" 3; "~*(?:\b)Genieo(?:\b)" 3; @@ -753,6 +754,8 @@ map $http_user_agent $bad_bot { "~*(?:\b)muhstik-scan(?:\b)" 3; "~*(?:\b)netEstate\ NE\ Crawler(?:\b)" 3; "~*(?:\b)oBot(?:\b)" 3; + "~*(?:\b)openai(?:\b)" 3; + "~*(?:\b)openai.com(?:\b)" 3; "~*(?:\b)page\ scorer(?:\b)" 3; "~*(?:\b)pcBrowser(?:\b)" 3; "~*(?:\b)plumanalytics(?:\b)" 3; diff --git a/dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf b/dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf index 56a00ac24..af7aacb1b 100644 --- a/dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf +++ b/dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.121 1; +10.1.0.65 1; 127.0.0.1 1; diff --git a/dev-tools/test1_conf_files_whitelist/bots.d/blacklist-user-agents.conf b/dev-tools/test1_conf_files_whitelist/bots.d/blacklist-user-agents.conf index 1a184825e..27901341e 100644 --- a/dev-tools/test1_conf_files_whitelist/bots.d/blacklist-user-agents.conf +++ b/dev-tools/test1_conf_files_whitelist/bots.d/blacklist-user-agents.conf @@ -220,6 +220,7 @@ "~*(?:\b)FyberSpider(?:\b|)" 0; "~*(?:\b)Fyrebot(?:\b|)" 0; "~*(?:\b)G-i-g-a-b-o-t(?:\b|)" 0; + "~*(?:\b)GPTBot(?:\b|)" 0; "~*(?:\b)GT::WWW(?:\b|)" 0; "~*(?:\b)GalaxyBot(?:\b|)" 0; "~*(?:\b)Genieo(?:\b|)" 0; @@ -665,6 +666,8 @@ "~*(?:\b)muhstik-scan(?:\b|)" 0; "~*(?:\b)netEstate\ NE\ Crawler(?:\b|)" 0; "~*(?:\b)oBot(?:\b|)" 0; + "~*(?:\b)openai(?:\b|)" 0; + "~*(?:\b)openai.com(?:\b|)" 0; "~*(?:\b)page\ scorer(?:\b|)" 0; "~*(?:\b)pcBrowser(?:\b|)" 0; "~*(?:\b)plumanalytics(?:\b|)" 0; diff --git a/dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf b/dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf index 3e6546af9..96ba8d603 100644 --- a/dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf +++ b/dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.121 0; +10.1.0.65 0; 127.0.0.1 0; diff --git a/dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf b/dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf index eaad9db32..7a307e699 100644 --- a/dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf +++ b/dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2023.08.3918 -### Updated: Mon Aug 7 22:01:24 UTC 2023 +### Version: V4.2023.08.3919 +### Updated: Tue Aug 8 08:59:23 UTC 2023 ### Bad Referrer Count: 7104 -### Bad Bot Count: 642 +### Bad Bot Count: 645 ################################################### ### VERSION INFORMATION ## @@ -308,6 +308,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)FyberSpider(?:\b)" 3; "~*(?:\b)Fyrebot(?:\b)" 3; "~*(?:\b)G-i-g-a-b-o-t(?:\b)" 3; + "~*(?:\b)GPTBot(?:\b)" 3; "~*(?:\b)GT::WWW(?:\b)" 3; "~*(?:\b)GalaxyBot(?:\b)" 3; "~*(?:\b)Genieo(?:\b)" 3; @@ -753,6 +754,8 @@ map $http_user_agent $bad_bot { "~*(?:\b)muhstik-scan(?:\b)" 3; "~*(?:\b)netEstate\ NE\ Crawler(?:\b)" 3; "~*(?:\b)oBot(?:\b)" 3; + "~*(?:\b)openai(?:\b)" 3; + "~*(?:\b)openai.com(?:\b)" 3; "~*(?:\b)page\ scorer(?:\b)" 3; "~*(?:\b)pcBrowser(?:\b)" 3; "~*(?:\b)plumanalytics(?:\b)" 3; diff --git a/dev-tools/test2_conf_files/bots.d/whitelist-ips.conf b/dev-tools/test2_conf_files/bots.d/whitelist-ips.conf index 97d79a509..3cbe6f240 100644 --- a/dev-tools/test2_conf_files/bots.d/whitelist-ips.conf +++ b/dev-tools/test2_conf_files/bots.d/whitelist-ips.conf @@ -45,4 +45,4 @@ # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST -138.91.87.24 0; +104.210.132.191 0; diff --git a/dev-tools/test2_conf_files/conf.d/globalblacklist.conf b/dev-tools/test2_conf_files/conf.d/globalblacklist.conf index 187c6d369..872ff735e 100644 --- a/dev-tools/test2_conf_files/conf.d/globalblacklist.conf +++ b/dev-tools/test2_conf_files/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2023.08.3918 -### Updated: Mon Aug 7 22:01:24 UTC 2023 +### Version: V4.2023.08.3919 +### Updated: Tue Aug 8 08:59:23 UTC 2023 ### Bad Referrer Count: 7104 -### Bad Bot Count: 642 +### Bad Bot Count: 645 ################################################### ### VERSION INFORMATION ## @@ -308,6 +308,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)FyberSpider(?:\b)" 3; "~*(?:\b)Fyrebot(?:\b)" 3; "~*(?:\b)G-i-g-a-b-o-t(?:\b)" 3; + "~*(?:\b)GPTBot(?:\b)" 3; "~*(?:\b)GT::WWW(?:\b)" 3; "~*(?:\b)GalaxyBot(?:\b)" 3; "~*(?:\b)Genieo(?:\b)" 3; @@ -753,6 +754,8 @@ map $http_user_agent $bad_bot { "~*(?:\b)muhstik-scan(?:\b)" 3; "~*(?:\b)netEstate\ NE\ Crawler(?:\b)" 3; "~*(?:\b)oBot(?:\b)" 3; + "~*(?:\b)openai(?:\b)" 3; + "~*(?:\b)openai.com(?:\b)" 3; "~*(?:\b)page\ scorer(?:\b)" 3; "~*(?:\b)pcBrowser(?:\b)" 3; "~*(?:\b)plumanalytics(?:\b)" 3; diff --git a/dev-tools/test3_conf_files/bots.d/whitelist-ips.conf b/dev-tools/test3_conf_files/bots.d/whitelist-ips.conf index 97d79a509..3cbe6f240 100644 --- a/dev-tools/test3_conf_files/bots.d/whitelist-ips.conf +++ b/dev-tools/test3_conf_files/bots.d/whitelist-ips.conf @@ -45,4 +45,4 @@ # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST -138.91.87.24 0; +104.210.132.191 0; diff --git a/dev-tools/test3_conf_files/conf.d/globalblacklist.conf b/dev-tools/test3_conf_files/conf.d/globalblacklist.conf index d91a9953f..0b3a41ee7 100644 --- a/dev-tools/test3_conf_files/conf.d/globalblacklist.conf +++ b/dev-tools/test3_conf_files/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2023.08.3918 -### Updated: Mon Aug 7 22:01:24 UTC 2023 +### Version: V4.2023.08.3919 +### Updated: Tue Aug 8 08:59:23 UTC 2023 ### Bad Referrer Count: 7104 -### Bad Bot Count: 642 +### Bad Bot Count: 645 ################################################### ### VERSION INFORMATION ## @@ -308,6 +308,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)FyberSpider(?:\b)" 3; "~*(?:\b)Fyrebot(?:\b)" 3; "~*(?:\b)G-i-g-a-b-o-t(?:\b)" 3; + "~*(?:\b)GPTBot(?:\b)" 3; "~*(?:\b)GT::WWW(?:\b)" 3; "~*(?:\b)GalaxyBot(?:\b)" 3; "~*(?:\b)Genieo(?:\b)" 3; @@ -753,6 +754,8 @@ map $http_user_agent $bad_bot { "~*(?:\b)muhstik-scan(?:\b)" 3; "~*(?:\b)netEstate\ NE\ Crawler(?:\b)" 3; "~*(?:\b)oBot(?:\b)" 3; + "~*(?:\b)openai(?:\b)" 3; + "~*(?:\b)openai.com(?:\b)" 3; "~*(?:\b)page\ scorer(?:\b)" 3; "~*(?:\b)pcBrowser(?:\b)" 3; "~*(?:\b)plumanalytics(?:\b)" 3; diff --git a/dev-tools/test_units/blacklist-ips.conf b/dev-tools/test_units/blacklist-ips.conf index 56a00ac24..af7aacb1b 100644 --- a/dev-tools/test_units/blacklist-ips.conf +++ b/dev-tools/test_units/blacklist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.121 1; +10.1.0.65 1; 127.0.0.1 1; diff --git a/dev-tools/test_units/blacklist-user-agents.conf b/dev-tools/test_units/blacklist-user-agents.conf index 1a184825e..27901341e 100644 --- a/dev-tools/test_units/blacklist-user-agents.conf +++ b/dev-tools/test_units/blacklist-user-agents.conf @@ -220,6 +220,7 @@ "~*(?:\b)FyberSpider(?:\b|)" 0; "~*(?:\b)Fyrebot(?:\b|)" 0; "~*(?:\b)G-i-g-a-b-o-t(?:\b|)" 0; + "~*(?:\b)GPTBot(?:\b|)" 0; "~*(?:\b)GT::WWW(?:\b|)" 0; "~*(?:\b)GalaxyBot(?:\b|)" 0; "~*(?:\b)Genieo(?:\b|)" 0; @@ -665,6 +666,8 @@ "~*(?:\b)muhstik-scan(?:\b|)" 0; "~*(?:\b)netEstate\ NE\ Crawler(?:\b|)" 0; "~*(?:\b)oBot(?:\b|)" 0; + "~*(?:\b)openai(?:\b|)" 0; + "~*(?:\b)openai.com(?:\b|)" 0; "~*(?:\b)page\ scorer(?:\b|)" 0; "~*(?:\b)pcBrowser(?:\b|)" 0; "~*(?:\b)plumanalytics(?:\b|)" 0; diff --git a/dev-tools/test_units/random-bots-for-test-quick.list b/dev-tools/test_units/random-bots-for-test-quick.list index 85afaff93..f78f4d885 100644 --- a/dev-tools/test_units/random-bots-for-test-quick.list +++ b/dev-tools/test_units/random-bots-for-test-quick.list @@ -1,10 +1,10 @@ -Anthill -Calculon -Jorgee -MolokaiBot +CATExplorador +InternetSeer +Mozlila +OpenVAS RocketCrawler -SlySearch -Spammen -WallpapersHD -Webalta -seocompany.store +SiteSnagger +SuperHTTP +WeSEE +netEstate NE Crawler +tAkeOut diff --git a/dev-tools/test_units/random-bots-for-whitelist-test.list b/dev-tools/test_units/random-bots-for-whitelist-test.list index e84002997..29e31078a 100644 --- a/dev-tools/test_units/random-bots-for-whitelist-test.list +++ b/dev-tools/test_units/random-bots-for-whitelist-test.list @@ -1,100 +1,100 @@ -01h4x.com -80legs -AIBOT -ASPSeek -Aipbot -Alligator -BBBike -BDFetch +ALittle Client +Abonti +Aboundex +Aboundexbot +AhrefsBot +AiHitBot +Asterias +AwarioRssBot +AwarioSmartBot +BackDoorBot BackStreet -Buck +Bigfoot BunnySlippers -BuzzSumo -CheeseBot -ChinaClaw -Crawling at Home Project -DatabaseDriverMysqli -Disco -DnBCrawler-Analytics -DomainStatsBot -Download Wonder -Drip -Express WebPictures +Bytespider +Calculon +Crescent +Custo +DTS Agent +DataCha0s +DomainAppender +Dotbot +EasyDL Ezooms -Fimap -FlashGet -Flunky -Foobot -FyberSpider -HEADMasterSEO -HTMLparser -InfoTekies -Intelliseek -Jbrofuzz -JikeSpider -JustView -Jyxobot -Kenjin Spider -Keyword Density -Kozmosbot +G-i-g-a-b-o-t +Getintent +Grabber +HTTrack +Harvest +InterGET +JOC Web Spider +Jorgee Larbin -LeechFTP -LexiBot -LinkWalker -LinksManager -LinqiaMetadataDownloaderBot -Ltx71 -MFC_Tear_Sample -MTRobot +LieBaoFast +LinkScan +Lmspider +MIDown tool +MarkMonitor Mass Downloader -MauiBot -MeanPath Bot -MolokaiBot -Name Intelligence +Mata Hari +Meanpathbot +Mojeek +Musobot +NICErsPRO NearSite -Nutch -OrangeSpider -PageScorer +NetMechanic +NetZIP +Offline Navigator Picscout -PleaseCrawl -Quick-Crawler +Pockey +ProPowerBot +Probethenet +Pump +RankingBot2 +RankurBot +ReGet Reaper Ripper -Scanbot -ScrepyBot -Semrush -SiteSucker -Spanner -Spbot +RocketCrawler +Rogerbot +SMTBot +Snapbot Sqlmap -Sqlworm -Sucker +Stripper T8Abot +TheNomad Thumbor -TightTwatBot -Toata -TwengaBot -VCI +Trendiction +True_Robot +TurnitinBot +Vagabondo Voil +Voltron WEBDAV -WWW::Mechanize -Web Fetch -Web Fuck +WWW-Collector-E +WWWOFFLE +Wallpapers/3.0 +WallpapersHD +Web Auto +Web Sucker WebCollage -WebEnhancer -WebReaper -Widow -Wonderbot -adscanner -cah.io.community +WebFuck +WebStripper +Webster +Whatweb +Woobot +Wprecon +ZoomBot coccocbot cognitiveseo -crawler.feedback -gopher -lwp-request +dataforseobot +ips-agent +iubenda-radar mediawords -oBot -siteripz -ubermetrics-technologies.com -webgains-bot -webprosbot +openai.com +page scorer +pcBrowser +plumanalytics +seoscanners +sogouspider +sp_auditbot diff --git a/dev-tools/test_units/random-referrers-for-test-quick.list b/dev-tools/test_units/random-referrers-for-test-quick.list index bb80c488e..ed9e79898 100644 --- a/dev-tools/test_units/random-referrers-for-test-quick.list +++ b/dev-tools/test_units/random-referrers-for-test-quick.list @@ -1,10 +1,10 @@ -adloads.net -alessandraleone.com -alice.it -gigixo.com -justkillingti.me -kino2018.club -mostantikor.ru -priscilarodrigues.com.br -weburlopener.com -xwatt.ru +advertex.info +bannerswap.com +cheapjerseysbizwholesale.us +domainanalyzing.xyz +eyes-on-you.ga +houtings.xyz +nagdak.ru +obiavo.net +volume-pills.biz +www.kabbalah-red-bracelets.com diff --git a/dev-tools/test_units/random-referrers-for-whitelist-test.list b/dev-tools/test_units/random-referrers-for-whitelist-test.list index 1de3c65cc..4f62f78e9 100644 --- a/dev-tools/test_units/random-referrers-for-whitelist-test.list +++ b/dev-tools/test_units/random-referrers-for-whitelist-test.list @@ -1,100 +1,100 @@ -2girls1cup-free.com -3xforum.ro -a96527gi.beget.tech -aanapa.ru -acheterviagrafr24.com -acortarurl.es -apibatbrowsecom-a.akamaihd.net -apimountainbikei-a.akamaihd.net -beatifullandscape.co -beauty-clinic.ru -best-coupon-offer.com -best-way.men -bestcalovebracelet.cn -bingoporno.com -blogqpot.com -briomotor.co -bristolhotel.com -buehne-fuer-menschenrechte.de -cfacarrosserie74.com -clips.ua.ac.be -commoncrawl.org -coslab.club -crystalslot.com -dataloading.net -detki-opt.ru -domaincheck.io -e2click.com -elektrischezigarette2.devhub.com -elektronischezigarettekaufen1.myblog.de -ezigarettenkaufen1.deviantart.com -freemags.cc -freetangodownload.com -geckoandfly.com -goosefishpost.bid -hkladys.com -homemade.gq -houseofrose.com -i-midias.net.br -inet-traffic.com -iptool.xyz -iqs.biz.ua -jackwolfskinoutlet.online -janettabridal.com -katadhin.co -kino-filmi.com -kinozapas.org -konoplisemena.com -kovesszucs.atw.hu +123locker.com +2728fb936f0.com +40cg.com +7zap.com +8lufu.com +accessoires-mode.in +actulite.com +adidas.frwebs.fr +admanaerofoil.com +adprotect.net +aloofly.com +alphaforum.ru +amazon-seo-service.com +app-ready.xyz +arendatora.ru +autorn.ru +azazaporn.com +bg6s0.com +biketank.ga +brainboostingsupplements.org +bravegirlsclub.com +brianhenry.co +casablancamanor.co.za +cbb1smartlist12.click +ccjp.eu +chatroulette.online +cmd.kz +continent-e.tv +crazyprotocol.com +crynet.cc +czeshop.info +dipstar.org +doyouknowtheword-flummox.ml +dtm-spain.com +elektrischezi.canalblog.com +enskedesquashclub.se +europeanwatches.ru +excitacao.com +filmbokep69.com +findpik.com +forum.poker4life.ru +forums.toucharcade.com +handicapbathtubarea.com +hdxnxxtube.mobi +humbmosquina.tk +importchinacoach-teach.com +inboxdollars.com +jav-way.site +jennyfire.ru +kellyonline.xyz +kimcurlrvsms.com +kinky-fetishes.com +kinobest.pl +languagecode.com layola.biz.tc -letmacwork.world -linkpulse.com -magicalfind-a.akamaihd.net -mir-limuzinov.ru -mmofreegames.online -mwtpludn.review -najaden.no -netpics.org -new7ob.com -nextconseil.com -of-ireland.info -offergroup.info -partnersafe.men -pomoc-drogowa.cba.pl -porodasobak.net -powitania.pl -printingpeach.com -pronorm.fr -prostitutki-tyumeni.org -quangcaons.com -rankingchart.de -raschtextil.com.ua -reeyanaturopathy.com -relayblog.com -risparmiocasa.bz.it -ruclicks.com -sanjosestartups.com -savetubevideo.com -sayyoethe.blogspot.co.za -searchaddis.com -sexfreepornoxxx.com -shoppingmiracles.co.uk -sotkal.lark.ru -sports-supplements.us -sticken.co -stillmiracle.com -teguh.info -thenetinfo.com -trafficmp.com -uginekologa.com -via-gra.webstarts.com -viagra.pp.ua -videochat.life -videochat.ph -vipsexfinders.com -wdrake.com -wfb.hatedriveapart.com -wikes.20fr.com -xn--80aaaks3bbhabgbigamdr2h.xn--p1ai -yaderenergy.ru -zzlgxh.com +lineshops.biz +lyrster.com +machicon-ueno.info +magicdiet.gq +maxthon.com +monclerboots.xyz +mydearest.co +myhydros.org +nakozhe.com +naval.jislaaik.com +pdns.cz +perfection-pleasure.ru +photo.houseofgaga.ru +picmoonco.pw +pladform.ru +platezhka.net +rankexperience.com +seolab.top +sexsearch.com +signx.info +simple-share-buttons.com +spy-app.info +stromerrealty.com +suhanpacktech.com +superstats.com +sweepstakes.rewardit.com +taraz.xkaz.org +textads.men +thaisamkok.com +time-japan.ru +toon-families.com +turbabitload.weebly.com +urzedowski.eu +videojam.tv +videositename.com +vvpg.ru +waysbetter.cn +wrz0iuebwhp5fg.freeddns.com +xcc24.pl +xfire.com +y8games-free.com +your-bearings.com +yourmovies.pl +yubikk.info +zscaler.net diff --git a/dev-tools/test_units/whitelist-ips.conf b/dev-tools/test_units/whitelist-ips.conf index 3e6546af9..96ba8d603 100644 --- a/dev-tools/test_units/whitelist-ips.conf +++ b/dev-tools/test_units/whitelist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.121 0; +10.1.0.65 0; 127.0.0.1 0; diff --git a/dev-tools/tmp/robots.txt b/dev-tools/tmp/robots.txt index b0eb05018..e9fdb5d52 100644 --- a/dev-tools/tmp/robots.txt +++ b/dev-tools/tmp/robots.txt @@ -6,9 +6,9 @@ ### Version Information # ################################################### -### Version: V4.2023.08.3918 -### Updated: Mon Aug 7 22:01:26 UTC 2023 -### Bad Bot Count: 642 +### Version: V4.2023.08.3919 +### Updated: Tue Aug 8 08:59:24 UTC 2023 +### Bad Bot Count: 645 ################################################### ### Version Information ## @@ -337,6 +337,8 @@ User-agent: Fyrebot Disallow:/ User-agent: G-i-g-a-b-o-t Disallow:/ +User-agent: GPTBot +Disallow:/ User-agent: GT::WWW Disallow:/ User-agent: GalaxyBot @@ -1227,6 +1229,10 @@ User-agent: netEstate NE Crawler Disallow:/ User-agent: oBot Disallow:/ +User-agent: openai +Disallow:/ +User-agent: openai.com +Disallow:/ User-agent: page scorer Disallow:/ User-agent: pcBrowser diff --git a/robots.txt/robots.txt b/robots.txt/robots.txt index b0eb05018..e9fdb5d52 100644 --- a/robots.txt/robots.txt +++ b/robots.txt/robots.txt @@ -6,9 +6,9 @@ ### Version Information # ################################################### -### Version: V4.2023.08.3918 -### Updated: Mon Aug 7 22:01:26 UTC 2023 -### Bad Bot Count: 642 +### Version: V4.2023.08.3919 +### Updated: Tue Aug 8 08:59:24 UTC 2023 +### Bad Bot Count: 645 ################################################### ### Version Information ## @@ -337,6 +337,8 @@ User-agent: Fyrebot Disallow:/ User-agent: G-i-g-a-b-o-t Disallow:/ +User-agent: GPTBot +Disallow:/ User-agent: GT::WWW Disallow:/ User-agent: GalaxyBot @@ -1227,6 +1229,10 @@ User-agent: netEstate NE Crawler Disallow:/ User-agent: oBot Disallow:/ +User-agent: openai +Disallow:/ +User-agent: openai.com +Disallow:/ User-agent: page scorer Disallow:/ User-agent: pcBrowser