From 777b44bba415512383f903422386e80110709109 Mon Sep 17 00:00:00 2001 From: mitchellkrogza Date: Sun, 9 Jan 2022 07:06:17 +0000 Subject: [PATCH] V4.2022.01.2841 [ci skip] --- AUTO-CONFIGURATION.md | 4 +- MANUAL-CONFIGURATION.md | 4 +- README.md | 4 +- _generator_lists/bad-user-agents.list | 2 +- _generator_lists/good-user-agents.list | 6 +- _google_analytics_ghost_spam/README.md | 4 +- .../etc/nginx/conf.d/globalblacklist.conf | 9 +- conf.d/globalblacklist-testing-version.conf | 9 +- conf.d/globalblacklist.conf | 9 +- dev-tools/_robots_input/robots-input.txt | 1 + .../conf.d/globalblacklist.conf | 9 +- .../bots.d/blacklist-ips.conf | 2 +- .../bots.d/whitelist-ips.conf | 2 +- .../conf.d/globalblacklist.conf | 9 +- .../bots.d/blacklist-ips.conf | 2 +- .../bots.d/whitelist-ips.conf | 2 +- .../conf.d/globalblacklist.conf | 9 +- .../bots.d/blacklist-ips.conf | 2 +- .../bots.d/blacklist-user-agents.conf | 1 + .../bots.d/whitelist-ips.conf | 2 +- .../conf.d/globalblacklist.conf | 9 +- dev-tools/buildnumber | 2 +- dev-tools/globalblacklist-testing.template | 9 +- dev-tools/globalblacklist.template | 9 +- .../bots.d/whitelist-ips.conf | 2 +- .../conf.d/globalblacklist.conf | 9 +- .../bots.d/blacklist-ips.conf | 2 +- .../bots.d/whitelist-ips.conf | 2 +- .../conf.d/globalblacklist.conf | 9 +- .../bots.d/blacklist-ips.conf | 2 +- .../bots.d/whitelist-ips.conf | 2 +- .../conf.d/globalblacklist.conf | 9 +- .../bots.d/blacklist-ips.conf | 2 +- .../bots.d/blacklist-user-agents.conf | 1 + .../bots.d/whitelist-ips.conf | 2 +- .../conf.d/globalblacklist.conf | 9 +- .../bots.d/whitelist-ips.conf | 2 +- .../conf.d/globalblacklist.conf | 9 +- .../bots.d/whitelist-ips.conf | 2 +- .../conf.d/globalblacklist.conf | 9 +- dev-tools/test_units/blacklist-ips.conf | 2 +- .../test_units/blacklist-user-agents.conf | 1 + dev-tools/test_units/good-bots-for-test.list | 2 + .../random-bots-for-test-quick.list | 20 +- .../test_units/random-bots-for-test.list | 306 +++--- .../random-bots-for-whitelist-test.list | 308 +++--- .../random-referrers-for-test-quick.list | 20 +- .../test_units/random-referrers-for-test.list | 936 +++++++++--------- .../random-referrers-for-whitelist-test.list | 474 ++++----- dev-tools/test_units/ratelimittest-master.txt | 14 +- dev-tools/test_units/whitelist-ips.conf | 2 +- dev-tools/tmp/robots.txt | 8 +- robots.txt/robots.txt | 8 +- 53 files changed, 1170 insertions(+), 1125 deletions(-) diff --git a/AUTO-CONFIGURATION.md b/AUTO-CONFIGURATION.md index f6d7a8ae7..500f85d26 100755 --- a/AUTO-CONFIGURATION.md +++ b/AUTO-CONFIGURATION.md @@ -4,9 +4,9 @@ ### PLEASE READ CONFIGURATION INSTRUCTIONS BELOW THOROUGHLY :exclamation: _______________ -#### Version: V4.2022.01.2840 +#### Version: V4.2022.01.2841 #### Bad Referrers Blocked: [7087](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-referrers.list) -#### Bad User-Agents (Bots) Blocked: [631](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) +#### Bad User-Agents (Bots) Blocked: [632](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) #### Fake Googlebots Blocked: [217](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/fake-googlebots.list) ____________________ diff --git a/MANUAL-CONFIGURATION.md b/MANUAL-CONFIGURATION.md index d66b83c1d..14a1bce39 100755 --- a/MANUAL-CONFIGURATION.md +++ b/MANUAL-CONFIGURATION.md @@ -4,9 +4,9 @@ ### PLEASE READ CONFIGURATION INSTRUCTIONS BELOW THOROUGHLY :exclamation: _______________ -#### Version: V4.2022.01.2840 +#### Version: V4.2022.01.2841 #### Bad Referrers Blocked: [7087](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-referrers.list) -#### Bad User-Agents (Bots) Blocked: [631](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) +#### Bad User-Agents (Bots) Blocked: [632](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) #### Fake Googlebots Blocked: [217](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/fake-googlebots.list) ____________________ diff --git a/README.md b/README.md index 56921c8a7..96c7eadad 100755 --- a/README.md +++ b/README.md @@ -13,9 +13,9 @@ ##### The Ultimate Nginx Bad Bot, User-Agent, Spam Referrer Blocker, Adware, Malware and Ransomware Blocker, Clickjacking Blocker, Click Re-Directing Blocker, SEO Companies and Bad IP Blocker with Anti DDOS System, Nginx Rate Limiting and Wordpress Theme Detector Blocking. Stop and Block all kinds of bad internet traffic even Fake Googlebots from ever reaching your web sites. [PLEASE SEE: Definition of Bad Bots](#define-bad-bots) _______________ -#### Version: V4.2022.01.2840 +#### Version: V4.2022.01.2841 #### Bad Referrers Blocked: [7087](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-referrers.list) -#### Bad User-Agents (Bots) Blocked: [631](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) +#### Bad User-Agents (Bots) Blocked: [632](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) #### Fake Googlebots Blocked: [217](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/fake-googlebots.list) ____________________ diff --git a/_generator_lists/bad-user-agents.list b/_generator_lists/bad-user-agents.list index bf50c1942..27b6bcd8c 100755 --- a/_generator_lists/bad-user-agents.list +++ b/_generator_lists/bad-user-agents.list @@ -1,4 +1,3 @@ -Seekport 01h4x.com 360Spider 404checker @@ -400,6 +399,7 @@ ScreenerBot ScrepyBot Searchestate SearchmetricsBot +Seekport SemanticJuice Semrush SemrushBot diff --git a/_generator_lists/good-user-agents.list b/_generator_lists/good-user-agents.list index 6e692089a..2101d3013 100755 --- a/_generator_lists/good-user-agents.list +++ b/_generator_lists/good-user-agents.list @@ -1,7 +1,5 @@ AdsBot-Google Applebot -adidxbot -aolbuild DoCoMo Feedfetcher-Google Google-HTTP-Java-Client @@ -21,6 +19,9 @@ Slackbot Slackbot-LinkExpanding TwitterBot Wordpress +adidxbot +aolbuild +bing bingbot bingpreview developers.facebook.com @@ -33,4 +34,3 @@ msnbot-media slurp teoma yahoo -bing diff --git a/_google_analytics_ghost_spam/README.md b/_google_analytics_ghost_spam/README.md index 4889b2d03..5c975c50a 100644 --- a/_google_analytics_ghost_spam/README.md +++ b/_google_analytics_ghost_spam/README.md @@ -3,9 +3,9 @@ # EASY CONFIGURATION INSTRUCTIONS FOR STOPPING GOOGLE ANALYTICS "GHOST" SPAM _______________ -#### Version: V4.2022.01.2840 +#### Version: V4.2022.01.2841 #### Bad Referrers Blocked: [7087](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-referrers.list) -#### Bad User-Agents (Bots) Blocked: [631](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) +#### Bad User-Agents (Bots) Blocked: [632](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) #### Fake Googlebots Blocked: [217](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/fake-googlebots.list) ____________________ diff --git a/_sample_config_files/Engintron_for_cPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf b/_sample_config_files/Engintron_for_cPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf index 4124b1550..26b2894e0 100755 --- a/_sample_config_files/Engintron_for_cPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf +++ b/_sample_config_files/Engintron_for_cPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2022.01.2840 -### Updated: Sun Jan 9 06:33:51 UTC 2022 +### Version: V4.2022.01.2841 +### Updated: Sun Jan 9 07:00:22 UTC 2022 ### Bad Referrer Count: 7087 -### Bad Bot Count: 631 +### Bad Bot Count: 632 ################################################### ### VERSION INFORMATION ## @@ -548,6 +548,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)ScrepyBot(?:\b)" 3; "~*(?:\b)Searchestate(?:\b)" 3; "~*(?:\b)SearchmetricsBot(?:\b)" 3; + "~*(?:\b)Seekport(?:\b)" 3; "~*(?:\b)SemanticJuice(?:\b)" 3; "~*(?:\b)Semrush(?:\b)" 3; "~*(?:\b)SemrushBot(?:\b)" 3; @@ -793,6 +794,7 @@ map $http_user_agent $bad_bot { # START GOOD BOTS ### DO NOT EDIT THIS LINE AT ALL ### "~*(?:\b)AdsBot-Google(?:\b)" 0; + "~*(?:\b)Applebot(?:\b)" 0; "~*(?:\b)DoCoMo(?:\b)" 0; "~*(?:\b)Feedfetcher-Google(?:\b)" 0; "~*(?:\b)Google-HTTP-Java-Client(?:\b)" 0; @@ -814,6 +816,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)Wordpress(?:\b)" 0; "~*(?:\b)adidxbot(?:\b)" 0; "~*(?:\b)aolbuild(?:\b)" 0; + "~*(?:\b)bing(?:\b)" 0; "~*(?:\b)bingbot(?:\b)" 0; "~*(?:\b)bingpreview(?:\b)" 0; "~*(?:\b)developers.facebook.com(?:\b)" 0; diff --git a/conf.d/globalblacklist-testing-version.conf b/conf.d/globalblacklist-testing-version.conf index 9cda5ea40..372417727 100644 --- a/conf.d/globalblacklist-testing-version.conf +++ b/conf.d/globalblacklist-testing-version.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2022.01.2840 -### Updated: Sun Jan 9 06:33:52 UTC 2022 +### Version: V3.2022.01.2841 +### Updated: Sun Jan 9 07:00:23 UTC 2022 ### Bad Referrer Count: 7087 -### Bad Bot Count: 631 +### Bad Bot Count: 632 ################################################### ### VERSION INFORMATION ## @@ -548,6 +548,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)ScrepyBot(?:\b)" 3; "~*(?:\b)Searchestate(?:\b)" 3; "~*(?:\b)SearchmetricsBot(?:\b)" 3; + "~*(?:\b)Seekport(?:\b)" 3; "~*(?:\b)SemanticJuice(?:\b)" 3; "~*(?:\b)Semrush(?:\b)" 3; "~*(?:\b)SemrushBot(?:\b)" 3; @@ -793,6 +794,7 @@ map $http_user_agent $bad_bot { # START GOOD BOTS ### DO NOT EDIT THIS LINE AT ALL ### "~*(?:\b)AdsBot-Google(?:\b)" 0; + "~*(?:\b)Applebot(?:\b)" 0; "~*(?:\b)DoCoMo(?:\b)" 0; "~*(?:\b)Feedfetcher-Google(?:\b)" 0; "~*(?:\b)Google-HTTP-Java-Client(?:\b)" 0; @@ -814,6 +816,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)Wordpress(?:\b)" 0; "~*(?:\b)adidxbot(?:\b)" 0; "~*(?:\b)aolbuild(?:\b)" 0; + "~*(?:\b)bing(?:\b)" 0; "~*(?:\b)bingbot(?:\b)" 0; "~*(?:\b)bingpreview(?:\b)" 0; "~*(?:\b)developers.facebook.com(?:\b)" 0; diff --git a/conf.d/globalblacklist.conf b/conf.d/globalblacklist.conf index 4124b1550..26b2894e0 100644 --- a/conf.d/globalblacklist.conf +++ b/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2022.01.2840 -### Updated: Sun Jan 9 06:33:51 UTC 2022 +### Version: V4.2022.01.2841 +### Updated: Sun Jan 9 07:00:22 UTC 2022 ### Bad Referrer Count: 7087 -### Bad Bot Count: 631 +### Bad Bot Count: 632 ################################################### ### VERSION INFORMATION ## @@ -548,6 +548,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)ScrepyBot(?:\b)" 3; "~*(?:\b)Searchestate(?:\b)" 3; "~*(?:\b)SearchmetricsBot(?:\b)" 3; + "~*(?:\b)Seekport(?:\b)" 3; "~*(?:\b)SemanticJuice(?:\b)" 3; "~*(?:\b)Semrush(?:\b)" 3; "~*(?:\b)SemrushBot(?:\b)" 3; @@ -793,6 +794,7 @@ map $http_user_agent $bad_bot { # START GOOD BOTS ### DO NOT EDIT THIS LINE AT ALL ### "~*(?:\b)AdsBot-Google(?:\b)" 0; + "~*(?:\b)Applebot(?:\b)" 0; "~*(?:\b)DoCoMo(?:\b)" 0; "~*(?:\b)Feedfetcher-Google(?:\b)" 0; "~*(?:\b)Google-HTTP-Java-Client(?:\b)" 0; @@ -814,6 +816,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)Wordpress(?:\b)" 0; "~*(?:\b)adidxbot(?:\b)" 0; "~*(?:\b)aolbuild(?:\b)" 0; + "~*(?:\b)bing(?:\b)" 0; "~*(?:\b)bingbot(?:\b)" 0; "~*(?:\b)bingpreview(?:\b)" 0; "~*(?:\b)developers.facebook.com(?:\b)" 0; diff --git a/dev-tools/_robots_input/robots-input.txt b/dev-tools/_robots_input/robots-input.txt index c6c595b3a..753d72c73 100644 --- a/dev-tools/_robots_input/robots-input.txt +++ b/dev-tools/_robots_input/robots-input.txt @@ -399,6 +399,7 @@ ScreenerBot ScrepyBot Searchestate SearchmetricsBot +Seekport SemanticJuice Semrush SemrushBot diff --git a/dev-tools/beta_conf_files/conf.d/globalblacklist.conf b/dev-tools/beta_conf_files/conf.d/globalblacklist.conf index 9cda5ea40..372417727 100644 --- a/dev-tools/beta_conf_files/conf.d/globalblacklist.conf +++ b/dev-tools/beta_conf_files/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2022.01.2840 -### Updated: Sun Jan 9 06:33:52 UTC 2022 +### Version: V3.2022.01.2841 +### Updated: Sun Jan 9 07:00:23 UTC 2022 ### Bad Referrer Count: 7087 -### Bad Bot Count: 631 +### Bad Bot Count: 632 ################################################### ### VERSION INFORMATION ## @@ -548,6 +548,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)ScrepyBot(?:\b)" 3; "~*(?:\b)Searchestate(?:\b)" 3; "~*(?:\b)SearchmetricsBot(?:\b)" 3; + "~*(?:\b)Seekport(?:\b)" 3; "~*(?:\b)SemanticJuice(?:\b)" 3; "~*(?:\b)Semrush(?:\b)" 3; "~*(?:\b)SemrushBot(?:\b)" 3; @@ -793,6 +794,7 @@ map $http_user_agent $bad_bot { # START GOOD BOTS ### DO NOT EDIT THIS LINE AT ALL ### "~*(?:\b)AdsBot-Google(?:\b)" 0; + "~*(?:\b)Applebot(?:\b)" 0; "~*(?:\b)DoCoMo(?:\b)" 0; "~*(?:\b)Feedfetcher-Google(?:\b)" 0; "~*(?:\b)Google-HTTP-Java-Client(?:\b)" 0; @@ -814,6 +816,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)Wordpress(?:\b)" 0; "~*(?:\b)adidxbot(?:\b)" 0; "~*(?:\b)aolbuild(?:\b)" 0; + "~*(?:\b)bing(?:\b)" 0; "~*(?:\b)bingbot(?:\b)" 0; "~*(?:\b)bingpreview(?:\b)" 0; "~*(?:\b)developers.facebook.com(?:\b)" 0; diff --git a/dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf b/dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf index 560fb94d1..35f734417 100644 --- a/dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf +++ b/dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.69 1; +10.1.0.222 1; 127.0.0.1 1; diff --git a/dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf b/dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf index 447da2bf0..3951616fb 100644 --- a/dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf +++ b/dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.69 0; +10.1.0.222 0; 127.0.0.1 0; diff --git a/dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf b/dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf index 9cda5ea40..372417727 100644 --- a/dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf +++ b/dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2022.01.2840 -### Updated: Sun Jan 9 06:33:52 UTC 2022 +### Version: V3.2022.01.2841 +### Updated: Sun Jan 9 07:00:23 UTC 2022 ### Bad Referrer Count: 7087 -### Bad Bot Count: 631 +### Bad Bot Count: 632 ################################################### ### VERSION INFORMATION ## @@ -548,6 +548,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)ScrepyBot(?:\b)" 3; "~*(?:\b)Searchestate(?:\b)" 3; "~*(?:\b)SearchmetricsBot(?:\b)" 3; + "~*(?:\b)Seekport(?:\b)" 3; "~*(?:\b)SemanticJuice(?:\b)" 3; "~*(?:\b)Semrush(?:\b)" 3; "~*(?:\b)SemrushBot(?:\b)" 3; @@ -793,6 +794,7 @@ map $http_user_agent $bad_bot { # START GOOD BOTS ### DO NOT EDIT THIS LINE AT ALL ### "~*(?:\b)AdsBot-Google(?:\b)" 0; + "~*(?:\b)Applebot(?:\b)" 0; "~*(?:\b)DoCoMo(?:\b)" 0; "~*(?:\b)Feedfetcher-Google(?:\b)" 0; "~*(?:\b)Google-HTTP-Java-Client(?:\b)" 0; @@ -814,6 +816,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)Wordpress(?:\b)" 0; "~*(?:\b)adidxbot(?:\b)" 0; "~*(?:\b)aolbuild(?:\b)" 0; + "~*(?:\b)bing(?:\b)" 0; "~*(?:\b)bingbot(?:\b)" 0; "~*(?:\b)bingpreview(?:\b)" 0; "~*(?:\b)developers.facebook.com(?:\b)" 0; diff --git a/dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf b/dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf index 560fb94d1..35f734417 100644 --- a/dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf +++ b/dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.69 1; +10.1.0.222 1; 127.0.0.1 1; diff --git a/dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf b/dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf index 447da2bf0..3951616fb 100644 --- a/dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf +++ b/dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.69 0; +10.1.0.222 0; 127.0.0.1 0; diff --git a/dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf b/dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf index 9cda5ea40..372417727 100644 --- a/dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf +++ b/dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2022.01.2840 -### Updated: Sun Jan 9 06:33:52 UTC 2022 +### Version: V3.2022.01.2841 +### Updated: Sun Jan 9 07:00:23 UTC 2022 ### Bad Referrer Count: 7087 -### Bad Bot Count: 631 +### Bad Bot Count: 632 ################################################### ### VERSION INFORMATION ## @@ -548,6 +548,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)ScrepyBot(?:\b)" 3; "~*(?:\b)Searchestate(?:\b)" 3; "~*(?:\b)SearchmetricsBot(?:\b)" 3; + "~*(?:\b)Seekport(?:\b)" 3; "~*(?:\b)SemanticJuice(?:\b)" 3; "~*(?:\b)Semrush(?:\b)" 3; "~*(?:\b)SemrushBot(?:\b)" 3; @@ -793,6 +794,7 @@ map $http_user_agent $bad_bot { # START GOOD BOTS ### DO NOT EDIT THIS LINE AT ALL ### "~*(?:\b)AdsBot-Google(?:\b)" 0; + "~*(?:\b)Applebot(?:\b)" 0; "~*(?:\b)DoCoMo(?:\b)" 0; "~*(?:\b)Feedfetcher-Google(?:\b)" 0; "~*(?:\b)Google-HTTP-Java-Client(?:\b)" 0; @@ -814,6 +816,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)Wordpress(?:\b)" 0; "~*(?:\b)adidxbot(?:\b)" 0; "~*(?:\b)aolbuild(?:\b)" 0; + "~*(?:\b)bing(?:\b)" 0; "~*(?:\b)bingbot(?:\b)" 0; "~*(?:\b)bingpreview(?:\b)" 0; "~*(?:\b)developers.facebook.com(?:\b)" 0; diff --git a/dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf b/dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf index 560fb94d1..35f734417 100644 --- a/dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf +++ b/dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.69 1; +10.1.0.222 1; 127.0.0.1 1; diff --git a/dev-tools/beta_conf_files_whitelist/bots.d/blacklist-user-agents.conf b/dev-tools/beta_conf_files_whitelist/bots.d/blacklist-user-agents.conf index c0d6a5c08..0bd3b22e8 100644 --- a/dev-tools/beta_conf_files_whitelist/bots.d/blacklist-user-agents.conf +++ b/dev-tools/beta_conf_files_whitelist/bots.d/blacklist-user-agents.conf @@ -460,6 +460,7 @@ "~*(?:\b)ScrepyBot(?:\b|)" 0; "~*(?:\b)Searchestate(?:\b|)" 0; "~*(?:\b)SearchmetricsBot(?:\b|)" 0; + "~*(?:\b)Seekport(?:\b|)" 0; "~*(?:\b)SemanticJuice(?:\b|)" 0; "~*(?:\b)Semrush(?:\b|)" 0; "~*(?:\b)SemrushBot(?:\b|)" 0; diff --git a/dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf b/dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf index 447da2bf0..3951616fb 100644 --- a/dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf +++ b/dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.69 0; +10.1.0.222 0; 127.0.0.1 0; diff --git a/dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf b/dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf index 9cda5ea40..372417727 100644 --- a/dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf +++ b/dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2022.01.2840 -### Updated: Sun Jan 9 06:33:52 UTC 2022 +### Version: V3.2022.01.2841 +### Updated: Sun Jan 9 07:00:23 UTC 2022 ### Bad Referrer Count: 7087 -### Bad Bot Count: 631 +### Bad Bot Count: 632 ################################################### ### VERSION INFORMATION ## @@ -548,6 +548,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)ScrepyBot(?:\b)" 3; "~*(?:\b)Searchestate(?:\b)" 3; "~*(?:\b)SearchmetricsBot(?:\b)" 3; + "~*(?:\b)Seekport(?:\b)" 3; "~*(?:\b)SemanticJuice(?:\b)" 3; "~*(?:\b)Semrush(?:\b)" 3; "~*(?:\b)SemrushBot(?:\b)" 3; @@ -793,6 +794,7 @@ map $http_user_agent $bad_bot { # START GOOD BOTS ### DO NOT EDIT THIS LINE AT ALL ### "~*(?:\b)AdsBot-Google(?:\b)" 0; + "~*(?:\b)Applebot(?:\b)" 0; "~*(?:\b)DoCoMo(?:\b)" 0; "~*(?:\b)Feedfetcher-Google(?:\b)" 0; "~*(?:\b)Google-HTTP-Java-Client(?:\b)" 0; @@ -814,6 +816,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)Wordpress(?:\b)" 0; "~*(?:\b)adidxbot(?:\b)" 0; "~*(?:\b)aolbuild(?:\b)" 0; + "~*(?:\b)bing(?:\b)" 0; "~*(?:\b)bingbot(?:\b)" 0; "~*(?:\b)bingpreview(?:\b)" 0; "~*(?:\b)developers.facebook.com(?:\b)" 0; diff --git a/dev-tools/buildnumber b/dev-tools/buildnumber index 8a30791f6..6cda91b39 100644 --- a/dev-tools/buildnumber +++ b/dev-tools/buildnumber @@ -1 +1 @@ -2840 +2841 diff --git a/dev-tools/globalblacklist-testing.template b/dev-tools/globalblacklist-testing.template index 9cda5ea40..372417727 100644 --- a/dev-tools/globalblacklist-testing.template +++ b/dev-tools/globalblacklist-testing.template @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2022.01.2840 -### Updated: Sun Jan 9 06:33:52 UTC 2022 +### Version: V3.2022.01.2841 +### Updated: Sun Jan 9 07:00:23 UTC 2022 ### Bad Referrer Count: 7087 -### Bad Bot Count: 631 +### Bad Bot Count: 632 ################################################### ### VERSION INFORMATION ## @@ -548,6 +548,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)ScrepyBot(?:\b)" 3; "~*(?:\b)Searchestate(?:\b)" 3; "~*(?:\b)SearchmetricsBot(?:\b)" 3; + "~*(?:\b)Seekport(?:\b)" 3; "~*(?:\b)SemanticJuice(?:\b)" 3; "~*(?:\b)Semrush(?:\b)" 3; "~*(?:\b)SemrushBot(?:\b)" 3; @@ -793,6 +794,7 @@ map $http_user_agent $bad_bot { # START GOOD BOTS ### DO NOT EDIT THIS LINE AT ALL ### "~*(?:\b)AdsBot-Google(?:\b)" 0; + "~*(?:\b)Applebot(?:\b)" 0; "~*(?:\b)DoCoMo(?:\b)" 0; "~*(?:\b)Feedfetcher-Google(?:\b)" 0; "~*(?:\b)Google-HTTP-Java-Client(?:\b)" 0; @@ -814,6 +816,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)Wordpress(?:\b)" 0; "~*(?:\b)adidxbot(?:\b)" 0; "~*(?:\b)aolbuild(?:\b)" 0; + "~*(?:\b)bing(?:\b)" 0; "~*(?:\b)bingbot(?:\b)" 0; "~*(?:\b)bingpreview(?:\b)" 0; "~*(?:\b)developers.facebook.com(?:\b)" 0; diff --git a/dev-tools/globalblacklist.template b/dev-tools/globalblacklist.template index 4124b1550..26b2894e0 100755 --- a/dev-tools/globalblacklist.template +++ b/dev-tools/globalblacklist.template @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2022.01.2840 -### Updated: Sun Jan 9 06:33:51 UTC 2022 +### Version: V4.2022.01.2841 +### Updated: Sun Jan 9 07:00:22 UTC 2022 ### Bad Referrer Count: 7087 -### Bad Bot Count: 631 +### Bad Bot Count: 632 ################################################### ### VERSION INFORMATION ## @@ -548,6 +548,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)ScrepyBot(?:\b)" 3; "~*(?:\b)Searchestate(?:\b)" 3; "~*(?:\b)SearchmetricsBot(?:\b)" 3; + "~*(?:\b)Seekport(?:\b)" 3; "~*(?:\b)SemanticJuice(?:\b)" 3; "~*(?:\b)Semrush(?:\b)" 3; "~*(?:\b)SemrushBot(?:\b)" 3; @@ -793,6 +794,7 @@ map $http_user_agent $bad_bot { # START GOOD BOTS ### DO NOT EDIT THIS LINE AT ALL ### "~*(?:\b)AdsBot-Google(?:\b)" 0; + "~*(?:\b)Applebot(?:\b)" 0; "~*(?:\b)DoCoMo(?:\b)" 0; "~*(?:\b)Feedfetcher-Google(?:\b)" 0; "~*(?:\b)Google-HTTP-Java-Client(?:\b)" 0; @@ -814,6 +816,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)Wordpress(?:\b)" 0; "~*(?:\b)adidxbot(?:\b)" 0; "~*(?:\b)aolbuild(?:\b)" 0; + "~*(?:\b)bing(?:\b)" 0; "~*(?:\b)bingbot(?:\b)" 0; "~*(?:\b)bingpreview(?:\b)" 0; "~*(?:\b)developers.facebook.com(?:\b)" 0; diff --git a/dev-tools/test1_conf_files/bots.d/whitelist-ips.conf b/dev-tools/test1_conf_files/bots.d/whitelist-ips.conf index 58434e3d0..e72cb9e76 100644 --- a/dev-tools/test1_conf_files/bots.d/whitelist-ips.conf +++ b/dev-tools/test1_conf_files/bots.d/whitelist-ips.conf @@ -45,4 +45,4 @@ # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST -20.112.84.204 0; +20.127.127.68 0; diff --git a/dev-tools/test1_conf_files/conf.d/globalblacklist.conf b/dev-tools/test1_conf_files/conf.d/globalblacklist.conf index 4124b1550..26b2894e0 100644 --- a/dev-tools/test1_conf_files/conf.d/globalblacklist.conf +++ b/dev-tools/test1_conf_files/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2022.01.2840 -### Updated: Sun Jan 9 06:33:51 UTC 2022 +### Version: V4.2022.01.2841 +### Updated: Sun Jan 9 07:00:22 UTC 2022 ### Bad Referrer Count: 7087 -### Bad Bot Count: 631 +### Bad Bot Count: 632 ################################################### ### VERSION INFORMATION ## @@ -548,6 +548,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)ScrepyBot(?:\b)" 3; "~*(?:\b)Searchestate(?:\b)" 3; "~*(?:\b)SearchmetricsBot(?:\b)" 3; + "~*(?:\b)Seekport(?:\b)" 3; "~*(?:\b)SemanticJuice(?:\b)" 3; "~*(?:\b)Semrush(?:\b)" 3; "~*(?:\b)SemrushBot(?:\b)" 3; @@ -793,6 +794,7 @@ map $http_user_agent $bad_bot { # START GOOD BOTS ### DO NOT EDIT THIS LINE AT ALL ### "~*(?:\b)AdsBot-Google(?:\b)" 0; + "~*(?:\b)Applebot(?:\b)" 0; "~*(?:\b)DoCoMo(?:\b)" 0; "~*(?:\b)Feedfetcher-Google(?:\b)" 0; "~*(?:\b)Google-HTTP-Java-Client(?:\b)" 0; @@ -814,6 +816,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)Wordpress(?:\b)" 0; "~*(?:\b)adidxbot(?:\b)" 0; "~*(?:\b)aolbuild(?:\b)" 0; + "~*(?:\b)bing(?:\b)" 0; "~*(?:\b)bingbot(?:\b)" 0; "~*(?:\b)bingpreview(?:\b)" 0; "~*(?:\b)developers.facebook.com(?:\b)" 0; diff --git a/dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf b/dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf index 560fb94d1..35f734417 100644 --- a/dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf +++ b/dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.69 1; +10.1.0.222 1; 127.0.0.1 1; diff --git a/dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf b/dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf index 447da2bf0..3951616fb 100644 --- a/dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf +++ b/dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.69 0; +10.1.0.222 0; 127.0.0.1 0; diff --git a/dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf b/dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf index 4124b1550..26b2894e0 100644 --- a/dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf +++ b/dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2022.01.2840 -### Updated: Sun Jan 9 06:33:51 UTC 2022 +### Version: V4.2022.01.2841 +### Updated: Sun Jan 9 07:00:22 UTC 2022 ### Bad Referrer Count: 7087 -### Bad Bot Count: 631 +### Bad Bot Count: 632 ################################################### ### VERSION INFORMATION ## @@ -548,6 +548,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)ScrepyBot(?:\b)" 3; "~*(?:\b)Searchestate(?:\b)" 3; "~*(?:\b)SearchmetricsBot(?:\b)" 3; + "~*(?:\b)Seekport(?:\b)" 3; "~*(?:\b)SemanticJuice(?:\b)" 3; "~*(?:\b)Semrush(?:\b)" 3; "~*(?:\b)SemrushBot(?:\b)" 3; @@ -793,6 +794,7 @@ map $http_user_agent $bad_bot { # START GOOD BOTS ### DO NOT EDIT THIS LINE AT ALL ### "~*(?:\b)AdsBot-Google(?:\b)" 0; + "~*(?:\b)Applebot(?:\b)" 0; "~*(?:\b)DoCoMo(?:\b)" 0; "~*(?:\b)Feedfetcher-Google(?:\b)" 0; "~*(?:\b)Google-HTTP-Java-Client(?:\b)" 0; @@ -814,6 +816,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)Wordpress(?:\b)" 0; "~*(?:\b)adidxbot(?:\b)" 0; "~*(?:\b)aolbuild(?:\b)" 0; + "~*(?:\b)bing(?:\b)" 0; "~*(?:\b)bingbot(?:\b)" 0; "~*(?:\b)bingpreview(?:\b)" 0; "~*(?:\b)developers.facebook.com(?:\b)" 0; diff --git a/dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf b/dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf index 560fb94d1..35f734417 100644 --- a/dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf +++ b/dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.69 1; +10.1.0.222 1; 127.0.0.1 1; diff --git a/dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf b/dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf index 447da2bf0..3951616fb 100644 --- a/dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf +++ b/dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.69 0; +10.1.0.222 0; 127.0.0.1 0; diff --git a/dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf b/dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf index 4124b1550..26b2894e0 100644 --- a/dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf +++ b/dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2022.01.2840 -### Updated: Sun Jan 9 06:33:51 UTC 2022 +### Version: V4.2022.01.2841 +### Updated: Sun Jan 9 07:00:22 UTC 2022 ### Bad Referrer Count: 7087 -### Bad Bot Count: 631 +### Bad Bot Count: 632 ################################################### ### VERSION INFORMATION ## @@ -548,6 +548,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)ScrepyBot(?:\b)" 3; "~*(?:\b)Searchestate(?:\b)" 3; "~*(?:\b)SearchmetricsBot(?:\b)" 3; + "~*(?:\b)Seekport(?:\b)" 3; "~*(?:\b)SemanticJuice(?:\b)" 3; "~*(?:\b)Semrush(?:\b)" 3; "~*(?:\b)SemrushBot(?:\b)" 3; @@ -793,6 +794,7 @@ map $http_user_agent $bad_bot { # START GOOD BOTS ### DO NOT EDIT THIS LINE AT ALL ### "~*(?:\b)AdsBot-Google(?:\b)" 0; + "~*(?:\b)Applebot(?:\b)" 0; "~*(?:\b)DoCoMo(?:\b)" 0; "~*(?:\b)Feedfetcher-Google(?:\b)" 0; "~*(?:\b)Google-HTTP-Java-Client(?:\b)" 0; @@ -814,6 +816,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)Wordpress(?:\b)" 0; "~*(?:\b)adidxbot(?:\b)" 0; "~*(?:\b)aolbuild(?:\b)" 0; + "~*(?:\b)bing(?:\b)" 0; "~*(?:\b)bingbot(?:\b)" 0; "~*(?:\b)bingpreview(?:\b)" 0; "~*(?:\b)developers.facebook.com(?:\b)" 0; diff --git a/dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf b/dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf index 560fb94d1..35f734417 100644 --- a/dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf +++ b/dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.69 1; +10.1.0.222 1; 127.0.0.1 1; diff --git a/dev-tools/test1_conf_files_whitelist/bots.d/blacklist-user-agents.conf b/dev-tools/test1_conf_files_whitelist/bots.d/blacklist-user-agents.conf index c0d6a5c08..0bd3b22e8 100644 --- a/dev-tools/test1_conf_files_whitelist/bots.d/blacklist-user-agents.conf +++ b/dev-tools/test1_conf_files_whitelist/bots.d/blacklist-user-agents.conf @@ -460,6 +460,7 @@ "~*(?:\b)ScrepyBot(?:\b|)" 0; "~*(?:\b)Searchestate(?:\b|)" 0; "~*(?:\b)SearchmetricsBot(?:\b|)" 0; + "~*(?:\b)Seekport(?:\b|)" 0; "~*(?:\b)SemanticJuice(?:\b|)" 0; "~*(?:\b)Semrush(?:\b|)" 0; "~*(?:\b)SemrushBot(?:\b|)" 0; diff --git a/dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf b/dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf index 447da2bf0..3951616fb 100644 --- a/dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf +++ b/dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.69 0; +10.1.0.222 0; 127.0.0.1 0; diff --git a/dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf b/dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf index 4124b1550..26b2894e0 100644 --- a/dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf +++ b/dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2022.01.2840 -### Updated: Sun Jan 9 06:33:51 UTC 2022 +### Version: V4.2022.01.2841 +### Updated: Sun Jan 9 07:00:22 UTC 2022 ### Bad Referrer Count: 7087 -### Bad Bot Count: 631 +### Bad Bot Count: 632 ################################################### ### VERSION INFORMATION ## @@ -548,6 +548,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)ScrepyBot(?:\b)" 3; "~*(?:\b)Searchestate(?:\b)" 3; "~*(?:\b)SearchmetricsBot(?:\b)" 3; + "~*(?:\b)Seekport(?:\b)" 3; "~*(?:\b)SemanticJuice(?:\b)" 3; "~*(?:\b)Semrush(?:\b)" 3; "~*(?:\b)SemrushBot(?:\b)" 3; @@ -793,6 +794,7 @@ map $http_user_agent $bad_bot { # START GOOD BOTS ### DO NOT EDIT THIS LINE AT ALL ### "~*(?:\b)AdsBot-Google(?:\b)" 0; + "~*(?:\b)Applebot(?:\b)" 0; "~*(?:\b)DoCoMo(?:\b)" 0; "~*(?:\b)Feedfetcher-Google(?:\b)" 0; "~*(?:\b)Google-HTTP-Java-Client(?:\b)" 0; @@ -814,6 +816,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)Wordpress(?:\b)" 0; "~*(?:\b)adidxbot(?:\b)" 0; "~*(?:\b)aolbuild(?:\b)" 0; + "~*(?:\b)bing(?:\b)" 0; "~*(?:\b)bingbot(?:\b)" 0; "~*(?:\b)bingpreview(?:\b)" 0; "~*(?:\b)developers.facebook.com(?:\b)" 0; diff --git a/dev-tools/test2_conf_files/bots.d/whitelist-ips.conf b/dev-tools/test2_conf_files/bots.d/whitelist-ips.conf index 58434e3d0..e72cb9e76 100644 --- a/dev-tools/test2_conf_files/bots.d/whitelist-ips.conf +++ b/dev-tools/test2_conf_files/bots.d/whitelist-ips.conf @@ -45,4 +45,4 @@ # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST -20.112.84.204 0; +20.127.127.68 0; diff --git a/dev-tools/test2_conf_files/conf.d/globalblacklist.conf b/dev-tools/test2_conf_files/conf.d/globalblacklist.conf index 661a44e51..1dce6b387 100644 --- a/dev-tools/test2_conf_files/conf.d/globalblacklist.conf +++ b/dev-tools/test2_conf_files/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2022.01.2840 -### Updated: Sun Jan 9 06:33:51 UTC 2022 +### Version: V4.2022.01.2841 +### Updated: Sun Jan 9 07:00:22 UTC 2022 ### Bad Referrer Count: 7087 -### Bad Bot Count: 631 +### Bad Bot Count: 632 ################################################### ### VERSION INFORMATION ## @@ -548,6 +548,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)ScrepyBot(?:\b)" 3; "~*(?:\b)Searchestate(?:\b)" 3; "~*(?:\b)SearchmetricsBot(?:\b)" 3; + "~*(?:\b)Seekport(?:\b)" 3; "~*(?:\b)SemanticJuice(?:\b)" 3; "~*(?:\b)Semrush(?:\b)" 3; "~*(?:\b)SemrushBot(?:\b)" 3; @@ -793,6 +794,7 @@ map $http_user_agent $bad_bot { # START GOOD BOTS ### DO NOT EDIT THIS LINE AT ALL ### "~*(?:\b)AdsBot-Google(?:\b)" 0; + "~*(?:\b)Applebot(?:\b)" 0; "~*(?:\b)DoCoMo(?:\b)" 0; "~*(?:\b)Feedfetcher-Google(?:\b)" 0; "~*(?:\b)Google-HTTP-Java-Client(?:\b)" 0; @@ -814,6 +816,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)Wordpress(?:\b)" 0; "~*(?:\b)adidxbot(?:\b)" 0; "~*(?:\b)aolbuild(?:\b)" 0; + "~*(?:\b)bing(?:\b)" 0; "~*(?:\b)bingbot(?:\b)" 0; "~*(?:\b)bingpreview(?:\b)" 0; "~*(?:\b)developers.facebook.com(?:\b)" 0; diff --git a/dev-tools/test3_conf_files/bots.d/whitelist-ips.conf b/dev-tools/test3_conf_files/bots.d/whitelist-ips.conf index 58434e3d0..e72cb9e76 100644 --- a/dev-tools/test3_conf_files/bots.d/whitelist-ips.conf +++ b/dev-tools/test3_conf_files/bots.d/whitelist-ips.conf @@ -45,4 +45,4 @@ # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST -20.112.84.204 0; +20.127.127.68 0; diff --git a/dev-tools/test3_conf_files/conf.d/globalblacklist.conf b/dev-tools/test3_conf_files/conf.d/globalblacklist.conf index 71d82c4c3..46452191c 100644 --- a/dev-tools/test3_conf_files/conf.d/globalblacklist.conf +++ b/dev-tools/test3_conf_files/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2022.01.2840 -### Updated: Sun Jan 9 06:33:51 UTC 2022 +### Version: V4.2022.01.2841 +### Updated: Sun Jan 9 07:00:22 UTC 2022 ### Bad Referrer Count: 7087 -### Bad Bot Count: 631 +### Bad Bot Count: 632 ################################################### ### VERSION INFORMATION ## @@ -548,6 +548,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)ScrepyBot(?:\b)" 3; "~*(?:\b)Searchestate(?:\b)" 3; "~*(?:\b)SearchmetricsBot(?:\b)" 3; + "~*(?:\b)Seekport(?:\b)" 3; "~*(?:\b)SemanticJuice(?:\b)" 3; "~*(?:\b)Semrush(?:\b)" 3; "~*(?:\b)SemrushBot(?:\b)" 3; @@ -793,6 +794,7 @@ map $http_user_agent $bad_bot { # START GOOD BOTS ### DO NOT EDIT THIS LINE AT ALL ### "~*(?:\b)AdsBot-Google(?:\b)" 0; + "~*(?:\b)Applebot(?:\b)" 0; "~*(?:\b)DoCoMo(?:\b)" 0; "~*(?:\b)Feedfetcher-Google(?:\b)" 0; "~*(?:\b)Google-HTTP-Java-Client(?:\b)" 0; @@ -814,6 +816,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)Wordpress(?:\b)" 0; "~*(?:\b)adidxbot(?:\b)" 0; "~*(?:\b)aolbuild(?:\b)" 0; + "~*(?:\b)bing(?:\b)" 0; "~*(?:\b)bingbot(?:\b)" 0; "~*(?:\b)bingpreview(?:\b)" 0; "~*(?:\b)developers.facebook.com(?:\b)" 0; diff --git a/dev-tools/test_units/blacklist-ips.conf b/dev-tools/test_units/blacklist-ips.conf index 560fb94d1..35f734417 100644 --- a/dev-tools/test_units/blacklist-ips.conf +++ b/dev-tools/test_units/blacklist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.69 1; +10.1.0.222 1; 127.0.0.1 1; diff --git a/dev-tools/test_units/blacklist-user-agents.conf b/dev-tools/test_units/blacklist-user-agents.conf index c0d6a5c08..0bd3b22e8 100644 --- a/dev-tools/test_units/blacklist-user-agents.conf +++ b/dev-tools/test_units/blacklist-user-agents.conf @@ -460,6 +460,7 @@ "~*(?:\b)ScrepyBot(?:\b|)" 0; "~*(?:\b)Searchestate(?:\b|)" 0; "~*(?:\b)SearchmetricsBot(?:\b|)" 0; + "~*(?:\b)Seekport(?:\b|)" 0; "~*(?:\b)SemanticJuice(?:\b|)" 0; "~*(?:\b)Semrush(?:\b|)" 0; "~*(?:\b)SemrushBot(?:\b|)" 0; diff --git a/dev-tools/test_units/good-bots-for-test.list b/dev-tools/test_units/good-bots-for-test.list index d2553a3e7..ec6f8ee58 100644 --- a/dev-tools/test_units/good-bots-for-test.list +++ b/dev-tools/test_units/good-bots-for-test.list @@ -1,4 +1,5 @@ AdsBot-Google +Applebot DoCoMo Feedfetcher-Google Google-HTTP-Java-Client @@ -20,6 +21,7 @@ TwitterBot Wordpress adidxbot aolbuild +bing bingbot bingpreview developers.facebook.com diff --git a/dev-tools/test_units/random-bots-for-test-quick.list b/dev-tools/test_units/random-bots-for-test-quick.list index e83e34793..1ed5ccb3a 100644 --- a/dev-tools/test_units/random-bots-for-test-quick.list +++ b/dev-tools/test_units/random-bots-for-test-quick.list @@ -1,10 +1,10 @@ -CSHttp -CherryPicker -Dispatch -Larbin -Scrapy -Sitebeam -Sucuri -WEBDAV -Web Fuck -WebReaper +Alexibot +BetaBot +Crawling at Home Project +Genieo +GetRight +Grabber +Image Sucker +Iria +Rankivabot +Re-re diff --git a/dev-tools/test_units/random-bots-for-test.list b/dev-tools/test_units/random-bots-for-test.list index 43e6e4946..b7d330e1a 100644 --- a/dev-tools/test_units/random-bots-for-test.list +++ b/dev-tools/test_units/random-bots-for-test.list @@ -1,250 +1,250 @@ -80legs -Abonti -Aboundexbot +360Spider +404checker +ADmantX +AIBOT +ALittle Client +Aboundex +Acunetix +AhrefsBot AiHitBot -Aipbot -Alexibot -AlphaBot +AllSubmitter Anarchy99 -Ankit -Anthill -Asterias AwarioRssBot AwarioSmartBot -BBBike -BackDoorBot -BackWeb +BackStreet +Backlink-Ceck BacklinkCrawler Badass +Bandit +Battleztar Bazinga +Bigfoot +Bitacle Blackboard -BlowFish +Blow Boardreader -Brandprotect -Bullseye -BunnySlippers -BuzzSumo +Brandwatch +Buck CATExplorador -Calculon +CCBot +CSHttp +Cegbfeieh +ChinaClaw +Chlooe Citoid Claritybot -Cocolyzebot +Cliqzbot Cogentbot -Cosmos -Craftbot -Crawling at Home Project -Crescent +Copier +CrazyWebCrawler +CrunchBot Curious -Custo -CyotekWebCopy -DSearch -DTS Agent +DataCha0s +Demon +Devil +Dirbuster Disco -Dispatch +DittoSpyder DnBCrawler-Analytics DnyzBot -DomainAppender -DomainCrawler DomainSigmaCrawler -DomainStatsBot -ECCP/1.0 +Domains Project +Dotbot +EMail Wolf +EasyDL Ebingbong -Exabot +Ecxi +EirGrabber +EroCrawler +Express WebPictures +ExtLinksBot +Extractor +ExtractorPro +EyeNetIE Ezooms FDM FHscan -Firefox/7.0 +FemtosearchBot +Fimap FlashGet -Foobot -Fuzz -FyberSpider +G-i-g-a-b-o-t GT::WWW -Genieo -GetRight -GetWeb -Gotit -GrabNet +Getintent Grabber -Grafula GrapeFX -GridBot -HTTrack +HaosouSpider +Harvest Havij Heritrix Hloader +HybridBot +IDBTE4M IDBot -IlseBot -Indy Library +Image Fetch +IndeedBot +InfoNaviRobot InfoTekies InterGET -Internet Ninja -InternetSeer +Iria +Iskanie IstellaBot +JamesBOT +Jbrofuzz +JennyBot +JetCar +JikeSpider +Joomla Jorgee -Kenjin Spider Keyword Density +Kinza LNSpiderguy Lanshanbot Larbin +LeechFTP LeechGet -LexiBot -Lftp Libwhisker -LieBaoFast -Lightspeedsystems -LinksManager -LinqiaRSSBot -Lipperhey -MFC_Tear_Sample +Linkdexbot +LinkextractorPro +LinqiaScrapeBot +Ltx71 MIDown tool -MSIECrawler -MTRobot -Magnet +MQQBrowser +Mag-Net Mail.RU_Bot +Majestic-SEO +Majestic12 +MarkMonitor MarkWatch Mass Downloader -Mata Hari Mb2345Browser -MeanPath Bot Meanpathbot -Mediatoolkitbot -Metauri -MicroMessenger -Microsoft Data Access -Mojeek -Morfeus Fucking Scanner -Mozlila -Mr.4x3 -Msrabot +Mojolicious +NICErsPRO +Navroad Nessus NetSpider NetZIP +Netcraft Netvibes -NextGenSearchBot -Nibbler -NimbleCrawler -Ninja +Nikto +Nutch Octopus +Offline Explorer Offline Navigator -OrangeBot +OpenLinkProfiler +OpenVAS +Openfind +OrangeSpider PHPCrawl -PageThing.com +PageAnalyzer +PageScorer Panscient -Papa Foto -PeoplePal -Pi-Monster -Pimonster -PleaseCrawl -Pockey -Probethenet -Pu_iN -QueryN Metasearch +Pavuk +Petalbot +Picscout +PictureFinder +Piepmatz +ProPowerBot +ProWebWalker +Psbot +PyCurl +Quick-Crawler +RSSingBot RankActive RankFlex RankingBot -RankurBot -RealDownload +Rankivabot +Reaper Recorder -RedesScrapy -Rogerbot -SBIder -SMTBot -ScanAlert -Scanbot +RepoMonkey +SEOkicks +SISTRIX +SalesIntelligent Scrapy -Searchestate -SemanticJuice -SemrushBot -SeobilityBot -Siphon +SearchmetricsBot +Seomoz +Shodan Site Sucker -SiteCheckerBotCrawler -SiteLockSpider -SiteSnagger +SiteExplorer SiteSucker -Sitebeam -Siteimprove Sitevigil -SmartDownload -SocialRankIOBot -Sogou web spider -SpankBot -Spbot -SputnikBot +Snoopy +Sosospider +Spinn3r Sqlmap -Sucker +Sucuri +SuperBot SuperHTTP SurveyBot Suzuran -T0PHackTeam -Telesoft +TeleportPro Telesphorep Thumbor Titan Toata Toweyabot -Tracemyfile Trendiction -True_Robot Turingos -TurnitinBot -TwengaBot -Twice +URLy Warning +URLy.Warning Upflow -VelenPublicWebCrawler +VB Project +Vacuum +VeriCiteCrawler VidibleScraper -VoidEYE +Voil WEBDAV -WWW-Collector-E -WWWOFFLE +WPScan +WWW::Mechanize WallpapersHD +WeSEE Web Auto -Web Collage +Web Fuck Web Sauger WebAuto WebBandit -WebGo IS -WebImageCollector +WebCollage +WebCopier +WebEnhancer +WebFetch WebLeacher WebPix WebReaper -WebSauger +WebStripper +WebSucker +WebWhacker WebZIP -WinHTTrack -Woobot -Xaldon_WebSpider +Webshag +Webster +Whatweb +Wotbox Xenu -YoudaoBot -Zade -Zauba Zermelo -Zeus -Zitebot -ZoominfoBot ZumBot ZyBorg -arquivo.pt -autoemailspider -cah.io.community -clark-crawler -cognitiveseo +adscanner +archive.org_bot +arquivo-web-crawler +crawler.feedback crawler4j -heritrix +demandbase-bot +domainsproject.org +evc-batch +facebookscraper +instabid internetVista monitor ips-agent +lwp-request +meanpathbot netEstate NE Crawler oBot -page scorer -pcBrowser -polaris version -scalaj-http +plumanalytics seocompany.store -serpstatbot -sitechecker.pro -sogouspider -sysscan -trendiction.com +seoscanners +siteripz +spyfu +tAkeOut trendiction.de -webgains-bot -webmeup-crawler +ubermetrics-technologies.com x09Mozilla +zauba.io zgrab diff --git a/dev-tools/test_units/random-bots-for-whitelist-test.list b/dev-tools/test_units/random-bots-for-whitelist-test.list index 04cf514f9..af8710872 100644 --- a/dev-tools/test_units/random-bots-for-whitelist-test.list +++ b/dev-tools/test_units/random-bots-for-whitelist-test.list @@ -1,250 +1,250 @@ -404enemy -80legs -AIBOT -Aboundex -Aboundexbot +01h4x.com +ADmantX +ALittle Client +Abonti Acunetix -AfD-Verbotsverfahren -Alexibot +AhrefsBot +Aipbot +AllSubmitter Alligator -Anarchie -AwarioRssBot -BBBike +AlphaBot +Apexoo +Aspiegel +Attach BDFetch -BackDoorBot +BLEXBot Backlink-Ceck -BacklinkCrawler -Badass -BatchFTP +Battleztar Bazinga +BetaBot +Bitacle Black Hole -BlackWidow -Blackboard +Blow Bolt -BotALot -Brandwatch Buck +Buddy +BuiltBotTough +BunnySlippers CATExplorador +CCBot +CODE87 CSHttp Calculon -CheTeam -Claritybot +CazoodleBot +Cegbfeieh +CensysInspect +CheeseBot +CherryPicker Cloud mapping -Cocolyzebot -Collector -CopyRightCheck -Cosmos +Cogentbot +Copyscape Crawling at Home Project +CrazyWebCrawler +CrunchBot Curious -CyotekWebCopy -DTS Agent -Deusu -Devil -Discoverybot -DittoSpyder +DIIbot +DSearch +DatabaseDriverMysqli +Demon +Digincore +Dirbuster +DnBCrawler-Analytics +DnyzBot +DomCopBot DomainAppender -Dragonfly -Drip +DomainCrawler +DomainSigmaCrawler +DomainStatsBot +Dotbot +EMail Siphon EMail Wolf -EasyDL -Exabot -Ezooms +EroCrawler +Extractor +ExtractorPro +Extreme Picture Finder +EyeNetIE FemtosearchBot -Fimap FlashGet -Flunky -Fuzz +Foobot G-i-g-a-b-o-t GT::WWW GalaxyBot +Genieo +GetRight +GetWeb +Go!Zilla GoZilla -Gotit -GrabNet -Grafula -HMView +GrapeshotCrawler +HTMLparser HTTP::Lite -Haansoft -HaosouSpider -Harvest -Humanlinks HybridBot +IDBTE4M +IDBot IRLbot +Iblog +Id-search +IlseBot Image Fetch Image Sucker -InfoNaviRobot -Intelliseek +Indy Library Internet Ninja Iria +IstellaBot JOC Web Spider -JamesBOT -JennyBot -JetCar +Jbrofuzz Jetty +JikeSpider Joomla -Jyxobot -Keyword Density -Kozmosbot +Jorgee LNSpiderguy LWP::Simple Larbin -LeechFTP -LexiBot Lftp LibWeb Libwhisker Lightspeedsystems +Likse +Linkdexbot +LinqiaMetadataDownloaderBot Lipperhey -Litemage_walker -Lmspider -MFC_Tear_Sample -MIDown tool -MIIxpc -MJ12bot +Ltx71 MQQBrowser MSFrontPage MSIECrawler -Mag-Net -Magnet +MTRobot Mail.RU_Bot -Majestic SEO -Majestic12 +Majestic-SEO +MarkMonitor +MarkWatch +Mass Downloader +MauiBot MeanPath Bot Meanpathbot -Metauri -Microsoft Data Access -Minefield -Mister PiX -Moblie Safari -Morfeus Fucking Scanner -Mozlila -Msrabot -Nameprotect -Navroad +MicroMessenger +Microsoft URL Control +Mr.4x3 +Needle NetAnts NetSpider NetZIP -Netcraft Netvibes -Ninja +NimbleCrawler Not -Nuclei -Octopus Offline Explorer -Offline Navigator -OpenLinkProfiler -OutfoxBot -PECL::HTTP +OnCrawl +Openfind +OrangeBot PHPCrawl -POE-Component-Client-HTTP +Page Analyzer PageAnalyzer -PageScorer -Pandalytics Panscient PeoplePal +Petalbot Pi-Monster -Piepmatz +PictureFinder Pimonster +PleaseCrawl +Pockey +ProPowerBot ProWebWalker -PxBroker +Psbot +Pu_iN PyCurl -Quick-Crawler -RankingBot +QueryN Metasearch +RSSingBot +Re-re ReGet RealDownload -SEOkicks-Robot +Reaper +RebelMouse +Rogerbot +SEOstats SISTRIX -SMTBot +SalesIntelligent ScanAlert -ScoutJet +Scanbot +Scrapy +Screaming ScreenerBot -ScrepyBot -Searchestate -Semrush -SemrushBot -SiteCheckerBotCrawler -SiteSnagger +Seekport +SentiBot +SeoSiteCheckup +Shodan +Site Sucker +SiteLockSpider +Sitebeam Siteimprove -Sitevigil -SmartDownload -Snoopy +Snake +Sociscraper Sottopop -SpaceBison -Spanner Spinn3r +Sqlworm +Steeler Stripper +Sucuri +SuperHTTP Surfbot -Swiftbot -T0PHackTeam -Teleport -TeleportPro -Telesphorep +Suzuran +Telesphoreo +TheNomad +Thumbor Toweyabot -Turingos -TurnitinBot +Tracemyfile +Trendiction +Trendictionbot TwengaBot Typhoeus -URLy Warning -V-BOT VCI +Vacuum Vagabondo -VeriCiteCrawler -VidibleScraper +Voltron WASALive-Bot +WEBDAV WISENutbot -WWW-Collector-E WWW-Mechanize WWW::Mechanize -WWWOFFLE Wallpapers Wallpapers/3.0 -WeSEE -Web Enhancer -Web Fuck +Web Auto +Web Collage Web Pix Web Sauger -WebAuto -WebBandit -WebEnhancer -WebFuck -WebImageCollector -WebSauger -WebSucker -WebmasterWorldForumBot +Web Sucker +WebLeacher +WebPix +WebReaper +WebWhacker +Webshag WebsiteExtractor -WebsiteQuester Webster -Whack -Who.is Bot -Widow -Woobot +WinHTTrack +Xaldon WebSpider +Xaldon_WebSpider +Xenu YoudaoBot Zade -Zauba Zeus Zitebot -ZmEu -ZoomBot -ZoominfoBot -ZumBot +ZyBorg adscanner +backlink-check cah.io.community coccocbot-web cognitiveseo -crawler.feedback +crawler4j +domainsproject.org eCatch +evc-batch +gopher internetVista monitor -ips-agent -isitwp.com -magpie-crawler meanpathbot -muhstik-scan -netEstate NE Crawler -oBot -pcBrowser -polaris version -probe-image-size -ripz -seoscanners -seostar -siteripz +plumanalytics +satoristudio.net +scalaj-http +seocompany.store +sogouspider +spyfu tAkeOut -x09Mozilla +trendiction.com +trendiction.de +webpros.com +zgrab diff --git a/dev-tools/test_units/random-referrers-for-test-quick.list b/dev-tools/test_units/random-referrers-for-test-quick.list index 8bacbb16f..77af2b05b 100644 --- a/dev-tools/test_units/random-referrers-for-test-quick.list +++ b/dev-tools/test_units/random-referrers-for-test-quick.list @@ -1,10 +1,10 @@ -1millionusd.xyz -cartierbracelet.xyz -gigixo.com -i-service.kz -konoplisemena.com -m0r0zk0-krava.ru -medmajor.ru -play.leadzupc.com -swinger-mobil.net -touchmods.fr +28n2gl3wfyb0.ru +cindymatches.com +dekorkeramik.ru +panamaforbeginners.com +pogodnyyeavarii.gq +sluganarodu.ru +vladhistory.com +watracker.net +xn--80ak6aa92e.com +xrus.org diff --git a/dev-tools/test_units/random-referrers-for-test.list b/dev-tools/test_units/random-referrers-for-test.list index dad0b63cc..fafd453ae 100644 --- a/dev-tools/test_units/random-referrers-for-test.list +++ b/dev-tools/test_units/random-referrers-for-test.list @@ -1,500 +1,500 @@ -007angels.com -0912701309f8ce.com 11235813.webzdarma.cz -4kepics.com -4replicawatch.net -4webmasters.org -6hopping.com -76brighton.co.uk -absolute-s.ru -absugars.com -academiacsmendoza.org -actulite.com -adultfullhd.com -aero2.ru +1314dh.com +178evakuator178.ru +1qingdao.com +24x7-server-support.site +2rich4bitches.com +40cg.com +8jn.dating-app.ru +91abcw.com +991mostfm.co.id +999webdesign.com +a96527gi.beget.tech +aadroid.net +acortarurl.es +ad-words.ru +adsland.men +affordablewebsitesandmobileapps.com +agapovdg.ru aksonural.ru -akusherok.ru -aliexpresscashback.club -allnews24.in +akuhni.by +alert-fjg.xyz +alienwheel.es +allbizne.co.ua allsilver925.co.il -alyeskaresort.com -amanda-porn.ga -amigobulls.com +am-se.com +ameblo.jp androids-store.com -anonymousfox.co -apishops.ru +animaltoplist.com +animenime.ru +apiallgeniusinfo-a.akamaihd.net +apioutoboxnet-a.akamaihd.net appartement-stumm.at -apple.com-webbrowsing-security.review apple.com-webbrowsing-security.science -arraty.altervista.org -ausergrubhof.info -autobudpostach.club -autodo.info -autoplate.info -aviav.eu +applicationg29.com +appmsr.org +arcadeplayhouse.com +arkartex.ru +ascat.porn +autoloans.com +autonew.biz +aversis.net +avon-novosib.ru +avtolombard-krasnodar.com +azazaporn.com +azazu.ru azte.ch -b00kmarks.com -bala.getenjoyment.net -bangla-chat-uk-paralud.ga -barnfurnituremart.com -beauty-things.com -beclean-nn.ru -bedandbreakfast.com -best-games.today -bestempresas.es -bibys.com -bildsuche.ru -bio-japan.net -bjgugu.net.ua -blobar.org -blog.koreadaily.com -blogig.org -blpmovies.com -bmwhighperformers.com -bobba.dzaba.com -bodybuilding-shop.biz -bolitgorlo.net -brk-rti.ru -builtwith.com -buketeg.xyz -buntube.net -burger-imperia.com -burn-fat.ga -buy-cheap-online.info -buytizanidineonline.blogspot.com -bycontext.com -c1.onedmp.com -cacheimages.com -calcularpagerank.com -camdolls.com -cartierbracelet.xyz -cartierlovestore.com -cbox.ws -cdncash.com -chatroulette.si -chee-by.biz -cigarpass.com -cinemaflix.website -cjs.com.ru -clicksor.com -clips.ua.ac.be -co.lumb.co +backlinkwatch.com +baltgem.com +banan.tv +banksrf.ru +bannerswap.com +batanga.net +battle.net +battlecarnival.su +bausparen.bz.it +beatifullandscape.co +best-coupon-offer.com +betune.onlinewebshop.net +bif-ru.info +bio-market.kz +blackcurranthumidifiers.site +blog.f00kclan.de +bloggen.be +blogqpot.com +bnt-team.com +bonanza-fortune.men +brateg.xyz +btvn.ru +buyantiviralwp.com +buydissertation.net +buyviagraa.blogspot.com +cablecar.us +cah.io.community +calc-for-credit.ru +candlewooddentalcentre.com.au +canzoni.ru +capture-room.com +cardsharp1.ru +carmuffler.net +carstrends2015.com +cartierring.xyz +cashkitten-a.akamaihd.net +cashonads.com +catalogs-parts.com +cayado.snn.gr +cezartabac.ro +cfsrating.sonicwall.com +cheapmoncler.pw +checktext.org +chocolateslim-u-srbiji.com +christianlouboutinoutlet.win +cindymatches.com +clevernt.com +click2pawn.com +clipartnew.com +clknsee.com +clksite.com +clothesforcash.com +codysbbq.com comeondog.info -comissionka.net -compliance-elena.xyz -conboy.us -connectionstrenth.com -consorzioilmosaico.org -contextualyield.com -copro.pw -craftburg.ru -creams.makeforum.eu -creatives.adbetclickin.pink -czat.wp.pl -date-withme.com -dayibiao.com -ddsoldes.fr -dear-diary.ru -dgfitness.co -diesel-parts28.ru -digest-project.ru -dikx.gdn -dimkino.ru -dipstar.org -directrev.com -doktoronline.no -dokumentalkino.net -domain-submit.info -domainsphoto.com -domcran.net -donvito.unas.cz -dood.live -downloaddy.net -dpihatinh.gov.vn -draniki.org -droidlook.net -drugspowerstore.com -drugstoreforyou.com -dscaas.website -duawitchrarato.tk +compliance-alexa.xyz +compliance-don.top +compliance-margo.top +cool-wedding.net +corta.co +crackzplanet.com +creativehutindia.com +cvety24.by +dandiyabeats.in +darmebel.com.ua +datadepths.com +daymusam.com +db.speedup.gdn +degerlund.net +dekorkeramik.ru +delitime.info +deluxedumps.com +derevesendeco.com +designdevise.com +detskie-konstruktory.ru +dev-seo.blog +diarioaconcagua.com +dickssportinggoods.com +dikqlyremy.info +dispo.de +disruptingdinnerparties.com +divci-hry.info +dmmspy.com +dodlive.mil +dojki365.online +domainsatcost.com +drupa.com +dvd-famille.com dyt.net -ec-file.info -ecommerce-seo.com -edwinkonijn.com.au -elektrischezigarette2.onsugar.com -elektronischezigarette2.mex.tl -energydiet-info.ru -enews.tech -eralph.tk -erot.co -erotag.com -es5.com -everflora.ru -exhibitionplus.eu -extstat.com -ezigarettenshop2.postbit.com -f1nder.org -failingmarriege.blogspot.com -falopicm.pw -fashionavenuegame.com -fealq.com -filesdatabase.net -filesmonster.porn -filkhbr.com -filmci.pro +e-buyeasy.com +earncash.com.ua +ecommerce-seo.org +ednorxmed.com +educatemetv.com +ee77ee.com +elektrischezigarette2.devhub.com +elektrozigarette1.dreamwidth.org +elektrozigarettekaufen2.blogse.nl +elite-sex-finders.com +elmacho.xyz +elvenar.com +en.altezza.travel +enbersoft.com +energy-ua.com +eorogo.top +escort-russian.com +eu-cookie-law.info +event-tracking.com +everypony.ru +expertblog.info +eyes-on-you.ga +ezigarettekaufen2.blox.pl +ezigarettenshop1.yolasite.com +ezrvrentals.com +f0815.de +face.hostingx.eu +familienzahnaerzte.com +fantasticpixcool.com +feeriaclub.ru +ferretsoft.com +ff30236ddef1465f88547e760973d70a.com +filmania-x.ru +filmfanatic.com +financehint.eu finansov.info -findpics.pw -finteks.ru -fix-website-errors.com -forms-mtm.ru +finstroy.net +fishtauto.ru +floridahuntingfishingadventures.com +foodcrafts.website +foodgid.net forums.d2jsp.org -free-gluten.ru -freemags.cc -front.ru -fungamelands.com -galaxy-family.ru -gallerylisting.com -gamingspark.com -geoads.com -gfaq.ru -gg.zzyjxs.com -gifspics.com -gigapeta.com -gok-kasten.net -goldadpremium.com -goldandcard.ru +freenode.info +freshdz.com +froggytube.com +fullfileaccess.com +fx-brokers-review.com +g7m.pl +gambarkatabaru.com +game300.ru +gateway.zscalerone.net +gaz-voshod.ru +gazoblok.net.ua +ge0ip.org +gearcraft.us +genericviagrasildenafiled.net +germes-trans.com +glcomputers.ru +goldpanningtools.com +golfresa.lucania.se googleseo.com.tr -green-tea.tv -growboxbl.ru -h2monline.com -hd720kino.ru -helicalpile.us -hermesreplica.win -heroz.fr -hostnow.men -hotchatdate.com +gowreckdiving.com +gratuitbaise.com +greatzip.com +groupmoney.ru +gsasearchengineranker.top +gta-top.ru +hasfun.com +havepussy.com +hdfreeporno.net +hmmm.cz +hobbyhuren-datenbank.com +home-task.com +homeandhealth.ru +homelygarden.com +hop.clickbank.net hotdl.in -hothot.ru -hspline.com -hyhj.info -hystersister.com -i-hobot.ru +hotelcrocenzi.sm +hoverboard360.de iblogpress.xyz -ifmo.ru -images.gyffu.com -in-tandem.co -indiakino.net -indianmedicaltourismshop.com -insomniagamingfestival.com -interfucks.net -iptool.xyz -ivanstroi.ru +ideashome.id +igrovyeavtomaty777.ru +iinstalll-fii1leis.jus0wil.pp.ua +ilmen.net +imagez.co +imgarit.pw +incep.imagine-ex.co +incest-ru.com +infowarcraft.ru +innoslicon.com +intimshop-fantasy.ru +ipchicken.com +isistaylorporn.info +jacago.com +jackwolfskinoutlet.online jagg.info -jeremyeaton.co -jerseysbizwholesalecheap.com -jewelryandfiligree.com -justkillingti.me +jbl-charge.info +jennyfire.ru +jerseyschinabizwholesale.us +jerseywholesaleelitestore.com +jobmarket.com.ua +jose.mulinohouse.co jyrxd.com -kanimage.com -karpun-iris.ru -karting196.ru -kiwe-analytics.com -konteiner24.com -krafte.ru -kupit-adenu.ru +kalandranis.gr +karadene.com +kareliatobacco.ru +katadhin.co +kazan.zrus.org +keki.info +kerch.site +kerei.ru +keyword-suggestions.com +kfon.eu +kino-key.info +kleine-titten.biz +kmgamex.cf +kndxbkdx.bloger.index.hr +kometars.xyz +konica.kz +krasivye-devushki.net +kumuk.info kvartira-sutochno.com -la-fa.ru -lapolis.it -law-check-nine.xyz -law-check-seven.xyz -lazy-z.com -lechenie-gemorroya.com -lechtaczka.net -lesbian.xyz -lignofix.ua -link.ac +larose.jb2c.me +layola.biz.tc +lernur.net +levaquin750mg.blogspot.com +limads.men +limtu.ifmo.ru linkrr.com -lion.misba.us -liupis.com -livingcanarias.com -llastbuy.ru -lomb.co -lovi-moment.com.ua -luckyshop.net.ua -m-google.xyz -m.b00kmarks.com -magento-crew.net -makedo.ru +livejasmin.com +livesex-amateure.info +lockerz.com +locustdesign.co +lulea-auktionsverk.se +macdamaged.space +makemoneyonline.com malls.com -marmitaco.cat -marwer.info -masturbate.co.uk -mathgym.com.au -matsdale.com -mbiologi.ru -mebel-vstroika.ru -mebelcomplekt.ru -meccadumps.net -megakino.net -mhi-systems.ru -mikrobiologies.ru -minneapoliscopiers.com -misslike.ru -mlf.hordo.win -mnogolok.info -mobplayer.ru -mole.pluto.ro -monclerjacketsoutlet.win -moncleroutletonline.xyz -montredemarque.nl -moz.com -mp3ringtone.info -msk-diplomat.com -muschisexbilder.com -musezone.ru -musicktab.com -mybackgroundlandscape.blogspot.com -mydeathspace.com -myftpupload.com -myyour.eu -napalm51.nut.cc -naperehresti.info -nardulan.com -new-apps.ru -normalegal.ru -novodigs.com -npoet.ru -nudo.ca -nzfilecloud.weebly.com -o00.in -obesidadealgarve.com -obiavo.in -offf.info -ohmyrings.com -online-podarki.com -online-templatestore.com -onlinefilmz.net -onstrapon.purplesphere.in -orenburg-gsm.ru -orhonit.com -paclitor.com -paintingplanet.ru -pandarastore.top -paparazzistudios.com.au -parajumpersstore.online -parlament.biz -partner-stop.men -pcboa.se -pensplan.com -pepperstyle.ru -perfection-pleasure.ru -photochki.com -pic2fly.com -pinkduck.ga -pirateday.ru -pistonclasico.com -pk-services.ru -pladform.ru +mantramusic.ru +marcogrup.com +masterseek.com +mature.free-websites.com +medicovi.com +medosmotr-ufa.ru +meetingrainstorm.bid +mejoresfotos.eu +member-quiz.com +memberty.com +meteocast.net +meteostate.com +midst.eu +mindeyegames.com +minecraft-neo.ru +mompussy.net +monclervests.xyz +moomi-daeri.com +more-letom.ru +most-kerch.org +mouselink.co +musflashtv.com +musicpro.monster +musicspire.online +muztops.ru +myonigroup.com +myperiod.club +na15.ru +naturalpharm.com.ua +nina.az +nrv.co.za +nsatc.net +nw-servis.ru +obiavo.ru +ochistka-stokov.ru +offer.camp +offside2.5v.pl +oivcvx.website +okout.ru +old-rock.com +omgtnc.com +omoikiri-japan.ru +onefilms.net +oranga.host.sk +orsonet.ru +paccohichetoti.ml +pastaleads.com +pewit.pw +phpdevops.com +picurams.pw +pintattoos.com +pinwallpaper.top +pisanieprac.info +pix-hd.com +pl.aasoldes.fr +placid-rounded-coast.glitch.me platezhka.net -play-movie.pl -plusnetwork.com -pogodnyyeavarii.gq -pokemongooo.ml -pornhub-forum.ga -pornmania.pl -porno-video-chati.ru -pornoklad.net -pornokorol.com -portside.xyz -prefersurvey.net -pretty-mart.com -priceg.com -privetsochi.ru +playlott.com +pod-muzyku.club +podshipniki-nsk.ru +polybuild.ru +portadd.men +predominant-invent.tk +privatov-zapisi.ru +privhosting.com prize44.com -prizefestival.mobi -prod2016.com -profitkode.com -proftests.net -progress-upakovka.ru -projectforte.ru -prom23.ru -prostitutki-rostova.org -prpops.com +prodess.ru +promalp-universal.ru +prostitutki-yaroslavlya.org +proxyradar.com +push-ad.com putevka24.ru -qualitymarketzone.com -quick-seeker.com -ranksonic.com -razleton.com -refererx.com -regionshop.biz -remedyotc.com +qcstrtvt.bloger.index.hr +qexyfu.bugs3.com +qitt.ru +questionmarque.ch +rankia.com +registry-cleaner.net +registrysweeper.com +reining.lovasszovetseg.hu +reklama1.ru +remont-mobile-phones.ru +remontvsamare.su +rentalcarnavi.info resultshub-a.akamaihd.net -retailwith.com -reyel1985.webnode.fr +retreatia.com +revistaindustria.com +rfserial.net +rightenergysolutions.com.au rimedia.org -ritlweb.com -rniaeba.ga -robotixix.com -rocis.site -rockma.se -roma-kukareku.livejournal.com -sa-live.com -sabaapress.com -samo-soznanie.ru -sampleletters.net -saneitconsulting.com -saneyes.com -sanjosestartups.com -saugatuck.com -scanner-margo.top -scenarii-1-sentyabrya.uroki.org.ua -scrnet.biz.ua -secret.xn--oogle-wmc.com -semalt.com -sendearnings.com -seoboxes.com -seolab.top -serptehnika.ru -service.adtech.us -sexpartygirls.net -sfj-ror.no -sharebutton.to -sheerseo.com -shtaketniki.ru +ruclicks.com +rvtv.ru +saecsa.co +salutmontreal.com +santasgift.ml +sarf3omlat.com +sbetodiodnye-lampy.ru +sbtdesign.co.uk +sbwealthsolutions.ca +scanhub.ru +scanner-alex.top +scanner-john.top +scrapinghub.com +scrapy.org +sdelatmebel.ru +searchimage.co +searchinterneat-a.akamaihd.net +seksotur.ru +seonetwizard.com +serpstat.com +servisural.ru +sexcamamateurchat.com +sexvporno.ru +sharebutton.net +sharpchallenge.com +shikiso.info +ship-marvel.co.ua +shop.xz618.com +shopfishing.com.ua +shoppingjequiti.com.br sibtest.ru -simplepooltips.com -simul.co -sisi-go.ru -skanninge.se -smart-scripts.com -smartpet.ru +sicfor.bcu.cc +sideeffectsoftizanidine.blogspot.com +sildenafilcitratemed.com +simple-share-buttons.com +sims-sims.ru +sitevalued.com +skuteczna-dieta.co.pl +slavic-magic.ru +sledstvie-veli.net +smart-balancewheel.com +snip.tw snow.nvr163.com -snts.shell-pmr.ru -socialmonkee.com +social-fun.ru +social-s-ggg.xyz +social-s-iii.xyz +socialmediasuggest.com +socialsignals24.com +soft-terminal.ru +soft1.ru +soheavyblog.com +solinf.co +solmarket.by +soochi.co +space-worry.ml space2019.top -sportbetfair.com -sps-shop.com -spy-app.info -spymac.net -stariy-baku.com -starpages.net -strangeduckfilms.com -strfls.com -stroy-portal22.ru -studentguide.ru +squidoo.com +stroymonolit.su +stroyplus.ru +stuff-about-money.com subj.ukr-lit.com -supers.com.ua -suralink.com -sureone.pro -surflinksmedical.com -suture.co -svtrd.com -swsociety.se -tacbelarus.ru -tagil.zrus.org -talant-factory.ru +suchenindeutschland.com +sukirgenk.dvrlists.com +survival.betteroffers.review +svbur.ru +t-bygg.com tamada69.com -taximytishi.ru -tdsing.ru -tehngr.ru -terrafootwear.us -thaismartloan.com -the-torrent-tracker.blogspot.com -the-usa-games.blogspot.com -thebestweightlosspills.ovh -thebluenoodle.com -thegameriders.com -thepantonpractice.co.uk -timeallnews.ru -timhost.ru -tizanidineandcipro.blogspot.com -tizanidinehcl2mgtablet.blogspot.com -tizanidinehcl4mgtablet.blogspot.com -tizanidineonline.blogspot.com -tizanidineoral.blogspot.com -tizanidinewithdrawal.blogspot.com -toondinsey.com -top10registrycleaners.com -totu.info -toy-shop.top -tozup.com -tpu.ru -tracfone.com -trackmedia101.com -trafficfactory.biz -trafficgenius.xyz -trafficmania.com -traxdom.ru -truemfilelj.gq -trustl.life -tuberkulezanet.ru -tupper-shop.ru -twincitiescarservice.com -uchetunet.su -uginekologa.com -unblocksit.es -undergroundcityphoto.com -universals.com.ua -uprour.com -uptime-delta.net -uptime-gamma.net -urccvfmc.bloger.index.hr -urengoy.pro -ustion.ru -uvozdeckych.info -v-doc.co -valmetrundan.se -vandrie-ict.nl +technika-remont.ru +teenbbw.yopoint.in +theautoprofit.ml +theheroes.ru +timecrimea.ru +tizanidinehcl4mgtabinfo.blogspot.com +tizanidinehclsideeffects.blogspot.com +tizanidineinfo.blogspot.com +tizanidinerecreationaluse.blogspot.com +tizanidinevssoma.blogspot.com +top-l2.com +topflownews.com +torrentdownloadhub.com +tourismvictoria.com +traffictrade.life +traffique.net +traffpartners.com +tramadolandtizanidine.blogspot.com +tryrating.com +tubo360.com +ukkelberg.no +umnovocaminho.com +unrealcommander.com +upproar.com +uroffer.link +vabasa.inwtrade.com veerotech.com -vegascosmetics.ru -vesnatehno.com -videochat.ph -videochat.world -virtuagirl.com -vseigru.one -vzlomsn.org +vegan-foods.us +velen.io +venta-prom.ru +ventelnos.com +video-camer.com +video-chat.love +vip-parfumeria.ru +vipsexfinders.com +vkmusics.ru +voloslove.ru +voluumtrk.com +voronezh.xrus.org w3data.co -wait3sec.org -wanker.us -webalan.ru -webcamdevochka.com -weclipart.com -weightatraining.com -westum.se -wildnatureimages.com -wjgony.com -worldhistory.biz -worldmusicfests.com -worldoffiles.ru -www.labves.ru -www.wohnkabinen-shop.de -wygraj-teraz.com -x-diesel.org -xaijo.com -xbaboon.com -xblog.in -xblognetwork.com -xlolitka.com -xmlinde.com -xn----7sbahjd3btneuw1joc.xn--p1ai -xn--80aafb2a.xn--p1ai -xn--80aeahghtf8ac5i.xn--p1ai -xn--hxazdsfy.blogspot.com -xvideospornoru.com +wallpaperaccess.com +walpaperlist.com +waterefficiency.co +website-analyzer.info +websites-reviews.com +wejdz-tu.pl +wellcome2slovenia.ru +westvilletowingservices.co.za +wholesalejerseychinashop.com +wholinkstome.com +writersgroup580.web.fc2.com +wsgames.ru +x-diesel.com +xclicks.net +xgftnlrt.bloger.index.hr +xitjw.info +xjlottery.com +xn-----8kcatubaocd1bneepefojs1h2e.xn--p1ai +xn----8sbarihbihxpxqgaf0g1e.xn--80adxhks +xn--lifehacer-1rb.com +xtubeporno.net +xxxdatinglocal.us yebocasino.co.za -yottos.com -yourmovies.pl -youtube-downloader.savetubevideo.com -yun56.co -za-music.mymobiplanet.com -zahvat.ru -zaloadi.ru -zdesformula.ru -zebramart.ru -zigarettenonlinekaufen2.bloog.pl -znakomstva-piter78.ru -zoogdiesney.com -zoominfo.com -zscaler.net -zvetki.ru -zzlgxh.com +yellowstonesafaritours.com +yoopsie.com +youraticles.pl +yourporngay.com +youtoner.it +ypmuseum.ru +zdesoboi.com +zigarettenonlinekaufen1.bloog.pl +zrelaya.pw diff --git a/dev-tools/test_units/random-referrers-for-whitelist-test.list b/dev-tools/test_units/random-referrers-for-whitelist-test.list index 922a53bde..068030095 100644 --- a/dev-tools/test_units/random-referrers-for-whitelist-test.list +++ b/dev-tools/test_units/random-referrers-for-whitelist-test.list @@ -1,250 +1,250 @@ -2daytrendingnews.com -2rich4bitches.com -40cg.com -51unlim.ru -5ws.dating-app.ru -9med.net -aanapa.ru -academia-nsk.org -adcash.com -adclickservice.com -adf.ly -ads-seo.men -adultfriendfinder.com -affordablewebsitesandmobileapps.com -albamargroup.com -ald2014.org -animal36.com -appsaurus.com -aruplighting.com -avkzarabotok.com -avtocredit-legko.ru -azartmsl.com -bambi.ck.ua -bestssaker.com -beta.hotkeys.com -blackcurranthumidifiers.site -blpmovies.com +1flag.co.za +2000k.ru +300richmond.co.nz +45en.ru +4webmasters.org +aac52006ec82a24e08b665f4db2b5013f7.com +abwa.tk +actionnooz.com +adelly.bg +agardomains.com +albuteroli.com +alienwheel.es +alif-ba-ta.com +amateurgalls.com +anal-acrobats.hol.es +anticrawler.org +apiholdingmypage-a.akamaihd.net +apimegabrowsebiz-a.akamaihd.net +approved.su +arewater.com +arius.tech +artpicso.com +ausergrubhof.info +becuo.com +best-seo-offer.com +bestwebsiteawards.com +bet365.com +bizcheapjerseyswholesalechina.com +blog.axant.it +bloggerads.net boanonihaca.tk -boltalko.xyz -bonzibuddi.com -breget74.ru -burkesales.com -buy-cheap-pills-order-online.com -californianews.cf -cannibalcheerleader.com -carloans.com -cars-modification.net -cartierlove2u.com -cenokos.ru -chatroulette.online -chorus.terakeet.com -citizenclsdriveri7.pen.io -cityadspix.com -claytransformations.info +boc.kz +braindaily.xyz +braip.com.br +bwlx.prepedu.cn +carmuffler.net +casinos4dummies.co.uk +cheapmoncler.win +chiblackhawks-jerseys.com +cialisndbrx.com +circlesl.com +clickhype.com +com-supportcenter.website communisave.co.za -community.allhiphop.com -concordexoticrentals.com -cordstrap.cc -cxpromote.com -dafatiri.com -dailyfinancefix.com -daisye.top -danashop.ru +compliance-alexa.top +compliance-don.xyz +contentlook.co +crackguru.tk +customcollegeessays.net +customerpromos-a.akamaihd.net demian.kz -divatraffic.com -domaincdn.xyz -driving.kiev.ua -dtm-spain.com -dvrlists.com -egvar.net -elektronischezigarettekaufen1.myblog.de -emediate.eu -envaseslotusama.com -eorogo.top -evening-dating-club.info -expolicenciaslatam.co -extremepornos.net -eyemagination.com -failingmarriege.blogspot.com -fetishinside.com -fetlifeblog.com -fil.ru -firma-legion.ru -firstdrugmall.ru -forexgb.ru -forodvd.com -forsex.info -free-traffic.xyz -freewebs.com -fungirlsgames.net -gambarxkata.co -gamezblox.com -gay-site.store -gelezki.com -generic-pills-online.com -get-your-social-buttons.info -getoutofdebtfree.org -ggiaro.com -gilsonchiro.xyz -girlspicsa.com -goblacked.com -golden-praga.ru -googleseo.com.tr -grtyi.com -handsandlegs.ru -homelygarden.com -hotcore.info -hotgirlhdwallpaper.com -housemilan.ru -hspline.com +doctissimo.fr +e-collantes.com +e90post.com +earn-from-articles.com +editmedios.com +ekto.ee +el-nation.com +elektrischezigaretten1.blogse.nl +elmifarhangi.com +eropho.net +erotik0049.com +european-torches.ru +euroskat.ru +excitacao.com +factorynightclub.com +faz99.com +feeriaclub.ru +financeloan.us +findthe.pet +florida-tourism.net +fm-upgrade.ru +footbalive.org +free-floating-buttons.com +fz139.ttk.ru +gallery.rennlist.com +game-top.su +gaming-journal.com +ge0ip.com +geilehausfrauen.net +given2.com +gk-atlant.info +globalscam.ga +gok-kasten.net +goodnightjournal.com +gosreg.amchs.ru +green-tea.tv +growshop.es +gsasearchengineranker.space +gunsvicceadadebt.tk +guruofcasino.com +hermesbelts.xyz +hesteel.pl +home-task.com +hosting-tracker.com +hothor.se huimin764128.com -i-midias.net.br -iamsport.org -imguramx.pw -immobiliaremassaro.com -importchinacoach-teach.com -inbabes.sexushost.com -incest-ru.com -infazavr.ru -inmate-locator.us -investsuccess.org -ispac.org -itsdp3.com -japan-bearings.ru -jerseywholesalebizchina.com -jerseywholesaleelitestore.com -joy-penguin.com -kashubadesign.ru -kazinogames.lv -ketoanhanoi.info -keywordsdoctor.com -keywordsuggest.org -kino-rf.ru -kinotorka.ru -lawyersinfo.org -liveinternet.ro -lostaruban.com -lottospring.com -magnetic-bracelets.ru -mbiologi.ru -medanestesia.ru -microstatic.pl -mini.7zap.com -misswell.net -mixed-wrestling.ru -mobile-appster.ru -moneyteam24.com -movies-in-theaters.net -mp3ritm.top -musicstock.me -mycaf.it -mylida.org -myplaycity.com -nakrutka.cc -net-radar.com -niroo.info -nofreezingmac.click -o-dachnik.ru -ochistka-stokov.ru -okonich.com.ua -okuos.com -oops-cinema.ru -orenburg-gsm.ru +ifirestarter.ru +iframe-toloka.com +ihtec2019.org +ilmexico.com +import-sales.com +increasewwwtraffic.info +informatiecentro.be +irunfar.com +jose.mulinohouse.co +katadhin.co +katushka.net +kiinomaniak.pl +kiskinhouse.com +kvartir-remont.biz +lacave.ntic.fr +lanadelreyfans.us +lapitec.eu +laudit.ru +lawyers.cafe +leadwayau.com +letolove.ru +lexixxx.com +livingroomdecor.info +magicdiet.gq +mb140.ru +medicovi.com +medkritika.ru +misandesign.se +missclub.info +mlvc4zzw.space +modenamebel.ru +moneyviking-a.akamaihd.net +mosrif.ru +mrpornogratis.xxx +mydirtystuff.com +myhydros.org +mzdish.site +naturalpharm.com.ua +netvouz.com +nicefloor.co.uk +nikhilbahl.com +nisuturnetdgu.tk +nodup.ru +norththeface.store +novgorod.xrus.org +okeinfo.online orhonit.com -orion-v.com -pekori.to -poiskzakona.ru -poptool.net -porn-w.org -pornosexrolik.com -portadd.men -portal-eu.ru -priscilarodrigues.com.br -profit-opportunity.com -profitkode.com -progress-upakovka.ru -prokotov.com -proposal-engine.com -radiodigital.co -rebelmouse.com -rfd-split.hr +parfusale.se +patol01.pw +pdns.download +perfectpracticeweb.com +photo.houseofgaga.ru +picphotos.net +piulatte.cz +pixelrz.com +pk-pomosch.ru +plastweb.ru +play-mp3.com +plohaya-kreditnaya-istoriya.ru +pochtovyi-index.ru +porno-chaman.info +poshiv-chehol.ru +prizeestates.cricket +projectforte.ru +ratemodels.net +regionshop.biz +registrationdomainsite.com +reklamuss.ru +remybutler.fr ribieiendom.no -ring4rhino.com -rockprogblog.com -rybalka-opt.ru +ringporno.com +royal-investments.net +ruex.org.ua +rutor.vip +salmonfishingsacramentoriver.com samoiedo.it -savingsslider-a.akamaihd.net -seo-traffic-ranking.info -seonetwizard.com -seozoom.it -servicecenter.co.ua -servisural.ru -sexstream.pl -sibdevice.ru -site-auditor.online -sizeplus.work -sotechco.co -spymac.net -sribno.net -stop-gepatit.te.ua -sumo.com -supervesti.ru -svarbit.com -symphonyintegratedhealthcare.com -tagil.zrus.org -taraz.xkaz.org -taxi-v-eisk.ru -tehngr.ru -teslathemes.com -tizanidinehcl4mgtabinfo.blogspot.com -tizanidinemusclerelaxant.blogspot.com -tour-line.net -triplepanda.xyz +sbornik-zakonov.ru +scanner-elena.top +scanner-ivan.top +sdsjweb.com +searchimpression.com +semprofile.com +serpstat.com +sexvporno.ru +sexy-screen-savers.com +sim-service.net +simplesite.com +skanninge.se +smartshoppymac.com +snjatie-geroinovoy-lomki.ru +snts.shell-pmr.ru +soblaznu.net +social-button.xyz +socialmadesimple.com +softonicads.com +spylog.com +spywarebegone.com +squidoo.com +studiofmp.com +success-seo.com +summerlinhomes411.info +sygraem.com +tarad.com +taranerymagesswa.blogspot.com +tattomedia.com +theguardlan.com +thenews-today.info +threecolumnblogger.com +tivolibasket.it +topclickguru.com +topdownloads.ru +tripper.de trubywriting.com -tupper-shop.ru -u17795.netangels.ru -uchil.net +trustaffs.com +tuckermktg.com +turbabitload.weebly.com +turbo-suslik.org +ugogo.info uhodzalijami.ru -ukrtvory.in.ua -umekana.ru -uptimebot.net -utrolive.ru -valmetrundan.se -vapsy.com -varbergsvind.se -venerologiya.com -vereo.eu -via-energy-acquistare.com -vigrx-original.ru -vitoriacabos.com -voluumtrk.com -vriel.batcave.net -vykup-avto-krasnodar.ru +umityangin.net +urlcut.ru +vapomnoncri.tk +viagroid.ru +video-chat.love +videochat.cafe +vip-parfumeria.ru +virus-respirators.com +viveresaniesnelli.it +voloslove.ru +volume-pills.biz +voronezh.xrus.org wardreapptokone.tk -warezkeeper.com -webix.biz -webix.me -website-analyzer.info -website-speed-check.site +waterpurifier.club website-speed-up.site +wesharepics.info +westen-z.life whengirlsgowild.com -winterclassichockeyjerseys.com -wiosenny-bon-1500.pl -wladimirpayen.com -wleuaprpxuvr.ga -wordpresscore.com -xclicks.net -xn----7sbifcamovvfggw9d.xn--p1ai -xn----8sblgmbj1a1bk8l.xn----161-4vemb6cjl7anbaea3afninj.xn--p1ai -xn----itbkqkfiq.xn--p1ai -xn--80aagddcgkbcqbad7amllnejg6dya.xn--p1ai -yaderenergy.ru -yoursearch.me -youtubologia.it -zastroyka.org -zigarettenonlinekaufen2.drupalgardens.com -zona-aqua.ru -zoompegs.com -zoosexart.com +wholesalenfljerseys.us.com +wingsofrefuge.net +woman-tampon.ru +womensterritory.ru +worst-sites.online +wstroika.ru +x-lime.com +xkaz.org +yhit.press +youdao.com +yourtemplatefinder.com +z-master.ru +zhacker.net +znaniyapolza.ru +zophim.me zs2vm.top diff --git a/dev-tools/test_units/ratelimittest-master.txt b/dev-tools/test_units/ratelimittest-master.txt index 3656720f2..b2122ba99 100644 --- a/dev-tools/test_units/ratelimittest-master.txt +++ b/dev-tools/test_units/ratelimittest-master.txt @@ -1,21 +1,11 @@ - - - -Welcome to NginX - - -
-

Welcome to the NginX Ultimate Bad Bot Blocker

-

This is merely used for testing !!!

- - + 503 Service Temporarily Unavailable

503 Service Temporarily Unavailable

nginx/1.18.0 (Ubuntu)
- +ml> 503 Service Temporarily Unavailable

503 Service Temporarily Unavailable

diff --git a/dev-tools/test_units/whitelist-ips.conf b/dev-tools/test_units/whitelist-ips.conf index 447da2bf0..3951616fb 100644 --- a/dev-tools/test_units/whitelist-ips.conf +++ b/dev-tools/test_units/whitelist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.69 0; +10.1.0.222 0; 127.0.0.1 0; diff --git a/dev-tools/tmp/robots.txt b/dev-tools/tmp/robots.txt index 1c315205b..ba480a4e1 100644 --- a/dev-tools/tmp/robots.txt +++ b/dev-tools/tmp/robots.txt @@ -6,9 +6,9 @@ ### Version Information # ################################################### -### Version: V4.2022.01.2840 -### Updated: Sun Jan 9 06:33:53 UTC 2022 -### Bad Bot Count: 631 +### Version: V4.2022.01.2841 +### Updated: Sun Jan 9 07:00:24 UTC 2022 +### Bad Bot Count: 632 ################################################### ### Version Information ## @@ -817,6 +817,8 @@ User-agent: Searchestate Disallow:/ User-agent: SearchmetricsBot Disallow:/ +User-agent: Seekport +Disallow:/ User-agent: SemanticJuice Disallow:/ User-agent: Semrush diff --git a/robots.txt/robots.txt b/robots.txt/robots.txt index 1c315205b..ba480a4e1 100644 --- a/robots.txt/robots.txt +++ b/robots.txt/robots.txt @@ -6,9 +6,9 @@ ### Version Information # ################################################### -### Version: V4.2022.01.2840 -### Updated: Sun Jan 9 06:33:53 UTC 2022 -### Bad Bot Count: 631 +### Version: V4.2022.01.2841 +### Updated: Sun Jan 9 07:00:24 UTC 2022 +### Bad Bot Count: 632 ################################################### ### Version Information ## @@ -817,6 +817,8 @@ User-agent: Searchestate Disallow:/ User-agent: SearchmetricsBot Disallow:/ +User-agent: Seekport +Disallow:/ User-agent: SemanticJuice Disallow:/ User-agent: Semrush