diff --git a/AUTO-CONFIGURATION.md b/AUTO-CONFIGURATION.md index 80c623ca9..c3dc16db5 100755 --- a/AUTO-CONFIGURATION.md +++ b/AUTO-CONFIGURATION.md @@ -4,7 +4,7 @@ ### PLEASE READ CONFIGURATION INSTRUCTIONS BELOW THOROUGHLY :exclamation: _______________ -#### Version: V4.2022.02.2930 +#### Version: V4.2022.02.2931 #### Bad Referrers Blocked: [7090](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-referrers.list) #### Bad User-Agents (Bots) Blocked: [636](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) #### Fake Googlebots Blocked: [217](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/fake-googlebots.list) diff --git a/MANUAL-CONFIGURATION.md b/MANUAL-CONFIGURATION.md index 11383c621..f07dbd7e3 100755 --- a/MANUAL-CONFIGURATION.md +++ b/MANUAL-CONFIGURATION.md @@ -4,7 +4,7 @@ ### PLEASE READ CONFIGURATION INSTRUCTIONS BELOW THOROUGHLY :exclamation: _______________ -#### Version: V4.2022.02.2930 +#### Version: V4.2022.02.2931 #### Bad Referrers Blocked: [7090](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-referrers.list) #### Bad User-Agents (Bots) Blocked: [636](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) #### Fake Googlebots Blocked: [217](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/fake-googlebots.list) diff --git a/README.md b/README.md index e76ba9f17..da3251ef2 100755 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ ##### The Ultimate Nginx Bad Bot, User-Agent, Spam Referrer Blocker, Adware, Malware and Ransomware Blocker, Clickjacking Blocker, Click Re-Directing Blocker, SEO Companies and Bad IP Blocker with Anti DDOS System, Nginx Rate Limiting and Wordpress Theme Detector Blocking. Stop and Block all kinds of bad internet traffic even Fake Googlebots from ever reaching your web sites. [PLEASE SEE: Definition of Bad Bots](#define-bad-bots) _______________ -#### Version: V4.2022.02.2930 +#### Version: V4.2022.02.2931 #### Bad Referrers Blocked: [7090](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-referrers.list) #### Bad User-Agents (Bots) Blocked: [636](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) #### Fake Googlebots Blocked: [217](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/fake-googlebots.list) diff --git a/_google_analytics_ghost_spam/README.md b/_google_analytics_ghost_spam/README.md index e5e48ae69..d4304c20e 100644 --- a/_google_analytics_ghost_spam/README.md +++ b/_google_analytics_ghost_spam/README.md @@ -3,7 +3,7 @@ # EASY CONFIGURATION INSTRUCTIONS FOR STOPPING GOOGLE ANALYTICS "GHOST" SPAM _______________ -#### Version: V4.2022.02.2930 +#### Version: V4.2022.02.2931 #### Bad Referrers Blocked: [7090](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-referrers.list) #### Bad User-Agents (Bots) Blocked: [636](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) #### Fake Googlebots Blocked: [217](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/fake-googlebots.list) diff --git a/_sample_config_files/Engintron_for_cPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf b/_sample_config_files/Engintron_for_cPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf index 638437452..5d842bf96 100755 --- a/_sample_config_files/Engintron_for_cPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf +++ b/_sample_config_files/Engintron_for_cPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2022.02.2930 -### Updated: Sun Feb 20 10:05:18 UTC 2022 +### Version: V4.2022.02.2931 +### Updated: Sun Feb 20 10:31:52 UTC 2022 ### Bad Referrer Count: 7090 ### Bad Bot Count: 636 ################################################### @@ -720,7 +720,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)cah.io.community(?:\b)" 3; "~*(?:\b)check1.exe(?:\b)" 3; "~*(?:\b)clark-crawler(?:\b)" 3; - "~*(?:\b)coccocbot-web(?:\b)" 3; + "~*(?:\b)coccocbot(?:\b)" 3; "~*(?:\b)cognitiveseo(?:\b)" 3; "~*(?:\b)com.plumanalytics(?:\b)" 3; "~*(?:\b)crawl.sogou.com(?:\b)" 3; diff --git a/conf.d/globalblacklist-testing-version.conf b/conf.d/globalblacklist-testing-version.conf index 0e7027ec2..a09c44c4a 100644 --- a/conf.d/globalblacklist-testing-version.conf +++ b/conf.d/globalblacklist-testing-version.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2022.02.2930 -### Updated: Sun Feb 20 10:05:18 UTC 2022 +### Version: V3.2022.02.2931 +### Updated: Sun Feb 20 10:31:53 UTC 2022 ### Bad Referrer Count: 7090 ### Bad Bot Count: 636 ################################################### @@ -720,7 +720,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)cah.io.community(?:\b)" 3; "~*(?:\b)check1.exe(?:\b)" 3; "~*(?:\b)clark-crawler(?:\b)" 3; - "~*(?:\b)coccocbot-web(?:\b)" 3; + "~*(?:\b)coccocbot(?:\b)" 3; "~*(?:\b)cognitiveseo(?:\b)" 3; "~*(?:\b)com.plumanalytics(?:\b)" 3; "~*(?:\b)crawl.sogou.com(?:\b)" 3; diff --git a/conf.d/globalblacklist.conf b/conf.d/globalblacklist.conf index 638437452..5d842bf96 100644 --- a/conf.d/globalblacklist.conf +++ b/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2022.02.2930 -### Updated: Sun Feb 20 10:05:18 UTC 2022 +### Version: V4.2022.02.2931 +### Updated: Sun Feb 20 10:31:52 UTC 2022 ### Bad Referrer Count: 7090 ### Bad Bot Count: 636 ################################################### @@ -720,7 +720,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)cah.io.community(?:\b)" 3; "~*(?:\b)check1.exe(?:\b)" 3; "~*(?:\b)clark-crawler(?:\b)" 3; - "~*(?:\b)coccocbot-web(?:\b)" 3; + "~*(?:\b)coccocbot(?:\b)" 3; "~*(?:\b)cognitiveseo(?:\b)" 3; "~*(?:\b)com.plumanalytics(?:\b)" 3; "~*(?:\b)crawl.sogou.com(?:\b)" 3; diff --git a/dev-tools/_robots_input/robots-input.txt b/dev-tools/_robots_input/robots-input.txt index c7603d971..1c6970e17 100644 --- a/dev-tools/_robots_input/robots-input.txt +++ b/dev-tools/_robots_input/robots-input.txt @@ -571,7 +571,7 @@ backlink-check cah.io.community check1.exe clark-crawler -coccocbot-web +coccocbot cognitiveseo com.plumanalytics crawl.sogou.com diff --git a/dev-tools/beta_conf_files/conf.d/globalblacklist.conf b/dev-tools/beta_conf_files/conf.d/globalblacklist.conf index 0e7027ec2..a09c44c4a 100644 --- a/dev-tools/beta_conf_files/conf.d/globalblacklist.conf +++ b/dev-tools/beta_conf_files/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2022.02.2930 -### Updated: Sun Feb 20 10:05:18 UTC 2022 +### Version: V3.2022.02.2931 +### Updated: Sun Feb 20 10:31:53 UTC 2022 ### Bad Referrer Count: 7090 ### Bad Bot Count: 636 ################################################### @@ -720,7 +720,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)cah.io.community(?:\b)" 3; "~*(?:\b)check1.exe(?:\b)" 3; "~*(?:\b)clark-crawler(?:\b)" 3; - "~*(?:\b)coccocbot-web(?:\b)" 3; + "~*(?:\b)coccocbot(?:\b)" 3; "~*(?:\b)cognitiveseo(?:\b)" 3; "~*(?:\b)com.plumanalytics(?:\b)" 3; "~*(?:\b)crawl.sogou.com(?:\b)" 3; diff --git a/dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf b/dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf index 560fb94d1..c690223f2 100644 --- a/dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf +++ b/dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.69 1; +10.1.0.183 1; 127.0.0.1 1; diff --git a/dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf b/dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf index 447da2bf0..324b1a098 100644 --- a/dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf +++ b/dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.69 0; +10.1.0.183 0; 127.0.0.1 0; diff --git a/dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf b/dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf index 0e7027ec2..a09c44c4a 100644 --- a/dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf +++ b/dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2022.02.2930 -### Updated: Sun Feb 20 10:05:18 UTC 2022 +### Version: V3.2022.02.2931 +### Updated: Sun Feb 20 10:31:53 UTC 2022 ### Bad Referrer Count: 7090 ### Bad Bot Count: 636 ################################################### @@ -720,7 +720,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)cah.io.community(?:\b)" 3; "~*(?:\b)check1.exe(?:\b)" 3; "~*(?:\b)clark-crawler(?:\b)" 3; - "~*(?:\b)coccocbot-web(?:\b)" 3; + "~*(?:\b)coccocbot(?:\b)" 3; "~*(?:\b)cognitiveseo(?:\b)" 3; "~*(?:\b)com.plumanalytics(?:\b)" 3; "~*(?:\b)crawl.sogou.com(?:\b)" 3; diff --git a/dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf b/dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf index 560fb94d1..c690223f2 100644 --- a/dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf +++ b/dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.69 1; +10.1.0.183 1; 127.0.0.1 1; diff --git a/dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf b/dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf index 447da2bf0..324b1a098 100644 --- a/dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf +++ b/dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.69 0; +10.1.0.183 0; 127.0.0.1 0; diff --git a/dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf b/dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf index 0e7027ec2..a09c44c4a 100644 --- a/dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf +++ b/dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2022.02.2930 -### Updated: Sun Feb 20 10:05:18 UTC 2022 +### Version: V3.2022.02.2931 +### Updated: Sun Feb 20 10:31:53 UTC 2022 ### Bad Referrer Count: 7090 ### Bad Bot Count: 636 ################################################### @@ -720,7 +720,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)cah.io.community(?:\b)" 3; "~*(?:\b)check1.exe(?:\b)" 3; "~*(?:\b)clark-crawler(?:\b)" 3; - "~*(?:\b)coccocbot-web(?:\b)" 3; + "~*(?:\b)coccocbot(?:\b)" 3; "~*(?:\b)cognitiveseo(?:\b)" 3; "~*(?:\b)com.plumanalytics(?:\b)" 3; "~*(?:\b)crawl.sogou.com(?:\b)" 3; diff --git a/dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf b/dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf index 560fb94d1..c690223f2 100644 --- a/dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf +++ b/dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.69 1; +10.1.0.183 1; 127.0.0.1 1; diff --git a/dev-tools/beta_conf_files_whitelist/bots.d/blacklist-user-agents.conf b/dev-tools/beta_conf_files_whitelist/bots.d/blacklist-user-agents.conf index 722f01c05..80f37f87d 100644 --- a/dev-tools/beta_conf_files_whitelist/bots.d/blacklist-user-agents.conf +++ b/dev-tools/beta_conf_files_whitelist/bots.d/blacklist-user-agents.conf @@ -632,7 +632,7 @@ "~*(?:\b)cah.io.community(?:\b|)" 0; "~*(?:\b)check1.exe(?:\b|)" 0; "~*(?:\b)clark-crawler(?:\b|)" 0; - "~*(?:\b)coccocbot-web(?:\b|)" 0; + "~*(?:\b)coccocbot(?:\b|)" 0; "~*(?:\b)cognitiveseo(?:\b|)" 0; "~*(?:\b)com.plumanalytics(?:\b|)" 0; "~*(?:\b)crawl.sogou.com(?:\b|)" 0; diff --git a/dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf b/dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf index 447da2bf0..324b1a098 100644 --- a/dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf +++ b/dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.69 0; +10.1.0.183 0; 127.0.0.1 0; diff --git a/dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf b/dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf index 0e7027ec2..a09c44c4a 100644 --- a/dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf +++ b/dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2022.02.2930 -### Updated: Sun Feb 20 10:05:18 UTC 2022 +### Version: V3.2022.02.2931 +### Updated: Sun Feb 20 10:31:53 UTC 2022 ### Bad Referrer Count: 7090 ### Bad Bot Count: 636 ################################################### @@ -720,7 +720,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)cah.io.community(?:\b)" 3; "~*(?:\b)check1.exe(?:\b)" 3; "~*(?:\b)clark-crawler(?:\b)" 3; - "~*(?:\b)coccocbot-web(?:\b)" 3; + "~*(?:\b)coccocbot(?:\b)" 3; "~*(?:\b)cognitiveseo(?:\b)" 3; "~*(?:\b)com.plumanalytics(?:\b)" 3; "~*(?:\b)crawl.sogou.com(?:\b)" 3; diff --git a/dev-tools/buildnumber b/dev-tools/buildnumber index ca6b28543..62c5e351a 100644 --- a/dev-tools/buildnumber +++ b/dev-tools/buildnumber @@ -1 +1 @@ -2930 +2931 diff --git a/dev-tools/globalblacklist-testing.template b/dev-tools/globalblacklist-testing.template index 0e7027ec2..a09c44c4a 100644 --- a/dev-tools/globalblacklist-testing.template +++ b/dev-tools/globalblacklist-testing.template @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2022.02.2930 -### Updated: Sun Feb 20 10:05:18 UTC 2022 +### Version: V3.2022.02.2931 +### Updated: Sun Feb 20 10:31:53 UTC 2022 ### Bad Referrer Count: 7090 ### Bad Bot Count: 636 ################################################### @@ -720,7 +720,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)cah.io.community(?:\b)" 3; "~*(?:\b)check1.exe(?:\b)" 3; "~*(?:\b)clark-crawler(?:\b)" 3; - "~*(?:\b)coccocbot-web(?:\b)" 3; + "~*(?:\b)coccocbot(?:\b)" 3; "~*(?:\b)cognitiveseo(?:\b)" 3; "~*(?:\b)com.plumanalytics(?:\b)" 3; "~*(?:\b)crawl.sogou.com(?:\b)" 3; diff --git a/dev-tools/globalblacklist.template b/dev-tools/globalblacklist.template index 638437452..5d842bf96 100755 --- a/dev-tools/globalblacklist.template +++ b/dev-tools/globalblacklist.template @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2022.02.2930 -### Updated: Sun Feb 20 10:05:18 UTC 2022 +### Version: V4.2022.02.2931 +### Updated: Sun Feb 20 10:31:52 UTC 2022 ### Bad Referrer Count: 7090 ### Bad Bot Count: 636 ################################################### @@ -720,7 +720,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)cah.io.community(?:\b)" 3; "~*(?:\b)check1.exe(?:\b)" 3; "~*(?:\b)clark-crawler(?:\b)" 3; - "~*(?:\b)coccocbot-web(?:\b)" 3; + "~*(?:\b)coccocbot(?:\b)" 3; "~*(?:\b)cognitiveseo(?:\b)" 3; "~*(?:\b)com.plumanalytics(?:\b)" 3; "~*(?:\b)crawl.sogou.com(?:\b)" 3; diff --git a/dev-tools/test1_conf_files/bots.d/whitelist-ips.conf b/dev-tools/test1_conf_files/bots.d/whitelist-ips.conf index a3ecb2a6d..a466b4227 100644 --- a/dev-tools/test1_conf_files/bots.d/whitelist-ips.conf +++ b/dev-tools/test1_conf_files/bots.d/whitelist-ips.conf @@ -45,4 +45,4 @@ # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST -20.102.58.114 0; +20.83.105.253 0; diff --git a/dev-tools/test1_conf_files/conf.d/globalblacklist.conf b/dev-tools/test1_conf_files/conf.d/globalblacklist.conf index 638437452..5d842bf96 100644 --- a/dev-tools/test1_conf_files/conf.d/globalblacklist.conf +++ b/dev-tools/test1_conf_files/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2022.02.2930 -### Updated: Sun Feb 20 10:05:18 UTC 2022 +### Version: V4.2022.02.2931 +### Updated: Sun Feb 20 10:31:52 UTC 2022 ### Bad Referrer Count: 7090 ### Bad Bot Count: 636 ################################################### @@ -720,7 +720,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)cah.io.community(?:\b)" 3; "~*(?:\b)check1.exe(?:\b)" 3; "~*(?:\b)clark-crawler(?:\b)" 3; - "~*(?:\b)coccocbot-web(?:\b)" 3; + "~*(?:\b)coccocbot(?:\b)" 3; "~*(?:\b)cognitiveseo(?:\b)" 3; "~*(?:\b)com.plumanalytics(?:\b)" 3; "~*(?:\b)crawl.sogou.com(?:\b)" 3; diff --git a/dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf b/dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf index 560fb94d1..c690223f2 100644 --- a/dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf +++ b/dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.69 1; +10.1.0.183 1; 127.0.0.1 1; diff --git a/dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf b/dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf index 447da2bf0..324b1a098 100644 --- a/dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf +++ b/dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.69 0; +10.1.0.183 0; 127.0.0.1 0; diff --git a/dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf b/dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf index 638437452..5d842bf96 100644 --- a/dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf +++ b/dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2022.02.2930 -### Updated: Sun Feb 20 10:05:18 UTC 2022 +### Version: V4.2022.02.2931 +### Updated: Sun Feb 20 10:31:52 UTC 2022 ### Bad Referrer Count: 7090 ### Bad Bot Count: 636 ################################################### @@ -720,7 +720,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)cah.io.community(?:\b)" 3; "~*(?:\b)check1.exe(?:\b)" 3; "~*(?:\b)clark-crawler(?:\b)" 3; - "~*(?:\b)coccocbot-web(?:\b)" 3; + "~*(?:\b)coccocbot(?:\b)" 3; "~*(?:\b)cognitiveseo(?:\b)" 3; "~*(?:\b)com.plumanalytics(?:\b)" 3; "~*(?:\b)crawl.sogou.com(?:\b)" 3; diff --git a/dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf b/dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf index 560fb94d1..c690223f2 100644 --- a/dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf +++ b/dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.69 1; +10.1.0.183 1; 127.0.0.1 1; diff --git a/dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf b/dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf index 447da2bf0..324b1a098 100644 --- a/dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf +++ b/dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.69 0; +10.1.0.183 0; 127.0.0.1 0; diff --git a/dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf b/dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf index 638437452..5d842bf96 100644 --- a/dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf +++ b/dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2022.02.2930 -### Updated: Sun Feb 20 10:05:18 UTC 2022 +### Version: V4.2022.02.2931 +### Updated: Sun Feb 20 10:31:52 UTC 2022 ### Bad Referrer Count: 7090 ### Bad Bot Count: 636 ################################################### @@ -720,7 +720,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)cah.io.community(?:\b)" 3; "~*(?:\b)check1.exe(?:\b)" 3; "~*(?:\b)clark-crawler(?:\b)" 3; - "~*(?:\b)coccocbot-web(?:\b)" 3; + "~*(?:\b)coccocbot(?:\b)" 3; "~*(?:\b)cognitiveseo(?:\b)" 3; "~*(?:\b)com.plumanalytics(?:\b)" 3; "~*(?:\b)crawl.sogou.com(?:\b)" 3; diff --git a/dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf b/dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf index 560fb94d1..c690223f2 100644 --- a/dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf +++ b/dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.69 1; +10.1.0.183 1; 127.0.0.1 1; diff --git a/dev-tools/test1_conf_files_whitelist/bots.d/blacklist-user-agents.conf b/dev-tools/test1_conf_files_whitelist/bots.d/blacklist-user-agents.conf index 722f01c05..80f37f87d 100644 --- a/dev-tools/test1_conf_files_whitelist/bots.d/blacklist-user-agents.conf +++ b/dev-tools/test1_conf_files_whitelist/bots.d/blacklist-user-agents.conf @@ -632,7 +632,7 @@ "~*(?:\b)cah.io.community(?:\b|)" 0; "~*(?:\b)check1.exe(?:\b|)" 0; "~*(?:\b)clark-crawler(?:\b|)" 0; - "~*(?:\b)coccocbot-web(?:\b|)" 0; + "~*(?:\b)coccocbot(?:\b|)" 0; "~*(?:\b)cognitiveseo(?:\b|)" 0; "~*(?:\b)com.plumanalytics(?:\b|)" 0; "~*(?:\b)crawl.sogou.com(?:\b|)" 0; diff --git a/dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf b/dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf index 447da2bf0..324b1a098 100644 --- a/dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf +++ b/dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.69 0; +10.1.0.183 0; 127.0.0.1 0; diff --git a/dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf b/dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf index 638437452..5d842bf96 100644 --- a/dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf +++ b/dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2022.02.2930 -### Updated: Sun Feb 20 10:05:18 UTC 2022 +### Version: V4.2022.02.2931 +### Updated: Sun Feb 20 10:31:52 UTC 2022 ### Bad Referrer Count: 7090 ### Bad Bot Count: 636 ################################################### @@ -720,7 +720,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)cah.io.community(?:\b)" 3; "~*(?:\b)check1.exe(?:\b)" 3; "~*(?:\b)clark-crawler(?:\b)" 3; - "~*(?:\b)coccocbot-web(?:\b)" 3; + "~*(?:\b)coccocbot(?:\b)" 3; "~*(?:\b)cognitiveseo(?:\b)" 3; "~*(?:\b)com.plumanalytics(?:\b)" 3; "~*(?:\b)crawl.sogou.com(?:\b)" 3; diff --git a/dev-tools/test2_conf_files/bots.d/whitelist-ips.conf b/dev-tools/test2_conf_files/bots.d/whitelist-ips.conf index a3ecb2a6d..a466b4227 100644 --- a/dev-tools/test2_conf_files/bots.d/whitelist-ips.conf +++ b/dev-tools/test2_conf_files/bots.d/whitelist-ips.conf @@ -45,4 +45,4 @@ # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST -20.102.58.114 0; +20.83.105.253 0; diff --git a/dev-tools/test2_conf_files/conf.d/globalblacklist.conf b/dev-tools/test2_conf_files/conf.d/globalblacklist.conf index 622658fd2..e46f8e54a 100644 --- a/dev-tools/test2_conf_files/conf.d/globalblacklist.conf +++ b/dev-tools/test2_conf_files/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2022.02.2930 -### Updated: Sun Feb 20 10:05:18 UTC 2022 +### Version: V4.2022.02.2931 +### Updated: Sun Feb 20 10:31:52 UTC 2022 ### Bad Referrer Count: 7090 ### Bad Bot Count: 636 ################################################### @@ -720,7 +720,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)cah.io.community(?:\b)" 3; "~*(?:\b)check1.exe(?:\b)" 3; "~*(?:\b)clark-crawler(?:\b)" 3; - "~*(?:\b)coccocbot-web(?:\b)" 3; + "~*(?:\b)coccocbot(?:\b)" 3; "~*(?:\b)cognitiveseo(?:\b)" 3; "~*(?:\b)com.plumanalytics(?:\b)" 3; "~*(?:\b)crawl.sogou.com(?:\b)" 3; diff --git a/dev-tools/test3_conf_files/bots.d/whitelist-ips.conf b/dev-tools/test3_conf_files/bots.d/whitelist-ips.conf index a3ecb2a6d..a466b4227 100644 --- a/dev-tools/test3_conf_files/bots.d/whitelist-ips.conf +++ b/dev-tools/test3_conf_files/bots.d/whitelist-ips.conf @@ -45,4 +45,4 @@ # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST -20.102.58.114 0; +20.83.105.253 0; diff --git a/dev-tools/test3_conf_files/conf.d/globalblacklist.conf b/dev-tools/test3_conf_files/conf.d/globalblacklist.conf index 13a6442ae..15b515f2c 100644 --- a/dev-tools/test3_conf_files/conf.d/globalblacklist.conf +++ b/dev-tools/test3_conf_files/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2022.02.2930 -### Updated: Sun Feb 20 10:05:18 UTC 2022 +### Version: V4.2022.02.2931 +### Updated: Sun Feb 20 10:31:52 UTC 2022 ### Bad Referrer Count: 7090 ### Bad Bot Count: 636 ################################################### @@ -720,7 +720,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)cah.io.community(?:\b)" 3; "~*(?:\b)check1.exe(?:\b)" 3; "~*(?:\b)clark-crawler(?:\b)" 3; - "~*(?:\b)coccocbot-web(?:\b)" 3; + "~*(?:\b)coccocbot(?:\b)" 3; "~*(?:\b)cognitiveseo(?:\b)" 3; "~*(?:\b)com.plumanalytics(?:\b)" 3; "~*(?:\b)crawl.sogou.com(?:\b)" 3; diff --git a/dev-tools/test_units/blacklist-ips.conf b/dev-tools/test_units/blacklist-ips.conf index 560fb94d1..c690223f2 100644 --- a/dev-tools/test_units/blacklist-ips.conf +++ b/dev-tools/test_units/blacklist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.69 1; +10.1.0.183 1; 127.0.0.1 1; diff --git a/dev-tools/test_units/blacklist-user-agents.conf b/dev-tools/test_units/blacklist-user-agents.conf index 722f01c05..80f37f87d 100644 --- a/dev-tools/test_units/blacklist-user-agents.conf +++ b/dev-tools/test_units/blacklist-user-agents.conf @@ -632,7 +632,7 @@ "~*(?:\b)cah.io.community(?:\b|)" 0; "~*(?:\b)check1.exe(?:\b|)" 0; "~*(?:\b)clark-crawler(?:\b|)" 0; - "~*(?:\b)coccocbot-web(?:\b|)" 0; + "~*(?:\b)coccocbot(?:\b|)" 0; "~*(?:\b)cognitiveseo(?:\b|)" 0; "~*(?:\b)com.plumanalytics(?:\b|)" 0; "~*(?:\b)crawl.sogou.com(?:\b|)" 0; diff --git a/dev-tools/test_units/random-bots-for-test-quick.list b/dev-tools/test_units/random-bots-for-test-quick.list index 9d49f7f92..c8f1adf15 100644 --- a/dev-tools/test_units/random-bots-for-test-quick.list +++ b/dev-tools/test_units/random-bots-for-test-quick.list @@ -1,10 +1,10 @@ -Battleztar Bazinga -Cloud mapping -Majestic12 -Moblie Safari -SEOkicks -Shodan -Sucker -ips-agent -page scorer -sp_auditbot +Anarchy99 +EroCrawler +Lipperhey +MarkWatch +Mata Hari +ScanAlert +SpankBot +evc-batch +seoscanners +ubermetrics-technologies.com diff --git a/dev-tools/test_units/random-bots-for-test.list b/dev-tools/test_units/random-bots-for-test.list index bdb6d36c9..2cbb4cc73 100644 --- a/dev-tools/test_units/random-bots-for-test.list +++ b/dev-tools/test_units/random-bots-for-test.list @@ -1,250 +1,250 @@ +01h4x.com ADmantX AIBOT -ALittle Client -ASPSeek +Abonti Aboundex +Aboundexbot AhrefsBot -AllSubmitter -Anarchie +Alexibot Anarchy99 +Ankit Apexoo -Asterias -Atomseobot -AwarioRssBot +Attach +AwarioSmartBot BDFetch +BLEXBot +Backlink-Ceck +Badass Bandit Barkrowler -Battleztar Bazinga -Bigfoot +BetaBot Black Hole Blackboard -Brandwatch -BuiltBotTough +BotALot +Brandprotect +BuiltWith +BunnySlippers CATExplorador -CODE87 +CSHttp +Calculon Cegbfeieh -CensysInspect CheTeam -CheeseBot -Chlooe -Citoid -Collector +CherryPicker +Claritybot +Cloud mapping Copier -CopyRightCheck -Copyscape -CrunchBot -DBLBot -DIIbot -Dirbuster -DnBCrawler-Analytics -DomCopBot +Crawling at Home Project +Curious +CyotekWebCopy +DatabaseDriverMysqli +Demon +Deusu +Devil +Disco +Discobot +Dispatch +DomainAppender +DomainCrawler +DomainStatsBot +Dotbot +Drip +EMail Siphon +EasyDL +Ebingbong Ecxi Evil -Exabot -ExtLinksBot -Extractor ExtractorPro -Extreme Picture Finder -Ezooms -FDM -FHscan -FemtosearchBot +Fimap Firefox/7.0 -Foobot -FrontPage Fuzz -FyberSpider -G-i-g-a-b-o-t -GalaxyBot +Fyrebot +GetWeb Go!Zilla -Go-Ahead-Got-It GoZilla -Gotit -Grabber -GrapeFX -GrapeshotCrawler -HEADMasterSEO -HTMLparser +HTTrack +Haansoft +Harvest +Havij +Heritrix Hloader Humanlinks HybridBot -IDBTE4M Iblog -IndeedBot -Intelliseek -InternetSeer -JOC Web Spider +Id-search +InfoTekies +InterGET +JamesBOT +Jbrofuzz JennyBot +JetCar JikeSpider -Jyxobot -Keyword Density -Kozmosbot -Larbin -LeechFTP +JustView +Kenjin Spider +LWP::Simple +Lanshanbot +Leap LeechGet -LexiBot Lftp -Likse -LinqiaMetadataDownloaderBot -LinqiaRSSBot -LinqiaScrapeBot -MQQBrowser +LieBaoFast +Lightspeedsystems +LinkWalker +Linkbot +Linkdexbot +LinkextractorPro +LinksManager +Lipperhey Spider +Litemage_walker +Lmspider +Ltx71 +MIIxpc +MJ12bot MTRobot -MarkMonitor -Mass Downloader -MauiBot +Majestic12 +Mata Hari MeanPath Bot Meanpathbot MegaIndex.ru -Metauri Microsoft Data Access -Moblie Safari +Minefield Mojeek -Mojolicious -NICErsPRO -NPbot +Name Intelligence +Nameprotect Navroad -NearSite -Needle -NetLyzer +Nessus +Net Vampire +NetAnts NetSpider -Netcraft -NextGenSearchBot +Nettrack +Netvibes Nibbler -Nikto Nimbostratus Ninja -Nmap +Not Nutch Offline Explorer -OnCrawl +Offline Navigator OpenLinkProfiler +OpenVAS Openvas OutclicksBot -PECL::HTTP -Page Analyzer PageAnalyzer PageGrabber +PageScorer PageThing.com Pandalytics Panscient -Pavuk Petalbot -PictureFinder -PleaseCrawl +Pi-Monster +Picsearch +Piepmatz +Pimonster Pockey +Psbot Pu_iN -Pump -PxBroker -Quick-Crawler +QueryN Metasearch +RSSingBot RankFlex -RankingBot -Rankivabot +RankingBot2 RankurBot Reaper -RepoMonkey -Ripper -Rogerbot -SBIder -SEOlyticsCrawler -ScanAlert -Scanbot -ScoutJet -Scrapy +SISTRIX +SalesIntelligent +Screaming Searchestate -SearchmetricsBot -Seekport Semrush -Siphon -SiteCheckerBotCrawler +SentiBot +SeoSiteCheckup +Seomoz +Site Sucker SiteLockSpider SiteSnagger -SiteSucker Siteimprove -SmartDownload +Sitevigil Snake -Snapbot Snoopy SocialRankIOBot +Sociscraper Sogou web spider -SpankBot -Spanner +Sosospider +Spammen Spinn3r -SputnikBot Sqlmap -Sqlworm -Sqworm -SuperBot +Steeler +Sucker Surfbot -SurveyBot -Swiftbot -Szukacz -T8Abot -Teleport +Suzuran +TeleportPro +Telesoft Telesphoreo The Intraformant +TheNomad +Thumbor Toata -Trendiction +Toweyabot +Tracemyfile +Trendictionbot True_Robot +Turnitin +TurnitinBot +Twice Typhoeus -URLy Warning -Upflow -VB Project -VelenPublicWebCrawler +URLy.Warning +V-BOT +VCI +Vagabondo +VeriCiteCrawler +VidibleScraper Virusdie -VoidEYE +Voil +WISENutbot WWW-Collector-E -WWW-Mechanize WWW::Mechanize -Wallpapers -Wallpapers/3.0 WeSEE -Web Collage -Web Fetch +Web Auto +Web Pix +Web Sauger +WebAuto WebBandit -WebSauger +WebCollage +WebGo IS +WebLeacher +WebPix WebStripper -WebZIP -WebmasterWorldForumBot +Webalta Webshag -Whack +Webster Whacker -Whatweb Who.is Bot -Widow -Wonderbot +WinHTTrack Woobot -Wprecon Xaldon WebSpider -Xenu -Zauba -ZoomBot +Xaldon_WebSpider +Zade +Zitebot +ZmEu ZyBorg adscanner -arquivo-web-crawler arquivo.pt -autoemailspider -coccocbot-web -com.plumanalytics -demandbase-bot -domainsproject.org +cah.io.community +clark-crawler +crawl.sogou.com +eCatch facebookscraper -heritrix internetVista monitor +ips-agent lwp-request lwp-trivial -magpie-crawler -muhstik-scan -plumanalytics -polaris version +netEstate NE Crawler +oBot +s1z.ru scan.lol -sexsearcher +seoscanners +serpstatbot sitechecker.pro +siteripz sp_auditbot -spyfu -trendiction.com -trendiction.de webpros.com webprosbot -x22Mozilla -xpymep1.exe +x09Mozilla +zauba.io +zgrab diff --git a/dev-tools/test_units/random-bots-for-whitelist-test.list b/dev-tools/test_units/random-bots-for-whitelist-test.list index 282eece05..555a9bdee 100644 --- a/dev-tools/test_units/random-bots-for-whitelist-test.list +++ b/dev-tools/test_units/random-bots-for-whitelist-test.list @@ -1,250 +1,250 @@ -01h4x.com +404checker +404enemy +ADmantX ALittle Client -Aboundexbot -Acunetix -AfD-Verbotsverfahren -AhrefsBot AiHitBot -Alexibot -AlphaBot -Ankit +Aipbot +Alligator +Anarchie +Anarchy +Anthill Apexoo -Asterias +Atomseobot AwarioSmartBot -BBBike -BDFetch BackDoorBot -BackStreet Backlink-Ceck -BatchFTP +BacklinkCrawler +Bandit +Black Hole BlackWidow Blow -Bolt +BlowFish +BotALot Buck Buddy -CheTeam -ChinaClaw -Citoid +CODE87 +Calculon +CazoodleBot +CensysInspect Claritybot -Copier +Cliqzbot +Cloud mapping +Cogentbot +Collector +CopyRightCheck Copyscape +Craftbot +CrazyWebCrawler Crescent -Custo +Curious CyotekWebCopy -DBLBot -DSearch -DTS Agent -Deusu -Devil -DigitalPebble -Dirbuster -Disco +DataCha0s Discoverybot -Dispatch DittoSpyder -DomainSigmaCrawler +DomainAppender +DomainCrawler +Domains Project Dotbot -Download Wonder Drip -ECCP/1.0 -Ebingbong -Ecxi +EMail Wolf EirGrabber EroCrawler -Express WebPictures -Extreme Picture Finder +Evil +Extractor EyeNetIE -Ezooms -Firefox/7.0 -Flunky -Freeuploader -FrontPage +FHscan +FlashGet Fuzz -G-i-g-a-b-o-t +FyberSpider +Fyrebot +GT::WWW GalaxyBot -GermCrawler GetWeb -Getintent Go-Ahead-Got-It -Gotit +GoZilla +GrabNet +Grabber Grafula -GrapeFX +GridBot HEADMasterSEO -HMView HTTP::Lite -HTTrack -Havij Heritrix -Hloader -HybridBot +Humanlinks IDBTE4M IDBot -Id-search -Image Sucker -Indy Library -Intelliseek -InterGET -InternetSeer +Iblog +IndeedBot +InfoNaviRobot +InfoTekies Iskanie JamesBOT +JennyBot +JetCar +Jetty +JikeSpider +JustView Jyxobot +Kenjin Spider +Kinza LNSpiderguy +LWP::Simple Larbin -Leap +LeechFTP LeechGet LexiBot Lftp -Libwhisker -Lightspeedsystems -Likse LinkWalker -Linkdexbot -LinkpadBot LinksManager -Litemage_walker -Lmspider -Ltx71 +LinqiaMetadataDownloaderBot +LinqiaScrapeBot +Lipperhey +Lipperhey Spider MFC_Tear_Sample -MIDown tool -MSFrontPage +MIIxpc +MJ12bot MSIECrawler -MTRobot -Majestic SEO +Mag-Net Majestic-SEO MarkWatch -Mass Downloader -Meanpathbot MegaIndex.ru -Metauri MicroMessenger -Mister PiX -Moblie Safari -MolokaiBot -Name Intelligence -NetLyzer -NetSpider -NetZIP -Netvibes -Niki-bot +Microsoft URL Control +Minefield +Mojolicious +Musobot +NICErsPRO +Nameprotect +Navroad +NearSite +Needle +Nessus +Netcraft +Nettrack NimbleCrawler -Nimbostratus -Ninja -Nmap -OnCrawl -OpenLinkProfiler -OpenVAS -OutclicksBot -OutfoxBot +Not +Offline Explorer +Offline Navigator +OrangeSpider +PECL::HTTP PHPCrawl -PageGrabber -PageThing.com +Page Analyzer +Pandalytics +Papa Foto Pavuk +Petalbot +Pi-Monster Picscout -PictureFinder -Pimonster -ProWebWalker -Psbot +Pump +PxBroker PyCurl +QueryN Metasearch +RankActiveLinkBot RankFlex -ReGet -Reaper -RebelMouse +RankingBot +RankingBot2 +RealDownload RedesScrapy -RepoMonkey -SEOkicks -SEOlyticsCrawler -SMTBot -SalesIntelligent -ScanAlert -ScreenerBot -ScrepyBot -Searchestate +Rogerbot +SBIder +SEOkicks-Robot +SISTRIX +Scanbot +Scrapy SearchmetricsBot -SemanticJuice +Seekport +SentiBot Seomoz -Shodan SiteCheckerBotCrawler -SiteSucker -Sitebeam +SiteLockSpider +SiteSnagger +Siteimprove SmartDownload -Snake Snapbot -Snoopy -SocialRankIOBot -Sosospider SpaceBison -Spanner -Spbot -Steeler -Sucker +SpankBot +Spinn3r SuperHTTP -Surfbot SurveyBot -Szukacz +T0PHackTeam T8Abot -Teleport -Telesphoreo +TeleportPro The Intraformant -Toweyabot +Titan Trendiction -Turnitin -TurnitinBot +Trendictionbot +Turingos Twice -URLy Warning -URLy.Warning UnisterBot -VB Project +V-BOT +VCI +Vacuum Vagabondo VelenPublicWebCrawler +VeriCiteCrawler VidibleScraper Virusdie -Voil -Voltron +VoidEYE WASALive-Bot -WPScan -WWW-Mechanize -WWW::Mechanize -Wallpapers/3.0 -WallpapersHD +WEBDAV WeSEE +Web Auto Web Collage +Web Enhancer +Web Pix +WebAuto WebCollage -WebCopier +WebEnhancer WebFuck -WebPix -WebSauger -WebStripper -WebSucker +WebGo IS +WebLeacher +WebReaper WebWhacker -WebZIP -WebsiteExtractor +Website Quester Whack -Woobot -Wotbox -Wprecon +Who.is Bot +Widow +WiseGuys Robot +Xaldon WebSpider +Xaldon_WebSpider +Xenu Zade -Zeus +Zauba ZmEu -adscanner +ZoomBot +ZoominfoBot +ZumBot archive.org_bot +arquivo-web-crawler arquivo.pt -autoemailspider -coccocbot-web -crawler.feedback -crawler4j -domainsproject.org +backlink-check +check1.exe +clark-crawler +coccocbot +dataforseo.com +demandbase-bot +eCatch instabid -ips-agent isitwp.com -iubenda-radar -meanpathbot -mediawords +lwp-request +lwp-trivial +muhstik-scan +netEstate NE Crawler +oBot +page scorer +polaris version +ripz s1z.ru satoristudio.net -scalaj-http +scan.lol +seocompany.store seoscanners +seostar +serpstatbot sogouspider sp_auditbot -spyfu -ubermetrics-technologies.com +sysscan +tAkeOut +webgains-bot +webmeup-crawler +xpymep1.exe zauba.io diff --git a/dev-tools/test_units/random-referrers-for-test-quick.list b/dev-tools/test_units/random-referrers-for-test-quick.list index 83229b092..dcc465652 100644 --- a/dev-tools/test_units/random-referrers-for-test-quick.list +++ b/dev-tools/test_units/random-referrers-for-test-quick.list @@ -1,10 +1,10 @@ -bjetjt.com -buqyxa.rincian.info -donna7753191.ru -myonigroup.com -sexreliz.com -tivolibasket.it -topbestgames.com -xn----7sbahjd3btneuw1joc.xn--p1ai -xn----btbdvdh4aafrfciljm6k.xn--p1ai -xxxnatelefon.ru +androids-store.com +blackplanet.com +finuse.com +formulaantiuban.com +isitwp.com +solution4u.com +tfxiq.com +voronezh.xrus.org +zeroredirect7.com +zigarettenonlinekaufen2.drupalgardens.com diff --git a/dev-tools/test_units/random-referrers-for-test.list b/dev-tools/test_units/random-referrers-for-test.list index d68d1afe5..e32e53bd1 100644 --- a/dev-tools/test_units/random-referrers-for-test.list +++ b/dev-tools/test_units/random-referrers-for-test.list @@ -1,500 +1,500 @@ -11pikachu.ru -1kdailyprofit.me -20pascals.nl -24-ak.ru -256bit.by -2ads.co.uk -2drittel.de -3rbseyes.com -3xforum.ro -45en.ru +007angels.com +01apple.com +0daymusic.org +1-99seo.com +1-free-share-buttons.com +1hwy.com +1j7740kd.website +1webmaster.ml +34.gs 4k-player.pl -4replicawatch.net -5tey7463.weebly.com -83net.jp -a.pr-cy.ru -aanapa.ru -abcguru.xyz -absolute-s.ru -academia-nsk.org -accessoires-mode.in -adanih.com -adcash.com -addbin.men -adloads.net -adtology.com -adult3dgames.com -agreda.pluto.ro -ahmedabadwebs.com -albamargroup.com -alert-jdh.xyz -alevinefesleri.com -algerianembassy.co.in -alibestsale.com -alienwheels.de -allcredits.su -allproblog.com -alphaforum.ru -amateurgalls.com -amazon-adsystem.com -anaksma.info -anatomiy.com -angkortours.vn -anticrawler.org -antons-transporte.de +4kepics.com +4rent.ru +5-steps-to-start-business.com +51unlim.ru +70casino.online +72-news.com +7minuteworkout.com +8gold.com +8jn.dating-app.ru +91abcw.com +adamoads.com +adobereader-free.ru +adserver-e7.com +adservme.com +adultfullhd.com +adultmeetonline.info +adultwebhosting.info +afslankpillen2017nl.eu +akuhni.by +alert-fjg.xyz +allboard.xobor.de +alldezservice.kz +allnews.md +alltheviews.com +alphahoverboards.com +alyeskaresort.com +amoi.tn +amospalla.es +amyfoxfitness.com +ananumous.ru +android-vsem.org +anlimebel.kiev.ua +anonymizeme.pro +anonymousfox.co +anti-virus-removal.info +apartment.ru apelsinnik.website -apiadanaknet-a.akamaihd.net -apibrowsesmartne-a.akamaihd.net -apifasterlightin-a.akamaihd.net -apispringsmartne-a.akamaihd.net -appearance-cool.com -arate.ru -arenda-yeisk.ru -artdeko.info -artefakct.com -as5000.com -asrv-a.akamoihd.net -asscenihotosidea.blogspot.co.za -asseenontvonline.ru -atdedinotuho.tk -auto-moto-elektronika.cz -autoplate.info -aviav.ru.com -avkzarabotok.com -awaybird.ru -ayerbo.xhost.ro -azartmix.com -azbaseimages.net -back.dog -badavit.com.ua -bagsonsale.online -banki76.ru -basedecor.ru -bausparen.bz.it -belinvestdom.by -belstaffstore.online -beremenyashka.com -berrymall.ru -bestmouthwash.club -bget.ru -bimatoprost-careprost.com -bitnote.co -blagovest-med.ru -blockety.co -blogig.org +apiluckyleapnet-a.akamaihd.net +apimegabrowsebiz-a.akamaihd.net +app-ready.xyz +appiq.mobi +arendakvartir.kz +arpelsreplica.xyz +arquivo.pt +atlant-auto.info +automatic-seo.com +automobile-spec.com +babyfactory.fr +backiee.com +battle.net +bensbargains.net +bestcalovebracelet.cn +bestcurs.org +bestssaker.com +bitcoin-ua.top +biteg.xyz +bkgr.se +blog-hits.com blogs.rediff.com -bmw-ark.ru -bnt-team.com -bonanza-fortune.men -boo-arts.com -boom-celebs.com -brandehk.dk -brandov.ru -bronzeaid-a.akamaihd.net -bus-offer.com +bodybuilding-shop.biz +boobsimge.com +bookmark4you.com +bosefux.esy.es +bugshoot.cn +bum.com.ru buyantiviralwp.com -buyhoverboard.com -buyparajumpers.online -buytizanidineonlinenoprescription.blogspot.com -candytiner.com -cardsharp1.ru -carina-sy.de -cartierjusteunclou.xyz -cbcseward.com -cbox.ws -cctva.tv -centre-indigo.org.ua -cete.ru +buytizanidineonline.blogspot.com +buzzurl.jp +cacheimages.com +calc-for-credit.ru +callejondelpozo.es +callstevens.com +canrioloadm.gq +carabela.com.do +carsplate.com +cartierbracelet.xyz +cartierreplica.xyz +castingbank.ru +centrumcoachingu.com cfsrating.sonicwall.com -cheapjerseysbizwholesale.us -chocolateslim-original.com -citetick.com -cleanallspyware.com -clickbanksites.info -clodo.ru -coaching-netz.info -cocolyze.com -cocyq.inwtrade.com -commoncrawl.org -compliance-barak.top -concealthyself.com -constantaservice.net -cookielawblog.wordpress.com -copyrightclaims.org -cornomase.win -covadhosting.biz -crackguru.tk -cranly.net -customerguru.in -dairyindia.in -damedingel.ya.ru -dawlenie.com -decenttools.com -dekorkeramik.ru -delayreferat.ru -deluxedumps.com +checkpagerank.net +chidporn.com +chinaelitecheapjerseys.com +chocolateslim-en-espana.com +cialgenisrx.com +classiquebijoux.ru +clicksor.com +com-secure.download +compliance-alexa.xyz +compliance-brian.top +compliance-irvin.xyz +compliance-jessica.top +compliance-julia.top +compliance-olga.top +compucelunlock.net +comsysnet.com +contentsexpress.com +cookingmeat.ru +coolwallpapers-hd.com +coolwallpapers4k.info +coprofam.org +cowblog.fr +customergrowthsystems.com +cymbaltaandtizanidine.blogspot.com +czat.wp.pl +d2jsp.org +delta-line.men +demian.kz +dentfidemountpreach.tk deplim.com -design-ideas.info -detskie-zabavi.ru -dfiles.me -dickssportinggoods.com -dinosaurus.site -dokfilms.net +descargar-musica-gratis.net +div.as +diy-handmade-ideas.com +dneprsvet.com.ua +doctissimo.fr domainanalyzing.xyz -domaineaneblanc.com -duawitchrarato.tk -dyshagi.ru -ebooknovel.club -eduinfosite.com -ege-essay.ru -ekn-art.se -elektrozigaretten1.webs.com -elitedollars.com -elitepcgames.com -elstal.com.pl -en.altezza.travel -energy-ua.com -enternet.ee -espaceinventoristes.com -essaypro.com -european-torches.ru -eurosamodelki.ru -evidencecleanergold.com -exdocsfiles.com -exoneration-project.us -exportshop.us -eywords-monitoring-your-success.com -ezigarettenshop1.yolasite.com -f5mtrack.com +domainsatcost.com +domcran.net +donvito.unas.cz +dosug-lux.ru +doublepimp.com +doxysexy.com +doyouknowtheword-flummox.ml +drupa.com +dstroy.su +e-collantes.com +easytuningshop.ru +electronicadirect.com +elektrozigaretten1.postbit.com +emergencyneeds.org +eng-lyrics.com +etm-consult.de +everflora.ru +exchanges-bet.com +excitacion.info +extractorandburner.com +extstat.com +f-loaded.de facebook-mobile.xyz -facecup.top -failingmarriege.blogspot.com -fanoboi.com -faracontrol.ir -fashion.stellaconstance.co -fashionindeed.ml -fealq.com -fickenprivat.info -film-one.ru -filmgo.ru -finuse.com -formularz-konkurs.tk -forodvd.com -freegamesplay.online -front.to -gambarkatabaru.com -gameplexcity.com -gamingspark.com -gcup.ru -get-your-social-buttons.info -ghostvisitor.com -gigapeta.com -glavprofit.ru -gne8.com -gomusix.com -googglet.com -gotwebsite1.com -gox.com.ua -greenshop.su -gribkovye-zabolevaniya.com -guge.io -guiadeserraazul.com -guitar-master.org -h2monline.com +facialporntube.com +fastfixing.tech +fearcrow.com +fefo.gdn +filesvine.com +findpik.com +fitness-video.net +flooringinstallation-edmonton.com +forex-procto.ru +forminecrafters.ru +formseo.com +free-share-buttons.blogspot.com +freemaintenancesysforpcandmac.top +freshberry.com.ua +fsakhalin.ru +funcrushgames.com +fx-brokers-review.com +gardene.ru +gaz-voshod.ru +gazoblok.net.ua +gdcentre.ru +geilehausfrauen.net +genetworx.com +gidonline.one +glall.ru +globalscam.ga +globetrotting-culture.ru +goblacked.com +greatdealshop.com +gribokstop.com +growmyfunds.ca +gsasearchengineranker.top +guigyverpo.cf +gwhwpxbw.bloger.index.hr +hahashka.ru hamilton.ca -hanwei.us -happy.new.yeartwit.com hauleddes.com -hd720kino.ru -healbio.ru -heartofpayne.xyz -hentaiheroes.com -hermesreplica.pw -hesteel.pl -hetmanship.xyz -himazin.info -hop.clickbank.net -hotel-mkad.ru -houseofrose.com -howtowhitenteethfast.xyz -icaseclub.ru -iccornacircri.cf -ie.57883.net -ihtec2019.org -ilte.info -imguramx.pw -incolors.club -integritylandscapeservices.com +hccoder.info +helicalpile.us +herehloadibs.cf +hermesbelts.xyz +hermesreplica.win +homedecoguide.info +houseofgaga.ru +hoverboard360.at +hoztorg-opt.ru +hplaserjetpdriver8y.pen.io +hut1.ru +ikritikimou.gr +ilovevitaly.ru +imagez.co +imperiafilm.ru +inboxdollars.com +incest-ru.com +infobabki.ru +intellektmedia.at interferencer.ru -interior-stickers.ru -intim-uslugi.info -invest-pamm.ru -investmac.com -investpamm.ru -ipchicken.com -irkutsk.online-podarki.com -istizanidineanopiate.blogspot.com -j33x.com -javatex.co.id +internetartfair.com +invitefashion.com +iqupdatetmz.win +irkutsk.zrus.org +ispac.org +istripper.com +iusstf.org +izismile.com +izoll.ru jbl-charge.info -jerseychinabizwholesale.us -jpcycles.com -jrcigars.com -junglenet-a.akamaihd.net -justkillingti.me -kazan.xrus.org -kchaxton.com -kerei.ru -kerwinandcariza.com -kino-rating.ru -kino2018.club +je7.us +job.icivil.ir +jobgirl24.ru +kamagragelusa.net +keki.info +keywesthideaways.co +kino-rf.ru +kinosed.net kinozapas.com -kletkimehan.ru -konoplisemena.com -koopilka.com -kredit-pod-zalog-krasnodar.ru -kskjco.club -lacasamorett.com -lazymanyoga.com -ledpolice.ru -lida-ru.com -lincolntheatre.com -linkdebrideur.xyz -littlecity.ch -lollypopgaming.com -lombia.co -losangeles-ads.com -louisvuittonoutletstore.net -lubetube.com -m3gadownload.pl -macrotek.ru -mainhp.net -manifestyourmillion.com -mansparskats.com -marblestyle.ru -massage-info.nl -mazda-roadsters.com -me-ke.com -mebel-vstroika.ru -meddesk.ru -medicine-4u.org -medizinreisen.de -members.ghanaweb.com -metabar.ru -michaelkorssaleoutletonline.net -mirobuvi.com.ua -mirzonru.net -missis.top -mojaocena.com -moje-recenze.cz -monarchfind-a.akamaihd.net -monclercoats.xyz -more-letom.ru -mossmesi.com -mototsikl.org +kinozapas.org +kol-energo.ru +kosova.de +ktotut.net +kumuk.info +la-fa.ru +landoftracking.com +lasvegaslockandsafe.com +lazy-z.com +lightinghomes.net +liquimondo.com +listiseltemournlan.gq +livingcanarias.com +locationdesaison.com +loftdigital.eu +lolnex.us +luxmagazine.cf +m.b00kmarks.com +magento-crew.net +maladot.com +manifestation.betteroffers.review +manimpotence.com +mb140.ru +meble-bogart.info +mecash.ru +med-zdorovie.com.ua +medicines-choice.com +megaapteka.ru +megagrabber.ru +mexicotravelnet.com +micasainvest.com +minharevisao.com +minyetki.ru +misandesign.se +misslike.ru +missvietnam.org +moesonce.com +monthlywinners.com +morlat.altervista.org moviezbonkerssk.cf -mp3downloadhq.com -mustat.com -mvpicton.co.uk myanyone.net -mycaf.it -myhealthcare.com -mymobilemoneypages.com -mzdish.site -narutonaruto.ru -national-today-winning-winner.club -naval.jislaaik.com -nerudlogistik.ru -nikhilbahl.com -nisuturnetdgu.tk -nl.netlog.com -ntic.fr -nw-servis.ru -office2web.com -okayimage.com -oksrv.com -optitrade24.com -orion-code-access.net -paclitor.com -paighambot.com -panicatack.com -paphoselectricianandplumber.com -partnerads.men -partnerline.men -pdfprof.com -perederni.net -perfection-pleasure.ru -photosaga.info -piccshare.com -pio.polytopesexempt.com -piski.top +mybinaryoptionsrobot.com +mylovelibrabry.com +mysex21.com +mysexpics.ru +mystats.xyz +nalogovyy-kodeks.ru +naperehresti.info +nardulan.com +narosty.com +neodownload.webcam +nero-us.com +netallergy.ru +netanalytics.xyz +nfljerseysforsalewholesaler.com +nightvision746.weebly.com +notasprensa.info +nudepatch.net +o-o-11-o-o.com +o-o-8-o-o.com +oakridgemo.com +obiavo.net +ochistka-stokov.ru +oda.as +one.net.in +onetravelguides.com +online-x.ru +onlywoman.org +ontargetseo.us +ooo-gotovie.ru +oranga.host.sk +ourtherapy.ru +oxford-book.com.ua +oz-offers.com +parajumpersoutlet.online +patterntrader-en.com +pavlodar.xkaz.org +pechikamini.ru +penisvergrotendepillennl.ovh +perm.xrus.org +phuketscreen.com +pic-re.blogspot.com +pic2fly.com +picsforkeywordsuggestion.com +pipki.r.acdnpro.com +pistonclasico.com piulatte.cz -piuminiita.com -pizdeishn.com -pl-vouchers.com -plastweb.ru -pochtovyi-index.ru -pomoc-drogowa.cba.pl -pony-business.com -popads.net -poptool.net +plaff-go.ru +pokemongooo.ml +polimga.pw +pooleroadmedicalcentre.co.uk +popmarker.com popugaychiki.com -popup-jdh.xyz -porno-transsexuals.ru -pornosmola.info -preparevideosafesystem4unow.space -pro-okis.ru -prokotov.com -promoforum.ru -pst2017.onlinewebshop.net -psvita.ru -purplestats.com -pushdata.sendpulse.com -putevka24.ru +pornhub-forum.ga +pornohub.me +pornoted.com +postclass.com +powitania.pl +princeadvantagesales.com +producm.ru +prointer.net.ua +pron.pro +purplesphere.in quelle.ru -qwertty.net -rapidsites.pro -reelheroes.net -registrationdomainsite.com -remontgruzovik.ru -rennlist.com -rewardpoll.com -ric.info -richinvestmonitor.com -rusexy.xyz -saddiechoua.com -sajatvelemeny.com -salut-camp.ru -sarm.tk -satellite.maps.ilovevitaly.com -scanhub.ru -schlampen-treffen.com -scoopquest.com -seolab.top -serialsway.ucoz.ru -servethis.com -servicecenter.co.ua -sethrollins.net -sexkontakteao.info -sexspornotub.com -sexxdate.net -shinikiev.com.ua -shitmovs.com -shodanhq.com -shoppytoolmac.com +quickloanbank.com +radiodigital.co +rankscanner.com +ranksonic.org +rarbg.to +ratemodels.net +rcb101.ru +realitykings.com +redirectme.net +referencemoi.com +refudiatethissarah.info +registratciya-v-moskve.ru +remontbiz.ru +renecaovilla.online +renecaovillasale.online +rent2spb.ru +ring4rhino.com +roll123.com +rumamba.com +ruscopybook.com +sack.net +sale-japan.com +scanner-ivan.top +scanner-julia.top +scanner-mary.top +scanner-viktor.top +scrapy.org +sdsjweb.com +searchinquire.com +seasaltwithfood.com +seccioncontrabajo.com +seoexperimenty.ru +seomarketings.online +seorussian.ru +seozoom.it +sex-dating.co +sex-sex-sex5.com +sexphoto.site +sexpornotales.net +sexy-pings.com +shijian.ac.cn +shitting.pro +shlyahten.ru sientalyric.co -signx.info -silvermature.net -simon3.ru -site-speed-check.site -skytraf.xyz -slavkokacunko.de -sledstvie-veli.net -smartbalanceworld.com -snworks.com -snyatie-lomki-v-stacionare.ru -social-s-iii.xyz -socialmediasuggest.com +silvercash.com +simplepooltips.com +sinel.info +slkrm.ru +slonechka.ru +slujbauborki.ru +smailik.org +snip.tw +socialbutton.xyz +socialsignals24.com sofit-dmd.ru -softxaker.ru -sohoindia.net solitaire-game.ru -soserfis.com -soviet-portal.do.am -spasswelt.xyz -spin2016.cf -splendorsearch-a.akamaihd.net -sps-shop.com -stackthatbucks.com -staff.prairiesouth.ca +souvenirua.com +spamnuker.com +spitfiremusic.com +spl63.fr +spravka-medosmotr.ru +spy-app.info +spylog.com +srdrvp.com +srecorder.com +sstroy44.ru +stair.registrydomainservices.com stariy-baku.com -steame.ru -strigkaomsk.ru -stroicol.net -stroy-portal22.ru -sugarkun.com -suggest-keywords.com -superlist.biz -supermesta.ru -symphonyintegratedhealthcare.com +steamoff.net +stroimajor.ru +stuffpride.com +sucsesofinspiration.com +sukirgenk.dvrlists.com +superiends.org +surflinksmedical.com +svtrd.com +swagbucks.com tacbelarus.ru -tako3.com -tanieaukcje.com.pl +tastyfoodideas.com +tds-advert005.info +tecnoteakviareggio.it +teenbbw.yopoint.in telemetryverification.net tetracsaudi.com -tgtclick.com -thebitcoincode.com -thebluffs.com -thefarmergame.com -thefotosgratis.eu -thruport.com -ticketsys.inetwd.com -tizanidineandhydrocodone.blogspot.com -tizanidinecost.blogspot.com -tizanidinehclsideeffects.blogspot.com -tizanidinerestlesslegsyndrome.blogspot.com -token-lab.org -toxicwap.com -tupper-shop.ru -twittruth.com -tzritel.tk -ucoz.ru -umnovocaminho.com -underthesite.com +theendivechronicles.com +thegalerie.eu +thepokertimer.com +tiandeural.ru +tizanidinehcl2mgsideeffects.blogspot.com +tizanidineingredients.blogspot.com +tizanidinepain.blogspot.com +tizanidinezanaflex.blogspot.com +tongkatmadura.info +top-karkas.ru +topcasinoratings.ru +toursmaps.com +trafaret74.ru +trahvid.com +traxdom.ru +trustaffs.com +tsan.net +tsyndicate.com +tuckermktg.com +tvnewsclips.info +txxx.com +ublaze.ru +uggbootsoutletsale.us +uhod-za-sobakoj.ru +ultimateclassicrock.com univerfiles.com -unlimitdocs.net -uptime-delta.net -urccvfmc.bloger.index.hr -urlcut.ru -vacances-voyages.info +uptimebot.net +ural-buldozer.ru +usiad.net valuado.com -vandrie-ict.nl -vbtracker.net -velen.io -video--production.com -video-woman.com -videogamesecrets.com -videojam.tv -vikistars.com -virtuagirl.com -vita.com.hr -viteonlusarezzo.it -vodaodessa.com -vozbujdenie.com -vse-pesni.com -vtc.pw -vzlomfb.com -w3javascript.com -wesharepics.info -whatzmyip.net -wholesalejerseychinashop.com +varbergsvind.se +varikozdok.ru +vchulkax.com +venerologiya.com +versaut.xxx-cam.webcam +vgoloveboli.net +via-energy-cumpara.com +via-energy-order.com +viagroid.ru +video-camer.com +video-hollywood.ru +viel.su +vigrx-original.ru +vilingstore.net +vizag.kharkov.ua +voluumtracker1.com +vpnhowto.info +walpaperlist.com +wdrake.com +web-analytics.date +webads.co.nz +webjam.com +weblibrary.win +wedding0venues.tk +whosonmyserver.com witherrom55.eklablog.fr -wladimirpayen.com -worldoffiles.ru +wonderfulflowers.biz +worldis.me wowas31.ucoz.ru woweb.com.ua -wrona.it -www.bookmaker-bets.com -www.jbetting.com -www.kabbalah-red-bracelets.com -www.wohnkabinen-shop.de -x-lime.com -xn--1-8sbcpb0bdm8k6a.xn--p1ai -xn--80aanaardaperhcem4a6i.com -xn--80ab4aa2g.xn--p1ai -xn--e1afanlbnfckd7c3d.xn--p1ai -xnxx699.com -xtubeporno.net -xz618.com -yebocasino.com -yellowproxy.net +wpsecurity.website +ww1943.ru +wyniki-lista.pl +wzgyyq.com +x-stars.ru +xn----itbeirbjbi7bc6bh2d.xn--p1ai +xn--80aikhbrhr.net +xn--80ajbshivpvn2i.xn--p1ai +xn--b1addnj3cah.xn--p1ai +yhit.press ynymnwbm.bloger.index.hr -zagadki.in.ua -zaim-pod-zalog-krasnodar.ru -zdesoboi.com -zerocash.msk.ru -zeroredirect2.com -zigarettenonlinekaufen.tumblr.com -znaturaloriginal.com +yoluxuryevents.com +youbloodyripper.com +your-bearings.com +yourserverisdown.com +yourtemplatefinder.com +yousense.info +za-music.mymobiplanet.com +zaimhelp.ru +zakazfutbolki.com +zawyna.ua +zeleznobeton.ru +zero1.it +zeroredirect10.com +zixizop.net.ru zooggames.com -zrizvtrnpale.tk +zootoplist.com diff --git a/dev-tools/test_units/random-referrers-for-whitelist-test.list b/dev-tools/test_units/random-referrers-for-whitelist-test.list index 234537336..1a43a0e38 100644 --- a/dev-tools/test_units/random-referrers-for-whitelist-test.list +++ b/dev-tools/test_units/random-referrers-for-whitelist-test.list @@ -1,250 +1,250 @@ -123any.com -6128786.com -7xc4n.com -8gold.com -9icmzvn6.website -aa81bf391151884adfa3dd677e41f94be1.com -above.com -addbin.men -adsland.men -agadelha.com.br -agardomains.com -akama.com -altamayoztourism.com -aludecor.info -am-se.com -arclk.net -astana.xxxkaz.org -autoseo-traffic.com -aviapanda.ru -bablonow.ru -back.dog -balla-boo.se -baltgem.com -basisches-wasser.net -batanga.net -beauby.ru +1001watch.com.ua +10bet.com +4rent.ru +4webmasters.com +9med.net +a14download.com +abclauncher.com +adloads.net +albamargroup.com +alfa-medosmotr.ru +alienwheels.de +allnews24.in +alyeskaresort.com +amateurmatch.com +apelsinnik.website +appearance-cool.com +artparquet.ru +ascat.porn +asrv-a.akamoihd.net +avtolombard-krasnodar.com +baersaratov.ru +bank.uz +bannerconnect.net +bannerspace.com +bayareaaeroclub.org +berlininsl.com +bestempresas.es bestwaystogetpaid.us -bioca.org -bjanshee.ru -boleznikogi.com +biznesluxe.ru +blackwitchcraft.ru +bmsco.co +brimstonehillfortress.org +brus-vsem.ru +bugshoot.cn bupropion-sr-150-mg.us -buybest1.biz -buynorxx.com -buzzsumo.com -cablecar.us -cannibalcheerleader.com -cercacamion.it -cerev.info -certifywebsite.win -checktext.org -christianlouboutinreplica.pw -cibpenonptib.flu.cc -cleansearch.net -clickbank.net -comeondog.info -compliance-fred.top -comsysnet.com -coolgramgoods.com -copenergo.ru -coprofam.org -cs-passion.pl -cscwtalkto.site -cyclobenzaprinevstizanidine.blogspot.com -danschawbel.com -datadepths.com -dekoration.us -digitalassetmanagement.site -discovertreasure-a.akamaihd.net -dispo.de -dscaas.website -dvr.biz.ua -earn-from-articles.com -ee77ee.com -ekspertmed.com -el-nation.com -elektrischeziga.livejournal.com -elektrischezigarette2.devhub.com -elite-sex-finders.com -elitepcgames.com -emailaccountlogin.co -europeanwatches.ru -excaliburfilms.com -expolicenciaslatam.co -failingmarriege.blogspot.com -fashionindeed.ml -fetlifeblog.com -findsexguide.com -franecki.net -franklinfire.co -freza-sverlo.ru -fuck-paid-share-buttons.xyz -funponsel.com -fusoradio.info -fx-brokers-review.com -g7m.pl -gamevalue7.weebly.com -gay-site.store +buron.pw +bus-offer.com +buy-meds24.com +buyfriend.ru +buyscabiescream.com +calcularpagerank.com +callejondelpozo.es +canrioloadm.gq +casinorewards.info +cgi2.nintendo.co.jp +chastnoeporno.com +chatroulette.video +chidporn.com +clean-start.net +clippingphotoindia.com +compliance-alexa.top +compucelunlock.net +coslab.club +crackguru.tk +dairyindia.in +date-withme.com +dawlenie.com +discovertreasurenow.com +divisioncore.com +drev.biz +eaplay.ru +easycommerce.cf +elektrischezi.canalblog.com +emergencyneeds.org +ereko.ru +erotiktreff24.info +essay-writing.work +exportshop.us +ezigarettekaufen2.mpbloggar.se +ezigarettenkaufen1.deviantart.com +fashion.stellaconstance.co +filerockstar298.weebly.com +financeloan.us +findthe.pet +flavors.me +formulaf1results.blogspot.com +forum.doctissimo.fr +forum.tvmir.org +forum20.smailik.org +freevpn.space +front.ru +garciniaxt.us gdzkurokam.ru -gheus.altervista.org -gidonline.one -gk170.ru -globatur.ru -globetrotting-culture.ru -googlemare.com -gopro-online.info -greenidesign.co -homegardenlova.com -hotel-mkad.ru -hotxnights.info -imadedinner.net -infokonkurs.ru -ipchicken.com -iplusbit.blogspot.co.za -itrevolution.cf -jackwolfskinoutlet.online -juliadiets.com -keywords-monitoring-success.com -kinogonew.ru -klin3952.ru -kochanelli.com -konica.kz -krassh.ru -lashstudia.ru -letmacwork.world -letsrepair.in -lineavabit.it -linkpulse.com -linkredirect.biz -lion.conboy.us -lion.misba.us +ge0ip.com +geileweiber.tk +getyourimage.club +golden-praga.ru +gosmeb.ru +gowreckdiving.com +handicapvansarea.com +hidefiles.org +hmmm.cz +homeart.space +hop.clickbank.net +housekuba.org +htmlcorner.com +hustoon.over-blog.com +hydropump.su +ibb.com.ua +ibmdatamanagement.co +images-graphics-pics.com +imediadesk.com +immigrational.info +incomekey.net +istripper.com +james13prix.info +jav-fetish.site +juliaworld.net +junglenet-a.akamaihd.net +jyvopys.com +kam-dom.ru +kamagragelusa.net +karpun-iris.ru +keywordspay.com +kit-opt.ru +lalalove.ru +laptop-4-less.com +law-check-eight.xyz +law-check-seven.xyz +laxob.com llastbuy.ru -longgreen.info -madot.onlinewebshop.net -manualterap.roleforum.ru -matpre.top -max-p.men -maximilitary.ru -mb140.ru -michellblog.online -monetizer.com-01.site -moneytop.ru -montazhnic.ru -moonci.ru -moscow-region.ru -mtmtv.info -myghillie.info -mypets.by -negociosdasha.com -new-post.tk -nikitsyringedrivelg.pen.io -nodup.ru -notasprensa.info -nucia.biz.ly -nuup.info -okayimage.com -onlythegames.com -openlibrary.org -outpersonals.com -oxford-book.com.ua -pensplan4u.com -picturesboss.com -playlott.com -pornoblood.com -pp-budpostach.com.ua -priceg.com -prizeestates.cricket -prosmibank.ru -push-ad.com -pxhdwsm.com -rankscanner.com -rcb101.ru -recipedays.com +long-beach-air-conditioning.com +lostaruban.com +louboutinbooties.xyz +maranbrinfo.com.br +mebeldekor.com.ua +mediafresh.online +missclub.info +mobicover.com.ua +moviemail-online.co.uk +mxgetcode.com +national-today-winning-winner.club +nbsproject.ru +nextlnk12.com +ntic.fr +nw-servis.ru +offf.info +one-gear.com +orsonet.ru +outshop.ru +oyster-green.com +panchro.co.uk +peekyou.com +pekori.to +perl.dp.ua +physfunc.ru +pierrehardysale.online +podshipniki-ntn.ru +poligon.com +polybuild.ru +popander.mobi +popcash.net +pornhive.org +pornogratisdiario.com +pornoklad.ru +pornosee.info +pornosexrolik.com +pornpost.in +pregnant.guru +privatov-zapisi.ru +pro-poly.ru +prokotov.com +promover.org +prostitutki-kharkova.org +prostitutki-tolyatti.org +pyramidlitho.webs.com +quebec-bin.com +ranking2017.ga +regionshop.biz registry-clean-up.net -registrysweeper.com -rockma.se -rybalka-opt.ru -sammlungfotos.online -santasgift.ml -scanmarine.info -scanner-marwin.top -screpy.com -scripted.com -seeingmeerkat.com -seemoreresultshu-a.akamaihd.net -seo-prof1.xyz -seozoom.it -service-core.ru -sexyteens.hol.es -sharpchallenge.com -shymkent.xkaz.org -silverage.ru -siteripz.net -skanninge.se -skatestick.bid -smart-balancewheel.com -social-s-ggg.xyz -softomix.net -ssconstruction.co -stairliftstrue.com -stopnarco.ru -streha-metalko.si -superiends.org -supernew.org -surgut.zrus.org -svetlotorg.ru -taxi-v-eisk.ru -teguh.info -thecoolimages.net -tizanidinehydrochloride2mg.blogspot.com -tizanidineorflexeril.blogspot.com -tizanidinestreetvalue.blogspot.com -topcar-krasnodar.ru -topkarkas.com -torrents.life -trafficmp.com -treasuretrack-a.akamaihd.net -trichizobswiv.agddns.net -trumpetedextremes.com +remont-ustanovka-tehniki.ru +roma-kukareku.livejournal.com +rome2rio.com +rozalli.com +ruclicks.com +rusprostitute.com +rutor.group +rvtv.ru +sadaholding.com +sardinie.us +sbf441.com +sbricur.com +scat.porn +seasaltwithfood.com +selectads.men +seocdvig.ru +sexgalleries.top +sexyali.com +share-buttons-for-free.com +shemalegalls.blogporn.in +shoppytoolmac.com +shtaketniki.kz +simul.co +snyatie-lomki-v-stacionare.ru +space4updating.win +stanthonyscatholicchurch.org +stretchingabuckblog.com +stroydetali.ru +sudexpert66.ru +superstats.com +the-torrent-tracker.blogspot.com +theallgirlarcade.com +thegameriders.com +tizanidinehcl2mgsideeffects.blogspot.com +tomatis.gospartner.com +torrnada.ru +trackmedia101.com +truemfilelj.gq trustedhealthtips.com -tubo360.com -turizm.bz -twojebook.pl -um-razum.ru -uzporno.mobi -via-gra.webstarts.com -vip2ch.com -vksaver-all.ru -vod.com.ua -vodkoved.ru -voprosotvet24.ru -vostoktrade.info -wait3sec.org -warningzscaler.heraeus.com -webenlace.com.ar -webmasterhome.cn -weboptimizes.com -wedding-salon.net -wedding0venues.tk -westen-v.life -wfb.hatedriveapart.com -whatzmyip.net -wingsoffury2.com -withstandingheartwarming.com -woman-h.ru -worldoffiles.ru -x-mix.info -xn----8sbarihbihxpxqgaf0g1e.xn--80adxhks -xn----9sbebi2bvzr7h.xn--p1ai -xn----9sbubg3ambdfl1j.xn--p1ai -xn--80aafb2a.xn--p1ai -xn--b1agm2d.net -xvideosporn.biz -xxxnatelefon.ru -youtoner.it -youtube-downloader.savetubevideo.com -zatjmuzu.info -zigarettenonlinekaufen1.bloog.pl -zoogdiesney.com -zoominfo.com -zx6.ru +tvorozhnaja-zapekanka-recept.ru +twittruth.com +udsgame.online +ukrtextbook.com +ultimateclassicrock.com +univerfiles.com +valkiria-tk.ru +vapeface.club +viandpet.com +videochat.tv.br +villakohlanta.nu +voluumtrk.com +vzubah.com +wallpaperaccess.com +wapsite.me +waterpurifier.club +webnode.me +wejdz-tu.pl +whatsupinfoley.com +wildcattube.com +wordkeyhelper.com +worst-sites.online +wygraj-teraz.com +wzgyyq.com +xchangetrak.com +xchat26.myfreecams.com +xjrul.com +xlolitka.com +xn-----7kcabaipgeakzcss7bjdqdwpfnhv.xn--p1ai +xn----7sbaaabaei0cc8aj5bj0bncejx.xn--p1ai +xn----8sblgmbj1a1bk8l.xn----161-4vemb6cjl7anbaea3afninj.xn--p1ai +xn----ctbigni3aj4h.xn--p1ai +xn--b1ag5cfn.xn--p1ai +xn--h1ahbi.com.ua +xnxx-n.com +xtubeporno.net +xuki.us +ytmnd.com +zastroyka.org +zocaparj.kz +zryydi.com diff --git a/dev-tools/test_units/whitelist-ips.conf b/dev-tools/test_units/whitelist-ips.conf index 447da2bf0..324b1a098 100644 --- a/dev-tools/test_units/whitelist-ips.conf +++ b/dev-tools/test_units/whitelist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.69 0; +10.1.0.183 0; 127.0.0.1 0; diff --git a/dev-tools/tmp/robots.txt b/dev-tools/tmp/robots.txt index b13c86282..0539ff17a 100644 --- a/dev-tools/tmp/robots.txt +++ b/dev-tools/tmp/robots.txt @@ -6,8 +6,8 @@ ### Version Information # ################################################### -### Version: V4.2022.02.2930 -### Updated: Sun Feb 20 10:05:19 UTC 2022 +### Version: V4.2022.02.2931 +### Updated: Sun Feb 20 10:31:54 UTC 2022 ### Bad Bot Count: 636 ################################################### ### Version Information ## @@ -1161,7 +1161,7 @@ User-agent: check1.exe Disallow:/ User-agent: clark-crawler Disallow:/ -User-agent: coccocbot-web +User-agent: coccocbot Disallow:/ User-agent: cognitiveseo Disallow:/ diff --git a/robots.txt/robots.txt b/robots.txt/robots.txt index b13c86282..0539ff17a 100644 --- a/robots.txt/robots.txt +++ b/robots.txt/robots.txt @@ -6,8 +6,8 @@ ### Version Information # ################################################### -### Version: V4.2022.02.2930 -### Updated: Sun Feb 20 10:05:19 UTC 2022 +### Version: V4.2022.02.2931 +### Updated: Sun Feb 20 10:31:54 UTC 2022 ### Bad Bot Count: 636 ################################################### ### Version Information ## @@ -1161,7 +1161,7 @@ User-agent: check1.exe Disallow:/ User-agent: clark-crawler Disallow:/ -User-agent: coccocbot-web +User-agent: coccocbot Disallow:/ User-agent: cognitiveseo Disallow:/