diff --git a/.dev-tools/beta_conf_files/conf.d/globalblacklist.conf b/.dev-tools/beta_conf_files/conf.d/globalblacklist.conf index 1ac3d7dbe..dfc19e253 100644 --- a/.dev-tools/beta_conf_files/conf.d/globalblacklist.conf +++ b/.dev-tools/beta_conf_files/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2019.07.1776 -### Updated: Wed Jul 31 09:58:07 SAST 2019 +### Version: V3.2019.07.1777 +### Updated: Wed Jul 31 12:28:46 SAST 2019 ### Bad Referrer Count: 6740 ### Bad Bot Count: 556 ################################################### diff --git a/.dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf b/.dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf index 6893ce0fc..2e2965ffc 100644 --- a/.dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf +++ b/.dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -35.192.136.167 1; +35.188.73.34 1; 127.0.0.1 1; diff --git a/.dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf b/.dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf index 879967dc9..d3d331ed4 100644 --- a/.dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf +++ b/.dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -35.192.136.167 0; +35.188.73.34 0; 127.0.0.1 0; diff --git a/.dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf b/.dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf index 1ac3d7dbe..dfc19e253 100644 --- a/.dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf +++ b/.dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2019.07.1776 -### Updated: Wed Jul 31 09:58:07 SAST 2019 +### Version: V3.2019.07.1777 +### Updated: Wed Jul 31 12:28:46 SAST 2019 ### Bad Referrer Count: 6740 ### Bad Bot Count: 556 ################################################### diff --git a/.dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf b/.dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf index 6893ce0fc..2e2965ffc 100644 --- a/.dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf +++ b/.dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -35.192.136.167 1; +35.188.73.34 1; 127.0.0.1 1; diff --git a/.dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf b/.dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf index 879967dc9..d3d331ed4 100644 --- a/.dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf +++ b/.dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -35.192.136.167 0; +35.188.73.34 0; 127.0.0.1 0; diff --git a/.dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf b/.dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf index 1ac3d7dbe..dfc19e253 100644 --- a/.dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf +++ b/.dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2019.07.1776 -### Updated: Wed Jul 31 09:58:07 SAST 2019 +### Version: V3.2019.07.1777 +### Updated: Wed Jul 31 12:28:46 SAST 2019 ### Bad Referrer Count: 6740 ### Bad Bot Count: 556 ################################################### diff --git a/.dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf b/.dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf index 6893ce0fc..2e2965ffc 100644 --- a/.dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf +++ b/.dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -35.192.136.167 1; +35.188.73.34 1; 127.0.0.1 1; diff --git a/.dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf b/.dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf index 879967dc9..d3d331ed4 100644 --- a/.dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf +++ b/.dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -35.192.136.167 0; +35.188.73.34 0; 127.0.0.1 0; diff --git a/.dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf b/.dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf index 1ac3d7dbe..dfc19e253 100644 --- a/.dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf +++ b/.dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2019.07.1776 -### Updated: Wed Jul 31 09:58:07 SAST 2019 +### Version: V3.2019.07.1777 +### Updated: Wed Jul 31 12:28:46 SAST 2019 ### Bad Referrer Count: 6740 ### Bad Bot Count: 556 ################################################### diff --git a/.dev-tools/globalblacklist-testing.template b/.dev-tools/globalblacklist-testing.template index 1ac3d7dbe..dfc19e253 100644 --- a/.dev-tools/globalblacklist-testing.template +++ b/.dev-tools/globalblacklist-testing.template @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2019.07.1776 -### Updated: Wed Jul 31 09:58:07 SAST 2019 +### Version: V3.2019.07.1777 +### Updated: Wed Jul 31 12:28:46 SAST 2019 ### Bad Referrer Count: 6740 ### Bad Bot Count: 556 ################################################### diff --git a/.dev-tools/globalblacklist.template b/.dev-tools/globalblacklist.template index a6fec1418..113f5f1f8 100755 --- a/.dev-tools/globalblacklist.template +++ b/.dev-tools/globalblacklist.template @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2019.07.1776 -### Updated: Wed Jul 31 09:58:06 SAST 2019 +### Version: V4.2019.07.1777 +### Updated: Wed Jul 31 12:28:45 SAST 2019 ### Bad Referrer Count: 6740 ### Bad Bot Count: 556 ################################################### diff --git a/.dev-tools/setup-ngxblocker/commented/botblocker-nginx-settings.conf b/.dev-tools/setup-ngxblocker/commented/botblocker-nginx-settings.conf index e69de29bb..883bc7688 100644 --- a/.dev-tools/setup-ngxblocker/commented/botblocker-nginx-settings.conf +++ b/.dev-tools/setup-ngxblocker/commented/botblocker-nginx-settings.conf @@ -0,0 +1,31 @@ +############################################################################## +# _ __ _ # +# / |/ /__ _(_)__ __ __ # +# / / _ `/ / _ \\ \ / # +# /_/|_/\_, /_/_//_/_\_\ # +# __/___/ __ ___ __ ___ __ __ # +# / _ )___ ____/ / / _ )___ / /_ / _ )/ /__ ____/ /_____ ____ # +# / _ / _ `/ _ / / _ / _ \/ __/ / _ / / _ \/ __/ '_/ -_) __/ # +# /____/\_,_/\_,_/ /____/\___/\__/ /____/_/\___/\__/_/\_\\__/_/ # +# # +############################################################################## + +# Version 1.1 +# ! new directives also to be added to include_filelist.txt ! # + +server_names_hash_bucket_size 256; +server_names_hash_max_size 4096; +variables_hash_max_size 4096; +variables_hash_bucket_size 4096; +limit_req_zone $binary_remote_addr zone=flood:50m rate=90r/s; +limit_conn_zone $binary_remote_addr zone=addr:50m; + +# **************************************************************************** +# NOTE: IF you are using a system like Nginx-Proxy from @JWilder +# **************************************************************************** +# Repo URL: https://github.com/jwilder/nginx-proxy +# You will need to comment out the first line here as follows. +# #server_names_hash_bucket_size 128; +# You will also need to modify the nginx.tmpl file to add the default include +# include /etc/nginx/conf.d/* +# **************************************************************************** diff --git a/.dev-tools/test1_conf_files/bots.d/whitelist-ips.conf b/.dev-tools/test1_conf_files/bots.d/whitelist-ips.conf index f0d7ffecc..0f09ea3af 100644 --- a/.dev-tools/test1_conf_files/bots.d/whitelist-ips.conf +++ b/.dev-tools/test1_conf_files/bots.d/whitelist-ips.conf @@ -45,4 +45,4 @@ # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST -35.192.187.174 0; +35.192.136.167 0; diff --git a/.dev-tools/test1_conf_files/conf.d/globalblacklist.conf b/.dev-tools/test1_conf_files/conf.d/globalblacklist.conf index a6fec1418..113f5f1f8 100644 --- a/.dev-tools/test1_conf_files/conf.d/globalblacklist.conf +++ b/.dev-tools/test1_conf_files/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2019.07.1776 -### Updated: Wed Jul 31 09:58:06 SAST 2019 +### Version: V4.2019.07.1777 +### Updated: Wed Jul 31 12:28:45 SAST 2019 ### Bad Referrer Count: 6740 ### Bad Bot Count: 556 ################################################### diff --git a/.dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf b/.dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf index b9d24324a..eabb2e30b 100644 --- a/.dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf +++ b/.dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -34.68.144.114 1; +104.198.131.58 1; 127.0.0.1 1; diff --git a/.dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf b/.dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf index ea117da9f..6bad13743 100644 --- a/.dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf +++ b/.dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -34.68.144.114 0; +104.198.131.58 0; 127.0.0.1 0; diff --git a/.dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf b/.dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf index a6fec1418..113f5f1f8 100644 --- a/.dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf +++ b/.dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2019.07.1776 -### Updated: Wed Jul 31 09:58:06 SAST 2019 +### Version: V4.2019.07.1777 +### Updated: Wed Jul 31 12:28:45 SAST 2019 ### Bad Referrer Count: 6740 ### Bad Bot Count: 556 ################################################### diff --git a/.dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf b/.dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf index b9d24324a..eabb2e30b 100644 --- a/.dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf +++ b/.dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -34.68.144.114 1; +104.198.131.58 1; 127.0.0.1 1; diff --git a/.dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf b/.dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf index ea117da9f..6bad13743 100644 --- a/.dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf +++ b/.dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -34.68.144.114 0; +104.198.131.58 0; 127.0.0.1 0; diff --git a/.dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf b/.dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf index a6fec1418..113f5f1f8 100644 --- a/.dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf +++ b/.dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2019.07.1776 -### Updated: Wed Jul 31 09:58:06 SAST 2019 +### Version: V4.2019.07.1777 +### Updated: Wed Jul 31 12:28:45 SAST 2019 ### Bad Referrer Count: 6740 ### Bad Bot Count: 556 ################################################### diff --git a/.dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf b/.dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf index b9d24324a..eabb2e30b 100644 --- a/.dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf +++ b/.dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -34.68.144.114 1; +104.198.131.58 1; 127.0.0.1 1; diff --git a/.dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf b/.dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf index ea117da9f..6bad13743 100644 --- a/.dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf +++ b/.dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -34.68.144.114 0; +104.198.131.58 0; 127.0.0.1 0; diff --git a/.dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf b/.dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf index a6fec1418..113f5f1f8 100644 --- a/.dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf +++ b/.dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2019.07.1776 -### Updated: Wed Jul 31 09:58:06 SAST 2019 +### Version: V4.2019.07.1777 +### Updated: Wed Jul 31 12:28:45 SAST 2019 ### Bad Referrer Count: 6740 ### Bad Bot Count: 556 ################################################### diff --git a/.dev-tools/test2_conf_files/bots.d/whitelist-ips.conf b/.dev-tools/test2_conf_files/bots.d/whitelist-ips.conf index 84fb2a6f7..137da33fd 100644 --- a/.dev-tools/test2_conf_files/bots.d/whitelist-ips.conf +++ b/.dev-tools/test2_conf_files/bots.d/whitelist-ips.conf @@ -45,5 +45,5 @@ # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST -104.154.113.151 0; -104.198.131.58 0; +34.66.178.120 0; +34.68.144.114 0; diff --git a/.dev-tools/test2_conf_files/conf.d/globalblacklist.conf b/.dev-tools/test2_conf_files/conf.d/globalblacklist.conf index 557afeffa..558afe553 100644 --- a/.dev-tools/test2_conf_files/conf.d/globalblacklist.conf +++ b/.dev-tools/test2_conf_files/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2019.07.1776 -### Updated: Wed Jul 31 09:58:06 SAST 2019 +### Version: V4.2019.07.1777 +### Updated: Wed Jul 31 12:28:45 SAST 2019 ### Bad Referrer Count: 6740 ### Bad Bot Count: 556 ################################################### diff --git a/.dev-tools/test3_conf_files/bots.d/whitelist-ips.conf b/.dev-tools/test3_conf_files/bots.d/whitelist-ips.conf index 84fb2a6f7..e521bd11b 100644 --- a/.dev-tools/test3_conf_files/bots.d/whitelist-ips.conf +++ b/.dev-tools/test3_conf_files/bots.d/whitelist-ips.conf @@ -45,5 +45,5 @@ # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST -104.154.113.151 0; -104.198.131.58 0; +35.192.136.167 0; +35.192.187.174 0; diff --git a/.dev-tools/test3_conf_files/conf.d/globalblacklist.conf b/.dev-tools/test3_conf_files/conf.d/globalblacklist.conf index e1d61c996..e36d30523 100644 --- a/.dev-tools/test3_conf_files/conf.d/globalblacklist.conf +++ b/.dev-tools/test3_conf_files/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2019.07.1776 -### Updated: Wed Jul 31 09:58:06 SAST 2019 +### Version: V4.2019.07.1777 +### Updated: Wed Jul 31 12:28:45 SAST 2019 ### Bad Referrer Count: 6740 ### Bad Bot Count: 556 ################################################### diff --git a/.dev-tools/test4_conf_files/bots.d/whitelist-ips.conf b/.dev-tools/test4_conf_files/bots.d/whitelist-ips.conf index 81d44ffba..f1c0e43d7 100644 --- a/.dev-tools/test4_conf_files/bots.d/whitelist-ips.conf +++ b/.dev-tools/test4_conf_files/bots.d/whitelist-ips.conf @@ -45,5 +45,5 @@ # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST -35.188.73.34 0; 35.193.14.140 0; +35.192.136.167 0; diff --git a/.dev-tools/test4_conf_files/conf.d/globalblacklist.conf b/.dev-tools/test4_conf_files/conf.d/globalblacklist.conf index a6fec1418..113f5f1f8 100644 --- a/.dev-tools/test4_conf_files/conf.d/globalblacklist.conf +++ b/.dev-tools/test4_conf_files/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2019.07.1776 -### Updated: Wed Jul 31 09:58:06 SAST 2019 +### Version: V4.2019.07.1777 +### Updated: Wed Jul 31 12:28:45 SAST 2019 ### Bad Referrer Count: 6740 ### Bad Bot Count: 556 ################################################### diff --git a/.dev-tools/test5_conf_files/bots.d/whitelist-ips.conf b/.dev-tools/test5_conf_files/bots.d/whitelist-ips.conf index 1e1c557a4..7a9d57444 100644 --- a/.dev-tools/test5_conf_files/bots.d/whitelist-ips.conf +++ b/.dev-tools/test5_conf_files/bots.d/whitelist-ips.conf @@ -45,5 +45,5 @@ # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST -34.68.144.114 0; -35.192.136.167 0; +104.198.131.58 0; +35.188.73.34 0; diff --git a/.dev-tools/test5_conf_files/conf.d/globalblacklist.conf b/.dev-tools/test5_conf_files/conf.d/globalblacklist.conf index a6fec1418..113f5f1f8 100644 --- a/.dev-tools/test5_conf_files/conf.d/globalblacklist.conf +++ b/.dev-tools/test5_conf_files/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2019.07.1776 -### Updated: Wed Jul 31 09:58:06 SAST 2019 +### Version: V4.2019.07.1777 +### Updated: Wed Jul 31 12:28:45 SAST 2019 ### Bad Referrer Count: 6740 ### Bad Bot Count: 556 ################################################### diff --git a/.dev-tools/test_units/blacklist-ips.conf b/.dev-tools/test_units/blacklist-ips.conf index 6893ce0fc..2e2965ffc 100644 --- a/.dev-tools/test_units/blacklist-ips.conf +++ b/.dev-tools/test_units/blacklist-ips.conf @@ -1,2 +1,2 @@ -35.192.136.167 1; +35.188.73.34 1; 127.0.0.1 1; diff --git a/.dev-tools/test_units/random-bots-for-test-quick.list b/.dev-tools/test_units/random-bots-for-test-quick.list index 34fa0b8fa..625d5f7a2 100644 --- a/.dev-tools/test_units/random-bots-for-test-quick.list +++ b/.dev-tools/test_units/random-bots-for-test-quick.list @@ -1,10 +1,10 @@ -ASPSeek -DomainSigmaCrawler -MarkWatch -Morfeus Fucking Scanner -MSIECrawler -PageScorer -PxBroker -Quick-Crawler +Abonti +ChinaClaw +Craftbot +FrontPage +IDBot +Microsoft Data Access +Nibbler Titan -WinHTTrack +UnisterBot +zauba.io diff --git a/.dev-tools/test_units/random-bots-for-test.list b/.dev-tools/test_units/random-bots-for-test.list index 538c22aa7..ff7c106cb 100644 --- a/.dev-tools/test_units/random-bots-for-test.list +++ b/.dev-tools/test_units/random-bots-for-test.list @@ -1,250 +1,250 @@ +360Spider 404checker -404enemy -Abonti -Aboundexbot -Acunetix +80legs +Aboundex AfD-Verbotsverfahren -AIBOT -AiHitBot -Alexibot +Aipbot +Alligator AlphaBot -Anarchie -Apexoo archive.org_bot -ASPSeek -Attach -autoemailspider -Backlink-Ceck +Asterias BacklinkCrawler -BackWeb -Badass -Bandit -BatchFTP +Barkrowler +BBBike BDCbot -BDFetch -Bigfoot Black Hole -BLEXBot -BotALot +BlackWidow Brandprotect Brandwatch +BuiltWith Bullseye -BunnySlippers BuzzSumo -Calculon +CATExplorador CazoodleBot CCBot -CheTeam -Chlooe +CheeseBot +CherryPicker +ChinaClaw Cliqzbot -cognitiveseo -Craftbot -crawler4j +coccocbot-web +Cogentbot +Copier +CopyRightCheck crawler.feedback -crawl.sogou.com CrazyWebCrawler -Custo -DatabaseDriverMysqli -demandbase-bot -Demon -Deusu +Crescent +CSHttp +Devil Digincore +DigitalPebble +Dirbuster Disco Dispatch DittoSpyder +DnyzBot DomainAppender DomainCrawler -Dotbot -Download Wonder -DTS Agent -ECCP/1.0 -EMail Wolf +DSearch +Ebingbong +Ecxi +EirGrabber +evc-batch +Exabot +Express WebPictures Extractor +ExtractorPro Extreme Picture Finder +EyeNetIE +FDM FHscan -FlashGet -Flunky -Foobot +Fimap +FrontPage +FyberSpider Fyrebot -Genieo GermCrawler +Getintent Gigabot GoZilla -Grabber +Go!Zilla +GrabNet +GrapeFX GrapeshotCrawler GridBot -GT::WWW -Haansoft -HaosouSpider +Harvest Heritrix Hloader -HMView +HTMLparser HTTP::Lite Humanlinks -Image Fetch +HybridBot +Iblog +IDBot +Image Sucker IndeedBot Indy Library +InfoNaviRobot +instabid Intelliseek -Internet Ninja +InternetSeer internetVista monitor -ips-agent -Iria IRLbot IstellaBot Jbrofuzz -JetCar +Jetty +JikeSpider +Joomla JustView -Jyxobot -Kenjin Spider -Keyword Density -Larbin -LeechGet +Lanshanbot +LexiBot +Lftp +LibWeb +Libwhisker Lightspeedsystems Likse -Linkdexbot -LinkextractorPro -LinkpadBot -LinksManager LinkWalker -LinqiaRSSBot -Lipperhey -Lipperhey Spider +LinqiaScrapeBot +Litemage_walker Lmspider -LNSpiderguy Ltx71 lwp-request LWP::Simple lwp-trivial +Magnet Mag-Net magpie-crawler -Mail.RU_Bot -Majestic12 Majestic SEO -Majestic-SEO -MarkWatch -Masscan +MarkMonitor Mass Downloader +Mata Hari MauiBot -Meanpathbot -Metauri +MeanPath Bot +Mediatoolkitbot +MFC_Tear_Sample MIDown tool -MIIxpc Mister PiX -Mojeek Mr.4x3 -MSFrontPage +MS Web Services Client Protocol muhstik-scan +Musobot +Name Intelligence +Nameprotect Navroad NearSite +Needle +netEstate NE Crawler NetLyzer +NetSpider Net Vampire -Niki-bot -Ninja +NextGenSearchBot +Nibbler +NimbleCrawler +Nmap +NPbot oBot -Offline Explorer -Offline Navigator OnCrawl OpenLinkProfiler Openvas OpenVAS -OutclicksBot +OrangeBot +OrangeSpider PageAnalyzer -Panscient -Papa Foto -PeoplePal -PHPCrawl -Picscout +PageGrabber +page scorer +Pandalytics +PictureFinder Pimonster +Pi-Monster +Pixray plumanalytics -POE-Component-Client-HTTP -Probethenet -Psbot +Pockey +ProPowerBot +PyCurl Quick-Crawler -RankActive +RankActiveLinkBot RankFlex -RankingBot Rankivabot -RankurBot -Reaper +RebelMouse Recorder -RepoMonkey +RedesScrapy Ripper -SBIder +SalesIntelligent ScanAlert scan.lol +ScoutJet Scrapy Screaming -ScreenerBot -Searchestate -Semrush +SearchmetricsBot SemrushBot -SEOkicks-Robot -SEOstats +SEOkicks +seoscanners +serpstatbot sexsearcher -Siphon -SiteExplorer +Sitebeam +Siteimprove +SiteLockSpider +SiteSnagger +SiteSucker Site Sucker -SlySearch -Snake +Sitevigil +SmartDownload +SMTBot sogouspider -Sogou web spider -SpaceBison -Spanner -sp_auditbot -Spinn3r -spyfu +Sosospider +Spammen +SpankBot +SputnikBot Sqlmap -Sqworm -Sucker -Sucuri +Sqlworm +Stripper +SuperBot SurveyBot -sysscan -Szukacz -T8Abot tAkeOut +Telesoft Telesphoreo +Telesphorep +The Intraformant Thumbor -Toweyabot -Trendiction +Tracemyfile Trendictionbot +Turingos +Turnitin TurnitinBot +Twice Typhoeus +UnisterBot Upflow +URLy Warning +URLy.Warning +Vacuum +Vagabondo VCI -VidibleScraper -Virusdie -VoidEYE -Voltron +Wallpapers/3.0 WallpapersHD WBSearchBot -Webalta -WebCollage -WEBDAV +Web Collage WebEnhancer -Web Fuck -WebLeacher -WebmasterWorldForumBot +Web Fetch +WebPix WebReaper Web Sauger Webshag +WebsiteExtractor WebsiteQuester Webster -WebWhacker -WebZIP -WeSEE +Web Sucker +Whack Who.is Bot -Widow +WinHTTrack +WiseGuys Robot +WISENutbot Woobot -Wotbox WPScan +WWW-Collector-E +WWW::Mechanize +WWWOFFLE x22Mozilla -Xaldon WebSpider Xaldon_WebSpider -Xenu YoudaoBot -Zade -zauba.io -Zermelo zgrab -Zitebot +ZumBot ZyBorg diff --git a/.dev-tools/test_units/random-bots-for-whitelist-test.list b/.dev-tools/test_units/random-bots-for-whitelist-test.list index a86bbd5a8..df2307c3b 100644 --- a/.dev-tools/test_units/random-bots-for-whitelist-test.list +++ b/.dev-tools/test_units/random-bots-for-whitelist-test.list @@ -1,250 +1,250 @@ 360Spider -404enemy -Aboundex +404checker Aboundexbot -ADmantX AfD-Verbotsverfahren +AhrefsBot AllSubmitter -AlphaBot -Attach -autoemailspider +Anarchie +Apexoo +ASPSeek +Asterias BackDoorBot +Backlink-Ceck BacklinkCrawler BackStreet -Badass -BatchFTP -BDCbot -Bigfoot -Black Hole -BlackWidow +Barkrowler +BBBike +BDFetch +BetaBot +Bitacle +Blow +BlowFish Boardreader +Bolt +BotALot Brandprotect -Buddy -BuiltBotTough +Brandwatch BuiltWith +BunnySlippers +Calculon CATExplorador -CazoodleBot -CCBot -CheeseBot +Cegbfeieh CheTeam ChinaClaw Chlooe -Claritybot -coccocbot-web -Cogentbot +cognitiveseo Collector +Copier CopyRightCheck -Cosmos -Craftbot -crawler.feedback -crawl.sogou.com -Crescent -Curious -DatabaseDriverMysqli +Copyscape +CrunchBot +CSHttp DataCha0s +DBLBot demandbase-bot Deusu -Devil -Dirbuster -Discobot -DomainAppender -DomainStatsBot +Digincore +DigitalPebble +DIIbot +Dispatch +DittoSpyder +DomainCrawler +DomainSigmaCrawler Dotbot -Dragonfly +Download Wonder Drip -DTS Agent EasyDL -evc-batch -Evil -Exabot -ExtractorPro -Extreme Picture Finder -EyeNetIE -facebookscraper +eCatch +EMail Wolf FemtosearchBot Flunky -FrontPage +GalaxyBot Genieo -GermCrawler -GetWeb -Gigabot +Getintent +GetRight +Gigablast Go-Ahead-Got-It -Gotit GoZilla +Grabber +GrapeFX GT::WWW -Haansoft -HaosouSpider -Harvest -HTMLparser -HTTP::Lite +Havij +HEADMasterSEO +Heritrix +Hloader HTTrack Humanlinks IDBot -Image Sucker -Indy Library +IlseBot +Image Fetch InfoNaviRobot -Intelliseek -Internet Ninja +InfoTekies +InternetSeer internetVista monitor ips-agent -Iria -IRLbot -Iskanie IstellaBot -JamesBOT -Jorgee -JustView +Jbrofuzz +JennyBot +Jetty +JikeSpider Kenjin Spider -Kozmosbot Larbin +LeechFTP +LeechGet LexiBot -Lftp -LibWeb -Likse -Linkdexbot +Libwhisker LinkScan -LinkWalker -Lipperhey +LinksManager +LinqiaMetadataDownloaderBot Lmspider -Ltx71 +lwp-request LWP::Simple -lwp-trivial -magpie-crawler +Magnet Mail.RU_Bot Majestic12 -Majestic-SEO +Majestic SEO MarkMonitor -MarkWatch -Masscan -meanpathbot -Meanpathbot -mediawords +Mass Downloader +Mata Hari +Mediatoolkitbot MegaIndex.ru -Metauri MFC_Tear_Sample -MIDown tool -MIIxpc +Microsoft URL Control Mister PiX +MJ12bot Morfeus Fucking Scanner +Mr.4x3 +MSFrontPage +MSIECrawler +Msrabot MS Web Services Client Protocol -Nameprotect -NearSite +muhstik-scan +Musobot +Name Intelligence Nessus NetAnts -Netvibes -NetZIP -Nibbler -NICErsPRO -NimbleCrawler -OnCrawl -Openfind +Netcraft +NetMechanic +NetSpider +Net Vampire +NextGenSearchBot +Nikto +Ninja +oBot OpenLinkProfiler -Openvas +OpenVAS +OrangeBot OrangeSpider +OutclicksBot OutfoxBot +PageAnalyzer Page Analyzer -page scorer -Panscient -PECL::HTTP +PageScorer +Pandalytics +Pavuk PHPCrawl Picscout +Picsearch PictureFinder -Pi-Monster +Pimonster +Pixray +plumanalytics Pockey -POE-Component-Client-HTTP -Psbot -QueryN Metasearch +Probethenet +ProPowerBot +PxBroker Quick-Crawler +RankActive RankFlex -RankingBot2 -RankurBot +RankingBot RealDownload Reaper -RedesScrapy -ReGet +Recorder RepoMonkey -Ripper -scan.lol +RocketCrawler +Rogerbot +s1z.ru +SalesIntelligent Scrapy -ScreenerBot +Screaming Searchestate -Semrush -SEOlyticsCrawler -SEOprofiler -SEOstats -sexsearcher +SEOkicks +SeoSiteCheckup +serpstatbot Shodan Siphon SISTRIX +Sitebeam Siteimprove -SiteSnagger -SiteSucker -Sitevigil -SlySearch +SiteLockSpider Snake Snapbot +Snoopy +SocialRankIOBot +Sociscraper sogouspider +Sogou web spider Sosospider +SpaceBison Spammen -sp_auditbot -Spbot -SputnikBot +Spinn3r spyfu -Steeler +Sqworm Sucuri -SuperBot -Surfbot -Suzuran -Swiftbot +SuperHTTP +SurveyBot +sysscan Szukacz +T0PHackTeam +T8Abot tAkeOut +Teleport TeleportPro +Telesoft Telesphoreo The Intraformant -Toata -Trendiction +TheNomad +TightTwatBot +Titan +Trendictionbot +trendiction.com +trendiction.de +Turingos Turnitin -TurnitinBot -Twice Upflow -URLy Warning -Vagabondo -VCI VidibleScraper Voil -WallpapersHD +WASALive-Bot +WBSearchBot +Webalta WebAuto Web Auto WebBandit -WebFetch -Web Fetch +WebEnhancer +Web Enhancer WebFuck Web Fuck WebGo IS -WebmasterWorldForumBot webmeup-crawler -WebReaper +WebPix +Web Pix Web Sauger Webshag -WebsiteExtractor Webster +WebStripper WebSucker -WebWhacker -WebZIP -WeSEE -Whacker -Whatweb -WinHTTrack -WiseGuys Robot +Who.is Bot +Widow Woobot -x09Mozilla +Wotbox +Wprecon +WPScan +WWW-Mechanize x22Mozilla -Xaldon WebSpider -xpymep1.exe -YoudaoBot -Zade +Xaldon_WebSpider +Xenu Zauba zauba.io -Zermelo -Zitebot -ZmEu +Zeus +ZumBot +ZyBorg diff --git a/.dev-tools/test_units/random-referrers-for-test-quick.list b/.dev-tools/test_units/random-referrers-for-test-quick.list index a4e5528f3..408fb0671 100644 --- a/.dev-tools/test_units/random-referrers-for-test-quick.list +++ b/.dev-tools/test_units/random-referrers-for-test-quick.list @@ -1,10 +1,10 @@ -asacopaco.tk -bestpornuha.com -businesn.men -cialisovercounteratwalmartusa.com -golden-praga.ru -hasfun.com -hem.passagen.se -slowmacfaster.trade -tradgardspartner.se -turbabitload.weebly.com +gribokstop.com +kolotiloff.ru +naturalshair.site +nonews.co +pornoforadult.com +pozdrawleniya.com +rentehno.ru +vozbujdenie.com +websiteaccountant.de +y8games-free.com diff --git a/.dev-tools/test_units/random-referrers-for-test.list b/.dev-tools/test_units/random-referrers-for-test.list index a9717bf5b..ba6623915 100644 --- a/.dev-tools/test_units/random-referrers-for-test.list +++ b/.dev-tools/test_units/random-referrers-for-test.list @@ -1,500 +1,500 @@ -00go.com -03p.info -08800.top -0c47f8422d3f.com -101raccoon.ru -123any.com -1kdailyprofit.me -2000k.ru -214jaluwobapef.cf -24-ak.ru +000free.us +00it.com +12u.info +178evakuator178.ru +202ch.com +2728fb936f0.com +2drittel.de +2girls1cup-free.com +2nt.ru +2pxg8bcf.top +2x2fan.ru +34.gs +4kepics.com 4rent.ru +4senses.co +51.la 57883.net -5kstore.com -72-news.com -7search.com -acgs.tk -acmebtn.ml -adinterax.com -adloads.com -adserver-e7.com -adultwebhosting.info -advokat-grodno.by -agecheckadult.com -akama.com -aktobe.xkaz.org -alfabot.xyz -alienwheel.es -aliexpress.com -all4invest.info -allblogroll.com -allcredits.su -alphavisions.net -angkortours.vn -anniemation.com +7yue.org +8xv8.com +activecampaign.dreamhosters.com +adidas.frwebs.fr +adimmix.com +agadelha.com.br +agahinameh.com +agardomains.com +ahrefs.com +aimayangzhi.com +air-edem.ru +akusherok.ru +albumsuper.info +alekseevec.ru +alert-jdh.xyz +alert.scansafe.net +alf-img.com +amateurgalls.com +amt-k.ru +amung.us +android4fun.org +anti-virus-removal.info apessay.com -apidiamondatanet-a.akamaihd.net -arabseyes.com -as5000.com -astrochicks.com -australia-opening-times.com -auto4style.ru -autorn.ru -aviva-limoux.com +apiqualitinknet-a.akamaihd.net +app5.letmacworkfaster.world +appartement-stumm.at +app-ready.xyz +appsaurus.com +apptester.tk +arendas.net +arendovalka.xyz +asrvrep-a.akamaihd.net +asseenontvonline.ru +atlasvkusov.ru +audiobangout.com +autoseo-traffic.com +auto-zapchasti.org +avirasecureserver.com +avkzarabotok.com +avon-ukraine.com awaybird.ru -azte.ch -babs.com.ua -bannerpower.com -baoxaydung.com.vn -battleforupdating.site -beachfix.co -beauty-bracelet.com -becuo.com -beetpics.pw +azartclub.org +azazaporn.com +bala.getenjoyment.net +bbtec.net +best-coupon-offer.com +best-deals-products.com bestmarriages.com -bestofupload.info -bestsexyblog.com -best-way.men -beztuberkuleza.ru -bjetjt.com -blog-hits.com -blogig.org -bochemit.com.ua -boeuklad.com -bolegapakistan.com -bonanza-fortune.men -bonzibuddi.com -boom-celebs.com -bosefux.esy.es +bestserials.com +bestwebsiteawards.com +bfz.biz +bigcities.org +bioscorp.ru +biznesrealnost.ru +biznesschat.net +bkgr.se +blackcurranthumidifiers.site +blavia.00author.com +bloggedporn.com +bloggerads.net +bloggers.nl +blogos.kz +blogrankers.com +boltalko.xyz +bongacams.com +bonky.biz +boole.onlinewebshop.net +bosman.pluto.ro +bouda.kvalitne.cz brakehawk.com -brandbucket.com -brandov.ru -brianhenry.co -budpost.com.ua -bumascloset.com -bupropion-sr-150-mg.us -businesn.men -buybest1.biz -buy-forum.ru -buyk.host.sk -buzzsumo.com -c1.onedmp.com -cakemediahq.com.au -calc-for-credit.ru -callmd5map.com -cartierlove2u.xyz -cbozhe.com -cdnanalytics.xyz -chatroulette.si -cheapwebsitehoster.com -cheatcode-lita12.rhcloud.com -chorus.terakeet.com -cialgenisrx.com -cinemaenergy-hd.ru -citysecurity.nu -clickso.com -clmforexeu.com -codq.info -codysbbq.com -companies-catalog.com -compliance-andrew.xyz -compliance-checker.info -compliance-don.xyz -compliance-jess.top -compliance-mary.top -coolgramgoods.com -coolyarddecorations.com -cranly.net -crest-poloski.ru -cryptoswap.biz -czeshop.info -daisye.top -damasarenai.info -darmebel.com.ua -data1.scopich.com -davebestdeals.com -day-news.info -deanmoore.ie -dentalpearls.com.au -derevesendeco.com -design-lands.ru -dev78.dioniqlabb.se -dildofotzen.net -directrev.com -divku.ru -dnsrsearch.com -doeco.ru -dotmass.top -dotnetdotcom.org -doxyporno.com -drev.biz -drunkmoms.net -dvr.biz.ua -e-buyeasy.com +bristolhostel.com +brk-rti.ru +buckspar.gq +buigas.00it.com +buketeg.xyz +bum.com.ru +buyessaynow.biz +buy-loft.ru +buypanicdisorderpill.com +buzz-porno.info +buzzurl.jp +carapics.com +cartierring.xyz +caulderwoodkennels.com +chatroulette.online +cheapmarmot.online +cheapmoncler.pw +chinaelitecheapjerseys.com +cialisndbrx.com +cindymatches.com +cineacademy.ru +clean-virus-mac.com +clicksotrk.com +conocer-sanabria.com +continent-e.tv +countercrazy.com +creditwell.ru +cypernhuset.se +deplim.com +design-ideas.info +dickssportinggoods.com +dikqlyremy.info +dogsrun.net +doska-vsem.ru +doublepimp.com +drupa.com +druzhininevgeniy63.blogspot.com +duplicashapp.com +dyshagi.ru +earn-from-articles.com education-cz.ru -ege-russian.ru -elektrischezi.canalblog.com -elitepcgames.com -en.altezza.travel -e-poker-2005.com -erotik0049.com -essayassist.com -etenininrade.ga -ezigarettenshop2.myblog.de -face.hostingx.eu -failingmarriege.blogspot.com -falllow.gq -familyholiday.ml -fantasticpixcool.com -fastcrawl.com -fast-wordpress-start.com -feel-planet.com -felizporno.com -fenoyl.batcave.net -financetip.eu -findacheaplawyers.com -fireads.men -fishingwholesale.us -flash4fun.com -flashahead.info -floating-share-buttons.com -florida-tourism.net -fortevidyoze.net -foto-sisek.porngalleries.top -foxydeal.com -freejabs.com -free-stock-illustration.com -free-video-tool.com -freshmac.space -freza-sverlo.ru -front.ru -fullgirl.ru -gameplexcity.com -gay.adultgalls.com -getlaid-xxxhookupdirect.com -getmyads24.com -gg-arena.ru -ghostvisitor.com -gilbertbanda.net -gktt.ru -gkvector.ru -goforexvps.com -goldenggames.com -gombita.info -goodprotein.ru -gopro-online.info -gotwebsite1.com +edudocs.net +electrik-avenue.com +elektrozigaretten1.postbit.com +elvel.com.ua +elvenar.com +erectile.bid +eropornosex.ru +essay-writing.work +este-line.com.ua +evogarage.com +ezigarettenkaufen1.deviantart.com +famix.xyz +favornews.com +feargames.ru +ferrotodo.com +filesdatabase.net +fillmewithhappiness.com +filmci.pro +filmgo.ru +find1friend.com +findpik.com +findsexguide.com +firstdrugmall.ru +flirt4free.com +floridamobilebillboards.com +flowwwers.com +fok.nl +forensicpsychiatry.ru +forsex.info +fotoxxxru.com +fotzen-ficken.com +foxjuegos.com +foxtechfpv.com +freetangodownload.com +free-traffic.xyz +fsalas.com +fungirlsgames.net +fusoradio.info +fyxabomiw.ru +galaxyflowers.ru +gameprimary.com +gamerscorps.com +gaming-journal.com +gardene.ru +gay-site.store +gcup.ru +ge0ip.org +gentamicineyedrops.blogspot.com +getoutofdebtfree.org +getridofstretchmarks.org +gk170.ru +gogalleryawesome.com +golden-praga.ru +gorodservis.ru +gourcy.altervista.org +gox.com.ua +graphics8.info +gratis-sexkontakte.com green-tea.tv -growshop.es -grupografico-pilar.com.ar -gsasearchengineranker.top -hanink.biz.ly -hard-porn.mobi -hdapp1008-a.akamaihd.net -hdseriale.pl -healbio.ru -healing-dysplasia.ru -heroz.fr -high-speed1.net -hitmuzik.ru -homeart.space -homelygarden.com -homik.pw -house.sieraddns.com -hoverboardforsaledirect.com -humbmosquina.tk -i4track.net -ifirestarter.ru -ikritikimou.gr -imagui.eu -imperiafilm.ru -impisr.ru -inbabes.sexushost.com -incolors.club -inet-traffic.com -intellekt21.ru -invest-pamm.ru -investpamm.ru -investzalog.ru -iron-age.info -itag.pw -itsdp3.com -janavibekken.no -jasonpartington.com -javlibrary.cc -javspace.net -jaxcube.info -jerseyssportsshop.com -jikoman.info -johannesburgsingles.co.za +gsasearchengineranker.space +hackers-crackers.tk +haveinc.xyz +hawaiielectriclight.com +hawaiisurf.com +hdimagegallery.net +heartofpayne.xyz +homemature.net +hornymatches.com +hotchatdate.com +humanorightswatch.org +hyipmanager.in +igrovyeavtomaty777.ru +ihc2015.info +ilikevitaly.com +iloveitaly.ru +images-graphics-pics.com +imguramx.pw +inboundlinks.win +incanto.in.ua +incest-ru.com +indianmedicaltourismshop.com +infokonkurs.ru +interesnie-faktu.ru +internetartfair.com +islandminingsupply.wordpress.com +istock-mebel.ru +iwanttodeliver.com +izismile.com +joingames.org jovencitas.gratis -kaac.ru -kazrent.com -keywordglobal.co.za -keyword-suggestions.com -keywordsuggest.org -kinogolos.ru -kinogonew.ru -kino-key.info -kiwi237au.tk -kleine-titten.biz -komputers-best.ru -konturkrasoty.ru -krasivoe-hd.com -krasnodar.xrus.org -kredit-pod-zalog-krasnodar.ru -kretpicf.pw -ks1234.com -laapp.com -lafriore.ru -laudit.ru -letmacworkfaster.site -lifehacĸer.com -lightinghomes.net -linerdrilling.com -lineshops.biz -live-sexchat.ru -loadingpages.me -locksmith.jp -lolnex.us -lomza.info -lovi-moment.com.ua -lulea-auktionsverk.se -magnetpress.sk -make-money-online.com -manimpotence.com -mararoom.ru -marketland.ml -marmitaco.cat -masserect.com -matematikus.info +junglenet-a.akamaihd.net +kaidalibor.de +kalb.ru +kanimage.com +kareliatobacco.ru +kicknights.gq +kihi.gdn +kinobest.pl +klin3952.ru +klitimg.pw +konkursowo-24.pl +kriokomora.info +kvartiry-remont.ucoz.ru +labplus.ru +lampokrat.ws +lanadelreyfans.us +lancheck.net +landoftracking.com +laparfumotec.com +lawyers.cafe +laxob.com +leboard.ru +lechenie-gemorroya.com +lernur.net +lescinq.com +letmacwork.world +lexaprogeneric.link +life.biz.ua +life-instyle.com +limtu.ifmo.ru +linksharingt.com +liumimgx.pw +liveinternet.ru +localflirtbuddies.com +lostaruban.com +louboutinreplica.xyz +love-baby.cz +lowephotos.info +lubetube.com +mafa-free.com +manipulyator-peterburg.ru +mapquestz.us +maranbrinfo.com.br +marsgatan.com +matchpal-a.akamaihd.net +max-eclat.men mebelcomplekt.ru -medmajor.ru -memberty.com -message-warning.net -michaelkorssaleoutletonline.net -microstatic.pl -mir-betting.ru -mirobuvi.com.ua -mirzonru.net -missclub.info -missis.top -missvietnam.org -mister-shop.com -mitrasound.ru -mixtapetorrent.com -mobifunapp.weebly.com -mobioffertrck.com -mojowhois.com -mojpregled.com +mediawhirl.net +mega-polis.biz.ua +meltwater.com +metabar.ru +mil-stak.com +mindeyegames.com +miniads.ca +mobsfun.net +moi-glazki.ru +moncleroutletonline.pw +moncleroutletonline.xyz monclervests.xyz monsterdivx.com -moonci.ru -mosrif.ru -mountainstream.ms -mp3downloadhq.com -msk-diplomat.com -musicstock.me +morefastermac.trade +more-letom.ru +morpicert.pw +msk.afora.ru +mug-na-chas-moscow.ru +musicktab.com mustat.com -muz-baza.net -mydirtystuff.com -myindospace.com -na15.ru -nagdak.ru -nationalbreakdown.com -netallergy.ru -next-dentists.tk -nottyu.xyz -novosibirsk.xrus.org -obesidadealgarve.com -oda.as -odesproperty.com -ohmyrings.com -okel.co -old-rock.com -onefilms.net -onlineku.com -online-templatestore.com -o-o-8-o-o.com -oops-cinema.ru -openlibrary.org -opinionreelle.com -optom-deshevo.ru -outshop.ru -panamaforbeginners.com -parfusale.se -pateaswing.com +muzaporn.com +mydirtyhobby.com +myperiod.club +myplaycity.com +naperehresti.info +narkologiya-voronezh.ru +nfljerseys.online +nhl17coinsforps3.gratisblog.biz +nibbler.silktide.com +nikkiewart.ru +ningessaybe.me +noclegonline.info +northfacestore.online +oballergiya.ru +oconto.ru +o.light.d0t.ru +omsk.xrus.org +onclickpredictiv.com +onclkads.com +onetravelguides.com +onlinemegax.com +orgasmus-virtual.com +osoznanie-narkotikam.net +oynat.info +paintingplanet.ru +pandroid.co +partner-pop.men +partner-stop.men paydayloanslocal.com -piccshare.com -picturesify.com -pila.pl -pillscheap24h.com -pinkduck.ga -pinsdaddy.com -pk-services.ru -platinumdeals.gr -playmsn.com -pl-top.pl -pochemychka.net -pontiacsolstice.info -poquoson.org +pensplan4u.com +pewit.pw +piccdata.com +picscout.com +pinwallpaper.top +pistonclasico.com +piter.xrus.org +pixgood.com +plastjulgranar.se +plastweb.ru +playlott.com +play-movie.pl +pokemon-go-play.online +porndairy.in +pornogratisdiario.com +pornokajf.com pornoklad.ru -pornonik.com -porno-transsexuals.ru -preparevideosafesystem4unow.site -preparevideosafesystem4unow.space -privatamateure.com +pornosemki.info +portadd.men +predmety.in.ua +prefersurvey.net privatefx.all4invest.info -prizeestates.cricket -prlog.ru prod2016.com -productarium.com -proekt-mos.ru -professionalwritingservices15.blogspot.ru -proftests.net -prostitutki-belgoroda.org -prostitutki-kiev.org -prostitutki-novgoroda.org -puzo2arbuza.ru -qcstrtvt.bloger.index.hr -qor360.com -raisedseo.com -ranksonic.org -reckonstat.info -remedyotc.com -remont-comp-pomosh.ru -remote-dba.de -remybutler.fr -revistaindustria.com -reward-survey.net +profitfx.online +pulseonclick.com +pussyfleet.com +py100.ru +quangcaons.com +rainbowice.ru +rangoman.date +rapidokbrain.com +rasteniya-vs-zombi.ru +ratemodels.net +recipedays.com +registrydomainservices.com +remont-fridge-tv.ru +remont-mobile-phones.ru +renecaovilla.online +resellerclub.com +rfd-split.hr ric.info -roleforum.ru -roll123.com -rp9.ru -rubbed.us -russian-postindex.ru -rustic-quiver.win -s1z.ru -sack.net -saecsa.co -sale-japan.com -salut-camp.ru -sandhillsonline.com -saneyes.com -saveindex.xyz -scanner-elena.top -searchimage.co -seo-prof1.xyz +rogervivierforsale.com +royalvegascasino.com +ruf777.com +ruscoininvest.company +rusenvironmental.net +rusprostitute.com +sakhboard.ru +sankt-peterburg.nodup.ru +sbf441.com +sbornik-zakonov.ru +scopich.com +search-error.com +search.xtconnect.com +securityallianceservices.com +seksvideoonlain.com +seo-platform.com seorussian.ru -service-core.ru -serviporno.com -sevendays.com.ua -sexspornotub.com -ship-marvel.co.ua -shoesonlinebuy.cn +sergiorossistore.online +sexblog.pw +sexs-foto.com +sexstream.pl +sexytrend.ru +sharebutton.org +shlyahten.ru +shopfishing.com.ua +shoplvlv.us +shtaketniki.ru silverage.ru -siteheart.net -skatestick.bid -sluganarodu.ru -socialbutton.xyz -socialsignals24.com -sodexo.com -softlinesolutions.me -solicita.info -spravka-medosmotr.ru -spymac.net +sinel.info +sirpornogratis.xxx +skytraf.xyz +sladkoevideo.com +slavic-magic.ru +s.lollypopgaming.com +slomm.ru +sms2x2.ru +sonnikforme.ru +sosdepotdebilan.com +spaceshipad.com +spammen.de +spb.ru +spravka130.ru +spywarenuker.com +ssconstruction.co +stairliftsarea.com +startufa.ru stathat.com -strangeduckfilms.com +stockspmb.info +stop-gepatit.te.ua streha-metalko.si -stretchmate.net -stroimajor.ru -stromerrealty.com -strongsignal-a.akamaihd.net -success-seo.com -surffoundation.nl -t3chtonic.com -tanieaukcje.com.pl +superinterstitial.com +svolze.com +taihouse.ru +takeflyte.com +taxi-v-eisk.ru +tech4master.com technopellet.gr -telefonsexsofort.tk +teenbbw.yopoint.in +texbaza.by thaisamkok.com -thiegs.reco.ws -tizanidine4mguses.blogspot.com -tizanidinenarcotic.blogspot.com -tizanidinerecreationaluse.blogspot.com -tizanidinewithdrawal.blogspot.com -tnctrx.com -tobeyouday.win -todohr.com -topanasex.com -topkarkas.com -top-study.work +thebluffs.com +thejournal.ru +themeforest.net +themestotal.com +the-torrent-tracker.blogspot.com +thewebsitetemplate.info +tiens2010.ru +timdreby.com +titelhelden.eu +tizanidineandgabapentin.blogspot.com +tizanidinerestlesslegsyndrome.blogspot.com +tizanidinesleepaid.blogspot.com +tjkckpytpnje.com +token-lab.org torrentgamer.net -toys.erolove.in -tpu.ru -track-rankings.online -tracxn.com -tradedeals.biz -treasuretrack-a.akamaihd.net -turvgori.ru -twelvevisionspartyofcolorado.com -ua.tc -ugogo.info -uhdtv.website -unpredictable.ga -uzungil.com -video-camer.com -video-chat.love -videositename.com -videospornogratisx.net -vip-dom.in +torrents.cd +totu.info +trafficfactory.biz +trafficmp.com +trustedmaccleaner.com +try-rx.com +ttrraacckkrr.com +ttsq.fr +tubeline.biz +tvand.ru +ua-company.ru +uamtrk.com +uchebavchehii.ru +uggbootsoutletsale.us +ukrobstep.com +uogonline.com +uptime-alpha.net +uptime.com +uptime-gamma.net +urlcut.ru +utrolive.ru +uytmaster.ru +validdomain.xyz +vashsvet.com +venta-prom.ru +vepad.com +video-chat.in +videochat.mx +videooko.weebly.com +vitoriacabos.com vodaodessa.com -voluumtracker1.com -vostoktrade.info -vtc.pw -wapsite.me -warezkeeper.com -waterefficiency.co -watracker.net -webmonetizer.net -webtherapy.ru -wedding-salon.net -weekes.biz.tc -wesharepics.site -whiteelephantwellington.com -whiteproduct.com -wholinkstome.com +vsdshnik.com +vzubah.com +warningwar.ru +waysbetter.cn +we-are-gamers.com +webextract.profound.net +webmasterhome.cn +website-analyzer.info +weburok.com +wellcome2slovenia.ru +weprik.ru +wetgames.ru +whatistizanidinehclusedfor.blogspot.com +wholesalejerseys-cheapest.com wingsoffury2.com -winx-play.ru -wma-x.com -womama.ru -woman-orgasm.ru -works.if.ua -wowas31.ucoz.ru -writingservices17.blogspot.ru -wtsindia.in -www.event-tracking.com -xep.info -xerox-douglas.cf -x-musics.com -xn----7sbifcamovvfggw9d.xn--p1ai -xn----9sbebi2bvzr7h.xn--p1ai -xn--d1aifoe0a9a.top -xn----dtbndd4ae7eub.top -xn--h1ahbi.com.ua -xn----itbkqkfiq.xn--p1ai -xvideosporn.biz -y8games-free.com -yellowads.men -yes-com.com -yoopsie.com -yopoint.in -yottos.com -youdao.com -youraticles.pl -zakazvzloma.com -zaobao.com.sg -zastroyka.org -zeikopay.com -zeroredirect2.com -zixizop.net.ru -zoogdisany.com +withstandingheartwarming.com +woman-h.ru +wonderfulflowers.biz +worst-sites.online +xboxster.ru +xcombear.ru +xn----7sbahjd3btneuw1joc.xn--p1ai +xn----7sbbahaq9bb5afgiqfliv4m.xn--p1ai +xn--80aaafbn2bc2ahdfrfkln6l.xn--p1ai +xn--b1ag5cfn.xn--p1ai +xtraffic.plus +yellowstonesafaritours.com +youbloodyripper.com +zaim-pod-zalog-krasnodar.ru +zakon-ob-obrazovanii.ru +zambini.ru +zapiszto.pl +zarabotat-na-sajte.ru +zdesformula.ru +zeroredirect6.com +znakomstvaonlain.ru +znaniyapolza.ru +zona-aqua.ru +zoolubimets.ru diff --git a/.dev-tools/test_units/random-referrers-for-whitelist-test.list b/.dev-tools/test_units/random-referrers-for-whitelist-test.list index f371a4630..94344b53d 100644 --- a/.dev-tools/test_units/random-referrers-for-whitelist-test.list +++ b/.dev-tools/test_units/random-referrers-for-whitelist-test.list @@ -1,250 +1,250 @@ -00webcams.com -01apple.com -0lovespells0.blogspot.com -1bet.com -2daytrendingnews.com -34.gs -3w24.com -4kpics.rocks -4kplayer.pl -4rent.ru -5tey7463.weebly.com -abcdeg.xyz -academiacsmendoza.org -adultmeetonline.info -agecheckadult.com -alert.scansafe.net -alevinefesleri.com -all-number.com -allwidewallpapers.com -animali.deagostinipassion.it -asiavirtualsolutions.com -asrv-a.akamoihd.net -asrvrep-a.akamaihd.net -azartniy-bonus.com -bala.getenjoyment.net -bannerswap.com -basisches-wasser.net -beetpics.pw +03p.info +2015god.org +3dracergames.com +4inn.ru +4istoshop.com +57883.net +5i2.net +7fon.ru +acgs.tk +adservme.com +advertisingtag.net +advokateg.xyz +algarveglobal.com +alvtank.se +animal36.com +asdfz.pro +asociatia-tipografilor-transilvania.ro +audiofree.ru +auto.rusvile.lt +avtovolop.ru +babosas.com +banksrf.ru +bestbrainprod.win bestpornuha.com -bif-ru.info -binaryoptionscops.info -bingoporno.com -biznesschat.net -bloggen.be -braintobe.win -brandehk.dk -bristolhotel.com.ua -bscodecs.com -businescoop.men -buyscabiescream.com -buzzsumo.com -cactussoft.biz -canrioloadm.gq -capture-room.com -carfax.com.ua -cartujano-pre.de -cdnnetwok.xyz -cfcl.co.uk -cgi2.nintendo.co.jp -chatroulette.world -cheapestjerseysonwholesale.com -cheapestjerseys-wholesale.com -choosecuisine.com -cigarpass.com -cineacademy.ru -clickzzs.nl +bestwebsitesawards.com +biographiya.com +biteg.xyz +bocoarchives.org +brainboosting.club +buy-forum.ru +buyfriend.ru +byme.se +callcafe.info +canrole.com +capacitacionyaprendizaje.com +cartierlovebracelet.xyz +cartierlovestore.xyz +cementaresearch.se +cezartabac.ro +charmstroy.info +chatroulette.life +clknsee.com +clodo.ru cl.s7.exct.net -compliance-don.xyz -compliance-elena.xyz -conciergegroup.org -cordstrap.cc -cryptoswap.biz -dailystrength.org -dispo.de -disruptingdinnerparties.com -dividendo.ru -downloads-whatsapp.com -druzhininevgeniy63.blogspot.com -dvd-famille.com -efnor-ac.com -elektir.ru -elstal.com.pl -emergencyneeds.org -empathica.com -esopini.com -estelight.ru -evidencecleanergold.com -exdocsfiles.com -ezigarettenshop1.yolasite.com -fefo.gdn -fellowshipoftheminds.com -fetishinside.com -filmci.pro -finansov.info -finuse.com -fix-website-errors.com -fodelsedagspresenter.nu -forpostlock.ru -forums.toucharcade.com -franecki.net +com-cleaner.systems +communisave.co.za +compliance-checker.info +compliance-margo.top +coolwallpapers-hd.com +danashop.ru +designdevise.com +destinationrealestate.com +dgfitness.co +diffbot.com +digital-video-processing.com +distver.ru +dnsrsearch.com +dostavka-v-krym.com +doyouknowtheword-flummox.ml +e-commerce-seo1.com +editmedios.com +educontest.net +ege-essay.ru +ekto.ee +elektronischezigarettekaufen1.myblog.de +el-nation.com +elvenar.com +eralph.tk +extractorandburner.com +ezigarettenkaufen1.pagina.gr +f1nder.org +falllow.gq +fashion-stickers.ru +fickkontakte.org +filkhbr.com +findthe.pet +flashbannernow.com +footballfarrago.com +forex.osobye.ru +forminecrafters.ru +fotzen-ficken.com +frameimage.org free-gluten.ru -freeuploader.ml -gamerextra.com -gaming-journal.com -gay-site.store -gazobeton-p.com.ua -geckoandfly.com -go2mike.ru -good-mummy.ru -goo.ne.jp -grtyi.com -guruofcasino.com -hanink.biz.ly -hanwei.us -hayate.biz -hebr.myddns-flir.com -ibb.com.ua -ideawheel.com -images.gyffu.com -impisr.ru -intermesh.net -istizanidineacontrolledsubstance.blogspot.com -ivoiretechnocom.ci -jerseychinabizwholesale.us -johannesburgsingles.co.za -jose.mulinohouse.co -katadhin.co -labelwater.se -lacasamorett.com -landmania.ru -laxdrills.com -lennatin.info -letmacworkfaster.world -letsrepair.in -linkwithin.com -lion.conboy.us -llastbuy.ru -losangeles-ads.com -lostaruban.com -m1media.net -mackeeper-land-672695126.us-east-1.elb.amazonaws.com -mac-shield.com -mafa-free.com -magnetic-bracelets.ru +freejabs.com +freelotto.com +freshsuperbloop.com +gamewrath.com +gg-arena.ru +ghostvisitor.com +gomusix.com +gorodservis.ru +gourcy.altervista.org +growshop.es +hamacapty.com +haveinc.xyz +helicalpile.us +help.tpu.ru +hermesbelts.xyz +hifidesign.ru +highland-homes.com +hotkeys.com +howopen.ru +igithab.com +importchinacoach-teach.com +impresagaia.it +infoupdate.org +jacago.com +james13prix.info +japan-bearings.ru +javstock.com +journeydownthescale.info +kamagragelusa.net +kamalsinha.com +kchaxton.com +keywordblocks.com +keywordsuggest.org +keywordteam.net +kmgamex.cf +ktotut.net +lampokrat.ws +languagecode.com +lapitec.eu +larchik.net +letolove.ru +linkarena.com +lirunet.ru +liverpool.gsofootball.com +live-sexcam.tk +localmatchbook.com +lombia.co +m0r0zk0-krava.ru +magicplayer-s.acestream.net +manifestation.betteroffers.review marcogrup.com -maximilitary.ru -medkritika.ru -mesto-x.com -middlerush-a.akamaihd.net -mielec.pl -minyetki.ru -mmoguider.ru -mncrftpcs.com -mobioffertrck.com -mole.pluto.ro -monclercheap.xyz -moncleroutletonline.win -moncleroutletonline.xyz -moneyzzz.ru -mosdverka.ru -mustwineblog.com -mydearest.co -myftpupload.com -myhealthcare.com -newtechspb.ru +mastershef.club +mebeldekor.com.ua +member-quiz.com +mhi-systems.ru +mhtr.be +miniature.io +mixx.com +moyakuhnia.ru +mttwtrack.com +musflashtv.com +musicktab.com +mvpicton.co.uk +myfreecams.com +mypets.by +news-readers.ru nhl09.ru -nl.netlog.com -nordstar.pro -oballergiya.ru -onlinedomains.ru -onstrapon.purplesphere.in -ontargetseo.us -onthemarch.co -orenburg-gsm.ru -orhonit.com -ourville.info -paintingplanet.ru -park.above.com -pechikamini.ru -pewit.pw -pics-group.com -pliks.pl -popserve.adscpm.net -porno-video-chati.ru -pospr.waw.pl -privatefx.all4invest.info -purplesphere.in -quick-seeker.com -ranksonic.com +nsatc.net +nullrefer.com +oecnhs.info +okout.ru +onko-24.com +oops-cinema.ru +panicatack.com +panouri-solare-acoperis.com +parajumpersoutlet.online +partnerads.men +pattersonsweb.com +paydayloanslocal.com +pcboa.se +perfection-pleasure.ru +petitions.whitehouse.gov +picturesmania.com +pix24x7.com +pixell.club +plaff-go.ru +plastweb.ru +play-mp3.com +pornhive.org +prostitutki-tyumeni.org +prpops.com +quickbuck.com realtytimes.com -reklamuss.ru -remont-comp-pomosh.ru -renecaovilla.online -ribieiendom.no -rp9.ru -ruspdd.com -sammlungfotos.online -sandhillsonline.com -sarf3omlat.com -sbornik-zakonov.ru -scat.porn -semalt.com -seocheki.net -sexphoto.site -sexy-pings.com -shop-electron.ru -signal03.ru -sims-sims.ru +redirlock.com +remontgruzovik.ru +rent2spb.ru +rethinkwasteni.info +retreatia.com +rfserial.net +rockma.se +rossanasaavedra.net +royal-investments.net +schlampen-treffen.com +school-diplomat.ru +scottbywater.com +seadragonherbery.com +sel-hoz.com +seofirmreviewsus.info +seorankinglinks.com +seo-smm.kz +shop.xz618.com +sibecoprom.ru +signoredom.com +simplepooltips.com +simul.co +site-speed-checker.site +sixcooler.de skuteczna-dieta.co.pl -softlinesolutions.me -spymac.net -srgwebmail.nl -staynplay.net -sterva.cc -stroydetali.ru -sudexpert66.ru -symbaloo.com -tattomedia.com -thecoolimages.net -thefotosgratis.eu -thejournal.ru -timecrimea.ru -timer4web.com -tizanidinesleepaid.blogspot.com -tizanidinevscyclobenzaprine.blogspot.com -topkarkas.com -topshef.ru -tripper.de -trustedhealthtips.com -turvgori.ru -ublaze.ru -uchebavchehii.ru -unece.org -univerfiles.com -uvozdeckych.info -vacances-voyages.info -vashsvet.com -verymes.xyz -viktoria-center.ru -vip-file.com -viven.host.sk -vrotike.ru -warningzscaler.heraeus.com -watchdogs-2.ru -webmasters.stackexchange.com -weightatraining.com -wemedinc.com -woman-tampon.ru -womens-journal.net -wttavern.com -xaylapdiendanang.com -xn----dtbndd4ae7eub.top -yaaknaa.info -yougetsignal.com -yousense.info -zigarettenonl.canalblog.com -zigarettenonlinekaufen2.bloog.pl -zigarettenonlinekaufen.tumblr.com -znakom.sibtest.ru -zoomovies.org +slutloadlive.com +sofit-dmd.ru +softwaretrend.net +solmarket.by +spabali.org +stickers-market.ru +stromerrealty.com +strongsignal-a.akamaihd.net +suchenindeutschland.com +sugarlyflex.pw +sukarame.net +surffoundation.nl +svtrd.com +swimpool.ca +takeflyte.com +taraz.xkaz.org +theallgirlarcade.com +tizanidinevsvalium.blogspot.com +track2.shop +tramadolandtizanidine.blogspot.com +tuberkulezanet.ru +turbodsp.com +turn-up-life.life +tv-spoty.info +tx41tclega.ru +umg-stroy.ru +upstore.me +uptime-delta.net +urdoot.win +uroki.net +v24s.net +vielporno.net +vip-dom.in +waterefficiency.co +weburlopener.com +whatsappdownload10.com +winwotgold.pl +womensplay.net +wonderfulflowers.biz +wurr.voila.net +xn--24-glceagatoq7c2a6ioc.xn--p1ai +xn--80ajjbdhgmudixfjc8c5a9df8b.xn--p1ai +xn--b1agm2d.net +xnxxandxvideos.com +xxxrus.org +zeroredirect8.com +znakomstva-moskva77.ru +zolotoy-lis.ru +zytpirwai.net diff --git a/.dev-tools/test_units/ratelimittest-beta.txt b/.dev-tools/test_units/ratelimittest-beta.txt index afb1627e8..6b86856ae 100644 --- a/.dev-tools/test_units/ratelimittest-beta.txt +++ b/.dev-tools/test_units/ratelimittest-beta.txt @@ -5,13 +5,6 @@