diff --git a/.dev-tools/beta_conf_files/conf.d/globalblacklist.conf b/.dev-tools/beta_conf_files/conf.d/globalblacklist.conf index e98ecf378..756641cad 100644 --- a/.dev-tools/beta_conf_files/conf.d/globalblacklist.conf +++ b/.dev-tools/beta_conf_files/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2019.08.1803 -### Updated: Thu Aug 15 14:19:14 SAST 2019 +### Version: V3.2019.08.1804 +### Updated: Thu Aug 15 18:43:51 SAST 2019 ### Bad Referrer Count: 6746 ### Bad Bot Count: 557 ################################################### diff --git a/.dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf b/.dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf index 0f27f0dd9..e22596857 100644 --- a/.dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf +++ b/.dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -104.154.113.151 1; +34.66.25.221 1; 127.0.0.1 1; diff --git a/.dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf b/.dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf index 0e0848852..e6677be79 100644 --- a/.dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf +++ b/.dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -104.154.113.151 0; +34.66.25.221 0; 127.0.0.1 0; diff --git a/.dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf b/.dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf index e98ecf378..756641cad 100644 --- a/.dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf +++ b/.dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2019.08.1803 -### Updated: Thu Aug 15 14:19:14 SAST 2019 +### Version: V3.2019.08.1804 +### Updated: Thu Aug 15 18:43:51 SAST 2019 ### Bad Referrer Count: 6746 ### Bad Bot Count: 557 ################################################### diff --git a/.dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf b/.dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf index 0f27f0dd9..e22596857 100644 --- a/.dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf +++ b/.dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -104.154.113.151 1; +34.66.25.221 1; 127.0.0.1 1; diff --git a/.dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf b/.dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf index 0e0848852..e6677be79 100644 --- a/.dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf +++ b/.dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -104.154.113.151 0; +34.66.25.221 0; 127.0.0.1 0; diff --git a/.dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf b/.dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf index e98ecf378..756641cad 100644 --- a/.dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf +++ b/.dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2019.08.1803 -### Updated: Thu Aug 15 14:19:14 SAST 2019 +### Version: V3.2019.08.1804 +### Updated: Thu Aug 15 18:43:51 SAST 2019 ### Bad Referrer Count: 6746 ### Bad Bot Count: 557 ################################################### diff --git a/.dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf b/.dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf index 0f27f0dd9..e22596857 100644 --- a/.dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf +++ b/.dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -104.154.113.151 1; +34.66.25.221 1; 127.0.0.1 1; diff --git a/.dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf b/.dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf index 0e0848852..e6677be79 100644 --- a/.dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf +++ b/.dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -104.154.113.151 0; +34.66.25.221 0; 127.0.0.1 0; diff --git a/.dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf b/.dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf index e98ecf378..756641cad 100644 --- a/.dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf +++ b/.dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2019.08.1803 -### Updated: Thu Aug 15 14:19:14 SAST 2019 +### Version: V3.2019.08.1804 +### Updated: Thu Aug 15 18:43:51 SAST 2019 ### Bad Referrer Count: 6746 ### Bad Bot Count: 557 ################################################### diff --git a/.dev-tools/globalblacklist-testing.template b/.dev-tools/globalblacklist-testing.template index e98ecf378..756641cad 100644 --- a/.dev-tools/globalblacklist-testing.template +++ b/.dev-tools/globalblacklist-testing.template @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2019.08.1803 -### Updated: Thu Aug 15 14:19:14 SAST 2019 +### Version: V3.2019.08.1804 +### Updated: Thu Aug 15 18:43:51 SAST 2019 ### Bad Referrer Count: 6746 ### Bad Bot Count: 557 ################################################### diff --git a/.dev-tools/globalblacklist.template b/.dev-tools/globalblacklist.template index 8fc85aede..4021ace13 100755 --- a/.dev-tools/globalblacklist.template +++ b/.dev-tools/globalblacklist.template @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2019.08.1803 -### Updated: Thu Aug 15 14:19:12 SAST 2019 +### Version: V4.2019.08.1804 +### Updated: Thu Aug 15 18:43:50 SAST 2019 ### Bad Referrer Count: 6746 ### Bad Bot Count: 557 ################################################### diff --git a/.dev-tools/setup-ngxblocker/modified/debian-8-jessie.conf.modified b/.dev-tools/setup-ngxblocker/modified/debian-8-jessie.conf.modified index 6d6e2323f..1b98fdcef 100644 --- a/.dev-tools/setup-ngxblocker/modified/debian-8-jessie.conf.modified +++ b/.dev-tools/setup-ngxblocker/modified/debian-8-jessie.conf.modified @@ -115,7 +115,5 @@ http { # nginx-naxsi config # include /etc/nginx/naxsi_core.rules; - # vhost - include /etc/nginx/conf.d/*.conf; - include /etc/nginx/sites-enabled/*; + /etc/nginx/sites-enabled/*; } diff --git a/.dev-tools/test1_conf_files/bots.d/whitelist-ips.conf b/.dev-tools/test1_conf_files/bots.d/whitelist-ips.conf index 6073098c1..91396c033 100644 --- a/.dev-tools/test1_conf_files/bots.d/whitelist-ips.conf +++ b/.dev-tools/test1_conf_files/bots.d/whitelist-ips.conf @@ -45,4 +45,4 @@ # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST -34.66.178.120 0; +34.66.200.49 0; diff --git a/.dev-tools/test1_conf_files/conf.d/globalblacklist.conf b/.dev-tools/test1_conf_files/conf.d/globalblacklist.conf index 8fc85aede..4021ace13 100644 --- a/.dev-tools/test1_conf_files/conf.d/globalblacklist.conf +++ b/.dev-tools/test1_conf_files/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2019.08.1803 -### Updated: Thu Aug 15 14:19:12 SAST 2019 +### Version: V4.2019.08.1804 +### Updated: Thu Aug 15 18:43:50 SAST 2019 ### Bad Referrer Count: 6746 ### Bad Bot Count: 557 ################################################### diff --git a/.dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf b/.dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf index b9d24324a..58acf0a1d 100644 --- a/.dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf +++ b/.dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -34.68.144.114 1; +35.202.245.105 1; 127.0.0.1 1; diff --git a/.dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf b/.dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf index ea117da9f..a69bef7d4 100644 --- a/.dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf +++ b/.dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -34.68.144.114 0; +35.202.245.105 0; 127.0.0.1 0; diff --git a/.dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf b/.dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf index 8fc85aede..4021ace13 100644 --- a/.dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf +++ b/.dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2019.08.1803 -### Updated: Thu Aug 15 14:19:12 SAST 2019 +### Version: V4.2019.08.1804 +### Updated: Thu Aug 15 18:43:50 SAST 2019 ### Bad Referrer Count: 6746 ### Bad Bot Count: 557 ################################################### diff --git a/.dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf b/.dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf index b9d24324a..58acf0a1d 100644 --- a/.dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf +++ b/.dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -34.68.144.114 1; +35.202.245.105 1; 127.0.0.1 1; diff --git a/.dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf b/.dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf index ea117da9f..a69bef7d4 100644 --- a/.dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf +++ b/.dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -34.68.144.114 0; +35.202.245.105 0; 127.0.0.1 0; diff --git a/.dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf b/.dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf index 8fc85aede..4021ace13 100644 --- a/.dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf +++ b/.dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2019.08.1803 -### Updated: Thu Aug 15 14:19:12 SAST 2019 +### Version: V4.2019.08.1804 +### Updated: Thu Aug 15 18:43:50 SAST 2019 ### Bad Referrer Count: 6746 ### Bad Bot Count: 557 ################################################### diff --git a/.dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf b/.dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf index b9d24324a..58acf0a1d 100644 --- a/.dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf +++ b/.dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -34.68.144.114 1; +35.202.245.105 1; 127.0.0.1 1; diff --git a/.dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf b/.dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf index ea117da9f..a69bef7d4 100644 --- a/.dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf +++ b/.dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -34.68.144.114 0; +35.202.245.105 0; 127.0.0.1 0; diff --git a/.dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf b/.dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf index 8fc85aede..4021ace13 100644 --- a/.dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf +++ b/.dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2019.08.1803 -### Updated: Thu Aug 15 14:19:12 SAST 2019 +### Version: V4.2019.08.1804 +### Updated: Thu Aug 15 18:43:50 SAST 2019 ### Bad Referrer Count: 6746 ### Bad Bot Count: 557 ################################################### diff --git a/.dev-tools/test2_conf_files/bots.d/whitelist-ips.conf b/.dev-tools/test2_conf_files/bots.d/whitelist-ips.conf index 2d4b2cec8..40b4b6f56 100644 --- a/.dev-tools/test2_conf_files/bots.d/whitelist-ips.conf +++ b/.dev-tools/test2_conf_files/bots.d/whitelist-ips.conf @@ -45,5 +45,5 @@ # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST -104.154.113.151 0; -35.192.187.174 0; +34.66.50.208 0; +35.202.245.105 0; diff --git a/.dev-tools/test2_conf_files/conf.d/globalblacklist.conf b/.dev-tools/test2_conf_files/conf.d/globalblacklist.conf index 4ce54077f..945d4da92 100644 --- a/.dev-tools/test2_conf_files/conf.d/globalblacklist.conf +++ b/.dev-tools/test2_conf_files/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2019.08.1803 -### Updated: Thu Aug 15 14:19:12 SAST 2019 +### Version: V4.2019.08.1804 +### Updated: Thu Aug 15 18:43:50 SAST 2019 ### Bad Referrer Count: 6746 ### Bad Bot Count: 557 ################################################### diff --git a/.dev-tools/test3_conf_files/bots.d/whitelist-ips.conf b/.dev-tools/test3_conf_files/bots.d/whitelist-ips.conf index 47a2b49b3..26bb5ecb7 100644 --- a/.dev-tools/test3_conf_files/bots.d/whitelist-ips.conf +++ b/.dev-tools/test3_conf_files/bots.d/whitelist-ips.conf @@ -45,5 +45,4 @@ # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST -35.192.187.174 0; -34.66.178.120 0; +34.66.50.208 0; diff --git a/.dev-tools/test3_conf_files/conf.d/globalblacklist.conf b/.dev-tools/test3_conf_files/conf.d/globalblacklist.conf index 828cdaf2c..94281779e 100644 --- a/.dev-tools/test3_conf_files/conf.d/globalblacklist.conf +++ b/.dev-tools/test3_conf_files/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2019.08.1803 -### Updated: Thu Aug 15 14:19:12 SAST 2019 +### Version: V4.2019.08.1804 +### Updated: Thu Aug 15 18:43:50 SAST 2019 ### Bad Referrer Count: 6746 ### Bad Bot Count: 557 ################################################### diff --git a/.dev-tools/test4_conf_files/bots.d/whitelist-ips.conf b/.dev-tools/test4_conf_files/bots.d/whitelist-ips.conf index 8395b3e7c..72ed0c26a 100644 --- a/.dev-tools/test4_conf_files/bots.d/whitelist-ips.conf +++ b/.dev-tools/test4_conf_files/bots.d/whitelist-ips.conf @@ -45,6 +45,6 @@ # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST -35.192.187.174 0; -35.192.136.167 0; -34.66.178.120 0; +34.66.200.49 0; +35.202.245.105 0; +34.66.25.221 0; diff --git a/.dev-tools/test4_conf_files/conf.d/globalblacklist.conf b/.dev-tools/test4_conf_files/conf.d/globalblacklist.conf index 8fc85aede..4021ace13 100644 --- a/.dev-tools/test4_conf_files/conf.d/globalblacklist.conf +++ b/.dev-tools/test4_conf_files/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2019.08.1803 -### Updated: Thu Aug 15 14:19:12 SAST 2019 +### Version: V4.2019.08.1804 +### Updated: Thu Aug 15 18:43:50 SAST 2019 ### Bad Referrer Count: 6746 ### Bad Bot Count: 557 ################################################### diff --git a/.dev-tools/test5_conf_files/bots.d/whitelist-ips.conf b/.dev-tools/test5_conf_files/bots.d/whitelist-ips.conf index 3871f94c4..29f60a72f 100644 --- a/.dev-tools/test5_conf_files/bots.d/whitelist-ips.conf +++ b/.dev-tools/test5_conf_files/bots.d/whitelist-ips.conf @@ -45,6 +45,6 @@ # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST -35.193.14.140 0; -35.192.187.174 0; -104.154.113.151 0; +34.66.25.221 0; +34.66.50.208 0; +35.202.245.105 0; diff --git a/.dev-tools/test5_conf_files/conf.d/globalblacklist.conf b/.dev-tools/test5_conf_files/conf.d/globalblacklist.conf index 8fc85aede..4021ace13 100644 --- a/.dev-tools/test5_conf_files/conf.d/globalblacklist.conf +++ b/.dev-tools/test5_conf_files/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2019.08.1803 -### Updated: Thu Aug 15 14:19:12 SAST 2019 +### Version: V4.2019.08.1804 +### Updated: Thu Aug 15 18:43:50 SAST 2019 ### Bad Referrer Count: 6746 ### Bad Bot Count: 557 ################################################### diff --git a/.dev-tools/test_units/blacklist-ips.conf b/.dev-tools/test_units/blacklist-ips.conf index 0f27f0dd9..e22596857 100644 --- a/.dev-tools/test_units/blacklist-ips.conf +++ b/.dev-tools/test_units/blacklist-ips.conf @@ -1,2 +1,2 @@ -104.154.113.151 1; +34.66.25.221 1; 127.0.0.1 1; diff --git a/.dev-tools/test_units/random-bots-for-test-quick.list b/.dev-tools/test_units/random-bots-for-test-quick.list index 7fb254a57..4e1e3ee13 100644 --- a/.dev-tools/test_units/random-bots-for-test-quick.list +++ b/.dev-tools/test_units/random-bots-for-test-quick.list @@ -1,10 +1,10 @@ -Blackboard -BuiltWith -CATExplorador -crawler.feedback -Morfeus Fucking Scanner -RankingBot2 -RebelMouse -RepoMonkey -Snapbot -Steeler +CSHttp +Custo +Go-Ahead-Got-It +muhstik-scan +Musobot +Netvibes +Octopus +Offline Explorer +Offline Navigator +Ripper diff --git a/.dev-tools/test_units/random-bots-for-test.list b/.dev-tools/test_units/random-bots-for-test.list index 8a66c8e57..811a45c96 100644 --- a/.dev-tools/test_units/random-bots-for-test.list +++ b/.dev-tools/test_units/random-bots-for-test.list @@ -1,250 +1,250 @@ 360Spider -Abonti -ADmantX +404checker +404enemy +80legs +Aboundexbot AfD-Verbotsverfahren -AIBOT AiHitBot Alexibot Alligator AlphaBot Apexoo -autoemailspider -BackDoorBot Backlink-Ceck -BacklinkCrawler +backlink-check BackStreet Bandit Barkrowler -Bitacle -Black Hole +BatchFTP +BDCbot +BDFetch +Blackboard BlackWidow -BLEXBot Blow -BlowFish -Boardreader Brandprotect Brandwatch -BuiltWith +BuiltBotTough Bullseye -BunnySlippers BuzzSumo -Calculon -Cegbfeieh +CCBot CheeseBot -CherryPicker -ChinaClaw Chlooe -Claritybot -Cliqzbot +coccocbot-web Collector -CopyRightCheck -Copyscape +crawler4j crawler.feedback -Curious -Custo -DBLBot +crawl.sogou.com +CrunchBot +CSHttp +DatabaseDriverMysqli +DataCha0s +demandbase-bot Demon -Disco +Deusu +Devil +DIIbot +Dirbuster +Discobot +Discoverybot Dispatch DnyzBot -DomainStatsBot +DomainAppender +DomainCrawler Dotbot Download Wonder Dragonfly -Drip -DSearch +DTS Agent +EasyDL Ebingbong eCatch -EMail Siphon -EMail Wolf -EroCrawler -evc-batch -Express WebPictures +Ecxi ExtLinksBot Extractor -EyeNetIE -FDM -FemtosearchBot -FrontPage +ExtractorPro +Extreme Picture Finder +Ezooms +FHscan +Fimap +Flunky +Foobot FyberSpider -Fyrebot Genieo GermCrawler -Getintent -Go-Ahead-Got-It +GetRight +GetWeb +G-i-g-a-b-o-t Gotit Go!Zilla Grabber -GrabNet +Grafula +GrapeFX +GrapeshotCrawler +GridBot GT::WWW -Harvest -Havij -HEADMasterSEO -Heritrix +Haansoft HTMLparser -Iblog +Humanlinks Id-search -IlseBot Image Fetch -Indy Library -InfoNaviRobot +Image Sucker InfoTekies -InternetSeer +instabid ips-agent Iskanie Jbrofuzz JennyBot -JikeSpider -JOC Web Spider -Joomla -JustView +Jorgee +Jyxobot Kenjin Spider -Keyword Density -Lanshanbot +LeechFTP +LeechGet +LexiBot Lftp -Libwhisker +LibWeb +Likse Linkdexbot -LinkextractorPro LinkpadBot LinkScan LinqiaMetadataDownloaderBot -LinqiaRSSBot -LinqiaScrapeBot -Lipperhey Spider -Lmspider -LNSpiderguy LWP::Simple -lwp-trivial -Magnet magpie-crawler +Mail.RU_Bot Majestic SEO -MarkWatch +Mata Hari +MauiBot +meanpathbot MeanPath Bot +Mediatoolkitbot mediawords MegaIndex.ru Metauri -MIIxpc +MFC_Tear_Sample +MIDown tool +Mister PiX +Mr.4x3 MSFrontPage +MSIECrawler Msrabot -Musobot +MS Web Services Client Protocol Name Intelligence -Nameprotect +Navroad NearSite +NetAnts Netcraft -netEstate NE Crawler NetLyzer -NetSpider -Nettrack -Net Vampire -NextGenSearchBot +NetMechanic +Nibbler NICErsPRO -Nikto +Niki-bot +Nimbostratus +Ninja Nmap -NPbot -Nutch +oBot Octopus Offline Explorer Offline Navigator -Openfind -Openvas +OnCrawl +OpenLinkProfiler OpenVAS -OutclicksBot -OutfoxBot +OrangeBot +OrangeSpider PageAnalyzer Page Analyzer -PageScorer -Pandalytics -Panscient -Pavuk +PageGrabber PECL::HTTP +Picscout +PictureFinder Pimonster +Pixray PleaseCrawl plumanalytics -POE-Component-Client-HTTP Probethenet -ProPowerBot -Psbot Pump -RankingBot +PyCurl +QueryN Metasearch +RankActive +RankActiveLinkBot RankingBot2 -Rankivabot RealDownload -Reaper -RebelMouse -RocketCrawler +Recorder +ReGet +SalesIntelligent ScanAlert -Scanbot -scan.lol -ScoutJet +Scrapy Screaming -Searchestate -SemrushBot SEOkicks SEOkicks-Robot -SEOlyticsCrawler seoscanners -SeoSiteCheckup -serpstatbot sexsearcher -Shodan -Siphon -Sitevigil -SlySearch -SmartDownload -SMTBot +SISTRIX +SiteExplorer +SiteLockSpider +SiteSnagger +Site Sucker Snake Snapbot Sociscraper +Sosospider Sottopop +SpaceBison Spammen -SputnikBot -spyfu -Sqlmap +SpankBot +sp_auditbot Sqworm Steeler +Sucker SuperBot SurveyBot +sysscan Szukacz -T8Abot +T0PHackTeam Teleport -TeleportPro Telesoft -The Intraformant TheNomad -Thumbor -TightTwatBot +Toata Toweyabot -Tracemyfile -Trendiction +Trendictionbot +trendiction.com trendiction.de -Turingos -Turnitin +True_Robot TurnitinBot -TwengaBot URLy Warning Vacuum -VB Project -VCI -VeriCiteCrawler +Vagabondo Virusdie -Voil -Wallpapers/3.0 -WebAuto +Voltron +Webalta +WebBandit +WebCollage +WebCopier WebEnhancer -WebFuck WebGo IS +WebImageCollector +webmeup-crawler +WebPix Web Pix WebReaper +WebSauger +Web Sauger Webshag -WebsiteExtractor Website Quester WebStripper +WebSucker +WebWhacker +WebZIP WeSEE -Widow +Whack WinHTTrack -WWWOFFLE -Xaldon WebSpider +WiseGuys Robot +Wotbox +Wprecon +WPScan +WWW-Mechanize +WWW::Mechanize +x09Mozilla Xenu -xpymep1.exe Zauba +zauba.io +Zermelo Zeus +ZmEu +ZumBot diff --git a/.dev-tools/test_units/random-bots-for-whitelist-test.list b/.dev-tools/test_units/random-bots-for-whitelist-test.list index 1b90d1566..8186d58ce 100644 --- a/.dev-tools/test_units/random-bots-for-whitelist-test.list +++ b/.dev-tools/test_units/random-bots-for-whitelist-test.list @@ -1,250 +1,250 @@ +360Spider 404enemy +80legs Abonti -AfD-Verbotsverfahren +Aboundexbot +Acunetix Aipbot -AllSubmitter -AlphaBot +Alexibot +Alligator Anarchie -archive.org_bot +ASPSeek Asterias Attach Backlink-Ceck -BackStreet -Bandit +BackWeb +Barkrowler +Battleztar Bazinga BDCbot -Bigfoot -Bitacle +BetaBot BlackWidow -Boardreader -BotALot +BLEXBot +Blow +Bolt +Brandwatch Buddy BuiltBotTough BuiltWith Bullseye +BuzzSumo +CATExplorador CCBot -CheeseBot CherryPicker Claritybot -Cogentbot -Collector -Copier -CopyRightCheck -Cosmos -crawler.feedback -CrazyWebCrawler +Cliqzbot +Copyscape +Craftbot +crawler4j CrunchBot -CSHttp DataCha0s -Demon -DigitalPebble -Discobot -DittoSpyder -DomainSigmaCrawler +DBLBot +demandbase-bot +Deusu +Digincore +DIIbot +Dispatch +DomainCrawler Download Wonder -Dragonfly -Drip +DTS Agent +EasyDL Ebingbong -EirGrabber EMail Siphon +EMail Wolf EroCrawler Exabot -Express WebPictures -Extractor ExtractorPro +Extreme Picture Finder +facebookscraper FDM -FemtosearchBot -FlashGet +Firefox/7.0 Flunky Foobot -FrontPage Fyrebot +Genieo +GermCrawler +Getintent GetRight -GetWeb Gigablast -G-i-g-a-b-o-t +Gigabot +Go-Ahead-Got-It +Gotit GoZilla -Go!Zilla +Grabber GrabNet -GrapeFX -GrapeshotCrawler +Grafula +GridBot +GT::WWW +HaosouSpider HEADMasterSEO -HTMLparser -HTTP::Lite -HTTrack -Humanlinks HybridBot Iblog -Image Fetch -Image Sucker -IndeedBot -InfoTekies -InterGET -Internet Ninja +Id-search +IlseBot +Indy Library +instabid +Intelliseek ips-agent -Iria -Iskanie IstellaBot -JetCar -Jetty +Jbrofuzz Jorgee +JustView Jyxobot -Keyword Density -LibWeb -Likse +Kozmosbot +Larbin +LeechFTP +LeechGet +LexiBot +Lftp +Libwhisker +Lightspeedsystems Linkdexbot +LinkextractorPro LinkpadBot LinkScan -LinqiaMetadataDownloaderBot +LinksManager +LinqiaScrapeBot Lipperhey Litemage_walker Lmspider LNSpiderguy +Ltx71 +lwp-request LWP::Simple -lwp-trivial -Magnet -Mail.RU_Bot -Majestic SEO -Majestic-SEO -MarkMonitor -Mass Downloader -meanpathbot +Mag-Net +magpie-crawler +Majestic12 +Mata Hari Meanpathbot -MeanPath Bot -MegaIndex.ru -Metauri MFC_Tear_Sample -Microsoft URL Control -MIDown tool MIIxpc Mister PiX -Mojeek +MJ12bot +Morfeus Fucking Scanner Mr.4x3 -muhstik-scan +MSIECrawler +Msrabot +Musobot +Name Intelligence Navroad -Needle -NetAnts -Netcraft -NetMechanic -NetSpider -Nettrack +NetLyzer +Net Vampire +Netvibes +NetZIP +Nibbler +NICErsPRO +NimbleCrawler Nimbostratus -Nmap NPbot +Nutch +Offline Navigator OnCrawl +Openfind +OpenLinkProfiler OpenVAS -OrangeSpider +OutclicksBot PageAnalyzer Page Analyzer +PageScorer Panscient +Pavuk pcBrowser PeoplePal -PHPCrawl -Picscout +Picsearch PictureFinder -plumanalytics +Pimonster +PleaseCrawl Pockey POE-Component-Client-HTTP +ProWebWalker +Psbot PxBroker PyCurl +QueryN Metasearch +RankActive RankFlex -RankingBot RankingBot2 -Reaper -RebelMouse +RealDownload Recorder -RedesScrapy -ReGet RepoMonkey -RocketCrawler -SBIder -scan.lol -Screaming -Searchestate +Ripper +RSSingBot +s1z.ru SearchmetricsBot Semrush SemrushBot SEOkicks-Robot -SEOprofiler +SEOlyticsCrawler +Seomoz +seoscanners SeoSiteCheckup SEOstats -serpstatbot sexsearcher -Shodan Siphon -SISTRIX -Sitebeam -SiteExplorer -SiteSnagger -SiteSucker +SiteLockSpider Site Sucker Sitevigil -SmartDownload +SlySearch SMTBot Snake SocialRankIOBot -Sociscraper Sogou web spider -Sosospider -Sottopop -SpankBot -Spanner -sp_auditbot Spbot -SputnikBot +Spinn3r spyfu Sqlmap -Sqlworm -Sqworm +Stripper Sucker -SuperBot -SuperHTTP +Sucuri Surfbot Suzuran sysscan -T8Abot -TeleportPro -Telesphoreo -TheNomad +Szukacz +tAkeOut +Teleport +The Intraformant TightTwatBot +Titan +Tracemyfile Trendiction trendiction.com +True_Robot +Turingos +Turnitin TurnitinBot -Twice -Typhoeus +TwengaBot +Upflow URLy.Warning -Vacuum Vagabondo -VB Project +VCI VeriCiteCrawler -VoidEYE -Voil +VidibleScraper +Virusdie Voltron -Wallpapers/3.0 +WallpapersHD +WASALive-Bot +WBSearchBot +Webalta WebAuto Web Enhancer WebFetch WebFuck -WebGo IS +Web Fuck +WebImageCollector WebmasterWorldForumBot webmeup-crawler -WebReaper +Webshag WebsiteExtractor Website Quester Webster -WebWhacker +WebZIP Whatweb WiseGuys Robot -WISENutbot -Wotbox -WPScan +Wprecon WWW-Collector-E -WWW-Mechanize -x09Mozilla -x22Mozilla +WWW::Mechanize +WWWOFFLE Xaldon WebSpider Xaldon_WebSpider +xpymep1.exe +Zade Zauba -zauba.io -Zermelo -ZumBot -ZyBorg diff --git a/.dev-tools/test_units/random-referrers-for-test-quick.list b/.dev-tools/test_units/random-referrers-for-test-quick.list index 6305aebaa..a03b76f54 100644 --- a/.dev-tools/test_units/random-referrers-for-test-quick.list +++ b/.dev-tools/test_units/random-referrers-for-test-quick.list @@ -1,10 +1,10 @@ -2itech.ru -alphahoverboards.com -bestcoin.cc -concealthyself.com -dailyfinancefix.com +98oi.ru +dignitasdata.se +findanysex.com +gagrasector.ru hetmanship.xyz -sobecjvuwa.com.ru -steamoff.net -tanieaukcje.com.pl -xxlargepop.com +qitt.ru +ruscopybook.com +sexs-foto.com +video-chat.love +yginekologa.com diff --git a/.dev-tools/test_units/random-referrers-for-test.list b/.dev-tools/test_units/random-referrers-for-test.list index a2fbbfdb1..afaaec743 100644 --- a/.dev-tools/test_units/random-referrers-for-test.list +++ b/.dev-tools/test_units/random-referrers-for-test.list @@ -1,500 +1,500 @@ -00it.com -101billion.com -101raccoon.ru -12u.info -1webmaster.ml -24x7-server-support.site -34.gs +101flag.ru +123cha.com +13tabs.com +1-free-share-buttons.com +2728fb936f0.com +2ads.co.uk +2pxg8bcf.top 4kpics.rocks -4ureyesonly.com -6128786.com -8xv8.com -98oi.ru +4kplayer.pl 999webdesign.com -aaa38852e886ac4af1a3cff9b47cab6272.com -aarbur.com -abbanreddy.com -abiente.ru -absolute-s.ru -addbin.men -addray.pro -adidas.frwebs.fr +aa14ab57a3339c4064bd9ae6fad7495b5f.com +aac52006ec82a24e08b665f4db2b5013f7.com +aad1f4acb0a373420d9b0c4202d38d94fa.com +aaronabel.com +absolutelyanalog.com +adclickthru.net +addtoadd.men +adman.gr admatic.com.tr -adultfriendfinder.com -agysya.ru -akita.kz -albuteroli.com -algarveglobal.com -alibestsale.com -aliexpresscashback.club -allcredits.su -amazon-adsystem.com -apartamentwroclaw.eu +adserver-e7.com +adsland.men +adult3dgames.com +advancedcleaner.com +alert-fjg.xyz +algerianembassy.co.in +alienwheels.de +allpdfmags.net +allwomen.info +alyeskaresort.com +am15.net +amanda-porn.ga +ameblo.top +anime.dougasouko.com +anonymous-redirect.com +apiadanaknet-a.akamaihd.net apibatbrowsecom-a.akamaihd.net -apilinkswiftco-a.akamaihd.net -apiportalmorecom-a.akamaihd.net -apple.com-cleaner.systems -approved.su -arewater.com -arius.tech -arraty.altervista.org -artefakct.com -articlesdirectoryme.info -aruplighting.com -as5000.com +apple.com-webbrowsing-security.review +ascat.porn asrvrep-a.akamaihd.net -atmagroup.ru -autobudpostach.club -automatic-seo.com -avtocredit-legko.ru -azazaporn.com -backlinkwatch.com -balkanfarma.org -bankiem.pl -bannerads.de -bannerconnect.net -basedecor.ru -bdsmgalls.net -bharatdefencekavach.com -biglistofwebsites.com -billiard-classic.com.ua -bitnote.co -bizcheapjerseyswholesalechina.com -biznesrealnost.ru -blogsfunky672.weebly.com -bochemit.com.ua -boole.onlinewebshop.net -braintobe.win -brainzpod.win -brandbucket.com +asscenihotosidea.blogspot.com +audiofree.ru +australia-opening-times.com +aversis.net +aviva-limoux.com +azartmix.com +bag77.ru +bannerspace.com +beachpics.com +beauty-b0x.pl +becuo.com +besthomemadepornsites.com +best-mam.ru +best-way.men +bestwrinklecreamnow.com +betune.onlinewebshop.net +bin-brokers.com +binomo.kz +bio.trade-jp.net +biteg.xyz +bizru.info +bjetjt.com +blackcurranthumidifiers.site +blavia.00author.com +bloggers.nl +bluesalt.co +bonux.nextview.ru +bookmark4you.com +bookmark4you.com.biz +boostmyppc.com +brainboostingsupplements.org +brandehk.dk brewdom.ru -brg8.com -buketeg.xyz -bzero1jewelry.net -callmd5map.com -canacopegdl.com -cardiosport.com.ua +brillianty.info +buyscabiescream.com +cactussoft.biz +camdolls.com +cand.jp +candlewooddentalcentre.com.au carmuffler.net -carsplate.com -cartierring.xyz -casinos4dummies.co.uk -cejewelry.xyz -centre-indigo.org.ua -cheapjerseysbizwholesale.us -cheapjerseysfootballshop.com +cartierlovebraceletreplica.xyz +cctva.tv +cementaresearch.se +cezartabac.ro +c-gainsbourg.com +chatroulette.video +chelnytruck.ru +chelyabinsk.xrus.org +chinaelitecheapjerseys.com +chiptuninger.com +chlooe.com +choice-credit.ru choosecuisine.com -cialiswithoutadoctor.net -cienum.fr -citysecurity.nu -click2pawn.com -coldfilm.ru -com-onlinesupport.site -compliance-alexa.top -compliance-jessica.top -compliance-walt.top -conocer-sanabria.com -cool-wedding.net -corta.co -cosmeticswomens-womensports.rhcloud.com -counterbot.com -countercrazy.com -crackguru.tk -creatives.adbetclickin.pink -cutalltheshit.com -darmebel.com.ua +cialisndbrx.com +cialis-samples.com +clarithromycin500mg.com +classiquebijoux.ru +cmhomestayagency.com +coin-hive.com +compliance-andrew.xyz +compliance-george.xyz +constantaservice.net +contentlook.co +courtshipgift.com +creams.makeforum.eu +creditwell.ru +crystalslot.com +cxpromote.com +cyprusbuyproperties.com +d0t.ru +danashop.ru +daneshetabiat.com dawlenie.com -decorazilla.com -deutschehobbyhuren.net -diarioaconcagua.com -diesel-parts28.ru -djonwatch.ru -dmmspy.com -doeco.ru -dogbreedspicture.net -dojki-hd.com -domoysshop.ru +deplim.com +descargar-musica-gratis.net +designdevise.com +destinationrealestate.com +detki-opt.ru +de.zapmeta.com +diegolopezcastan.com +digilander.libero.it +dkvorota.ru +docs4all.com +dogoimage.com +domcran.net domznaniy.ru -donna7753191.ru -downloader12.ru +downvids.net dptaughtme.com -drugs-no-rx.info -dvrlists.com -educatemetv.com -efnor-ac.com -ekspertmed.com -elektrischezigarette1.onsugar.com +e705.net +easyukraine.com +ecommerce-seo.org +ege-essay.ru +egovaleo.it +ekaterinburg.xrus.org +ekn-art.se +elektrischezigarette2.onsugar.com elektronischezigarettekaufen1.beeplog.com elektronischezigarettekaufen2.tumblr.com elektrozigarette2.wordpressy.pl -enews.tech -eorogo.top -erotikstories.ru +elmifarhangi.com +elvenmachine.com +encodable.com +engines-usa.com +epicbrogaming.com +ereko.ru eshop.md -estelight.ru +etur.ru +eu-cookie-law.blogspot.com +eugenevaultstorage.com +euroskat.ru +exmasters.com +extlabs.io extlinks.com -ezigarettekaufen1.hpage.com -f1nder.org -falllow.gq -fertilitetsradgivningen.se -fickenbumsen.net -filmfanatic.com +eyelike.com.ua +eyessurgery.ru +factorynightclub.com +fandlr.com +fanoboi.com +fantasticpixcool.com +faracontrol.ir +farmingworm.com +fastfixing.tech +fearcrow.com +fenoyl.batcave.net filmgo.ru -fil.ru -financepoints.eu -finansov.info -fishtauto.ru -flavors.me -flooringinstallation-edmonton.com -flowersbazar.com -food.dtu.dk -formulaantiuban.com -forodvd.com -fotopop.club -foto-telok.net -fototravel.eu -franch.info -franecki.net +filunika.com.ru +findpik.com +finstroy.net +firma-legion.ru +floridahuntingfishingadventures.com +forum-engineering.ru frcls.fr -freecamdollars.com -free-deals.faith -freenode.info -fuckmill.com -futbolkisales.ru -fxtips.ru +freemags.cc +freenom.link +freewhatsappload.com +fuel-gas.com +fullfileaccess.com +fun-mobi.pl +galaxyflowers.ru galeria-zdjec.com -gamebackyard.com -gamedayhouse.com -gameprimary.com -gdcentre.ru -generalporn.org +gaz-voshod.ru +geckoandfly.com +gembird.com gepezz.info -ggiaro.com -gigablast.com -gilsonchiro.xyz -girlgamerdaily.com -girls-ufa.ru -glall.ru -glass-msk.ru -go2jump.org -gomusix.com -goodwriterssales.com -gosarhivrt.ru -graphics8.info -gsasearchenginerankerdiscount.com -gsasearchengineranker.site -gsasearchengineranker.space -gstatey.net -guardlink.com -havepussy.com -hazardky.net -hdwallpapers-free.com -heartofbeijing.blogspot.com -hermesreplica.pw -high-speed1.net -hildinghr.se -histock.info -homeinns.com -homelygarden.com +get-seo-domain.com +gheus.altervista.org +gidonline.one +gifspics.com +glavprofit.ru +glopages.ru +gobongo.info +gok-kasten.net +golden-catalog.pro +gonextmedia.com +goodprotein.ru +gox.com.ua +groupmoney.ru +guarrasdelporno.xxx +guitar-master.org +gwebtools.com.br +gz2.bbsoldes.fr +hauleddes.com +healthcarestore.info +health-medical-portal.info +helpmymacfaster.trade +highspeed5.net +historichometeam.com +hobild.net +homedo.fabpage.com +honyaku.yahoofs.jp +hostsshop.ru +hotenergy.ru +hothot.ru hotkeys.com -idealtits.net +hotxnights.info +houseofrose.com +hyiphunter.org +icaseclub.ru +ico.re ideashome.id -idegenvezeto.eu -igithab.com -ilovevitaly.com -imagecoolpub.com -indetiske.ya.ru -indo-export.ru -infogame.name -inmate-locator.us -innodgfdriverhm.aircus.com -intellektmedia.at -interferencer.ru -inzn.ru -ip-guide.com -iqbazar.ru -irkutsk.zrus.org -isotoner.com -itrevolution.cf -izhevsk.zrus.org -jav-fetish.site -jeremyeaton.co -jus0wil.pp.ua -justprofit.xyz -jyvopys.com -kasino-money.pw -katjimej.blog.fc2.com -kidd.reunionwatch.com -kiinomaniak.pl -kimcurlrvsms.com -kinky-fetishes.com -kinogolos.ru -kinogonew.ru -kladrus.ru -klin3952.ru +ie.57883.net +iideaidekonkatu.info +imagefinder.site +imguramx.pw +impisr.edunsk.ru +importchinacoach-teach.com +indiasourcemart.in +intermesh.net +istmira.ru +itag.pw +it-max.com.ua +ivoiretechnocom.ci +jagg.info +jobmarket.com.ua +jongose.ninja +journalhome.com +justbcause.com +kaac.ru +kandidos.com +kazinogames.lv +kerwinandcariza.com +keywordglobal.co.za +keyword-suggestions.com +kinotorka.ru +kiwe-analytics.com +kiwi237au.tk +klumba55.ru kovesszucs.atw.hu -krynica.info -kupit-adenu.ru -kustanay.kz +kozhniebolezni.com +kraljeva-sutjeska.com +krasivye-devushki.net +krasnodar.ru +kredit-pod-zalog-krasnodar.ru +kuzinsp.ru +kvartiry-remont.ucoz.ru +lacloop.info laexotic.com -laptoper.net -laxob.com -leboard.ru -letmacworkfaster.site -lexixxx.com -lifehacĸer.com -lightinghomes.net -likesuccess.com -linkbux.ru -linksharingt.com -livefixer.com +lazymanyoga.com +ledpolice.ru +lietaer.com +liffa.ru +limtu.ifmo.ru +littlesunraiser.com livejasmin.com -livetsomudvekslingstudent.bloggersdelight.dk -lizace.com ljusihus.se -lkbennettstore.online -loginduepunti.it -lollypopgaming.com -lol-smurfs.com -lonerangergames.com +lomb.co lookingglassemb.com -louboutinshoes.xyz -love-baby.cz -luxup.ru -mactechinfo.info -magda-gadalka.ru -magicdiet.gq -manipulyator-peterburg.ru -manve.info -matb3aa.com -maximpartnerspr.com -mb140.ru -mebelcomplekt.ru -metabar.ru -mindeyegames.com -minet.club -mini.7zap.com -mir-betting.ru -missvietnam.org -mobile.ok.ru -moneytop.ru -monitorwebsitespeed.com -monsterdivx.tv -moscow-region.ru -moyakuhnia.ru -mp3ringtone.info -mrwhite.biz -na15.ru -naturalbreakthroughsresearch.com -netpics.org -nibbler.silktide.com -nightvision746.weebly.com -njkmznnb.ru -nvformula.ru -omoikiri-japan.ru -one-gear.com -openmediasoft.com -opinionreelle.com -orsonet.ru -otbelivanie-zubov.com -paperwritingservice17.blogspot.ru +lunamedia.co +luxmagazine.cf +mabdoola.blogspot.com +macdamaged.space +machicon-akihabara.info +machicon-ueno.info +macresource.co.uk +mafa-free.com +makis.nu +mbiologi.ru +med-zdorovie.com.ua +megahdporno.net +megaindex.ru +megavolt.net.ua +members.ghanaweb.com +merryhouse.co.uk +message-warning.net +microstatic.pl +mindbox.co.za +miniads.ca +mirzonru.net +modabutik.ru +moneymaster.ru +montredemarque.nl +morefastermac.trade +mossmesi.com +mturkcontent.com +musirc.com +muztops.ru +my-aladin.com +mycouponizemac.com +mydoctorok.ru +mymercy.info +myplaycity.com +nate.com +needtosellmyhousefast.com +nero-us.com +netallergy.ru +netanalytics.xyz +new7ob.com +newsperuse.com +newtechspb.ru +nextlnk12.com +ningessaybe.me +no-fuel.org +nordvpn.com +novodigs.com +nowtorrents.com +ochistka-stokov.ru +offers.bycontext.com +okout.ru +omsk.xrus.org +onlyforemont.ru +o-o-8-o-o.ru +osagonline.ru +ourville.info +outshop.ru +ownshop.cf +page2rss.com +panchro.xyz pavlodar.xkaz.org -paydayonlinecom.com -pcgroup.com.uy -pdn-4.com -pechikamini.ru -perl.dp.ua -petitions.whitehouse.gov -phormchina.com -pipki.r.acdnpro.com -plaff-go.ru -plastweb.ru -plohaya-kreditnaya-istoriya.ru +picscout.com +piluli.info +pinkduck.ga +pix-hd.com +play.leadzupc.com +playlott.com pl-vouchers.com -pogosh.com -popmarker.com -pornogad.com -pornokajf.com -pornolook.net -pornonik.com +polybuild.ru +popads.net +porndroids.com +pornoblood.com porodasobak.net -porto.abuilder.net -portside.xyz -postclass.com -printingpeach.com -professionalwritingservices15.blogspot.ru -profitkode.com -proftests.net -promalp-universal.ru -prombudpostach.com.ua -promgirldresses.xyz -pronorm.fr -pro-okis.ru -prostitutki-kiev.org -prostitutki-odessa.org -proxyradar.com -prpops.com -psa48.ru -pushdata.sendpulse.com -qualitymarketzone.com -quebec-bin.com -quickloanbank.com -r-control.ru -realitykings.com -realtytimes.com -recipedays.com -redbottomheels.xyz -refudiatethissarah.info -reklama1.ru +prchecker.info +pregnant.guru +preparevideosafesystem4unow.site +pricheski-video.com +privatbank46.ru +prosperent.com +prostitutki-belgoroda.org +pussysaga.com +puzo2arbuza.ru +qld10000.net +quizzitch.net +rabot.host.sk +razyboard.com +reeyanaturopathy.com +refererx.com +registry-cleaner.net retailwith.com -rewardpoll.com -rezeptiblud.ru -ringtonepartner.com -rn-to-bsn.com -rock-cafe.info +ricorsogiustizia.org rockma.se -rp9.ru -rubbed.us -rusoft-zone.ru -s8-nowy-wygraj.comli.com -sa-live.com -sammlungfotos.online -sankt-peterburg.nodup.ru -santechnik.jimdo.com -savetubevideo.com -scanmyphones.com -scanner-ivan.top -scanner-olga.top +royal-investments.net +roznica.com.ua +ruscoininvest.company +saneyes.com +saugatuck.com +sbf441.com +scanner-julianna.top scoopquest.com -screentoolkit.com -searchencrypt.com -search.pch.com -seeresultshub-a.akamaihd.net -semprofile.com -seo-platform.com -servingnotice.com -shell-pmr.ru -shillyourcoins.com -shoesonlinebuy.cn -shopcheermakeup.info -shopsellcardsdumps.com -sideeffectsoftizanidine.blogspot.com -sinestesia.host.sk -sisi-go.ru -sitebeam.net -sitevalued.com -skutecznetabletkinaporostwlosow.pl -sky-mine.ru -skyway24.ru -sladkoevideo.com -slim.sellany.ru -slkrm.ru -slonechka.ru -snsdeainavi.info -sovinsteel.ru -spamnuker.com +scrapy.org +sc-specialhost.com +seansonline24.pl +searchimage.co +secretscook.ru +seinterface.com +semrush.com +servethis.com +se-welding.ru +sexad.net +sex-tracker.com +sharpchallenge.com +sherlock.se +signoredom.com +siteimprove.com +site.ru +site-speed-checker.site +skatestick.bid +slomm.ru +smart-scripts.com +socialtrade.biz +sodexo.com +softomix.com +sotechco.co +souvenirua.com +sovetogorod.ru +soviet-portal.do.am +spacash.com +spb.afora.ru speechfoodie.com -spin2016.cf -splendorsearch-a.akamaihd.net -sps-shop.com -stairliftstrue.com -stair.registrydomainservices.com -startwp.org -steame.ru -stickers-market.ru -stockspmb.info -suggest-keywords.com -surgut.zrus.org -svarkagid.com -taaaak.com -talant-factory.ru -tarad.com -tasteidea.com -taxi-v-eisk.ru -technopellet.gr -terraclicks.com -thaisamkok.com -thecoolimages.net -themeforest.net -theprofitsmaker.net -thetoiletpaper.com -the-usa-games.blogspot.com -thruport.com -tizanidinedrugtest.blogspot.com -tizanidinestreetvalue.blogspot.com -top10-way.com -top250movies.ru -topanasex.com -torrent-to-magnet.com -totu.us -tracfone.com -traffic2cash.org -traffic2money.com -trafficgenius.xyz -trainoffend.ml -truemfilelj.gq -trumpetedextremes.com -trustaffs.com -tsan.net -tuberkulezanet.ru +spravka-medosmotr.ru +stairliftsarea.com +stats-collector.org +streetfire.net +strfls.com +stroilka.info +stromerrealty.com +sukirgenk.dvrlists.com +superoboi.com.ua +sureone.pro +szucs.ru +taihouse.ru +taximytishi.ru +tech4master.com +tedy.su +templates.radiodigital.co +testingads.pro +thebestweightlosspills.ovh +thebitcoincode.com +theendivechronicles.com +thefds.net +thelottosecrets.com +thetardistimes.ovh +the-trader.net +titan-cloud.life +tizanidine6mg.blogspot.com +tizanidinecost.blogspot.com +tizanidinefibromyalgia.blogspot.com +tizanidinerestlesslegsyndrome.blogspot.com +tizanidinewithdrawal.blogspot.com +toolsky.com +torrentgamer.net +touchmods.fr +toys.erolove.in +toy-shop.top +tozup.com +trafficstars.com +truck-rece.life +trucri.me tupper-shop.ru +tv-spoty.info +twelvevisionspartyofcolorado.com twincitiescarservice.com -ua-company.ru -u-cheats.ru -unrealcommander.biz -uogonline.com -url-img.link +twitlinks.com +twodollarshows.com +uk-zheu20.ru +ultimateclassicrock.com +umekana.ru +unacittaconte.org +unionmarkt.de +upstore.me +upupa.net +urdoot.win +urlcut.ru uroffer.link -v720hd.ru -versaut.xxx-cam.webcam -viddyoze.com -videochat.life -viel.su -vip2ch.com -vip-file.com -vipms.ru -vltai.com -vseigru.one -vucms.com -vysigy.su -wanker.us -watchdogs-2.ru -watchmygf.net -waysbetter.cn -webalan.ru -webix.biz -webmasterhome.cn -wesharepics.site -wildcattube.com -win-spy.com -wma-x.com -wnoz.de -word-vorlagen.net +vancleefreplica.pw +varikoz24.com +velpanex.ru +vesnatehno.ru +via-energy-acquistare.com +viagra-soft.ru +videogamesecrets.com +vintontech.info +vipromoffers.com +vod.com.ua +voloo.ru +voprosotvet24.ru +vzglyadriv.kg +warningzscaler.heraeus.com +waterefficiency.co +watracker.net +web-betting.ru +website-datenbank.de +website-speed-up.site +weburlopener.com +weprik.ru +wesharepics.info +wingsofrefuge.net +wleuaprpxuvr.ga world-mmo.com -woweb.com.ua -wygraj-skiny.win +worldtraveler.world +worst-sites.online +wowcasinoonline.ooo +www.get-free-traffic-now.com +x69ty.ru +xchangetrak.com xjrul.com -xlolitka.com -xn------7cdbapdecfd4ak1bn0amjffj7afu3y.xn--p1ai -xn----7sbahjd3btneuw1joc.xn--p1ai -xn--80aaagvmjabrs1aoc9luc.xn--p1ai -xn--80ajbshivpvn2i.xn--p1ai -xn-----8kcatubaocd1bneepefojs1h2e.xn--p1ai -xn----8sbarihbihxpxqgaf0g1e.xn--80adxhks +xmronta.com +xn--80aikhbrhr.net +xn--b1adccaf1bzj.xn--p1ai x-porno.video x-rates.ru -xuki.us -xxx-cam.webcam -xzlive.com -y8games-free.com -yaminecraft.ru -yeartwit.com -yogamatsexpert.com -you-shall-not-pass.is74.ru -yuarra.pluto.ro +xsion.net +xxlargepop.com +youtoner.it zajm-zalog-krasnodar.ru -zarenica.net -zeroredirect1.com -zeroredirect6.com -zooggames.com -zryydi.com -zvezdagedon.ru +zaobao.com.sg +zeroredirect10.com +zigzog.ru +zkjovpdgxivg.ga +znakomstvaonlain.ru +znaniyapolza.ru +zrelaya.pw +zzbroya.com.ua diff --git a/.dev-tools/test_units/random-referrers-for-whitelist-test.list b/.dev-tools/test_units/random-referrers-for-whitelist-test.list index 23522a6cb..a38cf32e0 100644 --- a/.dev-tools/test_units/random-referrers-for-whitelist-test.list +++ b/.dev-tools/test_units/random-referrers-for-whitelist-test.list @@ -1,250 +1,250 @@ -101flag.ru -20pascals.nl -2345.com -acads.net -acgs.tk +0n-line.tv +4kepics.com +4webmasters.com +51unlim.ru +83net.jp +actices.com +addray.pro +adserver-e7.com adtech.de -adverster.com -adviceforum.info -agahinameh.com -akama.com -akusherok.ru -alloysteel.ru -allproblog.com -altamayoztourism.com -alvtank.se -amazingpic.net -animalrank.com -anmysite.com -apisecretsaucebi-a.akamaihd.net -aramaicmedia.org -arenda-yeisk.ru -atlant-auto.info -atout-energie-69.com -australia-opening-times.com -avek.ru -aviav.co -avto-oligarh.ru -babosas.com -backgroundpictures.net -balans.shahterworld.org -balla-boo.se -bashtime.ru -bedandbreakfast.com -bestbuy.ca -bestcalovebracelet.cn -bestserials.com -bezlimitko.xyz -biznesschat.net -bluerobot.info -bonzbuddy.com -bosefux.esy.es +advertisingtag.net +alevinefesleri.com +allkrim.com +alot.com +anal-acrobats.hol.es +anticrawler.org +apiadanaknet-a.akamaihd.net +arcadepages.com +arcteryxstore.online +ardimobileinfo.ml +artpicso.com +autotop.com.ua +auto-zapchasti.org +avitocars.ru +avramstroy.ru +backlink4u.net +balois.worldbreak.com +bankiem.pl +bannerbank.ru +bannerpower.com +basedecor.ru +bcmp.org +beauty-b0x.pl +belstaffstore.online +bestcurs.org +best-mam.ru +bitcoins-live.ru +blockworld.ru +bravegirlsclub.com +burn-fat.ga +buybest1.biz +buy-loft.ru buzz-porno.info -cakemediahq.com.au cakesplus.com.au -cardiosport.com.ua -cartechnic.ru -cartierlovebraceletreplica.xyz -cdncash.com -cete.ru -chastnoeporno.com -cibpenonptib.flu.cc -compliance-john.top -compliance-julianna.top -crackguru.tk -custom-electric-guitar.com -damedingel.ya.ru -danschawbel.com -deluxedumps.com -diarioaconcagua.com -dickssportinggoods.com -documentbase.net +callejondelpozo.es +calstaterealty.us +camdenmemorials.com +cheapjerseysap.com +clarithromycin500mg.com +classiquebijoux.ru +cleansearch.net +clean-virus-mac.com +clmforexeu.com +coindirect.io +compliance-nelson.top +cornerstone-countertops.com +cxpromote.com +czeshop.info +dgfitness.co +djekxa.ru +doctormakarova.ru domainsatcost.com -domashneeruporno.com -downloader12.ru -dptaughtme.com -dverimegapolis.ru -election.interferencer.ru -elektronischezigarettekaufen1.myblog.de -enews.tech -ereko.ru -executehosting.com -ezigarettenkaufen2.dreamwidth.org -f0815.de -familienzahnaerzte.com -fbdownloader.com -ferrotodo.com -floating-share-buttons.com -forexgb.ru -forex-indextop20.ru -forexunion.net -formularz-konkurs.tk -fortunejack.com -forum-engineering.ru -freeuploader.com -free-video-tool.com -front.to -fyl.com.ru -ge0ip.net -geilehausfrauen.net -getmiro.com -gigixo.com -gilsonchiro.xyz -gobongo.info -gogalleryawesome.com -goldadpremium.com -googlemare.com -gwhwpxbw.bloger.index.hr -hd-film.pl -homedo.fabpage.com -hostingclub.lk -host-protection.com -housediz.com -houston-vikings.com -hoverboard360.nl -humbmosquina.tk +domik-derevne.ru +donvito.unas.cz +druzhbany.ru +e-biznes.info +elektroniksigaraankara.info +elektronischezigarette2.mex.tl +elitedollars.com +elitepcgames.com +energydiet-info.ru +englishdictionaryfree.com +eorogo.top +essaypro.com +euronis-free.com +ezaz.info +ezigarettekaufen2.yolasite.com +fashion-mk.net +financehint.eu +finuse.com +fitness-video.net +flytourisme.org +focalink.com +fordsonmajbor.cf +foto-basa.com +franecki.net +freewebs.com +freewhatsappload.com +gambarkataku.co +game300.ru +gameprimary.com +gelezki.com +globetrotting-culture.ru +golden-praga.ru +gourcy.altervista.org +greenidesign.co +gymi.name +hatdc.org +hauleddes.com +health-medical-portal.info +hermesbelts.xyz +hopeonthestreet.co.uk +hptwaakw.blog.fc2.com +hunthillfarmtrust.org +ibmdatamanagement.co +idc.com.ua ideoworld.org -imediadesk.com -inbabes.sexushost.com -incomekey.net -infobabki.ru -infostatsvc.com -intellego.info -iplusbit.blogspot.co.za -javcoast.com -javspace.net -jikoman.info -kaliningrad.zrus.org -karachev-city.ru -karusel-market.ru -kazrent.com -keenoutlet.online -khafre.us -kinoplen.ru -kleine-titten.biz -ktotut.net -lacave.ntic.fr -laexotic.com -lakomka.com.ua +igithab.com +inet-traffic.com +infoupdate.org +infowarcraft.ru +innoslicon.com +integritylandscapeservices.com +izhevsk.xrus.org +jackpotchances.com +javatex.co.id +kfon.eu +kichenaid.ru +kinobest.pl +koleso24.com.ua +kollesa.ru +komputernaya-pomosh-moscow.ru +krasivye-devushki.net +lafriore.ru larose.jb2c.me -le-clos-des-alouettes.com -lyrics.home-task.com -m1media.net -magazin-pics.ru -makedo.ru -mebel-vstroika.ru -michaelkorssaleoutletonline.net -mikozstop.com -millionare.com -mil-stak.com -minharevisao.com -mrsdalloways.com -mypets.by -myprintscreen.com -myshopmatemac.com -nastydollars.com -netanalytics.xyz -netvouz.com -nicefloor.co.uk -nowtorrents.com -o00.in -offergroup.info -okeinfo.online -ooo-gotovie.ru -openmediasoft.com -orel-reshka.net -oxotl.com -paradontozanet.ru -partner-trustworthy.men -pavlodar.xkaz.org -pestomou.info -pewit.pw -photo-clip.ru -pictures-and-images.net -picturesmania.com -play.leadzupc.com -pochemychka.net -pogruztehnik.ru -polytopesexempt.com -pooleroadmedicalcentre.co.uk +lechenie-gemorroya.com +lesbian.xyz +letsrepair.in +littleberry.ru +louboutinbooties.xyz +lowephotos.info +luckyshop.net.ua +magda-gadalka.ru +masserect.com +mebelcomplekt.ru +medispainstitute.com.au +megahdporno.net +mesbuta.info +metarip.ru +milblueprint.com +mirobuvi.com.ua +mister-shop.com +mobile-appster.ru +monclerboots.xyz +moncleroutletonline.pw +moviezbonkerssk.cf +musicas.baixar-musicas-gratis.com +muzaporn.com +muztops.ru +myfreecams.com +mygameplus.com +mygameplus.ru +mykings.pw +mylesosibirsk.ru +mymobilemoneypages.com +myplaycity.com +myxdate.info +onclickpredictiv.com +online-x.ru +ordernorxx.com +page2rss.com +palocco.it +pinstake.com +platinumdeals.gr +pojdelo.weebly.com pornofiljmi.com -pornosmola.info -prchecker.info -proekt-gaz.ru -psoriasis-file.trade -pssucai.info -ptr.ruvds.com -pulse33.ru -pussysaga.com -quality-traffic.com -razleton.com -rep-am.com -rixpix.ru -rumamba.com -running-line.ru -runofilms.ru -rvzr-a.akamaihd.net -ryetaw.com -sa-live.com -sbornik-zakonov.ru -scanner-ivan.top -scmor.ilxc.cc -seo-tools-optimizing.com -sexcamamateurchat.com -sexspornotub.com -sildenafilcitratemed.com -snowplanes.com -spy-sts.com -stanthonyscatholicchurch.org -steamoff.net -stuff-about-money.com -superinterstitial.com -supermesta.ru -sureone.pro -susanholtphotography.com -swiped.su -tabletkinaodchudzanie.com.pl -tamada69.com -telegramdownload10.com -thegamerznetwork.com -the-trader.net -tizanidineandhydrocodone.blogspot.com -tizanidinehydrochloride2mg.blogspot.com -tizanidinerecreationaluse.blogspot.com -tizanidinerestlesslegsyndrome.blogspot.com -tizanidinesideeffects.blogspot.com -tizanidineusedfor.blogspot.com -topcar-krasnodar.ru -toproadrunner5.info +porno-transsexuals.ru +positive2b.ru +pr0fit-b0x.com +prodvigator.ua +projefrio.com.br +promalp-universal.ru +promoforum.ru +pron.pro +prostitutki-kiev.org +purplesphere.in +pushdata.sendpulse.com +rankingchart.de +rcb101.ru +registry-cleaner.net +remmling.de +research.ifmo.ru +rff-cfal.info +ribieiendom.no +rubbed.us +rucrypt.com +sanjosestartups.com +sankt-peterburg.nodup.ru +saratov.xrus.org +scanner-andrew.top +scanner-walt.top +sexiporno.net +shoppingmiracles.co.uk +shoppytoolmac.com +shopsellcardsdumps.com +shop.xz618.com +space-worry.ml +spitfiremusic.com +spywareit.com +sribno.net +stairliftstrue.com +storehouse.ua +suhanpacktech.com +tabakur77.com +tech4master.com +tedy.su +telesvoboda.ru +thebestphotos.eu +tizanidinehclsideeffects.blogspot.com +tizanidinezanaflex.blogspot.com +top250movies.ru track-rankings.online -traffic2money.com -trafficmania.com -trafficmonetize.org -tzritel.tk -ucoz.ru -unisexjewelry.org -v720hd.ru -vapomnoncri.tk -vbabule.net -viagraneggrx.com -viagra-soft.ru -vip-parfumeria.ru -vkontarkte.com -vonradio.com -vote-up.ru -wareseeker.com -webradiology.ru -wedding-salon.net -weprik.ru -wttavern.com -www.kabbalah-red-bracelets.com -x-diesel.info -xn--80aebbcbcdemfkhba4byaehoejh8dza3v.xn--p1ai -xn----itbeirbjbi7bc6bh2d.xn--p1ai -xsion.net -yellowads.men -zebramart.ru -zeroredirect1.com -zetmaster.ru -znakomstva-moskva77.ru -zona-aqua.ru +tracxn.com +tri-slona.org +trk-4.net +truck-addzilla.life +tuberkuleznik.ru +tubo360.com +twu.com.ua +tyumen.xrus.org +uasb.ru +ugguk.online +uogonline.com +urlcut.ru +userequip.com +varikozdok.ru +viberdownload10.com +vip.51.la +voluumtracker1.com +voprosotvet24.ru +voronezh.xrus.org +vysigy.su +weclipart.com +wemarketing.se +wesharepics.com +westvilletowingservices.co.za +winner7777.net +wordpresscore.com +wordpress-crew.net +wowcasinoonline.ooo +ws.ampower.me +xerox-douglas.cf +xtraffic.plus +yadro.ru +yellocloud.be +youdesigner.kz +yun56.co +zarabotat-na-sajte.ru +zhorapankratov7.blogspot.com +zoomovies.org +zrelaya.pw diff --git a/.dev-tools/test_units/ratelimittest-master.txt b/.dev-tools/test_units/ratelimittest-master.txt index 2b1a6d95e..afb1627e8 100644 --- a/.dev-tools/test_units/ratelimittest-master.txt +++ b/.dev-tools/test_units/ratelimittest-master.txt @@ -5,6 +5,97 @@