diff --git a/.dev-tools/beta_conf_files/conf.d/globalblacklist.conf b/.dev-tools/beta_conf_files/conf.d/globalblacklist.conf index 4a3c5363b..0037f9038 100644 --- a/.dev-tools/beta_conf_files/conf.d/globalblacklist.conf +++ b/.dev-tools/beta_conf_files/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2020.01.1942 -### Updated: Tue Jan 28 13:12:07 SAST 2020 +### Version: V3.2020.01.1945 +### Updated: Tue Jan 28 14:26:16 SAST 2020 ### Bad Referrer Count: 6814 ### Bad Bot Count: 562 ################################################### diff --git a/.dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf b/.dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf index cc3de7218..38f5c32c1 100644 --- a/.dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf +++ b/.dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -34.66.200.49 1; +34.73.34.132 1; 127.0.0.1 1; diff --git a/.dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf b/.dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf index 1d3684d1d..c7afa9b6e 100644 --- a/.dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf +++ b/.dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -34.66.200.49 0; +34.73.34.132 0; 127.0.0.1 0; diff --git a/.dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf b/.dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf index 4a3c5363b..0037f9038 100644 --- a/.dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf +++ b/.dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2020.01.1942 -### Updated: Tue Jan 28 13:12:07 SAST 2020 +### Version: V3.2020.01.1945 +### Updated: Tue Jan 28 14:26:16 SAST 2020 ### Bad Referrer Count: 6814 ### Bad Bot Count: 562 ################################################### diff --git a/.dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf b/.dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf index cc3de7218..38f5c32c1 100644 --- a/.dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf +++ b/.dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -34.66.200.49 1; +34.73.34.132 1; 127.0.0.1 1; diff --git a/.dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf b/.dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf index 1d3684d1d..c7afa9b6e 100644 --- a/.dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf +++ b/.dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -34.66.200.49 0; +34.73.34.132 0; 127.0.0.1 0; diff --git a/.dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf b/.dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf index 4a3c5363b..0037f9038 100644 --- a/.dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf +++ b/.dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2020.01.1942 -### Updated: Tue Jan 28 13:12:07 SAST 2020 +### Version: V3.2020.01.1945 +### Updated: Tue Jan 28 14:26:16 SAST 2020 ### Bad Referrer Count: 6814 ### Bad Bot Count: 562 ################################################### diff --git a/.dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf b/.dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf index cc3de7218..38f5c32c1 100644 --- a/.dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf +++ b/.dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -34.66.200.49 1; +34.73.34.132 1; 127.0.0.1 1; diff --git a/.dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf b/.dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf index 1d3684d1d..c7afa9b6e 100644 --- a/.dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf +++ b/.dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -34.66.200.49 0; +34.73.34.132 0; 127.0.0.1 0; diff --git a/.dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf b/.dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf index 4a3c5363b..0037f9038 100644 --- a/.dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf +++ b/.dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2020.01.1942 -### Updated: Tue Jan 28 13:12:07 SAST 2020 +### Version: V3.2020.01.1945 +### Updated: Tue Jan 28 14:26:16 SAST 2020 ### Bad Referrer Count: 6814 ### Bad Bot Count: 562 ################################################### diff --git a/.dev-tools/globalblacklist-testing.template b/.dev-tools/globalblacklist-testing.template index 4a3c5363b..0037f9038 100644 --- a/.dev-tools/globalblacklist-testing.template +++ b/.dev-tools/globalblacklist-testing.template @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2020.01.1942 -### Updated: Tue Jan 28 13:12:07 SAST 2020 +### Version: V3.2020.01.1945 +### Updated: Tue Jan 28 14:26:16 SAST 2020 ### Bad Referrer Count: 6814 ### Bad Bot Count: 562 ################################################### diff --git a/.dev-tools/globalblacklist.template b/.dev-tools/globalblacklist.template index 3542d45ce..16705b611 100755 --- a/.dev-tools/globalblacklist.template +++ b/.dev-tools/globalblacklist.template @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2020.01.1942 -### Updated: Tue Jan 28 13:12:06 SAST 2020 +### Version: V4.2020.01.1945 +### Updated: Tue Jan 28 14:26:15 SAST 2020 ### Bad Referrer Count: 6814 ### Bad Bot Count: 562 ################################################### diff --git a/.dev-tools/test1_conf_files/bots.d/whitelist-ips.conf b/.dev-tools/test1_conf_files/bots.d/whitelist-ips.conf index 1749803d6..5f04cdae8 100644 --- a/.dev-tools/test1_conf_files/bots.d/whitelist-ips.conf +++ b/.dev-tools/test1_conf_files/bots.d/whitelist-ips.conf @@ -45,4 +45,4 @@ # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST -35.188.15.155 0; +34.73.34.132 0; diff --git a/.dev-tools/test1_conf_files/conf.d/globalblacklist.conf b/.dev-tools/test1_conf_files/conf.d/globalblacklist.conf index 3542d45ce..16705b611 100644 --- a/.dev-tools/test1_conf_files/conf.d/globalblacklist.conf +++ b/.dev-tools/test1_conf_files/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2020.01.1942 -### Updated: Tue Jan 28 13:12:06 SAST 2020 +### Version: V4.2020.01.1945 +### Updated: Tue Jan 28 14:26:15 SAST 2020 ### Bad Referrer Count: 6814 ### Bad Bot Count: 562 ################################################### diff --git a/.dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf b/.dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf index e22596857..c6cc367c2 100644 --- a/.dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf +++ b/.dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -34.66.25.221 1; +35.227.58.83 1; 127.0.0.1 1; diff --git a/.dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf b/.dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf index e6677be79..8d5a67df5 100644 --- a/.dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf +++ b/.dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -34.66.25.221 0; +35.227.58.83 0; 127.0.0.1 0; diff --git a/.dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf b/.dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf index 3542d45ce..16705b611 100644 --- a/.dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf +++ b/.dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2020.01.1942 -### Updated: Tue Jan 28 13:12:06 SAST 2020 +### Version: V4.2020.01.1945 +### Updated: Tue Jan 28 14:26:15 SAST 2020 ### Bad Referrer Count: 6814 ### Bad Bot Count: 562 ################################################### diff --git a/.dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf b/.dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf index e22596857..c6cc367c2 100644 --- a/.dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf +++ b/.dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -34.66.25.221 1; +35.227.58.83 1; 127.0.0.1 1; diff --git a/.dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf b/.dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf index e6677be79..8d5a67df5 100644 --- a/.dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf +++ b/.dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -34.66.25.221 0; +35.227.58.83 0; 127.0.0.1 0; diff --git a/.dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf b/.dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf index 3542d45ce..16705b611 100644 --- a/.dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf +++ b/.dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2020.01.1942 -### Updated: Tue Jan 28 13:12:06 SAST 2020 +### Version: V4.2020.01.1945 +### Updated: Tue Jan 28 14:26:15 SAST 2020 ### Bad Referrer Count: 6814 ### Bad Bot Count: 562 ################################################### diff --git a/.dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf b/.dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf index e22596857..c6cc367c2 100644 --- a/.dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf +++ b/.dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -34.66.25.221 1; +35.227.58.83 1; 127.0.0.1 1; diff --git a/.dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf b/.dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf index e6677be79..8d5a67df5 100644 --- a/.dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf +++ b/.dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -34.66.25.221 0; +35.227.58.83 0; 127.0.0.1 0; diff --git a/.dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf b/.dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf index 3542d45ce..16705b611 100644 --- a/.dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf +++ b/.dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2020.01.1942 -### Updated: Tue Jan 28 13:12:06 SAST 2020 +### Version: V4.2020.01.1945 +### Updated: Tue Jan 28 14:26:15 SAST 2020 ### Bad Referrer Count: 6814 ### Bad Bot Count: 562 ################################################### diff --git a/.dev-tools/test2_conf_files/bots.d/whitelist-ips.conf b/.dev-tools/test2_conf_files/bots.d/whitelist-ips.conf index 8cf4f00e8..dcd038a5e 100644 --- a/.dev-tools/test2_conf_files/bots.d/whitelist-ips.conf +++ b/.dev-tools/test2_conf_files/bots.d/whitelist-ips.conf @@ -45,5 +45,5 @@ # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST -35.192.10.37 0; -34.66.50.208 0; +35.231.58.0 0; +35.227.58.83 0; diff --git a/.dev-tools/test2_conf_files/conf.d/globalblacklist.conf b/.dev-tools/test2_conf_files/conf.d/globalblacklist.conf index 76d4f6694..d45b32425 100644 --- a/.dev-tools/test2_conf_files/conf.d/globalblacklist.conf +++ b/.dev-tools/test2_conf_files/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2020.01.1942 -### Updated: Tue Jan 28 13:12:06 SAST 2020 +### Version: V4.2020.01.1945 +### Updated: Tue Jan 28 14:26:15 SAST 2020 ### Bad Referrer Count: 6814 ### Bad Bot Count: 562 ################################################### diff --git a/.dev-tools/test3_conf_files/bots.d/whitelist-ips.conf b/.dev-tools/test3_conf_files/bots.d/whitelist-ips.conf index 71e2e0bdd..28f694873 100644 --- a/.dev-tools/test3_conf_files/bots.d/whitelist-ips.conf +++ b/.dev-tools/test3_conf_files/bots.d/whitelist-ips.conf @@ -45,5 +45,5 @@ # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST -34.66.25.221 0; -104.197.122.201 0; +35.196.82.30 0; +34.73.34.132 0; diff --git a/.dev-tools/test3_conf_files/conf.d/globalblacklist.conf b/.dev-tools/test3_conf_files/conf.d/globalblacklist.conf index 075012f5b..11fc89a11 100644 --- a/.dev-tools/test3_conf_files/conf.d/globalblacklist.conf +++ b/.dev-tools/test3_conf_files/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2020.01.1942 -### Updated: Tue Jan 28 13:12:06 SAST 2020 +### Version: V4.2020.01.1945 +### Updated: Tue Jan 28 14:26:15 SAST 2020 ### Bad Referrer Count: 6814 ### Bad Bot Count: 562 ################################################### diff --git a/.dev-tools/test4_conf_files/bots.d/whitelist-ips.conf b/.dev-tools/test4_conf_files/bots.d/whitelist-ips.conf index 6b5601ae4..efc95de51 100644 --- a/.dev-tools/test4_conf_files/bots.d/whitelist-ips.conf +++ b/.dev-tools/test4_conf_files/bots.d/whitelist-ips.conf @@ -45,5 +45,6 @@ # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST -104.197.122.201 0; -35.192.10.37 0; +35.227.58.83 0; +35.196.82.30 0; +34.73.66.97 0; diff --git a/.dev-tools/test4_conf_files/conf.d/globalblacklist.conf b/.dev-tools/test4_conf_files/conf.d/globalblacklist.conf index 3542d45ce..16705b611 100644 --- a/.dev-tools/test4_conf_files/conf.d/globalblacklist.conf +++ b/.dev-tools/test4_conf_files/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2020.01.1942 -### Updated: Tue Jan 28 13:12:06 SAST 2020 +### Version: V4.2020.01.1945 +### Updated: Tue Jan 28 14:26:15 SAST 2020 ### Bad Referrer Count: 6814 ### Bad Bot Count: 562 ################################################### diff --git a/.dev-tools/test5_conf_files/bots.d/whitelist-ips.conf b/.dev-tools/test5_conf_files/bots.d/whitelist-ips.conf index de880b6d6..d2df6a91c 100644 --- a/.dev-tools/test5_conf_files/bots.d/whitelist-ips.conf +++ b/.dev-tools/test5_conf_files/bots.d/whitelist-ips.conf @@ -45,5 +45,5 @@ # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST -35.192.10.37 0; -34.66.200.49 0; +34.73.34.132 0; +34.74.91.53 0; diff --git a/.dev-tools/test5_conf_files/conf.d/globalblacklist.conf b/.dev-tools/test5_conf_files/conf.d/globalblacklist.conf index 3542d45ce..16705b611 100644 --- a/.dev-tools/test5_conf_files/conf.d/globalblacklist.conf +++ b/.dev-tools/test5_conf_files/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2020.01.1942 -### Updated: Tue Jan 28 13:12:06 SAST 2020 +### Version: V4.2020.01.1945 +### Updated: Tue Jan 28 14:26:15 SAST 2020 ### Bad Referrer Count: 6814 ### Bad Bot Count: 562 ################################################### diff --git a/.dev-tools/test_units/blacklist-ips.conf b/.dev-tools/test_units/blacklist-ips.conf index cc3de7218..38f5c32c1 100644 --- a/.dev-tools/test_units/blacklist-ips.conf +++ b/.dev-tools/test_units/blacklist-ips.conf @@ -1,2 +1,2 @@ -34.66.200.49 1; +34.73.34.132 1; 127.0.0.1 1; diff --git a/.dev-tools/test_units/random-bots-for-test-quick.list b/.dev-tools/test_units/random-bots-for-test-quick.list index 735cd0cf7..466254fb4 100644 --- a/.dev-tools/test_units/random-bots-for-test-quick.list +++ b/.dev-tools/test_units/random-bots-for-test-quick.list @@ -1,10 +1,10 @@ -AfD-Verbotsverfahren -Attach -CopyRightCheck -FlashGet -Kozmosbot -PxBroker -Sqworm -WebCollage -WebFuck -ZyBorg +BDCbot +FrontPage +GalaxyBot +InfoNaviRobot +Msrabot +trendiction.de +UnisterBot +VidibleScraper +Web Fuck +Wprecon diff --git a/.dev-tools/test_units/random-bots-for-test.list b/.dev-tools/test_units/random-bots-for-test.list index f43492f98..1e08ed4f5 100644 --- a/.dev-tools/test_units/random-bots-for-test.list +++ b/.dev-tools/test_units/random-bots-for-test.list @@ -1,250 +1,250 @@ 404checker -Abonti +404enemy +80legs Aboundex -Acunetix +Aboundexbot +ADmantX AfD-Verbotsverfahren -AhrefsBot -Alexibot -Alligator AllSubmitter -AlphaBot Anarchie Apexoo arquivo.pt -Asterias +arquivo-web-crawler +ASPSeek Attach -autoemailspider AwarioRssBot -backlink-check -BacklinkCrawler -BackWeb -Bandit +Badass BatchFTP Battleztar Bazinga -BBBike BDCbot -BetaBot -BlackWidow -BLEXBot -Boardreader -Bolt -BotALot +BDFetch +Bitacle Brandprotect Brandwatch Buddy -Bullseye -BunnySlippers -Calculon +BuiltBotTough +BuzzSumo CazoodleBot -ChinaClaw -Claritybot +CCBot +CheeseBot +CheTeam +Cogentbot cognitiveseo -Copyscape +com.plumanalytics Cosmos +Craftbot crawler4j crawler.feedback crawl.sogou.com -CrazyWebCrawler -Crescent -CrunchBot -CSHttp -Curious +Custo DBLBot +demandbase-bot Demon +Devil +DigitalPebble DIIbot -Discoverybot +Dirbuster +Disco Dispatch DittoSpyder -DomainAppender -DomainSigmaCrawler +DomainStatsBot Dotbot +Download Wonder Dragonfly -Drip +DSearch +DTS Agent +EasyDL Ebingbong eCatch -EroCrawler -evc-batch -Express WebPictures +ECCP/1.0 +Exabot +ExtLinksBot Extractor ExtractorPro -Extreme Picture Finder -EyeNetIE +Ezooms +facebookscraper FDM -FlashGet +FemtosearchBot +FHscan +Fimap +Firefox/7.0 Flunky +Foobot Freeuploader -FrontPage -FyberSpider -Fyrebot -Getintent -GetRight -Gigabot +GalaxyBot +GermCrawler +Gigablast +G-i-g-a-b-o-t +Go-Ahead-Got-It Gotit -GoZilla Go!Zilla -Grafula -GrapeFX -GridBot +Grabber Haansoft -Harvest -HEADMasterSEO +heritrix +Heritrix HMView HTMLparser -HTTP::Lite -HTTrack -HybridBot -Iblog +Humanlinks +IDBot IlseBot -Image Fetch IndeedBot +InfoNaviRobot +instabid InterGET -InternetSeer -IRLbot -Jbrofuzz +internetVista monitor +Iria +IstellaBot +JamesBOT JennyBot JikeSpider -JOC Web Spider -Joomla +JustView Lanshanbot -LeechGet +Larbin +LeechFTP Lftp Libwhisker Lightspeedsystems -Likse -LinkextractorPro -LinkpadBot -LinkScan -LinkWalker -Litemage_walker -Lmspider +LinksManager +LinqiaMetadataDownloaderBot +LinqiaRSSBot +Lipperhey Spider +Ltx71 +LWP::Simple +lwp-trivial +Magnet +Mag-Net magpie-crawler -Majestic12 +Mail.RU_Bot Majestic SEO -Masscan -Mass Downloader +Majestic-SEO +MarkMonitor +MarkWatch Mata Hari meanpathbot +Meanpathbot Mediatoolkitbot +mediawords Metauri -MFC_Tear_Sample -Microsoft Data Access +Microsoft URL Control MIDown tool +Mister PiX +Mojeek Msrabot -Musobot -Name Intelligence Nameprotect -Navroad +NearSite Needle -NetSpider -Net Vampire +Nessus +Nettrack NetZIP +NextGenSearchBot Nibbler -Niki-bot -Nimbostratus -NPbot +Nikto +NimbleCrawler +Ninja +Nmap +Nutch Offline Navigator OnCrawl -OpenVAS +Openfind +Openvas +OrangeBot OrangeSpider -PageAnalyzer -Page Analyzer -page scorer -Pandalytics -Papa Foto -PeoplePal +OutclicksBot +OutfoxBot +PageScorer +pcBrowser +PECL::HTTP +PictureFinder Pi-Monster -ProWebWalker -Psbot +Pixray +POE-Component-Client-HTTP +Probethenet +ProPowerBot +PxBroker PyCurl +QueryN Metasearch +RankActive +RankActiveLinkBot RankFlex -RankingBot -RankurBot +Rankivabot RebelMouse -Recorder -RedesScrapy ReGet -Rogerbot -s1z.ru +Ripper +RSSingBot SalesIntelligent -SBIder -ScanAlert -scan.lol -ScoutJet -Scrapy -Screaming -Searchestate -SemrushBot +Scanbot +SearchmetricsBot +Semrush SEOkicks -SEOlyticsCrawler -Seomoz seoscanners -SeoSiteCheckup -SEOstats +sexsearcher +Shodan Siphon -SISTRIX SiteExplorer -Siteimprove +SiteLockSpider SiteSucker -Site Sucker Sitevigil -SlySearch -SMTBot -Snake +SmartDownload Snapbot -SocialRankIOBot -Sociscraper -Sogou web spider -Sosospider +Snoopy +sogouspider SpaceBison -SputnikBot -spyfu -Sqlworm -Sucuri -SuperBot +Spammen +SpankBot +Spanner +Spbot +Stripper +Sucker +SuperHTTP Surfbot -Swiftbot +Suzuran +sysscan Szukacz -T0PHackTeam -Teleport -TeleportPro -Telesphoreo +Telesoft TheNomad -Tracemyfile +Thumbor +Toata +Toweyabot +Trendiction trendiction.com -trendiction.de Turingos -TurnitinBot -Typhoeus -Upflow +TwengaBot +Twice +UnisterBot URLy.Warning -Vacuum -VeriCiteCrawler -Virusdie +VB Project VoidEYE +Voltron Wallpapers/3.0 Webalta +Web Auto WebBandit Web Collage -WEBDAV +WebCopier WebEnhancer +Web Enhancer WebFetch +Web Fetch +Web Fuck WebGo IS WebmasterWorldForumBot -WebReaper +Web Pix WebSauger -Webshag WebsiteExtractor +Website Quester WebSucker -WebWhacker -WebZIP +Web Sucker +WeSEE +Whatweb +Widow +Wonderbot Woobot -Wprecon -WPScan +WWW-Mechanize +WWW::Mechanize WWWOFFLE -Xaldon WebSpider +x22Mozilla Xaldon_WebSpider Xenu xpymep1.exe -YoudaoBot Zermelo -ZmEu +zgrab diff --git a/.dev-tools/test_units/random-bots-for-whitelist-test.list b/.dev-tools/test_units/random-bots-for-whitelist-test.list index 50b47168a..fcf14d206 100644 --- a/.dev-tools/test_units/random-bots-for-whitelist-test.list +++ b/.dev-tools/test_units/random-bots-for-whitelist-test.list @@ -1,250 +1,250 @@ -Aboundex -ADmantX -AIBOT -Aipbot +404checker +404enemy +Aboundexbot +AhrefsBot +Alexibot Alligator AlphaBot Apexoo -archive.org_bot arquivo.pt ASPSeek -Attach -autoemailspider +Asterias AwarioSmartBot -BackDoorBot -Backlink-Ceck -backlink-check -BackStreet -BatchFTP +Badass BBBike BDCbot BDFetch -Blackboard +BetaBot +Bigfoot +Bitacle Black Hole -BlowFish -Boardreader Bolt +BotALot Brandprotect -Buddy BuiltBotTough BuiltWith -Bullseye BunnySlippers BuzzSumo +Calculon +CATExplorador CazoodleBot +CCBot CheTeam -Claritybot -Cliqzbot -Cloud mapping -coccocbot-web -Copyscape -Cosmos +Chlooe +Cogentbot +cognitiveseo +Collector +com.plumanalytics Craftbot +crawler.feedback CrunchBot CSHttp +Custo DataCha0s -DBLBot -Demon -Devil +demandbase-bot Digincore -DigitalPebble DIIbot +Dirbuster +Disco Discobot -DittoSpyder +DnyzBot DomainAppender DomainCrawler +DomainSigmaCrawler DomainStatsBot Dotbot +Download Wonder +Dragonfly Drip DTS Agent -EasyDL +Ebingbong EirGrabber +EMail Siphon EroCrawler +evc-batch +Express WebPictures +ExtLinksBot Extractor -ExtractorPro EyeNetIE -Ezooms -facebookscraper +FHscan +Fimap Firefox/7.0 -Flunky -Freeuploader +Foobot +FyberSpider Fyrebot +GalaxyBot Genieo GermCrawler -Getintent -GetRight +GetWeb +G-i-g-a-b-o-t +Go-Ahead-Got-It Gotit GoZilla -Grafula GrapeFX -GrapeshotCrawler +GT::WWW +Haansoft HaosouSpider HEADMasterSEO -heritrix -HybridBot -IDBot +Hloader +HMView +HTMLparser +HTTP::Lite +HTTrack Id-search -IlseBot -Image Fetch IndeedBot InfoNaviRobot -InternetSeer -internetVista monitor +Intelliseek +InterGET +Internet Ninja Iria -IRLbot -IstellaBot -JamesBOT -Jbrofuzz +Iskanie +JetCar Jetty -JikeSpider +JOC Web Spider Joomla -Jorgee Jyxobot Kenjin Spider -LeechFTP Lftp -Libwhisker -Linkdexbot -LinkWalker -LinqiaMetadataDownloaderBot -LinqiaScrapeBot +Lightspeedsystems +LinkpadBot +LinkScan Lipperhey Spider +Lmspider LNSpiderguy Ltx71 +lwp-trivial +Magnet Mag-Net -Mail.RU_Bot -Majestic12 -Majestic SEO +magpie-crawler +MarkWatch Masscan -Mata Hari MauiBot -Meanpathbot -MeanPath Bot -MFC_Tear_Sample +meanpathbot +Mediatoolkitbot +MegaIndex.ru Microsoft Data Access Microsoft URL Control -MIDown tool -MJ12bot +MIIxpc Mojeek -Morfeus Fucking Scanner +Mr.4x3 MSFrontPage -muhstik-scan -Name Intelligence -Nameprotect -Needle +Msrabot +NearSite NetAnts -NetMechanic -NetSpider -Nettrack -Net Vampire +netEstate NE Crawler Netvibes +NextGenSearchBot NICErsPRO -NPbot +Nikto +Nimbostratus +Ninja +Nmap Nutch oBot -Octopus Offline Explorer Offline Navigator Openvas +OpenVAS +OrangeBot OutfoxBot -PageAnalyzer +PageGrabber Pandalytics -Pavuk -PECL::HTTP -PHPCrawl +PeoplePal Picscout -PictureFinder -Pimonster -plumanalytics +Picsearch +Pi-Monster POE-Component-Client-HTTP Probethenet -ProWebWalker Pump +PxBroker +PyCurl +Quick-Crawler +RankActive RankActiveLinkBot -RankFlex +RankingBot RankingBot2 -Rankivabot -RankurBot -RedesScrapy -RSSingBot +RealDownload +Reaper +Recorder +ReGet +RepoMonkey +Ripper SalesIntelligent -scan.lol +Scanbot ScoutJet -Scrapy Screaming -Searchestate -SemrushBot +ScreenerBot SEOkicks -SEOlyticsCrawler +SEOprofiler +seoscanners +SeoSiteCheckup SEOstats serpstatbot -sexsearcher +Shodan +Siphon SISTRIX -Sitebeam -SiteLockSpider -SiteSnagger -SiteSucker -Site Sucker +SlySearch +SMTBot +Snoopy +SocialRankIOBot Sociscraper sogouspider -Sogou web spider -Sosospider Sottopop SpankBot Spanner +sp_auditbot +Spbot +SputnikBot spyfu +Sqlmap Sqlworm -Steeler -Sucker +Sqworm +Stripper Sucuri -SuperHTTP -SurveyBot +SuperBot Suzuran Swiftbot -Szukacz -T8Abot -Telesoft -Telesphorep +TeleportPro +Telesphoreo +TheNomad TightTwatBot Toweyabot -Tracemyfile -Trendiction Trendictionbot trendiction.com -True_Robot -TurnitinBot -TwengaBot -Twice -UnisterBot -URLy Warning +Typhoeus Vacuum +Vagabondo VB Project -VoidEYE -Voil -Wallpapers/3.0 +VCI +VidibleScraper +Virusdie +Voltron +WASALive-Bot WBSearchBot -Webalta -WebBandit -WebFetch -WebPix +Web Auto +Web Collage +WEBDAV +Web Enhancer +WebFuck +Web Fuck +WebImageCollector +WebmasterWorldForumBot +webmeup-crawler Web Pix -WebsiteExtractor -WebsiteQuester -Website Quester -WebStripper +WebReaper +Web Sauger +WebSucker +Web Sucker WebZIP Whacker +Widow WinHTTrack -WiseGuys Robot -Woobot -Wprecon +Wonderbot WPScan -WWW-Collector-E -x09Mozilla +WWW-Mechanize +Xaldon WebSpider Xenu -xpymep1.exe +YoudaoBot +Zauba Zermelo Zeus -zgrab -ZmEu -ZumBot diff --git a/.dev-tools/test_units/random-referrers-for-test-quick.list b/.dev-tools/test_units/random-referrers-for-test-quick.list index 10d1e3f21..ece02a368 100644 --- a/.dev-tools/test_units/random-referrers-for-test-quick.list +++ b/.dev-tools/test_units/random-referrers-for-test-quick.list @@ -1,10 +1,10 @@ -23kw.ru -balook.com -fetishinside.com -house.sieraddns.com -picmoonco.pw -responsive-test.net -sanidumps.com -warezaccess.com -ww2awards.info -zahvat.ru +compliance-nelson.top +detskie-konstruktory.ru +emailaccountlogin.co +hasfun.com +healgastro.com +hermesbirkinhandbagoutlets.com +mamasuper.prom.ua +meccadumps.net +trkdf.com +vapsy.com diff --git a/.dev-tools/test_units/random-referrers-for-test.list b/.dev-tools/test_units/random-referrers-for-test.list index 53d7ff411..2d403f3d3 100644 --- a/.dev-tools/test_units/random-referrers-for-test.list +++ b/.dev-tools/test_units/random-referrers-for-test.list @@ -1,500 +1,500 @@ -100dollars-seo.com -11235813.webzdarma.cz -123kuma.com -123locker.com -12masterov.com -1314dh.com +11pikachu.ru +1bet.com +1kinobig.ru +1webmaster.ml 2000k.ru -24videos.tv -24x7-server-support.site 2kata.ru -300richmond.co.nz -7a2rnuey1tw9ar.ru -91abcw.com -991mostfm.co.id -aa81bf391151884adfa3dd677e41f94be1.com +4senses.co +4ur.click +5000-cotydzien.com +5forex.ru +5i2.net +5tey7463.weebly.com +8558l.jobs.net aad1f4acb0a373420d9b0c4202d38d94fa.com -add-add.men -addbin.men -addblueoff.com.ua -admanaerofoil.com -adpremium.org -adventureparkcostarica.com -agahinameh.com -agreda.pluto.ro -ahrntal.verymes.xyz -air-edem.ru -airlandsea.info -aksonural.ru -akusherok.ru -albamargroup.com +acunetix-referrer.com +adbetclickin.pink +adman.gr +adprotect.net +adultactioncam.com +advancedsoftwaresupport.com +ageofclones.com +aibolita.com alert-fdm.xyz -algerianembassy.co.in -alienwheel.es -alienwheels.de -aliexpress.com -allblogroll.com -alldownload.pw -allstatesugarbowl.org -alphacarolinas.com -alvtank.se -ample-awards-today.us -anabolics.shop -analytics-ads.xyz -angry-fermi-7633.arukascloud.io -animalrank.com +alert-fjg.xyz +alfapro.ru +all4invest.info +allkrim.com +aludecor.info +am-se.com +animal-drawings.com +animali.deagostinipassion.it anlimebel.kiev.ua anniemation.com -apartamentwroclaw.eu -apibatbrowsecom-a.akamaihd.net -apibrowsesmartne-a.akamaihd.net -apidigidocketnet-a.akamaihd.net -apioutoboxnet-a.akamaihd.net -approved.su -arcteryxsale.online +apessay.com +apiitsthirteende-a.akamaihd.net +apiqualitinknet-a.akamaihd.net +arate.ru +architecturebest.com +arendas.net +arenda-yeisk.ru +arendovalka.xyz asophoto.com -autodo.info -autogrep.ru -autoloans.com +ausergrubhof.info +aussie-prizes.com +autoblogger24.info automatic-seo.com -auto-moto-elektronika.cz -avramstroy.ru -avtolombard-krasnodar.ru -ayodhya.co -azadnegar.com -babyfactory.fr -bang-hotties.com -bankcrediti.ru -baoxaydung.com.vn +aviapanda.ru +azartclub.org +bad-stars.net +baksman.com +barbourjackets.online bayareaaeroclub.org -bbtec.net -bear.gotcher.us -benchmarkcommunications.co.uk -bestempresas.es -best-way.men -bestwrinklecreamnow.com -betteroffers.review -betwinservice.com -bikini-image.com -bioinnovate.co -blogporn.in +beauty-bracelet.com +belstaffstore.online +besthoro.ru +bestofpicture.com +bestwebsiteawards.com +bfz.biz +bibys.com +bigcareer.info +bimatoprost-careprost.com.ua +biteg.xyz +bjanshee.ru +bloggers.nl +blog-hits.com +blog.koorg.ru blog.yam.com -bonkers.name -bonzybuddy.com -brakehawk.com -brewdom.ru -brimstonehillfortress.org -brus.city +boltalko.xyz +bongiornos.info +bonky.biz +bonvillan.com +bookmark4you.com +braindaily.xyz +brandedleadgeneration.com +bristolhostel.com btnativenav.com -btvn.ru -budpost.com.ua -builtwith.com -bum.com.ru -buqayy0.livejournal.com -buttons-for-website.com -bwlx.prepedu.cn -bytimedance.ru +buchananshardware.com +bugof.gq +bumascloset.com +buy-cheap-online.info +buypuppies.ca +buyscabiescream.com +bvlgaribracelet.xyz cacheimages.com -calcularpagerank.com -callawaygolfoutlet.online -callawaygolfstore.online -calstaterealty.us -carina-sy.de -carstrends2015.com -cartierjusteunclou.xyz -cartierlove2u.xyz -cartierlove.xyz +californianews.cf +candlehandmade.com +candy-glam-hp.com +carabela.com.do +carrauterie.be +cartechnic.ru cartujano-pre.de -cctva.tv -cellfun.mobi -cfcl.co.uk -chatroulette.world -cheapestjerseysonwholesale.com -checkpagerank.net -check-this-out-now.online -christianlouboutinshoes.xyz -cialiswithoutadoctor.net -cleanersoft.com -clknsee.com -clothesforcash.com -coderstate.com -companies-catalog.com -compiko.info -compliance-checker-7.info -compliance-fred.xyz -compliance-ivan.top -compliance-jack.top -com-supportcenter.website -conboy.us -cosmeticswomens-womensports.rhcloud.com -country-chic.ru +cashkitten-a.akamaihd.net +centraletermice.us +cfsrating.sonicwall.com +chatroulette.video +cheapmarmot.online +check-host.net +checktext.org +cherrypointplace.ca +clickaider.com +clickso.com +compliance-jane.top +compliance-julia.top +compliance-olga.top +confib.ifmo.ru +coolingoods.com +coprofam.org +cornomase.win +countercrazy.com +cowblog.fr +cupday.com +cvety24.by cxpromote.com -dafatiri.com -dailyrank.net -daretodonate.co -deda-moroza-zakaz.ru -demian.kz -dimfour.com -divku.ru -documentbase.net -dodlive.mil -domainsatcost.com -dominateforex.ml -downloadeer.net -doxysexy.com -drugspowerstore.com -dvervmoskvu.ru -dwomlink.info -dyt.net -earncash.com.ua +czat.wp.pl +d2jsp.org +damedingel.ya.ru +danschawbel.com +datsun-do.com +daymusam.com +detoxmed24.ru +dfiles.me +dfwu1013.info +divci-hry.info +domainanalyzing.xyz +domain.webkeyit.com +dosug-lux.ru +downloadkakaotalk.com +doxyporno.com +eachdayisagift.review easytuningshop.ru -easyukraine.com -elektrischezigarettekaufen2.cowblog.fr -elektrischezigaretten2.beeplog.com -elektrozigarette2.wordpressy.pl -elektrozigaretten1.webs.com -erotische-geschichten-xxl.com -eugenevaultstorage.com -eurosamodelki.ru -extractorandburner.com -ezigaretteshop2.vefblog.net -family1st.ca -feargames.ru -filmci.pro -findpik.com -firma-legion.ru -footbalive.org -forumprofi.de -foto-basa.com -fotzen-ficken.com -frameimage.org -frbizlist.com -freelinkbuilding.website.tk -freewareseek.com -freewlan.info +e-commerce-seo1.com +educontest.net +elektrischeziga.livejournal.com +eng-lyrics.com +epicbrogaming.com +eropornosex.ru +erotikchat-24.com +ertelecom.ru +es5.com +event-tracking.com +exmasters.com +extremez.net +ezigarettekaufen2.blox.pl +ezigarettenshop1.yolasite.com +face.hostingx.eu +felizporno.com +filesdatabase.net +filunika.com.ru +find1friend.com +findanysex.com +findsexguide.com +fitness-video.net +flash4fun.com +flex4launch.ru +floridahuntingfishingadventures.com +free-gluten.ru +free-traffic.xyz +frighteningremain.cf frustrated-favorable.gq -fsakhalin.ru -fullgirl.ru -funnel.co.za +fxgallery.com +fyxabomiw.ru +gallerily.com gambarxkata.co -gamerscorps.com +gamedayhouse.com games.kolossale.ru -genericlowlatencyasiodriverhq.aircus.com -globalscam.ga -gok-kasten.net -gomusix.com -googlefeud.com -google-liar.ru -googlepositions.com +generic-pills-online.com +gerhardhealer.com +glall.ru +gloverid.site +gogps.me gopro-online.info -gratis-sexkontakte.com -green-tea.tv -gstatey.net -guardlink.org -gyffu.com -gymi.name +gourcy.altervista.org +gox.com.ua +gpirate.com +gribkovye-zabolevaniya.com +growmyfunds.ca +gta-club.ru +hamptonoaks.ca hazardky.net -herokuapp.com -highspeed5.net -homedecorpicture.us +hccoder.info +hd-filmy.net +hdimagegallery.net +hebr.myddns-flir.com +hentai-manga.porn +hermesbelts.xyz +holidaypics.org homemade.gq -hoterika.com -hothot.ru -housemilan.ru -houston-vikings.com -hoverboard360.se -howlongdoestizanidinestayinyoursystem.blogspot.com -howtowhitenteethfast.xyz +honyaku.yahoofs.jp +hoste.octopis.com +hothor.se +hptwaakw.blog.fc2.com hubbble.com -hulfingtonpost.com -ido3.com -ilovevitaly.com -ilovevitaly.xyz -images-graphics-pics.com -imguramx.pw -impisr.edunsk.ru -inboundlinks.win -insider.pro +icaseclub.ru +ictizanidinehcl4mg.blogspot.com +ifirestarter.ru +i-hobot.ru +iideaidekonkatu.info +imediadesk.com +imk.com.ua +incolors.club +inmate-locator.us +instasexyblog.com interfucks.net -interior-stickers.ru -intimshop-fantasy.ru -int.search.mywebsearch.com -int.search.tb.ask.com +inzn.ru iomoio.net -istanbulit.com -istizanidineanopiate.blogspot.com -itbc.kiev.ua -itservicesthatworkforyou.com -javrip.net -jillepille.com -jolic2.com -journalhome.com -jyvopys.com -kakadu-interior.com.ua -kamagragelusa.net -kamalsinha.com -karusel-market.ru -katushka.net -keywordsuggests.com -kiwe-analytics.com -kleine-titten.biz +iplogger.org +iqoption-bin.com +irkutsk.zrus.org +ispaniya-costa-blanca.ru +istizanidinelikexanax.blogspot.com +ix20.ru +iyasimasennka.com +juliadiets.com +kaliningrad.zrus.org +karaganda.xkaz.org +keyword-suggestions.com +kino2018.club +kinopolet.net +kiprinform.com +klejonka.info +kmd-pto.ru knogg.net +komp-pomosch.ru kongruan.com -koptims.tiu.ru -kosmetyki.tm.pl -kostenloser-sex.com +konkursov.net kountrylife.com -kovesszucs.atw.hu -krasnodar-avtolombards.ru -kreativperlen.ch -kretpicf.pw +kriokomora.info +kupiproday.com.ua kupit-adenu.ru -kutikomi.net -labplus.ru -ladsblue.com -landoftracking.com -lawyers.cafe -laxob.com -letmacwork.world -letsrepair.in -lightinghomes.net -linkpulse.com -linkredirect.biz -live-sexcam.tk -localflirtbuddies.com +laptop-4-less.com +law-check-seven.xyz +leadn.pl +lerporn.info +lexixxx.com +light.ifmo.ru +linkarena.com +livingcanarias.com +locatellicorretor.com.br +lodki-pvh.dp.ua +lolnex.us +lombia.co +lomza.info lostfilm-online.ru -lottospring.com -louboutinreplica.pw -lsp-awak-perikanan.com -lunamedia.co -macdamaged.tech -machicon-ueno.info -madot.onlinewebshop.net -mafa-free.com -mahnwachen-helfen.info -managerpak204.weebly.com -mantramusic.ru -masturbate.co.uk -matb3aa.com -mcadamssupplyco.com -mcnamaratech.com -medosmotr-ufa.ru +louisvuittonoutletstore.net +lovelycraftyhome.com +luckyday.world +m3gadownload.pl +machicon-akihabara.info +magda-gadalka.ru +maju.bluesalt.co +makemoneyonline.com +manipulyator-peterburg.ru +manyresultshub-a.akamaihd.net +mararoom.ru +marinetraffic.com +marketingvici.com +markjaybeefractal.com +masserect.com +mb140.ru +mebelcomplekt.ru +mebel-ekb.com meduza-consult.ru -meetingrainstorm.bid -megavolt.net.ua -menetie.ru -mhi-systems.ru -minneapoliscopiers.com -misterjtbarbers.com -mmofreegames.online -mnogolok.info -moneyviking-a.akamaihd.net -morocco-nomad-excursions.com -moz.com -mrbitsandbytes.com -muizre.ru -nakozhe.com -naphukete.ru -nardulan.com -narkologiya-peterburg.ru -nbsproject.ru -needtosellmyhousefast.com -negociosdasha.com -npoet.ru -ntic.fr -offtopic.biz +megaindex.ru +memberty.com +mensandals.xyz +michaelkorssaleoutletonline.net +mir-betting.ru +mmoguider.ru +mmostrike.ru +mojpreskumanie.com +monetizationking.net +mouselink.co +moyakuhnia.ru +msfsaar.de +muznachas-service.ru +muz-shoes.ru +mydownlodablefiles.com +myiptest.com +myliveblog.ru +mymercy.info +myseoconsultant.com +na15.ru +nalogovyykodeks.ru +narutonaruto.ru +na-telefon.biz +netpics.org +newstraveller.ru +nfljerseyscheapbiz.us +nordstar.pro +northfacestore.online +notaria-desalas.com +novosti-hi-tech.ru +nsatc.net +odesproperty.com +odoratus.net okeinfo.online -okmusic.jp -onlywoman.org +oksrv.com +olvanto.ru +oneshotdate.com +online-hit.info +onlineporno.site +onload.pw +onlyforemont.ru o-o-6-o-o.com -o-o-8-o-o.ru -oren-cats.ru -origin-my.ru -paidonlinesites.com -paintingplanet.ru -panchro.xyz -paparazzistudios.com.au -partner-cdn.men -partner-stop.men -pavlodar.xkaz.org -pawli.eu -pc4download.co -pcboa.se +o-o-8-o-o.com +oops-cinema.ru +ostroike.org +ourville.info +outpersonals.com +owathemes.com +partybunny.ru +pcgroup.com.uy +pc-services.ru +pc-virus-d0l92j2.pw pechikamini.ru -pensplan4u.com -petrovka-online.com -petsblogroll.com -phelissota.xyz -phpdevops.com -picsearch.com -picsfair.com -pillscheap24h.com -pinsdaddy.com -pinwallpaper.top +picsforkeywordsuggestion.com +pio.polytopesexempt.com +piratecams.com +pix24x7.com pizdopletka.club -plusnetwork.com -pochemychka.net -poffet.net -pogruztehnik.ru -pons-presse.com -porno-dojki.net -pornofiljmi.com -pornokorol.com -portside.xyz -poshiv-chehol.ru -prefersurvey.net -privat-girl.net -privatov-zapisi.ru -proekt-mos.ru -pulseonclick.com -pushdata.sendpulse.com -pussyfleet.com -qcstrtvt.bloger.index.hr -rankings-analytics.com -realizmobi.com -realnye-otzyvy.info -refudiatethissarah.info -registrydomainservices.com -remontgruzovik.ru -rentehno.ru -resant.ru -retailwith.com -rewardit.com -rozalli.com -russia-tao.ru -rvzr-a.akamaihd.net -saecsa.co -saletool.ru -salutmontreal.com -sbf441.com +pkr1hand.com +platezhka.net +plohaya-kreditnaya-istoriya.ru +poems.com.ua +pony-business.com +popugauka.ru +popunder.net +pornophoto.xyz +porno-transsexuals.ru +pr0fit-b0x.com +printdirectforless.com +privatbank46.ru +prizefestival.mobi +profitfx.online +prointer.net.ua +proxyradar.com +purchasepillsnorx.com +pussyspace.net +pxhdwsm.com +qexyfu.bugs3.com +quelle.ru +randalljhoward.com +ranking2017.ga +rarbg.to +raschtextil.com.ua +ratemodels.net +redirectingat.com +rednise.com +registrysweeper.com +reklama1.ru +respublica-otel.ru +reversing.cc +rfserial.net +rockprogblog.com +roll123.com +rossanasaavedra.net +s1z.ru +s8-nowy-wygraj.comli.com +sajatvelemeny.com +saneitconsulting.com +sankt-peterburg.nodup.ru +sarafangel.ru scanner-elena.top -scanner-josh.top -scanner-nelson.top -scanspyware.net -scat.porn -sdsjweb.com -search-error.com -se.bnt-team.com -secret.xn--oogle-wmc.com -sel-hoz.com -semrush.com +scanner-jessica.top +scansafe.net +search.xtconnect.com +seasonvar.ru +selectads.men +seojokes.net +seo-platform.com +seorussian.ru +serving.adbetclickin.pink sexpartygirls.net sexpornotales.net -sexreliz.com -sextracker.de -shakhtar-doneck.ru +sexsearch.com +sexs-foto.top +sexxdate.net +sexy-screen-savers.com +sexystrippe.info +sfd-chess.ru +shisha-swag.de +shivafurnishings.com +shopcheermakeup.info +shopsellcardsdumps.com shopwme.ru -sibdevice.ru -simpoed.ufop.br -sisiynas.ru -site3.free-share-buttons.com -siteimprove.com -sitevaluation.org -skanninge.se -sledstvie-veli.net -s.lollypopgaming.com -smallseotools.com -smartbalanceworld.com -sneakyboy.com -socialmadesimple.com -soda.media -sonnikforme.ru -soserfis.com -sotechco.co -souvenirua.com -spabali.org -spb-plitka.ru -spinnerco.ca -sports-supplements.us -sptslmtrafms.com -spy-app.info -spywarebegone.com -ssconstruction.co -statistici.ro -stauga.altervista.org -steebook.com -stockquotes.wooeb.com -strana-solnca.ru -studiokamyk.com.pl -summerlinhomes411.info -supervesti.ru -surcentro.com -survival.betteroffers.review -svolze.com -swinger-mobil.net -tagil.zrus.org -tdsing.ru -technika-remont.ru -tehngr.ru -telefonsex-ohne0900.net -textads.men -tgtclick.com -thebestphotos.eu -thegamerznetwork.com -thin.me.pn +sim-service.net +si-unique.com +skladvaz.ru +slonechka.ru +smart-balancewheel.com +sms2x2.ru +soblaznu.net +social-button.xyz +solitaire-game.ru +spaceshipad.com +spitfiremusic.com +spywarenuker.com +starpages.net +stats-public.grammarly.io +statustroll.com +stoki.ru +streetfire.net +stroyplus.ru +studiofmp.com +sukirgenk.dvrlists.com +superiends.org +superoboi.com.ua +sureone.pro +svtrd.com +swiped.su +tabletkinaodchudzanie.com.pl +tattomedia.com +tech4master.com +telegraf.by +telesvoboda.ru +thecounter.com +thegameriders.com +thetoiletpaper.com threecolumnblogger.com -thruport.com -tizanidine4mg.blogspot.com +time-japan.ru +titangel-vietnam.com +tizanidineandcipro.blogspot.com tizanidineorflexeril.blogspot.com -tizanidinerecreationaluse.blogspot.com -tizanidinevssoma.blogspot.com -tizanidinezanaflex.blogspot.com -tkanorganizma.ru -tmearegion26.com -track112.site -traffic100.com -traffic2money.com -trion.od.ua -truebeauty.cc -tupper-shop.ru -udayavani.com -uhdtv.website -upproar.com -uptime-delta.net -urdoot.win -usswrite.com -vapsy.com -vchulkax.com -viagraneggrx.com -videojam.tv -videotuber.ru -vinylvault.co.uk -vip2ch.com -vvon.co.uk -vysigy.su -w3data.co -w7s.ru -wanker.us -warningzscaler.heraeus.com -watchdogs-2.ru -waterefficiency.co -webalan.ru -web-betting.ru -webix.biz -websiteexplorer.info -websitevaluebot.com -wedding0venues.tk -wineration.com +tnaionline.org +token-lab.org +toondinsey.com +tootoo.to +torrent-to-magnet.com +track2.shop +trackmedia101.com +trackzapper.com +traffic2cash.org +trafficmania.com +trafficreceiver.club +tramadolandtizanidine.blogspot.com +trubywriting.com +tuckpointingmasonrysystems.com +turizmus.us +turn-up-life.life +tvand.ru +tx41tclega.ru +u-cheats.ru +uchetunet.su +ufa.xrus.org +ukkelberg.no +uk-zheu20.ru +ultimatesetnewfreeallsoftupgradesystems.pw +umaseh.com +unece.org +uptimebot.net +urldelivery.com +urlopener.blogspot.com.au +us-america.ru +vabasa.inwtrade.com +valmetrundan.se +valuado.com +vapeface.club +vintontech.info +vipms.ru +visionwell.com.cn +visitcambridge.org +vkontarkte.com +vladimir.xrus.org +vod.com.ua +voloslove.ru +wallpapersist.com +waysbetter.cn +webcamdevochka.com +webradiology.ru +website-datenbank.de +weburok.com +westsextube.com +westvilletowingservices.co.za +where-toget.com +wholesalejerseychinaoutlet.com +wholesalejerseysgaa.com +wholinkstome.com wingsoffury2.com -wnoz.de -writingservices17.blogspot.ru -xn----7sbahjd3btneuw1joc.xn--p1ai -xn----7sbifcamovvfggw9d.xn--p1ai -xn--80adaggc5bdhlfamsfdij4p7b.xn--p1ai +womensplay.net +x-diesel.org +xkaz.org +xmlinde.com +xn-----6kcaabbafhu7cskl7akvongwpo7hvjj.xn--p1ai +xn----8sbarihbihxpxqgaf0g1e.xn--80adxhks xn----ctbbcjd3dbsehgi.xn--p1ai -x-true.info -y8games-free.com -yadro.ru -yellowstonesafaritours.com -yes-com.com -yougotanewdomain.com +xolodremont.ru +xtube.com +yottos.com +youdesigner.kz +youjizz.com youporn-forum.ga -youthreaders.com -youtoner.it -yubikk.info -yunque.pluto.ro -zajm-pod-zalog-nedvizhimosti.ru +zapatosenventa.info zarabotat-na-sajte.ru -zastenchivosti.net -zeroredirect6.com -zetmaster.ru +zarepta.com +zeg-distribution.com +zeroredirect10.com +zeroredirect9.com +zigarettenonlinekaufen2.bloog.pl +znaniyapolza.ru +zooggames.com diff --git a/.dev-tools/test_units/random-referrers-for-whitelist-test.list b/.dev-tools/test_units/random-referrers-for-whitelist-test.list index 6cb7e1696..41df5a27e 100644 --- a/.dev-tools/test_units/random-referrers-for-whitelist-test.list +++ b/.dev-tools/test_units/random-referrers-for-whitelist-test.list @@ -1,250 +1,250 @@ -1flag.co.za -20pascals.nl -28n2gl3wfyb0.ru -57883.net -7wind.ru -8558l.jobs.net -adloads.com -ads-seo.men -agreda.pluto.ro -agysya.ru -aktobe.xkaz.org -appfastplay.com -arpelsreplica.xyz -arquapetrarca.info -autonew.biz -avtovolop.ru -babieca.com -backlinkwatch.com -bbsoldes.fr -beachtoday.ru -beauty-things.com -best-seo-offer.com -best-way.men -bgviagrachrx.com -bigames.online -bird1.ru -biznesrealnost.ru -bookmaker-bet.com -brainzpod.win -briomotor.co -bristolhostel.com -brus-vsem.ru -bum.com.ru -capacitacionyaprendizaje.com -carapics.com -carina-sy.de -cdncash.net -c-english.ru -cheapjerseysa.com -checkhit.com -chinajerseyswholesalecoupons.com -cineacademy.ru -com-onlinesupport.site -compliance-brian.top -copypaste.traderzplanet.in -crazyboost.pro -crynet.cc -dangphoto.trade -daooda.com -darmebel.com.ua -davidovic.info -ddsoldes.fr -dichvuvesinhhanoi.com -dickssportinggoods.com -dimkino.ru -distver.ru -domoysshop.ru -doska-vsem.ru -drugstoreforyou.com -eas-seo.com +0c47f8422d3f.com +3-letter-domains.net +51.la +5ws.dating-app.ru +abctoppictures.net +absugars.com +addtoadd.men +adimmix.com +advancedmassagebysara.com +alfabot.xyz +alphavisions.net +am-se.com +arabgirls.adultgalls.com +arkkivoltti.net +artel116.ru +ausergrubhof.info +bangla-chat-uk-paralud.ga +banki76.ru +belstaffstore.online +best-games.today +besthomemadepornsites.com +bidbuy.co.kr +bif-ru.info +blog.f00kclan.de +callejondelpozo.es +cardul.ru +cartechnic.ru +cbb1smartlist12.click +ccbill.com +cherrypointplace.ca +chinacheapelitejerseys.com +chorus.terakeet.com +christianlouboutinoutlet.win +cibpenonptib.flu.cc +clickso.com +codysbbq.com +coinsspb.com +com-cleaner.systems +compliance-george.top +compliance-jack.top +compliance-mark.top +compliance-viktor.top +counciltally.com +customchocolate.business-for-home.com +custom-electric-guitar.com +datract.com +davebestdeals.com +deluxedumps.com +dirtpics.pw +djekxa.ru +doeco.ru +domain.webkeyit.com +dprtb.com +draniki.org +editors.choice6912650.hulfingtonpost.com +educatemetv.com +ekobata.ru +electricwheelchairsarea.com elektrozigaretten1.postbit.com -elidelcream.weebly.com -elmifarhangi.com -escort-russian.com -essaypro.com -eurosamodelki.ru -evidencecleanergold.com -ezigarettenshop2.postbit.com -finansov.info -fishtauto.ru -flavors.me -flirt4free.com -forsex.info -fsalas.com -ftns.ru -funnypica.com -geoads.com -ghernnqr.skyrock.com -gidonline.one -gilbertbanda.net -girls-ufa.ru -go2jump.org -gogps.me -goldadpremium.com -gpirate.com +englate.com +envaseslotusama.com +espaceinventoristes.com +essay-edu.biz +essayservicewriting.org +extener.org +factorynightclub.com +faracontrol.ir +fickenprivat.info +findacheaplawyers.com +findanysex.com +flash4fun.com +florida-tourism.net +foxtechfpv.com +franch.info +free-deals.faith +freegamesplay.online +g33.org +galaxyflowers.ru +galleryawesome.com +gamerextra.com +girlporn.ru +global-ics.co.za +gnuetella.com +golaya.pw +gomusix.com +gq-catalog.gq gsasearchengineranker.pw -hdfreeporno.net -hdwallpapers-free.com -healthcarestore.info -heartofpayne.xyz +hamilton.ca hebr.myddns-flir.com -hjaoopoa.top -humbmosquina.tk -hustoon.over-blog.com -hvd-store.com -hystersister.com -ibb.com.ua -ie.57883.net -iflycapetown.co.za -igtools.club -imagerydatabase.com -internetproviderstucson.com -intervsem.ru -ipornox.xxx -japfm.com -jennyfire.ru -journalhome.com -kanimage.com -kerch.site -kit-opt.ru +help.tpu.ru +hetmanship.xyz +heygidday.biz +hotblognetwork.com +hotsocialz.com +housediz.com +howtoclean.club +hubbble.com +huhn.altervista.org +humanorightswatch.org +ilovevitaly.com +imperiafilm.ru +impresagaia.it +infektsii.com +infogame.name +insurple.com +intermesh.net +intim-uslugi.info +jackwolfskinoutlet.online +jamiembrown.com +jimmychoosale.online +jumptap.com +kamalsinha.com +keyword-suggestions.com +kihi.gdn +kirov.zrus.org knowsitall.info -kongoultry.net -krafte.ru -kursy-ege.ru -kvartira-sutochno.com -law-check-eight.xyz -lawyers.cafe -lechtaczka.net -letslowbefast.today -longadventure.com -lost-alpha.ru -luckyday.world -lulea-auktionsverk.se -lutherstable.org -matematikus.info -mearns-tractors.co.uk -medmajor.ru -meltwater.com -memberty.com -micasainvest.com -middlerush-a.akamaihd.net -missclub.info -moneyteam24.com -morf.snn.gr -mountainstream.ms -mrvideospornogratis.xxx -musflashtv.com -musicstock.me -mustwineblog.com -narkologiya-penza.ru -nbsproject.ru -netallergy.ru -nextrent-crimea.ru -nicefloor.co.uk -njkmznnb.ru -norththeface.store -oconto.ru -officedocuments.net -ok-ua.info -online-hd.pl -onlythegames.com -openfrost.com -optibuymac.com -outclicks.net -paleohub.info -parlament.biz -partnerline.men -partybunny.ru -pillscheap24h.com -pinsdaddy.com -pio.polytopesexempt.com -piuminiita.com -plugingeorgia.com -polcin.de -popugauka.ru +koopilka.com +kovesszucs.atw.hu +kupiproday.com.ua +languagecode.com +lapitec.eu +laudit.ru +law-check-nine.xyz +linkredirect.biz +lion.conboy.us +lion.playtap.us +liverpool.gsofootball.com +live-sexchat.ru +localflirtbuddies.com +lowephotos.info +make-money-online.com +mansparskats.com +maranbrinfo.com.br +marinetraffic.com +meddesk.ru +medicinacom.ru +megavolt.net.ua +menstennisforums.com +m-google.xyz +millionare.com +mir-betting.ru +mnimmigrantrights.net +moi-glazki.ru +mojowhois.com +mojpreskumanie.com +moncleroutletonline.win +moonci.ru +mrbitsandbytes.com +mydeathspace.com +mylesosibirsk.ru +mylida.org +mymercy.info +nardulan.com +nero-us.com +newstudio.tv +nhl17coins.exblog.jp +northfacestore.online +odesproperty.com +office-windows.ru +onlainbesplatno.ru +onlinetvseries.me +onstrapon.purplesphere.in +ooo-olni.ru +osoznanie-narkotikam.net +ostrovtaxi.ru +paphoselectricianandplumber.com +pharmacyincity.com +piluli.info +porndl.org pornhub-forum.ga -prospekt-st.ru -prosperent.com -pro-tec.kz -pyramidlitho.webs.com -radiogambling.com -realnye-otzyvy.info -rebuildermedical.com -registrationdomainsite.com -rentehno.ru -retailwith.com -ringporno.com -ringtonepartner.com -rixpix.ru -ruclicks.com -rucrypt.com -sabizonline.com -sanjosestartups.com -sayyoethe.blogspot.co.za +porn-w.org +pornzone.tv +pretty-mart.com +primedice.com +qitt.ru +qld10000.net +rangoman.date +razleton.com +real-time-analytics.com +redbottomheels.xyz +remont-fridge-tv.ru +resant.ru +russkie-gorki.ru +saveindex.xyz +sbf441.com +scanner-alex.top scanner-ivan.top -scopich.com -searchaddis.com -search.alot.com -sel-hoz.com -serialsway.ucoz.ru -sex-pr.net -shoppytoolmac.com -siteimprove.com -skytraf.xyz -socialbookmarksubmission.org -sophang8.com -space4update.pw -space4updating.win -steelmaster.lv -stefanbakosab.se -strag-invest.ru -strangeduckfilms.com -sundrugstore.com -susanholtphotography.com -syvertsen-da.no -tattomedia.com -teenbbw.yopoint.in -thefotosgratis.eu -tkanorganizma.ru -toloka.hurtom.com -torrent-to-magnet.com -twiclub.in -typer.one -uchebavchehii.ru +sdelatmebel.ru +secretscook.ru +securityallianceservices.com +sexsearch.com +sexs-foto.com +sfd-chess.ru +shymkent.xkaz.org +silvercash.com +sinestesia.host.sk +siteripz.net +soaksoak.ru +soda.media +softwaretrend.net +sohoindia.net +spb.ru +specialfinanceoffers.com +spitfiremusic.com +stairliftstrue.com +td-33.ru +textads.men +thaismartloan.com +titangel-vietnam.com +tizanidinehydrochloride2mg.blogspot.com +tootoo.to +torrentred.games +totu.us +trafficmonetize.org +traxdom.ru +truebeauty.cc +tuberkuleznik.ru ucoz.ru -underthesite.com -universals.com.ua -unlimitdocs.net -urlopener.blogspot.com.au -uroki.net -vabasa.inwtrade.com -validccseller.com -vesnatehno.ru -viddyoze.com +ugogo.info +umg-stroy.ru +unitexindia.com +uogonline.com +uptime-alpha.net +userequip.com +viagra-soft.ru +viandpet.com videochat.tv.br -videos-for-your-business.com -vipms.ru -vk-mus.ru -voprosotvet24.ru -w3javascript.com -webnode.me -whipme.yopoint.in -wladimirpayen.com -www.888.com -x-diesel.org -xgames-04.com +video-hollywood.ru +videotuber.ru +vip-file.com +warezkeeper.com +wcb.su +wellcome2slovenia.ru +wleuaprpxuvr.ga +wnhjavlhezp.gq +woman-tampon.ru +xblog.in +x-diesel.info +xgftnlrt.bloger.index.hr xmronta.com -xn----7sbbahaq9bb5afgiqfliv4m.xn--p1ai -xn--80aaagvmjabrs1aoc9luc.xn--p1ai -xn--80aeahghtf8ac5i.xn--p1ai -xn--l1aengat.xn--p1ai -x-true.info -xwatt.ru -xxxguitars.com -yellowads.men -zaloadi.ru -zeroredirect11.com -zigarettenonlinekaufen1.blox.pl -zocaparj.kz -zoogdinsney.com +xn--1-8sbcpb0bdm8k6a.xn--p1ai +xn--80aikhbrhr.net +youdao.com +zarabiaj-dzis.pl +zarabotok--doma.ru +zeroredirect12.com +zigarettenonlinekaufen2.bloog.pl +zoolubimets.ru +zoominfo.com +zverokruh-shop.cz diff --git a/.dev-tools/test_units/ratelimittest-beta.txt b/.dev-tools/test_units/ratelimittest-beta.txt index b3987bb03..a683b3cc9 100644 --- a/.dev-tools/test_units/ratelimittest-beta.txt +++ b/.dev-tools/test_units/ratelimittest-beta.txt @@ -1,21 +1,12 @@ - - -
-This is merely used for testing !!!
- - +