diff --git a/.dev-tools/beta_conf_files/conf.d/globalblacklist.conf b/.dev-tools/beta_conf_files/conf.d/globalblacklist.conf index 4c325c9a7..20be832b2 100644 --- a/.dev-tools/beta_conf_files/conf.d/globalblacklist.conf +++ b/.dev-tools/beta_conf_files/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2020.03.2009 -### Updated: Tue Mar 10 12:49:19 SAST 2020 +### Version: V3.2020.03.2010 +### Updated: Tue Mar 10 13:12:15 SAST 2020 ### Bad Referrer Count: 6829 ### Bad Bot Count: 564 ################################################### diff --git a/.dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf b/.dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf index fb87e15aa..8a7a5941a 100644 --- a/.dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf +++ b/.dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -35.196.158.85 1; +104.196.57.92 1; 127.0.0.1 1; diff --git a/.dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf b/.dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf index 755513aa7..9a0a5abca 100644 --- a/.dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf +++ b/.dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -35.196.158.85 0; +104.196.57.92 0; 127.0.0.1 0; diff --git a/.dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf b/.dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf index 4c325c9a7..20be832b2 100644 --- a/.dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf +++ b/.dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2020.03.2009 -### Updated: Tue Mar 10 12:49:19 SAST 2020 +### Version: V3.2020.03.2010 +### Updated: Tue Mar 10 13:12:15 SAST 2020 ### Bad Referrer Count: 6829 ### Bad Bot Count: 564 ################################################### diff --git a/.dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf b/.dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf index fb87e15aa..8a7a5941a 100644 --- a/.dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf +++ b/.dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -35.196.158.85 1; +104.196.57.92 1; 127.0.0.1 1; diff --git a/.dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf b/.dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf index 755513aa7..9a0a5abca 100644 --- a/.dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf +++ b/.dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -35.196.158.85 0; +104.196.57.92 0; 127.0.0.1 0; diff --git a/.dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf b/.dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf index 4c325c9a7..20be832b2 100644 --- a/.dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf +++ b/.dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2020.03.2009 -### Updated: Tue Mar 10 12:49:19 SAST 2020 +### Version: V3.2020.03.2010 +### Updated: Tue Mar 10 13:12:15 SAST 2020 ### Bad Referrer Count: 6829 ### Bad Bot Count: 564 ################################################### diff --git a/.dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf b/.dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf index fb87e15aa..8a7a5941a 100644 --- a/.dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf +++ b/.dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -35.196.158.85 1; +104.196.57.92 1; 127.0.0.1 1; diff --git a/.dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf b/.dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf index 755513aa7..9a0a5abca 100644 --- a/.dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf +++ b/.dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -35.196.158.85 0; +104.196.57.92 0; 127.0.0.1 0; diff --git a/.dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf b/.dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf index 4c325c9a7..20be832b2 100644 --- a/.dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf +++ b/.dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2020.03.2009 -### Updated: Tue Mar 10 12:49:19 SAST 2020 +### Version: V3.2020.03.2010 +### Updated: Tue Mar 10 13:12:15 SAST 2020 ### Bad Referrer Count: 6829 ### Bad Bot Count: 564 ################################################### diff --git a/.dev-tools/globalblacklist-testing.template b/.dev-tools/globalblacklist-testing.template index 4c325c9a7..20be832b2 100644 --- a/.dev-tools/globalblacklist-testing.template +++ b/.dev-tools/globalblacklist-testing.template @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2020.03.2009 -### Updated: Tue Mar 10 12:49:19 SAST 2020 +### Version: V3.2020.03.2010 +### Updated: Tue Mar 10 13:12:15 SAST 2020 ### Bad Referrer Count: 6829 ### Bad Bot Count: 564 ################################################### diff --git a/.dev-tools/globalblacklist.template b/.dev-tools/globalblacklist.template index 282be3837..38a75351f 100755 --- a/.dev-tools/globalblacklist.template +++ b/.dev-tools/globalblacklist.template @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2020.03.2009 -### Updated: Tue Mar 10 12:49:18 SAST 2020 +### Version: V4.2020.03.2010 +### Updated: Tue Mar 10 13:12:14 SAST 2020 ### Bad Referrer Count: 6829 ### Bad Bot Count: 564 ################################################### diff --git a/.dev-tools/test1_conf_files/conf.d/globalblacklist.conf b/.dev-tools/test1_conf_files/conf.d/globalblacklist.conf index 282be3837..38a75351f 100644 --- a/.dev-tools/test1_conf_files/conf.d/globalblacklist.conf +++ b/.dev-tools/test1_conf_files/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2020.03.2009 -### Updated: Tue Mar 10 12:49:18 SAST 2020 +### Version: V4.2020.03.2010 +### Updated: Tue Mar 10 13:12:14 SAST 2020 ### Bad Referrer Count: 6829 ### Bad Bot Count: 564 ################################################### diff --git a/.dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf b/.dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf index 2a28fc9d0..fb87e15aa 100644 --- a/.dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf +++ b/.dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -34.74.79.111 1; +35.196.158.85 1; 127.0.0.1 1; diff --git a/.dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf b/.dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf index a8c6bf5bf..755513aa7 100644 --- a/.dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf +++ b/.dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -34.74.79.111 0; +35.196.158.85 0; 127.0.0.1 0; diff --git a/.dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf b/.dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf index 282be3837..38a75351f 100644 --- a/.dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf +++ b/.dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2020.03.2009 -### Updated: Tue Mar 10 12:49:18 SAST 2020 +### Version: V4.2020.03.2010 +### Updated: Tue Mar 10 13:12:14 SAST 2020 ### Bad Referrer Count: 6829 ### Bad Bot Count: 564 ################################################### diff --git a/.dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf b/.dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf index 2a28fc9d0..fb87e15aa 100644 --- a/.dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf +++ b/.dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -34.74.79.111 1; +35.196.158.85 1; 127.0.0.1 1; diff --git a/.dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf b/.dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf index a8c6bf5bf..755513aa7 100644 --- a/.dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf +++ b/.dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -34.74.79.111 0; +35.196.158.85 0; 127.0.0.1 0; diff --git a/.dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf b/.dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf index 282be3837..38a75351f 100644 --- a/.dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf +++ b/.dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2020.03.2009 -### Updated: Tue Mar 10 12:49:18 SAST 2020 +### Version: V4.2020.03.2010 +### Updated: Tue Mar 10 13:12:14 SAST 2020 ### Bad Referrer Count: 6829 ### Bad Bot Count: 564 ################################################### diff --git a/.dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf b/.dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf index 2a28fc9d0..fb87e15aa 100644 --- a/.dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf +++ b/.dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -34.74.79.111 1; +35.196.158.85 1; 127.0.0.1 1; diff --git a/.dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf b/.dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf index a8c6bf5bf..755513aa7 100644 --- a/.dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf +++ b/.dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -34.74.79.111 0; +35.196.158.85 0; 127.0.0.1 0; diff --git a/.dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf b/.dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf index 282be3837..38a75351f 100644 --- a/.dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf +++ b/.dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2020.03.2009 -### Updated: Tue Mar 10 12:49:18 SAST 2020 +### Version: V4.2020.03.2010 +### Updated: Tue Mar 10 13:12:14 SAST 2020 ### Bad Referrer Count: 6829 ### Bad Bot Count: 564 ################################################### diff --git a/.dev-tools/test2_conf_files/bots.d/whitelist-ips.conf b/.dev-tools/test2_conf_files/bots.d/whitelist-ips.conf index 6d9ce9433..b9e2062b5 100644 --- a/.dev-tools/test2_conf_files/bots.d/whitelist-ips.conf +++ b/.dev-tools/test2_conf_files/bots.d/whitelist-ips.conf @@ -45,5 +45,4 @@ # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST -34.74.79.111 0; 35.237.56.208 0; diff --git a/.dev-tools/test2_conf_files/conf.d/globalblacklist.conf b/.dev-tools/test2_conf_files/conf.d/globalblacklist.conf index bc46c3f42..e578cf3c7 100644 --- a/.dev-tools/test2_conf_files/conf.d/globalblacklist.conf +++ b/.dev-tools/test2_conf_files/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2020.03.2009 -### Updated: Tue Mar 10 12:49:18 SAST 2020 +### Version: V4.2020.03.2010 +### Updated: Tue Mar 10 13:12:14 SAST 2020 ### Bad Referrer Count: 6829 ### Bad Bot Count: 564 ################################################### diff --git a/.dev-tools/test3_conf_files/bots.d/whitelist-ips.conf b/.dev-tools/test3_conf_files/bots.d/whitelist-ips.conf index e38166a52..ce4d992b8 100644 --- a/.dev-tools/test3_conf_files/bots.d/whitelist-ips.conf +++ b/.dev-tools/test3_conf_files/bots.d/whitelist-ips.conf @@ -45,5 +45,5 @@ # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST -35.227.97.188 0; +104.196.57.92 0; 35.196.158.85 0; diff --git a/.dev-tools/test3_conf_files/conf.d/globalblacklist.conf b/.dev-tools/test3_conf_files/conf.d/globalblacklist.conf index 87c16912e..e63f394ae 100644 --- a/.dev-tools/test3_conf_files/conf.d/globalblacklist.conf +++ b/.dev-tools/test3_conf_files/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2020.03.2009 -### Updated: Tue Mar 10 12:49:18 SAST 2020 +### Version: V4.2020.03.2010 +### Updated: Tue Mar 10 13:12:14 SAST 2020 ### Bad Referrer Count: 6829 ### Bad Bot Count: 564 ################################################### diff --git a/.dev-tools/test4_conf_files/bots.d/whitelist-ips.conf b/.dev-tools/test4_conf_files/bots.d/whitelist-ips.conf index 51ead413c..a70ed478f 100644 --- a/.dev-tools/test4_conf_files/bots.d/whitelist-ips.conf +++ b/.dev-tools/test4_conf_files/bots.d/whitelist-ips.conf @@ -45,6 +45,5 @@ # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST -104.196.57.92 0; -34.74.79.111 0; +35.227.97.188 0; 35.237.56.208 0; diff --git a/.dev-tools/test4_conf_files/conf.d/globalblacklist.conf b/.dev-tools/test4_conf_files/conf.d/globalblacklist.conf index 282be3837..38a75351f 100644 --- a/.dev-tools/test4_conf_files/conf.d/globalblacklist.conf +++ b/.dev-tools/test4_conf_files/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2020.03.2009 -### Updated: Tue Mar 10 12:49:18 SAST 2020 +### Version: V4.2020.03.2010 +### Updated: Tue Mar 10 13:12:14 SAST 2020 ### Bad Referrer Count: 6829 ### Bad Bot Count: 564 ################################################### diff --git a/.dev-tools/test5_conf_files/bots.d/whitelist-ips.conf b/.dev-tools/test5_conf_files/bots.d/whitelist-ips.conf index 0579cf4ff..48856a84f 100644 --- a/.dev-tools/test5_conf_files/bots.d/whitelist-ips.conf +++ b/.dev-tools/test5_conf_files/bots.d/whitelist-ips.conf @@ -45,6 +45,6 @@ # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST -35.196.72.151 0; +34.74.79.111 0; 35.237.56.208 0; -35.227.97.188 0; +35.196.72.151 0; diff --git a/.dev-tools/test5_conf_files/conf.d/globalblacklist.conf b/.dev-tools/test5_conf_files/conf.d/globalblacklist.conf index 282be3837..38a75351f 100644 --- a/.dev-tools/test5_conf_files/conf.d/globalblacklist.conf +++ b/.dev-tools/test5_conf_files/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2020.03.2009 -### Updated: Tue Mar 10 12:49:18 SAST 2020 +### Version: V4.2020.03.2010 +### Updated: Tue Mar 10 13:12:14 SAST 2020 ### Bad Referrer Count: 6829 ### Bad Bot Count: 564 ################################################### diff --git a/.dev-tools/test_units/blacklist-ips.conf b/.dev-tools/test_units/blacklist-ips.conf index fb87e15aa..8a7a5941a 100644 --- a/.dev-tools/test_units/blacklist-ips.conf +++ b/.dev-tools/test_units/blacklist-ips.conf @@ -1,2 +1,2 @@ -35.196.158.85 1; +104.196.57.92 1; 127.0.0.1 1; diff --git a/.dev-tools/test_units/random-bots-for-test-quick.list b/.dev-tools/test_units/random-bots-for-test-quick.list index f2137650c..1ea1b28e1 100644 --- a/.dev-tools/test_units/random-bots-for-test-quick.list +++ b/.dev-tools/test_units/random-bots-for-test-quick.list @@ -1,10 +1,10 @@ -BlowFish -EasyDL -Express WebPictures -EyeNetIE -Humanlinks -InternetSeer -Pump -Virusdie -Woobot -Zitebot +AlphaBot +BatchFTP +CheTeam +GalaxyBot +Openvas +ProWebWalker +RankurBot +serpstatbot +Steeler +WWW::Mechanize diff --git a/.dev-tools/test_units/random-bots-for-test.list b/.dev-tools/test_units/random-bots-for-test.list index f935d962c..6d31567d9 100644 --- a/.dev-tools/test_units/random-bots-for-test.list +++ b/.dev-tools/test_units/random-bots-for-test.list @@ -1,58 +1,53 @@ 360Spider -Abonti -Acunetix -ADmantX +404checker +404enemy AfD-Verbotsverfahren AhrefsBot -Aipbot -Apexoo -arquivo.pt -ASPSeek -Attach -backlink-check -BacklinkCrawler +Alligator +AllSubmitter +Ankit +AwarioRssBot +BackDoorBot BackStreet -BackWeb +Badass BatchFTP +Battleztar Bazinga BBBike -BDFetch BetaBot Bigfoot -Bitacle Blackboard +Black Hole BlackWidow BLEXBot Blow -Boardreader -Bolt -Brandwatch Buddy -BuiltBotTough CazoodleBot -Cegbfeieh -CheeseBot -CherryPicker +CCBot CheTeam +ChinaClaw Chlooe +Claritybot +Cliqzbot Cogentbot -Collector +cognitiveseo +com.plumanalytics Copyscape Cosmos -crawler4j +Craftbot crawler.feedback crawl.sogou.com -Crescent -CrunchBot +CrazyWebCrawler CSHttp -Custo +Curious +DatabaseDriverMysqli DataCha0s -DBLBot demandbase-bot Deusu Devil +Digincore +DigitalPebble +DIIbot Dirbuster -Disco -Discobot Discoverybot Dispatch DittoSpyder @@ -61,190 +56,195 @@ DomainCrawler DomainStatsBot Dotbot Download Wonder -Dragonfly -Drip -DSearch -Ebingbong +EasyDL ECCP/1.0 Ecxi EMail Siphon EMail Wolf +EroCrawler Evil ExtractorPro -FDM -Fimap +Extreme Picture Finder +EyeNetIE +Ezooms +FHscan +Firefox/7.0 +FlashGet Flunky -Foobot -Freeuploader -Fyrebot -Genieo -Getintent +GalaxyBot +GetWeb Gigablast Gigabot -Go-Ahead-Got-It -GoZilla +Gotit Go!Zilla +Grabber Grafula +GrapeshotCrawler +GridBot GT::WWW HaosouSpider -Harvest -Hloader +heritrix +HTMLparser HTTrack Humanlinks +HybridBot Iblog Indy Library -InfoTekies +Internet Ninja +InternetSeer +ips-agent Iria -Iskanie +Jbrofuzz +JetCar +JikeSpider Joomla -JustView +Jorgee Jyxobot +Keyword Density Lanshanbot -LeechGet +LexiBot Libwhisker +Lightspeedsystems Likse -LinkextractorPro -LinkpadBot LinkScan +LinksManager LinkWalker -LinqiaMetadataDownloaderBot -Lmspider -Mag-Net -magpie-crawler +LinqiaScrapeBot +Lipperhey +Lipperhey Spider +Ltx71 +lwp-request +Mail.RU_Bot Majestic SEO Majestic-SEO -Mata Hari +MarkMonitor +Mass Downloader +meanpathbot Meanpathbot -MeanPath Bot +Mediatoolkitbot Metauri MFC_Tear_Sample -MIIxpc -Mister PiX +MIDown tool +MJ12bot Mojeek Mojolicious -Morfeus Fucking Scanner Mr.4x3 -MSIECrawler -Msrabot -muhstik-scan Musobot Name Intelligence Nameprotect -NetAnts +netEstate NE Crawler +NetMechanic +NetSpider Nettrack +Net Vampire +Netvibes +NetZIP Niki-bot +Nikto +NimbleCrawler Nimbostratus -Ninja -oBot -Octopus +Nmap +NPbot Offline Explorer -Offline Navigator Openfind -OrangeBot OutclicksBot +OutfoxBot +PageAnalyzer Page Analyzer -page scorer -Pandalytics -Panscient -PHPCrawl +PageGrabber +pcBrowser +Picscout Pi-Monster PleaseCrawl -Pockey +plumanalytics POE-Component-Client-HTTP polaris version -Probethenet -PxBroker +Psbot +QueryN Metasearch Quick-Crawler -RankActive RankActiveLinkBot +RankFlex RankingBot -RankingBot2 -RebelMouse +Reaper +Recorder RepoMonkey -Ripper -RocketCrawler -ScanAlert -ScoutJet +RSSingBot +Scanbot +Scrapy +Screaming ScreenerBot -Searchestate -SearchmetricsBot Semrush +SemrushBot SEOkicks -SEOkicks-Robot -Seomoz -SEOprofiler -serpstatbot +SEOlyticsCrawler +SeoSiteCheckup Siphon SISTRIX Sitebeam -SiteExplorer Siteimprove -SiteSnagger SiteSucker -Sitevigil -SMTBot -Snake +Site Sucker +SlySearch SocialRankIOBot sogouspider Sogou web spider +Sosospider +Sottopop +SpaceBison Spammen Spanner -Spbot -Sqlmap -Stripper -Sucuri +sp_auditbot +SputnikBot +spyfu +Sqworm +SuperBot Suzuran Swiftbot sysscan +T8Abot tAkeOut TeleportPro Telesoft -Telesphoreo Telesphorep The Intraformant +TheNomad +TightTwatBot +Titan +Toata +Toweyabot Tracemyfile Trendictionbot -trendiction.de +True_Robot +Turnitin +TurnitinBot TwengaBot -Twice -Typhoeus -URLy.Warning -Vacuum -VCI -VeriCiteCrawler -Virusdie -Voltron +UnisterBot +Vagabondo +Wallpapers/3.0 WallpapersHD WBSearchBot +Webalta WebAuto Web Auto +WebBandit WebCollage -WebFuck -WebImageCollector -WebLeacher -webmeup-crawler -Web Pix +Web Collage +Web Fetch +WebmasterWorldForumBot +WebPix WebReaper WebSauger -Web Sauger -WebsiteQuester -Website Quester -Web Sucker -WebWhacker +Webster +WebSucker WebZIP -WeSEE +Whatweb +Who.is Bot Widow +Wonderbot +Woobot +Wprecon WPScan -WWW-Collector-E -WWW-Mechanize -WWW::Mechanize x09Mozilla -x22Mozilla -Xaldon_WebSpider -YoudaoBot -Zade -Zauba -Zermelo -zgrab -Zitebot ZmEu +ZumBot +ZyBorg diff --git a/.dev-tools/test_units/random-bots-for-whitelist-test.list b/.dev-tools/test_units/random-bots-for-whitelist-test.list index 5580b7923..320eb02ae 100644 --- a/.dev-tools/test_units/random-bots-for-whitelist-test.list +++ b/.dev-tools/test_units/random-bots-for-whitelist-test.list @@ -1,250 +1,250 @@ 360Spider 404checker 80legs +Abonti Aboundex -ADmantX -AhrefsBot -AIBOT -AiHitBot -Aipbot +Aboundexbot +AfD-Verbotsverfahren +Alexibot +AllSubmitter +AlphaBot +Apexoo archive.org_bot arquivo-web-crawler ASPSeek +Asterias autoemailspider -AwarioRssBot +AwarioSmartBot +BackDoorBot Backlink-Ceck -backlink-check BackWeb -BBBike -BDFetch -BetaBot +BDCbot Bigfoot -Blackboard Black Hole BlackWidow -BLEXBot -Boardreader +BlowFish Bolt +BotALot +Brandprotect +Brandwatch Buddy BuiltBotTough -BuiltWith -CazoodleBot +Bullseye +Calculon +CATExplorador +CCBot Cegbfeieh -CherryPicker CheTeam ChinaClaw Chlooe Claritybot Cliqzbot -coccocbot-web +Cloud mapping +Cogentbot +cognitiveseo Copier -CopyRightCheck -crawler.feedback +Copyscape crawl.sogou.com -Crescent -CrunchBot -DatabaseDriverMysqli +CSHttp +Curious +Custo DBLBot demandbase-bot -Demon Deusu DigitalPebble -DIIbot -Discobot DnyzBot -DomainCrawler +DomainAppender DomainSigmaCrawler DomainStatsBot -Dotbot -Download Wonder Dragonfly +Drip eCatch +ECCP/1.0 EMail Siphon EMail Wolf EroCrawler -evc-batch -ExtLinksBot -Extractor ExtractorPro -Extreme Picture Finder -EyeNetIE Ezooms facebookscraper -FDM -Fimap -Flunky +FemtosearchBot +Firefox/7.0 +Foobot Freeuploader -FrontPage +GalaxyBot Genieo GermCrawler +GetWeb Gigabot Go-Ahead-Got-It -Gotit -GoZilla -Go!Zilla -Grabber Grafula GrapeshotCrawler +GridBot +GT::WWW Haansoft -HaosouSpider Havij -HEADMasterSEO Heritrix Hloader -HTMLparser -Iblog -IDBot +HTTP::Lite +HTTrack +HybridBot Id-search -IlseBot +Image Fetch +Image Sucker IndeedBot +InfoTekies +instabid Intelliseek -InternetSeer -ips-agent -Iria -IstellaBot -JennyBot +Internet Ninja +IRLbot +JamesBOT JetCar Joomla Jorgee -Kenjin Spider -Kozmosbot +Larbin LeechGet Lftp +LibWeb Libwhisker -Lightspeedsystems -Likse -LinkextractorPro +Linkdexbot LinkWalker LinqiaMetadataDownloaderBot -LinqiaScrapeBot -Lipperhey +LinqiaRSSBot +Lipperhey Spider Lmspider LNSpiderguy lwp-trivial +Mail.RU_Bot Majestic12 -MarkMonitor -MauiBot +Majestic-SEO +Mata Hari meanpathbot -Meanpathbot -MegaIndex.ru -Metauri -Microsoft Data Access -MIIxpc +MeanPath Bot +Mediatoolkitbot +MFC_Tear_Sample +MIDown tool +Mister PiX +Mojeek Mojolicious Mr.4x3 -MSFrontPage -muhstik-scan -Navroad -NearSite +Musobot +Name Intelligence NetAnts netEstate NE Crawler NetMechanic -Nettrack +NetSpider +Netvibes NetZIP NICErsPRO -Nikto -Nutch -oBot -Octopus +Niki-bot +Ninja +NPbot +Offline Explorer +Offline Navigator OnCrawl Openfind +OpenLinkProfiler +OrangeBot OrangeSpider -OutfoxBot +OutclicksBot +PageAnalyzer +Page Analyzer page scorer PageScorer +Pandalytics Panscient -Papa Foto +PECL::HTTP PHPCrawl -Picsearch PictureFinder -Pi-Monster PleaseCrawl -plumanalytics Pockey +Probethenet +ProPowerBot +ProWebWalker Psbot -Pump -PxBroker -PyCurl +QueryN Metasearch RankActive -RankingBot -Rankivabot +RankActiveLinkBot +RankingBot2 RankurBot -RealDownload -Recorder -RedesScrapy +Reaper +RebelMouse RepoMonkey Ripper -Rogerbot SalesIntelligent -SBIder ScanAlert -ScreenerBot -Semrush -SEOlyticsCrawler +Scrapy +Searchestate +SemrushBot +Seomoz +SEOstats serpstatbot -Siphon -Siteimprove +sexsearcher +Shodan +SISTRIX +Sitebeam +SiteExplorer SiteSnagger +SiteSucker SmartDownload -SMTBot -Snapbot -SocialRankIOBot +Snoopy +Sosospider +Sottopop +SpaceBison +Spammen SpankBot -Spanner -sp_auditbot -Spbot -SputnikBot spyfu +Sqlmap Sqworm -Steeler -Stripper +Sucker +Sucuri +SuperHTTP +Surfbot +Suzuran +Swiftbot +sysscan Szukacz -Teleport +T0PHackTeam +tAkeOut +TeleportPro Telesphorep The Intraformant -TheNomad +Thumbor +TightTwatBot Titan -Toata Toweyabot -Tracemyfile +True_Robot Turingos -Twice +TurnitinBot +TwengaBot Typhoeus -Upflow -URLy Warning +UnisterBot URLy.Warning -Vagabondo -VB Project -VCI +VidibleScraper Virusdie -Voil -Voltron -WallpapersHD -Web Collage +Wallpapers/3.0 +WASALive-Bot +Web Auto WEBDAV -WebEnhancer -WebFuck +Web Enhancer +WebFetch WebGo IS WebImageCollector WebLeacher -webmeup-crawler -WebPix -Web Pix +WebmasterWorldForumBot +WebReaper Webshag -WebsiteExtractor -Website Quester +WebsiteQuester +Webster +WebStripper Web Sucker -WebZIP +WeSEE Whack +Widow WiseGuys Robot -Wotbox -Wprecon -WWW-Mechanize -WWW::Mechanize +Woobot +WWWOFFLE x09Mozilla -x22Mozilla Xaldon WebSpider Zauba -Zitebot ZmEu -ZumBot +ZyBorg diff --git a/.dev-tools/test_units/random-referrers-for-test-quick.list b/.dev-tools/test_units/random-referrers-for-test-quick.list index a29950d69..d04295480 100644 --- a/.dev-tools/test_units/random-referrers-for-test-quick.list +++ b/.dev-tools/test_units/random-referrers-for-test-quick.list @@ -1,10 +1,10 @@ -betwinservice.com -fsalas.com -gentamicineyedrops.blogspot.com -guruofcasino.com -hoztorg-opt.ru -kerei.ru -moyakuhnia.ru -paphoselectricianandplumber.com +100searchengines.com +alert-jdh.xyz +apple.com-webbrowsing-security.review +fickenprivat.info +gopro-online.info +jennyfire.ru +searchkut.com +sovinsteel.ru +tizanidinehcl2mg.blogspot.com vvpg.ru -xn----7sbbagbq7bd5aheftfllo4m.xn--p1ai diff --git a/.dev-tools/test_units/random-referrers-for-test.list b/.dev-tools/test_units/random-referrers-for-test.list index 42aa1346a..09de281af 100644 --- a/.dev-tools/test_units/random-referrers-for-test.list +++ b/.dev-tools/test_units/random-referrers-for-test.list @@ -1,500 +1,500 @@ -0daymusic.org -1flag.co.za -1qingdao.com -45en.ru -51.la +00author.com +00webcams.com +03e.info +0n-line.tv +1000n1.ru +100searchengines.com +101flag.ru +24videos.tv +24x7-server-support.site +2728fb936f0.com +273-fz.ru +2kata.ru +3th.co.in +3xforum.ro +4pp13.com +4ureyesonly.com +5000-cotydzien.com 5tey7463.weebly.com -5ws.dating-app.ru -6128786.com -7wind.ru -8xv8.com +9med.net a14download.com -aa81bf391151884adfa3dd677e41f94be1.com -aab94f698f36684c5a852a2ef272e031bb.com -aad1f4acb0a373420d9b0c4202d38d94fa.com -academia-nsk.org -adserver-e7.com -adsfresh.men -adsloads.com -adtech.fr -adultnet.in -advertisingtag.net +aa9d046aab36af4ff182f097f840430d51.com +abiente.ru +absolutelyanalog.com +academiacsmendoza.org +activecampaign.dreamhosters.com +adidas.frwebs.fr +admanaerofoil.com +adult3dgames.com +advancedmassagebysara.com +advancedsoftwaresupport.com aero2.ru -agecheckadult.com -air-edem.ru -akama.com -aksonural.ru -alfa-img.com -alice.it -alkoravto.ru -allblogroll.com -amt-k.ru -animebox.com.ua -anniemation.com -apifasterlightin-a.akamaihd.net -apiluckyleapnet-a.akamaihd.net -apimegabrowsebiz-a.akamaihd.net -apptester.tk -apxeo.info -arclk.net -asia-forum.ru -asrvrep-a.akamaihd.net +afslankpillen2017nl.eu +alldownload.pw +allmarketsnewdayli.gdn +alveris.ru +amateurgalls.com +am-se.com +apiallgeniusinfo-a.akamaihd.net +apispringsmartne-a.akamaihd.net +app5.letmacworkfaster.world +appsecurityr.com +ap.senai.br +arendakvartir.kz +arendatora.ru +artavenuegardenstudios.com +asiavirtualsolutions.com atelielembrancaqueencanta.com.br -atlasvkusov.ru -audiobangout.com -autochoixspinelli.com -autotrends.today -avek.ru -avitocars.ru -avon-ukraine.com -azartmsl.com -azazaporn.com -azbaseimages.net +ausergrubhof.info +aussie-prizes.com +aviav.co +aviva-limoux.com +azbukafree.com +babieca.com baditri.com -baixar-musicas-gratis.com +bad-stars.net balla-boo.se -baoxaydung.com.vn -beachtoday.ru -bear.gotcher.us -beget.tech -bestpriceninja.com -best-seo-software.xyz -bigames.online -big-cash.net -biketank.ga -bizcheapjerseyswholesalechina.com -biznesschat.net -bjorkbacken.nu -blog.yam.com -blue-square.biz -bobinoz.com -bolitgorlo.net -bonkers.name -briomotor.co -bscodecs.com -b-styles.xyz -business-made-fun.com -buydissertation.net -buyessaynow.biz -buyhoverboard.com -buypillsorderonline.com -buzzsumo.com -bycontext.com -bytimedance.ru -candcstuccoandstone.com -candlewooddentalcentre.com.au +bankofthewext.com +bannerspace.com +best-deals-products.com +bestempresas.es +besthomemadepornsites.com +beta.hotkeys.com +bidr.trellian.com +bkns.vn +blogig.org +boeuklad.com +bouda.kvalitne.cz +brakehawk.com +bravegirlsclub.com +brothers-smaller.ru +btnativenav.com +burger-tycoon.com +bus-offer.com +buy-loft.ru +buzzonclick.com +callejondelpozo.es +canoncdriverq3.pen.io canrioloadm.gq -canyougethighofftizanidine.blogspot.com -canzoni.ru -carabela.com.do -cardsdumps.com carloans.com -cartierbraceletsreplica.pw -caveavins.fr -cbcseward.com -cheapcigarettesc.info -children-learningreading.info -chinaelitecheapjerseys.com -ciceros.co +carrauterie.be +cashkitten-a.akamaihd.net +cheapwebsitehoster.com +cheatcode-lita12.rhcloud.com +checkm8.com +checktext.org +check-this-out-now.online +chelnytruck.ru +cicaki.net +cindymatches.com circlesl.com -citetick.com -ci.ua -civilwartheater.com -clksite.com -clubfashionista.com -codysbbq.com -compliance-alexa.xyz -compliance-barak.top -compliance-don.xyz -compliance-fred.top -compliance-julia.top +clarithromycin500mg.com +clean-virus-mac.com +clickcash.com +clickprotects.com +club-lukojl.ru +coaching-netz.info +compliance-jessica.top compliance-margo.top -contentlook.co -cookielawblog.wordpress.com -coolwallpapers-hd.com -countercrazy.com -covadhosting.biz -creatives.adbetclickin.pink -culmimg.pw -dame-ns.kz -danceuniverse.ru -daneshetabiat.com -daooda.com -datsun-do.com -dawlenie.com -ddospanel.com -decorazilla.com -deshevo-nedorogo.ru -detalizaciya-tut.biz -devochki.top -dieswaene.com -dikx.gdn -dlya-android.org -dms-sw.ru -docsportal.net -documentsite.net -druzhbany.ru -dvr.biz.ua -dwomlink.info -dyshagi.ru -earl-brown.info -earncash.com.ua -ec-file.info -e-commerce-seo1.com -ecxtracking.com -eduserver.net -efnor-ac.com -ekobata.ru -elektrozigarette1.dreamwidth.org -elektrozigaretten1.tumblr.com -emailaccountlogin.co -emergencyneeds.org -enews.tech -enhand.se -eorogo.top -excitacao.com -exportshop.us -extrabot.com -eyessurgery.ru -ezigarettenkaufen1.pagina.gr -face.hostingx.eu -factorynightclub.com -falllow.gq -fashion-mk.net -feargames.ru -feorina.ru -ferretsoft.com -ferrotodo.com -fettefrauen.net -findsexguide.com -finejewelryshop.ru -firstdrugmall.ru -flashlarevista.com -flex4launch.ru -foodcrafts.website -fordsonmajbor.cf -forexgb.ru -forsex.info -fortevidyoze.net -foto-telok.net -foxydeal.com -free-floating-buttons.com -free-share-buttons.top -frvo.alptandem.ru -fullgirl.ru -fxgallery.com +compucelunlock.net +construmac.com.mx +contextualyield.com +cool-wedding.net +cpabegins.ru +crystalslot.com +culad.com +cupday.com +customcollegeessays.net +custom-electric-guitar.com +danschawbel.com +data.vtc.pw +db.speedup.gdn +dekoration.us +dekorkeramik.ru +den-noch24.ru +dentalpearls.com.au +dfwu1019.info +dgfitness.co +dildofotzen.net +dktr.ru +dnsrsearch.com +domaincrawler.com +domaineaneblanc.com +domainsphoto.com +dostavka-v-krym.com +driving.kiev.ua +drugstoreforyou.com +dustyorate.com +dvrlists.com +easyukraine.com +e-avon.ru +educatemetv.com +egovaleo.it +elektrischezigaretten2.beeplog.com +elektronischezi.livejournal.com +elektrozigarette2.wordpressy.pl +elitedollars.com +elitepcgames.com +elitesportsadvisor.com +en.altezza.travel +erotische-geschichten-xxl.com +esopini.com +etm-consult.de +etotupo.ru +ezigaretteshop2.mywapblog.com +f5mtrack.com +fakehandbags.xyz +farm26.ru +fastcrawl.com +feel-planet.com +fickluder69.com +filesclub.net +filmi-v.online +finansov.info +finstroy.net +firesub.pl +fitness-video.net +flash4fun.com +flprog.com +fodelsedagspresenter.nu +forum20.smailik.org +forum.doctissimo.fr +forum-engineering.ru +freecamdollars.com +freelotto.com +frequiry.com +fuck-paid-share-buttons.xyz +funnymama.com +funnypica.com +funtoonez.com +fz139.ttk.ru +g33.org g7m.pl -gay.adultgalls.com -gaytube.com -gaz-voshod.ru -gearcraft.us +gamezblox.com +gaygalls.net geilehausfrauen.net -generalporn.org -getlaid-xxxhookupdirect.com -ggiaro.com -girlgamerdaily.com -glastecfilms.com.my +gemara.com +generic-pills-online.com +girlspicsa.com +given2.com +gktt.ru +glavtral.ru glicol.kz -golmau.host.sk -gstatey.net -guildebzh.info -gwebtools.com.br -hackers-crackers.tk -hdfreeporno.net -hdhc.ru -heartofbeijing.blogspot.com -herokuapp.com -hexpilot.com -himazin.info +gobongo.info +gomusix.com +googlefeud.com +gosarhivrt.ru +gotwebsite1.com +gox.com.ua +gpms.org.my +gq-catalog.gq +gratuitbaise.com +gratuitxblcodes.com +greendream.com.ua +growshop.es +gwagka.com +gwebtools.com +hacktougroup.ru +hamptonoaks.ca +hdimagelib.com himgaws.pw -histats.com -hiwibyh.bugs3.com -hjaoopoa.top -hunterboots.online -ifirestarter.ru -impresagaia.it -infobabki.ru -innodgfdriverhm.aircus.com -inspiring-desperate.tk -instabid.tech -interesnie-faktu.ru -iomoio.net -j33x.com -janerikholst.se -javatex.co.id -javcoast.com -javlibrary.cc -javrip.net -javspace.net -jerseysbizwholesalecheap.com -jillepille.com -johannesburgsingles.co.za -joingames.org -jolic2.com -jose.mulinohouse.co -justprofit.xyz -kalandranis.gr -kasino-money.pw -kazka.ru -kazrent.com +homegardenlova.com +homeinns.com +home.myplaycity.com +houston-vikings.com +ido3.com +igadgetsworld.com +ilo134uloh.com +imagecoolpub.com +images-graphics-pics.com +imgata.com +impisr.edunsk.ru +inboundlinks.win +inet-traffic.com +infazavr.ru +interfucks.net +iscblog.info +istizanidineanarcoticdrug.blogspot.com +ivearchenceinflu.cf +jackpotchances.com +jerseyssportsshop.com +jobmarket.com.ua +jrpmakati.com +karachev-city.ru +karusel-market.ru +kashubadesign.ru keki.info -keywordsking.com -keywordsuggests.com -kino2018.club -kinofak.net -kinozapas.org -kit-opt.ru -klejonka.info -kletkimehan.ru -kol-energo.ru -kollekcioner.ru -kongruan.com -konkursov.net -koptims.tiu.ru -kosova.de -kosynka-games.ru +kevblog.top +kicknights.gq +kiinomaniak.pl +kinoplen.ru +kinozapas.com +kleine-titten.biz +knogg.net +kollesa.ru +kongoultry.net +konoplisemena.com +koral.se +koronirealestate.gr +krasivoe-hd.net ktotut.net -kustanay.kz -lacapilla.info -lampokrat.ws -landoftracking.com -lasvegaslockandsafe.com -lflash.ru -lightinghomes.net -lion.misba.us -liumimgx.pw +ku6.com +kupit-adenu.ru +ladov.ru +landmania.ru +landreferat.ru +lankarns.com +lawyers.cafe +ldrtrack.com +leboard.ru +lexaprogeneric.link +lida-ru.com +linkbolic.com +lion.bolegapakistan.com +liran-locks.com liveinternet.ru -lkbennettoutlet.online +loftdigital.eu lottospring.com -louboutinbooties.xyz -magda-gadalka.ru -mahnwachen-helfen.info -managerpak204.weebly.com -mansparskats.com -manuscript.su -maridan.com.ua -marketingtechniques.info -masserect.com -master-muznachas.ru -mastershef.club -medanestesia.ru -mensandals.xyz -metallosajding.ru -mezaruk.info -millionare.com -minneapoliscopiers.com -mir-limuzinov.ru -misandesign.se -mlvc4zzw.space -monetizationking.net -monitorwebsitespeed.com -montredemarque.nl -mosrif.ru +ltvperf.com +luciddiagnostics.in +mackeeper-center.club +macnewtech.com +macresource.co.uk +mafa-free.com +magazin-pics.ru +marcowebonyodziezowe.pl +maslenka.kz +meduza-consult.ru +megaindex.ru +middlerush-a.akamaihd.net +mixed-wrestling.ru +modenamebel.ru +monclerboots.xyz +moneymaster.ru +morf.snn.gr +moscow-clining.ru moviezbonkerssk.cf -mundoaberrante.com -muznachas-service.ru -mvpicton.co.uk -my-big-family.com -mybinaryoptionsrobot.com +mrcsa.com.au +musflashtv.com +musicas.baixar-musicas-gratis.com +my-aladin.com +myblogregistercm.tk mydownlodablefiles.com -myfreemp3.eu -myftpupload.com -narutonaruto.ru -naturalbreakthroughsresearch.com -newhairstylesformen2014.com -nfljerseysforsalewholesaler.com -nikhilbahl.com -ningessaybe.me +mygameplus.com +mykings.pw +nalogovyy-kodeks.ru +namenectar.com +naturalshair.site +new-post.tk +newtechspb.ru +nikitsyringedrivelg.pen.io normalegal.ru -nosecret.com.ua -novatech.vn +novosti-hi-tech.ru +nrjmobile.fr nullrefer.com -nvssf.com -odesproperty.com -ohmyrings.com -oklogistic.ru -ok-ua.info -onescreen.cc -onlinedomains.ru -o-o-11-o-o.com +oakleyglassesonline.us +oballergiya.ru +ochistka-stokov.ru +offergroup.info +offf.info +offtopic.biz +omsk.xrus.org +o-o-6-o-o.ru +ooomeru.ru ooo-olni.ru -owohho.com -oxotl.com -paidonlinesites.com -partnersafe.men -pc4download.co -pcboa.se -penisvergrotendepillennl.ovh -photorepair.ru -picsforkeywordsuggestion.com -pics-group.com -pictures-and-images.net -picurams.pw -pillscheap24h.com -pinstake.com +open-odyssey.org +oralsexfilme.net +ororodnik.goodbb.ru +partner-cdn.men +pdns.cz +perfectpracticeweb.com +perosan.com +petrovka-online.com +phormchina.com +phuketscreen.com +piccshare.com +pintattoos.com +pistonclasico.com +pix24x7.com +pixelrz.com pizdeishn.com -pizdopletka.club -pizza-imperia.com -pladform.ru -playmsn.com +pl.aasoldes.fr pliks.pl -pobeiranie.pl +plohaya-kreditnaya-istoriya.ru +pl-vouchers.com pochemychka.net -poffet.net -popugauka.ru -pornosemki.info -portnoff.od.ua -powc.r.ca.d.sendibm2.com -pr0fit-b0x.com -printie.com -privatefx.all4invest.info -privatov-zapisi.ru -prizestohandle.club -produkto.net -prostitutki-astana.org -prostitutki-kiev.org +porno-dojki.net +porn-w.org +pospr.waw.pl +powenlite24.ru +preparevideosafesystem4unow.space +presleycollectibles.com +privatamateure.com +profolan.pl +proftests.net +pro-okis.ru prostitutki-novgoroda.org +prostitutki-rostova.org pssucai.info +pst2017.onlinewebshop.net +pts163.ru +quality-traffic.com +quangcaons.com quizzitch.net -raavidesigns.com -ragecash.com -rainbowice.ru -rarbg.to -razorweb-a.akamaihd.net -realizmobi.com -recipedays.com -redirectingat.com -renecaovilla.online -rep-am.com -responsive-test.net -retreatia.com +radiodigital.co +razyboard.com +realmonte.net +reeyanaturopathy.com +refudiatethissarah.info +reklamuss.ru +remont-mobile-phones.ru +resellerclub.com +responsinator.com +restaurantlescampi.com +restorator-msk.ru rff-cfal.info -rfid-locker.co -ritlweb.com -rocketchange.ru -roleforum.ru -russia-today-video.ru -s1z.ru -santechnik.jimdo.com +richinvestmonitor.com +rockprogblog.com +royal-investments.net +runtnc.net +rusenvironmental.net +russian-postindex.ru +rvtv.ru +s8-nowy-wygraj.comli.com +sack.net +samara.rosfirm.ru sarf3omlat.com sbdl.no -sbf441.com -scalerite.co.za -scanner-john.top -scanner-nelson.top +scanmyphones.com +scanner-barak.top +scanner-brian.top school-diplomat.ru -schoolfiles.net -searchtooknow-a.akamaihd.net -securesmrt-dt.com -seemoreresultshu-a.akamaihd.net -seminarygeorgia59.ga -seoholding.com +searchinquire.com +seasaltwithfood.com +seccioncontrabajo.com +seksotur.ru +seksvideoonlain.com +semrush.com +seogadget.ru serdcenebolit.com -servethis.com -serviporno.com -sex-foto.pw -sexvideo-sex.com -sex-watch.com -sexyteens.hol.es -sharebutton.org -shoesonlinebuy.cn -sideeffectsoftizanidine.blogspot.com -silvercash.com -sinel.info -site.ru -sklad-24.ru -smart-scripts.com -smokewithrabbits.com -snowplanes.com -socialbutton.xyz -sockshare.net +serialsway.ucoz.ru +sex-dating.co +shopvilleroyboch.com.ua +shymkent.xkaz.org +signal03.ru +signoredom.com +silktide.com +simple-share-buttons.com +sims-sims.ru +site-auditor.online +sky-mine.ru +skypasss.com +slutloadlive.com +smartpet.ru +social-fun.ru +socialtrade.biz +soft1.ru softlinesolutions.me -sogimlecal.tk -sophang8.com -space4updating.win +soft-terminal.ru +solicita.info +solmarket.by +songplanet.ru +sonnikforme.ru +soochi.co +spammen.de +spbchampionat.ru +spidtest.org spymac.net -start.myplaycity.com -staticfs.host -statoutlook.info -store-rx.com -students-cheapskate.ml -survival.betteroffers.review -swsociety.se -symphonyintegratedhealthcare.com -tamada69.com +startufa.ru +startwp.org +steelmaster.lv +stillmiracle.com +stpicks.com +stroilka.info +superinterstitial.com +superstats.com +surfbuyermac.com +surflinksmedical.com +svbur.ru +svetlotorg.ru +swingerseiten.com +swiped.su +tackletarts.co +takeprofitsystem.com tam-gde-more.ru +tanieaukcje.com.pl tarad.com -t-bygg.com -teslathemes.com -thebestweightlosspills.ovh -thegameriders.com -thepantonpractice.co.uk -time-japan.ru -titslove.yopoint.in -tizanidinehcl4mgtabinfo.blogspot.com -tizanidinehydrochloride2mg.blogspot.com -tizanidineshowupondrugtest.blogspot.com -tizanidinesleepaid.blogspot.com -tomatis.gospartner.com -toy-shop.top +teenbbw.yopoint.in +terrafootwear.us +theautoprofit.ml +thebestphotos.eu +thefotosgratis.eu +thegamerznetwork.com +thejournal.ru +thewebsitetemplate.info +tiandeural.ru +tizanidineandhydrocodone.blogspot.com +tizanidinedosage.blogspot.com +tizanidineforopiatewithdrawal.blogspot.com +tizanidineorflexeril.blogspot.com +toondinsey.com +topkarkas.com +torrnada.ru +torture.ml +toxicwap.com +tracxn.com traderzplanet.in -traffic100.com -trafficmania.com -trahic.ru -trahvid.com -trkdf.com -turist-strani.ru -turkeyreport.tk -turn-up-life.life -umg-stroy.ru -umityangin.net -ural-buldozer.ru -urccvfmc.bloger.index.hr -uytmaster.ru -vashsvet.com -v-doc.co -venerologiya.com -vezuviy.su -viagra-soft.ru -videooko.weebly.com -vietimgy.pw -vipms.ru -vipsexfinders.com -vitalads.net -viveresaniesnelli.it +traffic2money.com +truck-rece.life +trucri.me +trustedhealthtips.com +tuckpointingmasonrysystems.com +tupper-posuda.ru +twojebook.pl +ua-company.ru +uamtrk.com +uchetunet.su +undergroundcityphoto.com +underthesite.com +urengoy.pro +validdomain.xyz +valoresito.com +vancleefreplica.pw +vapeface.club +vedomstvo.net +video-chat.in +viktoria-center.ru voloomoney.com -vzglyadriv.kg -wallpapers-best.com -waterefficiency.co -wdfdocando.com -we-are-gamers.com -webadvance.club -webextract.profound.net +voluumtrk.com +vzube.com +wahicbefa31.soup.io +wallpaperstock.net +watracker.net +webalan.ru webix.me -weburok.com -weightatraining.com -welck.octopis.com -westsextube.com -westum.se -womensterritory.ru -word-vorlagen.net -www.arenda-yeisk.ru -xclicks.net -xjlottery.com -xn--80abgj3a5acid6ghs.top -xn----ctbbcjd3dbsehgi.xn--p1ai -xn--q1a.xn--b1aube0e.xn--c1acygb.xn--p1ai -xwatt.ru +weblo.com +witherrom55.eklablog.fr +withstandingheartwarming.com +woman-tampon.ru +x5market.ru +xcc24.pl +xmlinde.com +xn-----7kceclhb4abre1b4a0ccl2fxch1a.xn--p1ai +xn--80ajjbdhgmudixfjc8c5a9df8b.xn--p1ai +xn----8sbarihbihxpxqgaf0g1e.xn--80adxhks +xn--h1aakne2ba.xn--p1ai +xn--hxazdsfy.blogspot.com +xnxxandxvideos.com +xportvusbdriver8i.snack.ws +xxxtube69.com xxxtubesafari.com -yebocasino.co.za -yogamatsexpert.com -youandcredit.ru -youthreaders.com -yuarra.pluto.ro -yugk.net +yourothersite.com +yubikk.info zaapplesales.blogspot.com -zastenchivosti.net -z-master.ru -zoogdinsney.com -zscaler.net -zvetki.ru +zapiszto.pl +zero1.it +zeroredirect12.com +znaturaloriginal.com +zooggames.com +zverokruh-shop.cz +zx6.ru diff --git a/.dev-tools/test_units/random-referrers-for-whitelist-test.list b/.dev-tools/test_units/random-referrers-for-whitelist-test.list index caa021ae0..703e97e90 100644 --- a/.dev-tools/test_units/random-referrers-for-whitelist-test.list +++ b/.dev-tools/test_units/random-referrers-for-whitelist-test.list @@ -1,250 +1,250 @@ -76brighton.co.uk -abctoppictures.net -activecampaign.dreamhosters.com -adobereader-free.ru -adsloads.com -adult3dgames.com -adultmeetonline.info -adzerg.com -aliexpresscashback.club -alldownload.pw -allforminecraft.ru -all-number.com -alvtank.se -amoi.tn -ample-awards-today.us -aosheng-tech.com -apccargo.com +00webcams.com +100dollars-seo.com +12bet.com +202ch.com +2rich4bitches.com +acunetix-referrer.com +admitad.com +adpremium.org +advancedsoftwaresupport.com +afora.ru +aktobe.xkaz.org +amehdaily.com +am-se.com +animalia-life.club +animali.deagostinipassion.it +animalrank.com +apibrowsesmartne-a.akamaihd.net applyneedy.xyz -arenanews.com.ua -arendas.net -aruplighting.com -autogrep.ru -aviav.eu -bagcionderlab.com -bankcrediti.ru -bannerbank.ru -begalka.xyz -betterhdporn.com -bizcheapjerseyswholesalechina.com -bizlist.com.de -blockworld.ru -blogos.kz -bongiornos.info -bonzibuddi.com -brokergid.ru -buypuppies.ca -buyviagraa.blogspot.com -calstaterealty.us -camel-beach.com +arabsexxxtube.com +arendovalka.xyz +arkartex.ru +asia-forum.ru +bagsonsale.online +balkanfarma.org +bannerspace.com +beauby.ru +beauty-clinic.ru +best-mam.ru +bestsexyblog.com +bidbuy.co.kr +bingoporno.com +biographiya.com +bio.trade-jp.net +blogs.rediff.com +bolitgorlo.net +bonky.biz +brandbucket.com +businesn.men +cactussoft.biz +cam-kontakte.org +candcstuccoandstone.com +canvas.gloverid.site +centraletermice.us centre-indigo.org.ua -cezartabac.ro -children-learningreading.info +chocolateslim-original.com circlesl.com -citetick.com -clickintext.net -clothesforcash.com -club-samodelkin.ru -compliance-alex.xyz -compliance-walter.top -converse.ddsoldes.fr -counter.bloke.com -countercrazy.com -crackguru.tk -cutalltheshit.com -ddlmega.net -dfiles.me -dikx.gdn -distver.ru -domain2008.com -domain.webkeyit.com -dprtb.com -duplicashapp.com +clodo.ru +cntravelre.com +colehaanoutlet.store +companies-catalog.com +compliance-barak.xyz +confib.ifmo.ru +contentlook.co +creams.makeforum.eu +cymbaltaandtizanidine.blogspot.com +dairyindia.in +dayibiao.com +digilander.libero.it +dipstar.org +docs4all.com +doktoronline.no +dostavka-v-krym.com +dosugrostov.site +drugstoreforyou.com earthmagic.info -easycommerce.cf -easyukraine.com -e-commerce-seo.com -ednorxmed.com -edwinkonijn.com.au +eas-seo.com +e-biznes.info +ec-file.info +edelstahlschornstein-123.de +elexies.info +elkacentr.ru +en.altezza.travel enews.tech -eng-lyrics.com -enthuse.computernetworksonline.com -evidencecleanergold.com -exhibitionplus.eu -ezigarettenshop1.yolasite.com -ezigarettenshop2.myblog.de -family1st.ca -femdom.twiclub.in -filmetricsasia.com -finemanteam.com -footbalive.org -forexgb.ru -free-deals.faith -fsakhalin.ru -galaxyflowers.ru -gamesprite.me -gaming-journal.com -gaytube.com -gearcraft.us -getdot.ru -gilsonchiro.xyz +envaseslotusama.com +este-line.com.ua +europages.com.ru +evening-dating-club.info +exonline.info +favoritemoney.ru +filmgo.ru +flex4launch.ru +flprog.com +fototravel.eu +freejabs.com +freemaintenancesysforpcandmac.top +freeseedsonline.com +free-stock-illustration.com +friendflnder.com +frustrated-favorable.gq +g7m.pl +gameplexcity.com +gateway.zscalerone.net +gembird.com +get-your-social-buttons.info +golaya.pw golden-praga.ru -googlemare.com -googlsucks.com -gopixdatabase.com -green-tea.tv -gribkovye-zabolevaniya.com -grizzlysgrill.com -guardlink.com -healthcarestore.info -hem.passagen.se -hexpilot.com -high-speed1.net -hkdiiohi.skyrock.com -hkladys.com -hmywwogw.bloger.index.hr -hostnow.men -hotloans.ru -huhn.altervista.org -infektsii.com -investyb.com -istizanidinelikexanax.blogspot.com -janettabridal.com -jerseywholesalechinabiz.com -kazka.ru -kinocccp.net -kino-filmi.com -klejonka.info -kongruan.com -koronirealestate.gr -krasivoe-hd.com -krasnodar-avtolombards.ru -kredit-blog.ru -kung-fu-ru.com -lerporn.info -lexixxx.com -lifebyleese.com -light.ifmo.ru -lineshops.biz -liquimondo.com -lmrauction.com -loadopia.com -lotto6888.com -lutherstable.org -macresource.co.uk -matpre.top +golfresa.lucania.se +golmau.host.sk +goodhousekeeping.com +greatfind-a.akamaihd.net +gta-top.ru +guardlink.org +harmonyglen.us +himgaws.pw +hostsshop.ru +hustoon.over-blog.com +impisr.edunsk.ru +io9.com +ivanstroi.ru +javlibrary.cc +javstock.com +jerseywholesalebizchina.com +jerseywholesaleelitestore.com +keywords-monitoring-success.com +kino-rating.ru +kinosed.net +kiskinhouse.com +konpax.com +kretpicf.pw +kursy-ege.ru +lampokrat.ws +ledis.top +lineavabit.it +live-sexchat.ru +luxmagazine.cf +mackeeper-land-672695126.us-east-1.elb.amazonaws.com +mahnwachen-helfen.info +mainlinehobby.net +malwareremovalcenter.com +manifestation.betteroffers.review +marcogrup.com +marketingtechniques.info +marmotstore.online m.b00kmarks.com -mbiologi.ru -medtherapy.ru -med-zdorovie.com.ua -merryhouse.co.uk -moi-glazki.ru -mojpreskumanie.com -montredemarque.nl -most-kerch.org -moyakuhnia.ru -mrbojikobi4.biz -msfsaar.de -muztops.ru -mydoctorok.ru -myshopmatemac.com -namenectar.com -nbsproject.ru -nicovideo.jp -nightvision746.weebly.com -nuker.com -offers.bycontext.com -onlythegames.com -o-o-6-o-o.com -openmultipleurl.com -orion-code-access.net -oxford-book.com.ua -paleohub.info -pandroid.co -pavlodar.xkaz.org -paydayonlinecom.com -pc-services.ru -penisvergrotendepillennl.ovh -photochki.com -picturesfrom.com -popander.mobi -popup-jdh.xyz -pornoreino.com -powc.r.ca.d.sendibm2.com -pozdrawleniya.com -pravoholding.ru -princeadvantagesales.com -proekt-mos.ru -proxyradar.com -pts163.ru -py100.ru -qexyfu.bugs3.com -qor360.com -ranksonic.info -rapevideosmovies.com -raschtextil.com.ua -remontvsamare.su -rossanasaavedra.net -ruscoininvest.company -rvzr-a.akamaihd.net -scanner-walter.top -sdrescher.net -seksotur.ru -seksvideoonlain.com -sell-fb-group-here.com -semp.net -seoheap.com -seonetwizard.com -sexphoto.site -sexuria.net -sideeffectsoftizanidine.blogspot.com -simpoed.ufop.br -snow.nvr163.com -star61.de -stevemonsen.com -talant-factory.ru -tamada69.com -taraz.xkaz.org -thebitcoincode.com -thegamerznetwork.com -thejournal.ru -time-japan.ru -tizanidine4mgstreetprice.blogspot.com -tizanidinehydrochloride2mg.blogspot.com -topshef.ru -torrentdownloadhub.com -torrent-newgames.com -tourismvictoria.com -try-rx.com +mebel-alait.ru +mezaruk.info +mirtorrent.net +mobplayer.net +moesen-ficken.com +money-for-placing-articles.com +monsterdivx.com +moscow-clining.ru +mrwhite.biz +multgo.ru +musicspire.online +netfacet.net +new-apps.ru +nibbler.silktide.com +onblastblog.online +onefilms.net +onemactrckr.com +osnova3.ru +pacificair.com +paphoselectricianandplumber.com +paramountmarble.co.uk +par-fallen.ga +partner-cdn.men +pigrafix.at +pila.pl +piuminiita.com +plugingeorgia.com +podshipniki-ntn.ru +pomoc-drogowa.cba.pl +porn555.com +pornhub-ru.com +pornmania.pl +pornstartits.xblog.in +porn-w.org +pozdravleniya-c.ru +privacyassistant.net +proftests.net +pulseonclick.com +quickchange.cc +rainbowice.ru +rebuildermedical.com +recordpage-a.akamaihd.net +resultshub-a.akamaihd.net +retailwith.com +ric.info +robertefuller.com +roofers.org.uk +rosbalt.com.ua +royal-betting.net +ru-dety.ru +rusoft-zone.ru +rybalka-opt.ru +saletool.ru +sc-specialhost.com +searchengineranker.email +seoboxes.com +seofirmreviewsus.info +seolab.top +sexcamamateurchat.com +sexkontakte-seite.com +sexobzor.info +sexreliz.com +sex-tracker.com +shemale-sex.net +sibtest.ru +sibvitr.ru +sledstvie-veli.net +smstraf.ru +spabali.org +spb.afora.ru +sportbetfair.com +stauga.altervista.org +stockspmb.info +storehouse.ua +surfbuyermac.com +surgut.zrus.org +svnuppsalaorebro.se +swimpool.ca +szucs.ru +tampabaywatch.org +tech4master.com +tgtclick.com +thetoiletpaper.com +tizanidinestreetprice.blogspot.com +top-karkas.ru +totu.us +twittrading.com u17795.netangels.ru -ukkelberg.no -unionmarkt.de -uptimechecker.com -uvozdeckych.info -uzungil.com -valoresito.com -vedomstvo.net -vksex.ru -vzlomsn.org -webjam.com -website-speed-check.site -websitevaluebot.com -welck.octopis.com -weprik.ru -wheelchairliftsarea.com -wholesalecheapjerseysfree.com -wholesalejerseychinaoutlet.com -wowas31.ucoz.ru -xn--80aaafbn2bc2ahdfrfkln6l.xn--p1ai -xn--80ahvj9e.xn--p1ai -xwatt.ru -xxx-cam.webcam -yaoguangdj.com -yoopsie.com -youdao.com -youtubedownload.org -ytmnd.com -zazagames.org -zebramart.ru -zeroredirect9.com +uchebavchehii.ru +unisexjewelry.org +unpredictable.ga +uogonline.com +vancleefreplica.pw +vektorpress.ru +versaut.xxx-cam.webcam +viagengrarx.com +viagra.pp.ua +videotuber.ru +voronezh.xrus.org +vriel.batcave.net +vut.com.ru +w7s.ru +watchmygf.net +weburok.com +woodyguthrie.se +worst-sites.online +wowcasinoonline.ooo +wygraj-teraz.com +xaijo.com +x-lime.com +xsion.net +zojirushi-products.ru +zoogdisany.com +zoominfo.com zophim.me +zvuker.net diff --git a/.dev-tools/test_units/ratelimittest-master.txt b/.dev-tools/test_units/ratelimittest-master.txt index ed30c44f7..a683b3cc9 100644 --- a/.dev-tools/test_units/ratelimittest-master.txt +++ b/.dev-tools/test_units/ratelimittest-master.txt @@ -5,6 +5,14 @@