diff --git a/.dev-tools/beta_conf_files/conf.d/globalblacklist.conf b/.dev-tools/beta_conf_files/conf.d/globalblacklist.conf index 20be832b2..31007368b 100644 --- a/.dev-tools/beta_conf_files/conf.d/globalblacklist.conf +++ b/.dev-tools/beta_conf_files/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2020.03.2010 -### Updated: Tue Mar 10 13:12:15 SAST 2020 +### Version: V3.2020.03.2011 +### Updated: Tue Mar 10 13:35:24 SAST 2020 ### Bad Referrer Count: 6829 ### Bad Bot Count: 564 ################################################### diff --git a/.dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf b/.dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf index 8a7a5941a..241e97626 100644 --- a/.dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf +++ b/.dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -104.196.57.92 1; +34.74.253.255 1; 127.0.0.1 1; diff --git a/.dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf b/.dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf index 9a0a5abca..85130057a 100644 --- a/.dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf +++ b/.dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -104.196.57.92 0; +34.74.253.255 0; 127.0.0.1 0; diff --git a/.dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf b/.dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf index 20be832b2..31007368b 100644 --- a/.dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf +++ b/.dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2020.03.2010 -### Updated: Tue Mar 10 13:12:15 SAST 2020 +### Version: V3.2020.03.2011 +### Updated: Tue Mar 10 13:35:24 SAST 2020 ### Bad Referrer Count: 6829 ### Bad Bot Count: 564 ################################################### diff --git a/.dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf b/.dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf index 8a7a5941a..241e97626 100644 --- a/.dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf +++ b/.dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -104.196.57.92 1; +34.74.253.255 1; 127.0.0.1 1; diff --git a/.dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf b/.dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf index 9a0a5abca..85130057a 100644 --- a/.dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf +++ b/.dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -104.196.57.92 0; +34.74.253.255 0; 127.0.0.1 0; diff --git a/.dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf b/.dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf index 20be832b2..31007368b 100644 --- a/.dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf +++ b/.dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2020.03.2010 -### Updated: Tue Mar 10 13:12:15 SAST 2020 +### Version: V3.2020.03.2011 +### Updated: Tue Mar 10 13:35:24 SAST 2020 ### Bad Referrer Count: 6829 ### Bad Bot Count: 564 ################################################### diff --git a/.dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf b/.dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf index 8a7a5941a..241e97626 100644 --- a/.dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf +++ b/.dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -104.196.57.92 1; +34.74.253.255 1; 127.0.0.1 1; diff --git a/.dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf b/.dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf index 9a0a5abca..85130057a 100644 --- a/.dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf +++ b/.dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -104.196.57.92 0; +34.74.253.255 0; 127.0.0.1 0; diff --git a/.dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf b/.dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf index 20be832b2..31007368b 100644 --- a/.dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf +++ b/.dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2020.03.2010 -### Updated: Tue Mar 10 13:12:15 SAST 2020 +### Version: V3.2020.03.2011 +### Updated: Tue Mar 10 13:35:24 SAST 2020 ### Bad Referrer Count: 6829 ### Bad Bot Count: 564 ################################################### diff --git a/.dev-tools/globalblacklist-testing.template b/.dev-tools/globalblacklist-testing.template index 20be832b2..31007368b 100644 --- a/.dev-tools/globalblacklist-testing.template +++ b/.dev-tools/globalblacklist-testing.template @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2020.03.2010 -### Updated: Tue Mar 10 13:12:15 SAST 2020 +### Version: V3.2020.03.2011 +### Updated: Tue Mar 10 13:35:24 SAST 2020 ### Bad Referrer Count: 6829 ### Bad Bot Count: 564 ################################################### diff --git a/.dev-tools/globalblacklist.template b/.dev-tools/globalblacklist.template index 38a75351f..1fbd2ee1e 100755 --- a/.dev-tools/globalblacklist.template +++ b/.dev-tools/globalblacklist.template @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2020.03.2010 -### Updated: Tue Mar 10 13:12:14 SAST 2020 +### Version: V4.2020.03.2011 +### Updated: Tue Mar 10 13:35:23 SAST 2020 ### Bad Referrer Count: 6829 ### Bad Bot Count: 564 ################################################### diff --git a/.dev-tools/test1_conf_files/bots.d/whitelist-ips.conf b/.dev-tools/test1_conf_files/bots.d/whitelist-ips.conf index 719bd636b..e55d207d1 100644 --- a/.dev-tools/test1_conf_files/bots.d/whitelist-ips.conf +++ b/.dev-tools/test1_conf_files/bots.d/whitelist-ips.conf @@ -45,4 +45,4 @@ # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST -35.196.72.151 0; +35.237.8.208 0; diff --git a/.dev-tools/test1_conf_files/conf.d/globalblacklist.conf b/.dev-tools/test1_conf_files/conf.d/globalblacklist.conf index 38a75351f..1fbd2ee1e 100644 --- a/.dev-tools/test1_conf_files/conf.d/globalblacklist.conf +++ b/.dev-tools/test1_conf_files/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2020.03.2010 -### Updated: Tue Mar 10 13:12:14 SAST 2020 +### Version: V4.2020.03.2011 +### Updated: Tue Mar 10 13:35:23 SAST 2020 ### Bad Referrer Count: 6829 ### Bad Bot Count: 564 ################################################### diff --git a/.dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf b/.dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf index fb87e15aa..b4e441f5b 100644 --- a/.dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf +++ b/.dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -35.196.158.85 1; +35.237.212.185 1; 127.0.0.1 1; diff --git a/.dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf b/.dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf index 755513aa7..f707957e4 100644 --- a/.dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf +++ b/.dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -35.196.158.85 0; +35.237.212.185 0; 127.0.0.1 0; diff --git a/.dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf b/.dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf index 38a75351f..1fbd2ee1e 100644 --- a/.dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf +++ b/.dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2020.03.2010 -### Updated: Tue Mar 10 13:12:14 SAST 2020 +### Version: V4.2020.03.2011 +### Updated: Tue Mar 10 13:35:23 SAST 2020 ### Bad Referrer Count: 6829 ### Bad Bot Count: 564 ################################################### diff --git a/.dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf b/.dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf index fb87e15aa..b4e441f5b 100644 --- a/.dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf +++ b/.dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -35.196.158.85 1; +35.237.212.185 1; 127.0.0.1 1; diff --git a/.dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf b/.dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf index 755513aa7..f707957e4 100644 --- a/.dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf +++ b/.dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -35.196.158.85 0; +35.237.212.185 0; 127.0.0.1 0; diff --git a/.dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf b/.dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf index 38a75351f..1fbd2ee1e 100644 --- a/.dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf +++ b/.dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2020.03.2010 -### Updated: Tue Mar 10 13:12:14 SAST 2020 +### Version: V4.2020.03.2011 +### Updated: Tue Mar 10 13:35:23 SAST 2020 ### Bad Referrer Count: 6829 ### Bad Bot Count: 564 ################################################### diff --git a/.dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf b/.dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf index fb87e15aa..b4e441f5b 100644 --- a/.dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf +++ b/.dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -35.196.158.85 1; +35.237.212.185 1; 127.0.0.1 1; diff --git a/.dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf b/.dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf index 755513aa7..f707957e4 100644 --- a/.dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf +++ b/.dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -35.196.158.85 0; +35.237.212.185 0; 127.0.0.1 0; diff --git a/.dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf b/.dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf index 38a75351f..1fbd2ee1e 100644 --- a/.dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf +++ b/.dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2020.03.2010 -### Updated: Tue Mar 10 13:12:14 SAST 2020 +### Version: V4.2020.03.2011 +### Updated: Tue Mar 10 13:35:23 SAST 2020 ### Bad Referrer Count: 6829 ### Bad Bot Count: 564 ################################################### diff --git a/.dev-tools/test2_conf_files/bots.d/whitelist-ips.conf b/.dev-tools/test2_conf_files/bots.d/whitelist-ips.conf index b9e2062b5..c323b2cee 100644 --- a/.dev-tools/test2_conf_files/bots.d/whitelist-ips.conf +++ b/.dev-tools/test2_conf_files/bots.d/whitelist-ips.conf @@ -45,4 +45,5 @@ # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST -35.237.56.208 0; +34.74.253.255 0; +35.237.8.208 0; diff --git a/.dev-tools/test2_conf_files/conf.d/globalblacklist.conf b/.dev-tools/test2_conf_files/conf.d/globalblacklist.conf index e578cf3c7..7672266cc 100644 --- a/.dev-tools/test2_conf_files/conf.d/globalblacklist.conf +++ b/.dev-tools/test2_conf_files/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2020.03.2010 -### Updated: Tue Mar 10 13:12:14 SAST 2020 +### Version: V4.2020.03.2011 +### Updated: Tue Mar 10 13:35:23 SAST 2020 ### Bad Referrer Count: 6829 ### Bad Bot Count: 564 ################################################### diff --git a/.dev-tools/test3_conf_files/bots.d/whitelist-ips.conf b/.dev-tools/test3_conf_files/bots.d/whitelist-ips.conf index ce4d992b8..90131b140 100644 --- a/.dev-tools/test3_conf_files/bots.d/whitelist-ips.conf +++ b/.dev-tools/test3_conf_files/bots.d/whitelist-ips.conf @@ -45,5 +45,5 @@ # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST -104.196.57.92 0; -35.196.158.85 0; +35.237.212.185 0; +35.185.97.135 0; diff --git a/.dev-tools/test3_conf_files/conf.d/globalblacklist.conf b/.dev-tools/test3_conf_files/conf.d/globalblacklist.conf index e63f394ae..626d89137 100644 --- a/.dev-tools/test3_conf_files/conf.d/globalblacklist.conf +++ b/.dev-tools/test3_conf_files/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2020.03.2010 -### Updated: Tue Mar 10 13:12:14 SAST 2020 +### Version: V4.2020.03.2011 +### Updated: Tue Mar 10 13:35:23 SAST 2020 ### Bad Referrer Count: 6829 ### Bad Bot Count: 564 ################################################### diff --git a/.dev-tools/test4_conf_files/bots.d/whitelist-ips.conf b/.dev-tools/test4_conf_files/bots.d/whitelist-ips.conf index a70ed478f..f3833dbc6 100644 --- a/.dev-tools/test4_conf_files/bots.d/whitelist-ips.conf +++ b/.dev-tools/test4_conf_files/bots.d/whitelist-ips.conf @@ -45,5 +45,6 @@ # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST -35.227.97.188 0; -35.237.56.208 0; +104.196.213.122 0; +34.74.253.255 0; +35.185.97.135 0; diff --git a/.dev-tools/test4_conf_files/conf.d/globalblacklist.conf b/.dev-tools/test4_conf_files/conf.d/globalblacklist.conf index 38a75351f..1fbd2ee1e 100644 --- a/.dev-tools/test4_conf_files/conf.d/globalblacklist.conf +++ b/.dev-tools/test4_conf_files/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2020.03.2010 -### Updated: Tue Mar 10 13:12:14 SAST 2020 +### Version: V4.2020.03.2011 +### Updated: Tue Mar 10 13:35:23 SAST 2020 ### Bad Referrer Count: 6829 ### Bad Bot Count: 564 ################################################### diff --git a/.dev-tools/test5_conf_files/bots.d/whitelist-ips.conf b/.dev-tools/test5_conf_files/bots.d/whitelist-ips.conf index 48856a84f..0e435278f 100644 --- a/.dev-tools/test5_conf_files/bots.d/whitelist-ips.conf +++ b/.dev-tools/test5_conf_files/bots.d/whitelist-ips.conf @@ -45,6 +45,6 @@ # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST -34.74.79.111 0; -35.237.56.208 0; -35.196.72.151 0; +35.237.8.208 0; +104.196.213.122 0; +35.237.212.185 0; diff --git a/.dev-tools/test5_conf_files/conf.d/globalblacklist.conf b/.dev-tools/test5_conf_files/conf.d/globalblacklist.conf index 38a75351f..1fbd2ee1e 100644 --- a/.dev-tools/test5_conf_files/conf.d/globalblacklist.conf +++ b/.dev-tools/test5_conf_files/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2020.03.2010 -### Updated: Tue Mar 10 13:12:14 SAST 2020 +### Version: V4.2020.03.2011 +### Updated: Tue Mar 10 13:35:23 SAST 2020 ### Bad Referrer Count: 6829 ### Bad Bot Count: 564 ################################################### diff --git a/.dev-tools/test_units/blacklist-ips.conf b/.dev-tools/test_units/blacklist-ips.conf index 8a7a5941a..241e97626 100644 --- a/.dev-tools/test_units/blacklist-ips.conf +++ b/.dev-tools/test_units/blacklist-ips.conf @@ -1,2 +1,2 @@ -104.196.57.92 1; +34.74.253.255 1; 127.0.0.1 1; diff --git a/.dev-tools/test_units/random-bots-for-test-quick.list b/.dev-tools/test_units/random-bots-for-test-quick.list index 1ea1b28e1..ef1af6883 100644 --- a/.dev-tools/test_units/random-bots-for-test-quick.list +++ b/.dev-tools/test_units/random-bots-for-test-quick.list @@ -1,10 +1,10 @@ -AlphaBot -BatchFTP -CheTeam -GalaxyBot -Openvas -ProWebWalker -RankurBot -serpstatbot -Steeler -WWW::Mechanize +DBLBot +InternetSeer +Jyxobot +Mediatoolkitbot +Msrabot +Swiftbot +Trendiction +Webalta +Xaldon_WebSpider +Zauba diff --git a/.dev-tools/test_units/random-bots-for-test.list b/.dev-tools/test_units/random-bots-for-test.list index 6d31567d9..cabec7c03 100644 --- a/.dev-tools/test_units/random-bots-for-test.list +++ b/.dev-tools/test_units/random-bots-for-test.list @@ -1,42 +1,45 @@ 360Spider -404checker -404enemy -AfD-Verbotsverfahren -AhrefsBot +Aboundex +Aboundexbot +ADmantX +AiHitBot +Aipbot Alligator AllSubmitter -Ankit +AlphaBot +Anarchie +Apexoo +Attach +autoemailspider AwarioRssBot -BackDoorBot +AwarioSmartBot +Backlink-Ceck BackStreet -Badass BatchFTP -Battleztar Bazinga -BBBike -BetaBot +BDFetch Bigfoot -Blackboard +Bitacle Black Hole -BlackWidow BLEXBot -Blow -Buddy +BlowFish +Bolt +BotALot +Brandprotect +Brandwatch +BunnySlippers +BuzzSumo CazoodleBot -CCBot -CheTeam -ChinaClaw +Cegbfeieh +CheeseBot +CherryPicker Chlooe -Claritybot Cliqzbot -Cogentbot +Cloud mapping +coccocbot-web cognitiveseo -com.plumanalytics -Copyscape -Cosmos -Craftbot +CopyRightCheck +crawler4j crawler.feedback -crawl.sogou.com -CrazyWebCrawler CSHttp Curious DatabaseDriverMysqli @@ -44,207 +47,204 @@ DataCha0s demandbase-bot Deusu Devil -Digincore DigitalPebble -DIIbot -Dirbuster +Disco Discoverybot -Dispatch -DittoSpyder DnyzBot +DomainAppender DomainCrawler -DomainStatsBot -Dotbot -Download Wonder -EasyDL +Dragonfly +DSearch +Ebingbong ECCP/1.0 -Ecxi +EirGrabber EMail Siphon -EMail Wolf -EroCrawler +evc-batch Evil -ExtractorPro +ExtLinksBot +Extractor Extreme Picture Finder -EyeNetIE Ezooms +FDM +FemtosearchBot FHscan Firefox/7.0 FlashGet -Flunky -GalaxyBot -GetWeb -Gigablast -Gigabot +FyberSpider +GermCrawler +G-i-g-a-b-o-t Gotit Go!Zilla Grabber +GrabNet Grafula -GrapeshotCrawler -GridBot GT::WWW -HaosouSpider +Haansoft +Havij +HEADMasterSEO heritrix -HTMLparser -HTTrack -Humanlinks -HybridBot -Iblog +Hloader +IndeedBot Indy Library +InfoTekies Internet Ninja InternetSeer -ips-agent -Iria -Jbrofuzz -JetCar +internetVista monitor +Iskanie +IstellaBot +JamesBOT +Jetty JikeSpider -Joomla -Jorgee -Jyxobot -Keyword Density -Lanshanbot +JOC Web Spider +LeechFTP LexiBot -Libwhisker +Lftp +LibWeb Lightspeedsystems -Likse +LinkpadBot LinkScan -LinksManager LinkWalker -LinqiaScrapeBot -Lipperhey Lipperhey Spider -Ltx71 +Lmspider lwp-request -Mail.RU_Bot +lwp-trivial +Majestic12 Majestic SEO -Majestic-SEO -MarkMonitor Mass Downloader +Mata Hari meanpathbot -Meanpathbot -Mediatoolkitbot -Metauri +MeanPath Bot +mediawords +MegaIndex.ru MFC_Tear_Sample +Microsoft URL Control MIDown tool MJ12bot -Mojeek Mojolicious +Morfeus Fucking Scanner Mr.4x3 +Msrabot +muhstik-scan Musobot Name Intelligence -Nameprotect +NearSite +NetAnts netEstate NE Crawler -NetMechanic -NetSpider Nettrack -Net Vampire Netvibes -NetZIP -Niki-bot +Nibbler +NICErsPRO Nikto -NimbleCrawler -Nimbostratus -Nmap -NPbot +Ninja +Nutch Offline Explorer +OnCrawl Openfind +OpenVAS +OrangeBot OutclicksBot OutfoxBot PageAnalyzer Page Analyzer PageGrabber +Pandalytics +Pavuk pcBrowser -Picscout +PECL::HTTP +PeoplePal +PHPCrawl +Picsearch +PictureFinder Pi-Monster -PleaseCrawl -plumanalytics -POE-Component-Client-HTTP polaris version -Psbot -QueryN Metasearch +ProWebWalker +Pump +PxBroker Quick-Crawler -RankActiveLinkBot RankFlex -RankingBot -Reaper +Rankivabot Recorder +RedesScrapy RepoMonkey -RSSingBot -Scanbot -Scrapy -Screaming +Ripper +RocketCrawler +Rogerbot ScreenerBot -Semrush +SearchmetricsBot SemrushBot -SEOkicks SEOlyticsCrawler -SeoSiteCheckup -Siphon +SEOprofiler +seoscanners +Shodan SISTRIX -Sitebeam +SiteExplorer Siteimprove -SiteSucker -Site Sucker -SlySearch -SocialRankIOBot -sogouspider -Sogou web spider -Sosospider -Sottopop +SiteSnagger +SmartDownload +Snake +Snoopy +Sociscraper SpaceBison Spammen -Spanner sp_auditbot +Spbot +Spinn3r SputnikBot -spyfu -Sqworm -SuperBot +Sqlmap +Sqlworm +Sucker +SurveyBot Suzuran Swiftbot -sysscan +Szukacz T8Abot tAkeOut -TeleportPro Telesoft Telesphorep -The Intraformant TheNomad +Thumbor TightTwatBot Titan Toata Toweyabot -Tracemyfile Trendictionbot -True_Robot +trendiction.de +Turingos Turnitin -TurnitinBot TwengaBot +Twice +Typhoeus UnisterBot +URLy Warning +URLy.Warning Vagabondo -Wallpapers/3.0 +VB Project +VidibleScraper +Voltron WallpapersHD -WBSearchBot -Webalta -WebAuto Web Auto -WebBandit WebCollage -Web Collage -Web Fetch +WEBDAV +WebEnhancer +WebFetch +WebFuck +WebGo IS WebmasterWorldForumBot -WebPix +Web Pix WebReaper -WebSauger -Webster -WebSucker +Website Quester +Web Sucker +WebWhacker WebZIP -Whatweb -Who.is Bot -Widow -Wonderbot +Whacker +WinHTTrack +WISENutbot Woobot Wprecon -WPScan -x09Mozilla -ZmEu +WWW::Mechanize +x22Mozilla +Zade +Zauba +Zermelo +Zitebot ZumBot -ZyBorg diff --git a/.dev-tools/test_units/random-bots-for-whitelist-test.list b/.dev-tools/test_units/random-bots-for-whitelist-test.list index 320eb02ae..7ba4e6e0e 100644 --- a/.dev-tools/test_units/random-bots-for-whitelist-test.list +++ b/.dev-tools/test_units/random-bots-for-whitelist-test.list @@ -1,250 +1,250 @@ -360Spider 404checker -80legs -Abonti -Aboundex -Aboundexbot -AfD-Verbotsverfahren -Alexibot -AllSubmitter -AlphaBot +ADmantX +AhrefsBot +AiHitBot +Ankit Apexoo -archive.org_bot +arquivo.pt arquivo-web-crawler -ASPSeek Asterias autoemailspider -AwarioSmartBot BackDoorBot Backlink-Ceck -BackWeb +BacklinkCrawler +BackStreet BDCbot -Bigfoot +BDFetch +BetaBot +Blackboard Black Hole BlackWidow +BLEXBot +Blow BlowFish -Bolt -BotALot -Brandprotect -Brandwatch Buddy -BuiltBotTough -Bullseye -Calculon -CATExplorador -CCBot -Cegbfeieh -CheTeam -ChinaClaw -Chlooe -Claritybot +BunnySlippers +BuzzSumo +CazoodleBot Cliqzbot Cloud mapping -Cogentbot +coccocbot-web cognitiveseo +com.plumanalytics Copier -Copyscape +CopyRightCheck +crawler4j +crawler.feedback crawl.sogou.com +Crescent +CrunchBot CSHttp Curious -Custo +DataCha0s DBLBot -demandbase-bot -Deusu +Demon +Devil DigitalPebble +DIIbot +Dispatch DnyzBot -DomainAppender -DomainSigmaCrawler -DomainStatsBot -Dragonfly +Dotbot +Download Wonder Drip +DSearch eCatch -ECCP/1.0 EMail Siphon -EMail Wolf -EroCrawler -ExtractorPro -Ezooms -facebookscraper +Evil +Express WebPictures +ExtLinksBot +FDM FemtosearchBot Firefox/7.0 -Foobot +FlashGet Freeuploader +Fyrebot GalaxyBot -Genieo -GermCrawler +GetRight GetWeb +Gigablast Gigabot -Go-Ahead-Got-It +Gotit +GoZilla +GrabNet Grafula -GrapeshotCrawler -GridBot -GT::WWW Haansoft +HaosouSpider Havij Heritrix -Hloader +HMView +HTMLparser HTTP::Lite HTTrack -HybridBot -Id-search -Image Fetch +IlseBot Image Sucker IndeedBot -InfoTekies -instabid Intelliseek -Internet Ninja +InternetSeer IRLbot +Iskanie JamesBOT -JetCar -Joomla -Jorgee +JikeSpider +JustView +Keyword Density +Kozmosbot Larbin LeechGet -Lftp -LibWeb +LexiBot Libwhisker +Lightspeedsystems +Likse Linkdexbot -LinkWalker +LinkextractorPro +LinkpadBot +LinkScan LinqiaMetadataDownloaderBot -LinqiaRSSBot +LinqiaScrapeBot +Lipperhey Lipperhey Spider -Lmspider LNSpiderguy -lwp-trivial -Mail.RU_Bot +LWP::Simple +Magnet +Mag-Net +magpie-crawler Majestic12 +Majestic SEO Majestic-SEO -Mata Hari -meanpathbot -MeanPath Bot -Mediatoolkitbot +MarkWatch +MauiBot +MegaIndex.ru MFC_Tear_Sample -MIDown tool +Microsoft Data Access +Microsoft URL Control +MIIxpc Mister PiX -Mojeek -Mojolicious +MJ12bot +Morfeus Fucking Scanner Mr.4x3 +MSFrontPage Musobot -Name Intelligence +Navroad +NearSite NetAnts -netEstate NE Crawler +NetLyzer NetMechanic -NetSpider -Netvibes -NetZIP -NICErsPRO +Nettrack +Net Vampire Niki-bot -Ninja +Nimbostratus NPbot -Offline Explorer -Offline Navigator +oBot OnCrawl -Openfind -OpenLinkProfiler +OpenVAS OrangeBot OrangeSpider OutclicksBot PageAnalyzer -Page Analyzer +PageGrabber page scorer PageScorer Pandalytics -Panscient -PECL::HTTP -PHPCrawl +Papa Foto +Pavuk +pcBrowser +Picsearch PictureFinder -PleaseCrawl Pockey +polaris version Probethenet -ProPowerBot ProWebWalker Psbot +Pump +PxBroker +PyCurl QueryN Metasearch +Quick-Crawler RankActive RankActiveLinkBot -RankingBot2 +Rankivabot RankurBot Reaper -RebelMouse -RepoMonkey -Ripper -SalesIntelligent -ScanAlert +ReGet +RocketCrawler +s1z.ru +Scanbot Scrapy -Searchestate +Screaming +ScreenerBot +Semrush SemrushBot -Seomoz +SEOprofiler SEOstats -serpstatbot sexsearcher Shodan SISTRIX -Sitebeam SiteExplorer -SiteSnagger -SiteSucker -SmartDownload -Snoopy -Sosospider +Snapbot +SocialRankIOBot +Sociscraper Sottopop SpaceBison -Spammen SpankBot -spyfu -Sqlmap -Sqworm +Spanner +sp_auditbot +Spbot +SputnikBot +Stripper Sucker -Sucuri +SuperBot SuperHTTP -Surfbot Suzuran -Swiftbot -sysscan -Szukacz T0PHackTeam -tAkeOut -TeleportPro -Telesphorep -The Intraformant -Thumbor +T8Abot +Telesoft TightTwatBot Titan -Toweyabot -True_Robot Turingos TurnitinBot TwengaBot -Typhoeus -UnisterBot -URLy.Warning -VidibleScraper -Virusdie +Twice +Upflow +URLy Warning +Vagabondo +VoidEYE +Voil Wallpapers/3.0 -WASALive-Bot +WBSearchBot Web Auto -WEBDAV -Web Enhancer +Web Collage +WebEnhancer WebFetch -WebGo IS WebImageCollector WebLeacher -WebmasterWorldForumBot +WebPix WebReaper +Web Sauger Webshag -WebsiteQuester +WebsiteExtractor +Website Quester Webster WebStripper Web Sucker -WeSEE +WebWhacker +WebZIP Whack +Whacker Widow +WinHTTrack WiseGuys Robot +WISENutbot Woobot +Wotbox +WWW::Mechanize WWWOFFLE -x09Mozilla +x22Mozilla Xaldon WebSpider +Xaldon_WebSpider +YoudaoBot +Zade Zauba -ZmEu +zauba.io ZyBorg diff --git a/.dev-tools/test_units/random-referrers-for-test-quick.list b/.dev-tools/test_units/random-referrers-for-test-quick.list index d04295480..0163a2c71 100644 --- a/.dev-tools/test_units/random-referrers-for-test-quick.list +++ b/.dev-tools/test_units/random-referrers-for-test-quick.list @@ -1,10 +1,10 @@ -100searchengines.com -alert-jdh.xyz -apple.com-webbrowsing-security.review -fickenprivat.info -gopro-online.info -jennyfire.ru -searchkut.com -sovinsteel.ru -tizanidinehcl2mg.blogspot.com -vvpg.ru +beauty-things.com +cakemediahq.com.au +cannibalcheerleader.com +dickssportinggoods.com +ikearugs.xyz +istock-mebel.ru +kinozapas.org +socialtrade.biz +wieseversa.no +zaloro.com diff --git a/.dev-tools/test_units/random-referrers-for-test.list b/.dev-tools/test_units/random-referrers-for-test.list index 09de281af..6272a3bea 100644 --- a/.dev-tools/test_units/random-referrers-for-test.list +++ b/.dev-tools/test_units/random-referrers-for-test.list @@ -1,500 +1,500 @@ -00author.com -00webcams.com -03e.info -0n-line.tv -1000n1.ru -100searchengines.com -101flag.ru -24videos.tv -24x7-server-support.site -2728fb936f0.com -273-fz.ru -2kata.ru -3th.co.in +100dollars-seo.com +12bet.com 3xforum.ro -4pp13.com -4ureyesonly.com -5000-cotydzien.com -5tey7463.weebly.com -9med.net -a14download.com -aa9d046aab36af4ff182f097f840430d51.com -abiente.ru -absolutelyanalog.com -academiacsmendoza.org -activecampaign.dreamhosters.com -adidas.frwebs.fr -admanaerofoil.com -adult3dgames.com -advancedmassagebysara.com -advancedsoftwaresupport.com -aero2.ru -afslankpillen2017nl.eu -alldownload.pw -allmarketsnewdayli.gdn -alveris.ru -amateurgalls.com -am-se.com -apiallgeniusinfo-a.akamaihd.net -apispringsmartne-a.akamaihd.net -app5.letmacworkfaster.world -appsecurityr.com -ap.senai.br -arendakvartir.kz -arendatora.ru -artavenuegardenstudios.com -asiavirtualsolutions.com -atelielembrancaqueencanta.com.br -ausergrubhof.info -aussie-prizes.com -aviav.co -aviva-limoux.com -azbukafree.com -babieca.com -baditri.com -bad-stars.net -balla-boo.se -bankofthewext.com -bannerspace.com -best-deals-products.com -bestempresas.es -besthomemadepornsites.com -beta.hotkeys.com -bidr.trellian.com -bkns.vn -blogig.org -boeuklad.com -bouda.kvalitne.cz -brakehawk.com -bravegirlsclub.com -brothers-smaller.ru -btnativenav.com -burger-tycoon.com -bus-offer.com -buy-loft.ru -buzzonclick.com -callejondelpozo.es +4istoshop.com +6hopping.com +7zap.com +83net.jp +aa08daf7e13b6345e09e92f771507fa5f4.com +abctoppictures.net +acmebtn.ml +addblueoff.com.ua +adinterax.com +admatic.com.tr +adultmeetonline.info +adult-shop.com.ua +advertisingtag.net +ad-words.ru +aimayangzhi.com +akuhni.by +alborzan.com +albumsuper.info +allcredits.su +alldubai.biz +allnews24.in +amazon-seo-service.com +anal-acrobats.com +anal-acrobats.hol.es +animal-drawings.com +animebox.com.ua +antons-transporte.de +apilinkswiftco-a.akamaihd.net +apiluckyleapnet-a.akamaihd.net +apimegabrowsebiz-a.akamaihd.net +apishops.ru +arate.ru +arcadepages.com +artefakct.com +as5000.com +asrv-a.akamoihd.net +asseenontv.ru +autoloans.com +auto-moto-elektronika.cz +autotop.com.ua +autoua.top +avtolombard-krasnodar.ru +backlinkwatch.com +balitouroffice.com +bannerpower.com +bausparen.bz.it +bcmp.org +beachdriveblog.com +berdasovivan.ru +bestchoice.cf +bestmouthwash.club +bg6s0.com +biglistofwebsites.com +bitcoinremote.com +bobinoz.com +bocoarchives.org +bonux.nextview.ru +bookhome.info +bryansk.zrus.org +buketeg.xyz +bupropion-sr-150-mg.us +buqayy0.livejournal.com +businesxxl.com +buypuppies.ca +buyscabiescream.com +buytizanidineonline.blogspot.com +cablecar.us +calc-for-credit.ru +californianews.cf canoncdriverq3.pen.io -canrioloadm.gq -carloans.com -carrauterie.be -cashkitten-a.akamaihd.net -cheapwebsitehoster.com -cheatcode-lita12.rhcloud.com -checkm8.com -checktext.org -check-this-out-now.online -chelnytruck.ru -cicaki.net -cindymatches.com -circlesl.com -clarithromycin500mg.com -clean-virus-mac.com -clickcash.com -clickprotects.com -club-lukojl.ru -coaching-netz.info -compliance-jessica.top -compliance-margo.top -compucelunlock.net -construmac.com.mx -contextualyield.com +canyougethighofftizanidine.blogspot.com +canzoni.ru +cartierlovestore.com +cartiertrinity.xyz +cbozhe.com +cenoval.ru +centrumcoachingu.com +chatroulette.si +chelyabinsk.xrus.org +chinacheapelitejerseys.com +chlooe.com +chocoslim.pro +christianlouboutinoutlet.win +citetick.com +claytransformations.info +clicksor.com +clipartnew.com +cl.s7.exct.net +compliance-barak.top +compliance-donald.xyz +cookie-law-enforcement-ee.xyz +coolgramgoods.com cool-wedding.net -cpabegins.ru +country-chic.ru +courtshipgift.com +cowlmash.com +cpajunkies.com +cryptoswap.biz crystalslot.com -culad.com -cupday.com -customcollegeessays.net -custom-electric-guitar.com -danschawbel.com -data.vtc.pw +cutalltheshit.com +cxpromote.com +czat.wp.pl +czeshop.info +daneshetabiat.com +darkages.info +dataloading.net +date-withme.com db.speedup.gdn -dekoration.us -dekorkeramik.ru +dcj-nn.ru +decorationspcs.com den-noch24.ru -dentalpearls.com.au -dfwu1019.info -dgfitness.co -dildofotzen.net -dktr.ru -dnsrsearch.com -domaincrawler.com -domaineaneblanc.com -domainsphoto.com -dostavka-v-krym.com -driving.kiev.ua +deplim.com +designdevise.com +design-ideas.info +detoxmed24.ru +dickssportinggoods.com +digital-sale.su +diminishedvalueoforegon.com +div.as +divku.ru +djonwatch.ru +docs4all.com +dokfilms.net +domznaniy.ru +downloader12.ru +drpornogratisx.xxx drugstoreforyou.com -dustyorate.com -dvrlists.com -easyukraine.com -e-avon.ru -educatemetv.com +druzhininevgeniy63.blogspot.com egovaleo.it -elektrischezigaretten2.beeplog.com -elektronischezi.livejournal.com -elektrozigarette2.wordpressy.pl -elitedollars.com -elitepcgames.com -elitesportsadvisor.com -en.altezza.travel -erotische-geschichten-xxl.com -esopini.com -etm-consult.de -etotupo.ru -ezigaretteshop2.mywapblog.com -f5mtrack.com -fakehandbags.xyz -farm26.ru -fastcrawl.com -feel-planet.com -fickluder69.com -filesclub.net +ekto.ee +elmifarhangi.com +enews.tech +escortplius.com +european-torches.ru +evaashop.ru +evening-dating-club.info +exct.net +extremal-blog.com +eyelike.com.ua +eyes-on-you.ga +f012.de +fable.in.ua +fashion.stellaconstance.co +fatfasts-4tmz.com +favorcosmetics.com +faz99.com +feorina.ru +fickenprivat.info filmi-v.online -finansov.info -finstroy.net -firesub.pl -fitness-video.net -flash4fun.com -flprog.com -fodelsedagspresenter.nu -forum20.smailik.org -forum.doctissimo.fr -forum-engineering.ru -freecamdollars.com -freelotto.com -frequiry.com -fuck-paid-share-buttons.xyz -funnymama.com -funnypica.com -funtoonez.com -fz139.ttk.ru -g33.org -g7m.pl -gamezblox.com -gaygalls.net -geilehausfrauen.net -gemara.com -generic-pills-online.com -girlspicsa.com -given2.com -gktt.ru -glavtral.ru -glicol.kz -gobongo.info -gomusix.com +fivedwld.cf +focalink.com +fotosfotos.eu +fotoxxxru.com +fotzen-ficken.com +foxinsocks.ru +freelifetimefuckbook.com +freemags.cc +freenom.link +funnel.co.za +fyl.com.ru +gallery.rennlist.com +gambarkataku.co +gamerscorps.com +games.kolossale.ru +gay.adultgalls.com +getdot.ru +getmiro.com +getprismatic.com +gigixo.com +glavprofit.ru +gok-kasten.net googlefeud.com -gosarhivrt.ru -gotwebsite1.com -gox.com.ua -gpms.org.my -gq-catalog.gq -gratuitbaise.com -gratuitxblcodes.com -greendream.com.ua -growshop.es -gwagka.com -gwebtools.com -hacktougroup.ru -hamptonoaks.ca -hdimagelib.com -himgaws.pw -homegardenlova.com -homeinns.com -home.myplaycity.com -houston-vikings.com -ido3.com -igadgetsworld.com -ilo134uloh.com -imagecoolpub.com -images-graphics-pics.com -imgata.com -impisr.edunsk.ru -inboundlinks.win -inet-traffic.com -infazavr.ru -interfucks.net -iscblog.info -istizanidineanarcoticdrug.blogspot.com -ivearchenceinflu.cf -jackpotchances.com -jerseyssportsshop.com -jobmarket.com.ua -jrpmakati.com -karachev-city.ru -karusel-market.ru -kashubadesign.ru +goroda-vsego-mira.ru +gotorussia.com +gov.yanao.ru +guardlink.com +guigyverpo.cf +guitar-master.org +hdhc.ru +hdimagegallery.net +headpharmacy.com +health-medical-portal.info +heartofpayne.xyz +helpmymacfaster.trade +homedecorpicture.us +hostcritique.com +hostingclub.lk +howtowhitenteethfast.xyz +hully.altervista.org +ihc2015.info +imperiafilm.ru +impisr.ru +interferencer.ru +intermesh.net +intimshop-fantasy.ru +iopeninghours.co.uk +iplogger.org +iqoption.com +islamtoday.co.za +islandminingsupply.wordpress.com +ispaniya-costa-blanca.ru +itronics.ca +izhevsk.xrus.org +jackwolfskinoutlet.online +javidol.site +jewelryandfiligree.com +jjbabskoe.ru +johnrobertsoninc.com +journeydownthescale.info +juliadiets.com +jumptap.com +kalandranis.gr +kamagragelusa.net +kchaxton.com keki.info -kevblog.top -kicknights.gq -kiinomaniak.pl -kinoplen.ru -kinozapas.com -kleine-titten.biz -knogg.net +kenaba.su +keywordsking.com +kihi.gdn +kino2018.cc +kino2018.club +kinobest.pl +kinotorka.ru kollesa.ru -kongoultry.net -konoplisemena.com -koral.se -koronirealestate.gr -krasivoe-hd.net -ktotut.net -ku6.com -kupit-adenu.ru -ladov.ru -landmania.ru -landreferat.ru -lankarns.com -lawyers.cafe -ldrtrack.com -leboard.ru -lexaprogeneric.link -lida-ru.com -linkbolic.com -lion.bolegapakistan.com -liran-locks.com -liveinternet.ru -loftdigital.eu +konteiner24.com +krasnodar.ru +la-fa.ru +laparfumotec.com +law-check-nine.xyz +lecbter-relationships.ga +levaquin750mg.blogspot.com +lincolntheatre.com +linksharingt.com +liupis.com +locimge.pw +lonely-mature.com lottospring.com ltvperf.com -luciddiagnostics.in -mackeeper-center.club -macnewtech.com -macresource.co.uk -mafa-free.com -magazin-pics.ru -marcowebonyodziezowe.pl -maslenka.kz -meduza-consult.ru -megaindex.ru -middlerush-a.akamaihd.net -mixed-wrestling.ru -modenamebel.ru -monclerboots.xyz -moneymaster.ru -morf.snn.gr -moscow-clining.ru -moviezbonkerssk.cf -mrcsa.com.au -musflashtv.com -musicas.baixar-musicas-gratis.com -my-aladin.com -myblogregistercm.tk -mydownlodablefiles.com -mygameplus.com -mykings.pw -nalogovyy-kodeks.ru -namenectar.com -naturalshair.site -new-post.tk -newtechspb.ru -nikitsyringedrivelg.pen.io -normalegal.ru -novosti-hi-tech.ru -nrjmobile.fr -nullrefer.com -oakleyglassesonline.us -oballergiya.ru -ochistka-stokov.ru -offergroup.info -offf.info -offtopic.biz -omsk.xrus.org -o-o-6-o-o.ru -ooomeru.ru -ooo-olni.ru -open-odyssey.org -oralsexfilme.net -ororodnik.goodbb.ru -partner-cdn.men -pdns.cz -perfectpracticeweb.com -perosan.com -petrovka-online.com -phormchina.com -phuketscreen.com -piccshare.com -pintattoos.com -pistonclasico.com +luckyday.world +luckyshop.net.ua +lunamedia.co +luxup.ru +m5home.ru +macrotek.ru +mac-shield.com +magnetic-bracelets.ru +maju.bluesalt.co +makis.nu +mamasuper.prom.ua +mansparskats.com +manve.info +mararoom.ru +marketland.ml +matchpal-a.akamaihd.net +maxthon.com +mediafresh.online +megahdporno.net +mere.host.sk +m-google.xyz +mirtorrent.net +missvietnam.org +mixtapetorrent.com +mnogabukaff.net +mojowhois.com +monclervests.xyz +moneyteam24.com +moroccosurfadventures.com +moskva.nodup.ru +mp3ringtone.info +mrmoneymustache.com +msk.afora.ru +musezone.ru +my-cash-bot.co +myindospace.com +myiptest.com +nalogovyykodeks.ru +nash-krym.info +naturalbreakthroughsresearch.com +neobux-bg.info +netallergy.ru +newstudio.tv +nextconseil.com +nightvision746.weebly.com +niki-mlt.ru +niroo.info +noumeda.com +nvformula.ru +o-dachnik.ru +offtime.ru +olvanto.ru +onlinemegax.com +online-podarki.com +onlythegames.com +o-o-6-o-o.com +optibuymac.com +pacificair.com +panouri-solare-acoperis.com +papasdelivery.ru +partner-print.men +partnersafe.men +partybunny.ru +parvezmia.xyz +pawli.eu +pay2me.pl +pdamods.ru +pelfind.me +penisvergrotendepillennl.ovh +percin.biz.ly +petedrummond.com +picphotos.net +pictures-and-images.com +pio.polytopesexempt.com +piski.top pix24x7.com -pixelrz.com -pizdeishn.com -pl.aasoldes.fr -pliks.pl -plohaya-kreditnaya-istoriya.ru -pl-vouchers.com -pochemychka.net -porno-dojki.net -porn-w.org -pospr.waw.pl +pixshark.com +playlott.com +pl.id-forex.com +ploenjitmedia.azurewebsites.net +podshipniki-nsk.ru +popserve.adscpm.net +porn555.com +pornmania.pl +pornoxxx.com.mx +porodasobak.net powenlite24.ru -preparevideosafesystem4unow.space -presleycollectibles.com -privatamateure.com -profolan.pl -proftests.net +preparevideosafesystem4unow.site +priscilarodrigues.com.br +privacyassistant.net +prize44.com +prizestohandle.club +productarium.com +prohoster.info +promodj.com pro-okis.ru -prostitutki-novgoroda.org -prostitutki-rostova.org -pssucai.info -pst2017.onlinewebshop.net -pts163.ru -quality-traffic.com -quangcaons.com -quizzitch.net -radiodigital.co -razyboard.com -realmonte.net -reeyanaturopathy.com -refudiatethissarah.info -reklamuss.ru -remont-mobile-phones.ru -resellerclub.com -responsinator.com -restaurantlescampi.com +proposal-engine.com +pskcijdc.bloger.index.hr +purplesphere.in +queerspace.com +ratemodels.net +razleton.com +refererx.com +registrationdomainsite.com +registry-cleaner.net +responsive-test.net restorator-msk.ru +retailwith.com rff-cfal.info -richinvestmonitor.com -rockprogblog.com -royal-investments.net -runtnc.net -rusenvironmental.net +riders.ro +royalads.net +royalcar-ufa.ru +royalvegascasino.com russian-postindex.ru -rvtv.ru -s8-nowy-wygraj.comli.com -sack.net -samara.rosfirm.ru -sarf3omlat.com -sbdl.no -scanmyphones.com -scanner-barak.top +safe-app.net +sandhillsonline.com +santechnik.jimdo.com +sashagreyblog.ga +saveriopiazza.it +savingsslider-a.akamaihd.net +sawin.beth.webd.pl scanner-brian.top -school-diplomat.ru -searchinquire.com -seasaltwithfood.com -seccioncontrabajo.com -seksotur.ru -seksvideoonlain.com -semrush.com -seogadget.ru -serdcenebolit.com -serialsway.ucoz.ru -sex-dating.co -shopvilleroyboch.com.ua -shymkent.xkaz.org -signal03.ru -signoredom.com -silktide.com -simple-share-buttons.com -sims-sims.ru -site-auditor.online +scanner-elena.top +scanner-fred.top +scanner-viktor.top +scoopquest.com +sdi-pme.com +seolab.top +seo-tools-optimizing.com +serpstat.com +sexflirtbook.com +sexs-foto.com +sex-tracker.de +sexy-pings.com +sharebutton.org +sharebutton.to +shinikiev.com.ua +shtaketniki.ru +sientalyric.co +simple-image.com.ua +simply.net +site-analyzer.com +site-speed-check.site +sitevalued.com +sitiz.club sky-mine.ru -skypasss.com -slutloadlive.com -smartpet.ru -social-fun.ru -socialtrade.biz -soft1.ru -softlinesolutions.me -soft-terminal.ru -solicita.info -solmarket.by -songplanet.ru -sonnikforme.ru -soochi.co -spammen.de -spbchampionat.ru -spidtest.org -spymac.net -startufa.ru -startwp.org -steelmaster.lv -stillmiracle.com -stpicks.com -stroilka.info -superinterstitial.com -superstats.com -surfbuyermac.com -surflinksmedical.com -svbur.ru -svetlotorg.ru -swingerseiten.com -swiped.su -tackletarts.co -takeprofitsystem.com -tam-gde-more.ru -tanieaukcje.com.pl -tarad.com -teenbbw.yopoint.in +sledstvie-veli.net +smailik.org +smartadserver.com +snip.tw +socialseet.ru +sofit-dmd.ru +space4update.pw +spinnerco.ca +spitfiremusic.com +spl63.fr +srdrvp.com +stairliftstrue.com +stanthonyscatholicchurch.org +statoutlook.info +sterva.cc +stevemonsen.com +stjamesschool.info +stpolice.com +stroicol.net +summerlinhomes411.info +supers.com.ua +szucs.ru +takeflyte.com +techart24.com +teesdaleflyballclub.co.uk +telegraf.by terrafootwear.us -theautoprofit.ml -thebestphotos.eu -thefotosgratis.eu -thegamerznetwork.com -thejournal.ru -thewebsitetemplate.info -tiandeural.ru +tfxiq.com +thegioixekhach.com +thenetinfo.com +thetoiletpaper.com +ticketsys.inetwd.com +titangel-vietnam.com tizanidineandhydrocodone.blogspot.com -tizanidinedosage.blogspot.com +tizanidinecapsules.blogspot.com tizanidineforopiatewithdrawal.blogspot.com -tizanidineorflexeril.blogspot.com -toondinsey.com -topkarkas.com -torrnada.ru -torture.ml -toxicwap.com -tracxn.com -traderzplanet.in -traffic2money.com -truck-rece.life -trucri.me -trustedhealthtips.com -tuckpointingmasonrysystems.com -tupper-posuda.ru -twojebook.pl -ua-company.ru -uamtrk.com -uchetunet.su -undergroundcityphoto.com -underthesite.com -urengoy.pro -validdomain.xyz -valoresito.com -vancleefreplica.pw -vapeface.club -vedomstvo.net -video-chat.in -viktoria-center.ru -voloomoney.com -voluumtrk.com -vzube.com -wahicbefa31.soup.io -wallpaperstock.net -watracker.net -webalan.ru +tizanidinewithdrawalsymptoms.blogspot.com +tksn.ru +tmm-kurs.ru +tmtrck.com +toonfamilies.net +topappspro.com +topmira.com +topshef.ru +trafficjunky.net +tripper.de +triumf-realty.ru +truck-addzilla.life +truck-land.life +trustl.life +tv-spoty.info +typer.one +tyumen.xrus.org +u17795.netangels.ru +ucsol.ru +udayavani.com +uggbootsoutletsale.us +uginekologa.com +ugogo.info +umaseh.com +unrealcommander.org +uptimebot.net +urlopener.com +ussearche.cf +uzungil.com +videochat.mx +videochaty.ru +viel.su +vinsit.ru +vip-parfumeria.ru +virtuagirl.com +visitcambridge.org +voluumtracker1.com +vzlomtw.com +watchmyfb.pl +webadvance.club +web.cvut.cz webix.me -weblo.com -witherrom55.eklablog.fr -withstandingheartwarming.com -woman-tampon.ru -x5market.ru -xcc24.pl -xmlinde.com -xn-----7kceclhb4abre1b4a0ccl2fxch1a.xn--p1ai -xn--80ajjbdhgmudixfjc8c5a9df8b.xn--p1ai -xn----8sbarihbihxpxqgaf0g1e.xn--80adxhks -xn--h1aakne2ba.xn--p1ai -xn--hxazdsfy.blogspot.com -xnxxandxvideos.com -xportvusbdriver8i.snack.ws -xxxtube69.com -xxxtubesafari.com -yourothersite.com -yubikk.info -zaapplesales.blogspot.com -zapiszto.pl -zero1.it -zeroredirect12.com -znaturaloriginal.com -zooggames.com -zverokruh-shop.cz -zx6.ru +webkeyit.com +webs.com +wemedinc.com +wesharepics.com +wholesalenfljerseys.us.com +womensplay.net +writingservices17.blogspot.ru +ws.ampower.me +www.timer4web.com +xchangetrak.com +xgames-04.com +xn-----6kccaibs5cb8afhjrfmix2n.xn--p1ai +xn----7sbbahaq9bb5afgiqfliv4m.xn--p1ai +xn--80aanaardaperhcem4a6i.com +xn----8sblgmbj1a1bk8l.xn----161-4vemb6cjl7anbaea3afninj.xn--p1ai +xn----dtbndd4ae7eub.top +xn--oogle-wmc.com +x-porno.video +yachts-cruise.info +yebocasino.com +yellowproxy.net +yoshkarola.zrus.org +yourporngay.com +youtuhe.com +yugk.net +yurgorod.ru +za-fun-offer.com +zagadki.in.ua +zaimhelp.ru +zakonobosago.ru +zarajbuilders.com +zawyna.ua +zeroredirect6.com +znakomstva-piter78.ru +zoogdinsney.com +zskdla.site diff --git a/.dev-tools/test_units/random-referrers-for-whitelist-test.list b/.dev-tools/test_units/random-referrers-for-whitelist-test.list index 703e97e90..7a691efa4 100644 --- a/.dev-tools/test_units/random-referrers-for-whitelist-test.list +++ b/.dev-tools/test_units/random-referrers-for-whitelist-test.list @@ -1,250 +1,250 @@ -00webcams.com -100dollars-seo.com -12bet.com -202ch.com -2rich4bitches.com -acunetix-referrer.com -admitad.com -adpremium.org -advancedsoftwaresupport.com -afora.ru -aktobe.xkaz.org -amehdaily.com +0912701309f8ce.com +101billion.com +101lesbian.xyz +1hwy.com +24x7-server-support.site +2kata.ru +51unlim.ru +7a2rnuey1tw9ar.ru +7makemoneyonline.com +8xv8.com +9med.net +aa9bd78f328a6a41279d0fad0a88df1901.com +abcdefh.xyz +abclauncher.com +abusora.com +acheterviagrafr24.com +adimmix.com +adultwebhosting.info +advokateg.xyz +aerogo.com +aihelen.net +alert-fjg.xyz +allforminecraft.ru +alta-realestate.com +amateurlivechat.org am-se.com -animalia-life.club -animali.deagostinipassion.it -animalrank.com +an-donut.com +anime.dougasouko.com +anmysite.com +anti-virus-removal.info +apartmentratings.com +apccargo.com apibrowsesmartne-a.akamaihd.net -applyneedy.xyz -arabsexxxtube.com -arendovalka.xyz -arkartex.ru -asia-forum.ru -bagsonsale.online -balkanfarma.org -bannerspace.com -beauby.ru -beauty-clinic.ru -best-mam.ru -bestsexyblog.com -bidbuy.co.kr -bingoporno.com -biographiya.com -bio.trade-jp.net -blogs.rediff.com -bolitgorlo.net -bonky.biz -brandbucket.com -businesn.men -cactussoft.biz -cam-kontakte.org -candcstuccoandstone.com -canvas.gloverid.site -centraletermice.us -centre-indigo.org.ua -chocolateslim-original.com -circlesl.com -clodo.ru -cntravelre.com -colehaanoutlet.store -companies-catalog.com -compliance-barak.xyz -confib.ifmo.ru -contentlook.co -creams.makeforum.eu -cymbaltaandtizanidine.blogspot.com -dairyindia.in -dayibiao.com -digilander.libero.it -dipstar.org -docs4all.com -doktoronline.no -dostavka-v-krym.com -dosugrostov.site -drugstoreforyou.com +apilinkswiftco-a.akamaihd.net +app-ready.xyz +apptester.tk +arcadepages.com +arcteryxsale.online +arius.tech +atlasvkusov.ru +audiofree.ru +ausergrubhof.info +bag77.ru +bambi.ck.ua +bannerconnect.net +bestofupload.info +best-seo-offer.com +best-seo-software.xyz +betterhdporn.com +billiard-classic.com.ua +bitporno.sx +blackplanet.com +bluerobot.info +bmwhighperformers.com +brandehk.dk +brianhenry.co +btvn.ru +carmuffler.net +casite-513049.cloudaccess.net +celebsopics.com +ceotrk.com +chinaelitecheapjerseys.com +chorus.terakeet.com +cibpenonptib.flu.cc +cienum.fr +cleanpcnow.com +clickprotects.com +compliance-walter.top +contextualyield.com +cool-wedding.net +customcollegeessays.net +cxpromote.com +cyclobenzaprinevstizanidine.blogspot.com +cyse.tk +deanmoore.ie +defenderxtactical.com +delfin-aqua.com.ua +divatraffic.com +divku.ru +domainroam.win +downloadmefiranaratb1972.xpg.com.br +drunkenstepfather.com +dtnlyss.com +dyt.net +eachdayisagift.review earthmagic.info -eas-seo.com -e-biznes.info -ec-file.info -edelstahlschornstein-123.de -elexies.info -elkacentr.ru -en.altezza.travel -enews.tech -envaseslotusama.com -este-line.com.ua -europages.com.ru -evening-dating-club.info -exonline.info -favoritemoney.ru -filmgo.ru -flex4launch.ru -flprog.com -fototravel.eu -freejabs.com -freemaintenancesysforpcandmac.top -freeseedsonline.com -free-stock-illustration.com -friendflnder.com -frustrated-favorable.gq -g7m.pl -gameplexcity.com -gateway.zscalerone.net -gembird.com -get-your-social-buttons.info -golaya.pw -golden-praga.ru +ekn-art.se +en.home-task.com +enthuse.computernetworksonline.com +etenininrade.ga +exmasters.com +f1nder.org +fdzone.org +fettefrauen.net +find1friend.com +fivedwld.cf +flavors.me +forex21.ru +foto-telok.net +free-fb-traffic.com +freelifetimefuckbook.com +free-traffic.xyz +freshberry.com.ua +fuckingawesome.com +fugarif.ga +fxgallery.com +gael-s.ru +gamerextra.com +gay-file.com +ge0ip.net +genericlowlatencyasiodriverhq.aircus.com +getoutofdebtfree.org +getresponse.com +ghernnqr.skyrock.com +girlsatgames.ru +globalscam.ga +goldenggames.com golfresa.lucania.se -golmau.host.sk -goodhousekeeping.com -greatfind-a.akamaihd.net -gta-top.ru -guardlink.org -harmonyglen.us -himgaws.pw -hostsshop.ru -hustoon.over-blog.com -impisr.edunsk.ru -io9.com -ivanstroi.ru -javlibrary.cc -javstock.com -jerseywholesalebizchina.com -jerseywholesaleelitestore.com -keywords-monitoring-success.com -kino-rating.ru -kinosed.net -kiskinhouse.com -konpax.com -kretpicf.pw -kursy-ege.ru -lampokrat.ws -ledis.top +googglet.com +gosreg.amchs.ru +gribkovye-zabolevaniya.com +grizzlysgrill.com +groupmoney.ru +gsasearchengineranker.pw +hdxnxxtube.mobi +herokuapp.com +heygidday.biz +highstairs-a.akamaihd.net +hiwibyh.bugs3.com +hosting-tracker.com +htmlcorner.com +huimin764128.com +intellektmedia.at +investyb.com +iphantom.com +iqoption.pro +iron-age.info +isistaylorporn.info +iweblist.info +jackpotchances.com +komputers-best.ru +kw21.org +laulini.soclog.se +law-check-nine.xyz lineavabit.it -live-sexchat.ru +lion.bolegapakistan.com +loadopia.com +loginduepunti.it +louboutinreplica.pw luxmagazine.cf -mackeeper-land-672695126.us-east-1.elb.amazonaws.com -mahnwachen-helfen.info -mainlinehobby.net -malwareremovalcenter.com -manifestation.betteroffers.review -marcogrup.com -marketingtechniques.info -marmotstore.online -m.b00kmarks.com -mebel-alait.ru -mezaruk.info -mirtorrent.net -mobplayer.net -moesen-ficken.com -money-for-placing-articles.com -monsterdivx.com -moscow-clining.ru -mrwhite.biz -multgo.ru -musicspire.online -netfacet.net -new-apps.ru -nibbler.silktide.com -onblastblog.online -onefilms.net -onemactrckr.com -osnova3.ru -pacificair.com -paphoselectricianandplumber.com -paramountmarble.co.uk -par-fallen.ga -partner-cdn.men -pigrafix.at -pila.pl -piuminiita.com -plugingeorgia.com -podshipniki-ntn.ru -pomoc-drogowa.cba.pl -porn555.com -pornhub-ru.com -pornmania.pl -pornstartits.xblog.in -porn-w.org -pozdravleniya-c.ru -privacyassistant.net -proftests.net -pulseonclick.com -quickchange.cc -rainbowice.ru -rebuildermedical.com -recordpage-a.akamaihd.net -resultshub-a.akamaihd.net -retailwith.com -ric.info -robertefuller.com -roofers.org.uk -rosbalt.com.ua -royal-betting.net -ru-dety.ru -rusoft-zone.ru -rybalka-opt.ru -saletool.ru +m5home.ru +magazintiande.ru +managerpak204.weebly.com +maridan.com.ua +marketland.ml +matb3aa.com +mebeldekor.com.ua +mebel-vstroika.ru +metalonly.info +miniature.io +misterjtbarbers.com +modabutik.ru +mojpregled.com +motherboard.vice.com +moyaterapiya.ru +musicdaddy.net +muztops.ru +mycouponizemac.com +myonigroup.com +narkologiya-peterburg.ru +naturalbreakthroughsresearch.com +neodownload.webcam +nikkiewart.ru +niroo.info +nosecret.com.ua +of-ireland.info +oktube.ru +online-sbank.ru +oralsexfilme.net +parajumpersoutlet.online +park.above.com +pathwhelp.org +pkr1hand.com +plastgran.com +platinumdeals.gr +porno-video-chati.ru +proekt-mos.ru +profitwithalex.info +prointer.net.ua +proxyelite.biz +puzzleweb.ru +quebec-bin.com +raavidesigns.com +rankings-analytics.com +recipedays.com +redbottomheels.xyz +redirectme.net +registry-clean-up.net +reklamuss.ru +robot-forex.biz +santechnik.jimdo.com sc-specialhost.com -searchengineranker.email -seoboxes.com -seofirmreviewsus.info -seolab.top -sexcamamateurchat.com -sexkontakte-seite.com -sexobzor.info -sexreliz.com -sex-tracker.com +seadragonherbery.com +searchinterneat-a.akamaihd.net +seminarygeorgia59.ga +share-buttons-for-free.com shemale-sex.net -sibtest.ru -sibvitr.ru -sledstvie-veli.net -smstraf.ru -spabali.org -spb.afora.ru -sportbetfair.com -stauga.altervista.org -stockspmb.info -storehouse.ua -surfbuyermac.com -surgut.zrus.org -svnuppsalaorebro.se -swimpool.ca -szucs.ru -tampabaywatch.org -tech4master.com -tgtclick.com -thetoiletpaper.com -tizanidinestreetprice.blogspot.com -top-karkas.ru -totu.us -twittrading.com -u17795.netangels.ru -uchebavchehii.ru -unisexjewelry.org -unpredictable.ga -uogonline.com -vancleefreplica.pw -vektorpress.ru -versaut.xxx-cam.webcam -viagengrarx.com -viagra.pp.ua -videotuber.ru -voronezh.xrus.org -vriel.batcave.net -vut.com.ru -w7s.ru +shopvilleroyboch.com.ua +simply.net +sisi-go.ru +skypasss.com +slim.sellany.ru +snomer1.ru +snyatie-lomki-v-stacionare.ru +spamnuker.com +sprttrack.com +stariy-baku.com +studworks.org +swinger-mobil.net +taaaak.com +tecspb.ru +tedxrj.com +thegamerznetwork.com +thfox.com +tizanidinehydrochloride2mg.blogspot.com +topmira.com +torrentdownloadhub.com +torrentred.games +torrents-tracker.com +tradgardspartner.se +trafficjunky.net +tsstcorpcddvdwshbbdriverfb.aircus.com +tupper-shop.ru +vapeface.club +viandpet.com +videosbox.ru +wapsite.me watchmygf.net -weburok.com -woodyguthrie.se -worst-sites.online -wowcasinoonline.ooo -wygraj-teraz.com -xaijo.com -x-lime.com -xsion.net -zojirushi-products.ru -zoogdisany.com -zoominfo.com -zophim.me -zvuker.net +web-betting.ru +while.cheapwebsitehoster.com +woman-tampon.ru +worldtraveler.world +www.get-free-traffic-now.com +xblog.in +xn----8sblgmbj1a1bk8l.xn----161-4vemb6cjl7anbaea3afninj.xn--p1ai +xxxdatinglocal.us +xz618.com +yugk.net +zero1.it +zone-kev717.info diff --git a/.dev-tools/test_units/ratelimittest-beta.txt b/.dev-tools/test_units/ratelimittest-beta.txt index a683b3cc9..d4141bdc1 100644 --- a/.dev-tools/test_units/ratelimittest-beta.txt +++ b/.dev-tools/test_units/ratelimittest-beta.txt @@ -5,7 +5,114 @@