diff --git a/.dev-tools/beta_conf_files/conf.d/globalblacklist.conf b/.dev-tools/beta_conf_files/conf.d/globalblacklist.conf index 96f57ae80..368112be6 100644 --- a/.dev-tools/beta_conf_files/conf.d/globalblacklist.conf +++ b/.dev-tools/beta_conf_files/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2019.07.1712 -### Updated: Mon Jul 1 11:24:53 SAST 2019 +### Version: V3.2019.07.1713 +### Updated: Mon Jul 1 11:47:20 SAST 2019 ### Bad Referrer Count: 6717 ### Bad Bot Count: 556 ################################################### diff --git a/.dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf b/.dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf index b9d24324a..0fcb73d00 100644 --- a/.dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf +++ b/.dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -34.68.144.114 1; +35.192.187.174 1; 127.0.0.1 1; diff --git a/.dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf b/.dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf index ea117da9f..3dac4f84e 100644 --- a/.dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf +++ b/.dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -34.68.144.114 0; +35.192.187.174 0; 127.0.0.1 0; diff --git a/.dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf b/.dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf index 96f57ae80..368112be6 100644 --- a/.dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf +++ b/.dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2019.07.1712 -### Updated: Mon Jul 1 11:24:53 SAST 2019 +### Version: V3.2019.07.1713 +### Updated: Mon Jul 1 11:47:20 SAST 2019 ### Bad Referrer Count: 6717 ### Bad Bot Count: 556 ################################################### diff --git a/.dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf b/.dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf index b9d24324a..0fcb73d00 100644 --- a/.dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf +++ b/.dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -34.68.144.114 1; +35.192.187.174 1; 127.0.0.1 1; diff --git a/.dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf b/.dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf index ea117da9f..3dac4f84e 100644 --- a/.dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf +++ b/.dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -34.68.144.114 0; +35.192.187.174 0; 127.0.0.1 0; diff --git a/.dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf b/.dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf index 96f57ae80..368112be6 100644 --- a/.dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf +++ b/.dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2019.07.1712 -### Updated: Mon Jul 1 11:24:53 SAST 2019 +### Version: V3.2019.07.1713 +### Updated: Mon Jul 1 11:47:20 SAST 2019 ### Bad Referrer Count: 6717 ### Bad Bot Count: 556 ################################################### diff --git a/.dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf b/.dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf index b9d24324a..0fcb73d00 100644 --- a/.dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf +++ b/.dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -34.68.144.114 1; +35.192.187.174 1; 127.0.0.1 1; diff --git a/.dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf b/.dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf index ea117da9f..3dac4f84e 100644 --- a/.dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf +++ b/.dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -34.68.144.114 0; +35.192.187.174 0; 127.0.0.1 0; diff --git a/.dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf b/.dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf index 96f57ae80..368112be6 100644 --- a/.dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf +++ b/.dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2019.07.1712 -### Updated: Mon Jul 1 11:24:53 SAST 2019 +### Version: V3.2019.07.1713 +### Updated: Mon Jul 1 11:47:20 SAST 2019 ### Bad Referrer Count: 6717 ### Bad Bot Count: 556 ################################################### diff --git a/.dev-tools/globalblacklist-testing.template b/.dev-tools/globalblacklist-testing.template index 96f57ae80..368112be6 100644 --- a/.dev-tools/globalblacklist-testing.template +++ b/.dev-tools/globalblacklist-testing.template @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2019.07.1712 -### Updated: Mon Jul 1 11:24:53 SAST 2019 +### Version: V3.2019.07.1713 +### Updated: Mon Jul 1 11:47:20 SAST 2019 ### Bad Referrer Count: 6717 ### Bad Bot Count: 556 ################################################### diff --git a/.dev-tools/globalblacklist.template b/.dev-tools/globalblacklist.template index a1999739a..56f176a79 100755 --- a/.dev-tools/globalblacklist.template +++ b/.dev-tools/globalblacklist.template @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2019.07.1712 -### Updated: Mon Jul 1 11:24:52 SAST 2019 +### Version: V4.2019.07.1713 +### Updated: Mon Jul 1 11:47:19 SAST 2019 ### Bad Referrer Count: 6717 ### Bad Bot Count: 556 ################################################### diff --git a/.dev-tools/test1_conf_files/bots.d/whitelist-ips.conf b/.dev-tools/test1_conf_files/bots.d/whitelist-ips.conf index 2b15f263d..ca4fcdf54 100644 --- a/.dev-tools/test1_conf_files/bots.d/whitelist-ips.conf +++ b/.dev-tools/test1_conf_files/bots.d/whitelist-ips.conf @@ -45,4 +45,4 @@ # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST -34.68.144.114 0; +35.193.14.140 0; diff --git a/.dev-tools/test1_conf_files/conf.d/globalblacklist.conf b/.dev-tools/test1_conf_files/conf.d/globalblacklist.conf index a1999739a..56f176a79 100644 --- a/.dev-tools/test1_conf_files/conf.d/globalblacklist.conf +++ b/.dev-tools/test1_conf_files/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2019.07.1712 -### Updated: Mon Jul 1 11:24:52 SAST 2019 +### Version: V4.2019.07.1713 +### Updated: Mon Jul 1 11:47:19 SAST 2019 ### Bad Referrer Count: 6717 ### Bad Bot Count: 556 ################################################### diff --git a/.dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf b/.dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf index 0f27f0dd9..6893ce0fc 100644 --- a/.dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf +++ b/.dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -104.154.113.151 1; +35.192.136.167 1; 127.0.0.1 1; diff --git a/.dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf b/.dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf index 0e0848852..879967dc9 100644 --- a/.dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf +++ b/.dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -104.154.113.151 0; +35.192.136.167 0; 127.0.0.1 0; diff --git a/.dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf b/.dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf index a1999739a..56f176a79 100644 --- a/.dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf +++ b/.dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2019.07.1712 -### Updated: Mon Jul 1 11:24:52 SAST 2019 +### Version: V4.2019.07.1713 +### Updated: Mon Jul 1 11:47:19 SAST 2019 ### Bad Referrer Count: 6717 ### Bad Bot Count: 556 ################################################### diff --git a/.dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf b/.dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf index 0f27f0dd9..6893ce0fc 100644 --- a/.dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf +++ b/.dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -104.154.113.151 1; +35.192.136.167 1; 127.0.0.1 1; diff --git a/.dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf b/.dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf index 0e0848852..879967dc9 100644 --- a/.dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf +++ b/.dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -104.154.113.151 0; +35.192.136.167 0; 127.0.0.1 0; diff --git a/.dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf b/.dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf index a1999739a..56f176a79 100644 --- a/.dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf +++ b/.dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2019.07.1712 -### Updated: Mon Jul 1 11:24:52 SAST 2019 +### Version: V4.2019.07.1713 +### Updated: Mon Jul 1 11:47:19 SAST 2019 ### Bad Referrer Count: 6717 ### Bad Bot Count: 556 ################################################### diff --git a/.dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf b/.dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf index 0f27f0dd9..6893ce0fc 100644 --- a/.dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf +++ b/.dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -104.154.113.151 1; +35.192.136.167 1; 127.0.0.1 1; diff --git a/.dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf b/.dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf index 0e0848852..879967dc9 100644 --- a/.dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf +++ b/.dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -104.154.113.151 0; +35.192.136.167 0; 127.0.0.1 0; diff --git a/.dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf b/.dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf index a1999739a..56f176a79 100644 --- a/.dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf +++ b/.dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2019.07.1712 -### Updated: Mon Jul 1 11:24:52 SAST 2019 +### Version: V4.2019.07.1713 +### Updated: Mon Jul 1 11:47:19 SAST 2019 ### Bad Referrer Count: 6717 ### Bad Bot Count: 556 ################################################### diff --git a/.dev-tools/test2_conf_files/bots.d/whitelist-ips.conf b/.dev-tools/test2_conf_files/bots.d/whitelist-ips.conf index 81d44ffba..f5e1a7808 100644 --- a/.dev-tools/test2_conf_files/bots.d/whitelist-ips.conf +++ b/.dev-tools/test2_conf_files/bots.d/whitelist-ips.conf @@ -45,5 +45,5 @@ # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST -35.188.73.34 0; +35.192.136.167 0; 35.193.14.140 0; diff --git a/.dev-tools/test2_conf_files/conf.d/globalblacklist.conf b/.dev-tools/test2_conf_files/conf.d/globalblacklist.conf index 3116b23ce..00b05696f 100644 --- a/.dev-tools/test2_conf_files/conf.d/globalblacklist.conf +++ b/.dev-tools/test2_conf_files/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2019.07.1712 -### Updated: Mon Jul 1 11:24:52 SAST 2019 +### Version: V4.2019.07.1713 +### Updated: Mon Jul 1 11:47:19 SAST 2019 ### Bad Referrer Count: 6717 ### Bad Bot Count: 556 ################################################### diff --git a/.dev-tools/test4_conf_files/bots.d/whitelist-ips.conf b/.dev-tools/test4_conf_files/bots.d/whitelist-ips.conf index ebe757f93..4851d6475 100644 --- a/.dev-tools/test4_conf_files/bots.d/whitelist-ips.conf +++ b/.dev-tools/test4_conf_files/bots.d/whitelist-ips.conf @@ -45,6 +45,5 @@ # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST -35.192.136.167 0; -34.66.178.120 0; +35.188.73.34 0; 34.68.144.114 0; diff --git a/.dev-tools/test4_conf_files/conf.d/globalblacklist.conf b/.dev-tools/test4_conf_files/conf.d/globalblacklist.conf index a1999739a..56f176a79 100644 --- a/.dev-tools/test4_conf_files/conf.d/globalblacklist.conf +++ b/.dev-tools/test4_conf_files/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2019.07.1712 -### Updated: Mon Jul 1 11:24:52 SAST 2019 +### Version: V4.2019.07.1713 +### Updated: Mon Jul 1 11:47:19 SAST 2019 ### Bad Referrer Count: 6717 ### Bad Bot Count: 556 ################################################### diff --git a/.dev-tools/test5_conf_files/bots.d/whitelist-ips.conf b/.dev-tools/test5_conf_files/bots.d/whitelist-ips.conf index d21a1eaca..761a4bfb2 100644 --- a/.dev-tools/test5_conf_files/bots.d/whitelist-ips.conf +++ b/.dev-tools/test5_conf_files/bots.d/whitelist-ips.conf @@ -45,6 +45,6 @@ # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST +35.188.73.34 0; +34.68.144.114 0; 35.193.14.140 0; -34.66.178.120 0; -104.198.131.58 0; diff --git a/.dev-tools/test5_conf_files/conf.d/globalblacklist.conf b/.dev-tools/test5_conf_files/conf.d/globalblacklist.conf index a1999739a..56f176a79 100644 --- a/.dev-tools/test5_conf_files/conf.d/globalblacklist.conf +++ b/.dev-tools/test5_conf_files/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2019.07.1712 -### Updated: Mon Jul 1 11:24:52 SAST 2019 +### Version: V4.2019.07.1713 +### Updated: Mon Jul 1 11:47:19 SAST 2019 ### Bad Referrer Count: 6717 ### Bad Bot Count: 556 ################################################### diff --git a/.dev-tools/test_units/blacklist-ips.conf b/.dev-tools/test_units/blacklist-ips.conf index b9d24324a..0fcb73d00 100644 --- a/.dev-tools/test_units/blacklist-ips.conf +++ b/.dev-tools/test_units/blacklist-ips.conf @@ -1,2 +1,2 @@ -34.68.144.114 1; +35.192.187.174 1; 127.0.0.1 1; diff --git a/.dev-tools/test_units/random-bots-for-test-quick.list b/.dev-tools/test_units/random-bots-for-test-quick.list index d91f5f655..2cb7de62e 100644 --- a/.dev-tools/test_units/random-bots-for-test-quick.list +++ b/.dev-tools/test_units/random-bots-for-test-quick.list @@ -1,10 +1,10 @@ -BackDoorBot -Gigablast -NextGenSearchBot -Nibbler -pcBrowser -Pixray -Reaper -SEOlyticsCrawler -Sitevigil -Sottopop +evc-batch +Extractor +Ninja +SurveyBot +Vacuum +Vagabondo +WebCollage +WEBDAV +webmeup-crawler +Web Pix diff --git a/.dev-tools/test_units/random-bots-for-test.list b/.dev-tools/test_units/random-bots-for-test.list index 396bccc27..07d37a4f6 100644 --- a/.dev-tools/test_units/random-bots-for-test.list +++ b/.dev-tools/test_units/random-bots-for-test.list @@ -1,250 +1,250 @@ +360Spider 404enemy -80legs -Acunetix +Abonti +Aboundexbot ADmantX -Alligator +AIBOT +Alexibot +AllSubmitter AlphaBot -Apexoo +archive.org_bot ASPSeek Asterias Attach autoemailspider -Backlink-Ceck +BackDoorBot backlink-check +BacklinkCrawler BackStreet BackWeb -Badass +Barkrowler +BatchFTP Battleztar Bazinga -BetaBot -Black Hole -BlackWidow -BotALot -Brandprotect -Brandwatch +BDFetch +BlowFish BuiltBotTough BuiltWith +Bullseye BunnySlippers Calculon -CazoodleBot +CCBot CheeseBot +CherryPicker CheTeam -ChinaClaw Claritybot Cliqzbot -coccocbot-web +Cogentbot +com.plumanalytics Copyscape -Craftbot -crawler.feedback -Custo -DBLBot +Cosmos +CSHttp +DataCha0s +demandbase-bot Demon -Deusu -Digincore -Dirbuster +Devil +DIIbot Disco -Discobot Discoverybot -DomainSigmaCrawler -EasyDL -ECCP/1.0 +Dispatch +DittoSpyder EirGrabber EMail Siphon -EMail Wolf Express WebPictures -ExtLinksBot Extractor -ExtractorPro -Ezooms +Extreme Picture Finder facebookscraper FDM -FHscan Fimap -FlashGet -Freeuploader -GalaxyBot -GetRight -Gigabot -G-i-g-a-b-o-t +Flunky +FyberSpider +Fyrebot +Genieo +Getintent +GetWeb GoZilla Go!Zilla GrabNet -GrapeFX -GrapeshotCrawler -GridBot +Grafula GT::WWW -HaosouSpider -Harvest +Haansoft +HEADMasterSEO Heritrix Hloader -HTMLparser +HMView +HTTrack Humanlinks -Iblog -Id-search -Image Fetch +IDBot Image Sucker -Indy Library -InfoNaviRobot -Intelliseek InterGET -internetVista monitor -ips-agent -Iria -IRLbot -IstellaBot -JamesBOT +Internet Ninja +InternetSeer Jbrofuzz JennyBot -JikeSpider +JetCar Joomla -Jorgee Jyxobot +Keyword Density Kozmosbot +LeechFTP +LeechGet LexiBot -LibWeb -Linkdexbot +Lftp +LinkWalker LinqiaMetadataDownloaderBot LinqiaRSSBot +LinqiaScrapeBot Lipperhey +Lipperhey Spider Litemage_walker -Lmspider -LNSpiderguy Ltx71 -lwp-request -lwp-trivial -Magnet +LWP::Simple +magpie-crawler +Mail.RU_Bot +Majestic12 +Majestic SEO Majestic-SEO -MarkMonitor -MarkWatch Masscan -Mata Hari -meanpathbot +Mass Downloader +MauiBot +Meanpathbot +MeanPath Bot Mediatoolkitbot mediawords Metauri +Microsoft Data Access MIIxpc Mister PiX MJ12bot Mojeek -MSIECrawler -Msrabot +Mr.4x3 +MS Web Services Client Protocol +muhstik-scan Musobot Name Intelligence -Nameprotect -Navroad -Needle +NearSite +NetAnts Netcraft -netEstate NE Crawler +NetLyzer NetSpider Net Vampire +Netvibes +NetZIP NICErsPRO -Nikto -NimbleCrawler -Nimbostratus +Ninja Nmap -oBot -Offline Explorer +NPbot +Offline Navigator OnCrawl OpenLinkProfiler -Openvas -OpenVAS OrangeBot -OrangeSpider OutclicksBot OutfoxBot page scorer -PageScorer -Papa Foto +Panscient +PECL::HTTP PeoplePal -Picscout -Pimonster +Picsearch +PictureFinder +Pi-Monster PleaseCrawl -plumanalytics -ProPowerBot +ProWebWalker +Psbot +Pump PxBroker -PyCurl QueryN Metasearch -RankActiveLinkBot -RankingBot -RankingBot2 -ReGet -Ripper +Quick-Crawler +RankurBot +Reaper +Recorder +RedesScrapy +RepoMonkey +RocketCrawler s1z.ru -ScanAlert -scan.lol +SBIder +Scanbot +Screaming ScreenerBot -Semrush +Searchestate +SearchmetricsBot SemrushBot -Seomoz +SEOkicks +SEOkicks-Robot +SEOlyticsCrawler +seoscanners +SeoSiteCheckup SEOstats serpstatbot -SISTRIX +sexsearcher +Shodan +Siphon +Sitebeam SiteExplorer +SiteLockSpider +SiteSnagger SiteSucker +SlySearch SmartDownload -SMTBot +Snake SocialRankIOBot +Sociscraper sogouspider Sogou web spider Sottopop -Spammen -sp_auditbot -Spinn3r +Spbot spyfu Sqlmap -Steeler -Sucuri SuperBot -Surfbot +SuperHTTP SurveyBot -Swiftbot -T0PHackTeam -TheNomad +sysscan +tAkeOut +TeleportPro +Telesoft +Telesphorep TightTwatBot +Titan Toata Toweyabot -Trendictionbot +Tracemyfile trendiction.com -TwengaBot +True_Robot +Turingos +TurnitinBot Twice -URLy Warning +Upflow +Vacuum +Vagabondo +VB Project VidibleScraper -Voil -WallpapersHD +VoidEYE WASALive-Bot -Webalta -WebAuto -Web Auto -WebBandit +WBSearchBot WebCollage Web Collage +WEBDAV WebEnhancer +Web Enhancer WebFetch -Web Fetch Web Fuck WebGo IS -WebImageCollector -WebLeacher -WebPix -WebReaper -WebSauger -WebsiteQuester +webmeup-crawler +Web Pix Website Quester -Webster WebStripper -WebWhacker +WebSucker +WeSEE Whacker Whatweb -WiseGuys Robot +Who.is Bot +Widow +WinHTTrack WISENutbot -Wonderbot Woobot Wotbox -WPScan WWW-Mechanize -WWW::Mechanize -WWWOFFLE -Xaldon WebSpider -Xaldon_WebSpider -Xenu -Zauba -ZyBorg +x09Mozilla +x22Mozilla +xpymep1.exe +YoudaoBot +Zeus +ZumBot diff --git a/.dev-tools/test_units/random-bots-for-whitelist-test.list b/.dev-tools/test_units/random-bots-for-whitelist-test.list index 157672892..1bac71e0e 100644 --- a/.dev-tools/test_units/random-bots-for-whitelist-test.list +++ b/.dev-tools/test_units/random-bots-for-whitelist-test.list @@ -1,172 +1,175 @@ 360Spider 404enemy +80legs +Abonti +Aboundex Aboundexbot -AfD-Verbotsverfahren -AhrefsBot +ADmantX Aipbot Alexibot -Alligator AlphaBot +Anarchie archive.org_bot -Attach -BackDoorBot -BacklinkCrawler -BackStreet +autoemailspider +Backlink-Ceck +backlink-check BackWeb +Badass Bandit -Barkrowler -Battleztar Bazinga -BDCbot -Bigfoot -BlackWidow +BetaBot +BlowFish +Boardreader BotALot +Buddy BuiltBotTough BuiltWith -Calculon -CazoodleBot -CCBot -Cegbfeieh -CherryPicker +BunnySlippers +CATExplorador +CheeseBot Chlooe -Claritybot +Cloud mapping coccocbot-web -Cogentbot -cognitiveseo -com.plumanalytics -crawler.feedback +Collector +Copier +CopyRightCheck +Craftbot +Crescent CSHttp -Custo +Curious DatabaseDriverMysqli -demandbase-bot -Deusu -Dirbuster -Discoverybot -DomainAppender +DBLBot +Demon +Digincore +Disco +Dispatch +DittoSpyder +DnyzBot DomainCrawler +DomainSigmaCrawler +DomainStatsBot Dotbot -Download Wonder Dragonfly -Ebingbong +Drip +ECCP/1.0 Ecxi EirGrabber -EroCrawler -Evil +EMail Siphon +evc-batch +Express WebPictures ExtLinksBot -Extractor -Extreme Picture Finder -facebookscraper +EyeNetIE FDM -Firefox/7.0 -FlashGet -Flunky -Gigabot -Go-Ahead-Got-It +FHscan +Fimap +FrontPage +Getintent +Gigablast +G-i-g-a-b-o-t GoZilla +Go!Zilla Grabber -GrapeshotCrawler +GrabNet +Grafula +GrapeFX Haansoft HaosouSpider -Havij +Harvest HEADMasterSEO -Heritrix -Hloader +HMView HTTP::Lite -HTTrack -Humanlinks +HybridBot Iblog IDBot -IndeedBot +Id-search +Image Fetch +InfoNaviRobot InfoTekies +instabid InterGET +InternetSeer internetVista monitor ips-agent -Iria -IRLbot IstellaBot -JustView -Kenjin Spider +JamesBOT +JennyBot +JetCar +Jorgee +Jyxobot Kozmosbot Larbin -LexiBot -Lftp -Linkdexbot -LinkpadBot -LinkWalker +Libwhisker +Likse +LinkextractorPro +LinkScan +LinksManager LinqiaMetadataDownloaderBot +LinqiaScrapeBot Lipperhey Litemage_walker Lmspider -Magnet -Mag-Net -magpie-crawler -Majestic12 -Majestic SEO -Majestic-SEO -Mass Downloader -Mata Hari +Ltx71 +LWP::Simple +Mail.RU_Bot +MarkWatch MeanPath Bot -mediawords -MFC_Tear_Sample +Mediatoolkitbot +MegaIndex.ru +Metauri +Microsoft Data Access Microsoft URL Control -MIDown tool MIIxpc -Morfeus Fucking Scanner -Mr.4x3 +Mister PiX +Mojeek +MSIECrawler Msrabot +muhstik-scan Name Intelligence -Nameprotect +Navroad NetAnts -netEstate NE Crawler -NetMechanic -Nettrack -Net Vampire +NetLyzer Netvibes -Nibbler -NICErsPRO +NextGenSearchBot Niki-bot -Nikto -Nimbostratus -Nmap +NimbleCrawler +Ninja NPbot -Nutch -Octopus +oBot Offline Navigator -Openfind -OpenLinkProfiler +OnCrawl OpenVAS OrangeBot -OutclicksBot -OutfoxBot -PageGrabber -page scorer -Pandalytics +OrangeSpider +PageAnalyzer +Page Analyzer Panscient Pavuk PECL::HTTP +PeoplePal PHPCrawl -Picscout -Picsearch Pi-Monster -PleaseCrawl -plumanalytics -Pockey -POE-Component-Client-HTTP ProPowerBot +ProWebWalker +PxBroker +Quick-Crawler RankActive RankFlex -RankingBot2 -Rankivabot +RankingBot +RankurBot RealDownload -RebelMouse Recorder RedesScrapy -Ripper +ReGet +RepoMonkey +SalesIntelligent +SBIder ScanAlert -Scrapy +scan.lol +ScoutJet Screaming SearchmetricsBot -SemrushBot -SEOkicks-Robot +Semrush +Seomoz +SEOprofiler seoscanners SeoSiteCheckup SEOstats @@ -174,77 +177,74 @@ serpstatbot sexsearcher Shodan Siphon -SISTRIX -Siteimprove +Sitebeam +SiteExplorer SiteSucker +Site Sucker +Sitevigil SlySearch -SmartDownload -Snake -Snoopy -SocialRankIOBot -sogouspider -Sogou web spider -SpaceBison -Spanner -Spinn3r +Sociscraper +Sosospider +Sottopop +sp_auditbot +Spbot +SputnikBot Sqlmap -Sqlworm -Steeler Stripper -Sucker +Surfbot Suzuran Swiftbot sysscan -T0PHackTeam T8Abot tAkeOut -Telesphoreo -Telesphorep +Telesoft The Intraformant +TheNomad +TightTwatBot Titan -Trendictionbot -Twice -Typhoeus -Upflow +Toata +Trendiction +TurnitinBot +TwengaBot Vacuum +Vagabondo VCI -VeriCiteCrawler Virusdie -Voil -WBSearchBot -Web Auto +VoidEYE +Voltron +WallpapersHD +Webalta +WebAuto WebBandit Web Collage -WebCopier WEBDAV -Web Enhancer -WebFuck +WebEnhancer +Web Fetch +Web Fuck WebGo IS WebImageCollector WebLeacher -WebmasterWorldForumBot -WebPix +Web Pix WebReaper WebSauger +Web Sauger +Webshag WebsiteExtractor -WebsiteQuester -Webster +Website Quester +WebStripper WebSucker +Web Sucker WebZIP -Whacker -Widow +Whatweb +Who.is Bot WiseGuys Robot -Wonderbot -Woobot Wotbox -WPScan -WWW-Collector-E -x09Mozilla -Xaldon_WebSpider -Xenu -xpymep1.exe +Wprecon +WWW::Mechanize +WWWOFFLE +x22Mozilla +Xaldon WebSpider +YoudaoBot Zade -zauba.io -Zeus +zgrab Zitebot -ZmEu diff --git a/.dev-tools/test_units/random-referrers-for-test-quick.list b/.dev-tools/test_units/random-referrers-for-test-quick.list index 08b44b088..753434f08 100644 --- a/.dev-tools/test_units/random-referrers-for-test-quick.list +++ b/.dev-tools/test_units/random-referrers-for-test-quick.list @@ -1,10 +1,10 @@ -citetick.com -distver.ru -elektrozigarette2.wordpressy.pl -fillmewithhappiness.com -freakycheats.com -lotto6888.com -mosrif.ru -safe-app.net -s-iwantyou.com -toon-families.com +azazu.ru +boobsimge.com +digilander.libero.it +facebook-mobile.xyz +himazin.info +pornodojd.ru +socialbookmarksubmission.org +taqplayer.info +theautoprofit.ml +vip-file.com diff --git a/.dev-tools/test_units/random-referrers-for-test.list b/.dev-tools/test_units/random-referrers-for-test.list index 91782ef3d..92ba3ff48 100644 --- a/.dev-tools/test_units/random-referrers-for-test.list +++ b/.dev-tools/test_units/random-referrers-for-test.list @@ -1,500 +1,500 @@ -000free.us -101flag.ru -11235813.webzdarma.cz -123kuma.com -1kdailyprofit.me -1millionusd.xyz -273-fz.ru -2pxg8bcf.top -4kepics.com -4kpics.rocks -8kisses.eu -8lufu.com -aasoldes.fr -accessoires-mode.in -activecampaign.dreamhosters.com +101raccoon.ru +11pikachu.ru +1688.com +1j7740kd.website +34.gs +40cg.com +57883.net +5tey7463.weebly.com +aab94f698f36684c5a852a2ef272e031bb.com +abcguru.xyz +abctoppictures.net +actionnooz.com +adclickthru.net addbin.men -addblueoff.com.ua -adidas.frwebs.fr -adsloads.com -adsref.men -ads-seo.men -adtech.us +addtoadd.men +adultfriendfinder.com adultgalls.com -adultmeetonline.info -adultnet.in -advancedmassagebysara.com -advokateg.xyz -ad-words.ru +adultwebhosting.info +advancedsoftwaresupport.com aerogo.com -agahinameh.com -agapovdg.ru +afftrack001.com +agroeconom.kz ahmedabadwebs.com -air-edem.ru -akusherok.ru -alibestsale.com -allnews24.in -allpdfmags.net -all-streaming-media.com -alphahoverboards.com -amateurlivechat.org -amateurmatch.com -ampower.me -animal-drawings.com -aosheng-tech.com -aquarium-pro.ru -arabsexxxtube.com -arendatora.ru -asacopaco.tk -astana.xxxkaz.org -atlasvkusov.ru -atley.eu.pn -auto4style.ru -autotrends.today -auto-zapchasti.org +albuteroli.com +allboard.xobor.de +allesohnegirls.net +aludecor.info +alveris.ru +am-se.com +antons-transporte.de +apimountainbikei-a.akamaihd.net +apiportalmorecom-a.akamaihd.net +apishops.ru +a.pr-cy.ru +arkkivoltti.net +artel116.ru +asseenontv.ru +autoplate.info +autorn.ru avitocars.ru -avon-novosib.ru -avtoarenda.by -avtochehli.by -backgroundpictures.net -bala.getenjoyment.net -banan.tv -bannerads.de -belinka.com.ua -benchmarkcommunications.co.uk -berlininsl.com -bestchoice.cf -bestempresas.es -bestsexyblog.com -beta.hotkeys.com -bezlimitko.xyz -bezsporno.ru -big-boards.info -big-cash.net -biznesluxe.ru -bluerobot.info -bmwhighperformers.com -boeuklad.com -bosman.pluto.ro -briomotor.co -bristolhostel.com -bryansk.zrus.org -buyclomidonlaine.com -buyk.host.sk -bvlgariring.xyz -callcafe.info -carsplate.com -cartierlovebraceletreplica.xyz -cartierloveringreplica.xyz -casinorewards.info -caveavins.fr -cbox.ws -cellfun.mobi -cementaresearch.se -cheapjerseysbizwholesale.us -cheapwebsitehoster.com -chelyabinsk.xrus.org -children-learningreading.info -chocoslim.pro -click2pawn.com -clipartnew.com -clksite.com -clodo.ru -companies-catalog.com -compliance-brian.top -compliance-fred.xyz -comsysnet.com -concealthyself.com -confib.ifmo.ru -cosmetic.donna7753191.ru -cosmeticswomens-womensports.rhcloud.com -creams.makeforum.eu -creativehutindia.com -customcollegeessays.net -custom-product-labels.com -cxpromote.com -cyse.tk -dandiyabeats.in -deanmoore.ie -decoratinghomes.ga -delta-line.men -den-noch24.ru +avon-ukraine.com +avtolombard-krasnodar.ru +ayakino.net +babespcs.com +babs.com.ua +balitouroffice.com +balkanfarma.ru +banki76.ru +bedcapdealers.com +betterscooter.com +bio-market.kz +bizfly.info +bjetjt.com +blobar.org +bloggedporn.com +blogrankers.com +blog.remote-computer.de +boazpower.com +boltalko.xyz +bombla.org +bongacams.com +bookmark4you.com.biz +bpro1.top +brandbucket.com +brandov.ru +brianhenry.co +buyessay3.blogspot.ru +buynorxx.com +buyparajumpers.online +buytizanidineonline.blogspot.com +cablecar.us +cand.jp +carson.getenjoyment.net +cejewelry.xyz +centrdebut.ru +c-gainsbourg.com +chatroulette.world +chinese-amezon.com +chocolateslim-en-france.com +choice-credit.ru +cialischmrx.com +ciceros.co +cigarpass.com +clash-clans.ru +claytransformations.info +clickaider.com +clickso.com +clicksotrk.com +club-samodelkin.ru +cognitiveseo.com +coldfilm.ru +co.lumb.co +compiko.info +compliance-don.top +compliance-george.xyz +compliance-john.top +compliance-willy.top +cookie-law-enforcement-hh.xyz +copypaste.traderzplanet.in +cottageofgrace.com +countbertwistdisp26.soup.io +cpabegins.ru +cpajunkies.com +cracksplanet.com +crd.clan.su +crystalslot.com +custodieva.ru +customcatchcan.com +customerguru.in +d0t.ru +dailyrank.net +danashop.ru +danceuniverse.ru +decorationspcs.com +defenderxtactical.com +dekoration.us +deniven.1bb.ru design-ideas.info -detalizaciya-tut.biz -detki-opt.ru -dev.citetick.com devochki.top -de.zapmeta.com -diesel-parts28.ru -dieta-personalna.pl +dfwu1019.info +diffbot.com +digitalfaq.com +discountliv.com +discovertreasure-a.akamaihd.net +discovertreasurenow.com divatraffic.com -divisioncore.com -dmmspy.com -documentsite.net dogoimage.com -domaincheck.io -domain-tracker.com -domination.ml -domoysshop.ru -doska-vsem.ru -downloadeer.net +dojki-hd.com +dosugrostov.site +downloader12.ru downloadkakaotalk.com -download-of-the-warez.blogspot.com -dpihatinh.gov.vn -dstroy.su -dvd-famille.com -dverimegapolis.ru +doxyporno.com +druzhbany.ru +dvervmoskvu.ru e2click.com -easytuningshop.ru -e-biznes.info +eas-seo.com +easyshoppermac.com e-commerce-seo.com -educontest.net -elektrozigaretten1.postbit.com -elmacho.xyz -empis.magix.net +ecommerce-seo.org +elektronischezi.livejournal.com +elite-sex-finders.com +elvenar.com +elvenmachine.com +empirepoker.com engines-usa.com -enternet.ee -eorogo.top -eredijovon.com -ereko.ru -erotikchat-24.com -etur.ru -everflora.ru -exonline.info -expresstoplivo.ru -extrabot.com -fakehandbags.xyz -familyphysician.ru -fashion.stellaconstance.co -femdom.twiclub.in -ferrotodo.com -fickenprivat.info -filerockstar298.weebly.com -financepoints.eu -finemanteam.com -fivedwld.cf -floridahuntingfishingadventures.com +englishdictionaryfree.com +eng-lyrics.com +eropornosex.ru +erotiktreff24.info +essay-edu.biz +euronis-free.com +european-torches.ru +eurosamodelki.ru +exchangeit.gq +ezigarettenshop1.yolasite.com +f012.de +facebook-mobile.xyz +face.hostingx.eu +farmingworm.com +fbdownloader.com +filmi-onlain.info +financeloan.us +firesub.pl +fitfloponline.store +fitness-video.net +fix-website-errors.com +floridamobilebillboards.com +flowwwers.com fodelsedagspresenter.nu -forex.osobye.ru -freesitetest.com -freetangodownload.com -freewhatsappload.com +fok.nl +foxtechfpv.com +franecki.net +free-stock-illustration.com +freewebs.com freshwallpapers.info -fuckbuddybestgilf.info -fuckmill.com -fungamelands.com -fungirlsgames.net -gabeshop.ru -gamedayassist.com -games.kolossale.ru -gamingspark.com -gaz-voshod.ru -ge0ip.net -generalporn.org -gilsonchiro.xyz -girlspicsa.com -given2.com -globetrotting-culture.ru -gloverid.site -go2jump.org -goodhousekeeping.com -gopixdatabase.com -graphics8.info -greatdealshop.com -gribkovye-zabolevaniya.com -gwhwpxbw.bloger.index.hr -gz2.bbsoldes.fr +frivgame250.com +fuckingawesome.com +gagrasector.ru +gamebackyard.com +gameonasia.com +getpopunder.com +gfaq.ru +gktt.ru +gogps.me +goldadpremium.com +goodnightjournal.com +gratuitbaise.com +greatzip.com +greenidesign.co +gsmlab.pl hackers-crackers.tk -hd-film.pl -hdhc.ru -healbio.ru -helpmymacfaster.trade -hem.passagen.se -histats.com -hitcpm.com -hitsbox.info -hmywwogw.bloger.index.hr +hao123.com +hatedriveapart.com +hcate.com +headpress.ru +health-medical-portal.info +hildinghr.se +hit-men.men +hmmm.cz +homeandhealth.ru homelygarden.com -homemature.net -hop.clickbank.net -horoshieokna.com -hotelcrocenzi.sm -howtoclean.club -https-legalrc.biz -humanelydrew.com -humbmosquina.tk -hundejo.com -idegenvezeto.eu -immobiliaremassaro.com -infogame.name -infospot.pt -insta-add.pro -internet-apteka.ru -iomoio.net +homik.pw +hotshoppymac.com +htmlcorner.com +humanorightswatch.org +hustoon.over-blog.com +hydropump.su +hyhj.info +hyipmanager.in +ibmdatamanagement.co +iklysha.ml +ilbe.club +ilovevitaly.ru +imagecoolpub.com +img.wallpaperstock.net +imperia31.ru +impisr.ru +importchinacoach-teach.com +indetiske.ya.ru +informatiecentro.be +instasexyblog.com +investingclub.ru ip-guide.com -irkutsk.zrus.org -istanbulit.com -istizanidineacontrolledsubstance.blogspot.com +iphantom.com +iqoption.pro izoll.ru -jav-fetish.site -jjbabskoe.ru -jus0wil.pp.ua -kaidalibor.de -kambasoft.com -kam-dom.ru -katadhin.co +jerseywholesaleelitestore.com +johnrobertsoninc.com +jurajskie.info kazan.zrus.org -keywordhouse.com -kino-filmi.com -knogg.net -knowsitall.info -kosova.de -kuzinsp.ru -lakomka.com.ua -laulini.soclog.se -lechenie-gemorroya.com -li-er.ru -linksharingt.com -lion.snapmap.us -liveinternet.ru -longlifelomilomi.info +keywordglobal.co.za +khafre.us +kicknights.gq +kinomaniatv.pl +kinopolet.net +kmgamex.cf +komputers-best.ru +konturkrasoty.ru +krasivye-devushki.net +labelwater.se +lacasamorett.com +landoftracking.com +larger.io +laxdrills.com +laxob.com +leboard.ru +ledpolice.ru +likesuccess.com +limads.men +littleberry.ru +live-cam6.info +live-sexcam.tk +ljusihus.se +lomb.co +lostfilm-online.ru +lotto6888.com lsp-awak-perikanan.com -lyrics.home-task.com -m0r0zk0-krava.ru -mackeeper-land-672695126.us-east-1.elb.amazonaws.com -mail.allnews24.in -makedo.ru -mall.uk -manifestation.betteroffers.review -marcoislandvacations.net -masturbate.co.uk -max-p.men -medanestesia.ru -medosmotr-ufa.ru -meetlocalchicks.com -menetie.ru -migente.com -mir-business-24.ru -missvietnam.org -mnogolok.info -mojowhois.com -monclervests.xyz -money-for-placing-articles.com -mouselink.co -moxo.com -mozello.ru -musezone.su -mwtpludn.review -my-cash-bot.co -mydeathspace.com -myftpupload.com -myhitmp3.club -mykings.pw -myonigroup.com -najaden.no -naperehresti.info -naphukete.ru -naturalpharm.com.ua -new7ob.com -new-post.tk -newsperuse.com -newyorkhotelsmotels.info -nicefloor.co.uk -nightvision746.weebly.com -nikitsyringedrivelg.pen.io -njpalletremoval.com -november-lax.com -novosibirsk.xrus.org -nrjmobile.fr -nuit-artisanale.com -nuker.com -offer.wpsecurity.website -okroshki.ru -ok.ru -ok-ua.info -onetravelguides.com -online-templatestore.com -ontargetseo.us -o-o-6-o-o.ru -openlibrary.org -openmultipleurl.com +machicon-akihabara.info +machicon-ueno.info +macrotek.ru +magicplayer-s.acestream.net +maridan.com.ua +marktforschung-stuttgart.com +marmotstore.online +martlinker.com +mature.free-websites.com +mebeldekor.com.ua +medicineseasybuy.com +mesto-x.com +metabar.ru +methodsmarketing.com +m-google.xyz +millionare.com +mindeyegames.com +minegam.com +mirmedinfo.ru +missis.top +misswell.net +misterjtbarbers.com +mlvc4zzw.space +mobilemedia.md +moesen-ficken.com +moetomnenie.com +mokrayakiska.com +monclerjacketsoutlet.pw +monclerjacketsoutlet.win +moomi-daeri.com +moyakuhnia.ru +my-bc.ru +myfreecams.com +myindospace.com +nailsimg.com +narosty.com +na-telefon.biz +nationalbreakdown.com +negociosdasha.com +networkcheck.xyz +new-apps.ru +nextlnk12.com +nzfilecloud.weebly.com +offergroup.info +olvanto.ru +olympescort.com +onescreen.cc +online7777.com +onlinemeetingnow.com +openmediasoft.com open-odyssey.org -oranga.host.sk -orel-reshka.net -orgasmus-virtual.com -orion-v.com -ostroike.org -ownshop.win +ops.picscout.com +orsonet.ru +ozoz.it pagesense.com -paleohub.info -partners-ship.pro -pb-dv.ru -pcboa.se +pammik.ru +park.above.com +partner-pop.men +pathwhelp.org +paydayloanslocal.com +pcads.ru +pcimforum.com +pdns.download +pensplan.com pepperstyle.ru -perosan.com -pflexads.com -phormchina.com +perfectpracticeweb.com +petsblogroll.com picsfair.com -picsforkeywordsuggestion.com -picturesfrom.com -picturesmania.com -pila.pl -pillscheap24h.com -pintattoos.com -plastgranar.nu -plastjulgranar.se +pirateday.ru pobeiranie.pl -pornoklad.net -portal-eu.ru -portside.xyz -pravoholding.ru -preparevideosafesystem4unow.site -pricheski-video.com -prodess.ru -professionaldieselcare.com -proftests.net -promover.org -propranolol40mg.blogspot.com -psa48.ru -psvita.ru -push-ad.com -pussyspace.net -pxhdwsm.com -questionmarque.ch -ranksonic.org -rapidsites.pro -remontbiz.ru -remont-fridge-tv.ru -rethinkwasteni.info +pokerniydom.ru +pornozhara.com +pretty-mart.com +printie.com +proekt-gaz.ru +proprostatit.com +prospekt-st.ru +psoriasis-file.trade +pussysaga.com +qpypcx.com +quality-traffic.com +quickcashlimited.com +quickchange.cc +rabotaetvse.ru +redirectme.net +r-e-f-e-r-e-r.com +registry-clean-up.net +relayblog.com +remontgruzovik.ru rewardit.com -ruclicks.com -rukino.org -ruscopybook.com -rvi.biz -rybalka-opt.ru -sad-torg.com.ua +reyel1985.webnode.fr +rff-cfal.info +rfid-locker.co +rixpix.ru +rockingclicks.com +roll123.com +royal-betting.net +rozalli.com +rutor.vip +saecsa.co +salmonfishingsacramentoriver.com sampleletters.net -sbt-aqua.ru -scanner-george.top -scansafe.net -searchimage.co -sel-hoz.com -seocdvig.ru -seocheki.net -seomarketings.online -seo-platform.com -sexstream.pl -sex-tracker.com -sextracker.de -sexvideo-sex.com -shakhtar-doneck.ru -shell-pmr.ru -shemalegalls.blogporn.in -shlyahten.ru -sibecoprom.ru -siha.de +sankt-peterburg.nodup.ru +sarafangel.ru +sa-rewards.co.za +satellite.maps.ilovevitaly.com +sayyoethe.blogspot.co.za +sbf441.com +sbwealthsolutions.ca +scanner-josh.top +scanner-julia.top +school-diplomat.ru +scmor.ilxc.cc +searchinterneat-a.akamaihd.net +secret.xn--oogle-wmc.com +sexsearch.com +sfd-chess.ru +sharebutton.org +shikiso.info +shodanhq.com +shopcheermakeup.info +sibtest.ru +signx.info sildenafilcitratemed.com sildenafil-tadalafil.info -simplepooltips.com -sims-sims.ru -sindragosa.comxa.com -sinel.info -sitevaluation.com -sitevalued.com -skuteczna-dieta.co.pl -slimcdn.com -slutloadlive.com -smadihome.com -snegozaderzhatel.ru -snow.nvr163.com -snts.shell-pmr.ru -socialmadesimple.com -socialmonkee.com -sortthemesitesby.com -spb.afora.ru -sport-video-obzor.ru -srecorder.com -stairliftsarea.com -stillmiracle.com -strag-invest.ru -streetfire.net -stretchmate.net -superstarfloraluk.com -sweepstakes.rewardit.com -tam-gde-more.ru -teesdaleflyballclub.co.uk -template-kid.com -tengohydar.tk -thaisamkok.com -thefotosgratis.eu -thegolfclub.info -theporndude.com -timhost.ru -tmtrck.com -token-lab.org -traderzplanet.in -trahvid.com -tuberkulezanet.ru -tubo360.com -turbabitload.weebly.com -tvnewsclips.info -twincitiescarservice.com -ucsol.ru -uggsale.online -uginekologa.com -uhod-za-sobakoj.ru -ultimateclassicrock.com -underthesite.com -unece.org -uptimebot.net -urldelivery.com -utrolive.ru -valmetrundan.se -venta-prom.ru -ventelnos.com -verymes.xyz -viagengrarx.com -video-chat.in -vip-file.com -vitalads.net -vmnmvzsmn.over-blog.com -voloslove.ru -vtc.pw -vykup-avto-krasnodar.ru -wapsite.me -warezkeeper.com -waterpurifier.club -webscouter.net -wedding-salon.net -welcomeauto.ru -westum.se -wetgames.ru -whatistizanidinehclusedfor.blogspot.com -winterclassichockeyjerseys.com -woman-tampon.ru -wonderfulflowers.biz -www.888.com -xn-----6kccaibs5cb8afhjrfmix2n.xn--p1ai -xn------7cdbapdecfd4ak1bn0amjffj7afu3y.xn--p1ai -xn----7sbaaabaei0cc8aj5bj0bncejx.xn--p1ai -xn--80ahdheogk5l.xn--p1ai -xolodremont.ru -x-porno.video -x-stars.ru -xxxrus.org +silvercash.com +sivs.ru +slomm.ru +smallseotools.com +smartpet.ru +sneakyboy.com +sobecjvuwa.com.ru +soda.media +softlinesolutions.me +sotechco.co +soviet-portal.do.am +spaceshipad.com +space-worry.ml +spammen.de +sprttrack.com +spywarebegone.com +spywarespy.com +srgwebmail.nl +stanthonyscatholicchurch.org +stard.shop +statistici.ro +steamoff.net +stjamesschool.info +stoki.ru +sudexpert66.ru +summerlinhomes411.info +swingerseiten.com +tasteidea.com +tattomedia.com +taximytishi.ru +taxi-v-eisk.ru +t-bygg.com +telefonsex-ohne0900.net +terraclicks.com +thedownloadfreeonlinegames.blogspot.com +thepokertimer.com +thetoiletpaper.com +thexart.club +timecrimea.ru +tizanidinecost.blogspot.com +tizanidinedruginteractions.blogspot.com +tizanidinehcl2mgtablet.blogspot.com +top10registrycleaners.com +top-deal.com.pl +torrnada.ru +toyota.7zap.com +trafficjunky.com +trafficreceiver.club +trahic.ru +trubywriting.com +trucri.me +truemfilelj.gq +trumpetedextremes.com +tvorozhnaja-zapekanka-recept.ru +u-cheats.ru +uchebavchehii.ru +ugguk.online +umg-stroy.ru +uni.me +unitexindia.com +universals.com.ua +ural-buldozer.ru +urdoot.win +urll.eu +usadacha.net +vacuumscleaner.com +validdomain.xyz +vapsy.com +vereo.eu +via-energy-cumpara.com +viagra.pp.ua +via-gra.webstarts.com +videochat.bz +videochat.world +vikistars.com +vkontarkte.com +vladhistory.com +voloomoney.com +wareseeker.com +wdfdocando.com +webenlace.com.ar +weblo.com +weboptimizes.com +webs.com +website-speed-up.site +weclipart.com +wejdz-tu.pl +whiteproduct.com +wholesalejerseysgaa.com +whos.amung.us +wikes.20fr.com +wladimirpayen.com +woweb.com.ua +www.kabbalah-red-bracelets.com +xaylapdiendanang.com +xchangetrak.com +xchat26.myfreecams.com +x-lime.com +x-lime.net +xmlinde.com +xn-----6kcaacnblni5c5bicdpcmficy.xn--p1ai +xrus.org +xxxtube69.com yeartwit.com -yebocasino.com -yellowstonevisitortours.com -yginekologa.com -yopoint.in -yottos.com -youandcredit.ru -yougetsignal.com -youjizz.com -yoursearch.me -yun56.co -zarabiaj-dzis.pl -zatjmuzu.info -zerocash.msk.ru -zog.link -zootoplist.com +ynymnwbm.bloger.index.hr +youradulthosting.com +yourtemplatefinder.com +youtubedownload.org +zaloro.com +za-music.mymobiplanet.com +zarepta.com +zolotoy-lis.ru +zoolubimets.ru +zoosexart.com +zryydi.com +zscalertwo.net +zynax.ua +zytpirwai.net diff --git a/.dev-tools/test_units/random-referrers-for-whitelist-test.list b/.dev-tools/test_units/random-referrers-for-whitelist-test.list index d55b52151..e8686b4dd 100644 --- a/.dev-tools/test_units/random-referrers-for-whitelist-test.list +++ b/.dev-tools/test_units/random-referrers-for-whitelist-test.list @@ -1,250 +1,250 @@ -01apple.com -178evakuator178.ru +00webcams.com +123kuma.com +13tabs.com +1bet.com 1pamm.ru -4webmasters.org -8kisses.eu -8xv8.com -a1security.com.ua -aa81bf391151884adfa3dd677e41f94be1.com -abcdefh.xyz -abcguru.xyz -accessoires-mode.in -adultactioncam.com -adultmeetonline.info -advancedsoftwaresupport.com -adviceforum.info -advocatemsk.ru -advokateg.xyz -advokat-grodno.by +2ads.co.uk +3w24.com +5i2.net +888.com +8jn.dating-app.ru +9med.net +abcdeg.xyz +academiacsmendoza.org +addray.pro +ahrefs.com alarmobninsk.ru -alert.scansafe.net -alldezservice.kz -alldownload.pw -alt-servis.ru -apartamentwroclaw.eu -apartmentbay.ru +alessandraleone.com +alfa-img.com +alphacarolinas.com +angigreene.com +animalrank.com +anonymous-redirect.com +apparel-offer.com a.pr-cy.ru -artavenuegardenstudios.com -audiobangout.com -ayerbo.xhost.ro -bambi.ck.ua -begalka.xyz -bestcurs.org -bikini-image.com -buron.pw -buyessay3.blogspot.ru -carloans.com -carmuffler.net -cartierlovestore.com -chatseo.com -cheapjerseysa.com -cheapmoncler.xyz -checkm8.com -chee-by.biz -chiptuninger.com -cineacademy.ru -club-samodelkin.ru -cocyq.inwtrade.com -compliance-julia.top -corta.co -cubs-jerseys.us -czeshop.info -day-news.info -directivepub.com -dktr.ru -donvito.unas.cz -downloaddy.net -driving.kiev.ua -elektrischeziga.livejournal.com -enhand.se -eugenevaultstorage.com -ewebarticle.info -expertnaya-ocenka.ru -ezb.elvenmachine.com -ezigarettenkaufen1.deviantart.com -facecup.top -falopicm.pw -fast-wordpress-start.com +asophoto.com +autoblogger24.info +avkzarabotok.com +avkzarabotok.info +banksrf.ru +bestkfiledxd.cf +besttorrentknifta.weebly.com +bet365.com +betterhdporn.com +biketank.ga +bimatoprost-careprost.com.ua +biznesrealnost.ru +bjorkbacken.nu +bkns.vn +blog.koreadaily.com +blogsfunky672.weebly.com +bmw.afora.ru +bonzibuddi.com +boole.onlinewebshop.net +braintobe.win +brokergid.ru +buckspar.gq +buqyxa.rincian.info +buyantiviralwp.com +buydissertation.net +cartierloveringreplica.xyz +cartierreplica.win +cdnanalytics.xyz +cerev.info +cete.ru +cheapmoncler.pw +cheapsergiorossi.online +cherrypointplace.ca +chinacheapelitejerseys.com +chinaelitecheapjerseys.com +cjs.com.ru +coaching-netz.info +community.allhiphop.com +compliance-alex.xyz +compliance-jess.top +compliance-margo.top +craftburg.ru +crystalslot.com +cscwtalkto.site +cs-passion.pl +daptravel.com +davidovic.info +ddsoldes.fr +discountliv.com +domaincheck.io +educontest.net +elektrozigaretten1.tumblr.com +energydiet-info.ru +espaceinventoristes.com +event-tracking.com +executehosting.com +extlinks.com +f07.de +fetroshok.ru +filmi-onlain.info financehint.eu -financepoints.eu -flashlarevista.com -floating-share-buttons.com -footballfarrago.com -forum-engineering.ru -foxjuegos.com -frcls.fr -freejabs.com -fuel-gas.com -fx-brokers-review.com -gamedayhouse.com -gameplexcity.com -gateway.zscalerone.net -gazobeton-p.com.ua -gerhardhealer.com -gettpromos.com -ghazel.ru -glall.ru -globetrotting-culture.ru -gloverid.site -good-mummy.ru -goodprotein.ru -gopixdatabase.com -gsasearchengineranker.site -handicapbathtubarea.com -happy.new.yeartwit.com -hard-porn.mobi -homik.pw -htmlcorner.com -hunterboots.online -ikearugs.xyz -imadedinner.net -infokonkurs.ru -internetproviderstucson.com -investzalog.ru -iomoio.net -islandminingsupply.wordpress.com -jav-fetish.site -jaxcube.info -j-times.ru +finstroy.net +firesub.pl +fitnesspiks.com +forensicpsychiatry.ru +free-social-buttons.com +g33.org +gallerylisting.com +getpopunder.com +girls-ufa.ru +gta-top.ru +hacktougroup.ru +hkdiiohi.skyrock.com +houseofrose.com +huimin764128.com +imagine-ex.co +inboxdollars.com +inet-traffic.com +isistaylorporn.info +itronics.ca +jerseyswholesalechinalimited.com +jewelryandfiligree.com +jjbabskoe.ru juliadiets.com -kamen-e.ru -kareliatobacco.ru -keki.info -kenaba.su -ketrzyn.pl -keywordteam.net -kinoplen.ru -konpax.com -koptims.tiu.ru -kosynka-games.ru -kw21.org -la-fa.ru -lampokrat.ws -leboard.ru -levaquin750mg.blogspot.com -loadopia.com +jwss.cc +keywordsking.com +khafre.us +kimcurlrvsms.com +kinosed.net +konoplisemena.com +kutikomi.net +laexotic.com +law-check-two.xyz +lesbian.xyz +letolove.ru +loadingpages.me +locustdesign.co +lottospring.com +m0r0zk0-krava.ru +machicon-ueno.info macotool.com -magda-gadalka.ru -magicdiet.gq -mailemedicinals.com -malls.com -marblestyle.ru -maxxtor.eu -medicines-choice.com -medtherapy.ru -merryhouse.co.uk -middlerush-a.akamaihd.net -minecraft-rus.org -mnogolok.info -monetizer.com-01.site -morlat.altervista.org -mundoaberrante.com -musicvidz.ru -mykings.pw -nakozhe.com -naturtreenspicerx.pw -netfacet.net -nikitsyringedrivelg.pen.io -njpalletremoval.com -notebook-pro.ru -nullrefer.com -odesproperty.com -omoikiri-japan.ru -omsk.xrus.org -onlinemegax.com -opinionreelle.com -outpersonals.com -paparazzistudios.com.au -parfusale.se -partnerworkroom.men -petitions.whitehouse.gov -pictures-and-images.net -platesauto.com -pogosh.com -pogruztehnik.ru -powenlite24.ru -pozdravleniya-c.ru +maglid.ru +malwareremovalcenter.com +manuscript.su +marcowebonyodziezowe.pl +marmotstore.online +missis.top +mmostrike.ru +mobplayer.ru +mole.pluto.ro +mouselink.co +myhealthcare.com +myphotopipe.com +narkologiya-belgorod.ru +natali-forex.com +naturalpharm.com.ua +newyorkhotelsmotels.info +ntic.fr +odywpjtw.bloger.index.hr +office2web.com +okonich.com.ua +onclickpredictiv.com +oneclickfiles.com +onlinemeetingnow.com +osb.se11.ru +outshop.ru +owathemes.com +pavlodar.xkaz.org +pb-dv.ru +pdamods.ru +perl.dp.ua +picurams.pw +pizza-imperia.com +pl-top.pl +pornokajf.com pozdrawleniya.ru -praisong.net -pulseonclick.com -randalljhoward.com -relax.ru -remorcicomerciale.ro -rezeptiblud.ru -richinvestmonitor.com -ring4rhino.com -rutor.vip -sabizonline.com -sanatorrii.ru -saugatuck.com -sbt-aqua.ru -scanner-willy.top -scmor.ilxc.cc +prefersurvey.net +printingpeach.com +privetsochi.ru +prointer.net.ua +propranolol40mg.blogspot.com +prosperent.com +prostitutki-tyumeni.org +puteshestvennik.com +rankingchart.de +rapidgator-porn.ga +realitykings.com +rednise.com +responsinator.com +rocketchange.ru +rockma.se +roma-kukareku.livejournal.com +sady-urala.ru +saratov.xrus.org +satellite.maps.ilovevitaly.com +scanner-julia.top +scanner-viktor.top +scenicmissouri.us +sdelai-prosto.ru +searchimage.co selectads.men -seofied.com -seorankinglinks.xyz -seo-tools-optimizing.com -sergiorossistore.online -sex-pr.net -sharpchallenge.com -shivafurnishings.com +sexkontakteao.info +shitmovs.com shodanhq.com -shymkent.xkaz.org -sildenafilcitratemed.com -slimcdn.com -slowmac.tech -sluganarodu.ru -smokewithrabbits.com -snegozaderzhatel.ru -social-buttons.xyz -sosdepotdebilan.com -souvenir.cc -spamnuker.com -spb-plitka.ru -spidtest.space -sportbetfair.com -spywarebegone.com -sribno.net -stpolice.com -studiofmp.com -sugarkun.com -superlist.biz -tam-gde-more.ru -tattooha.com -telefonsex-ohne0900.net -thecoral.com.br -thewomenlife.com -tizanidinedrugtest.blogspot.com -tongkatmadura.info -topcar-krasnodar.ru -track112.site -tradedeals.biz -traffic-club.info -traffictrade.life -try-rx.com -tsstcorpcddvdwshbbdriverfb.aircus.com -tsyndicate.com -tuberkulezanet.ru -ukkelberg.no -ultimatesetnewfreeallsoftupgradesystems.pw -unpredictable.ga -urccvfmc.bloger.index.hr -uytmaster.ru -vacances-voyages.info -valoresito.com -venta-prom.ru -via-energy-cumpara.com -vk-mus.ru -wallpaperstock.net -wechatdownload10.com -wholesalecheapjerseysfree.com -w-journal.ru -wosik-dach.service-for-web.de -xev.ru -xn----7sbaphztdjeboffeiof6c.xn--p1ai -xn--80aodinpgi.xn--p1ai -xxxrus.org -xxx-treker.ru -yellowads.men -zarabotok--doma.ru -zeroredirect11.com +sicfor.bcu.cc +sierraapps.com +signoredom.com +snjack.info +solicita.info +solinf.co +sptslmtrafms.com +stal-rulon.ru +stjamesschool.info +strv.se +sundrugstore.com +surffoundation.nl +swagbucks.com +swinginwithme.ru +tedy.su +telefonsexi.com +thedownloadfromwarez.blogspot.com +theporndude.com +tiandeural.ru +tiens2010.ru +tizanidine6mg.blogspot.com +tizanidinehcl4mgtab.blogspot.com +tomatis.gospartner.com +toon-families.com +toyota.7zap.com +tozup.com +tracfone.com +trumpetedextremes.com +tuckpointingmasonrysystems.com +uac.net.au +uasb.ru +ua.tc +u-cheats.ru +uggbootsoutletsale.us +umekana.ru +underthesite.com +unitygame3d.com +universals.com.ua +upupa.net +uvozdeckych.info +valueclick.com +veselokloun.ru +videochat.life +videooko.weebly.com +vodaodessa.com +vut.com.ru +vzube.com +webnode.me +withstandingheartwarming.com +word-vorlagen.net +x-diesel.biz +xn-----6kcaabbafhu7cskl7akvongwpo7hvjj.xn--p1ai +xn-----7kceclhb4abre1b4a0ccl2fxch1a.xn--p1ai +xn----7sbaaabaei0cc8aj5bj0bncejx.xn--p1ai +xn--lifehacer-1rb.com +xxxdatinglocal.us +yaoguangdj.com +youtoner.it +youtube-downloader.savetubevideo.com +zakazvzloma.com +zigarettenonlinekaufen1.bloog.pl +zkjovpdgxivg.ga diff --git a/.dev-tools/test_units/ratelimittest-beta.txt b/.dev-tools/test_units/ratelimittest-beta.txt index f371d9e5f..525a76fd6 100644 --- a/.dev-tools/test_units/ratelimittest-beta.txt +++ b/.dev-tools/test_units/ratelimittest-beta.txt @@ -1,3 +1,24 @@ + +
This is merely used for testing !!!
- -