diff --git a/.dev-tools/beta-test-blocker-false-positives.sh b/.dev-tools/beta-test-blocker-false-positives.sh old mode 100644 new mode 100755 diff --git a/.dev-tools/beta_conf_files/conf.d/globalblacklist.conf b/.dev-tools/beta_conf_files/conf.d/globalblacklist.conf index 65f28ba04..e0433aa10 100644 --- a/.dev-tools/beta_conf_files/conf.d/globalblacklist.conf +++ b/.dev-tools/beta_conf_files/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2019.07.1705 -### Updated: Mon Jul 1 08:42:46 SAST 2019 +### Version: V3.2019.07.1706 +### Updated: Mon Jul 1 09:31:50 SAST 2019 ### Bad Referrer Count: 6717 ### Bad Bot Count: 556 ################################################### diff --git a/.dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf b/.dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf index 12f18b91b..0fcb73d00 100644 --- a/.dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf +++ b/.dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -35.202.145.110 1; +35.192.187.174 1; 127.0.0.1 1; diff --git a/.dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf b/.dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf index c3c284efc..3dac4f84e 100644 --- a/.dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf +++ b/.dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -35.202.145.110 0; +35.192.187.174 0; 127.0.0.1 0; diff --git a/.dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf b/.dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf index 65f28ba04..e0433aa10 100644 --- a/.dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf +++ b/.dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2019.07.1705 -### Updated: Mon Jul 1 08:42:46 SAST 2019 +### Version: V3.2019.07.1706 +### Updated: Mon Jul 1 09:31:50 SAST 2019 ### Bad Referrer Count: 6717 ### Bad Bot Count: 556 ################################################### diff --git a/.dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf b/.dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf index 12f18b91b..0fcb73d00 100644 --- a/.dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf +++ b/.dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -35.202.145.110 1; +35.192.187.174 1; 127.0.0.1 1; diff --git a/.dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf b/.dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf index c3c284efc..3dac4f84e 100644 --- a/.dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf +++ b/.dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -35.202.145.110 0; +35.192.187.174 0; 127.0.0.1 0; diff --git a/.dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf b/.dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf index 65f28ba04..e0433aa10 100644 --- a/.dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf +++ b/.dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2019.07.1705 -### Updated: Mon Jul 1 08:42:46 SAST 2019 +### Version: V3.2019.07.1706 +### Updated: Mon Jul 1 09:31:50 SAST 2019 ### Bad Referrer Count: 6717 ### Bad Bot Count: 556 ################################################### diff --git a/.dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf b/.dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf index 12f18b91b..0fcb73d00 100644 --- a/.dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf +++ b/.dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -35.202.145.110 1; +35.192.187.174 1; 127.0.0.1 1; diff --git a/.dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf b/.dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf index c3c284efc..3dac4f84e 100644 --- a/.dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf +++ b/.dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -35.202.145.110 0; +35.192.187.174 0; 127.0.0.1 0; diff --git a/.dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf b/.dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf index 65f28ba04..e0433aa10 100644 --- a/.dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf +++ b/.dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2019.07.1705 -### Updated: Mon Jul 1 08:42:46 SAST 2019 +### Version: V3.2019.07.1706 +### Updated: Mon Jul 1 09:31:50 SAST 2019 ### Bad Referrer Count: 6717 ### Bad Bot Count: 556 ################################################### diff --git a/.dev-tools/globalblacklist-testing.template b/.dev-tools/globalblacklist-testing.template index 65f28ba04..e0433aa10 100644 --- a/.dev-tools/globalblacklist-testing.template +++ b/.dev-tools/globalblacklist-testing.template @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2019.07.1705 -### Updated: Mon Jul 1 08:42:46 SAST 2019 +### Version: V3.2019.07.1706 +### Updated: Mon Jul 1 09:31:50 SAST 2019 ### Bad Referrer Count: 6717 ### Bad Bot Count: 556 ################################################### diff --git a/.dev-tools/globalblacklist.template b/.dev-tools/globalblacklist.template index 5bddc3b6e..94c91beec 100755 --- a/.dev-tools/globalblacklist.template +++ b/.dev-tools/globalblacklist.template @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2019.07.1705 -### Updated: Mon Jul 1 08:42:45 SAST 2019 +### Version: V4.2019.07.1706 +### Updated: Mon Jul 1 09:31:48 SAST 2019 ### Bad Referrer Count: 6717 ### Bad Bot Count: 556 ################################################### diff --git a/.dev-tools/test1_conf_files/bots.d/whitelist-ips.conf b/.dev-tools/test1_conf_files/bots.d/whitelist-ips.conf index deb2992bb..6073098c1 100644 --- a/.dev-tools/test1_conf_files/bots.d/whitelist-ips.conf +++ b/.dev-tools/test1_conf_files/bots.d/whitelist-ips.conf @@ -45,4 +45,4 @@ # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST -35.202.145.110 0; +34.66.178.120 0; diff --git a/.dev-tools/test1_conf_files/conf.d/globalblacklist.conf b/.dev-tools/test1_conf_files/conf.d/globalblacklist.conf index 5bddc3b6e..94c91beec 100644 --- a/.dev-tools/test1_conf_files/conf.d/globalblacklist.conf +++ b/.dev-tools/test1_conf_files/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2019.07.1705 -### Updated: Mon Jul 1 08:42:45 SAST 2019 +### Version: V4.2019.07.1706 +### Updated: Mon Jul 1 09:31:48 SAST 2019 ### Bad Referrer Count: 6717 ### Bad Bot Count: 556 ################################################### diff --git a/.dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf b/.dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf index 12f18b91b..0f27f0dd9 100644 --- a/.dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf +++ b/.dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -35.202.145.110 1; +104.154.113.151 1; 127.0.0.1 1; diff --git a/.dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf b/.dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf index c3c284efc..0e0848852 100644 --- a/.dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf +++ b/.dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -35.202.145.110 0; +104.154.113.151 0; 127.0.0.1 0; diff --git a/.dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf b/.dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf index 5bddc3b6e..94c91beec 100644 --- a/.dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf +++ b/.dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2019.07.1705 -### Updated: Mon Jul 1 08:42:45 SAST 2019 +### Version: V4.2019.07.1706 +### Updated: Mon Jul 1 09:31:48 SAST 2019 ### Bad Referrer Count: 6717 ### Bad Bot Count: 556 ################################################### diff --git a/.dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf b/.dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf index 12f18b91b..0f27f0dd9 100644 --- a/.dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf +++ b/.dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -35.202.145.110 1; +104.154.113.151 1; 127.0.0.1 1; diff --git a/.dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf b/.dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf index c3c284efc..0e0848852 100644 --- a/.dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf +++ b/.dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -35.202.145.110 0; +104.154.113.151 0; 127.0.0.1 0; diff --git a/.dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf b/.dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf index 5bddc3b6e..94c91beec 100644 --- a/.dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf +++ b/.dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2019.07.1705 -### Updated: Mon Jul 1 08:42:45 SAST 2019 +### Version: V4.2019.07.1706 +### Updated: Mon Jul 1 09:31:48 SAST 2019 ### Bad Referrer Count: 6717 ### Bad Bot Count: 556 ################################################### diff --git a/.dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf b/.dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf index 12f18b91b..0f27f0dd9 100644 --- a/.dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf +++ b/.dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -35.202.145.110 1; +104.154.113.151 1; 127.0.0.1 1; diff --git a/.dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf b/.dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf index c3c284efc..0e0848852 100644 --- a/.dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf +++ b/.dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -35.202.145.110 0; +104.154.113.151 0; 127.0.0.1 0; diff --git a/.dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf b/.dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf index 5bddc3b6e..94c91beec 100644 --- a/.dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf +++ b/.dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2019.07.1705 -### Updated: Mon Jul 1 08:42:45 SAST 2019 +### Version: V4.2019.07.1706 +### Updated: Mon Jul 1 09:31:48 SAST 2019 ### Bad Referrer Count: 6717 ### Bad Bot Count: 556 ################################################### diff --git a/.dev-tools/test2_conf_files/bots.d/whitelist-ips.conf b/.dev-tools/test2_conf_files/bots.d/whitelist-ips.conf index 6ff35261d..27c6e6297 100644 --- a/.dev-tools/test2_conf_files/bots.d/whitelist-ips.conf +++ b/.dev-tools/test2_conf_files/bots.d/whitelist-ips.conf @@ -45,5 +45,5 @@ # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST -35.192.85.2 0; -35.193.7.13 0; +104.198.131.58 0; +35.192.136.167 0; diff --git a/.dev-tools/test2_conf_files/conf.d/globalblacklist.conf b/.dev-tools/test2_conf_files/conf.d/globalblacklist.conf index 85e523c84..e23b06a87 100644 --- a/.dev-tools/test2_conf_files/conf.d/globalblacklist.conf +++ b/.dev-tools/test2_conf_files/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2019.07.1705 -### Updated: Mon Jul 1 08:42:45 SAST 2019 +### Version: V4.2019.07.1706 +### Updated: Mon Jul 1 09:31:48 SAST 2019 ### Bad Referrer Count: 6717 ### Bad Bot Count: 556 ################################################### diff --git a/.dev-tools/test4_conf_files/bots.d/whitelist-ips.conf b/.dev-tools/test4_conf_files/bots.d/whitelist-ips.conf index ecdc0d878..9496d4553 100644 --- a/.dev-tools/test4_conf_files/bots.d/whitelist-ips.conf +++ b/.dev-tools/test4_conf_files/bots.d/whitelist-ips.conf @@ -45,6 +45,5 @@ # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST -35.192.85.2 0; -35.224.112.202 0; -35.184.96.71 0; +34.66.178.120 0; +35.192.187.174 0; diff --git a/.dev-tools/test4_conf_files/conf.d/globalblacklist.conf b/.dev-tools/test4_conf_files/conf.d/globalblacklist.conf index 5bddc3b6e..94c91beec 100644 --- a/.dev-tools/test4_conf_files/conf.d/globalblacklist.conf +++ b/.dev-tools/test4_conf_files/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2019.07.1705 -### Updated: Mon Jul 1 08:42:45 SAST 2019 +### Version: V4.2019.07.1706 +### Updated: Mon Jul 1 09:31:48 SAST 2019 ### Bad Referrer Count: 6717 ### Bad Bot Count: 556 ################################################### diff --git a/.dev-tools/test5_conf_files/bots.d/whitelist-ips.conf b/.dev-tools/test5_conf_files/bots.d/whitelist-ips.conf index 16f37cf31..38e52c511 100644 --- a/.dev-tools/test5_conf_files/bots.d/whitelist-ips.conf +++ b/.dev-tools/test5_conf_files/bots.d/whitelist-ips.conf @@ -45,5 +45,6 @@ # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST -35.193.7.13 0; -35.192.85.2 0; +35.192.187.174 0; +35.193.14.140 0; +34.68.144.114 0; diff --git a/.dev-tools/test5_conf_files/conf.d/globalblacklist.conf b/.dev-tools/test5_conf_files/conf.d/globalblacklist.conf index 5bddc3b6e..94c91beec 100644 --- a/.dev-tools/test5_conf_files/conf.d/globalblacklist.conf +++ b/.dev-tools/test5_conf_files/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2019.07.1705 -### Updated: Mon Jul 1 08:42:45 SAST 2019 +### Version: V4.2019.07.1706 +### Updated: Mon Jul 1 09:31:48 SAST 2019 ### Bad Referrer Count: 6717 ### Bad Bot Count: 556 ################################################### diff --git a/.dev-tools/test_units/blacklist-ips.conf b/.dev-tools/test_units/blacklist-ips.conf index 12f18b91b..0fcb73d00 100644 --- a/.dev-tools/test_units/blacklist-ips.conf +++ b/.dev-tools/test_units/blacklist-ips.conf @@ -1,2 +1,2 @@ -35.202.145.110 1; +35.192.187.174 1; 127.0.0.1 1; diff --git a/.dev-tools/test_units/random-bots-for-test-quick.list b/.dev-tools/test_units/random-bots-for-test-quick.list index 9c485c1fa..3cd6c3c7f 100644 --- a/.dev-tools/test_units/random-bots-for-test-quick.list +++ b/.dev-tools/test_units/random-bots-for-test-quick.list @@ -1,10 +1,10 @@ -Anarchie -Barkrowler -Brandwatch -Copier -DomainAppender -FrontPage -RealDownload -SocialRankIOBot -Webalta -Xenu +DomainCrawler +Extractor +IlseBot +Iskanie +Mediatoolkitbot +Microsoft Data Access +Spinn3r +The Intraformant +VoidEYE +WebBandit diff --git a/.dev-tools/test_units/random-bots-for-test.list b/.dev-tools/test_units/random-bots-for-test.list index 6ee370cbd..6cd8cb470 100644 --- a/.dev-tools/test_units/random-bots-for-test.list +++ b/.dev-tools/test_units/random-bots-for-test.list @@ -1,250 +1,250 @@ 404enemy 80legs +Abonti Acunetix ADmantX -AfD-Verbotsverfahren +AIBOT +AiHitBot Aipbot Alligator AllSubmitter AlphaBot Anarchie -Apexoo -Attach -backlink-check -BacklinkCrawler -BackStreet -Battleztar Bazinga +ASPSeek +Asterias +Badass +Bandit +Barkrowler BBBike -BDCbot -BetaBot -Blackboard -Black Hole +Bigfoot +Bitacle BlackWidow BLEXBot -Brandprotect -BunnySlippers -BuzzSumo +BlowFish +Boardreader +Brandwatch +Buddy +BuiltBotTough +Bullseye CATExplorador -CazoodleBot CCBot Cegbfeieh -CheeseBot CherryPicker -Claritybot -Cloud mapping +CheTeam +cognitiveseo Collector com.plumanalytics -crawler4j -crawl.sogou.com +Copier +CopyRightCheck +Cosmos +Craftbot CrazyWebCrawler +CrunchBot +CSHttp +DatabaseDriverMysqli DataCha0s -Demon -Deusu -Digincore +DBLBot +Devil DIIbot -Dirbuster -Disco -Discoverybot -DomainAppender +Dispatch +DomainCrawler DomainSigmaCrawler -DomainStatsBot -Dragonfly -DSearch -EasyDL -eCatch -Ecxi +Dotbot EirGrabber -EMail Siphon +EroCrawler evc-batch -Evil +Exabot Express WebPictures ExtLinksBot Extractor -ExtractorPro +Extreme Picture Finder +EyeNetIE Ezooms -FemtosearchBot +FHscan Fimap Firefox/7.0 FlashGet Flunky Foobot +FrontPage FyberSpider -Genieo +Fyrebot +GalaxyBot GermCrawler -Getintent -Gigablast -G-i-g-a-b-o-t +GetRight Gotit -GoZilla Go!Zilla +Grabber GrabNet -Grafula -GT::WWW +GrapeshotCrawler +GridBot HaosouSpider -HEADMasterSEO -HTTP::Lite -HTTrack +Heritrix +Hloader +Humanlinks Iblog IlseBot -Indy Library +Image Sucker +IndeedBot InfoNaviRobot -instabid -Intelliseek +InfoTekies InterGET Internet Ninja -InternetSeer -Iria +internetVista monitor +ips-agent IRLbot Iskanie IstellaBot +JetCar JikeSpider -JOC Web Spider -Lanshanbot +Joomla +Jorgee +JustView +Kenjin Spider LeechFTP -Libwhisker +LeechGet +Lftp +LibWeb +Lightspeedsystems Likse -LinkextractorPro -LinkpadBot -LinkScan LinksManager -LinqiaScrapeBot -Lmspider +Lipperhey +Litemage_walker LNSpiderguy +Ltx71 lwp-request -Magnet +LWP::Simple +lwp-trivial Mag-Net magpie-crawler Majestic12 -Majestic SEO +Majestic-SEO +MarkWatch +Masscan Mata Hari -meanpathbot -Meanpathbot -Metauri -Microsoft URL Control +MeanPath Bot +Mediatoolkitbot +Microsoft Data Access MIIxpc Mister PiX Mojeek Morfeus Fucking Scanner Mr.4x3 MSFrontPage +MSIECrawler Msrabot -Navroad -Needle -NetAnts +NearSite +Nessus Netcraft netEstate NE Crawler -NetLyzer -NetZIP NextGenSearchBot +Nibbler NICErsPRO -Nikto -NimbleCrawler -Ninja -NPbot -OnCrawl +Niki-bot +Nutch +Octopus +Offline Explorer +Offline Navigator Openfind -Openvas +OpenLinkProfiler OpenVAS OrangeBot OutclicksBot +OutfoxBot +PageAnalyzer Page Analyzer PageGrabber page scorer +Pandalytics +Panscient +pcBrowser PeoplePal -PHPCrawl -Picscout Picsearch -Pimonster -Pixray -POE-Component-Client-HTTP -ProPowerBot +plumanalytics +Pockey +Probethenet Pump -PxBroker -PyCurl QueryN Metasearch RankActive RankActiveLinkBot +RankFlex Rankivabot -RankurBot Reaper +RebelMouse Recorder +RedesScrapy ReGet Ripper -Rogerbot s1z.ru -ScanAlert -Scanbot -scan.lol -SEOlyticsCrawler +SalesIntelligent +SBIder +Scrapy +Screaming +ScreenerBot +Seomoz +SEOprofiler seoscanners SEOstats -sexsearcher -SiteLockSpider -SiteSucker +Siteimprove Site Sucker -SlySearch -SmartDownload -Sociscraper +Sitevigil +Snake +Snapbot sogouspider -Sogou web spider -Sosospider +Sottopop Spammen -Spinn3r -Sqlmap +sp_auditbot +Sqlworm Sqworm Steeler Stripper Sucker +Sucuri SuperBot -SurveyBot -Swiftbot -sysscan Szukacz +tAkeOut TeleportPro -Telesoft Telesphorep -TightTwatBot -Tracemyfile -Trendiction +The Intraformant +Titan +Toata +Toweyabot +trendiction.de Turnitin TurnitinBot -TwengaBot Twice -Typhoeus +Upflow +URLy Warning Vacuum -Vagabondo VB Project +VCI +Virusdie +Voil Voltron -WASALive-Bot -Webalta +Wallpapers/3.0 +WallpapersHD +WebBandit WebCollage Web Collage -WebFuck -Web Fuck -WebGo IS -WebImageCollector -WebLeacher -WebmasterWorldForumBot +WEBDAV +Web Fetch +webmeup-crawler +WebReaper Web Sauger -Webshag -Website Quester -Webster -WebSucker -WebZIP -Whack +WebsiteQuester +WeSEE +Whacker Whatweb -Widow -WinHTTrack +WISENutbot +Wotbox Wprecon -WPScan -WWW::Mechanize -x09Mozilla +WWWOFFLE x22Mozilla -Xaldon_WebSpider -Xenu -Zauba -zauba.io +xpymep1.exe +YoudaoBot +Zade Zermelo +Zeus Zitebot ZmEu -ZyBorg +ZumBot diff --git a/.dev-tools/test_units/random-bots-for-whitelist-test.list b/.dev-tools/test_units/random-bots-for-whitelist-test.list index 59cfc2d69..25d5351db 100644 --- a/.dev-tools/test_units/random-bots-for-whitelist-test.list +++ b/.dev-tools/test_units/random-bots-for-whitelist-test.list @@ -1,250 +1,250 @@ -360Spider -404checker -404enemy +80legs Abonti -Acunetix -ADmantX -AiHitBot +Aboundex +Aboundexbot +AfD-Verbotsverfahren Alexibot AllSubmitter -AlphaBot -Anarchie -Apexoo ASPSeek -autoemailspider BackDoorBot -Backlink-Ceck backlink-check -BacklinkCrawler -Bandit -BDCbot +BackStreet +BatchFTP +Battleztar Bazinga +BBBike +BetaBot Bigfoot -Bitacle +Blackboard +Black Hole +Boardreader +BotALot +Brandprotect +Brandwatch Buddy BuiltWith +Bullseye BunnySlippers -BuzzSumo -Calculon -CCBot -ChinaClaw -Chlooe -Claritybot -Cliqzbot -coccocbot-web +CazoodleBot +CheeseBot +CherryPicker +CheTeam +Cloud mapping +Cogentbot com.plumanalytics Copier -Cosmos +Copyscape Craftbot crawler.feedback crawl.sogou.com CrazyWebCrawler Crescent CrunchBot -Curious -DataCha0s -DBLBot -Demon -Deusu -Devil -Digincore -Dirbuster -Discobot -Dispatch -DnyzBot +CSHttp +Custo +DigitalPebble +DIIbot +Discoverybot +DittoSpyder DomainAppender DomainCrawler +DomainSigmaCrawler Dotbot Download Wonder -Dragonfly Drip DSearch DTS Agent -eCatch -ECCP/1.0 +EasyDL +Ecxi +EirGrabber EMail Siphon -Exabot -ExtractorPro -Extreme Picture Finder +EroCrawler +evc-batch EyeNetIE +Ezooms FemtosearchBot -Firefox/7.0 -Flunky -Foobot FrontPage FyberSpider -GalaxyBot +Fyrebot Genieo -GetRight -GetWeb +GermCrawler +Gigablast Gigabot -G-i-g-a-b-o-t +Go-Ahead-Got-It GoZilla -Grabber Grafula GrapeFX -GrapeshotCrawler +GridBot GT::WWW Haansoft -HaosouSpider -Havij -HTMLparser -Humanlinks -HybridBot -IDBot -Indy Library -Internet Ninja -Iria +Harvest +HEADMasterSEO +Heritrix +HMView +HTTP::Lite +Id-search +IlseBot +Image Sucker +InfoNaviRobot +InfoTekies +instabid +internetVista monitor +ips-agent IRLbot JamesBOT JennyBot +JetCar Jetty JikeSpider +JOC Web Spider Joomla -JustView +Jorgee +Kenjin Spider +Kozmosbot Lanshanbot -LexiBot +LeechFTP +Lftp +LibWeb +Libwhisker Lightspeedsystems Linkdexbot LinkextractorPro +LinkpadBot LinksManager -LinkWalker -LinqiaMetadataDownloaderBot +LinqiaRSSBot LinqiaScrapeBot Lipperhey Spider -lwp-request -lwp-trivial -Mail.RU_Bot -Majestic12 -MarkWatch -Masscan +Mag-Net +magpie-crawler +Majestic SEO +MarkMonitor Mata Hari +Meanpathbot MeanPath Bot -Mediatoolkitbot Metauri Microsoft Data Access -MIDown tool -MIIxpc -Mojeek -Morfeus Fucking Scanner -MS Web Services Client Protocol -muhstik-scan -Musobot -Nameprotect -NearSite -Needle -Nessus -Netcraft -netEstate NE Crawler -NetSpider +Microsoft URL Control +MSIECrawler +NetAnts Nettrack -Net Vampire -NetZIP -NICErsPRO -NimbleCrawler +NextGenSearchBot +Niki-bot Ninja -Nmap -oBot -Octopus -Offline Explorer -Offline Navigator -OpenVAS +NPbot +Nutch PageAnalyzer -PageGrabber page scorer PageScorer -Pavuk -PECL::HTTP +Pandalytics +Panscient +Papa Foto PHPCrawl Picscout -Picsearch -Pi-Monster +PictureFinder +Pixray PleaseCrawl plumanalytics +POE-Component-Client-HTTP Probethenet -ProPowerBot -Pump +ProWebWalker PxBroker -QueryN Metasearch +Quick-Crawler RankActive +RankFlex RankingBot +RankingBot2 RealDownload Reaper -RebelMouse -ReGet -Ripper +Recorder +RepoMonkey RocketCrawler +s1z.ru +SalesIntelligent ScanAlert -ScoutJet +Scanbot Scrapy -SearchmetricsBot -SemrushBot +Semrush SEOkicks -Seomoz +seoscanners +SEOstats sexsearcher +Shodan +Siphon Sitebeam -Siteimprove -SiteSnagger +SiteExplorer SiteSucker -Site Sucker +Sitevigil SlySearch SMTBot -Snoopy +Snake +SocialRankIOBot +sogouspider Sogou web spider +Sosospider Sottopop Spammen -sp_auditbot -Spbot -Sqlmap +SpankBot +spyfu +Sqlworm Steeler -Stripper +Sucker Sucuri SuperBot Surfbot Suzuran +Swiftbot Szukacz -T0PHackTeam -tAkeOut -Teleport -Telesphoreo +T8Abot +Telesoft Telesphorep The Intraformant TightTwatBot -Toata Toweyabot -Tracemyfile -trendiction.com -trendiction.de -True_Robot -UnisterBot +Trendiction +Turingos +TurnitinBot Upflow URLy.Warning -VB Project +Vacuum +Vagabondo VCI +VidibleScraper Virusdie -Voil -WallpapersHD +VoidEYE +Voltron +Wallpapers/3.0 WBSearchBot +WebAuto +Web Auto +WebBandit +WebCollage +Web Collage +WebCopier +WEBDAV WebEnhancer -Web Fetch +Web Enhancer WebGo IS -WebLeacher -WebmasterWorldForumBot -WebReaper +WebImageCollector Web Sauger +Webshag WebsiteExtractor +WebsiteQuester +Website Quester +WebSucker Web Sucker -WebWhacker -Whacker +WebZIP Whatweb -Who.is Bot Widow +WinHTTrack +WiseGuys Robot Wonderbot -WWW-Collector-E +Wotbox +WPScan WWW-Mechanize +WWWOFFLE x22Mozilla -YoudaoBot +Xaldon WebSpider +Xenu +xpymep1.exe Zauba +Zermelo Zeus -zgrab -Zitebot -ZyBorg diff --git a/.dev-tools/test_units/random-referrers-for-test-quick.list b/.dev-tools/test_units/random-referrers-for-test-quick.list index fe39b5a44..0c75820cb 100644 --- a/.dev-tools/test_units/random-referrers-for-test-quick.list +++ b/.dev-tools/test_units/random-referrers-for-test-quick.list @@ -1,10 +1,10 @@ -adloads.com -continent-e.tv -findclan.org -game300.ru -photorepair.ru -puteshestvennik.com -spamnuker.com -visa-china.ru -xtube.com -zmoda.hostreo.com +activecampaign.dreamhosters.com +empirepoker.com +favoritemoney.ru +handicapbathtubarea.com +mil-stak.com +petitions.whitehouse.gov +rapidsites.pro +rcpmda.ikan1080.xyz +token-lab.org +wladimirpayen.com diff --git a/.dev-tools/test_units/random-referrers-for-test.list b/.dev-tools/test_units/random-referrers-for-test.list index 2b9fcd8be..cde6a2eaf 100644 --- a/.dev-tools/test_units/random-referrers-for-test.list +++ b/.dev-tools/test_units/random-referrers-for-test.list @@ -1,500 +1,500 @@ -0n-line.tv -1314dh.com -2kata.ru -2pxg8bcf.top -51.la +00author.com +101raccoon.ru +1millionusd.xyz +1webmaster.ml +2728fb936f0.com +2x2fan.ru +3dracergames.com +5kstore.com 5-steps-to-start-business.com -6hopping.com -991mostfm.co.id -aaa38852e886ac4af1a3cff9b47cab6272.com -abbanreddy.com -abcguru.xyz -adconscious.com -addbin.men -adloads.net -adman.se +7fon.ru +aa9d046aab36af4ff182f097f840430d51.com +aac500b7a15b2646968f6bd8c6305869d7.com +above.com +accessoires-mode.in +adelly.bg +admeasures.com admitad.com -ad-words.ru -adzpower.com -aghanyna.com -aibolita.com -alldubai.biz -alta-realestate.com -alvtank.se -amateurgalls.com -amazon-adsystem.com -anal-acrobats.com -androids-store.com +adsland.men +ads.tremorhub.com +advertisingtag.net +adzerg.com +agadelha.com.br +alert-jdh.xyz +allbizne.co.ua +all-number.com +alphahoverboards.com angry-fermi-7633.arukascloud.io -anime.dougasouko.com +anonymizeme.pro antons-transporte.de -aosexkontakte.net -apiqualitinknet-a.akamaihd.net -apiwebwebgetcom-a.akamaihd.net -arendas.net -arenda-yeisk.ru -artdestshop.eu -asdfz.pro -ausergrubhof.info -autoblogger24.info -autogrep.ru -autorn.ru -availit.weebly.com -bad-stars.net -balitouroffice.com -bamo.xsl.pt +apartmentbay.ru +apessay.com +apiallgeniusinfo-a.akamaihd.net +apioasisspacenet-a.akamaihd.net +appfixing.space +appiq.mobi +aramaicmedia.org +arate.ru +arendatora.ru +artdeko.info +asrvvv-a.akamaihd.net +avirasecureserver.com +azadnegar.com +azazu.ru +babieca.com +bag77.ru +baltgem.com bankcrediti.ru -barbourjackets.online -bedcapdealers.com -bestwaystogetpaid.us -bingoporno.com -bio.trade-jp.net -birzha-truda.eu -bkns.vn -blackhatworth.com +bannerads.de +bannerspace.com +baoxaydung.com.vn +begalka.xyz +best-businessman.ru +bestdooz.com +bestimagecoollibrary.com +best-seo-offer.com +bezcmexa.ru +bharatdefencekavach.com +bigames.online +big-cash.net +bigcities.org +bimatoprost-careprost.com.ua +biznesrealnost.ru +blobar.org +blog.axant.it blog.f00kclan.de -bloggedporn.com -bloggerads.net -blpmovies.com -boltalko.xyz -bongiornos.info -boostmyppc.com -brandedleadgeneration.com -brk-rti.ru -brus.city -buigas.00it.com -buyessaynow.biz -buzzurl.jp +bmsco.co +boole.onlinewebshop.net +bosefux.esy.es +bosman.pluto.ro +browsepulse-a.akamaihd.net +buron.pw +businesn.men +buyessayonline19.blogspot.ru +buyk.host.sk +buytizanidineonline.blogspot.com bvlgariring.xyz -bytimedance.ru -cablecar.us -cam-kontakte.org -candytiner.com -canoncdriverq3.pen.io -carina-sy.de -carsdined.org -carsplate.com -cartierlovestore.xyz -cartierwatch.xyz -cenoval.ru -charmstroy.info +calstaterealty.us +canacopegdl.com +capture-room.com +carson.getenjoyment.net +cartierreplica.xyz +cbb1smartlist12.click +cdncash.net +centrumcoachingu.com chastnoeporno.com -chcu.net -children-learningreading.info -chimiver.info -chinajerseyswholesalecoupons.com -cialischstgerts.com -cibpenonptib.flu.cc -cineacademy.ru -ci.ua +cheapbarbour.online +cheapcigarettesc.info +cheapmoncler.pw +cheap-trusted-backlinks.com +check-host.net +christianlouboutinoutlet.win +cinemaflix.website +citysecurity.nu clarithromycin500mg.com -cleanmypc.com +clayvasedesigns.tk clicksor.net -clmforexeu.com -colehaanoutlet.store -compliance-alex.top -compliance-alex.xyz -compliance-donald.xyz -compliance-elena.top -compliance-mary.top -cool-wedding.net -coslab.club -counter.yadro.ru -crazyprotocol.com -creams.makeforum.eu -creativehutindia.com -csgo4.win -cupday.com -custodieva.ru -dandiyabeats.in -daptravel.com -daretodonate.co -data.vtc.pw +cocyq.inwtrade.com +coinsspb.com +com-onlinesupport.host +compliance-elena.xyz +compliance-fred.top +compliance-jane.top +compliance-josh.top +compliance-margo.top +contextualyield.com +cosmetic.donna7753191.ru +crystalslot.com +customchocolate.business-for-home.com +data-mining.tk +datract.com dcj-nn.ru -dellalimov.com -dentalpearls.com.au -depositfiles-porn.ga -derevesendeco.com -descargar-musica-gratis.net -deshevo-nedorogo.ru -detskie-zabavi.ru -de.zapmeta.com -dgfitness.co -dieswaene.com -dikqlyremy.info -discovertreasurenow.com -divatraffic.com -divci-hry.info +dear-diary.ru +deplim.com +destinationrealestate.com +digitalassetmanagement.site +dignitasdata.se +dlya-android.org documentsite.net -dodlive.mil -dogclothing.store -domaincdn.xyz -domcran.net -doska-vsem.ru -dosugrostov.site +dominateforex.ml doublepimp.com -downloadeer.net -download-wallpaper.net -draniki.org -drunkmoms.net -easytuningshop.ru -e-buyeasy.com -econom.co +dtm-spain.com +dvervmoskvu.ru +dvr.biz.ua +e705.net +ebooknovel.club +ecommerce-seo.org eduinfosite.com -eduserver.net +ege-essay.ru eldiariodeguadalajara.com -elektrozigaretten1.postbit.com -enbersoft.com -energydiet-info.ru +elidelcream.weebly.com +empathica.com +englate.com +englishgamer.com +enskedesquashclub.se erotik-kostenlos.net -eyessurgery.ru -ezigarettekaufen1.hpage.com -family1st.ca -familyphysician.ru -fashion-stickers.ru -fertilitetsradgivningen.se -fetlifeblog.com -fickblock18.com -fickenbumsen.net -fifa-coins.online -filesmonster.porn -firstdrugmall.ru -folowsite.com -forexgb.ru -forex-indextop20.ru -formulaantiuban.com -fortunejack.com -fotosfotos.eu -fotoxxxru.com -frameimage.org +escortslet.net +evrotekhservis.ru +extremepornos.net +ezigarettenkaufen1.pagina.gr +ezigaretteshop2.mywapblog.com +ezigaretteshop.webs.com +familienzahnaerzte.com +fantasticpixcool.com +fbdownloader.com +filerockstar298.weebly.com +filmgo.ru +fil.ru +finejewelryshop.ru +food.dtu.dk +fortwosmartcar.pw freeseedsonline.com +froggytube.com +fullfileaccess.com fullgirl.ru -fusoradio.info -g7m.pl -galaxyflowers.ru -gamedayhouse.com -gazoblok.net.ua -getdot.ru -getyourimage.club -gigablast.com -gok-kasten.net -goodwinmetals.co -gotomontenegro.net -greenidesign.co -gribkovye-zabolevaniya.com -groupmoney.ru -hasshe.com -hausfrauensex18.com -hesteel.pl -high-speed1.net -himgaws.pw -homeandhealth.ru -homeart.space -honyaku.yahoofs.jp -host-protection.com -hotblog.top -hotenergy.ru -housekuba.org -hspline.com -hut1.ru -hyhj.info -iboss.com -ideoworld.org -igrovyeavtomaty777.ru -ilmexico.com -img.wallpaperstock.net -inbabes.sexushost.com -inclk.com -indiasourcemart.in -innoslicon.com -insomniagamingfestival.com -in-tandem.co -investsuccess.org -investyb.com -iqupdatetmz.win -iwantmyfreecash.com -jagg.info -javlibrary.cc -javxxx18.com -journeydownthescale.info -justbcause.com -justdating.online +fyl.com.ru +gamedayassist.com +gaming-journal.com +gaytube.com +gdebestkupit.ru +getoutofdebtfree.org +getresponse.com +girlgamerdaily.com +girlspicsa.com +golmau.host.sk +goodnightjournal.com +googlefeud.com +goroda-vsego-mira.ru +gourcy.altervista.org +greatzip.com +greendream.com.ua +gsmtlf.ru +haikuware.com +hamilton.ca +handicapbathtubarea.com +hawaiisurf.com +hazardky.net +hd720kino.ru +headpharmacy.com +healbio.ru +helicalpile.us +hem.passagen.se +hifidesign.ru +highspeed5.net +hobbyhuren24.net +hoholikik.club +homedecorpicture.us +homeinns.com +hotxnights.info +houseofrose.com +hoverboard360.de +hoverboard360.nl +howmuchdoestizanidinecost.blogspot.com +hunthillfarmtrust.org +ictizanidinehcl4mg.blogspot.com +imguramx.pw +iminent.com +imperia31.ru +inet-traffic.com +investingclub.ru +iqoption-bin.com +isitpaleo.info +isotoner.com +jackpotchances.com +japan-bearings.ru +jose.mulinohouse.co +jovencitas.gratis +juliadiets.com jwss.cc -kamen-e.ru -keyword-suggestions.com -keywordsuggests.com -kinobest.pl -kinocccp.net -kino-ecran.ru -kinohit1.ru -kit-opt.ru -klikbonus.com -komp-pomosch.ru -krasnodar.zrus.org -lernur.net -letsart.ru +kareliatobacco.ru +karting196.ru +katushka.net +kazinogames.lv +kenaba.su +keywordhut.com +kino-rating.ru +kiwe-analytics.com +kletkimehan.ru +klin3952.ru +kollesa.ru +kovesszucs.atw.hu +kredit-blog.ru +kumuk.info +ladov.ru +lampokrat.ws +lasvegaslockandsafe.com +laudit.ru +laulini.soclog.se +letslowbefast.today +lexaprogeneric.link +life.biz.ua +lifebyleese.com +lifehacĸer.com +life-instyle.com +liffa.ru +lignofix.ua +link.ac liran-locks.com -liveinternet.ru -liver-chintai.org -llastbuy.ru +liumimgx.pw +liveinternet.ro localflirtbuddies.com -low-format.ru -macnewtech.com -macrotek.ru -magazin-pics.ru -magazintiande.ru -magicplayer-s.acestream.net -makemoneyonline.com -marketingtechniques.info -marwer.info -masthopehomes.com -matpre.top -mature.free-websites.com -medicineseasybuy.com -meduza-consult.ru -meendo-free-traffic.ga -metaxalonevstizanidine.blogspot.com -milblueprint.com -minharevisao.com -more-letom.ru -moskva.nodup.ru -mostantikor.ru +lonerangergames.com +long-beach-air-conditioning.com +lookingglassemb.com +lovasszovetseg.hu +love-baby.cz +lulea-auktionsverk.se +magicdiet.gq +makis.nu +manifestation.betteroffers.review +marketland.ml +mecash.ru +medosmotr-ufa.ru +members.ghanaweb.com +menhealed.net +methodsmarketing.com +microsearch.ru +miniads.ca +mirmedinfo.ru +mister-shop.com +moesen-ficken.com +moetomnenie.com +mojowhois.com +moncleronline.xyz +mosdverka.ru +moviezbonkerssk.cf mozello.ru -mp3downloadhq.com -musezone.ru -musicvidz.ru -muz-baza.net -muztops.ru -mydoctorok.ru -myonigroup.com -naperehresti.info -narkologiya-peterburg.ru -narutonaruto.ru -neks.info -netanalytics.xyz -newtechspb.ru -nextlnk12.com -nfljerseys.online -nikkiewart.ru -noclegonline.info -november-lax.com -nvformula.ru +muabancantho.info +music7s.me +muz-shoes.ru +myonlinepayday.co +mypornfree.ru +mystats.xyz +nelc.edu.eg +networkcheck.xyz +news-speaker.com +newstaffadsshop.club +nikhilbahl.com +nootrino.com +nordvpn.com +northfacestore.online +notasprensa.info +nrv.co.za +nsatc.net oddamzadarmo.eu +oecnhs.info offer.camp -olvanto.ru -onlywoman.org +okmusic.jp +ok.ru +olympescort.com +onlinadverts.com +online-sbank.ru +o-o-6-o-o.com openfrost.com -open-odyssey.org -optitrade24.com -orgasmatrix.com -ossmalta.com -otbelivanie-zubov.com -owohho.com -paclitor.com -park.above.com -paydayloanslocal.com -p-business.ru -pelfind.me -pensplan4u.com -perfection-pleasure.ru -physfunc.ru -piccdata.com -piluli.info -pinstake.com -plaff-go.ru -pochtovyi-index.ru -podshipniki-ntn.ru -polybuild.ru -pontiacsolstice.info -popcash.net -popugauka.ru -popugaychiki.com -porn555.com -pornodojd.ru -pornogratisdiario.com -pornolook.net -pornonik.com -porno-raskazy.ru +palvira.com.ua +pammik.ru +patterntrader-en.com +pcads.ru +perosan.com +pic2fly.com +piccshare.com +picsfair.com +picturesboss.com +pijoto.net +pills24h.com +pinkduck.ga +playfortuna-play.ru +pliks.pl +pogodnyyeavarii.gq +popander.mobi +porno-play.net +porno.simple-image.com.ua pornowarp.info -pornoxxx.com.mx pornstartits.xblog.in -pravoholding.ru -prefersurvey.net -preg.marketingvici.com -preparevideosafesystem4unow.space -princeadvantagesales.com -printie.com -privacyassistant.net -privatbank46.ru -producm.ru -proekt-mos.ru -profitkode.com +portadd.men +pourvous.info +predominant-invent.tk +prizestohandle.club +prodvigator.ua profitsport.club -promgirldresses.xyz -prostitutki-yaroslavlya.org -pts163.ru -pulseonclick.com -pushdata.sendpulse.com -puzzleweb.ru -rabot.host.sk -randalljhoward.com -randki-sex.com -rankings-analytics.com -ranksignals.com -r-control.ru +promoheads.com +promover.org +proprostatit.com +pro-tec.kz +proxyradar.com +psa48.ru +q-moto.ru +questionmarque.ch +raavidesigns.com +rabotaetvse.ru +rainbowice.ru +ranking2017.ga +ranksonic.info realitykings.com -reckonstat.info -remont-fridge-tv.ru -renecaovilla.online -replica-watch.ru -rfid-locker.co -ringporno.com -ritlweb.com -rp9.ru -runofilms.ru -sabaapress.com -sale-japan.com -sa-live.com +realizmobi.com +recinziireale.com +remote-dba.de +rep-am.com +resant.ru +reyel1985.webnode.fr +richinvestmonitor.com +riders.ro +rightenergysolutions.com.au +ringtonepartner.com +rique.host.sk +rogervivierforsale.com +royalads.net +roznica.com.ua +running-line.ru +rusoft-zone.ru +rus-teh.narod.ru +rybalka-opt.ru +sad-torg.com.ua sampleletters.net -santechnik.jimdo.com -sape.top +saratov.xrus.org sbwealthsolutions.ca -scanmarine.info -scanmyphones.com -scanner-don.top -scanner-george.top -scanner-john.top -screentoolkit.com -secret.xn--oogle-wmc.com -seemoreresultshu-a.akamaihd.net -sell-fb-group-here.com -seminarygeorgia59.ga -serpstat.com -sexflirtbook.com -sexs-foto.com -sex-tracker.de -sexyali.com -shillyourcoins.com -sibecoprom.ru +scanner-willy.top +scenarii-1-sentyabrya.uroki.org.ua +search.1and1.com +search-error.com +searchkut.com +seoholding.com +seojokes.net +sexad.net +sexfreepornoxxx.com +sex.hotblog.top +sexvideo-sex.com +sexy-screen-savers.com +sexyteens.hol.es +sexytrend.ru +shopperifymac.com sideeffectsoftizanidine.blogspot.com -siha.de -sisiynas.ru -site3.free-share-buttons.com -smartpet.ru -smokewithrabbits.com -sms2x2.ru -smsactivator.ru -social-s-hhh.xyz -soft-terminal.ru -solicita.info -spaceshipad.com -spamnuker.com -squidoo.com -staff.prairiesouth.ca -star61.de -stefanbakosab.se -sterva.cc +silvercash.com +similardeals.net +simple-image.com.ua +sinestesia.host.sk +sisi-go.ru +sispe.com.br +site-auditor.online +siteripz.net +site.ru +s-iwantyou.com +sixcooler.de +snip.to +socialseet.ru +softomix.net +solitaire-game.ru +sonnikforme.ru +sovetogorod.ru +space4update.pw +spasswelt.xyz +spb.afora.ru +spidtest.org +spywarebegone.com +srecorder.com +sr-rekneskap.no +ssconstruction.co +statoutlook.info +steelmaster.lv stillmiracle.com +streetfooduncovered.com +strigkaomsk.ru stroyhelp-dv.ru -stroymonolit.su -styro.ru -sudexpert66.ru -suggest-keywords.com -superinterstitial.com -svetlotorg.ru -svetodiodoff.ru -sygraem.com -symphonyintegratedhealthcare.com -syvertsen-da.no -takeprofitsystem.com -td-l-market.ru -techart24.com -thaisamkok.com +studiofaca.com +studiofmp.com +sugarlyflex.pw +supernew.org +supers.com.ua +svarbit.com +svtrd.com +tackletarts.co +talant-factory.ru +taqywu51.soup.io +tattooreligion.ru +tdsing.ru thebestphotos.eu -thecounter.com -thin.me.pn -tiens2010.ru -tizanidine4mgstreetprice.blogspot.com -tizanidinedosage.blogspot.com -tizanidinedosageforsleep.blogspot.com -tizanidinehcl2mgtablet.blogspot.com -tizanidineoral.blogspot.com -tizanidineorflexeril.blogspot.com -tjkckpytpnje.com -tmm-kurs.ru -tootoo.to -top10-online-games.com -topsiteminecraft.com -torrents.cd -tracksz.co -trafficgenius.xyz -triumf-realty.ru +thebluenoodle.com +thetattoohut.com +titan-cloud.life +titslove.yopoint.in +tizanidine4mg.blogspot.com +tizanidinehcl2mgsideeffects.blogspot.com +tizanidinepills.blogspot.com +token-lab.org +top-deal.com.pl +topsy.com +totu.us +tour-line.net +trackmedia101.com +trafficmonetize.org +traffmonster.info truck-land.life -ttsq.fr -tuberkulezanet.ru +tube8.com +tubo360.com +tuckermktg.com +tupper-posuda.ru +tupper-shop.ru +tvnewsclips.info +tvorozhnaja-zapekanka-recept.ru +twincitiescarservice.com +uasb.ru +ufa.xrus.org +ultimateclassicrock.com unblocksit.es -unrealcommander.org -uptime-delta.net -urlopener.com -urzedowski.eu -ustion.ru -validdomain.xyz -vesnatehno.ru -vezuviy.su -videochat.world -video-woman.com +upstore.me +uptime.com +upupa.net +usdx.us +valuado.com +vepad.com +vereo.eu +via-energy-order.com +via-gra.webstarts.com +video--production.com +videos-for-your-business.com +vigrx-original.ru vinsit.ru -vintontech.info -visa-pasport.ru +vip-file.com +visitcambridge.org +vitalads.net vitanail.ru -vodkoved.ru -vonradio.com +viteonlusarezzo.it +voloomoney.com voprosotvet24.ru -vostoktrade.info -vozbujdenie.com -vrnelectro.ru -vseuznaem.com +vpnmouse.com vvon.co.uk -vykup-avto-krasnodar.ru -w7s.ru -watch-movies.ru -watchmyfb.pl -webalan.ru -webenlace.com.ar -webmonetizer.net -webpromotion.ae -webshoppermac.com -website-analyzer.info -websites-reviews.com -wheelchairliftsarea.com -wholesalejerseychinaoutlet.com -witherrom55.eklablog.fr -wnoz.de -wordkeyhelper.com -wpsecurity.website -wpthemedetector.co.uk -writingservices17.blogspot.ru -www.timer4web.com -x5market.ru -x-lime.com -xmronta.com -xn------7cdbapdecfd4ak1bn0amjffj7afu3y.xn--p1ai +vykupavto-krasnodar.ru +vzlom-na-zakaz.com +watchdogs-2.ru +webadvance.club +webs.com +webscouter.net +wejdz-tu.pl +westum.se +white-truck.life +windowssearch-exp.com +wladimirpayen.com +womama.ru +wonderfulflowers.biz +worldhistory.biz +www.get-free-traffic-now.com +x-diesel.info +x-mix.info +xn-----6kcaacnblni5c5bicdpcmficy.xn--p1ai xn----7sbahjd3btneuw1joc.xn--p1ai -xn--80aeahghtf8ac5i.xn--p1ai -xn--80aikhbrhr.net -xportvusbdriver8i.snack.ws -xxxtube69.com -yeartwit.com -youandcredit.ru -youradexchange.com -youraticles.pl -yourothersite.com -yoursearch.me -youthreaders.com -yuweng.info -zakon-ob-obrazovanii.ru +xn--90acenikpebbdd4f6d.xn--p1ai +xn--b1adccaf1bzj.xn--p1ai +xn--d1aifoe0a9a.top +xn--q1a.xn--b1aube0e.xn--c1acygb.xn--p1ai +yellocloud.be +yougame.biz +yunque.pluto.ro +yurgorod.ru za-music.mymobiplanet.com -zarepta.com -zrelaya.pw -zs2vm.top -zzlgxh.com +zapatosenventa.info +zhorapankratov7.blogspot.com +zigarettenonl.canalblog.com +zigarettenonlinekaufen1.blox.pl +znakomstvaonlain.ru +znaturaloriginal.com +zynax.ua diff --git a/.dev-tools/test_units/random-referrers-for-whitelist-test.list b/.dev-tools/test_units/random-referrers-for-whitelist-test.list index 114c440b8..aff621f47 100644 --- a/.dev-tools/test_units/random-referrers-for-whitelist-test.list +++ b/.dev-tools/test_units/random-referrers-for-whitelist-test.list @@ -1,250 +1,250 @@ -101lesbian.xyz -2x2fan.ru -aa8b68101d388c446389283820863176e7.com -abbanreddy.com -adinterax.com -adultfriendfinder.com +03e.info +11235813.webzdarma.cz +2000k.ru +20pascals.nl +51unlim.ru +5elementov.ru +7yue.org +98oi.ru +actionnooz.com +adloads.net +adminshop.com +adprotect.net aero2.ru -aihelen.net -albamargroup.com -alborzan.com -alert-hgd.xyz -algerianembassy.co.in -allsilver925.co.il -alot.com -altamayoztourism.com -animali.deagostinipassion.it -approved.su -arendas.net -asdfg.pro -asmxsatadriverin.aircus.com -autotop.com.ua -autotrends.today -azbukadiets.ru -baladur.ru -bala.getenjoyment.net -bazaronline24.ru -besthomemadepornsites.com -bestwebsitesawards.com -bezlimitko.xyz -binomo.kz -bio-japan.net -bitcoins-live.ru -bizlist.com.de -blackcurranthumidifiers.site -blog.koorg.ru -bluerobot.info -bmwhighperformers.com -board.f00d.de -bonzybuddy.com -boostmyppc.com -bristolhotel.com -buyk.host.sk -buytizanidineonline.blogspot.com -cakesplus.com.au -casablancamanor.co.za -cercacamion.it -chocoslim.pro -cjs.com.ru -cleanersoft.com -com-onlinesupport.host -companies-catalog.com -compliance-jess.top -compliance-josh.top -counciltally.com -customerpromos-a.akamaihd.net +algarveglobal.com +allnews.md +alt-servis.ru +amazingninja.com +angry-fermi-7633.arukascloud.io +apartment.ru +arate.ru +arewater.com +astana.xxxkaz.org +axbocz.net +azartmix.com +baixar-musicas-gratis.com +bankiem.pl +barnfurnituremart.com +beclean-nn.ru +berrymall.ru +bestbuy.ca +bestcalovebracelet.cn +bestkfiledxd.cf +best-seo-solution.com +best-way.men +bingoporno.com +bioinnovate.co +bizru.info +bmw-ark.ru +bodybuilding-shop.biz +bookmark4you.com +brianhenry.co +briomotor.co +browsepulse-a.akamaihd.net +cartierbracelet.xyz +cartierlovebracelet.xyz +cashonads.com +cheap-trusted-backlinks.com +cherubinimobili.it +cobaltpro.ru +colehaanoutlet.store +community.allhiphop.com +compliance-fred.top +confib.ifmo.ru +coolingoods.com +coolyarddecorations.com +covadhosting.biz +cpabegins.ru cyprusbuyproperties.com -daisye.top -darodar.com -davidovic.info -dokumentalkino.net -dood.live -drunkenstepfather.com -dvrlists.com -edelstahlschornstein-123.de -efnor-ac.com -elektronischezigarettekaufen1.myblog.de -elektrozigaretten1.postbit.com -elektrozigaretten1.tumblr.com -enews.tech +czat.wp.pl +dafatiri.com +darkages.info +data.vtc.pw +dating-time-now.com +ddpills.com +dev-seo.blog +dichvuvesinhhanoi.com +dirtpics.pw +discountbarbour.online +dotmass.top +downloadkakaotalk.com +downloads-whatsapp.com +download-wallpaper.net +drhomes.biz enginebay.ru -etotupo.ru -europages.com.ru +enternet.ee +e-poker-2005.com exlarseva.webblog.es -expertblog.info -facebook-mobile.xyz -famix.xyz -filesclub.net -forum-engineering.ru -frbizlist.com -fr.netlog.com -front.to -fyxabomiw.ru +fantasticpixcool.com +faptitans.com +fashion-stickers.ru +fatmaelgarny.com +femdom.twiclub.in +ferieboligkbh.dk +firstdrugmall.ru +flavors.me +floridahuntingfishingadventures.com +flowwwers.com +food.dtu.dk +forensicpsychiatry.ru +freesitetest.com +funcrushgames.com +g7m.pl +gambarxkata.co +games.kolossale.ru gaming-journal.com -gaytube.com -geckoandfly.com -getmiro.com -getridofstretchmarks.org -ghazel.ru -girlsatgames.ru -glasof.es -gnuetella.com -go2mike.ru -gonextmedia.com -guigyverpo.cf -havepussy.com -hcate.com -hd-film.pl -healbio.ru -hildinghr.se -hollywoodactress.info -hyiphunter.org -icaseclub.ru -ilte.info -imagefinder.site -inboxdollars.com -indetiske.ya.ru -internetartfair.com -ip-guide.com -jagg.info -javxxx18.com -jwss.cc -kamorel.com -klin3952.ru -krasnodar.xrus.org -laexotic.com -larger.io -law-check-seven.xyz -lennatin.info -letmacworkfaster.world -letslowbefast.site +gay-site.store +gemgrab-a.akamaihd.net +get-your-social-buttons.info +ggiaro.com +ghostvisitor.com +glavprofit.ru +gne8.com +goatse.ru +gotwebsite1.com +gsbs.com.ua +guardlink.com +guiadeserraazul.com +hasfun.com +home.myplaycity.com +housekuba.org +https-legalrc.biz +ibb.com.ua +imagerydatabase.com +inet-traffic.com +iyasimasennka.com +jerseyswholesalechinalimited.com +juliaworld.net +karaganda.xkaz.org +kazrent.com +ketoanhanoi.info +kinoflux.net +kliksaya.com +krynica.info +kvartira-sutochno.com +labelwater.se +lanasshop.ru life.biz.ua -lincolntheatre.com lirunet.ru -locustdesign.co -manimpotence.com -med-zdorovie.com.ua -meltwater.com -memberty.com -menetie.ru -mil-stak.com -moesonce.com -mokrayakiska.com -monclercheap.xyz -mostantikor.ru -musezone.su -mxgetcode.com +lolnex.us +louboutinbooties.xyz +lyrics.home-task.com +m5home.ru +machicon-ueno.info +macresource.co.uk +masturbate.co.uk +matematikus.info +mcadamssupplyco.com +mearns-tractors.co.uk +meget.co.za +mex-annushka.ru +microsearch.ru +monetizer.com-01.site +musezone.ru +my-cash-bot.co mymercy.info -myperiod.club -myyour.eu -nalogovyykodeks.ru -new-post.tk -nicefloor.co.uk -nordvpn.com -nzfilecloud.weebly.com -obuv-kupit.ru -odesproperty.com -one-gear.com -onemactrckr.com -onescreen.cc -onlainbesplatno.ru +ngps1.ru +no-fuel.org +nordstar.pro +okel.co oren-cats.ru -ourville.info -panchro.co.uk -paphoselectricianandplumber.com -peekyou.com -pensplan4u.com -petrovka-online.com -picsforkeywordsuggestion.com -picswe.com -pkr1hand.com -poppen-nw.net -popugaychiki.com -pornowarp.info -prize44.com -prodvigator.ua -profitkode.com -quality-traffic.com -rankingchart.de +orhonit.com +ovirus.ru +owathemes.com +oynat.info +p-business.ru +pensplan.com +picquery.com +pics-group.com +pintattoos.com +piter.xrus.org +pizdopletka.club +pl.aasoldes.fr +play-mp3.com +pornofeuer.com +pornokorol.com +pornoplen.com +praisong.net +priscilarodrigues.com.br +projectforte.ru +projefrio.com.br +pro-poly.ru +push-ad.com +rabot.host.sk rankings-analytics.com ranksonic.info -replicaclub.ru -ribieiendom.no -ric.info -rome2rio.com -rostov.xrus.org +rapevideosmovies.com +razamicroelectronics.com +redbottomheels.xyz +remedyotc.com +retailwith.com +ritlweb.com +royalads.net royalcar-ufa.ru -rus-pornuha.com -saddiechoua.com -saneyes.com -saugatuck.com -scanner-barak.top -scanner-viktor.top -scrapinghub.com -searchlock.com -searchtooknow-a.akamaihd.net -seocheki.net -shakhtar-doneck.ru -sharebutton.org -share-buttons-for-free.com -siteimprove.com +roznica.com.ua +scanner-mark.top +scanner-walter.top +scanner-willy.top +search-error.com +seminarykansas904.ml +semp.net +sexspornotub.com +sexuria.net +sharebutton.to +shodanhq.com sitevaluation.com -sivs.ru -soda.media -sohoindia.net -sortthemesitesby.com -spymac.net -srdrvp.com -starpages.net -statoutlook.info +skincrate.net +soblaznu.net +socialbuttons.xyz +space2019.top +speechfoodie.com +speedup-my.site +spywareit.com +staynplay.net stevemonsen.com -stillmiracle.com -studiofaca.com -sureone.pro -tds-advert002.info -tfxiq.com -thedownloadfromwarez.blogspot.com -theendivechronicles.com -titan-ads.life -titan-cloud.life -tizanidinehcl4mgtablet.blogspot.com -tpu.ru -trafaret74.ru -traffmonster.info -traffpartners.com -tula.mdverey.ru -umg-stroy.ru -ussearche.cf -veerotech.com -videosbox.ru -vip-file.com -vkgaleria.com -vkonche.com -vote-up.ru -vut.com.ru -vzube.com +stickers-market.ru +supermesta.ru +tehngr.ru +thelottosecrets.com +tizanidine4mg.blogspot.com +tizanidinedosageforsleep.blogspot.com +tizanidinehcl2mg.blogspot.com +tizanidinehcl2mgtablet.blogspot.com +tizanidineingredients.blogspot.com +tizanidinemusclerelaxant.blogspot.com +tk-assortiment.ru +top10-way.com +traffic2money.com +trafficmonetize.org +trkdf.com +udayavani.com +uggbootsoutletsale.us +uggsale.online +unitygame3d.com +unrealcommander.com +usdx.us +vapeface.club +vapsy.com +veles.shop +velobikestock.com +videogamesecrets.com +video--production.com +vikistars.com +voloomoney.com w3data.co -watchmygf.net -wdrake.com -webpromotion.ae -website-speed-checker.site -wechatdownload10.com -welcomeauto.ru -westsextube.com -wieseversa.no +warezkeeper.com +weekes.biz.tc wikes.20fr.com -wleuaprpxuvr.ga -wordpress-crew.net -worldhistory.biz -wowcasinoonline.ooo -ww1943.ru -x5market.ru -xn--h1ahbi.com.ua -xn--l1aengat.xn--p1ai -xtraffic.plus -xxlargepop.com +word-vorlagen.xyz +xjrul.com +xn--1-8sbcpb0bdm8k6a.xn--p1ai +xxxkaz.org +yaaknaa.info +youbrainboost.asia +yougetsignal.com +youradexchange.com yoursite.com -zajm-zalog-krasnodar.ru -zapatosenventa.info -zastenchivosti.net -zx6.ru +yubikk.info +zaloro.com +za-music.mymobiplanet.com +zerocash.msk.ru diff --git a/.dev-tools/test_units/ratelimittest-beta.txt b/.dev-tools/test_units/ratelimittest-beta.txt index 525a76fd6..a7215fb74 100644 --- a/.dev-tools/test_units/ratelimittest-beta.txt +++ b/.dev-tools/test_units/ratelimittest-beta.txt @@ -5,6 +5,13 @@