diff --git a/.dev-tools/beta_conf_files/conf.d/globalblacklist.conf b/.dev-tools/beta_conf_files/conf.d/globalblacklist.conf index 368112be6..fa04cf626 100644 --- a/.dev-tools/beta_conf_files/conf.d/globalblacklist.conf +++ b/.dev-tools/beta_conf_files/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2019.07.1713 -### Updated: Mon Jul 1 11:47:20 SAST 2019 +### Version: V3.2019.07.1714 +### Updated: Mon Jul 1 12:05:47 SAST 2019 ### Bad Referrer Count: 6717 ### Bad Bot Count: 556 ################################################### diff --git a/.dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf b/.dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf index 0fcb73d00..479de1217 100644 --- a/.dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf +++ b/.dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -35.192.187.174 1; +35.192.85.2 1; 127.0.0.1 1; diff --git a/.dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf b/.dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf index 3dac4f84e..d9f0a2c04 100644 --- a/.dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf +++ b/.dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -35.192.187.174 0; +35.192.85.2 0; 127.0.0.1 0; diff --git a/.dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf b/.dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf index 368112be6..fa04cf626 100644 --- a/.dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf +++ b/.dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2019.07.1713 -### Updated: Mon Jul 1 11:47:20 SAST 2019 +### Version: V3.2019.07.1714 +### Updated: Mon Jul 1 12:05:47 SAST 2019 ### Bad Referrer Count: 6717 ### Bad Bot Count: 556 ################################################### diff --git a/.dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf b/.dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf index 0fcb73d00..479de1217 100644 --- a/.dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf +++ b/.dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -35.192.187.174 1; +35.192.85.2 1; 127.0.0.1 1; diff --git a/.dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf b/.dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf index 3dac4f84e..d9f0a2c04 100644 --- a/.dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf +++ b/.dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -35.192.187.174 0; +35.192.85.2 0; 127.0.0.1 0; diff --git a/.dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf b/.dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf index 368112be6..fa04cf626 100644 --- a/.dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf +++ b/.dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2019.07.1713 -### Updated: Mon Jul 1 11:47:20 SAST 2019 +### Version: V3.2019.07.1714 +### Updated: Mon Jul 1 12:05:47 SAST 2019 ### Bad Referrer Count: 6717 ### Bad Bot Count: 556 ################################################### diff --git a/.dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf b/.dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf index 0fcb73d00..479de1217 100644 --- a/.dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf +++ b/.dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -35.192.187.174 1; +35.192.85.2 1; 127.0.0.1 1; diff --git a/.dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf b/.dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf index 3dac4f84e..d9f0a2c04 100644 --- a/.dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf +++ b/.dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -35.192.187.174 0; +35.192.85.2 0; 127.0.0.1 0; diff --git a/.dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf b/.dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf index 368112be6..fa04cf626 100644 --- a/.dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf +++ b/.dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2019.07.1713 -### Updated: Mon Jul 1 11:47:20 SAST 2019 +### Version: V3.2019.07.1714 +### Updated: Mon Jul 1 12:05:47 SAST 2019 ### Bad Referrer Count: 6717 ### Bad Bot Count: 556 ################################################### diff --git a/.dev-tools/globalblacklist-testing.template b/.dev-tools/globalblacklist-testing.template index 368112be6..fa04cf626 100644 --- a/.dev-tools/globalblacklist-testing.template +++ b/.dev-tools/globalblacklist-testing.template @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2019.07.1713 -### Updated: Mon Jul 1 11:47:20 SAST 2019 +### Version: V3.2019.07.1714 +### Updated: Mon Jul 1 12:05:47 SAST 2019 ### Bad Referrer Count: 6717 ### Bad Bot Count: 556 ################################################### diff --git a/.dev-tools/globalblacklist.template b/.dev-tools/globalblacklist.template index 56f176a79..367ed7080 100755 --- a/.dev-tools/globalblacklist.template +++ b/.dev-tools/globalblacklist.template @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2019.07.1713 -### Updated: Mon Jul 1 11:47:19 SAST 2019 +### Version: V4.2019.07.1714 +### Updated: Mon Jul 1 12:05:46 SAST 2019 ### Bad Referrer Count: 6717 ### Bad Bot Count: 556 ################################################### diff --git a/.dev-tools/test1_conf_files/bots.d/whitelist-ips.conf b/.dev-tools/test1_conf_files/bots.d/whitelist-ips.conf index ca4fcdf54..c0fc69668 100644 --- a/.dev-tools/test1_conf_files/bots.d/whitelist-ips.conf +++ b/.dev-tools/test1_conf_files/bots.d/whitelist-ips.conf @@ -45,4 +45,4 @@ # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST -35.193.14.140 0; +35.192.85.2 0; diff --git a/.dev-tools/test1_conf_files/conf.d/globalblacklist.conf b/.dev-tools/test1_conf_files/conf.d/globalblacklist.conf index 56f176a79..367ed7080 100644 --- a/.dev-tools/test1_conf_files/conf.d/globalblacklist.conf +++ b/.dev-tools/test1_conf_files/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2019.07.1713 -### Updated: Mon Jul 1 11:47:19 SAST 2019 +### Version: V4.2019.07.1714 +### Updated: Mon Jul 1 12:05:46 SAST 2019 ### Bad Referrer Count: 6717 ### Bad Bot Count: 556 ################################################### diff --git a/.dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf b/.dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf index 6893ce0fc..5e7c0ed8e 100644 --- a/.dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf +++ b/.dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -35.192.136.167 1; +35.184.226.236 1; 127.0.0.1 1; diff --git a/.dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf b/.dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf index 879967dc9..3e60bde86 100644 --- a/.dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf +++ b/.dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -35.192.136.167 0; +35.184.226.236 0; 127.0.0.1 0; diff --git a/.dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf b/.dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf index 56f176a79..367ed7080 100644 --- a/.dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf +++ b/.dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2019.07.1713 -### Updated: Mon Jul 1 11:47:19 SAST 2019 +### Version: V4.2019.07.1714 +### Updated: Mon Jul 1 12:05:46 SAST 2019 ### Bad Referrer Count: 6717 ### Bad Bot Count: 556 ################################################### diff --git a/.dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf b/.dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf index 6893ce0fc..5e7c0ed8e 100644 --- a/.dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf +++ b/.dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -35.192.136.167 1; +35.184.226.236 1; 127.0.0.1 1; diff --git a/.dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf b/.dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf index 879967dc9..3e60bde86 100644 --- a/.dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf +++ b/.dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -35.192.136.167 0; +35.184.226.236 0; 127.0.0.1 0; diff --git a/.dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf b/.dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf index 56f176a79..367ed7080 100644 --- a/.dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf +++ b/.dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2019.07.1713 -### Updated: Mon Jul 1 11:47:19 SAST 2019 +### Version: V4.2019.07.1714 +### Updated: Mon Jul 1 12:05:46 SAST 2019 ### Bad Referrer Count: 6717 ### Bad Bot Count: 556 ################################################### diff --git a/.dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf b/.dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf index 6893ce0fc..5e7c0ed8e 100644 --- a/.dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf +++ b/.dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -35.192.136.167 1; +35.184.226.236 1; 127.0.0.1 1; diff --git a/.dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf b/.dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf index 879967dc9..3e60bde86 100644 --- a/.dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf +++ b/.dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -35.192.136.167 0; +35.184.226.236 0; 127.0.0.1 0; diff --git a/.dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf b/.dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf index 56f176a79..367ed7080 100644 --- a/.dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf +++ b/.dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2019.07.1713 -### Updated: Mon Jul 1 11:47:19 SAST 2019 +### Version: V4.2019.07.1714 +### Updated: Mon Jul 1 12:05:46 SAST 2019 ### Bad Referrer Count: 6717 ### Bad Bot Count: 556 ################################################### diff --git a/.dev-tools/test2_conf_files/bots.d/whitelist-ips.conf b/.dev-tools/test2_conf_files/bots.d/whitelist-ips.conf index f5e1a7808..f6b36a541 100644 --- a/.dev-tools/test2_conf_files/bots.d/whitelist-ips.conf +++ b/.dev-tools/test2_conf_files/bots.d/whitelist-ips.conf @@ -45,5 +45,5 @@ # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST -35.192.136.167 0; -35.193.14.140 0; +35.224.112.202 0; +35.184.96.71 0; diff --git a/.dev-tools/test2_conf_files/conf.d/globalblacklist.conf b/.dev-tools/test2_conf_files/conf.d/globalblacklist.conf index 00b05696f..64fc71cfb 100644 --- a/.dev-tools/test2_conf_files/conf.d/globalblacklist.conf +++ b/.dev-tools/test2_conf_files/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2019.07.1713 -### Updated: Mon Jul 1 11:47:19 SAST 2019 +### Version: V4.2019.07.1714 +### Updated: Mon Jul 1 12:05:46 SAST 2019 ### Bad Referrer Count: 6717 ### Bad Bot Count: 556 ################################################### diff --git a/.dev-tools/test4_conf_files/bots.d/whitelist-ips.conf b/.dev-tools/test4_conf_files/bots.d/whitelist-ips.conf index 4851d6475..2c66352d8 100644 --- a/.dev-tools/test4_conf_files/bots.d/whitelist-ips.conf +++ b/.dev-tools/test4_conf_files/bots.d/whitelist-ips.conf @@ -45,5 +45,6 @@ # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST -35.188.73.34 0; -34.68.144.114 0; +35.192.85.2 0; +104.154.120.187 0; +35.184.96.71 0; diff --git a/.dev-tools/test4_conf_files/conf.d/globalblacklist.conf b/.dev-tools/test4_conf_files/conf.d/globalblacklist.conf index 56f176a79..367ed7080 100644 --- a/.dev-tools/test4_conf_files/conf.d/globalblacklist.conf +++ b/.dev-tools/test4_conf_files/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2019.07.1713 -### Updated: Mon Jul 1 11:47:19 SAST 2019 +### Version: V4.2019.07.1714 +### Updated: Mon Jul 1 12:05:46 SAST 2019 ### Bad Referrer Count: 6717 ### Bad Bot Count: 556 ################################################### diff --git a/.dev-tools/test5_conf_files/bots.d/whitelist-ips.conf b/.dev-tools/test5_conf_files/bots.d/whitelist-ips.conf index 761a4bfb2..79a52fe66 100644 --- a/.dev-tools/test5_conf_files/bots.d/whitelist-ips.conf +++ b/.dev-tools/test5_conf_files/bots.d/whitelist-ips.conf @@ -45,6 +45,6 @@ # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST -35.188.73.34 0; -34.68.144.114 0; -35.193.14.140 0; +35.184.226.236 0; +35.184.96.71 0; +35.224.112.202 0; diff --git a/.dev-tools/test5_conf_files/conf.d/globalblacklist.conf b/.dev-tools/test5_conf_files/conf.d/globalblacklist.conf index 56f176a79..367ed7080 100644 --- a/.dev-tools/test5_conf_files/conf.d/globalblacklist.conf +++ b/.dev-tools/test5_conf_files/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2019.07.1713 -### Updated: Mon Jul 1 11:47:19 SAST 2019 +### Version: V4.2019.07.1714 +### Updated: Mon Jul 1 12:05:46 SAST 2019 ### Bad Referrer Count: 6717 ### Bad Bot Count: 556 ################################################### diff --git a/.dev-tools/test_units/blacklist-ips.conf b/.dev-tools/test_units/blacklist-ips.conf index 0fcb73d00..479de1217 100644 --- a/.dev-tools/test_units/blacklist-ips.conf +++ b/.dev-tools/test_units/blacklist-ips.conf @@ -1,2 +1,2 @@ -35.192.187.174 1; +35.192.85.2 1; 127.0.0.1 1; diff --git a/.dev-tools/test_units/random-bots-for-test-quick.list b/.dev-tools/test_units/random-bots-for-test-quick.list index 2cb7de62e..e3b3a0e9b 100644 --- a/.dev-tools/test_units/random-bots-for-test-quick.list +++ b/.dev-tools/test_units/random-bots-for-test-quick.list @@ -1,10 +1,10 @@ -evc-batch -Extractor -Ninja -SurveyBot -Vacuum -Vagabondo -WebCollage -WEBDAV -webmeup-crawler -Web Pix +Buddy +LinksManager +Mass Downloader +Nikto +Nimbostratus +Suzuran +Szukacz +WebPix +WebSucker +Who.is Bot diff --git a/.dev-tools/test_units/random-bots-for-test.list b/.dev-tools/test_units/random-bots-for-test.list index 07d37a4f6..6ebfc0f92 100644 --- a/.dev-tools/test_units/random-bots-for-test.list +++ b/.dev-tools/test_units/random-bots-for-test.list @@ -1,250 +1,250 @@ -360Spider +404checker 404enemy Abonti -Aboundexbot -ADmantX +Aboundex +Acunetix AIBOT +Aipbot Alexibot -AllSubmitter -AlphaBot +Alligator archive.org_bot ASPSeek Asterias -Attach -autoemailspider BackDoorBot -backlink-check +Backlink-Ceck BacklinkCrawler -BackStreet -BackWeb -Barkrowler +Badass BatchFTP -Battleztar Bazinga -BDFetch -BlowFish -BuiltBotTough -BuiltWith -Bullseye -BunnySlippers +BetaBot +Bigfoot +Bitacle +Blackboard +BotALot +Brandprotect +Buddy Calculon CCBot CheeseBot -CherryPicker CheTeam -Claritybot -Cliqzbot -Cogentbot -com.plumanalytics -Copyscape -Cosmos +Copier +CopyRightCheck +Craftbot +crawler.feedback +crawl.sogou.com +CrazyWebCrawler +CrunchBot CSHttp +Curious +DatabaseDriverMysqli DataCha0s -demandbase-bot -Demon Devil DIIbot -Disco -Discoverybot -Dispatch DittoSpyder +DomainAppender +DomainSigmaCrawler +DomainStatsBot +Dotbot +Drip +DTS Agent +Ebingbong EirGrabber -EMail Siphon +EroCrawler +evc-batch Express WebPictures -Extractor +ExtLinksBot Extreme Picture Finder +EyeNetIE facebookscraper FDM -Fimap +FemtosearchBot +FHscan +Firefox/7.0 +FlashGet Flunky -FyberSpider +Freeuploader +FrontPage Fyrebot +GalaxyBot Genieo -Getintent GetWeb -GoZilla -Go!Zilla -GrabNet +Gigablast +Grabber Grafula +GridBot GT::WWW Haansoft +Havij HEADMasterSEO Heritrix -Hloader -HMView +HTMLparser +HTTP::Lite HTTrack -Humanlinks -IDBot -Image Sucker -InterGET -Internet Ninja +Image Fetch +Indy Library +instabid InternetSeer -Jbrofuzz -JennyBot +internetVista monitor +ips-agent +IRLbot +IstellaBot JetCar -Joomla -Jyxobot +JOC Web Spider +JustView Keyword Density -Kozmosbot +Larbin LeechFTP LeechGet -LexiBot -Lftp -LinkWalker -LinqiaMetadataDownloaderBot -LinqiaRSSBot -LinqiaScrapeBot -Lipperhey +Lightspeedsystems +LinkpadBot +LinksManager Lipperhey Spider Litemage_walker -Ltx71 -LWP::Simple -magpie-crawler -Mail.RU_Bot -Majestic12 -Majestic SEO +LNSpiderguy +lwp-request +lwp-trivial +Mag-Net Majestic-SEO -Masscan +MarkMonitor +MarkWatch Mass Downloader MauiBot +meanpathbot Meanpathbot MeanPath Bot Mediatoolkitbot -mediawords -Metauri +MFC_Tear_Sample Microsoft Data Access MIIxpc Mister PiX -MJ12bot Mojeek -Mr.4x3 +MSIECrawler +Msrabot MS Web Services Client Protocol muhstik-scan Musobot -Name Intelligence -NearSite +Nameprotect +Navroad +Needle +Nessus NetAnts -Netcraft +netEstate NE Crawler NetLyzer -NetSpider -Net Vampire Netvibes NetZIP NICErsPRO -Ninja -Nmap -NPbot +Niki-bot +Nikto +NimbleCrawler +oBot Offline Navigator -OnCrawl -OpenLinkProfiler -OrangeBot +OrangeSpider OutclicksBot -OutfoxBot +PageGrabber page scorer Panscient +Pavuk PECL::HTTP -PeoplePal -Picsearch +Picscout PictureFinder Pi-Monster -PleaseCrawl +Pixray +Probethenet ProWebWalker Psbot Pump -PxBroker -QueryN Metasearch -Quick-Crawler +PyCurl +RankingBot2 RankurBot Reaper +RebelMouse Recorder RedesScrapy +ReGet RepoMonkey RocketCrawler s1z.ru -SBIder +ScanAlert Scanbot +Scrapy Screaming -ScreenerBot -Searchestate -SearchmetricsBot -SemrushBot -SEOkicks SEOkicks-Robot SEOlyticsCrawler -seoscanners SeoSiteCheckup SEOstats -serpstatbot sexsearcher -Shodan -Siphon +SISTRIX Sitebeam -SiteExplorer -SiteLockSpider +Siteimprove SiteSnagger SiteSucker -SlySearch +Sitevigil SmartDownload +SMTBot Snake -SocialRankIOBot -Sociscraper -sogouspider -Sogou web spider -Sottopop +Snoopy +sp_auditbot Spbot spyfu Sqlmap +Stripper +Sucker +Sucuri SuperBot -SuperHTTP -SurveyBot -sysscan -tAkeOut +Suzuran +Szukacz +T0PHackTeam +Teleport TeleportPro Telesoft +Telesphoreo Telesphorep +TheNomad +Thumbor TightTwatBot Titan -Toata Toweyabot -Tracemyfile -trendiction.com +Trendiction +trendiction.de True_Robot Turingos +Turnitin TurnitinBot -Twice +TwengaBot Upflow Vacuum -Vagabondo VB Project -VidibleScraper -VoidEYE -WASALive-Bot +VCI +Virusdie +Voltron +WallpapersHD WBSearchBot -WebCollage +Webalta +Web Auto Web Collage -WEBDAV +WebCopier WebEnhancer -Web Enhancer -WebFetch -Web Fuck +WebFuck WebGo IS -webmeup-crawler -Web Pix -Website Quester +WebImageCollector +WebLeacher +WebmasterWorldForumBot +WebPix +Webshag WebStripper -WebSucker +WebWhacker WeSEE -Whacker -Whatweb +Whack Who.is Bot -Widow WinHTTrack WISENutbot -Woobot -Wotbox +WPScan +WWW-Collector-E WWW-Mechanize -x09Mozilla +WWWOFFLE x22Mozilla +Xaldon WebSpider +Xenu xpymep1.exe -YoudaoBot -Zeus -ZumBot +zauba.io +Zermelo +ZmEu diff --git a/.dev-tools/test_units/random-bots-for-whitelist-test.list b/.dev-tools/test_units/random-bots-for-whitelist-test.list index 1bac71e0e..6f4714f74 100644 --- a/.dev-tools/test_units/random-bots-for-whitelist-test.list +++ b/.dev-tools/test_units/random-bots-for-whitelist-test.list @@ -1,250 +1,250 @@ -360Spider -404enemy +404checker 80legs -Abonti Aboundex Aboundexbot ADmantX -Aipbot -Alexibot -AlphaBot -Anarchie +AiHitBot +Alligator archive.org_bot +ASPSeek +Attach autoemailspider -Backlink-Ceck backlink-check -BackWeb +BacklinkCrawler Badass Bandit +Barkrowler +BBBike +BDCbot +BDFetch BetaBot -BlowFish -Boardreader -BotALot +Bigfoot +Bitacle +Black Hole +Bolt Buddy BuiltBotTough -BuiltWith BunnySlippers -CATExplorador +BuzzSumo +CazoodleBot +CCBot +Cegbfeieh CheeseBot -Chlooe -Cloud mapping -coccocbot-web +CheTeam +Claritybot Collector +com.plumanalytics Copier -CopyRightCheck -Craftbot +Copyscape +crawler.feedback +CrazyWebCrawler Crescent +CrunchBot CSHttp Curious -DatabaseDriverMysqli -DBLBot +demandbase-bot Demon Digincore -Disco +DIIbot +Discobot Dispatch DittoSpyder DnyzBot DomainCrawler DomainSigmaCrawler -DomainStatsBot Dotbot -Dragonfly +Download Wonder Drip -ECCP/1.0 -Ecxi +eCatch EirGrabber -EMail Siphon -evc-batch -Express WebPictures +EroCrawler +Evil ExtLinksBot -EyeNetIE -FDM +Ezooms FHscan -Fimap -FrontPage -Getintent -Gigablast +Firefox/7.0 +Freeuploader +GalaxyBot +GermCrawler +GetRight +Gigabot G-i-g-a-b-o-t -GoZilla -Go!Zilla -Grabber +Go-Ahead-Got-It +Gotit GrabNet Grafula -GrapeFX -Haansoft -HaosouSpider -Harvest -HEADMasterSEO -HMView -HTTP::Lite +GrapeshotCrawler +Heritrix +HTTrack HybridBot -Iblog -IDBot -Id-search -Image Fetch -InfoNaviRobot -InfoTekies -instabid -InterGET -InternetSeer +IndeedBot +Indy Library internetVista monitor ips-agent +IRLbot +Iskanie IstellaBot JamesBOT JennyBot -JetCar +Jetty +JOC Web Spider +Joomla Jorgee +JustView Jyxobot Kozmosbot Larbin +LexiBot Libwhisker +Lightspeedsystems Likse -LinkextractorPro +Linkdexbot LinkScan LinksManager LinqiaMetadataDownloaderBot -LinqiaScrapeBot Lipperhey +Lipperhey Spider Litemage_walker -Lmspider -Ltx71 -LWP::Simple -Mail.RU_Bot +LNSpiderguy +lwp-request +lwp-trivial +magpie-crawler +Majestic12 +Majestic SEO +Majestic-SEO MarkWatch +Mata Hari +meanpathbot +Meanpathbot MeanPath Bot -Mediatoolkitbot -MegaIndex.ru +mediawords Metauri -Microsoft Data Access -Microsoft URL Control -MIIxpc -Mister PiX Mojeek +Morfeus Fucking Scanner +MSFrontPage MSIECrawler -Msrabot muhstik-scan -Name Intelligence -Navroad -NetAnts -NetLyzer +NearSite +Needle +netEstate NE Crawler +NetMechanic +Nettrack Netvibes -NextGenSearchBot -Niki-bot +NetZIP +Nibbler NimbleCrawler +Nimbostratus Ninja NPbot -oBot +Nutch +Octopus Offline Navigator -OnCrawl -OpenVAS -OrangeBot -OrangeSpider -PageAnalyzer +Openfind +Openvas +OutclicksBot +OutfoxBot Page Analyzer -Panscient -Pavuk +page scorer +Pandalytics +pcBrowser PECL::HTTP PeoplePal PHPCrawl -Pi-Monster -ProPowerBot -ProWebWalker -PxBroker -Quick-Crawler +Pimonster +PleaseCrawl +plumanalytics +Pockey +POE-Component-Client-HTTP +Probethenet RankActive -RankFlex -RankingBot -RankurBot -RealDownload Recorder RedesScrapy ReGet RepoMonkey +Ripper +RocketCrawler +Rogerbot +s1z.ru SalesIntelligent -SBIder -ScanAlert +Scanbot scan.lol -ScoutJet Screaming -SearchmetricsBot +ScreenerBot Semrush -Seomoz -SEOprofiler +SemrushBot +SEOkicks +SEOlyticsCrawler seoscanners SeoSiteCheckup -SEOstats -serpstatbot -sexsearcher Shodan Siphon -Sitebeam -SiteExplorer +SISTRIX SiteSucker -Site Sucker -Sitevigil SlySearch +Snapbot +Snoopy Sociscraper -Sosospider +Sogou web spider Sottopop +SpaceBison +Spammen +SpankBot +Spanner sp_auditbot Spbot -SputnikBot -Sqlmap -Stripper -Surfbot -Suzuran +Spinn3r +spyfu +SuperBot +SuperHTTP Swiftbot sysscan T8Abot -tAkeOut +Teleport +TeleportPro Telesoft -The Intraformant -TheNomad -TightTwatBot +Telesphoreo +Telesphorep Titan -Toata Trendiction +trendiction.com +True_Robot TurnitinBot TwengaBot -Vacuum -Vagabondo -VCI -Virusdie -VoidEYE +Twice +Typhoeus +VidibleScraper Voltron WallpapersHD -Webalta -WebAuto -WebBandit Web Collage +WebCopier WEBDAV -WebEnhancer +Web Enhancer +WebFetch Web Fetch -Web Fuck -WebGo IS +WebFuck WebImageCollector -WebLeacher -Web Pix -WebReaper +WebmasterWorldForumBot +webmeup-crawler +WebPix WebSauger Web Sauger Webshag -WebsiteExtractor +WebsiteQuester Website Quester +Webster WebStripper -WebSucker -Web Sucker -WebZIP +Whack Whatweb -Who.is Bot WiseGuys Robot -Wotbox +Wonderbot Wprecon -WWW::Mechanize +WWW-Collector-E +WWW-Mechanize WWWOFFLE -x22Mozilla -Xaldon WebSpider -YoudaoBot +Xaldon_WebSpider +Xenu +xpymep1.exe Zade +Zauba +zauba.io zgrab -Zitebot diff --git a/.dev-tools/test_units/random-referrers-for-test-quick.list b/.dev-tools/test_units/random-referrers-for-test-quick.list index 753434f08..f75640f75 100644 --- a/.dev-tools/test_units/random-referrers-for-test-quick.list +++ b/.dev-tools/test_units/random-referrers-for-test-quick.list @@ -1,10 +1,10 @@ -azazu.ru -boobsimge.com -digilander.libero.it -facebook-mobile.xyz -himazin.info -pornodojd.ru -socialbookmarksubmission.org -taqplayer.info -theautoprofit.ml -vip-file.com +bedcapdealers.com +frighteningremain.cf +investzalog.ru +konkursov.net +letmacwork.world +msk-diplomat.com +pospr.waw.pl +top-deal.com.pl +wesharepics.info +zdesformula.ru diff --git a/.dev-tools/test_units/random-referrers-for-test.list b/.dev-tools/test_units/random-referrers-for-test.list index 92ba3ff48..0c0345493 100644 --- a/.dev-tools/test_units/random-referrers-for-test.list +++ b/.dev-tools/test_units/random-referrers-for-test.list @@ -1,500 +1,500 @@ +00go.com +03p.info +100dollars-seo.com 101raccoon.ru -11pikachu.ru -1688.com -1j7740kd.website -34.gs -40cg.com -57883.net -5tey7463.weebly.com -aab94f698f36684c5a852a2ef272e031bb.com -abcguru.xyz -abctoppictures.net -actionnooz.com -adclickthru.net -addbin.men -addtoadd.men -adultfriendfinder.com -adultgalls.com -adultwebhosting.info -advancedsoftwaresupport.com -aerogo.com -afftrack001.com -agroeconom.kz -ahmedabadwebs.com -albuteroli.com -allboard.xobor.de -allesohnegirls.net -aludecor.info -alveris.ru +123locker.com +12masterov.com +2020iscoming.info +4inn.ru +9med.net +aa625d84f1587749c1ab011d6f269f7d64.com +aa9bd78f328a6a41279d0fad0a88df1901.com +aarbur.com +acgs.tk +ahrefs.com +alert.scansafe.net +alice.it +aliexpresscashback.club +alive-ua.com +all4invest.info +allpdfmags.net +am15.net am-se.com -antons-transporte.de -apimountainbikei-a.akamaihd.net -apiportalmorecom-a.akamaihd.net +anjalika.co.in +apartment.ru +apimegabrowsebiz-a.akamaihd.net apishops.ru -a.pr-cy.ru -arkkivoltti.net -artel116.ru -asseenontv.ru -autoplate.info -autorn.ru -avitocars.ru +apispringsmartne-a.akamaihd.net +appfastplay.com +aquarium-pro.ru +aramaicmedia.org +arcadepages.com +ardimobileinfo.ml +arenda-avtoprokat-krasnodar.ru +arewater.com +artefakct.com +ascat.porn +asiengirls.net +asrvrep-a.akamaihd.net +audiofree.ru +autodo.info +auto.rusvile.lt +autoua.top avon-ukraine.com -avtolombard-krasnodar.ru -ayakino.net -babespcs.com +ayeartoforget.com +azartniy-bonus.com babs.com.ua -balitouroffice.com -balkanfarma.ru -banki76.ru -bedcapdealers.com -betterscooter.com -bio-market.kz -bizfly.info -bjetjt.com +baladur.ru +balla-boo.se +bank.uz +bannerbank.ru +bazaronline24.ru +bestbuy.ca +bfz.biz +bharatdefencekavach.com +bigcareer.info +binomo.com +bitcoins-live.ru +biznesrealnost.ru +blagovest-med.ru blobar.org bloggedporn.com -blogrankers.com -blog.remote-computer.de -boazpower.com -boltalko.xyz -bombla.org -bongacams.com -bookmark4you.com.biz -bpro1.top -brandbucket.com -brandov.ru -brianhenry.co -buyessay3.blogspot.ru -buynorxx.com -buyparajumpers.online -buytizanidineonline.blogspot.com +blogig.org +blogporn.in +blogsfunky672.weebly.com +blogs.rediff.com +boobsimge.com +bristolhostel.com +brokergid.ru +business-made-fun.com +buzz-porno.info cablecar.us -cand.jp -carson.getenjoyment.net -cejewelry.xyz -centrdebut.ru -c-gainsbourg.com -chatroulette.world -chinese-amezon.com -chocolateslim-en-france.com -choice-credit.ru -cialischmrx.com -ciceros.co -cigarpass.com -clash-clans.ru -claytransformations.info -clickaider.com -clickso.com -clicksotrk.com -club-samodelkin.ru -cognitiveseo.com -coldfilm.ru +calc-for-credit.ru +callawaygolfstore.online +camdolls.com +candlehandmade.com +canyougethighofftizanidine.blogspot.com +carstrends2015.com +catalogs-parts.com +cellfun.mobi +chee-by.biz +chinajerseyswholesalecoupons.com +cicaki.net +classiquebijoux.ru +clayvasedesigns.tk +clksite.com +cmd.kz co.lumb.co -compiko.info -compliance-don.top -compliance-george.xyz -compliance-john.top -compliance-willy.top -cookie-law-enforcement-hh.xyz -copypaste.traderzplanet.in -cottageofgrace.com -countbertwistdisp26.soup.io +confib.ifmo.ru +continent-e.tv +counciltally.com +courtshipgift.com cpabegins.ru -cpajunkies.com cracksplanet.com -crd.clan.su -crystalslot.com -custodieva.ru -customcatchcan.com -customerguru.in -d0t.ru -dailyrank.net -danashop.ru -danceuniverse.ru -decorationspcs.com +craftburg.ru +damedingel.ya.ru +data.vtc.pw +dating-app.ru +dcj-nn.ru +ddsoldes.fr defenderxtactical.com -dekoration.us -deniven.1bb.ru -design-ideas.info -devochki.top -dfwu1019.info -diffbot.com -digitalfaq.com -discountliv.com -discovertreasure-a.akamaihd.net -discovertreasurenow.com -divatraffic.com -dogoimage.com -dojki-hd.com -dosugrostov.site -downloader12.ru +delfin-aqua.com.ua +dgfitness.co +dinosaurus.site +djonwatch.ru +dogclothing.store +dollartree.info +dolohen.com +domain2008.com +dominterior.org +donvito.unas.cz downloadkakaotalk.com -doxyporno.com -druzhbany.ru -dvervmoskvu.ru -e2click.com -eas-seo.com -easyshoppermac.com -e-commerce-seo.com +download-wallpaper.net +drhomes.biz +drivotracker.com +droidlook.net +drpornogratisx.xxx +drunkmoms.net ecommerce-seo.org +electro-prom.com elektronischezi.livejournal.com -elite-sex-finders.com -elvenar.com -elvenmachine.com -empirepoker.com -engines-usa.com -englishdictionaryfree.com -eng-lyrics.com -eropornosex.ru -erotiktreff24.info -essay-edu.biz -euronis-free.com -european-torches.ru -eurosamodelki.ru -exchangeit.gq -ezigarettenshop1.yolasite.com -f012.de +elidelcream.weebly.com +elitepcgames.com +empathica.com +encodable.com +envaseslotusama.com +eropho.com +erotikstories.ru +esnm.ru +exchanges-bet.com +expertblog.info +expertnaya-ocenka.ru +ezigaretteshop2.mywapblog.com +ezigaretteshop.webs.com +fable.in.ua facebook-mobile.xyz -face.hostingx.eu -farmingworm.com -fbdownloader.com -filmi-onlain.info -financeloan.us -firesub.pl -fitfloponline.store -fitness-video.net -fix-website-errors.com -floridamobilebillboards.com -flowwwers.com -fodelsedagspresenter.nu -fok.nl -foxtechfpv.com -franecki.net -free-stock-illustration.com +failingmarriege.blogspot.com +femmesdenudees.com +fickblock18.com +fil.ru +findacheaplawyers.com +fishtauto.ru +floating-share-buttons.com +flprog.com +foodcrafts.website +foxjuegos.com freewebs.com -freshwallpapers.info -frivgame250.com -fuckingawesome.com -gagrasector.ru -gamebackyard.com -gameonasia.com -getpopunder.com -gfaq.ru -gktt.ru -gogps.me -goldadpremium.com -goodnightjournal.com +frequiry.com +fugarif.ga +fungirlsgames.net +fusoradio.info +g33.org +g7m.pl +gabeshop.ru +gambarkatabaru.com +gamedayhouse.com +games.kolossale.ru +gentamicineyedrops.blogspot.com +getaclueamerica.com +gg.zzyjxs.com +girlporn.ru +globetrotting-culture.ru +golfresa.lucania.se +goodwinmetals.co +gopixdatabase.com +gosmeb.ru +gourcy.altervista.org gratuitbaise.com -greatzip.com -greenidesign.co +greatfind-a.akamaihd.net +groupmoney.ru +grupografico-pilar.com.ar +gsasearchenginerankerhelp.com gsmlab.pl -hackers-crackers.tk -hao123.com -hatedriveapart.com -hcate.com -headpress.ru -health-medical-portal.info -hildinghr.se -hit-men.men -hmmm.cz -homeandhealth.ru -homelygarden.com -homik.pw -hotshoppymac.com -htmlcorner.com -humanorightswatch.org -hustoon.over-blog.com -hydropump.su -hyhj.info -hyipmanager.in -ibmdatamanagement.co -iklysha.ml -ilbe.club -ilovevitaly.ru -imagecoolpub.com +g.starmoe.xyz +gta-club.ru +guiadeserraazul.com +hamacapty.com +hasfun.com +hausfrauensex18.com +hawaiielectriclight.com +hccoder.info +hd-filmy.net +hetmanship.xyz +hitcpm.com +hjaoopoa.top +homemade.gq +homemature.net +hop.clickbank.net +hostnow.men +hotblog.top +huhn.altervista.org +ideoworld.org +ido3.com img.wallpaperstock.net -imperia31.ru impisr.ru -importchinacoach-teach.com -indetiske.ya.ru -informatiecentro.be -instasexyblog.com -investingclub.ru -ip-guide.com -iphantom.com -iqoption.pro -izoll.ru -jerseywholesaleelitestore.com -johnrobertsoninc.com -jurajskie.info -kazan.zrus.org +infektsii.com +innodgfdriverhm.aircus.com +insta-add.pro +intellego.info +interfucks.net +interior-stickers.ru +internet-apteka.ru +int.search.tb.ask.com +inzn.ru +iqs.biz.ua +islandminingsupply.wordpress.com +itbc.kiev.ua +iwanttodeliver.com +jolic2.com +jrpmakati.com +junglenet-a.akamaihd.net +justdating.online +kam-dom.ru +kartiniresto.com +karusel-market.ru +kashubadesign.ru +kemerovo.zrus.org keywordglobal.co.za -khafre.us -kicknights.gq -kinomaniatv.pl -kinopolet.net -kmgamex.cf -komputers-best.ru -konturkrasoty.ru -krasivye-devushki.net -labelwater.se -lacasamorett.com -landoftracking.com -larger.io -laxdrills.com -laxob.com -leboard.ru -ledpolice.ru -likesuccess.com -limads.men -littleberry.ru -live-cam6.info -live-sexcam.tk -ljusihus.se -lomb.co +kichenaid.ru +kinofak.net +kinotorka.ru +kirov.zrus.org +kmd-pto.ru +komputernaya-pomosh-moscow.ru +kongoultry.net +konoplisemena.com +kopihijau.info +kozhniebolezni.com +kwzf.net +law-check-eight.xyz +lesbian.xyz +letto.by +lflash.ru +livingcanarias.com lostfilm-online.ru -lotto6888.com -lsp-awak-perikanan.com -machicon-akihabara.info -machicon-ueno.info -macrotek.ru -magicplayer-s.acestream.net -maridan.com.ua -marktforschung-stuttgart.com -marmotstore.online -martlinker.com -mature.free-websites.com -mebeldekor.com.ua -medicineseasybuy.com -mesto-x.com -metabar.ru -methodsmarketing.com -m-google.xyz -millionare.com +ltvperf.com +luckyshop.net.ua +lunamedia.co +luxmagazine.cf +m0r0zk0-krava.ru +magazin-pics.ru +magnetpress.sk +manifestation.betteroffers.review +mantramusic.ru +map028.com +maximilitary.ru +mcnamaratech.com +mebel-alait.ru +mediawhirl.net +medispainstitute.com.au +mex-annushka.ru +mexicotravelnet.com +mikozstop.com mindeyegames.com -minegam.com -mirmedinfo.ru -missis.top -misswell.net -misterjtbarbers.com -mlvc4zzw.space -mobilemedia.md -moesen-ficken.com -moetomnenie.com +minecraft-rus.org +mixx.com +mmog-play.ru +mmoguider.ru +mobplayer.ru +mobsfun.net +modnie-futbolki.net +moi-glazki.ru mokrayakiska.com -monclerjacketsoutlet.pw -monclerjacketsoutlet.win -moomi-daeri.com -moyakuhnia.ru -my-bc.ru -myfreecams.com +monclercheap.xyz +moscow-clining.ru +motherboard.vice.com +mrinsidesales.com +mwtpludn.review +mydirtyhobby.com +mygameplus.com +myhealthcare.com myindospace.com -nailsimg.com -narosty.com -na-telefon.biz -nationalbreakdown.com -negociosdasha.com -networkcheck.xyz +mykings.pw +mymercy.info +myperiod.club +natali-forex.com +neobux-bg.info new-apps.ru +newhairstylesformen2014.com nextlnk12.com -nzfilecloud.weebly.com -offergroup.info -olvanto.ru -olympescort.com -onescreen.cc -online7777.com -onlinemeetingnow.com -openmediasoft.com -open-odyssey.org -ops.picscout.com -orsonet.ru -ozoz.it -pagesense.com -pammik.ru -park.above.com -partner-pop.men -pathwhelp.org -paydayloanslocal.com +nodup.ru +notebook-pro.ru +office-windows.ru +offtime.ru +okel.co +oktube.ru +old-rock.com +onlinemegax.com +online-templatestore.com +online-x.ru +o-o-6-o-o.ru +openfrost.net +ordernorxx.com +orhonit.com +orion-v.com +oynat.info +oyster-green.com +pandarastore.top +paradontozanet.ru +paramountmarble.co.uk +parfusale.se +partner-trustworthy.men +patol01.pw pcads.ru pcimforum.com -pdns.download -pensplan.com -pepperstyle.ru -perfectpracticeweb.com -petsblogroll.com -picsfair.com -pirateday.ru -pobeiranie.pl -pokerniydom.ru -pornozhara.com -pretty-mart.com -printie.com -proekt-gaz.ru -proprostatit.com -prospekt-st.ru -psoriasis-file.trade -pussysaga.com -qpypcx.com -quality-traffic.com -quickcashlimited.com -quickchange.cc -rabotaetvse.ru +picurams.pw +pijoto.net +piski.top +pix-hd.com +pl.id-forex.com +plohaya-kreditnaya-istoriya.ru +polimga.pw +pontiacsolstice.info +popugaychiki.com +popunder.net +pornhub-forum.ga +pornogad.com +praisong.net +pricheski-video.com +privetsochi.ru +prostitutki-kharkova.org +ptr.ruvds.com +putevka24.ru +putitin.me +qualitymarketzone.com +queerspace.com +quelle.ru +quick-offer.com +quizzitch.net +rabot.host.sk +rada.ru +randalljhoward.com +rankscanner.com +ranksignals.com +rapidokbrain.com +real-time-analytics.com redirectme.net -r-e-f-e-r-e-r.com -registry-clean-up.net -relayblog.com -remontgruzovik.ru -rewardit.com -reyel1985.webnode.fr -rff-cfal.info -rfid-locker.co -rixpix.ru -rockingclicks.com -roll123.com -royal-betting.net -rozalli.com -rutor.vip -saecsa.co -salmonfishingsacramentoriver.com -sampleletters.net +registry-cleaner.net +remote-dba.de +rentalcarnavi.info +repeatlogo.co.uk +resant.ru +respublica-otel.ru +retreatia.com +ringtonepartner.com +rique.host.sk +ruf777.com +rusprostitute.com +rvzr-a.akamaihd.net +s8-nowy-wygraj.comli.com sankt-peterburg.nodup.ru -sarafangel.ru -sa-rewards.co.za -satellite.maps.ilovevitaly.com -sayyoethe.blogspot.co.za -sbf441.com -sbwealthsolutions.ca -scanner-josh.top -scanner-julia.top -school-diplomat.ru -scmor.ilxc.cc -searchinterneat-a.akamaihd.net +saratov.xrus.org +scanner-julianna.top +screentoolkit.com +sdelai-prosto.ru +search.xtconnect.com +se.bnt-team.com secret.xn--oogle-wmc.com -sexsearch.com -sfd-chess.ru -sharebutton.org +seemoreresultshu-a.akamaihd.net +semalt.com +seoboxes.com +seorankinglinks.com +servingnotice.com +se-welding.ru +sexcamamateurchat.com +sextracker.com +sexyteens.hol.es +sharpchallenge.com +shemale-sex.net shikiso.info -shodanhq.com -shopcheermakeup.info -sibtest.ru -signx.info +ship-marvel.co.ua +shlyahten.ru +shopsellcardsdumps.com +shopwme.ru sildenafilcitratemed.com -sildenafil-tadalafil.info -silvercash.com -sivs.ru -slomm.ru -smallseotools.com -smartpet.ru -sneakyboy.com -sobecjvuwa.com.ru -soda.media -softlinesolutions.me -sotechco.co -soviet-portal.do.am +sinel.info +site5.com +siteonomy.com +sitevalued.com +s-iwantyou.com +sixcooler.de +skyway24.ru +slim.sellany.ru +s.lollypopgaming.com +sluganarodu.ru +smailik.org +smart-scripts.com +snts.shell-pmr.ru +soc-econom-problems.ru +socialbookmarksubmission.org +social-fun.ru +socialsignals24.com +soheavyblog.com +solinf.co +sortthemesitesby.com spaceshipad.com -space-worry.ml -spammen.de -sprttrack.com -spywarebegone.com -spywarespy.com -srgwebmail.nl -stanthonyscatholicchurch.org -stard.shop -statistici.ro -steamoff.net +spasswelt.net +spb.ru +speechfoodie.com +sport-video-obzor.ru +standardchartered-forex.com +stariy-baku.com +stauga.altervista.org stjamesschool.info -stoki.ru -sudexpert66.ru +stop-gepatit.te.ua +stretchingabuckblog.com +stroicol.net +stroydetali.ru summerlinhomes411.info -swingerseiten.com -tasteidea.com -tattomedia.com -taximytishi.ru -taxi-v-eisk.ru +superlist.biz +svetodiodoff.ru +svnuppsalaorebro.se +sygraem.com +taihouse.ru +tam-gde-more.ru t-bygg.com -telefonsex-ohne0900.net -terraclicks.com -thedownloadfreeonlinegames.blogspot.com +technopellet.gr +telesvoboda.ru +thefotosgratis.eu +thenetinfo.com thepokertimer.com -thetoiletpaper.com -thexart.club -timecrimea.ru -tizanidinecost.blogspot.com -tizanidinedruginteractions.blogspot.com -tizanidinehcl2mgtablet.blogspot.com -top10registrycleaners.com -top-deal.com.pl -torrnada.ru -toyota.7zap.com -trafficjunky.com -trafficreceiver.club -trahic.ru -trubywriting.com -trucri.me -truemfilelj.gq -trumpetedextremes.com -tvorozhnaja-zapekanka-recept.ru -u-cheats.ru -uchebavchehii.ru -ugguk.online -umg-stroy.ru -uni.me -unitexindia.com -universals.com.ua -ural-buldozer.ru -urdoot.win -urll.eu -usadacha.net -vacuumscleaner.com -validdomain.xyz -vapsy.com -vereo.eu -via-energy-cumpara.com -viagra.pp.ua -via-gra.webstarts.com -videochat.bz -videochat.world -vikistars.com -vkontarkte.com -vladhistory.com -voloomoney.com -wareseeker.com -wdfdocando.com -webenlace.com.ar -weblo.com -weboptimizes.com -webs.com -website-speed-up.site -weclipart.com -wejdz-tu.pl -whiteproduct.com -wholesalejerseysgaa.com -whos.amung.us -wikes.20fr.com -wladimirpayen.com -woweb.com.ua -www.kabbalah-red-bracelets.com -xaylapdiendanang.com -xchangetrak.com -xchat26.myfreecams.com -x-lime.com -x-lime.net -xmlinde.com -xn-----6kcaacnblni5c5bicdpcmficy.xn--p1ai -xrus.org -xxxtube69.com -yeartwit.com -ynymnwbm.bloger.index.hr -youradulthosting.com -yourtemplatefinder.com -youtubedownload.org -zaloro.com -za-music.mymobiplanet.com -zarepta.com -zolotoy-lis.ru -zoolubimets.ru -zoosexart.com -zryydi.com -zscalertwo.net -zynax.ua -zytpirwai.net +titan-cloud.life +tizanidine4mgprice.blogspot.com +tizanidineandcipro.blogspot.com +tizanidinehydrochloride2mg.blogspot.com +tizanidineinfo.blogspot.com +tizanidineonline.blogspot.com +tonerbox.kz +tooplay.com +tootoo.to +torrentgamer.net +torture.ml +tpu.ru +trafaret74.ru +trafficmp.com +tsstcorpcddvdwshbbdriverfb.aircus.com +tube8.com +tvand.ru +tvnewsclips.info +tvteleport.ru +twojebook.pl +umityangin.net +uogonline.com +urengoy.pro +urlopener.com +validccseller.com +valmetrundan.se +vegan-foods.us +vezuviy.su +viagra-soft.ru +viandpet.com +video-chat.love +videojam.tv +videositename.com +vip-file.com +vitalads.net +vitanail.ru +viven.host.sk +vkgaleria.com +vkontaktemusic.ru +voditeltrezviy.ru +w3data.co +wait3sec.org +wallpapersdesk.info +wallpaperstock.net +wanker.us +watchmygf.net +wcb.su +webextract.profound.net +webnode.me +website-analytics.online +website-analyzer.info +welck.octopis.com +weprik.ru +whipme.yopoint.in +wordpresscore.com +worst-sites.online +wowas31.ucoz.ru +www.timer4web.com +wygraj-skiny.win +wzgyyq.com +xfluro.com +xlolitka.com +xn--80aafb2a.xn--p1ai +xn-----8kcatubaocd1bneepefojs1h2e.xn--p1ai +xn--e1afanlbnfckd7c3d.xn--p1ai +xn--lifehacer-1rb.com +xsion.net +x-true.info +yogamatsexpert.com +yoopsie.com +youdesigner.kz +yousense.info +yuweng.info +za-fun-offer.com +zakazvzloma.com +zarabotat-na-sajte.ru +zeroredirect12.com +znakom.sibtest.ru +znakomstva-moskva77.ru +zootoplist.com diff --git a/.dev-tools/test_units/random-referrers-for-whitelist-test.list b/.dev-tools/test_units/random-referrers-for-whitelist-test.list index e8686b4dd..5b0bf9cab 100644 --- a/.dev-tools/test_units/random-referrers-for-whitelist-test.list +++ b/.dev-tools/test_units/random-referrers-for-whitelist-test.list @@ -1,250 +1,250 @@ -00webcams.com -123kuma.com -13tabs.com -1bet.com -1pamm.ru -2ads.co.uk -3w24.com -5i2.net -888.com -8jn.dating-app.ru -9med.net -abcdeg.xyz -academiacsmendoza.org -addray.pro -ahrefs.com -alarmobninsk.ru -alessandraleone.com -alfa-img.com -alphacarolinas.com -angigreene.com -animalrank.com -anonymous-redirect.com -apparel-offer.com -a.pr-cy.ru -asophoto.com -autoblogger24.info -avkzarabotok.com -avkzarabotok.info -banksrf.ru -bestkfiledxd.cf -besttorrentknifta.weebly.com -bet365.com -betterhdporn.com -biketank.ga -bimatoprost-careprost.com.ua -biznesrealnost.ru -bjorkbacken.nu -bkns.vn -blog.koreadaily.com -blogsfunky672.weebly.com -bmw.afora.ru -bonzibuddi.com -boole.onlinewebshop.net -braintobe.win -brokergid.ru -buckspar.gq -buqyxa.rincian.info -buyantiviralwp.com -buydissertation.net -cartierloveringreplica.xyz -cartierreplica.win -cdnanalytics.xyz -cerev.info -cete.ru +2345.com +273-fz.ru +4webmasters.org +5elementov.ru +8kisses.eu +abiente.ru +above.com +adprotect.net +adserver-e7.com +advancedsoftwaresupport.com +alienwheel.es +all-number.com +anaksma.info +anal-acrobats.com +appearance-cool.com +approvedlocal.co.za +aquarium-pro.ru +arcteryxsale.online +arenda-yeisk.ru +atlasvkusov.ru +aviapanda.ru +avtocredit-legko.ru +bard-real.com.ua +bio-market.kz +bnt-team.com +boc.kz +brus-vsem.ru +buntube.net +bupropion-sr-150-mg.us +burn-fat.ga +cartujano-pre.de +cheapjerseysa.com cheapmoncler.pw +cheap-pills-norx.com cheapsergiorossi.online -cherrypointplace.ca -chinacheapelitejerseys.com -chinaelitecheapjerseys.com -cjs.com.ru -coaching-netz.info -community.allhiphop.com +chocoslim.pro +cialischsrx.com +cityadspix.com compliance-alex.xyz -compliance-jess.top -compliance-margo.top -craftburg.ru -crystalslot.com -cscwtalkto.site -cs-passion.pl -daptravel.com -davidovic.info -ddsoldes.fr -discountliv.com -domaincheck.io -educontest.net -elektrozigaretten1.tumblr.com -energydiet-info.ru -espaceinventoristes.com -event-tracking.com -executehosting.com -extlinks.com -f07.de -fetroshok.ru -filmi-onlain.info -financehint.eu -finstroy.net -firesub.pl -fitnesspiks.com -forensicpsychiatry.ru -free-social-buttons.com -g33.org -gallerylisting.com -getpopunder.com -girls-ufa.ru -gta-top.ru -hacktougroup.ru -hkdiiohi.skyrock.com -houseofrose.com -huimin764128.com -imagine-ex.co -inboxdollars.com -inet-traffic.com -isistaylorporn.info -itronics.ca -jerseyswholesalechinalimited.com -jewelryandfiligree.com -jjbabskoe.ru -juliadiets.com -jwss.cc -keywordsking.com -khafre.us -kimcurlrvsms.com -kinosed.net -konoplisemena.com -kutikomi.net -laexotic.com -law-check-two.xyz -lesbian.xyz +compliance-jack.top +compliance-john.top +cookielawblog.wordpress.com +cpabegins.ru +cyclobenzaprinevstizanidine.blogspot.com +data1.scopich.com +dbmkfhqk.bloger.index.hr +deniven.1bb.ru +design-ideas.info +dev-seo.blog +de.zapmeta.com +dichvuvesinhhanoi.com +diminishedvalueoforegon.com +dnepropetrovsk.mistr-x.org +docs4all.com +domain2008.com +domainsphoto.com +dostavimvdom.ru +download-of-the-warez.blogspot.com +dwomlink.info +eldiariodeguadalajara.com +essay-edu.biz +eu-cookie-law.info +evening-dating-club.info +exchangeit.gq +excitacao.com +ezigaretteshop2.mywapblog.com +factorynightclub.com +familyholiday.ml +fanrto.com +faracontrol.ir +fastcrawl.com +fastfixing.tech +fellowshipoftheminds.com +fickluder69.com +financeloan.us +footbalive.org +forodvd.com +fuckingawesome.com +gay.adultgalls.com +gearsadspromo.club +generousdeal-a.akamaihd.net +gentamicineyedrops.blogspot.com +germes-trans.com +get-seo-domain.com +ggiaro.com +go2album.com +gratuitxblcodes.com +grizzlysgrill.com +guiadeserraazul.com +gwebtools.com.br +harmonyglen.us +headpharmacy.com +headpress.ru +heartofpayne.xyz +hesteel.pl +heygidday.biz +homeart.space +hyiphunter.org +insomniagamingfestival.com +intellego.info +irkutsk.online-podarki.com +iwantmyfreecash.com +jerseyschinabizwholesale.us +konkursov.net +kopihijau.info +lawyersinfo.org letolove.ru -loadingpages.me -locustdesign.co -lottospring.com -m0r0zk0-krava.ru -machicon-ueno.info -macotool.com -maglid.ru -malwareremovalcenter.com +lignofix.ua +linkwithin.com +liran-locks.com +locatellicorretor.com.br +lol-smurfs.com +louboutinshoes.xyz +luxmagazine.cf +m3gadownload.pl +mailemedicinals.com manuscript.su -marcowebonyodziezowe.pl -marmotstore.online +meetingrainstorm.bid missis.top +misterjtbarbers.com mmostrike.ru -mobplayer.ru -mole.pluto.ro -mouselink.co -myhealthcare.com -myphotopipe.com -narkologiya-belgorod.ru -natali-forex.com -naturalpharm.com.ua -newyorkhotelsmotels.info -ntic.fr -odywpjtw.bloger.index.hr -office2web.com -okonich.com.ua +mowser.com +mp3downloadhq.com +music.utrolive.ru +musirc.com +myshopmatemac.com +myyour.eu +nalogovyy-kodeks.ru +needtosellmyhousefast.com +netpics.org +news-speaker.com +nfvsz.com +noumeda.com +oecnhs.info +okuos.com +olgacvetmet.com onclickpredictiv.com -oneclickfiles.com -onlinemeetingnow.com -osb.se11.ru -outshop.ru -owathemes.com -pavlodar.xkaz.org -pb-dv.ru -pdamods.ru -perl.dp.ua -picurams.pw -pizza-imperia.com -pl-top.pl +onlineserialy.ru +orgasmus-virtual.com +ovirus.ru +panchro.xyz +parlament.biz +pdn-4.com +pirateday.ru +piski.top +pladform.ru +plastjulgranar.se +polimga.pw +popup-hgd.xyz +pornofiljmi.com +pornogig.com pornokajf.com -pozdrawleniya.ru -prefersurvey.net -printingpeach.com -privetsochi.ru -prointer.net.ua -propranolol40mg.blogspot.com -prosperent.com -prostitutki-tyumeni.org -puteshestvennik.com -rankingchart.de -rapidgator-porn.ga -realitykings.com -rednise.com -responsinator.com -rocketchange.ru -rockma.se -roma-kukareku.livejournal.com -sady-urala.ru -saratov.xrus.org -satellite.maps.ilovevitaly.com -scanner-julia.top -scanner-viktor.top -scenicmissouri.us -sdelai-prosto.ru -searchimage.co -selectads.men -sexkontakteao.info -shitmovs.com -shodanhq.com -sicfor.bcu.cc -sierraapps.com -signoredom.com -snjack.info -solicita.info -solinf.co -sptslmtrafms.com -stal-rulon.ru -stjamesschool.info -strv.se -sundrugstore.com -surffoundation.nl -swagbucks.com -swinginwithme.ru -tedy.su -telefonsexi.com -thedownloadfromwarez.blogspot.com -theporndude.com -tiandeural.ru -tiens2010.ru -tizanidine6mg.blogspot.com -tizanidinehcl4mgtab.blogspot.com -tomatis.gospartner.com -toon-families.com -toyota.7zap.com -tozup.com -tracfone.com -trumpetedextremes.com -tuckpointingmasonrysystems.com -uac.net.au -uasb.ru -ua.tc -u-cheats.ru -uggbootsoutletsale.us -umekana.ru -underthesite.com -unitygame3d.com -universals.com.ua -upupa.net -uvozdeckych.info -valueclick.com +pornophoto.xyz +powc.r.ca.d.sendibm2.com +priscilarodrigues.com.br +prokotov.com +prombudpostach.com.ua +psoriasis-file.trade +quelle.ru +ratemodels.net +regdefense.com +remedyotc.com +remont-comp-pomosh.ru +remont-komputerov-notebook.ru +remontvsamare.su +rennlist.com +rentalcarnavi.info +reversing.cc +ribieiendom.no +rimedia.org +ru-dety.ru +rusenvironmental.net +sa-live.com +sandhillsonline.com +scanner-olga.top +sc-specialhost.com +see-your-website-here.com +seocheki.net +servisural.ru +sexobzor.info +sexpornotales.com +sex-tracker.com +sexystrippe.info +shemale-sex.net +siha.de +sitebeam.net +slayerlife.com +sledstvie-veli.net +slujbauborki.ru +snomer1.ru +snsdeainavi.info +social-s-ggg.xyz +sosdepotdebilan.com +space4update.pw +spitfiremusic.com +suchenindeutschland.com +surfbuyermac.com +swinon.site +taqywu51.soup.io +template-kid.com +tengohydar.tk +texbaza.by +thebestphotos.eu +thewomenlife.com +titangel-vietnam.com +tizanidine4mgstreetvalue.blogspot.com +tizanidineandgabapentin.blogspot.com +tizanidineandhydrocodone.blogspot.com +traffic2money.com +traffpartners.com +treasuretrack-a.akamaihd.net +twittrading.com +ufa.xrus.org +uggsale.online +umaseh.com +umg-stroy.ru +uroki.net +validccseller.com +varbergsvind.se +vavilone.com +vbabule.net veselokloun.ru -videochat.life -videooko.weebly.com -vodaodessa.com -vut.com.ru -vzube.com -webnode.me -withstandingheartwarming.com -word-vorlagen.net -x-diesel.biz -xn-----6kcaabbafhu7cskl7akvongwpo7hvjj.xn--p1ai -xn-----7kceclhb4abre1b4a0ccl2fxch1a.xn--p1ai -xn----7sbaaabaei0cc8aj5bj0bncejx.xn--p1ai -xn--lifehacer-1rb.com -xxxdatinglocal.us -yaoguangdj.com -youtoner.it -youtube-downloader.savetubevideo.com -zakazvzloma.com -zigarettenonlinekaufen1.bloog.pl -zkjovpdgxivg.ga +videochat.cafe +voluumtrk.com +vucms.com +warezaccess.com +watsonrealtycorp.com +wcb.su +webalan.ru +webkeyit.com +wieseversa.no +wildworld.site +winx-play.ru +wonderfulflowers.biz +word-vorlagen.xyz +www.kabbalah-red-bracelets.com +x-diesel.info +xgftnlrt.bloger.index.hr +xn--80ab4aa2g.xn--p1ai +xn--80ajjbdhgmudixfjc8c5a9df8b.xn--p1ai +xn--d1abj0abs9d.in.ua +xtremeeagles.net +xtubeporno.net +yaderenergy.ru +yebocasino.com +yoshkarola.zrus.org +youjizz.vc +yourporn.com +yuweng.info +zhuravlev.info diff --git a/.dev-tools/test_units/ratelimittest-beta.txt b/.dev-tools/test_units/ratelimittest-beta.txt index 525a76fd6..3b5fea028 100644 --- a/.dev-tools/test_units/ratelimittest-beta.txt +++ b/.dev-tools/test_units/ratelimittest-beta.txt @@ -19,6 +19,13 @@