diff --git a/.dev-tools/beta_conf_files/conf.d/globalblacklist.conf b/.dev-tools/beta_conf_files/conf.d/globalblacklist.conf index e8454380d..14f9269bb 100644 --- a/.dev-tools/beta_conf_files/conf.d/globalblacklist.conf +++ b/.dev-tools/beta_conf_files/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2019.07.1715 -### Updated: Mon Jul 1 12:23:33 SAST 2019 +### Version: V3.2019.07.1716 +### Updated: Mon Jul 1 12:41:00 SAST 2019 ### Bad Referrer Count: 6717 ### Bad Bot Count: 556 ################################################### diff --git a/.dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf b/.dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf index 966492772..479de1217 100644 --- a/.dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf +++ b/.dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -35.184.96.71 1; +35.192.85.2 1; 127.0.0.1 1; diff --git a/.dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf b/.dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf index 2f692b069..d9f0a2c04 100644 --- a/.dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf +++ b/.dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -35.184.96.71 0; +35.192.85.2 0; 127.0.0.1 0; diff --git a/.dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf b/.dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf index e8454380d..14f9269bb 100644 --- a/.dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf +++ b/.dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2019.07.1715 -### Updated: Mon Jul 1 12:23:33 SAST 2019 +### Version: V3.2019.07.1716 +### Updated: Mon Jul 1 12:41:00 SAST 2019 ### Bad Referrer Count: 6717 ### Bad Bot Count: 556 ################################################### diff --git a/.dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf b/.dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf index 966492772..479de1217 100644 --- a/.dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf +++ b/.dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -35.184.96.71 1; +35.192.85.2 1; 127.0.0.1 1; diff --git a/.dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf b/.dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf index 2f692b069..d9f0a2c04 100644 --- a/.dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf +++ b/.dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -35.184.96.71 0; +35.192.85.2 0; 127.0.0.1 0; diff --git a/.dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf b/.dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf index e8454380d..14f9269bb 100644 --- a/.dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf +++ b/.dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2019.07.1715 -### Updated: Mon Jul 1 12:23:33 SAST 2019 +### Version: V3.2019.07.1716 +### Updated: Mon Jul 1 12:41:00 SAST 2019 ### Bad Referrer Count: 6717 ### Bad Bot Count: 556 ################################################### diff --git a/.dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf b/.dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf index 966492772..479de1217 100644 --- a/.dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf +++ b/.dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -35.184.96.71 1; +35.192.85.2 1; 127.0.0.1 1; diff --git a/.dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf b/.dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf index 2f692b069..d9f0a2c04 100644 --- a/.dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf +++ b/.dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -35.184.96.71 0; +35.192.85.2 0; 127.0.0.1 0; diff --git a/.dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf b/.dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf index e8454380d..14f9269bb 100644 --- a/.dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf +++ b/.dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2019.07.1715 -### Updated: Mon Jul 1 12:23:33 SAST 2019 +### Version: V3.2019.07.1716 +### Updated: Mon Jul 1 12:41:00 SAST 2019 ### Bad Referrer Count: 6717 ### Bad Bot Count: 556 ################################################### diff --git a/.dev-tools/globalblacklist-testing.template b/.dev-tools/globalblacklist-testing.template index e8454380d..14f9269bb 100644 --- a/.dev-tools/globalblacklist-testing.template +++ b/.dev-tools/globalblacklist-testing.template @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2019.07.1715 -### Updated: Mon Jul 1 12:23:33 SAST 2019 +### Version: V3.2019.07.1716 +### Updated: Mon Jul 1 12:41:00 SAST 2019 ### Bad Referrer Count: 6717 ### Bad Bot Count: 556 ################################################### diff --git a/.dev-tools/globalblacklist.template b/.dev-tools/globalblacklist.template index f2dc5ac9d..dfe11992c 100755 --- a/.dev-tools/globalblacklist.template +++ b/.dev-tools/globalblacklist.template @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2019.07.1715 -### Updated: Mon Jul 1 12:23:32 SAST 2019 +### Version: V4.2019.07.1716 +### Updated: Mon Jul 1 12:40:58 SAST 2019 ### Bad Referrer Count: 6717 ### Bad Bot Count: 556 ################################################### diff --git a/.dev-tools/test1_conf_files/bots.d/whitelist-ips.conf b/.dev-tools/test1_conf_files/bots.d/whitelist-ips.conf index 609536562..cb2425fb6 100644 --- a/.dev-tools/test1_conf_files/bots.d/whitelist-ips.conf +++ b/.dev-tools/test1_conf_files/bots.d/whitelist-ips.conf @@ -45,4 +45,4 @@ # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST -35.193.7.13 0; +35.184.96.71 0; diff --git a/.dev-tools/test1_conf_files/conf.d/globalblacklist.conf b/.dev-tools/test1_conf_files/conf.d/globalblacklist.conf index f2dc5ac9d..dfe11992c 100644 --- a/.dev-tools/test1_conf_files/conf.d/globalblacklist.conf +++ b/.dev-tools/test1_conf_files/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2019.07.1715 -### Updated: Mon Jul 1 12:23:32 SAST 2019 +### Version: V4.2019.07.1716 +### Updated: Mon Jul 1 12:40:58 SAST 2019 ### Bad Referrer Count: 6717 ### Bad Bot Count: 556 ################################################### diff --git a/.dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf b/.dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf index 479de1217..465ec1626 100644 --- a/.dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf +++ b/.dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -35.192.85.2 1; +35.188.1.99 1; 127.0.0.1 1; diff --git a/.dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf b/.dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf index d9f0a2c04..290c48e19 100644 --- a/.dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf +++ b/.dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -35.192.85.2 0; +35.188.1.99 0; 127.0.0.1 0; diff --git a/.dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf b/.dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf index f2dc5ac9d..dfe11992c 100644 --- a/.dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf +++ b/.dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2019.07.1715 -### Updated: Mon Jul 1 12:23:32 SAST 2019 +### Version: V4.2019.07.1716 +### Updated: Mon Jul 1 12:40:58 SAST 2019 ### Bad Referrer Count: 6717 ### Bad Bot Count: 556 ################################################### diff --git a/.dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf b/.dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf index 479de1217..465ec1626 100644 --- a/.dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf +++ b/.dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -35.192.85.2 1; +35.188.1.99 1; 127.0.0.1 1; diff --git a/.dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf b/.dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf index d9f0a2c04..290c48e19 100644 --- a/.dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf +++ b/.dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -35.192.85.2 0; +35.188.1.99 0; 127.0.0.1 0; diff --git a/.dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf b/.dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf index f2dc5ac9d..dfe11992c 100644 --- a/.dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf +++ b/.dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2019.07.1715 -### Updated: Mon Jul 1 12:23:32 SAST 2019 +### Version: V4.2019.07.1716 +### Updated: Mon Jul 1 12:40:58 SAST 2019 ### Bad Referrer Count: 6717 ### Bad Bot Count: 556 ################################################### diff --git a/.dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf b/.dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf index 479de1217..465ec1626 100644 --- a/.dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf +++ b/.dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -35.192.85.2 1; +35.188.1.99 1; 127.0.0.1 1; diff --git a/.dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf b/.dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf index d9f0a2c04..290c48e19 100644 --- a/.dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf +++ b/.dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -35.192.85.2 0; +35.188.1.99 0; 127.0.0.1 0; diff --git a/.dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf b/.dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf index f2dc5ac9d..dfe11992c 100644 --- a/.dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf +++ b/.dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2019.07.1715 -### Updated: Mon Jul 1 12:23:32 SAST 2019 +### Version: V4.2019.07.1716 +### Updated: Mon Jul 1 12:40:58 SAST 2019 ### Bad Referrer Count: 6717 ### Bad Bot Count: 556 ################################################### diff --git a/.dev-tools/test2_conf_files/bots.d/whitelist-ips.conf b/.dev-tools/test2_conf_files/bots.d/whitelist-ips.conf index 16a05ac6a..1e63b10f6 100644 --- a/.dev-tools/test2_conf_files/bots.d/whitelist-ips.conf +++ b/.dev-tools/test2_conf_files/bots.d/whitelist-ips.conf @@ -45,5 +45,5 @@ # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST +35.202.145.110 0; 104.154.120.187 0; -35.184.226.236 0; diff --git a/.dev-tools/test2_conf_files/conf.d/globalblacklist.conf b/.dev-tools/test2_conf_files/conf.d/globalblacklist.conf index 0f8a32572..579e0709d 100644 --- a/.dev-tools/test2_conf_files/conf.d/globalblacklist.conf +++ b/.dev-tools/test2_conf_files/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2019.07.1715 -### Updated: Mon Jul 1 12:23:32 SAST 2019 +### Version: V4.2019.07.1716 +### Updated: Mon Jul 1 12:40:58 SAST 2019 ### Bad Referrer Count: 6717 ### Bad Bot Count: 556 ################################################### diff --git a/.dev-tools/test4_conf_files/bots.d/whitelist-ips.conf b/.dev-tools/test4_conf_files/bots.d/whitelist-ips.conf index 51608598f..433084043 100644 --- a/.dev-tools/test4_conf_files/bots.d/whitelist-ips.conf +++ b/.dev-tools/test4_conf_files/bots.d/whitelist-ips.conf @@ -45,6 +45,5 @@ # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST -35.202.145.110 0; +35.188.1.99 0; 35.184.226.236 0; -35.224.112.202 0; diff --git a/.dev-tools/test4_conf_files/conf.d/globalblacklist.conf b/.dev-tools/test4_conf_files/conf.d/globalblacklist.conf index f2dc5ac9d..dfe11992c 100644 --- a/.dev-tools/test4_conf_files/conf.d/globalblacklist.conf +++ b/.dev-tools/test4_conf_files/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2019.07.1715 -### Updated: Mon Jul 1 12:23:32 SAST 2019 +### Version: V4.2019.07.1716 +### Updated: Mon Jul 1 12:40:58 SAST 2019 ### Bad Referrer Count: 6717 ### Bad Bot Count: 556 ################################################### diff --git a/.dev-tools/test5_conf_files/bots.d/whitelist-ips.conf b/.dev-tools/test5_conf_files/bots.d/whitelist-ips.conf index 93e724bb9..4568239e2 100644 --- a/.dev-tools/test5_conf_files/bots.d/whitelist-ips.conf +++ b/.dev-tools/test5_conf_files/bots.d/whitelist-ips.conf @@ -45,6 +45,6 @@ # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST -35.202.145.110 0; -104.154.120.187 0; -35.184.226.236 0; +35.224.112.202 0; +35.188.1.99 0; +35.193.7.13 0; diff --git a/.dev-tools/test5_conf_files/conf.d/globalblacklist.conf b/.dev-tools/test5_conf_files/conf.d/globalblacklist.conf index f2dc5ac9d..dfe11992c 100644 --- a/.dev-tools/test5_conf_files/conf.d/globalblacklist.conf +++ b/.dev-tools/test5_conf_files/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2019.07.1715 -### Updated: Mon Jul 1 12:23:32 SAST 2019 +### Version: V4.2019.07.1716 +### Updated: Mon Jul 1 12:40:58 SAST 2019 ### Bad Referrer Count: 6717 ### Bad Bot Count: 556 ################################################### diff --git a/.dev-tools/test_units/blacklist-ips.conf b/.dev-tools/test_units/blacklist-ips.conf index 966492772..479de1217 100644 --- a/.dev-tools/test_units/blacklist-ips.conf +++ b/.dev-tools/test_units/blacklist-ips.conf @@ -1,2 +1,2 @@ -35.184.96.71 1; +35.192.85.2 1; 127.0.0.1 1; diff --git a/.dev-tools/test_units/random-bots-for-test-quick.list b/.dev-tools/test_units/random-bots-for-test-quick.list index 18745877a..309207a99 100644 --- a/.dev-tools/test_units/random-bots-for-test-quick.list +++ b/.dev-tools/test_units/random-bots-for-test-quick.list @@ -1,10 +1,10 @@ -Bolt -Claritybot -LinqiaMetadataDownloaderBot -Nmap -Nutch -RankingBot2 -SiteExplorer -spyfu -SurveyBot -WISENutbot +404enemy +Brandwatch +BuiltWith +CopyRightCheck +Deusu +DomainStatsBot +MJ12bot +Siteimprove +UnisterBot +WASALive-Bot diff --git a/.dev-tools/test_units/random-bots-for-test.list b/.dev-tools/test_units/random-bots-for-test.list index 5789900b5..8c9ee7bcc 100644 --- a/.dev-tools/test_units/random-bots-for-test.list +++ b/.dev-tools/test_units/random-bots-for-test.list @@ -1,250 +1,250 @@ -360Spider -80legs +404enemy +Aboundex Acunetix -AIBOT +AhrefsBot AiHitBot Aipbot Alexibot -Anarchie +Alligator +AllSubmitter +archive.org_bot ASPSeek -Asterias -Attach +autoemailspider +Backlink-Ceck backlink-check -BacklinkCrawler -BackWeb -Bandit +Barkrowler BatchFTP Battleztar Bazinga +BBBike +BDCbot BetaBot Bigfoot +Blackboard +Black Hole +BlackWidow +BLEXBot Blow -BlowFish +Boardreader Bolt -Brandwatch -Bullseye BuzzSumo -Calculon -CheeseBot +CazoodleBot +Cegbfeieh CherryPicker ChinaClaw Claritybot -Cloud mapping -Cogentbot -CopyRightCheck -Cosmos -CrazyWebCrawler -CSHttp -Curious +com.plumanalytics +Copyscape +Craftbot +crawler4j +crawler.feedback +Crescent +CrunchBot +Custo DataCha0s +demandbase-bot Demon -Deusu -Devil +Digincore Dirbuster +Discobot Discoverybot Dispatch -DittoSpyder -DomainSigmaCrawler +DomainAppender Dragonfly -DTS Agent EasyDL -Ebingbong -eCatch +ECCP/1.0 Ecxi -EirGrabber +EMail Wolf evc-batch -Express WebPictures -facebookscraper -FemtosearchBot +Exabot +EyeNetIE +Ezooms Fimap -Firefox/7.0 +FlashGet Flunky -Foobot -FyberSpider +Freeuploader GalaxyBot Genieo GermCrawler -Getintent -GetRight -GetWeb G-i-g-a-b-o-t Go-Ahead-Got-It +Gotit GoZilla -GrabNet -Grafula -GridBot -GT::WWW -HaosouSpider +Go!Zilla +Haansoft Harvest +Havij HEADMasterSEO -HTTP::Lite +Heritrix +HMView HTTrack -Humanlinks +HybridBot Iblog IDBot Id-search -Image Fetch +Image Sucker +Indy Library InfoNaviRobot -InfoTekies -instabid -Internet Ninja +Intelliseek internetVista monitor -ips-agent Iria -IRLbot +IstellaBot JamesBOT -Jetty +Jbrofuzz +JennyBot JikeSpider Joomla Jorgee Jyxobot Keyword Density +Kozmosbot +Lanshanbot LexiBot Lftp -LibWeb -Lightspeedsystems -Likse -LinkScan +LinkpadBot LinkWalker -Lipperhey +LinqiaMetadataDownloaderBot +LinqiaRSSBot +LinqiaScrapeBot Lipperhey Spider -lwp-request -Magnet -Mag-Net -Majestic12 -Majestic-SEO +lwp-trivial MarkMonitor -Masscan -Mata Hari +MarkWatch +Mass Downloader MauiBot meanpathbot Meanpathbot MeanPath Bot Mediatoolkitbot -mediawords MegaIndex.ru -Metauri -MIIxpc -MJ12bot +Microsoft URL Control Mojeek +MSFrontPage MSIECrawler +Msrabot MS Web Services Client Protocol Musobot +Nameprotect Navroad +NearSite Needle Nessus Netcraft -NetMechanic +netEstate NE Crawler Nettrack +Net Vampire +Netvibes NetZIP NextGenSearchBot Nibbler -NimbleCrawler -Ninja -Nmap -NPbot +NICErsPRO +Niki-bot +oBot +Octopus Offline Explorer Offline Navigator -OnCrawl -OrangeSpider -OutfoxBot +PageAnalyzer +Page Analyzer PageGrabber -page scorer +PageScorer +Pandalytics +Panscient pcBrowser Picscout Picsearch PictureFinder -Pimonster -PleaseCrawl -Pockey -POE-Component-Client-HTTP +plumanalytics ProPowerBot -ProWebWalker -Quick-Crawler -RankFlex +Psbot +RankActive +RankingBot2 RankurBot RealDownload -RedesScrapy -RepoMonkey -Ripper +Reaper +Recorder +ReGet RocketCrawler -SalesIntelligent +Rogerbot +ScanAlert +Scanbot scan.lol ScoutJet -Screaming -SearchmetricsBot -Semrush -SemrushBot -SEOkicks +ScreenerBot +Searchestate SEOkicks-Robot SEOlyticsCrawler -SEOprofiler -SeoSiteCheckup +seoscanners SEOstats -serpstatbot -SiteExplorer -Siteimprove -SiteLockSpider +Shodan +Siphon +SISTRIX SiteSucker +Site Sucker +SlySearch SMTBot -Snapbot -Snoopy -Sociscraper -sogouspider -Sosospider -SpankBot +Sogou web spider +SpaceBison +Spammen Spanner +sp_auditbot Spbot +spyfu Sqlmap -Sqlworm -Steeler -Stripper +Sucker Sucuri -SuperHTTP +Surfbot +SurveyBot Swiftbot sysscan -Szukacz -T0PHackTeam T8Abot -Thumbor -TightTwatBot -Titan +TeleportPro +TheNomad +Toata +Tracemyfile +trendiction.com +Turingos +Turnitin TurnitinBot -TwengaBot +Twice Typhoeus -UnisterBot +Upflow URLy Warning -Vagabondo +Vacuum VB Project Virusdie -VoidEYE Voltron Wallpapers/3.0 +WallpapersHD +WASALive-Bot Webalta WebAuto -WebBandit WebCollage Web Collage WEBDAV -Web Fetch +WebEnhancer +Web Enhancer +Web Fuck WebGo IS -WebLeacher +WebImageCollector +WebmasterWorldForumBot +webmeup-crawler WebSauger Web Sauger -Website Quester +WebStripper +WebWhacker WebZIP -Whacker Whatweb +Who.is Bot Widow +WinHTTrack WiseGuys Robot -Woobot -WPScan -WWW-Collector-E -WWW::Mechanize -x22Mozilla +WISENutbot +Wotbox +WWW-Mechanize +Xaldon WebSpider +Xaldon_WebSpider Xenu -xpymep1.exe -YoudaoBot Zade Zermelo Zitebot +ZumBot ZyBorg diff --git a/.dev-tools/test_units/random-bots-for-whitelist-test.list b/.dev-tools/test_units/random-bots-for-whitelist-test.list index 77b12c4bc..61cde4a03 100644 --- a/.dev-tools/test_units/random-bots-for-whitelist-test.list +++ b/.dev-tools/test_units/random-bots-for-whitelist-test.list @@ -1,250 +1,250 @@ -80legs Aboundex +ADmantX +AIBOT AiHitBot -Aipbot -archive.org_bot -ASPSeek -Asterias +Alexibot +AllSubmitter +Anarchie +Apexoo Attach -autoemailspider -BackDoorBot -BackStreet -Barkrowler -BatchFTP +Badass BBBike +BDCbot BDFetch BetaBot +Bigfoot Bitacle Black Hole -BLEXBot BlowFish BotALot +BuiltBotTough BuiltWith -BunnySlippers +BuzzSumo +CCBot +Cegbfeieh +CheeseBot CheTeam Cliqzbot -Cloud mapping Cogentbot -Copier -crawler4j +Collector +Copyscape +Cosmos +Craftbot crawler.feedback +crawl.sogou.com CrazyWebCrawler Crescent -CrunchBot +CSHttp +Curious DatabaseDriverMysqli +DataCha0s DBLBot -Demon -Digincore -DigitalPebble +demandbase-bot +Deusu DIIbot +Dirbuster +Discobot Dispatch -DnyzBot -DomainAppender +DittoSpyder DomainCrawler -Download Wonder +DomainSigmaCrawler Drip DSearch DTS Agent -eCatch ECCP/1.0 -Ecxi -EirGrabber -EMail Wolf -Evil -Exabot +EroCrawler Express WebPictures ExtLinksBot -Extractor ExtractorPro -Extreme Picture Finder Ezooms -facebookscraper -FDM -FHscan -Fimap -Firefox/7.0 -Foobot Freeuploader FrontPage -FyberSpider -Genieo +Fyrebot GermCrawler -GetRight +Getintent Gigablast Gigabot -G-i-g-a-b-o-t Gotit GoZilla -Go!Zilla Grabber +GrabNet Grafula GrapeFX -Havij -HEADMasterSEO -Hloader +GridBot +GT::WWW +Haansoft HMView -HTTP::Lite Humanlinks -IDBot -Id-search -Image Fetch -InfoTekies +HybridBot +Iblog +Image Sucker Intelliseek -Internet Ninja +InternetSeer +internetVista monitor Iria Iskanie -IstellaBot Jbrofuzz -JetCar Jetty -JikeSpider -JOC Web Spider -Joomla -Jorgee -Jyxobot +Kenjin Spider Kozmosbot -LeechFTP LeechGet +LexiBot Lftp -LinkScan -LinqiaRSSBot -LinqiaScrapeBot -Lipperhey +LibWeb +Libwhisker +LinkpadBot +LinqiaMetadataDownloaderBot Lipperhey Spider +Ltx71 +lwp-request LWP::Simple lwp-trivial Magnet +Mag-Net magpie-crawler Mail.RU_Bot -MarkWatch +Majestic-SEO Masscan +Mass Downloader +Mata Hari MauiBot meanpathbot Meanpathbot -MeanPath Bot -Microsoft Data Access -MIIxpc +Mediatoolkitbot +MegaIndex.ru +MFC_Tear_Sample +Microsoft URL Control +Mister PiX +MJ12bot +Morfeus Fucking Scanner Mr.4x3 -MSFrontPage muhstik-scan Name Intelligence +Nameprotect Navroad -Needle +NearSite Nessus NetAnts -NetMechanic -NetSpider -Netvibes -NimbleCrawler -Nimbostratus -Ninja +Netcraft +netEstate NE Crawler +NetZIP +NextGenSearchBot +Nibbler +Niki-bot Nmap -Nutch +oBot Offline Explorer Offline Navigator -OnCrawl -OrangeBot -OutfoxBot -PageAnalyzer -Page Analyzer +Openvas +OpenVAS +OrangeSpider PageGrabber -Panscient -Papa Foto -Pavuk -pcBrowser +page scorer +PageScorer PeoplePal -Picscout -Pimonster -Pump -RankActiveLinkBot +Picsearch +PictureFinder +Pi-Monster +Pixray +PleaseCrawl +plumanalytics +Pockey +POE-Component-Client-HTTP +Probethenet +ProPowerBot +Psbot +PxBroker +Quick-Crawler RankingBot RankingBot2 -RealDownload -Reaper +RebelMouse Recorder ReGet Ripper -RocketCrawler +Rogerbot +s1z.ru +SBIder ScanAlert Scanbot scan.lol -SearchmetricsBot -SemrushBot -SEOkicks -seoscanners +Scrapy +Screaming +Semrush +SeoSiteCheckup +SEOstats serpstatbot -sexsearcher -Siphon +SiteExplorer Siteimprove SiteLockSpider -SiteSnagger -SiteSucker -Sitevigil +Site Sucker +SlySearch +SmartDownload SMTBot Snake -Snapbot Sosospider -Sottopop -SpaceBison -Spammen -Spanner +sp_auditbot +Spbot spyfu Sqlmap Sqworm Steeler -Sucuri -SuperHTTP +Stripper +SuperBot Surfbot SurveyBot -Suzuran -Swiftbot sysscan Szukacz -Teleport -TeleportPro +T0PHackTeam +tAkeOut Telesoft Telesphoreo -trendiction.com -Turingos +TightTwatBot +Toata +Toweyabot +Trendiction +True_Robot Twice Upflow URLy.Warning +Vacuum Vagabondo VB Project -VCI -Virusdie -Voil +VeriCiteCrawler +VoidEYE +Voltron +WallpapersHD +Webalta WebAuto WebCollage -WEBDAV +WebCopier Web Fetch +WebGo IS +WebImageCollector WebLeacher -WebmasterWorldForumBot -webmeup-crawler +WebPix +Web Pix WebReaper WebSauger Web Sauger Webshag -Website Quester -Webster +WebsiteQuester WebStripper WebSucker -WebZIP -WeSEE Whack +Whacker Whatweb -Widow -WiseGuys Robot -WISENutbot +Who.is Bot +WinHTTrack +Wonderbot Woobot Wotbox -WPScan +WWW-Collector-E WWW-Mechanize -x22Mozilla +WWW::Mechanize +WWWOFFLE +x09Mozilla Xenu -xpymep1.exe -Zade -zgrab +Zermelo +Zeus Zitebot ZmEu -ZyBorg diff --git a/.dev-tools/test_units/random-referrers-for-test-quick.list b/.dev-tools/test_units/random-referrers-for-test-quick.list index e7e416ff8..904e87312 100644 --- a/.dev-tools/test_units/random-referrers-for-test-quick.list +++ b/.dev-tools/test_units/random-referrers-for-test-quick.list @@ -1,10 +1,10 @@ -abtasty.com -boleznikogi.com -cheap-pills-norx.com -cscwtalkto.site -megaindex.ru -portnoff.od.ua -porto.abuilder.net -remont-ustanovka-tehniki.ru -ytmnd.com -znaturaloriginal.com +centrumcoachingu.com +club-lukojl.ru +gettpromos.com +magicalfind-a.akamaihd.net +panouri-solare-acoperis.com +pornotubexxx.name +princeadvantagesales.com +sukirgenk.dvrlists.com +wahicbefa31.soup.io +zeroredirect11.com diff --git a/.dev-tools/test_units/random-referrers-for-test.list b/.dev-tools/test_units/random-referrers-for-test.list index 61f16fc81..4985b4db1 100644 --- a/.dev-tools/test_units/random-referrers-for-test.list +++ b/.dev-tools/test_units/random-referrers-for-test.list @@ -1,500 +1,500 @@ -007angels.com -00go.com -1kdailyprofit.me -1stat.ru -2000k.ru -256bit.by +202ch.com +21h2o.com +2345.com +23kw.ru +2pxg8bcf.top 3-letter-domains.net -3rbseyes.com -3weekdiet.com -4k-player.pl -4kplayer.pl -55wmz.ru +40cg.com +5elementov.ru 5kstore.com -5ws.dating-app.ru -7a2rnuey1tw9ar.ru -aa08daf7e13b6345e09e92f771507fa5f4.com -aa8b68101d388c446389283820863176e7.com -abovetherivernc.com -actulite.com +7yue.org +8gold.com +aa9bd78f328a6a41279d0fad0a88df1901.com +acc.eu.org +activecampaign.dreamhosters.com adamoads.com -adhome.biz -adultwebhosting.info -advancedsoftwaresupport.com -advokateg.xyz +adelly.bg +adktrailmap.com +adloads.com +adsloads.com +adultfullhd.com +adverster.com +ad-words.ru +ahrntal.verymes.xyz akita.kz -albumsuper.info -alekseevec.ru -all4invest.ru -allknow.info -allnews.md -alloysteel.ru -alot.com -amazingninja.com -android-style.com -anlimebel.kiev.ua -anonymizeme.pro -apiwebwebgetcom-a.akamaihd.net -apple.com-webbrowsing-security.review -arcadepages.com -artel116.ru -asrvrep-a.akamaihd.net -asseenontv.ru -atlant-auto.info -autoplate.info -autorn.ru -avitocars.ru +alert-fjg.xyz +alert.scansafe.net +alienwheel.es +allsilver925.co.il +angry-fermi-7633.arukascloud.io +apartmentratings.com +apps-analytics.net +apxeo.info +arabgirls.adultgalls.com +arcadeplayhouse.com +arpelsreplica.xyz +articlesdirectoryme.info +artparquet.ru +autobrennero.it +autoloans.com +aviav.org avon-severozapad.ru -avon-ukraine.com -azazaporn.com -azte.ch -bagcionderlab.com -bank.uz -bannerswap.com -battleforupdating.site -bcmp.org -beauty-bracelet.com -bensbargains.net -best-seo-software.xyz -bfz.biz -bg6s0.com -bget.ru +axbocz.net +azbuka-mo.ru +babyfactory.fr +bagsonsale.online +balkanfarma.org +bankcrediti.ru +batanga.net +begalka.xyz +bestofupload.info bigpenisguide.com -billiard-classic.com.ua -biographiya.com -bizlist.com.de -biznesrealnost.ru -bloggen.be -bochemit.com.ua -bolezniorganov.ru -boobsimge.com -bpro1.top -bronzeaid-a.akamaihd.net -bugof.gq -burn-fat.ga -bus-offer.com -buzzurl.jp -bvlgariring.xyz -calc-for-credit.ru -candcstuccoandstone.com -candlewooddentalcentre.com.au -candypeople.se -canrole.com -canvas.gloverid.site -cardsdumps.com -cashonads.com -cdncash.org -cfacarrosserie74.com -chatroulette.world -cheapbarbour.online -christianlouboutinreplica.pw -cialisndbrx.com -ci.ua -clips.ua.ac.be +bio-optomarket.ru +bitcoinremote.com +bjetjt.com +bloggedporn.com +bodybuilding-shop.biz +bonzibuddi.com +boo-arts.com +bosefux.esy.es +bracketsmackdown.com +brainboosting.club +brillianty.info +buehne-fuer-menschenrechte.de +bum.com.ru +buqayy0.livejournal.com +buyclomidonlaine.com +buyessayonline19.blogspot.ru +buyhoverboard.com +buypanicdisorderpill.com +buyscabiescream.com +bytimedance.ru +cardsharp1.ru +cartierlovestore.com +cctva.tv +cementaresearch.se +c-english.ru +cheapestjerseys-wholesale.com +checkhit.com +checkm8.com +choice-credit.ru +christianlouboutinreplica.win +cialischsrx.com +cleansearch.net club-lukojl.ru -club-samodelkin.ru -compliance-andrew.top -compliance-andrew.xyz -compliance-brian.top -compliance-fred.xyz -compliance-irvin.xyz -compliance-ivan.xyz -compliance-john.top -com-supportcenter.website -coolyarddecorations.com -cracksplanet.com -crackzplanet.com -csgo4.win -cs-passion.pl -customboxes4less.com -d2jsp.org -dandiyabeats.in -deanmoore.ie -depositfiles-porn.ga -devochki.top -discountbarbour.online -diusyawiga.tk -doctissimo.fr -domaincheck.io +cmhomestayagency.com +collegerentals.com +compliance-brian.xyz +compliance-george.xyz +compliance-julia.top +compliance-nelson.top +construmac.com.mx +cookielawblog.wordpress.com +countercrazy.com +cpajunkies.com +creativehutindia.com +crynet.cc +cubook.supernew.org +cubs-jerseys.us +customcatchcan.com +custom-product-labels.com +customsua.com.ua +d0t.ru +dalavia.ru +d-black.bz +decorazilla.com +delayreferat.ru +dellalimov.com +detmebel.su +diesel-parts28.ru +dipstar.org +discountliv.com +diy-handmade-ideas.com +djstools.com +dkvorota.ru +domain2008.com +domainsphoto.com domcran.net -domznaniy.ru -dostavka-v-ukrainu.ru -eladkarako.com -elektrischezigarette2.devhub.com -elektronischezigarettekaufen1.myblog.de +download-of-the-warez.blogspot.com +doxysexy.com +druzhininevgeniy63.blogspot.com +dumpsmania24.com +easyshoppermac.com +e-avon.ru +edelstahlschornstein-123.de +ekto.ee +elektrozigaretten1.webs.com eloxal.ru -elstal.com.pl -elvenar.com emediate.eu +emerson-rus.ru enge-fotzen.info -engines-usa.com -en.home-task.com -eralph.tk -esopini.com -espaceinventoristes.com -estelight.ru -este-line.com.ua -euromasterclass.ru +eorogo.top +escortplius.com +escort-russian.com +essayservicewriting.org +eu-cookie-law.blogspot.com +eupornstar.info eurosamodelki.ru -exchangeit.gq -expertnaya-ocenka.ru -ezigarettekaufen2.yolasite.com -ezofest.sk -face.hostingx.eu -familienzahnaerzte.com -fastcrawl.com +euroskat.ru +f012.de +f07.de +fashionavenuegame.com +feeriaclub.ru +fefo.gdn +fenoyl.batcave.net feorina.ru -film-one.ru -food.dtu.dk -footballfarrago.com -foxinsocks.ru -foxtechfpv.com -freakycheats.com -free-share-buttons.blogspot.com -free-share-buttons.top -frvo.alptandem.ru -fungirlsgames.net +fettefrauen.net +fickkontaktehobbyhuren.com +figuringmoneyout.com +findpik.com +floridamobilebillboards.com +fortevidyoze.net +fortwosmartcar.pw +fotoxxxru.com +franecki.net +fr-bearings.ru +freeseedsonline.com +free-social-buttons.xyz +free-video-tool.com +freewlan.info +froggytube.com +fullgirl.ru +funnymama.com +funponsel.com +funtoonez.com +fusoradio.info game-top.su -gazporno.com -ge0ip.org -geoads.com -germetiki.com.ua -ghernnqr.skyrock.com -glavtral.ru -gok-kasten.net -googlepositions.com -gosmeb.ru -gotcher.us -gourcy.altervista.org -gpms.org.my -graphics8.info -grupografico-pilar.com.ar -gsasearchengineranker.pw -gsou.cf -guge.io -gz2.bbsoldes.fr -hamacapty.com -hamptonoaks.ca -herokuapp.com -himazin.info -hledejvshopech.cz -hobbyhuren24.net -house.sieraddns.com -houston-vikings.com -hoverboard360.at -howopen.ru -howtoclean.club -hoztorg-opt.ru -huhn.altervista.org -humanelydrew.com -iboss.com -ido3.com -imgarcade.com -i-midias.net.br -impisr.edunsk.ru -import-sales.com +gamingspark.com +gateway.zscalerone.net +gaytube.com +gearcraft.us +generalporn.org +genetworx.com +gettpromos.com +girlsatgames.ru +goldandcard.ru +goldpanningtools.com +gombita.info +goodhousekeeping.com +goodnightjournal.com +goroda-vsego-mira.ru +greendream.com.ua +growboxbl.ru +guardlink.com +hchha.com +hdimagelib.com +hdpixion.com +hdxnxxtube.mobi +hermesbracelets.xyz +heroz.fr +hifidesign.ru +hiwibyh.bugs3.com +horoshieokna.com +hostcritique.com +hothor.se +hotkeys.com +howmuchdoestizanidinecost.blogspot.com +hscsscotland.com +hybrid.ru +ibmdatamanagement.co +ictizanidinehcl4mg.blogspot.com +ikearugs.xyz +ilmen.net +iminent.com +incolors.club incomekey.net -indetiske.ya.ru -indo-export.ru -infazavr.ru -infektsii.com -infospot.pt -intim-uslugi.info -inzn.ru -iscblog.info -itch.io -j33x.com -japfm.com +increasewwwtraffic.info +instabid.tech +intellektmedia.at +intervsem.ru +invitefashion.com +ip-guide.com +ivanovo.zrus.org javcoast.com -javstock.com -jaxcube.info -jerseywholesalechinabiz.com +jerseychinabizwholesale.us +jerseywholesalebizchina.com jongose.ninja +jus0wil.pp.ua +justucalling32211123456789.tk +jyvopys.com kabbalah-red-bracelets.com -kamalsinha.com -kamorel.com -kanimage.com -katadhin.co -kemerovo.zrus.org -keywordhouse.com -kino-filmi.com -kirov.zrus.org -kiwi237au.tk -knowsitall.info -koral.se -kraljeva-sutjeska.com +kadashihotel.com +kazan.xrus.org +kaz.kz +kchaxton.com +kerch.site +ketoanhanoi.info +kidskunst.info +kihi.gdn +kiinomaniak.pl +kinofak.net +kino-fun.ru +kinoplen.ru +kinopolet.net +klejonka.info +komputers-best.ru +kongruan.com +kovrenok.ru ku6.com -kustanay.kz -kvartir-remont.biz -laserpen.club -law-check-eight.xyz -law-check-two.xyz -lazymanyoga.com -ledpolice.ru -lifehacĸer.com +labelwater.se +lakomka.com.ua +lanasshop.ru +lankarns.com +letmacwork.world +lida-ru.com +likesdesign.com +link.ac +liquimondo.com lisque.batcave.net littlecity.ch -livefixer.com -longgreen.info -lookingglassemb.com +lombardfinder.ru lovasszovetseg.hu -lovi-moment.com.ua -lsp-awak-perikanan.com -mackeeper-land-672695126.us-east-1.elb.amazonaws.com -magnetpress.sk -mazda-roadsters.com -mebel-vstroika.ru -meccadumps.net -mediawhirl.net -meduza-consult.ru -megaapteka.ru -megavolt.net.ua -metallosajding.ru -millionare.com -minecraft-rus.org -mini.7zap.com -minyetki.ru -mmog-play.ru -mojpreskumanie.com -morepoweronmac.trade -morlat.altervista.org -mouselink.co -mowser.com -mrinsidesales.com -mug-na-chas-moscow.ru -musicdaddy.net -musicstock.me -mustat.com -mustwineblog.com -muzaporn.com -my-big-family.com -mygameplus.com -nalogovyy-kodeks.ru -naphukete.ru -narco24.me -narkologiya-voronezh.ru -narosty.com -needtosellmyhousefast.com -nelc.edu.eg -netoil.no -news-readers.ru -nibbler.silktide.com -nikitabuch.com -ningessaybe.me -notaria-desalas.com -novosti-hi-tech.ru -nvssf.com -officedocuments.net -old-rock.com -onefilms.net -oneshotdate.com -online-sbank.ru -o-o-6-o-o.com -ostrovtaxi.ru -oynat.info -ozas.net -panouri-solare-acoperis.com -par-fallen.ga -pflexads.com -pila.pl -pinstake.com -pio.polytopesexempt.com -pipki.r.acdnpro.com -pisanieprac.info -pistonclasico.com +love-baby.cz +luciddiagnostics.in +m292.info +mafcards.ru +magicalfind-a.akamaihd.net +makis.nu +map028.com +maxthon.com +mb140.ru +mcar.in.ua +medicine-4u.org +medkritika.ru +menstennisforums.com +mindbox.co.za +minet.club +mirtorrent.net +missis.top +mixx.com +mnogabukaff.net +moivestiy.biz +monclerjacketsoutlet.pw +moneytop.ru +monsterdivx.com +monthlywinners.com +moomi-daeri.com +moscow-region.ru +moz.com +musflashtv.com +muznachas-service.ru +muz-shoes.ru +myanyone.net +mycaf.it +mydeathspace.com +mydownlodablefiles.com +my-floor.in.ua +myhealthcare.com +mylesosibirsk.ru +mywallpaper.top +myyour.eu +nagdak.ru +narkologiya-orel.ru +natali-forex.com +nippon-bearings.ru +niroo.info +nlfjjunb5.ru +novodigs.com +oddamzadarmo.eu +offf.info +offtime.ru +okmusic.jp +oktube.ru +olgacvetmet.com +onemactrckr.com +onlainbesplatno.ru +online-templatestore.com +onload.pw +onlyforemont.ru +o-o-8-o-o.ru +ooomeru.ru +ozoz.it +papasdelivery.ru +parfusale.se +pc4download.co +petitions.whitehouse.gov +pharmacyincity.com +piccdata.com +picsforkeywordsuggestion.com +pihl.se +pillscheap24h.com pizza-tycoon.com -playfortuna-play.ru -play.leadzupc.com -plohaya-kreditnaya-istoriya.ru -pogodnyyeavarii.gq -pogosh.com -poiskzakona.ru -pokemon-go-play.online -popcash.net -popup.matchmaker.com -pornogratisdiario.com -pornoxxx.com.mx -portside.cc -portside.xyz -pregnant.guru -princeadvantagesales.com +platezhka.net +plusnetwork.com +pochemychka.net +podshipniki-ntn.ru +pomoc-drogowa.cba.pl +porno-chaman.info +pornosemki.info +porno.simple-image.com.ua +pornoslive.net +pornotubexxx.name +porto.abuilder.net +pourvous.info +preparevideosafesystem4unow.site +priscilarodrigues.com.br proekt-gaz.ru -promalp-universal.ru -prostitutki-almata.org -prostitutki-belgoroda.org -pro-tec.kz -qwarckoine.com -rank-checker.online +promodj.com +promoforum.ru +pukaporn.com +putitin.me +quit-smoking.ga +rankexperience.com ratemodels.net -rcb101.ru -r-control.ru -refads.pro -registry-cleaner.net -relayblog.com -remote-dba.de -responsinator.com -ric.info -rightenergysolutions.com.au -roleforum.ru -ruclicks.com +razyboard.com +real-time-analytics.com +redirectingat.com +registrationdomainsite.com +registrysweeper.com +remmling.de +remontvsamare.su +remorcicomerciale.ro +rentehno.ru +replicaclub.ru +replicalouboutin.xyz +rethinkwasteni.info +rewardit.com +ringtonepartner.com +robertefuller.com +rossmark.ru ru-dety.ru +rukino.org rumamba.com -s8-nowy-wygraj.comli.com -sabizonline.com -saddiechoua.com -sale-japan.com -saltspray.ru -samo-soznanie.ru +rusenvironmental.net +santechnik.jimdo.com sarf3omlat.com -saugatuck.com -sbornik-zakonov.ru -sbwealthsolutions.ca -scanner-alexa.top -scanner-alex.top +sarm.tk +satellite.maps.ilovevitaly.com +savingsslider-a.akamaihd.net +scanner-andrew.top +scanner-elena.top scanner-julia.top -scanner-olga.top -scottbywater.com -seansonline24.pl -seemoreresultshu-a.akamaihd.net -seksotur.ru -semrush.com +scanner-willy.top +scansafe.net +scat.porn +sc-specialhost.com +sdsjweb.com +searchaddis.com +searchimage.co +semalt.com seo-2-0.com -seo-prof1.xyz -seorank.info -serdcenebolit.com -servethis.com -sevenstars7.com -sexcamamateurchat.com +seocdvig.ru +seocheki.net +serialsway.ucoz.ru +serpstat.com +service-core.ru sexiporno.net -sexphoto.site -sexsaoy.com -sexsearch.com -sexs-foto.com -sextracker.be -sexyteens.hol.es -sharebutton.to -sharpchallenge.com -sherlock.se -shopcheermakeup.info -shopsellcardsdumps.com -shopvilleroyboch.com.ua -sibecoprom.ru -sibvitr.ru +sexs-foto.top +sexy.babes.frontend-stack.top +shiksabd.com +shop.acim.org +shopperifymac.com +silverage.ru +simon3.ru +simply.net simpoed.ufop.br -sims-sims.ru -siteaero.com -slimcdn.com -s.lollypopgaming.com -smadihome.com -smartpet.ru -snomer1.ru -snow.nvr163.com -soc-econom-problems.ru -social-button.xyz -socialbutton.xyz -socialseet.ru -social-s-ggg.xyz -softomix.com -soft-terminal.ru -sovinsteel.ru -stats-public.grammarly.io -stevemonsen.com -stickers-market.ru -streetfooduncovered.com -stretchmate.net -stroyhelp-dv.ru -styro.ru -success-seo.com +sitopreferito.it +skachat-besplatno-obrazcy.ru +skutecznetabletkinaporostwlosow.pl +slkrm.ru +social-search.me +solmarket.by +solution4u.com +spacash.com +spb.afora.ru +spb.ru +sport7777.net +spravka130.ru +stal-rulon.ru +startwp.org +staticfs.host +stockspmb.info +streha-metalko.si +stroimajor.ru suchenindeutschland.com -sugarkun.com -suhanpacktech.com -superfish.com -superlist.biz -superstarfloraluk.com -svetodiodoff.ru -takethatad.com -tako3.com -tds-advert005.info -teslathemes.com -texbaza.by -textads.men -thaismartloan.com -thebluenoodle.com -thecoolimages.net -thedownloadfreeonlinegames.blogspot.com -theendivechronicles.com -thefotosgratis.eu -the-torrent-tracker.blogspot.com -tiandeural.ru -tizanidineduringpregnancy.blogspot.com -tizanidinehcl2mgtablet.blogspot.com -tizanidinemusclerelaxant.blogspot.com -tmm-kurs.ru -token-lab.org -topbestgames.com +supermama.top +suralink.com +suture.co +taaaak.com +tattoo33.ru +td-l-market.ru +tds-advert002.info +techart24.com +technika-remont.ru +telefonsexsofort.tk +templates.franklinfire.co +theautoprofit.ml +themeforest.net +thewomenlife.com +threecolumnblogger.com +timecrimea.ru +tizanidine4mg.blogspot.com +tizanidineandhydrocodone.blogspot.com +tizanidinevssoma.blogspot.com topquality.cf -torrents.cd -tracklead.net -tracksz.co -traderzplanet.in -trafaret74.ru -traffic2money.com -traffictrade.life -trustedmaccleaner.com -tt-ipd.info -tupper-shop.ru -turbodsp.com -twojebook.pl -typer.one -uggbootsoutletsale.us -ukkelberg.no -underthesite.com -unionmarkt.de -ussearche.cf -usswrite.com -v720hd.ru -vacances-voyages.info -valmetrundan.se -venta-prom.ru -vertaform.com -video-camer.com -video-chat.love -video-woman.com -vip-parfumeria.ru -visa-pasport.ru -vitanail.ru -vod.com.ua -vrotike.ru +torontoplumbinggroup.com +tracfone.com +track-rankings.online +tula.howotorg.ru +tv-spoty.info +twodollarshows.com +uggsale.online +uk-zheu20.ru +ural-buldozer.ru +userequip.com +usiad.net +varikoz24.com +vashsvet.com +veerotech.com +veopornogratis.xxx +via-energy-cumpara.com +viagra-soft.ru +viagroid.ru +videos-for-your-business.com +vip-file.com +vipromoffers.com +vksaver-all.ru +vvpg.ru +vysigy.su wallpapers-best.com -wareseeker.com -watchinf.com -webkeyit.com -website-speed-checker.site -wheelchairliftsarea.com -wikes.20fr.com -wjgony.com -worldhistory.biz -world-mmo.com -wzgyyq.com -x-diesel.info -xkaz.org -xn-----7kceclhb4abre1b4a0ccl2fxch1a.xn--p1ai -xn----7sbaphztdjeboffeiof6c.xn--p1ai -xn--b1adccaf1bzj.xn--p1ai -xnxx-n.com -x-stars.ru -xxxnatelefon.ru -yoshkarola.zrus.org -youjizz.com -yourtemplatefinder.com -you-shall-not-pass.is74.ru +watchmygf.net +waterpurifier.club +we-are-gamers.com +webenlace.com.ar +webinstantservice.com +weboptimizes.com +westen-z.life +wholesalejerseys-cheapest.com +wholinkstome.com +whosonmyserver.com +womens-journal.net +www.event-tracking.com +xchangetrak.com +x-diesel.com +xdoza.com +xep.info +xgames-04.com +xn-----6kcaacnblni5c5bicdpcmficy.xn--p1ai +xn----7sbahjd3btneuw1joc.xn--p1ai +xn--80aaagvmjabrs1aoc9luc.xn--p1ai +xn--80ahdheogk5l.xn--p1ai +xn--d1abj0abs9d.in.ua +xrus.org +xxlargepop.com +y8games-free.com +yellowstonevisitortours.com +youbrainboost.asia +yourads.website +youresponsive.com +yousense.info ypmuseum.ru -zaimite.ru -zapatosenventa.info -zarepta.com -zero1.it -zhacker.net -zlatnajesen.com -znakom.sibtest.ru -zoompegs.com -zrus.org +yugo-star.ru +zeroredirect9.com +zetmaster.ru +zhorapankratov7.blogspot.com +zigarettenonlinekaufen1.blox.pl +znaturaloriginal.com +zog.link +zojirushi-products.ru +zolotoy-lis.ru +zverokruh-shop.cz diff --git a/.dev-tools/test_units/random-referrers-for-whitelist-test.list b/.dev-tools/test_units/random-referrers-for-whitelist-test.list index e8767bf99..515aaa43c 100644 --- a/.dev-tools/test_units/random-referrers-for-whitelist-test.list +++ b/.dev-tools/test_units/random-referrers-for-whitelist-test.list @@ -1,250 +1,250 @@ -1hwy.com -1millionusd.xyz -1pamm.ru -4kplayer.pl -aa14ab57a3339c4064bd9ae6fad7495b5f.com -abwa.tk -adrunnr.com -ads-cool.pro -ads-seo.men -aerogo.com -alert-hgd.xyz -all4wap.ru -alldownload.pw -allmarketsnewdayli.gdn -amateurgalls.com -animenime.ru -apccargo.com -apiappenableinfo-a.akamaihd.net -apilinkswiftco-a.akamaihd.net -arabgirls.adultgalls.com -architecturebest.com -arendatora.ru -autoblogger24.info -babyfactory.fr -backuperwebcam.weebly.com -bad-stars.net -bagsonsale.online -bannerads.de -bannerconnect.net -beauty-bracelet.com -belinka.com.ua -bharatdefencekavach.com -bioinnovate.co -bjanshee.ru -bkns.vn -blog.yam.com -bmw-ark.ru -brandehk.dk -brianhenry.co -buron.pw -businesn.men -businesxxl.com -buyessayonline19.blogspot.ru -camel-beach.com -cbb1smartlist12.click -cellfun.mobi -cementaresearch.se -changde.58.com -christianlouboutinsaleonline.us -cialischstgerts.com -cleanpcnow.com -clickcash.com -construmac.com.mx -coolgamechannel.com -covetnica.com -crackzplanet.com -culmimg.pw -cyse.tk -dailyfinancefix.com -dating-time-now.com -delayreferat.ru -detmebel.su -digitalfaq.com -digital-sale.su -dikx.gdn -doeco.ru -domini.cat -downloader12.ru -download-wallpaper.net -dpihatinh.gov.vn -drhomes.biz -drillsaw.ru -drugspowerstore.com -e705.net -editors.choice6912650.hulfingtonpost.com -educatemetv.com -edudocs.net -ekobata.ru -eloconcream.blogspot.com -elstal.com.pl -engines-usa.com -extlabs.io -eyemagination.com -femdom.twiclub.in -fifa-coins.online -fordsonmajbor.cf -forsex.info -fotosfotos.eu -freeseedsonline.com -futbolkisales.ru -fxgallery.com -ge0ip.net -ghernnqr.skyrock.com -glass-msk.ru -go2jump.org -googlepositions.com -graphid.com -groupmoney.ru -guruofcasino.com -gyffu.com +101billion.com +123movies.love +1adult.com +1-free-share-buttons.com +1webmaster.ml +2000k.ru +2015god.org +23kw.ru +2girls1cup-free.com +7yue.org +aa9bd78f328a6a41279d0fad0a88df1901.com +adexprts.com +ads.tremorhub.com +agreda.pluto.ro +aihelen.net +allnews24.in +anjalika.co.in +apidiamondatanet-a.akamaihd.net +apiluckyleapnet-a.akamaihd.net +apioutoboxnet-a.akamaihd.net +app-ready.xyz +approved.su +arenda-yeisk.ru +aruplighting.com +ausergrubhof.info +azadnegar.com +azbukafree.com +belstaffstore.online +bigames.online +bigcareer.info +bjgugu.net.ua +bkgr.se +blogqpot.com +bolezniorganov.ru +btvn.ru +buqyxa.rincian.info +business-made-fun.com +buy-loft.ru +carabela.com.do +cercacamion.it +cfjump.com +cheapmarmot.online +choice-credit.ru +codq.info +communisave.co.za +cougfan.info +cranly.net +creditwell.ru +dalavia.ru +damasarenai.info +date-withme.com +dayibiao.com +degerlund.net +dfwu1019.info +discovertreasure-a.akamaihd.net +djekxa.ru +dktr.ru +donna7753191.ru +download-of-the-warez.blogspot.com +earn-from-articles.com +ec-file.info +ecookna.com.ua +eduserver.net +ekto.ee +electrik-avenue.com +embedle.com +enews.tech +escortplius.com +esfchat.tk +euronis-free.com +ezigarettekaufen2.blox.pl +falllow.gq +fatmaelgarny.com +floating-share-buttons.com +formulaantiuban.com +forpackningsutveckling.se +forum-engineering.ru +forums.d2jsp.org +free-fb-traffic.com +free-share-buttons.com +gamebackyard.com +games.kolossale.ru +gaygalls.net +gayxperience.com +gaz-voshod.ru +gembird.com +get-your-social-buttons.info +gk-atlant.info +glavtral.ru +golden-catalog.pro +good-mummy.ru +gpirate.com +graphics8.info +gribkovye-zabolevaniya.com +growshop.es hacktougroup.ru +hawaiielectriclight.com hdseriale.pl -headpharmacy.com -healmytrauma.info -health-medical-portal.info -heartofpayne.xyz -hidefiles.org -hidemyass.com -hit-kino.ru -hol.es -homelygarden.com -hotgirlhdwallpaper.com +healing-dysplasia.ru +himazin.info +hkdiiohi.skyrock.com +hothor.se hut1.ru -istizanidineanarcoticdrug.blogspot.com -it-max.com.ua -ivoiretechnocom.ci -jerseywholesalebizchina.com -journeydownthescale.info -justbcause.com -kambasoft.com -kino-filmi.com -kountrylife.com -ktotut.net +igithab.com +imagecoolpub.com +imagine-ex.co +i-midias.net.br +indiasourcemart.in +iqoption.com +ivanovo.zrus.org +izoll.ru +javxxx18.com +jennyfire.ru +jeremyeaton.co +jerseychinabizwholesale.com +jestr.org +kamalsinha.com +kino-key.info +kinosed.net +knigonosha.net +kunstaktien.info +ladov.ru +lazymanyoga.com ledpolice.ru -linkpulse.com -linkwithin.com -lostfilm-online.ru -louisvuittonoutletstore.net -lunchrock.co -lyngdalhudterapi.no +lion.bolegapakistan.com +lkbennettoutlet.online +loadopia.com +locationdesaison.com +lombardfinder.ru +lovi-moment.com.ua +magda-gadalka.ru +makedo.ru manifestation.betteroffers.review -maranbrinfo.com.br -marsgatan.com -master-muznachas.ru -mature.free-websites.com -medispainstitute.com.au -mindeyegames.com -mir-limuzinov.ru -mirtorrent.net -mirzonru.net -morefastermac.trade -morepoweronmac.trade -morlat.altervista.org -mrcsa.com.au +medicines-choice.com +mhi-systems.ru +mielec.pl +minecraft-neo.ru +mir-betting.ru +mirmedinfo.ru +mncrftpcs.com +mobot.site +moncleroutletonline.xyz +mosrif.ru music.utrolive.ru -muznachas-service.ru -mylida.org -net-radar.com -niki-mlt.ru -nl.netlog.com -nodding-passion.tk -no-fuel.org -nzfilecloud.weebly.com -o00.in -oecnhs.info -okuos.com -onclickpredictiv.com -online-hd.pl -ontargetseo.us -ooo-olni.ru -openmultipleurl.com -oralsexfilme.net -osnova3.ru -owohho.com -partners-ship.pro -pattersonsweb.com -pcgroup.com.uy -pdns.cz -perm-profnastil.ru -phormchina.com +musicvidz.ru +mycaf.it +myhealthcare.com +nalogovyykodeks.ru +navalwiki.info +newsperuse.com +newstraveller.ru +nextconseil.com +offer.camp +onemantrip.com +ons-add.men +openfrost.net +paceform.com +panchro.xyz +partner-high.men +pc-services.ru pictures-and-images.net -pizza-tycoon.com -pobeiranie.pl -podshipniki-nsk.ru -porngalleries.top -pornoslive.net -princeadvantagesales.com -projefrio.com.br -prosmibank.ru -pukaporn.com -putitin.me -q-moto.ru -quizzitch.net -randki-sex.com -razorweb-a.akamaihd.net -recipedays.com -respublica-otel.ru -ribieiendom.no -rock-cafe.info -royalads.net -rucrypt.com -scanmyphones.com -scanner-jess.top -sell-fb-group-here.com -semprofile.com -serialsway.ucoz.ru -serw.clicksor.com -shijian.ac.cn -shinikiev.com.ua -shop-electron.ru -shoppytoolmac.com -sideeffectsoftizanidine.blogspot.com -sigmund-freud.co.uk -siteimprove.com -social-buttons.com -social-search.me -soft-program.com -soheavyblog.com -space2019.top -spravka-medosmotr.ru -starwars.wikia.com -superfish.com -superinterstitial.com -taaaak.com -taxi-v-eisk.ru -tengohydar.tk -teslathemes.com -testingads.pro -tizanidinenarcotic.blogspot.com -topkarkas.com -torrent-newgames.com -traffic-club.info -traffpartners.com -trk-4.net +plaff-go.ru +plusnetwork.com +pornhive.org +pornhubforum.tk +pornotubs.com +pp-budpostach.com.ua +priscilarodrigues.com.br +rangjued.com +razleton.com +risparmiocasa.bz.it +sax-sex.com +scanner-julianna.top +scanner-viktor.top +seansonline24.pl +security60-e.com +selectads.men +semalt.com +semaltmedia.com +seofied.com +seolab.top +seo-traffic-ranking.info +sexsaoy.com +shodanhq.com +shtaketniki.kz +sirpornogratis.xxx +sitopreferito.it +slowmacfaster.trade +smadihome.com +snsdeainavi.info +socialbutton.xyz +soc-proof.su +sohoindia.net +stats-collector.org +statustroll.com +sundrugstore.com +suralink.com +surflinksmedical.com +szamponrevita.pl +takethatad.com +tampabaywatch.org +teastory.co +tecspb.ru +telefonsexkostenlos.tk +tetracsaudi.com +titelhelden.eu +tizanidinehcl4mgtablet.blogspot.com +tizanidinesleepaid.blogspot.com +toonfamilies.net +top-study.work +tracxn.com +traderzplanet.in +treasuretrack-a.akamaihd.net +twinderbella.com udayavani.com -ufa.xrus.org -ugogo.info -upproar.com -urlopener.com -vesnatehno.com -viagraneggrx.com -video-camer.com -vip-dom.in -webadvance.club -webs.com -wejdz-tu.pl +ugguk.online +uptime.com +uptime-delta.net +validccseller.com +vertaform.com +viagra.pp.ua +viberdownload10.com +viel.su +vipsiterip.org +vizag.kharkov.ua +vvon.co.uk +weblo.com +website-audit.com.ua +website-stealer.nufaq.com wholesalejerseychinaoutlet.com -x5market.ru -x-lime.com -xn--80aodinpgi.xn--p1ai -xn-----8kcatubaocd1bneepefojs1h2e.xn--p1ai -xn--hxazdsfy.blogspot.com -yougotanewdomain.com -youradexchange.com -yourothersite.com -yourtemplatefinder.com -zarabotok--doma.ru -zastenchivosti.net -zigarettenonlinekaufen1.bloog.pl -zigzog.ru +wholinkstome.com +wineitudes.wordpress.com +worldoffiles.ru +worldtraveler.world +writingservices17.blogspot.ru +xcc24.pl +xjlottery.com +xn----7sbbagbq7bd5aheftfllo4m.xn--p1ai +xtrafficplus.com +xxart.ru +yogamatsexpert.com +yoopsie.com +your-bearings.com +zakazfutbolki.com +zero1.it +zixizop.net.ru +z-master.ru +zoodrawings.com +zoominfo.com +zs2vm.top diff --git a/.dev-tools/test_units/ratelimittest-master.txt b/.dev-tools/test_units/ratelimittest-master.txt index f371d9e5f..525a76fd6 100644 --- a/.dev-tools/test_units/ratelimittest-master.txt +++ b/.dev-tools/test_units/ratelimittest-master.txt @@ -1,3 +1,24 @@ + +
This is merely used for testing !!!
- -