diff --git a/.dev-tools/beta_conf_files/conf.d/globalblacklist.conf b/.dev-tools/beta_conf_files/conf.d/globalblacklist.conf index 31007368b..7aeb3fee4 100644 --- a/.dev-tools/beta_conf_files/conf.d/globalblacklist.conf +++ b/.dev-tools/beta_conf_files/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2020.03.2011 -### Updated: Tue Mar 10 13:35:24 SAST 2020 +### Version: V3.2020.03.2014 +### Updated: Tue Mar 10 15:03:56 SAST 2020 ### Bad Referrer Count: 6829 ### Bad Bot Count: 564 ################################################### diff --git a/.dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf b/.dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf index 241e97626..49090b431 100644 --- a/.dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf +++ b/.dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -34.74.253.255 1; +34.74.91.53 1; 127.0.0.1 1; diff --git a/.dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf b/.dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf index 85130057a..272faad99 100644 --- a/.dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf +++ b/.dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -34.74.253.255 0; +34.74.91.53 0; 127.0.0.1 0; diff --git a/.dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf b/.dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf index 31007368b..7aeb3fee4 100644 --- a/.dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf +++ b/.dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2020.03.2011 -### Updated: Tue Mar 10 13:35:24 SAST 2020 +### Version: V3.2020.03.2014 +### Updated: Tue Mar 10 15:03:56 SAST 2020 ### Bad Referrer Count: 6829 ### Bad Bot Count: 564 ################################################### diff --git a/.dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf b/.dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf index 241e97626..49090b431 100644 --- a/.dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf +++ b/.dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -34.74.253.255 1; +34.74.91.53 1; 127.0.0.1 1; diff --git a/.dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf b/.dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf index 85130057a..272faad99 100644 --- a/.dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf +++ b/.dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -34.74.253.255 0; +34.74.91.53 0; 127.0.0.1 0; diff --git a/.dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf b/.dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf index 31007368b..7aeb3fee4 100644 --- a/.dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf +++ b/.dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2020.03.2011 -### Updated: Tue Mar 10 13:35:24 SAST 2020 +### Version: V3.2020.03.2014 +### Updated: Tue Mar 10 15:03:56 SAST 2020 ### Bad Referrer Count: 6829 ### Bad Bot Count: 564 ################################################### diff --git a/.dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf b/.dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf index 241e97626..49090b431 100644 --- a/.dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf +++ b/.dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -34.74.253.255 1; +34.74.91.53 1; 127.0.0.1 1; diff --git a/.dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf b/.dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf index 85130057a..272faad99 100644 --- a/.dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf +++ b/.dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -34.74.253.255 0; +34.74.91.53 0; 127.0.0.1 0; diff --git a/.dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf b/.dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf index 31007368b..7aeb3fee4 100644 --- a/.dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf +++ b/.dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2020.03.2011 -### Updated: Tue Mar 10 13:35:24 SAST 2020 +### Version: V3.2020.03.2014 +### Updated: Tue Mar 10 15:03:56 SAST 2020 ### Bad Referrer Count: 6829 ### Bad Bot Count: 564 ################################################### diff --git a/.dev-tools/globalblacklist-testing.template b/.dev-tools/globalblacklist-testing.template index 31007368b..7aeb3fee4 100644 --- a/.dev-tools/globalblacklist-testing.template +++ b/.dev-tools/globalblacklist-testing.template @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2020.03.2011 -### Updated: Tue Mar 10 13:35:24 SAST 2020 +### Version: V3.2020.03.2014 +### Updated: Tue Mar 10 15:03:56 SAST 2020 ### Bad Referrer Count: 6829 ### Bad Bot Count: 564 ################################################### diff --git a/.dev-tools/globalblacklist.template b/.dev-tools/globalblacklist.template index 1fbd2ee1e..bf462e765 100755 --- a/.dev-tools/globalblacklist.template +++ b/.dev-tools/globalblacklist.template @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2020.03.2011 -### Updated: Tue Mar 10 13:35:23 SAST 2020 +### Version: V4.2020.03.2014 +### Updated: Tue Mar 10 15:03:55 SAST 2020 ### Bad Referrer Count: 6829 ### Bad Bot Count: 564 ################################################### diff --git a/.dev-tools/test1_conf_files/bots.d/whitelist-ips.conf b/.dev-tools/test1_conf_files/bots.d/whitelist-ips.conf index e55d207d1..5f04cdae8 100644 --- a/.dev-tools/test1_conf_files/bots.d/whitelist-ips.conf +++ b/.dev-tools/test1_conf_files/bots.d/whitelist-ips.conf @@ -45,4 +45,4 @@ # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST -35.237.8.208 0; +34.73.34.132 0; diff --git a/.dev-tools/test1_conf_files/conf.d/globalblacklist.conf b/.dev-tools/test1_conf_files/conf.d/globalblacklist.conf index 1fbd2ee1e..bf462e765 100644 --- a/.dev-tools/test1_conf_files/conf.d/globalblacklist.conf +++ b/.dev-tools/test1_conf_files/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2020.03.2011 -### Updated: Tue Mar 10 13:35:23 SAST 2020 +### Version: V4.2020.03.2014 +### Updated: Tue Mar 10 15:03:55 SAST 2020 ### Bad Referrer Count: 6829 ### Bad Bot Count: 564 ################################################### diff --git a/.dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf b/.dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf index b4e441f5b..49090b431 100644 --- a/.dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf +++ b/.dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -35.237.212.185 1; +34.74.91.53 1; 127.0.0.1 1; diff --git a/.dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf b/.dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf index f707957e4..272faad99 100644 --- a/.dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf +++ b/.dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -35.237.212.185 0; +34.74.91.53 0; 127.0.0.1 0; diff --git a/.dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf b/.dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf index 1fbd2ee1e..bf462e765 100644 --- a/.dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf +++ b/.dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2020.03.2011 -### Updated: Tue Mar 10 13:35:23 SAST 2020 +### Version: V4.2020.03.2014 +### Updated: Tue Mar 10 15:03:55 SAST 2020 ### Bad Referrer Count: 6829 ### Bad Bot Count: 564 ################################################### diff --git a/.dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf b/.dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf index b4e441f5b..49090b431 100644 --- a/.dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf +++ b/.dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -35.237.212.185 1; +34.74.91.53 1; 127.0.0.1 1; diff --git a/.dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf b/.dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf index f707957e4..272faad99 100644 --- a/.dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf +++ b/.dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -35.237.212.185 0; +34.74.91.53 0; 127.0.0.1 0; diff --git a/.dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf b/.dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf index 1fbd2ee1e..bf462e765 100644 --- a/.dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf +++ b/.dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2020.03.2011 -### Updated: Tue Mar 10 13:35:23 SAST 2020 +### Version: V4.2020.03.2014 +### Updated: Tue Mar 10 15:03:55 SAST 2020 ### Bad Referrer Count: 6829 ### Bad Bot Count: 564 ################################################### diff --git a/.dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf b/.dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf index b4e441f5b..49090b431 100644 --- a/.dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf +++ b/.dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -35.237.212.185 1; +34.74.91.53 1; 127.0.0.1 1; diff --git a/.dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf b/.dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf index f707957e4..272faad99 100644 --- a/.dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf +++ b/.dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -35.237.212.185 0; +34.74.91.53 0; 127.0.0.1 0; diff --git a/.dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf b/.dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf index 1fbd2ee1e..bf462e765 100644 --- a/.dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf +++ b/.dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2020.03.2011 -### Updated: Tue Mar 10 13:35:23 SAST 2020 +### Version: V4.2020.03.2014 +### Updated: Tue Mar 10 15:03:55 SAST 2020 ### Bad Referrer Count: 6829 ### Bad Bot Count: 564 ################################################### diff --git a/.dev-tools/test2_conf_files/bots.d/whitelist-ips.conf b/.dev-tools/test2_conf_files/bots.d/whitelist-ips.conf index c323b2cee..2b5479faa 100644 --- a/.dev-tools/test2_conf_files/bots.d/whitelist-ips.conf +++ b/.dev-tools/test2_conf_files/bots.d/whitelist-ips.conf @@ -45,5 +45,5 @@ # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST -34.74.253.255 0; -35.237.8.208 0; +35.227.58.83 0; +35.196.82.30 0; diff --git a/.dev-tools/test2_conf_files/conf.d/globalblacklist.conf b/.dev-tools/test2_conf_files/conf.d/globalblacklist.conf index 7672266cc..986e43fa3 100644 --- a/.dev-tools/test2_conf_files/conf.d/globalblacklist.conf +++ b/.dev-tools/test2_conf_files/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2020.03.2011 -### Updated: Tue Mar 10 13:35:23 SAST 2020 +### Version: V4.2020.03.2014 +### Updated: Tue Mar 10 15:03:55 SAST 2020 ### Bad Referrer Count: 6829 ### Bad Bot Count: 564 ################################################### diff --git a/.dev-tools/test3_conf_files/bots.d/whitelist-ips.conf b/.dev-tools/test3_conf_files/bots.d/whitelist-ips.conf index 90131b140..21f4798b7 100644 --- a/.dev-tools/test3_conf_files/bots.d/whitelist-ips.conf +++ b/.dev-tools/test3_conf_files/bots.d/whitelist-ips.conf @@ -45,5 +45,5 @@ # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST -35.237.212.185 0; -35.185.97.135 0; +34.73.66.97 0; +34.73.34.132 0; diff --git a/.dev-tools/test3_conf_files/conf.d/globalblacklist.conf b/.dev-tools/test3_conf_files/conf.d/globalblacklist.conf index 626d89137..9bd5235f8 100644 --- a/.dev-tools/test3_conf_files/conf.d/globalblacklist.conf +++ b/.dev-tools/test3_conf_files/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2020.03.2011 -### Updated: Tue Mar 10 13:35:23 SAST 2020 +### Version: V4.2020.03.2014 +### Updated: Tue Mar 10 15:03:55 SAST 2020 ### Bad Referrer Count: 6829 ### Bad Bot Count: 564 ################################################### diff --git a/.dev-tools/test4_conf_files/bots.d/whitelist-ips.conf b/.dev-tools/test4_conf_files/bots.d/whitelist-ips.conf index f3833dbc6..5bfc5aa1c 100644 --- a/.dev-tools/test4_conf_files/bots.d/whitelist-ips.conf +++ b/.dev-tools/test4_conf_files/bots.d/whitelist-ips.conf @@ -45,6 +45,5 @@ # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST -104.196.213.122 0; -34.74.253.255 0; -35.185.97.135 0; +34.73.34.132 0; +35.231.58.0 0; diff --git a/.dev-tools/test4_conf_files/conf.d/globalblacklist.conf b/.dev-tools/test4_conf_files/conf.d/globalblacklist.conf index 1fbd2ee1e..bf462e765 100644 --- a/.dev-tools/test4_conf_files/conf.d/globalblacklist.conf +++ b/.dev-tools/test4_conf_files/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2020.03.2011 -### Updated: Tue Mar 10 13:35:23 SAST 2020 +### Version: V4.2020.03.2014 +### Updated: Tue Mar 10 15:03:55 SAST 2020 ### Bad Referrer Count: 6829 ### Bad Bot Count: 564 ################################################### diff --git a/.dev-tools/test5_conf_files/bots.d/whitelist-ips.conf b/.dev-tools/test5_conf_files/bots.d/whitelist-ips.conf index 0e435278f..9c130daca 100644 --- a/.dev-tools/test5_conf_files/bots.d/whitelist-ips.conf +++ b/.dev-tools/test5_conf_files/bots.d/whitelist-ips.conf @@ -45,6 +45,6 @@ # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST -35.237.8.208 0; -104.196.213.122 0; -35.237.212.185 0; +35.227.58.83 0; +35.231.58.0 0; +34.73.66.97 0; diff --git a/.dev-tools/test5_conf_files/conf.d/globalblacklist.conf b/.dev-tools/test5_conf_files/conf.d/globalblacklist.conf index 1fbd2ee1e..bf462e765 100644 --- a/.dev-tools/test5_conf_files/conf.d/globalblacklist.conf +++ b/.dev-tools/test5_conf_files/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2020.03.2011 -### Updated: Tue Mar 10 13:35:23 SAST 2020 +### Version: V4.2020.03.2014 +### Updated: Tue Mar 10 15:03:55 SAST 2020 ### Bad Referrer Count: 6829 ### Bad Bot Count: 564 ################################################### diff --git a/.dev-tools/test_units/blacklist-ips.conf b/.dev-tools/test_units/blacklist-ips.conf index 241e97626..49090b431 100644 --- a/.dev-tools/test_units/blacklist-ips.conf +++ b/.dev-tools/test_units/blacklist-ips.conf @@ -1,2 +1,2 @@ -34.74.253.255 1; +34.74.91.53 1; 127.0.0.1 1; diff --git a/.dev-tools/test_units/random-bots-for-test-quick.list b/.dev-tools/test_units/random-bots-for-test-quick.list index ef1af6883..a9d947682 100644 --- a/.dev-tools/test_units/random-bots-for-test-quick.list +++ b/.dev-tools/test_units/random-bots-for-test-quick.list @@ -1,10 +1,10 @@ -DBLBot -InternetSeer -Jyxobot -Mediatoolkitbot -Msrabot -Swiftbot -Trendiction -Webalta -Xaldon_WebSpider -Zauba +AIBOT +Grabber +OrangeSpider +Quick-Crawler +SISTRIX +Sitebeam +Sociscraper +Webster +WebSucker +WPScan diff --git a/.dev-tools/test_units/random-bots-for-test.list b/.dev-tools/test_units/random-bots-for-test.list index cabec7c03..93339124c 100644 --- a/.dev-tools/test_units/random-bots-for-test.list +++ b/.dev-tools/test_units/random-bots-for-test.list @@ -1,250 +1,250 @@ -360Spider +404enemy +Abonti Aboundex -Aboundexbot +Acunetix ADmantX -AiHitBot -Aipbot +AhrefsBot +Alexibot Alligator AllSubmitter -AlphaBot Anarchie -Apexoo -Attach -autoemailspider +archive.org_bot +arquivo.pt AwarioRssBot AwarioSmartBot +BackDoorBot Backlink-Ceck +backlink-check BackStreet +Badass +Barkrowler BatchFTP +Battleztar Bazinga +BBBike BDFetch -Bigfoot -Bitacle -Black Hole -BLEXBot +BetaBot +Blow BlowFish Bolt BotALot -Brandprotect -Brandwatch -BunnySlippers -BuzzSumo -CazoodleBot Cegbfeieh CheeseBot -CherryPicker -Chlooe +CheTeam +Claritybot Cliqzbot Cloud mapping coccocbot-web cognitiveseo +Copier CopyRightCheck crawler4j crawler.feedback +crawl.sogou.com +CrazyWebCrawler +Crescent +CrunchBot CSHttp Curious -DatabaseDriverMysqli +Custo DataCha0s -demandbase-bot Deusu -Devil +Digincore DigitalPebble -Disco -Discoverybot +Dirbuster +Discobot +Dispatch DnyzBot DomainAppender DomainCrawler -Dragonfly +Dotbot DSearch +EasyDL Ebingbong +eCatch ECCP/1.0 -EirGrabber +Ecxi EMail Siphon -evc-batch +EroCrawler Evil -ExtLinksBot -Extractor -Extreme Picture Finder -Ezooms -FDM FemtosearchBot FHscan -Firefox/7.0 -FlashGet +Fimap +Flunky +FrontPage FyberSpider -GermCrawler +Fyrebot +Genieo +Getintent G-i-g-a-b-o-t +Go-Ahead-Got-It Gotit Go!Zilla Grabber -GrabNet -Grafula -GT::WWW -Haansoft -Havij -HEADMasterSEO +GrapeFX +Harvest heritrix Hloader +Humanlinks +HybridBot +Iblog +Image Fetch IndeedBot Indy Library InfoTekies +instabid +Intelliseek Internet Ninja -InternetSeer internetVista monitor -Iskanie -IstellaBot +Iria +IRLbot JamesBOT -Jetty -JikeSpider JOC Web Spider -LeechFTP +Joomla +Jorgee +JustView +Keyword Density +LeechGet LexiBot -Lftp -LibWeb -Lightspeedsystems +Likse +Linkdexbot LinkpadBot -LinkScan +LinksManager LinkWalker +LinqiaMetadataDownloaderBot +Lipperhey Lipperhey Spider -Lmspider -lwp-request +Litemage_walker +LNSpiderguy +Ltx71 +LWP::Simple lwp-trivial +Mag-Net +magpie-crawler +Mail.RU_Bot Majestic12 -Majestic SEO +Majestic-SEO +MarkMonitor Mass Downloader Mata Hari -meanpathbot MeanPath Bot -mediawords -MegaIndex.ru +Metauri MFC_Tear_Sample Microsoft URL Control MIDown tool -MJ12bot +Mojeek Mojolicious -Morfeus Fucking Scanner -Mr.4x3 +MSIECrawler Msrabot muhstik-scan Musobot Name Intelligence -NearSite -NetAnts +Nameprotect +Navroad +Needle +Netcraft netEstate NE Crawler -Nettrack -Netvibes -Nibbler -NICErsPRO -Nikto -Ninja -Nutch -Offline Explorer +Net Vampire +Nimbostratus +oBot OnCrawl -Openfind -OpenVAS +Openvas OrangeBot -OutclicksBot -OutfoxBot -PageAnalyzer +OrangeSpider Page Analyzer -PageGrabber +page scorer Pandalytics -Pavuk +Panscient +Papa Foto pcBrowser -PECL::HTTP PeoplePal PHPCrawl +Picscout Picsearch -PictureFinder Pi-Monster -polaris version -ProWebWalker -Pump -PxBroker -Quick-Crawler -RankFlex -Rankivabot -Recorder +PleaseCrawl +plumanalytics +Pockey +POE-Component-Client-HTTP +Probethenet +PyCurl +RankActiveLinkBot +RankurBot +RealDownload RedesScrapy -RepoMonkey Ripper RocketCrawler Rogerbot -ScreenerBot -SearchmetricsBot -SemrushBot +s1z.ru +SalesIntelligent +ScoutJet +Scrapy +Screaming +Searchestate SEOlyticsCrawler SEOprofiler -seoscanners +SEOstats Shodan SISTRIX -SiteExplorer -Siteimprove +Sitebeam SiteSnagger -SmartDownload -Snake -Snoopy +Site Sucker +Sitevigil +SlySearch +Snapbot Sociscraper +sogouspider SpaceBison Spammen +SpankBot sp_auditbot Spbot -Spinn3r SputnikBot +spyfu Sqlmap -Sqlworm +Stripper Sucker +Sucuri +SuperHTTP SurveyBot Suzuran -Swiftbot Szukacz -T8Abot tAkeOut +Teleport Telesoft -Telesphorep +The Intraformant TheNomad -Thumbor -TightTwatBot Titan Toata -Toweyabot -Trendictionbot trendiction.de -Turingos Turnitin -TwengaBot -Twice -Typhoeus -UnisterBot +TurnitinBot +Upflow URLy Warning -URLy.Warning Vagabondo -VB Project +VeriCiteCrawler VidibleScraper -Voltron +VoidEYE WallpapersHD -Web Auto +Webalta +WebAuto +WebBandit WebCollage -WEBDAV -WebEnhancer -WebFetch -WebFuck -WebGo IS -WebmasterWorldForumBot -Web Pix -WebReaper -Website Quester -Web Sucker -WebWhacker -WebZIP -Whacker +WebCopier +Web Fuck +WebImageCollector +Webster +WebSucker +WeSEE WinHTTrack WISENutbot +Wonderbot Woobot Wprecon -WWW::Mechanize -x22Mozilla +WWW-Collector-E +WWW-Mechanize +WWWOFFLE +Xaldon_WebSpider Zade Zauba Zermelo -Zitebot -ZumBot +ZyBorg diff --git a/.dev-tools/test_units/random-bots-for-whitelist-test.list b/.dev-tools/test_units/random-bots-for-whitelist-test.list index 7ba4e6e0e..b80226bb4 100644 --- a/.dev-tools/test_units/random-bots-for-whitelist-test.list +++ b/.dev-tools/test_units/random-bots-for-whitelist-test.list @@ -1,250 +1,250 @@ -404checker +404enemy +Abonti +Aboundexbot +Acunetix ADmantX AhrefsBot +AIBOT AiHitBot -Ankit -Apexoo +Aipbot arquivo.pt arquivo-web-crawler +ASPSeek Asterias -autoemailspider -BackDoorBot +Attach +AwarioRssBot +AwarioSmartBot Backlink-Ceck +backlink-check BacklinkCrawler -BackStreet -BDCbot +Badass +BatchFTP +BBBike BDFetch -BetaBot -Blackboard Black Hole BlackWidow -BLEXBot Blow -BlowFish +Boardreader +Brandwatch Buddy -BunnySlippers +BuiltWith BuzzSumo -CazoodleBot +CATExplorador +Chlooe Cliqzbot Cloud mapping -coccocbot-web cognitiveseo -com.plumanalytics Copier CopyRightCheck -crawler4j -crawler.feedback +Cosmos crawl.sogou.com -Crescent -CrunchBot -CSHttp -Curious -DataCha0s -DBLBot -Demon +DatabaseDriverMysqli +Deusu Devil -DigitalPebble DIIbot +Discobot Dispatch -DnyzBot +DittoSpyder +DomainSigmaCrawler +DomainStatsBot Dotbot -Download Wonder -Drip +Dragonfly DSearch -eCatch +EasyDL +ECCP/1.0 EMail Siphon +EMail Wolf Evil -Express WebPictures ExtLinksBot +ExtractorPro +Extreme Picture Finder +EyeNetIE +Ezooms FDM -FemtosearchBot -Firefox/7.0 -FlashGet -Freeuploader +FHscan +Flunky +Foobot Fyrebot GalaxyBot -GetRight -GetWeb +Getintent Gigablast -Gigabot +G-i-g-a-b-o-t +Go-Ahead-Got-It Gotit -GoZilla -GrabNet +Go!Zilla +Grabber Grafula +GT::WWW Haansoft HaosouSpider +Harvest Havij -Heritrix +HEADMasterSEO +heritrix +Hloader HMView -HTMLparser -HTTP::Lite HTTrack +Humanlinks +IDBot IlseBot +Image Fetch Image Sucker -IndeedBot -Intelliseek -InternetSeer -IRLbot +InfoNaviRobot +InterGET +Internet Ninja Iskanie JamesBOT +Jbrofuzz +JennyBot +Jetty JikeSpider +Joomla +Jorgee JustView -Keyword Density +Jyxobot +Kenjin Spider Kozmosbot -Larbin +LeechFTP LeechGet -LexiBot -Libwhisker -Lightspeedsystems Likse -Linkdexbot LinkextractorPro LinkpadBot -LinkScan -LinqiaMetadataDownloaderBot LinqiaScrapeBot Lipperhey -Lipperhey Spider +Lmspider LNSpiderguy -LWP::Simple -Magnet -Mag-Net magpie-crawler -Majestic12 Majestic SEO -Majestic-SEO MarkWatch +Mass Downloader +Mata Hari MauiBot -MegaIndex.ru -MFC_Tear_Sample -Microsoft Data Access -Microsoft URL Control +Meanpathbot +mediawords +Metauri +MIDown tool MIIxpc Mister PiX MJ12bot +Mojeek +Mojolicious Morfeus Fucking Scanner -Mr.4x3 -MSFrontPage -Musobot +MSIECrawler +Msrabot +Name Intelligence +Nameprotect Navroad NearSite -NetAnts NetLyzer NetMechanic -Nettrack +NetSpider Net Vampire -Niki-bot Nimbostratus -NPbot -oBot +Ninja +Nmap +Nutch +Octopus OnCrawl +Openfind +OpenLinkProfiler +Openvas OpenVAS OrangeBot OrangeSpider -OutclicksBot +OutfoxBot PageAnalyzer PageGrabber page scorer -PageScorer -Pandalytics +Panscient Papa Foto Pavuk pcBrowser +PECL::HTTP Picsearch -PictureFinder Pockey -polaris version -Probethenet ProWebWalker -Psbot -Pump -PxBroker PyCurl QueryN Metasearch -Quick-Crawler -RankActive -RankActiveLinkBot -Rankivabot +RankingBot RankurBot -Reaper -ReGet +RealDownload +Recorder +RepoMonkey +Ripper RocketCrawler -s1z.ru +SalesIntelligent Scanbot -Scrapy -Screaming ScreenerBot Semrush -SemrushBot -SEOprofiler -SEOstats +Seomoz +serpstatbot sexsearcher -Shodan SISTRIX SiteExplorer -Snapbot -SocialRankIOBot -Sociscraper +SiteLockSpider +Site Sucker +Sitevigil +SlySearch +SmartDownload +SMTBot +sogouspider Sottopop SpaceBison +Spammen SpankBot -Spanner -sp_auditbot -Spbot -SputnikBot +Spinn3r +spyfu +Sqlmap +Sqlworm Stripper Sucker -SuperBot SuperHTTP -Suzuran -T0PHackTeam -T8Abot Telesoft +Telesphorep +Thumbor TightTwatBot -Titan +Tracemyfile +trendiction.de Turingos -TurnitinBot TwengaBot Twice -Upflow -URLy Warning +Typhoeus +UnisterBot +URLy.Warning Vagabondo -VoidEYE -Voil +VidibleScraper +Virusdie Wallpapers/3.0 +WallpapersHD +WASALive-Bot WBSearchBot +WebAuto Web Auto Web Collage WebEnhancer -WebFetch +Web Fetch +Web Fuck WebImageCollector -WebLeacher -WebPix +WebmasterWorldForumBot WebReaper -Web Sauger +WebSauger Webshag WebsiteExtractor -Website Quester Webster -WebStripper Web Sucker -WebWhacker -WebZIP +WeSEE Whack -Whacker Widow -WinHTTrack -WiseGuys Robot -WISENutbot -Woobot -Wotbox +Wprecon +WWW-Mechanize WWW::Mechanize WWWOFFLE x22Mozilla -Xaldon WebSpider Xaldon_WebSpider -YoudaoBot +Xenu +xpymep1.exe Zade -Zauba zauba.io +Zeus +ZmEu ZyBorg diff --git a/.dev-tools/test_units/random-referrers-for-test-quick.list b/.dev-tools/test_units/random-referrers-for-test-quick.list index 0163a2c71..81ce5aa0e 100644 --- a/.dev-tools/test_units/random-referrers-for-test-quick.list +++ b/.dev-tools/test_units/random-referrers-for-test-quick.list @@ -1,10 +1,10 @@ -beauty-things.com -cakemediahq.com.au -cannibalcheerleader.com -dickssportinggoods.com -ikearugs.xyz -istock-mebel.ru -kinozapas.org -socialtrade.biz -wieseversa.no -zaloro.com +aihelen.net +bestchoice.cf +decoratinghomes.ga +extlinks.com +muabancantho.info +porno-dojki.net +rapevideosmovies.com +surflinksmedical.com +ttsq.fr +xn----8sblgmbj1a1bk8l.xn----161-4vemb6cjl7anbaea3afninj.xn--p1ai diff --git a/.dev-tools/test_units/random-referrers-for-test.list b/.dev-tools/test_units/random-referrers-for-test.list index 6272a3bea..1d9808328 100644 --- a/.dev-tools/test_units/random-referrers-for-test.list +++ b/.dev-tools/test_units/random-referrers-for-test.list @@ -1,500 +1,500 @@ -100dollars-seo.com -12bet.com -3xforum.ro -4istoshop.com -6hopping.com -7zap.com -83net.jp -aa08daf7e13b6345e09e92f771507fa5f4.com -abctoppictures.net -acmebtn.ml -addblueoff.com.ua -adinterax.com -admatic.com.tr -adultmeetonline.info -adult-shop.com.ua -advertisingtag.net -ad-words.ru -aimayangzhi.com -akuhni.by +000free.us +1001watch.com.ua +1-free-share-buttons.com +2728fb936f0.com +2girls1cup-free.com +2rich4bitches.com +4kepics.com +51unlim.ru +57883.net +5forex.ru +8gold.com +98oi.ru +a1security.com.ua +aa8780bb28a1de4eb5bff33c28a218a930.com +acads.net +activecampaign.dreamhosters.com +adimmix.com +adloads.net +admanaerofoil.com +adsref.men +adultactioncam.com +adultfullhd.com +advancedsoftwaresupport.com +ageofclones.com +aghanyna.com +air-edem.ru alborzan.com -albumsuper.info +aliexpresscashback.club +alkoravto.ru +allboard.xobor.de allcredits.su -alldubai.biz -allnews24.in -amazon-seo-service.com -anal-acrobats.com -anal-acrobats.hol.es -animal-drawings.com -animebox.com.ua -antons-transporte.de -apilinkswiftco-a.akamaihd.net -apiluckyleapnet-a.akamaihd.net -apimegabrowsebiz-a.akamaihd.net -apishops.ru -arate.ru -arcadepages.com -artefakct.com -as5000.com -asrv-a.akamoihd.net -asseenontv.ru -autoloans.com -auto-moto-elektronika.cz -autotop.com.ua -autoua.top -avtolombard-krasnodar.ru -backlinkwatch.com -balitouroffice.com -bannerpower.com -bausparen.bz.it -bcmp.org -beachdriveblog.com +allforminecraft.ru +allkrim.com +alpharma.net +alveris.ru +amateurlivechat.org +animal36.com +anniemation.com +apiqualitinknet-a.akamaihd.net +apple.com-cleaner.systems +appsaurus.com +aramaicmedia.org +arkkivoltti.net +artdeko.info +asrvrep-a.akamaihd.net +atout-energie-69.com +audiobangout.com +autochoixspinelli.com +aviav.eu +avtointeres.ru +ayeartoforget.com +azartmix.com +baltgem.com +bank.uz +bbtec.net +beauby.ru +beauty-b0x.pl +bedcapdealers.com berdasovivan.ru -bestchoice.cf -bestmouthwash.club -bg6s0.com -biglistofwebsites.com -bitcoinremote.com -bobinoz.com -bocoarchives.org -bonux.nextview.ru -bookhome.info -bryansk.zrus.org -buketeg.xyz -bupropion-sr-150-mg.us -buqayy0.livejournal.com -businesxxl.com -buypuppies.ca -buyscabiescream.com -buytizanidineonline.blogspot.com -cablecar.us -calc-for-credit.ru -californianews.cf -canoncdriverq3.pen.io -canyougethighofftizanidine.blogspot.com -canzoni.ru -cartierlovestore.com -cartiertrinity.xyz -cbozhe.com -cenoval.ru -centrumcoachingu.com -chatroulette.si -chelyabinsk.xrus.org -chinacheapelitejerseys.com -chlooe.com -chocoslim.pro -christianlouboutinoutlet.win -citetick.com -claytransformations.info -clicksor.com -clipartnew.com -cl.s7.exct.net -compliance-barak.top -compliance-donald.xyz -cookie-law-enforcement-ee.xyz -coolgramgoods.com -cool-wedding.net -country-chic.ru -courtshipgift.com -cowlmash.com +bestbuy.ca +best-coupon-offer.com +bestempresas.es +bestfortraders.com +best-games.today +bet365.com +betterhdporn.com +beztuberkuleza.ru +bidr.trellian.com +bikini-image.com +billiard-classic.com.ua +bjgugu.net.ua +bkgr.se +blacktwhite.com +blog.koorg.ru +bluesalt.co +bmsco.co +boleznikogi.com +bombla.org +bonzybuddy.com +boole.onlinewebshop.net +breastaugmentation.co.za +bristolhostel.com +bukleteg.xyz +bus-offer.com +callawaygolfstore.online +calstaterealty.us +cannibalcheerleader.com +cartechnic.ru +cartierreplica.win +cdncash.com +chatroulette.online +chatroulette.world +cheatcode-lita12.rhcloud.com +check-host.net +cherubinimobili.it +cialisndbrx.com +cialisovercounteratwalmartusa.com +cigarpass.com +clarithromycin500mg.com +clean-start.net +clippingphotoindia.com +club-lukojl.ru +comissionka.net +compliance-alex.top +compliance-mary.top +com-supportcenter.website +conocer-sanabria.com +coolingoods.com +copenergo.ru +corta.co +cougfan.info cpajunkies.com -cryptoswap.biz -crystalslot.com -cutalltheshit.com -cxpromote.com -czat.wp.pl -czeshop.info -daneshetabiat.com +crackguru.tk +crazyboost.pro +crazyprotocol.com +customergrowthsystems.com +cv.wallhade.co +dailyfinancefix.com +dangphoto.trade darkages.info -dataloading.net -date-withme.com -db.speedup.gdn -dcj-nn.ru -decorationspcs.com -den-noch24.ru -deplim.com -designdevise.com -design-ideas.info -detoxmed24.ru +darodar.com +data1.scopich.com dickssportinggoods.com -digital-sale.su -diminishedvalueoforegon.com -div.as -divku.ru -djonwatch.ru -docs4all.com -dokfilms.net -domznaniy.ru -downloader12.ru -drpornogratisx.xxx -drugstoreforyou.com -druzhininevgeniy63.blogspot.com -egovaleo.it -ekto.ee -elmifarhangi.com -enews.tech -escortplius.com -european-torches.ru -evaashop.ru -evening-dating-club.info -exct.net -extremal-blog.com -eyelike.com.ua -eyes-on-you.ga -f012.de -fable.in.ua -fashion.stellaconstance.co -fatfasts-4tmz.com +dieta-personalna.pl +docstoc.com +doctissimo.fr +dogoimage.com +domain-submit.info +domini.cat +domoysshop.ru +dotmass.top +downvids.net +doyouknowtheword-flummox.ml +dstroy.su +dtm-spain.com +eachdayisagift.review +econom.co +educontest.net +ekn-art.se +elektrozigarettekaufen1.devhub.com +elitedollars.com +energydiet24.ru +enginebay.ru +erotik0049.com +etm-consult.de +etur.ru +eu-cookie-law.info +evogarage.com +excitacion.info +executehosting.com +ezigarettekaufen2.yolasite.com +ezigarettekaufen.myblog.de +ezigarettenshop1.yolasite.com +fanoboi.com +fashionindeed.ml favorcosmetics.com -faz99.com -feorina.ru -fickenprivat.info -filmi-v.online -fivedwld.cf -focalink.com -fotosfotos.eu +fealq.com +feel-planet.com +fermersovet.ru +filesdatabase.net +fishingwholesale.us +floridamhca.org +forminecrafters.ru +formseo.com fotoxxxru.com -fotzen-ficken.com -foxinsocks.ru -freelifetimefuckbook.com -freemags.cc -freenom.link -funnel.co.za -fyl.com.ru -gallery.rennlist.com -gambarkataku.co -gamerscorps.com -games.kolossale.ru -gay.adultgalls.com -getdot.ru -getmiro.com -getprismatic.com -gigixo.com -glavprofit.ru -gok-kasten.net -googlefeud.com -goroda-vsego-mira.ru -gotorussia.com -gov.yanao.ru -guardlink.com -guigyverpo.cf -guitar-master.org -hdhc.ru -hdimagegallery.net -headpharmacy.com -health-medical-portal.info -heartofpayne.xyz -helpmymacfaster.trade -homedecorpicture.us -hostcritique.com -hostingclub.lk -howtowhitenteethfast.xyz -hully.altervista.org -ihc2015.info -imperiafilm.ru -impisr.ru -interferencer.ru -intermesh.net -intimshop-fantasy.ru -iopeninghours.co.uk -iplogger.org +free-share-buttons.com +free-video-tool.com +fusoradio.info +fym.com.ru +g33.org +galeon.com +gameonasia.com +gay-site.store +gcup.ru +gearcraft.us +geile-lelly.eu +get-free-social-traffic.com +getyourimage.club +ghostvisitor.com +gk-atlant.info +gktt.ru +globalsurfari.com +goblacked.com +golaya.pw +goldadpremium.com +grizzlysgrill.com +groupmoney.ru +gsasearchengineranker.pw +gsasearchengineranker.site +guruofcasino.com +hamilton.ca +hard-porn.mobi +hatdc.org +hem.passagen.se +hesteel.pl +hitsbox.info +holidaypics.org +homeandhealth.ru +homedecoguide.info +hreade.com +huhn.altervista.org +ideoworld.org +iklysha.ml +ilikevitaly.com +imabase.com +imk.com.ua +incanto.in.ua +infektsii.com +interior-stickers.ru +iptool.xyz iqoption.com -islamtoday.co.za -islandminingsupply.wordpress.com -ispaniya-costa-blanca.ru -itronics.ca -izhevsk.xrus.org -jackwolfskinoutlet.online -javidol.site -jewelryandfiligree.com -jjbabskoe.ru -johnrobertsoninc.com -journeydownthescale.info -juliadiets.com -jumptap.com -kalandranis.gr -kamagragelusa.net +jbl-charge.info +junglenet-a.akamaihd.net +karachev-city.ru +karpun-iris.ru kchaxton.com -keki.info -kenaba.su -keywordsking.com -kihi.gdn -kino2018.cc -kino2018.club -kinobest.pl -kinotorka.ru -kollesa.ru -konteiner24.com -krasnodar.ru -la-fa.ru +keywordglobal.co.za +khafre.us +kino-filmi.com +kinosed.net +kirov.zrus.org +kladrus.ru +kleine-titten.biz +klumba55.ru +kredit-blog.ru +kung-fu-ru.com +kupit-adenu.ru +kurbappeal.info +kustanay.kz +kvartiry-remont.ucoz.ru +laapp.com +labvis.host.sk +landreferat.ru laparfumotec.com -law-check-nine.xyz -lecbter-relationships.ga -levaquin750mg.blogspot.com -lincolntheatre.com -linksharingt.com -liupis.com -locimge.pw -lonely-mature.com -lottospring.com -ltvperf.com -luckyday.world -luckyshop.net.ua -lunamedia.co -luxup.ru -m5home.ru -macrotek.ru +lasvegaslockandsafe.com +laulini.soclog.se +letmacworkfaster.world +lietaer.com +linkbuddies.com +linkrr.com +llastbuy.ru +lomza.info +lonfon.xyz +louboutinshoes.xyz +lovelycraftyhome.com +luciddiagnostics.in +lunchrock.co +macresource.co.uk mac-shield.com -magnetic-bracelets.ru -maju.bluesalt.co -makis.nu -mamasuper.prom.ua -mansparskats.com -manve.info -mararoom.ru -marketland.ml -matchpal-a.akamaihd.net -maxthon.com -mediafresh.online -megahdporno.net -mere.host.sk -m-google.xyz -mirtorrent.net -missvietnam.org -mixtapetorrent.com -mnogabukaff.net -mojowhois.com -monclervests.xyz -moneyteam24.com +magazin-pics.ru +magento-crew.net +magicplayer-s.acestream.net +mainhp.net +maridan.com.ua +marwer.info +masthopehomes.com +matrixalchemy.com +mazda-roadsters.com +mediawhirl.net +meltwater.com +mex-annushka.ru +mitrasound.ru +moesexy.com moroccosurfadventures.com -moskva.nodup.ru -mp3ringtone.info -mrmoneymustache.com -msk.afora.ru -musezone.ru -my-cash-bot.co -myindospace.com -myiptest.com -nalogovyykodeks.ru -nash-krym.info -naturalbreakthroughsresearch.com -neobux-bg.info -netallergy.ru -newstudio.tv -nextconseil.com -nightvision746.weebly.com -niki-mlt.ru -niroo.info -noumeda.com -nvformula.ru -o-dachnik.ru -offtime.ru -olvanto.ru -onlinemegax.com -online-podarki.com +mrvideospornogratis.xxx +mturkcontent.com +muizre.ru +mustwineblog.com +myanyone.net +my-big-family.com +myonlinepayday.co +myyour.eu +napalm51.nut.cc +narco24.me +narosty.com +nate.com +naughtyconnect.com +negociosdasha.com +nerudlogistik.ru +netoil.no +nfljerseyscheapchinabiz.com +nfljerseysforsalewholesaler.com +nikhilbahl.com +nikitsyringedrivelg.pen.io +njkmznnb.ru +november-lax.com +o00.in +odesproperty.com +omsk.xrus.org +onclickpredictiv.com +onemantrip.com +online-templatestore.com onlythegames.com -o-o-6-o-o.com -optibuymac.com -pacificair.com -panouri-solare-acoperis.com -papasdelivery.ru -partner-print.men -partnersafe.men -partybunny.ru -parvezmia.xyz -pawli.eu -pay2me.pl -pdamods.ru -pelfind.me -penisvergrotendepillennl.ovh -percin.biz.ly -petedrummond.com -picphotos.net -pictures-and-images.com -pio.polytopesexempt.com -piski.top -pix24x7.com -pixshark.com -playlott.com -pl.id-forex.com -ploenjitmedia.azurewebsites.net -podshipniki-nsk.ru -popserve.adscpm.net -porn555.com -pornmania.pl -pornoxxx.com.mx -porodasobak.net +onthemarch.co +openstat.com +orel-reshka.net +osagonline.ru +oxotl.com +pagesense.com +paydayonlinecom.com +pepperstyle.ru +perm.xrus.org +petrovka-online.com +phormchina.com +photochki.com +photo-clip.ru +photosaga.info +photostudiolightings.com +picturesify.com +picturesmania.com +pigrafix.at +piluli.info +pizda.lol +pk-pomosch.ru +pk-services.ru +pobeiranie.pl +popup-hgd.xyz +popup-jdh.xyz +porndroids.com +pornogig.com +pourvous.info powenlite24.ru -preparevideosafesystem4unow.site -priscilarodrigues.com.br +preg.marketingvici.com +preparevideosafesystem4unow.space +printie.com privacyassistant.net -prize44.com +privatefx-in.ru +privhosting.com +prizesbook.online prizestohandle.club -productarium.com -prohoster.info -promodj.com -pro-okis.ru -proposal-engine.com -pskcijdc.bloger.index.hr -purplesphere.in -queerspace.com -ratemodels.net -razleton.com -refererx.com -registrationdomainsite.com -registry-cleaner.net -responsive-test.net -restorator-msk.ru -retailwith.com +profitkode.com +progress-upakovka.ru +projefrio.com.br +prostitutki-almata.org +prostitutki-belgoroda.org +psa48.ru +puteshestvennik.com +pwwysydh.com +py100.ru +qualitymarketzone.com +rabotaetvse.ru +rada.ru +ragecash.com +rankings-analytics.com +ranksays.com +raschtextil.com.ua +realizmobi.com +rebelmouse.com +recipedays.ru +redbottomheels.xyz +redhotfreebies.co.uk +redirectme.net +registratciya-v-moskve.ru +remedyotc.com +renewablewealth.com rff-cfal.info -riders.ro -royalads.net -royalcar-ufa.ru -royalvegascasino.com -russian-postindex.ru -safe-app.net -sandhillsonline.com -santechnik.jimdo.com -sashagreyblog.ga -saveriopiazza.it -savingsslider-a.akamaihd.net -sawin.beth.webd.pl -scanner-brian.top -scanner-elena.top -scanner-fred.top -scanner-viktor.top -scoopquest.com -sdi-pme.com -seolab.top -seo-tools-optimizing.com -serpstat.com -sexflirtbook.com -sexs-foto.com -sex-tracker.de -sexy-pings.com +rfserial.net +ricorsogiustizia.org +ros-ctm.ru +rostov.xrus.org +rukino.org +ruscoininvest.company +rustic-quiver.win +sajatvelemeny.com +sammyweaver.com +sampleletters.net +sanjosestartups.com +saratov.xrus.org +sa-rewards.co.za +sax-sex.com +scanner-barak.top +scanner-jessica.top +scanner-john.top +scanner-willy.top +schoolfiles.net +scopich.com +sdsjweb.com +searchlock.com +seojokes.net +seorankinglinks.com +serving.adbetclickin.pink +sexcamamateurchat.com +sexs-foto.top +sexvporno.ru +sfd-chess.ru +shakhtar-doneck.ru sharebutton.org -sharebutton.to -shinikiev.com.ua -shtaketniki.ru -sientalyric.co -simple-image.com.ua -simply.net -site-analyzer.com -site-speed-check.site -sitevalued.com -sitiz.club +shop.acim.org +signal03.ru +sildenafilcitratemed.com +simul.co +sitevaluation.com +skylta.com sky-mine.ru sledstvie-veli.net -smailik.org -smartadserver.com -snip.tw -socialseet.ru -sofit-dmd.ru -space4update.pw -spinnerco.ca -spitfiremusic.com -spl63.fr -srdrvp.com -stairliftstrue.com -stanthonyscatholicchurch.org -statoutlook.info -sterva.cc -stevemonsen.com -stjamesschool.info -stpolice.com -stroicol.net -summerlinhomes411.info -supers.com.ua -szucs.ru -takeflyte.com -techart24.com -teesdaleflyballclub.co.uk -telegraf.by -terrafootwear.us -tfxiq.com -thegioixekhach.com -thenetinfo.com -thetoiletpaper.com +snomer1.ru +snow.nvr163.com +sodexo.com +solution4u.com +songoo.wz.cz +spidtest.space +sport-video-obzor.ru +spywarenuker.com +star61.de +starpages.net +stockspmb.info +store-rx.com +supermesta.ru +svnuppsalaorebro.se +sygraem.com +textads.men +theautoprofit.ml +theendivechronicles.com +thefarmergame.com +the-torrent-tracker.blogspot.com +the-trader.net ticketsys.inetwd.com -titangel-vietnam.com -tizanidineandhydrocodone.blogspot.com -tizanidinecapsules.blogspot.com +tizanidinedosage.blogspot.com +tizanidinedosageforsleep.blogspot.com tizanidineforopiatewithdrawal.blogspot.com -tizanidinewithdrawalsymptoms.blogspot.com -tksn.ru -tmm-kurs.ru +tizanidinevsvalium.blogspot.com tmtrck.com -toonfamilies.net +tobeyouday.win +top250movies.ru topappspro.com -topmira.com -topshef.ru +toyota.7zap.com +tpu.ru +track112.site +tracksz.co trafficjunky.net -tripper.de -triumf-realty.ru -truck-addzilla.life -truck-land.life -trustl.life -tv-spoty.info -typer.one -tyumen.xrus.org -u17795.netangels.ru -ucsol.ru +trafficnetzwerk.de +ttrraacckkrr.com +turbodsp.com +turist-strani.ru +tvorozhnaja-zapekanka-recept.ru udayavani.com -uggbootsoutletsale.us -uginekologa.com -ugogo.info -umaseh.com -unrealcommander.org -uptimebot.net -urlopener.com -ussearche.cf -uzungil.com -videochat.mx -videochaty.ru -viel.su -vinsit.ru -vip-parfumeria.ru -virtuagirl.com -visitcambridge.org -voluumtracker1.com -vzlomtw.com -watchmyfb.pl -webadvance.club +uggsale.online +ukrobstep.com +ultramart.biz +universals.com.ua +unlimitdocs.net +unrealcommander.biz +userequip.com +usswrite.com +validccseller.com +vbtracker.net +velobikestock.com +ventelnos.com +via-energy-acquistare.com +video-hollywood.ru +videokrik.net +villacoloniale.com +vinylvault.co.uk +vip-dom.in +vip-file.com +vipsiterip.org +vksex.ru +vmnmvzsmn.over-blog.com +vod.com.ua +vodkoved.ru +voluumtrk.com +vostoktrade.info +vut.com.ru +wahicbefa31.soup.io +watchinf.com +webads.co.nz web.cvut.cz webix.me -webkeyit.com -webs.com -wemedinc.com -wesharepics.com -wholesalenfljerseys.us.com -womensplay.net +weblo.com +website-speed-checker.site +websitevaluebot.com +wedding-salon.net +wejdz-tu.pl +wesharepics.info +while.cheapwebsitehoster.com +white-truck.life +wildcattube.com +works.if.ua writingservices17.blogspot.ru -ws.ampower.me -www.timer4web.com -xchangetrak.com -xgames-04.com -xn-----6kccaibs5cb8afhjrfmix2n.xn--p1ai -xn----7sbbahaq9bb5afgiqfliv4m.xn--p1ai -xn--80aanaardaperhcem4a6i.com -xn----8sblgmbj1a1bk8l.xn----161-4vemb6cjl7anbaea3afninj.xn--p1ai -xn----dtbndd4ae7eub.top -xn--oogle-wmc.com -x-porno.video -yachts-cruise.info +xblognetwork.com +xcombear.ru +xmlinde.com +xn-----6kcaacnblni5c5bicdpcmficy.xn--p1ai +xn----7sbbagbq7bd5aheftfllo4m.xn--p1ai +xn--d1abj0abs9d.in.ua +xolodremont.ru yebocasino.com -yellowproxy.net -yoshkarola.zrus.org -yourporngay.com -youtuhe.com -yugk.net -yurgorod.ru -za-fun-offer.com -zagadki.in.ua -zaimhelp.ru -zakonobosago.ru -zarajbuilders.com +youradexchange.com +youradulthosting.com +youresponsive.com +zajm-zalog-krasnodar.ru +zatjmuzu.info zawyna.ua -zeroredirect6.com -znakomstva-piter78.ru -zoogdinsney.com -zskdla.site +zeikopay.com +zeleznobeton.ru +zetmaster.ru +zocaparj.kz +zojirushi-products.ru +zoogdiesney.com +zrizvtrnpale.tk diff --git a/.dev-tools/test_units/random-referrers-for-whitelist-test.list b/.dev-tools/test_units/random-referrers-for-whitelist-test.list index 7a691efa4..43e5dc894 100644 --- a/.dev-tools/test_units/random-referrers-for-whitelist-test.list +++ b/.dev-tools/test_units/random-referrers-for-whitelist-test.list @@ -1,250 +1,250 @@ -0912701309f8ce.com -101billion.com -101lesbian.xyz -1hwy.com -24x7-server-support.site -2kata.ru -51unlim.ru -7a2rnuey1tw9ar.ru -7makemoneyonline.com -8xv8.com -9med.net -aa9bd78f328a6a41279d0fad0a88df1901.com -abcdefh.xyz -abclauncher.com -abusora.com -acheterviagrafr24.com -adimmix.com -adultwebhosting.info -advokateg.xyz -aerogo.com -aihelen.net +21h2o.com +2728fb936f0.com +300richmond.co.nz +3weekdiet.com +4pp13.com +999webdesign.com +above.com +adprotect.net +advocatemsk.ru +adzerg.com alert-fjg.xyz -allforminecraft.ru -alta-realestate.com +alfa-medosmotr.ru +allwomen.info +alpharma.net amateurlivechat.org -am-se.com -an-donut.com -anime.dougasouko.com -anmysite.com -anti-virus-removal.info -apartmentratings.com -apccargo.com -apibrowsesmartne-a.akamaihd.net -apilinkswiftco-a.akamaihd.net -app-ready.xyz -apptester.tk -arcadepages.com -arcteryxsale.online -arius.tech -atlasvkusov.ru -audiofree.ru -ausergrubhof.info -bag77.ru -bambi.ck.ua -bannerconnect.net -bestofupload.info -best-seo-offer.com -best-seo-software.xyz +ampower.me +antons-transporte.de +ardimobileinfo.ml +asdfg.pro +auto-zapchasti.org +azinoofficial777.ru +beachdriveblog.com +beget.tech +best-businessman.ru +bestchoice.cf +best-seo-solution.com +bestssaker.com betterhdporn.com -billiard-classic.com.ua +bezcmexa.ru +bezlimitko.xyz +bidbuy.co.kr +big-boards.info +bitcoins-live.ru bitporno.sx -blackplanet.com -bluerobot.info -bmwhighperformers.com -brandehk.dk -brianhenry.co -btvn.ru -carmuffler.net -casite-513049.cloudaccess.net -celebsopics.com -ceotrk.com -chinaelitecheapjerseys.com -chorus.terakeet.com -cibpenonptib.flu.cc -cienum.fr -cleanpcnow.com -clickprotects.com -compliance-walter.top -contextualyield.com -cool-wedding.net +biznesluxe.ru +blog.f00kclan.de +bloggers.nl +blog-hits.com +bodybuilding-shop.biz +bookmaker-bet.com +bookmark4you.com +b-styles.xyz +buyantiviralwp.com +buyparajumpers.online +camdenmemorials.com +canrole.com +cardsharp1.ru +cartierring.xyz +cartiertrinity.xyz +cfjump.com +cheapbarbour.online +cheapjerseysfootballshop.com +chocolateslim-original.com +clayvasedesigns.tk +community.allhiphop.com +compliance-viktor.top +connectionstrenth.com +cookielawblog.wordpress.com +covadhosting.biz +cupday.com +custodieva.ru customcollegeessays.net -cxpromote.com -cyclobenzaprinevstizanidine.blogspot.com -cyse.tk -deanmoore.ie -defenderxtactical.com +cwetochki.ru +cyprusbuyproperties.com +data-mining.tk +date-withme.com delfin-aqua.com.ua -divatraffic.com -divku.ru -domainroam.win -downloadmefiranaratb1972.xpg.com.br -drunkenstepfather.com -dtnlyss.com -dyt.net -eachdayisagift.review -earthmagic.info -ekn-art.se -en.home-task.com -enthuse.computernetworksonline.com -etenininrade.ga -exmasters.com -f1nder.org -fdzone.org -fettefrauen.net -find1friend.com -fivedwld.cf -flavors.me -forex21.ru -foto-telok.net -free-fb-traffic.com -freelifetimefuckbook.com -free-traffic.xyz -freshberry.com.ua -fuckingawesome.com -fugarif.ga -fxgallery.com -gael-s.ru -gamerextra.com -gay-file.com -ge0ip.net -genericlowlatencyasiodriverhq.aircus.com -getoutofdebtfree.org -getresponse.com -ghernnqr.skyrock.com +dieswaene.com +diminishedvalueoforegon.com +docs4all.com +dogsrun.net +doktoronline.no +domainsatcost.com +domashneeruporno.com +doska-vsem.ru +downloadeer.net +ecommerce-seo.com +edge.sharethis.com +elektrozigaretten1.postbit.com +elvenmachine.com +energy-ua.com +eurosamodelki.ru +everypony.ru +eywords-monitoring-your-success.com +ezigarettenkaufen1.deviantart.com +fashion-stickers.ru +findpik.com +findthe.pet +firstsiteguide.com +fishingwholesale.us +flashlarevista.com +fotopop.club +foxydeal.com +fungirlsgames.net +games.kolossale.ru +gaming-journal.com +get-seo-domain.com +ghazel.ru +gilbertbanda.net girlsatgames.ru -globalscam.ga -goldenggames.com -golfresa.lucania.se -googglet.com -gosreg.amchs.ru -gribkovye-zabolevaniya.com -grizzlysgrill.com -groupmoney.ru -gsasearchengineranker.pw -hdxnxxtube.mobi -herokuapp.com -heygidday.biz -highstairs-a.akamaihd.net -hiwibyh.bugs3.com -hosting-tracker.com -htmlcorner.com -huimin764128.com -intellektmedia.at -investyb.com -iphantom.com -iqoption.pro -iron-age.info -isistaylorporn.info -iweblist.info -jackpotchances.com -komputers-best.ru -kw21.org -laulini.soclog.se -law-check-nine.xyz -lineavabit.it -lion.bolegapakistan.com -loadopia.com -loginduepunti.it -louboutinreplica.pw -luxmagazine.cf -m5home.ru -magazintiande.ru -managerpak204.weebly.com -maridan.com.ua -marketland.ml -matb3aa.com -mebeldekor.com.ua -mebel-vstroika.ru -metalonly.info -miniature.io -misterjtbarbers.com -modabutik.ru -mojpregled.com -motherboard.vice.com -moyaterapiya.ru -musicdaddy.net +girlsfuckdick.com +gk170.ru +global-ics.co.za +google-liar.ru +gorodservis.ru +gov.yanao.ru +gpirate.com +gwhwpxbw.bloger.index.hr +hamacapty.com +hesteel.pl +hmywwogw.bloger.index.hr +hotelcrocenzi.sm +housekuba.org +invitefashion.com +iplusbit.blogspot.co.za +istripper.com +itis4you.com +jerseyssportsshop.com +joingames.org +jolic2.com +kemerovo.zrus.org +keywords-monitoring-your-success.com +keyword-suggestions.com +kinobest.pl +kinostorm.net +kunstaktien.info +layola.biz.tc +lenvred.org +lexixxx.com +limads.men +littlesunraiser.com +macrotek.ru +makemoneyonline.com +matpre.top +menhealed.net +minecraft-neo.ru +mnogabukaff.net +musicspire.online muztops.ru -mycouponizemac.com -myonigroup.com -narkologiya-peterburg.ru -naturalbreakthroughsresearch.com -neodownload.webcam +narkologiya-orel.ru +nbsproject.ru nikkiewart.ru -niroo.info -nosecret.com.ua -of-ireland.info -oktube.ru -online-sbank.ru -oralsexfilme.net -parajumpersoutlet.online -park.above.com -pathwhelp.org -pkr1hand.com -plastgran.com -platinumdeals.gr -porno-video-chati.ru -proekt-mos.ru -profitwithalex.info -prointer.net.ua -proxyelite.biz -puzzleweb.ru -quebec-bin.com -raavidesigns.com -rankings-analytics.com -recipedays.com -redbottomheels.xyz +notaria-desalas.com +nuit-artisanale.com +okel.co +okout.ru +onlinemegax.com +onlythegames.com +ourtherapy.ru +parlament.biz +parvezmia.xyz +pc-virus-d0l92j2.pw +perosan.com +photosaga.info +pigrafix.at +piski.top +pl-top.pl +pod-muzyku.club +poquoson.org +pornhub-forum.ga +pornoklad.net +pornosexrolik.com +portadd.men +primedice.com +princevc.com +purchasepillsnorx.com +pwwysydh.com +qcstrtvt.bloger.index.hr +quality-traffic.com +quickloanbank.com +rankscanner.com +rapevideosmovies.com redirectme.net -registry-clean-up.net -reklamuss.ru -robot-forex.biz -santechnik.jimdo.com -sc-specialhost.com -seadragonherbery.com -searchinterneat-a.akamaihd.net -seminarygeorgia59.ga -share-buttons-for-free.com -shemale-sex.net -shopvilleroyboch.com.ua -simply.net -sisi-go.ru -skypasss.com -slim.sellany.ru -snomer1.ru -snyatie-lomki-v-stacionare.ru -spamnuker.com -sprttrack.com -stariy-baku.com -studworks.org -swinger-mobil.net -taaaak.com -tecspb.ru -tedxrj.com -thegamerznetwork.com -thfox.com -tizanidinehydrochloride2mg.blogspot.com -topmira.com -torrentdownloadhub.com -torrentred.games -torrents-tracker.com -tradgardspartner.se -trafficjunky.net -tsstcorpcddvdwshbbdriverfb.aircus.com -tupper-shop.ru -vapeface.club -viandpet.com -videosbox.ru -wapsite.me -watchmygf.net +restaurantlescampi.com +resultshub-a.akamaihd.net +rootandroid.org +royalvegascasino.com +sbricur.com +scanner-alex.top +scat.porn +sdelai-prosto.ru +search-error.com +secretscook.ru +seo-platform.com +service.adtech.fr +shop-electron.ru +shoppytoolmac.com +signoredom.com +silktide.com +simon3.ru +sisiynas.ru +site-analyzer.com +sitebeam.net +skylta.com +social-buttons.xyz +soft-terminal.ru +softwaretrend.net +spbchampionat.ru +splendorsearch-a.akamaihd.net +supernew.org +szamponrevita.pl +takeflyte.com +taqywu51.soup.io +tech4master.com +teslathemes.com +testingads.pro +thebitcoincode.com +themeforest.net +the-trader.net +timhost.ru +tizanidinestreetprice.blogspot.com +tjkckpytpnje.com +todohr.com +topvidos.ru +tvnewsclips.info +tv-spoty.info +twiclub.in +twincitiescarservice.com +uasb.ru +vacances-voyages.info +veerotech.com +vektorpress.ru +veopornogratis.xxx +vezuviy.su +vita.com.hr +vostoktrade.info +warezaccess.com web-betting.ru -while.cheapwebsitehoster.com -woman-tampon.ru -worldtraveler.world -www.get-free-traffic-now.com -xblog.in -xn----8sblgmbj1a1bk8l.xn----161-4vemb6cjl7anbaea3afninj.xn--p1ai +webjam.com +website-speed-checker.site +wjgony.com +woman-h.ru +word-vorlagen.net +wurr.voila.net +x-mix.info +xmronta.com +xn----7sbbagbq7bd5aheftfllo4m.xn--p1ai +xn----8sbdbjgb1ap7a9c4czbh.xn--p1acf +xn----dtbndd4ae7eub.top +xvideosbay.com xxxdatinglocal.us -xz618.com -yugk.net -zero1.it +xxxguitars.com +xxxsiterips.xyz +yubikk.info +zakon-ob-obrazovanii.ru +zeroredirect2.com +zigarettenonl.canalblog.com +znakom.sibtest.ru zone-kev717.info +zvuker.net diff --git a/.dev-tools/test_units/ratelimittest-beta.txt b/.dev-tools/test_units/ratelimittest-beta.txt index d4141bdc1..a683b3cc9 100644 --- a/.dev-tools/test_units/ratelimittest-beta.txt +++ b/.dev-tools/test_units/ratelimittest-beta.txt @@ -5,114 +5,7 @@