diff --git a/.dev-tools/beta_conf_files/conf.d/globalblacklist.conf b/.dev-tools/beta_conf_files/conf.d/globalblacklist.conf index 4f495ce21..f01c2b4b5 100644 --- a/.dev-tools/beta_conf_files/conf.d/globalblacklist.conf +++ b/.dev-tools/beta_conf_files/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2020.01.1948 -### Updated: Tue Jan 28 15:13:04 SAST 2020 +### Version: V3.2020.01.1952 +### Updated: Tue Jan 28 16:23:20 SAST 2020 ### Bad Referrer Count: 6814 ### Bad Bot Count: 562 ################################################### diff --git a/.dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf b/.dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf index 30a5b9007..4a51004bf 100644 --- a/.dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf +++ b/.dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -104.197.122.201 1; +35.231.58.0 1; 127.0.0.1 1; diff --git a/.dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf b/.dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf index c31c197b9..e288dd266 100644 --- a/.dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf +++ b/.dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -104.197.122.201 0; +35.231.58.0 0; 127.0.0.1 0; diff --git a/.dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf b/.dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf index 4f495ce21..f01c2b4b5 100644 --- a/.dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf +++ b/.dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2020.01.1948 -### Updated: Tue Jan 28 15:13:04 SAST 2020 +### Version: V3.2020.01.1952 +### Updated: Tue Jan 28 16:23:20 SAST 2020 ### Bad Referrer Count: 6814 ### Bad Bot Count: 562 ################################################### diff --git a/.dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf b/.dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf index 30a5b9007..4a51004bf 100644 --- a/.dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf +++ b/.dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -104.197.122.201 1; +35.231.58.0 1; 127.0.0.1 1; diff --git a/.dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf b/.dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf index c31c197b9..e288dd266 100644 --- a/.dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf +++ b/.dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -104.197.122.201 0; +35.231.58.0 0; 127.0.0.1 0; diff --git a/.dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf b/.dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf index 4f495ce21..f01c2b4b5 100644 --- a/.dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf +++ b/.dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2020.01.1948 -### Updated: Tue Jan 28 15:13:04 SAST 2020 +### Version: V3.2020.01.1952 +### Updated: Tue Jan 28 16:23:20 SAST 2020 ### Bad Referrer Count: 6814 ### Bad Bot Count: 562 ################################################### diff --git a/.dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf b/.dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf index 30a5b9007..4a51004bf 100644 --- a/.dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf +++ b/.dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -104.197.122.201 1; +35.231.58.0 1; 127.0.0.1 1; diff --git a/.dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf b/.dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf index c31c197b9..e288dd266 100644 --- a/.dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf +++ b/.dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -104.197.122.201 0; +35.231.58.0 0; 127.0.0.1 0; diff --git a/.dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf b/.dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf index 4f495ce21..f01c2b4b5 100644 --- a/.dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf +++ b/.dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2020.01.1948 -### Updated: Tue Jan 28 15:13:04 SAST 2020 +### Version: V3.2020.01.1952 +### Updated: Tue Jan 28 16:23:20 SAST 2020 ### Bad Referrer Count: 6814 ### Bad Bot Count: 562 ################################################### diff --git a/.dev-tools/globalblacklist-testing.template b/.dev-tools/globalblacklist-testing.template index 4f495ce21..f01c2b4b5 100644 --- a/.dev-tools/globalblacklist-testing.template +++ b/.dev-tools/globalblacklist-testing.template @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2020.01.1948 -### Updated: Tue Jan 28 15:13:04 SAST 2020 +### Version: V3.2020.01.1952 +### Updated: Tue Jan 28 16:23:20 SAST 2020 ### Bad Referrer Count: 6814 ### Bad Bot Count: 562 ################################################### diff --git a/.dev-tools/globalblacklist.template b/.dev-tools/globalblacklist.template index 153433272..45b254319 100755 --- a/.dev-tools/globalblacklist.template +++ b/.dev-tools/globalblacklist.template @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2020.01.1948 -### Updated: Tue Jan 28 15:13:03 SAST 2020 +### Version: V4.2020.01.1952 +### Updated: Tue Jan 28 16:23:18 SAST 2020 ### Bad Referrer Count: 6814 ### Bad Bot Count: 562 ################################################### diff --git a/.dev-tools/test1_conf_files/bots.d/whitelist-ips.conf b/.dev-tools/test1_conf_files/bots.d/whitelist-ips.conf index a48c747b7..48ec7d3f6 100644 --- a/.dev-tools/test1_conf_files/bots.d/whitelist-ips.conf +++ b/.dev-tools/test1_conf_files/bots.d/whitelist-ips.conf @@ -45,4 +45,4 @@ # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST -35.202.245.105 0; +34.74.91.53 0; diff --git a/.dev-tools/test1_conf_files/conf.d/globalblacklist.conf b/.dev-tools/test1_conf_files/conf.d/globalblacklist.conf index 153433272..45b254319 100644 --- a/.dev-tools/test1_conf_files/conf.d/globalblacklist.conf +++ b/.dev-tools/test1_conf_files/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2020.01.1948 -### Updated: Tue Jan 28 15:13:03 SAST 2020 +### Version: V4.2020.01.1952 +### Updated: Tue Jan 28 16:23:18 SAST 2020 ### Bad Referrer Count: 6814 ### Bad Bot Count: 562 ################################################### diff --git a/.dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf b/.dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf index 30a5b9007..57162ae29 100644 --- a/.dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf +++ b/.dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -104.197.122.201 1; +34.73.66.97 1; 127.0.0.1 1; diff --git a/.dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf b/.dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf index c31c197b9..365333905 100644 --- a/.dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf +++ b/.dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -104.197.122.201 0; +34.73.66.97 0; 127.0.0.1 0; diff --git a/.dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf b/.dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf index 153433272..45b254319 100644 --- a/.dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf +++ b/.dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2020.01.1948 -### Updated: Tue Jan 28 15:13:03 SAST 2020 +### Version: V4.2020.01.1952 +### Updated: Tue Jan 28 16:23:18 SAST 2020 ### Bad Referrer Count: 6814 ### Bad Bot Count: 562 ################################################### diff --git a/.dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf b/.dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf index 30a5b9007..57162ae29 100644 --- a/.dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf +++ b/.dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -104.197.122.201 1; +34.73.66.97 1; 127.0.0.1 1; diff --git a/.dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf b/.dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf index c31c197b9..365333905 100644 --- a/.dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf +++ b/.dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -104.197.122.201 0; +34.73.66.97 0; 127.0.0.1 0; diff --git a/.dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf b/.dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf index 153433272..45b254319 100644 --- a/.dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf +++ b/.dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2020.01.1948 -### Updated: Tue Jan 28 15:13:03 SAST 2020 +### Version: V4.2020.01.1952 +### Updated: Tue Jan 28 16:23:18 SAST 2020 ### Bad Referrer Count: 6814 ### Bad Bot Count: 562 ################################################### diff --git a/.dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf b/.dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf index 30a5b9007..57162ae29 100644 --- a/.dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf +++ b/.dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -104.197.122.201 1; +34.73.66.97 1; 127.0.0.1 1; diff --git a/.dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf b/.dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf index c31c197b9..365333905 100644 --- a/.dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf +++ b/.dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -104.197.122.201 0; +34.73.66.97 0; 127.0.0.1 0; diff --git a/.dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf b/.dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf index 153433272..45b254319 100644 --- a/.dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf +++ b/.dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2020.01.1948 -### Updated: Tue Jan 28 15:13:03 SAST 2020 +### Version: V4.2020.01.1952 +### Updated: Tue Jan 28 16:23:18 SAST 2020 ### Bad Referrer Count: 6814 ### Bad Bot Count: 562 ################################################### diff --git a/.dev-tools/test2_conf_files/bots.d/whitelist-ips.conf b/.dev-tools/test2_conf_files/bots.d/whitelist-ips.conf index 1df7835ed..9e62a49ce 100644 --- a/.dev-tools/test2_conf_files/bots.d/whitelist-ips.conf +++ b/.dev-tools/test2_conf_files/bots.d/whitelist-ips.conf @@ -45,5 +45,4 @@ # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST -35.192.10.37 0; -35.202.245.105 0; +35.227.58.83 0; diff --git a/.dev-tools/test2_conf_files/conf.d/globalblacklist.conf b/.dev-tools/test2_conf_files/conf.d/globalblacklist.conf index 4ce008a42..1d8e7461a 100644 --- a/.dev-tools/test2_conf_files/conf.d/globalblacklist.conf +++ b/.dev-tools/test2_conf_files/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2020.01.1948 -### Updated: Tue Jan 28 15:13:03 SAST 2020 +### Version: V4.2020.01.1952 +### Updated: Tue Jan 28 16:23:18 SAST 2020 ### Bad Referrer Count: 6814 ### Bad Bot Count: 562 ################################################### diff --git a/.dev-tools/test3_conf_files/bots.d/whitelist-ips.conf b/.dev-tools/test3_conf_files/bots.d/whitelist-ips.conf index 219394936..2b5479faa 100644 --- a/.dev-tools/test3_conf_files/bots.d/whitelist-ips.conf +++ b/.dev-tools/test3_conf_files/bots.d/whitelist-ips.conf @@ -45,5 +45,5 @@ # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST -34.66.25.221 0; -35.202.245.105 0; +35.227.58.83 0; +35.196.82.30 0; diff --git a/.dev-tools/test3_conf_files/conf.d/globalblacklist.conf b/.dev-tools/test3_conf_files/conf.d/globalblacklist.conf index c44ed5335..dcc400b2b 100644 --- a/.dev-tools/test3_conf_files/conf.d/globalblacklist.conf +++ b/.dev-tools/test3_conf_files/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2020.01.1948 -### Updated: Tue Jan 28 15:13:03 SAST 2020 +### Version: V4.2020.01.1952 +### Updated: Tue Jan 28 16:23:18 SAST 2020 ### Bad Referrer Count: 6814 ### Bad Bot Count: 562 ################################################### diff --git a/.dev-tools/test4_conf_files/bots.d/whitelist-ips.conf b/.dev-tools/test4_conf_files/bots.d/whitelist-ips.conf index 7fd2f90fa..ccb2d2e4d 100644 --- a/.dev-tools/test4_conf_files/bots.d/whitelist-ips.conf +++ b/.dev-tools/test4_conf_files/bots.d/whitelist-ips.conf @@ -45,5 +45,6 @@ # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST -34.66.50.208 0; -35.192.91.101 0; +34.73.34.132 0; +34.74.91.53 0; +34.73.66.97 0; diff --git a/.dev-tools/test4_conf_files/conf.d/globalblacklist.conf b/.dev-tools/test4_conf_files/conf.d/globalblacklist.conf index 153433272..45b254319 100644 --- a/.dev-tools/test4_conf_files/conf.d/globalblacklist.conf +++ b/.dev-tools/test4_conf_files/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2020.01.1948 -### Updated: Tue Jan 28 15:13:03 SAST 2020 +### Version: V4.2020.01.1952 +### Updated: Tue Jan 28 16:23:18 SAST 2020 ### Bad Referrer Count: 6814 ### Bad Bot Count: 562 ################################################### diff --git a/.dev-tools/test5_conf_files/bots.d/whitelist-ips.conf b/.dev-tools/test5_conf_files/bots.d/whitelist-ips.conf index ee2c29f58..79eaa9e3c 100644 --- a/.dev-tools/test5_conf_files/bots.d/whitelist-ips.conf +++ b/.dev-tools/test5_conf_files/bots.d/whitelist-ips.conf @@ -45,6 +45,6 @@ # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST -34.66.50.208 0; -35.202.245.105 0; -34.66.200.49 0; +34.73.66.97 0; +35.231.58.0 0; +34.73.34.132 0; diff --git a/.dev-tools/test5_conf_files/conf.d/globalblacklist.conf b/.dev-tools/test5_conf_files/conf.d/globalblacklist.conf index 153433272..45b254319 100644 --- a/.dev-tools/test5_conf_files/conf.d/globalblacklist.conf +++ b/.dev-tools/test5_conf_files/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2020.01.1948 -### Updated: Tue Jan 28 15:13:03 SAST 2020 +### Version: V4.2020.01.1952 +### Updated: Tue Jan 28 16:23:18 SAST 2020 ### Bad Referrer Count: 6814 ### Bad Bot Count: 562 ################################################### diff --git a/.dev-tools/test_units/blacklist-ips.conf b/.dev-tools/test_units/blacklist-ips.conf index 30a5b9007..4a51004bf 100644 --- a/.dev-tools/test_units/blacklist-ips.conf +++ b/.dev-tools/test_units/blacklist-ips.conf @@ -1,2 +1,2 @@ -104.197.122.201 1; +35.231.58.0 1; 127.0.0.1 1; diff --git a/.dev-tools/test_units/random-bots-for-test-quick.list b/.dev-tools/test_units/random-bots-for-test-quick.list index 665010db3..8411237c3 100644 --- a/.dev-tools/test_units/random-bots-for-test-quick.list +++ b/.dev-tools/test_units/random-bots-for-test-quick.list @@ -1,10 +1,10 @@ -AIBOT -InternetSeer -ips-agent -NetLyzer -Nimbostratus -Spammen -WBSearchBot -Web Fetch -Xaldon_WebSpider -YoudaoBot +360Spider +Cegbfeieh +EasyDL +Majestic12 +plumanalytics +ReGet +SBIder +Sitevigil +VidibleScraper +WPScan diff --git a/.dev-tools/test_units/random-bots-for-test.list b/.dev-tools/test_units/random-bots-for-test.list index 46b62c15c..07336942b 100644 --- a/.dev-tools/test_units/random-bots-for-test.list +++ b/.dev-tools/test_units/random-bots-for-test.list @@ -1,250 +1,250 @@ 360Spider 404checker 404enemy -Abonti +80legs Aboundex -Aboundexbot +Acunetix +Aipbot +Alexibot AllSubmitter -arquivo.pt -ASPSeek -AwarioRssBot +AlphaBot +Anarchie +Apexoo +Asterias +Attach +autoemailspider BackDoorBot Backlink-Ceck backlink-check BacklinkCrawler -Barkrowler +BackStreet +Badass +Bandit BatchFTP -Battleztar Bazinga -BDCbot -BDFetch -Bitacle -Blackboard +Bigfoot +BlowFish Bolt BotALot -Brandprotect -Buddy -BuzzSumo +BuiltBotTough +Bullseye +Calculon CATExplorador -CCBot +CazoodleBot +Cegbfeieh CheeseBot -Cogentbot +CherryPicker +ChinaClaw +Chlooe +Cloud mapping cognitiveseo -com.plumanalytics -Copier CopyRightCheck -Copyscape +Craftbot +crawler.feedback CrazyWebCrawler Crescent -Curious -DatabaseDriverMysqli +CSHttp +DataCha0s +DBLBot +demandbase-bot Deusu -Digincore -DigitalPebble DIIbot -Disco -Discoverybot -Dispatch DittoSpyder DnyzBot +DomainAppender DomainSigmaCrawler Download Wonder -Dragonfly Drip -DSearch EasyDL -Ebingbong eCatch +ECCP/1.0 +Ecxi +EirGrabber EMail Siphon -EMail Wolf -EroCrawler -evc-batch +Express WebPictures ExtractorPro -Extreme Picture Finder Ezooms -FDM FemtosearchBot -Fimap Firefox/7.0 FlashGet -Flunky -Foobot -FrontPage -FyberSpider +Freeuploader Fyrebot -GermCrawler -Getintent +GalaxyBot +GetRight GetWeb +Gigabot Go-Ahead-Got-It -GrabNet +Go!Zilla +Grabber +Grafula GrapeFX GrapeshotCrawler -GridBot +GT::WWW +Haansoft +HaosouSpider Harvest Havij heritrix -Hloader -HTTP::Lite -HTTrack -Humanlinks -HybridBot -IDBot -IlseBot -Image Fetch +Heritrix +HMView +HTMLparser +Id-search InfoNaviRobot -InfoTekies -Intelliseek +InterGET +Internet Ninja +ips-agent Iria -IRLbot -JamesBOT +Iskanie JennyBot -Jetty JikeSpider -JustView +Keyword Density +Kozmosbot Larbin +LeechFTP LeechGet LexiBot Lftp -LibWeb -Libwhisker Lightspeedsystems -Likse -LinkpadBot +Linkdexbot +LinkextractorPro +LinkScan LinksManager -LinqiaMetadataDownloaderBot -Litemage_walker +LinkWalker +Lipperhey Spider Lmspider +LNSpiderguy lwp-request -LWP::Simple -lwp-trivial -Majestic12 -Majestic-SEO +Magnet +Majestic SEO +MarkMonitor +MarkWatch Mass Downloader Mata Hari -Meanpathbot -Mediatoolkitbot -MegaIndex.ru +MauiBot +mediawords Metauri -MIIxpc -Mister PiX -MJ12bot -Mojeek -Msrabot -muhstik-scan -Musobot +MFC_Tear_Sample +Morfeus Fucking Scanner +Mr.4x3 +MSIECrawler +Name Intelligence +Navroad +NearSite +Needle Nessus -Nettrack +NetAnts +NetMechanic Net Vampire -Netvibes -NextGenSearchBot +NetZIP Nibbler -NICErsPRO Niki-bot +Nikto +NimbleCrawler +NPbot oBot -Octopus -Openfind -Openvas -OpenVAS -OrangeSpider +OnCrawl OutclicksBot -Page Analyzer PageGrabber -page scorer PageScorer -Pandalytics -Panscient -pcBrowser -PHPCrawl -Pimonster -Pixray -Pockey -POE-Component-Client-HTTP +Papa Foto +PeoplePal +PictureFinder +Pi-Monster +PleaseCrawl +plumanalytics ProWebWalker -Psbot +Pump +PxBroker +PyCurl +QueryN Metasearch Quick-Crawler +RankActive +RankActiveLinkBot RankFlex -RebelMouse -Recorder +Rankivabot +RankurBot +RedesScrapy RepoMonkey Ripper -Rogerbot RSSingBot -s1z.ru -ScanAlert -Scanbot +SBIder +ScoutJet +Scrapy +Screaming +Searchestate +SearchmetricsBot Semrush SemrushBot SEOkicks SEOkicks-Robot -SEOlyticsCrawler -SEOprofiler -SeoSiteCheckup -serpstatbot +SEOstats sexsearcher Shodan -Siphon -Siteimprove -SiteLockSpider -SiteSucker -Site Sucker -SlySearch +Sitebeam +SiteExplorer +SiteSnagger +Sitevigil Snake -SocialRankIOBot -Spanner +Snapbot +Snoopy +Sociscraper +Sosospider +Sottopop +SpankBot sp_auditbot -Spinn3r -SputnikBot -Sqlmap -Sqworm +spyfu +Sqlworm +Steeler +SuperBot SuperHTTP +Surfbot SurveyBot -Szukacz +Suzuran +T0PHackTeam T8Abot +tAkeOut +Teleport TeleportPro Telesoft Telesphoreo -Telesphorep TheNomad -TightTwatBot -Toata +Titan Tracemyfile -trendiction.com -True_Robot +Trendiction +trendiction.de Turingos Turnitin TurnitinBot +Twice +Typhoeus URLy Warning -VidibleScraper -Voil +URLy.Warning +VCI +VeriCiteCrawler +Voltron +Wallpapers/3.0 WallpapersHD -WBSearchBot -Webalta -WebCollage -WEBDAV +WASALive-Bot +WebBandit Web Enhancer -Web Fetch -WebFuck -WebLeacher -webmeup-crawler +WebmasterWorldForumBot +Web Pix WebReaper Web Sauger -WebsiteQuester -Website Quester -Webster -WebSucker -WeSEE -Whack -Whatweb -Widow +WebZIP +Whacker +Who.is Bot +WinHTTrack WiseGuys Robot -WISENutbot Wonderbot -Woobot +Wprecon +WPScan WWW-Collector-E -WWW-Mechanize -WWWOFFLE -x09Mozilla +WWW::Mechanize Xenu xpymep1.exe YoudaoBot -zgrab +Zade +Zauba +zauba.io +ZmEu ZyBorg diff --git a/.dev-tools/test_units/random-bots-for-whitelist-test.list b/.dev-tools/test_units/random-bots-for-whitelist-test.list index 40363f4ae..07363aa39 100644 --- a/.dev-tools/test_units/random-bots-for-whitelist-test.list +++ b/.dev-tools/test_units/random-bots-for-whitelist-test.list @@ -1,250 +1,250 @@ -404checker +360Spider +404enemy +80legs +Abonti Aboundex +Aboundexbot +ADmantX AfD-Verbotsverfahren -AhrefsBot +AIBOT Aipbot -Alexibot -AllSubmitter +Alligator +AlphaBot Anarchie Apexoo +arquivo.pt ASPSeek -Attach -autoemailspider +Asterias AwarioRssBot -Backlink-Ceck +BackDoorBot backlink-check -BackStreet -Badass -Battleztar Bazinga +BacklinkCrawler +BackWeb +BatchFTP BDCbot BDFetch BetaBot Bigfoot -Bitacle -Blackboard +Black Hole +BlackWidow Blow -BlowFish -Boardreader -BotALot +Bolt Brandprotect -Buddy -BuiltBotTough -BuiltWith +Brandwatch Bullseye -BunnySlippers -BuzzSumo +CATExplorador CazoodleBot -CCBot +Cegbfeieh Chlooe +Claritybot +Cliqzbot Cloud mapping +Cogentbot cognitiveseo com.plumanalytics CopyRightCheck +Copyscape Cosmos Craftbot -CrazyWebCrawler +Crescent CSHttp -DBLBot -demandbase-bot -Deusu -Devil -Digincore +Custo +Demon DigitalPebble -Discobot -DittoSpyder -DomainCrawler -DomainSigmaCrawler -Drip -DSearch -EasyDL -eCatch +DIIbot +Dispatch +DnyzBot +DomainStatsBot +Dragonfly ECCP/1.0 -EMail Wolf +EirGrabber +EMail Siphon EroCrawler -evc-batch Evil -Extreme Picture Finder +Exabot +Extractor +EyeNetIE +Ezooms +facebookscraper +FHscan FlashGet Flunky +Foobot Freeuploader -FrontPage -Fyrebot -Getintent +FyberSpider +GalaxyBot +Genieo +GetRight GetWeb -Go-Ahead-Got-It GoZilla +Go!Zilla +Grabber +GrabNet GrapeFX GrapeshotCrawler GT::WWW -Harvest -Havij +Haansoft heritrix HMView -HTMLparser -HTTrack Humanlinks -Iblog -IDBot Id-search -IlseBot -Image Sucker -IndeedBot -InfoTekies -Intelliseek +Image Fetch +InfoNaviRobot +instabid +InterGET Internet Ninja -InternetSeer internetVista monitor -Iria +Iskanie +IstellaBot +JamesBOT +Jbrofuzz JennyBot -JOC Web Spider +Joomla Jorgee +JustView Jyxobot -Kenjin Spider -Likse -LinkpadBot +Keyword Density +LeechGet +Lftp LinkScan -Lipperhey +LinksManager +LinqiaMetadataDownloaderBot +Lipperhey Spider Lmspider -Ltx71 -lwp-request -LWP::Simple +LNSpiderguy Magnet -magpie-crawler -Mail.RU_Bot +Majestic SEO MarkMonitor Masscan -meanpathbot -MeanPath Bot -Mediatoolkitbot -mediawords -MegaIndex.ru -Microsoft Data Access -Microsoft URL Control -MIDown tool -MIIxpc -Mister PiX -MJ12bot -Morfeus Fucking Scanner -Mr.4x3 +Meanpathbot +Metauri +MFC_Tear_Sample MSFrontPage +MSIECrawler Msrabot -Name Intelligence -Nameprotect +Musobot +Navroad NearSite -Needle -Nessus -netEstate NE Crawler NetMechanic -Nettrack -Netvibes +NetSpider NetZIP -Nibbler +Nikto NimbleCrawler -NPbot +oBot +Octopus +Offline Explorer OnCrawl -Openfind -OpenLinkProfiler -Openvas -OrangeBot OrangeSpider OutfoxBot +PageAnalyzer Page Analyzer -PageGrabber -PageScorer +page scorer Pandalytics -Panscient -Pavuk +Papa Foto pcBrowser -PHPCrawl -Picscout -Pi-Monster -Pixray -plumanalytics +PeoplePal +Picsearch +Pimonster Pockey -Probethenet -Psbot -PxBroker +PyCurl +QueryN Metasearch Quick-Crawler -RankActive +RankingBot RankingBot2 +Rankivabot +RealDownload Reaper -RocketCrawler -RSSingBot -s1z.ru -SalesIntelligent +Recorder +ReGet +RepoMonkey +Ripper +Rogerbot ScanAlert -scan.lol +Scanbot ScoutJet -SEOkicks-Robot +Searchestate +SearchmetricsBot +SEOkicks SEOlyticsCrawler +seoscanners SEOstats serpstatbot sexsearcher -Shodan -SISTRIX +Siphon Sitebeam SiteExplorer Siteimprove SiteLockSpider +SiteSnagger +SiteSucker SlySearch SmartDownload -Sociscraper -sogouspider +Snake +Snoopy Sogou web spider Sosospider Sottopop SpaceBison -Spanner sp_auditbot -Spbot spyfu -Sqlworm -Stripper -Sucker +Sqlmap +Sqworm Sucuri -SuperHTTP +SuperBot +Surfbot +SurveyBot Swiftbot +Szukacz +T0PHackTeam T8Abot -Teleport -Telesoft +TeleportPro Telesphoreo Telesphorep -The Intraformant -TheNomad -Thumbor +Titan Toata -Tracemyfile +Trendictionbot trendiction.de -Turingos Turnitin TurnitinBot -UnisterBot +TwengaBot Upflow URLy Warning -Vagabondo -VCI -VeriCiteCrawler -WASALive-Bot -Web Auto +Vacuum +VB Project +Virusdie +Voil +Voltron +Wallpapers/3.0 +WBSearchBot +WebAuto WebBandit -WebCollage Web Collage +WEBDAV +WebEnhancer Web Enhancer WebFuck -Web Fuck +WebGo IS WebLeacher -webmeup-crawler -Web Sauger -Webshag +Web Pix +WebsiteExtractor +WebsiteQuester +Website Quester +WebStripper WebSucker WebWhacker WeSEE +Whack Whacker +Who.is Bot +Widow WinHTTrack WiseGuys Robot -WPScan x09Mozilla x22Mozilla -Xaldon WebSpider +YoudaoBot Zauba +Zermelo +Zeus +zgrab Zitebot +ZyBorg diff --git a/.dev-tools/test_units/random-referrers-for-test-quick.list b/.dev-tools/test_units/random-referrers-for-test-quick.list index 1f9f787ad..19cc85d3a 100644 --- a/.dev-tools/test_units/random-referrers-for-test-quick.list +++ b/.dev-tools/test_units/random-referrers-for-test-quick.list @@ -1,10 +1,10 @@ -arcadepages.com -lflash.ru -nrv.co.za -pacificair.com -panouri-solare-acoperis.com -penisvergrotendepillennl.ovh -russia-tao.ru -signx.info -xn----9sbebi2bvzr7h.xn--p1ai -xvideospornoru.com +articlesdirectoryme.info +community.allhiphop.com +copenergo.ru +dayibiao.com +meds-online24.com +mesbuta.info +ochistka-stokov.ru +paramountmarble.co.uk +sundrugstore.com +tootoo.to diff --git a/.dev-tools/test_units/random-referrers-for-test.list b/.dev-tools/test_units/random-referrers-for-test.list index 23ab73969..f90427ff3 100644 --- a/.dev-tools/test_units/random-referrers-for-test.list +++ b/.dev-tools/test_units/random-referrers-for-test.list @@ -1,500 +1,500 @@ -101lesbian.xyz -1688.com -1-free-share-buttons.com -5i2.net -7makemoneyonline.com -7zap.com -883zy.com -8jn.dating-app.ru -a96527gi.beget.tech -aad1f4acb0a373420d9b0c4202d38d94fa.com -accessoires-mode.in -acc.eu.org -admanaerofoil.com -ad-words.ru -aero2.ru -agadelha.com.br -agecheckadult.com +08800.top +100dollars-seo.com +11235813.webzdarma.cz +12masterov.com +2ads.co.uk +2girls1cup-free.com +4k-player.pl +5kstore.com +7a2rnuey1tw9ar.ru +a1security.com.ua +aa625d84f1587749c1ab011d6f269f7d64.com +aab94f698f36684c5a852a2ef272e031bb.com +absolute-s.ru +acunetix-referrer.com +adelly.bg +admatic.com.tr +ads-cool.pro +adviceforum.info +affiliate-fr.com ahrntal.verymes.xyz -aibolita.com -akvamaster.dp.ua -alfa9.com -alienwheel.es -alldubai.biz -allnews.md -allornamenti.com -allpdfmags.net -allstatesugarbowl.org -all-streaming-media.com -amehdaily.com -amospalla.es -anabolics.shop -analytics-ads.xyz -anatomiy.com -android4fun.org -android-systems.ru -angry-fermi-7633.arukascloud.io -anjalika.co.in -anniemation.com -apioutoboxnet-a.akamaihd.net -app5.letmacworkfaster.world +alfa-img.com +alloysteel.ru +allwomen.info +alphavisions.net +alvtank.se +amyfoxfitness.com +anal-acrobats.hol.es +animebox.com.ua +anticrawler.org +antons-transporte.de +apiholdingmypage-a.akamaihd.net +api.stathat.com apple.com-cleaner.systems -apple.com-webbrowsing-security.review -ap.senai.br -arenda-avtoprokat-krasnodar.ru +app-ready.xyz +apps-analytics.net +aquarium-pro.ru +arcadeplayhouse.com +arraty.altervista.org artdeko.info -auspolice.com -autobrennero.it -autonew.biz -autotop.com.ua -auto-zapchasti.org -aviav.ru.com -avirasecureserver.com -avkzarabotok.com -avtolombard-krasnodar.ru -ayeartoforget.com -azbukafree.com -backlink4u.net -bad-stars.net -balitouroffice.com -bannerspace.com +asscenihotosidea.blogspot.co.za +autochoixspinelli.com +autoplate.info +autorn.ru +aversis.net +azlex.uz +b2b-lounge.com +backgroundpictures.net +baditri.com +baltgem.com +bang-hotties.com +bank.uz +bannerbank.ru bannertesting.com -beachdriveblog.com -beachpics.com -beauty-clinic.ru -beetpics.pw -best-businessman.ru -best-coupon-offer.com -big-boards.info -bikini-image.com +beauty-bracelet.com +bestfortraders.com +bestserials.com +bestwaystogetpaid.us +bet365.com +betterhdporn.com +betune.onlinewebshop.net +bharatdefencekavach.com +bigames.online bimatoprost-careprost.com.ua -bizcheapjerseyswholesalechina.com -bjanshee.ru -blobar.org -blogqpot.com -bloke.com -board.f00d.de -bonvillan.com -bonzybuddy.com -bracketsmackdown.com -brothers-smaller.ru -bugof.gq -burkesales.com -business-suggested.tk -buy-cheap-pills-order-online.com -buypuppies.ca -cacheimages.com -callawaygolfstore.online -cannibalcheerleader.com -cartierlove2u.xyz -casablancamanor.co.za -charmstroy.info -chatroulette.world -chcu.net -check-this-out-now.online -chinaelitecheapjerseys.com -chocolateslim-en-espana.com -chocolateslim-en-france.com -christianlouboutinreplica.pw -chuckguilford.com -cineacademy.ru +binomo.kz +birzha-truda.eu +blackplanet.com +bloglag.com +blogs.rediff.com +brianhenry.co +brokergid.ru +bronzeaid-a.akamaihd.net +bumble.cheapwebsitehoster.com +buntube.net +buyk.host.sk +buytizanidineonline.blogspot.com +byme.se +callstevens.com +candy-glam-hp.com +carloans.com +cartierlovebraceletreplica.xyz +cartierlovestore.xyz +cartierreplica.pw +cartierreplica.win +cementaresearch.se +centraletermice.us +centrdebut.ru +changde.58.com +cheapmarmot.online +cheap-trusted-backlinks.com +cigarpass.com clarithromycin500mg.com -clickso.com -cmhomestayagency.com -communisave.co.za -compliance-elena.xyz -compliance-nelson.top -contentlook.co -coolgramgoods.com -cool-wedding.net -covetnica.com -crackguru.tk -craftinsta.ru -creatives.adbetclickin.pink -customergrowthsystems.com -customerguru.in -daretodonate.co -date-withme.com -deanmoore.ie +cleanmypc.com +cmd.kz +coin-hive.com +colobit.biz +com-cleaner.systems +compliance-checker.info +compliance-ivan.xyz +com-secure.download +concordexoticrentals.com +contextualyield.com +coop-gamers.ru +copypaste.traderzplanet.in +cosmeticswomens-womensports.rhcloud.com +craftburg.ru +creams.makeforum.eu +cryptoswap.biz +cymbaltaandtizanidine.blogspot.com +danashop.ru +dekoration.us dekorkeramik.ru -destinationrealestate.com -devochki.top -dieta-personalna.pl -digilander.libero.it +delitime.info +detective01.ru +dev-seo.blog +dfiles.me +diffbot.com +dimfour.com dinosaurus.site -disruptingdinnerparties.com -dnsrsearch.com -domainsphoto.com -donna7753191.ru -dptaughtme.com -duplicashapp.com -dverimegapolis.ru -eandsgallery.com -ecommerce-seo.com -ecxtracking.com -education-cz.ru -electrik-avenue.com +directrev.com +djihispano.com +dktr.ru +doeco.ru +dogsrun.net +dokfilms.net +domcran.net +doska-vsem.ru +downloadkakaotalk.com +driving.kiev.ua +druzhininevgeniy63.blogspot.com +dynainbox.com +dyt.net +eachdayisagift.review +e-buyeasy.com +e-c.al +educontest.net +ege-russian.ru elektrischeziga.livejournal.com -elektrozigarette1.dreamwidth.org -elektrozigaretten2.yn.lt -embedle.com -emerson-rus.ru -en.home-task.com -eropho.net -erotik0049.com -eshop4u.jp -eshop.md -exlarseva.webblog.es +elektrischezigarette1.onsugar.com +epicbrogaming.com +erotikstories.ru +esnm.ru +estibot.com +european-torches.ru +event-tracking.com exonline.info -extener.org -ezaz.info -ezigarettekaufen2.blox.pl -ezigarettekaufen.myblog.de -ezigaretteshop2.vefblog.net -f012.de -fermersovet.ru -filmi-v.online -filunika.com.ru -findacheaplawyers.com -firesub.pl -fix-website-errors.com -flowersbazar.com -food.dtu.dk -fotosfotos.eu -fotzen-ficken.com -franch.info +eyelike.com.ua +eywords-monitoring-your-success.com +famix.xyz +fandlr.com +fashionindeed.ml +fast-wordpress-start.com +feargames.ru +fettefrauen.net +fickenbumsen.net +findanysex.com +findpics.pw +finuse.com +firstdrugmall.ru +fm-upgrade.ru +fodelsedagspresenter.nu +foto-sisek.porngalleries.top +foxjuegos.com +frameimage.org free-floating-buttons.com -freegamesplay.online -freejabs.com -freeseedsonline.com -free-share-buttons.top -freshsuperbloop.com -gamblingnerd.com -game300.ru -gamedayassist.com -gameplexcity.com +freelifetimefuckbook.com +free-share-buttons.com +freesitetest.com +freetangodownload.com +game-mmorpg.net +gamezblox.com gate5.co.za -ge0ip.com -ge0ip.org -geileweiber.tk -gembird.com +gaytube.com +gdebestkupit.ru +generousdeal-a.akamaihd.net getmiro.com -getoutofdebtfree.org -gigapeta.com -gilsonchiro.xyz -girlgamerdaily.com -gkvector.ru -gogps.me -gojiberriess.apishops.ru -goldadpremium.com -goodnightjournal.com -googlefeud.com -google-liar.ru +getyourimage.club +ghazel.ru +gilbertbanda.net +glavtral.ru +gne8.com +goatse.ru +good-mummy.ru +gorodservis.ru gotcher.us -greatgrace.ru -gsasearchengineranker.xyz -gstatey.net -gtopstats.com -hacktougroup.ru -hamacapty.com -headpress.ru -healbio.ru -hentaiheroes.com -herehloadibs.cf -herokuapp.com -hidefiles.org -home-task.com -hopeonthestreet.co.uk -house.sieraddns.com -hyiphunter.org -iflycapetown.co.za -igtools.club -incolors.club -interferencer.ru -investmac.com -iplusbit.blogspot.co.za -istizanidineanopiate.blogspot.com -istripper.com -iyasimasennka.com -jabimgo.pw -jerseyswholesalechinalimited.com -jongose.ninja -jovencitas.gratis -jrcigars.com -kamalsinha.com -karpun-iris.ru -kaz.kz -keywordblocks.com -keywords-monitoring-your-success.com -khafre.us +gpms.org.my +green-tea.tv +gta-top.ru +guigyverpo.cf +gwhwpxbw.bloger.index.hr +hahashka.ru +hasfun.com +hawaiisurf.com +hebr.myddns-flir.com +hentai-manga.porn +hifidesign.ru +hitcpm.com +homedo.fabpage.com +homeinns.com +hotel-mkad.ru +hoterika.com +hreade.com +huimin764128.com +hunthillfarmtrust.org +ibb.com.ua +ideibiznesa2015.ru +igithab.com +igru-xbox.net +iklysha.ml +ilo134uloh.com +ilovevitaly.com +images.gyffu.com +import-sales.com +impresagaia.it +inet-traffic.com +infodocsportal.com +infospot.pt +infowarcraft.ru +instakink.com +intellektmedia.at +interior-stickers.ru +intervsem.ru +int.search.tb.ask.com +investsuccess.org +ix20.ru +jackwolfskinoutlet.online +janettabridal.com +javatex.co.id +juliaworld.net +jwss.cc +kamen-e.ru +karachev-city.ru +ketrzyn.pl +kidskunst.info +kinobaks.com +kinocccp.net kinohit1.ru -kiprinform.com +kinostorm.net +kinozapas.com +kliksaya.com klosetkitten.com -knigonosha.net -komputers-best.ru -konkursowo-24.pl -konturkrasoty.ru -koronirealestate.gr -kovesszucs.atw.hu -krasivoe-hd.net -krasnodar-avtolombards.ru -kursy-ege.ru -kustanay.kz -landreferat.ru -larose.jb2c.me -larutti.ru -ledis.top -ledpolice.ru -lernur.net -li-er.ru -likesuccess.com -likrot.com -lirunet.ru +knogg.net +kometars.xyz +konoplisemena.com +kosynka-games.ru +koversite.info +ks1234.com +kung-fu-ru.com +kuzinsp.ru +labplus.ru +lacloop.info +ladsup.com +laptoper.net +letmacworkfaster.site +letmacwork.world +lion.bolegapakistan.com +lion.playtap.us live-cam6.info -livefixer.com -liverpool.gsofootball.com -livetsomudvekslingstudent.bloggersdelight.dk -locatellicorretor.com.br -longadventure.com -lovi-moment.com.ua +live-sexchat.ru +lostfilm-online.ru +lsp-awak-perikanan.com +lubetube.com lunamedia.co -lyrster.com -machicon-ueno.info -magazintiande.ru -manifestation.betteroffers.review -mcar.in.ua -mebel-ekb.com -megaapteka.ru -megahdporno.net -mesto-x.com -metarip.ru -metascephe.com -methodsmarketing.com -michaelkorsoutletstore.net +m4ever.net +make-money-online.com +manve.info +marcogrup.com +marcowebonyodziezowe.pl +marinetraffic.com +mecash.ru +mediaoffers.click +meltwater.com +mexicosleevegastrectomy.com +micasainvest.com +minharevisao.com +mirmedinfo.ru misandesign.se -modnie-futbolki.net -moncleronline.xyz -moscow.xrus.org -mouselink.co -mrlmedia.net +mobot.site +modabutik.ru +montredemarque.nl +moomi-daeri.com +more-letom.ru +mriyadh.com +mrmoneymustache.com +mrsdalloways.com mrvideospornogratis.xxx -muizre.ru -multgo.ru -muzaporn.com -mylesosibirsk.ru -na15.ru -naphukete.ru -narco24.me -naval.jislaaik.com -nbsproject.ru -netpics.org -nfljerseysforsalewholesaler.com -nodding-passion.tk -nofreezingmac.work -no-fuel.org -norththeface.store -notaria-desalas.com -o-dachnik.ru -oil-td.ru -okmedia.sk +msfsaar.de +musicspire.online +mydownlodablefiles.com +myliveblog.ru +myonlinepayday.co +mystats.xyz +nalogovyykodeks.ru +narutonaruto.ru +netanalytics.xyz +nevansk.ru +nfljerseyscheapbiz.us +nibbler.silktide.com +niki-mlt.ru +nosecret.com.ua +notebook-pro.ru +notfastfood.ru +oballergiya.ru +offer.camp +offergroup.info +onefilms.net onemactrckr.com -onlinemegax.com -onlinetvseries.me -onlywoman.org -openfrost.net -optikremont.ru -ourtherapy.ru -paceform.com -paidonlinesites.com -p-business.ru -pcads.ru -pcimforum.com -perl.dp.ua -pflexads.com -photo-clip.ru -pic2fly.com -picphotos.net -picquery.com +onemantrip.com +onlineserialy.ru +ops.picscout.com +ororodnik.goodbb.ru +osoznanie-narkotikam.net +owohho.com +papasdelivery.ru +parajumpersoutlet.online +par-fallen.ga +park.above.com +partnerads.men +parvezmia.xyz +penisvergrotendepillennl.ovh +petedrummond.com +photorepair.ru +phuketscreen.com +physfunc.ru pistonclasico.com -plastgranar.nu -playboyfiles.xblog.in -playlott.com -pl-top.pl -podshipniki-ntn.ru +pixgood.com +pk-services.ru +play-movie.pl +playtap.us pogodnyyeavarii.gq -poisk-zakona.ru -popunder.net -popup-jdh.xyz -porno-play.net -pricheskaonline.ru -privatefx.all4invest.info +pokemon-go-play.online +pomoc-drogowa.cba.pl +popander.mobi +poptool.net +pornobrazzers.biz +porno-dojki.net +pornofeuer.com +pornogratisdiario.com +pornonik.com +porno-raskazy.ru +pornoreino.com +pornozhara.com +pozdrawleniya.ru +prchecker.info +preparevideosafesystem4unow.space prizefestival.mobi -produkto.net professionalwritingservices15.blogspot.ru -prokotov.com +profit-opportunity.com prom23.ru -prombudpostach.com.ua -prosperent.com -prostitutki-novgoroda.org -pskcijdc.bloger.index.hr -pst2017.onlinewebshop.net -pwwysydh.com -q-moto.ru -rainbowice.ru -ranksonic.info -recinziireale.com -redhotfreebies.co.uk -refudiatethissarah.info -registrationdomainsite.com -registry-clean-up.net -registrysweeper.com -reklama-i-rabota.ru -remedyotc.com -research.ifmo.ru -rfd-split.hr +promalp-universal.ru +purplesphere.in +pyramidlitho.webs.com +questionmarque.ch +quickchange.cc +randki-sex.com +rankscanner.com +ranksonic.org +rapidsites.pro +ratemodels.net +razleton.com +rcb101.ru +realtytimes.com +rff-cfal.info +rightenergysolutions.com.au +romhacking.ru +rostov.xrus.org +rp9.ru rubanners.com -rumamba.com -running-line.ru -ruspdd.com -russia-today-video.ru -russkoe-zdorovie.ru -sabaapress.com -saddiechoua.com -saitevpatorii.com -salutmontreal.com +rus-teh.narod.ru +sakhboard.ru +saltspray.ru samoiedo.it -sandhillsonline.com -sanyuprojects.com -sarf3omlat.com -savefrom.com -sawin.beth.webd.pl -sayyoethe.blogspot.co.za -sbtdesign.co.uk -scanner-barak.top -scanner-donald.top -scanspyware.net -searchencrypt.com -searchengineranker.email -securityallianceservices.com -seoheap.com -seoholding.com -seojokes.net -serptehnika.ru -sex-dating.co -sexreliz.com -shemalegalls.blogporn.in +sampleletters.net +sape.top +sbt-aqua.ru +scanmyphones.com +scanner-andrew.top +scanner-walter.top +sdi-pme.com +searchinquire.com +searchmywindow-a.akamaihd.net +securesmrt-dt.com +seksvideoonlain.com +senger.atspace.co.uk +sexgalleries.top +sexs-foto.com shitmovs.com -sildenafilcitratemed.com -silktide.com -simplepooltips.com -sims-sims.ru -simul.co -sitevaluation.com -skypasss.com -slavkokacunko.de -sluganarodu.ru -smart-scripts.com -snjack.info -softlinesolutions.me -softomix.com -softomix.net -softwaretrend.net -sotechco.co +shivafurnishings.com +shopperifymac.com +sibecoprom.ru +siteaero.com +site.ru +sklad-24.ru +slimcdn.com +s.lollypopgaming.com +slonechka.ru +snomer1.ru +snworks.com +soblaznu.net +social-s-ggg.xyz +sonnikforme.ru +soochi.co spamnuker.com -speeddream.xyz -spidtest.space -spy-app.info -start.myplaycity.com -statoutlook.info -stmassage.ru -store-rx.com -streha-metalko.si -strigkaomsk.ru -studiokamyk.com.pl -supernew.org +spb-plitka.ru +sports-supplements.us +spywareit.com +statustroll.com +steame.ru +strag-invest.ru +stromerrealty.com +strongholdsb.ru +sukarame.net surcentro.com -surfbuyermac.com -surflinksmedical.com -szamponrevita.pl -takeflyte.com -taqplayer.info -titan-ads.life -titslove.yopoint.in -tizanidine6mg.blogspot.com -tizanidinecost.blogspot.com -tizanidinehcl4mgisitanarcotic.blogspot.com -tizanidinepills.blogspot.com -tizanidinerestlesslegsyndrome.blogspot.com -topappspro.com -toursmaps.com -tracksz.co -traffic100.com -trafficgenius.xyz -traffique.net -trahvid.com -trailer.cinemaflix.website -tuckermktg.com -turbodsp.com -typimga.pw -ucoz.ru -undergroundcityphoto.com -underthesite.com -uni.me -univerfiles.com -unlimitdocs.net -ural-buldozer.ru -urll.eu -usbggettwku.ga -ussearche.cf -uzungil.com -ventelnos.com -videochat.life -video-production.com -vikistars.com -vilingstore.net -villakohlanta.nu -vinsit.ru -vitoriacabos.com -vmnmvzsmn.over-blog.com -vroze.com +suture.co +svolze.com +takethatad.com +tako3.com +tastyfoodideas.com +taxi-v-eisk.ru +tedxrj.com +teesdaleflyballclub.co.uk +texbaza.by +thebestweightlosspills.ovh +thedownloadfreeonlinegames.blogspot.com +thefarmergame.com +thexart.club +thiegs.reco.ws +tiandeural.ru +tonerbox.kz +toolsky.com +topclickguru.com +top-l2.com +top-study.work +torrents.cd +trustl.life +tsstcorpcddvdwshbbdriverfb.aircus.com +tt-ipd.info +tversvet.ru +urlopener.com +utiblog.fr +valsalud.com +vapsy.com +varikozdok.ru +vduplo.ru +velobikestock.com +venta-prom.ru +vertaform.com +videokrik.net +videosbox.ru +vip-file.com +viven.host.sk +vizag.kharkov.ua +vkgaleria.com +vk-mus.ru +vseigru.one vzlom-na-zakaz.com -warezkeeper.com +wahicbefa31.soup.io +wallpapersist.com +watracker.net wcb.su -wdfdocando.com -websiteaccountant.de -westsextube.com -wholesalecheapjerseysfree.com -wholinkstome.com -woman-h.ru -woman-tampon.ru -worldoffiles.ru -wowas31.ucoz.ru -www.888.com -wwwadultcheck.com -x-diesel.org -xedserver.com -xep.info +webcamdevochka.com +webmasterhome.cn +website-analytics.online +website-analyzer.info +weekes.biz.tc +weightatraining.com +wellcome2slovenia.ru +whatistizanidinehclusedfor.blogspot.com +wholesalejerseyscheapjerseys.us.com +williamrobsonproperty.com +witherrom55.eklablog.fr +wladimirpayen.com +wnhjavlhezp.gq +womensterritory.ru +ww1943.ru +xaylapdiendanang.com +xgames-04.com xmladserver.com -xn--80ak6aa92e.com -xn----8sblgmbj1a1bk8l.xn----161-4vemb6cjl7anbaea3afninj.xn--p1ai -xn--b1agm2d.net -xn--e1afanlbnfckd7c3d.xn--p1ai -xnxx699.com -x-porno.video -xtraffic.plus -xxlargepop.com -xzlive.com +xn--80aikhbrhr.net +xn--b1adccaf1bzj.xn--p1ai +xn----itbeirbjbi7bc6bh2d.xn--p1ai +xn--lifehacer-1rb.com +xxxguitars.com yaoguangdj.com -youdesigner.kz -youghbould.wordpress.com +yellowfootprints.com +ynymnwbm.bloger.index.hr +yourads.website +yourdesires.ru yourporn.com -you-shall-not-pass.is74.ru -yugk.net -zahvat.ru -zaim-pod-zalog-krasnodar.ru -zajm-zalog-krasnodar.ru -zastenchivosti.net -zebramart.ru -zeroredirect12.com -zeroredirect9.com -zojirushi-products.ru +youtube-downloader.savetubevideo.com +yubikk.info +yuweng.info +zakazvzloma.com +za-music.mymobiplanet.com +zarabotat-na-sajte.ru +zarabotok--doma.ru +zebradudka.com +zerocash.msk.ru +zeroredirect8.com +zixizop.net.ru +zona-aqua.ru diff --git a/.dev-tools/test_units/random-referrers-for-whitelist-test.list b/.dev-tools/test_units/random-referrers-for-whitelist-test.list index c5b4d811a..b0f269185 100644 --- a/.dev-tools/test_units/random-referrers-for-whitelist-test.list +++ b/.dev-tools/test_units/random-referrers-for-whitelist-test.list @@ -1,250 +1,250 @@ -0daymusic.org +0lovespells0.blogspot.com +108shot.com 12masterov.com -2000k.ru -2728fb936f0.com -4kepics.com -aac500b7a15b2646968f6bd8c6305869d7.com -absugars.com -addblueoff.com.ua -adtology.com -adultnet.in +3w24.com +4istoshop.com +4senses.co +7ooo.ru +abcdefh.xyz +academiacsmendoza.org +adrunnr.com +ads.gold +adultmeetonline.info advertisingtag.net -ad-words.ru -alif-ba-ta.com -allboard.xobor.de -antons-transporte.de -apidiamondatanet-a.akamaihd.net -apparel-offer.com -appleid-verification.com -arewater.com -asophoto.com -autovideobroadcast.com -auto-zapchasti.org -awency.com -ayanaspa.com -banki76.ru -bank.uz -becuo.com -bestpornuha.com -betterhdporn.com -bharatdefencekavach.com -bimatoprost-careprost.com -bimatoprost-careprost.com.ua -bizcheapjerseyswholesalechina.com -buron.pw -cartierbraceletsreplica.pw -cfsrating.sonicwall.com -chatseo.com -chelyabinsk.xrus.org -clicksor.com -comissionka.net -compiko.info -compliance-brian.xyz -compliance-jessica.top -coolgramgoods.com -cool-wedding.net -copenergo.ru -cubs-jerseys.us -cyclobenzaprinevstizanidine.blogspot.com -data1.scopich.com -degerlund.net -dekorkeramik.ru -delfin-aqua.com.ua -doctissimo.fr -domain2008.com -dosug-lux.ru -doublepimp.com -downloads-whatsapp.com -drunkmoms.net -easycommerce.cf -e-collantes.com -ekto.ee -elstal.com.pl -englishdictionaryfree.com -evehemming.blogspot.com.au -evogarage.com -eyelike.com.ua -eyessurgery.ru -ez8motelseaworldsandiego.com -ezigarettekaufen2.mpbloggar.se -fickblock18.com -fidelityfunding.com -floridamobilebillboards.com -foto-sisek.porngalleries.top -frcls.fr -freshberry.com.ua -fuckbuddybestgilf.info -gamevalue7.weebly.com -gaming-journal.com -germetiki.com.ua -gk170.ru -goodhousekeeping.com -gpms.org.my -greatdealshop.com -happy.new.yeartwit.com +algerianembassy.co.in +all4invest.info +alt-servis.ru +android-style.com +apccargo.com +apelsinnik.website +apibetweenlinesn-a.akamaihd.net +apiwebwebgetcom-a.akamaihd.net +aramaicmedia.org +artdeko.info +articlesdirectoryme.info +asdfg.pro +autoloans.com +avtolombard-krasnodar.com +azbuka-mo.ru +back.dog +bannerspace.com +bestplacetobuyeriacta.jw.lt +bio.trade-jp.net +blackcurranthumidifiers.site +blacktwhite.com +blog.axant.it +blog.f00kclan.de +blpmovies.com +bombla.org +bookmark4you.com +bracketsmackdown.com +bristolhotel.com.ua +btvn.ru +buigas.00it.com +byme.se +bzero1jewelry.net +cactussoft.biz +candiceloves.us +canzoni.ru +cbox.ws +cheapjerseysap.com +cleanpcnow.com +clubfashionista.com +coldfilm.ru +compliance-andrew.top +compliance-irvin.xyz +compucelunlock.net +contextualyield.com +countbertwistdisp26.soup.io +cpajunkies.com +crd.clan.su +creditnation.ru +customerguru.in +dairyindia.in +decorazilla.com +dekoration.us +diminishedvalueoforegon.com +djekxa.ru +doctormakarova.ru +domainanalyzing.xyz +drev.biz +dustyorate.com +eaplay.ru +elektrischezigarettekaufen2.cowblog.fr +elkacentr.ru +elvenmachine.com +erotikchat-24.com +es5.com +extener.org +extstat.com +fatfasts-4tmz.com +filmbokep69.com +filmidivx.com +findpik.com +firstdrugmall.ru +flavors.me +floridahuntingfishingadventures.com +franch.info +frankofficial.ru +freeseedsonline.com +galaxys6manual.info +gameonasia.com +gateway.zscalertwo.net +ge0ip.org +girlspicsa.com +googleseo.com.tr +grupografico-pilar.com.ar +g.starmoe.xyz +guardlink.org +guruofcasino.com +hackers-crackers.tk hatdc.org -hentai-manga.porn -hidemyass.com -hostnow.men -hostsshop.ru -humanorightswatch.org -igru-xbox.net -infogame.name -innoslicon.com -intimshop-fantasy.ru -istizanidineanarcoticdrug.blogspot.com -itservicesthatworkforyou.com -jagg.info -janettabridal.com -jaxcube.info -jerseychinabizwholesale.us -jerseyschinabizwholesale.us -kalandranis.gr -kamalsinha.com -kam-dom.ru -keywordteam.net -kolotiloff.ru -konteiner24.com -kopihijau.info -krafte.ru -kursy-ege.ru -laulini.soclog.se -lincolntheatre.com -locationdesaison.com +hermesbirkinhandbagoutlets.com +homedecoguide.info +hothot.ru +hscsscotland.com +iccornacircri.cf +igrovyeavtomaty777.ru +i-hobot.ru +ilmen.net +imagui.eu +imguramx.pw +izhevsk.xrus.org +jerseywholesalebizchina.com +kichenaid.ru +kndxbkdx.bloger.index.hr +koleso24.com.ua +koronirealestate.gr +kotaku.com +kskjco.club +lancheck.net +landscaping.center +law-check-two.xyz +link.ac +liumimgx.pw +liveu.infoteka.hu +loadingpages.me +loftdigital.eu lol-smurfs.com -lonfon.xyz -lookingglassemb.com -louisvuittonoutletstore.net -magazintiande.ru -malls.com -marcowebonyodziezowe.pl -maridan.com.ua -max-eclat.men -mazda-roadsters.com -memberty.com -menetie.ru -michaelkorsoutlet.store -morepoweronmac.trade -moyakuhnia.ru -myanyone.net -mygameplus.com -mymobilemoneypages.com -myphotopipe.com -myyour.eu +luciddiagnostics.in +luckyday.world +m5home.ru +macnewtech.com +mere.host.sk +microsearch.ru +mnogolok.info +moesexy.com +moonci.ru +moscow.xrus.org +muschisexbilder.com +musezone.ru +my-aladin.com +myftpupload.com narkologiya-penza.ru -netvouz.com -new7ob.com -newstaffadsshop.club -nippon-bearings.ru -nottyu.xyz -odesproperty.com -officedocuments.net +navalwiki.info +new-post.tk +nsatc.net +nvformula.ru +oconto.ru +odywpjtw.bloger.index.hr +office2web.com okonich.com.ua -onefilms.net -onload.pw -o-o-6-o-o.com -openmultipleurl.com -paperwritingservice17.blogspot.ru -parajumpersstore.online -partnerline.men -pavlodar.xkaz.org -perederni.net -perm-profnastil.ru -petedrummond.com -petitions.whitehouse.gov -pijoto.net -pooleroadmedicalcentre.co.uk -pornosmola.info -postclass.com +onblastblog.online +onemantrip.com +online-podarki.com +o-o-11-o-o.com +oranga.host.sk +outclicks.net +oxford-book.com.ua +pandroid.co +paparazzistudios.com.au +pawli.eu +phobia.us +phpdevops.com +picmoonco.pw +pkr1hand.com +plastjulgranar.se +pliks.pl +popugaychiki.com +pornotubs.com +positive2b.ru +praisong.net predominant-invent.tk -preventheadacheguide.info -produkto.net -profolan.pl -progonrumarket.ru -pst2017.onlinewebshop.net -pxhdwsm.com -qualitymarketzone.com -qwarckoine.com -rainbowice.ru -razamicroelectronics.com -rcpmda.ikan1080.xyz -realmonte.net -recinziireale.com -redirectme.net -rethinkwasteni.info -rfserial.net -rome2rio.com -rusexy.xyz -russkoe-zdorovie.ru -rvtv.ru -ryetaw.com -s8-nowy-wygraj.comli.com -sadaholding.com -saecsa.co -sanidumps.com -scrapinghub.com -sdi-pme.com -seeingmeerkat.com -seeresultshub-a.akamaihd.net -sendearnings.com -servethis.com -sexfreepornoxxx.com -sexobzor.info -sexxdate.net -sharpchallenge.com -shisha-swag.de -sibdevice.ru -simplepooltips.com -sispe.com.br -snomer1.ru -solitaire-game.ru -spaceshipad.com -spitfiremusic.com -starpages.net -sticken.co -sugarlyflex.pw -swingerseiten.com -swiped.su -tanieaukcje.com.pl -tasteidea.com -tdsing.ru -thebitcoincode.com -theendivechronicles.com -thetardistimes.ovh -tiens2010.ru -tizanidinefibromyalgia.blogspot.com -tizanidinehcl2mgsideeffects.blogspot.com -toon-families.com -topvidos.ru +preparevideosafesystem4unow.space +pro-okis.ru +qcstrtvt.bloger.index.hr +quangcaons.com +redirlock.com +rednise.com +registry-cleaner.net +remmling.de +retreatia.com +ribieiendom.no +royalvegascasino.com +ruex.org.ua +runofilms.ru +runstocks.com +rutor.group +saletool.ru +sarafangel.ru +sbdl.no +sbornik-zakonov.ru +scanner-nelson.top +seksotur.ru +semaltmedia.com +senger.atspace.co.uk +sharebutton.net +shopfishing.com.ua +sladkoevideo.com +slavic-magic.ru +smallseotools.com +social-buttons.xyz +social-button.xyz +soochi.co +soundfrost.org +sovinsteel.ru +staticfs.host +stmassage.ru +stoki.ru +studiofaca.com +studiokamyk.com.pl +sumo.com +td-33.ru +telefonsex-ohne0900.net +tizanidineusedfor.blogspot.com +tomatis.gospartner.com tourismvictoria.com -toxicwap.com -trackmedia101.com +traffic2money.com trafficnetzwerk.de -trailer.cinemaflix.website -ua.tc -upupa.net -ussearche.cf -vbtracker.net -vclicks.net -velpanex.ru -viagra.pp.ua -video-chat.love -videtubs.pl -vitanail.ru -vykupavto-krasnodar.ru -vzglyadriv.kg -websitevaluebot.com -wellcome2slovenia.ru -wesharepics.site -westum.se -williamrobsonproperty.com -withstandingheartwarming.com -wtsindia.in -ww2awards.info -xn----btbdvdh4aafrfciljm6k.xn--p1ai -xtremeeagles.net -xz618.com -youbloodyripper.com -yourothersite.com -zastroyka.org -zdesformula.ru +trustaffs.com +uchil.net +udayavani.com +umaseh.com +valkiria-tk.ru +vertaform.com +vesnatehno.com +video-hollywood.ru +vkonche.com +vodaodessa.com +voluumtracker1.com +webinstantservice.com +webmasters.stackexchange.com +website-stealer.nufaq.com +wejdz-tu.pl +witherrom55.eklablog.fr +xn-----6kcaacnblni5c5bicdpcmficy.xn--p1ai +xn--80aeahghtf8ac5i.xn--p1ai +xtraffic.plus +xxx-treker.ru +y8games-free.com +yaminecraft.ru +ypmuseum.ru +za-music.mymobiplanet.com +zeroredirect1.com +zhacker.net +zigarettenonlinekaufen1.blox.pl +zigzog.ru +zionstar.net +zlatnajesen.com +znakomstvaonlain.ru +zynax.ua diff --git a/.dev-tools/test_units/ratelimittest-beta.txt b/.dev-tools/test_units/ratelimittest-beta.txt index a683b3cc9..ed30c44f7 100644 --- a/.dev-tools/test_units/ratelimittest-beta.txt +++ b/.dev-tools/test_units/ratelimittest-beta.txt @@ -5,14 +5,6 @@