diff --git a/.dev-tools/beta_conf_files/conf.d/globalblacklist.conf b/.dev-tools/beta_conf_files/conf.d/globalblacklist.conf index cff31fcd2..b61f81baa 100644 --- a/.dev-tools/beta_conf_files/conf.d/globalblacklist.conf +++ b/.dev-tools/beta_conf_files/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2019.08.1793 -### Updated: Fri Aug 9 09:43:35 SAST 2019 +### Version: V3.2019.08.1794 +### Updated: Fri Aug 9 18:11:01 SAST 2019 ### Bad Referrer Count: 6740 ### Bad Bot Count: 557 ################################################### @@ -7752,6 +7752,7 @@ geo $validate_client { 104.131.138.121 1; 104.199.80.9 1; 105.247.240.34 1; + 109.226.37.39 1; 109.95.32.172 1; 110.78.146.220 1; 111.125.143.174 1; diff --git a/.dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf b/.dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf index cc3de7218..479de1217 100644 --- a/.dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf +++ b/.dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -34.66.200.49 1; +35.192.85.2 1; 127.0.0.1 1; diff --git a/.dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf b/.dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf index 1d3684d1d..d9f0a2c04 100644 --- a/.dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf +++ b/.dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -34.66.200.49 0; +35.192.85.2 0; 127.0.0.1 0; diff --git a/.dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf b/.dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf index cff31fcd2..b61f81baa 100644 --- a/.dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf +++ b/.dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2019.08.1793 -### Updated: Fri Aug 9 09:43:35 SAST 2019 +### Version: V3.2019.08.1794 +### Updated: Fri Aug 9 18:11:01 SAST 2019 ### Bad Referrer Count: 6740 ### Bad Bot Count: 557 ################################################### @@ -7752,6 +7752,7 @@ geo $validate_client { 104.131.138.121 1; 104.199.80.9 1; 105.247.240.34 1; + 109.226.37.39 1; 109.95.32.172 1; 110.78.146.220 1; 111.125.143.174 1; diff --git a/.dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf b/.dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf index cc3de7218..479de1217 100644 --- a/.dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf +++ b/.dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -34.66.200.49 1; +35.192.85.2 1; 127.0.0.1 1; diff --git a/.dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf b/.dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf index 1d3684d1d..d9f0a2c04 100644 --- a/.dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf +++ b/.dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -34.66.200.49 0; +35.192.85.2 0; 127.0.0.1 0; diff --git a/.dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf b/.dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf index cff31fcd2..b61f81baa 100644 --- a/.dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf +++ b/.dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2019.08.1793 -### Updated: Fri Aug 9 09:43:35 SAST 2019 +### Version: V3.2019.08.1794 +### Updated: Fri Aug 9 18:11:01 SAST 2019 ### Bad Referrer Count: 6740 ### Bad Bot Count: 557 ################################################### @@ -7752,6 +7752,7 @@ geo $validate_client { 104.131.138.121 1; 104.199.80.9 1; 105.247.240.34 1; + 109.226.37.39 1; 109.95.32.172 1; 110.78.146.220 1; 111.125.143.174 1; diff --git a/.dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf b/.dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf index cc3de7218..479de1217 100644 --- a/.dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf +++ b/.dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -34.66.200.49 1; +35.192.85.2 1; 127.0.0.1 1; diff --git a/.dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf b/.dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf index 1d3684d1d..d9f0a2c04 100644 --- a/.dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf +++ b/.dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -34.66.200.49 0; +35.192.85.2 0; 127.0.0.1 0; diff --git a/.dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf b/.dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf index cff31fcd2..b61f81baa 100644 --- a/.dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf +++ b/.dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2019.08.1793 -### Updated: Fri Aug 9 09:43:35 SAST 2019 +### Version: V3.2019.08.1794 +### Updated: Fri Aug 9 18:11:01 SAST 2019 ### Bad Referrer Count: 6740 ### Bad Bot Count: 557 ################################################### @@ -7752,6 +7752,7 @@ geo $validate_client { 104.131.138.121 1; 104.199.80.9 1; 105.247.240.34 1; + 109.226.37.39 1; 109.95.32.172 1; 110.78.146.220 1; 111.125.143.174 1; diff --git a/.dev-tools/globalblacklist-testing.template b/.dev-tools/globalblacklist-testing.template index cff31fcd2..b61f81baa 100644 --- a/.dev-tools/globalblacklist-testing.template +++ b/.dev-tools/globalblacklist-testing.template @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2019.08.1793 -### Updated: Fri Aug 9 09:43:35 SAST 2019 +### Version: V3.2019.08.1794 +### Updated: Fri Aug 9 18:11:01 SAST 2019 ### Bad Referrer Count: 6740 ### Bad Bot Count: 557 ################################################### @@ -7752,6 +7752,7 @@ geo $validate_client { 104.131.138.121 1; 104.199.80.9 1; 105.247.240.34 1; + 109.226.37.39 1; 109.95.32.172 1; 110.78.146.220 1; 111.125.143.174 1; diff --git a/.dev-tools/globalblacklist.template b/.dev-tools/globalblacklist.template index cb33e58d7..f425a55a9 100755 --- a/.dev-tools/globalblacklist.template +++ b/.dev-tools/globalblacklist.template @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2019.08.1793 -### Updated: Fri Aug 9 09:43:33 SAST 2019 +### Version: V4.2019.08.1794 +### Updated: Fri Aug 9 18:10:59 SAST 2019 ### Bad Referrer Count: 6740 ### Bad Bot Count: 557 ################################################### diff --git a/.dev-tools/test1_conf_files/bots.d/whitelist-ips.conf b/.dev-tools/test1_conf_files/bots.d/whitelist-ips.conf index a48c747b7..503b1de7a 100644 --- a/.dev-tools/test1_conf_files/bots.d/whitelist-ips.conf +++ b/.dev-tools/test1_conf_files/bots.d/whitelist-ips.conf @@ -45,4 +45,4 @@ # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST -35.202.245.105 0; +35.224.112.202 0; diff --git a/.dev-tools/test1_conf_files/conf.d/globalblacklist.conf b/.dev-tools/test1_conf_files/conf.d/globalblacklist.conf index cb33e58d7..f425a55a9 100644 --- a/.dev-tools/test1_conf_files/conf.d/globalblacklist.conf +++ b/.dev-tools/test1_conf_files/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2019.08.1793 -### Updated: Fri Aug 9 09:43:33 SAST 2019 +### Version: V4.2019.08.1794 +### Updated: Fri Aug 9 18:10:59 SAST 2019 ### Bad Referrer Count: 6740 ### Bad Bot Count: 557 ################################################### diff --git a/.dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf b/.dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf index 58acf0a1d..465ec1626 100644 --- a/.dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf +++ b/.dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -35.202.245.105 1; +35.188.1.99 1; 127.0.0.1 1; diff --git a/.dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf b/.dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf index a69bef7d4..290c48e19 100644 --- a/.dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf +++ b/.dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -35.202.245.105 0; +35.188.1.99 0; 127.0.0.1 0; diff --git a/.dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf b/.dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf index cb33e58d7..f425a55a9 100644 --- a/.dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf +++ b/.dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2019.08.1793 -### Updated: Fri Aug 9 09:43:33 SAST 2019 +### Version: V4.2019.08.1794 +### Updated: Fri Aug 9 18:10:59 SAST 2019 ### Bad Referrer Count: 6740 ### Bad Bot Count: 557 ################################################### diff --git a/.dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf b/.dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf index 58acf0a1d..465ec1626 100644 --- a/.dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf +++ b/.dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -35.202.245.105 1; +35.188.1.99 1; 127.0.0.1 1; diff --git a/.dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf b/.dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf index a69bef7d4..290c48e19 100644 --- a/.dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf +++ b/.dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -35.202.245.105 0; +35.188.1.99 0; 127.0.0.1 0; diff --git a/.dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf b/.dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf index cb33e58d7..f425a55a9 100644 --- a/.dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf +++ b/.dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2019.08.1793 -### Updated: Fri Aug 9 09:43:33 SAST 2019 +### Version: V4.2019.08.1794 +### Updated: Fri Aug 9 18:10:59 SAST 2019 ### Bad Referrer Count: 6740 ### Bad Bot Count: 557 ################################################### diff --git a/.dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf b/.dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf index 58acf0a1d..465ec1626 100644 --- a/.dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf +++ b/.dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -35.202.245.105 1; +35.188.1.99 1; 127.0.0.1 1; diff --git a/.dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf b/.dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf index a69bef7d4..290c48e19 100644 --- a/.dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf +++ b/.dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -35.202.245.105 0; +35.188.1.99 0; 127.0.0.1 0; diff --git a/.dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf b/.dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf index cb33e58d7..f425a55a9 100644 --- a/.dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf +++ b/.dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2019.08.1793 -### Updated: Fri Aug 9 09:43:33 SAST 2019 +### Version: V4.2019.08.1794 +### Updated: Fri Aug 9 18:10:59 SAST 2019 ### Bad Referrer Count: 6740 ### Bad Bot Count: 557 ################################################### diff --git a/.dev-tools/test2_conf_files/bots.d/whitelist-ips.conf b/.dev-tools/test2_conf_files/bots.d/whitelist-ips.conf index aa66ed960..1d5bdbf7a 100644 --- a/.dev-tools/test2_conf_files/bots.d/whitelist-ips.conf +++ b/.dev-tools/test2_conf_files/bots.d/whitelist-ips.conf @@ -45,5 +45,5 @@ # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST -34.66.200.49 0; -35.202.245.105 0; +35.193.7.13 0; +35.184.226.236 0; diff --git a/.dev-tools/test2_conf_files/conf.d/globalblacklist.conf b/.dev-tools/test2_conf_files/conf.d/globalblacklist.conf index 66be18c36..e8caa4896 100644 --- a/.dev-tools/test2_conf_files/conf.d/globalblacklist.conf +++ b/.dev-tools/test2_conf_files/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2019.08.1793 -### Updated: Fri Aug 9 09:43:33 SAST 2019 +### Version: V4.2019.08.1794 +### Updated: Fri Aug 9 18:10:59 SAST 2019 ### Bad Referrer Count: 6740 ### Bad Bot Count: 557 ################################################### diff --git a/.dev-tools/test3_conf_files/bots.d/whitelist-ips.conf b/.dev-tools/test3_conf_files/bots.d/whitelist-ips.conf index 40b4b6f56..1e63b10f6 100644 --- a/.dev-tools/test3_conf_files/bots.d/whitelist-ips.conf +++ b/.dev-tools/test3_conf_files/bots.d/whitelist-ips.conf @@ -45,5 +45,5 @@ # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST -34.66.50.208 0; -35.202.245.105 0; +35.202.145.110 0; +104.154.120.187 0; diff --git a/.dev-tools/test3_conf_files/conf.d/globalblacklist.conf b/.dev-tools/test3_conf_files/conf.d/globalblacklist.conf index ea301b1f0..6fc389fce 100644 --- a/.dev-tools/test3_conf_files/conf.d/globalblacklist.conf +++ b/.dev-tools/test3_conf_files/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2019.08.1793 -### Updated: Fri Aug 9 09:43:33 SAST 2019 +### Version: V4.2019.08.1794 +### Updated: Fri Aug 9 18:10:59 SAST 2019 ### Bad Referrer Count: 6740 ### Bad Bot Count: 557 ################################################### diff --git a/.dev-tools/test4_conf_files/bots.d/whitelist-ips.conf b/.dev-tools/test4_conf_files/bots.d/whitelist-ips.conf index 954539aff..30a52ec41 100644 --- a/.dev-tools/test4_conf_files/bots.d/whitelist-ips.conf +++ b/.dev-tools/test4_conf_files/bots.d/whitelist-ips.conf @@ -45,6 +45,5 @@ # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST -34.66.25.221 0; -34.66.50.208 0; -34.66.200.49 0; +35.192.85.2 0; +35.202.145.110 0; diff --git a/.dev-tools/test4_conf_files/conf.d/globalblacklist.conf b/.dev-tools/test4_conf_files/conf.d/globalblacklist.conf index cb33e58d7..f425a55a9 100644 --- a/.dev-tools/test4_conf_files/conf.d/globalblacklist.conf +++ b/.dev-tools/test4_conf_files/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2019.08.1793 -### Updated: Fri Aug 9 09:43:33 SAST 2019 +### Version: V4.2019.08.1794 +### Updated: Fri Aug 9 18:10:59 SAST 2019 ### Bad Referrer Count: 6740 ### Bad Bot Count: 557 ################################################### diff --git a/.dev-tools/test5_conf_files/bots.d/whitelist-ips.conf b/.dev-tools/test5_conf_files/bots.d/whitelist-ips.conf index 72c8979cd..91bbb2c51 100644 --- a/.dev-tools/test5_conf_files/bots.d/whitelist-ips.conf +++ b/.dev-tools/test5_conf_files/bots.d/whitelist-ips.conf @@ -45,6 +45,6 @@ # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST -34.66.200.49 0; -34.66.50.208 0; -34.66.25.221 0; +35.202.145.110 0; +35.224.112.202 0; +35.184.226.236 0; diff --git a/.dev-tools/test5_conf_files/conf.d/globalblacklist.conf b/.dev-tools/test5_conf_files/conf.d/globalblacklist.conf index cb33e58d7..f425a55a9 100644 --- a/.dev-tools/test5_conf_files/conf.d/globalblacklist.conf +++ b/.dev-tools/test5_conf_files/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2019.08.1793 -### Updated: Fri Aug 9 09:43:33 SAST 2019 +### Version: V4.2019.08.1794 +### Updated: Fri Aug 9 18:10:59 SAST 2019 ### Bad Referrer Count: 6740 ### Bad Bot Count: 557 ################################################### diff --git a/.dev-tools/test_units/blacklist-ips.conf b/.dev-tools/test_units/blacklist-ips.conf index cc3de7218..479de1217 100644 --- a/.dev-tools/test_units/blacklist-ips.conf +++ b/.dev-tools/test_units/blacklist-ips.conf @@ -1,2 +1,2 @@ -34.66.200.49 1; +35.192.85.2 1; 127.0.0.1 1; diff --git a/.dev-tools/test_units/random-bots-for-test-quick.list b/.dev-tools/test_units/random-bots-for-test-quick.list index ef4a86d28..22476cc46 100644 --- a/.dev-tools/test_units/random-bots-for-test-quick.list +++ b/.dev-tools/test_units/random-bots-for-test-quick.list @@ -1,10 +1,10 @@ -ADmantX -Asterias -BDCbot -Evil -Go-Ahead-Got-It +BatchFTP +crawl.sogou.com +FlashGet Hloader -OnCrawl -Telesphoreo -webmeup-crawler +Internet Ninja +SearchmetricsBot +SeoSiteCheckup +SpankBot +WebAuto Xaldon_WebSpider diff --git a/.dev-tools/test_units/random-bots-for-test.list b/.dev-tools/test_units/random-bots-for-test.list index b91d3f394..b3463683b 100644 --- a/.dev-tools/test_units/random-bots-for-test.list +++ b/.dev-tools/test_units/random-bots-for-test.list @@ -1,250 +1,250 @@ -360Spider -404checker -404enemy -80legs Aboundex -Aboundexbot +ADmantX +AfD-Verbotsverfahren AhrefsBot -AIBOT AiHitBot +Aipbot Alligator -AllSubmitter -Anarchie -ASPSeek -Backlink-Ceck -BacklinkCrawler +archive.org_bot +Attach +backlink-check BackStreet -BackWeb -BBBike -BDFetch -BetaBot +Badass Bigfoot Bitacle -Blackboard -BlackWidow BLEXBot Blow -BlowFish Boardreader Bolt BotALot -Bullseye -BuzzSumo -CazoodleBot +Brandwatch Cegbfeieh -CheeseBot -CheTeam -ChinaClaw -Cliqzbot -Cloud mapping -Cogentbot +CherryPicker +Chlooe +Claritybot +coccocbot-web Collector -Copier -Cosmos +com.plumanalytics +CopyRightCheck +Craftbot crawler.feedback +CrazyWebCrawler Crescent -DatabaseDriverMysqli -Digincore -DigitalPebble -DIIbot -Discobot -Discoverybot +Custo +demandbase-bot +Demon +Devil +Dirbuster +Dispatch DnyzBot DomainAppender -DomainStatsBot +DomainCrawler Dotbot -Download Wonder Dragonfly +Drip +DTS Agent +EasyDL Ebingbong ECCP/1.0 -Ecxi -EMail Wolf -EroCrawler -Evil +Exabot +ExtLinksBot Extractor -Ezooms FDM -Flunky -Foobot -FyberSpider +FemtosearchBot +FHscan +FlashGet +Freeuploader GalaxyBot Genieo +GermCrawler Getintent -GetWeb -Gigabot Gotit GoZilla +Go!Zilla Grabber -Haansoft -Heritrix +Grafula +GrapeshotCrawler +GridBot +Harvest +HEADMasterSEO +Hloader +HMView HTTP::Lite HTTrack +Humanlinks HybridBot -IDBot -Image Fetch +Iblog +Id-search +IlseBot Image Sucker IndeedBot -InfoNaviRobot +Indy Library +InfoTekies instabid -InterGET Internet Ninja -ips-agent +internetVista monitor Iskanie +JamesBOT Jbrofuzz JennyBot JetCar -JikeSpider -Jorgee -Jyxobot +Jetty Kenjin Spider -Lanshanbot LexiBot -Lftp -Libwhisker -Lightspeedsystems -Likse LinkpadBot -LinkWalker -LinqiaRSSBot -Lipperhey -Lipperhey Spider +LinksManager +LinqiaScrapeBot +Litemage_walker +LNSpiderguy +lwp-request lwp-trivial +Magnet +Mail.RU_Bot +Majestic12 Majestic-SEO +MarkMonitor MarkWatch Mass Downloader Mata Hari MauiBot -meanpathbot Meanpathbot +mediawords MegaIndex.ru -Metauri -MFC_Tear_Sample +Microsoft Data Access Microsoft URL Control -MJ12bot +MIDown tool +MIIxpc Mojeek -MSFrontPage +Morfeus Fucking Scanner MSIECrawler +Msrabot +MS Web Services Client Protocol muhstik-scan +Musobot +Nameprotect Navroad NearSite -Needle -Nessus NetAnts +Netcraft +netEstate NE Crawler NetMechanic NetSpider +Nettrack Net Vampire -Netvibes NetZIP -NPbot +Nikto +Ninja +Nmap +Nutch +oBot Octopus Offline Navigator -OnCrawl -Openfind OpenLinkProfiler Openvas -OrangeBot +OpenVAS +OutclicksBot +PageAnalyzer Page Analyzer -PageGrabber page scorer PageScorer -Panscient -Papa Foto -Pavuk -Picscout -Picsearch +pcBrowser +PeoplePal +PHPCrawl PictureFinder Pimonster -Pi-Monster +Pixray plumanalytics Pockey -POE-Component-Client-HTTP +Probethenet ProPowerBot -Psbot +ProWebWalker PyCurl -QueryN Metasearch -Rankivabot -RealDownload -RebelMouse -Recorder +RankActive +RankingBot2 +Reaper ReGet +RepoMonkey Ripper -s1z.ru -ScanAlert +RocketCrawler +SBIder +scan.lol ScoutJet -Scrapy +Screaming +ScreenerBot Searchestate -SemrushBot -SEOkicks-Robot +SearchmetricsBot Seomoz -SEOprofiler -serpstatbot +SeoSiteCheckup sexsearcher +Siphon SISTRIX -Sitebeam SiteExplorer -SiteSnagger +Siteimprove SiteSucker +Site Sucker +Sitevigil SlySearch -Snake -Snapbot +SocialRankIOBot Sociscraper -sogouspider -Sosospider +Sottopop SpaceBison +Spammen SpankBot -Spanner -Spbot -Sqlworm +Spinn3r +SputnikBot +spyfu Sqworm -Steeler -Surfbot -Swiftbot -T0PHackTeam +Sucker +Sucuri +Suzuran +sysscan T8Abot -Teleport TeleportPro Telesoft Telesphoreo Telesphorep -Thumbor -TightTwatBot -Titan -Toata +The Intraformant +TheNomad Toweyabot -Tracemyfile Trendiction -Turingos +True_Robot TurnitinBot +Twice +URLy Warning URLy.Warning Vagabondo -Virusdie +VB Project +VCI +VoidEYE +Voil Voltron +Wallpapers/3.0 +WallpapersHD WASALive-Bot +Webalta Web Collage WEBDAV -Web Enhancer WebFetch -WebFuck Web Fuck WebGo IS -WebLeacher +WebImageCollector webmeup-crawler -WebsiteExtractor +WebPix +Web Sauger WebsiteQuester -Webster WebStripper -Whatweb -WinHTTrack -WiseGuys Robot +Web Sucker +Who.is Bot Wonderbot -Wotbox +Wprecon WPScan -WWW-Collector-E -WWW::Mechanize WWWOFFLE +x09Mozilla +Xaldon WebSpider +Xaldon_WebSpider xpymep1.exe -Zade zauba.io Zermelo Zeus -zgrab +ZumBot +ZyBorg diff --git a/.dev-tools/test_units/random-bots-for-whitelist-test.list b/.dev-tools/test_units/random-bots-for-whitelist-test.list index ba4b0549a..d37442383 100644 --- a/.dev-tools/test_units/random-bots-for-whitelist-test.list +++ b/.dev-tools/test_units/random-bots-for-whitelist-test.list @@ -1,250 +1,250 @@ -Aboundex +360Spider +80legs ADmantX -AIBOT -Aipbot +AfD-Verbotsverfahren +Alexibot +Alligator AllSubmitter -Anarchie -Apexoo -ASPSeek -Asterias +AlphaBot +archive.org_bot Attach -BackDoorBot -Backlink-Ceck +autoemailspider backlink-check BackStreet BackWeb -BBBike -BlackWidow -BlowFish -Boardreader +BatchFTP +BDCbot +BetaBot +Bigfoot +BLEXBot Bolt Buddy -Bullseye -BunnySlippers +BuiltBotTough +BuiltWith Calculon -CATExplorador -CazoodleBot -CheTeam -Chlooe +CheeseBot +Claritybot +Cliqzbot Cloud mapping Cogentbot cognitiveseo -Collector -com.plumanalytics Copier -Copyscape -Cosmos -Craftbot crawler4j -crawler.feedback -crawl.sogou.com -Curious -DBLBot +CrazyWebCrawler +CrunchBot +CSHttp +Custo +Devil DigitalPebble -Disco -DomainAppender +Dirbuster +Discobot +Dispatch +DomainCrawler DomainSigmaCrawler +DomainStatsBot Dotbot Download Wonder -Dragonfly -DSearch -DTS Agent -EasyDL -Ebingbong +Drip eCatch ECCP/1.0 +Ecxi +EirGrabber EMail Siphon -EMail Wolf +EroCrawler +evc-batch Exabot Express WebPictures -ExtractorPro +ExtLinksBot Extreme Picture Finder EyeNetIE -Ezooms facebookscraper -Fimap +FemtosearchBot +FHscan Firefox/7.0 -FlashGet Flunky Foobot +Freeuploader FrontPage -Fyrebot -GermCrawler +FyberSpider +Genieo +Getintent GetRight +Gigablast +Gigabot +Gotit GoZilla Grabber GrabNet +GrapeFX +GrapeshotCrawler GridBot -GT::WWW -Haansoft HaosouSpider -Harvest -Havij +HEADMasterSEO Heritrix -HTTP::Lite -Humanlinks +Hloader +HTMLparser +HTTrack +HybridBot IDBot +Id-search IlseBot -Indy Library +Image Sucker InfoNaviRobot InfoTekies -instabid InterGET -Internet Ninja InternetSeer +internetVista monitor ips-agent -IstellaBot -JennyBot -JetCar -Jetty -JOC Web Spider +IRLbot +Iskanie +JikeSpider Joomla -Jyxobot -Kenjin Spider +Jorgee +JustView +Keyword Density +Kozmosbot Lanshanbot -LeechFTP -LeechGet +LexiBot Lftp -LinkpadBot -LinkWalker -LinqiaMetadataDownloaderBot +Lightspeedsystems +LinkScan +LinksManager +LinqiaRSSBot +LinqiaScrapeBot Lipperhey Lipperhey Spider -Lmspider +Litemage_walker +Ltx71 +lwp-request LWP::Simple -lwp-trivial +magpie-crawler Mail.RU_Bot -Majestic SEO MarkWatch +Mass Downloader MauiBot -meanpathbot Meanpathbot -MeanPath Bot -Mediatoolkitbot mediawords Metauri -MFC_Tear_Sample Microsoft Data Access +Microsoft URL Control Mister PiX -MJ12bot Mojeek +Morfeus Fucking Scanner +MSFrontPage +Msrabot Musobot Name Intelligence Navroad NearSite -Needle -Nessus +Netcraft NetLyzer NetMechanic -Net Vampire +NetSpider +Nettrack +Netvibes +NetZIP +NextGenSearchBot Nibbler -Niki-bot -Nikto -NimbleCrawler +NICErsPRO Nutch -oBot Octopus Offline Navigator +OnCrawl Openfind -OpenVAS +OrangeBot OutclicksBot -OutfoxBot -PageAnalyzer -Page Analyzer page scorer -Panscient -Papa Foto Pavuk -pcBrowser -Picscout -PictureFinder -Pimonster -Pixray +PeoplePal PleaseCrawl -Probethenet +plumanalytics +POE-Component-Client-HTTP ProPowerBot -PxBroker +Psbot RankActive RankActiveLinkBot RankFlex -RankingBot2 -RankurBot RealDownload -ReGet -Ripper -RocketCrawler +Reaper +Recorder +RedesScrapy +RepoMonkey +Rogerbot s1z.ru -Scanbot +SBIder +ScoutJet Screaming -Searchestate -SEOkicks-Robot -Seomoz +ScreenerBot +SearchmetricsBot +SemrushBot +SEOkicks SEOprofiler -SeoSiteCheckup +SEOstats +serpstatbot sexsearcher -Siphon -Sitebeam -Siteimprove -SiteLockSpider -SiteSnagger +SISTRIX +SiteExplorer SiteSucker +SmartDownload SMTBot -Snoopy -Sogou web spider +Snake +SocialRankIOBot +Sociscraper +Sosospider Sottopop -Spammen +SpaceBison SpankBot Spanner -sp_auditbot SputnikBot Sqlmap -Sqlworm Sqworm -Stripper +Steeler Sucker -Surfbot +SuperBot +SuperHTTP +SurveyBot Swiftbot +sysscan T0PHackTeam -T8Abot +tAkeOut Teleport -Telesoft -Telesphorep +The Intraformant +TheNomad +Thumbor Titan -Toweyabot +Trendiction trendiction.com -Turnitin -TurnitinBot +True_Robot TwengaBot +Typhoeus +UnisterBot Upflow -VB Project -VidibleScraper -VoidEYE +URLy.Warning +Virusdie Voil -Web Auto -Web Collage -WebCopier -WEBDAV +WBSearchBot +Webalta Web Fetch -Web Fuck -WebLeacher -webmeup-crawler -WebSauger -Web Sauger +WebmasterWorldForumBot +WebPix +WebReaper Webshag -WebsiteExtractor +WebsiteQuester +Website Quester Webster -Whack -Who.is Bot -WISENutbot +WebStripper +Web Sucker +WebWhacker +WeSEE +Whatweb +Widow Wonderbot -WPScan -WWW-Collector-E -WWW::Mechanize -x22Mozilla +Woobot +WWWOFFLE +Xaldon WebSpider YoudaoBot -zauba.io +Zade Zermelo -Zeus zgrab -Zitebot -ZmEu +ZumBot +ZyBorg diff --git a/.dev-tools/test_units/random-referrers-for-test-quick.list b/.dev-tools/test_units/random-referrers-for-test-quick.list index 62134f552..32a19529e 100644 --- a/.dev-tools/test_units/random-referrers-for-test-quick.list +++ b/.dev-tools/test_units/random-referrers-for-test-quick.list @@ -1,10 +1,10 @@ -appsaurus.com -dogclothing.store -editors.choice6912650.hulfingtonpost.com -freza-sverlo.ru -investpamm.ru -msk-diplomat.com -mturkcontent.com -prpops.com -tds-advert002.info -zeroredirect.com +adpremium.org +blogporn.in +datsun-do.com +enskedesquashclub.se +ezrvrentals.com +thegamerznetwork.com +tizanidinevssoma.blogspot.com +tourismvictoria.com +xwatt.ru +yogamatsexpert.com diff --git a/.dev-tools/test_units/random-referrers-for-test.list b/.dev-tools/test_units/random-referrers-for-test.list index aae308469..ef6f821a5 100644 --- a/.dev-tools/test_units/random-referrers-for-test.list +++ b/.dev-tools/test_units/random-referrers-for-test.list @@ -1,500 +1,500 @@ -123locker.com -1hwy.com -1kdailyprofit.me -2000k.ru -2015god.org -21h2o.com -23kw.ru -4webmasters.com -51unlim.ru -5kstore.com -888.com -8xv8.com +00it.com +12bet.com +2itech.ru +34.gs +4istoshop.com +4ur.click +6hopping.com +7fon.ru a14download.com -aaronabel.com -activecampaign.dreamhosters.com -adclickservice.com -add-add.men -admeasures.com -adminshop.com -admitad.com -adservme.com -adtiger.tk -adultmeetonline.info -aero2.ru -alarmobninsk.ru -albamargroup.com -albumsuper.info -albuteroli.com -alert-fdm.xyz +a1security.com.ua +aa14ab57a3339c4064bd9ae6fad7495b5f.com +actionnooz.com +addbin.men +adidas.frwebs.fr +adimmix.com +adserver-e7.com +adsref.men +adultactioncam.com +adultfullhd.com +adviceforum.info +afmuseum.com +airmaxshoes-2016.com +aktivator-windows10.blogspot.com alfa9.com -allbizne.co.ua -alldezservice.kz -allforminecraft.ru -allnews24.in +alfa-img.com +all4invest.info +allmarketsnewdayli.gdn +alloysteel.ru +amateurmatch.com +angkortours.vn animal36.com -animali.deagostinipassion.it +animalrank.com +animebox.com.ua animenime.ru -apccargo.com -apifasterlightin-a.akamaihd.net -arclk.net -arcteryxsale.online -ardimobileinfo.ml -arpelsreplica.xyz -asscenihotosidea.blogspot.com -astrochicks.com -atovh.local-finders.com -autoua.top -autowebmarket.com.ua -avkzarabotok.com -backlinkwatch.com -bankcrediti.ru -barnfurnituremart.com -bayareaaeroclub.org -benchmarkcommunications.co.uk -berlininsl.com -bestadbid.com -bestcoin.cc -best-deals-products.com -best-way.men -betune.onlinewebshop.net -bigcities.org -bikini-image.com -bildsuche.ru -bio.trade-jp.net -bitporno.sx -blavia.00author.com -blobar.org -blog.koreadaily.com -bmw-ark.ru -bombla.org -bonkers.name -brothers-smaller.ru -burger-imperia.com -businescoop.men -buyessaynow.biz +anlimebel.kiev.ua +anonymous-redirect.com +apimountainbikei-a.akamaihd.net +apple.com-cleaner.systems +arendovalka.xyz +asmxsatadriverin.aircus.com +asrv-a.akamaihd.net +asseenontv.ru +azazu.ru +babieca.com +balook.com +bannerbank.ru +bestbuy.ca +bestchoice.cf +bestpornuha.com +besttorrentknifta.weebly.com +bestwrinklecreamnow.com +bet-prognoz.com +beyan.host.sk +bfz.biz +billiard-classic.com.ua +bingo8888.com +bjorkbacken.nu +blogsfunky672.weebly.com +bolitgorlo.net +boltalko.xyz +bosman.pluto.ro +braindaily.xyz +braintobe.win +brateg.xyz +brg8.com +browsepulse-a.akamaihd.net +brus.city +brus-vsem.ru +burn-fat.ga buyessayonline19.blogspot.ru -buy-forum.ru -buypuppies.ca -callawaygolfstore.online -callcafe.info -camdolls.com -cardsharp1.ru -cartierjusteunclou.xyz +buyhoverboard.com +buyviagraa.blogspot.com +callejondelpozo.es +candytiner.com +cartierlovebraceletreplica.xyz cartierlovebracelet.xyz -cartierlove.xyz -cellfun.mobi -cgi2.nintendo.co.jp -cheatcode-lita12.rhcloud.com -checkhit.com -chiptuninger.com +cartiertrinity.xyz +cashkitten-a.akamaihd.net +cbox.ws +cctva.tv +cdnanalytics.xyz +ceotrk.com +cercacamion.it +cfcl.co.uk +cfsrating.sonicwall.com +chatroulette.si +cheapjerseysap.com +cheapmoncler.xyz chocolateslim-en-espana.com -chocolateslim-original.com -christianlouboutinsaleonline.us -cialgenisrx.com -cialischmrx.com -cialischsrx.com -cineacademy.ru -citizenclsdriveri7.pen.io -clarithromycin500mg.com -claytransformations.info -clean-start.net -clevernt.com -collegerentals.com -communisave.co.za -compliance-alex.top -compliance-donald.xyz -compliance-olga.top -concordexoticrentals.com -contentlook.co -coolwallpapers-hd.com -cordstrap.cc -dailyrank.net -dalavia.ru -daooda.com +choice-credit.ru +chorus.terakeet.com +ciekawinki.pl +cigarpass.com +cjs.com.ru +clickintext.net +clicksor.com +codq.info +compliance-irvin.top +compliance-jess.top +compliance-julianna.top +compliance-walt.top +conboy.us +conciergegroup.org +cookingmeat.ru +costablue.xyz +cowblog.fr +crafthubs.com +crazyprotocol.com +crest-poloski.ru +csgo4.win +culmimg.pw +cupday.com +customergrowthsystems.com darodar.com data1.scopich.com -dawlenie.com -dcj-nn.ru -degerlund.net -delayreferat.ru -demenageur.com -deplim.com -descargar-musica-gratis.net -design-ideas.info -detskie-zabavi.ru -dieswaene.com -digitalfaq.com -discountbarbour.online -djihispano.com -djstools.com -dominateforex.ml -domznaniy.ru -dood.live -dostavka-v-ukrainu.ru -droidlook.net -drugs-no-rx.info -dtnlyss.com -dumpsmania24.com -dyt.net -econom.co -ekaterinburg.xrus.org -elektroniksigaraankara.info -elektrozigaretten1.postbit.com -eluxer.net -empis.magix.net -encodable.com -energy-ua.com -eorogo.top +davidovic.info +daydream-studio.ru +den-noch24.ru +deutschland123.de +dfiles.me +diarioaconcagua.com +discovertreasure-a.akamaihd.net +distver.ru +divatraffic.com +domaineaneblanc.com +dotmass.top +downloader12.ru +doyouknowtheword-flummox.ml +drhomes.biz +drunkmoms.net +druzhbany.ru +edudocs.net +ee77ee.com +elektronischezigarette2.mex.tl +empirepoker.com epicbrogaming.com -erectile.bid -ereko.ru -erot.co -erotik0049.com -esopini.com -etur.ru -evehemming.blogspot.com.au -evening-dating-club.info -event-tracking.com +europages.com.ru evogarage.com -exmasters.com -eyelike.com.ua +ewebarticle.info +exdocsfiles.com +eyessurgery.ru ezigarettekaufen.myblog.de -familyphysician.ru -favoritemoney.ru +ezrvrentals.com +fashion-stickers.ru +favornews.com +fba-mexico.com femmesdenudees.com -ferrotodo.com -fidelityfunding.com -fix-website-errors.com -flashlarevista.com -forum.poker4life.ru -freecamdollars.com -freenom.link -free-social-buttons.xyz -freeuploader.ml -freewareseek.com -fsakhalin.ru +ferretsoft.com +ff30236ddef1465f88547e760973d70a.com +fickenbumsen.net +filesmonster.porn +filunika.com.ru +findclan.org +forexgb.ru +forum20.smailik.org +fototravel.eu +fr-bearings.ru +frbizlist.com +free-stock-illustration.com +freetangodownload.com +freeuploader.com +front.to fx-brokers-review.com -g33.org -g7m.pl -gaming-journal.com -geoads.com -getlamborghini.ga -getpopunder.com -get-your-social-buttons.info -girlspicsa.com -gkvector.ru -gne8.com -gnuetella.com +gabeshop.ru +gallerylisting.com +gaytube.com +gazoblok.net.ua +gearsadspromo.club +getridofstretchmarks.org +gettpromos.com +getyourimage.club +ghostvisitor.com +gilbertbanda.net +glavtral.ru gobongo.info -gojiberriess.apishops.ru -googst2.ru -greatfind-a.akamaihd.net -growboxbl.ru -gsou.cf -gwebtools.com -habermetre.com -haikuware.com -hamptonoaks.ca -handicapbathtubarea.com -hao123.com -hasfun.com +greatdealshop.com +grtyi.com +gsbs.com.ua +gsmlab.pl +guildebzh.info +gungamesz.com +hackers-crackers.tk +hannasolution.ru +hanwei.us +hazardky.net +hchha.com hdimagegallery.net -hentaiheroes.com -high-speed1.net -homeart.space -homeinns.com -hotxnights.info -howopen.ru -hoztorg-opt.ru -hut1.ru -iccornacircri.cf -ico.re -ideashome.id -ifmo.ru -iklysha.ml -imediadesk.com -imgarcade.com -indo-export.ru -infospot.pt -infoupdate.org -insurple.com -interesnie-faktu.ru -iradiology.ru -it-max.com.ua -ivanstroi.ru +hdimagelib.com +healbio.ru +healthcarestore.info +hifidesign.ru +highspeed5.net +histats.com +hobild.net +hoste.octopis.com +hotchatdate.com +hoverboard360.at +hspline.com +hulfingtonpost.com +iamsport.org +igithab.com +ikritikimou.gr +ilovevitaly.xyz +immobiliaremassaro.com +importchinacoach-teach.com +indiasourcemart.in +innodgfdriverhm.aircus.com +int.search.tb.ask.com +invivo.hu +itrevolution.cf ivearchenceinflu.cf -james13prix.info +ix20.ru +izhevsk.xrus.org javitas.info -javlibrary.cc -jerseywholesalebizchina.com +jennyfire.ru +jimmychoosale.online +jobmarket.com.ua +johnrobertsoninc.com jpcycles.com justkillingti.me -kadashihotel.com -kandidos.com -katjimej.blog.fc2.com -keywordsuggest.org +justucalling32211123456789.tk +kaidalibor.de +kakablog.net +kerei.ru keywordsuggests.com -kidd.reunionwatch.com -kino2018.cc -kino-filmi.com -kiwe-analytics.com -kladrus.ru -kletkimehan.ru -kolotiloff.ru -konturkrasoty.ru -koral.se -kovesszucs.atw.hu -krynica.info -kvartir-remont.biz -labelwater.se -lankarns.com -laptoper.net -law-check-eight.xyz -law-check-seven.xyz +kicknights.gq +kinky-fetishes.com +kinobaks.com +kinogonew.ru +kinohit1.ru +kinoplen.ru +klumba55.ru +knogg.net +knowsitall.info +kollesa.ru +komputernaya-pomosh-moscow.ru +kosmetyki.tm.pl +kupit-adenu.ru +ladov.ru +landoftracking.com +laudit.ru laxdrills.com -lerporn.info -limtu.ifmo.ru -lincolntheatre.com -link.ac -linkbuddies.com -linkredirect.biz -littlesunraiser.com -liveinternet.ro -live-sexcam.tk -lkbennettoutlet.online -lmrauction.com -locksmith.jp -louboutinreplica.xyz -low-format.ru -lyrics.home-task.com -macdamaged.space -machicon-ueno.info -mackeeper-land-672695126.us-east-1.elb.amazonaws.com -macnewtech.com -make-money-online.com -mansiondelrio.co -mapquestz.us -maranbrinfo.com.br -mcnamaratech.com -meddesk.ru -meetingrainstorm.bid -methodsmarketing.com -mezaruk.info -michaelkorsoutletstore.net -mielec.pl -mir-betting.ru +ldrtrack.com +likrot.com +linkbux.ru +liran-locks.com +loftdigital.eu +lolnex.us +lol-smurfs.com +lostaruban.com +lulea-auktionsverk.se +lumb.co +luxmagazine.cf +m3gadownload.pl +magicplayer-s.acestream.net +maju.bluesalt.co +makemoneyonline.com +managerpak204.weebly.com +mansparskats.com +marcogrup.com +mature.free-websites.com +m.b00kmarks.com +mb140.ru +mearns-tractors.co.uk +medkletki.ru +mega-bony-2017.pl +meget.co.za +menhealed.net +mesto-x.com +micasainvest.com +mindbox.co.za +mindeyegames.com +minecraft-rus.org +mirmedinfo.ru mirobuvi.com.ua -mjchamonix.org -mobile.ok.ru -monetizer.com-01.site -monsterdivx.com -morefastermac.trade +mmofreegames.online +mobile-appster.ru +mokrayakiska.com +monclercheap.xyz +moncleroutletonline.pw +morlat.altervista.org mowser.com -muznachas-service.ru -mwtpludn.review -my-aladin.com -my-cash-bot.co -mydoctorok.ru +mriyadh.com +mrpornogratis.xxx +mycaf.it myhitmp3.club -myiptest.com -myshopmatemac.com +myperiod.club +mypets.by +myxdate.info +nagdak.ru narkologiya-peterburg.ru -naturalshair.site -new-post.tk -nhl17coinsforps3.gratisblog.biz -nicovideo.jp -nikitsyringedrivelg.pen.io -nzfilecloud.weebly.com -obesidadealgarve.com -onlinemeetingnow.com -onlineporno.site -ooo-gotovie.ru -optibuymac.com -ossmalta.com -outpersonals.com -oxford-book.com.ua -paperwritingservice17.blogspot.ru +nbsproject.ru +nero-us.com +netanalytics.xyz +newsperuse.com +newstudio.tv +nextlnk12.com +nibbler.silktide.com +njpalletremoval.com +oconto.ru +oda.as +oddamzadarmo.eu +offers.bycontext.com +ohmyrings.com +okonich.com.ua +old-rock.com +oneshotdate.com +onlinebay.ru +onlineserialy.ru +optikremont.ru +ordernorxx.com +orenburg-gsm.ru +orgasmus-virtual.com +ownshop.win +owohho.com +oxotl.com +ozas.net +pacificair.com paradontozanet.ru -park.above.com -partner-cdn.men -paydayloanslocal.com -pc-services.ru -pc-virus-d0l92j2.pw -perl.dp.ua -petsblogroll.com -piccdata.com -piccshare.com -picmoonco.pw -picsforkeywordsuggestion.com -picturesmania.com -pistonclasico.com -piter.xrus.org -pix24x7.com -pixell.club -pix-hd.com -pladform.ru -ploenjitmedia.azurewebsites.net -pokemon-go-play.online -pokerniydom.ru +parajumpersstore.online +parfusale.se +paydayonlinecom.com +pcimforum.com +perfectpracticeweb.com +photorepair.ru +phpdevops.com +picture-group.com +pinsdaddy.com +pirateday.ru +pixelrz.com +pizdopletka.club +platinumdeals.gr +play.leadzupc.com +pobeiranie.pl +pochtovyi-index.ru +pons-presse.com popunder.net -porndroids.com -pornogad.com -portal-eu.ru -pozdrawleniya.com +poquoson.org +pornhive.org +porno-dojki.net +pornzone.tv +postclass.com +pozdrawleniya.ru +preparevideosafesystem4unow.site presleycollectibles.com -preventheadacheguide.info -profitsport.club -promalp-universal.ru -prostitutki-almata.org -quebec-bin.com -rada.ru -ranksonic.org -rasteniya-vs-zombi.ru -razorweb-a.akamaihd.net -reelheroes.net -registrysweeper.com -remont-comp-pomosh.ru -renecaovillasale.online -resant.ru -resellerclub.com -rethinkwasteni.info -rialp.getenjoyment.net -rniaeba.ga -rockma.se -rosbalt.com.ua -royalvegascasino.com -ruex.org.ua -rusexy.xyz -salut-camp.ru -saneyes.com -scanmyphones.com +printie.com +privacyassistant.net +privat-girl.net +privetsochi.ru +prostitutki-astana.org +proxyradar.com +ptr.ruvds.com +pulse33.ru +purplestats.com +pxhdwsm.com +pyrodesigns.com.au +qwarckoine.com +rabotaetvse.ru +rankingchart.de +rankscanner.com +ranksignals.com +r-control.ru +rcpmda.ikan1080.xyz +redbottomheels.xyz +regionshop.biz +registrydomainservices.com +rep-am.com +resultshub-a.akamaihd.net +rfd-split.hr +runstocks.com +rvi.biz +sakhboard.ru +samolet.fr +sbetodiodnye-lampy.ru +scanner-barak.top scanner-brian.top -scrapy.org -sc-specialhost.com -semp.net -seocheki.net -seo-tools-optimizing.com -serpstat.com -sextracker.be -shinikiev.com.ua -shtora66.ru -sideeffectsoftizanidine.blogspot.com -sim-service.net -skanninge.se -slavia.info -smart-balancewheel.com -social-vestnik.ru -sofit-dmd.ru -spin2016.cf -spy-sts.com -stackthatbucks.com -stair.registrydomainservices.com -staticfs.host -steame.ru +scanner-irvin.top +scanner-josh.top +scanner-mary.top +screentoolkit.com +sdelatmebel.ru +segol.tv +seorankinglinks.com +sexiporno.net +sexspornotub.com +sexyali.com +sexystrippe.info +shama-rc.net +shivafurnishings.com +shymkent.xkaz.org +silktide.com +site-analyzer.com +site-auditor.online +sivs.ru +sixcooler.de +smartpet.ru +snjack.info +snow.nvr163.com +socialbutton.xyz +social-fun.ru +sonata-arctica.wz.cz +space-worry.ml +sprttrack.com +ssconstruction.co +statistici.ro sticken.co -stop-gepatit.te.ua -studiofaca.com -svolze.com +stmassage.ru +stockspmb.info +stroymonolit.su +superstats.com +support.nopeas.sk +sygraem.com symbaloo.com -tabakur77.com -tampabaywatch.org -taximytishi.ru -teastory.co -technika-remont.ru -tedxrj.com -tehngr.ru -telefonsexkostenlos.tk +syndicate.fun +t3chtonic.com +taxi-v-eisk.ru +t-bygg.com +telegraf.by testingads.pro -thebestphotos.eu -theendivechronicles.com -thefarmergame.com -thegalerie.eu -thegameriders.com -thegioixekhach.com -themestotal.com -thetattoohut.com -timhost.ru +tfxiq.com +thebestweightlosspills.ovh +thebitcoincode.com +thebluffs.com +theplacetoupdating.pw +the-usa-games.blogspot.com +ticketsys.inetwd.com +timdreby.com +timecrimea.ru +timer4web.com titangel-vietnam.com -tizanidinehcl2mgsideeffects.blogspot.com -tizanidinerecreationaluse.blogspot.com -tizanidineshowupondrugtest.blogspot.com -tizanidinevssoma.blogspot.com -tizanidinevsvalium.blogspot.com -token-lab.org -tongkatmadura.info -topads.men -topappspro.com -torontoplumbinggroup.com -torrents-tracker.com -track-rankings.online -traffic2cash.org -trafficstars.com -truebeauty.cc -trumpetedextremes.com -trustedhealthtips.com -tt-ipd.info -tuberkulezanet.ru +tizanidine4mguses.blogspot.com +tizanidineoral.blogspot.com +tizanidinerestlesslegsyndrome.blogspot.com +tizanidineusedfor.blogspot.com +tk-assortiment.ru +tnaionline.org +toloka.hurtom.com +top10-way.com +topshef.ru +torrnada.ru +tour-line.net +tracksurf.daooda.com +trafaret74.ru +trafficmp.com +traffmonster.info +trion.od.ua +triumf-realty.ru +try-rx.com +ttrraacckkrr.com tuberkuleznik.ru -tuckpointingmasonrysystems.com -tv-spoty.info -twitlinks.com -twu.com.ua -ukrup.com -uni.me -unitygame3d.com -universals.com.ua -url-img.link -utrolive.ru +turbodsp.com +turbo-suslik.org +tvnewsclips.info +twinderbella.com +twojebook.pl +txxx.com +typer.one +u555u.info +uginekologa.com +unionmarkt.de +uptimebot.net +urccvfmc.bloger.index.hr +urdoot.win +v720hd.ru vadimkravtcov.ru -valuado.com -valueclick.com -vbabule.net -vellings.info -venerologiya.com -ventelnos.com -vesnatehno.ru -videochat.cafe -viteonlusarezzo.it -voltrknc1.com -vrnelectro.ru -warezkeeper.com -website-analytics.online -website-speed-up.site -website-stealer.nufaq.com -wemedinc.com -whatistizanidinehclusedfor.blogspot.com -wholesalejerseysgaa.com -whosonmyserver.com -wildnatureimages.com -womama.ru -wordkeyhelper.com -worldoffiles.ru -wowcasinoonline.ooo -wpthemedetector.co.uk -www.888.com -wwwadultcheck.com -www.get-free-traffic-now.com -x5market.ru -x-diesel.biz -xev.ru -xlovecam.com -xn--24-glceagatoq7c2a6ioc.xn--p1ai -xn--80adaggc5bdhlfamsfdij4p7b.xn--p1ai +validdomain.xyz +valsalud.com +vduplo.ru +venta-prom.ru +vezuviy.su +via-energy-acquistare.com +viagraneggrx.com +videos-for-your-business.com +vilingstore.net +vipps.com.my +voloo.ru +vrotike.ru +vzglyadriv.kg +vzube.com +wahicbefa31.soup.io +westen-z.life +while.cheapwebsitehoster.com +witherrom55.eklablog.fr +workle.website +worldtraveler.world +www.timer4web.com +x69ty.ru +xblog.in +xn--80ahdheogk5l.xn--p1ai xn--80ajbshivpvn2i.xn--p1ai -xn--80ak6aa92e.com -xn--80aodinpgi.xn--p1ai -xn----8sbarihbihxpxqgaf0g1e.xn--80adxhks -xn--b1adccaf1bzj.xn--p1ai -xn--b1agm2d.net -xnxx-n.com -xzlive.com -youdao.com -yourdesires.ru -yunque.pluto.ro -zarajbuilders.com -zebramart.ru -zed21.net -zixizop.net.ru +xsion.net +xuki.us +youdesigner.kz +yourserverisdown.com +yourtemplatefinder.com +yubikk.info +zaimite.ru +zerocash.msk.ru zoogdiesney.com -zoosexart.com +zrelaya.pw diff --git a/.dev-tools/test_units/random-referrers-for-whitelist-test.list b/.dev-tools/test_units/random-referrers-for-whitelist-test.list index cd92eb3a2..08ad9832e 100644 --- a/.dev-tools/test_units/random-referrers-for-whitelist-test.list +++ b/.dev-tools/test_units/random-referrers-for-whitelist-test.list @@ -1,250 +1,250 @@ +100searchengines.com +123cha.com 1stat.ru -2itech.ru -4kpics.rocks -7minuteworkout.com -8kisses.eu -acheterviagrafr24.com -adrenalin-stk.ru -airmaxshoes-2016.com -algerianembassy.co.in -alkoravto.ru -alpharma.net -amazon-adsystem.com -apiappenableinfo-a.akamaihd.net -auto4style.ru -ayanaspa.com -backgroundpictures.net -backuperwebcam.weebly.com -bag77.ru -baixar-musicas-gratis.com -bang-hotties.com -bear.gotcher.us -becuo.com -billiard-classic.com.ua -blog.axant.it -bloke.com -bmw.afora.ru -bolezniorganov.ru -brendbutik.ru -brimstonehillfortress.org -buy-forum.ru -buytizanidineonlinenoprescription.blogspot.com -canrioloadm.gq -cartechnic.ru -cartierreplica.pw -cartierwatch.xyz -chatroulette.world -cherrypointplace.ca -cialis-samples.com -ciproandtizanidine.blogspot.com -click2pawn.com -clodo.ru -cl.s7.exct.net -compliance-alexa.xyz -compliance-andrew.xyz -compliance-checker.info +2728fb936f0.com +2drittel.de +4kplayer.pl +8xv8.com +add-add.men +advocatemsk.ru +aktivator-windows10.blogspot.com +allkrim.com +all-streaming-media.com +alphahoverboards.com +apelsinnik.website +audiofree.ru +avtovolop.ru +azlex.uz +bagsonsale.online +bala.getenjoyment.net +bambi.ck.ua +berdasovivan.ru +bestofferswalkmydogouteveryday.gq +biznesluxe.ru +bloggedporn.com +boc.kz +bristolhotel.com +butstrap.space +cartierjusteunclou.xyz +cartiertrinity.xyz +cascadelink.org +casite-513049.cloudaccess.net +chiptuninger.com +chlooe.com +choice-credit.ru +christianlouboutinreplica.win +clickpapa.com +coin-hive.com +com-cleaner.systems +com-onlinesupport.host +companies-catalog.com +compliance-margo.top +compliance-viktor.top +compliance-walt.top +computernetworksonline.com connectionstrenth.com -continent-e.tv -country-chic.ru -cs-passion.pl -customboxes4less.com -customerpromos-a.akamaihd.net +copypaste.traderzplanet.in +copyrightclaims.org +cpajunkies.com +customchocolate.business-for-home.com +custom-product-labels.com +cymbaltaandtizanidine.blogspot.com cypernhuset.se -danceuniverse.ru -dangphoto.trade -dating-time-now.com -devochki.top -dirtpics.pw -disruptingdinnerparties.com -doctissimo.fr -doeco.ru -dojki-hd.com -druzhininevgeniy63.blogspot.com -e-c.al -e-collantes.com -election.interferencer.ru -elektrischezigarettekaufen2.cowblog.fr -elektrozigarette2.webs.com -elektrozigarettekaufen2.blogse.nl -elexies.info -elidelcream.weebly.com -energy-ua.com -erotik0049.com -etm-consult.de -eu-cookie-law.info -europeanwatches.ru +damasarenai.info +daydream-studio.ru +dbmkfhqk.bloger.index.hr +deshevo-nedorogo.ru +detskie-konstruktory.ru +dividendo.ru +dogbreedspicture.net +dolohen.com +domain2008.com +downloader12.ru +download-wallpaper.net +easync.io +ekobata.ru +elektronischezigarettekaufen2.tumblr.com +empirepoker.com +en.home-task.com +enskedesquashclub.se +epicbrogaming.com +erotische-geschichten-xxl.com +exchanges-bet.com exlarseva.webblog.es -exoneration-project.us -ezigarettenshop2.postbit.com -f1nder.org -fellowshipoftheminds.com +eyessurgery.ru +ezigarettekaufen2.yolasite.com +ezigarettekaufen.myblog.de +fatmaelgarny.com +feeriaclub.ru femmesdenudees.com -fickenbumsen.net -fodelsedagspresenter.nu -foodcrafts.website -forexgb.ru -formseo.com -frustrated-favorable.gq -funnymama.com -funnypica.com +filmgo.ru +filmi-onlain.info +financepoints.eu +forums.toucharcade.com +fotopop.club +freshberry.com.ua g33.org -game300.ru -gaytube.com -genetworx.com -gotomontenegro.net -gsasearchengineranker.space -guitar-master.org -hidemyass.com -homegardenlova.com -hptwaakw.blog.fc2.com -ibmdatamanagement.co -infowarcraft.ru -intellego.info -intermesh.net -itronics.ca -iusstf.org -iwantedmoney.com -iweblist.info -jacago.com -jillepille.com -j-times.ru -kanimage.com -kazrent.com -kinobest.pl -kiskinhouse.com -knigonosha.net -kol-energo.ru -krasnodar.xrus.org -kriokomora.info -ks1234.com -kupit-adenu.ru -kvartir-remont.biz +game-top.su +gamezblox.com +gemgrab-a.akamaihd.net +girlsfuckdick.com +glopages.ru +grand-chlen.ru +growmyfunds.ca +hahashka.ru +hdpixent.com +hetmanship.xyz +hitcpm.com +hosting-tracker.com +hscsscotland.com +hulfingtonpost.com +igrovyeavtomaty777.ru +imicrovision.com +impisr.ru +inboxdollars.com +intervsem.ru +iplusbit.blogspot.co.za +iyasimasennka.com +jav-fetish.site +kazka.ru +khafre.us +kongruan.com +kopihijau.info +kredit-pod-zalog-krasnodar.ru +kretpicf.pw +ladov.ru lecbter-relationships.ga -lego4x4.ru -lenvred.org -lernur.net -lmrauction.com -magnetic-bracelets.ru -mantramusic.ru -manyresultshub-a.akamaihd.net -meetingrainstorm.bid -minneapoliscopiers.com -moivestiy.biz -monclercheap.xyz -moneytop.ru -mosrif.ru -mundoaberrante.com -musicstock.me -mycaf.it -mydearest.co +linerdrilling.com +lion.bolegapakistan.com +liveu.infoteka.hu +livingroomdecor.info +locustdesign.co +lolnex.us +luckyday.world +lyrster.com +m292.info +master-muznachas.ru +mcar.in.ua +mesto-x.com +midst.eu +mncrftpcs.com +mole.pluto.ro mydownloadengine.com -naughtyconnect.com -nikitabuch.com -nikkiewart.ru -njkmznnb.ru -no-fuel.org -nudepatch.net -oakleyglassesonline.us -onefilms.net -onlinadverts.com -ororodnik.goodbb.ru +mymobilemoneypages.com +myprintscreen.com +naturtreenspicerx.pw +netvouz.com +news-speaker.com +newyorkhotelsmotels.info +nextlnk12.com +nlfjjunb5.ru +nowtorrents.com +nzfilecloud.weebly.com +offtopic.biz +okonich.com.ua +oneclickfiles.com +online-sbank.ru +ooo-gotovie.ru +optikremont.ru outpersonals.com -ownshop.win -page2rss.com +ownshop.cf +palvira.com.ua +parfusale.se partner-high.men -poffet.net -pornosexrolik.com -pornpost.in -portal-eu.ru -porto.abuilder.net -pozdrawleniya.ru -printdirectforless.com -priora-2.com -projectforte.ru -puzo2arbuza.ru -rcb101.ru -refererx.com -regdefense.com -remote-dba.de -reyel1985.webnode.fr -rospromtest.ru -rusenvironmental.net -rvzr-a.akamaihd.net -rybalka-opt.ru -s1z.ru -salmonfishingsacramentoriver.com -scanner-don.top -searchimage.co -servethis.com -sexkontakte-seite.com -sexytrend.ru -shijian.ac.cn +partybunny.ru +pharmacyincity.com +picquery.com +pizda.lol +popserve.adscpm.net +pornhub-ru.com +prodvigator.ua +progonrumarket.ru +pst2017.onlinewebshop.net +psvita.ru +putevka24.ru +quality-traffic.com +quick-seeker.com +reckonstat.info +redirectme.net +redirect.trafficreceiver.club +reining.lovasszovetseg.hu +ros-ctm.ru +rubbed.us +russia-today-video.ru +sabaapress.com +safe-app.net +sarf3omlat.com +sashagreyblog.ga +scanner-andrew.top +scanner-irvin.top +searchtooknow-a.akamaihd.net +see-your-website-here.com +seoboxes.com +seojokes.net sientalyric.co -si-unique.com -s.lollypopgaming.com -small-games.biz -smstraf.ru -snjack.info -socialbookmarksubmission.org -social-search.me -souvenirua.com -stair.registrydomainservices.com -standardchartered-forex.com -stariy-baku.com -superstats.com -tacbelarus.ru +sierraapps.com +sildenafil-tadalafil.info +simplepooltips.com +slavic-magic.ru +spacash.com +sprttrack.com +sps-shop.com +stats-collector.org +streetfooduncovered.com +strigkaomsk.ru +strongsignal-a.akamaihd.net +superfish.com +supers.com.ua tagil.zrus.org -tarad.com -textads.men -tfxiq.com -tizanidinedosageforsleep.blogspot.com -tizanidinefibromyalgia.blogspot.com +tandvardshuset.net +taximytishi.ru +technika-remont.ru +tecspb.ru +terraclicks.com +thecounter.com +thegolfclub.info +theprofitsmaker.net +tizanidine4mg.blogspot.com +tizanidineinfo.blogspot.com +tizanidinerestlesslegsyndrome.blogspot.com tnctrx.com -topshef.ru -torrents-tracker.com -tracksz.co +toloka.hurtom.com +topbestgames.com +track2.shop tradedeals.biz -truebeauty.cc -ttsq.fr -tubeoffline.com -typer.one -uac.net.au -uginekologa.com -unacittaconte.org -uptime.com -uptime-gamma.net -urll.eu -uroki.net -valsalud.com -vclicks.net -vedomstvo.net -video-chat.in -villacoloniale.com -visa-pasport.ru -voditeltrezviy.ru +trafficmp.com +tsstcorpcddvdwshbbdriverfb.aircus.com +turkeyreport.tk +tytoona.com +ukkelberg.no +uvozdeckych.info +vasileostrovsky-rayon.ru +videos-for-your-business.com +visa-china.ru +viteonlusarezzo.it +vizitki.net +voloslove.ru vostoktrade.info -vzlomfb.com -warezkeeper.com -warningzscaler.heraeus.com -waterpurifier.club -website-speed-check.site -weclipart.com -wetgames.ru -wholesalejerseys-cheapest.com -wikes.20fr.com -wingsoffury2.com -woodyguthrie.se -xev.ru -x-lime.com -xn-----6kcaacnblni5c5bicdpcmficy.xn--p1ai -xn----7sbbagbq7bd5aheftfllo4m.xn--p1ai -xn--d1aifoe0a9a.top -xwatt.ru -yaderenergy.ru -yottos.com -yourtemplatefinder.com -yugo-star.ru -yuweng.info -zeroredirect7.com -zhongwenlink.com -zone-kev717.info +vzubah.com +watchinf.com +webenlace.com.ar +webradiology.ru +websiteaccountant.de +wholesalecheapjerseysfree.com +wleuaprpxuvr.ga +wpthemedetector.co.uk +xkaz.org +x-lime.net +x-mix.info +xmladserver.com +xn--24-glceagatoq7c2a6ioc.xn--p1ai +xn-----7kceclhb4abre1b4a0ccl2fxch1a.xn--p1ai +xn----7sbaphztdjeboffeiof6c.xn--p1ai +xn----itbeirbjbi7bc6bh2d.xn--p1ai +xnxxandxvideos.com +xvideosporn.biz +yellowfootprints.com +yourads.website +zarepta.com +zawyna.ua +zeroredirect6.com diff --git a/.dev-tools/test_units/ratelimittest-beta.txt b/.dev-tools/test_units/ratelimittest-beta.txt index afb1627e8..6b86856ae 100644 --- a/.dev-tools/test_units/ratelimittest-beta.txt +++ b/.dev-tools/test_units/ratelimittest-beta.txt @@ -5,13 +5,6 @@