diff --git a/.dev-tools/beta_conf_files/conf.d/globalblacklist.conf b/.dev-tools/beta_conf_files/conf.d/globalblacklist.conf index 576b150cf..f491cd512 100644 --- a/.dev-tools/beta_conf_files/conf.d/globalblacklist.conf +++ b/.dev-tools/beta_conf_files/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2019.07.1780 -### Updated: Wed Jul 31 14:58:48 SAST 2019 +### Version: V3.2019.07.1781 +### Updated: Wed Jul 31 16:24:45 SAST 2019 ### Bad Referrer Count: 6740 ### Bad Bot Count: 556 ################################################### diff --git a/.dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf b/.dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf index 12f18b91b..90fe4f9d3 100644 --- a/.dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf +++ b/.dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -35.202.145.110 1; +104.154.120.187 1; 127.0.0.1 1; diff --git a/.dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf b/.dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf index c3c284efc..bc389ee32 100644 --- a/.dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf +++ b/.dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -35.202.145.110 0; +104.154.120.187 0; 127.0.0.1 0; diff --git a/.dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf b/.dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf index 576b150cf..f491cd512 100644 --- a/.dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf +++ b/.dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2019.07.1780 -### Updated: Wed Jul 31 14:58:48 SAST 2019 +### Version: V3.2019.07.1781 +### Updated: Wed Jul 31 16:24:45 SAST 2019 ### Bad Referrer Count: 6740 ### Bad Bot Count: 556 ################################################### diff --git a/.dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf b/.dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf index 12f18b91b..90fe4f9d3 100644 --- a/.dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf +++ b/.dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -35.202.145.110 1; +104.154.120.187 1; 127.0.0.1 1; diff --git a/.dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf b/.dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf index c3c284efc..bc389ee32 100644 --- a/.dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf +++ b/.dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -35.202.145.110 0; +104.154.120.187 0; 127.0.0.1 0; diff --git a/.dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf b/.dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf index 576b150cf..f491cd512 100644 --- a/.dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf +++ b/.dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2019.07.1780 -### Updated: Wed Jul 31 14:58:48 SAST 2019 +### Version: V3.2019.07.1781 +### Updated: Wed Jul 31 16:24:45 SAST 2019 ### Bad Referrer Count: 6740 ### Bad Bot Count: 556 ################################################### diff --git a/.dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf b/.dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf index 12f18b91b..90fe4f9d3 100644 --- a/.dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf +++ b/.dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -35.202.145.110 1; +104.154.120.187 1; 127.0.0.1 1; diff --git a/.dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf b/.dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf index c3c284efc..bc389ee32 100644 --- a/.dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf +++ b/.dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -35.202.145.110 0; +104.154.120.187 0; 127.0.0.1 0; diff --git a/.dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf b/.dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf index 576b150cf..f491cd512 100644 --- a/.dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf +++ b/.dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2019.07.1780 -### Updated: Wed Jul 31 14:58:48 SAST 2019 +### Version: V3.2019.07.1781 +### Updated: Wed Jul 31 16:24:45 SAST 2019 ### Bad Referrer Count: 6740 ### Bad Bot Count: 556 ################################################### diff --git a/.dev-tools/globalblacklist-testing.template b/.dev-tools/globalblacklist-testing.template index 576b150cf..f491cd512 100644 --- a/.dev-tools/globalblacklist-testing.template +++ b/.dev-tools/globalblacklist-testing.template @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2019.07.1780 -### Updated: Wed Jul 31 14:58:48 SAST 2019 +### Version: V3.2019.07.1781 +### Updated: Wed Jul 31 16:24:45 SAST 2019 ### Bad Referrer Count: 6740 ### Bad Bot Count: 556 ################################################### diff --git a/.dev-tools/globalblacklist.template b/.dev-tools/globalblacklist.template index 5bf255eba..43f9b946a 100755 --- a/.dev-tools/globalblacklist.template +++ b/.dev-tools/globalblacklist.template @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2019.07.1780 -### Updated: Wed Jul 31 14:58:47 SAST 2019 +### Version: V4.2019.07.1781 +### Updated: Wed Jul 31 16:24:44 SAST 2019 ### Bad Referrer Count: 6740 ### Bad Bot Count: 556 ################################################### diff --git a/.dev-tools/setup-ngxblocker/vhost-cases/ansible.vhost b/.dev-tools/setup-ngxblocker/vhost-cases/ansible.vhost index 3861d1c68..a44e35933 100644 --- a/.dev-tools/setup-ngxblocker/vhost-cases/ansible.vhost +++ b/.dev-tools/setup-ngxblocker/vhost-cases/ansible.vhost @@ -2,6 +2,10 @@ server { listen 8080; server_name xxxx; +### NGINX BOT BLOCKER INCLUDE +include /etc/nginx/bots.d/blockbots.conf; + + location / { return 301 https://$host$request_uri; } @@ -10,6 +14,10 @@ server { server { listen 443 ssl http2; server_name xxxxx + +### NGINX BOT BLOCKER INCLUDE +include /etc/nginx/bots.d/blockbots.conf; + root {{ vhost_www_directory }}/sites/public; include {{ nginx_blockbots_cfg_path }}; @@ -26,6 +34,10 @@ server { root {{ vhost_www_directory }}/admin/public; server_name admin.{{ domain_to_protect }}; +### NGINX BOT BLOCKER INCLUDE +include /etc/nginx/bots.d/blockbots.conf; + + include {{ nginx_yprox_security_path }}; include {{ nginx_yprox_ssl_path }}; @@ -40,6 +52,10 @@ server { listen 443 ssl http2; server_name alp.{{ domain_to_protect }}; +### NGINX BOT BLOCKER INCLUDE +include /etc/nginx/bots.d/blockbots.conf; + + root {{ alp_live_directory }}; include {{ nginx_bots_cfg_path }}/blockbots.conf; index index.html; diff --git a/.dev-tools/test1_conf_files/bots.d/whitelist-ips.conf b/.dev-tools/test1_conf_files/bots.d/whitelist-ips.conf index 798770665..896ffe576 100644 --- a/.dev-tools/test1_conf_files/bots.d/whitelist-ips.conf +++ b/.dev-tools/test1_conf_files/bots.d/whitelist-ips.conf @@ -45,4 +45,4 @@ # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST -35.188.1.99 0; +104.154.120.187 0; diff --git a/.dev-tools/test1_conf_files/conf.d/globalblacklist.conf b/.dev-tools/test1_conf_files/conf.d/globalblacklist.conf index 5bf255eba..43f9b946a 100644 --- a/.dev-tools/test1_conf_files/conf.d/globalblacklist.conf +++ b/.dev-tools/test1_conf_files/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2019.07.1780 -### Updated: Wed Jul 31 14:58:47 SAST 2019 +### Version: V4.2019.07.1781 +### Updated: Wed Jul 31 16:24:44 SAST 2019 ### Bad Referrer Count: 6740 ### Bad Bot Count: 556 ################################################### diff --git a/.dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf b/.dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf index 12f18b91b..479de1217 100644 --- a/.dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf +++ b/.dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -35.202.145.110 1; +35.192.85.2 1; 127.0.0.1 1; diff --git a/.dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf b/.dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf index c3c284efc..d9f0a2c04 100644 --- a/.dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf +++ b/.dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -35.202.145.110 0; +35.192.85.2 0; 127.0.0.1 0; diff --git a/.dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf b/.dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf index 5bf255eba..43f9b946a 100644 --- a/.dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf +++ b/.dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2019.07.1780 -### Updated: Wed Jul 31 14:58:47 SAST 2019 +### Version: V4.2019.07.1781 +### Updated: Wed Jul 31 16:24:44 SAST 2019 ### Bad Referrer Count: 6740 ### Bad Bot Count: 556 ################################################### diff --git a/.dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf b/.dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf index 12f18b91b..479de1217 100644 --- a/.dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf +++ b/.dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -35.202.145.110 1; +35.192.85.2 1; 127.0.0.1 1; diff --git a/.dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf b/.dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf index c3c284efc..d9f0a2c04 100644 --- a/.dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf +++ b/.dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -35.202.145.110 0; +35.192.85.2 0; 127.0.0.1 0; diff --git a/.dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf b/.dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf index 5bf255eba..43f9b946a 100644 --- a/.dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf +++ b/.dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2019.07.1780 -### Updated: Wed Jul 31 14:58:47 SAST 2019 +### Version: V4.2019.07.1781 +### Updated: Wed Jul 31 16:24:44 SAST 2019 ### Bad Referrer Count: 6740 ### Bad Bot Count: 556 ################################################### diff --git a/.dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf b/.dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf index 12f18b91b..479de1217 100644 --- a/.dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf +++ b/.dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -35.202.145.110 1; +35.192.85.2 1; 127.0.0.1 1; diff --git a/.dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf b/.dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf index c3c284efc..d9f0a2c04 100644 --- a/.dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf +++ b/.dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -35.202.145.110 0; +35.192.85.2 0; 127.0.0.1 0; diff --git a/.dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf b/.dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf index 5bf255eba..43f9b946a 100644 --- a/.dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf +++ b/.dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2019.07.1780 -### Updated: Wed Jul 31 14:58:47 SAST 2019 +### Version: V4.2019.07.1781 +### Updated: Wed Jul 31 16:24:44 SAST 2019 ### Bad Referrer Count: 6740 ### Bad Bot Count: 556 ################################################### diff --git a/.dev-tools/test2_conf_files/bots.d/whitelist-ips.conf b/.dev-tools/test2_conf_files/bots.d/whitelist-ips.conf index acb73c006..adf26e87b 100644 --- a/.dev-tools/test2_conf_files/bots.d/whitelist-ips.conf +++ b/.dev-tools/test2_conf_files/bots.d/whitelist-ips.conf @@ -45,5 +45,5 @@ # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST +104.154.120.187 0; 35.202.145.110 0; -35.188.1.99 0; diff --git a/.dev-tools/test2_conf_files/conf.d/globalblacklist.conf b/.dev-tools/test2_conf_files/conf.d/globalblacklist.conf index 35313ce8e..997612f63 100644 --- a/.dev-tools/test2_conf_files/conf.d/globalblacklist.conf +++ b/.dev-tools/test2_conf_files/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2019.07.1780 -### Updated: Wed Jul 31 14:58:47 SAST 2019 +### Version: V4.2019.07.1781 +### Updated: Wed Jul 31 16:24:44 SAST 2019 ### Bad Referrer Count: 6740 ### Bad Bot Count: 556 ################################################### diff --git a/.dev-tools/test3_conf_files/bots.d/whitelist-ips.conf b/.dev-tools/test3_conf_files/bots.d/whitelist-ips.conf index 607f5404f..a2156e437 100644 --- a/.dev-tools/test3_conf_files/bots.d/whitelist-ips.conf +++ b/.dev-tools/test3_conf_files/bots.d/whitelist-ips.conf @@ -45,5 +45,5 @@ # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST -35.184.226.236 0; -35.202.145.110 0; +104.154.120.187 0; +35.192.85.2 0; diff --git a/.dev-tools/test3_conf_files/conf.d/globalblacklist.conf b/.dev-tools/test3_conf_files/conf.d/globalblacklist.conf index 4ee67bea3..0390045b3 100644 --- a/.dev-tools/test3_conf_files/conf.d/globalblacklist.conf +++ b/.dev-tools/test3_conf_files/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2019.07.1780 -### Updated: Wed Jul 31 14:58:47 SAST 2019 +### Version: V4.2019.07.1781 +### Updated: Wed Jul 31 16:24:44 SAST 2019 ### Bad Referrer Count: 6740 ### Bad Bot Count: 556 ################################################### diff --git a/.dev-tools/test4_conf_files/bots.d/whitelist-ips.conf b/.dev-tools/test4_conf_files/bots.d/whitelist-ips.conf index ce96e6089..430e746ee 100644 --- a/.dev-tools/test4_conf_files/bots.d/whitelist-ips.conf +++ b/.dev-tools/test4_conf_files/bots.d/whitelist-ips.conf @@ -45,6 +45,6 @@ # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST -35.192.85.2 0; -35.193.7.13 0; -35.224.112.202 0; +104.154.120.187 0; +35.184.226.236 0; +35.202.145.110 0; diff --git a/.dev-tools/test4_conf_files/conf.d/globalblacklist.conf b/.dev-tools/test4_conf_files/conf.d/globalblacklist.conf index 5bf255eba..43f9b946a 100644 --- a/.dev-tools/test4_conf_files/conf.d/globalblacklist.conf +++ b/.dev-tools/test4_conf_files/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2019.07.1780 -### Updated: Wed Jul 31 14:58:47 SAST 2019 +### Version: V4.2019.07.1781 +### Updated: Wed Jul 31 16:24:44 SAST 2019 ### Bad Referrer Count: 6740 ### Bad Bot Count: 556 ################################################### diff --git a/.dev-tools/test5_conf_files/bots.d/whitelist-ips.conf b/.dev-tools/test5_conf_files/bots.d/whitelist-ips.conf index f26dff412..142e19c14 100644 --- a/.dev-tools/test5_conf_files/bots.d/whitelist-ips.conf +++ b/.dev-tools/test5_conf_files/bots.d/whitelist-ips.conf @@ -45,6 +45,6 @@ # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST -35.184.96.71 0; -35.184.226.236 0; +35.193.7.13 0; +35.192.85.2 0; 35.202.145.110 0; diff --git a/.dev-tools/test5_conf_files/conf.d/globalblacklist.conf b/.dev-tools/test5_conf_files/conf.d/globalblacklist.conf index 5bf255eba..43f9b946a 100644 --- a/.dev-tools/test5_conf_files/conf.d/globalblacklist.conf +++ b/.dev-tools/test5_conf_files/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2019.07.1780 -### Updated: Wed Jul 31 14:58:47 SAST 2019 +### Version: V4.2019.07.1781 +### Updated: Wed Jul 31 16:24:44 SAST 2019 ### Bad Referrer Count: 6740 ### Bad Bot Count: 556 ################################################### diff --git a/.dev-tools/test_units/blacklist-ips.conf b/.dev-tools/test_units/blacklist-ips.conf index 12f18b91b..90fe4f9d3 100644 --- a/.dev-tools/test_units/blacklist-ips.conf +++ b/.dev-tools/test_units/blacklist-ips.conf @@ -1,2 +1,2 @@ -35.202.145.110 1; +104.154.120.187 1; 127.0.0.1 1; diff --git a/.dev-tools/test_units/random-bots-for-test-quick.list b/.dev-tools/test_units/random-bots-for-test-quick.list index cf0fe7264..3ad5270da 100644 --- a/.dev-tools/test_units/random-bots-for-test-quick.list +++ b/.dev-tools/test_units/random-bots-for-test-quick.list @@ -1,10 +1,10 @@ -Alligator -autoemailspider -BotALot -Ezooms -IstellaBot -MS Web Services Client Protocol -SEOstats -SiteSnagger -Webster -Whatweb +Bandit +Blackboard +FHscan +HaosouSpider +Litemage_walker +Microsoft Data Access +seoscanners +Sosospider +Sqlworm +URLy Warning diff --git a/.dev-tools/test_units/random-bots-for-test.list b/.dev-tools/test_units/random-bots-for-test.list index c444082c4..76bc523aa 100644 --- a/.dev-tools/test_units/random-bots-for-test.list +++ b/.dev-tools/test_units/random-bots-for-test.list @@ -1,250 +1,250 @@ 360Spider +404checker +404enemy +80legs +Abonti +Acunetix ADmantX -AhrefsBot -AIBOT AiHitBot -Aipbot -Alexibot Alligator -Anarchie +AllSubmitter Apexoo -archive.org_bot -BackDoorBot +autoemailspider +backlink-check +BacklinkCrawler BackStreet -Battleztar Bazinga -Bitacle +Badass +Barkrowler +BBBike +BDCbot Blow +BlowFish Boardreader -BotALot -Brandwatch -BuiltWith -CazoodleBot +Brandprotect +Buddy +BuiltBotTough +Bullseye +BunnySlippers +BuzzSumo +CATExplorador +CCBot +Cegbfeieh +CheeseBot CherryPicker +CheTeam +ChinaClaw +Chlooe +Claritybot Cliqzbot Cloud mapping -Cogentbot Collector +com.plumanalytics +Copyscape Cosmos -crawler4j -crawler.feedback +crawl.sogou.com CrazyWebCrawler -CrunchBot +Crescent CSHttp -Curious -Custo +DBLBot +demandbase-bot Demon -Deusu Devil -DIIbot +DigitalPebble Disco -DomainSigmaCrawler -Drip -Ebingbong -ECCP/1.0 -EMail Siphon -EMail Wolf +Discoverybot +DittoSpyder +DomainStatsBot +Dotbot +EasyDL +eCatch +EroCrawler evc-batch -Evil -Express WebPictures +ExtLinksBot ExtractorPro +EyeNetIE Ezooms -facebookscraper -FemtosearchBot -Fimap -FlashGet -Flunky -Foobot +FDM +FHscan +Firefox/7.0 +Freeuploader Fyrebot GalaxyBot -GetRight -GetWeb -Gigablast -Gigabot Gotit GoZilla -Grabber Grafula -GrapeshotCrawler +GrapeFX GridBot -HaosouSpider +GT::WWW +Haansoft Harvest +Havij HEADMasterSEO Hloader -HTMLparser HTTP::Lite HTTrack -Humanlinks Iblog -IlseBot Image Sucker -IndeedBot -Indy Library -InfoNaviRobot -InfoTekies -InterGET -internetVista monitor -Iria +instabid +Intelliseek +Internet Ninja +ips-agent +IRLbot Iskanie -IstellaBot Jbrofuzz +JennyBot JetCar Jetty -JikeSpider JOC Web Spider -Joomla Jorgee -JustView -Kenjin Spider -Keyword Density +Jyxobot Kozmosbot -LeechFTP -LeechGet +LexiBot +Lftp Libwhisker +Lightspeedsystems Likse +Linkdexbot LinkpadBot -LinksManager +LinkWalker LinqiaMetadataDownloaderBot LinqiaRSSBot -LinqiaScrapeBot -Lipperhey Spider Lmspider +LNSpiderguy +Ltx71 lwp-request -LWP::Simple Magnet Mag-Net -magpie-crawler -Mail.RU_Bot -Majestic12 -Majestic-SEO -MarkMonitor Mass Downloader -Meanpathbot -MeanPath Bot +meanpathbot Mediatoolkitbot -Microsoft Data Access +mediawords +Metauri +Microsoft URL Control +MIDown tool +Mojeek +Morfeus Fucking Scanner Mr.4x3 -MSFrontPage -Msrabot -muhstik-scan -Navroad -Netcraft +MSIECrawler +MS Web Services Client Protocol +Musobot +Nameprotect +Needle +NetAnts NetMechanic -Nettrack -NICErsPRO -Nikto -Nimbostratus -NPbot -Nutch -OnCrawl -OpenVAS -OrangeBot -OrangeSpider -OutfoxBot -Page Analyzer +Nibbler +Offline Explorer +Openfind +OutclicksBot +PageAnalyzer PageGrabber -Pandalytics -Papa Foto +page scorer +PageScorer Pavuk -Pimonster -Pi-Monster +pcBrowser +PECL::HTTP +PHPCrawl +Picsearch PleaseCrawl +Pockey POE-Component-Client-HTTP -Probethenet +ProPowerBot +ProWebWalker +Psbot Pump -RankFlex -RankingBot2 +PyCurl +RankActive +RankActiveLinkBot +RankurBot RealDownload Reaper +RebelMouse +RedesScrapy Ripper +RocketCrawler Rogerbot -s1z.ru SalesIntelligent +SBIder +ScanAlert +Scanbot scan.lol -ScoutJet -ScreenerBot +Scrapy +Screaming Searchestate +SearchmetricsBot +Semrush SemrushBot -SEOkicks -SEOkicks-Robot +SEOlyticsCrawler SEOprofiler seoscanners -SEOstats +SeoSiteCheckup +sexsearcher Shodan +Siphon Sitebeam -SiteExplorer Siteimprove -SiteSnagger Snake -Snapbot -Snoopy -SocialRankIOBot -Sociscraper -sogouspider -SpankBot +Sogou web spider sp_auditbot +Spbot +Spinn3r +spyfu Sqlmap -Sqlworm +Steeler Stripper -SuperBot -SuperHTTP +Sucker +Sucuri +Surfbot SurveyBot -Swiftbot -sysscan -T8Abot +Szukacz +T0PHackTeam Teleport TeleportPro Telesphoreo -Telesphorep -The Intraformant -Thumbor -TightTwatBot +TheNomad Titan -Toata Toweyabot -Tracemyfile Trendiction trendiction.com -trendiction.de -Turnitin -TurnitinBot -TwengaBot -Twice -Typhoeus -Vagabondo -VidibleScraper +True_Robot +UnisterBot +Upflow +Vacuum +VB Project Virusdie +Voil Voltron -WASALive-Bot +Wallpapers/3.0 WBSearchBot -Webalta +WebAuto Web Auto -Web Collage -WEBDAV -WebEnhancer +WebBandit +WebCollage +WebCopier Web Enhancer -WebFetch Web Fetch WebFuck -Web Fuck -WebGo IS -WebImageCollector +WebmasterWorldForumBot webmeup-crawler WebPix -WebReaper -WebSucker +WebSauger +WebsiteQuester +Webster +WebStripper Web Sucker +WebWhacker +WebZIP WeSEE -Whack -Widow -WISENutbot -Woobot -Wotbox +Whacker +WinHTTrack +Wonderbot Wprecon -WWW-Collector-E -Xaldon WebSpider -Xaldon_WebSpider -xpymep1.exe +WWW-Mechanize +WWW::Mechanize +WWWOFFLE +x09Mozilla +YoudaoBot Zade -Zauba zauba.io Zermelo +Zeus +Zitebot diff --git a/.dev-tools/test_units/random-bots-for-whitelist-test.list b/.dev-tools/test_units/random-bots-for-whitelist-test.list index bfbae5059..b5dae62f9 100644 --- a/.dev-tools/test_units/random-bots-for-whitelist-test.list +++ b/.dev-tools/test_units/random-bots-for-whitelist-test.list @@ -1,250 +1,250 @@ +360Spider +404checker 80legs Abonti -Aboundexbot +Aboundex +AIBOT +AiHitBot +Aipbot Alexibot AllSubmitter Anarchie -Apexoo +archive.org_bot ASPSeek autoemailspider +Backlink-Ceck backlink-check -BatchFTP +BacklinkCrawler +BackWeb +Badass +Barkrowler +BBBike BDFetch BetaBot -Bigfoot -Bitacle -Blackboard Black Hole BlackWidow Blow -Boardreader -Bolt BotALot Brandwatch Buddy Bullseye -CazoodleBot +Calculon +CATExplorador +CCBot Cegbfeieh -CheeseBot -Claritybot Cloud mapping coccocbot-web -cognitiveseo -Copier -Craftbot -crawl.sogou.com +CopyRightCheck +Cosmos +CrazyWebCrawler CSHttp Custo DatabaseDriverMysqli -DataCha0s -demandbase-bot +DBLBot Demon -Deusu Devil -Digincore -Dirbuster -DomainAppender -DomainCrawler -DomainSigmaCrawler -DTS Agent -eCatch +DigitalPebble +DIIbot +Dotbot +Download Wonder +Dragonfly +ECCP/1.0 +Ecxi EMail Siphon -EMail Wolf EroCrawler evc-batch +Evil Exabot -ExtractorPro +Express WebPictures EyeNetIE +Ezooms facebookscraper -FHscan -Firefox/7.0 +Fimap Flunky Foobot Freeuploader FrontPage -FyberSpider -GalaxyBot -Genieo +GetRight GetWeb +G-i-g-a-b-o-t +Gotit +GoZilla Grabber GrabNet -Grafula +GrapeFX GrapeshotCrawler -GridBot +GT::WWW Haansoft -Harvest -Havij -HTMLparser +HaosouSpider +Heritrix +Hloader +HTTP::Lite +Humanlinks +HybridBot +Iblog IDBot +Image Sucker Indy Library InfoNaviRobot +InfoTekies instabid Intelliseek -InterGET InternetSeer internetVista monitor +Iria +IRLbot IstellaBot -Jbrofuzz JetCar -Joomla -JustView -Kenjin Spider -Keyword Density -Lanshanbot -Larbin -LexiBot -LibWeb +Jyxobot +LeechFTP Libwhisker -Lightspeedsystems -Linkdexbot +Likse LinkextractorPro -LinkScan -LinqiaMetadataDownloaderBot -Lipperhey Spider -Litemage_walker -lwp-request -LWP::Simple -magpie-crawler +LinkpadBot +LinkWalker +LinqiaRSSBot +LinqiaScrapeBot +LNSpiderguy +Magnet +Mag-Net Mail.RU_Bot -Mass Downloader -Mata Hari -Meanpathbot -MeanPath Bot -mediawords -MegaIndex.ru +Majestic SEO +Masscan +Mediatoolkitbot MFC_Tear_Sample Microsoft Data Access -MIDown tool +MIIxpc +MJ12bot Mojeek -MSFrontPage +Mr.4x3 +MSIECrawler Msrabot Musobot -Name Intelligence +Navroad Needle Nessus NetAnts netEstate NE Crawler -Nettrack -Net Vampire Netvibes NetZIP +NextGenSearchBot +Nibbler +NICErsPRO Niki-bot -Ninja +NimbleCrawler +Nimbostratus +Nmap +NPbot oBot +Offline Explorer Offline Navigator -Openfind -Openvas -OrangeBot +OnCrawl +OpenLinkProfiler +OrangeSpider +OutclicksBot OutfoxBot -Page Analyzer -page scorer Panscient Papa Foto pcBrowser PeoplePal -PHPCrawl -Pixray -PleaseCrawl -plumanalytics +Picsearch +Pimonster Pockey -ProPowerBot +Probethenet ProWebWalker Psbot -Quick-Crawler -RankFlex -RankingBot -RankingBot2 -Rankivabot +PyCurl RankurBot RealDownload -Reaper -Recorder +ReGet Rogerbot SalesIntelligent -scan.lol -ScoutJet +SBIder +ScanAlert +Scanbot Scrapy Screaming +Searchestate Semrush +SemrushBot SEOkicks SEOkicks-Robot -SEOlyticsCrawler Seomoz SEOprofiler +SeoSiteCheckup +SEOstats Shodan Siphon -SiteExplorer -SiteLockSpider +Sitebeam +Siteimprove +SiteSnagger SiteSucker -Site Sucker -SlySearch +Sitevigil +SMTBot +Snake Snapbot -Snoopy -Sociscraper -Sogou web spider +SocialRankIOBot +Sosospider +Sottopop +SpaceBison +Spammen SpankBot sp_auditbot -Spbot +SputnikBot +spyfu Sqlmap Sqlworm -Sqworm -Steeler -Sucuri -Surfbot -SurveyBot -Suzuran -Swiftbot -Szukacz -Teleport +SuperBot +T0PHackTeam +TeleportPro +Telesoft +Telesphoreo Telesphorep The Intraformant -TheNomad +Thumbor TightTwatBot -Toata +Tracemyfile +Trendiction Trendictionbot -True_Robot -Turingos +trendiction.com TurnitinBot -TwengaBot -Typhoeus UnisterBot +URLy Warning URLy.Warning -Vacuum Vagabondo -VB Project -VidibleScraper -Virusdie -Voltron +VCI +VoidEYE +Voil +WallpapersHD +WASALive-Bot WBSearchBot -WebAuto -WebBandit -WebCopier +Webalta +Web Collage WEBDAV WebEnhancer +WebFetch Web Fetch WebFuck -WebImageCollector +WebmasterWorldForumBot WebSauger -Web Sauger -WebsiteQuester -Webster -WebStripper -Web Sucker -WebWhacker +WebsiteExtractor +Website Quester +WebZIP WeSEE -Who.is Bot -WinHTTrack -Woobot -Wprecon +Whack +Widow +Wonderbot +Wotbox WPScan -WWW-Mechanize -x22Mozilla +WWW-Collector-E +WWW::Mechanize +WWWOFFLE +Xaldon WebSpider Xenu -Zade +xpymep1.exe +YoudaoBot Zauba -zauba.io +Zeus zgrab -Zitebot -ZmEu ZumBot ZyBorg diff --git a/.dev-tools/test_units/random-referrers-for-test-quick.list b/.dev-tools/test_units/random-referrers-for-test-quick.list index 355e0a39a..bdd0fc7da 100644 --- a/.dev-tools/test_units/random-referrers-for-test-quick.list +++ b/.dev-tools/test_units/random-referrers-for-test-quick.list @@ -1,10 +1,10 @@ -androids-store.com -editors.choice6912650.hulfingtonpost.com -hyiphunter.org -laudit.ru -marcowebonyodziezowe.pl -pornophoto.xyz -py100.ru -remmling.de -sophang8.com -supernew.org +apxeo.info +bmw-ark.ru +cartierbraceletsreplica.pw +fodelsedagspresenter.nu +graphid.com +myyour.eu +oil-td.ru +rocketchange.ru +sex.hotblog.top +xn-----6kccaibs5cb8afhjrfmix2n.xn--p1ai diff --git a/.dev-tools/test_units/random-referrers-for-test.list b/.dev-tools/test_units/random-referrers-for-test.list index 7c51b4745..36d008d22 100644 --- a/.dev-tools/test_units/random-referrers-for-test.list +++ b/.dev-tools/test_units/random-referrers-for-test.list @@ -1,500 +1,500 @@ -101billion.com -1adult.com -1pamm.ru -214jaluwobapef.cf -24-ak.ru -256bit.by -2itech.ru -2rich4bitches.com -3xforum.ro -4kpics.rocks -57883.net -8lufu.com -aac52006ec82a24e08b665f4db2b5013f7.com -abcdeg.xyz -abclauncher.com -above.com -absolute-s.ru -abusora.com -adloads.com -ads-seo.men -adult3dgames.com -advokat-grodno.by -adzerg.com -afmuseum.com -agahinameh.com -ahrntal.verymes.xyz -akuhni.by -alfa-img.com -allbizne.co.ua -all-number.com -allwomen.info -amazingninja.com -amyfoxfitness.com -anabolics.shop -apiadanaknet-a.akamaihd.net -apimegabrowsebiz-a.akamaihd.net -arraty.altervista.org -artdeko.info -as5000.com -asiengirls.net -auto4style.ru +007angels.com +00it.com +0n-line.tv +11pikachu.ru +123kuma.com +1flag.co.za +1stat.ru +2nt.ru +3w24.com +4pp13.com +4webmasters.org +51.la +991mostfm.co.id +aa14ab57a3339c4064bd9ae6fad7495b5f.com +abctoppictures.net +accessoires-mode.in +adserver-e7.com +ads.gold +adult-shop.com.ua +adultwebhosting.info +advancedsoftwaresupport.com +ahrefs.com +akusherok.ru +alf-img.com +algerianembassy.co.in +allcredits.su +allesohnegirls.net +allmarketsnewdayli.gdn +allnews.md +alphahoverboards.com +altamayoztourism.com +amazon-adsystem.com +an-donut.com +anti-virus-removal.info +antons-transporte.de +apilinkswiftco-a.akamaihd.net +apiqualitinknet-a.akamaihd.net +apispringsmartne-a.akamaihd.net +apiwebwebgetcom-a.akamaihd.net +arendakvartir.kz +asacopaco.tk +asiavirtualsolutions.com +atleticpharm.org +audiobangout.com +ausmepa.org.au autoseo-traffic.com -aviav.eu +autotrends.today avirasecureserver.com -ayakino.net -azazu.ru -babieca.com -backlink4u.net -bang-hotties.com -bankmib.ru -bank.uz -bannerconnect.net -bashtime.ru -bayareaaeroclub.org -beachtoday.ru -belstaffstore.online -bestcalovebracelet.cn -bestofferswalkmydogouteveryday.gq -bestofupload.info -best-seo-software.xyz -bet365.com -betteroffers.review -biteg.xyz -blackhatworth.com -blog.koreadaily.com -bochemit.com.ua -boltalko.xyz -bonvillan.com -bonzbuddy.com -boostmyppc.com -bosman.pluto.ro -brainzpod.win -brus.city -buigas.00it.com -builtwith.com -bum.com.ru -businescoop.men +babs.com.ua +baixar-musicas-gratis.com +baladur.ru +banksrf.ru +basedecor.ru +beachfix.co +becuo.com +begalka.xyz +beget.tech +best-businessman.ru +best-seo-offer.com +bestwebsiteawards.com +beta.hotkeys.com +betonka.pro +bibys.com +bigcareer.info +bitnote.co +biznesrealnost.ru +blacktwhite.com +blobar.org +bloke.com +bluesalt.co +board.f00d.de +bodybuilding-shop.biz +boeuklad.com +bolegapakistan.com +bombla.org +bonky.biz +bonzybuddy.com +brainboosting.club +brainsandeggs.blogspot.com +briomotor.co +budpost.com.ua +buntube.net +burkesales.com buzzonclick.com -carabela.com.do -cardsdumps.com -cars-modification.net -cartierlovebracelet.xyz -cartierlovestore.xyz -celejihad.info +bycontext.com +candiceloves.us +cand.jp +canvas.gloverid.site +canzoni.ru +cartierjusteunclou.xyz +cartierloveringreplica.xyz +cartierreplica.win +cbcseward.com +cdncash.com +cdncash.net +centrumcoachingu.com +cfacarrosserie74.com cfjump.com -chatroulette.si -cheapestjerseysonwholesale.com -chinacheapelitejerseys.com -chocolateslim-en-espana.com -cialischsrx.com -civilwartheater.com -cleanallvirus.com -clipartnew.com -club-samodelkin.ru -com-onlinesupport.site -compliance-ivan.xyz -compliance-jane.top -compliance-john.top -conocer-sanabria.com -contentsexpress.com -coolyarddecorations.com -copblock.org -cornomase.win +changde.58.com +charmstroy.info +check-host.net +cherrypointplace.ca +chimiver.info +chocolateslim-original.com +cicaki.net +citetick.com +cityadspix.com +click2pawn.com +clickhype.com +clickso.com +colobit.biz +compliance-brian.xyz +compliance-elena.xyz +compliance-julianna.top +compliance-viktor.top +comsysnet.com +connectionstrenth.com +cookie-law-enforcement-aa.xyz coslab.club -counter.yadro.ru -craftinsta.ru -culmimg.pw -customcollegeessays.net -cvety24.by -czat.wp.pl -dafatiri.com -dcj-nn.ru -deanmoore.ie +cougfan.info +crackzplanet.com +crazyboost.pro +creams.makeforum.eu +crest-poloski.ru +cypernhuset.se +dairyindia.in +darkbooks.org +darodar.com +dawlenie.com +ddpills.com +descargar-musica-gratis.net deshevo-nedorogo.ru -detsky-nabytek.info -dev33.dioniqlabb.se -dgfitness.co -diarioaconcagua.com -digital-video-processing.com -dinkolove.ya.ru -distver.ru -dkvorota.ru -doctormakarova.ru -documentsite.net -dstroy.su -e2click.com +detmebel.su +dev-seo.blog +dlya-android.org +dms-sw.ru +dnepr-avtospar.com.ua +doctissimo.fr +dogclothing.store +dojki-hd.com +dolohen.com +domain2008.com +domoysshop.ru +doxysexy.com earl-brown.info -e-c.al -eldiariodeguadalajara.com -elektrischezigaretten2.beeplog.com -elektrozigarette1.dreamwidth.org -elektrozigaretten2.yn.lt -envaseslotusama.com -eralph.tk -eredijovon.com -erotik0049.com -errorfixing.space -etur.ru -euromasterclass.ru -evaashop.ru -exonline.info -expertblog.info -ez8motelseaworldsandiego.com -ezigarettekaufen2.mpbloggar.se -face.hostingx.eu -fakehandbags.xyz -favorcosmetics.com -fellowshipoftheminds.com -fermersovet.ru -filefilter.weebly.com +easytuningshop.ru +ecommerce-seo.com +edelstahlschornstein-123.de +ednorxmed.com +eduinfosite.com +ekn-art.se +ekspresihati.info +elmacho.xyz +enskedesquashclub.se +epicbrogaming.com +erotiktreff24.info +esnm.ru +essay-edu.biz +etenininrade.ga +everflora.ru +ezigarettekaufen.myblog.de +family1st.ca +fastcrawl.com +fastfixing.tech +favoritemoney.ru +fba-mexico.com find1friend.com -flirt4free.com -fm-upgrade.ru -forensicpsychiatry.ru -forex-procto.ru -forum-engineering.ru -forum.tvmir.org -foxjuegos.com -foxtechfpv.com -foxweber.com -frameimage.org -freejabs.com -freeseedsonline.com -freshberry.com.ua -frustrated-favorable.gq -fsakhalin.ru -fungamelands.com -fun-mobi.pl -g7m.pl -gabeshop.ru -galleryawesome.com -gallery.rennlist.com -gepezz.info -getridofstretchmarks.org -gilbertbanda.net -gilsonchiro.xyz -girls-ufa.ru -global-ics.co.za -gopro-online.info -gpirate.com -gsasearchengineranker.top -gta-top.ru -gymi.name -hamptonoaks.ca -handicapvantoday.com -hazardky.net -hdxnxxtube.mobi -healing-dysplasia.ru -healthcarestore.info -heatpower.ru -helpmymacfaster.trade -herokuapp.com -hifidesign.ru -hmywwogw.bloger.index.hr -hobbyhuren24.net -hotchatdate.com -hotgirlhdwallpaper.com -hybrid.ru -idc.com.ua -iflycapetown.co.za -images.gyffu.com -imgarcade.com -indetiske.ya.ru -instasexyblog.com -int.search.tb.ask.com -iqbazar.ru -iqoption.com -iqoption.pro -iron-age.info -isistaylorporn.info -isitpaleo.info -izhevsk.xrus.org -jerseyswholesalechinalimited.com -jewelryandfiligree.com -jillepille.com -jongose.ninja -jovencitas.gratis -jurajskie.info -jus0wil.pp.ua -justdating.online -kamen-e.ru -kasino-money.pw -kemerovo.zrus.org -kinohit1.ru -klejonka.info -klosetkitten.com -knigonosha.net +finejewelryshop.ru +fireads.men +flipper.top +f-loaded.de +flowwwers.com +forms-mtm.ru +forsex.info +fototravel.eu +foxydeal.com +franecki.net +frbizlist.com +frcls.fr +freesitetest.com +free-today.com +freewareseek.com +fuel-gas.com +fxtips.ru +gamesprite.me +gamevalue7.weebly.com +gamewrath.com +genetworx.com +geoads.com +getaclueamerica.com +get-free-social-traffic.com +getresponse.com +girlgamerdaily.com +glopages.ru +gombita.info +gonextmedia.com +googlemare.com +greatfind-a.akamaihd.net +gribokstop.com +guruofcasino.com +gwhwpxbw.bloger.index.hr +hamacapty.com +happy.new.yeartwit.com +havepussy.com +hermesbelts.xyz +hoholikik.club +host-protection.com +hostsshop.ru +hotsocialz.com +hoverboard360.at +huimin764128.com +hunterboots.online +hyiphunter.org +ibb.com.ua +i-hobot.ru +imagerydatabase.com +immobiliaremassaro.com +impresagaia.it +incep.imagine-ex.co +insomniagamingfestival.com +intellekt21.ru +interesnie-faktu.ru +islandminingsupply.wordpress.com +istizanidineanopiate.blogspot.com +itsdp3.com +iusstf.org +ivearchenceinflu.cf +jagg.info +janerikholst.se +j-times.ru +justprofit.xyz +jwss.cc +kaac.ru +kadashihotel.com +kalb.ru +kartiniresto.com +keyword-suggestions.com +kino-ecran.ru +kol-energo.ru kolotiloff.ru -kostenloser-sex.com -koversite.info -kraljeva-sutjeska.com -krasivoe-hd.com -krasnodar.zrus.org -kreativperlen.ch -kredit-pod-zalog-krasnodar.ru ks1234.com +kupiproday.com.ua +kvartir-remont.biz +labplus.ru +lakomka.com.ua +landinez.co +landscaping.center +languagecode.com laparfumotec.com -laptoper.net -lasvegaslockandsafe.com -lecbter-relationships.ga -le-clos-des-alouettes.com -ledis.top -lennatin.info -letolove.ru -lexaprogeneric.link -linkrr.com -lion.conboy.us -lookingglassemb.com -louboutinbooties.xyz -lsp-awak-perikanan.com -lunamedia.co -macdamaged.tech -macotool.com -magicalfind-a.akamaihd.net -maglid.ru -manualterap.roleforum.ru -marketingtechniques.info -marketland.ml -marktforschung-stuttgart.com -mearns-tractors.co.uk -medmajor.ru -meduza-consult.ru -meget.co.za -me-ke.com -mesto-x.com -metarip.ru -microsoftportal.net -mir-business-24.ru -mirzonru.net +legionalpha.com +lenvred.org +lesbian.xyz +letmacworkfaster.site +lezbiyanki.net +linkarena.com +lion.snapmap.us +lockerz.com +louboutinreplica.pw +louisvuittonoutletstore.net +lowephotos.info +m3gadownload.pl +macrotek.ru +madot.onlinewebshop.net +matrixalchemy.com +mazda-roadsters.com +mb140.ru +mebelcomplekt.ru +megakino.net +mega-polis.biz.ua +methodsmarketing.com +mjchamonix.org mnogolok.info -mobilemedia.md -mobile.ok.ru -mrcsa.com.au -mug-na-chas-moscow.ru -muizre.ru -music7s.me -musicspire.online -muzaporn.com -muz-baza.net -muz-tracker.net -mydearest.co -mydownlodablefiles.com -nastydollars.com -national-today-winning-winner.club -naturalpharm.com.ua -netcheckcdn.xyz -nikhilbahl.com -normalegal.ru -notebook-pro.ru -nzfilecloud.weebly.com -omoikiri-japan.ru -onefilms.net -onemantrip.com -ons-add.men -o-o-8-o-o.ru -openstat.com -oren-cats.ru -otbelivanie-zubov.com -ownshop.cf +monclerboots.xyz +monclercheap.xyz +monclercoats.xyz +moneyviking-a.akamaihd.net +more-letom.ru +moxo.com +mp3downloadhq.com +mrpornogratis.xxx +msk-diplomat.com +mttwtrack.com +musezone.su +myfreecams.com +mypets.by +narutonaruto.ru +naturalshair.site +nbsproject.ru +neks.info +news-readers.ru +newstaffadsshop.club +nibbler.silktide.com +nicefloor.co.uk +noscrapleftbehind.co +novgorod.xrus.org +novosibirsk.xrus.org +nowtorrents.com +offer.camp +ok.ru +omsk.xrus.org +onlinefilmz.net +online-hd.pl +onlineslotmaschine.com +onlinewritingjobs17.blogspot.ru +ooomeru.ru +openfrost.net +oralsexfilme.net +ordernorxx.com paintingplanet.ru -palocco.it -partner-cdn.men -partnerline.men -partner-print.men -pharmacyincity.com -photo-clip.ru -pic2fly.com -pio.polytopesexempt.com -pix-hd.com -plaff-go.ru -plastjulgranar.se -platesauto.com -pokerniydom.ru +panouri-solare-acoperis.com +partnersafe.men +partners-ship.pro +pestomou.info +petrovka-online.com +peugeot-club.org +php-market.ru +pictures-and-images.com +pigrafix.at +pijoto.net +pinsdaddy.com +pinstake.com +pixshark.com +pizza-tycoon.com +play-movie.pl +ploenjitmedia.azurewebsites.net +pogosh.com +pony-business.com +pooleroadmedicalcentre.co.uk popup-jdh.xyz -porn555.com -pornoplen.com -pornowarp.info -pornozhara.com +popup.matchmaker.com +porndairy.in +porndl.org +pornoelita.info +pornoklad.net +pornoslive.net +pornosmola.info +pornzone.tv +powitania.pl pp-budpostach.com.ua +pravoholding.ru +pregnant.guru +printie.com printingpeach.com -privatbank46.ru -prom23.ru -promoforum.ru -purplestats.com -push-ad.com -pussysaga.com -pxhdwsm.com -quick-offer.com -raavidesigns.com -rainbowice.ru +prize44.com +proekt-gaz.ru +promalp-universal.ru +prostitutki-astana.org +qld10000.net +queerspace.com +raisedseo.com rankexperience.com -razorweb-a.akamaihd.net -refads.pro -referencemoi.com -refererx.com -registrationdomainsite.com -relayblog.com -rentalcarnavi.info -research.ifmo.ru -reversing.cc -rn-to-bsn.com -robertefuller.com -rogervivierforsale.com -royalvegascasino.com -rumamba.com -rusoft-zone.ru -russkie-gorki.ru -saletool.ru -samo-soznanie.ru -sampleletters.net -sbt-aqua.ru -sbtdesign.co.uk -scanmarine.info -scanner-elena.top -scenarii-1-sentyabrya.uroki.org.ua -screentoolkit.com -search.alot.com -secret.xn--oogle-wmc.com -selectads.men -semaltmedia.com -sextracker.de -sexvporno.ru -sexyali.com -shell-pmr.ru -shmetall.com.ua -shoppytoolmac.com -sicfor.bcu.cc -simpoed.ufop.br -sim-service.net -site-auditor.online -siteripz.net -site.ru -smartpet.ru +rankings-analytics.com +rapevideosmovies.com +realtytimes.com +recipedays.ru +remont-fridge-tv.ru +remont-mobile-phones.ru +remybutler.fr +resultshub-a.akamaihd.net +rewardpoll.com +rfserial.net +ringporno.com +robot-forex.biz +rock-cafe.info +rockma.se +russkoe-zdorovie.ru +saecsa.co +sakhboard.ru +sale-japan.com +sandhillsonline.com +savememoney.co.za +sbricur.com +sdelatmebel.ru +seccioncontrabajo.com +seksotur.ru +sel-hoz.com +seorussian.ru +seo-smm.kz +sexblog.pw +sex-pr.net +sexuria.net +sfd-chess.ru +sharebutton.net +shohanb.com +shoppingjequiti.com.br +shopvilleroyboch.com.ua +sientalyric.co +silvermature.net +simple-share-buttons.com +simply.net +sk.golden-praga.ru +sklad-24.ru +sledstvie-veli.net +slim.sellany.ru +smartbalanceworld.com +soc-econom-problems.ru +socialseet.ru sodexo.com -solmarket.by -sonnikforme.ru -sotkal.lark.ru -space4updating.win -spammen.de -spb-plitka.ru -spywareit.com -srecorder.com -stariy-baku.com -statustroll.com -stop-gepatit.te.ua -strag-invest.ru -stroiminsk.com -sundrugstore.com -surgut.zrus.org -surintech.ac.th -svensk-poesi.com -svetlotorg.ru -sygraem.com -tanieaukcje.com.pl -taximytishi.ru -tech4master.com -tedy.su -telefonsexkostenlos.tk -telesvoboda.ru -templates.radiodigital.co +softxaker.ru +sovinsteel.ru +spabali.org +speedup-my.site +sptslmtrafms.com +squidoo.com +staynplay.net +stevemonsen.com +storehouse.ua +strangeduckfilms.com +streetfooduncovered.com +stroy-portal22.ru +strv.se +suggest-keywords.com +svarbit.com +symbaloo.com +szamponrevita.pl +t3chtonic.com +tattooreligion.ru +td-33.ru +tdsing.ru +telegramdownload10.com tengohydar.tk -thebestweightlosspills.ovh -theplacetoupdating.pw -thewomenlife.com -timer4web.com -timetorelax.biz -tizanidineduringpregnancy.blogspot.com -tizanidinehcl2mg.blogspot.com -tizanidinehcl4mgisitanarcotic.blogspot.com -tizanidineshowupondrugtest.blogspot.com -tizanidinevssoma.blogspot.com +tgtclick.com +thelottosecrets.com +thiegs.reco.ws +timeallnews.ru +tizanidine4mgstreetprice.blogspot.com +tizanidine4mgtablets.blogspot.com +tizanidineforopiatewithdrawal.blogspot.com tizanidinevsvalium.blogspot.com -tkanorganizma.ru -tn811.us -tnaionline.org -todohr.com -topdownloads.ru -topkarkas.com -topsy.com -track112.site -trustaffs.com -tt-ipd.info -ttrraacckkrr.com -tubeoffline.com -udayavani.com -ufa.xrus.org -uhdtv.website -ukkelberg.no -ultimatesetnewfreeallsoftupgradesystems.pw -unitygame3d.com -unlimitdocs.net -unrealcommander.org -upstore.me -urdoot.win -urengoy.pro -url2image.com -usacasino.com -userequip.com -v720hd.ru -valmetrundan.se -varikoz24.com -vclicks.net -vduplo.ru -veselokloun.ru -vesnatehno.ru -viagroid.ru -videochaty.ru -videojam.tv -villacoloniale.com -vipsexfinders.com -vitalads.net -vpnhowto.info -vykupavto-krasnodar.ru +tizanidinewithdrawalsymptoms.blogspot.com +tmtrck.com +tomck.com +toolsky.com +top10registrycleaners.com +torontoplumbinggroup.com +touchmods.fr +tourcroatia.co.uk +tourismvictoria.com +track2.shop +traffixer.com +triumf-realty.ru +tula.mdverey.ru +tupper-shop.ru +turbo-suslik.org +tvteleport.ru +typimga.pw +u17795.netangels.ru +umaseh.com +unece.org +unionmarkt.de +ural-buldozer.ru +validccseller.com +vbtracker.net +velpanex.ru +vikistars.com +vilingstore.net +vip-parfumeria.ru +virtuagirl.com +visionwell.com.cn +visitcambridge.org +vozbujdenie.com +vsexkontakte.net wallpapers-best.com -waysbetter.cn -wcb.su -webinstantservice.com -webnode.me -website-audit.com.ua -website-stealer.nufaq.com -wejdz-tu.pl -white-truck.life -wordpress-crew.net -world-mmo.com -ww1943.ru -wwwadultcheck.com +wareseeker.com +warningzscaler.heraeus.com +wedding-salon.net +weightatraining.com +westen-v.life +wheelchairliftsarea.com +wholesalejerseysgaa.com +wineitudes.wordpress.com +winx-play.ru +worldlovers.ru +worst-sites.online +wpthemedetector.co.uk +www.kabbalah-red-bracelets.com wygraj-teraz.com -wyniki-lista.pl -xaijo.com -xclicks.net -xn-----6kcaacnblni5c5bicdpcmficy.xn--p1ai -xn----7sbaaabaei0cc8aj5bj0bncejx.xn--p1ai -xn----btbdvdh4aafrfciljm6k.xn--p1ai -xn--e1afanlbnfckd7c3d.xn--p1ai -xn--hxazdsfy.blogspot.com -xtubeporno.net -xxxrus.org -xxxtube69.com -yellowfootprints.com -yellowstonesafaritours.com -yes-com.com -yginekologa.com -yoshkarola.zrus.org -yottos.com -youbrainboost.asia -yougetsignal.com -yourads.website -youraticles.pl -yourserverisdown.com -za-fun-offer.com -zaidia.xhost.ro -zakazvzloma.com -zaloadi.ru -zeroredirect7.com -zhuravlev.info -znakomstva-moskva77.ru -znakomstva-piter78.ru -zoogdisany.com -zrizvtrnpale.tk -zynax.ua +xmlinde.com +xn------7cdbapdecfd4ak1bn0amjffj7afu3y.xn--p1ai +xn----7sbbagbq7bd5aheftfllo4m.xn--p1ai +xtube.com +yebocasino.co.za +yellocloud.be +youghbould.wordpress.com +youjizz.vc +youresponsive.com +yoursite.com +yourtemplatefinder.com +youthreaders.com +youtube-downloader.savetubevideo.com +zaobao.com.sg +zarepta.com +zkjovpdgxivg.ga +zocaparj.kz +zojirushi-products.ru +zona-aqua.ru +zoolubimets.ru +zoominfo.com +zrelaya.pw +zvetki.ru diff --git a/.dev-tools/test_units/random-referrers-for-whitelist-test.list b/.dev-tools/test_units/random-referrers-for-whitelist-test.list index ea3dbd9d0..3f8f9009c 100644 --- a/.dev-tools/test_units/random-referrers-for-whitelist-test.list +++ b/.dev-tools/test_units/random-referrers-for-whitelist-test.list @@ -1,250 +1,250 @@ -00go.com -0912701309f8ce.com -1flag.co.za -21h2o.com -5i2.net -98oi.ru -aanapa.ru -adloads.com -adrunnr.com -adult3dgames.com -advocatemsk.ru -alekseevec.ru -all-number.com -alt-servis.ru -angigreene.com -anmysite.com +3-letter-domains.net +5ws.dating-app.ru +991mostfm.co.id +aa9bd78f328a6a41279d0fad0a88df1901.com +aaronabel.com +abcguru.xyz +air-edem.ru +alevinefesleri.com +alldubai.biz +allknow.info +allnews24.in +allpdfmags.net +amazon-adsystem.com +ampower.me +am-se.com +animalrank.com apisecretsaucebi-a.akamaihd.net -arendas.net -artefakct.com -asiavirtualsolutions.com -asscenihotosidea.blogspot.co.za -audiofree.ru -ayerbo.xhost.ro -b00kmarks.com -babyfactory.fr -backlink4u.net -bag77.ru -baksman.com -bala.getenjoyment.net -baoxaydung.com.vn -beachpics.com -bestdooz.com -bio-optomarket.ru -blackwitchcraft.ru -blavia.00author.com -boc.kz -bolezniorganov.ru -brakehawk.com -brothers-smaller.ru -buyhoverboard.com -buy-meds24.com -buynorxx.com -cakesplus.com.au -cardul.ru -cartierjusteunclou.xyz -cerev.info -changde.58.com -chinajerseyswholesalecoupons.com -chlooe.com -chocolateslim-original.com -clarithromycin500mg.com -clean-virus-mac.com -clipartnew.com -compliance-irvin.xyz -compliance-nelson.top -compucelunlock.net -counter.bloke.com -covetnica.com -creatives.adbetclickin.pink -cubs-jerseys.us -customcollegeessays.net -dalavia.ru -damasarenai.info -data.vtc.pw -deadlinkchecker.com -design-lands.ru -dimkino.ru -dogclothing.store -donna7753191.ru -dostavka-v-krym.com -draniki.org -droidlook.net -ed-shop01.ru -ee77ee.com -election.interferencer.ru -elektrozigarette2.webs.com +apiwoodensealcom-a.akamaihd.net +arcadepages.com +artpicso.com +avek.ru +azadnegar.com +babieca.com +balla-boo.se +bank.uz +barnfurnituremart.com +basisches-wasser.net +bbsoldes.fr +best-coupon-offer.com +blackhatworth.com +boazpower.com +bonvillan.com +bonzybuddy.com +braip.com.br +brg8.com +brianhenry.co +buy-forum.ru +buytizanidineonline.blogspot.com +cartierlovestore.xyz +casinos4dummies.co.uk +cayado.snn.gr +cbozhe.com +cfsrating.sonicwall.com +cheapmoncler.pw +chelyabinsk.xrus.org +choice-credit.ru +christianlouboutinshoes.xyz +civilwartheater.com +club-samodelkin.ru +counciltally.com +cranly.net +customsua.com.ua +daydream-studio.ru +degerlund.net +derevesendeco.com +disruptingdinnerparties.com +diusyawiga.tk +djekxa.ru +djihispano.com +docsportal.net +doska-vsem.ru +downloads-whatsapp.com +download-wallpaper.net +dyt.net +educontest.net +elektrischezigarette2.onsugar.com +elektrozigarettekaufen2.blogse.nl +envaseslotusama.com esfchat.tk -euronis-free.com -evening-dating-club.info -extads.net -extractorandburner.com -ezigarettenshop1.yolasite.com -fearcrow.com -filkhbr.com -finansov.info -fireads.men -folowsite.com -foto-telok.net -fototravel.eu -free-floating-buttons.com -freenode.info -free-today.com -freshmac.space -gamebackyard.com -gavuer.ru -gaytube.com -get-your-social-buttons.info -globalsurfari.com +euroskat.ru +faracontrol.ir +fba-mexico.com +ferrotodo.com +fettefrauen.net +fikasound.tk +findthe.pet +forum69.info +fotzen-ficken.com +free-gluten.ru +free-traffic.xyz +gamingspark.com +gerhardhealer.com +gfaq.ru +gheus.altervista.org +gidonline.one +glcomputers.ru gojiberriess.apishops.ru -gox.com.ua -gpirate.com -gta-club.ru -hamptonoaks.ca -hoverboard360.nl -ifmo.ru -ilbe.club -ilmen.net -intervsem.ru -investyb.com -investzalog.ru -istripper.com -iusstf.org -iwantmyfreecash.com -jamiembrown.com -jmat.cn -joessmogtestonly.com -kakadu-interior.com.ua -kambasoft.com -kerei.ru -kollesa.ru -komputers-best.ru -kostenlos-sexvideos.com -kupit-adenu.ru -kvartiry-remont.ucoz.ru -legionalpha.com -lerporn.info -liupis.com -lomza.info +golden-catalog.pro +goroda-vsego-mira.ru +gorodservis.ru +growboxbl.ru +handsandlegs.ru +hatdc.org +helicalpile.us +herokuapp.com +himgaws.pw +hotdl.in +hothot.ru +hunthillfarmtrust.org +husky-shop.cz +idealtits.net +imadedinner.net +infobabki.ru +itbc.kiev.ua +janavibekken.no +jewelryandfiligree.com +jumptap.com +kalb.ru +katjimej.blog.fc2.com +khafre.us +kolotiloff.ru +labelwater.se +laserpen.club +laxdrills.com +life.biz.ua +littlesunraiser.com lotto6888.com -m3gadownload.pl -m5home.ru -manyresultshub-a.akamaihd.net -marktforschung-stuttgart.com -marmitaco.cat -matrixalchemy.com -mcnamaratech.com -meble-bogart.info -medmajor.ru -megahdporno.net -menhealed.net -methodsmarketing.com -mmgq.ru -mobot.site -money-every-day.com -mrsdalloways.com -muycerdas.xxx -myhealthcare.com -najaden.no -nationalbreakdown.com -nextlnk12.com -nfljerseyscheapbiz.us -nhl17coins.exblog.jp -nonews.co -offtopic.biz -oil-td.ru -onclkads.com -onlinemeetingnow.com -o-o-6-o-o.com -papasdelivery.ru -picphotos.net -pod-muzyku.club -pokemongooo.ml -pontiacsolstice.info -pornoxxx.com.mx -princevc.com -privhosting.com -profitwithalex.info -proftests.net -progress-upakovka.ru -prospekt-st.ru -pst2017.onlinewebshop.net -pulseonclick.com -rada.ru -razleton.com -redhotfreebies.co.uk -reklamuss.ru -roll123.com -russkoe-zdorovie.ru -salmonfishingsacramentoriver.com -salut-camp.ru -sanyuprojects.com -sbetodiodnye-lampy.ru -scalerite.co.za -scanner-jessica.top -scanner-julianna.top -scopich.com -sdrescher.net -searchinquire.com -seasonvar.ru -seeresultshub-a.akamaihd.net -seokicks.de -seonetwizard.com -seryeznie-znakomstva.ru -sibdevice.ru -skanninge.se -sladkoevideo.com -slonechka.ru -snsdeainavi.info -soaksoak.ru -social-s-iii.xyz -socialtrade.biz +lyngdalhudterapi.no +magicplayer-s.acestream.net +makis.nu +mamasuper.prom.ua +mearns-tractors.co.uk +mex-annushka.ru +minneapoliscopiers.com +mnogolok.info +mp3films.ru +mp3ringtone.info +multgo.ru +musicstock.me +mxgetcode.com +mydoctorok.ru +myhitmp3.club +nardulan.com +narkologiya-belgorod.ru +naturtreenspicerx.pw +nbsproject.ru +net-radar.com +newhairstylesformen2014.com +nootrino.com +nrjmobile.fr +nrv.co.za +offers.bycontext.com +onlainbesplatno.ru +ororodnik.goodbb.ru +paceform.com +patol01.pw +phobia.us +phormchina.com +piluli.info +pipki.r.acdnpro.com +pixshark.com +pornofiljmi.com +powenlite24.ru +pr0fit-b0x.com +proekt-gaz.ru +prostitutki-belgoroda.org +psvita.ru +raavidesigns.com +rasteniya-vs-zombi.ru +razamicroelectronics.com +registratciya-v-moskve.ru +rn-to-bsn.com +ros-ctm.ru +rospromtest.ru +royal-investments.net +rubbed.us +runofilms.ru +runstocks.com +russian-postindex.ru +rustic-quiver.win +s8-nowy-wygraj.comli.com +sadaholding.com +sady-urala.ru +sbtdesign.co.uk +scanner-jack.top +scottbywater.com +seocheckupx.com +seo-tools-optimizing.com +sex-pr.net +sexyali.com +shopfishing.com.ua +sk.golden-praga.ru +slkrm.ru +smallseotools.com +soblaznu.net +solicita.info +solitaire-game.ru soochi.co -spaceshipad.com -spb.ru -spitfiremusic.com -spy-app.info -start.myplaycity.com -stats-collector.org -studiofaca.com -superlist.biz -telegraf.by -thebluenoodle.com -thecounter.com -themeforest.net -tizanidinezanaflex.blogspot.com -totu.info -tozup.com -trafficreceiver.club -trichizobswiv.agddns.net -ttrraacckkrr.com -twincitiescarservice.com -uptime-as.net -urdoot.win +sophang8.com +sortthemesitesby.com +spamnuker.com +spywarenuker.com +steebook.com +stroydetali.ru +takethatad.com +tarad.com +tattooreligion.ru +teesdaleflyballclub.co.uk +teguh.info +telefonsex-ohne0900.net +theautoprofit.ml +thegolfclub.info +thewomenlife.com +timetorelax.biz +tizanidine4mgstreetprice.blogspot.com +tizanidinedruginteractions.blogspot.com +tizanidinenarcotic.blogspot.com +tn811.us +top250movies.ru +topvidos.ru +torrents-tracker.com +touchmods.fr +tracksurf.daooda.com +trafficmonetize.org +trkdf.com +tupper-shop.ru +twittruth.com +ucsol.ru +uginekologa.com +ultimatesetnewfreeallsoftupgradesystems.pw urlopener.blogspot.com.au -uvozdeckych.info -virtuagirl.com -viteonlusarezzo.it -vksaver-all.ru -vsesubwaysurfers.com -warningzscaler.heraeus.com -watchdogs-2.ru -webstatsdomain.org -wesharepics.site -westum.se -whatzmyip.net -whengirlsgowild.com -win-spy.com -witherrom55.eklablog.fr -womensplay.net -wowcasinoonline.ooo -wpsecurity.website -xaijo.com -xblog.in -xn----7sbifcamovvfggw9d.xn--p1ai -xn--d1abj0abs9d.in.ua -yoluxuryevents.com -youbrainboost.asia -youresponsive.com -yourporngay.com -zocaparj.kz +vekzdorov.ru +vellings.info +video-production.com +vitalads.net +vut.com.ru +w3javascript.com +warezkeeper.com +wdfdocando.com +webnode.me +white-truck.life +wildcattube.com +wordpress-crew.net +worldis.me +wurr.voila.net +www.kabbalah-red-bracelets.com +www.timer4web.com +xbaboon.com +xclicks.net +xev.ru +xwatt.ru +yachts-cruise.info +yaderenergy.ru +yellocloud.be +you-shall-not-pass.is74.ru +yurgorod.ru +zmoda.hostreo.com +zoogdinsney.com +zooggames.com +zrelaya.pw diff --git a/.dev-tools/test_units/ratelimittest-beta.txt b/.dev-tools/test_units/ratelimittest-beta.txt index 1ddae6201..afb1627e8 100644 --- a/.dev-tools/test_units/ratelimittest-beta.txt +++ b/.dev-tools/test_units/ratelimittest-beta.txt @@ -1,21 +1,11 @@ - - -
-This is merely used for testing !!!
- - +