diff --git a/.dev-tools/beta_conf_files/conf.d/globalblacklist.conf b/.dev-tools/beta_conf_files/conf.d/globalblacklist.conf index 70aae59df..20f14dd03 100644 --- a/.dev-tools/beta_conf_files/conf.d/globalblacklist.conf +++ b/.dev-tools/beta_conf_files/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2019.07.1750 -### Updated: Fri Jul 5 13:11:39 SAST 2019 +### Version: V3.2019.07.1751 +### Updated: Fri Jul 5 14:12:46 SAST 2019 ### Bad Referrer Count: 6719 ### Bad Bot Count: 556 ################################################### diff --git a/.dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf b/.dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf index 5e7c0ed8e..b9d24324a 100644 --- a/.dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf +++ b/.dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -35.184.226.236 1; +34.68.144.114 1; 127.0.0.1 1; diff --git a/.dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf b/.dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf index 3e60bde86..ea117da9f 100644 --- a/.dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf +++ b/.dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -35.184.226.236 0; +34.68.144.114 0; 127.0.0.1 0; diff --git a/.dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf b/.dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf index 70aae59df..20f14dd03 100644 --- a/.dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf +++ b/.dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2019.07.1750 -### Updated: Fri Jul 5 13:11:39 SAST 2019 +### Version: V3.2019.07.1751 +### Updated: Fri Jul 5 14:12:46 SAST 2019 ### Bad Referrer Count: 6719 ### Bad Bot Count: 556 ################################################### diff --git a/.dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf b/.dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf index 5e7c0ed8e..b9d24324a 100644 --- a/.dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf +++ b/.dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -35.184.226.236 1; +34.68.144.114 1; 127.0.0.1 1; diff --git a/.dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf b/.dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf index 3e60bde86..ea117da9f 100644 --- a/.dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf +++ b/.dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -35.184.226.236 0; +34.68.144.114 0; 127.0.0.1 0; diff --git a/.dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf b/.dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf index 70aae59df..20f14dd03 100644 --- a/.dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf +++ b/.dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2019.07.1750 -### Updated: Fri Jul 5 13:11:39 SAST 2019 +### Version: V3.2019.07.1751 +### Updated: Fri Jul 5 14:12:46 SAST 2019 ### Bad Referrer Count: 6719 ### Bad Bot Count: 556 ################################################### diff --git a/.dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf b/.dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf index 5e7c0ed8e..b9d24324a 100644 --- a/.dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf +++ b/.dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -35.184.226.236 1; +34.68.144.114 1; 127.0.0.1 1; diff --git a/.dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf b/.dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf index 3e60bde86..ea117da9f 100644 --- a/.dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf +++ b/.dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -35.184.226.236 0; +34.68.144.114 0; 127.0.0.1 0; diff --git a/.dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf b/.dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf index 70aae59df..20f14dd03 100644 --- a/.dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf +++ b/.dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2019.07.1750 -### Updated: Fri Jul 5 13:11:39 SAST 2019 +### Version: V3.2019.07.1751 +### Updated: Fri Jul 5 14:12:46 SAST 2019 ### Bad Referrer Count: 6719 ### Bad Bot Count: 556 ################################################### diff --git a/.dev-tools/globalblacklist-testing.template b/.dev-tools/globalblacklist-testing.template index 70aae59df..20f14dd03 100644 --- a/.dev-tools/globalblacklist-testing.template +++ b/.dev-tools/globalblacklist-testing.template @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2019.07.1750 -### Updated: Fri Jul 5 13:11:39 SAST 2019 +### Version: V3.2019.07.1751 +### Updated: Fri Jul 5 14:12:46 SAST 2019 ### Bad Referrer Count: 6719 ### Bad Bot Count: 556 ################################################### diff --git a/.dev-tools/globalblacklist.template b/.dev-tools/globalblacklist.template index 6deaedc5e..20a284438 100755 --- a/.dev-tools/globalblacklist.template +++ b/.dev-tools/globalblacklist.template @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2019.07.1750 -### Updated: Fri Jul 5 13:11:38 SAST 2019 +### Version: V4.2019.07.1751 +### Updated: Fri Jul 5 14:12:45 SAST 2019 ### Bad Referrer Count: 6719 ### Bad Bot Count: 556 ################################################### diff --git a/.dev-tools/test1_conf_files/bots.d/whitelist-ips.conf b/.dev-tools/test1_conf_files/bots.d/whitelist-ips.conf index 896ffe576..2b15f263d 100644 --- a/.dev-tools/test1_conf_files/bots.d/whitelist-ips.conf +++ b/.dev-tools/test1_conf_files/bots.d/whitelist-ips.conf @@ -45,4 +45,4 @@ # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST -104.154.120.187 0; +34.68.144.114 0; diff --git a/.dev-tools/test1_conf_files/conf.d/globalblacklist.conf b/.dev-tools/test1_conf_files/conf.d/globalblacklist.conf index 6deaedc5e..20a284438 100644 --- a/.dev-tools/test1_conf_files/conf.d/globalblacklist.conf +++ b/.dev-tools/test1_conf_files/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2019.07.1750 -### Updated: Fri Jul 5 13:11:38 SAST 2019 +### Version: V4.2019.07.1751 +### Updated: Fri Jul 5 14:12:45 SAST 2019 ### Bad Referrer Count: 6719 ### Bad Bot Count: 556 ################################################### diff --git a/.dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf b/.dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf index 479de1217..6893ce0fc 100644 --- a/.dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf +++ b/.dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -35.192.85.2 1; +35.192.136.167 1; 127.0.0.1 1; diff --git a/.dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf b/.dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf index d9f0a2c04..879967dc9 100644 --- a/.dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf +++ b/.dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -35.192.85.2 0; +35.192.136.167 0; 127.0.0.1 0; diff --git a/.dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf b/.dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf index 6deaedc5e..20a284438 100644 --- a/.dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf +++ b/.dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2019.07.1750 -### Updated: Fri Jul 5 13:11:38 SAST 2019 +### Version: V4.2019.07.1751 +### Updated: Fri Jul 5 14:12:45 SAST 2019 ### Bad Referrer Count: 6719 ### Bad Bot Count: 556 ################################################### diff --git a/.dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf b/.dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf index 479de1217..6893ce0fc 100644 --- a/.dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf +++ b/.dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -35.192.85.2 1; +35.192.136.167 1; 127.0.0.1 1; diff --git a/.dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf b/.dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf index d9f0a2c04..879967dc9 100644 --- a/.dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf +++ b/.dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -35.192.85.2 0; +35.192.136.167 0; 127.0.0.1 0; diff --git a/.dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf b/.dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf index 6deaedc5e..20a284438 100644 --- a/.dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf +++ b/.dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2019.07.1750 -### Updated: Fri Jul 5 13:11:38 SAST 2019 +### Version: V4.2019.07.1751 +### Updated: Fri Jul 5 14:12:45 SAST 2019 ### Bad Referrer Count: 6719 ### Bad Bot Count: 556 ################################################### diff --git a/.dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf b/.dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf index 479de1217..6893ce0fc 100644 --- a/.dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf +++ b/.dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -35.192.85.2 1; +35.192.136.167 1; 127.0.0.1 1; diff --git a/.dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf b/.dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf index d9f0a2c04..879967dc9 100644 --- a/.dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf +++ b/.dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -35.192.85.2 0; +35.192.136.167 0; 127.0.0.1 0; diff --git a/.dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf b/.dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf index 6deaedc5e..20a284438 100644 --- a/.dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf +++ b/.dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2019.07.1750 -### Updated: Fri Jul 5 13:11:38 SAST 2019 +### Version: V4.2019.07.1751 +### Updated: Fri Jul 5 14:12:45 SAST 2019 ### Bad Referrer Count: 6719 ### Bad Bot Count: 556 ################################################### diff --git a/.dev-tools/test2_conf_files/bots.d/whitelist-ips.conf b/.dev-tools/test2_conf_files/bots.d/whitelist-ips.conf index 896ffe576..e521bd11b 100644 --- a/.dev-tools/test2_conf_files/bots.d/whitelist-ips.conf +++ b/.dev-tools/test2_conf_files/bots.d/whitelist-ips.conf @@ -45,4 +45,5 @@ # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST -104.154.120.187 0; +35.192.136.167 0; +35.192.187.174 0; diff --git a/.dev-tools/test2_conf_files/conf.d/globalblacklist.conf b/.dev-tools/test2_conf_files/conf.d/globalblacklist.conf index c695ca095..9fe4d283f 100644 --- a/.dev-tools/test2_conf_files/conf.d/globalblacklist.conf +++ b/.dev-tools/test2_conf_files/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2019.07.1750 -### Updated: Fri Jul 5 13:11:38 SAST 2019 +### Version: V4.2019.07.1751 +### Updated: Fri Jul 5 14:12:45 SAST 2019 ### Bad Referrer Count: 6719 ### Bad Bot Count: 556 ################################################### diff --git a/.dev-tools/test3_conf_files/bots.d/whitelist-ips.conf b/.dev-tools/test3_conf_files/bots.d/whitelist-ips.conf index 496177b18..81d44ffba 100644 --- a/.dev-tools/test3_conf_files/bots.d/whitelist-ips.conf +++ b/.dev-tools/test3_conf_files/bots.d/whitelist-ips.conf @@ -45,5 +45,5 @@ # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST -35.192.85.2 0; -35.184.226.236 0; +35.188.73.34 0; +35.193.14.140 0; diff --git a/.dev-tools/test3_conf_files/conf.d/globalblacklist.conf b/.dev-tools/test3_conf_files/conf.d/globalblacklist.conf index f234c425a..fe094fba6 100644 --- a/.dev-tools/test3_conf_files/conf.d/globalblacklist.conf +++ b/.dev-tools/test3_conf_files/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2019.07.1750 -### Updated: Fri Jul 5 13:11:38 SAST 2019 +### Version: V4.2019.07.1751 +### Updated: Fri Jul 5 14:12:45 SAST 2019 ### Bad Referrer Count: 6719 ### Bad Bot Count: 556 ################################################### diff --git a/.dev-tools/test4_conf_files/bots.d/whitelist-ips.conf b/.dev-tools/test4_conf_files/bots.d/whitelist-ips.conf index e69630404..f495383e2 100644 --- a/.dev-tools/test4_conf_files/bots.d/whitelist-ips.conf +++ b/.dev-tools/test4_conf_files/bots.d/whitelist-ips.conf @@ -45,6 +45,6 @@ # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST -35.184.96.71 0; -35.202.145.110 0; -35.193.7.13 0; +34.68.144.114 0; +34.66.178.120 0; +35.192.187.174 0; diff --git a/.dev-tools/test4_conf_files/conf.d/globalblacklist.conf b/.dev-tools/test4_conf_files/conf.d/globalblacklist.conf index 6deaedc5e..20a284438 100644 --- a/.dev-tools/test4_conf_files/conf.d/globalblacklist.conf +++ b/.dev-tools/test4_conf_files/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2019.07.1750 -### Updated: Fri Jul 5 13:11:38 SAST 2019 +### Version: V4.2019.07.1751 +### Updated: Fri Jul 5 14:12:45 SAST 2019 ### Bad Referrer Count: 6719 ### Bad Bot Count: 556 ################################################### diff --git a/.dev-tools/test5_conf_files/bots.d/whitelist-ips.conf b/.dev-tools/test5_conf_files/bots.d/whitelist-ips.conf index bfc641eb0..f495383e2 100644 --- a/.dev-tools/test5_conf_files/bots.d/whitelist-ips.conf +++ b/.dev-tools/test5_conf_files/bots.d/whitelist-ips.conf @@ -45,6 +45,6 @@ # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST -104.154.120.187 0; -35.184.226.236 0; -35.193.7.13 0; +34.68.144.114 0; +34.66.178.120 0; +35.192.187.174 0; diff --git a/.dev-tools/test5_conf_files/conf.d/globalblacklist.conf b/.dev-tools/test5_conf_files/conf.d/globalblacklist.conf index 6deaedc5e..20a284438 100644 --- a/.dev-tools/test5_conf_files/conf.d/globalblacklist.conf +++ b/.dev-tools/test5_conf_files/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2019.07.1750 -### Updated: Fri Jul 5 13:11:38 SAST 2019 +### Version: V4.2019.07.1751 +### Updated: Fri Jul 5 14:12:45 SAST 2019 ### Bad Referrer Count: 6719 ### Bad Bot Count: 556 ################################################### diff --git a/.dev-tools/test_units/blacklist-ips.conf b/.dev-tools/test_units/blacklist-ips.conf index 5e7c0ed8e..b9d24324a 100644 --- a/.dev-tools/test_units/blacklist-ips.conf +++ b/.dev-tools/test_units/blacklist-ips.conf @@ -1,2 +1,2 @@ -35.184.226.236 1; +34.68.144.114 1; 127.0.0.1 1; diff --git a/.dev-tools/test_units/random-bots-for-test-quick.list b/.dev-tools/test_units/random-bots-for-test-quick.list index 00a35d3cf..b4dd810ec 100644 --- a/.dev-tools/test_units/random-bots-for-test-quick.list +++ b/.dev-tools/test_units/random-bots-for-test-quick.list @@ -1,10 +1,10 @@ -Boardreader +backlink-check Bolt -Cegbfeieh -Discoverybot -Offline Navigator +Curious +EyeNetIE +IstellaBot +Mail.RU_Bot +Rogerbot Scanbot -Seomoz -TurnitinBot -WebStripper -Widow +SiteSucker +SpankBot diff --git a/.dev-tools/test_units/random-bots-for-test.list b/.dev-tools/test_units/random-bots-for-test.list index 24dd81190..ab9b02678 100644 --- a/.dev-tools/test_units/random-bots-for-test.list +++ b/.dev-tools/test_units/random-bots-for-test.list @@ -1,250 +1,250 @@ -ADmantX -AfD-Verbotsverfahren +404enemy +Abonti +Aboundexbot +AhrefsBot +AIBOT Aipbot -Alligator -AlphaBot +AllSubmitter +Anarchie +Apexoo Asterias -Attach -autoemailspider BackDoorBot -backlink-check BacklinkCrawler -BackStreet -Badass +BackWeb Bandit -Barkrowler BatchFTP -Battleztar Bazinga BBBike BDFetch -Blackboard +BetaBot Black Hole +BlackWidow BLEXBot -Boardreader +Blow +BlowFish +Bolt Brandwatch -BuiltBotTough +BuiltWith Bullseye +BunnySlippers BuzzSumo Calculon -CazoodleBot -Cegbfeieh +CCBot CheeseBot CherryPicker Chlooe -Cliqzbot -coccocbot-web +Cloud mapping +cognitiveseo +Copier Copyscape -Cosmos -crawler4j +Craftbot crawler.feedback +Crescent CrunchBot CSHttp -DatabaseDriverMysqli -Demon +DataCha0s +DBLBot +demandbase-bot +Deusu +DigitalPebble DIIbot -DittoSpyder -DnyzBot -DomainCrawler +Dirbuster +Disco +Discoverybot +Dispatch +DomainAppender DomainSigmaCrawler -DomainStatsBot -Drip -EasyDL -Ecxi -EirGrabber -EMail Siphon +Dotbot +Dragonfly +DTS Agent +Ebingbong +eCatch EMail Wolf -EroCrawler evc-batch -Express WebPictures -ExtLinksBot -Extractor -Extreme Picture Finder +Evil FDM FemtosearchBot FHscan -Flunky -Foobot -GalaxyBot -Genieo -GermCrawler +Fimap +Firefox/7.0 +FlashGet +Fyrebot +Getintent GetRight -GetWeb -Gotit +Gigablast +Go-Ahead-Got-It GoZilla -Grafula -HaosouSpider +Grabber +GrabNet +GrapeFX +GrapeshotCrawler Harvest -HEADMasterSEO +Havij +Heritrix +Hloader HMView -HTMLparser -HTTrack +HybridBot +Iblog IlseBot -Image Fetch -IndeedBot -InfoNaviRobot +Image Sucker +Intelliseek InterGET InternetSeer ips-agent -Iria Iskanie -JamesBOT +IstellaBot JennyBot -Jetty +JikeSpider +Jorgee Jyxobot Kenjin Spider -Keyword Density -Kozmosbot +Lanshanbot Larbin -LexiBot -Lftp -Linkdexbot +LeechFTP +LinkextractorPro LinkScan +LinksManager LinkWalker -Lipperhey +LinqiaMetadataDownloaderBot +LinqiaScrapeBot Lipperhey Spider -lwp-request +Litemage_walker +lwp-trivial +Magnet +magpie-crawler Mail.RU_Bot -Majestic12 -Majestic SEO -MarkMonitor +MarkWatch Masscan +Mass Downloader +Mata Hari +meanpathbot +MeanPath Bot MegaIndex.ru -Metauri -MFC_Tear_Sample -Microsoft URL Control -MIIxpc MJ12bot -Mojeek Morfeus Fucking Scanner -Mr.4x3 +MSIECrawler MS Web Services Client Protocol Musobot -Name Intelligence -Nameprotect -Navroad NearSite -Needle Nessus NetAnts -Netcraft -NetLyzer -NetSpider Nettrack +Net Vampire +Netvibes +NextGenSearchBot Nibbler -Niki-bot +NICErsPRO Nikto -NimbleCrawler NPbot -Octopus +oBot Offline Explorer Openfind +OpenLinkProfiler OrangeBot OutfoxBot -Page Analyzer +PageAnalyzer page scorer -PageScorer Panscient -Pavuk +pcBrowser PHPCrawl +Picscout Picsearch PictureFinder Pi-Monster -plumanalytics +Pixray Pockey +POE-Component-Client-HTTP Probethenet -Pump -PyCurl -RankFlex -RankingBot2 -RankurBot -RealDownload +PxBroker +QueryN Metasearch +RankingBot RebelMouse -RedesScrapy -ReGet Ripper -Rogerbot +s1z.ru +SalesIntelligent ScanAlert -Scanbot -Searchestate -Semrush -SEOprofiler -SeoSiteCheckup -SEOstats -serpstatbot +ScoutJet +Screaming +ScreenerBot +SemrushBot +SEOkicks +SEOlyticsCrawler +Seomoz +seoscanners Shodan -SISTRIX +Siphon SiteExplorer -Siteimprove +SiteSnagger +SiteSucker Sitevigil -SlySearch +SmartDownload SMTBot -Snake Snoopy SocialRankIOBot -Sociscraper -sogouspider -Sogou web spider -Sosospider Sottopop Spammen SpankBot Spanner -SputnikBot +Spinn3r +spyfu Sqlmap Sqworm Stripper SuperBot -SuperHTTP -SurveyBot -Swiftbot +Suzuran +sysscan Szukacz T8Abot +tAkeOut TeleportPro -TheNomad -Titan -Toata -Turnitin +Telesphorep +The Intraformant +Thumbor +Trendictionbot +trendiction.com +True_Robot +TurnitinBot +TwengaBot Twice -Typhoeus Upflow -URLy Warning URLy.Warning -Vacuum -VidibleScraper -Voil -Webalta -WebAuto +Vagabondo +VeriCiteCrawler +VoidEYE +Wallpapers/3.0 +WBSearchBot Web Auto +WebCollage +Web Collage WebCopier Web Enhancer -Web Fetch +WebFetch WebGo IS -WebLeacher -WebmasterWorldForumBot +WebImageCollector webmeup-crawler Web Pix -WebSauger +WebReaper Web Sauger -WebsiteExtractor +Webshag Website Quester +Webster WebStripper +WebZIP WeSEE -Whacker +Whack Whatweb -WinHTTrack +Who.is Bot WiseGuys Robot -WISENutbot +Woobot Wotbox +Wprecon WWW-Mechanize -WWW::Mechanize -x09Mozilla x22Mozilla Xaldon WebSpider -Xenu -YoudaoBot +Xaldon_WebSpider Zade -zauba.io Zermelo Zeus +zgrab Zitebot -ZyBorg +ZmEu +ZumBot diff --git a/.dev-tools/test_units/random-bots-for-whitelist-test.list b/.dev-tools/test_units/random-bots-for-whitelist-test.list index e3da7c5a1..4bfd97c75 100644 --- a/.dev-tools/test_units/random-bots-for-whitelist-test.list +++ b/.dev-tools/test_units/random-bots-for-whitelist-test.list @@ -1,250 +1,250 @@ -404checker -404enemy +360Spider 80legs -Abonti +Aboundex Acunetix -ADmantX -AiHitBot +AhrefsBot +AIBOT Aipbot -Alligator -Anarchie +Alexibot +AllSubmitter +AlphaBot +Apexoo archive.org_bot +Asterias Attach -backlink-check -BackStreet +autoemailspider +BackDoorBot BackWeb -Badass Barkrowler -BDCbot -BetaBot -Bitacle -Black Hole -BlowFish -Boardreader +BBBike +BDFetch +Blackboard +BotALot Brandprotect -Brandwatch BuiltBotTough BuiltWith +Bullseye +BunnySlippers +BuzzSumo +Calculon CATExplorador +CCBot Cegbfeieh -CheeseBot ChinaClaw -Claritybot +Chlooe Cliqzbot -Cogentbot -Copier +CopyRightCheck Cosmos +Craftbot CrazyWebCrawler +Crescent CSHttp +DatabaseDriverMysqli Demon -Deusu +Devil +Digincore +DigitalPebble DIIbot +Dirbuster Discobot -Discoverybot -Dispatch +DomainAppender DomainCrawler DomainSigmaCrawler -Dotbot -Download Wonder -DSearch +Dragonfly +Drip DTS Agent -EasyDL +ECCP/1.0 +Ecxi +EirGrabber evc-batch Evil -ExtLinksBot +Exabot +Express WebPictures Extractor -ExtractorPro -Extreme Picture Finder -EyeNetIE -Ezooms -FDM -FHscan +FemtosearchBot Fimap +Firefox/7.0 FlashGet -Flunky -Foobot Freeuploader -Fyrebot -Genieo +FyberSpider GetRight GetWeb Gigabot -Go-Ahead-Got-It -Gotit +G-i-g-a-b-o-t +Go!Zilla GrabNet -Grafula -GrapeshotCrawler -Hloader +GridBot HMView -Humanlinks +HTMLparser +HTTP::Lite HybridBot +IDBot +Image Fetch +Image Sucker InfoNaviRobot -InternetSeer -Iria -JamesBOT +InfoTekies +Intelliseek +InterGET +ips-agent +IRLbot +Iskanie +IstellaBot +Jbrofuzz JetCar -Jorgee -JustView -Kenjin Spider -Kozmosbot +JikeSpider +Joomla +Jyxobot Lanshanbot Larbin +LeechFTP LexiBot -Lightspeedsystems +Libwhisker Likse -LinkextractorPro +Linkdexbot +LinkpadBot +LinkWalker LinqiaRSSBot LinqiaScrapeBot Lipperhey -Lmspider -Ltx71 +Lipperhey Spider lwp-request +LWP::Simple lwp-trivial -Magnet -magpie-crawler -Majestic SEO -Majestic-SEO -MarkMonitor -Masscan -Mass Downloader +Mag-Net +Mail.RU_Bot +Majestic12 MauiBot +Meanpathbot MeanPath Bot Mediatoolkitbot +mediawords +MegaIndex.ru +Metauri +Microsoft Data Access +MIIxpc Mister PiX +MJ12bot Morfeus Fucking Scanner Mr.4x3 MSFrontPage -MSIECrawler -Msrabot -MS Web Services Client Protocol -muhstik-scan Musobot -Navroad +Name Intelligence NearSite -NetAnts +Needle +Netcraft netEstate NE Crawler -NetLyzer -NetMechanic -NetSpider -Nettrack -Netvibes +Net Vampire NetZIP -Niki-bot -NimbleCrawler +NextGenSearchBot Nimbostratus -Nmap -NPbot -Offline Explorer -Offline Navigator -OnCrawl -Openfind -OutclicksBot +Nutch +oBot +Octopus +OpenVAS +OrangeBot OutfoxBot PageAnalyzer -Page Analyzer -PageGrabber -page scorer -PageScorer -Papa Foto +Pandalytics Pavuk -PeoplePal -PHPCrawl -PictureFinder -Pimonster -Pockey +pcBrowser +Picscout +Pixray +PleaseCrawl +plumanalytics Probethenet -ProPowerBot -Psbot -Pump -PxBroker +QueryN Metasearch Quick-Crawler -RankActiveLinkBot RankingBot2 RankurBot -RealDownload +Reaper RebelMouse ReGet -Rogerbot -SalesIntelligent -Scanbot +RepoMonkey +scan.lol ScoutJet -SemrushBot -SEOkicks-Robot +ScreenerBot +Searchestate +SearchmetricsBot +Semrush +SEOkicks SEOlyticsCrawler -Seomoz -SEOstats +seoscanners +sexsearcher SISTRIX -SiteLockSpider -SiteSnagger -Site Sucker +SiteExplorer +Siteimprove +SiteSucker SMTBot Snake -SocialRankIOBot -sogouspider -Sosospider -SpaceBison +Snoopy +Sociscraper +Sottopop Spammen SpankBot +Spbot Sqlworm Sqworm Steeler +Stripper +Sucker Sucuri -SuperBot +SuperHTTP Suzuran Swiftbot sysscan Szukacz T8Abot tAkeOut -Teleport +TeleportPro Telesphoreo Telesphorep +TheNomad Thumbor -TightTwatBot Titan -Toata -Trendiction -Trendictionbot -trendiction.com +Toweyabot +trendiction.de Turnitin +TurnitinBot Twice Typhoeus -Upflow -URLy.Warning Vacuum +Vagabondo +VCI +VidibleScraper Virusdie -WASALive-Bot -WebAuto +Wallpapers/3.0 +WallpapersHD Web Auto -WebBandit -WebCollage Web Collage -WebCopier WEBDAV WebEnhancer Web Enhancer -WebFetch +Web Fetch WebFuck -Web Fuck -WebmasterWorldForumBot +WebPix +WebReaper WebSauger +Web Sauger Webshag WebsiteExtractor +WebsiteQuester Website Quester -Webster -WebStripper -WebZIP -WeSEE +WebSucker +Web Sucker +WebWhacker Whacker Whatweb Who.is Bot -Wonderbot -Woobot +WinHTTrack +WiseGuys Robot +WISENutbot Wotbox Wprecon -x22Mozilla Xaldon WebSpider -Xaldon_WebSpider -xpymep1.exe -YoudaoBot Zade +Zermelo +Zeus +zgrab +ZmEu +ZumBot +ZyBorg diff --git a/.dev-tools/test_units/random-referrers-for-test-quick.list b/.dev-tools/test_units/random-referrers-for-test-quick.list index eab7561b9..f8a9760d4 100644 --- a/.dev-tools/test_units/random-referrers-for-test-quick.list +++ b/.dev-tools/test_units/random-referrers-for-test-quick.list @@ -1,10 +1,10 @@ -balkanfarma.org -bestplacetobuyeriacta.jw.lt -christianlouboutinreplica.pw -ekspertmed.com -metaxalonevstizanidine.blogspot.com -misandesign.se -onlinemegax.com -o-o-8-o-o.com -xchangetrak.com -zaloadi.ru +7minuteworkout.com +dood.live +erectile.bid +fashion-mk.net +naj-filmy24.pl +scrapinghub.com +sledstvie-veli.net +swsociety.se +tizanidine6mg.blogspot.com +twojebook.pl diff --git a/.dev-tools/test_units/random-referrers-for-test.list b/.dev-tools/test_units/random-referrers-for-test.list index 9e5a9c940..403a1bda6 100644 --- a/.dev-tools/test_units/random-referrers-for-test.list +++ b/.dev-tools/test_units/random-referrers-for-test.list @@ -1,500 +1,500 @@ -01apple.com -03e.info -10bet.com -123kuma.com -202ch.com -24videos.tv -2pxg8bcf.top -3-letter-domains.net +1001watch.com.ua +101raccoon.ru +2000k.ru +2ads.co.uk +3weekdiet.com +4inn.ru +4pp13.com 4webmasters.org -51.la -5elementov.ru -5tey7463.weebly.com -5ws.dating-app.ru -888.com -8kisses.eu -8xv8.com -91abcw.com -aac500b7a15b2646968f6bd8c6305869d7.com -aac52006ec82a24e08b665f4db2b5013f7.com -accessoires-mode.in -addtoadd.men -admitad.com -adservme.com -adtech.de -adzpower.com +7makemoneyonline.com +98oi.ru +a1security.com.ua +aasoldes.fr +absolutelyanalog.com +adelly.bg +adloads.net +adnotbad.com +adsloads.com +adultfriendfinder.com agardomains.com -aihelen.net -akuhni.by -alert-jdh.xyz -alfapro.ru -algerianembassy.co.in -amospalla.es -apiadanaknet-a.akamaihd.net -apidigidocketnet-a.akamaihd.net -apiholdingmypage-a.akamaihd.net -appartement-stumm.at -apps-analytics.net -arcteryxsale.online -arewater.com -asrvrep-a.akamaihd.net -autoblogger24.info -autorn.ru -autotop.com.ua -autotrends.today -avtolombard-krasnodar.ru -avto-oligarh.ru -babieca.com -baditri.com -baersaratov.ru -balitouroffice.com -banksrf.ru -barbourjackets.online -barnfurnituremart.com -beatifullandscape.co -beclean-nn.ru -beremenyashka.com -berrymall.ru +alarmobninsk.ru +albamargroup.com +alekseevec.ru +alfa-medosmotr.ru +all4wap.ru +allkrim.com +alphavisions.net +amt-k.ru +an-donut.com +angigreene.com +angry-fermi-7633.arukascloud.io +animal-drawings.com +anjalika.co.in +anticrawler.org +aosheng-tech.com +apccargo.com +apibetweenlinesn-a.akamaihd.net +apiwebwebgetcom-a.akamaihd.net +ap.senai.br +arkkivoltti.net +asia-forum.ru +asiavirtualsolutions.com +atleticpharm.org +ausmepa.org.au +autobudpostach.club +autoplate.info +autoua.top +avek.ru +avramstroy.ru +azazu.ru +balook.com +bannerbank.ru +bannerpower.com +battleforupdating.site +beachfix.co +beauby.ru +beauty-clinic.ru +bedcapdealers.com +berlininsl.com +bestcurs.org +best-games.today bestofpicture.com -best-seo-software.xyz -betterscooter.com -bikini-image.com -binomo.com +bestpriceninja.com +bestsexyblog.com +besttorrentknifta.weebly.com +bestwebsitesawards.com bio-market.kz -biteg.xyz -bkgr.se -blackhatworth.com -blagovest-med.ru -blogsfunky672.weebly.com -bmsco.co -boleznikogi.com -bookmark4you.com -bouda.kvalitne.cz -brandov.ru -bumascloset.com -buypuppies.ca -buytizanidineonlinenoprescription.blogspot.com -bwlx.prepedu.cn -camdolls.com -camel-beach.com -carfax.com.ua -carmuffler.net -carsdined.org -cartierbraceletsreplica.pw -cartierloveringreplica.xyz -cbozhe.com -celebsopics.com -cenokos.ru -cgi2.nintendo.co.jp -chatroulette.online -chatroulette.world -cheapcigarettesc.info -cheapjerseysap.com +blobar.org +blogig.org +bloglag.com +boltalko.xyz +braindaily.xyz +brainsandeggs.blogspot.com +brimstonehillfortress.org +briomotor.co +builtwith.com +buyantiviralwp.com +buybest1.biz +buy-forum.ru +cartechnic.ru +cartierlovestore.com +cartierring.xyz +caulderwoodkennels.com +cheapjerseysa.com +cheapmoncler.pw checkpagerank.net -cialischstgerts.com -clash-clans.ru -classicakuhni.ru -clickcash.com +chelnytruck.ru +chiblackhawks-jerseys.com +chidporn.com +chinajerseyswholesalecoupons.com +chocolateslim-u-srbiji.com +cineacademy.ru +claytransformations.info +clayvasedesigns.tk +cleanpcnow.com +clickhype.com +clicksor.com +clicksotrk.com +cocyq.inwtrade.com colehaanoutlet.store -collegerentals.com -companies-catalog.com -compliance-andrew.xyz -compliance-checker.info -compliance-irvin.xyz -compliance-viktor.top -coolgamechannel.com -cougfan.info -cs-passion.pl -customboxes4less.com -customergrowthsystems.com -dafatiri.com -daisye.top -danceuniverse.ru -dandiyabeats.in -darkages.info -davebestdeals.com -daydream-studio.ru +com-cleaner.systems +cometorussia.net +compliance-alexa.top +compliance-fred.top +compliance-george.top +compliance-willy.top +constantaservice.net +cornomase.win +crackguru.tk +crest-poloski.ru +cscwtalkto.site +custom-electric-guitar.com +custom-product-labels.com +cv.wallhade.co +daneshetabiat.com db.speedup.gdn -defenderxtactical.com -destinationrealestate.com -dev-seo.blog -dieta-personalna.pl -dipstar.org -disruptingdinnerparties.com +dcj-nn.ru +dealitright.click +depositfiles-porn.ga +dev33.dioniqlabb.se +diarioaconcagua.com dnepr-avtospar.com.ua -docstoc.com -dolohen.com -doska-vsem.ru -dostavka-v-krym.com +dodlive.mil +dogbreedspicture.net +domainsatcost.com +doxysexy.com drivotracker.com -drunkmoms.net -dwomlink.info -ekspresihati.info -eladkarako.com +eachdayisagift.review +earncash.com.ua +earthmagic.info +easync.io +ecookna.com.ua +edudocs.net +eduinfosite.com +elektrischeziga.livejournal.com elektronischezigarettekaufen1.beeplog.com -elektrozigarette2.wordpressy.pl -elite-sex-finders.com -enews.tech -enge-fotzen.info +elvenar.com engines-usa.com -epicbrogaming.com -eralph.tk -erectile.bid -erotikstories.ru -essay-edu.biz -etenininrade.ga -evidencecleanergold.com -evogarage.com -exchanges-bet.com -exonline.info -ezigaretteshop2.vefblog.net -f07.de -f0815.de -falcoware.com -familyphysician.ru -favorcosmetics.com -fearcrow.com -feargames.ru -femdom.twiclub.in -fickkontakte.org -fikasound.tk -filmidivx.com +esnm.ru +evrotekhservis.ru +exportshop.us +extads.net +extener.org +ezigarettenshop2.myblog.de +ezigaretteshop2.mywapblog.com +f5mtrack.com +family1st.ca +fastcrawl.com +fatmaelgarny.com +fdzone.org +fefo.gdn +ferieboligkbh.dk +fermersovet.ru +filkhbr.com +filmi-onlain.info financehint.eu -findanysex.com -finteks.ru +finemanteam.com +floating-share-buttons.com +flytourisme.org focalink.com -forex21.ru -formularz-konkurs.tk -fortwosmartcar.pw -fotzen-ficken.com -foxweber.com -frequiry.com -frighteningremain.cf -fr.netlog.com -fuel-gas.com -g7m.pl -gallerylisting.com -gamedayassist.com -gameonasia.com -gaygalls.net -gaz-voshod.ru -geileweiber.tk -geoads.com -gepezz.info -getmiro.com +forsex.info +fortevidyoze.net +fortunejack.com +forums.toucharcade.com +fotopop.club +frameimage.org +franecki.net +free-deals.faith +free-floating-buttons.com +freelifetimefuckbook.com +free-share-buttons.blogspot.com +funcrushgames.com +gabeshop.ru +galaxy-family.ru +gallictures.com +game-top.su +gate5.co.za +geckoandfly.com +get-free-social-traffic.com getpopunder.com -gilbertbanda.net -gne8.com -go2jump.org -golaya.pw -goldadpremium.com +getresponse.com +gidonline.one +gnuetella.com +goatse.ru +golfresa.lucania.se golmau.host.sk -googlefeud.com -gorabagrata.ru +goodhousekeeping.com +goodwinmetals.co goroda-vsego-mira.ru -gorodservis.ru +gov.yanao.ru gox.com.ua grand-chlen.ru -greatdealshop.com -gta-top.ru -guarrasdelporno.xxx -hahashka.ru -hamacapty.com -hausfrauensex18.com -himgaws.pw -hit-kino.ru -hol.es -homeandhealth.ru -homeinns.com -hostingclub.lk -hotkeys.com -howopen.ru -hubbble.com -hundejo.com -hyiphunter.org -ifmo.ru -i-hobot.ru -iloveitaly.ru -imitex-plus.ru -import-sales.com -indiakino.net -infodocsportal.com -interferencer.ru -iqs.biz.ua -irunfar.com -istripper.com -itbc.kiev.ua -ix20.ru -izhevsk.zrus.org -javcoast.com -jerseysbizwholesalecheap.com -jerseyschinabizwholesale.us -jestr.org +g.starmoe.xyz +guardlink.com +guigyverpo.cf +haikuware.com +hd720kino.ru +hd-filmy.net +himazin.info +historichometeam.com +hobbyhuren-datenbank.com +homedecorpicture.us +homedo.fabpage.com +home-task.com +hotblognetwork.com +hotloans.ru +hotsocialz.com +housediz.com +housemilan.ru +hoverboard360.de +hoztorg-opt.ru +ideoworld.org +ilovevitaly.xyz +imagui.eu +imgarcade.com +impisr.ru +incolors.club +infogame.name +inome.com.ua +instabid.tech +intelhdgraphicsgtdrive6w.metroblog.com +int.search.tb.ask.com +iphantom.com +irkutsk.online-podarki.com +isoveti.ru +iwantedmoney.com +jackwolfskinoutlet.online jolic2.com -kaac.ru -kandidos.com -katushka.net -kazka.ru +kamen-e.ru +karadene.com +karusel-market.ru +kchaxton.com ketoanhanoi.info -keywordsuggest.org -kinomaniatv.pl -kliksaya.com -kotaku.com -kovesszucs.atw.hu -kreativperlen.ch -ks1234.com -kung-fu-ru.com -labplus.ru -lakomka.com.ua -languagecode.com -lasvegaslockandsafe.com -leftofcentrist.com -lflash.ru -lifebyleese.com -light.ifmo.ru -linkredirect.biz -liquimondo.com -localmatchbook.com -lonely-mature.com -louboutinreplica.pw -lovelycraftyhome.com -lubetube.com -lunchrock.co -magazin-pics.ru -mainhp.net -make-money-online.com -marketingvici.com -marmotstore.online -master-muznachas.ru -maximpartnerspr.com -mebeldekor.com.ua -medi-fitt.hu -megaapteka.ru -mejoresfotos.eu -member-quiz.com -message-warning.net -mex-annushka.ru -micasainvest.com -migente.com -miniads.ca -mirobuvi.com.ua -mixed-wrestling.ru -mixtapetorrent.com -mobifunapp.weebly.com -mobot.site -mobsfun.net -monclerboots.xyz -moscow-clining.ru -moviezbonkerssk.cf +kino-ecran.ru +kmd-pto.ru +kochanelli.com +konoplisemena.com +konteiner24.com +kosmetyki.tm.pl +kountrylife.com +krasnodar.ru +labelwater.se +laexotic.com +lancheck.net +laparfumotec.com +legionalpha.com +lesbian.xyz +limtu.ifmo.ru +linkarena.com +lion.misba.us +liver-chintai.org +lomza.info +longlifelomilomi.info +low-format.ru +lsex.xyz +ltvperf.com +lunamedia.co +luxup.ru +m0r0zk0-krava.ru +mackeeper-land-672695126.us-east-1.elb.amazonaws.com +macotool.com +macresource.co.uk +maladot.com +marketingtechniques.info +masserect.com +max-p.men +mcar.in.ua +mebelcomplekt.ru +medictube.ru +meds-online24.com +mikozstop.com +mil-stak.com +mitrasound.ru +mjchamonix.org +mobilierland.com +modnie-futbolki.net +mojaocena.com +monetizationking.net +moxo.com mrinsidesales.com -mrvideospornogratis.xxx -mundoaberrante.com -musicspire.online -mvpicton.co.uk -mxgetcode.com -myghillie.info -myprintscreen.com -narkologiya-peterburg.ru -neodownload.webcam -news-speaker.com -newstudio.tv -nlfjjunb5.ru -notaria-desalas.com -o00.in -offtime.ru -o.light.d0t.ru -online.ktc45.ru -onlywoman.org -ooo-olni.ru -osb.se11.ru -oxford-book.com.ua -pacificair.com -parfusale.se -park.above.com -pcgroup.com.uy -perfectpracticeweb.com -picscout.com -pictures-and-images.com -pinkduck.ga -pintattoos.com -plastgran.com -pl.id-forex.com -pliks.pl -ploenjitmedia.azurewebsites.net -plusnetwork.com -pl-vouchers.com -pochemychka.net -polytopesexempt.com -pons-presse.com -poquoson.org -porngalleries.top -pornodojd.ru -pornokajf.com -porno-play.net +mrpornogratis.xxx +multgo.ru +musflashtv.com +muz-shoes.ru +my-bc.ru +my-floor.in.ua +myliveblog.ru +mymercy.info +myonigroup.com +naturalshair.site +nbsproject.ru +nero-us.com +netcheckcdn.xyz +nippon-bearings.ru +njkmznnb.ru +nordstar.pro +nottyu.xyz +nowtorrents.com +nrjmobile.fr +nufaq.com +oda.as +offergroup.info +okayimage.com +omsk.xrus.org +ops.picscout.com +otbelivanie-zubov.com +page2rss.com +parajumpersjakkesalgnorge.info +partners-ship.pro +pastaleads.com +pb-dv.ru +pdns.cz +pearlisland.ru +pelfind.me +petrovka-online.com +pistonclasico.com +pix24x7.com +pixshark.com +play.leadzupc.com +poisk-zakona.ru +pornhive.org +pornhubforum.tk +pornmania.pl +pornoreino.com +porno-transsexuals.ru pornstartits.xblog.in -porodasobak.net -printdirectforless.com -priora-2.com -priscilarodrigues.com.br -privat-girl.net -privetsochi.ru -produkto.net -profit-opportunity.com -prosmibank.ru -prostitutki-belgoroda.org -prpops.com -pts163.ru -pukaporn.com -pussyspace.net -qor360.com -quick-offer.com -quit-smoking.ga -randki-sex.com -rankingchart.de -razleton.com -realizmobi.com -recordpage-a.akamaihd.net -redirectingat.com -regdefense.com -registrationdomainsite.com +portside.cc +pr0fit-b0x.com +pravoholding.ru +preparevideosafesystem4unow.space +prizeestates.cricket +prostitutki-novgoroda.org +prostitutki-tyumeni.org +purplestats.com +pxhdwsm.com +raavidesigns.com +rabot.host.sk +rankings-analytics.com +rebrand.ly +rednise.com +reelheroes.net +registratciya-v-moskve.ru rep-am.com -research.ifmo.ru -responsive-test.net -respublica-otel.ru -restaurantlescampi.com -rn-to-bsn.com -rospromtest.ru -russia-today-video.ru -rustic-quiver.win -santasgift.ml -scanner-irvin.top -scoopquest.com -search-error.com +replicaclub.ru +rialp.getenjoyment.net +riders.ro +rocketchange.ru +rome2rio.com +roofers.org.uk +rus-pornuha.com +rvzr-a.akamaihd.net +s8-nowy-wygraj.comli.com +saletool.ru +salmonfishingsacramentoriver.com +sampleletters.net +sbwealthsolutions.ca +scanner-alex.top +scanner-donald.top +scanner-don.top +scanner-julianna.top +searchmywindow-a.akamaihd.net segol.tv -seoholding.com -seojokes.net +sei80.com seomarketings.online -seo-prof1.xyz +seorankinglinks.com +seorankinglinks.us seo-tools-optimizing.com -serpstat.com -servicecenter.co.ua -sethrollins.net -se-welding.ru -sexblog.pw -sex.hotblog.top -sexkrasivo.net -sharebutton.org -sharebutton.to -shijian.ac.cn -shopvilleroyboch.com.ua -siha.de -simple-share-buttons.com -sindragosa.comxa.com -sitevaluation.com -skuteczna-dieta.co.pl -sledstvie-veli.net -socialbuttons.xyz -social-s-iii.xyz -soft-terminal.ru -space4updating.win -spidtest.space -splendorsearch-a.akamaihd.net -spywarenuker.com -srecorder.com -srgwebmail.nl -stanthonyscatholicchurch.org -star61.de -sticken.co -stickers-market.ru -stpicks.com -strag-invest.ru -streha-metalko.si -suchenindeutschland.com -svensk-poesi.com -svetodiodoff.ru -swagbucks.com -swinger-mobil.net -tdsing.ru -tech4master.com -teesdaleflyballclub.co.uk -thedownloadfreeonlinegames.blogspot.com -theendivechronicles.com -themestotal.com +serialsway.ucoz.ru +serviporno.com +sexcamamateurchat.com +sexgalleries.top +sexreliz.com +sexspornotub.com +sexuria.net +sex-watch.com +shama-rc.net +ship-marvel.co.ua +shlyahten.ru +shmetall.com.ua +shopfishing.com.ua +sicfor.bcu.cc +sisi-go.ru +sitiz.club +sivs.ru +sklad-24.ru +skytraf.xyz +smart-balancewheel.com +smartshoppymac.com +snegozaderzhatel.ru +snsdeainavi.info +social-fun.ru +socialsignals24.com +sockshare.net +sosdepotdebilan.com +space2019.top +spinnerco.ca +sprttrack.com +stairliftstrue.com +starwars.wikia.com +staticfs.host +steamoff.net +stevemonsen.com +stmassage.ru +stroilka.info +sugarkun.com +sundrugstore.com +supermama.top +supers.com.ua +svarkagid.com +tackletarts.co +takeprofitsystem.com +tattooha.com +teastory.co +texbaza.by +tgtclick.com +thecoolimages.net +thecoral.com.br +theheroes.ru thepokertimer.com -timeallnews.ru -tizanidinedruginteractions.blogspot.com -tizanidineduringpregnancy.blogspot.com -tizanidinepain.blogspot.com -tmearegion26.com -todohr.com -totu.info -traffic-club.info -trafficfactory.biz +thewebsitetemplate.info +titan-cloud.life +tizanidine4mgstreetprice.blogspot.com +tizanidinecapsules.blogspot.com +tizanidinedosageforsleep.blogspot.com +tizanidinehydrochloride4mgstreetvalue.blogspot.com +tizanidineinteractions.blogspot.com +tizanidinevsvalium.blogspot.com +tooplay.com +toproadrunner5.info +topvidos.ru +torrents-tracker.com +trafficjunky.com trafficjunky.net -traffpartners.com -tubo360.com -tversvet.ru -txxx.com -uasb.ru -uchil.net +traffictrade.life +trahvid.com +truck-land.life +tuberkuleznik.ru +tupper-posuda.ru +turbo-suslik.org +uchebavchehii.ru ultimatesetnewfreeallsoftupgradesystems.pw -umityangin.net -unacittaconte.org -underthesite.com -unmaroll.ya.ru -veerotech.com -verymes.xyz -viagra-soft.ru -via-gra.webstarts.com -videochat.ph -videosbox.ru -vikistars.com -vintontech.info -vipps.com.my -vizag.kharkov.ua +um-razum.ru +uprour.com +uptime-gamma.net +us-america.ru +vapomnoncri.tk +velobikestock.com +videochat.mx +viktoria-center.ru +vilingstore.net +vip.51.la +viteonlusarezzo.it +vkontarkte.com +vodaodessa.com +vod.com.ua voprosotvet24.ru -vostoktrade.info -vvpg.ru -vzube.com -walkme.com -webextract.profound.net -webinstantservice.com -webjam.com -webpromotion.ae -wedding0venues.tk -welcomeauto.ru -wheelchairliftsarea.com -wholesalejerseys-cheapest.com -williamrobsonproperty.com -winterclassichockeyjerseys.com -worldhistory.biz -worldinternetauthority.com -wowas31.ucoz.ru -wufak.com -x69ty.ru -xaylapdiendanang.com -xclicks.net +vriel.batcave.net +vsesubwaysurfers.com +vucms.com +wallpapers-best.com +warningwar.ru +watchmyfb.pl +website-stealer.nufaq.com +websitevaluebot.com +whiteproduct.com +wholesalejerseychinashop.com +wineitudes.wordpress.com +writingservices17.blogspot.ru +wsgames.ru +wtsindia.in +wygraj-skiny.win +xaijo.com +xerox-douglas.cf +xgftnlrt.bloger.index.hr xingzi-vision.com -xmladserver.com -xn------7cdbapdecfd4ak1bn0amjffj7afu3y.xn--p1ai -xn--80aaaks3bbhabgbigamdr2h.xn--p1ai -xn--80ahvj9e.xn--p1ai -xn--oogle-wmc.com +x-lime.com +xn--1-8sbcpb0bdm8k6a.xn--p1ai +xn-----7kcabaipgeakzcss7bjdqdwpfnhv.xn--p1ai +xn----7sbahjd3btneuw1joc.xn--p1ai +xn--80aaagvmjabrs1aoc9luc.xn--p1ai +xn--80aeahghtf8ac5i.xn--p1ai +xn--80ajbshivpvn2i.xn--p1ai +xn--90acenikpebbdd4f6d.xn--p1ai +xn----ctbbcjd3dbsehgi.xn--p1ai xn--q1a.xn--b1aube0e.xn--c1acygb.xn--p1ai -x-stars.ru -xxxkaz.org -y8games-free.com -yaminecraft.ru -yebocasino.co.za +xnxx-n.com +xolodremont.ru yogamatsexpert.com -youradexchange.com -yourothersite.com -youtoner.it -zakazvzloma.com -zarajbuilders.com -zastroyka.org -zeroredirect11.com -zeroredirect.com -zetgie.com.pl -zhuravlev.info +yoshkarola.zrus.org +youdesigner.kz +yougotanewdomain.com +youporn-forum.ga +yourdesires.ru +zakonobosago.ru +zaobao.com.sg +zed21.net +zeroredirect12.com +zeroredirect5.com +zolotoy-lis.ru +zoogdinsney.com zscalerone.net diff --git a/.dev-tools/test_units/random-referrers-for-whitelist-test.list b/.dev-tools/test_units/random-referrers-for-whitelist-test.list index 73a636ba1..8936bb0e8 100644 --- a/.dev-tools/test_units/random-referrers-for-whitelist-test.list +++ b/.dev-tools/test_units/random-referrers-for-whitelist-test.list @@ -1,250 +1,250 @@ -00go.com -0n-line.tv -2nt.ru -5000-cotydzien.com -5tey7463.weebly.com -7search.com -above.com -academia-nsk.org -admatic.com.tr -adsref.men -advokat-grodno.by -alienwheels.de -alldezservice.kz -allnews24.in -android4fun.org -angry-fermi-7633.arukascloud.io -aosheng-tech.com -arkkivoltti.net -atlant-auto.info -autochoixspinelli.com -baikaleminer.com -basisches-wasser.net -bear.gotcher.us -best-games.today -bestofpicture.com -bestplacetobuyeriacta.jw.lt -blog.koorg.ru -boleznikogi.com -bracketsmackdown.com -brainsandeggs.blogspot.com -bristolhotel.com.ua -browsepulse-a.akamaihd.net -btvn.ru -buyessayonline19.blogspot.ru -cakemediahq.com.au -cardsdumps.com -cartierloveringreplica.xyz -catalogs-parts.com -cezartabac.ro -c-gainsbourg.com -cheapmarmot.online -clickso.com -clmforexeu.com -community.allhiphop.com -compliance-fred.xyz -compliance-irvin.top -computernetworksonline.com -conboy.us -contextualyield.com -cookie-law-enforcement-hh.xyz -coolbar.pro -coolgramgoods.com -customchocolate.business-for-home.com -cypernhuset.se -danceuniverse.ru -dawlenie.com -dengi-pod-zalog-nedvizhimosti.ru -digitalassetmanagement.site -dipstar.org -domaincrawler.com -donna7753191.ru -downloaddy.net -druzhininevgeniy63.blogspot.com -e2click.com -educontest.net -electro-prom.com -elektrozigaretten1.webs.com -elite-sex-finders.com -eloxal.ru -emailaccountlogin.co -engines-usa.com +00webcams.com +51unlim.ru +5forex.ru +7xc4n.com +aa81bf391151884adfa3dd677e41f94be1.com +aac500b7a15b2646968f6bd8c6305869d7.com +adconscious.com +adman.se +admeasures.com +adultfriendfinder.com +afmuseum.com +alfa9.com +allwomen.info +andpolice.com +apelsinnik.website +apessay.com +apiappenableinfo-a.akamaihd.net +apioutoboxnet-a.akamaihd.net +apxeo.info +arcteryxstore.online +asacopaco.tk +asmxsatadriverin.aircus.com +atley.eu.pn +autotop.com.ua +azbaseimages.net +beclean-nn.ru +besthomemadepornsites.com +bestpriceninja.com +betteroffers.review +bin-brokers.com +bjanshee.ru +bmw.afora.ru +bmwhighperformers.com +brainxs.us +builtwith.com +carapics.com +carloans.com +cashkitten-a.akamaihd.net +c-english.ru +chorus.terakeet.com +clarithromycin500mg.com +cleanmypc.com +clean-virus-mac.com +coldfilm.ru +compliance-elena.xyz +contentlook.co +copro.pw +counciltally.com +cpabegins.ru +creativehutindia.com +crynet.cc +csgo4.win +culad.com +culturevie.info +customcatchcan.com +cyse.tk +d0t.ru +ddsoldes.fr +deanmoore.ie +diusyawiga.tk +doctissimo.fr +dominateforex.ml +domination.ml +downloader12.ru +dvervmoskvu.ru +e-biznes.info +ed-shop01.ru +ekn-art.se +eladkarako.com +elektrischezigarette2.onsugar.com +elidelcream.weebly.com +englate.com epicbrogaming.com -exchanges-bet.com -expertnaya-ocenka.ru -ezigarettekaufen2.blox.pl -ezigarettenshop1.yolasite.com -fatfasts-4tmz.com -faz99.com -findanysex.com -firesub.pl -flipper.top -freeuploader.ml -freewebs.com -fr.netlog.com -frvo.alptandem.ru -fsalas.com -galeon.com -game-mmorpg.net -gameonasia.com -gamevalue7.weebly.com -generalporn.org -germes-trans.com -germetiki.com.ua -get-seo-domain.com +epngo.bz +europages.com.ru +ezigarettenkaufen1.pagina.gr +facebook-mobile.xyz +fandlr.com +femdom.twiclub.in +filmfanatic.com +flauntyoursite.com +free-traffic.xyz +gallictures.com +gambarkataku.co +gamblingnerd.com +gamezblox.com +getlamborghini.ga +getmiro.com glall.ru -goodhousekeeping.com -goodhumor24.com -gowreckdiving.com -gpirate.com -gsou.cf -hawaiielectriclight.com -hazardky.net -hchha.com -hdapp1008-a.akamaihd.net -hentai-manga.porn -hollywoodactress.info -hoverboard360.es -ideawheel.com -impresagaia.it -infowarcraft.ru -interesnie-faktu.ru -it-max.com.ua -izhevsk.zrus.org -javspace.net -jus0wil.pp.ua -kam-dom.ru -kazka.ru -ketrzyn.pl -kraljeva-sutjeska.com -lanadelreyfans.us -lashstudia.ru -law-check-seven.xyz -lider-zhaluzi.kiev.ua -linkdebrideur.xyz -locimge.pw -mactechinfo.info -magicdiet.gq -manve.info -matrixalchemy.com -mebel-vstroika.ru -mejoresfotos.eu -metascephe.com -minecraft-rus.org -mirzonru.net -mncrftpcs.com -moncleronline.xyz -monetizer.com-01.site -monitorwebsitespeed.com -morf.snn.gr -mrbitsandbytes.com -mulberryoutletonlineeu.com -myfreemp3.eu -namenectar.com -newstraveller.ru -njpalletremoval.com -oecnhs.info -office-windows.ru -ohmyrings.com -oksrv.com -onlineku.com -onlineserialy.ru -openfrost.net -origin-my.ru -osnova3.ru -ourville.info -paydayloanslocal.com -picphotos.net -pk-pomosch.ru -play.leadzupc.com -podshipniki-nsk.ru -popmarker.com -popserve.adscpm.net -poptool.net -pornolook.net -porodasobak.net -portadd.men -princevc.com -priora-2.com -privetsochi.ru -profitkode.com -progonrumarket.ru -promalp-universal.ru -prostitutki-belgoroda.org -pulse33.ru -qor360.com -ring4rhino.com -romhacking.ru -rus-pornuha.com +glastecfilms.com.my +gopixdatabase.com +gotcher.us +graphics8.info +gratuitbaise.com +growmyfunds.ca +h2monline.com +hackers-crackers.tk +hanwei.us +histock.info +hobbyhuren-datenbank.com +homedecoguide.info +home.myplaycity.com +hotchatdate.com +hoverboard360.se +hscsscotland.com +hunthillfarmtrust.org +ilmexico.com +ilovevitaly.ru +imagine-ex.co +imitex-plus.ru +infodocsportal.com +irkutsk.online-podarki.com +itsdp3.com +jerseychinabizwholesale.com +jovencitas.gratis +kinomaniatv.pl +kollekcioner.ru +koversite.info +lalalove.ru +landinez.co +laxdrills.com +leftofcentrist.com +lenvred.org +lerporn.info +likesuccess.com +likrot.com +live-sexcam.tk +locustdesign.co +lodki-pvh.dp.ua +mackeeper-land-672695126.us-east-1.elb.amazonaws.com +maridan.com.ua +missvietnam.org +misswell.net +mister-shop.com +mojpregled.com +moncleroutletonline.win +mossmesi.com +moz.com +musicas.baixar-musicas-gratis.com +muznachas-service.ru +net-radar.com +nightvision746.weebly.com +nodup.ru +nowtorrents.com +npoet.ru +nzfilecloud.weebly.com +o00.in +ok.ru +onlythegames.com +o-o-11-o-o.com +pacificair.com +partner-host.men +patol01.pw +peekyou.com +pizda.lol +polybuild.ru +poppen-nw.net +poquoson.org +porn555.com +porngalleries.top +pornhive.org +pornmania.pl +produkto.net +purplestats.com +rapidsites.pro +realmonte.net +reklama-i-rabota.ru +reklamuss.ru +remontbiz.ru +retreatia.com +rewardit.com +rn-to-bsn.com +rootandroid.org +rubanners.com +ruclicks.com +russian-postindex.ru saitevpatorii.com -sbprabooks.com -scanner-ivan.top -scanner-josh.top -scanner-mary.top -se.bnt-team.com -seeingmeerkat.com -seeresultshub-a.akamaihd.net -semalt.com -seofirmreviewsus.info -sex.hotblog.top -sexphoto.site -shemalegalls.blogporn.in -sherlock.se -shiksabd.com -shop-electron.ru -sicfor.bcu.cc -simon3.ru -siteaero.com -siteonomy.com -site-speed-check.site -sledstvie-veli.net -soft-terminal.ru -solmarket.by -spyfu.com -stackthatbucks.com -stefanbakosab.se -suggest-keywords.com +scanner-elena.top +scanner-jack.top +scenicmissouri.us +scopich.com +sdi-pme.com +searchaddis.com +seccioncontrabajo.com +seksvideoonlain.com +sexobzor.info +sigmund-freud.co.uk +simply.net +simpoed.ufop.br +sitevalued.com +skuteczna-dieta.co.pl +sladkoevideo.com +small-games.biz +snow.nvr163.com +solution4u.com +stauga.altervista.org +stevemonsen.com +strana-solnca.ru sugvant.ru -suhanpacktech.com -survival.betteroffers.review -svbur.ru -thebluenoodle.com -the-usa-games.blogspot.com -tizanidine4mgstreetvalue.blogspot.com -tizanidinerecreationaluse.blogspot.com -tizanidinewithdrawalsymptoms.blogspot.com -toon-families.com -tupper-posuda.ru -tvoystartup.ru -twitlinks.com -umekana.ru -undergroundcityphoto.com +tagil.zrus.org +tamada69.com +tam-gde-more.ru +terraclicks.com +thedownloadfromwarez.blogspot.com +theendivechronicles.com +thegamerznetwork.com +theplacetoupdating.pw +tizanidinehcl4mgtablet.blogspot.com +tizanidineingredients.blogspot.com +tizanidineusedfor.blogspot.com +tmm-kurs.ru +trafficmania.com +trafficmonetize.org +tsyndicate.com +turn-up-life.life +tvnewsclips.info +uggsale.online +umnovocaminho.com +uptimebot.net +ustion.ru vacances-voyages.info -veles.shop -vip-parfumeria.ru -vk-mus.ru -vladimir.xrus.org -vseuznaem.com -warningwar.ru -westsextube.com -wholesalejerseys-cheapest.com +versaut.xxx-cam.webcam +viberdownload10.com +videotuber.ru +vintontech.info +vkak.ru +vykupavto-krasnodar.ru +vzube.com +watchmygf.net +weboptimizes.com whosonmyserver.com -wieseversa.no -wingsofrefuge.net -worldhistory.biz -worldlovers.ru -world-mmo.com -ww2awards.info -www.kabbalah-red-bracelets.com -x69ty.ru -xfire.com -xmlinde.com -xmronta.com -xn------7cdbapdecfd4ak1bn0amjffj7afu3y.xn--p1ai -xn--80ak6aa92e.com -xnxx-n.com -zacreditom.ru -zawyna.ua -zeroredirect11.com -znaniyapolza.ru -zojirushi-products.ru +win-spy.com +worldinternetauthority.com +wosik-dach.service-for-web.de +ws.ampower.me +x-diesel.info +xjlottery.com +xn--80aikhbrhr.net +xn----8sbarihbihxpxqgaf0g1e.xn--80adxhks +xn--q1a.xn--b1aube0e.xn--c1acygb.xn--p1ai +xtremeeagles.net +yachts-cruise.info +yoshkarola.zrus.org +yottos.com +youbrainboost.asia +youdao.com +zarabotat-na-sajte.ru +zigzog.ru +zophim.me +zrus.org diff --git a/.dev-tools/test_units/ratelimittest-beta.txt b/.dev-tools/test_units/ratelimittest-beta.txt index 3b5fea028..f371d9e5f 100644 --- a/.dev-tools/test_units/ratelimittest-beta.txt +++ b/.dev-tools/test_units/ratelimittest-beta.txt @@ -1,4 +1,14 @@ - + + +
+This is merely used for testing !!!
+ +This is merely used for testing !!!
- - - -This is merely used for testing !!!
- \ No newline at end of file diff --git a/.dev-tools/test_units/ratelimittest-master.txt b/.dev-tools/test_units/ratelimittest-master.txt index f371d9e5f..4f32ca24e 100644 --- a/.dev-tools/test_units/ratelimittest-master.txt +++ b/.dev-tools/test_units/ratelimittest-master.txt @@ -96,17 +96,14 @@This is merely used for testing !!!
-