diff --git a/AUTO-CONFIGURATION.md b/AUTO-CONFIGURATION.md index 6543573e6..8e0d66335 100755 --- a/AUTO-CONFIGURATION.md +++ b/AUTO-CONFIGURATION.md @@ -4,7 +4,7 @@ ### PLEASE READ CONFIGURATION INSTRUCTIONS BELOW THOROUGHLY :exclamation: _______________ -#### Version: V4.2021.07.2452 +#### Version: V4.2021.07.2453 #### Bad Referrers Blocked: [7066](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-referrers.list) #### Bad User-Agents (Bots) Blocked: [615](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) #### Fake Googlebots Blocked: [217](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/fake-googlebots.list) diff --git a/MANUAL-CONFIGURATION.md b/MANUAL-CONFIGURATION.md index 896d6874a..b4def8a67 100755 --- a/MANUAL-CONFIGURATION.md +++ b/MANUAL-CONFIGURATION.md @@ -4,7 +4,7 @@ ### PLEASE READ CONFIGURATION INSTRUCTIONS BELOW THOROUGHLY :exclamation: _______________ -#### Version: V4.2021.07.2452 +#### Version: V4.2021.07.2453 #### Bad Referrers Blocked: [7066](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-referrers.list) #### Bad User-Agents (Bots) Blocked: [615](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) #### Fake Googlebots Blocked: [217](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/fake-googlebots.list) diff --git a/README.md b/README.md index 4e5abfee5..2d97b2380 100755 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ ##### The Ultimate Nginx Bad Bot, User-Agent, Spam Referrer Blocker, Adware, Malware and Ransomware Blocker, Clickjacking Blocker, Click Re-Directing Blocker, SEO Companies and Bad IP Blocker with Anti DDOS System, Nginx Rate Limiting and Wordpress Theme Detector Blocking. Stop and Block all kinds of bad internet traffic even Fake Googlebots from ever reaching your web sites. [PLEASE SEE: Definition of Bad Bots](#define-bad-bots) _______________ -#### Version: V4.2021.07.2452 +#### Version: V4.2021.07.2453 #### Bad Referrers Blocked: [7066](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-referrers.list) #### Bad User-Agents (Bots) Blocked: [615](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) #### Fake Googlebots Blocked: [217](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/fake-googlebots.list) @@ -28,9 +28,9 @@ ____________________ _______________________ ##### Tested On: ##### nginx version: nginx/1.10.x :heavy_check_mark: -##### nginx version: nginx/1.18.0 (Ubuntu) :heavy_check_mark: -##### nginx version: nginx/1.15.8 :heavy_check_mark: -##### nginx version: nginx/1.21.0 :heavy_check_mark: +##### :heavy_check_mark: +##### :heavy_check_mark: +##### :heavy_check_mark: __________________________ Not Using Nginx? See the [![Get the APACHE ULTIMATE BAD BOT BLOCKER](https://img.shields.io/badge/APACHE%20-%20ULTIMATE%20BAD%20BOT%20BLOCKER%20%E2%9B%94-blue.svg)](https://github.com/mitchellkrogza/apache-ultimate-bad-bot-blocker) diff --git a/_google_analytics_ghost_spam/README.md b/_google_analytics_ghost_spam/README.md index 778ef65fc..521348aa2 100644 --- a/_google_analytics_ghost_spam/README.md +++ b/_google_analytics_ghost_spam/README.md @@ -3,7 +3,7 @@ # EASY CONFIGURATION INSTRUCTIONS FOR STOPPING GOOGLE ANALYTICS "GHOST" SPAM _______________ -#### Version: V4.2021.07.2452 +#### Version: V4.2021.07.2453 #### Bad Referrers Blocked: [7066](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-referrers.list) #### Bad User-Agents (Bots) Blocked: [615](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) #### Fake Googlebots Blocked: [217](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/fake-googlebots.list) diff --git a/_sample_config_files/Engintron_for_cPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf b/_sample_config_files/Engintron_for_cPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf index 9eab654c0..692d7925a 100755 --- a/_sample_config_files/Engintron_for_cPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf +++ b/_sample_config_files/Engintron_for_cPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2021.07.2452 -### Updated: Mon Jul 5 11:13:41 UTC 2021 +### Version: V4.2021.07.2453 +### Updated: Mon Jul 5 11:27:48 UTC 2021 ### Bad Referrer Count: 7066 ### Bad Bot Count: 615 ################################################### diff --git a/conf.d/globalblacklist-testing-version.conf b/conf.d/globalblacklist-testing-version.conf index ef3cc9678..c84d8cf6c 100644 --- a/conf.d/globalblacklist-testing-version.conf +++ b/conf.d/globalblacklist-testing-version.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2021.07.2452 -### Updated: Mon Jul 5 11:13:42 UTC 2021 +### Version: V3.2021.07.2453 +### Updated: Mon Jul 5 11:27:48 UTC 2021 ### Bad Referrer Count: 7066 ### Bad Bot Count: 615 ################################################### diff --git a/conf.d/globalblacklist.conf b/conf.d/globalblacklist.conf index 9eab654c0..692d7925a 100644 --- a/conf.d/globalblacklist.conf +++ b/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2021.07.2452 -### Updated: Mon Jul 5 11:13:41 UTC 2021 +### Version: V4.2021.07.2453 +### Updated: Mon Jul 5 11:27:48 UTC 2021 ### Bad Referrer Count: 7066 ### Bad Bot Count: 615 ################################################### diff --git a/dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf b/dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf index a0b6c094d..1eced411f 100644 --- a/dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf +++ b/dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.134 1; +10.1.0.22 1; 127.0.0.1 1; diff --git a/dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf b/dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf index 2832e7149..c69218003 100644 --- a/dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf +++ b/dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.134 0; +10.1.0.22 0; 127.0.0.1 0; diff --git a/dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf b/dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf index a0b6c094d..1eced411f 100644 --- a/dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf +++ b/dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.134 1; +10.1.0.22 1; 127.0.0.1 1; diff --git a/dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf b/dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf index 2832e7149..c69218003 100644 --- a/dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf +++ b/dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.134 0; +10.1.0.22 0; 127.0.0.1 0; diff --git a/dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf b/dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf index a0b6c094d..1eced411f 100644 --- a/dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf +++ b/dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.134 1; +10.1.0.22 1; 127.0.0.1 1; diff --git a/dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf b/dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf index 2832e7149..c69218003 100644 --- a/dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf +++ b/dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.134 0; +10.1.0.22 0; 127.0.0.1 0; diff --git a/dev-tools/buildnumber b/dev-tools/buildnumber index 2e5b071d2..d9d672867 100644 --- a/dev-tools/buildnumber +++ b/dev-tools/buildnumber @@ -1 +1 @@ -2452 +2453 diff --git a/dev-tools/globalblacklist-testing.template b/dev-tools/globalblacklist-testing.template index ef3cc9678..c84d8cf6c 100644 --- a/dev-tools/globalblacklist-testing.template +++ b/dev-tools/globalblacklist-testing.template @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2021.07.2452 -### Updated: Mon Jul 5 11:13:42 UTC 2021 +### Version: V3.2021.07.2453 +### Updated: Mon Jul 5 11:27:48 UTC 2021 ### Bad Referrer Count: 7066 ### Bad Bot Count: 615 ################################################### diff --git a/dev-tools/globalblacklist.template b/dev-tools/globalblacklist.template index 9eab654c0..692d7925a 100755 --- a/dev-tools/globalblacklist.template +++ b/dev-tools/globalblacklist.template @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2021.07.2452 -### Updated: Mon Jul 5 11:13:41 UTC 2021 +### Version: V4.2021.07.2453 +### Updated: Mon Jul 5 11:27:48 UTC 2021 ### Bad Referrer Count: 7066 ### Bad Bot Count: 615 ################################################### diff --git a/dev-tools/test1_conf_files/bots.d/whitelist-ips.conf b/dev-tools/test1_conf_files/bots.d/whitelist-ips.conf index ae9c0c6de..72ce800b5 100644 --- a/dev-tools/test1_conf_files/bots.d/whitelist-ips.conf +++ b/dev-tools/test1_conf_files/bots.d/whitelist-ips.conf @@ -45,4 +45,4 @@ # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST -51.141.178.176 0; +52.183.126.106 0; diff --git a/dev-tools/test1_conf_files/conf.d/globalblacklist.conf b/dev-tools/test1_conf_files/conf.d/globalblacklist.conf index 9eab654c0..692d7925a 100644 --- a/dev-tools/test1_conf_files/conf.d/globalblacklist.conf +++ b/dev-tools/test1_conf_files/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2021.07.2452 -### Updated: Mon Jul 5 11:13:41 UTC 2021 +### Version: V4.2021.07.2453 +### Updated: Mon Jul 5 11:27:48 UTC 2021 ### Bad Referrer Count: 7066 ### Bad Bot Count: 615 ################################################### diff --git a/dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf b/dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf index a0b6c094d..1eced411f 100644 --- a/dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf +++ b/dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.134 1; +10.1.0.22 1; 127.0.0.1 1; diff --git a/dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf b/dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf index 2832e7149..c69218003 100644 --- a/dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf +++ b/dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.134 0; +10.1.0.22 0; 127.0.0.1 0; diff --git a/dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf b/dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf index 9eab654c0..692d7925a 100644 --- a/dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf +++ b/dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2021.07.2452 -### Updated: Mon Jul 5 11:13:41 UTC 2021 +### Version: V4.2021.07.2453 +### Updated: Mon Jul 5 11:27:48 UTC 2021 ### Bad Referrer Count: 7066 ### Bad Bot Count: 615 ################################################### diff --git a/dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf b/dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf index a0b6c094d..1eced411f 100644 --- a/dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf +++ b/dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.134 1; +10.1.0.22 1; 127.0.0.1 1; diff --git a/dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf b/dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf index 2832e7149..c69218003 100644 --- a/dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf +++ b/dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.134 0; +10.1.0.22 0; 127.0.0.1 0; diff --git a/dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf b/dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf index 9eab654c0..692d7925a 100644 --- a/dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf +++ b/dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2021.07.2452 -### Updated: Mon Jul 5 11:13:41 UTC 2021 +### Version: V4.2021.07.2453 +### Updated: Mon Jul 5 11:27:48 UTC 2021 ### Bad Referrer Count: 7066 ### Bad Bot Count: 615 ################################################### diff --git a/dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf b/dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf index a0b6c094d..1eced411f 100644 --- a/dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf +++ b/dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.134 1; +10.1.0.22 1; 127.0.0.1 1; diff --git a/dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf b/dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf index 2832e7149..c69218003 100644 --- a/dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf +++ b/dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.134 0; +10.1.0.22 0; 127.0.0.1 0; diff --git a/dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf b/dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf index 9eab654c0..692d7925a 100644 --- a/dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf +++ b/dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2021.07.2452 -### Updated: Mon Jul 5 11:13:41 UTC 2021 +### Version: V4.2021.07.2453 +### Updated: Mon Jul 5 11:27:48 UTC 2021 ### Bad Referrer Count: 7066 ### Bad Bot Count: 615 ################################################### diff --git a/dev-tools/test2_conf_files/bots.d/whitelist-ips.conf b/dev-tools/test2_conf_files/bots.d/whitelist-ips.conf index ae9c0c6de..72ce800b5 100644 --- a/dev-tools/test2_conf_files/bots.d/whitelist-ips.conf +++ b/dev-tools/test2_conf_files/bots.d/whitelist-ips.conf @@ -45,4 +45,4 @@ # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST -51.141.178.176 0; +52.183.126.106 0; diff --git a/dev-tools/test2_conf_files/conf.d/globalblacklist.conf b/dev-tools/test2_conf_files/conf.d/globalblacklist.conf index 9af207c3e..4fc1f3d62 100644 --- a/dev-tools/test2_conf_files/conf.d/globalblacklist.conf +++ b/dev-tools/test2_conf_files/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2021.07.2452 -### Updated: Mon Jul 5 11:13:41 UTC 2021 +### Version: V4.2021.07.2453 +### Updated: Mon Jul 5 11:27:48 UTC 2021 ### Bad Referrer Count: 7066 ### Bad Bot Count: 615 ################################################### diff --git a/dev-tools/test3_conf_files/bots.d/whitelist-ips.conf b/dev-tools/test3_conf_files/bots.d/whitelist-ips.conf index ae9c0c6de..72ce800b5 100644 --- a/dev-tools/test3_conf_files/bots.d/whitelist-ips.conf +++ b/dev-tools/test3_conf_files/bots.d/whitelist-ips.conf @@ -45,4 +45,4 @@ # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST -51.141.178.176 0; +52.183.126.106 0; diff --git a/dev-tools/test3_conf_files/conf.d/globalblacklist.conf b/dev-tools/test3_conf_files/conf.d/globalblacklist.conf index 08356e7ae..395efe28c 100644 --- a/dev-tools/test3_conf_files/conf.d/globalblacklist.conf +++ b/dev-tools/test3_conf_files/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2021.07.2452 -### Updated: Mon Jul 5 11:13:41 UTC 2021 +### Version: V4.2021.07.2453 +### Updated: Mon Jul 5 11:27:48 UTC 2021 ### Bad Referrer Count: 7066 ### Bad Bot Count: 615 ################################################### diff --git a/dev-tools/test_units/blacklist-ips.conf b/dev-tools/test_units/blacklist-ips.conf index a0b6c094d..1eced411f 100644 --- a/dev-tools/test_units/blacklist-ips.conf +++ b/dev-tools/test_units/blacklist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.134 1; +10.1.0.22 1; 127.0.0.1 1; diff --git a/dev-tools/test_units/random-bots-for-test-quick.list b/dev-tools/test_units/random-bots-for-test-quick.list index 5da7742e0..53713c40c 100644 --- a/dev-tools/test_units/random-bots-for-test-quick.list +++ b/dev-tools/test_units/random-bots-for-test-quick.list @@ -1,10 +1,10 @@ -BackWeb -BotALot -Buck -BuiltBotTough -GetRight -Gigablast -VB Project -Virusdie -WBSearchBot -x09Mozilla +404enemy +BDCbot +Cogentbot +CrazyWebCrawler +Image Sucker +JamesBOT +Morfeus Fucking Scanner +Probethenet +WebCollage +YoudaoBot diff --git a/dev-tools/test_units/random-bots-for-test.list b/dev-tools/test_units/random-bots-for-test.list index ffdf9d840..beb36d9e9 100644 --- a/dev-tools/test_units/random-bots-for-test.list +++ b/dev-tools/test_units/random-bots-for-test.list @@ -1,250 +1,250 @@ +01h4x.com 404checker -404enemy -Aboundex -AhrefsBot -AiHitBot +80legs +ADmantX Aipbot -Alexibot -Anarchie -Anarchy Ankit -Anthill -Asterias +Aspiegel Attach -AwarioRssBot AwarioSmartBot -BDCbot -BackDoorBot -BacklinkCrawler +BDFetch +BLEXBot Badass -Bandit -BlackWidow +Barkrowler +Bigfoot Blackboard -Blow -Boardreader -Buck +BlowFish +Brandprotect +Brandwatch +Buddy BunnySlippers -CATExplorador -CCBot CazoodleBot Cegbfeieh CheTeam -Chlooe -Cliqzbot -Collector +CheeseBot +Cocolyzebot +Copier CopyRightCheck -Copyscape +Craftbot +CrazyWebCrawler +Crescent +CrunchBot +Curious Custo -DBLBot +CyotekWebCopy DSearch -DTS Agent -DataCha0s -Demon -Deusu -Discobot -Discoverybot -DittoSpyder +Devil +Dirbuster +DnyzBot DomainAppender -Domains Project -Drip +Dotbot +Dragonfly EMail Siphon -EMail Wolf -EasyDL -Ecxi -Express WebPictures +Evil +Exabot +ExtLinksBot Extractor -ExtractorPro -EyeNetIE -Ezooms -FDM -FemtosearchBot +FHscan FlashGet -Flunky -Freeuploader -FyberSpider -G-i-g-a-b-o-t -GalaxyBot +Foobot +FrontPage +Fyrebot GetRight -GetWeb Getintent Go-Ahead-Got-It -GoZilla -Grafula +GrabNet GrapeFX +GridBot +HMView +HTMLparser HTTrack -Harvest -Havij -Hloader +Heritrix HybridBot -IRLbot -Iblog -Image Sucker -InterGET -Iria -IstellaBot -JamesBOT +IDBTE4M +IDBot +Id-search +Image Fetch +Indy Library +Internet Ninja +Iskanie Jbrofuzz +JennyBot +JetCar +Jetty +JikeSpider Jorgee +JustView LNSpiderguy Lanshanbot -LexiBot -LibWeb -LieBaoFast +LeechGet +Libwhisker Lightspeedsystems -Likse -Linkbot -Linkdexbot +LinkScan +LinkWalker LinkextractorPro +LinkpadBot LinksManager LinqiaMetadataDownloaderBot LinqiaRSSBot +Lipperhey +Lipperhey Spider MIDown tool -MIIxpc +MJ12bot +MSFrontPage MTRobot -Mag-Net Magnet -Majestic SEO -Majestic12 +Mail.RU_Bot +MarkMonitor +MarkWatch Masscan Mata Hari +Mb2345Browser +MeanPath Bot Mediatoolkitbot +MegaIndex.ru Metauri -Microsoft URL Control -Mojeek +Microsoft Data Access Mojolicious -Mozlila +Musobot NICErsPRO -NPbot -Nameprotect -Navroad -NetLyzer +Needle +Net Vampire +NetAnts NetMechanic NetSpider -Nettrack +Netcraft +NextGenSearchBot Nibbler Niki-bot -Nikto -Nmap -Offline Explorer +NimbleCrawler +Nimbostratus +Nutch +OPPO A33 Offline Navigator -OnCrawl OpenLinkProfiler +Openfind +Openvas OutclicksBot -PECL::HTTP +OutfoxBot +POE-Component-Client-HTTP +Page Analyzer PageAnalyzer PageGrabber +PageThing.com +Panscient Papa Foto +Pavuk Picscout +Picsearch PictureFinder -Pimonster -PleaseCrawl -ProPowerBot +Pockey +ProWebWalker Psbot -Pump -Quick-Crawler +PyCurl +QueryN Metasearch RankActive -RankActiveLinkBot -Rankivabot -RealDownload -RebelMouse +RankingBot Recorder +RepoMonkey Ripper -SEOkicks-Robot -SEOlyticsCrawler -SEOstats -SMTBot +SBIder +SISTRIX ScanAlert -SearchmetricsBot +ScoutJet +Scrapy +ScreenerBot +SentiBot SeoSiteCheckup -Shodan -SiteCheckerBotCrawler -SiteSnagger +Seomoz +SiteExplorer SiteSucker Sitebeam -Siteimprove -SlySearch -Snapbot +Sitevigil +SmartDownload +Snoopy Sogou web spider Sosospider -Spammen -SpankBot +Sottopop Spanner -Sqlworm -Sqworm -Steeler +Sqlmap +Stripper Sucker +Sucuri +SuperBot +SuperHTTP +Surfbot SurveyBot -T0PHackTeam -T8Abot -Teleport +Suzuran TeleportPro -Telesoft +Telesphoreo TheNomad -Thumbor +Toata Toweyabot Trendiction Trendictionbot -TurnitinBot -TwengaBot +Turingos +Twice Typhoeus +UnisterBot Upflow -VCI -Vacuum -Vagabondo -VelenPublicWebCrawler -VidibleScraper -VoidEYE +V-BOT +VB Project +VeriCiteCrawler +Virusdie Voil -WBSearchBot +WEBDAV +WISENutbot WWW-Collector-E -WWW-Mechanize -WWW::Mechanize -WeSEE +WallpapersHD +Web Collage Web Enhancer Web Fetch Web Fuck +Web Sucker +WebAuto WebBandit WebCollage -WebFetch -WebFuck -WebGo IS +WebEnhancer WebLeacher -WebPix +WebSauger +WebStripper +WebWhacker WebZIP Webalta -Website Quester -WebsiteQuester -Whatweb Widow +WinHTTrack WiseGuys Robot Wonderbot -Woobot -Wprecon Xaldon WebSpider +Zade +Zauba ZmEu ZoomBot -backlink-check +ZoominfoBot +ZyBorg +archive.org_bot +arquivo-web-crawler +autoemailspider +check1.exe +coccocbot-web cognitiveseo crawl.sogou.com domainsproject.org -gopher -instabid -internetVista monitor +evc-batch +facebookscraper +heritrix ips-agent -isitwp.com +magpie-crawler mediawords -muhstik-scan -netEstate NE Crawler -oBot -pcBrowser +page scorer plumanalytics polaris version +probe-image-size ripz -s1z.ru +satoristudio.net scalaj-http -scan.lol seobility -serpstatbot -sogouspider -sp_auditbot +sitechecker.pro +spyfu zauba.io +zgrab diff --git a/dev-tools/test_units/random-bots-for-whitelist-test.list b/dev-tools/test_units/random-bots-for-whitelist-test.list index bd76b0423..12a07792d 100644 --- a/dev-tools/test_units/random-bots-for-whitelist-test.list +++ b/dev-tools/test_units/random-bots-for-whitelist-test.list @@ -1,250 +1,250 @@ -404checker -Aboundexbot -AfD-Verbotsverfahren +360Spider +80legs +ASPSeek +Aboundex +Acunetix +AhrefsBot AiHitBot -Aipbot -Alexibot -AllSubmitter Alligator -Anarchie -Anarchy Anarchy99 -AwarioSmartBot -BBBike +Anthill +Attach +BDCbot +BDFetch +BackDoorBot BackStreet -BacklinkCrawler Badass -BatchFTP -BetaBot -Bigfoot -Bitacle -Black Hole +Barkrowler +Battleztar Bazinga +Blow +BlowFish Bolt Brandprotect Brandwatch Buddy +BuiltBotTough +BuiltWith Bullseye -CATExplorador CCBot CODE87 -CensysInspect -Cliqzbot -Cloud mapping -Copier +Cegbfeieh +CheeseBot +ChinaClaw +Claritybot +Cocolyzebot CopyRightCheck +Copyscape Cosmos -CrazyWebCrawler -CrunchBot -Curious +Custo +CyotekWebCopy DBLBot DTS Agent +DataCha0s +DatabaseDriverMysqli Deusu -Digincore +Devil +Dirbuster +Disco Discoverybot -Dispatch DittoSpyder -DomCopBot -DomainCrawler +DomainSigmaCrawler Dotbot +Drip +EMail Siphon EasyDL +Ecxi +EirGrabber EroCrawler Express WebPictures +ExtLinksBot ExtractorPro -FHscan -FemtosearchBot -Firefox/7.0 +EyeNetIE Flunky -Foobot -FrontPage -FyberSpider -G-i-g-a-b-o-t GT::WWW +GalaxyBot Genieo +GermCrawler +GetRight +GetWeb +Getintent +Gigablast Gigabot -Go!Zilla GoZilla Gotit -Grabber -GrapeFX -GrapeshotCrawler -GridBot -HEADMasterSEO +GrabNet HMView HTMLparser -HTTP::Lite -Heritrix +Haansoft +HaosouSpider +Havij Humanlinks -IDBTE4M -Image Fetch +HybridBot +IDBot +IlseBot +Image Sucker IndeedBot -InfoNaviRobot -InfoTekies -Intelliseek -InterGET -Iskanie +Iria +JOC Web Spider JamesBOT +Jbrofuzz +JennyBot JetCar -Joomla +Jetty JustView +Jyxobot Kenjin Spider -Larbin -LeechFTP -LeechGet +Keyword Density +LNSpiderguy +LexiBot Lftp -Libwhisker +LibWeb LieBaoFast +Lightspeedsystems Likse +LinkWalker Linkdexbot -Lipperhey +LinkpadBot +LinksManager +LinqiaRSSBot Lipperhey Spider Litemage_walker +Lmspider Ltx71 -MIDown tool -MJ12bot -MTRobot -Mail.RU_Bot -Majestic12 -Mata Hari -MeanPath Bot -Microsoft Data Access +MarkWatch +Mass Downloader +MauiBot +Mediatoolkitbot +MegaIndex.ru +Metauri Microsoft URL Control -Mister PiX -Mojeek +Moblie Safari +Mojolicious +Morfeus Fucking Scanner +Mozlila +Musobot NICErsPRO -Name Intelligence -Nameprotect -Nessus -Net Vampire -NetLyzer +Navroad +NearSite +Needle +NetMechanic NetZIP Netcraft -Nettrack NextGenSearchBot Nibbler Niki-bot -NimbleCrawler +Nikto +Ninja Nmap -OPPO A33 +Octopus Offline Explorer OnCrawl -OutclicksBot -OutfoxBot +Openfind +Openvas +OrangeBot +OrangeSpider +PECL::HTTP +PageAnalyzer PageGrabber -PageThing.com Pandalytics -Panscient PeoplePal +Petalbot Picscout -PictureFinder -Piepmatz -Pockey -ProWebWalker -Pump -QueryN Metasearch -RankActiveLinkBot -RankingBot2 -RankurBot -Re-re -ReGet -RealDownload -RocketCrawler -SEOkicks -SEOkicks-Robot -Screaming -ScreenerBot -ScrepyBot -SeoSiteCheckup -Seomoz +Picsearch +PleaseCrawl +Pu_iN +PyCurl +RankFlex +Rankivabot +Reaper +RedesScrapy +RepoMonkey +Ripper +SBIder +SEOlyticsCrawler +SEOprofiler +SEOstats +SMTBot +SalesIntelligent +SearchmetricsBot +Semrush Shodan SiteCheckerBotCrawler SiteExplorer SiteLockSpider -SiteSnagger SiteSucker -Sitebeam -SmartDownload -Snake +Siteimprove Sociscraper -Sogou web spider -Spammen -SpankBot -Spanner -Spbot -SputnikBot -Sqlmap +Sottopop +Spinn3r +Sqlworm Sqworm -Steeler Stripper SuperBot -SuperHTTP +SurveyBot Swiftbot -T8Abot +T0PHackTeam +TeleportPro +Telesoft +Telesphorep +The Intraformant TheNomad +TightTwatBot +Titan +Toweyabot Tracemyfile Trendiction Trendictionbot -Typhoeus +TwengaBot +Twice Upflow V-BOT VB Project -Vacuum +Vagabondo VelenPublicWebCrawler VidibleScraper -VoidEYE -Voltron +WASALive-Bot +WBSearchBot WISENutbot -WPScan -WWW::Mechanize WWWOFFLE -Web Auto -Web Collage +WeSEE Web Enhancer -Web Pix +Web Fuck Web Sauger -WebBandit -WebCopier -WebFuck -WebLeacher -WebPix -WebReaper -WebZIP -Webshag +WebAuto +WebEnhancer +WebFetch +WebStripper +Webalta +WebsiteExtractor Whack -Whacker -Who.is Bot WinHTTrack -Woobot +WiseGuys Robot Wotbox Wprecon Xaldon WebSpider -Xaldon_WebSpider -YoudaoBot -Zade +Zauba Zermelo -Zitebot -ZmEu -ZyBorg +Zeus +ZoomBot +ZoominfoBot +archive.org_bot +arquivo.pt +autoemailspider +backlink-check check1.exe -coccocbot-web cognitiveseo +crawl.sogou.com crawler4j demandbase-bot +eCatch evc-batch -facebookscraper -ips-agent -isitwp.com -lwp-request -magpie-crawler -meanpathbot -muhstik-scan +gopher netEstate NE Crawler -ripz -scalaj-http -sexsearcher -siteripz +page scorer +pcBrowser +satoristudio.net +seobility +sogouspider sp_auditbot -spyfu -trendiction.com -trendiction.de -voyagerx.com -x22Mozilla +sysscan +tAkeOut diff --git a/dev-tools/test_units/random-referrers-for-test-quick.list b/dev-tools/test_units/random-referrers-for-test-quick.list index 4d41a48cc..ee202ebf3 100644 --- a/dev-tools/test_units/random-referrers-for-test-quick.list +++ b/dev-tools/test_units/random-referrers-for-test-quick.list @@ -1,10 +1,10 @@ -enternet.ee -lost-alpha.ru -onlinadverts.com -pcads.ru -pizdeishn.com -pornoreino.com -prokotov.com -russkoe-zdorovie.ru -vesnatehno.com -yun56.co +cakemediahq.com.au +compliance-mary.top +diesel-parts28.ru +jerseychinabizwholesale.com +pawli.eu +ranksonic.org +reward-survey.net +studiofaca.com +tizanidinestreetprice.blogspot.com +vgoloveboli.net diff --git a/dev-tools/test_units/random-referrers-for-test.list b/dev-tools/test_units/random-referrers-for-test.list index f731669b9..ce1887a2d 100644 --- a/dev-tools/test_units/random-referrers-for-test.list +++ b/dev-tools/test_units/random-referrers-for-test.list @@ -1,100 +1,100 @@ -76brighton.co.uk -aa625d84f1587749c1ab011d6f269f7d64.com -adidas.frwebs.fr -adminshop.com -ahrefs.com -alborzan.com -allboard.xobor.de -avek.ru -babyguns.ru -bedandbreakfast.com -bestofupload.info -betterscooter.com -buzz-porno.info -buzzurl.jp -cheap-pills-norx.com -cjs.com.ru -clarithromycin500mg.com -clipartnew.com -compliance-john.top -countercrazy.com -cv.wallhade.co -cxpromote.com -dandiyabeats.in -ddlmega.net -dogsrun.net -domaincheck.io -dostavimvdom.ru -e-biznes.info -elektrozigarettekaufen2.blogse.nl -emerson-rus.ru -european-torches.ru -exdocsfiles.com -fable.in.ua -findclan.org -flowersbazar.com -gazoblok.net.ua -googglet.com -hesteel.pl -hoporno.com -imadedinner.net -infogame.name -intermesh.net -isotoner.com -janavibekken.no -jolic2.com -kaliningrad.zrus.org -labvis.host.sk -livingcanarias.com -lombia.co -macresource.co.uk -mansiondelrio.co -masserect.com -mazda-roadsters.com -mega-bony2017.pl -menetie.ru -mikozstop.com -moi-glazki.ru -mojpregled.com -most.gov.iq -mriyadh.com -nmrk.ru -olvanto.ru -onefilms.net -oops-cinema.ru -paintingplanet.ru -pharmacyincity.com -pirateday.ru -pornoxxx.com.mx -raavidesigns.com -rabot.host.sk -randalljhoward.com +18ps.ru +24videos.tv +883zy.com +aa9d046aab36af4ff182f097f840430d51.com +aanapa.ru +adtech.fr +amigobulls.com +apiallgeniusinfo-a.akamaihd.net +apple.com-cleaner.systems +atovh.local-finders.com +banksrf.ru +beauty-bracelet.com +cardul.ru +carrauterie.be +cdncash.net +checkm8.com +christianlouboutinsaleonline.us +cicaki.net +compliance-alexa.xyz +compliance-barak.xyz +country-chic.ru +dentfidemountpreach.tk +dieswaene.com +directivepub.com +dprtb.com +druzhbany.ru +dtnlyss.com +e2click.com +ecookna.com.ua +edelstahlschornstein-123.de +farm26.ru +fashion-mk.net +finansov.info +flytourisme.org +fotzen-ficken.com +fullfileaccess.com +galaxys6manual.info +gk170.ru +gungamesz.com +guod.me +hollywoodactress.info +homedecorpicture.us +hothor.se +hundejo.com +iboss.com +ilovevitaly.com +imperia31.ru +iqoption-bin.com +journeydownthescale.info +kiinomaniak.pl +kleine-titten.biz +kndxbkdx.bloger.index.hr +letsart.ru +luchshie-topcasino.ru +luckyshop.net.ua +medicineseasybuy.com +metabar.ru +morlat.altervista.org +musicspire.online +mymercy.info +nagdak.ru +narkologiya-penza.ru +nastroyke.net +nerudlogistik.ru +nevansk.ru +next-dentists.tk +nicefloor.co.uk +niroo.info +onlinadverts.com +onlinemeetingnow.com +onlywoman.org +pornpost.in +preconnubial.usuby.site +princeadvantagesales.com +printingpeach.com rangjued.com -reyel1985.webnode.fr -rfid-locker.co -rique.host.sk -rrutw.com -satellite.maps.ilovevitaly.com -scanner-viktor.top -sexyali.com -shopvilleroyboch.com.ua -siteexpress.co.il -slavia.info -slomm.ru -snomer1.ru -soaksoak.ru -storehouse.ua -svolze.com -taboola.com -tetracsaudi.com -thexart.club -tiens2010.ru -tripper.de -truck-land.life -tuberkulezanet.ru -vriel.batcave.net -webstatsdomain.org -wingsoffury2.com -xwatt.ru -yurgorod.ru -zeleznobeton.ru +reward-survey.net +rusenvironmental.net +russintv.fr +scansafe.net +seorankinglinks.xyz +shivafurnishings.com +skutecznetabletkinaporostwlosow.pl +streetfooduncovered.com +tagil.zrus.org +the-torrent-tracker.blogspot.com +the-usa-games.blogspot.com +uptimechecker.com +vipromoffers.com +vonradio.com +wallpaperstock.net +websites-reviews.com +winner7777.net +xn----7sbaphztdjeboffeiof6c.xn--p1ai +xn----ctbigni3aj4h.xn--p1ai +xn--80ajbshivpvn2i.xn--p1ai +xn--c1acygb.xn--p1ai +zarabotok--doma.ru +zeg-distribution.com +zynax.ua diff --git a/dev-tools/test_units/random-referrers-for-whitelist-test.list b/dev-tools/test_units/random-referrers-for-whitelist-test.list index a5656f257..bc5a52867 100644 --- a/dev-tools/test_units/random-referrers-for-whitelist-test.list +++ b/dev-tools/test_units/random-referrers-for-whitelist-test.list @@ -1,250 +1,250 @@ -123movies.love -1688.com -1j7740kd.website -24x7-server-support.site -3rbseyes.com -45en.ru -7makemoneyonline.com -abtasty.com -activecampaign.dreamhosters.com -adclickthru.net -advancedmassagebysara.com -allcredits.su -allforminecraft.ru -alphaforum.ru -alphahoverboards.com -angel1777.kz -animebox.com.ua -apple.com-webbrowsing-security.review -arquapetrarca.info -asacopaco.tk -asmxsatadriverin.aircus.com -asrvvv-a.akamaihd.net -atelielembrancaqueencanta.com.br -atmagroup.ru -atofilms.com -autonew.biz -azartmix.com -bablonow.ru -bad-stars.net -balook.com +1314dh.com +2000k.ru +404.mn +4rent.ru +4replicawatch.net +55wmz.ru +76brighton.co.uk +7ooo.ru +83net.jp +8lufu.com +991mostfm.co.id +abbanreddy.com +adbetclickin.pink +admeasures.com +adultfriendfinder.com +advokateg.xyz +ageofclones.com +alot.com +alpharma.net +animalia-life.club +anniemation.com +api.stathat.com +apiholdingmypage-a.akamaihd.net +artavenuegardenstudios.com +artefakct.com +autobrennero.it +avkzarabotok.info +b2b-lounge.com +banan.tv banksrf.ru -bedandbreakfast.com -best-way.men -bestadbid.com -bestmarriages.com -bestofpicture.com -bigcareer.info -boazpower.com -brandehk.dk -btc4u.ru -bum.com.ru -bumascloset.com -canrioloadm.gq -canrole.com -christianlouboutinshoes.xyz -cinemaflix.website -cl.s7.exct.net -cleanmypc.com -clickbank.net -clubfashionista.com -co.lumb.co -compliance-don.top -compliance-jess.top -coolingoods.com -counciltally.com -dangphoto.trade -dieta-personalna.pl -drhomes.biz -e-collantes.com -elektrischezigarette1.blog.pl -essayassist.com -eugenevaultstorage.com -excitacion.info -fetroshok.ru -figuringmoneyout.com -findclan.org -finuse.com -focalink.com -foto-weinberger.at -freemaintenancesysforpcandmac.top -frivgame250.com -funtoonez.com -fz139.ttk.ru -gallery.rennlist.com -gemgrab-a.akamaihd.net -goblacked.com -goldadpremium.com -gosarhivrt.ru -gotomontenegro.net -greenshop.su -gsmlab.pl -hannasolution.ru -hd-filmy.net -hjaoopoa.top -hollywoodactress.info -hydropump.su +beachdriveblog.com +bestbuy.ca +bestkfiledxd.cf +betteroffers.review +binomo.kz +bkgr.se +blagovest-med.ru +blogig.org +bongacams.com +brainboosting.club +brendbutik.ru +burger-imperia.com +buyhoverboard.com +cakesplus.com.au +cam-kontakte.org +candiceloves.us +carmuffler.net +carrauterie.be +carsdined.org +carsplate.com +cbb1smartlist12.click +cementaresearch.se +cheapmoncler.win +choice-credit.ru +cinemaenergy-hd.ru +cjs.com.ru +cleanersoft.com +cocolyze.com +communisave.co.za +compliance-julianna.top +compliance-margo.top +converse.ddsoldes.fr +counter.yadro.ru +dealwifi.com +dentalpearls.com.au +dikqlyremy.info +dikx.gdn +dirtpics.pw +doeco.ru +domain-tracker.com +domain2008.com +dood.live +druzhininevgeniy63.blogspot.com +e-biznes.info +egovaleo.it +elektrischezigaretten2.beeplog.com +emerson-rus.ru +en.home-task.com +eorogo.top +estibot.com +ezb.elvenmachine.com +fellowshipoftheminds.com +filefilter.weebly.com +findacheaplawyers.com +forum-engineering.ru +fsakhalin.ru +funcrushgames.com +gcup.ru +germes-trans.com +getpopunder.com +gigixo.com +girlsfuckdick.com +gloverid.site +goldpanningtools.com +gymi.name +hahashka.ru +haveinc.xyz +havepussy.com +help.tpu.ru +hitcpm.com +hoste.octopis.com +hotblog.top idegenvezeto.eu -ifmo.ru -ilbe.club -imitex-plus.ru -immobiliaremassaro.com -ispaniya-costa-blanca.ru -istock-mebel.ru -it-max.com.ua -iwantmyfreecash.com -jav-fetish.site -jbl-charge.info -joyceblog.top -karachev-city.ru -kashubadesign.ru -kenaba.su -kino-ecran.ru -kinosed.net -knowyournextmove.com -koopilka.com -kung-fu-ru.com -labelwater.se -ladsblue.com -laptoper.net -lazymanyoga.com -lecbter-relationships.ga -li-er.ru -lida-ru.com -light.ifmo.ru -limads.men -lineavabit.it -linkwithin.com -lomza.info -long-beach-air-conditioning.com -luchshie-topcasino.ru -lunchrock.co -macnewtech.com -managerpak204.weebly.com -matsdale.com -mebelcomplekt.ru -meccadumps.net -meendo-free-traffic.ga -mega-bony2017.pl -mlf.hordo.win -moesexy.com -moneymaster.ru -moviezbonkerssk.cf -muabancantho.info -muz-shoes.ru -mysexpics.ru -nationalbreakdown.com -naughtyconnect.com -networkad.net -nextlnk12.com -nfljerseysforsalewholesaler.com -nrjmobile.fr -o-dachnik.ru -o-o-8-o-o.ru -odesproperty.com -ok-ua.info +ido3.com +inclk.com +infektsii.com +innoslicon.com +istizanidinelikexanax.blogspot.com +joingames.org +jongose.ninja +jumptap.com +jwss.cc +kakadu-interior.com.ua +keywesthideaways.co +kihi.gdn +kletkimehan.ru +klosetkitten.com +kollekcioner.ru +konteiner24.com +lakomka.com.ua +lawyersinfo.org +lider-zhaluzi.kiev.ua +lineshops.biz +liveinternet.ro +loadopia.com +locustdesign.co +lomb.co +louboutinreplica.xyz +lrac.info +luckyshop.net.ua +machicon-akihabara.info +madot.onlinewebshop.net +manifestation.betteroffers.review +meetlocalchicks.com +mobicover.com.ua +moje-recenze.cz +moonci.ru +mrinsidesales.com +mypornfree.ru +nastroyke.net +news-readers.ru +nfljerseyscheapchinabiz.com +nrv.co.za +nw-servis.ru +offf.info oktube.ru -onemactrckr.com -onlythegames.com -onlywoman.org -otbelivanie-zubov.com -oz-offers.com -pc-test.net -photo.houseofgaga.ru -plastgran.com -play.leadzupc.com -pornobrazzers.biz -potoideas.us -pourvous.info -pricheski-video.com -privacyassistant.net -prlog.ru -prpops.com -pussyspace.net -putitin.me -qld10000.net -qor360.com -quality-traffic.com -r-control.ru -raisedseo.com -rating-bestcasino.com -redirectingat.com -registrationdomainsite.com -restorator-msk.ru -rock-cafe.info -ruclicks.com -sack.net -saneitconsulting.com -search-goo.com -search.pch.com -seasonvar.ru -shitmovs.com -shopfishing.com.ua -si-unique.com +onko-24.com +openstat.com +pattersonsweb.com +pavlodar.xkaz.org +pepperstyle.ru +photo-clip.ru +php-market.ru +pihl.se +plaff-go.ru +play-mp3.com +popserve.adscpm.net +porn-w.org +pornogratisdiario.com +pornosok.ru +pornotubexxx.name +prizestohandle.club +productarium.com +progress-upakovka.ru +projefrio.com.br +psvita.ru +purplestats.com +q-moto.ru +quick-offer.com +rada.ru +radiogambling.com +ragecash.com +remont-comp-pomosh.ru +sa-live.com +saitevpatorii.com +sax-sex.com +scanner-mark.top +screentoolkit.com +searchencrypt.com +searchimpression.com +secretscook.ru +seemoreresultshu-a.akamaihd.net +seocheki.net +serialsway.ucoz.ru +serptehnika.ru +sexspornotub.com +sexxdate.net +sexy-screen-savers.com +shikiso.info sierraapps.com -smart-balancewheel.com -smartpet.ru -soc-proof.su -socialtrade.biz -solicita.info +sildenafil-tadalafil.info +sims-sims.ru +site5.com +sitevalued.com +socialbutton.xyz space4update.pw -spaceshipad.com -special-porn.com -speechfoodie.com -stairliftstrue.com -streamin.to -success-seo.com -svarbit.com -szamponrevita.pl -tandvardshuset.net -tecspb.ru -telemetryverification.net -templates.franklinfire.co -tengohydar.tk -theallgirlarcade.com -thin.me.pn -tizanidineduringpregnancy.blogspot.com -tizanidinehcl4mgtabinfo.blogspot.com -tizanidineusedfor.blogspot.com -tnaionline.org -tobeyouday.win -toondinsey.com -topads.men -topcar-krasnodar.ru -topflownews.com -topshef.ru +sport-video-obzor.ru +spy-sts.com +starpages.net +statustroll.com +stroicol.net +stroyhelp-dv.ru +svtrd.com +t-bygg.com +t3chtonic.com +teastory.co +teencastingporn.com +telefonsexkostenlos.tk +tetracsaudi.com +theendivechronicles.com +tizanidinehcl2mgsideeffects.blogspot.com +toloka.hurtom.com torrnada.ru -traffic100.com -trafficshaper.com -trustedmaccleaner.com -twu.com.ua -ucsol.ru -ukkelberg.no -ukrtvir.com.ua -us-america.ru -vacuumcleanerguru.com -varbergsvind.se -velpanex.ru -vroze.com -vse-pesni.com -vvpg.ru -warningzscaler.heraeus.com -webcamtalk.net -whiteelephantwellington.com -wholesalejerseys-cheapest.com -word-vorlagen.net -worldhistory.biz -worldoffiles.ru -wpthemedetector.co.uk -xn-----7kceclhb4abre1b4a0ccl2fxch1a.xn--p1ai -xn----7sbaphztdjeboffeiof6c.xn--p1ai -xn--80aaagvmjabrs1aoc9luc.xn--p1ai -xnxxandxvideos.com -xsfetish.org -yaderenergy.ru -yellowstonesafaritours.com -yourdesires.ru -youtoner.it +tradgardspartner.se +traffic-club.info +trahvid.com +ttrraacckkrr.com +tube8.com +tvnewsclips.info +tytoona.com +ua-company.ru +uggbootsoutletsale.us +undergroundcityphoto.com +univerfiles.com +uptime-as.net +uptime-gamma.net +ussearche.cf +vedomstvo.net +ventelnos.com +via-energy-cumpara.com +vipps.com.my +vk-mus.ru +voprosotvet24.ru +wallpapers-best.com +website-datenbank.de +whatsupinfoley.com +wholinkstome.com +xgames-04.com +xn-----8kcatubaocd1bneepefojs1h2e.xn--p1ai +xn----8sbhefaln6acifdaon5c6f4axh.xn--p1ai +xn----btbdvdh4aafrfciljm6k.xn--p1ai +xn----dtbndd4ae7eub.top +xn--80aaafbn2bc2ahdfrfkln6l.xn--p1ai +xn--80aodinpgi.xn--p1ai +xxxtube69.com zacreditom.ru -zoompegs.com -zvuker.net +zaloadi.ru +zeroredirect2.com +znakomstvaonlain.ru +zoomovies.org diff --git a/dev-tools/test_units/whitelist-ips.conf b/dev-tools/test_units/whitelist-ips.conf index 2832e7149..c69218003 100644 --- a/dev-tools/test_units/whitelist-ips.conf +++ b/dev-tools/test_units/whitelist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.134 0; +10.1.0.22 0; 127.0.0.1 0; diff --git a/dev-tools/tmp/robots.txt b/dev-tools/tmp/robots.txt index 08276c436..f01c456fd 100644 --- a/dev-tools/tmp/robots.txt +++ b/dev-tools/tmp/robots.txt @@ -2494,3 +2494,1251 @@ Disallow:/ User-agent: zgrab Disallow:/ +################################################################### +# The Ultimate robots.txt Bot and User-Agent Blocker +# Copyright: +# https://github.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker +################################################################### + +### Version Information # +################################################### +### Version: V4.2021.07.2453 +### Updated: Mon Jul 5 11:27:49 UTC 2021 +### Bad Bot Count: 615 +################################################### +### Version Information ## + +User-agent: * +Disallow: /wp-admin/ +Allow: /wp-admin/admin-ajax.php +User-agent: 01h4x.com +Disallow:/ +User-agent: 360Spider +Disallow:/ +User-agent: 404checker +Disallow:/ +User-agent: 404enemy +Disallow:/ +User-agent: 80legs +Disallow:/ +User-agent: ADmantX +Disallow:/ +User-agent: AIBOT +Disallow:/ +User-agent: ASPSeek +Disallow:/ +User-agent: Abonti +Disallow:/ +User-agent: Aboundex +Disallow:/ +User-agent: Aboundexbot +Disallow:/ +User-agent: Acunetix +Disallow:/ +User-agent: AfD-Verbotsverfahren +Disallow:/ +User-agent: AhrefsBot +Disallow:/ +User-agent: AiHitBot +Disallow:/ +User-agent: Aipbot +Disallow:/ +User-agent: Alexibot +Disallow:/ +User-agent: AllSubmitter +Disallow:/ +User-agent: Alligator +Disallow:/ +User-agent: AlphaBot +Disallow:/ +User-agent: Anarchie +Disallow:/ +User-agent: Anarchy +Disallow:/ +User-agent: Anarchy99 +Disallow:/ +User-agent: Ankit +Disallow:/ +User-agent: Anthill +Disallow:/ +User-agent: Apexoo +Disallow:/ +User-agent: Aspiegel +Disallow:/ +User-agent: Asterias +Disallow:/ +User-agent: Attach +Disallow:/ +User-agent: AwarioRssBot +Disallow:/ +User-agent: AwarioSmartBot +Disallow:/ +User-agent: BBBike +Disallow:/ +User-agent: BDCbot +Disallow:/ +User-agent: BDFetch +Disallow:/ +User-agent: BLEXBot +Disallow:/ +User-agent: BackDoorBot +Disallow:/ +User-agent: BackStreet +Disallow:/ +User-agent: BackWeb +Disallow:/ +User-agent: Backlink-Ceck +Disallow:/ +User-agent: BacklinkCrawler +Disallow:/ +User-agent: Badass +Disallow:/ +User-agent: Bandit +Disallow:/ +User-agent: Barkrowler +Disallow:/ +User-agent: BatchFTP +Disallow:/ +User-agent: Battleztar Bazinga +Disallow:/ +User-agent: BetaBot +Disallow:/ +User-agent: Bigfoot +Disallow:/ +User-agent: Bitacle +Disallow:/ +User-agent: BlackWidow +Disallow:/ +User-agent: Black Hole +Disallow:/ +User-agent: Blackboard +Disallow:/ +User-agent: Blow +Disallow:/ +User-agent: BlowFish +Disallow:/ +User-agent: Boardreader +Disallow:/ +User-agent: Bolt +Disallow:/ +User-agent: BotALot +Disallow:/ +User-agent: Brandprotect +Disallow:/ +User-agent: Brandwatch +Disallow:/ +User-agent: Buck +Disallow:/ +User-agent: Buddy +Disallow:/ +User-agent: BuiltBotTough +Disallow:/ +User-agent: BuiltWith +Disallow:/ +User-agent: Bullseye +Disallow:/ +User-agent: BunnySlippers +Disallow:/ +User-agent: BuzzSumo +Disallow:/ +User-agent: CATExplorador +Disallow:/ +User-agent: CCBot +Disallow:/ +User-agent: CODE87 +Disallow:/ +User-agent: CSHttp +Disallow:/ +User-agent: Calculon +Disallow:/ +User-agent: CazoodleBot +Disallow:/ +User-agent: Cegbfeieh +Disallow:/ +User-agent: CensysInspect +Disallow:/ +User-agent: CheTeam +Disallow:/ +User-agent: CheeseBot +Disallow:/ +User-agent: CherryPicker +Disallow:/ +User-agent: ChinaClaw +Disallow:/ +User-agent: Chlooe +Disallow:/ +User-agent: Claritybot +Disallow:/ +User-agent: Cliqzbot +Disallow:/ +User-agent: Cloud mapping +Disallow:/ +User-agent: Cocolyzebot +Disallow:/ +User-agent: Cogentbot +Disallow:/ +User-agent: Collector +Disallow:/ +User-agent: Copier +Disallow:/ +User-agent: CopyRightCheck +Disallow:/ +User-agent: Copyscape +Disallow:/ +User-agent: Cosmos +Disallow:/ +User-agent: Craftbot +Disallow:/ +User-agent: CrazyWebCrawler +Disallow:/ +User-agent: Crescent +Disallow:/ +User-agent: CrunchBot +Disallow:/ +User-agent: Curious +Disallow:/ +User-agent: Custo +Disallow:/ +User-agent: CyotekWebCopy +Disallow:/ +User-agent: DBLBot +Disallow:/ +User-agent: DIIbot +Disallow:/ +User-agent: DSearch +Disallow:/ +User-agent: DTS Agent +Disallow:/ +User-agent: DataCha0s +Disallow:/ +User-agent: DatabaseDriverMysqli +Disallow:/ +User-agent: Demon +Disallow:/ +User-agent: Deusu +Disallow:/ +User-agent: Devil +Disallow:/ +User-agent: Digincore +Disallow:/ +User-agent: DigitalPebble +Disallow:/ +User-agent: Dirbuster +Disallow:/ +User-agent: Disco +Disallow:/ +User-agent: Discobot +Disallow:/ +User-agent: Discoverybot +Disallow:/ +User-agent: Dispatch +Disallow:/ +User-agent: DittoSpyder +Disallow:/ +User-agent: DnyzBot +Disallow:/ +User-agent: DomCopBot +Disallow:/ +User-agent: DomainAppender +Disallow:/ +User-agent: DomainCrawler +Disallow:/ +User-agent: DomainSigmaCrawler +Disallow:/ +User-agent: DomainStatsBot +Disallow:/ +User-agent: Domains Project +Disallow:/ +User-agent: Dotbot +Disallow:/ +User-agent: Download Wonder +Disallow:/ +User-agent: Dragonfly +Disallow:/ +User-agent: Drip +Disallow:/ +User-agent: ECCP/1.0 +Disallow:/ +User-agent: EMail Siphon +Disallow:/ +User-agent: EMail Wolf +Disallow:/ +User-agent: EasyDL +Disallow:/ +User-agent: Ebingbong +Disallow:/ +User-agent: Ecxi +Disallow:/ +User-agent: EirGrabber +Disallow:/ +User-agent: EroCrawler +Disallow:/ +User-agent: Evil +Disallow:/ +User-agent: Exabot +Disallow:/ +User-agent: Express WebPictures +Disallow:/ +User-agent: ExtLinksBot +Disallow:/ +User-agent: Extractor +Disallow:/ +User-agent: ExtractorPro +Disallow:/ +User-agent: Extreme Picture Finder +Disallow:/ +User-agent: EyeNetIE +Disallow:/ +User-agent: Ezooms +Disallow:/ +User-agent: FDM +Disallow:/ +User-agent: FHscan +Disallow:/ +User-agent: FemtosearchBot +Disallow:/ +User-agent: Fimap +Disallow:/ +User-agent: Firefox/7.0 +Disallow:/ +User-agent: FlashGet +Disallow:/ +User-agent: Flunky +Disallow:/ +User-agent: Foobot +Disallow:/ +User-agent: Freeuploader +Disallow:/ +User-agent: FrontPage +Disallow:/ +User-agent: Fuzz +Disallow:/ +User-agent: FyberSpider +Disallow:/ +User-agent: Fyrebot +Disallow:/ +User-agent: G-i-g-a-b-o-t +Disallow:/ +User-agent: GT::WWW +Disallow:/ +User-agent: GalaxyBot +Disallow:/ +User-agent: Genieo +Disallow:/ +User-agent: GermCrawler +Disallow:/ +User-agent: GetRight +Disallow:/ +User-agent: GetWeb +Disallow:/ +User-agent: Getintent +Disallow:/ +User-agent: Gigablast +Disallow:/ +User-agent: Gigabot +Disallow:/ +User-agent: Go!Zilla +Disallow:/ +User-agent: Go-Ahead-Got-It +Disallow:/ +User-agent: GoZilla +Disallow:/ +User-agent: Gotit +Disallow:/ +User-agent: GrabNet +Disallow:/ +User-agent: Grabber +Disallow:/ +User-agent: Grafula +Disallow:/ +User-agent: GrapeFX +Disallow:/ +User-agent: GrapeshotCrawler +Disallow:/ +User-agent: GridBot +Disallow:/ +User-agent: HEADMasterSEO +Disallow:/ +User-agent: HMView +Disallow:/ +User-agent: HTMLparser +Disallow:/ +User-agent: HTTP::Lite +Disallow:/ +User-agent: HTTrack +Disallow:/ +User-agent: Haansoft +Disallow:/ +User-agent: HaosouSpider +Disallow:/ +User-agent: Harvest +Disallow:/ +User-agent: Havij +Disallow:/ +User-agent: Heritrix +Disallow:/ +User-agent: Hloader +Disallow:/ +User-agent: Humanlinks +Disallow:/ +User-agent: HybridBot +Disallow:/ +User-agent: IDBTE4M +Disallow:/ +User-agent: IDBot +Disallow:/ +User-agent: IRLbot +Disallow:/ +User-agent: Iblog +Disallow:/ +User-agent: Id-search +Disallow:/ +User-agent: IlseBot +Disallow:/ +User-agent: Image Fetch +Disallow:/ +User-agent: Image Sucker +Disallow:/ +User-agent: IndeedBot +Disallow:/ +User-agent: Indy Library +Disallow:/ +User-agent: InfoNaviRobot +Disallow:/ +User-agent: InfoTekies +Disallow:/ +User-agent: Intelliseek +Disallow:/ +User-agent: InterGET +Disallow:/ +User-agent: InternetSeer +Disallow:/ +User-agent: Internet Ninja +Disallow:/ +User-agent: Iria +Disallow:/ +User-agent: Iskanie +Disallow:/ +User-agent: IstellaBot +Disallow:/ +User-agent: JOC Web Spider +Disallow:/ +User-agent: JamesBOT +Disallow:/ +User-agent: Jbrofuzz +Disallow:/ +User-agent: JennyBot +Disallow:/ +User-agent: JetCar +Disallow:/ +User-agent: Jetty +Disallow:/ +User-agent: JikeSpider +Disallow:/ +User-agent: Joomla +Disallow:/ +User-agent: Jorgee +Disallow:/ +User-agent: JustView +Disallow:/ +User-agent: Jyxobot +Disallow:/ +User-agent: Kenjin Spider +Disallow:/ +User-agent: Keyword Density +Disallow:/ +User-agent: Kinza +Disallow:/ +User-agent: Kozmosbot +Disallow:/ +User-agent: LNSpiderguy +Disallow:/ +User-agent: LWP::Simple +Disallow:/ +User-agent: Lanshanbot +Disallow:/ +User-agent: Larbin +Disallow:/ +User-agent: LeechFTP +Disallow:/ +User-agent: LeechGet +Disallow:/ +User-agent: LexiBot +Disallow:/ +User-agent: Lftp +Disallow:/ +User-agent: LibWeb +Disallow:/ +User-agent: Libwhisker +Disallow:/ +User-agent: LieBaoFast +Disallow:/ +User-agent: Lightspeedsystems +Disallow:/ +User-agent: Likse +Disallow:/ +User-agent: LinkScan +Disallow:/ +User-agent: LinkWalker +Disallow:/ +User-agent: Linkbot +Disallow:/ +User-agent: Linkdexbot +Disallow:/ +User-agent: LinkextractorPro +Disallow:/ +User-agent: LinkpadBot +Disallow:/ +User-agent: LinksManager +Disallow:/ +User-agent: LinqiaMetadataDownloaderBot +Disallow:/ +User-agent: LinqiaRSSBot +Disallow:/ +User-agent: LinqiaScrapeBot +Disallow:/ +User-agent: Lipperhey +Disallow:/ +User-agent: Lipperhey Spider +Disallow:/ +User-agent: Litemage_walker +Disallow:/ +User-agent: Lmspider +Disallow:/ +User-agent: Ltx71 +Disallow:/ +User-agent: MFC_Tear_Sample +Disallow:/ +User-agent: MIDown tool +Disallow:/ +User-agent: MIIxpc +Disallow:/ +User-agent: MJ12bot +Disallow:/ +User-agent: MQQBrowser +Disallow:/ +User-agent: MSFrontPage +Disallow:/ +User-agent: MSIECrawler +Disallow:/ +User-agent: MTRobot +Disallow:/ +User-agent: Mag-Net +Disallow:/ +User-agent: Magnet +Disallow:/ +User-agent: Mail.RU_Bot +Disallow:/ +User-agent: Majestic-SEO +Disallow:/ +User-agent: Majestic12 +Disallow:/ +User-agent: Majestic SEO +Disallow:/ +User-agent: MarkMonitor +Disallow:/ +User-agent: MarkWatch +Disallow:/ +User-agent: Mass Downloader +Disallow:/ +User-agent: Masscan +Disallow:/ +User-agent: Mata Hari +Disallow:/ +User-agent: MauiBot +Disallow:/ +User-agent: Mb2345Browser +Disallow:/ +User-agent: MeanPath Bot +Disallow:/ +User-agent: Meanpathbot +Disallow:/ +User-agent: Mediatoolkitbot +Disallow:/ +User-agent: MegaIndex.ru +Disallow:/ +User-agent: Metauri +Disallow:/ +User-agent: MicroMessenger +Disallow:/ +User-agent: Microsoft Data Access +Disallow:/ +User-agent: Microsoft URL Control +Disallow:/ +User-agent: Mister PiX +Disallow:/ +User-agent: Moblie Safari +Disallow:/ +User-agent: Mojeek +Disallow:/ +User-agent: Mojolicious +Disallow:/ +User-agent: Morfeus Fucking Scanner +Disallow:/ +User-agent: Mozlila +Disallow:/ +User-agent: Mr.4x3 +Disallow:/ +User-agent: Msrabot +Disallow:/ +User-agent: Musobot +Disallow:/ +User-agent: NICErsPRO +Disallow:/ +User-agent: NPbot +Disallow:/ +User-agent: Name Intelligence +Disallow:/ +User-agent: Nameprotect +Disallow:/ +User-agent: Navroad +Disallow:/ +User-agent: NearSite +Disallow:/ +User-agent: Needle +Disallow:/ +User-agent: Nessus +Disallow:/ +User-agent: NetAnts +Disallow:/ +User-agent: NetLyzer +Disallow:/ +User-agent: NetMechanic +Disallow:/ +User-agent: NetSpider +Disallow:/ +User-agent: NetZIP +Disallow:/ +User-agent: Net Vampire +Disallow:/ +User-agent: Netcraft +Disallow:/ +User-agent: Nettrack +Disallow:/ +User-agent: Netvibes +Disallow:/ +User-agent: NextGenSearchBot +Disallow:/ +User-agent: Nibbler +Disallow:/ +User-agent: Niki-bot +Disallow:/ +User-agent: Nikto +Disallow:/ +User-agent: NimbleCrawler +Disallow:/ +User-agent: Nimbostratus +Disallow:/ +User-agent: Ninja +Disallow:/ +User-agent: Nmap +Disallow:/ +User-agent: Nuclei +Disallow:/ +User-agent: Nutch +Disallow:/ +User-agent: OPPO A33 +Disallow:/ +User-agent: Octopus +Disallow:/ +User-agent: Offline Explorer +Disallow:/ +User-agent: Offline Navigator +Disallow:/ +User-agent: OnCrawl +Disallow:/ +User-agent: OpenLinkProfiler +Disallow:/ +User-agent: OpenVAS +Disallow:/ +User-agent: Openfind +Disallow:/ +User-agent: Openvas +Disallow:/ +User-agent: OrangeBot +Disallow:/ +User-agent: OrangeSpider +Disallow:/ +User-agent: OutclicksBot +Disallow:/ +User-agent: OutfoxBot +Disallow:/ +User-agent: PECL::HTTP +Disallow:/ +User-agent: PHPCrawl +Disallow:/ +User-agent: POE-Component-Client-HTTP +Disallow:/ +User-agent: PageAnalyzer +Disallow:/ +User-agent: PageGrabber +Disallow:/ +User-agent: PageScorer +Disallow:/ +User-agent: PageThing.com +Disallow:/ +User-agent: Page Analyzer +Disallow:/ +User-agent: Pandalytics +Disallow:/ +User-agent: Panscient +Disallow:/ +User-agent: Papa Foto +Disallow:/ +User-agent: Pavuk +Disallow:/ +User-agent: PeoplePal +Disallow:/ +User-agent: Petalbot +Disallow:/ +User-agent: Pi-Monster +Disallow:/ +User-agent: Picscout +Disallow:/ +User-agent: Picsearch +Disallow:/ +User-agent: PictureFinder +Disallow:/ +User-agent: Piepmatz +Disallow:/ +User-agent: Pimonster +Disallow:/ +User-agent: Pixray +Disallow:/ +User-agent: PleaseCrawl +Disallow:/ +User-agent: Pockey +Disallow:/ +User-agent: ProPowerBot +Disallow:/ +User-agent: ProWebWalker +Disallow:/ +User-agent: Probethenet +Disallow:/ +User-agent: Psbot +Disallow:/ +User-agent: Pu_iN +Disallow:/ +User-agent: Pump +Disallow:/ +User-agent: PxBroker +Disallow:/ +User-agent: PyCurl +Disallow:/ +User-agent: QueryN Metasearch +Disallow:/ +User-agent: Quick-Crawler +Disallow:/ +User-agent: RSSingBot +Disallow:/ +User-agent: RankActive +Disallow:/ +User-agent: RankActiveLinkBot +Disallow:/ +User-agent: RankFlex +Disallow:/ +User-agent: RankingBot +Disallow:/ +User-agent: RankingBot2 +Disallow:/ +User-agent: Rankivabot +Disallow:/ +User-agent: RankurBot +Disallow:/ +User-agent: Re-re +Disallow:/ +User-agent: ReGet +Disallow:/ +User-agent: RealDownload +Disallow:/ +User-agent: Reaper +Disallow:/ +User-agent: RebelMouse +Disallow:/ +User-agent: Recorder +Disallow:/ +User-agent: RedesScrapy +Disallow:/ +User-agent: RepoMonkey +Disallow:/ +User-agent: Ripper +Disallow:/ +User-agent: RocketCrawler +Disallow:/ +User-agent: Rogerbot +Disallow:/ +User-agent: SBIder +Disallow:/ +User-agent: SEOkicks +Disallow:/ +User-agent: SEOkicks-Robot +Disallow:/ +User-agent: SEOlyticsCrawler +Disallow:/ +User-agent: SEOprofiler +Disallow:/ +User-agent: SEOstats +Disallow:/ +User-agent: SISTRIX +Disallow:/ +User-agent: SMTBot +Disallow:/ +User-agent: SalesIntelligent +Disallow:/ +User-agent: ScanAlert +Disallow:/ +User-agent: Scanbot +Disallow:/ +User-agent: ScoutJet +Disallow:/ +User-agent: Scrapy +Disallow:/ +User-agent: Screaming +Disallow:/ +User-agent: ScreenerBot +Disallow:/ +User-agent: ScrepyBot +Disallow:/ +User-agent: Searchestate +Disallow:/ +User-agent: SearchmetricsBot +Disallow:/ +User-agent: Semrush +Disallow:/ +User-agent: SemrushBot +Disallow:/ +User-agent: SentiBot +Disallow:/ +User-agent: SeoSiteCheckup +Disallow:/ +User-agent: SeobilityBot +Disallow:/ +User-agent: Seomoz +Disallow:/ +User-agent: Shodan +Disallow:/ +User-agent: Siphon +Disallow:/ +User-agent: SiteCheckerBotCrawler +Disallow:/ +User-agent: SiteExplorer +Disallow:/ +User-agent: SiteLockSpider +Disallow:/ +User-agent: SiteSnagger +Disallow:/ +User-agent: SiteSucker +Disallow:/ +User-agent: Site Sucker +Disallow:/ +User-agent: Sitebeam +Disallow:/ +User-agent: Siteimprove +Disallow:/ +User-agent: Sitevigil +Disallow:/ +User-agent: SlySearch +Disallow:/ +User-agent: SmartDownload +Disallow:/ +User-agent: Snake +Disallow:/ +User-agent: Snapbot +Disallow:/ +User-agent: Snoopy +Disallow:/ +User-agent: SocialRankIOBot +Disallow:/ +User-agent: Sociscraper +Disallow:/ +User-agent: Sogou web spider +Disallow:/ +User-agent: Sosospider +Disallow:/ +User-agent: Sottopop +Disallow:/ +User-agent: SpaceBison +Disallow:/ +User-agent: Spammen +Disallow:/ +User-agent: SpankBot +Disallow:/ +User-agent: Spanner +Disallow:/ +User-agent: Spbot +Disallow:/ +User-agent: Spinn3r +Disallow:/ +User-agent: SputnikBot +Disallow:/ +User-agent: Sqlmap +Disallow:/ +User-agent: Sqlworm +Disallow:/ +User-agent: Sqworm +Disallow:/ +User-agent: Steeler +Disallow:/ +User-agent: Stripper +Disallow:/ +User-agent: Sucker +Disallow:/ +User-agent: Sucuri +Disallow:/ +User-agent: SuperBot +Disallow:/ +User-agent: SuperHTTP +Disallow:/ +User-agent: Surfbot +Disallow:/ +User-agent: SurveyBot +Disallow:/ +User-agent: Suzuran +Disallow:/ +User-agent: Swiftbot +Disallow:/ +User-agent: Szukacz +Disallow:/ +User-agent: T0PHackTeam +Disallow:/ +User-agent: T8Abot +Disallow:/ +User-agent: Teleport +Disallow:/ +User-agent: TeleportPro +Disallow:/ +User-agent: Telesoft +Disallow:/ +User-agent: Telesphoreo +Disallow:/ +User-agent: Telesphorep +Disallow:/ +User-agent: TheNomad +Disallow:/ +User-agent: The Intraformant +Disallow:/ +User-agent: Thumbor +Disallow:/ +User-agent: TightTwatBot +Disallow:/ +User-agent: Titan +Disallow:/ +User-agent: Toata +Disallow:/ +User-agent: Toweyabot +Disallow:/ +User-agent: Tracemyfile +Disallow:/ +User-agent: Trendiction +Disallow:/ +User-agent: Trendictionbot +Disallow:/ +User-agent: True_Robot +Disallow:/ +User-agent: Turingos +Disallow:/ +User-agent: Turnitin +Disallow:/ +User-agent: TurnitinBot +Disallow:/ +User-agent: TwengaBot +Disallow:/ +User-agent: Twice +Disallow:/ +User-agent: Typhoeus +Disallow:/ +User-agent: URLy.Warning +Disallow:/ +User-agent: URLy Warning +Disallow:/ +User-agent: UnisterBot +Disallow:/ +User-agent: Upflow +Disallow:/ +User-agent: V-BOT +Disallow:/ +User-agent: VB Project +Disallow:/ +User-agent: VCI +Disallow:/ +User-agent: Vacuum +Disallow:/ +User-agent: Vagabondo +Disallow:/ +User-agent: VelenPublicWebCrawler +Disallow:/ +User-agent: VeriCiteCrawler +Disallow:/ +User-agent: VidibleScraper +Disallow:/ +User-agent: Virusdie +Disallow:/ +User-agent: VoidEYE +Disallow:/ +User-agent: Voil +Disallow:/ +User-agent: Voltron +Disallow:/ +User-agent: WASALive-Bot +Disallow:/ +User-agent: WBSearchBot +Disallow:/ +User-agent: WEBDAV +Disallow:/ +User-agent: WISENutbot +Disallow:/ +User-agent: WPScan +Disallow:/ +User-agent: WWW-Collector-E +Disallow:/ +User-agent: WWW-Mechanize +Disallow:/ +User-agent: WWW::Mechanize +Disallow:/ +User-agent: WWWOFFLE +Disallow:/ +User-agent: Wallpapers/3.0 +Disallow:/ +User-agent: WallpapersHD +Disallow:/ +User-agent: WeSEE +Disallow:/ +User-agent: WebAuto +Disallow:/ +User-agent: WebBandit +Disallow:/ +User-agent: WebCollage +Disallow:/ +User-agent: WebCopier +Disallow:/ +User-agent: WebEnhancer +Disallow:/ +User-agent: WebFetch +Disallow:/ +User-agent: WebFuck +Disallow:/ +User-agent: WebGo IS +Disallow:/ +User-agent: WebImageCollector +Disallow:/ +User-agent: WebLeacher +Disallow:/ +User-agent: WebPix +Disallow:/ +User-agent: WebReaper +Disallow:/ +User-agent: WebSauger +Disallow:/ +User-agent: WebStripper +Disallow:/ +User-agent: WebSucker +Disallow:/ +User-agent: WebWhacker +Disallow:/ +User-agent: WebZIP +Disallow:/ +User-agent: Web Auto +Disallow:/ +User-agent: Web Collage +Disallow:/ +User-agent: Web Enhancer +Disallow:/ +User-agent: Web Fetch +Disallow:/ +User-agent: Web Fuck +Disallow:/ +User-agent: Web Pix +Disallow:/ +User-agent: Web Sauger +Disallow:/ +User-agent: Web Sucker +Disallow:/ +User-agent: Webalta +Disallow:/ +User-agent: WebmasterWorldForumBot +Disallow:/ +User-agent: Webshag +Disallow:/ +User-agent: WebsiteExtractor +Disallow:/ +User-agent: WebsiteQuester +Disallow:/ +User-agent: Website Quester +Disallow:/ +User-agent: Webster +Disallow:/ +User-agent: Whack +Disallow:/ +User-agent: Whacker +Disallow:/ +User-agent: Whatweb +Disallow:/ +User-agent: Who.is Bot +Disallow:/ +User-agent: Widow +Disallow:/ +User-agent: WinHTTrack +Disallow:/ +User-agent: WiseGuys Robot +Disallow:/ +User-agent: Wonderbot +Disallow:/ +User-agent: Woobot +Disallow:/ +User-agent: Wotbox +Disallow:/ +User-agent: Wprecon +Disallow:/ +User-agent: Xaldon WebSpider +Disallow:/ +User-agent: Xaldon_WebSpider +Disallow:/ +User-agent: Xenu +Disallow:/ +User-agent: YoudaoBot +Disallow:/ +User-agent: Zade +Disallow:/ +User-agent: Zauba +Disallow:/ +User-agent: Zermelo +Disallow:/ +User-agent: Zeus +Disallow:/ +User-agent: Zitebot +Disallow:/ +User-agent: ZmEu +Disallow:/ +User-agent: ZoomBot +Disallow:/ +User-agent: ZoominfoBot +Disallow:/ +User-agent: ZumBot +Disallow:/ +User-agent: ZyBorg +Disallow:/ +User-agent: archive.org_bot +Disallow:/ +User-agent: arquivo-web-crawler +Disallow:/ +User-agent: arquivo.pt +Disallow:/ +User-agent: autoemailspider +Disallow:/ +User-agent: backlink-check +Disallow:/ +User-agent: check1.exe +Disallow:/ +User-agent: coccocbot-web +Disallow:/ +User-agent: cognitiveseo +Disallow:/ +User-agent: com.plumanalytics +Disallow:/ +User-agent: crawl.sogou.com +Disallow:/ +User-agent: crawler.feedback +Disallow:/ +User-agent: crawler4j +Disallow:/ +User-agent: demandbase-bot +Disallow:/ +User-agent: domainsproject.org +Disallow:/ +User-agent: eCatch +Disallow:/ +User-agent: evc-batch +Disallow:/ +User-agent: facebookscraper +Disallow:/ +User-agent: gopher +Disallow:/ +User-agent: heritrix +Disallow:/ +User-agent: instabid +Disallow:/ +User-agent: internetVista monitor +Disallow:/ +User-agent: ips-agent +Disallow:/ +User-agent: isitwp.com +Disallow:/ +User-agent: lwp-request +Disallow:/ +User-agent: lwp-trivial +Disallow:/ +User-agent: magpie-crawler +Disallow:/ +User-agent: meanpathbot +Disallow:/ +User-agent: mediawords +Disallow:/ +User-agent: muhstik-scan +Disallow:/ +User-agent: netEstate NE Crawler +Disallow:/ +User-agent: oBot +Disallow:/ +User-agent: page scorer +Disallow:/ +User-agent: pcBrowser +Disallow:/ +User-agent: plumanalytics +Disallow:/ +User-agent: polaris version +Disallow:/ +User-agent: probe-image-size +Disallow:/ +User-agent: ripz +Disallow:/ +User-agent: s1z.ru +Disallow:/ +User-agent: satoristudio.net +Disallow:/ +User-agent: scalaj-http +Disallow:/ +User-agent: scan.lol +Disallow:/ +User-agent: seobility +Disallow:/ +User-agent: seoscanners +Disallow:/ +User-agent: seostar +Disallow:/ +User-agent: serpstatbot +Disallow:/ +User-agent: sexsearcher +Disallow:/ +User-agent: sitechecker.pro +Disallow:/ +User-agent: siteripz +Disallow:/ +User-agent: sogouspider +Disallow:/ +User-agent: sp_auditbot +Disallow:/ +User-agent: spyfu +Disallow:/ +User-agent: sysscan +Disallow:/ +User-agent: tAkeOut +Disallow:/ +User-agent: trendiction.com +Disallow:/ +User-agent: trendiction.de +Disallow:/ +User-agent: voyagerx.com +Disallow:/ +User-agent: webmeup-crawler +Disallow:/ +User-agent: x09Mozilla +Disallow:/ +User-agent: x22Mozilla +Disallow:/ +User-agent: xpymep1.exe +Disallow:/ +User-agent: zauba.io +Disallow:/ +User-agent: zgrab +Disallow:/ + diff --git a/robots.txt/robots.txt b/robots.txt/robots.txt index 08276c436..f01c456fd 100644 --- a/robots.txt/robots.txt +++ b/robots.txt/robots.txt @@ -2494,3 +2494,1251 @@ Disallow:/ User-agent: zgrab Disallow:/ +################################################################### +# The Ultimate robots.txt Bot and User-Agent Blocker +# Copyright: +# https://github.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker +################################################################### + +### Version Information # +################################################### +### Version: V4.2021.07.2453 +### Updated: Mon Jul 5 11:27:49 UTC 2021 +### Bad Bot Count: 615 +################################################### +### Version Information ## + +User-agent: * +Disallow: /wp-admin/ +Allow: /wp-admin/admin-ajax.php +User-agent: 01h4x.com +Disallow:/ +User-agent: 360Spider +Disallow:/ +User-agent: 404checker +Disallow:/ +User-agent: 404enemy +Disallow:/ +User-agent: 80legs +Disallow:/ +User-agent: ADmantX +Disallow:/ +User-agent: AIBOT +Disallow:/ +User-agent: ASPSeek +Disallow:/ +User-agent: Abonti +Disallow:/ +User-agent: Aboundex +Disallow:/ +User-agent: Aboundexbot +Disallow:/ +User-agent: Acunetix +Disallow:/ +User-agent: AfD-Verbotsverfahren +Disallow:/ +User-agent: AhrefsBot +Disallow:/ +User-agent: AiHitBot +Disallow:/ +User-agent: Aipbot +Disallow:/ +User-agent: Alexibot +Disallow:/ +User-agent: AllSubmitter +Disallow:/ +User-agent: Alligator +Disallow:/ +User-agent: AlphaBot +Disallow:/ +User-agent: Anarchie +Disallow:/ +User-agent: Anarchy +Disallow:/ +User-agent: Anarchy99 +Disallow:/ +User-agent: Ankit +Disallow:/ +User-agent: Anthill +Disallow:/ +User-agent: Apexoo +Disallow:/ +User-agent: Aspiegel +Disallow:/ +User-agent: Asterias +Disallow:/ +User-agent: Attach +Disallow:/ +User-agent: AwarioRssBot +Disallow:/ +User-agent: AwarioSmartBot +Disallow:/ +User-agent: BBBike +Disallow:/ +User-agent: BDCbot +Disallow:/ +User-agent: BDFetch +Disallow:/ +User-agent: BLEXBot +Disallow:/ +User-agent: BackDoorBot +Disallow:/ +User-agent: BackStreet +Disallow:/ +User-agent: BackWeb +Disallow:/ +User-agent: Backlink-Ceck +Disallow:/ +User-agent: BacklinkCrawler +Disallow:/ +User-agent: Badass +Disallow:/ +User-agent: Bandit +Disallow:/ +User-agent: Barkrowler +Disallow:/ +User-agent: BatchFTP +Disallow:/ +User-agent: Battleztar Bazinga +Disallow:/ +User-agent: BetaBot +Disallow:/ +User-agent: Bigfoot +Disallow:/ +User-agent: Bitacle +Disallow:/ +User-agent: BlackWidow +Disallow:/ +User-agent: Black Hole +Disallow:/ +User-agent: Blackboard +Disallow:/ +User-agent: Blow +Disallow:/ +User-agent: BlowFish +Disallow:/ +User-agent: Boardreader +Disallow:/ +User-agent: Bolt +Disallow:/ +User-agent: BotALot +Disallow:/ +User-agent: Brandprotect +Disallow:/ +User-agent: Brandwatch +Disallow:/ +User-agent: Buck +Disallow:/ +User-agent: Buddy +Disallow:/ +User-agent: BuiltBotTough +Disallow:/ +User-agent: BuiltWith +Disallow:/ +User-agent: Bullseye +Disallow:/ +User-agent: BunnySlippers +Disallow:/ +User-agent: BuzzSumo +Disallow:/ +User-agent: CATExplorador +Disallow:/ +User-agent: CCBot +Disallow:/ +User-agent: CODE87 +Disallow:/ +User-agent: CSHttp +Disallow:/ +User-agent: Calculon +Disallow:/ +User-agent: CazoodleBot +Disallow:/ +User-agent: Cegbfeieh +Disallow:/ +User-agent: CensysInspect +Disallow:/ +User-agent: CheTeam +Disallow:/ +User-agent: CheeseBot +Disallow:/ +User-agent: CherryPicker +Disallow:/ +User-agent: ChinaClaw +Disallow:/ +User-agent: Chlooe +Disallow:/ +User-agent: Claritybot +Disallow:/ +User-agent: Cliqzbot +Disallow:/ +User-agent: Cloud mapping +Disallow:/ +User-agent: Cocolyzebot +Disallow:/ +User-agent: Cogentbot +Disallow:/ +User-agent: Collector +Disallow:/ +User-agent: Copier +Disallow:/ +User-agent: CopyRightCheck +Disallow:/ +User-agent: Copyscape +Disallow:/ +User-agent: Cosmos +Disallow:/ +User-agent: Craftbot +Disallow:/ +User-agent: CrazyWebCrawler +Disallow:/ +User-agent: Crescent +Disallow:/ +User-agent: CrunchBot +Disallow:/ +User-agent: Curious +Disallow:/ +User-agent: Custo +Disallow:/ +User-agent: CyotekWebCopy +Disallow:/ +User-agent: DBLBot +Disallow:/ +User-agent: DIIbot +Disallow:/ +User-agent: DSearch +Disallow:/ +User-agent: DTS Agent +Disallow:/ +User-agent: DataCha0s +Disallow:/ +User-agent: DatabaseDriverMysqli +Disallow:/ +User-agent: Demon +Disallow:/ +User-agent: Deusu +Disallow:/ +User-agent: Devil +Disallow:/ +User-agent: Digincore +Disallow:/ +User-agent: DigitalPebble +Disallow:/ +User-agent: Dirbuster +Disallow:/ +User-agent: Disco +Disallow:/ +User-agent: Discobot +Disallow:/ +User-agent: Discoverybot +Disallow:/ +User-agent: Dispatch +Disallow:/ +User-agent: DittoSpyder +Disallow:/ +User-agent: DnyzBot +Disallow:/ +User-agent: DomCopBot +Disallow:/ +User-agent: DomainAppender +Disallow:/ +User-agent: DomainCrawler +Disallow:/ +User-agent: DomainSigmaCrawler +Disallow:/ +User-agent: DomainStatsBot +Disallow:/ +User-agent: Domains Project +Disallow:/ +User-agent: Dotbot +Disallow:/ +User-agent: Download Wonder +Disallow:/ +User-agent: Dragonfly +Disallow:/ +User-agent: Drip +Disallow:/ +User-agent: ECCP/1.0 +Disallow:/ +User-agent: EMail Siphon +Disallow:/ +User-agent: EMail Wolf +Disallow:/ +User-agent: EasyDL +Disallow:/ +User-agent: Ebingbong +Disallow:/ +User-agent: Ecxi +Disallow:/ +User-agent: EirGrabber +Disallow:/ +User-agent: EroCrawler +Disallow:/ +User-agent: Evil +Disallow:/ +User-agent: Exabot +Disallow:/ +User-agent: Express WebPictures +Disallow:/ +User-agent: ExtLinksBot +Disallow:/ +User-agent: Extractor +Disallow:/ +User-agent: ExtractorPro +Disallow:/ +User-agent: Extreme Picture Finder +Disallow:/ +User-agent: EyeNetIE +Disallow:/ +User-agent: Ezooms +Disallow:/ +User-agent: FDM +Disallow:/ +User-agent: FHscan +Disallow:/ +User-agent: FemtosearchBot +Disallow:/ +User-agent: Fimap +Disallow:/ +User-agent: Firefox/7.0 +Disallow:/ +User-agent: FlashGet +Disallow:/ +User-agent: Flunky +Disallow:/ +User-agent: Foobot +Disallow:/ +User-agent: Freeuploader +Disallow:/ +User-agent: FrontPage +Disallow:/ +User-agent: Fuzz +Disallow:/ +User-agent: FyberSpider +Disallow:/ +User-agent: Fyrebot +Disallow:/ +User-agent: G-i-g-a-b-o-t +Disallow:/ +User-agent: GT::WWW +Disallow:/ +User-agent: GalaxyBot +Disallow:/ +User-agent: Genieo +Disallow:/ +User-agent: GermCrawler +Disallow:/ +User-agent: GetRight +Disallow:/ +User-agent: GetWeb +Disallow:/ +User-agent: Getintent +Disallow:/ +User-agent: Gigablast +Disallow:/ +User-agent: Gigabot +Disallow:/ +User-agent: Go!Zilla +Disallow:/ +User-agent: Go-Ahead-Got-It +Disallow:/ +User-agent: GoZilla +Disallow:/ +User-agent: Gotit +Disallow:/ +User-agent: GrabNet +Disallow:/ +User-agent: Grabber +Disallow:/ +User-agent: Grafula +Disallow:/ +User-agent: GrapeFX +Disallow:/ +User-agent: GrapeshotCrawler +Disallow:/ +User-agent: GridBot +Disallow:/ +User-agent: HEADMasterSEO +Disallow:/ +User-agent: HMView +Disallow:/ +User-agent: HTMLparser +Disallow:/ +User-agent: HTTP::Lite +Disallow:/ +User-agent: HTTrack +Disallow:/ +User-agent: Haansoft +Disallow:/ +User-agent: HaosouSpider +Disallow:/ +User-agent: Harvest +Disallow:/ +User-agent: Havij +Disallow:/ +User-agent: Heritrix +Disallow:/ +User-agent: Hloader +Disallow:/ +User-agent: Humanlinks +Disallow:/ +User-agent: HybridBot +Disallow:/ +User-agent: IDBTE4M +Disallow:/ +User-agent: IDBot +Disallow:/ +User-agent: IRLbot +Disallow:/ +User-agent: Iblog +Disallow:/ +User-agent: Id-search +Disallow:/ +User-agent: IlseBot +Disallow:/ +User-agent: Image Fetch +Disallow:/ +User-agent: Image Sucker +Disallow:/ +User-agent: IndeedBot +Disallow:/ +User-agent: Indy Library +Disallow:/ +User-agent: InfoNaviRobot +Disallow:/ +User-agent: InfoTekies +Disallow:/ +User-agent: Intelliseek +Disallow:/ +User-agent: InterGET +Disallow:/ +User-agent: InternetSeer +Disallow:/ +User-agent: Internet Ninja +Disallow:/ +User-agent: Iria +Disallow:/ +User-agent: Iskanie +Disallow:/ +User-agent: IstellaBot +Disallow:/ +User-agent: JOC Web Spider +Disallow:/ +User-agent: JamesBOT +Disallow:/ +User-agent: Jbrofuzz +Disallow:/ +User-agent: JennyBot +Disallow:/ +User-agent: JetCar +Disallow:/ +User-agent: Jetty +Disallow:/ +User-agent: JikeSpider +Disallow:/ +User-agent: Joomla +Disallow:/ +User-agent: Jorgee +Disallow:/ +User-agent: JustView +Disallow:/ +User-agent: Jyxobot +Disallow:/ +User-agent: Kenjin Spider +Disallow:/ +User-agent: Keyword Density +Disallow:/ +User-agent: Kinza +Disallow:/ +User-agent: Kozmosbot +Disallow:/ +User-agent: LNSpiderguy +Disallow:/ +User-agent: LWP::Simple +Disallow:/ +User-agent: Lanshanbot +Disallow:/ +User-agent: Larbin +Disallow:/ +User-agent: LeechFTP +Disallow:/ +User-agent: LeechGet +Disallow:/ +User-agent: LexiBot +Disallow:/ +User-agent: Lftp +Disallow:/ +User-agent: LibWeb +Disallow:/ +User-agent: Libwhisker +Disallow:/ +User-agent: LieBaoFast +Disallow:/ +User-agent: Lightspeedsystems +Disallow:/ +User-agent: Likse +Disallow:/ +User-agent: LinkScan +Disallow:/ +User-agent: LinkWalker +Disallow:/ +User-agent: Linkbot +Disallow:/ +User-agent: Linkdexbot +Disallow:/ +User-agent: LinkextractorPro +Disallow:/ +User-agent: LinkpadBot +Disallow:/ +User-agent: LinksManager +Disallow:/ +User-agent: LinqiaMetadataDownloaderBot +Disallow:/ +User-agent: LinqiaRSSBot +Disallow:/ +User-agent: LinqiaScrapeBot +Disallow:/ +User-agent: Lipperhey +Disallow:/ +User-agent: Lipperhey Spider +Disallow:/ +User-agent: Litemage_walker +Disallow:/ +User-agent: Lmspider +Disallow:/ +User-agent: Ltx71 +Disallow:/ +User-agent: MFC_Tear_Sample +Disallow:/ +User-agent: MIDown tool +Disallow:/ +User-agent: MIIxpc +Disallow:/ +User-agent: MJ12bot +Disallow:/ +User-agent: MQQBrowser +Disallow:/ +User-agent: MSFrontPage +Disallow:/ +User-agent: MSIECrawler +Disallow:/ +User-agent: MTRobot +Disallow:/ +User-agent: Mag-Net +Disallow:/ +User-agent: Magnet +Disallow:/ +User-agent: Mail.RU_Bot +Disallow:/ +User-agent: Majestic-SEO +Disallow:/ +User-agent: Majestic12 +Disallow:/ +User-agent: Majestic SEO +Disallow:/ +User-agent: MarkMonitor +Disallow:/ +User-agent: MarkWatch +Disallow:/ +User-agent: Mass Downloader +Disallow:/ +User-agent: Masscan +Disallow:/ +User-agent: Mata Hari +Disallow:/ +User-agent: MauiBot +Disallow:/ +User-agent: Mb2345Browser +Disallow:/ +User-agent: MeanPath Bot +Disallow:/ +User-agent: Meanpathbot +Disallow:/ +User-agent: Mediatoolkitbot +Disallow:/ +User-agent: MegaIndex.ru +Disallow:/ +User-agent: Metauri +Disallow:/ +User-agent: MicroMessenger +Disallow:/ +User-agent: Microsoft Data Access +Disallow:/ +User-agent: Microsoft URL Control +Disallow:/ +User-agent: Mister PiX +Disallow:/ +User-agent: Moblie Safari +Disallow:/ +User-agent: Mojeek +Disallow:/ +User-agent: Mojolicious +Disallow:/ +User-agent: Morfeus Fucking Scanner +Disallow:/ +User-agent: Mozlila +Disallow:/ +User-agent: Mr.4x3 +Disallow:/ +User-agent: Msrabot +Disallow:/ +User-agent: Musobot +Disallow:/ +User-agent: NICErsPRO +Disallow:/ +User-agent: NPbot +Disallow:/ +User-agent: Name Intelligence +Disallow:/ +User-agent: Nameprotect +Disallow:/ +User-agent: Navroad +Disallow:/ +User-agent: NearSite +Disallow:/ +User-agent: Needle +Disallow:/ +User-agent: Nessus +Disallow:/ +User-agent: NetAnts +Disallow:/ +User-agent: NetLyzer +Disallow:/ +User-agent: NetMechanic +Disallow:/ +User-agent: NetSpider +Disallow:/ +User-agent: NetZIP +Disallow:/ +User-agent: Net Vampire +Disallow:/ +User-agent: Netcraft +Disallow:/ +User-agent: Nettrack +Disallow:/ +User-agent: Netvibes +Disallow:/ +User-agent: NextGenSearchBot +Disallow:/ +User-agent: Nibbler +Disallow:/ +User-agent: Niki-bot +Disallow:/ +User-agent: Nikto +Disallow:/ +User-agent: NimbleCrawler +Disallow:/ +User-agent: Nimbostratus +Disallow:/ +User-agent: Ninja +Disallow:/ +User-agent: Nmap +Disallow:/ +User-agent: Nuclei +Disallow:/ +User-agent: Nutch +Disallow:/ +User-agent: OPPO A33 +Disallow:/ +User-agent: Octopus +Disallow:/ +User-agent: Offline Explorer +Disallow:/ +User-agent: Offline Navigator +Disallow:/ +User-agent: OnCrawl +Disallow:/ +User-agent: OpenLinkProfiler +Disallow:/ +User-agent: OpenVAS +Disallow:/ +User-agent: Openfind +Disallow:/ +User-agent: Openvas +Disallow:/ +User-agent: OrangeBot +Disallow:/ +User-agent: OrangeSpider +Disallow:/ +User-agent: OutclicksBot +Disallow:/ +User-agent: OutfoxBot +Disallow:/ +User-agent: PECL::HTTP +Disallow:/ +User-agent: PHPCrawl +Disallow:/ +User-agent: POE-Component-Client-HTTP +Disallow:/ +User-agent: PageAnalyzer +Disallow:/ +User-agent: PageGrabber +Disallow:/ +User-agent: PageScorer +Disallow:/ +User-agent: PageThing.com +Disallow:/ +User-agent: Page Analyzer +Disallow:/ +User-agent: Pandalytics +Disallow:/ +User-agent: Panscient +Disallow:/ +User-agent: Papa Foto +Disallow:/ +User-agent: Pavuk +Disallow:/ +User-agent: PeoplePal +Disallow:/ +User-agent: Petalbot +Disallow:/ +User-agent: Pi-Monster +Disallow:/ +User-agent: Picscout +Disallow:/ +User-agent: Picsearch +Disallow:/ +User-agent: PictureFinder +Disallow:/ +User-agent: Piepmatz +Disallow:/ +User-agent: Pimonster +Disallow:/ +User-agent: Pixray +Disallow:/ +User-agent: PleaseCrawl +Disallow:/ +User-agent: Pockey +Disallow:/ +User-agent: ProPowerBot +Disallow:/ +User-agent: ProWebWalker +Disallow:/ +User-agent: Probethenet +Disallow:/ +User-agent: Psbot +Disallow:/ +User-agent: Pu_iN +Disallow:/ +User-agent: Pump +Disallow:/ +User-agent: PxBroker +Disallow:/ +User-agent: PyCurl +Disallow:/ +User-agent: QueryN Metasearch +Disallow:/ +User-agent: Quick-Crawler +Disallow:/ +User-agent: RSSingBot +Disallow:/ +User-agent: RankActive +Disallow:/ +User-agent: RankActiveLinkBot +Disallow:/ +User-agent: RankFlex +Disallow:/ +User-agent: RankingBot +Disallow:/ +User-agent: RankingBot2 +Disallow:/ +User-agent: Rankivabot +Disallow:/ +User-agent: RankurBot +Disallow:/ +User-agent: Re-re +Disallow:/ +User-agent: ReGet +Disallow:/ +User-agent: RealDownload +Disallow:/ +User-agent: Reaper +Disallow:/ +User-agent: RebelMouse +Disallow:/ +User-agent: Recorder +Disallow:/ +User-agent: RedesScrapy +Disallow:/ +User-agent: RepoMonkey +Disallow:/ +User-agent: Ripper +Disallow:/ +User-agent: RocketCrawler +Disallow:/ +User-agent: Rogerbot +Disallow:/ +User-agent: SBIder +Disallow:/ +User-agent: SEOkicks +Disallow:/ +User-agent: SEOkicks-Robot +Disallow:/ +User-agent: SEOlyticsCrawler +Disallow:/ +User-agent: SEOprofiler +Disallow:/ +User-agent: SEOstats +Disallow:/ +User-agent: SISTRIX +Disallow:/ +User-agent: SMTBot +Disallow:/ +User-agent: SalesIntelligent +Disallow:/ +User-agent: ScanAlert +Disallow:/ +User-agent: Scanbot +Disallow:/ +User-agent: ScoutJet +Disallow:/ +User-agent: Scrapy +Disallow:/ +User-agent: Screaming +Disallow:/ +User-agent: ScreenerBot +Disallow:/ +User-agent: ScrepyBot +Disallow:/ +User-agent: Searchestate +Disallow:/ +User-agent: SearchmetricsBot +Disallow:/ +User-agent: Semrush +Disallow:/ +User-agent: SemrushBot +Disallow:/ +User-agent: SentiBot +Disallow:/ +User-agent: SeoSiteCheckup +Disallow:/ +User-agent: SeobilityBot +Disallow:/ +User-agent: Seomoz +Disallow:/ +User-agent: Shodan +Disallow:/ +User-agent: Siphon +Disallow:/ +User-agent: SiteCheckerBotCrawler +Disallow:/ +User-agent: SiteExplorer +Disallow:/ +User-agent: SiteLockSpider +Disallow:/ +User-agent: SiteSnagger +Disallow:/ +User-agent: SiteSucker +Disallow:/ +User-agent: Site Sucker +Disallow:/ +User-agent: Sitebeam +Disallow:/ +User-agent: Siteimprove +Disallow:/ +User-agent: Sitevigil +Disallow:/ +User-agent: SlySearch +Disallow:/ +User-agent: SmartDownload +Disallow:/ +User-agent: Snake +Disallow:/ +User-agent: Snapbot +Disallow:/ +User-agent: Snoopy +Disallow:/ +User-agent: SocialRankIOBot +Disallow:/ +User-agent: Sociscraper +Disallow:/ +User-agent: Sogou web spider +Disallow:/ +User-agent: Sosospider +Disallow:/ +User-agent: Sottopop +Disallow:/ +User-agent: SpaceBison +Disallow:/ +User-agent: Spammen +Disallow:/ +User-agent: SpankBot +Disallow:/ +User-agent: Spanner +Disallow:/ +User-agent: Spbot +Disallow:/ +User-agent: Spinn3r +Disallow:/ +User-agent: SputnikBot +Disallow:/ +User-agent: Sqlmap +Disallow:/ +User-agent: Sqlworm +Disallow:/ +User-agent: Sqworm +Disallow:/ +User-agent: Steeler +Disallow:/ +User-agent: Stripper +Disallow:/ +User-agent: Sucker +Disallow:/ +User-agent: Sucuri +Disallow:/ +User-agent: SuperBot +Disallow:/ +User-agent: SuperHTTP +Disallow:/ +User-agent: Surfbot +Disallow:/ +User-agent: SurveyBot +Disallow:/ +User-agent: Suzuran +Disallow:/ +User-agent: Swiftbot +Disallow:/ +User-agent: Szukacz +Disallow:/ +User-agent: T0PHackTeam +Disallow:/ +User-agent: T8Abot +Disallow:/ +User-agent: Teleport +Disallow:/ +User-agent: TeleportPro +Disallow:/ +User-agent: Telesoft +Disallow:/ +User-agent: Telesphoreo +Disallow:/ +User-agent: Telesphorep +Disallow:/ +User-agent: TheNomad +Disallow:/ +User-agent: The Intraformant +Disallow:/ +User-agent: Thumbor +Disallow:/ +User-agent: TightTwatBot +Disallow:/ +User-agent: Titan +Disallow:/ +User-agent: Toata +Disallow:/ +User-agent: Toweyabot +Disallow:/ +User-agent: Tracemyfile +Disallow:/ +User-agent: Trendiction +Disallow:/ +User-agent: Trendictionbot +Disallow:/ +User-agent: True_Robot +Disallow:/ +User-agent: Turingos +Disallow:/ +User-agent: Turnitin +Disallow:/ +User-agent: TurnitinBot +Disallow:/ +User-agent: TwengaBot +Disallow:/ +User-agent: Twice +Disallow:/ +User-agent: Typhoeus +Disallow:/ +User-agent: URLy.Warning +Disallow:/ +User-agent: URLy Warning +Disallow:/ +User-agent: UnisterBot +Disallow:/ +User-agent: Upflow +Disallow:/ +User-agent: V-BOT +Disallow:/ +User-agent: VB Project +Disallow:/ +User-agent: VCI +Disallow:/ +User-agent: Vacuum +Disallow:/ +User-agent: Vagabondo +Disallow:/ +User-agent: VelenPublicWebCrawler +Disallow:/ +User-agent: VeriCiteCrawler +Disallow:/ +User-agent: VidibleScraper +Disallow:/ +User-agent: Virusdie +Disallow:/ +User-agent: VoidEYE +Disallow:/ +User-agent: Voil +Disallow:/ +User-agent: Voltron +Disallow:/ +User-agent: WASALive-Bot +Disallow:/ +User-agent: WBSearchBot +Disallow:/ +User-agent: WEBDAV +Disallow:/ +User-agent: WISENutbot +Disallow:/ +User-agent: WPScan +Disallow:/ +User-agent: WWW-Collector-E +Disallow:/ +User-agent: WWW-Mechanize +Disallow:/ +User-agent: WWW::Mechanize +Disallow:/ +User-agent: WWWOFFLE +Disallow:/ +User-agent: Wallpapers/3.0 +Disallow:/ +User-agent: WallpapersHD +Disallow:/ +User-agent: WeSEE +Disallow:/ +User-agent: WebAuto +Disallow:/ +User-agent: WebBandit +Disallow:/ +User-agent: WebCollage +Disallow:/ +User-agent: WebCopier +Disallow:/ +User-agent: WebEnhancer +Disallow:/ +User-agent: WebFetch +Disallow:/ +User-agent: WebFuck +Disallow:/ +User-agent: WebGo IS +Disallow:/ +User-agent: WebImageCollector +Disallow:/ +User-agent: WebLeacher +Disallow:/ +User-agent: WebPix +Disallow:/ +User-agent: WebReaper +Disallow:/ +User-agent: WebSauger +Disallow:/ +User-agent: WebStripper +Disallow:/ +User-agent: WebSucker +Disallow:/ +User-agent: WebWhacker +Disallow:/ +User-agent: WebZIP +Disallow:/ +User-agent: Web Auto +Disallow:/ +User-agent: Web Collage +Disallow:/ +User-agent: Web Enhancer +Disallow:/ +User-agent: Web Fetch +Disallow:/ +User-agent: Web Fuck +Disallow:/ +User-agent: Web Pix +Disallow:/ +User-agent: Web Sauger +Disallow:/ +User-agent: Web Sucker +Disallow:/ +User-agent: Webalta +Disallow:/ +User-agent: WebmasterWorldForumBot +Disallow:/ +User-agent: Webshag +Disallow:/ +User-agent: WebsiteExtractor +Disallow:/ +User-agent: WebsiteQuester +Disallow:/ +User-agent: Website Quester +Disallow:/ +User-agent: Webster +Disallow:/ +User-agent: Whack +Disallow:/ +User-agent: Whacker +Disallow:/ +User-agent: Whatweb +Disallow:/ +User-agent: Who.is Bot +Disallow:/ +User-agent: Widow +Disallow:/ +User-agent: WinHTTrack +Disallow:/ +User-agent: WiseGuys Robot +Disallow:/ +User-agent: Wonderbot +Disallow:/ +User-agent: Woobot +Disallow:/ +User-agent: Wotbox +Disallow:/ +User-agent: Wprecon +Disallow:/ +User-agent: Xaldon WebSpider +Disallow:/ +User-agent: Xaldon_WebSpider +Disallow:/ +User-agent: Xenu +Disallow:/ +User-agent: YoudaoBot +Disallow:/ +User-agent: Zade +Disallow:/ +User-agent: Zauba +Disallow:/ +User-agent: Zermelo +Disallow:/ +User-agent: Zeus +Disallow:/ +User-agent: Zitebot +Disallow:/ +User-agent: ZmEu +Disallow:/ +User-agent: ZoomBot +Disallow:/ +User-agent: ZoominfoBot +Disallow:/ +User-agent: ZumBot +Disallow:/ +User-agent: ZyBorg +Disallow:/ +User-agent: archive.org_bot +Disallow:/ +User-agent: arquivo-web-crawler +Disallow:/ +User-agent: arquivo.pt +Disallow:/ +User-agent: autoemailspider +Disallow:/ +User-agent: backlink-check +Disallow:/ +User-agent: check1.exe +Disallow:/ +User-agent: coccocbot-web +Disallow:/ +User-agent: cognitiveseo +Disallow:/ +User-agent: com.plumanalytics +Disallow:/ +User-agent: crawl.sogou.com +Disallow:/ +User-agent: crawler.feedback +Disallow:/ +User-agent: crawler4j +Disallow:/ +User-agent: demandbase-bot +Disallow:/ +User-agent: domainsproject.org +Disallow:/ +User-agent: eCatch +Disallow:/ +User-agent: evc-batch +Disallow:/ +User-agent: facebookscraper +Disallow:/ +User-agent: gopher +Disallow:/ +User-agent: heritrix +Disallow:/ +User-agent: instabid +Disallow:/ +User-agent: internetVista monitor +Disallow:/ +User-agent: ips-agent +Disallow:/ +User-agent: isitwp.com +Disallow:/ +User-agent: lwp-request +Disallow:/ +User-agent: lwp-trivial +Disallow:/ +User-agent: magpie-crawler +Disallow:/ +User-agent: meanpathbot +Disallow:/ +User-agent: mediawords +Disallow:/ +User-agent: muhstik-scan +Disallow:/ +User-agent: netEstate NE Crawler +Disallow:/ +User-agent: oBot +Disallow:/ +User-agent: page scorer +Disallow:/ +User-agent: pcBrowser +Disallow:/ +User-agent: plumanalytics +Disallow:/ +User-agent: polaris version +Disallow:/ +User-agent: probe-image-size +Disallow:/ +User-agent: ripz +Disallow:/ +User-agent: s1z.ru +Disallow:/ +User-agent: satoristudio.net +Disallow:/ +User-agent: scalaj-http +Disallow:/ +User-agent: scan.lol +Disallow:/ +User-agent: seobility +Disallow:/ +User-agent: seoscanners +Disallow:/ +User-agent: seostar +Disallow:/ +User-agent: serpstatbot +Disallow:/ +User-agent: sexsearcher +Disallow:/ +User-agent: sitechecker.pro +Disallow:/ +User-agent: siteripz +Disallow:/ +User-agent: sogouspider +Disallow:/ +User-agent: sp_auditbot +Disallow:/ +User-agent: spyfu +Disallow:/ +User-agent: sysscan +Disallow:/ +User-agent: tAkeOut +Disallow:/ +User-agent: trendiction.com +Disallow:/ +User-agent: trendiction.de +Disallow:/ +User-agent: voyagerx.com +Disallow:/ +User-agent: webmeup-crawler +Disallow:/ +User-agent: x09Mozilla +Disallow:/ +User-agent: x22Mozilla +Disallow:/ +User-agent: xpymep1.exe +Disallow:/ +User-agent: zauba.io +Disallow:/ +User-agent: zgrab +Disallow:/ +