vikarti.anatra/nginx-ultimate-bad-bot-blocker
1
0
Fork 0
mirror of https://github.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker.git synced 2025-09-15 17:49:49 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

V4.2021.07.2451 [ci skip]

Browse source
This commit is contained in:
mitchellkrogza 2021-07-05 11:05:03 +00:00
parent 606872502d
commit 41967724e3
77 changed files with 123172 additions and 122224 deletions
Download patch file Download diff file Expand all files Collapse all files

2
dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf
View file

@ -1,2 +1,2 @@
34.66.25.221 1;
10.1.0.59 1;
127.0.0.1 1;

368
dev-tools/beta_conf_files_whitelist/bots.d/blacklist-user-agents.conf
View file

@ -64,19 +64,20 @@
"~*(?:\b)404checker(?:\b|)" 0;
"~*(?:\b)404enemy(?:\b|)" 0;
"~*(?:\b)80legs(?:\b|)" 0;
"~*(?:\b)ADmantX(?:\b|)" 0;
"~*(?:\b)AIBOT(?:\b|)" 0;
"~*(?:\b)ASPSeek(?:\b|)" 0;
"~*(?:\b)Abonti(?:\b|)" 0;
"~*(?:\b)Aboundex(?:\b|)" 0;
"~*(?:\b)Aboundexbot(?:\b|)" 0;
"~*(?:\b)Acunetix(?:\b|)" 0;
"~*(?:\b)ADmantX(?:\b|)" 0;
"~*(?:\b)AfD-Verbotsverfahren(?:\b|)" 0;
"~*(?:\b)AhrefsBot(?:\b|)" 0;
"~*(?:\b)AIBOT(?:\b|)" 0;
"~*(?:\b)AiHitBot(?:\b|)" 0;
"~*(?:\b)Aipbot(?:\b|)" 0;
"~*(?:\b)Alexibot(?:\b|)" 0;
"~*(?:\b)Alligator(?:\b|)" 0;
"~*(?:\b)AllSubmitter(?:\b|)" 0;
"~*(?:\b)Alligator(?:\b|)" 0;
"~*(?:\b)AlphaBot(?:\b|)" 0;
"~*(?:\b)Anarchie(?:\b|)" 0;
"~*(?:\b)Anarchy(?:\b|)" 0;
@ -84,37 +85,31 @@
"~*(?:\b)Ankit(?:\b|)" 0;
"~*(?:\b)Anthill(?:\b|)" 0;
"~*(?:\b)Apexoo(?:\b|)" 0;
"~*(?:\b)archive.org_bot(?:\b|)" 0;
"~*(?:\b)arquivo.pt(?:\b|)" 0;
"~*(?:\b)arquivo-web-crawler(?:\b|)" 0;
"~*(?:\b)Aspiegel(?:\b|)" 0;
"~*(?:\b)ASPSeek(?:\b|)" 0;
"~*(?:\b)Asterias(?:\b|)" 0;
"~*(?:\b)Attach(?:\b|)" 0;
"~*(?:\b)autoemailspider(?:\b|)" 0;
"~*(?:\b)AwarioRssBot(?:\b|)" 0;
"~*(?:\b)AwarioSmartBot(?:\b|)" 0;
"~*(?:\b)BBBike(?:\b|)" 0;
"~*(?:\b)BDCbot(?:\b|)" 0;
"~*(?:\b)BDFetch(?:\b|)" 0;
"~*(?:\b)BLEXBot(?:\b|)" 0;
"~*(?:\b)BackDoorBot(?:\b|)" 0;
"~*(?:\b)Backlink-Ceck(?:\b|)" 0;
"~*(?:\b)backlink-check(?:\b|)" 0;
"~*(?:\b)BacklinkCrawler(?:\b|)" 0;
"~*(?:\b)BackStreet(?:\b|)" 0;
"~*(?:\b)BackWeb(?:\b|)" 0;
"~*(?:\b)Backlink-Ceck(?:\b|)" 0;
"~*(?:\b)BacklinkCrawler(?:\b|)" 0;
"~*(?:\b)Badass(?:\b|)" 0;
"~*(?:\b)Bandit(?:\b|)" 0;
"~*(?:\b)Barkrowler(?:\b|)" 0;
"~*(?:\b)BatchFTP(?:\b|)" 0;
"~*(?:\b)Battleztar\ Bazinga(?:\b|)" 0;
"~*(?:\b)BBBike(?:\b|)" 0;
"~*(?:\b)BDCbot(?:\b|)" 0;
"~*(?:\b)BDFetch(?:\b|)" 0;
"~*(?:\b)BetaBot(?:\b|)" 0;
"~*(?:\b)Bigfoot(?:\b|)" 0;
"~*(?:\b)Bitacle(?:\b|)" 0;
"~*(?:\b)Blackboard(?:\b|)" 0;
"~*(?:\b)Black\ Hole(?:\b|)" 0;
"~*(?:\b)BlackWidow(?:\b|)" 0;
"~*(?:\b)BLEXBot(?:\b|)" 0;
"~*(?:\b)Black\ Hole(?:\b|)" 0;
"~*(?:\b)Blackboard(?:\b|)" 0;
"~*(?:\b)Blow(?:\b|)" 0;
"~*(?:\b)BlowFish(?:\b|)" 0;
"~*(?:\b)Boardreader(?:\b|)" 0;
@ -129,53 +124,47 @@
"~*(?:\b)Bullseye(?:\b|)" 0;
"~*(?:\b)BunnySlippers(?:\b|)" 0;
"~*(?:\b)BuzzSumo(?:\b|)" 0;
"~*(?:\b)Calculon(?:\b|)" 0;
"~*(?:\b)CATExplorador(?:\b|)" 0;
"~*(?:\b)CazoodleBot(?:\b|)" 0;
"~*(?:\b)CCBot(?:\b|)" 0;
"~*(?:\b)CODE87(?:\b|)" 0;
"~*(?:\b)CSHttp(?:\b|)" 0;
"~*(?:\b)Calculon(?:\b|)" 0;
"~*(?:\b)CazoodleBot(?:\b|)" 0;
"~*(?:\b)Cegbfeieh(?:\b|)" 0;
"~*(?:\b)CensysInspect(?:\b|)" 0;
"~*(?:\b)check1.exe(?:\b|)" 0;
"~*(?:\b)CheTeam(?:\b|)" 0;
"~*(?:\b)CheeseBot(?:\b|)" 0;
"~*(?:\b)CherryPicker(?:\b|)" 0;
"~*(?:\b)CheTeam(?:\b|)" 0;
"~*(?:\b)ChinaClaw(?:\b|)" 0;
"~*(?:\b)Chlooe(?:\b|)" 0;
"~*(?:\b)Claritybot(?:\b|)" 0;
"~*(?:\b)Cliqzbot(?:\b|)" 0;
"~*(?:\b)Cloud\ mapping(?:\b|)" 0;
"~*(?:\b)coccocbot-web(?:\b|)" 0;
"~*(?:\b)Cocolyzebot(?:\b|)" 0;
"~*(?:\b)CODE87(?:\b|)" 0;
"~*(?:\b)Cogentbot(?:\b|)" 0;
"~*(?:\b)cognitiveseo(?:\b|)" 0;
"~*(?:\b)Collector(?:\b|)" 0;
"~*(?:\b)com.plumanalytics(?:\b|)" 0;
"~*(?:\b)Copier(?:\b|)" 0;
"~*(?:\b)CopyRightCheck(?:\b|)" 0;
"~*(?:\b)Copyscape(?:\b|)" 0;
"~*(?:\b)Cosmos(?:\b|)" 0;
"~*(?:\b)Craftbot(?:\b|)" 0;
"~*(?:\b)crawler4j(?:\b|)" 0;
"~*(?:\b)crawler.feedback(?:\b|)" 0;
"~*(?:\b)crawl.sogou.com(?:\b|)" 0;
"~*(?:\b)CrazyWebCrawler(?:\b|)" 0;
"~*(?:\b)Crescent(?:\b|)" 0;
"~*(?:\b)CrunchBot(?:\b|)" 0;
"~*(?:\b)CSHttp(?:\b|)" 0;
"~*(?:\b)Curious(?:\b|)" 0;
"~*(?:\b)Custo(?:\b|)" 0;
"~*(?:\b)CyotekWebCopy(?:\b|)" 0;
"~*(?:\b)DatabaseDriverMysqli(?:\b|)" 0;
"~*(?:\b)DataCha0s(?:\b|)" 0;
"~*(?:\b)DBLBot(?:\b|)" 0;
"~*(?:\b)demandbase-bot(?:\b|)" 0;
"~*(?:\b)DIIbot(?:\b|)" 0;
"~*(?:\b)DSearch(?:\b|)" 0;
"~*(?:\b)DTS\ Agent(?:\b|)" 0;
"~*(?:\b)DataCha0s(?:\b|)" 0;
"~*(?:\b)DatabaseDriverMysqli(?:\b|)" 0;
"~*(?:\b)Demon(?:\b|)" 0;
"~*(?:\b)Deusu(?:\b|)" 0;
"~*(?:\b)Devil(?:\b|)" 0;
"~*(?:\b)Digincore(?:\b|)" 0;
"~*(?:\b)DigitalPebble(?:\b|)" 0;
"~*(?:\b)DIIbot(?:\b|)" 0;
"~*(?:\b)Dirbuster(?:\b|)" 0;
"~*(?:\b)Disco(?:\b|)" 0;
"~*(?:\b)Discobot(?:\b|)" 0;
@ -183,29 +172,24 @@
"~*(?:\b)Dispatch(?:\b|)" 0;
"~*(?:\b)DittoSpyder(?:\b|)" 0;
"~*(?:\b)DnyzBot(?:\b|)" 0;
"~*(?:\b)DomCopBot(?:\b|)" 0;
"~*(?:\b)DomainAppender(?:\b|)" 0;
"~*(?:\b)DomainCrawler(?:\b|)" 0;
"~*(?:\b)DomainSigmaCrawler(?:\b|)" 0;
"~*(?:\b)Domains\ Project(?:\b|)" 0;
"~*(?:\b)domainsproject.org(?:\b|)" 0;
"~*(?:\b)DomainStatsBot(?:\b|)" 0;
"~*(?:\b)DomCopBot(?:\b|)" 0;
"~*(?:\b)Domains\ Project(?:\b|)" 0;
"~*(?:\b)Dotbot(?:\b|)" 0;
"~*(?:\b)Download\ Wonder(?:\b|)" 0;
"~*(?:\b)Dragonfly(?:\b|)" 0;
"~*(?:\b)Drip(?:\b|)" 0;
"~*(?:\b)DSearch(?:\b|)" 0;
"~*(?:\b)DTS\ Agent(?:\b|)" 0;
"~*(?:\b)EasyDL(?:\b|)" 0;
"~*(?:\b)Ebingbong(?:\b|)" 0;
"~*(?:\b)eCatch(?:\b|)" 0;
"~*(?:\b)ECCP/1.0(?:\b|)" 0;
"~*(?:\b)Ecxi(?:\b|)" 0;
"~*(?:\b)EirGrabber(?:\b|)" 0;
"~*(?:\b)EMail\ Siphon(?:\b|)" 0;
"~*(?:\b)EMail\ Wolf(?:\b|)" 0;
"~*(?:\b)EasyDL(?:\b|)" 0;
"~*(?:\b)Ebingbong(?:\b|)" 0;
"~*(?:\b)Ecxi(?:\b|)" 0;
"~*(?:\b)EirGrabber(?:\b|)" 0;
"~*(?:\b)EroCrawler(?:\b|)" 0;
"~*(?:\b)evc-batch(?:\b|)" 0;
"~*(?:\b)Evil(?:\b|)" 0;
"~*(?:\b)Exabot(?:\b|)" 0;
"~*(?:\b)Express\ WebPictures(?:\b|)" 0;
@ -215,10 +199,9 @@
"~*(?:\b)Extreme\ Picture\ Finder(?:\b|)" 0;
"~*(?:\b)EyeNetIE(?:\b|)" 0;
"~*(?:\b)Ezooms(?:\b|)" 0;
"~*(?:\b)facebookscraper(?:\b|)" 0;
"~*(?:\b)FDM(?:\b|)" 0;
"~*(?:\b)FemtosearchBot(?:\b|)" 0;
"~*(?:\b)FHscan(?:\b|)" 0;
"~*(?:\b)FemtosearchBot(?:\b|)" 0;
"~*(?:\b)Fimap(?:\b|)" 0;
"~*(?:\b)Firefox/7.0(?:\b|)" 0;
"~*(?:\b)FlashGet(?:\b|)" 0;
@ -229,44 +212,43 @@
"~*(?:\b)Fuzz(?:\b|)" 0;
"~*(?:\b)FyberSpider(?:\b|)" 0;
"~*(?:\b)Fyrebot(?:\b|)" 0;
"~*(?:\b)G-i-g-a-b-o-t(?:\b|)" 0;
"~*(?:\b)GT::WWW(?:\b|)" 0;
"~*(?:\b)GalaxyBot(?:\b|)" 0;
"~*(?:\b)Genieo(?:\b|)" 0;
"~*(?:\b)GermCrawler(?:\b|)" 0;
"~*(?:\b)Getintent(?:\b|)" 0;
"~*(?:\b)GetRight(?:\b|)" 0;
"~*(?:\b)GetWeb(?:\b|)" 0;
"~*(?:\b)Getintent(?:\b|)" 0;
"~*(?:\b)Gigablast(?:\b|)" 0;
"~*(?:\b)Gigabot(?:\b|)" 0;
"~*(?:\b)G-i-g-a-b-o-t(?:\b|)" 0;
"~*(?:\b)Go-Ahead-Got-It(?:\b|)" 0;
"~*(?:\b)gopher(?:\b|)" 0;
"~*(?:\b)Gotit(?:\b|)" 0;
"~*(?:\b)GoZilla(?:\b|)" 0;
"~*(?:\b)Go!Zilla(?:\b|)" 0;
"~*(?:\b)Grabber(?:\b|)" 0;
"~*(?:\b)Go-Ahead-Got-It(?:\b|)" 0;
"~*(?:\b)GoZilla(?:\b|)" 0;
"~*(?:\b)Gotit(?:\b|)" 0;
"~*(?:\b)GrabNet(?:\b|)" 0;
"~*(?:\b)Grabber(?:\b|)" 0;
"~*(?:\b)Grafula(?:\b|)" 0;
"~*(?:\b)GrapeFX(?:\b|)" 0;
"~*(?:\b)GrapeshotCrawler(?:\b|)" 0;
"~*(?:\b)GridBot(?:\b|)" 0;
"~*(?:\b)GT::WWW(?:\b|)" 0;
"~*(?:\b)Haansoft(?:\b|)" 0;
"~*(?:\b)HaosouSpider(?:\b|)" 0;
"~*(?:\b)Harvest(?:\b|)" 0;
"~*(?:\b)Havij(?:\b|)" 0;
"~*(?:\b)HEADMasterSEO(?:\b|)" 0;
"~*(?:\b)heritrix(?:\b|)" 0;
"~*(?:\b)Heritrix(?:\b|)" 0;
"~*(?:\b)Hloader(?:\b|)" 0;
"~*(?:\b)HMView(?:\b|)" 0;
"~*(?:\b)HTMLparser(?:\b|)" 0;
"~*(?:\b)HTTP::Lite(?:\b|)" 0;
"~*(?:\b)HTTrack(?:\b|)" 0;
"~*(?:\b)Haansoft(?:\b|)" 0;
"~*(?:\b)HaosouSpider(?:\b|)" 0;
"~*(?:\b)Harvest(?:\b|)" 0;
"~*(?:\b)Havij(?:\b|)" 0;
"~*(?:\b)Heritrix(?:\b|)" 0;
"~*(?:\b)Hloader(?:\b|)" 0;
"~*(?:\b)Humanlinks(?:\b|)" 0;
"~*(?:\b)HybridBot(?:\b|)" 0;
"~*(?:\b)Iblog(?:\b|)" 0;
"~*(?:\b)IDBot(?:\b|)" 0;
"~*(?:\b)IDBTE4M(?:\b|)" 0;
"~*(?:\b)IDBot(?:\b|)" 0;
"~*(?:\b)IRLbot(?:\b|)" 0;
"~*(?:\b)Iblog(?:\b|)" 0;
"~*(?:\b)Id-search(?:\b|)" 0;
"~*(?:\b)IlseBot(?:\b|)" 0;
"~*(?:\b)Image\ Fetch(?:\b|)" 0;
@ -275,25 +257,20 @@
"~*(?:\b)Indy\ Library(?:\b|)" 0;
"~*(?:\b)InfoNaviRobot(?:\b|)" 0;
"~*(?:\b)InfoTekies(?:\b|)" 0;
"~*(?:\b)instabid(?:\b|)" 0;
"~*(?:\b)Intelliseek(?:\b|)" 0;
"~*(?:\b)InterGET(?:\b|)" 0;
"~*(?:\b)Internet\ Ninja(?:\b|)" 0;
"~*(?:\b)InternetSeer(?:\b|)" 0;
"~*(?:\b)internetVista\ monitor(?:\b|)" 0;
"~*(?:\b)ips-agent(?:\b|)" 0;
"~*(?:\b)Internet\ Ninja(?:\b|)" 0;
"~*(?:\b)Iria(?:\b|)" 0;
"~*(?:\b)IRLbot(?:\b|)" 0;
"~*(?:\b)isitwp.com(?:\b|)" 0;
"~*(?:\b)Iskanie(?:\b|)" 0;
"~*(?:\b)IstellaBot(?:\b|)" 0;
"~*(?:\b)JOC\ Web\ Spider(?:\b|)" 0;
"~*(?:\b)JamesBOT(?:\b|)" 0;
"~*(?:\b)Jbrofuzz(?:\b|)" 0;
"~*(?:\b)JennyBot(?:\b|)" 0;
"~*(?:\b)JetCar(?:\b|)" 0;
"~*(?:\b)Jetty(?:\b|)" 0;
"~*(?:\b)JikeSpider(?:\b|)" 0;
"~*(?:\b)JOC\ Web\ Spider(?:\b|)" 0;
"~*(?:\b)Joomla(?:\b|)" 0;
"~*(?:\b)Jorgee(?:\b|)" 0;
"~*(?:\b)JustView(?:\b|)" 0;
@ -302,6 +279,8 @@
"~*(?:\b)Keyword\ Density(?:\b|)" 0;
"~*(?:\b)Kinza(?:\b|)" 0;
"~*(?:\b)Kozmosbot(?:\b|)" 0;
"~*(?:\b)LNSpiderguy(?:\b|)" 0;
"~*(?:\b)LWP::Simple(?:\b|)" 0;
"~*(?:\b)Lanshanbot(?:\b|)" 0;
"~*(?:\b)Larbin(?:\b|)" 0;
"~*(?:\b)LeechFTP(?:\b|)" 0;
@ -313,13 +292,13 @@
"~*(?:\b)LieBaoFast(?:\b|)" 0;
"~*(?:\b)Lightspeedsystems(?:\b|)" 0;
"~*(?:\b)Likse(?:\b|)" 0;
"~*(?:\b)LinkScan(?:\b|)" 0;
"~*(?:\b)LinkWalker(?:\b|)" 0;
"~*(?:\b)Linkbot(?:\b|)" 0;
"~*(?:\b)Linkdexbot(?:\b|)" 0;
"~*(?:\b)LinkextractorPro(?:\b|)" 0;
"~*(?:\b)LinkpadBot(?:\b|)" 0;
"~*(?:\b)LinkScan(?:\b|)" 0;
"~*(?:\b)LinksManager(?:\b|)" 0;
"~*(?:\b)LinkWalker(?:\b|)" 0;
"~*(?:\b)LinqiaMetadataDownloaderBot(?:\b|)" 0;
"~*(?:\b)LinqiaRSSBot(?:\b|)" 0;
"~*(?:\b)LinqiaScrapeBot(?:\b|)" 0;
@ -327,53 +306,47 @@
"~*(?:\b)Lipperhey\ Spider(?:\b|)" 0;
"~*(?:\b)Litemage_walker(?:\b|)" 0;
"~*(?:\b)Lmspider(?:\b|)" 0;
"~*(?:\b)LNSpiderguy(?:\b|)" 0;
"~*(?:\b)Ltx71(?:\b|)" 0;
"~*(?:\b)lwp-request(?:\b|)" 0;
"~*(?:\b)LWP::Simple(?:\b|)" 0;
"~*(?:\b)lwp-trivial(?:\b|)" 0;
"~*(?:\b)Magnet(?:\b|)" 0;
"~*(?:\b)MFC_Tear_Sample(?:\b|)" 0;
"~*(?:\b)MIDown\ tool(?:\b|)" 0;
"~*(?:\b)MIIxpc(?:\b|)" 0;
"~*(?:\b)MJ12bot(?:\b|)" 0;
"~*(?:\b)MQQBrowser(?:\b|)" 0;
"~*(?:\b)MSFrontPage(?:\b|)" 0;
"~*(?:\b)MSIECrawler(?:\b|)" 0;
"~*(?:\b)MTRobot(?:\b|)" 0;
"~*(?:\b)Mag-Net(?:\b|)" 0;
"~*(?:\b)magpie-crawler(?:\b|)" 0;
"~*(?:\b)Magnet(?:\b|)" 0;
"~*(?:\b)Mail.RU_Bot(?:\b|)" 0;
"~*(?:\b)Majestic12(?:\b|)" 0;
"~*(?:\b)Majestic-SEO(?:\b|)" 0;
"~*(?:\b)Majestic12(?:\b|)" 0;
"~*(?:\b)Majestic\ SEO(?:\b|)" 0;
"~*(?:\b)MarkMonitor(?:\b|)" 0;
"~*(?:\b)MarkWatch(?:\b|)" 0;
"~*(?:\b)Masscan(?:\b|)" 0;
"~*(?:\b)Mass\ Downloader(?:\b|)" 0;
"~*(?:\b)Masscan(?:\b|)" 0;
"~*(?:\b)Mata\ Hari(?:\b|)" 0;
"~*(?:\b)MauiBot(?:\b|)" 0;
"~*(?:\b)Mb2345Browser(?:\b|)" 0;
"~*(?:\b)meanpathbot(?:\b|)" 0;
"~*(?:\b)Meanpathbot(?:\b|)" 0;
"~*(?:\b)MeanPath\ Bot(?:\b|)" 0;
"~*(?:\b)Meanpathbot(?:\b|)" 0;
"~*(?:\b)Mediatoolkitbot(?:\b|)" 0;
"~*(?:\b)mediawords(?:\b|)" 0;
"~*(?:\b)MegaIndex.ru(?:\b|)" 0;
"~*(?:\b)Metauri(?:\b|)" 0;
"~*(?:\b)MFC_Tear_Sample(?:\b|)" 0;
"~*(?:\b)MicroMessenger(?:\b|)" 0;
"~*(?:\b)Microsoft\ Data\ Access(?:\b|)" 0;
"~*(?:\b)Microsoft\ URL\ Control(?:\b|)" 0;
"~*(?:\b)MIDown\ tool(?:\b|)" 0;
"~*(?:\b)MIIxpc(?:\b|)" 0;
"~*(?:\b)Mister\ PiX(?:\b|)" 0;
"~*(?:\b)MJ12bot(?:\b|)" 0;
"~*(?:\b)Moblie Safari(?:\b|)" 0;
"~*(?:\b)Mojeek(?:\b|)" 0;
"~*(?:\b)Mojolicious(?:\b|)" 0;
"~*(?:\b)Morfeus\ Fucking\ Scanner(?:\b|)" 0;
"~*(?:\b)Mozlila(?:\b|)" 0;
"~*(?:\b)MQQBrowser(?:\b|)" 0;
"~*(?:\b)Mr.4x3(?:\b|)" 0;
"~*(?:\b)MSFrontPage(?:\b|)" 0;
"~*(?:\b)MSIECrawler(?:\b|)" 0;
"~*(?:\b)Msrabot(?:\b|)" 0;
"~*(?:\b)MTRobot(?:\b|)" 0;
"~*(?:\b)muhstik-scan(?:\b|)" 0;
"~*(?:\b)Musobot(?:\b|)" 0;
"~*(?:\b)NICErsPRO(?:\b|)" 0;
"~*(?:\b)NPbot(?:\b|)" 0;
"~*(?:\b)Name\ Intelligence(?:\b|)" 0;
"~*(?:\b)Nameprotect(?:\b|)" 0;
"~*(?:\b)Navroad(?:\b|)" 0;
@ -381,72 +354,63 @@
"~*(?:\b)Needle(?:\b|)" 0;
"~*(?:\b)Nessus(?:\b|)" 0;
"~*(?:\b)NetAnts(?:\b|)" 0;
"~*(?:\b)Netcraft(?:\b|)" 0;
"~*(?:\b)netEstate\ NE\ Crawler(?:\b|)" 0;
"~*(?:\b)NetLyzer(?:\b|)" 0;
"~*(?:\b)NetMechanic(?:\b|)" 0;
"~*(?:\b)NetSpider(?:\b|)" 0;
"~*(?:\b)Nettrack(?:\b|)" 0;
"~*(?:\b)Net\ Vampire(?:\b|)" 0;
"~*(?:\b)Netvibes(?:\b|)" 0;
"~*(?:\b)NetZIP(?:\b|)" 0;
"~*(?:\b)Net\ Vampire(?:\b|)" 0;
"~*(?:\b)Netcraft(?:\b|)" 0;
"~*(?:\b)Nettrack(?:\b|)" 0;
"~*(?:\b)Netvibes(?:\b|)" 0;
"~*(?:\b)NextGenSearchBot(?:\b|)" 0;
"~*(?:\b)Nibbler(?:\b|)" 0;
"~*(?:\b)NICErsPRO(?:\b|)" 0;
"~*(?:\b)Niki-bot(?:\b|)" 0;
"~*(?:\b)Nikto(?:\b|)" 0;
"~*(?:\b)NimbleCrawler(?:\b|)" 0;
"~*(?:\b)Nimbostratus(?:\b|)" 0;
"~*(?:\b)Ninja(?:\b|)" 0;
"~*(?:\b)Nmap(?:\b|)" 0;
"~*(?:\b)NPbot(?:\b|)" 0;
"~*(?:\b)Nuclei(?:\b|)" 0;
"~*(?:\b)Nutch(?:\b|)" 0;
"~*(?:\b)oBot(?:\b|)" 0;
"~*(?:\b)OPPO\ A33(?:\b|)" 0;
"~*(?:\b)Octopus(?:\b|)" 0;
"~*(?:\b)Offline\ Explorer(?:\b|)" 0;
"~*(?:\b)Offline\ Navigator(?:\b|)" 0;
"~*(?:\b)OnCrawl(?:\b|)" 0;
"~*(?:\b)Openfind(?:\b|)" 0;
"~*(?:\b)OpenLinkProfiler(?:\b|)" 0;
"~*(?:\b)Openvas(?:\b|)" 0;
"~*(?:\b)OpenVAS(?:\b|)" 0;
"~*(?:\b)OPPO\ A33(?:\b|)" 0;
"~*(?:\b)Openfind(?:\b|)" 0;
"~*(?:\b)Openvas(?:\b|)" 0;
"~*(?:\b)OrangeBot(?:\b|)" 0;
"~*(?:\b)OrangeSpider(?:\b|)" 0;
"~*(?:\b)OutclicksBot(?:\b|)" 0;
"~*(?:\b)OutfoxBot(?:\b|)" 0;
"~*(?:\b)PECL::HTTP(?:\b|)" 0;
"~*(?:\b)PHPCrawl(?:\b|)" 0;
"~*(?:\b)POE-Component-Client-HTTP(?:\b|)" 0;
"~*(?:\b)PageAnalyzer(?:\b|)" 0;
"~*(?:\b)Page\ Analyzer(?:\b|)" 0;
"~*(?:\b)PageGrabber(?:\b|)" 0;
"~*(?:\b)page\ scorer(?:\b|)" 0;
"~*(?:\b)PageScorer(?:\b|)" 0;
"~*(?:\b)PageThing.com(?:\b|)" 0;
"~*(?:\b)Page\ Analyzer(?:\b|)" 0;
"~*(?:\b)Pandalytics(?:\b|)" 0;
"~*(?:\b)Panscient(?:\b|)" 0;
"~*(?:\b)Papa\ Foto(?:\b|)" 0;
"~*(?:\b)Pavuk(?:\b|)" 0;
"~*(?:\b)pcBrowser(?:\b|)" 0;
"~*(?:\b)PECL::HTTP(?:\b|)" 0;
"~*(?:\b)PeoplePal(?:\b|)" 0;
"~*(?:\b)Petalbot(?:\b|)" 0;
"~*(?:\b)PHPCrawl(?:\b|)" 0;
"~*(?:\b)Pi-Monster(?:\b|)" 0;
"~*(?:\b)Picscout(?:\b|)" 0;
"~*(?:\b)Picsearch(?:\b|)" 0;
"~*(?:\b)PictureFinder(?:\b|)" 0;
"~*(?:\b)Piepmatz(?:\b|)" 0;
"~*(?:\b)Pimonster(?:\b|)" 0;
"~*(?:\b)Pi-Monster(?:\b|)" 0;
"~*(?:\b)Pixray(?:\b|)" 0;
"~*(?:\b)PleaseCrawl(?:\b|)" 0;
"~*(?:\b)plumanalytics(?:\b|)" 0;
"~*(?:\b)Pockey(?:\b|)" 0;
"~*(?:\b)POE-Component-Client-HTTP(?:\b|)" 0;
"~*(?:\b)polaris\ version(?:\b|)" 0;
"~*(?:\b)probe-image-size(?:\b|)" 0;
"~*(?:\b)Probethenet(?:\b|)" 0;
"~*(?:\b)ProPowerBot(?:\b|)" 0;
"~*(?:\b)ProWebWalker(?:\b|)" 0;
"~*(?:\b)Probethenet(?:\b|)" 0;
"~*(?:\b)Psbot(?:\b|)" 0;
"~*(?:\b)Pu_iN(?:\b|)" 0;
"~*(?:\b)Pump(?:\b|)" 0;
@ -454,6 +418,7 @@
"~*(?:\b)PyCurl(?:\b|)" 0;
"~*(?:\b)QueryN\ Metasearch(?:\b|)" 0;
"~*(?:\b)Quick-Crawler(?:\b|)" 0;
"~*(?:\b)RSSingBot(?:\b|)" 0;
"~*(?:\b)RankActive(?:\b|)" 0;
"~*(?:\b)RankActiveLinkBot(?:\b|)" 0;
"~*(?:\b)RankFlex(?:\b|)" 0;
@ -461,27 +426,28 @@
"~*(?:\b)RankingBot2(?:\b|)" 0;
"~*(?:\b)Rankivabot(?:\b|)" 0;
"~*(?:\b)RankurBot(?:\b|)" 0;
"~*(?:\b)Re-re(?:\b|)" 0;
"~*(?:\b)ReGet(?:\b|)" 0;
"~*(?:\b)RealDownload(?:\b|)" 0;
"~*(?:\b)Reaper(?:\b|)" 0;
"~*(?:\b)RebelMouse(?:\b|)" 0;
"~*(?:\b)Recorder(?:\b|)" 0;
"~*(?:\b)RedesScrapy(?:\b|)" 0;
"~*(?:\b)ReGet(?:\b|)" 0;
"~*(?:\b)RepoMonkey(?:\b|)" 0;
"~*(?:\b)Re-re(?:\b|)" 0;
"~*(?:\b)Ripper(?:\b|)" 0;
"~*(?:\b)ripz(?:\b|)" 0;
"~*(?:\b)RocketCrawler(?:\b|)" 0;
"~*(?:\b)Rogerbot(?:\b|)" 0;
"~*(?:\b)RSSingBot(?:\b|)" 0;
"~*(?:\b)s1z.ru(?:\b|)" 0;
"~*(?:\b)SalesIntelligent(?:\b|)" 0;
"~*(?:\b)satoristudio.net(?:\b|)" 0;
"~*(?:\b)SBIder(?:\b|)" 0;
"~*(?:\b)scalaj-http(?:\b|)" 0;
"~*(?:\b)SEOkicks(?:\b|)" 0;
"~*(?:\b)SEOkicks-Robot(?:\b|)" 0;
"~*(?:\b)SEOlyticsCrawler(?:\b|)" 0;
"~*(?:\b)SEOprofiler(?:\b|)" 0;
"~*(?:\b)SEOstats(?:\b|)" 0;
"~*(?:\b)SISTRIX(?:\b|)" 0;
"~*(?:\b)SMTBot(?:\b|)" 0;
"~*(?:\b)SalesIntelligent(?:\b|)" 0;
"~*(?:\b)ScanAlert(?:\b|)" 0;
"~*(?:\b)Scanbot(?:\b|)" 0;
"~*(?:\b)scan.lol(?:\b|)" 0;
"~*(?:\b)ScoutJet(?:\b|)" 0;
"~*(?:\b)Scrapy(?:\b|)" 0;
"~*(?:\b)Screaming(?:\b|)" 0;
@ -492,42 +458,27 @@
"~*(?:\b)Semrush(?:\b|)" 0;
"~*(?:\b)SemrushBot(?:\b|)" 0;
"~*(?:\b)SentiBot(?:\b|)" 0;
"~*(?:\b)seobility(?:\b|)" 0;
"~*(?:\b)SeobilityBot(?:\b|)" 0;
"~*(?:\b)SEOkicks(?:\b|)" 0;
"~*(?:\b)SEOkicks-Robot(?:\b|)" 0;
"~*(?:\b)SEOlyticsCrawler(?:\b|)" 0;
"~*(?:\b)Seomoz(?:\b|)" 0;
"~*(?:\b)SEOprofiler(?:\b|)" 0;
"~*(?:\b)seoscanners(?:\b|)" 0;
"~*(?:\b)SeoSiteCheckup(?:\b|)" 0;
"~*(?:\b)seostar(?:\b|)" 0;
"~*(?:\b)SEOstats(?:\b|)" 0;
"~*(?:\b)serpstatbot(?:\b|)" 0;
"~*(?:\b)sexsearcher(?:\b|)" 0;
"~*(?:\b)SeobilityBot(?:\b|)" 0;
"~*(?:\b)Seomoz(?:\b|)" 0;
"~*(?:\b)Shodan(?:\b|)" 0;
"~*(?:\b)Siphon(?:\b|)" 0;
"~*(?:\b)SISTRIX(?:\b|)" 0;
"~*(?:\b)Sitebeam(?:\b|)" 0;
"~*(?:\b)SiteCheckerBotCrawler(?:\b|)" 0;
"~*(?:\b)sitechecker.pro(?:\b|)" 0;
"~*(?:\b)SiteExplorer(?:\b|)" 0;
"~*(?:\b)Siteimprove(?:\b|)" 0;
"~*(?:\b)SiteLockSpider(?:\b|)" 0;
"~*(?:\b)siteripz(?:\b|)" 0;
"~*(?:\b)SiteSnagger(?:\b|)" 0;
"~*(?:\b)SiteSucker(?:\b|)" 0;
"~*(?:\b)Site\ Sucker(?:\b|)" 0;
"~*(?:\b)Sitebeam(?:\b|)" 0;
"~*(?:\b)Siteimprove(?:\b|)" 0;
"~*(?:\b)Sitevigil(?:\b|)" 0;
"~*(?:\b)SlySearch(?:\b|)" 0;
"~*(?:\b)SmartDownload(?:\b|)" 0;
"~*(?:\b)SMTBot(?:\b|)" 0;
"~*(?:\b)Snake(?:\b|)" 0;
"~*(?:\b)Snapbot(?:\b|)" 0;
"~*(?:\b)Snoopy(?:\b|)" 0;
"~*(?:\b)SocialRankIOBot(?:\b|)" 0;
"~*(?:\b)Sociscraper(?:\b|)" 0;
"~*(?:\b)sogouspider(?:\b|)" 0;
"~*(?:\b)Sogou\ web\ spider(?:\b|)" 0;
"~*(?:\b)Sosospider(?:\b|)" 0;
"~*(?:\b)Sottopop(?:\b|)" 0;
@ -535,11 +486,9 @@
"~*(?:\b)Spammen(?:\b|)" 0;
"~*(?:\b)SpankBot(?:\b|)" 0;
"~*(?:\b)Spanner(?:\b|)" 0;
"~*(?:\b)sp_auditbot(?:\b|)" 0;
"~*(?:\b)Spbot(?:\b|)" 0;
"~*(?:\b)Spinn3r(?:\b|)" 0;
"~*(?:\b)SputnikBot(?:\b|)" 0;
"~*(?:\b)spyfu(?:\b|)" 0;
"~*(?:\b)Sqlmap(?:\b|)" 0;
"~*(?:\b)Sqlworm(?:\b|)" 0;
"~*(?:\b)Sqworm(?:\b|)" 0;
@ -553,18 +502,16 @@
"~*(?:\b)SurveyBot(?:\b|)" 0;
"~*(?:\b)Suzuran(?:\b|)" 0;
"~*(?:\b)Swiftbot(?:\b|)" 0;
"~*(?:\b)sysscan(?:\b|)" 0;
"~*(?:\b)Szukacz(?:\b|)" 0;
"~*(?:\b)T0PHackTeam(?:\b|)" 0;
"~*(?:\b)T8Abot(?:\b|)" 0;
"~*(?:\b)tAkeOut(?:\b|)" 0;
"~*(?:\b)Teleport(?:\b|)" 0;
"~*(?:\b)TeleportPro(?:\b|)" 0;
"~*(?:\b)Telesoft(?:\b|)" 0;
"~*(?:\b)Telesphoreo(?:\b|)" 0;
"~*(?:\b)Telesphorep(?:\b|)" 0;
"~*(?:\b)The\ Intraformant(?:\b|)" 0;
"~*(?:\b)TheNomad(?:\b|)" 0;
"~*(?:\b)The\ Intraformant(?:\b|)" 0;
"~*(?:\b)Thumbor(?:\b|)" 0;
"~*(?:\b)TightTwatBot(?:\b|)" 0;
"~*(?:\b)Titan(?:\b|)" 0;
@ -573,8 +520,6 @@
"~*(?:\b)Tracemyfile(?:\b|)" 0;
"~*(?:\b)Trendiction(?:\b|)" 0;
"~*(?:\b)Trendictionbot(?:\b|)" 0;
"~*(?:\b)trendiction.com(?:\b|)" 0;
"~*(?:\b)trendiction.de(?:\b|)" 0;
"~*(?:\b)True_Robot(?:\b|)" 0;
"~*(?:\b)Turingos(?:\b|)" 0;
"~*(?:\b)Turnitin(?:\b|)" 0;
@ -582,15 +527,15 @@
"~*(?:\b)TwengaBot(?:\b|)" 0;
"~*(?:\b)Twice(?:\b|)" 0;
"~*(?:\b)Typhoeus(?:\b|)" 0;
"~*(?:\b)UnisterBot(?:\b|)" 0;
"~*(?:\b)Upflow(?:\b|)" 0;
"~*(?:\b)URLy.Warning(?:\b|)" 0;
"~*(?:\b)URLy\ Warning(?:\b|)" 0;
"~*(?:\b)Vacuum(?:\b|)" 0;
"~*(?:\b)Vagabondo(?:\b|)" 0;
"~*(?:\b)UnisterBot(?:\b|)" 0;
"~*(?:\b)Upflow(?:\b|)" 0;
"~*(?:\b)V-BOT(?:\b|)" 0;
"~*(?:\b)VB\ Project(?:\b|)" 0;
"~*(?:\b)VCI(?:\b|)" 0;
"~*(?:\b)Vacuum(?:\b|)" 0;
"~*(?:\b)Vagabondo(?:\b|)" 0;
"~*(?:\b)VelenPublicWebCrawler(?:\b|)" 0;
"~*(?:\b)VeriCiteCrawler(?:\b|)" 0;
"~*(?:\b)VidibleScraper(?:\b|)" 0;
@ -598,46 +543,50 @@
"~*(?:\b)VoidEYE(?:\b|)" 0;
"~*(?:\b)Voil(?:\b|)" 0;
"~*(?:\b)Voltron(?:\b|)" 0;
"~*(?:\b)voyagerx.com(?:\b|)" 0;
"~*(?:\b)Wallpapers/3.0(?:\b|)" 0;
"~*(?:\b)WallpapersHD(?:\b|)" 0;
"~*(?:\b)WASALive-Bot(?:\b|)" 0;
"~*(?:\b)WBSearchBot(?:\b|)" 0;
"~*(?:\b)Webalta(?:\b|)" 0;
"~*(?:\b)WEBDAV(?:\b|)" 0;
"~*(?:\b)WISENutbot(?:\b|)" 0;
"~*(?:\b)WPScan(?:\b|)" 0;
"~*(?:\b)WWW-Collector-E(?:\b|)" 0;
"~*(?:\b)WWW-Mechanize(?:\b|)" 0;
"~*(?:\b)WWW::Mechanize(?:\b|)" 0;
"~*(?:\b)WWWOFFLE(?:\b|)" 0;
"~*(?:\b)Wallpapers/3.0(?:\b|)" 0;
"~*(?:\b)WallpapersHD(?:\b|)" 0;
"~*(?:\b)WeSEE(?:\b|)" 0;
"~*(?:\b)WebAuto(?:\b|)" 0;
"~*(?:\b)Web\ Auto(?:\b|)" 0;
"~*(?:\b)WebBandit(?:\b|)" 0;
"~*(?:\b)WebCollage(?:\b|)" 0;
"~*(?:\b)Web\ Collage(?:\b|)" 0;
"~*(?:\b)WebCopier(?:\b|)" 0;
"~*(?:\b)WEBDAV(?:\b|)" 0;
"~*(?:\b)WebEnhancer(?:\b|)" 0;
"~*(?:\b)Web\ Enhancer(?:\b|)" 0;
"~*(?:\b)WebFetch(?:\b|)" 0;
"~*(?:\b)Web\ Fetch(?:\b|)" 0;
"~*(?:\b)WebFuck(?:\b|)" 0;
"~*(?:\b)Web\ Fuck(?:\b|)" 0;
"~*(?:\b)WebGo\ IS(?:\b|)" 0;
"~*(?:\b)WebImageCollector(?:\b|)" 0;
"~*(?:\b)WebLeacher(?:\b|)" 0;
"~*(?:\b)WebmasterWorldForumBot(?:\b|)" 0;
"~*(?:\b)webmeup-crawler(?:\b|)" 0;
"~*(?:\b)WebPix(?:\b|)" 0;
"~*(?:\b)Web\ Pix(?:\b|)" 0;
"~*(?:\b)WebReaper(?:\b|)" 0;
"~*(?:\b)WebSauger(?:\b|)" 0;
"~*(?:\b)WebStripper(?:\b|)" 0;
"~*(?:\b)WebSucker(?:\b|)" 0;
"~*(?:\b)WebWhacker(?:\b|)" 0;
"~*(?:\b)WebZIP(?:\b|)" 0;
"~*(?:\b)Web\ Auto(?:\b|)" 0;
"~*(?:\b)Web\ Collage(?:\b|)" 0;
"~*(?:\b)Web\ Enhancer(?:\b|)" 0;
"~*(?:\b)Web\ Fetch(?:\b|)" 0;
"~*(?:\b)Web\ Fuck(?:\b|)" 0;
"~*(?:\b)Web\ Pix(?:\b|)" 0;
"~*(?:\b)Web\ Sauger(?:\b|)" 0;
"~*(?:\b)Web\ Sucker(?:\b|)" 0;
"~*(?:\b)Webalta(?:\b|)" 0;
"~*(?:\b)WebmasterWorldForumBot(?:\b|)" 0;
"~*(?:\b)Webshag(?:\b|)" 0;
"~*(?:\b)WebsiteExtractor(?:\b|)" 0;
"~*(?:\b)WebsiteQuester(?:\b|)" 0;
"~*(?:\b)Website\ Quester(?:\b|)" 0;
"~*(?:\b)Webster(?:\b|)" 0;
"~*(?:\b)WebStripper(?:\b|)" 0;
"~*(?:\b)WebSucker(?:\b|)" 0;
"~*(?:\b)Web\ Sucker(?:\b|)" 0;
"~*(?:\b)WebWhacker(?:\b|)" 0;
"~*(?:\b)WebZIP(?:\b|)" 0;
"~*(?:\b)WeSEE(?:\b|)" 0;
"~*(?:\b)Whack(?:\b|)" 0;
"~*(?:\b)Whacker(?:\b|)" 0;
"~*(?:\b)Whatweb(?:\b|)" 0;
@ -645,33 +594,84 @@
"~*(?:\b)Widow(?:\b|)" 0;
"~*(?:\b)WinHTTrack(?:\b|)" 0;
"~*(?:\b)WiseGuys\ Robot(?:\b|)" 0;
"~*(?:\b)WISENutbot(?:\b|)" 0;
"~*(?:\b)Wonderbot(?:\b|)" 0;
"~*(?:\b)Woobot(?:\b|)" 0;
"~*(?:\b)Wotbox(?:\b|)" 0;
"~*(?:\b)Wprecon(?:\b|)" 0;
"~*(?:\b)WPScan(?:\b|)" 0;
"~*(?:\b)WWW-Collector-E(?:\b|)" 0;
"~*(?:\b)WWW-Mechanize(?:\b|)" 0;
"~*(?:\b)WWW::Mechanize(?:\b|)" 0;
"~*(?:\b)WWWOFFLE(?:\b|)" 0;
"~*(?:\b)x09Mozilla(?:\b|)" 0;
"~*(?:\b)x22Mozilla(?:\b|)" 0;
"~*(?:\b)Xaldon_WebSpider(?:\b|)" 0;
"~*(?:\b)Xaldon\ WebSpider(?:\b|)" 0;
"~*(?:\b)Xaldon_WebSpider(?:\b|)" 0;
"~*(?:\b)Xenu(?:\b|)" 0;
"~*(?:\b)xpymep1.exe(?:\b|)" 0;
"~*(?:\b)YoudaoBot(?:\b|)" 0;
"~*(?:\b)Zade(?:\b|)" 0;
"~*(?:\b)Zauba(?:\b|)" 0;
"~*(?:\b)zauba.io(?:\b|)" 0;
"~*(?:\b)Zermelo(?:\b|)" 0;
"~*(?:\b)Zeus(?:\b|)" 0;
"~*(?:\b)zgrab(?:\b|)" 0;
"~*(?:\b)Zitebot(?:\b|)" 0;
"~*(?:\b)ZmEu(?:\b|)" 0;
"~*(?:\b)ZoomBot(?:\b|)" 0;
"~*(?:\b)ZoominfoBot(?:\b|)" 0;
"~*(?:\b)ZumBot(?:\b|)" 0;
"~*(?:\b)ZyBorg(?:\b|)" 0;
"~*(?:\b)archive.org_bot(?:\b|)" 0;
"~*(?:\b)arquivo-web-crawler(?:\b|)" 0;
"~*(?:\b)arquivo.pt(?:\b|)" 0;
"~*(?:\b)autoemailspider(?:\b|)" 0;
"~*(?:\b)backlink-check(?:\b|)" 0;
"~*(?:\b)check1.exe(?:\b|)" 0;
"~*(?:\b)coccocbot-web(?:\b|)" 0;
"~*(?:\b)cognitiveseo(?:\b|)" 0;
"~*(?:\b)com.plumanalytics(?:\b|)" 0;
"~*(?:\b)crawl.sogou.com(?:\b|)" 0;
"~*(?:\b)crawler.feedback(?:\b|)" 0;
"~*(?:\b)crawler4j(?:\b|)" 0;
"~*(?:\b)demandbase-bot(?:\b|)" 0;
"~*(?:\b)domainsproject.org(?:\b|)" 0;
"~*(?:\b)eCatch(?:\b|)" 0;
"~*(?:\b)evc-batch(?:\b|)" 0;
"~*(?:\b)facebookscraper(?:\b|)" 0;
"~*(?:\b)gopher(?:\b|)" 0;
"~*(?:\b)heritrix(?:\b|)" 0;
"~*(?:\b)instabid(?:\b|)" 0;
"~*(?:\b)internetVista\ monitor(?:\b|)" 0;
"~*(?:\b)ips-agent(?:\b|)" 0;
"~*(?:\b)isitwp.com(?:\b|)" 0;
"~*(?:\b)lwp-request(?:\b|)" 0;
"~*(?:\b)lwp-trivial(?:\b|)" 0;
"~*(?:\b)magpie-crawler(?:\b|)" 0;
"~*(?:\b)meanpathbot(?:\b|)" 0;
"~*(?:\b)mediawords(?:\b|)" 0;
"~*(?:\b)muhstik-scan(?:\b|)" 0;
"~*(?:\b)netEstate\ NE\ Crawler(?:\b|)" 0;
"~*(?:\b)oBot(?:\b|)" 0;
"~*(?:\b)page\ scorer(?:\b|)" 0;
"~*(?:\b)pcBrowser(?:\b|)" 0;
"~*(?:\b)plumanalytics(?:\b|)" 0;
"~*(?:\b)polaris\ version(?:\b|)" 0;
"~*(?:\b)probe-image-size(?:\b|)" 0;
"~*(?:\b)ripz(?:\b|)" 0;
"~*(?:\b)s1z.ru(?:\b|)" 0;
"~*(?:\b)satoristudio.net(?:\b|)" 0;
"~*(?:\b)scalaj-http(?:\b|)" 0;
"~*(?:\b)scan.lol(?:\b|)" 0;
"~*(?:\b)seobility(?:\b|)" 0;
"~*(?:\b)seoscanners(?:\b|)" 0;
"~*(?:\b)seostar(?:\b|)" 0;
"~*(?:\b)serpstatbot(?:\b|)" 0;
"~*(?:\b)sexsearcher(?:\b|)" 0;
"~*(?:\b)sitechecker.pro(?:\b|)" 0;
"~*(?:\b)siteripz(?:\b|)" 0;
"~*(?:\b)sogouspider(?:\b|)" 0;
"~*(?:\b)sp_auditbot(?:\b|)" 0;
"~*(?:\b)spyfu(?:\b|)" 0;
"~*(?:\b)sysscan(?:\b|)" 0;
"~*(?:\b)tAkeOut(?:\b|)" 0;
"~*(?:\b)trendiction.com(?:\b|)" 0;
"~*(?:\b)trendiction.de(?:\b|)" 0;
"~*(?:\b)voyagerx.com(?:\b|)" 0;
"~*(?:\b)webmeup-crawler(?:\b|)" 0;
"~*(?:\b)x09Mozilla(?:\b|)" 0;
"~*(?:\b)x22Mozilla(?:\b|)" 0;
"~*(?:\b)xpymep1.exe(?:\b|)" 0;
"~*(?:\b)zauba.io(?:\b|)" 0;
"~*(?:\b)zgrab(?:\b|)" 0;
# END MAKE BAD BOTS GOOD ### DO NOT EDIT THIS LINE AT ALL ###

4
dev-tools/beta_conf_files_whitelist/bots.d/custom-bad-referrers.conf
View file

@ -498,6 +498,7 @@
"~*(?:\b)anniemation\.com(?:\b)" 0;
"~*(?:\b)anonymizeme\.pro(?:\b)" 0;
"~*(?:\b)anonymous\-redirect\.com(?:\b)" 0;
"~*(?:\b)anonymousfox\.co(?:\b)" 0;
"~*(?:\b)anti\-virus\-removal\.info(?:\b)" 0;
"~*(?:\b)anticrawler\.org(?:\b)" 0;
"~*(?:\b)antons\-transporte\.de(?:\b)" 0;
@ -4637,6 +4638,7 @@
"~*(?:\b)pokemongooo\.ml(?:\b)" 0;
"~*(?:\b)pokerniydom\.ru(?:\b)" 0;
"~*(?:\b)polcin\.de(?:\b)" 0;
"~*(?:\b)poligon\.com(?:\b)" 0;
"~*(?:\b)polimga\.pw(?:\b)" 0;
"~*(?:\b)polska\-poezja\.com(?:\b)" 0;
"~*(?:\b)polybuild\.ru(?:\b)" 0;
@ -4693,6 +4695,7 @@
"~*(?:\b)pornogratisdiario\.com(?:\b)" 0;
"~*(?:\b)pornohd1080\.online(?:\b)" 0;
"~*(?:\b)pornohub\.me(?:\b)" 0;
"~*(?:\b)pornoinn\.com(?:\b)" 0;
"~*(?:\b)pornokajf\.com(?:\b)" 0;
"~*(?:\b)pornoklad\.net(?:\b)" 0;
"~*(?:\b)pornoklad\.ru(?:\b)" 0;
@ -6811,6 +6814,7 @@
"~*(?:\b)xlovecam\.com(?:\b)" 0;
"~*(?:\b)xmladserver\.com(?:\b)" 0;
"~*(?:\b)xmlinde\.com(?:\b)" 0;
"~*(?:\b)xmnb\.net(?:\b)" 0;
"~*(?:\b)xmronta\.com(?:\b)" 0;
"~*(?:\b)xn\-\-\-\-\-\-7cdbapdecfd4ak1bn0amjffj7afu3y\.xn\-\-p1ai(?:\b)" 0;
"~*(?:\b)xn\-\-\-\-\-6kcaabbafhu7cskl7akvongwpo7hvjj\.xn\-\-p1ai(?:\b)" 0;

2
dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf
View file

@ -1,2 +1,2 @@
34.66.25.221 0;
10.1.0.59 0;
127.0.0.1 0;

15472
dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf
View file

File diff suppressed because it is too large Load diff

9
dev-tools/beta_conf_files_whitelist/default.vhost
View file

@ -1,7 +1,6 @@
server {
listen *:9000;
root /var/www/html;
listen 80 default_server;
listen [::]:80 default_server;
server_name localhost;
charset UTF-8;
@ -14,6 +13,10 @@ server {
include /etc/nginx/bots.d/blockbots.conf;
index index.html;
root /var/www/html;
# Add index.php to the list if you are using PHP
index index.html index.htm index.nginx-debian.html;
location / {
root /var/www/html/;