V4.2021.10.2630 [ci skip]

dev-tools/test_units/random-bots-for-whitelist-test.list

@@ -1,250 +1,250 @@
 01h4x.com
-360Spider
-404checker
-AIBOT
-ASPSeek
+80legs
+ADmantX
+ALittle Client
+Aboundex
 Acunetix
 AfD-Verbotsverfahren
+AhrefsBot
 Aipbot
+Alexibot
 AllSubmitter
+AlphaBot
 Anarchie
-Anarchy
-Anarchy99
-Ankit
-Aspiegel
-Asterias
-AwarioRssBot
-BDCbot
+Anthill
+AwarioSmartBot
 BDFetch
-BackWeb
-Badass
 Bandit
-Barkrowler
 BatchFTP
 Battleztar Bazinga
-Black Hole
-Brandprotect
-BuiltBotTough
-Bullseye
+Blackboard
+Blow
+BlowFish
+Brandwatch
 BunnySlippers
+CCBot
 CODE87
-CazoodleBot
-CherryPicker
-Chlooe
+Calculon
+Cegbfeieh
+CheTeam
+ChinaClaw
 Claritybot
+Cliqzbot
 Cloud mapping
-Cocolyzebot
-Collector
+Cogentbot
 CopyRightCheck
+Copyscape
 Cosmos
-Crawling at Home Project
-DBLBot
-DIIbot
-DSearch
-DataCha0s
-DatabaseDriverMysqli
-Dirbuster
-Disco
-Dispatch
+Craftbot
+CrazyWebCrawler
+CrunchBot
+Curious
 DomainAppender
-Download Wonder
-Dragonfly
-EMail Siphon
+DomainSigmaCrawler
+DomainStatsBot
+Domains Project
+ECCP/1.0
+EMail Wolf
 EasyDL
 Ebingbong
-Ecxi
-EroCrawler
-Evil
+Express WebPictures
+ExtLinksBot
+Extractor
 EyeNetIE
 FDM
-FHscan
+FemtosearchBot
+Fimap
 FlashGet
-FyberSpider
-GetRight
-GetWeb
+Foobot
+Fuzz
+Fyrebot
+Genieo
+GermCrawler
 Getintent
-Gigabot
-Go!Zilla
-Go-Ahead-Got-It
-GrabNet
+GoZilla
 Grabber
-Grafula
-GridBot
-HTTrack
-Heritrix
-Hloader
-Iblog
+GrapeFX
+GrapeshotCrawler
+HEADMasterSEO
+HTTP::Lite
+Haansoft
+Harvest
+Havij
+IDBTE4M
+IlseBot
+Image Fetch
 Image Sucker
-IndeedBot
-Indy Library
+InfoNaviRobot
+Intelliseek
+InterGET
 Internet Ninja
-Iskanie
-IstellaBot
-JOC Web Spider
+InternetSeer
+Iria
 JamesBOT
 Jbrofuzz
-JetCar
-Jetty
-Joomla
+JikeSpider
 Jorgee
+JustView
+Kenjin Spider
 Keyword Density
-Kinza
-Kozmosbot
 LNSpiderguy
-LWP::Simple
-Lanshanbot
-LeechFTP
+Larbin
+Leap
+LeechGet
+LexiBot
 LibWeb
-Libwhisker
 LieBaoFast
-Lightspeedsystems
-LinkScan
-Linkbot
-Linkdexbot
+LinkWalker
 LinkpadBot
-LinqiaScrapeBot
+LinksManager
+Lipperhey Spider
 Litemage_walker
 MIDown tool
-MJ12bot
+MIIxpc
 MSFrontPage
 MSIECrawler
+MTRobot
 Mag-Net
+Magnet
 Majestic SEO
-Majestic-SEO
-MauiBot
+Majestic12
+MarkWatch
+Mb2345Browser
+Meanpathbot
+Mediatoolkitbot
 MegaIndex.ru
-Metauri
-Mister PiX
-Mojeek
-Mojolicious
-Morfeus Fucking Scanner
-Mozlila
+MicroMessenger
+Microsoft Data Access
+Microsoft URL Control
+Mr.4x3
+Musobot
+NPbot
 Name Intelligence
-Nameprotect
+Navroad
+NearSite
 Nessus
+NetAnts
+NetLyzer
+NetMechanic
+NetSpider
 NetZIP
-Netcraft
+Nettrack
 Netvibes
 Nibbler
-Niki-bot
-Nikto
-Nuclei
 Nutch
-Offline Explorer
+Octopus
+OnCrawl
 OpenLinkProfiler
-OpenVAS
 Openfind
-OutclicksBot
-PageScorer
-PageThing.com
-PeoplePal
+OutfoxBot
+PECL::HTTP
+Page Analyzer
+PageGrabber
+Pandalytics
+Picsearch
 Pimonster
-PleaseCrawl
-Probethenet
-Psbot
 Pu_iN
-QueryN Metasearch
-Quick-Crawler
-RankActive
-RankActiveLinkBot
+PyCurl
+RSSingBot
 RankFlex
+RankingBot2
 Rankivabot
 RankurBot
+Re-re
 ReGet
-Reaper
 RebelMouse
-RepoMonkey
-SEOkicks
+Recorder
+RedesScrapy
+Ripper
+SEOkicks-Robot
 SEOlyticsCrawler
 SEOprofiler
-SEOstats
 SMTBot
-SalesIntelligent
-Scanbot
+ScanAlert
 ScoutJet
 Scrapy
-Screaming
-ScreenerBot
-SearchmetricsBot
-SemrushBot
-SentiBot
-SeoSiteCheckup
-SeobilityBot
+Searchestate
 Seomoz
-SiteLockSpider
-Siteimprove
-Snake
-Sottopop
+Shodan
+SiteCheckerBotCrawler
+SiteExplorer
+SlySearch
+SmartDownload
+Snoopy
+SocialRankIOBot
 SpaceBison
-Spinn3r
+SpankBot
+Spbot
+SputnikBot
 Sqlmap
+Stripper
+Sucuri
 SuperHTTP
-SurveyBot
-Suzuran
-Swiftbot
-Telesphoreo
-Telesphorep
-The Intraformant
-Titan
+T0PHackTeam
+TheNomad
+Thumbor
 Toata
-Trendictionbot
+Tracemyfile
+Trendiction
 Turingos
 Turnitin
 TurnitinBot
-TwengaBot
-Twice
-Typhoeus
-URLy.Warning
-VidibleScraper
-Virusdie
+URLy Warning
+VCI
+VelenPublicWebCrawler
+VeriCiteCrawler
 VoidEYE
 Voltron
+WASALive-Bot
+WBSearchBot
 WISENutbot
-WWW-Collector-E
-WallpapersHD
-WeSEE
-Web Collage
+Wallpapers
+Web Auto
+Web Fuck
+WebAuto
 WebBandit
-WebFetch
-WebFuck
+WebGo IS
 WebImageCollector
-WebLeacher
+WebReaper
 WebSauger
 WebStripper
-WebZIP
-Webshag
+WebWhacker
+Webalta
 Webster
-Whack
-Whacker
-Who.is Bot
+Widow
+WinHTTrack
+Wonderbot
 Woobot
-Wotbox
-Xaldon WebSpider
+Xenu
 YoudaoBot
-Zade
-Zauba
-Zermelo
-Zeus
-Zitebot
-ZmEu
+ZoominfoBot
 archive.org_bot
 arquivo.pt
-cognitiveseo
+backlink-check
+clark-crawler
+com.plumanalytics
+crawl.sogou.com
+crawler.feedback
 crawler4j
 domainsproject.org
-facebookscraper
+evc-batch
+gopher
 isitwp.com
-lwp-request
 magpie-crawler
-netEstate NE Crawler
+meanpathbot
 oBot
 page scorer
-plumanalytics
+pcBrowser
+polaris version
 probe-image-size
-satoristudio.net
 scalaj-http
+seostar
+sitechecker.pro
+spyfu
 sysscan
 tAkeOut
+trendiction.de
+ubermetrics-technologies.com
+voyagerx.com
 webmeup-crawler
-webpros.com
-webprosbot
+x22Mozilla