V4.2021.04.2323 [ci skip]

.dev-tools/test_units/random-bots-for-whitelist-test.list

@@ -1,250 +1,250 @@
-404checker
-80legs
+01h4x.com
 Abonti
-Aboundex
-Acunetix
-AfD-Verbotsverfahren
-AhrefsBot
-AIBOT
-AiHitBot
-Aipbot
-Alexibot
+ADmantX
+AlphaBot
 Anarchy
-Apexoo
 arquivo.pt
 arquivo-web-crawler
-Aspiegel
-Attach
+ASPSeek
 autoemailspider
 AwarioSmartBot
 BackDoorBot
 Backlink-Ceck
-BacklinkCrawler
-BackStreet
-Badass
+Bandit
 BatchFTP
 Battleztar Bazinga
+BBBike
 BDCbot
 Blackboard
 Boardreader
-Bolt
-Brandprotect
-Brandwatch
-BuiltBotTough
-BuiltWith
+BotALot
+BuzzSumo
+CCBot
 Cegbfeieh
-CensysInspect
-CheeseBot
+check1.exe
 CherryPicker
 CheTeam
 ChinaClaw
 Chlooe
-Claritybot
 Cliqzbot
-coccocbot-web
+CODE87
+Cogentbot
 Collector
 com.plumanalytics
-Copyscape
-Cosmos
-Craftbot
-crawl.sogou.com
-CSHttp
+Copier
+crawler4j
+CrazyWebCrawler
+CrunchBot
+CyotekWebCopy
+DataCha0s
 DBLBot
-Devil
-Dirbuster
-Discobot
-DittoSpyder
-DomainCrawler
+Demon
+Deusu
+DnyzBot
+DomainAppender
+Domains Project
 domainsproject.org
+DomainStatsBot
 DomCopBot
-Dotbot
 Download Wonder
+Dragonfly
 DSearch
-DTS Agent
-eCatch
-Ecxi
-EirGrabber
-evc-batch
+EMail Wolf
+EroCrawler
+Evil
+Exabot
 Express WebPictures
-EyeNetIE
-Ezooms
-facebookscraper
-FHscan
+ExtractorPro
+FemtosearchBot
 Fimap
-Foobot
+Firefox/7.0
 Freeuploader
-Fyrebot
+FrontPage
+FyberSpider
 GalaxyBot
-Getintent
-Gigablast
+GetRight
+GetWeb
+Gigabot
 gopher
+Gotit
 GoZilla
 Grabber
 GrabNet
-Grafula
 GridBot
-Harvest
+HaosouSpider
 Havij
 HEADMasterSEO
-Hloader
-HTTP::Lite
-HTTrack
+heritrix
+Heritrix
+HTMLparser
+Humanlinks
+HybridBot
 IDBTE4M
-Image Sucker
-IndeedBot
 InfoNaviRobot
-InterGET
+Internet Ninja
 InternetSeer
-ips-agent
-IRLbot
-JennyBot
+internetVista monitor
+Iskanie
+JamesBOT
+Jbrofuzz
 JetCar
-JOC Web Spider
+Jetty
+JikeSpider
 Joomla
-JustView
+Jorgee
 Keyword Density
-Kozmosbot
+Lanshanbot
 Larbin
+LeechFTP
+LeechGet
+Lightspeedsystems
 Likse
-LinkextractorPro
-LinkWalker
-Lipperhey
+Linkbot
+Linkdexbot
+LinkpadBot
+LinkScan
+LinksManager
 Litemage_walker
-Lmspider
 LNSpiderguy
 Ltx71
-Majestic SEO
-Majestic-SEO
-MarkMonitor
+Magnet
+Mag-Net
+Mail.RU_Bot
+Majestic12
 Masscan
-Meanpathbot
+meanpathbot
+MeanPath Bot
 Mediatoolkitbot
-MicroMessenger
-Mister PiX
-MQQBrowser
+mediawords
+Microsoft Data Access
+MIDown tool
+MJ12bot
+Mozlila
 MSFrontPage
-MSIECrawler
+Msrabot
 MTRobot
-muhstik-scan
 Musobot
-Navroad
+Name Intelligence
+Nameprotect
 NearSite
-NetAnts
+Netcraft
 netEstate NE Crawler
-NetMechanic
-NetSpider
-Net Vampire
+Nettrack
 Netvibes
-NICErsPRO
+NetZIP
 Nikto
 NimbleCrawler
-Nmap
+Nuclei
 Nutch
 oBot
-Offline Explorer
-OnCrawl
-Openfind
-OpenLinkProfiler
+Openvas
+OpenVAS
 OPPO A33
 OrangeBot
+OrangeSpider
 Page Analyzer
+PageGrabber
+PageThing.com
 Pandalytics
 Panscient
-Petalbot
-Picscout
-PictureFinder
-Pimonster
-PleaseCrawl
+pcBrowser
+PECL::HTTP
 Pockey
 POE-Component-Client-HTTP
-Probethenet
+ProPowerBot
+ProWebWalker
 Pu_iN
-PyCurl
+Pump
+Quick-Crawler
 RankActive
-RankActiveLinkBot
-RankFlex
-RankingBot2
-Rankivabot
-RealDownload
-Reaper
+RankingBot
+RankurBot
+RebelMouse
 RedesScrapy
-Ripper
+ReGet
+RepoMonkey
+RocketCrawler
 RSSingBot
 s1z.ru
-satoristudio.net
+SBIder
+Scanbot
 scan.lol
-Screaming
+ScoutJet
+Scrapy
+ScreenerBot
+Searchestate
 SearchmetricsBot
 Semrush
+SemrushBot
 SentiBot
 SEOkicks
 SEOkicks-Robot
-SEOlyticsCrawler
 SEOprofiler
-SeoSiteCheckup
-serpstatbot
+SEOstats
 sexsearcher
-Siphon
+SISTRIX
+sitechecker.pro
+SiteExplorer
 Siteimprove
-Site Sucker
-Sitevigil
+SiteSnagger
+SiteSucker
 SmartDownload
-Snake
-Snapbot
-Snoopy
-Sociscraper
-sogouspider
+SocialRankIOBot
 Sosospider
-SpaceBison
-Spammen
 SpankBot
-sp_auditbot
+Spanner
+Spbot
 Sqlworm
+Sqworm
+Sucker
 SuperBot
-Suzuran
+Swiftbot
+Teleport
 TeleportPro
 Thumbor
-Trendiction
+Titan
+Toweyabot
+trendiction.com
 trendiction.de
-Turnitin
-TurnitinBot
-Twice
-Typhoeus
+Turingos
 UnisterBot
-URLy Warning
-VB Project
-VCI
-VelenPublicWebCrawler
-VeriCiteCrawler
-VoidEYE
+Vacuum
+Vagabondo
+Virusdie
 Voil
 Voltron
-Web Auto
-WebCollage
+Wallpapers/3.0
+WBSearchBot
+WebAuto
+WebBandit
+Web Collage
+WebCopier
 WebEnhancer
 WebFetch
-WebFuck
+Web Fetch
+Web Fuck
 WebImageCollector
+WebLeacher
 WebmasterWorldForumBot
-Web Sauger
-Website Quester
+WebPix
+Webshag
+WebsiteExtractor
 Webster
 Web Sucker
 WebZIP
 WeSEE
-Whack
 Whacker
-WinHTTrack
-WiseGuys Robot
-WISENutbot
-Woobot
+Who.is Bot
 Wotbox
-WPScan
-WWW-Mechanize
+Wprecon
+WWW::Mechanize
 WWWOFFLE
+Xaldon WebSpider
+Xaldon_WebSpider
+xpymep1.exe
 YoudaoBot
-Zade
+Zeus
 Zitebot
-ZmEu
 ZoomBot
 ZoominfoBot
+ZumBot