V4.2021.12.2786 [ci skip]

dev-tools/test_units/random-bots-for-whitelist-test.list

@@ -1,250 +1,250 @@
+404checker
 404enemy
 80legs
-AhrefsBot
-Aipbot
+ASPSeek
+Abonti
+Aboundexbot
 Alligator
+AlphaBot
 Anarchy
-Apexoo
-Aspiegel
-AwarioSmartBot
-BLEXBot
-BackDoorBot
-BackStreet
+Attach
+AwarioRssBot
+BDCbot
+BDFetch
+BackWeb
 BacklinkCrawler
-Badass
+Bandit
+Battleztar Bazinga
 BetaBot
-Blackboard
-BlowFish
+Bigfoot
 Boardreader
+Buck
 BuiltBotTough
 Bullseye
-CATExplorador
+BuzzSumo
 CCBot
-CODE87
 Calculon
-Cegbfeieh
 CheTeam
-CheeseBot
-Cloud mapping
+Chlooe
+Claritybot
+Cliqzbot
 Cocolyzebot
 Cogentbot
 Collector
-Craftbot
+CrazyWebCrawler
+Crescent
 CrunchBot
 Curious
+CyotekWebCopy
 DBLBot
-DSearch
-DTS Agent
-DataCha0s
-Demon
+DIIbot
 Deusu
-Devil
-Digincore
-DnBCrawler-Analytics
+Disco
+Discobot
+DittoSpyder
 DnyzBot
-Download Wonder
+DomainCrawler
+DomainStatsBot
+Domains Project
+Dragonfly
 Drip
 ECCP/1.0
-EasyDL
+EMail Siphon
 Ebingbong
-Exabot
-ExtractorPro
-EyeNetIE
-Ezooms
-FemtosearchBot
-Fimap
+Ecxi
+ExtLinksBot
+Extreme Picture Finder
 Foobot
-Freeuploader
-FrontPage
 FyberSpider
 Fyrebot
+G-i-g-a-b-o-t
 GT::WWW
 Genieo
-GermCrawler
 GetRight
 Getintent
+Gigabot
+Go!Zilla
 Go-Ahead-Got-It
+GoZilla
 Gotit
+GrabNet
 Grabber
 Grafula
-GrapeFX
-GrapeshotCrawler
 HMView
-HTMLparser
-Haansoft
-HaosouSpider
+HTTrack
 Havij
-Humanlinks
+Heritrix
+Hloader
 HybridBot
-IDBot
 IRLbot
+Iblog
 Id-search
-Iria
+IlseBot
+IndeedBot
+Indy Library
+InfoTekies
+InterGET
+InternetSeer
 JOC Web Spider
-Jbrofuzz
+JamesBOT
 JennyBot
-JikeSpider
 Jorgee
-Jyxobot
+Kinza
 LNSpiderguy
 LWP::Simple
-Leap
-LeechGet
-LexiBot
-Libwhisker
-LieBaoFast
-Likse
-LinkWalker
+Lanshanbot
+Lftp
+LibWeb
+Lightspeedsystems
+Linkbot
+Linkdexbot
 LinksManager
-Litemage_walker
+Ltx71
+MFC_Tear_Sample
 MIDown tool
-MIIxpc
-MQQBrowser
+MJ12bot
+MSFrontPage
+MTRobot
+Mag-Net
 Magnet
-Mail.RU_Bot
-Majestic SEO
 MarkMonitor
 MarkWatch
-Mass Downloader
-MauiBot
-Mb2345Browser
+Mata Hari
 MeanPath Bot
-Meanpathbot
-MegaIndex.ru
+Microsoft URL Control
+Moblie Safari
 Mojeek
-Morfeus Fucking Scanner
+Mojolicious
 Mozlila
-NICErsPRO
-NPbot
-Nameprotect
-Navroad
+Musobot
+NearSite
 NetAnts
 NetLyzer
-NetMechanic
 NetZIP
+Netcraft
 Netvibes
 NextGenSearchBot
-Ninja
+Nimbostratus
+Not
 Nuclei
-Octopus
-Offline Explorer
+Openfind
 OrangeBot
-OutfoxBot
-Page Analyzer
-PageScorer
+OrangeSpider
+PECL::HTTP
+PHPCrawl
+POE-Component-Client-HTTP
+PageGrabber
 PageThing.com
 Panscient
-Pimonster
+Papa Foto
+PeoplePal
+Pi-Monster
+ProPowerBot
 ProWebWalker
+Probethenet
 Psbot
+Pu_iN
+Pump
 PxBroker
-RSSingBot
+PyCurl
+Quick-Crawler
 RankActiveLinkBot
+RankingBot
 Rankivabot
-RebelMouse
-Ripper
-SBIder
-SEOkicks-Robot
-SEOlyticsCrawler
-SEOstats
+RankurBot
+Re-re
+RealDownload
+RepoMonkey
 SISTRIX
 SMTBot
 ScanAlert
 ScoutJet
 Scrapy
-ScreenerBot
-SearchmetricsBot
-SemrushBot
+Screaming
+ScrepyBot
+Semrush
+SeobilityBot
 Siphon
-Site Sucker
-SiteExplorer
-SiteLockSpider
-Sitevigil
 SlySearch
-Snake
 Snapbot
 Snoopy
-Sociscraper
+Sogou web spider
 Sosospider
 Sottopop
 SpankBot
-Spinn3r
+Spbot
 SputnikBot
-Sqlmap
-Steeler
 Sucker
-SuperBot
-Surfbot
-Suzuran
-Szukacz
 T0PHackTeam
-T8Abot
 Teleport
-Telesoft
-TheNomad
-Thumbor
-TightTwatBot
-Trendictionbot
-URLy Warning
-UnisterBot
+TeleportPro
+Telesphoreo
+The Intraformant
+Toweyabot
+True_Robot
+TurnitinBot
+TwengaBot
+Twice
+URLy.Warning
 Upflow
-VeriCiteCrawler
-Virusdie
+V-BOT
+VB Project
+VCI
+Vacuum
+Vagabondo
 VoidEYE
 Voil
-Voltron
-WASALive-Bot
+WEBDAV
+WPScan
 WWW-Collector-E
+WWW::Mechanize
 WWWOFFLE
-Wallpapers/3.0
-WeSEE
-Web Auto
+WallpapersHD
 Web Enhancer
-Web Fuck
+Web Fetch
 Web Pix
-Web Sucker
 WebAuto
-WebBandit
-WebGo IS
-WebPix
+WebCopier
 WebReaper
 WebSauger
+WebSucker
 Webalta
 WebmasterWorldForumBot
-Webshag
+WebsiteExtractor
 WebsiteQuester
 Webster
 Whacker
-Widow
-WinHTTrack
-Wonderbot
-Wotbox
+WiseGuys Robot
+Wprecon
 Xaldon WebSpider
-Zermelo
 Zitebot
-ZoomBot
-ZyBorg
-archive.org_bot
-arquivo-web-crawler
+ZmEu
+ZoominfoBot
+ZumBot
+adscanner
+backlink-check
+check1.exe
+coccocbot-web
+cognitiveseo
 crawler.feedback
 crawler4j
-dataforseo.com
 domainsproject.org
-facebookscraper
-gopher
-internetVista monitor
-meanpathbot
+eCatch
+heritrix
+isitwp.com
+lwp-request
 mediawords
-probe-image-size
+netEstate NE Crawler
+polaris version
+satoristudio.net
 scan.lol
 seobility
-seocompany.store
 seoscanners
-sitechecker.pro
-sp_auditbot
-spyfu
-trendiction.com
-x22Mozilla
+sexsearcher
+siteripz
+ubermetrics-technologies.com
+voyagerx.com
+webgains-bot
+webmeup-crawler
+x09Mozilla
 xpymep1.exe
-zauba.io
-zgrab