V4.2022.03.2951 [ci skip]

dev-tools/test_units/random-bots-for-whitelist-test.list

@@ -1,250 +1,250 @@
 01h4x.com
 404checker
-80legs
-ADmantX
+404enemy
 AIBOT
 ALittle Client
-Aboundexbot
-Acunetix
+ASPSeek
+Aboundex
+AhrefsBot
+AiHitBot
 Aipbot
-Alexibot
 AllSubmitter
-AlphaBot
-Anarchie
-Anarchy
+Alligator
 Anarchy99
-Ankit
-Atomseobot
-BDCbot
-BDFetch
-BackWeb
-Backlink-Ceck
+Apexoo
+Attach
+AwarioRssBot
+AwarioSmartBot
+BLEXBot
+BacklinkCrawler
 Bandit
-Black Hole
-BlackWidow
-Boardreader
-BotALot
+Barkrowler
+Blackboard
+Blow
+BlowFish
+Bolt
 Buck
 Buddy
-BuiltBotTough
-BuiltWith
+BuzzSumo
 CATExplorador
-CODE87
-Calculon
+CazoodleBot
 CheTeam
+CheeseBot
 CherryPicker
-Chlooe
-Citoid
-Claritybot
-Cliqzbot
+ChinaClaw
+Cogentbot
 Collector
 Copier
-Crawling at Home Project
-Curious
+Copyscape
+Cosmos
+CrazyWebCrawler
+Crescent
 Custo
-DTS Agent
-DatabaseDriverMysqli
-Discobot
+DIIbot
+DSearch
+Deusu
+DigitalPebble
+Dirbuster
+Disco
 Discoverybot
-DomainAppender
+Dispatch
+DnyzBot
+DomainCrawler
+DomainSigmaCrawler
+Domains Project
 Dotbot
-Dragonfly
-ECCP/1.0
-EasyDL
+Drip
 Ebingbong
-EirGrabber
-Evil
+Ecxi
+EroCrawler
 Exabot
-Express WebPictures
-EyeNetIE
+ExtLinksBot
+Extractor
+Extreme Picture Finder
 FDM
-Fimap
-FlashGet
+FHscan
+FemtosearchBot
 Flunky
-Freeuploader
-Fuzz
+Fyrebot
 G-i-g-a-b-o-t
 GalaxyBot
 Genieo
-Gigabot
+Getintent
 Go-Ahead-Got-It
 Gotit
+GrabNet
 Grabber
-HTTP::Lite
-Haansoft
-HaosouSpider
-Heritrix
-Hloader
-HybridBot
+Grafula
+GrapeshotCrawler
+GridBot
+HEADMasterSEO
+Harvest
+Havij
+HonoluluBot
+Humanlinks
+IDBTE4M
+IDBot
+IRLbot
+Iblog
 Image Sucker
-Indy Library
+IndeedBot
 InfoTekies
-Intelliseek
-Internet Ninja
-Iria
+Iskanie
+JOC Web Spider
 JennyBot
-Jetty
+JetCar
 JikeSpider
-Joomla
-JustView
-Kenjin Spider
-Kozmosbot
-LNSpiderguy
-LeechFTP
-Libwhisker
+Jyxobot
+Kinza
+Larbin
+Likse
+LinkScan
+LinkextractorPro
 LinkpadBot
 LinqiaMetadataDownloaderBot
 LinqiaScrapeBot
 Litemage_walker
-Ltx71
 MFC_Tear_Sample
 MIDown tool
 MIIxpc
-Mag-Net
-Magnet
+MQQBrowser
+MTRobot
+Majestic SEO
 Majestic-SEO
-Mass Downloader
 Masscan
-Mata Hari
+MauiBot
 MeanPath Bot
-Mediatoolkitbot
-Microsoft URL Control
-Minefield
-Mojolicious
+Mister PiX
 MolokaiBot
 Msrabot
-NPbot
-Name Intelligence
-Navroad
+Musobot
+NICErsPRO
+NearSite
+Needle
+Net Vampire
+NetAnts
 NetSpider
-NetZIP
+Netcraft
 Nettrack
-Netvibes
-Niki-bot
-NimbleCrawler
-Nimbostratus
+Nibbler
+Nikto
+Nmap
 Not
-Offline Navigator
+Nutch
+OpenLinkProfiler
 OpenVAS
+Openfind
 OrangeBot
-OutclicksBot
-OutfoxBot
-Page Analyzer
-PageScorer
-PageThing.com
-Panscient
-Petalbot
-Pi-Monster
-Picsearch
-Pimonster
+PECL::HTTP
+Papa Foto
+PeoplePal
+PictureFinder
 Pixray
-PleaseCrawl
 Psbot
-PyCurl
-Quick-Crawler
-RankingBot2
-RankurBot
+Pump
+PxBroker
+RSSingBot
+RankActiveLinkBot
+RankFlex
+Rankivabot
+Re-re
 RealDownload
 Recorder
-RepoMonkey
 Rogerbot
 SBIder
-SEOkicks
-SEOlyticsCrawler
 SISTRIX
-SalesIntelligent
-ScoutJet
-Scrapy
-Screaming
-ScrepyBot
+Searchestate
+SearchmetricsBot
+Seekport
 SemanticJuice
-Semrush
 SeoSiteCheckup
-SeobilityBot
+Seomoz
+Site Sucker
 SiteCheckerBotCrawler
+SiteLockSpider
 SiteSnagger
-Sitebeam
-Siteimprove
-Sitevigil
-SocialRankIOBot
+SiteSucker
+Snapbot
 Sosospider
-SpankBot
+Spanner
 Spbot
 Spinn3r
 Sqlmap
-Sucker
+Sqworm
+Stripper
 Sucuri
 SuperBot
-Surfbot
+SurveyBot
+Suzuran
+Szukacz
 T8Abot
-TeleportPro
-The Intraformant
+Telesoft
+Toata
+Toweyabot
+Tracemyfile
 Trendiction
-True_Robot
+Trendictionbot
 Turnitin
-TurnitinBot
-TwengaBot
-Twice
-Typhoeus
 URLy Warning
-URLy.Warning
-Upflow
+UnisterBot
 V-BOT
 VB Project
 VCI
-Vacuum
 VeriCiteCrawler
+VidibleScraper
+VoidEYE
+Voil
 WASALive-Bot
-WWW-Collector-E
+WEBDAV
+WISENutbot
 WWW-Mechanize
-WWW::Mechanize
-WWWOFFLE
+Wallpapers
+WeSEE
 Web Auto
-Web Enhancer
 Web Fetch
+WebCopier
 WebFetch
 WebGo IS
-WebStripper
-Webshag
+WebLeacher
+WebWhacker
+Webalta
 Website Quester
 WebsiteExtractor
 Webster
-Whatweb
 Who.is Bot
-Xenu
-Zauba
-Zermelo
+Widow
+YoudaoBot
+Zeus
 Zitebot
 ZoominfoBot
-ZyBorg
-adscanner
 arquivo-web-crawler
 backlink-check
-clark-crawler
-com.plumanalytics
+crawl.sogou.com
 crawler4j
-domainsproject.org
 evc-batch
 facebookscraper
+gopher
+heritrix
 instabid
+ips-agent
+iubenda-radar
 lwp-request
-lwp-trivial
+magpie-crawler
 mediawords
 muhstik-scan
-netEstate NE Crawler
-page scorer
-satoristudio.net
+polaris version
+ripz
 scalaj-http
-scan.lol
+seocompany.store
 seoscanners
-seostar
-sexsearcher
-sp_auditbot
-spyfu
+serpstatbot
+siteripz
 sysscan
 tAkeOut
-trendiction.com
+voyagerx.com
+webmeup-crawler
+webpros.com
 webprosbot
-x22Mozilla
+x09Mozilla
 xpymep1.exe
-zauba.io
+zgrab