V4.2022.10.3427 [ci skip]

dev-tools/test_units/random-bots-for-whitelist-test.list

@@ -1,250 +1,250 @@
-404checker
-ASPSeek
-AhrefsBot
+360Spider
+80legs
+ADmantX
+AIBOT
+ALittle Client
+Abonti
+AfD-Verbotsverfahren
+AiHitBot
 Alligator
+AlphaBot
 Anarchy99
-Ankit
-Aspiegel
-AwarioRssBot
-AwarioSmartBot
+Asterias
 BDCbot
-BackDoorBot
-BackStreet
-BackWeb
-Backlink-Ceck
-Badass
+Battleztar Bazinga
 Bigfoot
-Bitacle
+Blackboard
 Blow
-BlowFish
-BotALot
+Bolt
+Brandprotect
+BuiltBotTough
 BuiltWith
-Bullseye
-BunnySlippers
 Cegbfeieh
 CensysInspect
-CheTeam
 CherryPicker
-Chlooe
-Citoid
-Claritybot
-Cliqzbot
-Cloud mapping
+ChinaClaw
+Cocolyzebot
+Cogentbot
 Collector
-Copier
 CopyRightCheck
-Copyscape
-Cosmos
+CrazyWebCrawler
+Crescent
+Curious
+CyotekWebCopy
 DBLBot
+DSearch
 DTS Agent
 DatabaseDriverMysqli
 Demon
 Devil
-Digincore
-Dirbuster
 Discoverybot
+DnyzBot
 DomCopBot
-DomainSigmaCrawler
-DomainStatsBot
+Domains Project
 Download Wonder
+ECCP/1.0
+EMail Siphon
+EMail Wolf
+EasyDL
 Ebingbong
-Evil
-EyeNetIE
-Ezooms
-FemtosearchBot
+Ecxi
+Exabot
+Express WebPictures
+FDM
+FHscan
+FlashGet
+Flunky
+Foobot
 Freeuploader
+FrontPage
+Fuzz
 Fyrebot
 GT::WWW
+Getintent
+Gigabot
 Go!Zilla
-GoZilla
-Gotit
+Grafula
 GrapeFX
-GrapeshotCrawler
-HEADMasterSEO
+GridBot
+HTMLparser
+HTTP::Lite
 HaosouSpider
-Harvest
-Humanlinks
+Heritrix
 HybridBot
 IDBTE4M
-IDBot
-IRLbot
 IlseBot
-InfoNaviRobot
-Intelliseek
-InterGET
-Iria
+Indy Library
+Internet Ninja
 Iskanie
-JetCar
-Jetty
-Kenjin Spider
+IstellaBot
+JOC Web Spider
+Joomla
+Jorgee
 Keyword Density
-Kinza
-Kozmosbot
 LNSpiderguy
-Leap
-LexiBot
-Lftp
+Lanshanbot
+Larbin
+LeechFTP
+LeechGet
 LieBaoFast
 Lightspeedsystems
-LinkextractorPro
-LinksManager
-LinqiaScrapeBot
-Lipperhey
+Likse
+LinkWalker
+Linkbot
+LinqiaMetadataDownloaderBot
+LinqiaRSSBot
 Litemage_walker
-Lmspider
+MJ12bot
 MSFrontPage
-MSIECrawler
-Magnet
+MTRobot
 Majestic SEO
-Mass Downloader
-Mata Hari
+Majestic12
+MarkWatch
 MauiBot
+Mb2345Browser
 MeanPath Bot
-Mediatoolkitbot
-MegaIndex.ru
+Metauri
+Microsoft URL Control
+Moblie Safari
+Mojeek
 Mojolicious
+MolokaiBot
 Msrabot
 Musobot
+NICErsPRO
 NPbot
-Name Intelligence
-NearSite
+Nameprotect
+Navroad
 Nessus
-NetSpider
+Net Vampire
+NetAnts
+NetMechanic
 NetZIP
+Netcraft
 Netvibes
-NextGenSearchBot
 Nibbler
 Nikto
-NimbleCrawler
-Not
+Nimbostratus
+Ninja
+Nmap
 Nuclei
-Nutch
 Offline Navigator
+OnCrawl
 Openfind
-OrangeSpider
-OutclicksBot
-OutfoxBot
-PECL::HTTP
-PageScorer
-PageThing.com
+PHPCrawl
+Page Analyzer
+PageAnalyzer
 Pandalytics
 Panscient
-Papa Foto
-Pi-Monster
 Picscout
 PictureFinder
-Pimonster
-Pockey
-ProWebWalker
+Piepmatz
+PleaseCrawl
 Probethenet
-Psbot
-Pump
-PxBroker
 QueryN Metasearch
-Quick-Crawler
 RSSingBot
-RankActiveLinkBot
-RankFlex
+RankingBot
 RankingBot2
-ReGet
-Recorder
-RedesScrapy
-RepoMonkey
-Rogerbot
+RankurBot
+Re-re
+Reaper
+RocketCrawler
 SBIder
-SEOkicks-Robot
-SEOlyticsCrawler
-SEOprofiler
+SMTBot
 ScoutJet
 Scrapy
-Screaming
-ScreenerBot
-ScrepyBot
 Searchestate
+SemanticJuice
 SentiBot
 Shodan
 Site Sucker
-SiteCheckerBotCrawler
-SmartDownload
-SocialRankIOBot
-Sosospider
+SiteLockSpider
+SiteSucker
+Sitebeam
+Siteimprove
+Sitevigil
+Snapbot
+Snoopy
+Sociscraper
+Sottopop
 SpaceBison
-SputnikBot
-Sucker
-SuperBot
+SpankBot
+Sqlmap
+Sqworm
 SurveyBot
+Suzuran
 Swiftbot
 T8Abot
 Teleport
-Telesoft
-Telesphorep
+TeleportPro
+Telesphoreo
 The Intraformant
-Thumbor
+TheNomad
+Titan
 Tracemyfile
-Trendictionbot
-True_Robot
+Trendiction
 Turingos
-TwengaBot
+TurnitinBot
+Typhoeus
+URLy Warning
+URLy.Warning
 UnisterBot
-Voil
-WBSearchBot
+Upflow
+VelenPublicWebCrawler
+VidibleScraper
+Virusdie
+WASALive-Bot
+WEBDAV
+WISENutbot
 WPScan
 WWW-Collector-E
-WWW-Mechanize
-WWWOFFLE
-Wallpapers/3.0
-WallpapersHD
 WeSEE
-Web Auto
 Web Collage
-Web Fetch
+Web Enhancer
 Web Pix
 Web Sucker
+WebAuto
+WebBandit
+WebCollage
+WebCopier
 WebFuck
-WebGo IS
 WebImageCollector
-WebLeacher
-WebSucker
+WebPix
+WebReaper
+WebStripper
 WebZIP
-Webalta
+WebmasterWorldForumBot
 Webshag
-WebsiteExtractor
 WebsiteQuester
+Webster
 Whacker
-Whatweb
-WiseGuys Robot
-Wotbox
-Xaldon WebSpider
-Xaldon_WebSpider
-Zermelo
-Zitebot
-ZoominfoBot
+Who.is Bot
+WinHTTrack
+Wonderbot
+Xenu
+YoudaoBot
+ZmEu
 ZumBot
-ZyBorg
 adscanner
-cah.io.community
-coccocbot
-crawl.sogou.com
+arquivo-web-crawler
+arquivo.pt
+autoemailspider
+check1.exe
+cognitiveseo
 crawler.feedback
-dataforseo.com
-domainsproject.org
+crawler4j
+demandbase-bot
 eCatch
-heritrix
-internetVista monitor
+ips-agent
 isitwp.com
-lwp-request
+linkdexbot
 mediawords
-pcBrowser
-ripz
+muhstik-scan
+oBot
 satoristudio.net
 scalaj-http
-scan.lol
-seocompany.store
-serpstatbot
-sogouspider
-trendiction.de
+seobility
 webgains-bot
+webprosbot
 x22Mozilla
-xpymep1.exe
 zauba.io
-zgrab