V4.2022.07.3261 [ci skip]

dev-tools/test_units/random-bots-for-whitelist-test.list

@@ -1,250 +1,250 @@
 01h4x.com
 404checker
-80legs
+ASPSeek
 Aboundex
+Aboundexbot
+AfD-Verbotsverfahren
+AhrefsBot
 AiHitBot
-Aipbot
-Anarchy
+AllSubmitter
+Alligator
+AlphaBot
+Anarchie
 Anarchy99
-Ankit
-Anthill
+Aspiegel
+Asterias
+Atomseobot
 AwarioRssBot
 BBBike
-BLEXBot
-BackDoorBot
-BackWeb
 Bandit
-Barkrowler
 BatchFTP
 Battleztar Bazinga
-BetaBot
-Bigfoot
 Bitacle
-Black Hole
-Blow
+BlackWidow
+Boardreader
+Bolt
 BotALot
 Brandwatch
-Buddy
+Buck
+BuiltWith
+Bullseye
 BuzzSumo
-CCBot
+CATExplorador
 CSHttp
-CheTeam
-Claritybot
+Chlooe
 Cocolyzebot
 Cogentbot
 Collector
-Craftbot
 Crawling at Home Project
-CrazyWebCrawler
+CrunchBot
 Curious
+Custo
+DBLBot
 DSearch
 DataCha0s
-Disco
 Discobot
-DnBCrawler-Analytics
+DnyzBot
 DomCopBot
 DomainAppender
+Domains Project
 Dotbot
+Download Wonder
 Dragonfly
-EMail Siphon
-Exabot
-Extractor
-Extreme Picture Finder
-Ezooms
-FDM
-FHscan
+EMail Wolf
+EasyDL
+Ebingbong
+Ecxi
+EroCrawler
+EyeNetIE
 FemtosearchBot
-FlashGet
-Flunky
-Foobot
-FrontPage
-Fuzz
-Fyrebot
 G-i-g-a-b-o-t
-GT::WWW
-GetRight
-Go-Ahead-Got-It
+Genieo
+Getintent
+Gigabot
+GrabNet
+GrapeFX
 GrapeshotCrawler
-HEADMasterSEO
-HTMLparser
-HTTrack
-Haansoft
+GridBot
 Havij
+Heritrix
+Hloader
+Humanlinks
+HybridBot
 IDBTE4M
-Iblog
-Id-search
-Intelliseek
+InterGET
 Iskanie
-JOC Web Spider
-Kinza
-LNSpiderguy
-Lanshanbot
-Leap
-LeechFTP
-LexiBot
+JamesBOT
+Jbrofuzz
+JennyBot
+Jetty
+Joomla
+Jorgee
+Kenjin Spider
+Kozmosbot
+LWP::Simple
+LibWeb
+Libwhisker
 Likse
 LinkScan
 Linkbot
+LinkextractorPro
 LinksManager
+LinqiaScrapeBot
 Lipperhey
-MSIECrawler
-Majestic SEO
-Majestic-SEO
-MarkMonitor
+Lipperhey Spider
+Litemage_walker
+Ltx71
+Mag-Net
+Magnet
+Mail.RU_Bot
 MarkWatch
-Mass Downloader
-Masscan
-MauiBot
-Microsoft Data Access
-Minefield
-Mister PiX
-Mojeek
-Mozlila
+MegaIndex.ru
+Metauri
+Moblie Safari
 Mr.4x3
 Msrabot
 Musobot
-Name Intelligence
 Nameprotect
-Net Vampire
+Navroad
+NearSite
+NetAnts
 NetLyzer
-NetZIP
-Netcraft
+NetMechanic
 Nettrack
-Netvibes
-Nibbler
-Nimbostratus
-Nuclei
-Octopus
-Offline Explorer
-Offline Navigator
-Openfind
+NextGenSearchBot
+Niki-bot
+Ninja
+Not
+Nutch
 Openvas
-OrangeBot
-OrangeSpider
-OutfoxBot
 PECL::HTTP
-POE-Component-Client-HTTP
+PHPCrawl
+Page Analyzer
+PageAnalyzer
 PageGrabber
-Papa Foto
+PageScorer
+PageThing.com
+Pandalytics
 Pavuk
 PeoplePal
-Pi-Monster
 Picscout
-Pimonster
+Piepmatz
+Pixray
 PleaseCrawl
-ProWebWalker
 Probethenet
-Psbot
 Pu_iN
-Quick-Crawler
+Pump
+PxBroker
+QueryN Metasearch
 RankActive
-Rankivabot
+RankingBot
+RankingBot2
 RankurBot
+ReGet
+RealDownload
+Recorder
 RedesScrapy
-RepoMonkey
-SBIder
-SEOkicks
-SEOkicks-Robot
-SEOlyticsCrawler
-SISTRIX
-SMTBot
-Scanbot
+Ripper
+RocketCrawler
+Rogerbot
+SEOprofiler
+SalesIntelligent
+ScanAlert
+Screaming
 ScrepyBot
-Searchestate
-SearchmetricsBot
 SentiBot
-SeoSiteCheckup
-SeobilityBot
-Seomoz
-Shodan
-Site Sucker
 SiteExplorer
+SiteLockSpider
+SiteSnagger
 SiteSucker
-SlySearch
+Sitebeam
+SmartDownload
 Snapbot
-Snoopy
+Sociscraper
 Sosospider
 Sottopop
 SpaceBison
-Spinn3r
+Spanner
+Spbot
+Sqlworm
 Sqworm
 Steeler
-Stripper
-SuperHTTP
-Surfbot
+Sucuri
+SuperBot
 SurveyBot
-Suzuran
 Swiftbot
+TeleportPro
 Telesphoreo
-The Intraformant
+Telesphorep
+Titan
 Toweyabot
-Tracemyfile
-Trendiction
 Trendictionbot
-True_Robot
-TurnitinBot
+Turnitin
+Twice
 Typhoeus
-URLy.Warning
 Upflow
 V-BOT
 VCI
+Vacuum
 Vagabondo
 VeriCiteCrawler
 VoidEYE
+Voil
 WBSearchBot
-WWW-Collector-E
-WWW::Mechanize
-WWWOFFLE
-Wallpapers
-Wallpapers/3.0
+WEBDAV
+WPScan
+WallpapersHD
+WeSEE
 Web Auto
 Web Collage
 Web Enhancer
-Web Fetch
-Web Pix
-Web Sucker
-WebFetch
-WebLeacher
+Web Fuck
+Web Sauger
+WebCollage
+WebEnhancer
 WebPix
-WebSucker
-WebmasterWorldForumBot
+WebReaper
+WebStripper
+WebWhacker
+WebZIP
 Website Quester
-WebsiteExtractor
+WebsiteQuester
 Webster
 Whatweb
-Widow
+Who.is Bot
+Wonderbot
 Woobot
-Wprecon
-YoudaoBot
+Xaldon_WebSpider
+Xenu
 Zermelo
-ZyBorg
-adscanner
+Zeus
+ZumBot
+arquivo-web-crawler
 arquivo.pt
 autoemailspider
-crawler.feedback
+cognitiveseo
+com.plumanalytics
 crawler4j
-demandbase-bot
+dataforseo.com
+domainsproject.org
+eCatch
+facebookscraper
 heritrix
-ips-agent
+instabid
 isitwp.com
-oBot
+lwp-request
+lwp-trivial
+magpie-crawler
+meanpathbot
+netEstate NE Crawler
+page scorer
 plumanalytics
-polaris version
 ripz
 satoristudio.net
-scan.lol
-seobility
+scalaj-http
 seocompany.store
-seoscanners
-seostar
-serpstatbot
-sitechecker.pro
-siteripz
-spyfu
-trendiction.com
-trendiction.de
-voyagerx.com
+sysscan
+tAkeOut
+webgains-bot
 webmeup-crawler
-webpros.com
-xpymep1.exe
-zgrab
+zauba.io