V4.2022.05.3096 [ci skip]

dev-tools/test_units/random-bots-for-whitelist-test.list

@@ -1,250 +1,250 @@
-360Spider
-AIBOT
-AfD-Verbotsverfahren
-AiHitBot
-Aipbot
-AllSubmitter
+01h4x.com
+404checker
+ASPSeek
+Aboundex
+AhrefsBot
+Alexibot
 Anarchie
-Anarchy
-Apexoo
 Aspiegel
-Atomseobot
 Attach
 AwarioRssBot
-AwarioSmartBot
-BBBike
+BDCbot
 BDFetch
-BacklinkCrawler
-Bandit
-Barkrowler
+BackWeb
+Backlink-Ceck
+Badass
+BatchFTP
+Battleztar Bazinga
+BetaBot
+Bigfoot
 Bitacle
-Black Hole
-BlackWidow
-Blow
-Brandwatch
+Boardreader
+Brandprotect
+Buck
 Buddy
 BuiltBotTough
-BuiltWith
-BunnySlippers
-CATExplorador
+Bullseye
 CCBot
-CODE87
 CSHttp
 Cegbfeieh
+CheeseBot
 ChinaClaw
-Chlooe
-Cliqzbot
+Claritybot
 Cloud mapping
 Cocolyzebot
-Collector
+Copier
+Copyscape
+Cosmos
 Craftbot
-Crawling at Home Project
-CrazyWebCrawler
-Curious
-Custo
+Crescent
+CrunchBot
+CyotekWebCopy
 DBLBot
+DataCha0s
 DatabaseDriverMysqli
-Deusu
-Devil
-Digincore
+Demon
 Dirbuster
 Discobot
 Dispatch
-DittoSpyder
+DomCopBot
 DomainAppender
-Dotbot
-Dragonfly
-EMail Siphon
-EirGrabber
-EroCrawler
-Evil
-Express WebPictures
+DomainCrawler
+Domains Project
+ECCP/1.0
+EMail Wolf
 Extractor
 ExtractorPro
-Extreme Picture Finder
+Ezooms
 FHscan
 FemtosearchBot
 Firefox/7.0
-Foobot
-Freeuploader
-Fuzz
-FyberSpider
-Fyrebot
-GT::WWW
+FlashGet
+Flunky
+G-i-g-a-b-o-t
 GalaxyBot
-GetRight
-Gigabot
+Genieo
+Getintent
 Go!Zilla
-Go-Ahead-Got-It
-Gotit
-HTMLparser
+GoZilla
+Grabber
+GrapeshotCrawler
+HMView
 HTTP::Lite
-HTTrack
-Humanlinks
+Haansoft
+HaosouSpider
+HonoluluBot
 IDBTE4M
 IDBot
 IRLbot
 Iblog
-Id-search
 Image Fetch
-IndeedBot
-InfoTekies
-Iria
-JOC Web Spider
-JamesBOT
-JetCar
+Intelliseek
+InterGET
+JennyBot
+JikeSpider
+Joomla
 Jorgee
 JustView
-Kenjin Spider
-Keybot Translation-Search-Machine
 Keyword Density
-Kinza
+Lanshanbot
+Larbin
 Leap
-Lightspeedsystems
-LinkScan
+LexiBot
+Lftp
+Libwhisker
+LinkWalker
+Linkbot
 LinkpadBot
-LinksManager
-Lipperhey
+LinqiaMetadataDownloaderBot
+LinqiaRSSBot
 Lipperhey Spider
 Litemage_walker
-Ltx71
 MFC_Tear_Sample
+MIDown tool
+MIIxpc
 MQQBrowser
 MSIECrawler
-MTRobot
-Mail.RU_Bot
+Mag-Net
+Magnet
 Majestic SEO
-Majestic12
-MarkWatch
-Masscan
-Mata Hari
-MauiBot
-Mb2345Browser
+MeanPath Bot
+MegaIndex.ru
 MicroMessenger
 Mojeek
-Mojolicious
-Mr.4x3
+MolokaiBot
 Msrabot
-NPbot
-Name Intelligence
+NICErsPRO
 Navroad
 Needle
-NetAnts
-NetMechanic
+Nessus
+NetLyzer
 NetZIP
 Netvibes
-Not
-Nuclei
+NextGenSearchBot
+Niki-bot
+Nimbostratus
 Nutch
+Offline Navigator
 OnCrawl
-Openfind
-OrangeBot
-OrangeSpider
-PECL::HTTP
-POE-Component-Client-HTTP
-PageAnalyzer
-PageGrabber
-Panscient
+Page Analyzer
+PageScorer
+Papa Foto
+Pavuk
+Petalbot
 Pi-Monster
-Picscout
 Pimonster
+Pixray
 PleaseCrawl
-Probethenet
-Quick-Crawler
+ProPowerBot
+ProWebWalker
+QueryN Metasearch
 RankingBot2
-Re-re
+RealDownload
+RebelMouse
+RepoMonkey
 RocketCrawler
+Rogerbot
+SEOkicks
 SEOkicks-Robot
-SMTBot
+SEOlyticsCrawler
+SEOprofiler
+SEOstats
 ScanAlert
-Searchestate
-SemanticJuice
+Scanbot
+Screaming
+ScrepyBot
 Semrush
-SemrushBot
 SeoSiteCheckup
-Seomoz
+SeobilityBot
+Shodan
 Siphon
 SiteExplorer
-SiteSnagger
+SiteLockSpider
+SiteSucker
 Siteimprove
-Snake
-Snoopy
+Snapbot
 SocialRankIOBot
 Sociscraper
 Sogou web spider
+Sosospider
 SpaceBison
+Spammen
 Spanner
 Spbot
-SputnikBot
-Sqlworm
-Sqworm
-Steeler
-SuperBot
-Surfbot
-TeleportPro
-Toata
+Sucuri
+SurveyBot
+Teleport
+Telesphoreo
+Telesphorep
+Thumbor
 Toweyabot
 Tracemyfile
-Trendictionbot
+True_Robot
+Turingos
+Turnitin
+TurnitinBot
 Typhoeus
-URLy.Warning
+V-BOT
 VB Project
 Vacuum
-VeriCiteCrawler
-Voltron
-WEBDAV
+Vagabondo
+VelenPublicWebCrawler
+VidibleScraper
+Virusdie
+VoidEYE
+WASALive-Bot
+WBSearchBot
 WISENutbot
-WWW-Mechanize
 WWW::Mechanize
-WWWOFFLE
-Wallpapers
-Wallpapers/3.0
 WallpapersHD
 WeSEE
-Web Fuck
-WebAuto
-WebCollage
+Web Collage
+Web Pix
+WebEnhancer
 WebFetch
-WebFuck
+WebImageCollector
+WebLeacher
+WebPix
 WebReaper
 WebWhacker
-WebZIP
-Webalta
-WebmasterWorldForumBot
-Whack
-Widow
-WinHTTrack
-WiseGuys Robot
-Wonderbot
-Wotbox
-Xenu
+Webshag
+WebsiteExtractor
+WebsiteQuester
+Whacker
+Xaldon WebSpider
 YoudaoBot
-Zade
-Zermelo
-ZyBorg
-check1.exe
+Zeus
+Zitebot
+ZoomBot
+ZoominfoBot
+adscanner
+archive.org_bot
+arquivo.pt
+backlink-check
+cognitiveseo
 crawler4j
-evc-batch
-facebookscraper
-heritrix
+domainsproject.org
+eCatch
+instabid
 internetVista monitor
 isitwp.com
-iubenda-radar
 lwp-request
+meanpathbot
 mediawords
-muhstik-scan
-oBot
 polaris version
-ripz
-seocompany.store
+probe-image-size
+s1z.ru
+satoristudio.net
+scalaj-http
+seobility
 seoscanners
 seostar
-sexsearcher
-siteripz
-sogouspider
+sitechecker.pro
 sp_auditbot
+spyfu
+tAkeOut
 trendiction.com
-trendiction.de
+webmeup-crawler
+webprosbot
 x09Mozilla
-xpymep1.exe
 zgrab