V4.2022.07.3210 [ci skip]

dev-tools/test_units/random-bots-for-whitelist-test.list

@@ -1,162 +1,168 @@
-01h4x.com
-404enemy
+ADmantX
 AIBOT
-Acunetix
-AfD-Verbotsverfahren
+Aboundex
+Aboundexbot
+AhrefsBot
 AiHitBot
-AlphaBot
-Apexoo
-AwarioSmartBot
-BDCbot
-BDFetch
-Backlink-Ceck
+Aipbot
+AllSubmitter
+Anarchie
+Anarchy99
+Ankit
+BBBike
+BLEXBot
+BackStreet
 Badass
+Black Hole
 BlackWidow
+Blow
+BlowFish
 Boardreader
-Brandwatch
-Buck
+BotALot
+Brandprotect
 Buddy
-Bullseye
 BunnySlippers
 CCBot
+CSHttp
 CazoodleBot
+CensysInspect
+CheeseBot
 CherryPicker
 Chlooe
 Citoid
 Cliqzbot
-Cosmos
-Craftbot
-CrazyWebCrawler
-CrunchBot
-Custo
+Cloud mapping
+Cogentbot
+Copier
+CopyRightCheck
+Copyscape
 CyotekWebCopy
 DBLBot
-DIIbot
+DSearch
+Demon
+Deusu
 Devil
-Digincore
-DigitalPebble
-Dirbuster
+Disco
 Discobot
-Discoverybot
 DittoSpyder
-DomainAppender
-DomainCrawler
-Domains Project
-Dotbot
-Dragonfly
-ECCP/1.0
+DnyzBot
+DomCopBot
+DomainStatsBot
+EMail Siphon
+EMail Wolf
 EasyDL
-Ecxi
 EirGrabber
-EroCrawler
-Express WebPictures
-ExtLinksBot
+Exabot
 ExtractorPro
-Fimap
-FlashGet
+EyeNetIE
+Ezooms
+FHscan
+FemtosearchBot
 Flunky
 Freeuploader
 FrontPage
+Fuzz
 FyberSpider
 Fyrebot
-G-i-g-a-b-o-t
-GT::WWW
-Genieo
-Go-Ahead-Got-It
-Gotit
+GalaxyBot
+GetRight
+GetWeb
+GoZilla
 GrabNet
-GrapeFX
+Grabber
+Grafula
 GrapeshotCrawler
+HMView
+HTMLparser
+HTTP::Lite
 Haansoft
-HaosouSpider
+Havij
 Heritrix
-Hloader
-HybridBot
-IDBTE4M
-Iblog
-IlseBot
-IndeedBot
-Indy Library
-Intelliseek
+Humanlinks
+Image Fetch
+InfoNaviRobot
+InfoTekies
 InterGET
-InternetSeer
-Iria
+Internet Ninja
 Iskanie
+JOC Web Spider
 JamesBOT
-Jbrofuzz
-JetCar
-Jetty
+JennyBot
+Joomla
 Jorgee
-Kinza
+Kenjin Spider
 Lanshanbot
-Larbin
-Leap
-Lftp
-LibWeb
+LeechFTP
+LeechGet
+Libwhisker
 LieBaoFast
 Lightspeedsystems
+Likse
 LinkScan
 LinkWalker
+Linkbot
 LinkextractorPro
 LinkpadBot
-LinqiaMetadataDownloaderBot
 Ltx71
-MIDown tool
+MFC_Tear_Sample
 MJ12bot
 MQQBrowser
 MSIECrawler
-Mail.RU_Bot
-Majestic SEO
+Mag-Net
+Majestic-SEO
+Majestic12
+MarkMonitor
 MarkWatch
+Masscan
 MeanPath Bot
-Meanpathbot
-MegaIndex.ru
-Metauri
-MicroMessenger
-Microsoft URL Control
-Moblie Safari
+Mediatoolkitbot
+Minefield
+Mojeek
+Mojolicious
 Morfeus Fucking Scanner
-Mozlila
+Mr.4x3
+Msrabot
+NICErsPRO
+Nameprotect
+NearSite
 NetLyzer
-Netvibes
 Nibbler
+Niki-bot
+Nikto
 NimbleCrawler
-Nimbostratus
-Ninja
-Octopus
-OrangeBot
+Nmap
+Offline Explorer
+OnCrawl
+OpenLinkProfiler
+Openvas
+OutclicksBot
 OutfoxBot
-PHPCrawl
-PageAnalyzer
-PageGrabber
-PageScorer
-PageThing.com
+POE-Component-Client-HTTP
 Pandalytics
+Papa Foto
 Pavuk
+PeoplePal
+Petalbot
 Pi-Monster
 Picscout
-Piepmatz
+Picsearch
+PleaseCrawl
 ProPowerBot
 ProWebWalker
-Probethenet
-RankingBot
-RankingBot2
-RankurBot
-ReGet
-RebelMouse
-RedesScrapy
-Ripper
-RocketCrawler
-SEOkicks-Robot
-SEOlyticsCrawler
+PxBroker
+Re-re
+Reaper
+RepoMonkey
+SBIder
+SEOkicks
 SEOprofiler
-SISTRIX
-Scrapy
+SEOstats
+ScoutJet
 Screaming
-ScreenerBot
-ScrepyBot
 SearchmetricsBot
-SentiBot
+SemanticJuice
+SemrushBot
+SeobilityBot
 Shodan
 Site Sucker
 SiteExplorer
@@ -164,87 +170,81 @@ SiteLockSpider
 SiteSucker
 Sitebeam
 Siteimprove
-Snoopy
-Sociscraper
-SpaceBison
+SmartDownload
+SocialRankIOBot
+Spammen
 SpankBot
-Spbot
-Steeler
-SuperBot
-SuperHTTP
-Surfbot
-Swiftbot
-Szukacz
-TeleportPro
-The Intraformant
+Spinn3r
+Sqlmap
+Sucuri
+SurveyBot
+T0PHackTeam
+TheNomad
 TightTwatBot
-Toweyabot
-Trendiction
+Toata
+Turingos
 TurnitinBot
-URLy Warning
+Twice
 URLy.Warning
 UnisterBot
+Upflow
 V-BOT
-Vagabondo
+VeriCiteCrawler
 Virusdie
 Voil
 WASALive-Bot
-WPScan
+WWW-Collector-E
+WWW-Mechanize
 WWW::Mechanize
-WeSEE
-Web Enhancer
+Wallpapers
+Web Collage
 Web Fetch
-Web Pix
-Web Sucker
-WebCollage
-WebEnhancer
+Web Sauger
+WebAuto
 WebFetch
-WebFuck
 WebImageCollector
 WebLeacher
+WebPix
+WebReaper
+WebStripper
 WebSucker
-WebWhacker
-WebZIP
-Webalta
-WebmasterWorldForumBot
+Webshag
+Website Quester
 WebsiteExtractor
-WebsiteQuester
-Whatweb
+Webster
+Whacker
 Who.is Bot
-Widow
 Wonderbot
 Woobot
 Wotbox
-Xaldon WebSpider
+Xaldon_WebSpider
 Xenu
+Zauba
 Zermelo
-Zeus
 ZmEu
+ZoomBot
 ZumBot
-arquivo-web-crawler
+ZyBorg
+archive.org_bot
+backlink-check
+cah.io.community
 cognitiveseo
 com.plumanalytics
-crawl.sogou.com
-crawler.feedback
-dataforseo.com
-demandbase-bot
 domainsproject.org
-eCatch
-facebookscraper
 gopher
-heritrix
-internetVista monitor
-iubenda-radar
-linkdexbot
-meanpathbot
-muhstik-scan
-netEstate NE Crawler
+instabid
+ips-agent
+lwp-trivial
+page scorer
 polaris version
-ripz
+probe-image-size
 s1z.ru
 scalaj-http
-seoscanners
-sp_auditbot
-trendiction.de
-webgains-bot
-x09Mozilla
+seostar
+serpstatbot
+siteripz
+sogouspider
+voyagerx.com
+webmeup-crawler
+zauba.io
+zgrab