V4.2021.09.2608 [ci skip]

dev-tools/test_units/random-bots-for-whitelist-test.list

@@ -1,21 +1,15 @@
-404checker
-404enemy
 80legs
-ASPSeek
-Abonti
-Aboundexbot
-AhrefsBot
-Aipbot
-AllSubmitter
+ADmantX
+AIBOT
+Aboundex
+AiHitBot
+Alligator
 AlphaBot
-Anarchy
-Anarchy99
+Anthill
 Apexoo
-Aspiegel
-Attach
+Asterias
 AwarioRssBot
 AwarioSmartBot
-BBBike
 BDCbot
 BDFetch
 BackDoorBot
@@ -23,228 +17,234 @@ BackStreet
 Backlink-Ceck
 BacklinkCrawler
 Bandit
-Battleztar Bazinga
+BetaBot
 BlackWidow
-Blackboard
+Blow
 BlowFish
-Boardreader
-Bullseye
-BuzzSumo
-Calculon
+BotALot
+Buck
+BuiltBotTough
+CATExplorador
+CCBot
 CazoodleBot
 Cegbfeieh
-CensysInspect
-CheTeam
-ChinaClaw
 Claritybot
-Cloud mapping
-Cogentbot
 Collector
-Craftbot
+CopyRightCheck
 Crawling at Home Project
 CrazyWebCrawler
-CrunchBot
-CyotekWebCopy
-DBLBot
-DIIbot
-DSearch
+Crescent
+Curious
 DatabaseDriverMysqli
+Demon
 Deusu
-Devil
-Digincore
+DigitalPebble
 Disco
-Discoverybot
-DittoSpyder
-DnyzBot
+Dispatch
 DomainAppender
 DomainCrawler
 DomainSigmaCrawler
 DomainStatsBot
-EMail Siphon
+Domains Project
+Download Wonder
+ECCP/1.0
 Ebingbong
+Ecxi
+EirGrabber
+EroCrawler
 Exabot
+Express WebPictures
 ExtLinksBot
-Extractor
+ExtractorPro
+Extreme Picture Finder
 Ezooms
-FlashGet
+FemtosearchBot
+Fimap
+Firefox/7.0
+FrontPage
 Fuzz
 FyberSpider
-GT::WWW
-GalaxyBot
+G-i-g-a-b-o-t
 Genieo
-GermCrawler
-GetRight
 GetWeb
 Getintent
-Go-Ahead-Got-It
-HEADMasterSEO
-HMView
-Harvest
+Gigabot
+Go!Zilla
+GrabNet
+Grafula
+Haansoft
 Hloader
-HybridBot
+Humanlinks
+IDBTE4M
 IDBot
 IlseBot
-Image Sucker
 IndeedBot
-Intelliseek
-InterGET
 InternetSeer
+Iria
+JOC Web Spider
+JennyBot
 JetCar
-Jetty
-JikeSpider
-Keyword Density
+Kenjin Spider
 LNSpiderguy
+LWP::Simple
 Larbin
-Leap
-LeechFTP
-LeechGet
-LexiBot
-Lightspeedsystems
-Likse
+Lftp
+Libwhisker
+LieBaoFast
+Linkbot
+Linkdexbot
 LinkextractorPro
-LinqiaMetadataDownloaderBot
+LinkpadBot
 Lipperhey
-MFC_Tear_Sample
 MQQBrowser
-MSFrontPage
-MSIECrawler
-Mag-Net
-Mail.RU_Bot
-Mass Downloader
-MauiBot
-Meanpathbot
-Moblie Safari
+Magnet
+Majestic SEO
+MarkMonitor
+MeanPath Bot
+Microsoft Data Access
+Microsoft URL Control
 Mojeek
-Msrabot
-Navroad
+Morfeus Fucking Scanner
+NPbot
+Nameprotect
+NearSite
+Needle
 Nessus
-NetMechanic
-NextGenSearchBot
-Ninja
+NetAnts
+NetSpider
+Nettrack
+Netvibes
+Nibbler
+Nikto
+NimbleCrawler
+Nmap
+Nutch
+Octopus
+Offline Explorer
+OnCrawl
 OpenLinkProfiler
 OpenVAS
 Openvas
-OrangeBot
 OrangeSpider
-PHPCrawl
-POE-Component-Client-HTTP
-PageAnalyzer
-PageThing.com
+OutfoxBot
+Page Analyzer
+PageScorer
 Pandalytics
-Panscient
 Papa Foto
-Pavuk
-Pi-Monster
+PeoplePal
+Petalbot
+Picscout
 Picsearch
+PictureFinder
+Pimonster
+PleaseCrawl
+Pockey
 ProPowerBot
-ProWebWalker
-RankActiveLinkBot
-RankingBot2
+Pu_iN
+Pump
+PxBroker
+Quick-Crawler
+RankingBot
 Rankivabot
+ReGet
 Reaper
-Recorder
+RebelMouse
 RedesScrapy
-RocketCrawler
-SEOlyticsCrawler
+Ripper
+SBIder
+SEOkicks
 SMTBot
-Scanbot
-ScoutJet
+Scrapy
 ScreenerBot
-Semrush
 SeoSiteCheckup
+SeobilityBot
 Seomoz
-Shodan
-Site Sucker
+SiteCheckerBotCrawler
 SiteSnagger
 SiteSucker
-Sitebeam
 Siteimprove
-Sitevigil
+SlySearch
+Snake
 Snoopy
 Sociscraper
 Sosospider
 Sottopop
-SpaceBison
-Spammen
-Spanner
-Spbot
+SpankBot
+Spinn3r
+Sqlmap
 Sqlworm
-Steeler
-Stripper
+Sqworm
 Sucker
-Sucuri
-SuperHTTP
+SuperBot
 Surfbot
+Swiftbot
 Teleport
 Telesoft
-Telesphoreo
-The Intraformant
+Telesphorep
 Thumbor
 Toata
-Tracemyfile
 Trendiction
-Turnitin
+Trendictionbot
+True_Robot
+Turingos
 TurnitinBot
 TwengaBot
 Twice
-URLy Warning
+Typhoeus
+URLy.Warning
 UnisterBot
 Upflow
-Vagabondo
+VCI
 VelenPublicWebCrawler
-VeriCiteCrawler
-VidibleScraper
-Virusdie
-VoidEYE
-Voil
-Voltron
-WISENutbot
-WPScan
-WWW-Mechanize
+WASALive-Bot
 Wallpapers
 Web Auto
+Web Enhancer
 Web Fetch
-Web Pix
-Web Sauger
-WebAuto
-WebCollage
+Web Sucker
+WebBandit
 WebCopier
-WebGo IS
-WebSauger
-WebStripper
-WebSucker
-WebmasterWorldForumBot
+WebEnhancer
+WebFetch
+WebLeacher
+WebPix
 Webshag
-WebsiteExtractor
 WebsiteQuester
 Webster
-Whacker
+Widow
+WinHTTrack
+Wonderbot
+Wotbox
 Wprecon
-Xaldon WebSpider
+Xaldon_WebSpider
 Xenu
 Zauba
 Zeus
+Zitebot
+ZmEu
 ZoomBot
+ZoominfoBot
+ZyBorg
+archive.org_bot
+arquivo.pt
 autoemailspider
-backlink-check
-cah.io.community
-clark-crawler
-coccocbot-web
-cognitiveseo
-com.plumanalytics
+crawl.sogou.com
 crawler.feedback
-gopher
+crawler4j
+dataforseo.com
 internetVista monitor
-isitwp.com
-lwp-request
 netEstate NE Crawler
 oBot
-satoristudio.net
-seobility
-seostar
-serpstatbot
-sexsearcher
+pcBrowser
+ripz
+s1z.ru
+scalaj-http
 siteripz
+sogouspider
+sp_auditbot
+spyfu
 trendiction.com
+trendiction.de
 ubermetrics-technologies.com
-webmeup-crawler
+x09Mozilla
+x22Mozilla