V4.2021.07.2453 [ci skip]

dev-tools/test_units/random-bots-for-whitelist-test.list

@@ -1,250 +1,250 @@
-404checker
-Aboundexbot
-AfD-Verbotsverfahren
+360Spider
+80legs
+ASPSeek
+Aboundex
+Acunetix
+AhrefsBot
 AiHitBot
-Aipbot
-Alexibot
-AllSubmitter
 Alligator
-Anarchie
-Anarchy
 Anarchy99
-AwarioSmartBot
-BBBike
+Anthill
+Attach
+BDCbot
+BDFetch
+BackDoorBot
 BackStreet
-BacklinkCrawler
 Badass
-BatchFTP
-BetaBot
-Bigfoot
-Bitacle
-Black Hole
+Barkrowler
+Battleztar Bazinga
+Blow
+BlowFish
 Bolt
 Brandprotect
 Brandwatch
 Buddy
+BuiltBotTough
+BuiltWith
 Bullseye
-CATExplorador
 CCBot
 CODE87
-CensysInspect
-Cliqzbot
-Cloud mapping
-Copier
+Cegbfeieh
+CheeseBot
+ChinaClaw
+Claritybot
+Cocolyzebot
 CopyRightCheck
+Copyscape
 Cosmos
-CrazyWebCrawler
-CrunchBot
-Curious
+Custo
+CyotekWebCopy
 DBLBot
 DTS Agent
+DataCha0s
+DatabaseDriverMysqli
 Deusu
-Digincore
+Devil
+Dirbuster
+Disco
 Discoverybot
-Dispatch
 DittoSpyder
-DomCopBot
-DomainCrawler
+DomainSigmaCrawler
 Dotbot
+Drip
+EMail Siphon
 EasyDL
+Ecxi
+EirGrabber
 EroCrawler
 Express WebPictures
+ExtLinksBot
 ExtractorPro
-FHscan
-FemtosearchBot
-Firefox/7.0
+EyeNetIE
 Flunky
-Foobot
-FrontPage
-FyberSpider
-G-i-g-a-b-o-t
 GT::WWW
+GalaxyBot
 Genieo
+GermCrawler
+GetRight
+GetWeb
+Getintent
+Gigablast
 Gigabot
-Go!Zilla
 GoZilla
 Gotit
-Grabber
-GrapeFX
-GrapeshotCrawler
-GridBot
-HEADMasterSEO
+GrabNet
 HMView
 HTMLparser
-HTTP::Lite
-Heritrix
+Haansoft
+HaosouSpider
+Havij
 Humanlinks
-IDBTE4M
-Image Fetch
+HybridBot
+IDBot
+IlseBot
+Image Sucker
 IndeedBot
-InfoNaviRobot
-InfoTekies
-Intelliseek
-InterGET
-Iskanie
+Iria
+JOC Web Spider
 JamesBOT
+Jbrofuzz
+JennyBot
 JetCar
-Joomla
+Jetty
 JustView
+Jyxobot
 Kenjin Spider
-Larbin
-LeechFTP
-LeechGet
+Keyword Density
+LNSpiderguy
+LexiBot
 Lftp
-Libwhisker
+LibWeb
 LieBaoFast
+Lightspeedsystems
 Likse
+LinkWalker
 Linkdexbot
-Lipperhey
+LinkpadBot
+LinksManager
+LinqiaRSSBot
 Lipperhey Spider
 Litemage_walker
+Lmspider
 Ltx71
-MIDown tool
-MJ12bot
-MTRobot
-Mail.RU_Bot
-Majestic12
-Mata Hari
-MeanPath Bot
-Microsoft Data Access
+MarkWatch
+Mass Downloader
+MauiBot
+Mediatoolkitbot
+MegaIndex.ru
+Metauri
 Microsoft URL Control
-Mister PiX
-Mojeek
+Moblie Safari
+Mojolicious
+Morfeus Fucking Scanner
+Mozlila
+Musobot
 NICErsPRO
-Name Intelligence
-Nameprotect
-Nessus
-Net Vampire
-NetLyzer
+Navroad
+NearSite
+Needle
+NetMechanic
 NetZIP
 Netcraft
-Nettrack
 NextGenSearchBot
 Nibbler
 Niki-bot
-NimbleCrawler
+Nikto
+Ninja
 Nmap
-OPPO A33
+Octopus
 Offline Explorer
 OnCrawl
-OutclicksBot
-OutfoxBot
+Openfind
+Openvas
+OrangeBot
+OrangeSpider
+PECL::HTTP
+PageAnalyzer
 PageGrabber
-PageThing.com
 Pandalytics
-Panscient
 PeoplePal
+Petalbot
 Picscout
-PictureFinder
-Piepmatz
-Pockey
-ProWebWalker
-Pump
-QueryN Metasearch
-RankActiveLinkBot
-RankingBot2
-RankurBot
-Re-re
-ReGet
-RealDownload
-RocketCrawler
-SEOkicks
-SEOkicks-Robot
-Screaming
-ScreenerBot
-ScrepyBot
-SeoSiteCheckup
-Seomoz
+Picsearch
+PleaseCrawl
+Pu_iN
+PyCurl
+RankFlex
+Rankivabot
+Reaper
+RedesScrapy
+RepoMonkey
+Ripper
+SBIder
+SEOlyticsCrawler
+SEOprofiler
+SEOstats
+SMTBot
+SalesIntelligent
+SearchmetricsBot
+Semrush
 Shodan
 SiteCheckerBotCrawler
 SiteExplorer
 SiteLockSpider
-SiteSnagger
 SiteSucker
-Sitebeam
-SmartDownload
-Snake
+Siteimprove
 Sociscraper
-Sogou web spider
-Spammen
-SpankBot
-Spanner
-Spbot
-SputnikBot
-Sqlmap
+Sottopop
+Spinn3r
+Sqlworm
 Sqworm
-Steeler
 Stripper
 SuperBot
-SuperHTTP
+SurveyBot
 Swiftbot
-T8Abot
+T0PHackTeam
+TeleportPro
+Telesoft
+Telesphorep
+The Intraformant
 TheNomad
+TightTwatBot
+Titan
+Toweyabot
 Tracemyfile
 Trendiction
 Trendictionbot
-Typhoeus
+TwengaBot
+Twice
 Upflow
 V-BOT
 VB Project
-Vacuum
+Vagabondo
 VelenPublicWebCrawler
 VidibleScraper
-VoidEYE
-Voltron
+WASALive-Bot
+WBSearchBot
 WISENutbot
-WPScan
-WWW::Mechanize
 WWWOFFLE
-Web Auto
-Web Collage
+WeSEE
 Web Enhancer
-Web Pix
+Web Fuck
 Web Sauger
-WebBandit
-WebCopier
-WebFuck
-WebLeacher
-WebPix
-WebReaper
-WebZIP
-Webshag
+WebAuto
+WebEnhancer
+WebFetch
+WebStripper
+Webalta
+WebsiteExtractor
 Whack
-Whacker
-Who.is Bot
 WinHTTrack
-Woobot
+WiseGuys Robot
 Wotbox
 Wprecon
 Xaldon WebSpider
-Xaldon_WebSpider
-YoudaoBot
-Zade
+Zauba
 Zermelo
-Zitebot
-ZmEu
-ZyBorg
+Zeus
+ZoomBot
+ZoominfoBot
+archive.org_bot
+arquivo.pt
+autoemailspider
+backlink-check
 check1.exe
-coccocbot-web
 cognitiveseo
+crawl.sogou.com
 crawler4j
 demandbase-bot
+eCatch
 evc-batch
-facebookscraper
-ips-agent
-isitwp.com
-lwp-request
-magpie-crawler
-meanpathbot
-muhstik-scan
+gopher
 netEstate NE Crawler
-ripz
-scalaj-http
-sexsearcher
-siteripz
+page scorer
+pcBrowser
+satoristudio.net
+seobility
+sogouspider
 sp_auditbot
-spyfu
-trendiction.com
-trendiction.de
-voyagerx.com
-x22Mozilla
+sysscan
+tAkeOut