V4.2021.09.2582 [ci skip]

dev-tools/test_units/random-bots-for-whitelist-test.list

@@ -1,250 +1,250 @@
-404enemy
-80legs
 ASPSeek
-Aboundex
-Aboundexbot
 Aipbot
+AllSubmitter
+Alligator
 AlphaBot
+Anarchie
 Anarchy
-Attach
-BDCbot
+Anarchy99
+Ankit
+Aspiegel
+AwarioRssBot
+AwarioSmartBot
+BackDoorBot
 BackStreet
 BackWeb
-BacklinkCrawler
-Bandit
-BatchFTP
+Barkrowler
 Battleztar Bazinga
-BetaBot
 Bigfoot
-Bitacle
 Black Hole
-BlackWidow
-BlowFish
+Blackboard
 Boardreader
-Bolt
-Brandwatch
-BuiltWith
-Bullseye
+BotALot
+Buck
+Buddy
 BunnySlippers
-BuzzSumo
-CATExplorador
-Calculon
-CensysInspect
+CSHttp
+CazoodleBot
+Cegbfeieh
 CheTeam
 ChinaClaw
-Claritybot
-Cloud mapping
-Cocolyzebot
-Collector
+Chlooe
+Cogentbot
 Copier
-CopyRightCheck
 Cosmos
 Crawling at Home Project
-CrunchBot
-Custo
-DIIbot
+CrazyWebCrawler
+Crescent
+DBLBot
 DTS Agent
-DataCha0s
 DatabaseDriverMysqli
-Demon
-Dirbuster
-Disco
-Discobot
-Discoverybot
-DnyzBot
+Devil
+DigitalPebble
 DomainAppender
-DomainCrawler
-Drip
-EMail Wolf
+DomainSigmaCrawler
+DomainStatsBot
+EMail Siphon
 EasyDL
-EirGrabber
-ExtLinksBot
+Evil
 Extractor
 ExtractorPro
+Extreme Picture Finder
 EyeNetIE
-Ezooms
-FemtosearchBot
-Fimap
+FDM
+FHscan
 Firefox/7.0
-Flunky
-Foobot
-FrontPage
+FlashGet
+Fuzz
 FyberSpider
-GetWeb
+G-i-g-a-b-o-t
+GT::WWW
 Go!Zilla
 Go-Ahead-Got-It
-Gotit
 Grabber
-Grafula
-GridBot
+GrapeshotCrawler
+HMView
+HTMLparser
+HTTP::Lite
 HTTrack
 Haansoft
-Havij
+HaosouSpider
 Heritrix
-Humanlinks
+Hloader
+HybridBot
 IDBot
-IRLbot
 Iblog
-Id-search
-IlseBot
-Image Fetch
-InterGET
+Indy Library
+InfoTekies
+Intelliseek
 Internet Ninja
-InternetSeer
-Iskanie
-JOC Web Spider
 Jbrofuzz
+JennyBot
+JetCar
 Jetty
 Jorgee
 JustView
-Jyxobot
-Keyword Density
+Kenjin Spider
+Kinza
+Kozmosbot
+Larbin
 LeechFTP
 LeechGet
-LexiBot
-Libwhisker
-LinkWalker
-LinkextractorPro
+Lftp
+LieBaoFast
+Lightspeedsystems
+Linkbot
+Linkdexbot
 LinqiaMetadataDownloaderBot
-LinqiaScrapeBot
-Lipperhey
-Lipperhey Spider
-Litemage_walker
-Ltx71
 MJ12bot
-MQQBrowser
+MSFrontPage
+MSIECrawler
+MTRobot
 Mag-Net
-Mail.RU_Bot
-Mata Hari
-Mb2345Browser
-Mediatoolkitbot
-Microsoft Data Access
-Microsoft URL Control
+MarkMonitor
 Mister PiX
-Morfeus Fucking Scanner
+Mojeek
+Mojolicious
 Mr.4x3
-Msrabot
-Name Intelligence
+NICErsPRO
 Navroad
 NearSite
 Net Vampire
-NetAnts
-NetLyzer
-NetMechanic
-NetSpider
 NetZIP
-Netcraft
-Nettrack
-Niki-bot
-Offline Navigator
+NextGenSearchBot
+Nikto
+Nmap
+Nutch
 OnCrawl
-OpenVAS
-Openfind
-Openvas
 OrangeSpider
-POE-Component-Client-HTTP
-Pandalytics
+OutfoxBot
+PageScorer
+PageThing.com
+Panscient
+Papa Foto
 Pavuk
-Picsearch
+PeoplePal
+Picscout
+PictureFinder
+Pimonster
+Pixray
+Pockey
+ProPowerBot
 ProWebWalker
-Probethenet
 Psbot
 Pump
-PyCurl
 QueryN Metasearch
-RSSingBot
 RankActive
+RankActiveLinkBot
 RankingBot
+RankingBot2
 Rankivabot
 RankurBot
-RepoMonkey
+Re-re
+ReGet
+RebelMouse
+Recorder
+RedesScrapy
 Ripper
+Rogerbot
 SBIder
+SEOkicks
 SEOkicks-Robot
-SEOprofiler
-SEOstats
 SISTRIX
 SMTBot
 ScanAlert
-ScoutJet
+Scanbot
+Scrapy
+ScreenerBot
 ScrepyBot
-SemrushBot
-SeobilityBot
-Seomoz
+SeoSiteCheckup
 Siphon
-SiteCheckerBotCrawler
-SiteExplorer
 SiteLockSpider
 SiteSnagger
-Sitebeam
+Sitevigil
 SmartDownload
-Snake
+Snoopy
+SocialRankIOBot
 Sociscraper
+Sogou web spider
+Sottopop
 SpaceBison
-SputnikBot
+Spanner
+Spbot
 Sqlmap
-Sqworm
+Steeler
 Stripper
-Suzuran
+Sucuri
+SurveyBot
 Swiftbot
-TeleportPro
+Szukacz
+T0PHackTeam
 Telesoft
 Telesphoreo
-Toata
-Tracemyfile
-Trendiction
-True_Robot
+Telesphorep
+TheNomad
+Thumbor
+TightTwatBot
+Turingos
 Turnitin
+TurnitinBot
+Typhoeus
 URLy Warning
-URLy.Warning
 UnisterBot
+Upflow
 V-BOT
+VB Project
+VCI
+Vagabondo
 VelenPublicWebCrawler
 VidibleScraper
-Virusdie
-WEBDAV
 WPScan
+WWW-Collector-E
+WWWOFFLE
 WallpapersHD
-Web Auto
+Web Enhancer
 Web Pix
-Web Sucker
-WebFetch
+Web Sauger
+WebEnhancer
+WebFuck
 WebImageCollector
-WebLeacher
-WebPix
-WebSauger
-WebSucker
+WebStripper
 Webalta
-WebsiteExtractor
-Webster
-Whack
+WebmasterWorldForumBot
+Webshag
+Website Quester
+Whacker
 Widow
-WiseGuys Robot
-Wonderbot
+WinHTTrack
 Woobot
-Wotbox
+Wprecon
 Xaldon WebSpider
+Xenu
+Zade
+Zermelo
 Zitebot
-autoemailspider
-backlink-check
-coccocbot-web
+ZumBot
 cognitiveseo
-domainsproject.org
-eCatch
+com.plumanalytics
+crawler.feedback
+demandbase-bot
 facebookscraper
-heritrix
-ips-agent
-lwp-trivial
+gopher
+lwp-request
+meanpathbot
 mediawords
+muhstik-scan
 oBot
+page scorer
+pcBrowser
 polaris version
-satoristudio.net
-scan.lol
-seobility
+probe-image-size
+ripz
+s1z.ru
+scalaj-http
 seostar
 siteripz
 sogouspider
-tAkeOut
-trendiction.de
-webmeup-crawler
-x09Mozilla
+sp_auditbot
+trendiction.com
+webprosbot
+zgrab