V4.2022.03.3004 [ci skip]

dev-tools/test_units/random-bots-for-whitelist-test.list

@@ -1,250 +1,250 @@
-01h4x.com
 360Spider
-80legs
-ASPSeek
-Abonti
-Aboundexbot
-AfD-Verbotsverfahren
-AhrefsBot
+404checker
+ADmantX
+ALittle Client
+Acunetix
 AllSubmitter
-Alligator
 AlphaBot
-Anarchy99
+Anarchy
+Ankit
+Apexoo
+Aspiegel
 Attach
 AwarioRssBot
 AwarioSmartBot
-BDCbot
-BDFetch
+BLEXBot
 BackDoorBot
-BacklinkCrawler
-Badass
-Battleztar Bazinga
-BetaBot
+BackStreet
+Backlink-Ceck
 Bigfoot
-BlackWidow
-Boardreader
-Bolt
-BotALot
-Brandprotect
+Black Hole
+Blackboard
+Buck
 Buddy
+BuiltBotTough
 BuiltWith
-BunnySlippers
-Calculon
+Bullseye
+CODE87
 CazoodleBot
-ChinaClaw
-Chlooe
-Citoid
+CheTeam
+CherryPicker
 Cliqzbot
 Cloud mapping
-Craftbot
-Crawling at Home Project
-Crescent
-Curious
-Custo
+CopyRightCheck
+Cosmos
+CyotekWebCopy
+DBLBot
 DIIbot
+DSearch
+DTS Agent
+DataCha0s
+DatabaseDriverMysqli
 Demon
-DigitalPebble
 Disco
 Discobot
-DnBCrawler-Analytics
-DomCopBot
 DomainSigmaCrawler
-Dragonfly
-Drip
+DomainStatsBot
 EMail Siphon
 EMail Wolf
-EroCrawler
+Ecxi
+EirGrabber
 Evil
+ExtLinksBot
 Extractor
-ExtractorPro
 Extreme Picture Finder
-Fimap
+FDM
+FemtosearchBot
 Firefox/7.0
+FlashGet
 Flunky
-Fyrebot
+Freeuploader
+FrontPage
+FyberSpider
 G-i-g-a-b-o-t
-GalaxyBot
+GT::WWW
+Getintent
+Gigabot
 Go-Ahead-Got-It
-Grafula
+Gotit
 GridBot
-HMView
-HTMLparser
-HTTrack
-Havij
-Hloader
-Humanlinks
+HaosouSpider
+Harvest
+HonoluluBot
 IDBot
 IRLbot
+Iblog
 IlseBot
 Image Sucker
-Intelliseek
+IndeedBot
+InfoNaviRobot
 InterGET
-Iskanie
+Iria
+IstellaBot
+JamesBOT
 Jbrofuzz
 JetCar
+Jetty
 JikeSpider
 JustView
+Jyxobot
 Kenjin Spider
-Libwhisker
+Keyword Density
+Kinza
+Lanshanbot
+Larbin
 Lightspeedsystems
-LinkScan
-Linkbot
-LinksManager
+LinkpadBot
+LinqiaRSSBot
 LinqiaScrapeBot
 Lipperhey
+Litemage_walker
 Lmspider
+MFC_Tear_Sample
 MIDown tool
-MJ12bot
-MSFrontPage
-Majestic SEO
-Majestic-SEO
-Majestic12
-MarkMonitor
-MeanPath Bot
+MIIxpc
+MQQBrowser
+MSIECrawler
+Mag-Net
+Mail.RU_Bot
+Masscan
+Mata Hari
+MauiBot
+Mb2345Browser
+Meanpathbot
 Mediatoolkitbot
-Metauri
-MicroMessenger
-Microsoft URL Control
-Minefield
+MegaIndex.ru
+Microsoft Data Access
 Mister PiX
 Moblie Safari
 Mojeek
-Morfeus Fucking Scanner
 Mr.4x3
-Musobot
 NICErsPRO
-NPbot
-NetAnts
+Nameprotect
+Navroad
+Nessus
+Net Vampire
 NetLyzer
-NetSpider
+NetMechanic
+NetZIP
 Nettrack
+Netvibes
 NextGenSearchBot
-Nibbler
-Niki-bot
-Nuclei
+Nikto
+Nimbostratus
+Ninja
+Nmap
 Octopus
 Offline Explorer
+OnCrawl
+OpenLinkProfiler
+OpenVAS
+Openvas
+OrangeBot
+OrangeSpider
 OutclicksBot
-PageAnalyzer
-PageScorer
+OutfoxBot
+PECL::HTTP
 PageThing.com
-Pandalytics
 Pavuk
 PeoplePal
-Pi-Monster
-Picsearch
 Piepmatz
 Pimonster
 PleaseCrawl
 Pockey
-ProPowerBot
-ProWebWalker
-Psbot
-Pu_iN
-PxBroker
-QueryN Metasearch
+Pump
+Quick-Crawler
+RSSingBot
+RankActive
 RankingBot
-ReGet
-Reaper
-RebelMouse
-RedesScrapy
-RepoMonkey
+RankurBot
+Re-re
+RealDownload
 Ripper
-Rogerbot
-SBIder
 SEOkicks-Robot
-SISTRIX
+SEOprofiler
 Scrapy
-ScrepyBot
-Seekport
+ScreenerBot
+SemanticJuice
 Semrush
-SeobilityBot
-Shodan
-Siteimprove
+SentiBot
+Siphon
+Site Sucker
+SiteExplorer
+SiteSucker
+Sitevigil
+SmartDownload
+Snapbot
+Sogou web spider
+Sottopop
+SpaceBison
+SpankBot
 Spanner
-Sucuri
+Spbot
+Spinn3r
+Sqlworm
+Sucker
+SuperBot
 SuperHTTP
-Surfbot
-SurveyBot
-Swiftbot
-Szukacz
-T0PHackTeam
-T8Abot
-Telesoft
-Telesphoreo
-Thumbor
-TightTwatBot
-Titan
+Suzuran
+Telesphorep
+The Intraformant
+TheNomad
 Toata
 Toweyabot
-Tracemyfile
 Trendiction
-Turingos
 TurnitinBot
-Typhoeus
 URLy Warning
 URLy.Warning
-Upflow
-VB Project
-VCI
-VidibleScraper
-Virusdie
+UnisterBot
+VelenPublicWebCrawler
 Voil
 WBSearchBot
+WEBDAV
+WISENutbot
+WPScan
 WWW-Mechanize
-WWWOFFLE
-WallpapersHD
-Web Collage
+Wallpapers
+Wallpapers/3.0
+WeSEE
 Web Fetch
 Web Fuck
-Web Sauger
-WebFetch
+WebFuck
 WebGo IS
-WebImageCollector
-WebLeacher
+WebPix
+WebSauger
 WebStripper
-WebWhacker
 WebZIP
 Webshag
-WebsiteQuester
-Webster
+Website Quester
+Whacker
 Widow
-WinHTTrack
 Woobot
 Wotbox
-Wprecon
-Xenu
-Zermelo
-Zitebot
-ZmEu
-ZoominfoBot
-ZumBot
-ZyBorg
-adscanner
+Xaldon_WebSpider
+YoudaoBot
+Zeus
 arquivo-web-crawler
 arquivo.pt
-check1.exe
-coccocbot
+autoemailspider
 crawler.feedback
 crawler4j
-dataforseo.com
 evc-batch
-facebookscraper
-heritrix
-iubenda-radar
-magpie-crawler
-muhstik-scan
+isitwp.com
+lwp-request
+meanpathbot
+mediawords
 netEstate NE Crawler
-oBot
-plumanalytics
+page scorer
 polaris version
-s1z.ru
-satoristudio.net
-scalaj-http
-sitechecker.pro
-sysscan
+seobility
+seocompany.store
+seostar
+sexsearcher
+siteripz
 trendiction.com
-voyagerx.com
+ubermetrics-technologies.com
 webmeup-crawler
-xpymep1.exe
+webprosbot
+x09Mozilla
+x22Mozilla
 zgrab