V4.2022.02.2911 [ci skip]

dev-tools/test_units/random-bots-for-whitelist-test.list

@@ -1,250 +1,250 @@
-80legs
+360Spider
+404enemy
+ALittle Client
+ASPSeek
+Abonti
 Aboundex
-Aboundexbot
 Acunetix
-AfD-Verbotsverfahren
-Aipbot
-Alligator
-Anarchie
-Anarchy99
-Ankit
-Anthill
-Aspiegel
-Atomseobot
-AwarioRssBot
-BDFetch
+AhrefsBot
+Alexibot
+AlphaBot
+AwarioSmartBot
 BackDoorBot
-Backlink-Ceck
+BackStreet
+BackWeb
 BacklinkCrawler
-Badass
-Barkrowler
-BatchFTP
 BetaBot
-Bigfoot
-Bitacle
-BlackWidow
-BlowFish
+Black Hole
+Blackboard
+Blow
 BotALot
-Brandwatch
-Buck
-CCBot
+Brandprotect
 CODE87
-Calculon
-ChinaClaw
+CheeseBot
+Chlooe
 Citoid
-Claritybot
+Cliqzbot
 Cloud mapping
+Cogentbot
+Copyscape
+Cosmos
 Craftbot
 CrazyWebCrawler
 Crescent
-Curious
-Custo
+CrunchBot
 DIIbot
 DSearch
-DTS Agent
 DatabaseDriverMysqli
-Demon
-Digincore
-Disco
-DomCopBot
-DomainCrawler
+Deusu
+Devil
+DigitalPebble
 DomainSigmaCrawler
-Domains Project
 Download Wonder
-Dragonfly
-Drip
-EMail Siphon
-Ebingbong
-Ecxi
-EroCrawler
+ECCP/1.0
 Evil
 ExtLinksBot
-Extractor
+ExtractorPro
 Extreme Picture Finder
 Ezooms
+FemtosearchBot
+Firefox/7.0
 FlashGet
+Flunky
+Freeuploader
+FyberSpider
 Fyrebot
+G-i-g-a-b-o-t
 GT::WWW
-GalaxyBot
 Gigabot
+Go!Zilla
+GoZilla
+Gotit
 GrabNet
-GrapeshotCrawler
-HEADMasterSEO
-HTTrack
+GrapeFX
 Haansoft
 HaosouSpider
 Harvest
-Heritrix
-Hloader
-Humanlinks
-HybridBot
+IDBTE4M
+IDBot
+IRLbot
+Iblog
+Id-search
+IlseBot
 IndeedBot
-Indy Library
-InfoTekies
+Intelliseek
 InterGET
+InternetSeer
 Iria
 IstellaBot
 JOC Web Spider
 JamesBOT
-Jbrofuzz
-JustView
+JikeSpider
 Jyxobot
-Kozmosbot
+Keyword Density
+Kinza
+LNSpiderguy
 Lanshanbot
 Larbin
-LeechFTP
+Leap
 LeechGet
-Lftp
+LibWeb
+Libwhisker
+LieBaoFast
+LinkScan
+Linkdexbot
+LinkextractorPro
+LinkpadBot
+LinqiaMetadataDownloaderBot
 LinqiaRSSBot
+LinqiaScrapeBot
 Lipperhey Spider
-MSIECrawler
-MTRobot
-Mag-Net
-Majestic12
-Mata Hari
-Mb2345Browser
+Lmspider
+MFC_Tear_Sample
+MIIxpc
+MJ12bot
+MQQBrowser
+MSFrontPage
+Majestic-SEO
+MarkMonitor
+MarkWatch
 MeanPath Bot
-Metauri
-Microsoft URL Control
-Minefield
+Meanpathbot
 Mister PiX
-MolokaiBot
+Morfeus Fucking Scanner
 Mozlila
+Mr.4x3
 Msrabot
-NICErsPRO
 NPbot
-Navroad
-NetSpider
-Netcraft
+Nameprotect
+Nessus
 NextGenSearchBot
-Niki-bot
 NimbleCrawler
-Nmap
-Not
-Nuclei
-Nutch
 Octopus
 OnCrawl
 OpenLinkProfiler
-OpenVAS
-Openfind
-OrangeSpider
+Openvas
+OrangeBot
+PECL::HTTP
+PHPCrawl
 POE-Component-Client-HTTP
-PageGrabber
 PageScorer
-PageThing.com
-PeoplePal
-Pi-Monster
+Pavuk
+Petalbot
+Picsearch
 Piepmatz
+Pimonster
 Pixray
-Pockey
-Pump
+ProPowerBot
+Probethenet
+Psbot
 PxBroker
-PyCurl
+QueryN Metasearch
 Quick-Crawler
-RankActiveLinkBot
+RSSingBot
 RankFlex
 RankingBot
-Rankivabot
-Re-re
-RealDownload
+RankingBot2
+RankurBot
+Reaper
+RedesScrapy
+RepoMonkey
 RocketCrawler
-Rogerbot
-SEOkicks-Robot
-SEOlyticsCrawler
-SEOprofiler
+SEOstats
 SISTRIX
 SMTBot
-SalesIntelligent
-Scrapy
-ScrepyBot
+ScoutJet
+ScreenerBot
+SearchmetricsBot
+Seekport
 Semrush
+SeobilityBot
 Shodan
-Siphon
-Site Sucker
-SiteCheckerBotCrawler
 SiteExplorer
-SiteLockSpider
-Sitevigil
-SmartDownload
-Snoopy
-Sosospider
+SiteSnagger
+Siteimprove
+SlySearch
+Snake
+Sociscraper
+Sogou web spider
 SpankBot
-Spbot
-Sqlworm
-SuperBot
+Steeler
+Sucker
 SuperHTTP
-Surfbot
-Suzuran
 Szukacz
 T8Abot
 TeleportPro
-Telesoft
 Telesphoreo
-TheNomad
-TightTwatBot
+Telesphorep
 Toata
 Tracemyfile
 Trendiction
+Trendictionbot
 True_Robot
 Turingos
-Vacuum
+Turnitin
+TwengaBot
+Twice
+URLy Warning
+UnisterBot
+Upflow
+Vagabondo
 VelenPublicWebCrawler
 VeriCiteCrawler
-VidibleScraper
 VoidEYE
-Voltron
-WISENutbot
-WWW::Mechanize
+Voil
+WASALive-Bot
+WBSearchBot
+WPScan
+WWW-Mechanize
+Wallpapers
+Wallpapers/3.0
 WeSEE
-Web Enhancer
-Web Fuck
-Web Pix
-Web Sauger
+Web Auto
+Web Collage
 Web Sucker
-WebAuto
+WebBandit
+WebEnhancer
+WebFetch
 WebGo IS
 WebPix
-WebStripper
-WebWhacker
-Website Quester
-WebsiteExtractor
+WebSucker
 WebsiteQuester
-Whatweb
-Widow
+Whacker
+WiseGuys Robot
+Wonderbot
+Woobot
 Wprecon
-ZmEu
-ZoominfoBot
-ZumBot
-ZyBorg
-autoemailspider
+YoudaoBot
+Zade
+Zermelo
+Zeus
+archive.org_bot
 backlink-check
 cah.io.community
 check1.exe
-clark-crawler
-coccocbot-web
+cognitiveseo
 com.plumanalytics
 crawl.sogou.com
-crawler.feedback
-demandbase-bot
 eCatch
-gopher
-heritrix
+instabid
 internetVista monitor
 ips-agent
-lwp-request
 lwp-trivial
+meanpathbot
+mediawords
 oBot
 page scorer
-polaris version
-probe-image-size
+plumanalytics
 ripz
-scalaj-http
-scan.lol
-seobility
-serpstatbot
+seocompany.store
+seoscanners
+seostar
 sitechecker.pro
-spyfu
-voyagerx.com
-xpymep1.exe
-zauba.io
+sogouspider
+sp_auditbot
+tAkeOut
+trendiction.com
+ubermetrics-technologies.com
+webgains-bot
+webprosbot
+zgrab