V4.2022.03.2980 [ci skip]

dev-tools/test_units/random-bots-for-whitelist-test.list

@@ -1,250 +1,250 @@
 01h4x.com
-404checker
-80legs
-ADmantX
+360Spider
+404enemy
+AIBOT
 ALittle Client
 Abonti
-Aboundexbot
+AfD-Verbotsverfahren
 AhrefsBot
-AllSubmitter
-Anarchy
-Apexoo
-Attach
-AwarioSmartBot
+AiHitBot
+Alexibot
+Anthill
+Atomseobot
 BDCbot
-BackStreet
-Backlink-Ceck
+BDFetch
+BackWeb
 BacklinkCrawler
-Barkrowler
-BatchFTP
-Battleztar Bazinga
-BetaBot
-Black Hole
+Bandit
+Bigfoot
 Blackboard
 Boardreader
 Bolt
-Brandprotect
 Bullseye
-Calculon
-CheTeam
-Cosmos
-Craftbot
+BunnySlippers
+Chlooe
+CopyRightCheck
+Copyscape
 Crawling at Home Project
-CrazyWebCrawler
+CrunchBot
 Curious
+Custo
 DBLBot
 DIIbot
-DatabaseDriverMysqli
-Devil
+Demon
+Deusu
+Digincore
+DigitalPebble
 Discobot
 Discoverybot
 Dispatch
-DomainCrawler
-Domains Project
-Dragonfly
-EMail Siphon
+DnyzBot
+DomainAppender
+DomainStatsBot
+Download Wonder
 EMail Wolf
-Ebingbong
+EasyDL
 Ecxi
-EirGrabber
 EroCrawler
-Evil
-Extreme Picture Finder
+Exabot
+Express WebPictures
 EyeNetIE
-FDM
-FemtosearchBot
 Fimap
-Firefox/7.0
+Foobot
 Freeuploader
 FrontPage
-Fuzz
-Fyrebot
-G-i-g-a-b-o-t
+FyberSpider
 GT::WWW
-GalaxyBot
+GermCrawler
 GetWeb
 Getintent
 Gigabot
 Go-Ahead-Got-It
-Gotit
-Grabber
-Grafula
 GrapeFX
-GridBot
 HEADMasterSEO
-HMView
 HTMLparser
-HTTP::Lite
-Haansoft
-IDBTE4M
-Id-search
-Image Sucker
+HTTrack
+Heritrix
+Humanlinks
+IRLbot
+IlseBot
+IndeedBot
+InfoNaviRobot
+Intelliseek
+InterGET
+Internet Ninja
 Iria
 Iskanie
 IstellaBot
-JamesBOT
+JennyBot
 JetCar
-LeechFTP
-LeechGet
+Jetty
+JikeSpider
+Jorgee
+Kenjin Spider
+LNSpiderguy
+Leap
+LibWeb
 Libwhisker
-Likse
-Linkdexbot
+LieBaoFast
+Lightspeedsystems
+LinkScan
+Linkbot
+LinksManager
+LinqiaMetadataDownloaderBot
+LinqiaRSSBot
+LinqiaScrapeBot
+Lipperhey
+Lipperhey Spider
 Ltx71
-MFC_Tear_Sample
 MIDown tool
-MQQBrowser
-MSIECrawler
+MTRobot
 Mag-Net
 Majestic SEO
-Majestic-SEO
+Majestic12
 MarkMonitor
 Mass Downloader
+Mata Hari
 MauiBot
+Mb2345Browser
 MeanPath Bot
+Meanpathbot
+MegaIndex.ru
 Microsoft Data Access
-Microsoft URL Control
-Moblie Safari
-Mozlila
-Musobot
-Nameprotect
+Mister PiX
+MolokaiBot
+NICErsPRO
+NPbot
+Name Intelligence
 Navroad
-Needle
-Nessus
-NetAnts
-NetSpider
-NimbleCrawler
+NearSite
+NetMechanic
+NetZIP
+Netcraft
+NextGenSearchBot
+Nikto
 Nimbostratus
 Not
 Nuclei
 Nutch
 Offline Navigator
-OnCrawl
 Openfind
 Openvas
 OrangeSpider
-OutfoxBot
-POE-Component-Client-HTTP
-PageGrabber
-Pavuk
+PageScorer
+Pandalytics
 PeoplePal
+Petalbot
 Pi-Monster
-Picscout
-Picsearch
 PictureFinder
 Piepmatz
 Pixray
-Pockey
+ProPowerBot
 Psbot
-Pu_iN
 Pump
 QueryN Metasearch
+Quick-Crawler
+RSSingBot
 RankActive
+RankActiveLinkBot
+RankingBot
+RankingBot2
 Re-re
-ReGet
-Recorder
+Reaper
+RedesScrapy
+Ripper
 RocketCrawler
-SBIder
+SEOkicks
 SEOkicks-Robot
-SEOlyticsCrawler
-SalesIntelligent
+SEOprofiler
+SMTBot
 Scanbot
-ScoutJet
-Scrapy
 ScreenerBot
 Searchestate
-Seekport
+SearchmetricsBot
 SemanticJuice
-SemrushBot
 SentiBot
-SeobilityBot
+Seomoz
 Shodan
-Siphon
-Site Sucker
-SiteCheckerBotCrawler
-Sitebeam
+SiteSucker
+Siteimprove
 SlySearch
 SmartDownload
-Snapbot
-Sociscraper
-SpankBot
-Spanner
+Snoopy
+Sogou web spider
+Sosospider
+Spammen
+Spinn3r
 SputnikBot
 Sqlmap
-Steeler
-Sucker
+Sqworm
+Stripper
 SuperBot
-Swiftbot
+Surfbot
+Suzuran
+Szukacz
+TeleportPro
 Telesoft
 Telesphoreo
+Telesphorep
+TheNomad
 Thumbor
-Toata
-Toweyabot
+Trendiction
+Turingos
 TurnitinBot
-VB Project
-Vacuum
+TwengaBot
+UnisterBot
+Upflow
+V-BOT
 Vagabondo
-VidibleScraper
-VoidEYE
+Voil
 WASALive-Bot
-WBSearchBot
+WEBDAV
 WPScan
-Wallpapers
+WWW-Collector-E
+Wallpapers/3.0
+WeSEE
 Web Auto
 Web Pix
+Web Sauger
 Web Sucker
-WebAuto
-WebBandit
-WebCopier
-WebEnhancer
-WebFetch
+WebCollage
+WebFuck
+WebGo IS
 WebImageCollector
-WebLeacher
-WebReaper
-WebSucker
-WebWhacker
-WebZIP
-Website Quester
-Webster
-Widow
-Xaldon WebSpider
-Xenu
-YoudaoBot
+WebmasterWorldForumBot
+Webshag
+Whack
+Who.is Bot
+Wprecon
+Zade
+Zauba
 Zermelo
-ZmEu
-ZoomBot
+Zitebot
 ZumBot
 ZyBorg
 arquivo-web-crawler
-backlink-check
 cah.io.community
-coccocbot
 cognitiveseo
 crawl.sogou.com
-crawler4j
-dataforseo.com
 demandbase-bot
-domainsproject.org
 eCatch
-facebookscraper
-gopher
-internetVista monitor
+heritrix
+instabid
 lwp-trivial
 meanpathbot
 muhstik-scan
-probe-image-size
-s1z.ru
-scalaj-http
-scan.lol
+netEstate NE Crawler
+page scorer
+plumanalytics
+satoristudio.net
 seobility
-seoscanners
+seocompany.store
 serpstatbot
-sexsearcher
 siteripz
-spyfu
-tAkeOut
-trendiction.com
+sysscan
 webgains-bot
-webpros.com
-webprosbot
+webmeup-crawler
 x09Mozilla
-x22Mozilla
+xpymep1.exe
+zauba.io
+zgrab