V4.2021.09.2592 [ci skip]

dev-tools/test_units/random-bots-for-whitelist-test.list

@@ -1,250 +1,250 @@
-404enemy
-ADmantX
-AIBOT
-Aboundex
+01h4x.com
+360Spider
 Aboundexbot
-AiHitBot
+Acunetix
+AfD-Verbotsverfahren
+AhrefsBot
+Aipbot
 Alexibot
-AllSubmitter
-AlphaBot
 Anarchie
-Anarchy99
+Apexoo
 Asterias
-AwarioRssBot
-AwarioSmartBot
-BDCbot
-BLEXBot
-BackDoorBot
-Barkrowler
-Bitacle
+BackStreet
+BacklinkCrawler
+Badass
+Bigfoot
 Black Hole
-Blackboard
-Blow
+BlackWidow
 BotALot
 Brandprotect
-Buck
-Bullseye
+Brandwatch
 BunnySlippers
-BuzzSumo
-Cegbfeieh
-CensysInspect
-CheTeam
+Calculon
+CazoodleBot
+CheeseBot
 CherryPicker
+ChinaClaw
 Cocolyzebot
-Cogentbot
 Copier
+CopyRightCheck
 Cosmos
-CrunchBot
+Craftbot
+Crawling at Home Project
 Curious
+Custo
+CyotekWebCopy
 DIIbot
 DSearch
+DataCha0s
 Devil
-Disco
-Discobot
-DnyzBot
-DomainStatsBot
-Dotbot
-Drip
-EMail Wolf
-Ebingbong
-EirGrabber
-Evil
-ExtLinksBot
+Digincore
+DigitalPebble
+Dirbuster
+DittoSpyder
+DomCopBot
+DomainAppender
+DomainCrawler
+DomainSigmaCrawler
+EasyDL
+Ecxi
+EroCrawler
+Express WebPictures
 ExtractorPro
 EyeNetIE
-Ezooms
-FDM
-FHscan
+Firefox/7.0
 FlashGet
-Flunky
+Foobot
 Freeuploader
 FyberSpider
 G-i-g-a-b-o-t
+GT::WWW
+GalaxyBot
+Genieo
+GetRight
 Getintent
-Go!Zilla
-GoZilla
+Go-Ahead-Got-It
 Gotit
-GrabNet
-GrapeFX
-GrapeshotCrawler
-HMView
+HEADMasterSEO
+HTMLparser
+HTTP::Lite
 HTTrack
-Haansoft
-Havij
-Heritrix
-Humanlinks
-IRLbot
+Harvest
+Hloader
+IDBTE4M
+IDBot
 Iblog
+Image Fetch
 Image Sucker
+IndeedBot
 Indy Library
-InfoNaviRobot
+InfoTekies
 Intelliseek
+InterGET
 InternetSeer
-Iskanie
-JikeSpider
-JustView
-Kinza
-Kozmosbot
-Lanshanbot
+IstellaBot
+Jbrofuzz
+JennyBot
+JetCar
+Jetty
+Kenjin Spider
+Keyword Density
+LNSpiderguy
 Leap
-LeechFTP
-LeechGet
-LexiBot
+Lftp
+LibWeb
 Libwhisker
 LieBaoFast
-Likse
 LinkScan
-Linkbot
-LinqiaRSSBot
-Lipperhey
+LinkextractorPro
+LinksManager
 Lipperhey Spider
 Ltx71
 MFC_Tear_Sample
 MIDown tool
-MIIxpc
-MJ12bot
-MTRobot
+MQQBrowser
 Mag-Net
+Magnet
 Mail.RU_Bot
+Majestic-SEO
 MarkMonitor
-Mass Downloader
-Mata Hari
+MauiBot
 MeanPath Bot
-Meanpathbot
-Mediatoolkitbot
-MegaIndex.ru
-Mojeek
-Morfeus Fucking Scanner
-Mozlila
-Musobot
+MicroMessenger
+Mister PiX
+Moblie Safari
+Name Intelligence
+Nameprotect
 Navroad
-Nettrack
-Nikto
-NimbleCrawler
-Nimbostratus
-Ninja
-Nutch
+NearSite
+Nessus
+NetMechanic
+NetSpider
+Netvibes
+Nuclei
 Octopus
 Offline Explorer
-OnCrawl
-OpenLinkProfiler
-OrangeSpider
+Offline Navigator
+Openfind
 OutclicksBot
-PECL::HTTP
 PHPCrawl
-PageScorer
+POE-Component-Client-HTTP
 PageThing.com
-Pavuk
+Panscient
 PeoplePal
-Petalbot
 Picsearch
 PictureFinder
 Pimonster
-Pixray
 PleaseCrawl
 ProPowerBot
-Probethenet
+ProWebWalker
 Pu_iN
-QueryN Metasearch
-RankActiveLinkBot
-RankurBot
+PxBroker
+Quick-Crawler
+RankFlex
+RankingBot
 Re-re
-RebelMouse
+RealDownload
 Recorder
+RedesScrapy
+RocketCrawler
 SEOkicks-Robot
-SEOlyticsCrawler
 SEOprofiler
-SISTRIX
-SMTBot
 Scanbot
-ScoutJet
-ScreenerBot
-ScrepyBot
+Scrapy
 Semrush
 SemrushBot
-SeoSiteCheckup
+Shodan
+Site Sucker
 SiteCheckerBotCrawler
-SiteLockSpider
+SiteSnagger
 SiteSucker
 Sitebeam
-Sitevigil
-SlySearch
+Siteimprove
 Snake
 Snapbot
-SocialRankIOBot
-Sociscraper
-Spanner
-Sqlmap
-Sqworm
+Snoopy
+Sottopop
+Spammen
+Spbot
+Sqlworm
+Steeler
+Stripper
 SuperHTTP
 Surfbot
-Swiftbot
-T0PHackTeam
-Telesphoreo
-The Intraformant
-Thumbor
-Titan
-Trendictionbot
+Szukacz
+TeleportPro
+Telesoft
+TheNomad
+TightTwatBot
+Toata
+Toweyabot
+Trendiction
+True_Robot
 Turnitin
+TwengaBot
 Twice
-Typhoeus
+URLy.Warning
 Upflow
-V-BOT
+VCI
 Vacuum
-VelenPublicWebCrawler
 VeriCiteCrawler
 VidibleScraper
-Virusdie
-VoidEYE
-Voltron
-WPScan
-WWW-Mechanize
+WEBDAV
+WISENutbot
+WWW-Collector-E
 WWWOFFLE
-Wallpapers/3.0
 WallpapersHD
-Web Collage
-Web Sauger
+WeSEE
+Web Fuck
+Web Sucker
+WebAuto
 WebEnhancer
+WebFuck
+WebGo IS
 WebImageCollector
+WebPix
 WebReaper
-WebSucker
+WebZIP
 Webalta
 WebmasterWorldForumBot
-Whacker
+Webshag
+WebsiteQuester
+Whack
+Whatweb
 Widow
-WiseGuys Robot
+WinHTTrack
 Wonderbot
-Xaldon WebSpider
+Wotbox
+Xaldon_WebSpider
+YoudaoBot
+Zauba
+Zermelo
+Zeus
 Zitebot
-ZmEu
+ZoominfoBot
 ZyBorg
 archive.org_bot
-backlink-check
+arquivo.pt
 cah.io.community
-crawler.feedback
-dataforseo.com
+clark-crawler
+cognitiveseo
 demandbase-bot
+evc-batch
 facebookscraper
-gopher
-internetVista monitor
-isitwp.com
-lwp-request
+instabid
+lwp-trivial
+magpie-crawler
 mediawords
 muhstik-scan
 netEstate NE Crawler
-pcBrowser
-polaris version
-probe-image-size
 ripz
 s1z.ru
 satoristudio.net
-seostar
+seoscanners
+sexsearcher
 sitechecker.pro
-sysscan
-tAkeOut
-trendiction.de
+siteripz
+spyfu
 ubermetrics-technologies.com
-voyagerx.com
-webmeup-crawler
-x09Mozilla
-xpymep1.exe
+webprosbot
+x22Mozilla