V4.2021.08.2548 [ci skip]

dev-tools/test_units/random-bots-for-whitelist-test.list

@@ -1,250 +1,250 @@
-80legs
+01h4x.com
+404enemy
 AIBOT
-ALittle Client
 Abonti
 Aboundex
+AfD-Verbotsverfahren
 AiHitBot
+Aipbot
 Alexibot
 AllSubmitter
-AlphaBot
-Anarchie
-Ankit
-Apexoo
+Anarchy
+Anthill
+Asterias
 AwarioRssBot
 BBBike
-BDCbot
-BLEXBot
 BackDoorBot
-Backlink-Ceck
+BackStreet
 BacklinkCrawler
+Barkrowler
 Bigfoot
-Black Hole
 BlackWidow
-Boardreader
-Bolt
-BotALot
-Brandwatch
-Buddy
-BunnySlippers
+Brandprotect
+Buck
+BuiltBotTough
 BuzzSumo
-CATExplorador
-CCBot
-Calculon
+CSHttp
+CazoodleBot
 Cegbfeieh
-CheeseBot
+CensysInspect
+CheTeam
 CherryPicker
 ChinaClaw
-Chlooe
-Cliqzbot
-Cocolyzebot
-Cosmos
-Crawling at Home Project
+Cloud mapping
+Collector
+Copier
 CrunchBot
-Custo
+Curious
 CyotekWebCopy
 DBLBot
 DIIbot
 DSearch
-Deusu
-Devil
-DigitalPebble
-Disco
+DatabaseDriverMysqli
+Digincore
+Discobot
+Discoverybot
 Dispatch
 DittoSpyder
 DomCopBot
-DomainStatsBot
-Dotbot
-ECCP/1.0
+DomainSigmaCrawler
+Domains Project
+Dragonfly
 EMail Siphon
 EMail Wolf
 EasyDL
+Ebingbong
 Ecxi
-EirGrabber
+EroCrawler
+Evil
+Exabot
 ExtLinksBot
-Extreme Picture Finder
+ExtractorPro
 EyeNetIE
 Ezooms
-Fimap
+FemtosearchBot
+Firefox/7.0
 FlashGet
+FrontPage
 Fuzz
-GetWeb
+GalaxyBot
+GetRight
 Getintent
 Gigabot
-Go!Zilla
 Gotit
-GrabNet
-Grabber
-HaosouSpider
+Grafula
+GrapeshotCrawler
+GridBot
+HEADMasterSEO
+HMView
+HTTP::Lite
+Harvest
 Havij
-Heritrix
+Hloader
 Humanlinks
-IDBot
+IDBTE4M
+IRLbot
 Iblog
-Id-search
 Image Fetch
-Image Sucker
 IndeedBot
-Indy Library
-InfoNaviRobot
+InfoTekies
 InterGET
+Internet Ninja
 InternetSeer
 Iria
-JOC Web Spider
-JamesBOT
-Jbrofuzz
+JetCar
+JikeSpider
 Joomla
 JustView
-Jyxobot
-Kinza
-LWP::Simple
+Keyword Density
+Larbin
 Leap
-LeechFTP
-LexiBot
-LibWeb
 Libwhisker
-Lightspeedsystems
 Likse
-Linkbot
+LinkWalker
 Linkdexbot
-LinqiaRSSBot
-LinqiaScrapeBot
-Litemage_walker
-Lmspider
-MIDown tool
-MSFrontPage
-Mail.RU_Bot
-Majestic SEO
-Majestic-SEO
-MarkMonitor
+LinkextractorPro
+LinqiaMetadataDownloaderBot
+Lipperhey
+Lipperhey Spider
+Ltx71
+MSIECrawler
+Majestic12
 Masscan
-Meanpathbot
-Mediatoolkitbot
+Mata Hari
+Mb2345Browser
+MeanPath Bot
 Metauri
-Microsoft URL Control
 Mister PiX
 Mojolicious
+Morfeus Fucking Scanner
 Mozlila
-Nameprotect
-Navroad
-NearSite
-Needle
+Name Intelligence
 Nessus
 Net Vampire
-NetLyzer
-NetMechanic
-NextGenSearchBot
+NetSpider
+NetZIP
+Nettrack
+Netvibes
 Nibbler
-NimbleCrawler
+Niki-bot
+Nikto
+Nimbostratus
 Nuclei
-Nutch
 Octopus
-Offline Explorer
-Offline Navigator
-OpenLinkProfiler
+OnCrawl
+Openvas
+OutclicksBot
 OutfoxBot
-PageGrabber
-PageScorer
-Pandalytics
-Petalbot
+PECL::HTTP
+PHPCrawl
+Page Analyzer
+Panscient
+Papa Foto
+Pi-Monster
 Picscout
 Piepmatz
 Pimonster
 PleaseCrawl
+Pockey
 ProPowerBot
-Probethenet
-Pump
+Psbot
+Pu_iN
 PxBroker
-PyCurl
 QueryN Metasearch
-Quick-Crawler
+RankActive
 RankActiveLinkBot
-Rankivabot
-RankurBot
+RankingBot
+RankingBot2
 Re-re
+RealDownload
+Recorder
+RedesScrapy
 RepoMonkey
-RocketCrawler
-Rogerbot
-SEOprofiler
-ScanAlert
+Ripper
+SBIder
+SEOkicks-Robot
+SISTRIX
 ScoutJet
+Screaming
 ScreenerBot
-Searchestate
-Semrush
-SeoSiteCheckup
+SearchmetricsBot
+SemrushBot
 Shodan
-Siphon
-SiteSnagger
-SiteSucker
-Sitevigil
-SlySearch
-Snake
-SocialRankIOBot
+Siteimprove
 Sociscraper
-SpaceBison
+Sogou web spider
+Sosospider
 Spbot
-Sqlmap
-Sqlworm
-Stripper
-Sucker
-SurveyBot
-Suzuran
-Szukacz
+Spinn3r
+Sucuri
+SuperBot
+Swiftbot
+T0PHackTeam
 T8Abot
-Telesoft
-Thumbor
+Teleport
+Titan
+Toata
 Toweyabot
-Tracemyfile
 True_Robot
-Turingos
 Turnitin
 TwengaBot
-URLy Warning
-V-BOT
-Vacuum
+URLy.Warning
+Upflow
+VeriCiteCrawler
+VidibleScraper
 Voltron
 WISENutbot
+WWW::Mechanize
 WWWOFFLE
 Wallpapers
 Wallpapers/3.0
+WeSEE
+Web Collage
+Web Enhancer
 Web Fetch
-Web Sucker
-WebCopier
+Web Fuck
+Web Pix
+Web Sauger
+WebBandit
 WebFuck
-WebGo IS
-WebLeacher
-WebPix
-Webalta
-WebmasterWorldForumBot
-Webshag
+WebImageCollector
+Website Quester
 Webster
+Whack
 Whatweb
-Widow
+WiseGuys Robot
+Woobot
+Xaldon WebSpider
+Xenu
 YoudaoBot
-Zeus
+Zade
 Zitebot
 ZoomBot
-ZoominfoBot
 ZumBot
-arquivo.pt
-backlink-check
-cah.io.community
+archive.org_bot
+arquivo-web-crawler
 check1.exe
-crawl.sogou.com
-eCatch
+clark-crawler
+coccocbot-web
+cognitiveseo
+crawler4j
+demandbase-bot
+facebookscraper
+heritrix
 instabid
-internetVista monitor
-lwp-request
-meanpathbot
+lwp-trivial
 mediawords
-netEstate NE Crawler
-polaris version
+page scorer
 probe-image-size
+satoristudio.net
 scalaj-http
 scan.lol
-sexsearcher
+seoscanners
+seostar
+serpstatbot
 siteripz
 sogouspider
 sysscan
-tAkeOut
-trendiction.de
-webmeup-crawler
+ubermetrics-technologies.com
+webprosbot
+x22Mozilla