mirror of
https://github.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker.git
synced 2025-09-02 10:40:36 +00:00
Test Whitelisting
This commit is contained in:
Mitchell Krog
parent
230d7a2a19
commit
a517008738
6 changed files with 118 additions and 18 deletions
63
.dev-tools/blacklist-user-agents.conf
Normal file
63
.dev-tools/blacklist-user-agents.conf
Normal file
|
|
@ -0,0 +1,63 @@
|
|||
# EDIT THIS FILE AS YOU LIKE TO BLACKLIST OR WHITELIST ANY BAD USER-AGENT STRINGS YOU WANT TO SCAN FOR
|
||||
# ****************************************************************************************************
|
||||
|
||||
### VERSION INFORMATION #
|
||||
###################################################
|
||||
### Version: V3.2018.05
|
||||
### Updated: 2018-08-21
|
||||
###################################################
|
||||
### VERSION INFORMATION ##
|
||||
|
||||
##############################################################################
|
||||
# _ __ _ #
|
||||
# / |/ /__ _(_)__ __ __ #
|
||||
# / / _ `/ / _ \\ \ / #
|
||||
# /_/|_/\_, /_/_//_/_\_\ #
|
||||
# __/___/ __ ___ __ ___ __ __ #
|
||||
# / _ )___ ____/ / / _ )___ / /_ / _ )/ /__ ____/ /_____ ____ #
|
||||
# / _ / _ `/ _ / / _ / _ \/ __/ / _ / / _ \/ __/ '_/ -_) __/ #
|
||||
# /____/\_,_/\_,_/ /____/\___/\__/ /____/_/\___/\__/_/\_\\__/_/ #
|
||||
# #
|
||||
##############################################################################
|
||||
|
||||
# Add One Entry Per Line - List all the extra bad User-Agents you want to permanently block or whitelist.
|
||||
|
||||
# This is for User-Agents that are not included in the main list of the bot blocker
|
||||
# This file must exist on your system or Nginx will fail a reload due to a missing file
|
||||
# This allows you finer control of keeping certain bots blocked and automatic updates will
|
||||
# Never be able to remove this custom list of yours
|
||||
|
||||
# Please note this include file loads first before any of the already whitelisted User-Agents
|
||||
# in the bad bot blocker. By loading first in line it over-rides anything below it so for instance
|
||||
# if you want to block Baidu, Google or Bing for any reason you add them to this file which loads
|
||||
# first and takes precedence over anything below it. This now allows even finer control over the
|
||||
# bad bot blocker. Enjoy !!!
|
||||
|
||||
# Even though this file is called blacklist-user-agents, as mentioned it can also be used to whitelist user agents
|
||||
# By adding them below and setting the 3; to 0; this will permanently whitelist the User-Agent.
|
||||
|
||||
# Make sure any words that contain special characters are escaped and include word boundaries as per the Regex examples below.
|
||||
|
||||
# Example the User-Agent name "someverybaduseragentname1" is entered as "\bsomeverybaduseragentname1\b"
|
||||
# Example the User-Agent name "some-very-bad-useragentname2" is entered as "\bsome\-very\-bad\-useragentname1\b"
|
||||
# the "\b" are word boundaries which prevents partial matching and false positives.
|
||||
|
||||
# BY DEFAULT ALL THE EXAMPLES BELOW ARE COMMENTED OUT AND HENCE NOT ENABLED
|
||||
|
||||
# ---------------------
|
||||
# WHITELISTING EXAMPLES
|
||||
# ---------------------
|
||||
# "~*\bsomeverygooduseragentname1\b" 0;
|
||||
# "~*\bsomeverygooduseragentname2\b" 0;
|
||||
# "~*\bsome\-very\-good\-useragentname2\b" 0;
|
||||
|
||||
# ---------------------
|
||||
# BLACKLISTING EXAMPLES
|
||||
# ---------------------
|
||||
# "~*\bsomeverybaduseragentname1\b" 3;
|
||||
# "~*\bsomeverybaduseragentname2\b" 3;
|
||||
# "~*\bsome\-very\-bad\-useragentname2\b" 3;
|
||||
|
||||
"~*(?:\b)Nutch(?:\b|)" 0;
|
||||
|
||||
|
||||
|
|
@ -54,6 +54,7 @@ sudo chmod +x ${TRAVIS_BUILD_DIR}/.dev-tools/generate-regex-format-referrers.php
|
|||
sudo chmod +x ${TRAVIS_BUILD_DIR}/.dev-tools/modify-config-readme-files.sh
|
||||
sudo chmod +x ${TRAVIS_BUILD_DIR}/.dev-tools/modify-files-and-commit.sh
|
||||
sudo chmod +x ${TRAVIS_BUILD_DIR}/.dev-tools/test-blocker.sh
|
||||
sudo chmod +x ${TRAVIS_BUILD_DIR}/.dev-tools/test-blocker-whitelist.sh
|
||||
sudo chmod +x ${TRAVIS_BUILD_DIR}/.dev-tools/prepare-robots-input.sh
|
||||
sudo chmod +x ${TRAVIS_BUILD_DIR}/.dev-tools/install-nginx-1.sh
|
||||
sudo chmod +x ${TRAVIS_BUILD_DIR}/.dev-tools/install-nginx-2.sh
|
||||
|
|
|
|||
|
|
@ -131,8 +131,6 @@ map $http_user_agent $bad_bot {
|
|||
# END CUSTOM BLACKLISTED USER AGENTS ### DO NOT EDIT OR REMOVE THIS LINE AT ALL ###
|
||||
# *********************************************************************************
|
||||
|
||||
# Test User Whitelisting
|
||||
"~*(?:\b)Nutch(?:\b|)" 0;
|
||||
|
||||
# *********************************************
|
||||
# Bad User-Agent Strings That We Block Outright
|
||||
|
|
|
|||
52
.dev-tools/test-blocker-whitelist.sh
Normal file
52
.dev-tools/test-blocker-whitelist.sh
Normal file
|
|
@ -0,0 +1,52 @@
|
|||
#!/bin/bash
|
||||
# Curl Testing Script for Nginx Ultimate Bad Bot Blocker
|
||||
# Created by: Mitchell Krog (mitchellkrog@gmail.com)
|
||||
# Copyright: Mitchell Krog - https://github.com/mitchellkrogza
|
||||
# Repo Url: https://github.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker
|
||||
|
||||
##############################################################################
|
||||
# _ __ _ #
|
||||
# / |/ /__ _(_)__ __ __ #
|
||||
# / / _ `/ / _ \\ \ / #
|
||||
# /_/|_/\_, /_/_//_/_\_\ #
|
||||
# __/___/ __ ___ __ ___ __ __ #
|
||||
# / _ )___ ____/ / / _ )___ / /_ / _ )/ /__ ____/ /_____ ____ #
|
||||
# / _ / _ `/ _ / / _ / _ \/ __/ / _ / / _ \/ __/ '_/ -_) __/ #
|
||||
# /____/\_,_/\_,_/ /____/\___/\__/ /____/_/\___/\__/_/\_\\__/_/ #
|
||||
# #
|
||||
##############################################################################
|
||||
|
||||
echo "Whitelist Tests Starting"
|
||||
|
||||
echo "Activating User Whitelist/Blacklist"
|
||||
sudo cp ${TRAVIS_BUILD_DIR}/.dev-tools/blacklist-user-agents.conf /etc/nginx/bots.d/blacklist-user-agents.conf
|
||||
echo "Reloading Nginx"
|
||||
sudo nginx -t && sudo nginx -s reload
|
||||
|
||||
|
||||
# *******************************************************
|
||||
# Function Curl Test 16 - Test User Whitelist for "Nutch"
|
||||
# *******************************************************
|
||||
|
||||
run_curltest1 () {
|
||||
printf '\n%s\n%s\n%s\n\n' "#########################" "TESTING USER WHITELIST" "#########################"
|
||||
if curl -v -A "Nutch" http://localhost:9000/index.php 2>&1 | grep -i 'Welcome'; then
|
||||
echo "$(tput setaf 2)BAD BOT ALLOWED - TEST PASSED"
|
||||
else
|
||||
echo "$(tput setaf 1)WHITELISTING FAILED - TEST FAILED"
|
||||
#exit 1
|
||||
curl -v -A "Nutch" http://localhost:9000/index.php 2>&1
|
||||
fi
|
||||
}
|
||||
run_curltest1
|
||||
|
||||
|
||||
echo "Whitelist Tests Completed"
|
||||
|
||||
# **********************
|
||||
# Exit With Error Number
|
||||
# **********************
|
||||
|
||||
exit ${?}
|
||||
|
||||
|
||||
|
|
@ -251,22 +251,6 @@ fi
|
|||
}
|
||||
run_curltest15
|
||||
|
||||
# *******************************************************
|
||||
# Function Curl Test 16 - Test User Whitelist for "Nutch"
|
||||
# *******************************************************
|
||||
|
||||
run_curltest16 () {
|
||||
printf '\n%s\n%s\n%s\n\n' "#########################" "TESTING USER WHITELIST" "#########################"
|
||||
if curl -v -A "Nutch" http://localhost:9000/index.php 2>&1 | grep -i 'Welcome'; then
|
||||
echo "$(tput setaf 2)BAD BOT ALLOWED - TEST PASSED"
|
||||
else
|
||||
echo "$(tput setaf 1)WHITELISTING FAILED - TEST FAILED"
|
||||
#exit 1
|
||||
curl -v -A "Nutch" http://localhost:9000/index.php 2>&1
|
||||
fi
|
||||
}
|
||||
run_curltest16
|
||||
|
||||
|
||||
echo "Tests Completed"
|
||||
|
||||
|
|
|
|||
|
|
@ -40,8 +40,10 @@ script:
|
|||
- bash .dev-tools/generate-files.sh
|
||||
- bash .dev-tools/install-nginx-1.sh
|
||||
- bash .dev-tools/test-blocker.sh
|
||||
- bash .dev-tools/test-blocker-whitelist.sh
|
||||
- bash .dev-tools/install-nginx-testing-of-changes.sh
|
||||
- bash .dev-tools/test-blocker.sh
|
||||
- bash .dev-tools/test-blocker-whitelist.sh
|
||||
#- bash .dev-tools/install-nginx-2.sh
|
||||
#- bash .dev-tools/test-blocker.sh
|
||||
#- bash .dev-tools/install-nginx-3.sh
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue