vikarti.anatra/nginx-ultimate-bad-bot-blocker
1
0
Fork 0
mirror of https://github.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker.git synced 2025-09-02 10:40:36 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Improves Tests Commenting

Browse source
This commit is contained in:
Mitchell Krog 2019-06-24 14:17:49 +02:00
parent ce88019c6d
commit ff53848bae
No known key found for this signature in database
GPG key ID: C243C388553EDE5D
3 changed files with 74 additions and 55 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

38
.dev-tools/install-nginx-testing-of-changes.sh
View file

@ -41,30 +41,62 @@
# Run testing of test version # Run testing of test version
# ************************
# Set Terminal Font Colors
# ************************
bold=$(tput bold)
red=$(tput setaf 1)
green=$(tput setaf 2)
yellow=$(tput setaf 3)
blue=$(tput setaf 4)
magenta=$(tput setaf 5)
cyan=$(tput setaf 6)
white=$(tput setaf 7)
defaultcolor=$(tput setaf default)
# *********************************************************************** # ***********************************************************************
# Make sure we test latest generated globalblacklist-testing-version.conf # Make sure we test latest generated globalblacklist-testing-version.conf
# *********************************************************************** # ***********************************************************************
printf '\n%s\n%s\n%s\n\n' "##############################################################################" "Copy Test Unit of globalblacklist.conf file to test for any compile errors" "##############################################################################" printf "\n"
echo "${bold}${yellow}---------------------------------------------------"
echo "${bold}${yellow}Copy Test Unit of globalblacklist.conf file to test"
echo "${bold}${yellow}---------------------------------------------------"
printf "\n"
sudo cp ${TRAVIS_BUILD_DIR}/conf.d/globalblacklist-testing-version.conf /etc/nginx/conf.d/globalblacklist.conf sudo cp ${TRAVIS_BUILD_DIR}/conf.d/globalblacklist-testing-version.conf /etc/nginx/conf.d/globalblacklist.conf
# ********************* # *********************
# Force reload of Nginx # Force reload of Nginx
# ********************* # *********************
printf '\n%s\n%s\n%s\n\n' "########################" "Force Reloading of Nginx" "########################" printf "\n"
echo "${bold}${green}------------"
echo "${bold}${green}Reload Nginx"
echo "${bold}${green}------------"
printf "\n"
sudo nginx -t && sudo nginx -s reload sudo nginx -t && sudo nginx -s reload
# ********************** # **********************
# Now Run our Curl Tests # Now Run our Curl Tests
# ********************** # **********************
printf '\n%s\n%s\n%s\n\n' "######################" "Now Run our Curl Tests" "######################" printf "\n"
echo "${bold}${green}---------------------------"
echo "${bold}${green}Run Bot and Referrer Checks"
echo "${bold}${green}---------------------------"
printf "\n"
# ************************************************************* # *************************************************************
# Copy all .conf files used in Testing to a folder for checking # Copy all .conf files used in Testing to a folder for checking
# ************************************************************* # *************************************************************
printf "\n"
echo "${bold}${green}------------------------------------------------------------"
echo "${bold}${green}Make Backup all conf files and folders used during this test"
echo "${bold}${green}------------------------------------------------------------"
printf "\n"
sudo cp /etc/nginx/bots.d/* ${TRAVIS_BUILD_DIR}/.dev-tools/_conf_files_testing_changes/bots.d/ sudo cp /etc/nginx/bots.d/* ${TRAVIS_BUILD_DIR}/.dev-tools/_conf_files_testing_changes/bots.d/
sudo cp /etc/nginx/conf.d/* ${TRAVIS_BUILD_DIR}/.dev-tools/_conf_files_testing_changes/conf.d/ sudo cp /etc/nginx/conf.d/* ${TRAVIS_BUILD_DIR}/.dev-tools/_conf_files_testing_changes/conf.d/
sudo cp /etc/nginx/sites-available/default.vhost ${TRAVIS_BUILD_DIR}/.dev-tools/_conf_files_testing_changes/default.vhost sudo cp /etc/nginx/sites-available/default.vhost ${TRAVIS_BUILD_DIR}/.dev-tools/_conf_files_testing_changes/default.vhost

89
.dev-tools/test-blocker-quick.sh
View file

@ -63,11 +63,10 @@ printf "\n\n"
# ************************************************* # *************************************************
run_curltest1 () { run_curltest1 () {
printf '\n%s\n%s\n%s\n\n' "#########################" "TESTING BAD BOT IS DENIED" "#########################"
if curl -v -A "80legs" http://localhost:9000 2>&1 | grep -i '(52)'; then if curl -v -A "80legs" http://localhost:9000 2>&1 | grep -i '(52)'; then
echo "${bold}${green}PASSED - ${red}BAD BOT DETECTED" echo "${bold}${green}PASSED - ${red}80legs BAD BOT DETECTED"
else else
echo "${bold}${red}FAILED - ${red}BAD BOT NOT DETECTED" echo "${bold}${red}FAILED - ${red}80legs BAD BOT NOT DETECTED"
exit 1 exit 1
fi fi
} }
@ -79,11 +78,10 @@ run_curltest1
# ************************************************** # **************************************************
run_curltest2 () { run_curltest2 () {
printf '\n%s\n%s\n%s\n\n' "#########################" "TESTING BAD BOT IS DENIED" "#########################"
if curl -v -A "Nutch" http://localhost:9000 2>&1 | grep -i '(52)'; then if curl -v -A "Nutch" http://localhost:9000 2>&1 | grep -i '(52)'; then
echo "${bold}${green}PASSED - ${red}BAD BOT DETECTED" echo "${bold}${green}PASSED - ${red}Nutch BAD BOT DETECTED"
else else
echo "${bold}${red}FAILED - ${red}BAD BOT NOT DETECTED" echo "${bold}${red}FAILED - ${red}Nutch BAD BOT NOT DETECTED"
exit 1 exit 1
fi fi
} }
@ -95,11 +93,10 @@ run_curltest2
# ****************************************************************** # ******************************************************************
run_curltest3 () { run_curltest3 () {
printf '\n%s\n%s\n%s\n\n' "##############################" "TESTING BAD REFERRER IS DENIED" "##############################"
if curl -I http://localhost:9000 -e http://100dollars-seo.com 2>&1 | grep -i '(52)'; then if curl -I http://localhost:9000 -e http://100dollars-seo.com 2>&1 | grep -i '(52)'; then
echo "${bold}${green}PASSED - ${red}BAD REFERRER DETECTED" echo "${bold}${green}PASSED - ${red}100dollars-seo.com BAD REFERRER DETECTED"
else else
echo "${bold}${red}FAILED - ${red}BAD REFERRER NOT DETECTED" echo "${bold}${red}FAILED - ${red}100dollars-seo.com BAD REFERRER NOT DETECTED"
exit 1 exit 1
fi fi
} }
@ -111,11 +108,10 @@ run_curltest3
# ****************************************************** # ******************************************************
run_curltest4 () { run_curltest4 () {
printf '\n%s\n%s\n%s\n\n' "##############################" "TESTING BAD REFERRER IS DENIED" "##############################"
if curl -I http://localhost:9000 -e http://zx6.ru 2>&1 | grep -i '(52)'; then if curl -I http://localhost:9000 -e http://zx6.ru 2>&1 | grep -i '(52)'; then
echo "${bold}${green}PASSED - ${red}BAD REFERRER DETECTED" echo "${bold}${green}PASSED - ${red}zx6.ru BAD REFERRER DETECTED"
else else
echo "${bold}${red}FAILED - ${red}BAD REFERRER NOT DETECTED" echo "${bold}${red}FAILED - ${red}zx6.ru BAD REFERRER NOT DETECTED"
exit 1 exit 1
fi fi
} }
@ -126,11 +122,10 @@ run_curltest4
# ***************************************************** # *****************************************************
run_curltest5 () { run_curltest5 () {
printf '\n%s\n%s\n%s\n\n' "###########################" "TESTING GOOD BOT IS ALLOWED" "###########################"
if curl -v -A "GoogleBot" http://localhost:9000 2>&1 | grep -i 'Welcome'; then if curl -v -A "GoogleBot" http://localhost:9000 2>&1 | grep -i 'Welcome'; then
echo "${bold}${green}PASSED - ${green}GOOD BOT ALLOWED THROUGH" echo "${bold}${green}PASSED - ${green}GoogleBot GOOD BOT ALLOWED THROUGH"
else else
echo "${bold}${red}FAILED - ${red}GOOD BOT NOT ALLOWED THROUGH" echo "${bold}${red}FAILED - ${red}GoogleBot GOOD BOT NOT ALLOWED THROUGH"
exit 1 exit 1
fi fi
} }
@ -141,11 +136,10 @@ run_curltest5
# *************************************************** # ***************************************************
run_curltest6 () { run_curltest6 () {
printf '\n%s\n%s\n%s\n\n' "###########################" "TESTING GOOD BOT IS ALLOWED" "###########################"
if curl -v -A "BingBot" http://localhost:9000 2>&1 | grep -i 'Welcome'; then if curl -v -A "BingBot" http://localhost:9000 2>&1 | grep -i 'Welcome'; then
echo "${bold}${green}PASSED - ${green}GOOD BOT ALLOWED THROUGH" echo "${bold}${green}PASSED - ${green}BingBot GOOD BOT ALLOWED THROUGH"
else else
echo "${bold}${red}FAILED - ${red}GOOD BOT NOT ALLOWED THROUGH" echo "${bold}${red}FAILED - ${red}BingBot GOOD BOT NOT ALLOWED THROUGH"
exit 1 exit 1
fi fi
} }
@ -156,11 +150,10 @@ run_curltest6
# *********************************************************** # ***********************************************************
run_curltest7 () { run_curltest7 () {
printf '\n%s\n%s\n%s\n\n' "################################" "TESTING GOOD REFERRER IS ALLOWED" "################################"
if curl http://localhost:9000 -e http://google.com 2>&1 | grep -i 'Welcome'; then if curl http://localhost:9000 -e http://google.com 2>&1 | grep -i 'Welcome'; then
echo "${bold}${green}PASSED - ${green}GOOD REFERRER DETECTED" echo "${bold}${green}PASSED - ${green}google.com GOOD REFERRER DETECTED"
else else
echo "${bold}${red}FAILED - ${red}GOOD REFERRER NOT DETECTED" echo "${bold}${red}FAILED - ${red}google.com GOOD REFERRER NOT DETECTED"
exit 1 exit 1
fi fi
} }
@ -171,11 +164,10 @@ run_curltest7
# *********************************************************** # ***********************************************************
run_curltest8 () { run_curltest8 () {
printf '\n%s\n%s\n%s\n\n' "################################" "TESTING GOOD REFERRER IS ALLOWED" "################################"
if curl http://localhost:9000 -e http://bing.com 2>&1 | grep -i 'Welcome'; then if curl http://localhost:9000 -e http://bing.com 2>&1 | grep -i 'Welcome'; then
echo "${bold}${green}PASSED - ${red}GOOD REFERRER DETECTED" echo "${bold}${green}PASSED - ${red}bing.com GOOD REFERRER DETECTED"
else else
echo "${bold}${red}FAILED - ${red}GOOD REFERRER NOT DETECTED" echo "${bold}${red}FAILED - ${red}bing.com GOOD REFERRER NOT DETECTED"
exit 1 exit 1
fi fi
} }
@ -186,11 +178,10 @@ run_curltest8
# ************************************************** # **************************************************
run_curltest9 () { run_curltest9 () {
printf '\n%s\n%s\n%s\n\n' "#########################" "TESTING BAD BOT IS DENIED" "#########################"
if curl -A "Googlebot/Nutch-1.7" http://localhost:9000 2>&1 | grep -i '(52)'; then if curl -A "Googlebot/Nutch-1.7" http://localhost:9000 2>&1 | grep -i '(52)'; then
echo "${bold}${green}PASSED - ${red}BAD BOT DETECTED" echo "${bold}${green}PASSED - ${red}Googlebot/Nutch-1.7 BAD BOT DETECTED"
else else
echo "${bold}${red}FAILED - ${red}BAD BOT NOT DETECTED" echo "${bold}${red}FAILED - ${red}Googlebot/Nutch-1.7 BAD BOT NOT DETECTED"
#exit 1 #exit 1
fi fi
} }
@ -201,11 +192,10 @@ run_curltest9
# ************************************************** # **************************************************
run_curltest10 () { run_curltest10 () {
printf '\n%s\n%s\n%s\n\n' "#########################" "TESTING BAD BOT IS DENIED" "#########################"
if curl -A "Mozilla/5.0 (compatible; Googlebot/Nutch2.1; +http://www.google.com/bot.html)" http://localhost:9000 2>&1 | grep -i '(52)'; then if curl -A "Mozilla/5.0 (compatible; Googlebot/Nutch2.1; +http://www.google.com/bot.html)" http://localhost:9000 2>&1 | grep -i '(52)'; then
echo "${bold}${green}PASSED - ${red}BAD BOT DETECTED" echo "${bold}${green}PASSED - ${red}Googlebot/Nutch2.1 BAD BOT DETECTED"
else else
echo "${bold}${red}FAILED - ${red}BAD BOT NOT DETECTED" echo "${bold}${red}FAILED - ${red}Googlebot/Nutch2.1 BAD BOT NOT DETECTED"
#exit 1 #exit 1
fi fi
} }
@ -216,11 +206,10 @@ run_curltest10
# ************************************************** # **************************************************
run_curltest11 () { run_curltest11 () {
printf '\n%s\n%s\n%s\n\n' "#########################" "TESTING BAD BOT IS DENIED" "#########################"
if curl -A "Mozilla/5.0 (compatible; Googlebot/nutch/-2.1; +http://www.google.com/bot.html)" http://localhost:9000 2>&1 | grep -i '(52)'; then if curl -A "Mozilla/5.0 (compatible; Googlebot/nutch/-2.1; +http://www.google.com/bot.html)" http://localhost:9000 2>&1 | grep -i '(52)'; then
echo "${bold}${green}PASSED - ${red}BAD BOT DETECTED" echo "${bold}${green}PASSED - ${red}Googlebot/nutch/-2.1 BAD BOT DETECTED"
else else
echo "${bold}${red}FAILED - BAD BOT NOT DETECTED" echo "${bold}${red}FAILED - Googlebot/nutch/-2.1 BAD BOT NOT DETECTED"
#exit 1 #exit 1
fi fi
} }
@ -231,11 +220,10 @@ run_curltest11
# ************************************************** # **************************************************
run_curltest12 () { run_curltest12 () {
printf '\n%s\n%s\n%s\n\n' "############################" "TESTING FALSE POSITIVE CASES" "############################"
if curl -v -A "Mozilla/5.0 (compatible; Googlebot-Image/Snutch\-/-2.1; +http://www.google.com/bot.html)" http://localhost:9000 2>&1 | grep -i 'Welcome'; then if curl -v -A "Mozilla/5.0 (compatible; Googlebot-Image/Snutch\-/-2.1; +http://www.google.com/bot.html)" http://localhost:9000 2>&1 | grep -i 'Welcome'; then
echo "${bold}${green}PASSED - NO FALSE POSITIVE" echo "${bold}${green}PASSED - NO FALSE POSITIVE on Googlebot-Image/Snutch\-/-2.1"
else else
echo "${bold}${red}FAILED - FALSE POSITIVE FOUND" echo "${bold}${red}FAILED - FALSE POSITIVE FOUND on Googlebot-Image/Snutch\-/-2.1"
#exit 1 #exit 1
fi fi
} }
@ -246,11 +234,10 @@ run_curltest12
# ************************************************** # **************************************************
run_curltest13 () { run_curltest13 () {
printf '\n%s\n%s\n%s\n\n' "############################" "TESTING FALSE POSITIVE CASES" "############################"
if curl -v -A "SnutchMozilla/5.0 (compatible; Googlebot-Image/SMutch\-/-2.1; +http://www.google.com/bot.html)" http://localhost:9000 2>&1 | grep -i 'Welcome'; then if curl -v -A "SnutchMozilla/5.0 (compatible; Googlebot-Image/SMutch\-/-2.1; +http://www.google.com/bot.html)" http://localhost:9000 2>&1 | grep -i 'Welcome'; then
echo "${bold}${green}PASSED - NO FALSE POSITIVE" echo "${bold}${green}PASSED - NO FALSE POSITIVE on SnutchMozilla/5.0"
else else
echo "${bold}${red}FAILED - FALSE POSITIVE FOUND" echo "${bold}${red}FAILED - FALSE POSITIVE FOUND on SnutchMozilla/5.0"
#exit 1 #exit 1
fi fi
} }
@ -261,11 +248,10 @@ run_curltest13
# ************************************************** # **************************************************
run_curltest14 () { run_curltest14 () {
printf '\n%s\n%s\n%s\n\n' "############################" "TESTING FALSE POSITIVE CASES" "############################"
if curl -v -A "Mozilla/5.0 (X11; Linux x86_64; rv:67.0) Gecko/20100101 Firefox/67.0" http://localhost:9000 2>&1 | grep -i 'Welcome'; then if curl -v -A "Mozilla/5.0 (X11; Linux x86_64; rv:67.0) Gecko/20100101 Firefox/67.0" http://localhost:9000 2>&1 | grep -i 'Welcome'; then
echo "${bold}${green}PASSED - NO FALSE POSITIVE" echo "${bold}${green}PASSED - NO FALSE POSITIVE on Mozilla/5.0"
else else
echo "${bold}${red}FAILED - FALSE POSITIVE FOUND" echo "${bold}${red}FAILED - FALSE POSITIVE FOUND on Mozilla/5.0"
#exit 1 #exit 1
fi fi
} }
@ -277,16 +263,17 @@ run_curltest14
# ************************************************** # **************************************************
run_curltest15 () { run_curltest15 () {
printf '\n%s\n%s\n%s\n\n' "############################" "TESTING FALSE POSITIVE CASES" "############################" if curl -v -A "Mozilla/5.0 (iPhone; CPU iPhone OS 12_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.1 Mobile/15E148 Safari/604.1" http://localhost:9000 2>&1 | grep -i 'Welcome'; then
if curl -v -A "Mozilla/5.0 (X11; Linux x86_64; rv:67.0) Gecko/20100101 Firefox/67.0" http://localhost:9000 2>&1 | grep -i 'Welcome'; then echo "${bold}${green}PASSED - NO FALSE POSITIVE on Safari"
echo "${bold}${green}PASSED - NO FALSE POSITIVE"
else else
echo "${bold}${red}FAILED - FALSE POSITIVE FOUND" echo "${bold}${red}FAILED - FALSE POSITIVE FOUND on Safari"
#exit 1 #exit 1
fi fi
} }
run_curltest15 run_curltest15
# Add Any More Tests Here for False Positives
# ************************************************ # ************************************************
# Test 50 User-Agents from Bad-User-Agents # Test 50 User-Agents from Bad-User-Agents
# ************************************************ # ************************************************
@ -390,12 +377,12 @@ shuf -n 100 ${TRAVIS_BUILD_DIR}/_generator_lists/bad-referrers.list > ${TRAVIS_B
sort -u ${TRAVIS_BUILD_DIR}/.dev-tools/test_units/random-referrers-for-test-quick.list -o ${TRAVIS_BUILD_DIR}/.dev-tools/test_units/random-referrers-for-test-quick.list sort -u ${TRAVIS_BUILD_DIR}/.dev-tools/test_units/random-referrers-for-test-quick.list -o ${TRAVIS_BUILD_DIR}/.dev-tools/test_units/random-referrers-for-test-quick.list
printf "\n\n" printf "\n\n"
echo "${bold}${magenta}--------------------------------" echo "${bold}${magenta}---------------------------------"
echo "${bold}${magenta}Testing 100 Random Bad Referrers" echo "${bold}${magenta}Testing 100 Random Bad Referrers"
echo "${bold}${magenta}--------------------------------" echo "${bold}${magenta}---------------------------------"
printf "\n\n" printf "\n\n"
IFS=$'\n' IFS=$'\n'
file=${TRAVIS_BUILD_DIR}/.dev-tools/test_units/random-referrers-for-test.list file=${TRAVIS_BUILD_DIR}/.dev-tools/test_units/random-referrers-for-test-quick.list
lines=$(cat ${file}) lines=$(cat ${file})
for line in ${lines}; do for line in ${lines}; do
if if
@ -432,8 +419,8 @@ IFS=""
printf "\n" printf "\n"
echo "${bold}${cyan}Tests Completed" echo "${bold}${cyan}Tests Completed"
echo "${bold}${gren}All Tests Passed" echo "${bold}${green}All Tests Passed"
echo "${bold}${magenta}Now We Test Whitelisting and then Release a new build" echo "${bold}${cyan}Move to Next Test"
# ********************** # **********************
# Exit With Error Number # Exit With Error Number

2
.dev-tools/test-blocker.sh
View file

@ -419,7 +419,7 @@ IFS=""
printf "\n" printf "\n"
echo "${bold}${cyan}Tests Completed" echo "${bold}${cyan}Tests Completed"
echo "${bold}${gren}All Tests Passed" echo "${bold}${green}All Tests Passed"
echo "${bold}${magenta}Now We Test Whitelisting and then Release a new build" echo "${bold}${magenta}Now We Test Whitelisting and then Release a new build"
# ********************** # **********************