UPDATE Test Blocker [MASTER]

This commit is contained in:
Mitchell Krog 2019-06-28 15:34:14 +02:00
parent d55a726430
commit 71bddc77ca
No known key found for this signature in database
GPG key ID: C243C388553EDE5D

View file

@ -16,9 +16,34 @@
# # # #
############################################################################## ##############################################################################
# ************************ # ------------------------------------------------------------------------------
# MIT License
# ------------------------------------------------------------------------------
# Copyright (c) 2017 Mitchell Krog - mitchellkrog@gmail.com
# https://github.com/mitchellkrogza
# ------------------------------------------------------------------------------
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
# ------------------------------------------------------------------------------
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
# ------------------------------------------------------------------------------
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
# ------------------------------------------------------------------------------
# ------------------------
# Set Terminal Font Colors # Set Terminal Font Colors
# ************************ # ------------------------
bold=$(tput bold) bold=$(tput bold)
red=$(tput setaf 1) red=$(tput setaf 1)
@ -30,37 +55,34 @@ cyan=$(tput setaf 6)
white=$(tput setaf 7) white=$(tput setaf 7)
defaultcolor=$(tput setaf default) defaultcolor=$(tput setaf default)
# ******************************************* # ---------
# Set Location of our Curl Test Results Files # FUNCTIONS
# ******************************************* # ---------
resetNginx () {
echo "${bold}${green}------------------------------------------------" echo "${bold}${green}------------------------------------------------"
echo "${bold}${green}Disable any User Whitelisting and set to Default" echo "${bold}${green}Disable any User Whitelisting and set to Default"
echo "${bold}${green}------------------------------------------------" echo "${bold}${green}------------------------------------------------"
printf "\n\n" printf "\n\n"
sudo cp ${TRAVIS_BUILD_DIR}/.dev-tools/test_units/blacklist-user-agents-none.conf /etc/nginx/bots.d/blacklist-user-agents.conf sudo cp ${TRAVIS_BUILD_DIR}/.dev-tools/test_units/blacklist-user-agents-none.conf /etc/nginx/bots.d/blacklist-user-agents.conf
}
reloadNginX () {
echo "${bold}${green}---------------" echo "${bold}${green}---------------"
echo "${bold}${green}Reloading Nginx" echo "${bold}${green}Reloading Nginx"
echo "${bold}${green}---------------" echo "${bold}${green}---------------"
printf "\n\n" printf "\n\n"
sudo nginx -t && sudo nginx -s reload sudo nginx -t && sudo nginx -s reload
}
waitforReload () {
echo "${bold}${yellow}-----------------------------------------------------------------------" echo "${bold}${yellow}-----------------------------------------------------------------------"
echo "${bold}${yellow}Sleeping for 10 seconds to allow Nginx to Properly Reload inside Travis" echo "${bold}${yellow}Sleeping for 10 seconds to allow Nginx to Properly Reload inside Travis"
echo "${bold}${yellow}-----------------------------------------------------------------------" echo "${bold}${yellow}-----------------------------------------------------------------------"
printf "\n\n" printf "\n\n"
sleep 10s sleep 10s
}
echo "${bold}${green}--------------"
echo "${bold}${green}Tests Starting"
echo "${bold}${green}--------------"
printf "\n\n"
# *************************************************
# Function Curl Test 1 - Check for Bad Bot "80legs"
# *************************************************
run_curltest1 () { run_curltest1 () {
if curl -v -A "80legs" http://localhost:9000 2>&1 | grep -i '(52)'; then if curl -v -A "80legs" http://localhost:9000 2>&1 | grep -i '(52)'; then
@ -70,12 +92,6 @@ else
exit 1 exit 1
fi fi
} }
run_curltest1
# **************************************************
# Function Curl Test 2 - Check for Bad Bot "Nutch"
# **************************************************
run_curltest2 () { run_curltest2 () {
if curl -v -A "Nutch" http://localhost:9000 2>&1 | grep -i '(52)'; then if curl -v -A "Nutch" http://localhost:9000 2>&1 | grep -i '(52)'; then
@ -85,12 +101,6 @@ else
exit 1 exit 1
fi fi
} }
run_curltest2
# ******************************************************************
# Function Curl Test 3 - Check for Bad Referrer "100dollars-seo.com"
# ******************************************************************
run_curltest3 () { run_curltest3 () {
if curl -I http://localhost:9000 -e http://100dollars-seo.com 2>&1 | grep -i '(52)'; then if curl -I http://localhost:9000 -e http://100dollars-seo.com 2>&1 | grep -i '(52)'; then
@ -100,12 +110,6 @@ else
exit 1 exit 1
fi fi
} }
run_curltest3
# ******************************************************
# Function Curl Test 4 - Check for Bad Referrer "zx6.ru"
# ******************************************************
run_curltest4 () { run_curltest4 () {
if curl -I http://localhost:9000 -e http://zx6.ru 2>&1 | grep -i '(52)'; then if curl -I http://localhost:9000 -e http://zx6.ru 2>&1 | grep -i '(52)'; then
@ -115,11 +119,6 @@ else
exit 1 exit 1
fi fi
} }
run_curltest4
# *****************************************************
# Function Curl Test 5 - Check for Good Bot "GoogleBot"
# *****************************************************
run_curltest5 () { run_curltest5 () {
if curl -v -A "GoogleBot" http://localhost:9000 2>&1 | grep -i 'Welcome'; then if curl -v -A "GoogleBot" http://localhost:9000 2>&1 | grep -i 'Welcome'; then
@ -129,11 +128,6 @@ else
exit 1 exit 1
fi fi
} }
run_curltest5
# ***************************************************
# Function Curl Test 6 - Check for Good Bot "BingBot"
# ***************************************************
run_curltest6 () { run_curltest6 () {
if curl -v -A "BingBot" http://localhost:9000 2>&1 | grep -i 'Welcome'; then if curl -v -A "BingBot" http://localhost:9000 2>&1 | grep -i 'Welcome'; then
@ -143,11 +137,6 @@ else
exit 1 exit 1
fi fi
} }
run_curltest6
# ***********************************************************
# Function Curl Test 7 - Check for Good Referrer "google.com"
# ***********************************************************
run_curltest7 () { run_curltest7 () {
if curl http://localhost:9000 -e http://google.com 2>&1 | grep -i 'Welcome'; then if curl http://localhost:9000 -e http://google.com 2>&1 | grep -i 'Welcome'; then
@ -157,11 +146,6 @@ else
exit 1 exit 1
fi fi
} }
run_curltest7
# ***********************************************************
# Function Curl Test 8 - Check for Good Referrer "bing.com"
# ***********************************************************
run_curltest8 () { run_curltest8 () {
if curl http://localhost:9000 -e http://bing.com 2>&1 | grep -i 'Welcome'; then if curl http://localhost:9000 -e http://bing.com 2>&1 | grep -i 'Welcome'; then
@ -171,11 +155,6 @@ else
exit 1 exit 1
fi fi
} }
run_curltest8
# **************************************************
# Function Curl Test 9 - Check for Bad Bot "Nutch"
# **************************************************
run_curltest9 () { run_curltest9 () {
if curl -A "Googlebot/Nutch-1.7" http://localhost:9000 2>&1 | grep -i '(52)'; then if curl -A "Googlebot/Nutch-1.7" http://localhost:9000 2>&1 | grep -i '(52)'; then
@ -185,11 +164,6 @@ else
exit 1 exit 1
fi fi
} }
run_curltest9
# **************************************************
# Function Curl Test 10 - Check for Bad Bot "Nutch"
# **************************************************
run_curltest10 () { run_curltest10 () {
if curl -A "Mozilla/5.0 (compatible; Googlebot/Nutch2.1; +http://www.google.com/bot.html)" http://localhost:9000 2>&1 | grep -i '(52)'; then if curl -A "Mozilla/5.0 (compatible; Googlebot/Nutch2.1; +http://www.google.com/bot.html)" http://localhost:9000 2>&1 | grep -i '(52)'; then
@ -199,11 +173,6 @@ else
exit 1 exit 1
fi fi
} }
run_curltest10
# **************************************************
# Function Curl Test 11 - Check for Bad Bot "Nutch"
# **************************************************
run_curltest11 () { run_curltest11 () {
if curl -A "Mozilla/5.0 (compatible; Googlebot/nutch/-2.1; +http://www.google.com/bot.html)" http://localhost:9000 2>&1 | grep -i '(52)'; then if curl -A "Mozilla/5.0 (compatible; Googlebot/nutch/-2.1; +http://www.google.com/bot.html)" http://localhost:9000 2>&1 | grep -i '(52)'; then
@ -213,11 +182,6 @@ else
exit 1 exit 1
fi fi
} }
run_curltest11
# **************************************************
# Function Curl Test 12 - Check for Bad Bot "Nutch"
# **************************************************
run_curltest12 () { run_curltest12 () {
if curl -v -A "Mozilla/5.0 (compatible; Googlebot-Image/Snutch\-/-2.1; +http://www.google.com/bot.html)" http://localhost:9000 2>&1 | grep -i 'Welcome'; then if curl -v -A "Mozilla/5.0 (compatible; Googlebot-Image/Snutch\-/-2.1; +http://www.google.com/bot.html)" http://localhost:9000 2>&1 | grep -i 'Welcome'; then
@ -227,11 +191,6 @@ else
exit 1 exit 1
fi fi
} }
run_curltest12
# **************************************************
# Function Curl Test 13 - Check for Bad Bot "Nutch"
# **************************************************
run_curltest13 () { run_curltest13 () {
if curl -v -A "SnutchMozilla/5.0 (compatible; Googlebot-Image/SMutch\-/-2.1; +http://www.google.com/bot.html)" http://localhost:9000 2>&1 | grep -i 'Welcome'; then if curl -v -A "SnutchMozilla/5.0 (compatible; Googlebot-Image/SMutch\-/-2.1; +http://www.google.com/bot.html)" http://localhost:9000 2>&1 | grep -i 'Welcome'; then
@ -241,11 +200,6 @@ else
exit 1 exit 1
fi fi
} }
run_curltest13
# **************************************************
# Function Curl Test 14 - Check for Good Bot
# **************************************************
run_curltest14 () { run_curltest14 () {
if curl -v -A "Mozilla/5.0 (X11; Linux x86_64; rv:67.0) Gecko/20100101 Firefox/67.0" http://localhost:9000 2>&1 | grep -i 'Welcome'; then if curl -v -A "Mozilla/5.0 (X11; Linux x86_64; rv:67.0) Gecko/20100101 Firefox/67.0" http://localhost:9000 2>&1 | grep -i 'Welcome'; then
@ -255,12 +209,6 @@ else
exit 1 exit 1
fi fi
} }
run_curltest14
# **************************************************
# Function Curl Test 15 - TEST CASES
# **************************************************
run_curltest15 () { run_curltest15 () {
if curl -v -A "Mozilla/5.0 (iPhone; CPU iPhone OS 12_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.1 Mobile/15E148 Safari/604.1" http://localhost:9000 2>&1 | grep -i 'Welcome'; then if curl -v -A "Mozilla/5.0 (iPhone; CPU iPhone OS 12_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.1 Mobile/15E148 Safari/604.1" http://localhost:9000 2>&1 | grep -i 'Welcome'; then
@ -270,18 +218,12 @@ else
exit 1 exit 1
fi fi
} }
run_curltest15
# Add Any More Tests Here for False Positives testBadUserAgents () {
# ************************************************
# Test 50 User-Agents from Bad-User-Agents
# ************************************************
shuf -n 50 ${TRAVIS_BUILD_DIR}/_generator_lists/bad-user-agents.list > ${TRAVIS_BUILD_DIR}/.dev-tools/test_units/random-bots-for-test-quick.tmp shuf -n 50 ${TRAVIS_BUILD_DIR}/_generator_lists/bad-user-agents.list > ${TRAVIS_BUILD_DIR}/.dev-tools/test_units/random-bots-for-test-quick.tmp
sed 's/\\//g' ${TRAVIS_BUILD_DIR}/.dev-tools/test_units/random-bots-for-test-quick.tmp > ${TRAVIS_BUILD_DIR}/.dev-tools/test_units/random-bots-for-test-quick.list sed 's/\\//g' ${TRAVIS_BUILD_DIR}/.dev-tools/test_units/random-bots-for-test-quick.tmp > ${TRAVIS_BUILD_DIR}/.dev-tools/test_units/random-bots-for-test-quick.list
sudo rm ${TRAVIS_BUILD_DIR}/.dev-tools/test_units/random-bots-for-test-quick.tmp sudo rm ${TRAVIS_BUILD_DIR}/.dev-tools/test_units/random-bots-for-test-quick.tmp
sort -u ${TRAVIS_BUILD_DIR}/.dev-tools/test_units/random-bots-for-test-quick.list -o ${TRAVIS_BUILD_DIR}/.dev-tools/test_units/random-bots-for-test-quick.list sort -u ${TRAVIS_BUILD_DIR}/.dev-tools/test_units/random-bots-for-test-quick.list -o ${TRAVIS_BUILD_DIR}/.dev-tools/test_units/random-bots-for-test-quick.list
printf "\n\n" printf "\n\n"
echo "${bold}${magenta}---------------------------" echo "${bold}${magenta}---------------------------"
echo "${bold}${magenta}Testing 50 Random Bad Bots" echo "${bold}${magenta}Testing 50 Random Bad Bots"
@ -299,12 +241,10 @@ for line in ${lines}; do
fi fi
done done
IFS="" IFS=""
}
# ************************************************ testGoodUserAgents () {
# Test All Good User-Agents from Good-User-Agents
# ************************************************
sed 's/\\//g' ${TRAVIS_BUILD_DIR}/_generator_lists/good-user-agents.list > ${TRAVIS_BUILD_DIR}/.dev-tools/test_units/good-bots-for-test.list sed 's/\\//g' ${TRAVIS_BUILD_DIR}/_generator_lists/good-user-agents.list > ${TRAVIS_BUILD_DIR}/.dev-tools/test_units/good-bots-for-test.list
printf "\n\n" printf "\n\n"
echo "${bold}${magenta}---------------------" echo "${bold}${magenta}---------------------"
echo "${bold}${magenta}Testing All Good Bots" echo "${bold}${magenta}Testing All Good Bots"
@ -322,13 +262,10 @@ for line in ${lines}; do
fi fi
done done
IFS="" IFS=""
}
testAllowedUserAgents () {
# *****************************************************
# Test All Allowed User-Agents from Allowed-User-Agents
# *****************************************************
sed 's/\\//g' ${TRAVIS_BUILD_DIR}/_generator_lists/allowed-user-agents.list > ${TRAVIS_BUILD_DIR}/.dev-tools/test_units/allowed-bots-for-test.list sed 's/\\//g' ${TRAVIS_BUILD_DIR}/_generator_lists/allowed-user-agents.list > ${TRAVIS_BUILD_DIR}/.dev-tools/test_units/allowed-bots-for-test.list
printf "\n\n" printf "\n\n"
echo "${bold}${magenta}------------------------" echo "${bold}${magenta}------------------------"
echo "${bold}${magenta}Testing All Allowed Bots" echo "${bold}${magenta}Testing All Allowed Bots"
@ -346,12 +283,10 @@ for line in ${lines}; do
fi fi
done done
IFS="" IFS=""
}
# ***************************************************** testLimitedUserAgents () {
# Test All Limited User-Agents from Limited-User-Agents
# *****************************************************
sed 's/\\//g' ${TRAVIS_BUILD_DIR}/_generator_lists/limited-user-agents.list > ${TRAVIS_BUILD_DIR}/.dev-tools/test_units/limited-bots-for-test.list sed 's/\\//g' ${TRAVIS_BUILD_DIR}/_generator_lists/limited-user-agents.list > ${TRAVIS_BUILD_DIR}/.dev-tools/test_units/limited-bots-for-test.list
printf "\n\n" printf "\n\n"
echo "${bold}${magenta}------------------------" echo "${bold}${magenta}------------------------"
echo "${bold}${magenta}Testing All Limited Bots" echo "${bold}${magenta}Testing All Limited Bots"
@ -369,13 +304,11 @@ for line in ${lines}; do
fi fi
done done
IFS="" IFS=""
}
# ******************************************** testRandomReferrers () {
# Test 100 Random Referrers from Bad-Referrers
# ********************************************
shuf -n 100 ${TRAVIS_BUILD_DIR}/_generator_lists/bad-referrers.list > ${TRAVIS_BUILD_DIR}/.dev-tools/test_units/random-referrers-for-test-quick.list shuf -n 100 ${TRAVIS_BUILD_DIR}/_generator_lists/bad-referrers.list > ${TRAVIS_BUILD_DIR}/.dev-tools/test_units/random-referrers-for-test-quick.list
sort -u ${TRAVIS_BUILD_DIR}/.dev-tools/test_units/random-referrers-for-test-quick.list -o ${TRAVIS_BUILD_DIR}/.dev-tools/test_units/random-referrers-for-test-quick.list sort -u ${TRAVIS_BUILD_DIR}/.dev-tools/test_units/random-referrers-for-test-quick.list -o ${TRAVIS_BUILD_DIR}/.dev-tools/test_units/random-referrers-for-test-quick.list
printf "\n\n" printf "\n\n"
echo "${bold}${magenta}---------------------------------" echo "${bold}${magenta}---------------------------------"
echo "${bold}${magenta}Testing 100 Random Bad Referrers" echo "${bold}${magenta}Testing 100 Random Bad Referrers"
@ -393,11 +326,9 @@ for line in ${lines}; do
fi fi
done done
IFS="" IFS=""
}
# ************************** testGoodReferrers () {
# Test Good Referrer Domains
# **************************
printf "\n\n" printf "\n\n"
echo "${bold}${magenta}----------------------" echo "${bold}${magenta}----------------------"
echo "${bold}${magenta}Testing Good Referrers" echo "${bold}${magenta}Testing Good Referrers"
@ -415,17 +346,74 @@ for line in ${lines}; do
fi fi
done done
IFS="" IFS=""
}
# -----------------
# Trigger Functions
# -----------------
echo "${bold}${green}--------------"
echo "${bold}${green}Tests Starting"
echo "${bold}${green}--------------"
printf "\n\n"
resetNginx
reloadNginX
waitforReload
run_curltest1
run_curltest2
run_curltest3
run_curltest4
run_curltest5
run_curltest6
run_curltest7
run_curltest8
run_curltest9
run_curltest10
run_curltest11
run_curltest12
run_curltest13
run_curltest14
run_curltest15
testBadUserAgents
testGoodUserAgents
testAllowedUserAgents
testLimitedUserAgents
testRandomReferrers
testGoodReferrers
printf "\n" printf "\n"
echo "${bold}${cyan}Tests Completed" echo "${bold}${cyan}Tests Completed"
echo "${bold}${green}All Tests Passed" echo "${bold}${green}All Tests Passed"
echo "${bold}${cyan}Move to Next Test"
# ********************** # ----------------------
# Exit With Error Number # Exit With Error Number
# ********************** # ----------------------
exit ${?} exit ${?}
# ------------------------------------------------------------------------------
# MIT License
# ------------------------------------------------------------------------------
# Copyright (c) 2017 Mitchell Krog - mitchellkrog@gmail.com
# https://github.com/mitchellkrogza
# ------------------------------------------------------------------------------
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
# ------------------------------------------------------------------------------
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
# ------------------------------------------------------------------------------
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
# ------------------------------------------------------------------------------