From 813737284ebefc9048b1c036babfac199abf3d6c Mon Sep 17 00:00:00 2001 From: Mitchell Krog Date: Thu, 27 Jun 2019 14:44:43 +0200 Subject: [PATCH] TEST new template REF: #201 --- .../beta-install-nginx-testing-of-changes.sh | 133 ++++++ .dev-tools/beta-test-blocker-badwords.sh | 96 ++++ .dev-tools/beta-test-blocker-rate-limiting.sh | 117 +++++ .../beta-test-blocker-whitelist-domains.sh | 96 ++++ .dev-tools/beta-test-blocker-whitelist-ips.sh | 127 +++++ .dev-tools/beta-test-blocker-whitelist.sh | 264 +++++++++++ .dev-tools/beta-test-blocker.sh | 443 ++++++++++++++++++ 7 files changed, 1276 insertions(+) create mode 100644 .dev-tools/beta-install-nginx-testing-of-changes.sh create mode 100755 .dev-tools/beta-test-blocker-badwords.sh create mode 100755 .dev-tools/beta-test-blocker-rate-limiting.sh create mode 100755 .dev-tools/beta-test-blocker-whitelist-domains.sh create mode 100755 .dev-tools/beta-test-blocker-whitelist-ips.sh create mode 100755 .dev-tools/beta-test-blocker-whitelist.sh create mode 100755 .dev-tools/beta-test-blocker.sh diff --git a/.dev-tools/beta-install-nginx-testing-of-changes.sh b/.dev-tools/beta-install-nginx-testing-of-changes.sh new file mode 100644 index 000000000..0ae1e776f --- /dev/null +++ b/.dev-tools/beta-install-nginx-testing-of-changes.sh @@ -0,0 +1,133 @@ +#!/bin/bash +# Travis CI Generating and Building for the Nginx Ultimate Bad Bot Blocker (using non standard folder locations) +# Created by: Mitchell Krog (mitchellkrog@gmail.com) +# Copyright: Mitchell Krog - https://github.com/mitchellkrogza +# Repo Url: https://github.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker + +############################################################################## +# _ __ _ # +# / |/ /__ _(_)__ __ __ # +# / / _ `/ / _ \\ \ / # +# /_/|_/\_, /_/_//_/_\_\ # +# __/___/ __ ___ __ ___ __ __ # +# / _ )___ ____/ / / _ )___ / /_ / _ )/ /__ ____/ /_____ ____ # +# / _ / _ `/ _ / / _ / _ \/ __/ / _ / / _ \/ __/ '_/ -_) __/ # +# /____/\_,_/\_,_/ /____/\___/\__/ /____/_/\___/\__/_/\_\\__/_/ # +# # +############################################################################## + +# MIT License + +# Copyright (c) 2017 Mitchell Krog - mitchellkrog@gmail.com +# https://github.com/mitchellkrogza + +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: + +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. + +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +# Run testing of test version + +# ************************ +# Set Terminal Font Colors +# ************************ + +bold=$(tput bold) +red=$(tput setaf 1) +green=$(tput setaf 2) +yellow=$(tput setaf 3) +blue=$(tput setaf 4) +magenta=$(tput setaf 5) +cyan=$(tput setaf 6) +white=$(tput setaf 7) +defaultcolor=$(tput setaf default) + +# *********************************************************************** +# Make sure we test latest generated globalblacklist-testing-version.conf +# *********************************************************************** + +printf "\n" +echo "${bold}${yellow}---------------------------------------------------" +echo "${bold}${yellow}Copy Test Unit of globalblacklist.conf file to test" +echo "${bold}${yellow}---------------------------------------------------" +printf "\n" + +sudo cp ${TRAVIS_BUILD_DIR}/conf.d/globalblacklist-testing-version.conf /etc/nginx/conf.d/globalblacklist.conf + +# ********************* +# Force reload of Nginx +# ********************* + +printf "\n" +echo "${bold}${green}------------" +echo "${bold}${green}Reload Nginx" +echo "${bold}${green}------------" +printf "\n" +sudo nginx -t && sudo nginx -s reload + +# ********************** +# Now Run our Curl Tests +# ********************** + +printf "\n" +echo "${bold}${green}---------------------------" +echo "${bold}${green}Run Bot and Referrer Checks" +echo "${bold}${green}---------------------------" +printf "\n" + +# ************************************************************* +# Copy all .conf files used in Testing to a folder for checking +# ************************************************************* + +printf "\n" +echo "${bold}${green}------------------------------------------------------------" +echo "${bold}${green}Make Backup all conf files and folders used during this test" +echo "${bold}${green}------------------------------------------------------------" +printf "\n" +sudo cp /etc/nginx/bots.d/* ${TRAVIS_BUILD_DIR}/.dev-tools/_conf_files_testing_changes/bots.d/ +sudo cp /etc/nginx/conf.d/* ${TRAVIS_BUILD_DIR}/.dev-tools/_conf_files_testing_changes/conf.d/ +sudo cp /etc/nginx/sites-available/default.vhost ${TRAVIS_BUILD_DIR}/.dev-tools/_conf_files_testing_changes/default.vhost +sudo cp /etc/nginx/nginx.conf ${TRAVIS_BUILD_DIR}/.dev-tools/_conf_files_testing_changes/nginx.conf + +# ********************** +# Exit With Error Number +# ********************** + +exit ${?} + + +# MIT License + +# Copyright (c) 2017 Mitchell Krog - mitchellkrog@gmail.com +# https://github.com/mitchellkrogza + +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: + +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. + +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. \ No newline at end of file diff --git a/.dev-tools/beta-test-blocker-badwords.sh b/.dev-tools/beta-test-blocker-badwords.sh new file mode 100755 index 000000000..1bd616c79 --- /dev/null +++ b/.dev-tools/beta-test-blocker-badwords.sh @@ -0,0 +1,96 @@ +#!/bin/bash +# Curl Testing Script for Nginx Ultimate Bad Bot Blocker +# Created by: Mitchell Krog (mitchellkrog@gmail.com) +# Copyright: Mitchell Krog - https://github.com/mitchellkrogza +# Repo Url: https://github.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker + +############################################################################## +# _ __ _ # +# / |/ /__ _(_)__ __ __ # +# / / _ `/ / _ \\ \ / # +# /_/|_/\_, /_/_//_/_\_\ # +# __/___/ __ ___ __ ___ __ __ # +# / _ )___ ____/ / / _ )___ / /_ / _ )/ /__ ____/ /_____ ____ # +# / _ / _ `/ _ / / _ / _ \/ __/ / _ / / _ \/ __/ '_/ -_) __/ # +# /____/\_,_/\_,_/ /____/\___/\__/ /____/_/\___/\__/_/\_\\__/_/ # +# # +############################################################################## + +# ************************ +# Set Terminal Font Colors +# ************************ + +bold=$(tput bold) +red=$(tput setaf 1) +green=$(tput setaf 2) +yellow=$(tput setaf 3) +blue=$(tput setaf 4) +magenta=$(tput setaf 5) +cyan=$(tput setaf 6) +white=$(tput setaf 7) +defaultcolor=$(tput setaf default) + +echo "${bold}${green}--------------------------------" +echo "${bold}${green}Bad Referrer Words Test Starting" +echo "${bold}${green}--------------------------------" +printf "\n\n" + +echo "${bold}${green}----------------------------------------" +echo "${bold}${green}Activating Users bad-referrer-words.conf" +echo "${bold}${green}----------------------------------------" +printf "\n\n" +sudo cp ${TRAVIS_BUILD_DIR}/.dev-tools/test_units/bad-referrer-words.conf /etc/nginx/bots.d/bad-referrer-words.conf +echo "${bold}${green}---------------" +echo "${bold}${green}Reloading Nginx" +echo "${bold}${green}---------------" +printf "\n\n" +sudo nginx -t && sudo nginx -s reload + + +echo "${bold}${yellow}-----------------------------------------------------------------------" +echo "${bold}${yellow}Sleeping for 10 seconds to allow Nginx to Properly Reload inside Travis" +echo "${bold}${yellow}-----------------------------------------------------------------------" +printf "\n\n" +sleep 10s + +# ******************************************************* +# Function Curl Test 1 - Test User Whitelist for "Nutch" +# ******************************************************* + +run_curltest1 () { +if curl -I http://localhost:9000 -e "thisisabadword" 2>&1 | grep -i '(52)'; then + echo "${bold}${green}PASSED - User bad-referrer-words.conf working" +else + echo "${bold}${red}FAILED - User bad-referrer-words.conf NOT working" + #exit 1 +fi +} +run_curltest1 + +# ************************************************************** +# Function Curl Test 2 - Check for Whitelisted Referrer "zx6.ru" +# ************************************************************** + +run_curltest2 () { +if curl -I http://localhost:9000 -e "thisisanotherbadword" 2>&1 | grep -i '(52)'; then + echo "${bold}${green}PASSED - User bad-referrer-words.conf working" +else + echo "${bold}${red}FAILED - User bad-referrer-words.conf NOT working" + #exit 1 +fi +} +run_curltest2 + + +echo "${bold}${green}--------------------------------" +echo "${bold}${green}Bad Referrer Words Test Complete" +echo "${bold}${green}--------------------------------" +printf "\n\n" + +# ********************** +# Exit With Error Number +# ********************** + +exit ${?} + + diff --git a/.dev-tools/beta-test-blocker-rate-limiting.sh b/.dev-tools/beta-test-blocker-rate-limiting.sh new file mode 100755 index 000000000..7ce7afa57 --- /dev/null +++ b/.dev-tools/beta-test-blocker-rate-limiting.sh @@ -0,0 +1,117 @@ +#!/bin/bash +# Curl Testing Script for Nginx Ultimate Bad Bot Blocker +# Created by: Mitchell Krog (mitchellkrog@gmail.com) +# Copyright: Mitchell Krog - https://github.com/mitchellkrogza +# Repo Url: https://github.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker + +############################################################################## +# _ __ _ # +# / |/ /__ _(_)__ __ __ # +# / / _ `/ / _ \\ \ / # +# /_/|_/\_, /_/_//_/_\_\ # +# __/___/ __ ___ __ ___ __ __ # +# / _ )___ ____/ / / _ )___ / /_ / _ )/ /__ ____/ /_____ ____ # +# / _ / _ `/ _ / / _ / _ \/ __/ / _ / / _ \/ __/ '_/ -_) __/ # +# /____/\_,_/\_,_/ /____/\___/\__/ /____/_/\___/\__/_/\_\\__/_/ # +# # +############################################################################## + +# ************************ +# Set Terminal Font Colors +# ************************ + +bold=$(tput bold) +red=$(tput setaf 1) +green=$(tput setaf 2) +yellow=$(tput setaf 3) +blue=$(tput setaf 4) +magenta=$(tput setaf 5) +cyan=$(tput setaf 6) +white=$(tput setaf 7) +defaultcolor=$(tput setaf default) + +# ****************** +# TEST RATE LIMITING +# ****************** + +printf "\n" +echo "${bold}${green}---------------------------" +echo "${bold}${green}Starting Rate Limiting Test" +echo "${bold}${green}---------------------------" +printf "\n\n" + +# *************************** +# Make GoogleBot Rate Limited +# *************************** + +echo "${bold}${yellow}-----------------------------" +echo "${bold}${yellow}Making GoogleBot ${yellow}RATE LIMITED" +echo "${bold}${yellow}-----------------------------" +printf "\n\n" +sudo cp ${TRAVIS_BUILD_DIR}/.dev-tools/test_units/rate-limiting-user-agents.conf /etc/nginx/bots.d/blacklist-user-agents.conf + +echo "${bold}${green}---------------" +echo "${bold}${green}Reloading Nginx" +echo "${bold}${green}---------------" +printf "\n\n" +sudo nginx -t && sudo nginx -s reload + +echo "${bold}${yellow}-----------------------------------------------------------------------" +echo "${bold}${yellow}Sleeping for 10 seconds to allow Nginx to Properly Reload inside Travis" +echo "${bold}${yellow}-----------------------------------------------------------------------" +printf "\n\n" +sleep 10s + +ratelimittestfile=${TRAVIS_BUILD_DIR}/.dev-tools/test_units/ratelimittest.txt +truncate -s 0 ${ratelimittestfile} +curl -A "GoogleBot" http://localhost:9000 2>&1 > ${ratelimittestfile} & +curl -A "GoogleBot" http://localhost:9000 2>&1 >> ${ratelimittestfile} & +curl -A "GoogleBot" http://localhost:9000 2>&1 >> ${ratelimittestfile} & +curl -A "GoogleBot" http://localhost:9000 2>&1 >> ${ratelimittestfile} & +curl -A "GoogleBot" http://localhost:9000 2>&1 >> ${ratelimittestfile} & +curl -A "GoogleBot" http://localhost:9000 2>&1 >> ${ratelimittestfile} & +curl -A "GoogleBot" http://localhost:9000 2>&1 >> ${ratelimittestfile} & +curl -A "GoogleBot" http://localhost:9000 2>&1 >> ${ratelimittestfile} & +curl -A "GoogleBot" http://localhost:9000 2>&1 >> ${ratelimittestfile} & +curl -A "GoogleBot" http://localhost:9000 2>&1 >> ${ratelimittestfile} & +curl -A "GoogleBot" http://localhost:9000 2>&1 >> ${ratelimittestfile} & +curl -A "GoogleBot" http://localhost:9000 2>&1 >> ${ratelimittestfile} & +curl -A "GoogleBot" http://localhost:9000 2>&1 >> ${ratelimittestfile} & +curl -A "GoogleBot" http://localhost:9000 2>&1 >> ${ratelimittestfile} & +curl -A "GoogleBot" http://localhost:9000 2>&1 >> ${ratelimittestfile} + +if grep -i 'Unavailable' < ${ratelimittestfile}; then + echo "${bold}${green}PASSED - ${red}GoogleBot was ${bold}${red}RATE LIMITED" + else + echo "${bold}${red}FAILED - ${red}GoogleBot was ${bold}${red}NOT RATE LIMITED" + fi + +# ************************************************************* +# Copy all .conf files used in Testing to a folder for checking +# ************************************************************* + +printf "\n" +echo "${bold}${green}------------------------------------------------------------" +echo "${bold}${green}Make Backup all conf files and folders used during this test" +echo "${bold}${green}------------------------------------------------------------" +printf "\n" +sudo cp /etc/nginx/bots.d/* ${TRAVIS_BUILD_DIR}/.dev-tools/_conf_files_testing_changes_ratelimiting/bots.d/ +sudo cp /etc/nginx/conf.d/* ${TRAVIS_BUILD_DIR}/.dev-tools/_conf_files_testing_changes_ratelimiting/conf.d/ +sudo cp /etc/nginx/sites-available/default.vhost ${TRAVIS_BUILD_DIR}/.dev-tools/_conf_files_testing_changes_ratelimiting/default.vhost +sudo cp /etc/nginx/nginx.conf ${TRAVIS_BUILD_DIR}/.dev-tools/_conf_files_testing_changes_ratelimiting/nginx.conf + + + +printf "\n" +echo "${bold}${green}---------------------------" +echo "${bold}${green}Rate Limiting Test Complete" +echo "${bold}${green}---------------------------" +printf "\n\n" + +# ********************** +# Exit With Error Number +# ********************** + +exit ${?} + + diff --git a/.dev-tools/beta-test-blocker-whitelist-domains.sh b/.dev-tools/beta-test-blocker-whitelist-domains.sh new file mode 100755 index 000000000..2f3dfc080 --- /dev/null +++ b/.dev-tools/beta-test-blocker-whitelist-domains.sh @@ -0,0 +1,96 @@ +#!/bin/bash +# Curl Testing Script for Nginx Ultimate Bad Bot Blocker +# Created by: Mitchell Krog (mitchellkrog@gmail.com) +# Copyright: Mitchell Krog - https://github.com/mitchellkrogza +# Repo Url: https://github.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker + +############################################################################## +# _ __ _ # +# / |/ /__ _(_)__ __ __ # +# / / _ `/ / _ \\ \ / # +# /_/|_/\_, /_/_//_/_\_\ # +# __/___/ __ ___ __ ___ __ __ # +# / _ )___ ____/ / / _ )___ / /_ / _ )/ /__ ____/ /_____ ____ # +# / _ / _ `/ _ / / _ / _ \/ __/ / _ / / _ \/ __/ '_/ -_) __/ # +# /____/\_,_/\_,_/ /____/\___/\__/ /____/_/\___/\__/_/\_\\__/_/ # +# # +############################################################################## + +# ************************ +# Set Terminal Font Colors +# ************************ + +bold=$(tput bold) +red=$(tput setaf 1) +green=$(tput setaf 2) +yellow=$(tput setaf 3) +blue=$(tput setaf 4) +magenta=$(tput setaf 5) +cyan=$(tput setaf 6) +white=$(tput setaf 7) +defaultcolor=$(tput setaf default) + +echo "${bold}${green}-------------------------------" +echo "${bold}${green}Whitelist Domains Test Starting" +echo "${bold}${green}-------------------------------" +printf "\n\n" + +echo "${bold}${green}---------------------------------------" +echo "${bold}${green}Activating Users whitelist-domains.conf" +echo "${bold}${green}---------------------------------------" +printf "\n\n" +sudo cp ${TRAVIS_BUILD_DIR}/.dev-tools/test_units/whitelist-domains.conf /etc/nginx/bots.d/whitelist-domains.conf +echo "${bold}${green}---------------" +echo "${bold}${green}Reloading Nginx" +echo "${bold}${green}---------------" +printf "\n\n" +sudo nginx -t && sudo nginx -s reload + + +echo "${bold}${yellow}-----------------------------------------------------------------------" +echo "${bold}${yellow}Sleeping for 10 seconds to allow Nginx to Properly Reload inside Travis" +echo "${bold}${yellow}-----------------------------------------------------------------------" +printf "\n\n" +sleep 10s + +# ************************************************* +# Function Curl Test 1 - Test User Domain Whitelist +# ************************************************* + +run_curltest1 () { +if curl http://localhost:9000 -e http://www.myowndomain.com 2>&1 | grep -i 'Welcome'; then + echo "${bold}${green}PASSED - User whitelist-domains.conf working" +else + echo "${bold}${red}FAILED - User whitelist-domains.conf NOT working" + #exit 1 +fi +} +run_curltest1 + +# ************************************************* +# Function Curl Test 2 - Test User Domain Whitelist +# ************************************************* + +run_curltest2 () { +if curl http://localhost:9000 -e http://www.myotherdomain.com 2>&1 | grep -i 'Welcome'; then + echo "${bold}${green}PASSED - User whitelist-domains.conf working" +else + echo "${bold}${red}FAILED - User whitelist-domains.conf NOT working" + #exit 1 +fi +} +run_curltest2 + + +echo "${bold}${green}-------------------------------" +echo "${bold}${green}Whitelist Domains Test Complete" +echo "${bold}${green}-------------------------------" +printf "\n\n" + +# ********************** +# Exit With Error Number +# ********************** + +exit ${?} + + diff --git a/.dev-tools/beta-test-blocker-whitelist-ips.sh b/.dev-tools/beta-test-blocker-whitelist-ips.sh new file mode 100755 index 000000000..bd56782b3 --- /dev/null +++ b/.dev-tools/beta-test-blocker-whitelist-ips.sh @@ -0,0 +1,127 @@ +#!/bin/bash +# Curl Testing Script for Nginx Ultimate Bad Bot Blocker +# Created by: Mitchell Krog (mitchellkrog@gmail.com) +# Copyright: Mitchell Krog - https://github.com/mitchellkrogza +# Repo Url: https://github.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker + +############################################################################## +# _ __ _ # +# / |/ /__ _(_)__ __ __ # +# / / _ `/ / _ \\ \ / # +# /_/|_/\_, /_/_//_/_\_\ # +# __/___/ __ ___ __ ___ __ __ # +# / _ )___ ____/ / / _ )___ / /_ / _ )/ /__ ____/ /_____ ____ # +# / _ / _ `/ _ / / _ / _ \/ __/ / _ / / _ \/ __/ '_/ -_) __/ # +# /____/\_,_/\_,_/ /____/\___/\__/ /____/_/\___/\__/_/\_\\__/_/ # +# # +############################################################################## + +# ************************ +# Set Terminal Font Colors +# ************************ + +bold=$(tput bold) +red=$(tput setaf 1) +green=$(tput setaf 2) +yellow=$(tput setaf 3) +blue=$(tput setaf 4) +magenta=$(tput setaf 5) +cyan=$(tput setaf 6) +white=$(tput setaf 7) +defaultcolor=$(tput setaf default) +thisip=$(curl -s ifconfig.co) + +echo "${bold}${green}--------------------------" +echo "${bold}${green}Whitelist IP Test Starting" +echo "${bold}${green}--------------------------" +printf "\n\n" + +echo "${bold}${green}-------------------------" +echo "${bold}${green}Blacklisting own IP First" +echo "${bold}${green}-------------------------" +printf "\n\n" + +sudo truncate -s 0 ${TRAVIS_BUILD_DIR}/.dev-tools/test_units/blacklist-ips.conf +printf '%s\t%s\n' "${thisip}" "1;" >> ${TRAVIS_BUILD_DIR}/.dev-tools/test_units/blacklist-ips.conf +printf '%s\t%s\n' "127.0.0.1" "1;" >> ${TRAVIS_BUILD_DIR}/.dev-tools/test_units/blacklist-ips.conf +sudo cp ${TRAVIS_BUILD_DIR}/.dev-tools/test_units/blacklist-ips.conf /etc/nginx/bots.d/blacklist-ips.conf + +echo "${bold}${green}--------------------" +echo "${bold}${green}Now Whitelist own IP" +echo "${bold}${green}--------------------" +printf "\n\n" + +sudo truncate -s 0 ${TRAVIS_BUILD_DIR}/.dev-tools/test_units/whitelist-ips.conf +printf '%s\t%s\n' "${thisip}" "0;" >> ${TRAVIS_BUILD_DIR}/.dev-tools/test_units/whitelist-ips.conf +printf '%s\t%s\n' "127.0.0.1" "1;" >> ${TRAVIS_BUILD_DIR}/.dev-tools/test_units/whitelist-ips.conf +printf '%s\t%s\n' "127.0.0.1" "1;" >> ${TRAVIS_BUILD_DIR}/.dev-tools/test_units/whitelist-ips.conf +printf '%s\t%s\n' "127.0.0.1" "1;" >> ${TRAVIS_BUILD_DIR}/.dev-tools/test_units/whitelist-ips.conf +printf '%s\t%s\n' "127.0.0.1" "0;" >> ${TRAVIS_BUILD_DIR}/.dev-tools/test_units/whitelist-ips.conf +sudo cp ${TRAVIS_BUILD_DIR}/.dev-tools/test_units/whitelist-ips.conf /etc/nginx/bots.d/whitelist-ips.conf + +sudo cp ${TRAVIS_BUILD_DIR}/.dev-tools/test_units/botblocker-nginx-settings.conf /etc/nginx/conf.d/botblocker-nginx-settings.conf + +#printf "\n" +#echo "${bold}${yellow}---------------------------------------------------" +#echo "${bold}${yellow}Copy Test Unit of globalblacklist.conf file to test" +#echo "${bold}${yellow}---------------------------------------------------" +#printf "\n" + +#sudo cp ${TRAVIS_BUILD_DIR}/conf.d/globalblacklist-testing-version.conf /etc/nginx/conf.d/globalblacklist.conf + + +echo "${bold}${green}---------------" +echo "${bold}${green}Reloading Nginx" +echo "${bold}${green}---------------" +printf "\n\n" +sudo nginx -t && sudo nginx -s reload + + +echo "${bold}${yellow}-----------------------------------------------------------------------" +echo "${bold}${yellow}Sleeping for 10 seconds to allow Nginx to Properly Reload inside Travis" +echo "${bold}${yellow}-----------------------------------------------------------------------" +printf "\n\n" +sleep 30s + +# ************************************************* +# Function Curl Test 1 - Test User Domain Whitelist +# ************************************************* + +run_curltest1 () { +if curl http://localhost:9000 2>&1 | grep -i 'Welcome'; then + echo "${bold}${green}PASSED - whitelist own ip is WORKING" +else + echo "${bold}${red}FAILED - whitelist own ip is NOT working" + curl http://localhost:9000 +fi +} +run_curltest1 + + +echo "${bold}${green}-----------------------------" +echo "${bold}${green}Whitelisting IP Test Complete" +echo "${bold}${green}-----------------------------" +printf "\n\n" + +# ************************************************************* +# Copy all .conf files used in Testing to a folder for checking +# ************************************************************* + +printf "\n" +echo "${bold}${green}------------------------------------------------------------" +echo "${bold}${green}Make Backup all conf files and folders used during this test" +echo "${bold}${green}------------------------------------------------------------" +printf "\n" +sudo cp /etc/nginx/bots.d/* ${TRAVIS_BUILD_DIR}/.dev-tools/_conf_files_testing_changes_ip_whitelist/bots.d/ +sudo cp /etc/nginx/conf.d/* ${TRAVIS_BUILD_DIR}/.dev-tools/_conf_files_testing_changes_ip_whitelist/conf.d/ +sudo cp /etc/nginx/sites-available/default.vhost ${TRAVIS_BUILD_DIR}/.dev-tools/_conf_files_testing_changes_ip_whitelist/default.vhost +sudo cp /etc/nginx/nginx.conf ${TRAVIS_BUILD_DIR}/.dev-tools/_conf_files_testing_changes_ip_whitelist/nginx.conf + + +# ********************** +# Exit With Error Number +# ********************** + +exit ${?} + + diff --git a/.dev-tools/beta-test-blocker-whitelist.sh b/.dev-tools/beta-test-blocker-whitelist.sh new file mode 100755 index 000000000..420fac700 --- /dev/null +++ b/.dev-tools/beta-test-blocker-whitelist.sh @@ -0,0 +1,264 @@ +#!/bin/bash +# Curl Testing Script for Nginx Ultimate Bad Bot Blocker +# Created by: Mitchell Krog (mitchellkrog@gmail.com) +# Copyright: Mitchell Krog - https://github.com/mitchellkrogza +# Repo Url: https://github.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker + +############################################################################## +# _ __ _ # +# / |/ /__ _(_)__ __ __ # +# / / _ `/ / _ \\ \ / # +# /_/|_/\_, /_/_//_/_\_\ # +# __/___/ __ ___ __ ___ __ __ # +# / _ )___ ____/ / / _ )___ / /_ / _ )/ /__ ____/ /_____ ____ # +# / _ / _ `/ _ / / _ / _ \/ __/ / _ / / _ \/ __/ '_/ -_) __/ # +# /____/\_,_/\_,_/ /____/\___/\__/ /____/_/\___/\__/_/\_\\__/_/ # +# # +############################################################################## + +# ************************ +# Set Terminal Font Colors +# ************************ + +bold=$(tput bold) +red=$(tput setaf 1) +green=$(tput setaf 2) +yellow=$(tput setaf 3) +blue=$(tput setaf 4) +magenta=$(tput setaf 5) +cyan=$(tput setaf 6) +white=$(tput setaf 7) +defaultcolor=$(tput setaf default) + +echo "${bold}${green}--------------------------" +echo "${bold}${green}Whitelisting Test Starting" +echo "${bold}${green}--------------------------" +printf "\n\n" + +echo "${bold}${magenta}-----------------------------------------------------------------" +echo "${bold}${magenta}Generating blacklist-user-agents.conf & custom-bad-referrers.conf" +echo "${bold}${magenta}-----------------------------------------------------------------" +printf "\n\n" + + +# ************************************************************************** +# Generate our custom test units turning all good to bad and all bad to good +# ************************************************************************** +_input1=${TRAVIS_BUILD_DIR}/_generator_lists/bad-user-agents.list +_input2=${TRAVIS_BUILD_DIR}/.dev-tools/referrers-regex-format-whitelist-test.txt +_inputdb1=${TRAVIS_BUILD_DIR}/.dev-tools/good-bad-user-agents.db +_inputdb2=${TRAVIS_BUILD_DIR}/.dev-tools/good-bad-referrers.db +_tmpnginx1=_tmpnginx1 +_tmpnginx2=_tmpnginx2 +_start1="# START MAKE BAD BOTS GOOD ### DO NOT EDIT THIS LINE AT ALL ###" +_end1="# END MAKE BAD BOTS GOOD ### DO NOT EDIT THIS LINE AT ALL ###" +_start2="# START MAKE BAD REFERRERS GOOD ### DO NOT EDIT THIS LINE AT ALL ###" +_end2="# END MAKE BAD REFERRERS GOOD ### DO NOT EDIT THIS LINE AT ALL ###" +_action1="0;" + +# **************************** +# BAD BOTS - Create and Insert +# **************************** + +echo "${bold}${yellow}------------------------" +echo "${bold}${green}Making all ${red}Bad Bots ${green}GOOD" +echo "${bold}${yellow}------------------------" +printf "\n\n" + +printf '%s\n' "$_start1" >> ${_tmpnginx1} +while IFS= read -r LINE +do +printf '\t"~*%s%s"\t\t%s\n' "(?:\b)${LINE}" "(?:\b|)" "$_action1" >> ${_tmpnginx1} +done < ${_input1} +printf '%s\n' "$_end1" >> ${_tmpnginx1} +mv ${_tmpnginx1} ${_inputdb1} +ed -s ${_inputdb1}<<\IN +1,/# START MAKE BAD BOTS GOOD ### DO NOT EDIT THIS LINE AT ALL ###/d +/# END MAKE BAD BOTS GOOD ### DO NOT EDIT THIS LINE AT ALL ###/,$d +,d +.r /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/.dev-tools/test_units/blacklist-user-agents.conf +/# START MAKE BAD BOTS GOOD ### DO NOT EDIT THIS LINE AT ALL ###/x +.t. +.,/# END MAKE BAD BOTS GOOD ### DO NOT EDIT THIS LINE AT ALL ###/-d +w /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/.dev-tools/test_units/blacklist-user-agents.conf +q +IN +rm ${_inputdb1} + +# ******************************** +# BAD REFERERS - Create and Insert +# ******************************** + +echo "${bold}${yellow}-----------------------------" +echo "${bold}${green}Making all ${red}Bad Referrers ${green}GOOD" +echo "${bold}${yellow}-----------------------------" +printf "\n\n" + + +printf '%s\n' "$_start2" >> ${_tmpnginx2} +while IFS= read -r LINE +do +printf '\t%s\n' "${LINE}" >> ${_tmpnginx2} +done < ${_input2} +printf '%s\n' "$_end2" >> ${_tmpnginx2} +mv ${_tmpnginx2} ${_inputdb2} +ed -s ${_inputdb2}<<\IN +1,/# START MAKE BAD REFERRERS GOOD ### DO NOT EDIT THIS LINE AT ALL ###/d +/# END MAKE BAD REFERRERS GOOD ### DO NOT EDIT THIS LINE AT ALL ###/,$d +,d +.r /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/.dev-tools/test_units/custom-bad-referrers.conf +/# START MAKE BAD REFERRERS GOOD ### DO NOT EDIT THIS LINE AT ALL ###/x +.t. +.,/# END MAKE BAD REFERRERS GOOD ### DO NOT EDIT THIS LINE AT ALL ###/-d +w /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/.dev-tools/test_units/custom-bad-referrers.conf +q +IN +rm ${_inputdb2} + + +echo "${bold}${green}------------------------------------------------" +echo "${bold}${green}Activating Users User-Agents Whitelist/Blacklist" +echo "${bold}${green}------------------------------------------------" +printf "\n\n" +sudo cp ${TRAVIS_BUILD_DIR}/.dev-tools/test_units/blacklist-user-agents.conf /etc/nginx/bots.d/blacklist-user-agents.conf +echo "${bold}${green}----------------------------------------------" +echo "${bold}${green}Activating Users Referrers Whitelist/Blacklist" +echo "${bold}${green}----------------------------------------------" +printf "\n\n" +sudo cp ${TRAVIS_BUILD_DIR}/.dev-tools/test_units/custom-bad-referrers.conf /etc/nginx/bots.d/custom-bad-referrers.conf +echo "${bold}${green}---------------" +echo "${bold}${green}Reloading Nginx" +echo "${bold}${green}---------------" +printf "\n\n" +sudo nginx -t && sudo nginx -s reload + + +echo "${bold}${yellow}-----------------------------------------------------------------------" +echo "${bold}${yellow}Sleeping for 10 seconds to allow Nginx to Properly Reload inside Travis" +echo "${bold}${yellow}-----------------------------------------------------------------------" +printf "\n\n" +sleep 10s + +# ******************************************************* +# Function Curl Test 1 - Test User Whitelist for "Nutch" +# ******************************************************* + +run_curltest1 () { +if curl -v -A "Nutch" http://localhost:9000 2>&1 | grep -i 'Welcome'; then + echo "${bold}${green}PASSED - WHITELISTING OF BAD BOT Nutch ALLOWED" +else + echo "${bold}${red}FAILED - WHITELISTING of BAD BOT Nutch FAILED" + #exit 1 + curl -v -A "Nutch" http://localhost:9000 2>&1 +fi +} +run_curltest1 + +# ************************************************************** +# Function Curl Test 2 - Check for Whitelisted Referrer "zx6.ru" +# ************************************************************** + +run_curltest2 () { +if curl http://localhost:9000 -e http://zx6.ru 2>&1 | grep -i 'Welcome'; then + echo "${bold}${green}PASSED - WHITELISTING OF REFERRER zx6.ru ALLOWED" +else + echo "${bold}${red}FAILED - WHITELISTING OF REFERRER zx6.ru FAILED" + #exit 1 + curl http://localhost:9000 -e http://zx6.ru 2>&1 +fi +} +run_curltest2 + +# ************************************************ +# Test 250 User-Agents from Bad-User-Agents +# ************************************************ + +shuf -n 250 ${TRAVIS_BUILD_DIR}/_generator_lists/bad-user-agents.list > ${TRAVIS_BUILD_DIR}/.dev-tools/test_units/random-bots-for-whitelist-test.tmp +sed 's/\\//g' ${TRAVIS_BUILD_DIR}/.dev-tools/test_units/random-bots-for-whitelist-test.tmp > ${TRAVIS_BUILD_DIR}/.dev-tools/test_units/random-bots-for-whitelist-test.list +sudo rm ${TRAVIS_BUILD_DIR}/.dev-tools/test_units/random-bots-for-whitelist-test.tmp +sort -u ${TRAVIS_BUILD_DIR}/.dev-tools/test_units/random-bots-for-whitelist-test.list -o ${TRAVIS_BUILD_DIR}/.dev-tools/test_units/random-bots-for-whitelist-test.list + +printf "\n\n" +echo "${bold}${magenta}---------------------------" +echo "${bold}${magenta}Testing 250 Random Bad Bots" +echo "${bold}${magenta}---------------------------" +printf "\n\n" +IFS=$'\n' +file=${TRAVIS_BUILD_DIR}/.dev-tools/test_units/random-bots-for-whitelist-test.list +lines=$(cat ${file}) +for line in ${lines}; do + if + curl -v -A "${line}" http://localhost:9000 2>&1 | grep -i 'Welcome'; then + echo "${bold}${green}PASSED - ${red}${line} was ${bold}${green}ALLOWED" + else + echo "${bold}${red}FAILED - ${red}${line} was ${bold}${red}NOT ALLOWED" + fi +done +IFS="" + +# ************************************************ +# Test 250 Referrers from Bad-Referrers +# ************************************************ + +shuf -n 250 ${TRAVIS_BUILD_DIR}/_generator_lists/bad-referrers.list > ${TRAVIS_BUILD_DIR}/.dev-tools/test_units/random-referrers-for-whitelist-test.tmp +sed 's/\\//g' ${TRAVIS_BUILD_DIR}/.dev-tools/test_units/random-referrers-for-whitelist-test.tmp > ${TRAVIS_BUILD_DIR}/.dev-tools/test_units/random-referrers-for-whitelist-test.list +sudo rm ${TRAVIS_BUILD_DIR}/.dev-tools/test_units/random-referrers-for-whitelist-test.tmp +sort -u ${TRAVIS_BUILD_DIR}/.dev-tools/test_units/random-referrers-for-whitelist-test.list -o ${TRAVIS_BUILD_DIR}/.dev-tools/test_units/random-referrers-for-whitelist-test.list + +printf "\n\n" +echo "${bold}${magenta}----------------------------" +echo "${bold}${magenta}Testing 250 Random Referrers" +echo "${bold}${magenta}----------------------------" +printf "\n\n" +IFS=$'\n' +file=${TRAVIS_BUILD_DIR}/.dev-tools/test_units/random-referrers-for-whitelist-test.list +lines=$(cat ${file}) +for line in ${lines}; do + if + curl http://localhost:9000 -e "http://${line}" 2>&1 | grep -i 'Welcome'; then + echo "${bold}${green}PASSED - ${red}${line} was ${bold}${green}ALLOWED" + else + echo "${bold}${red}FAILED - ${red}${line} was ${bold}${red}NOT ALLOWED" + fi +done +IFS="" + +printf "\n" +echo "${bold}${green}--------------------------" +echo "${bold}${green}Whitelisting Test Complete" +echo "${bold}${green}--------------------------" +printf "\n\n" + +# ************************************************************* +# Copy all .conf files used in Testing to a folder for checking +# ************************************************************* + +printf "\n" +echo "${bold}${green}------------------------------------------------------------" +echo "${bold}${green}Make Backup all conf files and folders used during this test" +echo "${bold}${green}------------------------------------------------------------" +printf "\n" +sudo cp /etc/nginx/bots.d/* ${TRAVIS_BUILD_DIR}/.dev-tools/_conf_files_testing_changes_whitelist/bots.d/ +sudo cp /etc/nginx/conf.d/* ${TRAVIS_BUILD_DIR}/.dev-tools/_conf_files_testing_changes_whitelist/conf.d/ +sudo cp /etc/nginx/sites-available/default.vhost ${TRAVIS_BUILD_DIR}/.dev-tools/_conf_files_testing_changes_whitelist/default.vhost +sudo cp /etc/nginx/nginx.conf ${TRAVIS_BUILD_DIR}/.dev-tools/_conf_files_testing_changes_whitelist/nginx.conf + +# ******************* +# RELEASE NEW VERSION +# ******************* +YEAR=$(date +"%Y") +MONTH=$(date +"%m") +latestbuild=V4.${YEAR}.${MONTH}.${TRAVIS_BUILD_NUMBER} +printf "\n" +echo "${bold}${green}All Nginx Tests Completed" +echo "${bold}${green}All Bot and Referrer Testing Completed" +echo "${bold}${green}All Function Testing Completed" +echo "${bold}${magenta}Releasing ${latestbuild}" + +# ********************** +# Exit With Error Number +# ********************** + +exit ${?} + + diff --git a/.dev-tools/beta-test-blocker.sh b/.dev-tools/beta-test-blocker.sh new file mode 100755 index 000000000..c84477712 --- /dev/null +++ b/.dev-tools/beta-test-blocker.sh @@ -0,0 +1,443 @@ +#!/bin/bash +# Curl Testing Script for Nginx Ultimate Bad Bot Blocker +# Created by: Mitchell Krog (mitchellkrog@gmail.com) +# Copyright: Mitchell Krog - https://github.com/mitchellkrogza +# Repo Url: https://github.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker + +############################################################################## +# _ __ _ # +# / |/ /__ _(_)__ __ __ # +# / / _ `/ / _ \\ \ / # +# /_/|_/\_, /_/_//_/_\_\ # +# __/___/ __ ___ __ ___ __ __ # +# / _ )___ ____/ / / _ )___ / /_ / _ )/ /__ ____/ /_____ ____ # +# / _ / _ `/ _ / / _ / _ \/ __/ / _ / / _ \/ __/ '_/ -_) __/ # +# /____/\_,_/\_,_/ /____/\___/\__/ /____/_/\___/\__/_/\_\\__/_/ # +# # +############################################################################## + +# ************************ +# Set Terminal Font Colors +# ************************ + +bold=$(tput bold) +red=$(tput setaf 1) +green=$(tput setaf 2) +yellow=$(tput setaf 3) +blue=$(tput setaf 4) +magenta=$(tput setaf 5) +cyan=$(tput setaf 6) +white=$(tput setaf 7) +defaultcolor=$(tput setaf default) + +# ******************************************* +# Set Location of our Curl Test Results Files +# ******************************************* + +#echo "${bold}${green}------------------------------------------------" +#echo "${bold}${green}Disable any User Whitelisting and set to Default" +#echo "${bold}${green}------------------------------------------------" +#printf "\n\n" + +#sudo rm /etc/nginx/conf.d/*.conf +#sudo rm /etc/nginx/bots.d/*.conf +#ls -la /etc/nginx/conf.d/ +#ls -la /etc/nginx/bots.d/ + +#printf "\n" +#echo "${bold}${magenta}-------------------------" +#echo "${bold}${magenta}Execute update-ngxblocker" +#echo "${bold}${magenta}-------------------------" +#printf "\n" +#cd /usr/sbin +#sudo bash ./update-ngxblocker -n + +#echo "${bold}${green}---------------" +#echo "${bold}${green}Reloading Nginx" +#echo "${bold}${green}---------------" +#printf "\n\n" +#sudo nginx -t && sudo nginx -s reload + +#echo "${bold}${yellow}-----------------------------------------------------------------------" +#echo "${bold}${yellow}Sleeping for 10 seconds to allow Nginx to Properly Reload inside Travis" +#echo "${bold}${yellow}-----------------------------------------------------------------------" +#printf "\n\n" +#sleep 10s + +echo "${bold}${green}--------------" +echo "${bold}${green}Tests Starting" +echo "${bold}${green}--------------" +printf "\n\n" + + +# ************************************************* +# Function Curl Test 1 - Check for Bad Bot "80legs" +# ************************************************* + +run_curltest1 () { +if curl -v -A "80legs" http://localhost:9000 2>&1 | grep -i '(52)'; then + echo "${bold}${green}PASSED - ${red}80legs BAD BOT DETECTED" +else + echo "${bold}${red}FAILED - ${red}80legs BAD BOT NOT DETECTED" + exit 1 +fi +} +run_curltest1 + + +# ************************************************** +# Function Curl Test 2 - Check for Bad Bot "Nutch" +# ************************************************** + +run_curltest2 () { +if curl -v -A "Nutch" http://localhost:9000 2>&1 | grep -i '(52)'; then + echo "${bold}${green}PASSED - ${red}Nutch BAD BOT DETECTED" +else + echo "${bold}${red}FAILED - ${red}Nutch BAD BOT NOT DETECTED" + exit 1 +fi +} +run_curltest2 + + +# ****************************************************************** +# Function Curl Test 3 - Check for Bad Referrer "100dollars-seo.com" +# ****************************************************************** + +run_curltest3 () { +if curl -I http://localhost:9000 -e http://100dollars-seo.com 2>&1 | grep -i '(52)'; then + echo "${bold}${green}PASSED - ${red}100dollars-seo.com BAD REFERRER DETECTED" +else + echo "${bold}${red}FAILED - ${red}100dollars-seo.com BAD REFERRER NOT DETECTED" + exit 1 +fi +} +run_curltest3 + + +# ****************************************************** +# Function Curl Test 4 - Check for Bad Referrer "zx6.ru" +# ****************************************************** + +run_curltest4 () { +if curl -I http://localhost:9000 -e http://zx6.ru 2>&1 | grep -i '(52)'; then + echo "${bold}${green}PASSED - ${red}zx6.ru BAD REFERRER DETECTED" +else + echo "${bold}${red}FAILED - ${red}zx6.ru BAD REFERRER NOT DETECTED" + exit 1 +fi +} +run_curltest4 + +# ***************************************************** +# Function Curl Test 5 - Check for Good Bot "GoogleBot" +# ***************************************************** + +run_curltest5 () { +if curl -v -A "GoogleBot" http://localhost:9000 2>&1 | grep -i 'Welcome'; then + echo "${bold}${green}PASSED - ${green}GoogleBot GOOD BOT ALLOWED THROUGH" +else + echo "${bold}${red}FAILED - ${red}GoogleBot GOOD BOT NOT ALLOWED THROUGH" + exit 1 +fi +} +run_curltest5 + +# *************************************************** +# Function Curl Test 6 - Check for Good Bot "BingBot" +# *************************************************** + +run_curltest6 () { +if curl -v -A "BingBot" http://localhost:9000 2>&1 | grep -i 'Welcome'; then + echo "${bold}${green}PASSED - ${green}BingBot GOOD BOT ALLOWED THROUGH" +else + echo "${bold}${red}FAILED - ${red}BingBot GOOD BOT NOT ALLOWED THROUGH" + exit 1 +fi +} +run_curltest6 + +# *********************************************************** +# Function Curl Test 7 - Check for Good Referrer "google.com" +# *********************************************************** + +run_curltest7 () { +if curl http://localhost:9000 -e http://google.com 2>&1 | grep -i 'Welcome'; then + echo "${bold}${green}PASSED - ${green}google.com GOOD REFERRER DETECTED" +else + echo "${bold}${red}FAILED - ${red}google.com GOOD REFERRER NOT DETECTED" + exit 1 +fi +} +run_curltest7 + +# *********************************************************** +# Function Curl Test 8 - Check for Good Referrer "bing.com" +# *********************************************************** + +run_curltest8 () { +if curl http://localhost:9000 -e http://bing.com 2>&1 | grep -i 'Welcome'; then + echo "${bold}${green}PASSED - ${red}bing.com GOOD REFERRER DETECTED" +else + echo "${bold}${red}FAILED - ${red}bing.com GOOD REFERRER NOT DETECTED" + exit 1 +fi +} +run_curltest8 + +# ************************************************** +# Function Curl Test 9 - Check for Bad Bot "Nutch" +# ************************************************** + +run_curltest9 () { +if curl -A "Googlebot/Nutch-1.7" http://localhost:9000 2>&1 | grep -i '(52)'; then + echo "${bold}${green}PASSED - ${red}Googlebot/Nutch-1.7 BAD BOT DETECTED" +else + echo "${bold}${red}FAILED - ${red}Googlebot/Nutch-1.7 BAD BOT NOT DETECTED" + #exit 1 +fi +} +run_curltest9 + +# ************************************************** +# Function Curl Test 10 - Check for Bad Bot "Nutch" +# ************************************************** + +run_curltest10 () { +if curl -A "Mozilla/5.0 (compatible; Googlebot/Nutch2.1; +http://www.google.com/bot.html)" http://localhost:9000 2>&1 | grep -i '(52)'; then + echo "${bold}${green}PASSED - ${red}Googlebot/Nutch2.1 BAD BOT DETECTED" +else + echo "${bold}${red}FAILED - ${red}Googlebot/Nutch2.1 BAD BOT NOT DETECTED" + #exit 1 +fi +} +run_curltest10 + +# ************************************************** +# Function Curl Test 11 - Check for Bad Bot "Nutch" +# ************************************************** + +run_curltest11 () { +if curl -A "Mozilla/5.0 (compatible; Googlebot/nutch/-2.1; +http://www.google.com/bot.html)" http://localhost:9000 2>&1 | grep -i '(52)'; then + echo "${bold}${green}PASSED - ${red}Googlebot/nutch/-2.1 BAD BOT DETECTED" +else + echo "${bold}${red}FAILED - Googlebot/nutch/-2.1 BAD BOT NOT DETECTED" + #exit 1 +fi +} +run_curltest11 + +# ************************************************** +# Function Curl Test 12 - Check for Bad Bot "Nutch" +# ************************************************** + +run_curltest12 () { +if curl -v -A "Mozilla/5.0 (compatible; Googlebot-Image/Snutch\-/-2.1; +http://www.google.com/bot.html)" http://localhost:9000 2>&1 | grep -i 'Welcome'; then + echo "${bold}${green}PASSED - NO FALSE POSITIVE on Googlebot-Image/Snutch\-/-2.1" +else + echo "${bold}${red}FAILED - FALSE POSITIVE FOUND on Googlebot-Image/Snutch\-/-2.1" + #exit 1 +fi +} +run_curltest12 + +# ************************************************** +# Function Curl Test 13 - Check for Bad Bot "Nutch" +# ************************************************** + +run_curltest13 () { +if curl -v -A "SnutchMozilla/5.0 (compatible; Googlebot-Image/SMutch\-/-2.1; +http://www.google.com/bot.html)" http://localhost:9000 2>&1 | grep -i 'Welcome'; then + echo "${bold}${green}PASSED - NO FALSE POSITIVE on SnutchMozilla/5.0" +else + echo "${bold}${red}FAILED - FALSE POSITIVE FOUND on SnutchMozilla/5.0" + #exit 1 +fi +} +run_curltest13 + +# ************************************************** +# Function Curl Test 14 - Check for Good Bot +# ************************************************** + +run_curltest14 () { +if curl -v -A "Mozilla/5.0 (X11; Linux x86_64; rv:67.0) Gecko/20100101 Firefox/67.0" http://localhost:9000 2>&1 | grep -i 'Welcome'; then + echo "${bold}${green}PASSED - NO FALSE POSITIVE on Mozilla/5.0" +else + echo "${bold}${red}FAILED - FALSE POSITIVE FOUND on Mozilla/5.0" + #exit 1 +fi +} +run_curltest14 + + +# ************************************************** +# Function Curl Test 15 - TEST CASES +# ************************************************** + +run_curltest15 () { +if curl -v -A "Mozilla/5.0 (iPhone; CPU iPhone OS 12_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.1 Mobile/15E148 Safari/604.1" http://localhost:9000 2>&1 | grep -i 'Welcome'; then + echo "${bold}${green}PASSED - NO FALSE POSITIVE on Safari" +else + echo "${bold}${red}FAILED - FALSE POSITIVE FOUND on Safari" + #exit 1 +fi +} +run_curltest15 + +# Add Any More Tests Here for False Positives + +# ************************************************ +# Test 250 User-Agents from Bad-User-Agents +# ************************************************ +shuf -n 250 ${TRAVIS_BUILD_DIR}/_generator_lists/bad-user-agents.list > ${TRAVIS_BUILD_DIR}/.dev-tools/test_units/random-bots-for-test.tmp +sed 's/\\//g' ${TRAVIS_BUILD_DIR}/.dev-tools/test_units/random-bots-for-test.tmp > ${TRAVIS_BUILD_DIR}/.dev-tools/test_units/random-bots-for-test.list +sudo rm ${TRAVIS_BUILD_DIR}/.dev-tools/test_units/random-bots-for-test.tmp +sort -u ${TRAVIS_BUILD_DIR}/.dev-tools/test_units/random-bots-for-test.list -o ${TRAVIS_BUILD_DIR}/.dev-tools/test_units/random-bots-for-test.list + +printf "\n\n" +echo "${bold}${magenta}---------------------------" +echo "${bold}${magenta}Testing 250 Random Bad Bots" +echo "${bold}${magenta}---------------------------" +printf "\n\n" +IFS=$'\n' +file=${TRAVIS_BUILD_DIR}/.dev-tools/test_units/random-bots-for-test.list +lines=$(cat ${file}) +for line in ${lines}; do + if + curl -v -A "${line}" http://localhost:9000 2>&1 | grep -i '(52)'; then + echo "${bold}${green}PASSED - ${red}${line} was ${bold}${red}BLOCKED" + else + echo "${bold}${red}FAILED - ${red}${line} was ${bold}${red}NOT BLOCKED" + fi +done +IFS="" + +# ************************************************ +# Test All Good User-Agents from Good-User-Agents +# ************************************************ +sed 's/\\//g' ${TRAVIS_BUILD_DIR}/_generator_lists/good-user-agents.list > ${TRAVIS_BUILD_DIR}/.dev-tools/test_units/good-bots-for-test.list + +printf "\n\n" +echo "${bold}${magenta}---------------------" +echo "${bold}${magenta}Testing All Good Bots" +echo "${bold}${magenta}---------------------" +printf "\n\n" +IFS=$'\n' +file=${TRAVIS_BUILD_DIR}/.dev-tools/test_units/good-bots-for-test.list +lines=$(cat ${file}) +for line in ${lines}; do + if + curl -v -A "${line}" http://localhost:9000 2>&1 | grep -i 'Welcome'; then + echo "${bold}${green}PASSED - ${green}${line} was ${bold}${green}ALLOWED" + else + echo "${bold}${red}FAILED - ${red}${line} was ${bold}${red}BLOCKED" + fi +done +IFS="" + + +# ***************************************************** +# Test All Allowed User-Agents from Allowed-User-Agents +# ***************************************************** +sed 's/\\//g' ${TRAVIS_BUILD_DIR}/_generator_lists/allowed-user-agents.list > ${TRAVIS_BUILD_DIR}/.dev-tools/test_units/allowed-bots-for-test.list + +printf "\n\n" +echo "${bold}${magenta}------------------------" +echo "${bold}${magenta}Testing All Allowed Bots" +echo "${bold}${magenta}------------------------" +printf "\n\n" +IFS=$'\n' +file=${TRAVIS_BUILD_DIR}/.dev-tools/test_units/allowed-bots-for-test.list +lines=$(cat ${file}) +for line in ${lines}; do + if + curl -v -A "${line}" http://localhost:9000 2>&1 | grep -i 'Welcome'; then + echo "${bold}${green}PASSED - ${green}${line} was ${bold}${green}ALLOWED" + else + echo "${bold}${red}FAILED - ${red}${line} was ${bold}${red}BLOCKED" + fi +done +IFS="" + +# ***************************************************** +# Test All Limited User-Agents from Limited-User-Agents +# ***************************************************** +sed 's/\\//g' ${TRAVIS_BUILD_DIR}/_generator_lists/limited-user-agents.list > ${TRAVIS_BUILD_DIR}/.dev-tools/test_units/limited-bots-for-test.list + +printf "\n\n" +echo "${bold}${magenta}------------------------" +echo "${bold}${magenta}Testing All Limited Bots" +echo "${bold}${magenta}------------------------" +printf "\n\n" +IFS=$'\n' +file=${TRAVIS_BUILD_DIR}/.dev-tools/test_units/limited-bots-for-test.list +lines=$(cat ${file}) +for line in ${lines}; do + if + curl -v -A "${line}" http://localhost:9000 2>&1 | grep -i 'Welcome'; then + echo "${bold}${green}PASSED - ${green}${line} was ${bold}${green}ALLOWED" + else + echo "${bold}${green}FAILED - ${red}${line} was ${bold}${red}BLOCKED" + fi +done +IFS="" + +# ******************************************** +# Test 1000 Random Referrers from Bad-Referrers +# ******************************************** +shuf -n 1000 ${TRAVIS_BUILD_DIR}/_generator_lists/bad-referrers.list > ${TRAVIS_BUILD_DIR}/.dev-tools/test_units/random-referrers-for-test.list +sort -u ${TRAVIS_BUILD_DIR}/.dev-tools/test_units/random-referrers-for-test.list -o ${TRAVIS_BUILD_DIR}/.dev-tools/test_units/random-referrers-for-test.list + +printf "\n\n" +echo "${bold}${magenta}---------------------------------" +echo "${bold}${magenta}Testing 1000 Random Bad Referrers" +echo "${bold}${magenta}---------------------------------" +printf "\n\n" +IFS=$'\n' +file=${TRAVIS_BUILD_DIR}/.dev-tools/test_units/random-referrers-for-test.list +lines=$(cat ${file}) +for line in ${lines}; do + if + curl -I http://localhost:9000 -e "http://${line}" 2>&1 | grep -i '(52)'; then + echo "${bold}${green}PASSED - ${red}${line} was ${bold}${red}BLOCKED" + else + echo "${bold}${red}FAILED - ${red}${line} was ${bold}${red}NOT BLOCKED" + fi +done +IFS="" + +# ************************** +# Test Good Referrer Domains +# ************************** + +printf "\n\n" +echo "${bold}${magenta}----------------------" +echo "${bold}${magenta}Testing Good Referrers" +echo "${bold}${magenta}----------------------" +printf "\n\n" +IFS=$'\n' +file=${TRAVIS_BUILD_DIR}/.dev-tools/test_units/good-referrers-for-test.list +lines=$(cat ${file}) +for line in ${lines}; do + if + curl -v -A "${line}" http://localhost:9000 2>&1 | grep -i 'Welcome'; then + echo "${bold}${green}PASSED - ${green}${line} was ${bold}${green}ALLOWED" + else + echo "${bold}${green}FAILED - ${red}${line} was ${bold}${red}BLOCKED" + fi +done +IFS="" + + +printf "\n" +echo "${bold}${cyan}Tests Completed" +echo "${bold}${green}All Tests Passed" +echo "${bold}${magenta}Now We Test Whitelisting and then Release a new build" + +# ********************** +# Exit With Error Number +# ********************** + +exit ${?} + +