vikarti.anatra/nginx-ultimate-bad-bot-blocker
1
0
Fork 0
mirror of https://github.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker.git synced 2025-09-04 03:29:57 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

GHA workflow - Beta Tests

Browse source 
Ref: #419
This commit is contained in:
Mitchell Krog 2021-07-05 11:24:35 +02:00
parent 515a553f93
commit 5f0f19316a
No known key found for this signature in database
GPG key ID: E02E9D10427B6A43
4 changed files with 150 additions and 142 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

176
dev-tools/beta-test-blocker-rate-limiting.sh
View file

@ -16,6 +16,8 @@
# # # #
############################################################################## ##############################################################################
export TERM=xterm
# ------------------------------------------------------------------------------ # ------------------------------------------------------------------------------
# MIT License # MIT License
# ------------------------------------------------------------------------------ # ------------------------------------------------------------------------------
@ -64,7 +66,7 @@ echo "${bold}${green}---------------"
echo "${bold}${green}Reloading Nginx" echo "${bold}${green}Reloading Nginx"
echo "${bold}${green}---------------" echo "${bold}${green}---------------"
printf "\n\n" printf "\n\n"
sudo nginx -t && sudo nginx -s reload sudo nginx -t && sudo systemctl reload nginx
} }
waitforReload () { waitforReload () {
@ -80,92 +82,92 @@ echo "${bold}${yellow}-----------------------------"
echo "${bold}${yellow}Making GoogleBot ${yellow}RATE LIMITED" echo "${bold}${yellow}Making GoogleBot ${yellow}RATE LIMITED"
echo "${bold}${yellow}-----------------------------" echo "${bold}${yellow}-----------------------------"
printf "\n\n" printf "\n\n"
sudo cp ${TRAVIS_BUILD_DIR}/.dev-tools/test_units/rate-limiting-user-agents.conf /etc/nginx/bots.d/blacklist-user-agents.conf sudo cp ./dev-tools/test_units/rate-limiting-user-agents.conf /etc/nginx/bots.d/blacklist-user-agents.conf
} }
testRateLimiting () { testRateLimiting () {
ratelimittestfile=${TRAVIS_BUILD_DIR}/.dev-tools/test_units/ratelimittest-beta.txt ratelimittestfile=./dev-tools/test_units/ratelimittest-beta.txt
truncate -s 0 ${ratelimittestfile} truncate -s 0 ${ratelimittestfile}
curl -A "GoogleBot" http://localhost:9000 2>&1 > ${ratelimittestfile} & curl -A "GoogleBot" http://localhost:80 2>&1 > ${ratelimittestfile} &
curl -A "GoogleBot" http://localhost:9000 2>&1 >> ${ratelimittestfile} & curl -A "GoogleBot" http://localhost:80 2>&1 >> ${ratelimittestfile} &
curl -A "GoogleBot" http://localhost:9000 2>&1 >> ${ratelimittestfile} & curl -A "GoogleBot" http://localhost:80 2>&1 >> ${ratelimittestfile} &
curl -A "GoogleBot" http://localhost:9000 2>&1 >> ${ratelimittestfile} & curl -A "GoogleBot" http://localhost:80 2>&1 >> ${ratelimittestfile} &
curl -A "GoogleBot" http://localhost:9000 2>&1 >> ${ratelimittestfile} & curl -A "GoogleBot" http://localhost:80 2>&1 >> ${ratelimittestfile} &
curl -A "GoogleBot" http://localhost:9000 2>&1 >> ${ratelimittestfile} & curl -A "GoogleBot" http://localhost:80 2>&1 >> ${ratelimittestfile} &
curl -A "GoogleBot" http://localhost:9000 2>&1 >> ${ratelimittestfile} & curl -A "GoogleBot" http://localhost:80 2>&1 >> ${ratelimittestfile} &
curl -A "GoogleBot" http://localhost:9000 2>&1 >> ${ratelimittestfile} & curl -A "GoogleBot" http://localhost:80 2>&1 >> ${ratelimittestfile} &
curl -A "GoogleBot" http://localhost:9000 2>&1 >> ${ratelimittestfile} & curl -A "GoogleBot" http://localhost:80 2>&1 >> ${ratelimittestfile} &
curl -A "GoogleBot" http://localhost:9000 2>&1 >> ${ratelimittestfile} & curl -A "GoogleBot" http://localhost:80 2>&1 >> ${ratelimittestfile} &
curl -A "GoogleBot" http://localhost:9000 2>&1 >> ${ratelimittestfile} & curl -A "GoogleBot" http://localhost:80 2>&1 >> ${ratelimittestfile} &
curl -A "GoogleBot" http://localhost:9000 2>&1 >> ${ratelimittestfile} & curl -A "GoogleBot" http://localhost:80 2>&1 >> ${ratelimittestfile} &
curl -A "GoogleBot" http://localhost:9000 2>&1 >> ${ratelimittestfile} & curl -A "GoogleBot" http://localhost:80 2>&1 >> ${ratelimittestfile} &
curl -A "GoogleBot" http://localhost:9000 2>&1 >> ${ratelimittestfile} & curl -A "GoogleBot" http://localhost:80 2>&1 >> ${ratelimittestfile} &
curl -A "GoogleBot" http://localhost:9000 2>&1 >> ${ratelimittestfile} & curl -A "GoogleBot" http://localhost:80 2>&1 >> ${ratelimittestfile} &
curl -A "GoogleBot" http://localhost:9000 2>&1 >> ${ratelimittestfile} & curl -A "GoogleBot" http://localhost:80 2>&1 >> ${ratelimittestfile} &
curl -A "GoogleBot" http://localhost:9000 2>&1 >> ${ratelimittestfile} & curl -A "GoogleBot" http://localhost:80 2>&1 >> ${ratelimittestfile} &
curl -A "GoogleBot" http://localhost:9000 2>&1 >> ${ratelimittestfile} & curl -A "GoogleBot" http://localhost:80 2>&1 >> ${ratelimittestfile} &
curl -A "GoogleBot" http://localhost:9000 2>&1 >> ${ratelimittestfile} & curl -A "GoogleBot" http://localhost:80 2>&1 >> ${ratelimittestfile} &
curl -A "GoogleBot" http://localhost:9000 2>&1 >> ${ratelimittestfile} & curl -A "GoogleBot" http://localhost:80 2>&1 >> ${ratelimittestfile} &
curl -A "GoogleBot" http://localhost:9000 2>&1 >> ${ratelimittestfile} & curl -A "GoogleBot" http://localhost:80 2>&1 >> ${ratelimittestfile} &
curl -A "GoogleBot" http://localhost:9000 2>&1 >> ${ratelimittestfile} & curl -A "GoogleBot" http://localhost:80 2>&1 >> ${ratelimittestfile} &
curl -A "GoogleBot" http://localhost:9000 2>&1 >> ${ratelimittestfile} & curl -A "GoogleBot" http://localhost:80 2>&1 >> ${ratelimittestfile} &
curl -A "GoogleBot" http://localhost:9000 2>&1 >> ${ratelimittestfile} & curl -A "GoogleBot" http://localhost:80 2>&1 >> ${ratelimittestfile} &
curl -A "GoogleBot" http://localhost:9000 2>&1 >> ${ratelimittestfile} & curl -A "GoogleBot" http://localhost:80 2>&1 >> ${ratelimittestfile} &
curl -A "GoogleBot" http://localhost:9000 2>&1 >> ${ratelimittestfile} & curl -A "GoogleBot" http://localhost:80 2>&1 >> ${ratelimittestfile} &
curl -A "GoogleBot" http://localhost:9000 2>&1 >> ${ratelimittestfile} & curl -A "GoogleBot" http://localhost:80 2>&1 >> ${ratelimittestfile} &
curl -A "GoogleBot" http://localhost:9000 2>&1 >> ${ratelimittestfile} & curl -A "GoogleBot" http://localhost:80 2>&1 >> ${ratelimittestfile} &
curl -A "GoogleBot" http://localhost:9000 2>&1 >> ${ratelimittestfile} & curl -A "GoogleBot" http://localhost:80 2>&1 >> ${ratelimittestfile} &
curl -A "GoogleBot" http://localhost:9000 2>&1 >> ${ratelimittestfile} & curl -A "GoogleBot" http://localhost:80 2>&1 >> ${ratelimittestfile} &
curl -A "GoogleBot" http://localhost:9000 2>&1 >> ${ratelimittestfile} & curl -A "GoogleBot" http://localhost:80 2>&1 >> ${ratelimittestfile} &
curl -A "GoogleBot" http://localhost:9000 2>&1 >> ${ratelimittestfile} & curl -A "GoogleBot" http://localhost:80 2>&1 >> ${ratelimittestfile} &
curl -A "GoogleBot" http://localhost:9000 2>&1 >> ${ratelimittestfile} & curl -A "GoogleBot" http://localhost:80 2>&1 >> ${ratelimittestfile} &
curl -A "GoogleBot" http://localhost:9000 2>&1 >> ${ratelimittestfile} & curl -A "GoogleBot" http://localhost:80 2>&1 >> ${ratelimittestfile} &
curl -A "GoogleBot" http://localhost:9000 2>&1 >> ${ratelimittestfile} & curl -A "GoogleBot" http://localhost:80 2>&1 >> ${ratelimittestfile} &
curl -A "GoogleBot" http://localhost:9000 2>&1 >> ${ratelimittestfile} & curl -A "GoogleBot" http://localhost:80 2>&1 >> ${ratelimittestfile} &
curl -A "GoogleBot" http://localhost:9000 2>&1 >> ${ratelimittestfile} & curl -A "GoogleBot" http://localhost:80 2>&1 >> ${ratelimittestfile} &
curl -A "GoogleBot" http://localhost:9000 2>&1 >> ${ratelimittestfile} & curl -A "GoogleBot" http://localhost:80 2>&1 >> ${ratelimittestfile} &
curl -A "GoogleBot" http://localhost:9000 2>&1 >> ${ratelimittestfile} & curl -A "GoogleBot" http://localhost:80 2>&1 >> ${ratelimittestfile} &
curl -A "GoogleBot" http://localhost:9000 2>&1 >> ${ratelimittestfile} & curl -A "GoogleBot" http://localhost:80 2>&1 >> ${ratelimittestfile} &
curl -A "GoogleBot" http://localhost:9000 2>&1 >> ${ratelimittestfile} & curl -A "GoogleBot" http://localhost:80 2>&1 >> ${ratelimittestfile} &
curl -A "GoogleBot" http://localhost:9000 2>&1 >> ${ratelimittestfile} & curl -A "GoogleBot" http://localhost:80 2>&1 >> ${ratelimittestfile} &
curl -A "GoogleBot" http://localhost:9000 2>&1 >> ${ratelimittestfile} & curl -A "GoogleBot" http://localhost:80 2>&1 >> ${ratelimittestfile} &
curl -A "GoogleBot" http://localhost:9000 2>&1 >> ${ratelimittestfile} & curl -A "GoogleBot" http://localhost:80 2>&1 >> ${ratelimittestfile} &
curl -A "GoogleBot" http://localhost:9000 2>&1 >> ${ratelimittestfile} & curl -A "GoogleBot" http://localhost:80 2>&1 >> ${ratelimittestfile} &
curl -A "GoogleBot" http://localhost:9000 2>&1 >> ${ratelimittestfile} & curl -A "GoogleBot" http://localhost:80 2>&1 >> ${ratelimittestfile} &
curl -A "GoogleBot" http://localhost:9000 2>&1 >> ${ratelimittestfile} & curl -A "GoogleBot" http://localhost:80 2>&1 >> ${ratelimittestfile} &
curl -A "GoogleBot" http://localhost:9000 2>&1 >> ${ratelimittestfile} & curl -A "GoogleBot" http://localhost:80 2>&1 >> ${ratelimittestfile} &
curl -A "GoogleBot" http://localhost:9000 2>&1 >> ${ratelimittestfile} & curl -A "GoogleBot" http://localhost:80 2>&1 >> ${ratelimittestfile} &
curl -A "GoogleBot" http://localhost:9000 2>&1 >> ${ratelimittestfile} & curl -A "GoogleBot" http://localhost:80 2>&1 >> ${ratelimittestfile} &
curl -A "GoogleBot" http://localhost:9000 2>&1 >> ${ratelimittestfile} & curl -A "GoogleBot" http://localhost:80 2>&1 >> ${ratelimittestfile} &
curl -A "GoogleBot" http://localhost:9000 2>&1 >> ${ratelimittestfile} & curl -A "GoogleBot" http://localhost:80 2>&1 >> ${ratelimittestfile} &
curl -A "GoogleBot" http://localhost:9000 2>&1 >> ${ratelimittestfile} & curl -A "GoogleBot" http://localhost:80 2>&1 >> ${ratelimittestfile} &
curl -A "GoogleBot" http://localhost:9000 2>&1 >> ${ratelimittestfile} & curl -A "GoogleBot" http://localhost:80 2>&1 >> ${ratelimittestfile} &
curl -A "GoogleBot" http://localhost:9000 2>&1 >> ${ratelimittestfile} & curl -A "GoogleBot" http://localhost:80 2>&1 >> ${ratelimittestfile} &
curl -A "GoogleBot" http://localhost:9000 2>&1 >> ${ratelimittestfile} & curl -A "GoogleBot" http://localhost:80 2>&1 >> ${ratelimittestfile} &
curl -A "GoogleBot" http://localhost:9000 2>&1 >> ${ratelimittestfile} & curl -A "GoogleBot" http://localhost:80 2>&1 >> ${ratelimittestfile} &
curl -A "GoogleBot" http://localhost:9000 2>&1 >> ${ratelimittestfile} & curl -A "GoogleBot" http://localhost:80 2>&1 >> ${ratelimittestfile} &
curl -A "GoogleBot" http://localhost:9000 2>&1 >> ${ratelimittestfile} & curl -A "GoogleBot" http://localhost:80 2>&1 >> ${ratelimittestfile} &
curl -A "GoogleBot" http://localhost:9000 2>&1 >> ${ratelimittestfile} & curl -A "GoogleBot" http://localhost:80 2>&1 >> ${ratelimittestfile} &
curl -A "GoogleBot" http://localhost:9000 2>&1 >> ${ratelimittestfile} & curl -A "GoogleBot" http://localhost:80 2>&1 >> ${ratelimittestfile} &
curl -A "GoogleBot" http://localhost:9000 2>&1 >> ${ratelimittestfile} & curl -A "GoogleBot" http://localhost:80 2>&1 >> ${ratelimittestfile} &
curl -A "GoogleBot" http://localhost:9000 2>&1 >> ${ratelimittestfile} & curl -A "GoogleBot" http://localhost:80 2>&1 >> ${ratelimittestfile} &
curl -A "GoogleBot" http://localhost:9000 2>&1 >> ${ratelimittestfile} & curl -A "GoogleBot" http://localhost:80 2>&1 >> ${ratelimittestfile} &
curl -A "GoogleBot" http://localhost:9000 2>&1 >> ${ratelimittestfile} & curl -A "GoogleBot" http://localhost:80 2>&1 >> ${ratelimittestfile} &
curl -A "GoogleBot" http://localhost:9000 2>&1 >> ${ratelimittestfile} & curl -A "GoogleBot" http://localhost:80 2>&1 >> ${ratelimittestfile} &
curl -A "GoogleBot" http://localhost:9000 2>&1 >> ${ratelimittestfile} & curl -A "GoogleBot" http://localhost:80 2>&1 >> ${ratelimittestfile} &
curl -A "GoogleBot" http://localhost:9000 2>&1 >> ${ratelimittestfile} & curl -A "GoogleBot" http://localhost:80 2>&1 >> ${ratelimittestfile} &
curl -A "GoogleBot" http://localhost:9000 2>&1 >> ${ratelimittestfile} & curl -A "GoogleBot" http://localhost:80 2>&1 >> ${ratelimittestfile} &
curl -A "GoogleBot" http://localhost:9000 2>&1 >> ${ratelimittestfile} & curl -A "GoogleBot" http://localhost:80 2>&1 >> ${ratelimittestfile} &
curl -A "GoogleBot" http://localhost:9000 2>&1 >> ${ratelimittestfile} & curl -A "GoogleBot" http://localhost:80 2>&1 >> ${ratelimittestfile} &
curl -A "GoogleBot" http://localhost:9000 2>&1 >> ${ratelimittestfile} & curl -A "GoogleBot" http://localhost:80 2>&1 >> ${ratelimittestfile} &
curl -A "GoogleBot" http://localhost:9000 2>&1 >> ${ratelimittestfile} & curl -A "GoogleBot" http://localhost:80 2>&1 >> ${ratelimittestfile} &
curl -A "GoogleBot" http://localhost:9000 2>&1 >> ${ratelimittestfile} & curl -A "GoogleBot" http://localhost:80 2>&1 >> ${ratelimittestfile} &
curl -A "GoogleBot" http://localhost:9000 2>&1 >> ${ratelimittestfile} & curl -A "GoogleBot" http://localhost:80 2>&1 >> ${ratelimittestfile} &
curl -A "GoogleBot" http://localhost:9000 2>&1 >> ${ratelimittestfile} & curl -A "GoogleBot" http://localhost:80 2>&1 >> ${ratelimittestfile} &
curl -A "GoogleBot" http://localhost:9000 2>&1 >> ${ratelimittestfile} & curl -A "GoogleBot" http://localhost:80 2>&1 >> ${ratelimittestfile} &
curl -A "GoogleBot" http://localhost:9000 2>&1 >> ${ratelimittestfile} & curl -A "GoogleBot" http://localhost:80 2>&1 >> ${ratelimittestfile} &
curl -A "GoogleBot" http://localhost:9000 2>&1 >> ${ratelimittestfile} & curl -A "GoogleBot" http://localhost:80 2>&1 >> ${ratelimittestfile} &
curl -A "GoogleBot" http://localhost:9000 2>&1 >> ${ratelimittestfile} curl -A "GoogleBot" http://localhost:80 2>&1 >> ${ratelimittestfile}
if grep -i 'Unavailable' < ${ratelimittestfile}; then if grep -i 'Unavailable' < ${ratelimittestfile}; then
echo "${bold}${green}PASSED - ${red}GoogleBot was ${bold}${red}RATE LIMITED" echo "${bold}${green}PASSED - ${red}GoogleBot was ${bold}${red}RATE LIMITED"
@ -181,10 +183,10 @@ echo "${bold}${green}-----------------------------------------------------------
echo "${bold}${green}Make Backup all conf files and folders used during this test" echo "${bold}${green}Make Backup all conf files and folders used during this test"
echo "${bold}${green}------------------------------------------------------------" echo "${bold}${green}------------------------------------------------------------"
printf "\n" printf "\n"
sudo cp /etc/nginx/bots.d/* ${TRAVIS_BUILD_DIR}/.dev-tools/beta_conf_files_ratelimiting/bots.d/ sudo cp /etc/nginx/bots.d/* ./dev-tools/beta_conf_files_ratelimiting/bots.d/
sudo cp /etc/nginx/conf.d/* ${TRAVIS_BUILD_DIR}/.dev-tools/beta_conf_files_ratelimiting/conf.d/ sudo cp /etc/nginx/conf.d/* ./dev-tools/beta_conf_files_ratelimiting/conf.d/
sudo cp /etc/nginx/sites-available/default.vhost ${TRAVIS_BUILD_DIR}/.dev-tools/beta_conf_files_ratelimiting/default.vhost sudo cp /etc/nginx/sites-available/default.vhost ./dev-tools/beta_conf_files_ratelimiting/default.vhost
sudo cp /etc/nginx/nginx.conf ${TRAVIS_BUILD_DIR}/.dev-tools/beta_conf_files_ratelimiting/nginx.conf sudo cp /etc/nginx/nginx.conf ./dev-tools/beta_conf_files_ratelimiting/nginx.conf
} }
# ----------- # -----------

10
dev-tools/beta-test-blocker-whitelist-domains.sh
View file

@ -16,6 +16,8 @@
# # # #
############################################################################## ##############################################################################
export TERM=xterm
# ------------------------------------------------------------------------------ # ------------------------------------------------------------------------------
# MIT License # MIT License
# ------------------------------------------------------------------------------ # ------------------------------------------------------------------------------
@ -64,7 +66,7 @@ echo "${bold}${green}---------------"
echo "${bold}${green}Reloading Nginx" echo "${bold}${green}Reloading Nginx"
echo "${bold}${green}---------------" echo "${bold}${green}---------------"
printf "\n\n" printf "\n\n"
sudo nginx -t && sudo nginx -s reload sudo nginx -t && sudo systemctl reload nginx
} }
waitforReload () { waitforReload () {
@ -80,11 +82,11 @@ echo "${bold}${green}---------------------------------------"
echo "${bold}${green}Activating Users whitelist-domains.conf" echo "${bold}${green}Activating Users whitelist-domains.conf"
echo "${bold}${green}---------------------------------------" echo "${bold}${green}---------------------------------------"
printf "\n\n" printf "\n\n"
sudo cp ${TRAVIS_BUILD_DIR}/.dev-tools/test_units/whitelist-domains.conf /etc/nginx/bots.d/whitelist-domains.conf sudo cp ./dev-tools/test_units/whitelist-domains.conf /etc/nginx/bots.d/whitelist-domains.conf
} }
run_curltest1 () { run_curltest1 () {
if curl http://localhost:9000 -e http://www.myowndomain.com 2>&1 | grep -i 'Welcome'; then if curl http://localhost:80 -e http://www.myowndomain.com 2>&1 | grep -i 'Welcome'; then
echo "${bold}${green}PASSED - User whitelist-domains.conf working" echo "${bold}${green}PASSED - User whitelist-domains.conf working"
else else
echo "${bold}${red}FAILED - User whitelist-domains.conf NOT working" echo "${bold}${red}FAILED - User whitelist-domains.conf NOT working"
@ -93,7 +95,7 @@ fi
} }
run_curltest2 () { run_curltest2 () {
if curl http://localhost:9000 -e http://www.myotherdomain.com 2>&1 | grep -i 'Welcome'; then if curl http://localhost:80 -e http://www.myotherdomain.com 2>&1 | grep -i 'Welcome'; then
echo "${bold}${green}PASSED - User whitelist-domains.conf working" echo "${bold}${green}PASSED - User whitelist-domains.conf working"
else else
echo "${bold}${red}FAILED - User whitelist-domains.conf NOT working" echo "${bold}${red}FAILED - User whitelist-domains.conf NOT working"

42
dev-tools/beta-test-blocker-whitelist-ips.sh
View file

@ -16,6 +16,8 @@
# # # #
############################################################################## ##############################################################################
export TERM=xterm
# ------------------------------------------------------------------------------ # ------------------------------------------------------------------------------
# MIT License # MIT License
# ------------------------------------------------------------------------------ # ------------------------------------------------------------------------------
@ -54,7 +56,7 @@ magenta=$(tput setaf 5)
cyan=$(tput setaf 6) cyan=$(tput setaf 6)
white=$(tput setaf 7) white=$(tput setaf 7)
defaultcolor=$(tput setaf default) defaultcolor=$(tput setaf default)
thisip=$(dig +short myip.opendns.com @resolver1.opendns.com) thisip=$(ip -o route get to 8.8.8.8 | sed -n 's/.*src \([0-9.]\+\).*/\1/p')
# --------- # ---------
# FUNCTIONS # FUNCTIONS
@ -65,7 +67,7 @@ echo "${bold}${green}---------------"
echo "${bold}${green}Reloading Nginx" echo "${bold}${green}Reloading Nginx"
echo "${bold}${green}---------------" echo "${bold}${green}---------------"
printf "\n\n" printf "\n\n"
sudo nginx -t && sudo nginx -s reload sudo nginx -t && sudo systemctl reload nginx
} }
waitforReload () { waitforReload () {
@ -77,7 +79,7 @@ sleep 10s
} }
run_curltest1 () { run_curltest1 () {
if curl http://localhost:9000 2>&1 | grep -i '(52)'; then if curl http://${thisip}:80 2>&1 | grep -i '(52)'; then
echo "${bold}${green}PASSED - ${bold}${red}blacklist own ip is WORKING" echo "${bold}${green}PASSED - ${bold}${red}blacklist own ip is WORKING"
else else
echo "${bold}${red}FAILED - blacklist own ip is NOT working" echo "${bold}${red}FAILED - blacklist own ip is NOT working"
@ -86,11 +88,11 @@ fi
} }
run_curltest2 () { run_curltest2 () {
if curl http://localhost:9000 2>&1 | grep -i 'Welcome'; then if curl http://${thisip}:80 2>&1 | grep -i 'Welcome'; then
echo "${bold}${green}PASSED - whitelist own ip is WORKING" echo "${bold}${green}PASSED - whitelist own ip is WORKING"
else else
echo "${bold}${red}FAILED - whitelist own ip is NOT working" echo "${bold}${red}FAILED - whitelist own ip is NOT working"
curl http://localhost:9000 curl http://${thisip}:80
exit 1 exit 1
fi fi
} }
@ -101,28 +103,28 @@ echo "${bold}${green}-----------------------------------------------------------
echo "${bold}${green}Make Backup all conf files and folders used during this test" echo "${bold}${green}Make Backup all conf files and folders used during this test"
echo "${bold}${green}------------------------------------------------------------" echo "${bold}${green}------------------------------------------------------------"
printf "\n" printf "\n"
sudo cp /etc/nginx/bots.d/* ${TRAVIS_BUILD_DIR}/.dev-tools/beta_conf_files_ip_whitelist/bots.d/ sudo cp /etc/nginx/bots.d/* ./dev-tools/beta_conf_files_ip_whitelist/bots.d/
sudo cp /etc/nginx/conf.d/* ${TRAVIS_BUILD_DIR}/.dev-tools/beta_conf_files_ip_whitelist/conf.d/ sudo cp /etc/nginx/conf.d/* ./dev-tools/beta_conf_files_ip_whitelist/conf.d/
sudo cp /etc/nginx/sites-available/default.vhost ${TRAVIS_BUILD_DIR}/.dev-tools/beta_conf_files_ip_whitelist/default.vhost sudo cp /etc/nginx/sites-available/default.vhost ./dev-tools/beta_conf_files_ip_whitelist/default.vhost
sudo cp /etc/nginx/nginx.conf ${TRAVIS_BUILD_DIR}/.dev-tools/beta_conf_files_ip_whitelist/nginx.conf sudo cp /etc/nginx/nginx.conf ./dev-tools/beta_conf_files_ip_whitelist/nginx.conf
} }
blacklistOwnIP () { blacklistOwnIP () {
sudo truncate -s 0 ${TRAVIS_BUILD_DIR}/.dev-tools/test_units/blacklist-ips.conf sudo truncate -s 0 ./dev-tools/test_units/blacklist-ips.conf
printf '%s\t%s\n' "${thisip}" "1;" > ${TRAVIS_BUILD_DIR}/.dev-tools/test_units/blacklist-ips.conf printf '%s\t%s\n' "${thisip}" "1;" > ./dev-tools/test_units/blacklist-ips.conf
printf '%s\t%s\n' "127.0.0.1" "1;" >> ${TRAVIS_BUILD_DIR}/.dev-tools/test_units/blacklist-ips.conf printf '%s\t%s\n' "127.0.0.1" "1;" >> ./dev-tools/test_units/blacklist-ips.conf
sudo cp ${TRAVIS_BUILD_DIR}/.dev-tools/test_units/blacklist-ips.conf /etc/nginx/bots.d/blacklist-ips.conf sudo cp ./dev-tools/test_units/blacklist-ips.conf /etc/nginx/bots.d/blacklist-ips.conf
sudo truncate -s 0 ${TRAVIS_BUILD_DIR}/.dev-tools/test_units/whitelist-ips.conf sudo truncate -s 0 ./dev-tools/test_units/whitelist-ips.conf
sudo cp ${TRAVIS_BUILD_DIR}/.dev-tools/test_units/whitelist-ips.conf /etc/nginx/bots.d/whitelist-ips.conf sudo cp ./dev-tools/test_units/whitelist-ips.conf /etc/nginx/bots.d/whitelist-ips.conf
} }
whitelistOwnIP () { whitelistOwnIP () {
sudo truncate -s 0 ${TRAVIS_BUILD_DIR}/.dev-tools/test_units/whitelist-ips.conf sudo truncate -s 0 ./dev-tools/test_units/whitelist-ips.conf
printf '%s\t%s\n' "${thisip}" "0;" > ${TRAVIS_BUILD_DIR}/.dev-tools/test_units/whitelist-ips.conf printf '%s\t%s\n' "${thisip}" "0;" > ./dev-tools/test_units/whitelist-ips.conf
printf '%s\t%s\n' "127.0.0.1" "0;" >> ${TRAVIS_BUILD_DIR}/.dev-tools/test_units/whitelist-ips.conf printf '%s\t%s\n' "127.0.0.1" "0;" >> ./dev-tools/test_units/whitelist-ips.conf
sudo cp ${TRAVIS_BUILD_DIR}/.dev-tools/test_units/whitelist-ips.conf /etc/nginx/bots.d/whitelist-ips.conf sudo cp ./dev-tools/test_units/whitelist-ips.conf /etc/nginx/bots.d/whitelist-ips.conf
# TEST ANY CHANGES TO botblocker-nginx-settings.conf # TEST ANY CHANGES TO botblocker-nginx-settings.conf
sudo cp ${TRAVIS_BUILD_DIR}/.dev-tools/test_units/botblocker-nginx-settings.conf /etc/nginx/conf.d/botblocker-nginx-settings.conf sudo cp ./dev-tools/test_units/botblocker-nginx-settings.conf /etc/nginx/conf.d/botblocker-nginx-settings.conf
} }
# ----------- # -----------

64
dev-tools/beta-test-blocker-whitelist.sh
View file

@ -16,6 +16,8 @@
# # # #
############################################################################## ##############################################################################
export TERM=xterm
# ------------------------------------------------------------------------------ # ------------------------------------------------------------------------------
# MIT License # MIT License
# ------------------------------------------------------------------------------ # ------------------------------------------------------------------------------
@ -64,7 +66,7 @@ echo "${bold}${green}---------------"
echo "${bold}${green}Reloading Nginx" echo "${bold}${green}Reloading Nginx"
echo "${bold}${green}---------------" echo "${bold}${green}---------------"
printf "\n\n" printf "\n\n"
sudo nginx -t && sudo nginx -s reload sudo nginx -t && sudo systemctl reload nginx
} }
waitforReload () { waitforReload () {
@ -76,22 +78,22 @@ sleep 10s
} }
run_curltest1 () { run_curltest1 () {
if curl -v -A "Nutch" http://localhost:9000 2>&1 | grep -i 'Welcome'; then if curl -v -A "Nutch" http://localhost:80 2>&1 | grep -i 'Welcome'; then
echo "${bold}${green}PASSED - WHITELISTING OF BAD BOT Nutch ALLOWED" echo "${bold}${green}PASSED - WHITELISTING OF BAD BOT Nutch ALLOWED"
else else
echo "${bold}${red}FAILED - WHITELISTING of BAD BOT Nutch FAILED" echo "${bold}${red}FAILED - WHITELISTING of BAD BOT Nutch FAILED"
#exit 1 #exit 1
curl -v -A "Nutch" http://localhost:9000 2>&1 curl -v -A "Nutch" http://localhost:80 2>&1
fi fi
} }
run_curltest2 () { run_curltest2 () {
if curl http://localhost:9000 -e http://zx6.ru 2>&1 | grep -i 'Welcome'; then if curl http://localhost:80 -e http://zx6.ru 2>&1 | grep -i 'Welcome'; then
echo "${bold}${green}PASSED - WHITELISTING OF REFERRER zx6.ru ALLOWED" echo "${bold}${green}PASSED - WHITELISTING OF REFERRER zx6.ru ALLOWED"
else else
echo "${bold}${red}FAILED - WHITELISTING OF REFERRER zx6.ru FAILED" echo "${bold}${red}FAILED - WHITELISTING OF REFERRER zx6.ru FAILED"
#exit 1 #exit 1
curl http://localhost:9000 -e http://zx6.ru 2>&1 curl http://localhost:80 -e http://zx6.ru 2>&1
fi fi
} }
@ -100,10 +102,10 @@ echo "${bold}${magenta}---------------------------------------------------------
echo "${bold}${magenta}Generating blacklist-user-agents.conf & custom-bad-referrers.conf" echo "${bold}${magenta}Generating blacklist-user-agents.conf & custom-bad-referrers.conf"
echo "${bold}${magenta}-----------------------------------------------------------------" echo "${bold}${magenta}-----------------------------------------------------------------"
printf "\n\n" printf "\n\n"
_input1=${TRAVIS_BUILD_DIR}/_generator_lists/bad-user-agents.list _input1=./_generator_lists/bad-user-agents.list
_input2=${TRAVIS_BUILD_DIR}/.dev-tools/referrers-regex-format-whitelist-test.txt _input2=./dev-tools/referrers-regex-format-whitelist-test.txt
_inputdb1=${TRAVIS_BUILD_DIR}/.dev-tools/good-bad-user-agents.db _inputdb1=./dev-tools/good-bad-user-agents.db
_inputdb2=${TRAVIS_BUILD_DIR}/.dev-tools/good-bad-referrers.db _inputdb2=./dev-tools/good-bad-referrers.db
_tmpnginx1=_tmpnginx1 _tmpnginx1=_tmpnginx1
_tmpnginx2=_tmpnginx2 _tmpnginx2=_tmpnginx2
_start1="# START MAKE BAD BOTS GOOD ### DO NOT EDIT THIS LINE AT ALL ###" _start1="# START MAKE BAD BOTS GOOD ### DO NOT EDIT THIS LINE AT ALL ###"
@ -132,11 +134,11 @@ ed -s ${_inputdb1}<<\IN
1,/# START MAKE BAD BOTS GOOD ### DO NOT EDIT THIS LINE AT ALL ###/d 1,/# START MAKE BAD BOTS GOOD ### DO NOT EDIT THIS LINE AT ALL ###/d
/# END MAKE BAD BOTS GOOD ### DO NOT EDIT THIS LINE AT ALL ###/,$d /# END MAKE BAD BOTS GOOD ### DO NOT EDIT THIS LINE AT ALL ###/,$d
,d ,d
.r /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/.dev-tools/test_units/blacklist-user-agents.conf .r ./dev-tools/test_units/blacklist-user-agents.conf
/# START MAKE BAD BOTS GOOD ### DO NOT EDIT THIS LINE AT ALL ###/x /# START MAKE BAD BOTS GOOD ### DO NOT EDIT THIS LINE AT ALL ###/x
.t. .t.
.,/# END MAKE BAD BOTS GOOD ### DO NOT EDIT THIS LINE AT ALL ###/-d .,/# END MAKE BAD BOTS GOOD ### DO NOT EDIT THIS LINE AT ALL ###/-d
w /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/.dev-tools/test_units/blacklist-user-agents.conf w ./dev-tools/test_units/blacklist-user-agents.conf
q q
IN IN
rm ${_inputdb1} rm ${_inputdb1}
@ -162,11 +164,11 @@ ed -s ${_inputdb2}<<\IN
1,/# START MAKE BAD REFERRERS GOOD ### DO NOT EDIT THIS LINE AT ALL ###/d 1,/# START MAKE BAD REFERRERS GOOD ### DO NOT EDIT THIS LINE AT ALL ###/d
/# END MAKE BAD REFERRERS GOOD ### DO NOT EDIT THIS LINE AT ALL ###/,$d /# END MAKE BAD REFERRERS GOOD ### DO NOT EDIT THIS LINE AT ALL ###/,$d
,d ,d
.r /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/.dev-tools/test_units/custom-bad-referrers.conf .r ./dev-tools/test_units/custom-bad-referrers.conf
/# START MAKE BAD REFERRERS GOOD ### DO NOT EDIT THIS LINE AT ALL ###/x /# START MAKE BAD REFERRERS GOOD ### DO NOT EDIT THIS LINE AT ALL ###/x
.t. .t.
.,/# END MAKE BAD REFERRERS GOOD ### DO NOT EDIT THIS LINE AT ALL ###/-d .,/# END MAKE BAD REFERRERS GOOD ### DO NOT EDIT THIS LINE AT ALL ###/-d
w /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/.dev-tools/test_units/custom-bad-referrers.conf w ./dev-tools/test_units/custom-bad-referrers.conf
q q
IN IN
rm ${_inputdb2} rm ${_inputdb2}
@ -177,30 +179,30 @@ echo "${bold}${green}------------------------------------------------"
echo "${bold}${green}Activating Users User-Agents Whitelist/Blacklist" echo "${bold}${green}Activating Users User-Agents Whitelist/Blacklist"
echo "${bold}${green}------------------------------------------------" echo "${bold}${green}------------------------------------------------"
printf "\n\n" printf "\n\n"
sudo cp ${TRAVIS_BUILD_DIR}/.dev-tools/test_units/blacklist-user-agents.conf /etc/nginx/bots.d/blacklist-user-agents.conf sudo cp ./dev-tools/test_units/blacklist-user-agents.conf /etc/nginx/bots.d/blacklist-user-agents.conf
echo "${bold}${green}----------------------------------------------" echo "${bold}${green}----------------------------------------------"
echo "${bold}${green}Activating Users Referrers Whitelist/Blacklist" echo "${bold}${green}Activating Users Referrers Whitelist/Blacklist"
echo "${bold}${green}----------------------------------------------" echo "${bold}${green}----------------------------------------------"
printf "\n\n" printf "\n\n"
sudo cp ${TRAVIS_BUILD_DIR}/.dev-tools/test_units/custom-bad-referrers.conf /etc/nginx/bots.d/custom-bad-referrers.conf sudo cp ./dev-tools/test_units/custom-bad-referrers.conf /etc/nginx/bots.d/custom-bad-referrers.conf
} }
testBadUserAgents () { testBadUserAgents () {
shuf -n 250 ${TRAVIS_BUILD_DIR}/_generator_lists/bad-user-agents.list > ${TRAVIS_BUILD_DIR}/.dev-tools/test_units/random-bots-for-whitelist-test.tmp shuf -n 250 ./_generator_lists/bad-user-agents.list > ./dev-tools/test_units/random-bots-for-whitelist-test.tmp
sed 's/\\//g' ${TRAVIS_BUILD_DIR}/.dev-tools/test_units/random-bots-for-whitelist-test.tmp > ${TRAVIS_BUILD_DIR}/.dev-tools/test_units/random-bots-for-whitelist-test.list sed 's/\\//g' ./dev-tools/test_units/random-bots-for-whitelist-test.tmp > ./dev-tools/test_units/random-bots-for-whitelist-test.list
sudo rm ${TRAVIS_BUILD_DIR}/.dev-tools/test_units/random-bots-for-whitelist-test.tmp sudo rm ./dev-tools/test_units/random-bots-for-whitelist-test.tmp
sort -u ${TRAVIS_BUILD_DIR}/.dev-tools/test_units/random-bots-for-whitelist-test.list -o ${TRAVIS_BUILD_DIR}/.dev-tools/test_units/random-bots-for-whitelist-test.list sort -u ./dev-tools/test_units/random-bots-for-whitelist-test.list -o ./dev-tools/test_units/random-bots-for-whitelist-test.list
printf "\n\n" printf "\n\n"
echo "${bold}${magenta}---------------------------" echo "${bold}${magenta}---------------------------"
echo "${bold}${magenta}Testing 250 Random Bad Bots" echo "${bold}${magenta}Testing 250 Random Bad Bots"
echo "${bold}${magenta}---------------------------" echo "${bold}${magenta}---------------------------"
printf "\n\n" printf "\n\n"
IFS=$'\n' IFS=$'\n'
file=${TRAVIS_BUILD_DIR}/.dev-tools/test_units/random-bots-for-whitelist-test.list file=./dev-tools/test_units/random-bots-for-whitelist-test.list
lines=$(cat ${file}) lines=$(cat ${file})
for line in ${lines}; do for line in ${lines}; do
if if
curl -v -A "${line}" http://localhost:9000 2>&1 | grep -i 'Welcome'; then curl -v -A "${line}" http://localhost:80 2>&1 | grep -i 'Welcome'; then
echo "${bold}${green}PASSED - ${red}${line} was ${bold}${green}ALLOWED" echo "${bold}${green}PASSED - ${red}${line} was ${bold}${green}ALLOWED"
else else
echo "${bold}${red}FAILED - ${red}${line} was ${bold}${red}NOT ALLOWED" echo "${bold}${red}FAILED - ${red}${line} was ${bold}${red}NOT ALLOWED"
@ -211,21 +213,21 @@ IFS=""
} }
testBadReferrers () { testBadReferrers () {
shuf -n 250 ${TRAVIS_BUILD_DIR}/_generator_lists/bad-referrers.list > ${TRAVIS_BUILD_DIR}/.dev-tools/test_units/random-referrers-for-whitelist-test.tmp shuf -n 250 ./_generator_lists/bad-referrers.list > ./dev-tools/test_units/random-referrers-for-whitelist-test.tmp
sed 's/\\//g' ${TRAVIS_BUILD_DIR}/.dev-tools/test_units/random-referrers-for-whitelist-test.tmp > ${TRAVIS_BUILD_DIR}/.dev-tools/test_units/random-referrers-for-whitelist-test.list sed 's/\\//g' ./dev-tools/test_units/random-referrers-for-whitelist-test.tmp > ./dev-tools/test_units/random-referrers-for-whitelist-test.list
sudo rm ${TRAVIS_BUILD_DIR}/.dev-tools/test_units/random-referrers-for-whitelist-test.tmp sudo rm ./dev-tools/test_units/random-referrers-for-whitelist-test.tmp
sort -u ${TRAVIS_BUILD_DIR}/.dev-tools/test_units/random-referrers-for-whitelist-test.list -o ${TRAVIS_BUILD_DIR}/.dev-tools/test_units/random-referrers-for-whitelist-test.list sort -u ./dev-tools/test_units/random-referrers-for-whitelist-test.list -o ./dev-tools/test_units/random-referrers-for-whitelist-test.list
printf "\n\n" printf "\n\n"
echo "${bold}${magenta}----------------------------" echo "${bold}${magenta}----------------------------"
echo "${bold}${magenta}Testing 250 Random Referrers" echo "${bold}${magenta}Testing 250 Random Referrers"
echo "${bold}${magenta}----------------------------" echo "${bold}${magenta}----------------------------"
printf "\n\n" printf "\n\n"
IFS=$'\n' IFS=$'\n'
file=${TRAVIS_BUILD_DIR}/.dev-tools/test_units/random-referrers-for-whitelist-test.list file=./dev-tools/test_units/random-referrers-for-whitelist-test.list
lines=$(cat ${file}) lines=$(cat ${file})
for line in ${lines}; do for line in ${lines}; do
if if
curl http://localhost:9000 -e "http://${line}" 2>&1 | grep -i 'Welcome'; then curl http://localhost:80 -e "http://${line}" 2>&1 | grep -i 'Welcome'; then
echo "${bold}${green}PASSED - ${red}${line} was ${bold}${green}ALLOWED" echo "${bold}${green}PASSED - ${red}${line} was ${bold}${green}ALLOWED"
else else
echo "${bold}${red}FAILED - ${red}${line} was ${bold}${red}NOT ALLOWED" echo "${bold}${red}FAILED - ${red}${line} was ${bold}${red}NOT ALLOWED"
@ -241,10 +243,10 @@ echo "${bold}${green}-----------------------------------------------------------
echo "${bold}${green}Make Backup all conf files and folders used during this test" echo "${bold}${green}Make Backup all conf files and folders used during this test"
echo "${bold}${green}------------------------------------------------------------" echo "${bold}${green}------------------------------------------------------------"
printf "\n" printf "\n"
sudo cp /etc/nginx/bots.d/* ${TRAVIS_BUILD_DIR}/.dev-tools/beta_conf_files_whitelist/bots.d/ sudo cp /etc/nginx/bots.d/* ./dev-tools/beta_conf_files_whitelist/bots.d/
sudo cp /etc/nginx/conf.d/* ${TRAVIS_BUILD_DIR}/.dev-tools/beta_conf_files_whitelist/conf.d/ sudo cp /etc/nginx/conf.d/* ./dev-tools/beta_conf_files_whitelist/conf.d/
sudo cp /etc/nginx/sites-available/default.vhost ${TRAVIS_BUILD_DIR}/.dev-tools/beta_conf_files_whitelist/default.vhost sudo cp /etc/nginx/sites-available/default.vhost ./dev-tools/beta_conf_files_whitelist/default.vhost
sudo cp /etc/nginx/nginx.conf ${TRAVIS_BUILD_DIR}/.dev-tools/beta_conf_files_whitelist/nginx.conf sudo cp /etc/nginx/nginx.conf ./dev-tools/beta_conf_files_whitelist/nginx.conf
} }
# ----------- # -----------