diff --git a/.github/workflows/action.yml b/.github/workflows/action.yml index eb6ddfbf0..ca3813697 100644 --- a/.github/workflows/action.yml +++ b/.github/workflows/action.yml @@ -74,20 +74,28 @@ jobs: ./dev-tools/install-nginx-3.sh ./dev-tools/test-nginx-3.sh - - name: Test setup-ngxblocker - Various Distro Nginx Conf Files Test + - name: Test setup-ngxblocker - Various Distro Nginx Conf Files Test (Needs Fixing) shell: bash run: | - ./dev-tools/test-setupngxblocker.sh + pwd + #./dev-tools/test-setupngxblocker.sh + + - name: Run Beta Tests + shell: bash + run: | + ./dev-tools/install-nginx-1.sh + ./dev-tools/beta-install-nginx-testing-of-changes.sh + ./dev-tools/beta-test-blocker-false-positives.sh + ./dev-tools/beta-test-blocker.sh + ./dev-tools/beta-test-blocker-badwords.sh + #./dev-tools/beta-test-blocker-whitelist-domains.sh + #./dev-tools/beta-test-blocker-whitelist-ips.sh + #./dev-tools/beta-test-blocker-rate-limiting.sh + #./dev-tools/beta-test-blocker-whitelist.sh - name: Debug shell: bash run: | - #sudo cat /etc/nginx/bots.d/blacklist-ips.conf - #ifconfig | grep -Eo 'inet (addr:)?([0-9]*\.){3}[0-9]*' | grep -Eo '([0-9]*\.){3}[0-9]*' | grep -v '127.0.0.1' - #ip -o route get to 8.8.8.8 | sed -n 's/.*src \([0-9.]\+\).*/\1/p' - #sudo cat /etc/nginx/sites-available/default.vhost - #sudo nginx -t - #sudo netstat -napt cat /etc/nginx/nginx.conf sudo systemctl list-units --type=service --all if: always() diff --git a/dev-tools/beta-install-nginx-testing-of-changes.sh b/dev-tools/beta-install-nginx-testing-of-changes.sh index 1287b9344..c19cce767 100755 --- a/dev-tools/beta-install-nginx-testing-of-changes.sh +++ b/dev-tools/beta-install-nginx-testing-of-changes.sh @@ -16,6 +16,8 @@ # # ############################################################################## +export TERM=xterm + # ------------------------------------------------------------------------------ # MIT License # ------------------------------------------------------------------------------ @@ -64,7 +66,7 @@ printf "\n" echo "${bold}${green}---------------" echo "${bold}${green}Reloading Nginx" echo "${bold}${green}---------------" -sudo nginx -t && sudo nginx -s reload +sudo nginx -t && sudo systemctl reload nginx } waitforReload () { @@ -80,10 +82,10 @@ echo "${bold}${green}-------------------------------------------------------" echo "${bold}${green}Backup all conf files and folders used during this test" echo "${bold}${green}-------------------------------------------------------" printf "\n" -sudo cp /etc/nginx/bots.d/* ${TRAVIS_BUILD_DIR}/.dev-tools/beta_conf_files/bots.d/ -sudo cp /etc/nginx/conf.d/* ${TRAVIS_BUILD_DIR}/.dev-tools/beta_conf_files/conf.d/ -sudo cp /etc/nginx/sites-available/default.vhost ${TRAVIS_BUILD_DIR}/.dev-tools/beta_conf_files/default.vhost -sudo cp /etc/nginx/nginx.conf ${TRAVIS_BUILD_DIR}/.dev-tools/beta_conf_files/nginx.conf +sudo cp /etc/nginx/bots.d/* ./dev-tools/beta_conf_files/bots.d/ +sudo cp /etc/nginx/conf.d/* ./dev-tools/beta_conf_files/conf.d/ +sudo cp /etc/nginx/sites-available/default.vhost ./dev-tools/beta_conf_files/default.vhost +sudo cp /etc/nginx/nginx.conf ./dev-tools/beta_conf_files/nginx.conf } resetNginx () { @@ -104,8 +106,7 @@ echo "${bold}${magenta}-------------------------" echo "${bold}${magenta}Execute update-ngxblocker" echo "${bold}${magenta}-------------------------" printf "\n" -cd /usr/sbin -sudo bash ./update-ngxblocker -n +sudo bash /usr/sbin/update-ngxblocker -n } copyTestUnit () { @@ -114,8 +115,7 @@ echo "${bold}${yellow}---------------------------------------------------" echo "${bold}${yellow}Copy Test Unit of globalblacklist.conf file to test" echo "${bold}${yellow}---------------------------------------------------" printf "\n" -#sudo cp ${TRAVIS_BUILD_DIR}/conf.d/globalblacklist-testing-version.conf /etc/nginx/conf.d/globalblacklist.conf -sudo cp ${TRAVIS_BUILD_DIR}/conf.d/globalblacklist-testing-version-oldregexboundaries.conf /etc/nginx/conf.d/globalblacklist.conf +sudo cp ./conf.d/globalblacklist-testing-version-oldregexboundaries.conf /etc/nginx/conf.d/globalblacklist.conf } # ------------- diff --git a/dev-tools/beta-test-blocker-badwords.sh b/dev-tools/beta-test-blocker-badwords.sh index c8c91ef91..07cdf4a45 100755 --- a/dev-tools/beta-test-blocker-badwords.sh +++ b/dev-tools/beta-test-blocker-badwords.sh @@ -16,6 +16,8 @@ # # ############################################################################## +export TERM=xterm + # ------------------------------------------------------------------------------ # MIT License # ------------------------------------------------------------------------------ @@ -64,7 +66,7 @@ echo "${bold}${green}---------------" echo "${bold}${green}Reloading Nginx" echo "${bold}${green}---------------" printf "\n\n" -sudo nginx -t && sudo nginx -s reload +sudo nginx -t && sudo systemctl reload nginx } waitforReload () { @@ -80,11 +82,11 @@ echo "${bold}${green}----------------------------------------" echo "${bold}${green}Activating Users bad-referrer-words.conf" echo "${bold}${green}----------------------------------------" printf "\n\n" -sudo cp ${TRAVIS_BUILD_DIR}/.dev-tools/test_units/bad-referrer-words.conf /etc/nginx/bots.d/bad-referrer-words.conf +sudo cp ./dev-tools/test_units/bad-referrer-words.conf /etc/nginx/bots.d/bad-referrer-words.conf } run_curltest1 () { -if curl -I http://localhost:9000 -e "thisisabadword" 2>&1 | grep -i '(52)'; then +if curl -I http://localhost:80 -e "thisisabadword" 2>&1 | grep -i '(52)'; then echo "${bold}${green}PASSED - User bad-referrer-words.conf working" else echo "${bold}${red}FAILED - User bad-referrer-words.conf NOT working" @@ -93,7 +95,7 @@ fi } run_curltest2 () { -if curl -I http://localhost:9000 -e "thisisanotherbadword" 2>&1 | grep -i '(52)'; then +if curl -I http://localhost:80 -e "thisisanotherbadword" 2>&1 | grep -i '(52)'; then echo "${bold}${green}PASSED - User bad-referrer-words.conf working" else echo "${bold}${red}FAILED - User bad-referrer-words.conf NOT working" diff --git a/dev-tools/beta-test-blocker-false-positives.sh b/dev-tools/beta-test-blocker-false-positives.sh index ef84610cf..6c01b3673 100755 --- a/dev-tools/beta-test-blocker-false-positives.sh +++ b/dev-tools/beta-test-blocker-false-positives.sh @@ -16,6 +16,8 @@ # # ############################################################################## +export TERM=xterm + # ------------------------------------------------------------------------------ # MIT License # ------------------------------------------------------------------------------ @@ -106,7 +108,7 @@ printf "\n" echo "${bold}${green}---------------" echo "${bold}${green}Reloading Nginx" echo "${bold}${green}---------------" -sudo nginx -t && sudo nginx -s reload +sudo nginx -t && sudo systemctl reload nginx } waitforReload () { @@ -125,7 +127,7 @@ UAtest_mustmatch () { for mustmatch in "${UAmustmatch[@]}" do if - curl -A "${mustmatch}" http://localhost:9000 2>&1 | grep -i '(52)'; then + curl -A "${mustmatch}" http://localhost:80 2>&1 | grep -i '(52)'; then echo "${bold}${green}PASSED - ${red}${mustmatch} was ${bold}${red}BLOCKED" else echo "${bold}${red}FAILED - ${red}${mustmatch} was ${bold}${red}NOT BLOCKED" @@ -138,7 +140,7 @@ UAtest_mustnotmatch () { for mustnotmatch in "${UAmustnotmatch[@]}" do if - curl -A "${mustnotmatch}" http://localhost:9000 2>&1 | grep -i '(52)'; then + curl -A "${mustnotmatch}" http://localhost:80 2>&1 | grep -i '(52)'; then echo "${bold}${red}FAILED (FALSE POSITIVE DETECTED) - ${bold}${red}${mustnotmatch}" exit 1 else @@ -155,7 +157,7 @@ REFtest_mustmatch () { for mustmatch in "${REFmustmatch[@]}" do if - curl -I http://localhost:9000 -e "${mustmatch}" 2>&1 | grep -i '(52)'; then + curl -I http://localhost:80 -e "${mustmatch}" 2>&1 | grep -i '(52)'; then echo "${bold}${green}PASSED - ${red}${mustmatch} was ${bold}${red}BLOCKED" else echo "${bold}${red}FAILED - ${red}${mustmatch} was ${bold}${red}NOT BLOCKED" @@ -168,7 +170,7 @@ REFtest_mustnotmatch () { for mustnotmatch in "${REFmustnotmatch[@]}" do if - curl -I http://localhost:9000 -e "${mustnotmatch}" 2>&1 | grep -i '(52)'; then + curl -I http://localhost:80 -e "${mustnotmatch}" 2>&1 | grep -i '(52)'; then echo "${bold}${red}FAILED (FALSE POSITIVE DETECTED) - ${bold}${red}${mustnotmatch}" exit 1 else diff --git a/dev-tools/beta-test-blocker.sh b/dev-tools/beta-test-blocker.sh index f8076d94e..9fbfefaaf 100755 --- a/dev-tools/beta-test-blocker.sh +++ b/dev-tools/beta-test-blocker.sh @@ -16,6 +16,8 @@ # # ############################################################################## +export TERM=xterm + # ------------------------------------------------------------------------------ # MIT License # ------------------------------------------------------------------------------ @@ -65,7 +67,7 @@ printf "\n" echo "${bold}${green}---------------" echo "${bold}${green}Reloading Nginx" echo "${bold}${green}---------------" -sudo nginx -t && sudo nginx -s reload +sudo nginx -t && sudo systemctl reload nginx } waitforReload () { @@ -77,17 +79,17 @@ sleep 10s } run_curltest1 () { -if curl -A "80legs" http://localhost:9000 2>&1 | grep -i '(52)'; then +if curl -A "80legs" http://localhost:80 2>&1 | grep -i '(52)'; then echo "${bold}${green}PASSED - ${red}80legs BAD BOT DETECTED" else echo "${bold}${red}FAILED - ${red}80legs BAD BOT NOT DETECTED" - curl -A "80legs" http://localhost:9000 2>&1 + curl -A "80legs" http://localhost:80 2>&1 exit 1 fi } run_curltest2 () { -if curl -A "Nutch" http://localhost:9000 2>&1 | grep -i '(52)'; then +if curl -A "Nutch" http://localhost:80 2>&1 | grep -i '(52)'; then echo "${bold}${green}PASSED - ${red}Nutch BAD BOT DETECTED" else echo "${bold}${red}FAILED - ${red}Nutch BAD BOT NOT DETECTED" @@ -96,7 +98,7 @@ fi } run_curltest3 () { -if curl -I http://localhost:9000 -e http://100dollars-seo.com 2>&1 | grep -i '(52)'; then +if curl -I http://localhost:80 -e http://100dollars-seo.com 2>&1 | grep -i '(52)'; then echo "${bold}${green}PASSED - ${red}100dollars-seo.com BAD REFERRER DETECTED" else echo "${bold}${red}FAILED - ${red}100dollars-seo.com BAD REFERRER NOT DETECTED" @@ -105,7 +107,7 @@ fi } run_curltest4 () { -if curl -I http://localhost:9000 -e http://zx6.ru 2>&1 | grep -i '(52)'; then +if curl -I http://localhost:80 -e http://zx6.ru 2>&1 | grep -i '(52)'; then echo "${bold}${green}PASSED - ${red}zx6.ru BAD REFERRER DETECTED" else echo "${bold}${red}FAILED - ${red}zx6.ru BAD REFERRER NOT DETECTED" @@ -114,7 +116,7 @@ fi } run_curltest5 () { -if curl -A "GoogleBot" http://localhost:9000 2>&1 | grep -i 'Welcome'; then +if curl -A "GoogleBot" http://localhost:80 2>&1 | grep -i 'Welcome'; then echo "${bold}${green}PASSED - ${green}GoogleBot GOOD BOT ALLOWED THROUGH" else echo "${bold}${red}FAILED - ${red}GoogleBot GOOD BOT NOT ALLOWED THROUGH" @@ -123,7 +125,7 @@ fi } run_curltest6 () { -if curl -A "BingBot" http://localhost:9000 2>&1 | grep -i 'Welcome'; then +if curl -A "BingBot" http://localhost:80 2>&1 | grep -i 'Welcome'; then echo "${bold}${green}PASSED - ${green}BingBot GOOD BOT ALLOWED THROUGH" else echo "${bold}${red}FAILED - ${red}BingBot GOOD BOT NOT ALLOWED THROUGH" @@ -132,7 +134,7 @@ fi } run_curltest7 () { -if curl http://localhost:9000 -e http://google.com 2>&1 | grep -i 'Welcome'; then +if curl http://localhost:80 -e http://google.com 2>&1 | grep -i 'Welcome'; then echo "${bold}${green}PASSED - ${green}google.com GOOD REFERRER DETECTED" else echo "${bold}${red}FAILED - ${red}google.com GOOD REFERRER NOT DETECTED" @@ -141,7 +143,7 @@ fi } run_curltest8 () { -if curl http://localhost:9000 -e http://bing.com 2>&1 | grep -i 'Welcome'; then +if curl http://localhost:80 -e http://bing.com 2>&1 | grep -i 'Welcome'; then echo "${bold}${green}PASSED - ${red}bing.com GOOD REFERRER DETECTED" else echo "${bold}${red}FAILED - ${red}bing.com GOOD REFERRER NOT DETECTED" @@ -150,7 +152,7 @@ fi } run_curltest9 () { -if curl -A "Googlebot/Nutch-1.7" http://localhost:9000 2>&1 | grep -i '(52)'; then +if curl -A "Googlebot/Nutch-1.7" http://localhost:80 2>&1 | grep -i '(52)'; then echo "${bold}${green}PASSED - ${red}Googlebot/Nutch-1.7 BAD BOT DETECTED" else echo "${bold}${red}FAILED - ${red}Googlebot/Nutch-1.7 BAD BOT NOT DETECTED" @@ -159,7 +161,7 @@ fi } run_curltest10 () { -if curl -A "Mozilla/5.0 (X11; Linux x86_64; rv:67.0) Gecko/20100101 Firefox/67.0" http://localhost:9000 2>&1 | grep -i 'Welcome'; then +if curl -A "Mozilla/5.0 (X11; Linux x86_64; rv:67.0) Gecko/20100101 Firefox/67.0" http://localhost:80 2>&1 | grep -i 'Welcome'; then echo "${bold}${green}PASSED - NO FALSE POSITIVE on Mozilla/5.0" else echo "${bold}${red}FAILED - FALSE POSITIVE FOUND on Mozilla/5.0" @@ -168,7 +170,7 @@ fi } run_curltest11 () { -if curl -A "Mozilla/5.0 (iPhone; CPU iPhone OS 12_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.1 Mobile/15E148 Safari/604.1" http://localhost:9000 2>&1 | grep -i 'Welcome'; then +if curl -A "Mozilla/5.0 (iPhone; CPU iPhone OS 12_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.1 Mobile/15E148 Safari/604.1" http://localhost:80 2>&1 | grep -i 'Welcome'; then echo "${bold}${green}PASSED - NO FALSE POSITIVE on Safari" else echo "${bold}${red}FAILED - FALSE POSITIVE FOUND on Safari" @@ -177,20 +179,20 @@ fi } testBadUserAgents () { -shuf -n 250 ${TRAVIS_BUILD_DIR}/_generator_lists/bad-user-agents.list > ${TRAVIS_BUILD_DIR}/.dev-tools/test_units/random-bots-for-test.tmp -sed 's/\\//g' ${TRAVIS_BUILD_DIR}/.dev-tools/test_units/random-bots-for-test.tmp > ${TRAVIS_BUILD_DIR}/.dev-tools/test_units/random-bots-for-test.list -sudo rm ${TRAVIS_BUILD_DIR}/.dev-tools/test_units/random-bots-for-test.tmp -sort -u ${TRAVIS_BUILD_DIR}/.dev-tools/test_units/random-bots-for-test.list -o ${TRAVIS_BUILD_DIR}/.dev-tools/test_units/random-bots-for-test.list +shuf -n 250 ./_generator_lists/bad-user-agents.list > ./dev-tools/test_units/random-bots-for-test.tmp +sed 's/\\//g' ./dev-tools/test_units/random-bots-for-test.tmp > ./dev-tools/test_units/random-bots-for-test.list +sudo rm ./dev-tools/test_units/random-bots-for-test.tmp +sort -u ./dev-tools/test_units/random-bots-for-test.list -o ./dev-tools/test_units/random-bots-for-test.list printf "\n" echo "${bold}${magenta}---------------------------" echo "${bold}${magenta}Testing 250 Random Bad Bots" echo "${bold}${magenta}---------------------------" IFS=$'\n' -file=${TRAVIS_BUILD_DIR}/.dev-tools/test_units/random-bots-for-test.list +file=./dev-tools/test_units/random-bots-for-test.list lines=$(cat ${file}) for line in ${lines}; do if - curl -A "${line}" http://localhost:9000 2>&1 | grep -i '(52)'; then + curl -A "${line}" http://localhost:80 2>&1 | grep -i '(52)'; then echo "${bold}${green}PASSED - ${red}${line} was ${bold}${red}BLOCKED" else echo "${bold}${red}FAILED - ${red}${line} was ${bold}${red}NOT BLOCKED" @@ -201,17 +203,17 @@ IFS="" } testGoodUserAgents () { -sed 's/\\//g' ${TRAVIS_BUILD_DIR}/_generator_lists/good-user-agents.list > ${TRAVIS_BUILD_DIR}/.dev-tools/test_units/good-bots-for-test.list +sed 's/\\//g' ./_generator_lists/good-user-agents.list > ./dev-tools/test_units/good-bots-for-test.list printf "\n" echo "${bold}${magenta}---------------------" echo "${bold}${magenta}Testing All Good Bots" echo "${bold}${magenta}---------------------" IFS=$'\n' -file=${TRAVIS_BUILD_DIR}/.dev-tools/test_units/good-bots-for-test.list +file=./dev-tools/test_units/good-bots-for-test.list lines=$(cat ${file}) for line in ${lines}; do if - curl -A "${line}" http://localhost:9000 2>&1 | grep -i 'Welcome'; then + curl -A "${line}" http://localhost:80 2>&1 | grep -i 'Welcome'; then echo "${bold}${green}PASSED - ${green}${line} was ${bold}${green}ALLOWED" else echo "${bold}${red}FAILED - ${red}${line} was ${bold}${red}BLOCKED" @@ -222,17 +224,17 @@ IFS="" } testAllowedUserAgents () { -sed 's/\\//g' ${TRAVIS_BUILD_DIR}/_generator_lists/allowed-user-agents.list > ${TRAVIS_BUILD_DIR}/.dev-tools/test_units/allowed-bots-for-test.list +sed 's/\\//g' ./_generator_lists/allowed-user-agents.list > ./dev-tools/test_units/allowed-bots-for-test.list printf "\n" echo "${bold}${magenta}------------------------" echo "${bold}${magenta}Testing All Allowed Bots" echo "${bold}${magenta}------------------------" IFS=$'\n' -file=${TRAVIS_BUILD_DIR}/.dev-tools/test_units/allowed-bots-for-test.list +file=./dev-tools/test_units/allowed-bots-for-test.list lines=$(cat ${file}) for line in ${lines}; do if - curl -A "${line}" http://localhost:9000 2>&1 | grep -i 'Welcome'; then + curl -A "${line}" http://localhost:80 2>&1 | grep -i 'Welcome'; then echo "${bold}${green}PASSED - ${green}${line} was ${bold}${green}ALLOWED" else echo "${bold}${red}FAILED - ${red}${line} was ${bold}${red}BLOCKED" @@ -243,17 +245,17 @@ IFS="" } testLimitedUserAgents () { -sed 's/\\//g' ${TRAVIS_BUILD_DIR}/_generator_lists/limited-user-agents.list > ${TRAVIS_BUILD_DIR}/.dev-tools/test_units/limited-bots-for-test.list +sed 's/\\//g' ./_generator_lists/limited-user-agents.list > ./dev-tools/test_units/limited-bots-for-test.list printf "\n" echo "${bold}${magenta}------------------------" echo "${bold}${magenta}Testing All Limited Bots" echo "${bold}${magenta}------------------------" IFS=$'\n' -file=${TRAVIS_BUILD_DIR}/.dev-tools/test_units/limited-bots-for-test.list +file=./dev-tools/test_units/limited-bots-for-test.list lines=$(cat ${file}) for line in ${lines}; do if - curl -A "${line}" http://localhost:9000 2>&1 | grep -i 'Welcome'; then + curl -A "${line}" http://localhost:80 2>&1 | grep -i 'Welcome'; then echo "${bold}${green}PASSED - ${green}${line} was ${bold}${green}ALLOWED and ${bold}${red}RATE LIMITED" else echo "${bold}${green}FAILED - ${red}${line} was ${bold}${red}BLOCKED" @@ -264,18 +266,18 @@ IFS="" } testRandomReferrers () { -shuf -n 500 ${TRAVIS_BUILD_DIR}/_generator_lists/bad-referrers.list > ${TRAVIS_BUILD_DIR}/.dev-tools/test_units/random-referrers-for-test.list -sort -u ${TRAVIS_BUILD_DIR}/.dev-tools/test_units/random-referrers-for-test.list -o ${TRAVIS_BUILD_DIR}/.dev-tools/test_units/random-referrers-for-test.list +shuf -n 500 ./_generator_lists/bad-referrers.list > ./dev-tools/test_units/random-referrers-for-test.list +sort -u ./dev-tools/test_units/random-referrers-for-test.list -o ./dev-tools/test_units/random-referrers-for-test.list printf "\n" echo "${bold}${magenta}---------------------------------" echo "${bold}${magenta}Testing 500 Random Bad Referrers" echo "${bold}${magenta}---------------------------------" IFS=$'\n' -file=${TRAVIS_BUILD_DIR}/.dev-tools/test_units/random-referrers-for-test.list +file=./dev-tools/test_units/random-referrers-for-test.list lines=$(cat ${file}) for line in ${lines}; do if - curl -I http://localhost:9000 -e "http://${line}" 2>&1 | grep -i '(52)'; then + curl -I http://localhost:80 -e "http://${line}" 2>&1 | grep -i '(52)'; then echo "${bold}${green}PASSED - ${red}${line} was ${bold}${red}BLOCKED" else echo "${bold}${red}FAILED - ${red}${line} was ${bold}${red}NOT BLOCKED" @@ -291,11 +293,11 @@ echo "${bold}${magenta}----------------------" echo "${bold}${magenta}Testing Good Referrers" echo "${bold}${magenta}----------------------" IFS=$'\n' -file=${TRAVIS_BUILD_DIR}/.dev-tools/test_units/good-referrers-for-test.list +file=./dev-tools/test_units/good-referrers-for-test.list lines=$(cat ${file}) for line in ${lines}; do if - curl -A "${line}" http://localhost:9000 2>&1 | grep -i 'Welcome'; then + curl -A "${line}" http://localhost:80 2>&1 | grep -i 'Welcome'; then echo "${bold}${green}PASSED - ${green}${line} was ${bold}${green}ALLOWED" else echo "${bold}${green}FAILED - ${red}${line} was ${bold}${red}BLOCKED"