diff --git a/.dev-tools/generate-files.sh b/.dev-tools/generate-files.sh index 05cfdd794..52fa3e030 100755 --- a/.dev-tools/generate-files.sh +++ b/.dev-tools/generate-files.sh @@ -57,7 +57,7 @@ sudo chmod +x ${TRAVIS_BUILD_DIR}/.dev-tools/run-curl-tests-1.sh sudo chmod +x ${TRAVIS_BUILD_DIR}/.dev-tools/run-curl-tests-2.sh sudo chmod +x ${TRAVIS_BUILD_DIR}/.dev-tools/run-curl-tests-3.sh sudo chmod +x ${TRAVIS_BUILD_DIR}/.dev-tools/run-curl-tests-4.sh -sudo chmod +x ${TRAVIS_BUILD_DIR}/.dev-tools/run-curl-testing-of-changes.sh +sudo chmod +x ${TRAVIS_BUILD_DIR}/.dev-tools/run-curl-tests-testing-of-changes.sh sudo chmod +x ${TRAVIS_BUILD_DIR}/.dev-tools/prepare-robots-input.sh sudo chmod +x ${TRAVIS_BUILD_DIR}/.dev-tools/install-nginx-1.sh sudo chmod +x ${TRAVIS_BUILD_DIR}/.dev-tools/install-nginx-2.sh diff --git a/.dev-tools/install-nginx-2.sh b/.dev-tools/install-nginx-2.sh index 02e45805d..e69903846 100755 --- a/.dev-tools/install-nginx-2.sh +++ b/.dev-tools/install-nginx-2.sh @@ -106,7 +106,7 @@ sudo chmod +x /usr/sbin/install-ngxblocker printf '\n%s\n%s\n%s\n\n' "######################" "Run install-ngxblocker" "######################" cd /usr/sbin -sudo bash ./install-ngxblocker -x -c /usr/local/nginx/conf.d -b /usr/local/nginx/bots.d +sudo bash -x ./install-ngxblocker -x -c /usr/local/nginx/conf.d -b /usr/local/nginx/bots.d # ************************************************** # Set our install and setup scripts to be executable @@ -150,7 +150,7 @@ sudo cp ${TRAVIS_BUILD_DIR}/.dev-tools/globalblacklist-dummy.conf /usr/local/ngi printf '\n%s\n%s\n%s\n\n' "######################" "Run update-ngxblocker" "######################" cd /usr/sbin -sudo bash ./update-ngxblocker -c /usr/local/nginx/conf.d -b /usr/local/nginx/bots.d -n +sudo bash -x ./update-ngxblocker -c /usr/local/nginx/conf.d -b /usr/local/nginx/bots.d -n # ********************* # Force reload of Nginx @@ -182,7 +182,7 @@ ls -la /usr/local/nginx/bots.d/ printf '\n%s\n%s\n%s\n\n' "######################" "Run update-ngxblocker" "######################" cd /usr/sbin -sudo bash ./update-ngxblocker -c /usr/local/nginx/conf.d -b /usr/local/nginx/bots.d -n +sudo bash -x ./update-ngxblocker -c /usr/local/nginx/conf.d -b /usr/local/nginx/bots.d -n # **************************************************** # List Directories to Confirm Downloaded Missing Files @@ -206,7 +206,7 @@ sudo cp ${TRAVIS_BUILD_DIR}/.dev-tools/globalblacklist-dummy.conf /usr/local/ngi printf '\n%s\n%s\n%s\n\n' "######################" "Run update-ngxblocker" "######################" cd /usr/sbin -sudo bash ./update-ngxblocker -c /usr/local/nginx/conf.d -b /usr/local/nginx/bots.d -n +sudo bash -x ./update-ngxblocker -c /usr/local/nginx/conf.d -b /usr/local/nginx/bots.d -n # ************************** # Run setup-ngxblocker Again @@ -214,7 +214,7 @@ sudo bash ./update-ngxblocker -c /usr/local/nginx/conf.d -b /usr/local/nginx/bot printf '\n%s\n%s\n%s\n\n' "####################" "Run setup-ngxblocker" "####################" cd /usr/sbin -sudo bash ./setup-ngxblocker -x -c /usr/local/nginx/conf.d -b /usr/local/nginx/bots.d +sudo bash -x ./setup-ngxblocker -x -c /usr/local/nginx/conf.d -b /usr/local/nginx/bots.d # ********************* # Force reload of Nginx @@ -236,7 +236,7 @@ sudo cp ${TRAVIS_BUILD_DIR}/conf.d/globalblacklist.conf /usr/local/nginx/conf.d/ printf '\n%s\n%s\n%s\n\n' "############################################################" "Run setup-ngxblocker to fix includes in globalblacklist.conf" "############################################################" cd /usr/sbin -sudo bash ./setup-ngxblocker -x -c /usr/local/nginx/conf.d -b /usr/local/nginx/bots.d +sudo bash -x ./setup-ngxblocker -x -c /usr/local/nginx/conf.d -b /usr/local/nginx/bots.d # ********************* # Force reload of Nginx diff --git a/.dev-tools/run-curl-tests-testing-of-changes.sh b/.dev-tools/run-curl-tests-testing-of-changes.sh index 7427abecd..198de6934 100644 --- a/.dev-tools/run-curl-tests-testing-of-changes.sh +++ b/.dev-tools/run-curl-tests-testing-of-changes.sh @@ -50,7 +50,7 @@ else #exit 1 fi } -run_curltest1 +#run_curltest1 # ************************************************** @@ -69,7 +69,7 @@ else #exit 1 fi } -run_curltest2 +#run_curltest2 # ****************************************************************** @@ -88,7 +88,7 @@ else #exit 1 fi } -run_curltest3 +#run_curltest3 # ****************************************************** @@ -107,7 +107,7 @@ else #exit 1 fi } -run_curltest4 +#run_curltest4 # ***************************************************** # Function Curl Test 5 - Check for Good Bot "GoogleBot" @@ -125,7 +125,7 @@ else #exit 1 fi } -run_curltest5 +#run_curltest5 # *************************************************** # Function Curl Test 6 - Check for Good Bot "BingBot" @@ -143,7 +143,7 @@ else #exit 1 fi } -run_curltest6 +#run_curltest6 # *********************************************************** # Function Curl Test 7 - Check for Good Referrer "google.com" @@ -161,7 +161,7 @@ else #exit 1 fi } -run_curltest7 +#run_curltest7 # *********************************************************** # Function Curl Test 8 - Check for Good Referrer "bing.com" @@ -180,7 +180,7 @@ else #exit 1 fi } -run_curltest8 +#run_curltest8 # ************************************************** # Function Curl Test 9 - Check for Bad Bot "Nutch" @@ -198,7 +198,7 @@ else #exit 1 fi } -run_curltest9 +#run_curltest9 # ************************************************** # Function Curl Test 10 - Check for Bad Bot "Nutch" @@ -216,12 +216,117 @@ else #exit 1 fi } -run_curltest10 +#run_curltest10 # ********************************* # Trigger our curl functions to run # ********************************* +runtest () { +truncate -s 0 ${_curltest1} +printf '\n%s\n%s\n%s\n\n' "#########################" "TESTING BAD BOT IS DENIED" "#########################" +printf '%s%s\n\n' "Last Tested: " "$_now" >> "${_curltest1}" +curl -A "80legs" http://localhost:9000/index.php 2>> ${_curltest1} +if grep -i '(52)' ${_curltest1}; then + echo 'BAD BOT DETECTED - TEST PASSED' +else + echo 'BAD BOT NOT DETECTED - TEST FAILED' + #exit 1 +fi +truncate -s 0 ${_curltest2} +printf '\n%s\n%s\n%s\n\n' "#########################" "TESTING BAD BOT IS DENIED" "#########################" +printf '%s%s\n\n' "Last Tested: " "$_now" >> "${_curltest2}" +curl -A "Nutch" http://localhost:9000/index.php 2>> ${_curltest2} +if grep -i '(52)' ${_curltest2}; then + echo 'BAD BOT DETECTED - TEST PASSED' +else + echo 'BAD BOT NOT DETECTED - TEST FAILED' + #exit 1 +fi +truncate -s 0 ${_curltest3} +printf '\n%s\n%s\n%s\n\n' "##############################" "TESTING BAD REFERRER IS DENIED" "##############################" +printf '%s%s\n\n' "Last Tested: " "$_now" >> "${_curltest3}" +curl -I http://localhost:9000/index.php -e http://100dollars-seo.com 2>> ${_curltest3} +if grep -i '(52)' ${_curltest3}; then + echo 'BAD REFERRER DETECTED - TEST PASSED' +else + echo 'BAD REFERRER NOT DETECTED - TEST FAILED' + #exit 1 +fi +truncate -s 0 ${_curltest4} +printf '\n%s\n%s\n%s\n\n' "##############################" "TESTING BAD REFERRER IS DENIED" "##############################" +printf '%s%s\n\n' "Last Tested: " "$_now" >> "${_curltest4}" +curl -I http://localhost:9000/index.php -e http://zx6.ru 2>> ${_curltest4} +if grep -i '(52)' ${_curltest4}; then + echo 'BAD REFERRER DETECTED - TEST PASSED' +else + echo 'BAD REFERRER NOT DETECTED - TEST FAILED' + #exit 1 +fi +truncate -s 0 ${_curltest5} +printf '\n%s\n%s\n%s\n\n' "###########################" "TESTING GOOD BOT IS ALLOWED" "###########################" +printf '%s%s\n\n' "Last Tested: " "$_now" >> "${_curltest5}" +curl -v -A "GoogleBot" http://localhost:9000/index.php 2>&1 >> ${_curltest5} +if grep -i 'Welcome' ${_curltest5}; then + echo 'GOOD BOT ALLOWED THROUGH - TEST PASSED' +else + echo 'GOOD BOT NOT ALLOWED THROUGH - TEST FAILED' + #exit 1 +fi +truncate -s 0 ${_curltest6} +printf '\n%s\n%s\n%s\n\n' "###########################" "TESTING GOOD BOT IS ALLOWED" "###########################" +printf '%s%s\n\n' "Last Tested: " "$_now" >> "${_curltest6}" +curl -v -A "BingBot" http://localhost:9000/index.php 2>&1 >> ${_curltest6} +if grep -i 'Welcome' ${_curltest6}; then + echo 'GOOD BOT ALLOWED THROUGH - TEST PASSED' +else + echo 'GOOD BOT NOT ALLOWED THROUGH - TEST FAILED' + #exit 1 +fi +truncate -s 0 ${_curltest7} +printf '\n%s\n%s\n%s\n\n' "################################" "TESTING GOOD REFERRER IS ALLOWED" "################################" +printf '%s%s\n\n' "Last Tested: " "$_now" >> "${_curltest7}" +curl http://localhost:9000/index.php -e http://google.com 2>&1 >> ${_curltest7} +if grep -i 'Welcome' ${_curltest7}; then + echo 'GOOD REFERRER DETECTED - TEST PASSED' +else + echo 'GOOD REFERRER NOT DETECTED - TEST FAILED' + #exit 1 +fi +truncate -s 0 ${_curltest8} +printf '\n%s\n%s\n%s\n\n' "################################" "TESTING GOOD REFERRER IS ALLOWED" "################################" +printf '%s%s\n\n' "Last Tested: " "$_now" >> "${_curltest8}" +curl http://localhost:9000/index.php -e http://bing.com 2>&1 >> ${_curltest8} +if grep -i 'Welcome' ${_curltest8}; then + echo 'GOOD REFERRER DETECTED - TEST PASSED' + exit 0 +else + echo 'GOOD REFERRER NOT DETECTED - TEST FAILED' + #exit 1 +fi +truncate -s 0 ${_curltest9} +printf '\n%s\n%s\n%s\n\n' "#########################" "TESTING BAD BOT IS DENIED" "#########################" +printf '%s%s\n\n' "Last Tested: " "$_now" >> "${_curltest9}" +curl -A "Googlebot/Nutch-1.7" http://localhost:9000/index.php 2>> ${_curltest9} +if grep -i '(52)' ${_curltest9}; then + echo 'BAD BOT DETECTED - TEST PASSED' +else + echo 'BAD BOT NOT DETECTED - TEST FAILED' + #exit 1 +fi +truncate -s 0 ${_curltest10} +printf '\n%s\n%s\n%s\n\n' "#########################" "TESTING BAD BOT IS DENIED" "#########################" +printf '%s%s\n\n' "Last Tested: " "$_now" >> "${_curltest10}" +curl -A "Mozilla/5.0 (compatible; Googlebot/Nutch2.1; +http://www.google.com/bot.html)" http://localhost:9000/index.php 2>> ${_curltest10} +if grep -i '(52)' ${_curltest10}; then + echo 'BAD BOT DETECTED - TEST PASSED' +else + echo 'BAD BOT NOT DETECTED - TEST FAILED' + #exit 1 +fi +} +runtest + echo "Tests Completed" diff --git a/.travis.yml b/.travis.yml index 28810bbcb..c083ec3d8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,5 +1,3 @@ -git: - depth: 1 addons: apt: packages: @@ -43,12 +41,12 @@ script: - bash .dev-tools/install-nginx-1.sh - bash .dev-tools/run-curl-tests-1.sh - bash .dev-tools/install-nginx-testing-of-changes.sh - - bash .dev-tools/run-curl-tests-testing-of-changes.sh - - bash .dev-tools/install-nginx-2.sh + - bash -x .dev-tools/run-curl-tests-testing-of-changes.sh + - bash -x .dev-tools/install-nginx-2.sh - bash .dev-tools/run-curl-tests-2.sh - - bash .dev-tools/install-nginx-3.sh + - bash -x .dev-tools/install-nginx-3.sh - bash .dev-tools/run-curl-tests-3.sh - - bash .dev-tools/install-nginx-4.sh + - bash -x .dev-tools/install-nginx-4.sh - bash .dev-tools/run-curl-tests-4.sh - bash .dev-tools/modify-files-and-commit.sh