Debug errors in Travis Builds with setup-ngxblocker and update-ngxblocker

.dev-tools/generate-files.sh

@@ -57,7 +57,7 @@ sudo chmod +x ${TRAVIS_BUILD_DIR}/.dev-tools/run-curl-tests-1.sh
 sudo chmod +x ${TRAVIS_BUILD_DIR}/.dev-tools/run-curl-tests-2.sh
 sudo chmod +x ${TRAVIS_BUILD_DIR}/.dev-tools/run-curl-tests-3.sh
 sudo chmod +x ${TRAVIS_BUILD_DIR}/.dev-tools/run-curl-tests-4.sh
-sudo chmod +x ${TRAVIS_BUILD_DIR}/.dev-tools/run-curl-testing-of-changes.sh
+sudo chmod +x ${TRAVIS_BUILD_DIR}/.dev-tools/run-curl-tests-testing-of-changes.sh
 sudo chmod +x ${TRAVIS_BUILD_DIR}/.dev-tools/prepare-robots-input.sh
 sudo chmod +x ${TRAVIS_BUILD_DIR}/.dev-tools/install-nginx-1.sh
 sudo chmod +x ${TRAVIS_BUILD_DIR}/.dev-tools/install-nginx-2.sh

.dev-tools/install-nginx-2.sh

@@ -106,7 +106,7 @@ sudo chmod +x /usr/sbin/install-ngxblocker
 
 printf '\n%s\n%s\n%s\n\n' "######################" "Run install-ngxblocker" "######################"
 cd /usr/sbin
-sudo bash ./install-ngxblocker -x -c /usr/local/nginx/conf.d -b /usr/local/nginx/bots.d
+sudo bash -x ./install-ngxblocker -x -c /usr/local/nginx/conf.d -b /usr/local/nginx/bots.d
 
 # **************************************************
 # Set our install and setup scripts to be executable
@@ -150,7 +150,7 @@ sudo cp ${TRAVIS_BUILD_DIR}/.dev-tools/globalblacklist-dummy.conf /usr/local/ngi
 
 printf '\n%s\n%s\n%s\n\n' "######################" "Run update-ngxblocker" "######################"
 cd /usr/sbin
-sudo bash ./update-ngxblocker -c /usr/local/nginx/conf.d -b /usr/local/nginx/bots.d -n
+sudo bash -x ./update-ngxblocker -c /usr/local/nginx/conf.d -b /usr/local/nginx/bots.d -n
 
 # *********************
 # Force reload of Nginx
@@ -182,7 +182,7 @@ ls -la /usr/local/nginx/bots.d/
 
 printf '\n%s\n%s\n%s\n\n' "######################" "Run update-ngxblocker" "######################"
 cd /usr/sbin
-sudo bash ./update-ngxblocker -c /usr/local/nginx/conf.d -b /usr/local/nginx/bots.d -n
+sudo bash -x ./update-ngxblocker -c /usr/local/nginx/conf.d -b /usr/local/nginx/bots.d -n
 
 # ****************************************************
 # List Directories to Confirm Downloaded Missing Files
@@ -206,7 +206,7 @@ sudo cp ${TRAVIS_BUILD_DIR}/.dev-tools/globalblacklist-dummy.conf /usr/local/ngi
 
 printf '\n%s\n%s\n%s\n\n' "######################" "Run update-ngxblocker" "######################"
 cd /usr/sbin
-sudo bash ./update-ngxblocker -c /usr/local/nginx/conf.d -b /usr/local/nginx/bots.d -n
+sudo bash -x ./update-ngxblocker -c /usr/local/nginx/conf.d -b /usr/local/nginx/bots.d -n
 
 # **************************
 # Run setup-ngxblocker Again
@@ -214,7 +214,7 @@ sudo bash ./update-ngxblocker -c /usr/local/nginx/conf.d -b /usr/local/nginx/bot
 
 printf '\n%s\n%s\n%s\n\n' "####################" "Run setup-ngxblocker" "####################"
 cd /usr/sbin
-sudo bash ./setup-ngxblocker -x -c /usr/local/nginx/conf.d -b /usr/local/nginx/bots.d
+sudo bash -x ./setup-ngxblocker -x -c /usr/local/nginx/conf.d -b /usr/local/nginx/bots.d
 
 # *********************
 # Force reload of Nginx
@@ -236,7 +236,7 @@ sudo cp ${TRAVIS_BUILD_DIR}/conf.d/globalblacklist.conf /usr/local/nginx/conf.d/
 
 printf '\n%s\n%s\n%s\n\n' "############################################################" "Run setup-ngxblocker to fix includes in globalblacklist.conf" "############################################################"
 cd /usr/sbin
-sudo bash ./setup-ngxblocker -x -c /usr/local/nginx/conf.d -b /usr/local/nginx/bots.d
+sudo bash -x ./setup-ngxblocker -x -c /usr/local/nginx/conf.d -b /usr/local/nginx/bots.d
 
 # *********************
 # Force reload of Nginx

.dev-tools/run-curl-tests-testing-of-changes.sh

@@ -50,7 +50,7 @@ else
    #exit 1
 fi
 }
-run_curltest1
+#run_curltest1
 
 
 # **************************************************
@@ -69,7 +69,7 @@ else
    #exit 1
 fi
 }
-run_curltest2
+#run_curltest2
 
 
 # ******************************************************************
@@ -88,7 +88,7 @@ else
    #exit 1
 fi
 }
-run_curltest3
+#run_curltest3
 
 
 # ******************************************************
@@ -107,7 +107,7 @@ else
    #exit 1
 fi
 }
-run_curltest4
+#run_curltest4
 
 # *****************************************************
 # Function Curl Test 5 - Check for Good Bot "GoogleBot"
@@ -125,7 +125,7 @@ else
    #exit 1
 fi
 }
-run_curltest5
+#run_curltest5
 
 # ***************************************************
 # Function Curl Test 6 - Check for Good Bot "BingBot"
@@ -143,7 +143,7 @@ else
    #exit 1
 fi
 }
-run_curltest6
+#run_curltest6
 
 # ***********************************************************
 # Function Curl Test 7 - Check for Good Referrer "google.com"
@@ -161,7 +161,7 @@ else
    #exit 1
 fi
 }
-run_curltest7
+#run_curltest7
 
 # ***********************************************************
 # Function Curl Test 8 - Check for Good Referrer "bing.com"
@@ -180,7 +180,7 @@ else
    #exit 1
 fi
 }
-run_curltest8
+#run_curltest8
 
 # **************************************************
 # Function Curl Test 9 - Check for Bad Bot "Nutch"
@@ -198,7 +198,7 @@ else
    #exit 1
 fi
 }
-run_curltest9
+#run_curltest9
 
 # **************************************************
 # Function Curl Test 10 - Check for Bad Bot "Nutch"
@@ -216,12 +216,117 @@ else
    #exit 1
 fi
 }
-run_curltest10
+#run_curltest10
 
 # *********************************
 # Trigger our curl functions to run
 # *********************************
 
+runtest () {
+truncate -s 0 ${_curltest1}
+printf '\n%s\n%s\n%s\n\n' "#########################" "TESTING BAD BOT IS DENIED" "#########################"
+printf '%s%s\n\n' "Last Tested: " "$_now" >> "${_curltest1}"
+curl -A "80legs" http://localhost:9000/index.php 2>> ${_curltest1}
+if grep -i '(52)' ${_curltest1}; then
+   echo 'BAD BOT DETECTED - TEST PASSED'
+else
+   echo 'BAD BOT NOT DETECTED - TEST FAILED'
+   #exit 1
+fi
+truncate -s 0 ${_curltest2}
+printf '\n%s\n%s\n%s\n\n' "#########################" "TESTING BAD BOT IS DENIED" "#########################"
+printf '%s%s\n\n' "Last Tested: " "$_now" >> "${_curltest2}"
+curl -A "Nutch" http://localhost:9000/index.php 2>> ${_curltest2}
+if grep -i '(52)' ${_curltest2}; then
+   echo 'BAD BOT DETECTED - TEST PASSED'
+else
+   echo 'BAD BOT NOT DETECTED - TEST FAILED'
+   #exit 1
+fi
+truncate -s 0 ${_curltest3}
+printf '\n%s\n%s\n%s\n\n' "##############################" "TESTING BAD REFERRER IS DENIED" "##############################"
+printf '%s%s\n\n' "Last Tested: " "$_now" >> "${_curltest3}"
+curl -I http://localhost:9000/index.php -e http://100dollars-seo.com 2>> ${_curltest3}
+if grep -i '(52)' ${_curltest3}; then
+   echo 'BAD REFERRER DETECTED - TEST PASSED'
+else
+   echo 'BAD REFERRER NOT DETECTED - TEST FAILED'
+   #exit 1
+fi
+truncate -s 0 ${_curltest4}
+printf '\n%s\n%s\n%s\n\n' "##############################" "TESTING BAD REFERRER IS DENIED" "##############################"
+printf '%s%s\n\n' "Last Tested: " "$_now" >> "${_curltest4}"
+curl -I http://localhost:9000/index.php -e http://zx6.ru 2>> ${_curltest4}
+if grep -i '(52)' ${_curltest4}; then
+   echo 'BAD REFERRER DETECTED - TEST PASSED'
+else
+   echo 'BAD REFERRER NOT DETECTED - TEST FAILED'
+   #exit 1
+fi
+truncate -s 0 ${_curltest5}
+printf '\n%s\n%s\n%s\n\n' "###########################" "TESTING GOOD BOT IS ALLOWED" "###########################"
+printf '%s%s\n\n' "Last Tested: " "$_now" >> "${_curltest5}"
+curl -v -A "GoogleBot" http://localhost:9000/index.php 2>&1 >> ${_curltest5}
+if grep -i 'Welcome' ${_curltest5}; then
+   echo 'GOOD BOT ALLOWED THROUGH - TEST PASSED'
+else
+   echo 'GOOD BOT NOT ALLOWED THROUGH - TEST FAILED'
+   #exit 1
+fi
+truncate -s 0 ${_curltest6}
+printf '\n%s\n%s\n%s\n\n' "###########################" "TESTING GOOD BOT IS ALLOWED" "###########################"
+printf '%s%s\n\n' "Last Tested: " "$_now" >> "${_curltest6}"
+curl -v -A "BingBot" http://localhost:9000/index.php 2>&1 >> ${_curltest6}
+if grep -i 'Welcome' ${_curltest6}; then
+   echo 'GOOD BOT ALLOWED THROUGH - TEST PASSED'
+else
+   echo 'GOOD BOT NOT ALLOWED THROUGH - TEST FAILED'
+   #exit 1
+fi
+truncate -s 0 ${_curltest7}
+printf '\n%s\n%s\n%s\n\n' "################################" "TESTING GOOD REFERRER IS ALLOWED" "################################"
+printf '%s%s\n\n' "Last Tested: " "$_now" >> "${_curltest7}"
+curl http://localhost:9000/index.php -e http://google.com 2>&1 >> ${_curltest7}
+if grep -i 'Welcome' ${_curltest7}; then
+   echo 'GOOD REFERRER DETECTED - TEST PASSED'
+else
+   echo 'GOOD REFERRER NOT DETECTED - TEST FAILED'
+   #exit 1
+fi
+truncate -s 0 ${_curltest8}
+printf '\n%s\n%s\n%s\n\n' "################################" "TESTING GOOD REFERRER IS ALLOWED" "################################"
+printf '%s%s\n\n' "Last Tested: " "$_now" >> "${_curltest8}"
+curl http://localhost:9000/index.php -e http://bing.com 2>&1 >> ${_curltest8}
+if grep -i 'Welcome' ${_curltest8}; then
+   echo 'GOOD REFERRER DETECTED - TEST PASSED'
+   exit 0
+else
+   echo 'GOOD REFERRER NOT DETECTED - TEST FAILED'
+   #exit 1
+fi
+truncate -s 0 ${_curltest9}
+printf '\n%s\n%s\n%s\n\n' "#########################" "TESTING BAD BOT IS DENIED" "#########################"
+printf '%s%s\n\n' "Last Tested: " "$_now" >> "${_curltest9}"
+curl -A "Googlebot/Nutch-1.7" http://localhost:9000/index.php 2>> ${_curltest9}
+if grep -i '(52)' ${_curltest9}; then
+   echo 'BAD BOT DETECTED - TEST PASSED'
+else
+   echo 'BAD BOT NOT DETECTED - TEST FAILED'
+   #exit 1
+fi
+truncate -s 0 ${_curltest10}
+printf '\n%s\n%s\n%s\n\n' "#########################" "TESTING BAD BOT IS DENIED" "#########################"
+printf '%s%s\n\n' "Last Tested: " "$_now" >> "${_curltest10}"
+curl -A "Mozilla/5.0 (compatible; Googlebot/Nutch2.1; +http://www.google.com/bot.html)" http://localhost:9000/index.php 2>> ${_curltest10}
+if grep -i '(52)' ${_curltest10}; then
+   echo 'BAD BOT DETECTED - TEST PASSED'
+else
+   echo 'BAD BOT NOT DETECTED - TEST FAILED'
+   #exit 1
+fi
+}
+runtest
+
 
 echo "Tests Completed"
 

.travis.yml

@@ -1,5 +1,3 @@
-git:
-  depth: 1
 addons:
     apt:
         packages:
@@ -43,12 +41,12 @@ script:
   - bash .dev-tools/install-nginx-1.sh
   - bash .dev-tools/run-curl-tests-1.sh
   - bash .dev-tools/install-nginx-testing-of-changes.sh
-  - bash .dev-tools/run-curl-tests-testing-of-changes.sh
+  - bash -x .dev-tools/run-curl-tests-testing-of-changes.sh
-  - bash .dev-tools/install-nginx-2.sh
+  - bash -x .dev-tools/install-nginx-2.sh
   - bash .dev-tools/run-curl-tests-2.sh
-  - bash .dev-tools/install-nginx-3.sh
+  - bash -x .dev-tools/install-nginx-3.sh
   - bash .dev-tools/run-curl-tests-3.sh
-  - bash .dev-tools/install-nginx-4.sh
+  - bash -x .dev-tools/install-nginx-4.sh
   - bash .dev-tools/run-curl-tests-4.sh
   - bash .dev-tools/modify-files-and-commit.sh