vikarti.anatra/nginx-ultimate-bad-bot-blocker
1
0
Fork 0
mirror of https://github.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker.git synced 2025-09-02 10:40:36 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Cleanup Build Scripts

Browse source
This commit is contained in:
Mitchell Krog 2018-08-21 16:52:53 +02:00
parent ec01b4bdfb
commit ce9b26a509
No known key found for this signature in database
GPG key ID: EA1DD210CACCC45E
16 changed files with 452 additions and 377 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
.dev-tools/deploy-package.sh
View file

@ -75,6 +75,13 @@ sudo git push origin master && git push origin master --tags
# Now TravisCI moves into the deploy: section of TravisCI - see .travis.yml
# *************************************************************************
# **********************
# Exit With Error Number
# **********************
exit ${?}
# MIT License
# Copyright (c) 2017 Mitchell Krog - mitchellkrog@gmail.com

166
.dev-tools/generate-blacklist.sh
View file

@ -45,10 +45,12 @@
YEAR=$(date +"%Y")
MONTH=$(date +"%m")
MY_GIT_TAG=V3.$YEAR.$MONTH.$TRAVIS_BUILD_NUMBER
BAD_REFERRERS=$(wc -l < $TRAVIS_BUILD_DIR/_generator_lists/bad-referrers.list)
BAD_BOTS=$(wc -l < $TRAVIS_BUILD_DIR/_generator_lists/bad-user-agents.list)
MY_GIT_TAG=V3.${YEAR}.${MONTH}.${TRAVIS_BUILD_NUMBER}
BAD_REFERRERS=$(wc -l < ${TRAVIS_BUILD_DIR}/_generator_lists/bad-referrers.list)
BAD_BOTS=$(wc -l < ${TRAVIS_BUILD_DIR}/_generator_lists/bad-user-agents.list)
_now="$(date)"
# NOTE:
# To get DATE output into uppercase format (if needed)
#_now=$(date | tr -s '[:lower:]' '[:upper:]')
@ -189,14 +191,14 @@ rm ${_inputdb1}
# ALLOWED BOTS - Create and Insert
# ********************************
printf '%s\n' "$_start2" >> "$_tmpnginx2"
printf '%s\n' "$_start2" >> ${_tmpnginx2}
while IFS= read -r LINE
do
printf '\t"~*%s%s%s"\t\t%s\n' "\b" "${LINE}" "\b" "$_action2" >> "$_tmpnginx2"
done < $_input2
printf '%s\n' "$_end2" >> "$_tmpnginx2"
mv $_tmpnginx2 $_inputdb2
ed -s $_inputdb2<<\IN
printf '\t"~*%s%s%s"\t\t%s\n' "\b" "${LINE}" "\b" "$_action2" >> ${_tmpnginx2}
done < ${_input2}
printf '%s\n' "$_end2" >> ${_tmpnginx2}
mv ${_tmpnginx2} ${_inputdb2}
ed -s ${_inputdb2}<<\IN
1,/# START ALLOWED BOTS ### DO NOT EDIT THIS LINE AT ALL ###/d
/# END ALLOWED BOTS ### DO NOT EDIT THIS LINE AT ALL ###/,$d
,d
@ -207,20 +209,20 @@ ed -s $_inputdb2<<\IN
w /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/.dev-tools/globalblacklist.template
q
IN
rm $_inputdb2
rm ${_inputdb2}
# ********************************
# LIMITED BOTS - Create and Insert
# ********************************
printf '%s\n' "$_start3" >> "$_tmpnginx3"
printf '%s\n' "$_start3" >> ${_tmpnginx3}
while IFS= read -r LINE
do
printf '\t"~*%s%s%s"\t\t%s\n' "\b" "${LINE}" "\b" "$_action3" >> "$_tmpnginx3"
done < $_input3
printf '%s\n' "$_end3" >> "$_tmpnginx3"
mv $_tmpnginx3 $_inputdb3
ed -s $_inputdb3<<\IN
printf '\t"~*%s%s%s"\t\t%s\n' "\b" "${LINE}" "\b" "$_action3" >> ${_tmpnginx3}
done < ${_input3}
printf '%s\n' "$_end3" >> ${_tmpnginx3}
mv ${_tmpnginx3} ${_inputdb3}
ed -s ${_inputdb3}<<\IN
1,/# START LIMITED BOTS ### DO NOT EDIT THIS LINE AT ALL ###/d
/# END LIMITED BOTS ### DO NOT EDIT THIS LINE AT ALL ###/,$d
,d
@ -231,20 +233,20 @@ ed -s $_inputdb3<<\IN
w /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/.dev-tools/globalblacklist.template
q
IN
rm $_inputdb3
rm ${_inputdb3}
# ****************************
# BAD BOTS - Create and Insert
# ****************************
printf '%s\n' "$_start4" >> "$_tmpnginx4"
printf '%s\n' "$_start4" >> ${_tmpnginx4}
while IFS= read -r LINE
do
printf '\t"~*%s%s%s"\t\t%s\n' "\b" "${LINE}" "\b" "$_action4" >> "$_tmpnginx4"
done < $_input4
printf '%s\n' "$_end4" >> "$_tmpnginx4"
mv $_tmpnginx4 $_inputdb4
ed -s $_inputdb4<<\IN
printf '\t"~*%s%s%s"\t\t%s\n' "\b" "${LINE}" "\b" "$_action4" >> ${_tmpnginx4}
done < ${_input4}
printf '%s\n' "$_end4" >> ${_tmpnginx4}
mv ${_tmpnginx4} ${_inputdb4}
ed -s ${_inputdb4}<<\IN
1,/# START BAD BOTS ### DO NOT EDIT THIS LINE AT ALL ###/d
/# END BAD BOTS ### DO NOT EDIT THIS LINE AT ALL ###/,$d
,d
@ -255,20 +257,20 @@ ed -s $_inputdb4<<\IN
w /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/.dev-tools/globalblacklist.template
q
IN
rm $_inputdb4
rm ${_inputdb4}
# ********************************
# BAD REFERERS - Create and Insert
# ********************************
printf '%s\n' "$_start5" >> "$_tmpnginx5"
printf '%s\n' "$_start5" >> ${_tmpnginx5}
while IFS= read -r LINE
do
printf '\t%s\n' "${LINE}" >> "$_tmpnginx5"
done < $_input5
printf '%s\n' "$_end5" >> "$_tmpnginx5"
mv $_tmpnginx5 $_inputdb5
ed -s $_inputdb5<<\IN
printf '\t%s\n' "${LINE}" >> ${_tmpnginx5}
done < ${_input5}
printf '%s\n' "$_end5" >> ${_tmpnginx5}
mv ${_tmpnginx5} ${_inputdb5}
ed -s ${_inputdb5}<<\IN
1,/# START BAD REFERRERS ### DO NOT EDIT THIS LINE AT ALL ###/d
/# END BAD REFERRERS ### DO NOT EDIT THIS LINE AT ALL ###/,$d
,d
@ -279,20 +281,20 @@ ed -s $_inputdb5<<\IN
w /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/.dev-tools/globalblacklist.template
q
IN
rm $_inputdb5
rm ${_inputdb5}
# ************************************
# GOOGLE IP RANGES - Create and Insert
# ************************************
printf '%s\n' "$_start6" >> "$_tmpnginx6"
printf '%s\n' "$_start6" >> ${_tmpnginx6}
while IFS= read -r LINE
do
printf '\t%s\t\t%s\n' "${LINE}" "$_action1" >> "$_tmpnginx6"
done < $_input6
printf '%s\n' "$_end6" >> "$_tmpnginx6"
mv $_tmpnginx6 $_inputdb6
ed -s $_inputdb6<<\IN
printf '\t%s\t\t%s\n' "${LINE}" "$_action1" >> ${_tmpnginx6}
done < ${_input6}
printf '%s\n' "$_end6" >> ${_tmpnginx6}
mv ${_tmpnginx6} ${_inputdb6}
ed -s ${_inputdb6}<<\IN
1,/# START GOOGLE IP RANGES ### DO NOT EDIT THIS LINE AT ALL ###/d
/# END GOOGLE IP RANGES ### DO NOT EDIT THIS LINE AT ALL ###/,$d
,d
@ -303,20 +305,20 @@ ed -s $_inputdb6<<\IN
w /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/.dev-tools/globalblacklist.template
q
IN
rm $_inputdb6
rm ${_inputdb6}
# **********************************
# BING IP RANGES - Create and Insert
# **********************************
printf '%s\n' "$_start7" >> "$_tmpnginx7"
printf '%s\n' "$_start7" >> ${_tmpnginx7}
while IFS= read -r LINE
do
printf '\t%s\t\t%s\n' "${LINE}" "$_action1" >> "$_tmpnginx7"
done < $_input7
printf '%s\n' "$_end7" >> "$_tmpnginx7"
mv $_tmpnginx7 $_inputdb7
ed -s $_inputdb7<<\IN
printf '\t%s\t\t%s\n' "${LINE}" "$_action1" >> ${_tmpnginx7}
done < ${_input7}
printf '%s\n' "$_end7" >> ${_tmpnginx7}
mv ${_tmpnginx7} ${_inputdb7}
ed -s ${_inputdb7}<<\IN
1,/# START BING IP RANGES ### DO NOT EDIT THIS LINE AT ALL ###/d
/# END BING IP RANGES ### DO NOT EDIT THIS LINE AT ALL ###/,$d
,d
@ -327,20 +329,20 @@ ed -s $_inputdb7<<\IN
w /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/.dev-tools/globalblacklist.template
q
IN
rm $_inputdb7
rm ${_inputdb7}
# *********************************************
# Wordpress Theme Detectors - Create and Insert
# *********************************************
printf '%s\n' "$_start8" >> "$_tmpnginx8"
printf '%s\n' "$_start8" >> ${_tmpnginx8}
while IFS= read -r LINE
do
printf '%s\n' "${LINE}" >> "$_tmpnginx8"
done < $_input8
printf '%s\n' "$_end8" >> "$_tmpnginx8"
mv $_tmpnginx8 $_inputdb8
ed -s $_inputdb8<<\IN
printf '%s\n' "${LINE}" >> ${_tmpnginx8}
done < ${_input8}
printf '%s\n' "$_end8" >> ${_tmpnginx8}
mv ${_tmpnginx8} ${_inputdb8}
ed -s ${_inputdb8}<<\IN
1,/# START WP THEME DETECTORS ### DO NOT EDIT THIS LINE AT ALL ###/d
/# END WP THEME DETECTORS ### DO NOT EDIT THIS LINE AT ALL ###/,$d
,d
@ -351,20 +353,20 @@ ed -s $_inputdb8<<\IN
w /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/.dev-tools/globalblacklist.template
q
IN
rm $_inputdb8
rm ${_inputdb8}
# *******************************
# Nibbler SEO - Create and Insert
# *******************************
printf '%s\n' "$_start9" >> "$_tmpnginx9"
printf '%s\n' "$_start9" >> ${_tmpnginx9}
while IFS= read -r LINE
do
printf '\t%s\t\t%s\n' "${LINE}" "$_action2" >> "$_tmpnginx9"
done < $_input9
printf '%s\n' "$_end9" >> "$_tmpnginx9"
mv $_tmpnginx9 $_inputdb9
ed -s $_inputdb9<<\IN
printf '\t%s\t\t%s\n' "${LINE}" "$_action2" >> ${_tmpnginx9}
done < ${_input9}
printf '%s\n' "$_end9" >> ${_tmpnginx9}
mv ${_tmpnginx9} ${_inputdb9}
ed -s ${_inputdb9}<<\IN
1,/# START NIBBLER ### DO NOT EDIT THIS LINE AT ALL ###/d
/# END NIBBLER ### DO NOT EDIT THIS LINE AT ALL ###/,$d
,d
@ -375,20 +377,20 @@ ed -s $_inputdb9<<\IN
w /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/.dev-tools/globalblacklist.template
q
IN
rm $_inputdb9
rm ${_inputdb9}
# ****************************************
# CLOUDFLARE IP RANGES - Create and Insert
# ****************************************
printf '%s\n' "$_start10" >> "$_tmpnginx10"
printf '%s\n' "$_start10" >> ${_tmpnginx10}
while IFS= read -r LINE
do
printf '\t%s\t\t%s\n' "${LINE}" "$_action1" >> "$_tmpnginx10"
done < $_input10
printf '%s\n' "$_end10" >> "$_tmpnginx10"
mv $_tmpnginx10 $_inputdb10
ed -s $_inputdb10<<\IN
printf '\t%s\t\t%s\n' "${LINE}" "$_action1" >> ${_tmpnginx10}
done < ${_input10}
printf '%s\n' "$_end10" >> ${_tmpnginx10}
mv ${_tmpnginx10} ${_inputdb10}
ed -s ${_inputdb10}<<\IN
1,/# START CLOUDFLARE IP RANGES ### DO NOT EDIT THIS LINE AT ALL ###/d
/# END CLOUDFLARE IP RANGES ### DO NOT EDIT THIS LINE AT ALL ###/,$d
,d
@ -399,20 +401,20 @@ ed -s $_inputdb10<<\IN
w /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/.dev-tools/globalblacklist.template
q
IN
rm $_inputdb10
rm ${_inputdb10}
# ****************************************
# KNOWN BAD IP RANGES - Create and Insert
# ****************************************
printf '%s\n' "$_start11" >> "$_tmpnginx11"
printf '%s\n' "$_start11" >> ${_tmpnginx11}
while IFS= read -r LINE
do
printf '\t%s\t\t%s\n' "${LINE}" "$_action2" >> "$_tmpnginx11"
done < $_input11
printf '%s\n' "$_end11" >> "$_tmpnginx11"
mv $_tmpnginx11 $_inputdb11
ed -s $_inputdb11<<\IN
printf '\t%s\t\t%s\n' "${LINE}" "$_action2" >> ${_tmpnginx11}
done < ${_input11}
printf '%s\n' "$_end11" >> ${_tmpnginx11}
mv ${_tmpnginx11} ${_inputdb11}
ed -s ${_inputdb11}<<\IN
1,/# START KNOWN BAD IP ADDRESSES ### DO NOT EDIT THIS LINE AT ALL ###/d
/# END KNOWN BAD IP ADDRESSES ### DO NOT EDIT THIS LINE AT ALL ###/,$d
,d
@ -423,15 +425,15 @@ ed -s $_inputdb11<<\IN
w /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/.dev-tools/globalblacklist.template
q
IN
rm $_inputdb11
rm ${_inputdb11}
# *******************************************************************************
# PRINT VERSION, SCRIPT RUNTIME and UPDATE INFORMATION INTO GLOBALBLACKLIST FILES
# *******************************************************************************
printf '%s\n%s\n%s%s\n%s%s\n%s%s\n%s%s\n%s\n%s\n' "$_startmarker" "###################################################" "### Version: " "$MY_GIT_TAG" "### Updated: " "$_now" "### Bad Referrer Count: " "$BAD_REFERRERS" "### Bad Bot Count: " "$BAD_BOTS" "###################################################" "$_endmarker" >> $_tmpnginxA
mv $_tmpnginxA $_inputdbA
ed -s $_inputdbA<<\IN
printf '%s\n%s\n%s%s\n%s%s\n%s%s\n%s%s\n%s\n%s\n' "$_startmarker" "###################################################" "### Version: " "$MY_GIT_TAG" "### Updated: " "$_now" "### Bad Referrer Count: " "$BAD_REFERRERS" "### Bad Bot Count: " "$BAD_BOTS" "###################################################" "$_endmarker" >> ${_tmpnginxA}
mv ${_tmpnginxA} ${_inputdbA}
ed -s ${_inputdbA}<<\IN
1,/### VERSION INFORMATION #/d
/### VERSION INFORMATION ##/,$d
,d
@ -444,16 +446,20 @@ ed -s $_inputdbA<<\IN
w /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/.dev-tools/globalblacklist.template
q
IN
rm $_inputdbA
rm ${_inputdbA}
# **************************************************
# Generate Additional Files and Copy Them to Folders
# **************************************************
sudo cp $_nginx $TRAVIS_BUILD_DIR/conf.d/globalblacklist.conf
sudo cp $_nginx $TRAVIS_BUILD_DIR/_sample_config_files/Engintron_for_cPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf
sudo cp ${_nginx} ${TRAVIS_BUILD_DIR}/conf.d/globalblacklist.conf
sudo cp ${_nginx} ${TRAVIS_BUILD_DIR}/_sample_config_files/Engintron_for_cPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf
exit 0
# **********************
# Exit With Error Number
# **********************
exit ${?}
# MIT License

63
.dev-tools/generate-files.sh
View file

@ -43,53 +43,58 @@
# Set all our setup and deploy scripts to be executable
# *****************************************************
sudo chmod +x $TRAVIS_BUILD_DIR/.dev-tools/deploy-package.sh
sudo chmod +x $TRAVIS_BUILD_DIR/.dev-tools/generate-blacklist.sh
sudo chmod +x $TRAVIS_BUILD_DIR/.dev-tools/generate-files.sh
sudo chmod +x $TRAVIS_BUILD_DIR/.dev-tools/generate-robots.sh
sudo chmod +x $TRAVIS_BUILD_DIR/.dev-tools/generate-google-disavow.sh
sudo chmod +x $TRAVIS_BUILD_DIR/.dev-tools/generate-google-exclude.php
sudo chmod +x $TRAVIS_BUILD_DIR/.dev-tools/generate-regex-format-referrers.php
sudo chmod +x $TRAVIS_BUILD_DIR/.dev-tools/modify-config-readme-files.sh
sudo chmod +x $TRAVIS_BUILD_DIR/.dev-tools/modify-files-and-commit.sh
sudo chmod +x $TRAVIS_BUILD_DIR/.dev-tools/run-curl-tests-1.sh
sudo chmod +x $TRAVIS_BUILD_DIR/.dev-tools/run-curl-tests-2.sh
sudo chmod +x $TRAVIS_BUILD_DIR/.dev-tools/run-curl-tests-3.sh
sudo chmod +x $TRAVIS_BUILD_DIR/.dev-tools/run-curl-tests-4.sh
sudo chmod +x $TRAVIS_BUILD_DIR/.dev-tools/prepare-robots-input.sh
sudo chmod +x $TRAVIS_BUILD_DIR/.dev-tools/install-nginx-1.sh
sudo chmod +x $TRAVIS_BUILD_DIR/.dev-tools/install-nginx-2.sh
sudo chmod +x $TRAVIS_BUILD_DIR/.dev-tools/install-nginx-3.sh
sudo chmod +x $TRAVIS_BUILD_DIR/.dev-tools/install-nginx-4.sh
sudo chmod +x $TRAVIS_BUILD_DIR/.dev-tools/pull-data.sh
sudo chmod +x ${TRAVIS_BUILD_DIR}/.dev-tools/deploy-package.sh
sudo chmod +x ${TRAVIS_BUILD_DIR}/.dev-tools/generate-blacklist.sh
sudo chmod +x ${TRAVIS_BUILD_DIR}/.dev-tools/generate-files.sh
sudo chmod +x ${TRAVIS_BUILD_DIR}/.dev-tools/generate-robots.sh
sudo chmod +x ${TRAVIS_BUILD_DIR}/.dev-tools/generate-google-disavow.sh
sudo chmod +x ${TRAVIS_BUILD_DIR}/.dev-tools/generate-google-exclude.php
sudo chmod +x ${TRAVIS_BUILD_DIR}/.dev-tools/generate-regex-format-referrers.php
sudo chmod +x ${TRAVIS_BUILD_DIR}/.dev-tools/modify-config-readme-files.sh
sudo chmod +x ${TRAVIS_BUILD_DIR}/.dev-tools/modify-files-and-commit.sh
sudo chmod +x ${TRAVIS_BUILD_DIR}/.dev-tools/run-curl-tests-1.sh
sudo chmod +x ${TRAVIS_BUILD_DIR}/.dev-tools/run-curl-tests-2.sh
sudo chmod +x ${TRAVIS_BUILD_DIR}/.dev-tools/run-curl-tests-3.sh
sudo chmod +x ${TRAVIS_BUILD_DIR}/.dev-tools/run-curl-tests-4.sh
sudo chmod +x ${TRAVIS_BUILD_DIR}/.dev-tools/prepare-robots-input.sh
sudo chmod +x ${TRAVIS_BUILD_DIR}/.dev-tools/install-nginx-1.sh
sudo chmod +x ${TRAVIS_BUILD_DIR}/.dev-tools/install-nginx-2.sh
sudo chmod +x ${TRAVIS_BUILD_DIR}/.dev-tools/install-nginx-3.sh
sudo chmod +x ${TRAVIS_BUILD_DIR}/.dev-tools/install-nginx-4.sh
sudo chmod +x ${TRAVIS_BUILD_DIR}/.dev-tools/pull-data.sh
cd $TRAVIS_BUILD_DIR
cd ${TRAVIS_BUILD_DIR}
# ***************************************************
# Modify our files with build and version information
# ***************************************************
php ./.dev-tools/generate-regex-format-referrers.php
sudo $TRAVIS_BUILD_DIR/.dev-tools/generate-blacklist.sh
sudo $TRAVIS_BUILD_DIR/.dev-tools/modify-config-readme-files.sh
sudo $TRAVIS_BUILD_DIR/.dev-tools/prepare-robots-input.sh
sudo $TRAVIS_BUILD_DIR/.dev-tools/generate-robots.sh
sudo $TRAVIS_BUILD_DIR/.dev-tools/generate-google-disavow.sh
sudo ${TRAVIS_BUILD_DIR}/.dev-tools/generate-blacklist.sh
sudo ${TRAVIS_BUILD_DIR}/.dev-tools/modify-config-readme-files.sh
sudo ${TRAVIS_BUILD_DIR}/.dev-tools/prepare-robots-input.sh
sudo ${TRAVIS_BUILD_DIR}/.dev-tools/generate-robots.sh
sudo ${TRAVIS_BUILD_DIR}/.dev-tools/generate-google-disavow.sh
php ./.dev-tools/generate-google-exclude.php
# ************************************
# Convert All Our Files to Unix Format
# ************************************
sudo dos2unix $TRAVIS_BUILD_DIR/bots.d/*.conf
sudo dos2unix $TRAVIS_BUILD_DIR/conf.d/*.conf
sudo dos2unix $TRAVIS_BUILD_DIR/_generator_lists/*.list
sudo dos2unix $TRAVIS_BUILD_DIR/.dev-tools/referrers-regex-format.txt
sudo dos2unix ${TRAVIS_BUILD_DIR}/bots.d/*.conf
sudo dos2unix ${TRAVIS_BUILD_DIR}/conf.d/*.conf
sudo dos2unix ${TRAVIS_BUILD_DIR}/_generator_lists/*.list
sudo dos2unix ${TRAVIS_BUILD_DIR}/.dev-tools/referrers-regex-format.txt
# *************************************************
# Move On to the Rest of Our TravisCI Build Testing
# *************************************************
# **********************
# Exit With Error Number
# **********************
exit ${?}
# MIT License

14
.dev-tools/generate-google-disavow.sh
View file

@ -43,19 +43,23 @@
# Specify input list for the generator
# ************************************
_input1=$TRAVIS_BUILD_DIR/_generator_lists/bad-referrers.list
_input1=${TRAVIS_BUILD_DIR}/_generator_lists/bad-referrers.list
# **************************
# Create Google Disavow File
# **************************
sudo truncate -s 0 $TRAVIS_BUILD_DIR/_google_webmaster_disavow_links/google-disavow.txt
sudo truncate -s 0 ${TRAVIS_BUILD_DIR}/_google_webmaster_disavow_links/google-disavow.txt
while IFS= read -r LINE
do
printf '%s%s\n' "domain:" "${LINE}" >> $TRAVIS_BUILD_DIR/_google_webmaster_disavow_links/google-disavow.txt
done < $_input1
printf '%s%s\n' "domain:" "${LINE}" >> ${TRAVIS_BUILD_DIR}/_google_webmaster_disavow_links/google-disavow.txt
done < ${_input1}
exit 0
# **********************
# Exit With Error Number
# **********************
exit ${?}
# MIT License

29
.dev-tools/generate-robots.sh
View file

@ -45,22 +45,22 @@
# Set Input Files
# ***************
_input1=$TRAVIS_BUILD_DIR/_generator_lists/bad-user-agents.list
_input1=${TRAVIS_BUILD_DIR}/_generator_lists/bad-user-agents.list
_tmprobots=/tmp/robots.txt
_inputtmp=$TRAVIS_BUILD_DIR/.dev-tools/_robots_input/robots.tmp
_output=$TRAVIS_BUILD_DIR/.dev-tools/_robots_input/robots-input.txt
_inputtmp=${TRAVIS_BUILD_DIR}/.dev-tools/_robots_input/robots.tmp
_output=${TRAVIS_BUILD_DIR}/.dev-tools/_robots_input/robots-input.txt
# ***********************
# Truncate our input file
# ***********************
sudo truncate -s 0 $_output
sudo truncate -s 0 ${_output}
# *************************************
# Use sed to prepare our new input file
# *************************************
cat $_input1 | sed 's/\\ / /g' > $_inputtmp && mv $_inputtmp $_output
cat ${_input1} | sed 's/\\ / /g' > ${_inputtmp} && mv ${_inputtmp} ${_output}
# ******************
# Set Some Variables
@ -68,9 +68,9 @@ cat $_input1 | sed 's/\\ / /g' > $_inputtmp && mv $_inputtmp $_output
YEAR=$(date +"%Y")
MONTH=$(date +"%m")
MY_GIT_TAG=V3.$YEAR.$MONTH.$TRAVIS_BUILD_NUMBER
BAD_REFERRERS=$(wc -l < $TRAVIS_BUILD_DIR/_generator_lists/bad-referrers.list)
BAD_BOTS=$(wc -l < $TRAVIS_BUILD_DIR/_generator_lists/bad-user-agents.list)
MY_GIT_TAG=V3.${YEAR}.${MONTH}.${TRAVIS_BUILD_NUMBER}
BAD_REFERRERS=$(wc -l < ${TRAVIS_BUILD_DIR}/_generator_lists/bad-referrers.list)
BAD_BOTS=$(wc -l < ${TRAVIS_BUILD_DIR}/_generator_lists/bad-user-agents.list)
_now="$(date)"
# *************************
@ -92,10 +92,15 @@ printf '%s\n%s\n%s%s\n%s%s\n%s%s\n%s\n%s\n\n%s\n%s\n%s\n' "$_startmarker" "#####
while IFS= read -r LINE
do
printf 'User-agent: %s\n%s\n' "${LINE}" "Disallow:/" >> "${_tmprobots}"
done < $_output
printf '\n' >> "${_tmprobots}"
sudo cp $_tmprobots $TRAVIS_BUILD_DIR/robots.txt/robots.txt
exit 0
done < ${_output}
printf '\n' >> ${_tmprobots}
sudo cp ${_tmprobots} ${TRAVIS_BUILD_DIR}/robots.txt/robots.txt
# **********************
# Exit With Error Number
# **********************
exit ${?}
# MIT License

23
.dev-tools/install-nginx-1.sh
View file

@ -51,7 +51,7 @@ printf '\n%s\n%s\n%s\n\n' "###################################" " STARTING
# ******************************************************
printf '\n%s\n%s\n%s\n\n' "###################################" "Making backup of default nginx.conf" "###################################"
sudo cp /etc/nginx/nginx.conf $TRAVIS_BUILD_DIR/.dev-tools/_nginx_conf_backup/nginx.conf
sudo cp /etc/nginx/nginx.conf ${TRAVIS_BUILD_DIR}/.dev-tools/_nginx_conf_backup/nginx.conf
# *************************************************
# Delete default site created by Nginx Installation
@ -68,7 +68,7 @@ printf '\n%s\n%s\n%s\n\n' "#######################################" "Nginx prepa
# ********************************************************
printf '\n%s\n%s\n%s\n\n' "################################" "Get Nginx Setup for Nginx Test 1" "################################"
sudo cp $TRAVIS_BUILD_DIR/.dev-tools/default.vhost /etc/nginx/sites-available/default.vhost
sudo cp ${TRAVIS_BUILD_DIR}/.dev-tools/default.vhost /etc/nginx/sites-available/default.vhost
# **********************************************
# Link the vhost file into Nginx /sites-enabled/
@ -80,7 +80,7 @@ sudo ln -s /etc/nginx/sites-available/default.vhost /etc/nginx/sites-enabled/def
# Copy our index.php file into the default site's root folder
# ***********************************************************
sudo cp $TRAVIS_BUILD_DIR/.dev-tools/index.php /var/www/html/index.php
sudo cp ${TRAVIS_BUILD_DIR}/.dev-tools/index.php /var/www/html/index.php
printf '\n%s\n%s\n%s\n\n' "#####################################" "Finished Nginx Setup for Nginx Test 1" "#####################################"
# ***********************************************
@ -127,7 +127,7 @@ sudo nginx -c /etc/nginx/nginx.conf
# ****************************************************************************************
printf '\n%s\n%s\n%s\n\n' "####################################################" "Copy older globalblacklist.conf file to force update" "####################################################"
sudo cp $TRAVIS_BUILD_DIR/.dev-tools/globalblacklist-dummy.conf /etc/nginx/conf.d/globalblacklist.conf
sudo cp ${TRAVIS_BUILD_DIR}/.dev-tools/globalblacklist-dummy.conf /etc/nginx/conf.d/globalblacklist.conf
# ****************************************************************************************
# Run update-ngxblocker test which downloads latest globalblacklist.conf and reloads Nginx
@ -183,7 +183,7 @@ ls -la /etc/nginx/bots.d/
# ****************************************************************************************
printf '\n%s\n%s\n%s\n\n' "####################################################" "Copy older globalblacklist.conf file to force update" "####################################################"
sudo cp $TRAVIS_BUILD_DIR/.dev-tools/globalblacklist-dummy.conf /etc/nginx/conf.d/globalblacklist.conf
sudo cp ${TRAVIS_BUILD_DIR}/.dev-tools/globalblacklist-dummy.conf /etc/nginx/conf.d/globalblacklist.conf
# *********************************************************************************************************
# Run update-ngxblocker to test for missing files and download latest globalblacklist.conf and reload Nginx
@ -213,7 +213,7 @@ sudo service nginx reload
# *******************************************************
printf '\n%s\n%s\n%s\n\n' "##############################################################################" "Copy latest generated globalblacklist.conf file to test for any compile errors" "##############################################################################"
sudo cp $TRAVIS_BUILD_DIR/conf.d/globalblacklist.conf /etc/nginx/conf.d/globalblacklist.conf
sudo cp ${TRAVIS_BUILD_DIR}/conf.d/globalblacklist.conf /etc/nginx/conf.d/globalblacklist.conf
# *********************
# Force reload of Nginx
@ -232,13 +232,20 @@ printf '\n%s\n%s\n%s\n\n' "######################" "Now Run our Curl Tests" "###
# Copy all .conf files used in Test 1 to a folder for checking
# ************************************************************
sudo cp /etc/nginx/bots.d/* $TRAVIS_BUILD_DIR/.dev-tools/_conf_files_test1/bots.d/
sudo cp /etc/nginx/conf.d/* $TRAVIS_BUILD_DIR/.dev-tools/_conf_files_test1/conf.d/
sudo cp /etc/nginx/bots.d/* ${TRAVIS_BUILD_DIR}/.dev-tools/_conf_files_test1/bots.d/
sudo cp /etc/nginx/conf.d/* ${TRAVIS_BUILD_DIR}/.dev-tools/_conf_files_test1/conf.d/
# *****************************************************************************************
# Travis now moves into running the rest of the tests in the script: section of .travis.yml
# *****************************************************************************************
# **********************
# Exit With Error Number
# **********************
exit ${?}
# MIT License
# Copyright (c) 2017 Mitchell Krog - mitchellkrog@gmail.com

20
.dev-tools/install-nginx-2.sh
View file

@ -77,7 +77,7 @@ printf '\n%s\n\n' " "
# ********************************************************
printf '\n%s\n%s\n%s\n\n' "################################" "Get Nginx Setup for Nginx Test 2" "################################"
sudo cp $TRAVIS_BUILD_DIR/.dev-tools/default.vhost /etc/nginx/sites-available/default.vhost
sudo cp ${TRAVIS_BUILD_DIR}/.dev-tools/default.vhost /etc/nginx/sites-available/default.vhost
# **********************************************
# Link the vhost file into Nginx /sites-enabled/
@ -89,7 +89,7 @@ sudo ln -s /etc/nginx/sites-available/default.vhost /etc/nginx/sites-enabled/def
# Copy our index.php file into the default site's root folder
# ***********************************************************
sudo cp $TRAVIS_BUILD_DIR/.dev-tools/index.php /var/www/html/index.php
sudo cp ${TRAVIS_BUILD_DIR}/.dev-tools/index.php /var/www/html/index.php
printf '\n%s\n%s\n%s\n\n' "#####################################" "Finished Nginx Setup for Nginx Test 2" "#####################################"
# ***********************************************
@ -142,7 +142,7 @@ sudo nginx -c /etc/nginx/nginx.conf
# ****************************************************************************************
printf '\n%s\n%s\n%s\n\n' "####################################################" "Copy older globalblacklist.conf file to force update" "####################################################"
sudo cp $TRAVIS_BUILD_DIR/.dev-tools/globalblacklist-dummy.conf /usr/local/nginx/conf.d/globalblacklist.conf
sudo cp ${TRAVIS_BUILD_DIR}/.dev-tools/globalblacklist-dummy.conf /usr/local/nginx/conf.d/globalblacklist.conf
# ****************************************************************************************
# Run update-ngxblocker test which downloads latest globalblacklist.conf and reloads Nginx
@ -198,7 +198,7 @@ ls -la /usr/local/nginx/bots.d/
# ****************************************************************************************
printf '\n%s\n%s\n%s\n\n' "####################################################" "Copy older globalblacklist.conf file to force update" "####################################################"
sudo cp $TRAVIS_BUILD_DIR/.dev-tools/globalblacklist-dummy.conf /usr/local/nginx/conf.d/globalblacklist.conf
sudo cp ${TRAVIS_BUILD_DIR}/.dev-tools/globalblacklist-dummy.conf /usr/local/nginx/conf.d/globalblacklist.conf
# *********************************************************************************************************
# Run update-ngxblocker to test for missing files and download latest globalblacklist.conf and reload Nginx
@ -228,7 +228,7 @@ sudo service nginx reload
# *******************************************************
printf '\n%s\n%s\n%s\n\n' "##############################################################################" "Copy latest generated globalblacklist.conf file to test for any compile errors" "##############################################################################"
sudo cp $TRAVIS_BUILD_DIR/conf.d/globalblacklist.conf /usr/local/nginx/conf.d/globalblacklist.conf
sudo cp ${TRAVIS_BUILD_DIR}/conf.d/globalblacklist.conf /usr/local/nginx/conf.d/globalblacklist.conf
# **************************
# Run setup-ngxblocker Again
@ -255,13 +255,19 @@ printf '\n%s\n%s\n%s\n\n' "######################" "Now Run our Curl Tests" "###
# Copy all .conf files used in Test 2 to a folder for checking
# ************************************************************
sudo cp /usr/local/nginx/bots.d/* $TRAVIS_BUILD_DIR/.dev-tools/_conf_files_test2/bots.d/
sudo cp /usr/local/nginx/conf.d/* $TRAVIS_BUILD_DIR/.dev-tools/_conf_files_test2/conf.d/
sudo cp /usr/local/nginx/bots.d/* ${TRAVIS_BUILD_DIR}/.dev-tools/_conf_files_test2/bots.d/
sudo cp /usr/local/nginx/conf.d/* ${TRAVIS_BUILD_DIR}/.dev-tools/_conf_files_test2/conf.d/
# *****************************************************************************************
# Travis now moves into running the rest of the tests in the script: section of .travis.yml
# *****************************************************************************************
# **********************
# Exit With Error Number
# **********************
exit ${?}
# MIT License
# Copyright (c) 2017 Mitchell Krog - mitchellkrog@gmail.com

26
.dev-tools/install-nginx-3.sh
View file

@ -61,7 +61,7 @@ printf '\n%s\n%s\n%s\n\n' "########################################" "Cleaning u
# ******************************************************
printf '\n%s\n%s\n%s\n\n' "#################################" "Restoring Default nginx.conf file" "#################################"
sudo cp $TRAVIS_BUILD_DIR/.dev-tools/_nginx_conf_backup/nginx.conf /etc/nginx/nginx.conf
sudo cp ${TRAVIS_BUILD_DIR}/.dev-tools/_nginx_conf_backup/nginx.conf /etc/nginx/nginx.conf
# *************************************
# List Directories to Confirm Deletions
@ -84,7 +84,7 @@ printf '\n%s\n\n' " "
# ********************************************************
printf '\n%s\n%s\n%s\n\n' "################################" "Get Nginx Setup for Nginx Test 3" "################################"
sudo cp $TRAVIS_BUILD_DIR/.dev-tools/default.vhost /etc/nginx/sites-available/default.vhost
sudo cp ${TRAVIS_BUILD_DIR}/.dev-tools/default.vhost /etc/nginx/sites-available/default.vhost
# **********************************************
# Link the vhost file into Nginx /sites-enabled/
@ -96,7 +96,7 @@ sudo ln -s /etc/nginx/sites-available/default.vhost /etc/nginx/sites-enabled/def
# Copy our index.php file into the default site's root folder
# ***********************************************************
sudo cp $TRAVIS_BUILD_DIR/.dev-tools/index.php /var/www/html/index.php
sudo cp ${TRAVIS_BUILD_DIR}/.dev-tools/index.php /var/www/html/index.php
printf '\n%s\n%s\n%s\n\n' "#####################################" "Finished Nginx Setup for Nginx Test 3" "#####################################"
# ***********************************************
@ -149,7 +149,7 @@ sudo nginx -c /etc/nginx/nginx.conf
# ****************************************************************************************
printf '\n%s\n%s\n%s\n\n' "####################################################" "Copy older globalblacklist.conf file to force update" "####################################################"
sudo cp $TRAVIS_BUILD_DIR/.dev-tools/globalblacklist-dummy.conf /etc/nginx/myconf.d/globalblacklist.conf
sudo cp ${TRAVIS_BUILD_DIR}/.dev-tools/globalblacklist-dummy.conf /etc/nginx/myconf.d/globalblacklist.conf
# ****************************************************************************************
# Run update-ngxblocker test which downloads latest globalblacklist.conf and reloads Nginx
@ -205,7 +205,7 @@ ls -la /etc/nginx/mybots.d/
# ****************************************************************************************
printf '\n%s\n%s\n%s\n\n' "####################################################" "Copy older globalblacklist.conf file to force update" "####################################################"
sudo cp $TRAVIS_BUILD_DIR/.dev-tools/globalblacklist-dummy.conf /etc/nginx/myconf.d/globalblacklist.conf
sudo cp ${TRAVIS_BUILD_DIR}/.dev-tools/globalblacklist-dummy.conf /etc/nginx/myconf.d/globalblacklist.conf
# *********************************************************************************************************
# Run update-ngxblocker to test for missing files and download latest globalblacklist.conf and reload Nginx
@ -235,7 +235,7 @@ sudo service nginx reload
# *******************************************************
printf '\n%s\n%s\n%s\n\n' "##############################################################################" "Copy latest generated globalblacklist.conf file to test for any compile errors" "##############################################################################"
sudo cp $TRAVIS_BUILD_DIR/conf.d/globalblacklist.conf /etc/nginx/myconf.d/globalblacklist.conf
sudo cp ${TRAVIS_BUILD_DIR}/conf.d/globalblacklist.conf /etc/nginx/myconf.d/globalblacklist.conf
# **************************
# Run setup-ngxblocker Again
@ -262,15 +262,21 @@ printf '\n%s\n%s\n%s\n\n' "######################" "Now Run our Curl Tests" "###
# Copy all .conf files used in Test 3 to a folder for checking
# ************************************************************
sudo chown -R travis:travis $TRAVIS_BUILD_DIR/
sudo chown -R travis:travis $TRAVIS_BUILD_DIR/.dev-tools/_conf_files_test3/
sudo cp /etc/nginx/mybots.d/* $TRAVIS_BUILD_DIR/.dev-tools/_conf_files_test3/bots.d/
sudo cp /etc/nginx/myconf.d/* $TRAVIS_BUILD_DIR/.dev-tools/_conf_files_test3/conf.d/
sudo chown -R travis:travis ${TRAVIS_BUILD_DIR}/
sudo chown -R travis:travis ${TRAVIS_BUILD_DIR}/.dev-tools/_conf_files_test3/
sudo cp /etc/nginx/mybots.d/* ${TRAVIS_BUILD_DIR}/.dev-tools/_conf_files_test3/bots.d/
sudo cp /etc/nginx/myconf.d/* ${TRAVIS_BUILD_DIR}/.dev-tools/_conf_files_test3/conf.d/
# *****************************************************************************************
# Travis now moves into running the rest of the tests in the script: section of .travis.yml
# *****************************************************************************************
# **********************
# Exit With Error Number
# **********************
exit ${?}
# MIT License
# Copyright (c) 2017 Mitchell Krog - mitchellkrog@gmail.com

35
.dev-tools/install-nginx-4.sh
View file

@ -54,7 +54,7 @@ sudo rm /etc/nginx/bots.d/*.conf
sudo rm /etc/nginx/sites-available/default
sudo rm /etc/nginx/nginx.conf
ls -la /etc/nginx/
sudo cp $TRAVIS_BUILD_DIR/.dev-tools/_nginx_conf_backup/nginx13.conf /etc/nginx/nginx.conf
sudo cp ${TRAVIS_BUILD_DIR}/.dev-tools/_nginx_conf_backup/nginx13.conf /etc/nginx/nginx.conf
ls -la /etc/nginx/
sudo apt-get purge nginx-full
sudo apt-get purge nginx-common
@ -65,7 +65,7 @@ sudo apt-get purge nginx*
# *****************************
mainstreamnginx=development
sudo add-apt-repository -y ppa:nginx/$mainstreamnginx
sudo add-apt-repository -y ppa:nginx/${mainstreamnginx}
sudo apt-get update
sudo apt-get install -y --assume-yes nginx-full
sudo nginx -V
@ -116,7 +116,7 @@ printf '\n%s\n\n' " "
# ********************************************************
printf '\n%s\n%s\n%s\n\n' "################################" "Get Nginx Setup for Nginx Test 4" "################################"
sudo cp $TRAVIS_BUILD_DIR/.dev-tools/default.vhost /etc/nginx/sites-available/default.vhost
sudo cp ${TRAVIS_BUILD_DIR}/.dev-tools/default.vhost /etc/nginx/sites-available/default.vhost
# **********************************************
# Link the vhost file into Nginx /sites-enabled/
@ -128,7 +128,7 @@ sudo ln -s /etc/nginx/sites-available/default.vhost /etc/nginx/sites-enabled/def
# Copy our index.php file into the default site's root folder
# ***********************************************************
sudo cp $TRAVIS_BUILD_DIR/.dev-tools/index.html /var/www/html/index.html
sudo cp ${TRAVIS_BUILD_DIR}/.dev-tools/index.html /var/www/html/index.html
printf '\n%s\n%s\n%s\n\n' "#####################################" "Finished Nginx Setup for Nginx Test 4" "#####################################"
# ***********************************************
@ -181,7 +181,7 @@ sudo nginx -c /etc/nginx/nginx.conf
# ****************************************************************************************
printf '\n%s\n%s\n%s\n\n' "####################################################" "Copy older globalblacklist.conf file to force update" "####################################################"
sudo cp $TRAVIS_BUILD_DIR/.dev-tools/globalblacklist-dummy.conf /etc/nginx/conf.d/globalblacklist.conf
sudo cp ${TRAVIS_BUILD_DIR}/.dev-tools/globalblacklist-dummy.conf /etc/nginx/conf.d/globalblacklist.conf
# ****************************************************************************************
# Run update-ngxblocker test which downloads latest globalblacklist.conf and reloads Nginx
@ -237,7 +237,7 @@ ls -la /etc/nginx/bots.d/
# ****************************************************************************************
printf '\n%s\n%s\n%s\n\n' "####################################################" "Copy older globalblacklist.conf file to force update" "####################################################"
sudo cp $TRAVIS_BUILD_DIR/.dev-tools/globalblacklist-dummy.conf /etc/nginx/conf.d/globalblacklist.conf
sudo cp ${TRAVIS_BUILD_DIR}/.dev-tools/globalblacklist-dummy.conf /etc/nginx/conf.d/globalblacklist.conf
# *********************************************************************************************************
# Run update-ngxblocker to test for missing files and download latest globalblacklist.conf and reload Nginx
@ -267,7 +267,7 @@ sudo service nginx reload
# *******************************************************
printf '\n%s\n%s\n%s\n\n' "##############################################################################" "Copy latest generated globalblacklist.conf file to test for any compile errors" "##############################################################################"
sudo cp $TRAVIS_BUILD_DIR/conf.d/globalblacklist.conf /etc/nginx/conf.d/globalblacklist.conf
sudo cp ${TRAVIS_BUILD_DIR}/conf.d/globalblacklist.conf /etc/nginx/conf.d/globalblacklist.conf
# **************************
# Run setup-ngxblocker Again
@ -284,7 +284,7 @@ sudo bash ./setup-ngxblocker -x
printf '\n%s\n%s\n%s\n\n' "########################" "Force Reloading of Nginx" "########################"
sudo service nginx reload
sudo cp $TRAVIS_BUILD_DIR/.dev-tools/default13.vhost /etc/nginx/sites-available/default.vhost
sudo cp ${TRAVIS_BUILD_DIR}/.dev-tools/default13.vhost /etc/nginx/sites-available/default.vhost
sudo service nginx reload
# **********************
@ -297,14 +297,14 @@ printf '\n%s\n%s\n%s\n\n' "######################" "Now Run our Curl Tests" "###
# Copy all .conf files used in Test 3 to a folder for checking
# ************************************************************
sudo chown -R travis:travis $TRAVIS_BUILD_DIR/
sudo chown -R travis:travis $TRAVIS_BUILD_DIR/.dev-tools/_conf_files_test4/
sudo cp /etc/nginx/bots.d/* $TRAVIS_BUILD_DIR/.dev-tools/_conf_files_test4/bots.d/
sudo cp /etc/nginx/conf.d/* $TRAVIS_BUILD_DIR/.dev-tools/_conf_files_test4/conf.d/
sudo cp /etc/nginx/sites-available/default.vhost $TRAVIS_BUILD_DIR/.dev-tools/_conf_files_test4/default.vhost
sudo chown -R travis:travis ${TRAVIS_BUILD_DIR}/
sudo chown -R travis:travis ${TRAVIS_BUILD_DIR}/.dev-tools/_conf_files_test4/
sudo cp /etc/nginx/bots.d/* ${TRAVIS_BUILD_DIR}/.dev-tools/_conf_files_test4/bots.d/
sudo cp /etc/nginx/conf.d/* ${TRAVIS_BUILD_DIR}/.dev-tools/_conf_files_test4/conf.d/
sudo cp /etc/nginx/sites-available/default.vhost ${TRAVIS_BUILD_DIR}/.dev-tools/_conf_files_test4/default.vhost
ls -la /etc/nginx/sites-available/
ls -la /var/www/
sudo chown -R travis:travis $TRAVIS_BUILD_DIR/
sudo chown -R travis:travis ${TRAVIS_BUILD_DIR}/
###
@ -312,6 +312,13 @@ sudo chown -R travis:travis $TRAVIS_BUILD_DIR/
# Travis now moves into running the rest of the tests in the script: section of .travis.yml
# *****************************************************************************************
# **********************
# Exit With Error Number
# **********************
exit ${?}
# MIT License
# Copyright (c) 2017 Mitchell Krog - mitchellkrog@gmail.com

44
.dev-tools/modify-config-readme-files.sh
View file

@ -45,9 +45,9 @@
YEAR=$(date +%Y)
MONTH=$(date +%m)
MY_GIT_TAG=V3.$YEAR.$MONTH.$TRAVIS_BUILD_NUMBER
BAD_REFERRERS=$(wc -l < $TRAVIS_BUILD_DIR/_generator_lists/bad-referrers.list)
BAD_BOTS=$(wc -l < $TRAVIS_BUILD_DIR/_generator_lists/bad-user-agents.list)
MY_GIT_TAG=V3.${YEAR}.${MONTH}.${TRAVIS_BUILD_NUMBER}
BAD_REFERRERS=$(wc -l < ${TRAVIS_BUILD_DIR}/_generator_lists/bad-referrers.list)
BAD_BOTS=$(wc -l < ${TRAVIS_BUILD_DIR}/_generator_lists/bad-user-agents.list)
# **********************************
# Temporary database files we create
@ -68,8 +68,8 @@ _endmarker="____________________"
# ****************************************
printf '%s\n%s%s\n%s%s\n%s%s\n%s' "$_startmarker" "#### Version: " "$MY_GIT_TAG" "#### Bad Referrer Count: " "$BAD_REFERRERS" "#### Bad Bot Count: " "$BAD_BOTS" "$_endmarker" >> "$_tmpnginxA"
mv $_tmpnginxA $_inputdbA
ed -s $_inputdbA<<\IN
mv ${_tmpnginxA} ${_inputdbA}
ed -s ${_inputdbA}<<\IN
1,/_______________/d
/____________________/,$d
,d
@ -82,15 +82,15 @@ ed -s $_inputdbA<<\IN
w /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/README.md
q
IN
rm $_inputdbA
rm ${_inputdbA}
# ****************************************************
# PRINT VERSION INFORMATION INTO AUTO-CONFIGURATION.md
# ****************************************************
printf '%s\n%s%s\n%s%s\n%s%s\n%s' "$_startmarker" "#### Version: " "$MY_GIT_TAG" "#### Bad Referrer Count: " "$BAD_REFERRERS" "#### Bad Bot Count: " "$BAD_BOTS" "$_endmarker" >> "$_tmpnginxA"
mv $_tmpnginxA $_inputdbA
ed -s $_inputdbA<<\IN
printf '%s\n%s%s\n%s%s\n%s%s\n%s' "$_startmarker" "#### Version: " "$MY_GIT_TAG" "#### Bad Referrer Count: " "$BAD_REFERRERS" "#### Bad Bot Count: " "$BAD_BOTS" "$_endmarker" >> "${_tmpnginxA}"
mv ${_tmpnginxA} ${_inputdbA}
ed -s ${_inputdbA}<<\IN
1,/_______________/d
/____________________/,$d
,d
@ -103,15 +103,15 @@ ed -s $_inputdbA<<\IN
w /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/AUTO-CONFIGURATION.md
q
IN
rm $_inputdbA
rm ${_inputdbA}
# ******************************************************
# PRINT VERSION INFORMATION INTO MANUAL-CONFIGURATION.md
# ******************************************************
printf '%s\n%s%s\n%s%s\n%s%s\n%s' "$_startmarker" "#### Version: " "$MY_GIT_TAG" "#### Bad Referrer Count: " "$BAD_REFERRERS" "#### Bad Bot Count: " "$BAD_BOTS" "$_endmarker" >> "$_tmpnginxA"
mv $_tmpnginxA $_inputdbA
ed -s $_inputdbA<<\IN
printf '%s\n%s%s\n%s%s\n%s%s\n%s' "$_startmarker" "#### Version: " "$MY_GIT_TAG" "#### Bad Referrer Count: " "$BAD_REFERRERS" "#### Bad Bot Count: " "$BAD_BOTS" "$_endmarker" >> "${_tmpnginxA}"
mv ${_tmpnginxA} ${_inputdbA}
ed -s ${_inputdbA}<<\IN
1,/_______________/d
/____________________/,$d
,d
@ -124,15 +124,15 @@ ed -s $_inputdbA<<\IN
w /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/MANUAL-CONFIGURATION.md
q
IN
rm $_inputdbA
rm ${_inputdbA}
# *******************************************************
# PRINT VERSION INFORMATION INTO Google Ghost Spam README
# *******************************************************
printf '%s\n%s%s\n%s%s\n%s%s\n%s' "$_startmarker" "#### Version: " "$MY_GIT_TAG" "#### Bad Referrer Count: " "$BAD_REFERRERS" "#### Bad Bot Count: " "$BAD_BOTS" "$_endmarker" >> "$_tmpnginxA"
mv $_tmpnginxA $_inputdbA
ed -s $_inputdbA<<\IN
printf '%s\n%s%s\n%s%s\n%s%s\n%s' "$_startmarker" "#### Version: " "$MY_GIT_TAG" "#### Bad Referrer Count: " "$BAD_REFERRERS" "#### Bad Bot Count: " "$BAD_BOTS" "$_endmarker" >> "${_tmpnginxA}"
mv ${_tmpnginxA} ${_inputdbA}
ed -s ${_inputdbA}<<\IN
1,/_______________/d
/____________________/,$d
,d
@ -145,10 +145,16 @@ ed -s $_inputdbA<<\IN
w /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/_google_analytics_ghost_spam/README.md
q
IN
rm $_inputdbA
rm ${_inputdbA}
# **********************
# Exit With Error Number
# **********************
exit ${?}
exit 0
# MIT License

34
.dev-tools/modify-files-and-commit.sh
View file

@ -45,7 +45,7 @@
YEAR=$(date +"%Y")
MONTH=$(date +"%m")
cd $TRAVIS_BUILD_DIR
cd ${TRAVIS_BUILD_DIR}
# *******************************
# Remove Remote Added by TravisCI
@ -77,39 +77,27 @@ git checkout master
# Gzip Our Latest Release So We can Include it the Travis Release
# ***************************************************************
cd $TRAVIS_BUILD_DIR/.latest_release/
tar -czf conf.d.tar.gz -C $TRAVIS_BUILD_DIR/conf.d/ .
tar -czf bots.d.tar.gz -C $TRAVIS_BUILD_DIR/bots.d/ .
cd ${TRAVIS_BUILD_DIR}/.latest_release/
tar -czf conf.d.tar.gz -C ${TRAVIS_BUILD_DIR}/conf.d/ .
tar -czf bots.d.tar.gz -C ${TRAVIS_BUILD_DIR}/bots.d/ .
# *************************************
# Add all the modified files and commit
# *************************************
git add -A
git commit -am "V3.$YEAR.$MONTH.$TRAVIS_BUILD_NUMBER [ci skip]"
# ***************************************************
# Try pushing changes to Google Ghost Spam Repository
# ***************************************************
#cd /tmp/
#sudo git clone https://github.com/mitchellkrogza/Stop.Google.Analytics.Ghost.Spam.HOWTO.git
#sudo cp $TRAVIS_BUILD_DIR/_generator_lists/bad-referrers.list /tmp/Stop.Google.Analytics.Ghost.Spam.HOWTO/.dev-tools/_input_source/bad-referrers.list
#cd /tmp/Stop.Google.Analytics.Ghost.Spam.HOWTO/
#ls -la
#sudo git remote -v
#sudo git remote rm origin
#sudo git remote add origin https://${GOOGLESPAM_TOKEN}@github.com/mitchellkrogza/Stop.Google.Analytics.Ghost.Spam.HOWTO.git
#sudo git remote -v
#sudo git add -A
#sudo git commit -am "Referrers (+)"
#sudo git push origin master
git commit -am "V3.${YEAR}.${MONTH}.${TRAVIS_BUILD_NUMBER} [ci skip]"
# *************************************************************
# Travis now moves to the before_deploy: section of .travis.yml
# *************************************************************
# **********************
# Exit With Error Number
# **********************
exit ${?}
# MIT License
# Copyright (c) 2017 Mitchell Krog - mitchellkrog@gmail.com

23
.dev-tools/prepare-robots-input.sh
View file

@ -20,28 +20,31 @@
# Specify Input and Output Files
# ******************************
_input=$TRAVIS_BUILD_DIR/_generator_lists/bad-user-agents.list
_output=$TRAVIS_BUILD_DIR/.dev-tools/_robots_input/robots-input.txt
_input=${TRAVIS_BUILD_DIR}/_generator_lists/bad-user-agents.list
_output=${TRAVIS_BUILD_DIR}/.dev-tools/_robots_input/robots-input.txt
# *****************
# Truncate our file
# *****************
sudo truncate -s 0 $_output
sudo truncate -s 0 ${_output}
# ********************************************
# Use sed to strip the \ out of the input file
# ********************************************
sed 's/\\ / /g' $_input > $_output
# ***************************************************
# Use cut to strip the domains out of the url strings
# ***************************************************
#cut -d'\' -f3 $_input > $_output
sed 's/\\ / /g' ${_input} > ${_output}
# *************************************
# Sort our output file and remove dupes
# *************************************
sort -u $_output -o $_output
sort -u ${_output} -o ${_output}
# **********************
# Exit With Error Number
# **********************
exit ${?}

87
.dev-tools/run-curl-tests-1.sh
View file

@ -20,14 +20,14 @@
# Set Location of our Curl Test Results Files
# *******************************************
_curltest1=$TRAVIS_BUILD_DIR/.dev-tools/_curl_tests/curltest1.txt
_curltest2=$TRAVIS_BUILD_DIR/.dev-tools/_curl_tests/curltest2.txt
_curltest3=$TRAVIS_BUILD_DIR/.dev-tools/_curl_tests/curltest3.txt
_curltest4=$TRAVIS_BUILD_DIR/.dev-tools/_curl_tests/curltest4.txt
_curltest5=$TRAVIS_BUILD_DIR/.dev-tools/_curl_tests/curltest5.txt
_curltest6=$TRAVIS_BUILD_DIR/.dev-tools/_curl_tests/curltest6.txt
_curltest7=$TRAVIS_BUILD_DIR/.dev-tools/_curl_tests/curltest7.txt
_curltest8=$TRAVIS_BUILD_DIR/.dev-tools/_curl_tests/curltest8.txt
_curltest1=${TRAVIS_BUILD_DIR}/.dev-tools/_curl_tests/curltest1.txt
_curltest2=${TRAVIS_BUILD_DIR}/.dev-tools/_curl_tests/curltest2.txt
_curltest3=${TRAVIS_BUILD_DIR}/.dev-tools/_curl_tests/curltest3.txt
_curltest4=${TRAVIS_BUILD_DIR}/.dev-tools/_curl_tests/curltest4.txt
_curltest5=${TRAVIS_BUILD_DIR}/.dev-tools/_curl_tests/curltest5.txt
_curltest6=${TRAVIS_BUILD_DIR}/.dev-tools/_curl_tests/curltest6.txt
_curltest7=${TRAVIS_BUILD_DIR}/.dev-tools/_curl_tests/curltest7.txt
_curltest8=${TRAVIS_BUILD_DIR}/.dev-tools/_curl_tests/curltest8.txt
_now="$(date)"
# *************************************************
@ -35,11 +35,11 @@ _now="$(date)"
# *************************************************
run_curltest1 () {
truncate -s 0 $_curltest1
truncate -s 0 ${_curltest1}
printf '\n%s\n%s\n%s\n\n' "#########################" "TESTING BAD BOT IS DENIED" "#########################"
printf '%s%s\n\n' "Last Tested: " "$_now" >> "$_curltest1"
curl -A "80legs" http://localhost:9000/index.php 2>> $_curltest1
if grep -i '(52)' $_curltest1; then
printf '%s%s\n\n' "Last Tested: " "$_now" >> "${_curltest1}"
curl -A "80legs" http://localhost:9000/index.php 2>> ${_curltest1}
if grep -i '(52)' ${_curltest1}; then
echo 'BAD BOT DETECTED - TEST PASSED'
else
echo 'BAD BOT NOT DETECTED - TEST FAILED'
@ -52,11 +52,11 @@ fi
# **************************************************
run_curltest2 () {
truncate -s 0 $_curltest2
truncate -s 0 ${_curltest2}
printf '\n%s\n%s\n%s\n\n' "#########################" "TESTING BAD BOT IS DENIED" "#########################"
printf '%s%s\n\n' "Last Tested: " "$_now" >> "$_curltest2"
curl -A "masscan" http://localhost:9000/index.php 2>> $_curltest2
if grep -i '(52)' $_curltest2; then
printf '%s%s\n\n' "Last Tested: " "$_now" >> "${_curltest2}"
curl -A "masscan" http://localhost:9000/index.php 2>> ${_curltest2}
if grep -i '(52)' ${_curltest2}; then
echo 'BAD BOT DETECTED - TEST PASSED'
else
echo 'BAD BOT NOT DETECTED - TEST FAILED'
@ -69,11 +69,11 @@ fi
# ******************************************************************
run_curltest3 () {
truncate -s 0 $_curltest3
truncate -s 0 ${_curltest3}
printf '\n%s\n%s\n%s\n\n' "##############################" "TESTING BAD REFERRER IS DENIED" "##############################"
printf '%s%s\n\n' "Last Tested: " "$_now" >> "$_curltest3"
curl -I http://localhost:9000/index.php -e http://100dollars-seo.com 2>> $_curltest3
if grep -i '(52)' $_curltest3; then
printf '%s%s\n\n' "Last Tested: " "$_now" >> "${_curltest3}"
curl -I http://localhost:9000/index.php -e http://100dollars-seo.com 2>> ${_curltest3}
if grep -i '(52)' ${_curltest3}; then
echo 'BAD REFERRER DETECTED - TEST PASSED'
else
echo 'BAD REFERRER NOT DETECTED - TEST FAILED'
@ -86,11 +86,11 @@ fi
# ******************************************************
run_curltest4 () {
truncate -s 0 $_curltest4
truncate -s 0 ${_curltest4}
printf '\n%s\n%s\n%s\n\n' "##############################" "TESTING BAD REFERRER IS DENIED" "##############################"
printf '%s%s\n\n' "Last Tested: " "$_now" >> "$_curltest4"
curl -I http://localhost:9000/index.php -e http://zx6.ru 2>> $_curltest4
if grep -i '(52)' $_curltest4; then
printf '%s%s\n\n' "Last Tested: " "$_now" >> "${_curltest4}"
curl -I http://localhost:9000/index.php -e http://zx6.ru 2>> ${_curltest4}
if grep -i '(52)' ${_curltest4}; then
echo 'BAD REFERRER DETECTED - TEST PASSED'
else
echo 'BAD REFERRER NOT DETECTED - TEST FAILED'
@ -103,11 +103,11 @@ fi
# *****************************************************
run_curltest5 () {
truncate -s 0 $_curltest5
truncate -s 0 ${_curltest5}
printf '\n%s\n%s\n%s\n\n' "###########################" "TESTING GOOD BOT IS ALLOWED" "###########################"
printf '%s%s\n\n' "Last Tested: " "$_now" >> "$_curltest5"
curl -v -A "GoogleBot" http://localhost:9000/index.php 2>&1 >> $_curltest5
if grep -i 'Welcome' $_curltest5; then
printf '%s%s\n\n' "Last Tested: " "$_now" >> "${_curltest5}"
curl -v -A "GoogleBot" http://localhost:9000/index.php 2>&1 >> ${_curltest5}
if grep -i 'Welcome' ${_curltest5}; then
echo 'GOOD BOT ALLOWED THROUGH - TEST PASSED'
else
echo 'GOOD BOT NOT ALLOWED THROUGH - TEST FAILED'
@ -120,11 +120,11 @@ fi
# ***************************************************
run_curltest6 () {
truncate -s 0 $_curltest6
truncate -s 0 ${_curltest6}
printf '\n%s\n%s\n%s\n\n' "###########################" "TESTING GOOD BOT IS ALLOWED" "###########################"
printf '%s%s\n\n' "Last Tested: " "$_now" >> "$_curltest6"
curl -v -A "BingBot" http://localhost:9000/index.php 2>&1 >> $_curltest6
if grep -i 'Welcome' $_curltest6; then
printf '%s%s\n\n' "Last Tested: " "$_now" >> "${_curltest6}"
curl -v -A "BingBot" http://localhost:9000/index.php 2>&1 >> ${_curltest6}
if grep -i 'Welcome' ${_curltest6}; then
echo 'GOOD BOT ALLOWED THROUGH - TEST PASSED'
else
echo 'GOOD BOT NOT ALLOWED THROUGH - TEST FAILED'
@ -137,11 +137,11 @@ fi
# ***********************************************************
run_curltest7 () {
truncate -s 0 $_curltest7
truncate -s 0 ${_curltest7}
printf '\n%s\n%s\n%s\n\n' "################################" "TESTING GOOD REFERRER IS ALLOWED" "################################"
printf '%s%s\n\n' "Last Tested: " "$_now" >> "$_curltest7"
curl http://localhost:9000/index.php -e http://google.com 2>&1 >> $_curltest7
if grep -i 'Welcome' $_curltest7; then
printf '%s%s\n\n' "Last Tested: " "$_now" >> "${_curltest7}"
curl http://localhost:9000/index.php -e http://google.com 2>&1 >> ${_curltest7}
if grep -i 'Welcome' ${_curltest7}; then
echo 'GOOD REFERRER DETECTED - TEST PASSED'
else
echo 'GOOD REFERRER NOT DETECTED - TEST FAILED'
@ -154,11 +154,11 @@ fi
# ***********************************************************
run_curltest8 () {
truncate -s 0 $_curltest8
truncate -s 0 ${_curltest8}
printf '\n%s\n%s\n%s\n\n' "################################" "TESTING GOOD REFERRER IS ALLOWED" "################################"
printf '%s%s\n\n' "Last Tested: " "$_now" >> "$_curltest8"
curl http://localhost:9000/index.php -e http://bing.com 2>&1 >> $_curltest8
if grep -i 'Welcome' $_curltest8; then
printf '%s%s\n\n' "Last Tested: " "$_now" >> "${_curltest8}"
curl http://localhost:9000/index.php -e http://bing.com 2>&1 >> ${_curltest8}
if grep -i 'Welcome' ${_curltest8}; then
echo 'GOOD REFERRER DETECTED - TEST PASSED'
exit 0
else
@ -180,3 +180,10 @@ run_curltest6
run_curltest7
run_curltest8
# **********************
# Exit With Error Number
# **********************
exit ${?}

86
.dev-tools/run-curl-tests-2.sh
View file

@ -20,14 +20,14 @@
# Set Location of our Curl Test Results Files
# *******************************************
_curltest1=$TRAVIS_BUILD_DIR/.dev-tools/_curl_tests_2/curltest1.txt
_curltest2=$TRAVIS_BUILD_DIR/.dev-tools/_curl_tests_2/curltest2.txt
_curltest3=$TRAVIS_BUILD_DIR/.dev-tools/_curl_tests_2/curltest3.txt
_curltest4=$TRAVIS_BUILD_DIR/.dev-tools/_curl_tests_2/curltest4.txt
_curltest5=$TRAVIS_BUILD_DIR/.dev-tools/_curl_tests_2/curltest5.txt
_curltest6=$TRAVIS_BUILD_DIR/.dev-tools/_curl_tests_2/curltest6.txt
_curltest7=$TRAVIS_BUILD_DIR/.dev-tools/_curl_tests_2/curltest7.txt
_curltest8=$TRAVIS_BUILD_DIR/.dev-tools/_curl_tests_2/curltest8.txt
_curltest1=${TRAVIS_BUILD_DIR}/.dev-tools/_curl_tests_2/curltest1.txt
_curltest2=${TRAVIS_BUILD_DIR}/.dev-tools/_curl_tests_2/curltest2.txt
_curltest3=${TRAVIS_BUILD_DIR}/.dev-tools/_curl_tests_2/curltest3.txt
_curltest4=${TRAVIS_BUILD_DIR}/.dev-tools/_curl_tests_2/curltest4.txt
_curltest5=${TRAVIS_BUILD_DIR}/.dev-tools/_curl_tests_2/curltest5.txt
_curltest6=${TRAVIS_BUILD_DIR}/.dev-tools/_curl_tests_2/curltest6.txt
_curltest7=${TRAVIS_BUILD_DIR}/.dev-tools/_curl_tests_2/curltest7.txt
_curltest8=${TRAVIS_BUILD_DIR}/.dev-tools/_curl_tests_2/curltest8.txt
_now="$(date)"
# *************************************************
@ -35,11 +35,11 @@ _now="$(date)"
# *************************************************
run_curltest1 () {
truncate -s 0 $_curltest1
truncate -s 0 ${_curltest1}
printf '\n%s\n%s\n%s\n\n' "#########################" "TESTING BAD BOT IS DENIED" "#########################"
printf '%s%s\n\n' "Last Tested: " "$_now" >> "$_curltest1"
curl -A "80legs" http://localhost:9000/index.php 2>> $_curltest1
if grep -i '(52)' $_curltest1; then
printf '%s%s\n\n' "Last Tested: " "$_now" >> "${_curltest1}"
curl -A "80legs" http://localhost:9000/index.php 2>> ${_curltest1}
if grep -i '(52)' ${_curltest1}; then
echo 'BAD BOT DETECTED - TEST PASSED'
else
echo 'BAD BOT NOT DETECTED - TEST FAILED'
@ -52,11 +52,11 @@ fi
# **************************************************
run_curltest2 () {
truncate -s 0 $_curltest2
truncate -s 0 ${_curltest2}
printf '\n%s\n%s\n%s\n\n' "#########################" "TESTING BAD BOT IS DENIED" "#########################"
printf '%s%s\n\n' "Last Tested: " "$_now" >> "$_curltest2"
curl -A "masscan" http://localhost:9000/index.php 2>> $_curltest2
if grep -i '(52)' $_curltest2; then
printf '%s%s\n\n' "Last Tested: " "$_now" >> "${_curltest2}"
curl -A "masscan" http://localhost:9000/index.php 2>> ${_curltest2}
if grep -i '(52)' ${_curltest2}; then
echo 'BAD BOT DETECTED - TEST PASSED'
else
echo 'BAD BOT NOT DETECTED - TEST FAILED'
@ -69,11 +69,11 @@ fi
# ******************************************************************
run_curltest3 () {
truncate -s 0 $_curltest3
truncate -s 0 ${_curltest3}
printf '\n%s\n%s\n%s\n\n' "##############################" "TESTING BAD REFERRER IS DENIED" "##############################"
printf '%s%s\n\n' "Last Tested: " "$_now" >> "$_curltest3"
curl -I http://localhost:9000/index.php -e http://100dollars-seo.com 2>> $_curltest3
if grep -i '(52)' $_curltest3; then
printf '%s%s\n\n' "Last Tested: " "$_now" >> "${_curltest3}"
curl -I http://localhost:9000/index.php -e http://100dollars-seo.com 2>> ${_curltest3}
if grep -i '(52)' ${_curltest3}; then
echo 'BAD REFERRER DETECTED - TEST PASSED'
else
echo 'BAD REFERRER NOT DETECTED - TEST FAILED'
@ -86,11 +86,11 @@ fi
# ******************************************************
run_curltest4 () {
truncate -s 0 $_curltest4
truncate -s 0 ${_curltest4}
printf '\n%s\n%s\n%s\n\n' "##############################" "TESTING BAD REFERRER IS DENIED" "##############################"
printf '%s%s\n\n' "Last Tested: " "$_now" >> "$_curltest4"
curl -I http://localhost:9000/index.php -e http://zx6.ru 2>> $_curltest4
if grep -i '(52)' $_curltest4; then
printf '%s%s\n\n' "Last Tested: " "$_now" >> "${_curltest4}"
curl -I http://localhost:9000/index.php -e http://zx6.ru 2>> ${_curltest4}
if grep -i '(52)' ${_curltest4}; then
echo 'BAD REFERRER DETECTED - TEST PASSED'
else
echo 'BAD REFERRER NOT DETECTED - TEST FAILED'
@ -103,11 +103,11 @@ fi
# *****************************************************
run_curltest5 () {
truncate -s 0 $_curltest5
truncate -s 0 ${_curltest5}
printf '\n%s\n%s\n%s\n\n' "###########################" "TESTING GOOD BOT IS ALLOWED" "###########################"
printf '%s%s\n\n' "Last Tested: " "$_now" >> "$_curltest5"
curl -v -A "GoogleBot" http://localhost:9000/index.php 2>&1 >> $_curltest5
if grep -i 'Welcome' $_curltest5; then
printf '%s%s\n\n' "Last Tested: " "$_now" >> "${_curltest5}"
curl -v -A "GoogleBot" http://localhost:9000/index.php 2>&1 >> ${_curltest5}
if grep -i 'Welcome' ${_curltest5}; then
echo 'GOOD BOT ALLOWED THROUGH - TEST PASSED'
else
echo 'GOOD BOT NOT ALLOWED THROUGH - TEST FAILED'
@ -120,11 +120,11 @@ fi
# ***************************************************
run_curltest6 () {
truncate -s 0 $_curltest6
truncate -s 0 ${_curltest6}
printf '\n%s\n%s\n%s\n\n' "###########################" "TESTING GOOD BOT IS ALLOWED" "###########################"
printf '%s%s\n\n' "Last Tested: " "$_now" >> "$_curltest6"
curl -v -A "BingBot" http://localhost:9000/index.php 2>&1 >> $_curltest6
if grep -i 'Welcome' $_curltest6; then
printf '%s%s\n\n' "Last Tested: " "$_now" >> "${_curltest6}"
curl -v -A "BingBot" http://localhost:9000/index.php 2>&1 >> ${_curltest6}
if grep -i 'Welcome' ${_curltest6}; then
echo 'GOOD BOT ALLOWED THROUGH - TEST PASSED'
else
echo 'GOOD BOT NOT ALLOWED THROUGH - TEST FAILED'
@ -137,11 +137,11 @@ fi
# ***********************************************************
run_curltest7 () {
truncate -s 0 $_curltest7
truncate -s 0 ${_curltest7}
printf '\n%s\n%s\n%s\n\n' "################################" "TESTING GOOD REFERRER IS ALLOWED" "################################"
printf '%s%s\n\n' "Last Tested: " "$_now" >> "$_curltest7"
curl http://localhost:9000/index.php -e http://google.com 2>&1 >> $_curltest7
if grep -i 'Welcome' $_curltest7; then
printf '%s%s\n\n' "Last Tested: " "$_now" >> "${_curltest7}"
curl http://localhost:9000/index.php -e http://google.com 2>&1 >> ${_curltest7}
if grep -i 'Welcome' ${_curltest7}; then
echo 'GOOD REFERRER DETECTED - TEST PASSED'
else
echo 'GOOD REFERRER NOT DETECTED - TEST FAILED'
@ -154,11 +154,11 @@ fi
# ***********************************************************
run_curltest8 () {
truncate -s 0 $_curltest8
truncate -s 0 ${_curltest8}
printf '\n%s\n%s\n%s\n\n' "################################" "TESTING GOOD REFERRER IS ALLOWED" "################################"
printf '%s%s\n\n' "Last Tested: " "$_now" >> "$_curltest8"
curl http://localhost:9000/index.php -e http://bing.com 2>&1 >> $_curltest8
if grep -i 'Welcome' $_curltest8; then
printf '%s%s\n\n' "Last Tested: " "$_now" >> "${_curltest8}"
curl http://localhost:9000/index.php -e http://bing.com 2>&1 >> ${_curltest8}
if grep -i 'Welcome' ${_curltest8}; then
echo 'GOOD REFERRER DETECTED - TEST PASSED'
exit 0
else
@ -180,3 +180,9 @@ run_curltest6
run_curltest7
run_curltest8
# **********************
# Exit With Error Number
# **********************
exit ${?}

86
.dev-tools/run-curl-tests-3.sh
View file

@ -20,14 +20,14 @@
# Set Location of our Curl Test Results Files
# *******************************************
_curltest1=$TRAVIS_BUILD_DIR/.dev-tools/_curl_tests_3/curltest1.txt
_curltest2=$TRAVIS_BUILD_DIR/.dev-tools/_curl_tests_3/curltest2.txt
_curltest3=$TRAVIS_BUILD_DIR/.dev-tools/_curl_tests_3/curltest3.txt
_curltest4=$TRAVIS_BUILD_DIR/.dev-tools/_curl_tests_3/curltest4.txt
_curltest5=$TRAVIS_BUILD_DIR/.dev-tools/_curl_tests_3/curltest5.txt
_curltest6=$TRAVIS_BUILD_DIR/.dev-tools/_curl_tests_3/curltest6.txt
_curltest7=$TRAVIS_BUILD_DIR/.dev-tools/_curl_tests_3/curltest7.txt
_curltest8=$TRAVIS_BUILD_DIR/.dev-tools/_curl_tests_3/curltest8.txt
_curltest1=${TRAVIS_BUILD_DIR}/.dev-tools/_curl_tests_3/curltest1.txt
_curltest2=${TRAVIS_BUILD_DIR}/.dev-tools/_curl_tests_3/curltest2.txt
_curltest3=${TRAVIS_BUILD_DIR}/.dev-tools/_curl_tests_3/curltest3.txt
_curltest4=${TRAVIS_BUILD_DIR}/.dev-tools/_curl_tests_3/curltest4.txt
_curltest5=${TRAVIS_BUILD_DIR}/.dev-tools/_curl_tests_3/curltest5.txt
_curltest6=${TRAVIS_BUILD_DIR}/.dev-tools/_curl_tests_3/curltest6.txt
_curltest7=${TRAVIS_BUILD_DIR}/.dev-tools/_curl_tests_3/curltest7.txt
_curltest8=${TRAVIS_BUILD_DIR}/.dev-tools/_curl_tests_3/curltest8.txt
_now="$(date)"
# *************************************************
@ -35,11 +35,11 @@ _now="$(date)"
# *************************************************
run_curltest1 () {
truncate -s 0 $_curltest1
truncate -s 0 ${_curltest1}
printf '\n%s\n%s\n%s\n\n' "#########################" "TESTING BAD BOT IS DENIED" "#########################"
printf '%s%s\n\n' "Last Tested: " "$_now" >> "$_curltest1"
curl -A "80legs" http://localhost:9000/index.php 2>> $_curltest1
if grep -i '(52)' $_curltest1; then
printf '%s%s\n\n' "Last Tested: " "$_now" >> "${_curltest1}"
curl -A "80legs" http://localhost:9000/index.php 2>> ${_curltest1}
if grep -i '(52)' ${_curltest1}; then
echo 'BAD BOT DETECTED - TEST PASSED'
else
echo 'BAD BOT NOT DETECTED - TEST FAILED'
@ -52,11 +52,11 @@ fi
# **************************************************
run_curltest2 () {
truncate -s 0 $_curltest2
truncate -s 0 ${_curltest2}
printf '\n%s\n%s\n%s\n\n' "#########################" "TESTING BAD BOT IS DENIED" "#########################"
printf '%s%s\n\n' "Last Tested: " "$_now" >> "$_curltest2"
curl -A "masscan" http://localhost:9000/index.php 2>> $_curltest2
if grep -i '(52)' $_curltest2; then
printf '%s%s\n\n' "Last Tested: " "$_now" >> "${_curltest2}"
curl -A "masscan" http://localhost:9000/index.php 2>> ${_curltest2}
if grep -i '(52)' ${_curltest2}; then
echo 'BAD BOT DETECTED - TEST PASSED'
else
echo 'BAD BOT NOT DETECTED - TEST FAILED'
@ -69,11 +69,11 @@ fi
# ******************************************************************
run_curltest3 () {
truncate -s 0 $_curltest3
truncate -s 0 ${_curltest3}
printf '\n%s\n%s\n%s\n\n' "##############################" "TESTING BAD REFERRER IS DENIED" "##############################"
printf '%s%s\n\n' "Last Tested: " "$_now" >> "$_curltest3"
curl -I http://localhost:9000/index.php -e http://100dollars-seo.com 2>> $_curltest3
if grep -i '(52)' $_curltest3; then
printf '%s%s\n\n' "Last Tested: " "$_now" >> "${_curltest3}"
curl -I http://localhost:9000/index.php -e http://100dollars-seo.com 2>> ${_curltest3}
if grep -i '(52)' ${_curltest3}; then
echo 'BAD REFERRER DETECTED - TEST PASSED'
else
echo 'BAD REFERRER NOT DETECTED - TEST FAILED'
@ -86,11 +86,11 @@ fi
# ******************************************************
run_curltest4 () {
truncate -s 0 $_curltest4
truncate -s 0 ${_curltest4}
printf '\n%s\n%s\n%s\n\n' "##############################" "TESTING BAD REFERRER IS DENIED" "##############################"
printf '%s%s\n\n' "Last Tested: " "$_now" >> "$_curltest4"
curl -I http://localhost:9000/index.php -e http://zx6.ru 2>> $_curltest4
if grep -i '(52)' $_curltest4; then
printf '%s%s\n\n' "Last Tested: " "$_now" >> "${_curltest4}"
curl -I http://localhost:9000/index.php -e http://zx6.ru 2>> ${_curltest4}
if grep -i '(52)' ${_curltest4}; then
echo 'BAD REFERRER DETECTED - TEST PASSED'
else
echo 'BAD REFERRER NOT DETECTED - TEST FAILED'
@ -103,11 +103,11 @@ fi
# *****************************************************
run_curltest5 () {
truncate -s 0 $_curltest5
truncate -s 0 ${_curltest5}
printf '\n%s\n%s\n%s\n\n' "###########################" "TESTING GOOD BOT IS ALLOWED" "###########################"
printf '%s%s\n\n' "Last Tested: " "$_now" >> "$_curltest5"
curl -v -A "GoogleBot" http://localhost:9000/index.php 2>&1 >> $_curltest5
if grep -i 'Welcome' $_curltest5; then
printf '%s%s\n\n' "Last Tested: " "$_now" >> "${_curltest5}"
curl -v -A "GoogleBot" http://localhost:9000/index.php 2>&1 >> ${_curltest5}
if grep -i 'Welcome' ${_curltest5}; then
echo 'GOOD BOT ALLOWED THROUGH - TEST PASSED'
else
echo 'GOOD BOT NOT ALLOWED THROUGH - TEST FAILED'
@ -120,11 +120,11 @@ fi
# ***************************************************
run_curltest6 () {
truncate -s 0 $_curltest6
truncate -s 0 ${_curltest6}
printf '\n%s\n%s\n%s\n\n' "###########################" "TESTING GOOD BOT IS ALLOWED" "###########################"
printf '%s%s\n\n' "Last Tested: " "$_now" >> "$_curltest6"
curl -v -A "BingBot" http://localhost:9000/index.php 2>&1 >> $_curltest6
if grep -i 'Welcome' $_curltest6; then
printf '%s%s\n\n' "Last Tested: " "$_now" >> "${_curltest6}"
curl -v -A "BingBot" http://localhost:9000/index.php 2>&1 >> ${_curltest6}
if grep -i 'Welcome' ${_curltest6}; then
echo 'GOOD BOT ALLOWED THROUGH - TEST PASSED'
else
echo 'GOOD BOT NOT ALLOWED THROUGH - TEST FAILED'
@ -137,11 +137,11 @@ fi
# ***********************************************************
run_curltest7 () {
truncate -s 0 $_curltest7
truncate -s 0 ${_curltest7}
printf '\n%s\n%s\n%s\n\n' "################################" "TESTING GOOD REFERRER IS ALLOWED" "################################"
printf '%s%s\n\n' "Last Tested: " "$_now" >> "$_curltest7"
curl http://localhost:9000/index.php -e http://google.com 2>&1 >> $_curltest7
if grep -i 'Welcome' $_curltest7; then
printf '%s%s\n\n' "Last Tested: " "$_now" >> "${_curltest7}"
curl http://localhost:9000/index.php -e http://google.com 2>&1 >> ${_curltest7}
if grep -i 'Welcome' ${_curltest7}; then
echo 'GOOD REFERRER DETECTED - TEST PASSED'
else
echo 'GOOD REFERRER NOT DETECTED - TEST FAILED'
@ -154,11 +154,11 @@ fi
# ***********************************************************
run_curltest8 () {
truncate -s 0 $_curltest8
truncate -s 0 ${_curltest8}
printf '\n%s\n%s\n%s\n\n' "################################" "TESTING GOOD REFERRER IS ALLOWED" "################################"
printf '%s%s\n\n' "Last Tested: " "$_now" >> "$_curltest8"
curl http://localhost:9000/index.php -e http://bing.com 2>&1 >> $_curltest8
if grep -i 'Welcome' $_curltest8; then
printf '%s%s\n\n' "Last Tested: " "$_now" >> "${_curltest8}"
curl http://localhost:9000/index.php -e http://bing.com 2>&1 >> ${_curltest8}
if grep -i 'Welcome' ${_curltest8}; then
echo 'GOOD REFERRER DETECTED - TEST PASSED'
exit 0
else
@ -180,4 +180,10 @@ run_curltest6
run_curltest7
run_curltest8
# **********************
# Exit With Error Number
# **********************
exit ${?}

86
.dev-tools/run-curl-tests-4.sh
View file

@ -20,14 +20,14 @@
# Set Location of our Curl Test Results Files
# *******************************************
_curltest1=$TRAVIS_BUILD_DIR/.dev-tools/_curl_tests_4/curltest1.txt
_curltest2=$TRAVIS_BUILD_DIR/.dev-tools/_curl_tests_4/curltest2.txt
_curltest3=$TRAVIS_BUILD_DIR/.dev-tools/_curl_tests_4/curltest3.txt
_curltest4=$TRAVIS_BUILD_DIR/.dev-tools/_curl_tests_4/curltest4.txt
_curltest5=$TRAVIS_BUILD_DIR/.dev-tools/_curl_tests_4/curltest5.txt
_curltest6=$TRAVIS_BUILD_DIR/.dev-tools/_curl_tests_4/curltest6.txt
_curltest7=$TRAVIS_BUILD_DIR/.dev-tools/_curl_tests_4/curltest7.txt
_curltest8=$TRAVIS_BUILD_DIR/.dev-tools/_curl_tests_4/curltest8.txt
_curltest1=${TRAVIS_BUILD_DIR}/.dev-tools/_curl_tests_4/curltest1.txt
_curltest2=${TRAVIS_BUILD_DIR}/.dev-tools/_curl_tests_4/curltest2.txt
_curltest3=${TRAVIS_BUILD_DIR}/.dev-tools/_curl_tests_4/curltest3.txt
_curltest4=${TRAVIS_BUILD_DIR}/.dev-tools/_curl_tests_4/curltest4.txt
_curltest5=${TRAVIS_BUILD_DIR}/.dev-tools/_curl_tests_4/curltest5.txt
_curltest6=${TRAVIS_BUILD_DIR}/.dev-tools/_curl_tests_4/curltest6.txt
_curltest7=${TRAVIS_BUILD_DIR}/.dev-tools/_curl_tests_4/curltest7.txt
_curltest8=${TRAVIS_BUILD_DIR}/.dev-tools/_curl_tests_4/curltest8.txt
_now="$(date)"
# *************************************************
@ -35,11 +35,11 @@ _now="$(date)"
# *************************************************
run_curltest1 () {
truncate -s 0 $_curltest1
truncate -s 0 ${_curltest1}
printf '\n%s\n%s\n%s\n\n' "#########################" "TESTING BAD BOT IS DENIED" "#########################"
printf '%s%s\n\n' "Last Tested: " "$_now" >> "$_curltest1"
curl -A "80legs" http://localhost:9000/index.html 2>> $_curltest1
if grep -i '(52)' $_curltest1; then
printf '%s%s\n\n' "Last Tested: " "$_now" >> "${_curltest1}"
curl -A "80legs" http://localhost:9000/index.html 2>> ${_curltest1}
if grep -i '(52)' ${_curltest1}; then
echo 'BAD BOT DETECTED - TEST PASSED'
else
echo 'BAD BOT NOT DETECTED - TEST FAILED'
@ -52,11 +52,11 @@ fi
# **************************************************
run_curltest2 () {
truncate -s 0 $_curltest2
truncate -s 0 ${_curltest2}
printf '\n%s\n%s\n%s\n\n' "#########################" "TESTING BAD BOT IS DENIED" "#########################"
printf '%s%s\n\n' "Last Tested: " "$_now" >> "$_curltest2"
curl -A "masscan" http://localhost:9000/index.html 2>> $_curltest2
if grep -i '(52)' $_curltest2; then
printf '%s%s\n\n' "Last Tested: " "$_now" >> "${_curltest2}"
curl -A "masscan" http://localhost:9000/index.html 2>> ${_curltest2}
if grep -i '(52)' ${_curltest2}; then
echo 'BAD BOT DETECTED - TEST PASSED'
else
echo 'BAD BOT NOT DETECTED - TEST FAILED'
@ -69,11 +69,11 @@ fi
# ******************************************************************
run_curltest3 () {
truncate -s 0 $_curltest3
truncate -s 0 ${_curltest3}
printf '\n%s\n%s\n%s\n\n' "##############################" "TESTING BAD REFERRER IS DENIED" "##############################"
printf '%s%s\n\n' "Last Tested: " "$_now" >> "$_curltest3"
curl -I http://localhost:9000/index.html -e http://100dollars-seo.com 2>> $_curltest3
if grep -i '(52)' $_curltest3; then
printf '%s%s\n\n' "Last Tested: " "$_now" >> "${_curltest3}"
curl -I http://localhost:9000/index.html -e http://100dollars-seo.com 2>> ${_curltest3}
if grep -i '(52)' ${_curltest3}; then
echo 'BAD REFERRER DETECTED - TEST PASSED'
else
echo 'BAD REFERRER NOT DETECTED - TEST FAILED'
@ -86,11 +86,11 @@ fi
# ******************************************************
run_curltest4 () {
truncate -s 0 $_curltest4
truncate -s 0 ${_curltest4}
printf '\n%s\n%s\n%s\n\n' "##############################" "TESTING BAD REFERRER IS DENIED" "##############################"
printf '%s%s\n\n' "Last Tested: " "$_now" >> "$_curltest4"
curl -I http://localhost:9000/index.html -e http://zx6.ru 2>> $_curltest4
if grep -i '(52)' $_curltest4; then
printf '%s%s\n\n' "Last Tested: " "$_now" >> "${_curltest4}"
curl -I http://localhost:9000/index.html -e http://zx6.ru 2>> ${_curltest4}
if grep -i '(52)' ${_curltest4}; then
echo 'BAD REFERRER DETECTED - TEST PASSED'
else
echo 'BAD REFERRER NOT DETECTED - TEST FAILED'
@ -103,11 +103,11 @@ fi
# *****************************************************
run_curltest5 () {
truncate -s 0 $_curltest5
truncate -s 0 ${_curltest5}
printf '\n%s\n%s\n%s\n\n' "###########################" "TESTING GOOD BOT IS ALLOWED" "###########################"
printf '%s%s\n\n' "Last Tested: " "$_now" >> "$_curltest5"
curl -v -A "GoogleBot" http://localhost:9000/index.html 2>&1 >> $_curltest5
if grep -i 'Welcome' $_curltest5; then
printf '%s%s\n\n' "Last Tested: " "$_now" >> "${_curltest5}"
curl -v -A "GoogleBot" http://localhost:9000/index.html 2>&1 >> ${_curltest5}
if grep -i 'Welcome' ${_curltest5}; then
echo 'GOOD BOT ALLOWED THROUGH - TEST PASSED'
else
echo 'GOOD BOT NOT ALLOWED THROUGH - TEST FAILED'
@ -120,11 +120,11 @@ fi
# ***************************************************
run_curltest6 () {
truncate -s 0 $_curltest6
truncate -s 0 ${_curltest6}
printf '\n%s\n%s\n%s\n\n' "###########################" "TESTING GOOD BOT IS ALLOWED" "###########################"
printf '%s%s\n\n' "Last Tested: " "$_now" >> "$_curltest6"
curl -v -A "BingBot" http://localhost:9000/index.html 2>&1 >> $_curltest6
if grep -i 'Welcome' $_curltest6; then
printf '%s%s\n\n' "Last Tested: " "$_now" >> "${_curltest6}"
curl -v -A "BingBot" http://localhost:9000/index.html 2>&1 >> ${_curltest6}
if grep -i 'Welcome' ${_curltest6}; then
echo 'GOOD BOT ALLOWED THROUGH - TEST PASSED'
else
echo 'GOOD BOT NOT ALLOWED THROUGH - TEST FAILED'
@ -137,11 +137,11 @@ fi
# ***********************************************************
run_curltest7 () {
truncate -s 0 $_curltest7
truncate -s 0 ${_curltest7}
printf '\n%s\n%s\n%s\n\n' "################################" "TESTING GOOD REFERRER IS ALLOWED" "################################"
printf '%s%s\n\n' "Last Tested: " "$_now" >> "$_curltest7"
curl http://localhost:9000/index.html -e http://google.com 2>&1 >> $_curltest7
if grep -i 'Welcome' $_curltest7; then
printf '%s%s\n\n' "Last Tested: " "$_now" >> "${_curltest7}"
curl http://localhost:9000/index.html -e http://google.com 2>&1 >> ${_curltest7}
if grep -i 'Welcome' ${_curltest7}; then
echo 'GOOD REFERRER DETECTED - TEST PASSED'
else
echo 'GOOD REFERRER NOT DETECTED - TEST FAILED'
@ -154,11 +154,11 @@ fi
# ***********************************************************
run_curltest8 () {
truncate -s 0 $_curltest8
truncate -s 0 ${_curltest8}
printf '\n%s\n%s\n%s\n\n' "################################" "TESTING GOOD REFERRER IS ALLOWED" "################################"
printf '%s%s\n\n' "Last Tested: " "$_now" >> "$_curltest8"
curl http://localhost:9000/index.html -e http://bing.com 2>&1 >> $_curltest8
if grep -i 'Welcome' $_curltest8; then
printf '%s%s\n\n' "Last Tested: " "$_now" >> "${_curltest8}"
curl http://localhost:9000/index.html -e http://bing.com 2>&1 >> ${_curltest8}
if grep -i 'Welcome' ${_curltest8}; then
echo 'GOOD REFERRER DETECTED - TEST PASSED'
exit 0
else
@ -180,3 +180,9 @@ run_curltest6
run_curltest7
run_curltest8
# **********************
# Exit With Error Number
# **********************
exit ${?}