vikarti.anatra/nginx-ultimate-bad-bot-blocker
1
0
Fork 0
mirror of https://github.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker.git synced 2025-09-04 19:50:17 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

First Test of TravisCI doing all generation of the bad blocker files using pull requests on _generator_lists

Browse source
This commit is contained in:
Mitchell Krog 2017-06-25 13:08:18 +02:00
parent e8a04af1ed
commit 1dfc535937
16 changed files with 7265 additions and 21 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

72
travisCI/deploy-package.sh
View file

@ -1,20 +1,86 @@
#!/bin/bash #!/bin/bash
# TravisCI Package Deploy Script for the Nginx Ultimate Bad Bot Blocker
# Created by: Mitchell Krog (mitchellkrog@gmail.com)
# Copyright: Mitchell Krog - https://github.com/mitchellkrogza
# Repo Url: https://github.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker
# MIT License
# Copyright (c) 2017 Mitchell Krog - mitchellkrog@gmail.com
# https://github.com/mitchellkrogza
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
# ******************
# Set Some Variables
# ******************
YEAR=$(date +"%Y") YEAR=$(date +"%Y")
MONTH=$(date +"%m") MONTH=$(date +"%m")
# ***************************************
# Make Sure we are in the Build Directory # Make Sure we are in the Build Directory
# ***************************************
cd $TRAVIS_BUILD_DIR cd $TRAVIS_BUILD_DIR
# *************************
# Create our Version Number # Create our Version Number
# *************************
export GIT_TAG=V3.$YEAR.$MONTH.$TRAVIS_BUILD_NUMBER export GIT_TAG=V3.$YEAR.$MONTH.$TRAVIS_BUILD_NUMBER
# ***************
# Tag our release # Tag our release
# ***************
git tag $GIT_TAG -a -m "V3.$YEAR.$MONTH.$TRAVIS_BUILD_NUMBER" git tag $GIT_TAG -a -m "V3.$YEAR.$MONTH.$TRAVIS_BUILD_NUMBER"
# *****************************************
# Push our commit and tags back to the repo # Push our commit and tags back to the repo
# *****************************************
sudo git push origin master && git push origin master --tags sudo git push origin master && git push origin master --tags
# Uncomment to list all git folders and modified files etc # *************************************************************************
#ls -aR
# Now TravisCI moves into the deploy: section of TravisCI - see .travis.yml # Now TravisCI moves into the deploy: section of TravisCI - see .travis.yml
# *************************************************************************
# MIT License
# Copyright (c) 2017 Mitchell Krog - mitchellkrog@gmail.com
# https://github.com/mitchellkrogza
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

472
travisCI/generate-blacklist.sh Executable file
View file

@ -0,0 +1,472 @@
#!/bin/bash
# Generator Script for the Nginx Ultimate Bad Bot Blocker
# Created by: Mitchell Krog (mitchellkrog@gmail.com)
# Copyright: Mitchell Krog - https://github.com/mitchellkrogza
# Repo Url: https://github.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker
# MIT License
# Copyright (c) 2017 Mitchell Krog - mitchellkrog@gmail.com
# https://github.com/mitchellkrogza
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
# ******************
# Set Some Variables
# ******************
YEAR=$(date +"%Y")
MONTH=$(date +"%m")
MY_GIT_TAG=V3.$YEAR.$MONTH.$TRAVIS_BUILD_NUMBER
BAD_REFERRERS=$(wc -l < $TRAVIS_BUILD_DIR/_generator_lists/bad-referrers.list)
BAD_BOTS=$(wc -l < $TRAVIS_BUILD_DIR/_generator_lists/bad-user-agents.list)
# *************************************
# Specify input lists for the generator
# *************************************
_input1=$TRAVIS_BUILD_DIR/_generator_lists/good-user-agents.list
_input2=$TRAVIS_BUILD_DIR/_generator_lists/allowed-user-agents.list
_input3=$TRAVIS_BUILD_DIR/_generator_lists/limited-user-agents.list
_input4=$TRAVIS_BUILD_DIR/_generator_lists/bad-user-agents.list
_input5=$TRAVIS_BUILD_DIR/_generator_lists/bad-referrers.list
_input6=$TRAVIS_BUILD_DIR/_generator_lists/google-ip-ranges.list
_input7=$TRAVIS_BUILD_DIR/_generator_lists/bing-ip-ranges.list
_input8=$TRAVIS_BUILD_DIR/_generator_lists/wordpress-theme-detectors.list
_input9=$TRAVIS_BUILD_DIR/_generator_lists/nibbler-seo.list
_input10=$TRAVIS_BUILD_DIR/_generator_lists/cloudflare-ip-ranges.list
# *******************************************************
# Declare temporary database files used during generation
# *******************************************************
_inputdbA=/tmp/version-information.db
_inputdb1=/tmp/good-user-agents.db
_inputdb2=/tmp/allowed-user-agents.db
_inputdb3=/tmp/limited-user-agents.db
_inputdb4=/tmp/bad-user-agents.db
_inputdb5=/tmp/bad-referers.db
_inputdb6=/tmp/google-ip-ranges.db
_inputdb7=/tmp/bing-ip-ranges.db
_inputdb8=/tmp/wordpress-theme-detectors.db
_inputdb9=/tmp/nibbler-seo.db
_inputdb10=/tmp/cloudflare-ip-ranges.db
# **************************************************
# Declare temporary variables used during generation
# **************************************************
_nginx=$TRAVIS_BUILD_DIR/travisCI/globalblacklist.template
_tmpnginxA=tmpnginxA
_tmpnginx1=tmpnginx1
_tmpnginx2=tmpnginx2
_tmpnginx3=tmpnginx3
_tmpnginx4=tmpnginx4
_tmpnginx5=tmpnginx5
_tmpnginx6=tmpnginx6
_tmpnginx7=tmpnginx7
_tmpnginx8=tmpnginx8
_tmpnginx9=tmpnginx9
_tmpnginx10=tmpnginx10
# *************************************************************
# Sort all input lists alphabetically and remove any duplicates
# *************************************************************
sort -u $_input1 -o $_input1
sort -u $_input2 -o $_input2
sort -u $_input3 -o $_input3
sort -u $_input4 -o $_input4
sort -u $_input5 -o $_input5
sort -u $_input6 -o $_input6
sort -u $_input7 -o $_input7
sort -u $_input8 -o $_input8
sort -u $_input9 -o $_input9
sort -u $_input10 -o $_input10
# ***************************************************************
# Start and End Strings to Search for to do inserts into template
# ***************************************************************
_start1="# START GOOD BOTS ### DO NOT EDIT THIS LINE AT ALL ###"
_end1="# END GOOD BOTS ### DO NOT EDIT THIS LINE AT ALL ###"
_start2="# START ALLOWED BOTS ### DO NOT EDIT THIS LINE AT ALL ###"
_end2="# END ALLOWED BOTS ### DO NOT EDIT THIS LINE AT ALL ###"
_start3="# START LIMITED BOTS ### DO NOT EDIT THIS LINE AT ALL ###"
_end3="# END LIMITED BOTS ### DO NOT EDIT THIS LINE AT ALL ###"
_start4="# START BAD BOTS ### DO NOT EDIT THIS LINE AT ALL ###"
_end4="# END BAD BOTS ### DO NOT EDIT THIS LINE AT ALL ###"
_start5="# START BAD REFERERS ### DO NOT EDIT THIS LINE AT ALL ###"
_end5="# END BAD REFERERS ### DO NOT EDIT THIS LINE AT ALL ###"
_start6="# START GOOGLE IP RANGES ### DO NOT EDIT THIS LINE AT ALL ###"
_end6="# END GOOGLE IP RANGES ### DO NOT EDIT THIS LINE AT ALL ###"
_start7="# START BING IP RANGES ### DO NOT EDIT THIS LINE AT ALL ###"
_end7="# END BING IP RANGES ### DO NOT EDIT THIS LINE AT ALL ###"
_start8="# START WP THEME DETECTORS ### DO NOT EDIT THIS LINE AT ALL ###"
_end8="# END WP THEME DETECTORS ### DO NOT EDIT THIS LINE AT ALL ###"
_start9="# START NIBBLER ### DO NOT EDIT THIS LINE AT ALL ###"
_end9="# END NIBBLER ### DO NOT EDIT THIS LINE AT ALL ###"
_start10="# START CLOUDFLARE IP RANGES ### DO NOT EDIT THIS LINE AT ALL ###"
_end10="# END CLOUDFLARE IP RANGES ### DO NOT EDIT THIS LINE AT ALL ###"
_startmarker="### Version Information #"
_endmarker="### Version Information ##"
# **********************************
# SET ALLOW OR DENY ACTION VARIABLES
# **********************************
ACTION1="0;"
ACTION2="1;"
ACTION3="2;"
ACTION4="3;"
# ************************************
# GOOD USER AGENTS - Create and Insert
# ************************************
GOODBOTSIFS=$IFS
IFS=$'\n'
echo $_start1 >> $_tmpnginx1
for line in $(cat $_input1); do
printf "\t\"~${line}\"\t\t$ACTION1\n" >> $_tmpnginx1
done
echo $_end1 >> $_tmpnginx1
IFS=$GOODBOTSIFS
mv $_tmpnginx1 $_inputdb1
ed -s $_inputdb1<<\IN
1,/# START GOOD BOTS ### DO NOT EDIT THIS LINE AT ALL ###/d
/# END GOOD BOTS ### DO NOT EDIT THIS LINE AT ALL ###/,$d
,d
.r /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/travisCI/globalblacklist.template
/# START GOOD BOTS ### DO NOT EDIT THIS LINE AT ALL ###/x
.t.
.,/# END GOOD BOTS ### DO NOT EDIT THIS LINE AT ALL ###/-d
#,p
#,p used to print output replaced with w below to write
w /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/travisCI/globalblacklist.template
q
IN
rm $_inputdb1
# ********************************
# ALLOWED BOTS - Create and Insert
# ********************************
ALLOWEDBOTSIFS=$IFS
IFS=$'\n'
echo $_start2 >> $_tmpnginx2
for line in $(cat $_input2); do
printf "\t\"~${line}\"\t\t$ACTION2\n" >> $_tmpnginx2
done
echo $_end2 >> $_tmpnginx2
IFS=$ALLOWEDBOTSIFS
mv $_tmpnginx2 $_inputdb2
ed -s $_inputdb2<<\IN
1,/# START ALLOWED BOTS ### DO NOT EDIT THIS LINE AT ALL ###/d
/# END ALLOWED BOTS ### DO NOT EDIT THIS LINE AT ALL ###/,$d
,d
.r /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/travisCI/globalblacklist.template
/# START ALLOWED BOTS ### DO NOT EDIT THIS LINE AT ALL ###/x
.t.
.,/# END ALLOWED BOTS ### DO NOT EDIT THIS LINE AT ALL ###/-d
w /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/travisCI/globalblacklist.template
q
IN
rm $_inputdb2
# ********************************
# LIMITED BOTS - Create and Insert
# ********************************
LIMITEDBOTSIFS=$IFS
IFS=$'\n'
echo $_start3 >> $_tmpnginx3
for line in $(cat $_input3); do
printf "\t\"~${line}\"\t\t$ACTION3\n" >> $_tmpnginx3
done
echo $_end3 >> $_tmpnginx3
IFS=$LIMITEDBOTSIFS
mv $_tmpnginx3 $_inputdb3
ed -s $_inputdb3<<\IN
1,/# START LIMITED BOTS ### DO NOT EDIT THIS LINE AT ALL ###/d
/# END LIMITED BOTS ### DO NOT EDIT THIS LINE AT ALL ###/,$d
,d
.r /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/travisCI/globalblacklist.template
/# START LIMITED BOTS ### DO NOT EDIT THIS LINE AT ALL ###/x
.t.
.,/# END LIMITED BOTS ### DO NOT EDIT THIS LINE AT ALL ###/-d
w /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/travisCI/globalblacklist.template
q
IN
rm $_inputdb3
# ****************************
# BAD BOTS - Create and Insert
# ****************************
BADBOTSIFS=$IFS
IFS=$'\n'
echo $_start4 >> $_tmpnginx4
for line in $(cat $_input4); do
printf "\t\"~*${line}\"\t\t$ACTION4\n" >> $_tmpnginx4
done
echo $_end4 >> $_tmpnginx4
IFS=$BADBOTSIFS
mv $_tmpnginx4 $_inputdb4
ed -s $_inputdb4<<\IN
1,/# START BAD BOTS ### DO NOT EDIT THIS LINE AT ALL ###/d
/# END BAD BOTS ### DO NOT EDIT THIS LINE AT ALL ###/,$d
,d
.r /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/travisCI/globalblacklist.template
/# START BAD BOTS ### DO NOT EDIT THIS LINE AT ALL ###/x
.t.
.,/# END BAD BOTS ### DO NOT EDIT THIS LINE AT ALL ###/-d
w /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/travisCI/globalblacklist.template
q
IN
rm $_inputdb4
# ********************************
# BAD REFERERS - Create and Insert
# ********************************
BADREFERER=$IFS
IFS=$'\n'
echo $_start5 >> $_tmpnginx5
for line in $(cat $_input5); do
printf "\t\"~*${line}\"\t\t$ACTION2\n" >> $_tmpnginx5
done
echo $_end5 >> $_tmpnginx5
IFS=$BADREFERER
mv $_tmpnginx5 $_inputdb5
ed -s $_inputdb5<<\IN
1,/# START BAD REFERERS ### DO NOT EDIT THIS LINE AT ALL ###/d
/# END BAD REFERERS ### DO NOT EDIT THIS LINE AT ALL ###/,$d
,d
.r /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/travisCI/globalblacklist.template
/# START BAD REFERERS ### DO NOT EDIT THIS LINE AT ALL ###/x
.t.
.,/# END BAD REFERERS ### DO NOT EDIT THIS LINE AT ALL ###/-d
w /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/travisCI/globalblacklist.template
q
IN
rm $_inputdb5
# ************************************
# GOOGLE IP RANGES - Create and Insert
# ************************************
GOOGLE=$IFS
IFS=$'\n'
echo $_start6 >> $_tmpnginx6
for line in $(cat $_input6); do
printf "\t${line}\t\t$ACTION1\n" >> $_tmpnginx6
done
echo $_end6 >> $_tmpnginx6
IFS=$GOOGLE
mv $_tmpnginx6 $_inputdb6
ed -s $_inputdb6<<\IN
1,/# START GOOGLE IP RANGES ### DO NOT EDIT THIS LINE AT ALL ###/d
/# END GOOGLE IP RANGES ### DO NOT EDIT THIS LINE AT ALL ###/,$d
,d
.r /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/travisCI/globalblacklist.template
/# START GOOGLE IP RANGES ### DO NOT EDIT THIS LINE AT ALL ###/x
.t.
.,/# END GOOGLE IP RANGES ### DO NOT EDIT THIS LINE AT ALL ###/-d
w /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/travisCI/globalblacklist.template
q
IN
rm $_inputdb6
# **********************************
# BING IP RANGES - Create and Insert
# **********************************
BING=$IFS
IFS=$'\n'
echo $_start7 >> $_tmpnginx7
for line in $(cat $_input7); do
printf "\t${line}\t\t$ACTION1\n" >> $_tmpnginx7
done
echo $_end7 >> $_tmpnginx7
IFS=$BING
mv $_tmpnginx7 $_inputdb7
ed -s $_inputdb7<<\IN
1,/# START BING IP RANGES ### DO NOT EDIT THIS LINE AT ALL ###/d
/# END BING IP RANGES ### DO NOT EDIT THIS LINE AT ALL ###/,$d
,d
.r /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/travisCI/globalblacklist.template
/# START BING IP RANGES ### DO NOT EDIT THIS LINE AT ALL ###/x
.t.
.,/# END BING IP RANGES ### DO NOT EDIT THIS LINE AT ALL ###/-d
w /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/travisCI/globalblacklist.template
q
IN
rm $_inputdb7
# *********************************************
# Wordpress Theme Detectors - Create and Insert
# *********************************************
WPTHEME=$IFS
IFS=$'\n'
echo $_start8 >> $_tmpnginx8
for line in $(cat $_input8); do
printf "\t${line}\n" >> $_tmpnginx8
done
echo $_end8 >> $_tmpnginx8
IFS=$WPTHEME
mv $_tmpnginx8 $_inputdb8
ed -s $_inputdb8<<\IN
1,/# START WP THEME DETECTORS ### DO NOT EDIT THIS LINE AT ALL ###/d
/# END WP THEME DETECTORS ### DO NOT EDIT THIS LINE AT ALL ###/,$d
,d
.r /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/travisCI/globalblacklist.template
/# START WP THEME DETECTORS ### DO NOT EDIT THIS LINE AT ALL ###/x
.t.
.,/# END WP THEME DETECTORS ### DO NOT EDIT THIS LINE AT ALL ###/-d
w /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/travisCI/globalblacklist.template
q
IN
rm $_inputdb8
# *******************************
# Nibbler SEO - Create and Insert
# *******************************
NIBBLER=$IFS
IFS=$'\n'
echo $_start9 >> $_tmpnginx9
for line in $(cat $_input9); do
printf "\t${line}\t\t$ACTION2\n" >> $_tmpnginx9
done
echo $_end9 >> $_tmpnginx9
IFS=$NIBBLER
mv $_tmpnginx9 $_inputdb9
ed -s $_inputdb9<<\IN
1,/# START NIBBLER ### DO NOT EDIT THIS LINE AT ALL ###/d
/# END NIBBLER ### DO NOT EDIT THIS LINE AT ALL ###/,$d
,d
.r /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/travisCI/globalblacklist.template
/# START NIBBLER ### DO NOT EDIT THIS LINE AT ALL ###/x
.t.
.,/# END NIBBLER ### DO NOT EDIT THIS LINE AT ALL ###/-d
w /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/travisCI/globalblacklist.template
q
IN
rm $_inputdb9
# ****************************************
# CLOUDFLARE IP RANGES - Create and Insert
# ****************************************
CLOUDFLARE=$IFS
IFS=$'\n'
echo $_start10 >> $_tmpnginx10
for line in $(cat $_input10); do
printf "\t${line}\t\t$ACTION1\n" >> $_tmpnginx10
done
echo $_end10 >> $_tmpnginx10
IFS=$CLOUDFLARE
mv $_tmpnginx10 $_inputdb10
ed -s $_inputdb10<<\IN
1,/# START CLOUDFLARE IP RANGES ### DO NOT EDIT THIS LINE AT ALL ###/d
/# END CLOUDFLARE IP RANGES ### DO NOT EDIT THIS LINE AT ALL ###/,$d
,d
.r /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/travisCI/globalblacklist.template
/# START CLOUDFLARE IP RANGES ### DO NOT EDIT THIS LINE AT ALL ###/x
.t.
.,/# END CLOUDFLARE IP RANGES ### DO NOT EDIT THIS LINE AT ALL ###/-d
w /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/travisCI/globalblacklist.template
q
IN
rm $_inputdb10
# *******************************************************************************
# PRINT VERSION, SCRIPT RUNTIME and UPDATE INFORMATION INTO GLOBALBLACKLIST FILES
# *******************************************************************************
LASTUPDATEIFS=$IFS
IFS=$'\n'
now="$(date)"
end=$(date +%s.%N)
echo $_startmarker >> $_tmpnginxA
printf "###################################################\n### Version: "$MY_GIT_TAG"\n### Updated: "$now"\n### Bad Referrer Count: "$BAD_REFERRERS"\n### Bad Bot Count: "$BAD_BOTS"\n###################################################\n" >> $_tmpnginxA
echo $_endmarker >> $_tmpnginxA
IFS=$LASTUPDATEIFS
mv $_tmpnginxA $_inputdbA
ed -s $_inputdbA<<\IN
1,/### Version Information #/d
/### Version Information ##/,$d
,d
.r /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/travisCI/globalblacklist.template
/### Version Information #/x
.t.
.,/### Version Information ##/-d
#,p
#,p used to print output replaced with w below to write
w /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/travisCI/globalblacklist.template
q
IN
rm $_inputdbA
# **************************************************
# Generate Additional Files and Copy Them to Folders
# **************************************************
sudo cp $_input1 $TRAVIS_BUILD_DIR/_generator_lists/_generator_lists/good-user-agents.list
sudo cp $_input2 $TRAVIS_BUILD_DIR/_generator_lists/_generator_lists/allowed-user-agents.list
sudo cp $_input3 $TRAVIS_BUILD_DIR/_generator_lists/_generator_lists/limited-user-agents.list
sudo cp $_input4 $TRAVIS_BUILD_DIR/_generator_lists/_generator_lists/bad-user-agents.list
sudo cp $_input5 $TRAVIS_BUILD_DIR/_generator_lists/_generator_lists/bad-referrers.list
sudo cp $_input6 $TRAVIS_BUILD_DIR/_generator_lists/_generator_lists/google-ip-ranges.list
sudo cp $_input7 $TRAVIS_BUILD_DIR/_generator_lists/_generator_lists/bing-ip-ranges.list
sudo cp $_input8 $TRAVIS_BUILD_DIR/_generator_lists/_generator_lists/wordpress-theme-detectors.list
sudo cp $_input9 $TRAVIS_BUILD_DIR/_generator_lists/_generator_lists/nibbler-seo.list
sudo cp $_input10 $TRAVIS_BUILD_DIR/_generator_lists/_generator_lists/cloudflare-ip-ranges.list
sudo cp $_nginx $TRAVIS_BUILD_DIR/conf.d/globalblacklist.conf
sudo cp $_nginx $TRAVIS_BUILD_DIR/_sample_config_files/Engintron_for_cPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf
exit 0
# MIT License
# Copyright (c) 2017 Mitchell Krog - mitchellkrog@gmail.com
# https://github.com/mitchellkrogza
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

68
travisCI/generate-google-disavow.sh Executable file
View file

@ -0,0 +1,68 @@
#!/bin/bash
# Google Disavow file generator for Nginx Ultimate Bad Bot Blocker
# Created by: Mitchell Krog (mitchellkrog@gmail.com)
# Copyright: Mitchell Krog - https://github.com/mitchellkrogza
# Repo Url: https://github.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker
# MIT License
# Copyright (c) 2017 Mitchell Krog - mitchellkrog@gmail.com
# https://github.com/mitchellkrogza
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
# ************************************
# Specify input list for the generator
# ************************************
_input1=$TRAVIS_BUILD_DIR/_generator_lists/bad-referrers.list
# **************************
# Create Google Disavow File
# **************************
sudo truncate -s 0 $TRAVIS_BUILD_DIR/google-disavow.txt
for line in $(cat $_input1); do
printf "domain:${line}\n" >> $TRAVIS_BUILD_DIR/google-disavow.txt
done
exit 0
# MIT License
# Copyright (c) 2017 Mitchell Krog - mitchellkrog@gmail.com
# https://github.com/mitchellkrogza
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

98
travisCI/generate-google-exclude.php Executable file
View file

@ -0,0 +1,98 @@
<?php
/* Google Exclude File Generator Script for the Nginx Ultimate Bad Bot Blocker
* Created by: Mitchell Krog (mitchellkrog@gmail.com)
* Copyright: Mitchell Krog - https://github.com/mitchellkrogza
* Repo Url: https://github.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker
* MIT License
* Copyright (c) 2017 Mitchell Krog - mitchellkrog@gmail.com
*/
class Generate
{
private $projectUrl = "https://github.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker";
public function generateFiles()
{
date_default_timezone_set('Africa/Johannesburg');
$date = date('Y-m-d H:i:s');
$lines = $this->domainWorker();
$this->createGoogleExclude($lines);
}
/**
* @return array
*/
public function domainWorker()
{
$domainsFile = "/home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/_generator_lists/bad-referrers.list";
$handle = fopen($domainsFile, "r");
if (!$handle) {
throw new \RuntimeException('Error opening file ' . $domainsFile);
}
$lines = array();
while (($line = fgets($handle)) !== false) {
$line = trim(preg_replace('/\s\s+/', ' ', $line));
// convert russian domains
if (preg_match('/[А-Яа-яЁё]/u', $line)) {
$IDN = new IdnaConvert();
$line = $IDN->encode($line);
}
if (empty($line)) {
continue;
}
$lines[] = $line;
}
fclose($handle);
$uniqueLines = array_unique($lines, SORT_STRING);
sort($uniqueLines, SORT_STRING);
if (is_writable($domainsFile)) {
file_put_contents($domainsFile, implode("\n", $uniqueLines));
} else {
trigger_error("Permission denied");
}
return $lines;
}
/**
* @param $file
* @param $data
*/
protected function writeToFile($file, $data)
{
if (is_writable($file)) {
file_put_contents($file, $data);
if (!chmod($file, 0755)) {
trigger_error("Couldn't not set " . basename($file) . " permissions to 755");
}
} else {
trigger_error("Permission denied");
}
}
public function createGoogleExclude(array $lines)
{
$regexLines = [];
foreach ($lines as $line) {
$regexLines[] = preg_quote($line);
}
$data = implode('|', $regexLines);
$googleLimit = 30000;
$dataLength = strlen($data);
// keep track of the last split
$lastPosition = 0;
for ($x = 1; $lastPosition < $dataLength; $x++) {
// already in the boundary limits?
if( ($dataLength-$lastPosition) >= $googleLimit){
// search for the last occurence of | in the boundary limits
$pipePosition = strrpos(substr($data, $lastPosition, $googleLimit), '|');
$dataSplit = substr($data, $lastPosition, $pipePosition);
// without trailing pipe at the beginning of next round
$lastPosition = $lastPosition + $pipePosition+1;
}else{
// Rest of the regex (no pipe at the end)
$dataSplit = substr($data, $lastPosition);
$lastPosition = $dataLength; // Break
}
$file = '/home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/google-exclude-0' . $x . '.txt';
$this->writeToFile($file, $dataSplit);
}
}
}
$generator = new Generate();
$generator->generateFiles();

99
travisCI/generate-robots.sh Executable file
View file

@ -0,0 +1,99 @@
#!/bin/bash
# Generator Script for the Nginx Ultimate Bad Bot Blocker
# Created by: Mitchell Krog (mitchellkrog@gmail.com)
# Copyright: Mitchell Krog - https://github.com/mitchellkrogza
# Repo Url: https://github.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker
# Generate a robots.txt file for those unable to use the full Nginx Ultimate Bad Bot Blocker
# MIT License
# Copyright (c) 2017 Mitchell Krog - mitchellkrog@gmail.com
# https://github.com/mitchellkrogza
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
# ***************
# Set Input Files
# ***************
_input4a=$TRAVIS_BUILD_DIR/_generator_lists/bad-user-agents.list
_robotsinput1=/tmp/robotsinput1.txt
_robotsinput1b=/tmp/robotsinput1b.txt
_tmprobots=/tmp/robots.txt
# ******************
# Set Some Variables
# ******************
YEAR=$(date +"%Y")
MONTH=$(date +"%m")
MY_GIT_TAG=V3.$YEAR.$MONTH.$TRAVIS_BUILD_NUMBER
BAD_REFERRERS=$(wc -l < $TRAVIS_BUILD_DIR/_generator_lists/bad-referrers.list)
BAD_BOTS=$(wc -l < $TRAVIS_BUILD_DIR/_generator_lists/bad-user-agents.list)
now="$(date)"
# *************************
# Set Start and End Markers
# *************************
_startmarker="### Version Information #"
_endmarker="### Version Information ##"
# **************************
# Create the robots.txt file
# **************************
cp $_input4a $_robotsinput1
sed 's/[\]//g' $_robotsinput1 > $_robotsinput1b
IFS=''
echo $_startmarker >> $_tmprobots
printf "###################################################\n### Version: "$MY_GIT_TAG"\n### Updated: "$now"\n### Bad Referrer Count: "$BAD_REFERRERS"\n### Bad Bot Count: "$BAD_BOTS"\n###################################################\n" >> $_tmprobots
echo $_endmarker >> $_tmprobots
printf "\n\n" >> $_tmprobots
cat $_robotsinput1b |
while read line; do
printf 'User-agent: '${line}'\n Disallow:/ \n' >> $_tmprobots
done
sudo cp $_tmprobots $TRAVIS_BUILD_DIR/robots.txt/robots.txt
exit 0
# MIT License
# Copyright (c) 2017 Mitchell Krog - mitchellkrog@gmail.com
# https://github.com/mitchellkrogza
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

5983
travisCI/globalblacklist.template Executable file
View file

File diff suppressed because it is too large Load diff

89
travisCI/install-nginx.sh
View file

@ -1,67 +1,144 @@
#!/bin/bash #!/bin/bash
# Travis CI Testing for Nginx Ultimate Bad Bot Blocker # Travis CI Generating and Building for the Nginx Ultimate Bad Bot Blocker
# Created by: Mitchell Krog (mitchellkrog@gmail.com)
# Copyright: Mitchell Krog - https://github.com/mitchellkrogza
# Repo Url: https://github.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker
# MIT License
# Copyright (c) 2017 Mitchell Krog - mitchellkrog@gmail.com
# https://github.com/mitchellkrogza # https://github.com/mitchellkrogza
set -x # Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
# ***************************************************************
# Start Getting Nginx Ready for Testing the Nginx Bad Bot Blocker # Start Getting Nginx Ready for Testing the Nginx Bad Bot Blocker
# ***************************************************************
# *************************************************
# Delete default site created by Nginx Installation # Delete default site created by Nginx Installation
# *************************************************
sudo rm /etc/nginx/sites-available/default sudo rm /etc/nginx/sites-available/default
# ***********************************************************************
# Download the Nginx Bad Bot Blocker setup files from the Live Repository # Download the Nginx Bad Bot Blocker setup files from the Live Repository
# ***********************************************************************
sudo wget https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/install-ngxblocker -O /usr/sbin/install-ngxblocker sudo wget https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/install-ngxblocker -O /usr/sbin/install-ngxblocker
sudo wget https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/setup-ngxblocker -O /usr/sbin/setup-ngxblocker sudo wget https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/setup-ngxblocker -O /usr/sbin/setup-ngxblocker
sudo wget https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/update-ngxblocker -O /usr/sbin/update-ngxblocker sudo wget https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/update-ngxblocker -O /usr/sbin/update-ngxblocker
# **************************************************
# Set our install and setup scripts to be executable # Set our install and setup scripts to be executable
# **************************************************
sudo chmod +x /usr/sbin/install-ngxblocker sudo chmod +x /usr/sbin/install-ngxblocker
sudo chmod +x /usr/sbin/setup-ngxblocker sudo chmod +x /usr/sbin/setup-ngxblocker
sudo chmod +x /usr/sbin/update-ngxblocker sudo chmod +x /usr/sbin/update-ngxblocker
# **********************
# Run Install-NgxBlocker # Run Install-NgxBlocker
# **********************
cd /usr/sbin cd /usr/sbin
sudo ./install-ngxblocker -x sudo ./install-ngxblocker -x
# ********************************************************
# Copy our default.vhost file into Nginx /sites-available/ # Copy our default.vhost file into Nginx /sites-available/
# ********************************************************
sudo cp $TRAVIS_BUILD_DIR/travisCI/default.vhost /etc/nginx/sites-available/default.vhost sudo cp $TRAVIS_BUILD_DIR/travisCI/default.vhost /etc/nginx/sites-available/default.vhost
# Link the vhost file into Nginx /sites-enabled/ and reload nginx # **********************************************
# Link the vhost file into Nginx /sites-enabled/
# **********************************************
sudo ln -s /etc/nginx/sites-available/default.vhost /etc/nginx/sites-enabled/default.vhost sudo ln -s /etc/nginx/sites-available/default.vhost /etc/nginx/sites-enabled/default.vhost
# ********************
# Run setup-ngxblocker # Run setup-ngxblocker
# ********************
cd /usr/sbin cd /usr/sbin
sudo ./setup-ngxblocker -x sudo ./setup-ngxblocker -x
# ******************************************************************************************************
# NOTE: for Verbose Testing of any shell scripts use below format adding sh -x before running the script # NOTE: for Verbose Testing of any shell scripts use below format adding sh -x before running the script
# this helps a lot inside the TravisCI environment to see where a shell script may be failing # this helps a lot inside the TravisCI environment to see where a shell script may be failing
# sudo sh -x ./setup-ngxblocker -x # sudo sh -x ./setup-ngxblocker -x
# *******************************************************************************************************
# ************************
# Load our Nginx.conf file # Load our Nginx.conf file
# ************************
sudo nginx -c /etc/nginx/nginx.conf sudo nginx -c /etc/nginx/nginx.conf
# ***********************************************************
# Copy our index.php file into the default site's root folder # Copy our index.php file into the default site's root folder
# ***********************************************************
sudo cp $TRAVIS_BUILD_DIR/travisCI/index.php /var/www/html/index.php sudo cp $TRAVIS_BUILD_DIR/travisCI/index.php /var/www/html/index.php
# Run update-ngxblocker test which downloads latest globalblacklist.conf and reload Nginx # ****************************************************************************************
# Run update-ngxblocker test which downloads latest globalblacklist.conf and reloads Nginx
# ****************************************************************************************
cd /usr/sbin cd /usr/sbin
sudo ./update-ngxblocker -e mitchellkrog@gmail.com sudo ./update-ngxblocker -e mitchellkrog@gmail.com
# ***********************************************************
# Set all our other setup and deploy scripts to be executable # Set all our other setup and deploy scripts to be executable
# ***********************************************************
sudo chmod +x $TRAVIS_BUILD_DIR/travisCI/modify-globalblacklist.sh
sudo chmod +x $TRAVIS_BUILD_DIR/travisCI/modify-readme.sh
sudo chmod +x $TRAVIS_BUILD_DIR/travisCI/deploy-package.sh sudo chmod +x $TRAVIS_BUILD_DIR/travisCI/deploy-package.sh
sudo chmod +x $TRAVIS_BUILD_DIR/travisCI/generate-blacklist.sh
sudo chmod +x $TRAVIS_BUILD_DIR/travisCI/generate-robots.sh
sudo chmod +x $TRAVIS_BUILD_DIR/travisCI/generate-google-disavow.sh
sudo chmod +x $TRAVIS_BUILD_DIR/travisCI/generate-google-exclude.php
sudo chmod +x $TRAVIS_BUILD_DIR/travisCI/modify-config-readme-files.sh
sudo chmod +x $TRAVIS_BUILD_DIR/travisCI/modify-files-and-commit.sh sudo chmod +x $TRAVIS_BUILD_DIR/travisCI/modify-files-and-commit.sh
# *****************************************************************************************
# Travis now moves into running the rest of the tests in the script: section of .travis.yml # Travis now moves into running the rest of the tests in the script: section of .travis.yml
# *****************************************************************************************
# MIT License
# Copyright (c) 2017 Mitchell Krog - mitchellkrog@gmail.com
# https://github.com/mitchellkrogza
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

165
travisCI/modify-config-readme-files.sh Executable file
View file

@ -0,0 +1,165 @@
#!/bin/bash
# Write Build / Version Number into README.md, AUTO-CONFIGURATION.md and MANUAL-CONFIGURATION.md files
# Created by: Mitchell Krog (mitchellkrog@gmail.com)
# Copyright: Mitchell Krog - https://github.com/mitchellkrogza
# Repo Url: https://github.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker
# MIT License
# Copyright (c) 2017 Mitchell Krog - mitchellkrog@gmail.com
# https://github.com/mitchellkrogza
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
# ******************
# Set Some Variables
# ******************
YEAR=$(date +%Y)
MONTH=$(date +%m)
MY_GIT_TAG=V3.$YEAR.$MONTH.$TRAVIS_BUILD_NUMBER
BAD_REFERRERS=$(wc -l < $TRAVIS_BUILD_DIR/_generator_lists/bad-referrers.list)
BAD_BOTS=$(wc -l < $TRAVIS_BUILD_DIR/_generator_lists/bad-user-agents.list)
# **********************************
# Temporary database files we create
# **********************************
_inputdbA=/tmp/lastupdated.db
_tmpnginxA=tmpnginxA
# ***************************************************************
# Start and End Strings to Search for to do inserts into template
# ***************************************************************
_startmarker="##### Version Information #"
_endmarker="##### Version Information ##"
# ****************************************
# PRINT VERSION INFORMATION INTO README.md
# ****************************************
LASTUPDATEIFS=$IFS
IFS=$'\n'
now="$(date)"
end=$(date +%s.%N)
echo $_startmarker >> $_tmpnginxA
runtime=$(python -c "print(${end} - ${start})")
printf "********************************************\n#### Version: "$MY_GIT_TAG"\n#### Bad Referrer Count: "$BAD_REFERRERS"\n#### Bad Bot Count: "$BAD_BOTS"\n********************************************\n" >> $_tmpnginxA
echo $_endmarker >> $_tmpnginxA
IFS=$LASTUPDATEIFS
mv $_tmpnginxA $_inputdbA
ed -s $_inputdbA<<\IN
1,/##### Version Information #/d
/##### Version Information ##/,$d
,d
.r /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/README.md
/##### Version Information #/x
.t.
.,/##### Version Information ##/-d
#,p
#,p used to print output replaced with w below to write
w /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/README.md
q
IN
rm $_inputdbA
# ****************************************************
# PRINT VERSION INFORMATION INTO AUTO-CONFIGURATION.md
# ****************************************************
LASTUPDATEIFS2=$IFS
IFS=$'\n'
now="$(date)"
end=$(date +%s.%N)
echo $_startmarker >> $_tmpnginxA
runtime=$(python -c "print(${end} - ${start})")
printf "********************************************\n#### Version: "$MY_GIT_TAG"\n#### Bad Referrer Count: "$BAD_REFERRERS"\n#### Bad Bot Count: "$BAD_BOTS"\n********************************************\n" >> $_tmpnginxA
echo $_endmarker >> $_tmpnginxA
IFS=$LASTUPDATEIFS2
mv $_tmpnginxA $_inputdbA
ed -s $_inputdbA<<\IN
1,/##### Version Information #/d
/##### Version Information ##/,$d
,d
.r /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/AUTO-CONFIGURATION.md
/##### Version Information #/x
.t.
.,/##### Version Information ##/-d
#,p
#,p used to print output replaced with w below to write
w /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/AUTO-CONFIGURATION.md
q
IN
rm $_inputdbA
# ******************************************************
# PRINT VERSION INFORMATION INTO MANUAL-CONFIGURATION.md
# ******************************************************
LASTUPDATEIFS3=$IFS
IFS=$'\n'
now="$(date)"
end=$(date +%s.%N)
echo $_startmarker >> $_tmpnginxA
runtime=$(python -c "print(${end} - ${start})")
printf "********************************************\n#### Version: "$MY_GIT_TAG"\n#### Bad Referrer Count: "$BAD_REFERRERS"\n#### Bad Bot Count: "$BAD_BOTS"\n********************************************\n" >> $_tmpnginxA
echo $_endmarker >> $_tmpnginxA
IFS=$LASTUPDATEIFS3
mv $_tmpnginxA $_inputdbA
ed -s $_inputdbA<<\IN
1,/##### Version Information #/d
/##### Version Information ##/,$d
,d
.r /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/MANUAL-CONFIGURATION.md
/##### Version Information #/x
.t.
.,/##### Version Information ##/-d
#,p
#,p used to print output replaced with w below to write
w /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/MANUAL-CONFIGURATION.md
q
IN
rm $_inputdbA
exit 0
# MIT License
# Copyright (c) 2017 Mitchell Krog - mitchellkrog@gmail.com
# https://github.com/mitchellkrogza
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

94
travisCI/modify-files-and-commit.sh
View file

@ -1,31 +1,113 @@
#!/bin/bash #!/bin/bash
# Travis CI Generating and Building for the Nginx Ultimate Bad Bot Blocker
# Created by: Mitchell Krog (mitchellkrog@gmail.com)
# Copyright: Mitchell Krog - https://github.com/mitchellkrogza
# Repo Url: https://github.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker
# MIT License
# Copyright (c) 2017 Mitchell Krog - mitchellkrog@gmail.com
# https://github.com/mitchellkrogza
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
# ******************
# Set Some Variables
# ******************
#set -v
YEAR=$(date +"%Y") YEAR=$(date +"%Y")
MONTH=$(date +"%m") MONTH=$(date +"%m")
cd $TRAVIS_BUILD_DIR cd $TRAVIS_BUILD_DIR
#Remove Remotes Added by TravisCI # *******************************
# Remove Remote Added by TravisCI
# *******************************
git remote rm origin git remote rm origin
# **************************
# Add Remote with Secure Key # Add Remote with Secure Key
# **************************
git remote add origin https://${GH_TOKEN}@github.com/${TRAVIS_REPO_SLUG}.git git remote add origin https://${GH_TOKEN}@github.com/${TRAVIS_REPO_SLUG}.git
# **********************************************************************************
# List Remotes ONLY DURING testing - do not do this on live repo / possible key leak # List Remotes ONLY DURING testing - do not do this on live repo / possible key leak
# git remote -v # git remote -v
# ***********************************************************************************
# *********************
# Set Our Git Variables
# *********************
# Set Git Variables
git config --global user.email "${GIT_EMAIL}" git config --global user.email "${GIT_EMAIL}"
git config --global user.name "${GIT_NAME}" git config --global user.name "${GIT_NAME}"
git config --global push.default simple git config --global push.default simple
# Make sure we have master branch checked out in Git # *******************************************
# Make sure we have checked out master branch
# *******************************************
git checkout master git checkout master
# ***************************************************
# Modify our files with build and version information # Modify our files with build and version information
sudo $TRAVIS_BUILD_DIR/travisCI/modify-globalblacklist.sh # ***************************************************
sudo $TRAVIS_BUILD_DIR/travisCI/modify-readme.sh
sudo $TRAVIS_BUILD_DIR/travisCI/generate-blacklist.sh
sudo $TRAVIS_BUILD_DIR/travisCI/modify-config-readme-files.sh
sudo $TRAVIS_BUILD_DIR/travisCI/generate-robots.sh
sudo $TRAVIS_BUILD_DIR/travisCI/generate-google-disavow.sh
php ./travisCI/generate-google-exclude.php
# *************************************
# Add all the modified files and commit
# *************************************
# Add the modified files and commit
git add -A git add -A
git commit -am "V3.$YEAR.$MONTH.$TRAVIS_BUILD_NUMBER [ci skip]" git commit -am "V3.$YEAR.$MONTH.$TRAVIS_BUILD_NUMBER [ci skip]"
# *************************************************************
# Travis now moves to the before_deploy: section of .travis.yml # Travis now moves to the before_deploy: section of .travis.yml
# *************************************************************
# MIT License
# Copyright (c) 2017 Mitchell Krog - mitchellkrog@gmail.com
# https://github.com/mitchellkrogza
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

11
travisCI/old_scripts/default.vhost Normal file
View file

@ -0,0 +1,11 @@
server {
listen *:9000;
root /var/www/html;
server_name localhost;
charset UTF-8;
location / {
root /var/www/html/;
}
}

20
travisCI/old_scripts/deploy-package.sh Executable file
View file

@ -0,0 +1,20 @@
#!/bin/bash
YEAR=$(date +"%Y")
MONTH=$(date +"%m")
# Make Sure we are in the Build Directory
cd $TRAVIS_BUILD_DIR
# Create our Version Number
export GIT_TAG=V3.$YEAR.$MONTH.$TRAVIS_BUILD_NUMBER
# Tag our release
git tag $GIT_TAG -a -m "V3.$YEAR.$MONTH.$TRAVIS_BUILD_NUMBER"
# Push our commit and tags back to the repo
sudo git push origin master && git push origin master --tags
# Uncomment to list all git folders and modified files etc
#ls -aR
# Now TravisCI moves into the deploy: section of TravisCI - see .travis.yml

5
travisCI/old_scripts/index.php Normal file
View file

@ -0,0 +1,5 @@
<?php
echo "Welcome to Nginx\n";
echo "This is the Travis Testing Environment for the Nginx Ultimate Bad Bot Blocker\n";
echo "Visit https://github.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker\n";

67
travisCI/old_scripts/install-nginx.sh Executable file
View file

@ -0,0 +1,67 @@
#!/bin/bash
# Travis CI Testing for Nginx Ultimate Bad Bot Blocker
# https://github.com/mitchellkrogza
set -x
# Start Getting Nginx Ready for Testing the Nginx Bad Bot Blocker
# Delete default site created by Nginx Installation
sudo rm /etc/nginx/sites-available/default
# Download the Nginx Bad Bot Blocker setup files from the Live Repository
sudo wget https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/install-ngxblocker -O /usr/sbin/install-ngxblocker
sudo wget https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/setup-ngxblocker -O /usr/sbin/setup-ngxblocker
sudo wget https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/update-ngxblocker -O /usr/sbin/update-ngxblocker
# Set our install and setup scripts to be executable
sudo chmod +x /usr/sbin/install-ngxblocker
sudo chmod +x /usr/sbin/setup-ngxblocker
sudo chmod +x /usr/sbin/update-ngxblocker
# Run Install-NgxBlocker
cd /usr/sbin
sudo ./install-ngxblocker -x
# Copy our default.vhost file into Nginx /sites-available/
sudo cp $TRAVIS_BUILD_DIR/travisCI/default.vhost /etc/nginx/sites-available/default.vhost
# Link the vhost file into Nginx /sites-enabled/ and reload nginx
sudo ln -s /etc/nginx/sites-available/default.vhost /etc/nginx/sites-enabled/default.vhost
# Run setup-ngxblocker
cd /usr/sbin
sudo ./setup-ngxblocker -x
# NOTE: for Verbose Testing of any shell scripts use below format adding sh -x before running the script
# this helps a lot inside the TravisCI environment to see where a shell script may be failing
#sudo sh -x ./setup-ngxblocker -x
# Load our Nginx.conf file
sudo nginx -c /etc/nginx/nginx.conf
# Copy our index.php file into the default site's root folder
sudo cp $TRAVIS_BUILD_DIR/travisCI/index.php /var/www/html/index.php
# Run update-ngxblocker test which downloads latest globalblacklist.conf and reload Nginx
cd /usr/sbin
sudo ./update-ngxblocker -e mitchellkrog@gmail.com
# Set all our other setup and deploy scripts to be executable
sudo chmod +x $TRAVIS_BUILD_DIR/travisCI/modify-globalblacklist.sh
sudo chmod +x $TRAVIS_BUILD_DIR/travisCI/modify-readme.sh
sudo chmod +x $TRAVIS_BUILD_DIR/travisCI/deploy-package.sh
sudo chmod +x $TRAVIS_BUILD_DIR/travisCI/modify-files-and-commit.sh
# Travis now moves into running the rest of the tests in the script: section of .travis.yml

31
travisCI/old_scripts/modify-files-and-commit.sh Executable file
View file

@ -0,0 +1,31 @@
#!/bin/bash
YEAR=$(date +"%Y")
MONTH=$(date +"%m")
cd $TRAVIS_BUILD_DIR
#Remove Remotes Added by TravisCI
git remote rm origin
#Add Remote with Secure Key
git remote add origin https://${GH_TOKEN}@github.com/${TRAVIS_REPO_SLUG}.git
#List Remotes ONLY DURING testing - do not do this on live repo / possible key leak
#git remote -v
# Set Git Variables
git config --global user.email "${GIT_EMAIL}"
git config --global user.name "${GIT_NAME}"
git config --global push.default simple
# Make sure we have master branch checked out in Git
git checkout master
# Modify our files with build and version information
sudo $TRAVIS_BUILD_DIR/travisCI/modify-globalblacklist.sh
sudo $TRAVIS_BUILD_DIR/travisCI/modify-readme.sh
# Add the modified files and commit
git add -A
git commit -am "V3.$YEAR.$MONTH.$TRAVIS_BUILD_NUMBER [ci skip]"
# Travis now moves to the before_deploy: section of .travis.yml

0
travisCI/modify-globalblacklist.sh → travisCI/old_scripts/modify-globalblacklist.sh
View file

0
travisCI/modify-readme.sh → travisCI/old_scripts/modify-readme.sh
View file