vikarti.anatra/nginx-ultimate-bad-bot-blocker
1
0
Fork 0
mirror of https://github.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker.git synced 2025-09-05 04:00:13 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

UPDATE Generate Robots.txt [MASTER]

Browse source
This commit is contained in:
Mitchell Krog 2019-06-28 14:53:53 +02:00
parent f79f5dc6d5
commit 60bd2efffa
No known key found for this signature in database
GPG key ID: C243C388553EDE5D
1 changed files with 49 additions and 40 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

87
.dev-tools/generate-robots.sh
View file

@ -18,21 +18,22 @@
# # # #
############################################################################## ##############################################################################
# ------------------------------------------------------------------------------
# MIT License # MIT License
# ------------------------------------------------------------------------------
# Copyright (c) 2017 Mitchell Krog - mitchellkrog@gmail.com # Copyright (c) 2017 Mitchell Krog - mitchellkrog@gmail.com
# https://github.com/mitchellkrogza # https://github.com/mitchellkrogza
# ------------------------------------------------------------------------------
# Permission is hereby granted, free of charge, to any person obtaining a copy # Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal # of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights # in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is # copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions: # furnished to do so, subject to the following conditions:
# ------------------------------------------------------------------------------
# The above copyright notice and this permission notice shall be included in all # The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software. # copies or substantial portions of the Software.
# ------------------------------------------------------------------------------
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
@ -40,31 +41,34 @@
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE. # SOFTWARE.
# ------------------------------------------------------------------------------
# *************** # ------------------------
# Set Input Files # Set Terminal Font Colors
# *************** # ------------------------
bold=$(tput bold)
red=$(tput setaf 1)
green=$(tput setaf 2)
yellow=$(tput setaf 3)
blue=$(tput setaf 4)
magenta=$(tput setaf 5)
cyan=$(tput setaf 6)
white=$(tput setaf 7)
defaultcolor=$(tput setaf default)
# --------------
# File Variables
# --------------
_input1=${TRAVIS_BUILD_DIR}/_generator_lists/bad-user-agents.list _input1=${TRAVIS_BUILD_DIR}/_generator_lists/bad-user-agents.list
_tmprobots=/tmp/robots.txt _tmprobots=/tmp/robots.txt
_inputtmp=${TRAVIS_BUILD_DIR}/.dev-tools/_robots_input/robots.tmp _inputtmp=${TRAVIS_BUILD_DIR}/.dev-tools/_robots_input/robots.tmp
_output=${TRAVIS_BUILD_DIR}/.dev-tools/_robots_input/robots-input.txt _output=${TRAVIS_BUILD_DIR}/.dev-tools/_robots_input/robots-input.txt
# *********************** # ---------
# Truncate our input file # Variables
# *********************** # ---------
sudo truncate -s 0 ${_output}
# *************************************
# Use sed to prepare our new input file
# *************************************
cat ${_input1} | sed 's/\\ / /g' > ${_inputtmp} && mv ${_inputtmp} ${_output}
# ******************
# Set Some Variables
# ******************
YEAR=$(date +"%Y") YEAR=$(date +"%Y")
MONTH=$(date +"%m") MONTH=$(date +"%m")
@ -72,51 +76,55 @@ MY_GIT_TAG=V4.${YEAR}.${MONTH}.${TRAVIS_BUILD_NUMBER}
BAD_REFERRERS=$(wc -l < ${TRAVIS_BUILD_DIR}/_generator_lists/bad-referrers.list) BAD_REFERRERS=$(wc -l < ${TRAVIS_BUILD_DIR}/_generator_lists/bad-referrers.list)
BAD_BOTS=$(wc -l < ${TRAVIS_BUILD_DIR}/_generator_lists/bad-user-agents.list) BAD_BOTS=$(wc -l < ${TRAVIS_BUILD_DIR}/_generator_lists/bad-user-agents.list)
_now="$(date)" _now="$(date)"
# *************************
# Set Start and End Markers
# *************************
_startmarker="### Version Information #" _startmarker="### Version Information #"
_endmarker="### Version Information ##" _endmarker="### Version Information ##"
# ---------
# FUNCTIONS
# ---------
# ************************** generateRobotsFile () {
# Create the robots.txt file sudo truncate -s 0 ${_output}
# ************************** # Use sed to prepare our new input file
cat ${_input1} | sed 's/\\ / /g' > ${_inputtmp} && mv ${_inputtmp} ${_output}
printf '###################################################################\n# The Ultimate robots.txt Bot and User-Agent Blocker\n# Copyright:\n# https://github.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker\n###################################################################\n\n' >> "${_tmprobots}" printf '###################################################################\n# The Ultimate robots.txt Bot and User-Agent Blocker\n# Copyright:\n# https://github.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker\n###################################################################\n\n' >> "${_tmprobots}"
printf '%s\n%s\n%s%s\n%s%s\n%s%s\n%s\n%s\n\n%s\n%s\n%s\n' "$_startmarker" "###################################################" "### Version: " "$MY_GIT_TAG" "### Updated: " "$_now" "### Bad Bot Count: " "$BAD_BOTS" "###################################################" "$_endmarker" "User-agent: *" "Disallow: /wp-admin/" "Allow: /wp-admin/admin-ajax.php" >> "${_tmprobots}" printf '%s\n%s\n%s%s\n%s%s\n%s%s\n%s\n%s\n\n%s\n%s\n%s\n' "$_startmarker" "###################################################" "### Version: " "$MY_GIT_TAG" "### Updated: " "$_now" "### Bad Bot Count: " "$BAD_BOTS" "###################################################" "$_endmarker" "User-agent: *" "Disallow: /wp-admin/" "Allow: /wp-admin/admin-ajax.php" >> "${_tmprobots}"
while IFS= read -r LINE while IFS= read -r LINE
do do
printf 'User-agent: %s\n%s\n' "${LINE}" "Disallow:/" >> "${_tmprobots}" printf 'User-agent: %s\n%s\n' "${LINE}" "Disallow:/" >> "${_tmprobots}"
done < ${_output} done < ${_output}
printf '\n' >> ${_tmprobots} printf '\n' >> ${_tmprobots}
sudo cp ${_tmprobots} ${TRAVIS_BUILD_DIR}/robots.txt/robots.txt sudo cp ${_tmprobots} ${TRAVIS_BUILD_DIR}/robots.txt/robots.txt
}
# ********************** # ----------------
# Trigger Function
# ----------------
generateRobotsFile
# ----------------------
# Exit With Error Number # Exit With Error Number
# ********************** # ----------------------
exit ${?} exit ${?}
# ------------------------------------------------------------------------------
# MIT License # MIT License
# ------------------------------------------------------------------------------
# Copyright (c) 2017 Mitchell Krog - mitchellkrog@gmail.com # Copyright (c) 2017 Mitchell Krog - mitchellkrog@gmail.com
# https://github.com/mitchellkrogza # https://github.com/mitchellkrogza
# ------------------------------------------------------------------------------
# Permission is hereby granted, free of charge, to any person obtaining a copy # Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal # of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights # in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is # copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions: # furnished to do so, subject to the following conditions:
# ------------------------------------------------------------------------------
# The above copyright notice and this permission notice shall be included in all # The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software. # copies or substantial portions of the Software.
# ------------------------------------------------------------------------------
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
@ -124,3 +132,4 @@ exit ${?}
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE. # SOFTWARE.
# ------------------------------------------------------------------------------