vikarti.anatra/nginx-ultimate-bad-bot-blocker
1
0
Fork 0
mirror of https://github.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker.git synced 2025-09-02 02:29:58 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

FIX robots.txt formatting errors - regex removal

Browse source
This commit is contained in:
Mitchell Krog 2017-10-15 15:39:08 +02:00
parent 861134f240
commit ae20be5ae6
No known key found for this signature in database
GPG key ID: 2BB9A077C8DD6994
1 changed files with 16 additions and 2 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

18
.dev-tools/generate-robots.sh
View file

@ -45,8 +45,22 @@
# Set Input Files
# ***************
_input1=$TRAVIS_BUILD_DIR/.dev-tools/_robots_input/robots-input.txt
_input1=$TRAVIS_BUILD_DIR/_generator_lists/bad-user-agents.list
_tmprobots=/tmp/robots.txt
_inputtmp=$TRAVIS_BUILD_DIR/.dev-tools/_robots_input/robots.tmp
_output=$TRAVIS_BUILD_DIR/.dev-tools/_robots_input/robots-input.txt
# ***********************
# Truncate our input file
# ***********************
sudo truncate -s 0 $_output
# *************************************
# Use sed to prepare our new input file
# *************************************
cat $_input1 | sed 's/\\ / /g' > $_inputtmp && mv $_inputtmp $_output
# ******************
# Set Some Variables
@ -75,7 +89,7 @@ printf '%s\n%s\n%s%s\n%s%s\n%s%s\n%s\n%s\n\n%s\n%s\n%s\n' "$_startmarker" "#####
while IFS= read -r LINE
do
printf 'User-agent: %s\n%s\n' "${LINE}" "Disallow:/" >> "$_tmprobots"
done < $_input1
done < $_output
printf '\n' >> "$_tmprobots"
sudo cp $_tmprobots $TRAVIS_BUILD_DIR/robots.txt/robots.txt
exit 0