From ae20be5ae6f2b01ab7bb8a2b9832e5beaff42d89 Mon Sep 17 00:00:00 2001
From: Mitchell Krog <mitchellkrog@gmail.com>
Date: Sun, 15 Oct 2017 15:39:08 +0200
Subject: [PATCH] FIX robots.txt formatting errors - regex removal

---
 .dev-tools/generate-robots.sh | 18 ++++++++++++++++--
 1 file changed, 16 insertions(+), 2 deletions(-)

diff --git a/.dev-tools/generate-robots.sh b/.dev-tools/generate-robots.sh
index 7f171ba3b..8030a9ce4 100755
--- a/.dev-tools/generate-robots.sh
+++ b/.dev-tools/generate-robots.sh
@@ -45,8 +45,22 @@
 # Set Input Files
 # ***************
 
-_input1=$TRAVIS_BUILD_DIR/.dev-tools/_robots_input/robots-input.txt
+_input1=$TRAVIS_BUILD_DIR/_generator_lists/bad-user-agents.list
 _tmprobots=/tmp/robots.txt
+_inputtmp=$TRAVIS_BUILD_DIR/.dev-tools/_robots_input/robots.tmp
+_output=$TRAVIS_BUILD_DIR/.dev-tools/_robots_input/robots-input.txt
+
+# ***********************
+# Truncate our input file
+# ***********************
+
+sudo truncate -s 0 $_output
+
+# *************************************
+# Use sed to prepare our new input file
+# *************************************
+
+cat $_input1 | sed 's/\\ / /g' > $_inputtmp && mv $_inputtmp $_output
 
 # ******************
 # Set Some Variables
@@ -75,7 +89,7 @@ printf '%s\n%s\n%s%s\n%s%s\n%s%s\n%s\n%s\n\n%s\n%s\n%s\n' "$_startmarker" "#####
 while IFS= read -r LINE
 do
 printf 'User-agent: %s\n%s\n' "${LINE}" "Disallow:/" >> "$_tmprobots"
-done < $_input1
+done < $_output
 printf '\n' >> "$_tmprobots"
 sudo cp $_tmprobots $TRAVIS_BUILD_DIR/robots.txt/robots.txt
 exit 0