From 6f5bb2be6e8d76349f52b7c11edab43ed8944448 Mon Sep 17 00:00:00 2001
From: Mitchell Krog <mitchellkrog@gmail.com>
Date: Mon, 10 Jul 2017 17:37:36 +0200
Subject: [PATCH] Regex formatting generator script test. - dots need to be
 escaped, leads to false positives.

---
 travisCI/generate-regex-format-referrers.php | 94 ++++++++++++++++++++
 travisCI/install-nginx.sh                    |  1 +
 travisCI/modify-files-and-commit.sh          |  1 +
 travisCI/referrers-regex-format.txt          |  0
 4 files changed, 96 insertions(+)
 create mode 100644 travisCI/generate-regex-format-referrers.php
 create mode 100644 travisCI/referrers-regex-format.txt

diff --git a/travisCI/generate-regex-format-referrers.php b/travisCI/generate-regex-format-referrers.php
new file mode 100644
index 000000000..46697568e
--- /dev/null
+++ b/travisCI/generate-regex-format-referrers.php
@@ -0,0 +1,94 @@
+<?php
+    /* Regex Formatting Generator Script for the Nginx Ultimate Bad Bot Blocker
+     * Adapted from Script Copyright (c) 2017 Stevie-Ray - https://github.com/Stevie-Ray
+     * Adapted by: Mitchell Krog (mitchellkrog@gmail.com) - https://github.com/mitchellkrogza
+     * Repo Url: https://github.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker
+     */
+namespace mitchellkrogza;
+
+use Mso\IdnaConvert\IdnaConvert;
+
+class Generator
+{
+    
+    private $projectUrl = "https://github.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker";
+    public function generateFiles()
+    {
+        date_default_timezone_set('Africa/Johannesburg');
+        $date = date('Y-m-d H:i:s');
+        $lines = $this->domainWorker();
+        $this->createNginx($lines);
+    }
+
+
+    /**
+     * @return array
+     */
+    public function domainWorker()
+    {
+        $domainsFile = "/home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/_generator_lists/bad-referrers.list";
+
+        $handle = fopen($domainsFile, "r");
+        if (!$handle) {
+            throw new \RuntimeException('Error opening file ' . $domainsFile);
+        }
+        $lines = array();
+        while (($line = fgets($handle)) !== false) {
+            $line = trim(preg_replace('/\s\s+/', ' ', $line));
+
+            // convert internationalized domain names
+            if (preg_match('/[А-Яа-яЁёɢ]/u', $line)) {
+
+                $IDN = new IdnaConvert();
+
+                $line = $IDN->encode($line);
+
+            }
+
+            if (empty($line)) {
+                continue;
+            }
+            $lines[] = $line;
+        }
+        fclose($handle);
+        $uniqueLines = array_unique($lines, SORT_STRING);
+        sort($uniqueLines, SORT_STRING);
+        if (is_writable($domainsFile)) {
+            file_put_contents($domainsFile, implode("\n", $uniqueLines));
+        } else {
+            trigger_error("Permission denied");
+        }
+
+        return $lines;
+    }
+
+	/**
+     * Write to File Function
+     * @param $filename
+     * @param $data
+     */
+    protected function writeToFile($filename, $data)
+    {
+    $file = "/home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/travisCI/referrers-regex-format.txt";
+	$handle = fopen($file, 'w') or die('Cannot open file:  '.$file);
+	fwrite($handle, $data);
+    }
+
+    /**
+     * @param string $date
+     * @param array  $lines
+     */
+    public function createNginx(array $lines) 
+    {
+    $file = "/home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/travisCI/referrers-regex-format.txt";
+        $data = "";
+    {
+        foreach ($lines as $line) {
+            $data .= "~*\"" . preg_quote($line) . "\" \t1;\n";
+        }
+        $this->writeToFile($file, $data);
+    	}
+	}
+}
+$generator = new Generator();
+$generator->generateFiles();
diff --git a/travisCI/install-nginx.sh b/travisCI/install-nginx.sh
index 19c1de2fe..aa3efa1a2 100755
--- a/travisCI/install-nginx.sh
+++ b/travisCI/install-nginx.sh
@@ -121,6 +121,7 @@ sudo chmod +x $TRAVIS_BUILD_DIR/travisCI/generate-blacklist.sh
 sudo chmod +x $TRAVIS_BUILD_DIR/travisCI/generate-robots.sh
 sudo chmod +x $TRAVIS_BUILD_DIR/travisCI/generate-google-disavow.sh
 sudo chmod +x $TRAVIS_BUILD_DIR/travisCI/generate-google-exclude.php
+sudo chmod +x $TRAVIS_BUILD_DIR/travisCI/generate-regex-format-referrers.php
 sudo chmod +x $TRAVIS_BUILD_DIR/travisCI/modify-config-readme-files.sh
 sudo chmod +x $TRAVIS_BUILD_DIR/travisCI/modify-files-and-commit.sh
 sudo chmod +x $TRAVIS_BUILD_DIR/travisCI/run-curl-tests.sh
diff --git a/travisCI/modify-files-and-commit.sh b/travisCI/modify-files-and-commit.sh
index af43d2863..b72e98d77 100755
--- a/travisCI/modify-files-and-commit.sh
+++ b/travisCI/modify-files-and-commit.sh
@@ -77,6 +77,7 @@ sudo $TRAVIS_BUILD_DIR/travisCI/modify-config-readme-files.sh
 sudo $TRAVIS_BUILD_DIR/travisCI/generate-robots.sh
 sudo $TRAVIS_BUILD_DIR/travisCI/generate-google-disavow.sh
 php ./travisCI/generate-google-exclude.php
+php ./travisCI/generate-regex-format-referrers.php
 
 # *************************************
 # Add all the modified files and commit
diff --git a/travisCI/referrers-regex-format.txt b/travisCI/referrers-regex-format.txt
new file mode 100644
index 000000000..e69de29bb