From 0514435f2a738f2f95dbdce0591ffd3ecaa9e507 Mon Sep 17 00:00:00 2001
From: Renan Bernordi <altendorfme@gmail.com>
Date: Thu, 9 Jan 2025 21:41:28 -0300
Subject: [PATCH] =?UTF-8?q?removida=20configura=C3=A7=C3=A3o=20de=20user?=
 =?UTF-8?q?=20agent?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 README.en.md             |  1 -
 README.md                |  1 -
 app/config.php           |  1 -
 app/data/user_agents.php | 25 -------------------------
 app/inc/URLAnalyzer.php  | 23 ++++++++++++++++++-----
 5 files changed, 18 insertions(+), 33 deletions(-)
 delete mode 100644 app/data/user_agents.php

diff --git a/README.en.md b/README.en.md
index 0408036..8960f3b 100644
--- a/README.en.md
+++ b/README.en.md
@@ -105,7 +105,6 @@ The configurations are organized in `data/`:
 - `domain_rules.php`: Site-specific rules
 - `global_rules.php`: Rules that apply to all sites
 - `blocked_domains.php`: List of blocked sites
-- `user_agents.php`: User Agents configurations
 
 ### Translations
 
diff --git a/README.md b/README.md
index 3be8096..a43d213 100644
--- a/README.md
+++ b/README.md
@@ -105,7 +105,6 @@ As configurações estão organizadas em `data/`:
 - `domain_rules.php`: Regras específicas para cada site
 - `global_rules.php`: Regras que se aplicam a todos os sites
 - `blocked_domains.php`: Lista de sites bloqueados
-- `user_agents.php`: Configurações de User Agents
 
 ### Traduções
 
diff --git a/app/config.php b/app/config.php
index c7ecc3b..e61cfe6 100644
--- a/app/config.php
+++ b/app/config.php
@@ -99,7 +99,6 @@ try {
      * Load system configurations
      * Carrega as configurações do sistema
      */
-    define('USER_AGENTS', require __DIR__ . '/data/user_agents.php');
     define('BLOCKED_DOMAINS', require __DIR__ . '/data/blocked_domains.php');
     define('DOMAIN_RULES', require __DIR__ . '/data/domain_rules.php');
     define('GLOBAL_RULES', require __DIR__ . '/data/global_rules.php');
diff --git a/app/data/user_agents.php b/app/data/user_agents.php
deleted file mode 100644
index ac8f8c1..0000000
--- a/app/data/user_agents.php
+++ /dev/null
@@ -1,25 +0,0 @@
-<?php
-
-/**
- * Bot configurations
- * Configurações dos bots
- * 
- * Defines user agents that can be used to make requests
- * Define os user agents que podem ser utilizados para fazer requisições
- * 
- * These user agents are used to simulate legitimate web crawlers
- * Estes user agents são usados para simular crawlers web legítimos
- */
-return [
-    // Google News bot
-    // Bot do Google News
-    'Googlebot-News',
-
-    // Mobile Googlebot
-    // Googlebot para dispositivos móveis
-    'Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/W.X.Y.Z Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)',
-
-    // Desktop Googlebot
-    // Googlebot para desktop
-    'Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Googlebot/2.1; +http://www.google.com/bot.html) Chrome/W.X.Y.Z Safari/537.36'
-];
diff --git a/app/inc/URLAnalyzer.php b/app/inc/URLAnalyzer.php
index 066f431..31c762b 100644
--- a/app/inc/URLAnalyzer.php
+++ b/app/inc/URLAnalyzer.php
@@ -35,17 +35,31 @@ class URLAnalyzer
      * @var array List of available User Agents for requests
      * @var array Lista de User Agents disponíveis para requisições
      */
-    private $userAgents;
+    private $userAgents = [
+        // Google News bot
+        // Bot do Google News
+        'Googlebot-News',
+        // Mobile Googlebot
+        // Googlebot para dispositivos móveis
+        'Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/W.X.Y.Z Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)',
+        // Desktop Googlebot
+        // Googlebot para desktop
+        'Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Googlebot/2.1; +http://www.google.com/bot.html) Chrome/W.X.Y.Z Safari/537.36'
+    ];
 
     /**
      * @var array List of social media referrers
      * @var array Lista de referenciadores de mídia social
      */
     private $socialReferrers = [
+        // Twitter
         'https://t.co/',
-        'https://www.google.com/',
-        'https://www.facebook.com/',
         'https://www.twitter.com/',
+        // Google
+        'https://www.google.com/',
+        // Facebook
+        'https://www.facebook.com/',
+        // Linkedin
         'https://www.linkedin.com/'
     ];
 
@@ -82,7 +96,6 @@ class URLAnalyzer
      */
     public function __construct()
     {
-        $this->userAgents = USER_AGENTS;
         $this->dnsServers = explode(',', DNS_SERVERS);
         $this->rules = new Rules();
         $this->cache = new Cache();
@@ -326,7 +339,7 @@ class URLAnalyzer
     private function fetchContent($url)
     {
         $curl = new Curl();
-        $this->setupBasicCurlOptions($curl);
+        $this->setupBasicCurlOptions($curl, $url);
         
         $host = parse_url($url, PHP_URL_HOST);
         $host = preg_replace('/^www\./', '', $host);