vikarti.anatra/main
1
0
Fork 0
mirror of https://github.com/manualdousuario/marreta.git synced 2025-09-03 19:20:52 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

adicionado o referer nas chamadas do selenium

Browse source
This commit is contained in:
Renan Bernordi 2024-12-09 15:37:17 -03:00
parent 5fbd39786d
commit d1582a351d
2 changed files with 64 additions and 20 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

62
app/data/domain_rules.php
View file

@ -129,16 +129,16 @@ return [
], ],
'scriptTagRemove' => ['wrapperMessagingWithoutDetection.js'], 'scriptTagRemove' => ['wrapperMessagingWithoutDetection.js'],
'customCode' => ' 'customCode' => '
var artBodyContainer = document.querySelector("article.article"); var artBodyContainer = document.querySelector("article.article");
var artBody = artBodyContainer.innerHTML; var artBody = artBodyContainer.innerHTML;
checkPaywall(); checkPaywall();
function checkPaywall() { function checkPaywall() {
let paywallBox = document.querySelector(".layout-article-regwall"); let paywallBox = document.querySelector(".layout-article-regwall");
if (paywallBox) { if (paywallBox) {
artBodyContainer.innerHTML = artBody; artBodyContainer.innerHTML = artBody;
}
} }
' }
'
], ],
'ft.com' => [ 'ft.com' => [
'cookies' => [ 'cookies' => [
@ -150,9 +150,47 @@ return [
] ]
], ],
'nytimes.com' => [ 'nytimes.com' => [
'cookies' => [ 'useSelenium' => true,
'nyt-gdpr' => '1', 'excludeGlobalRules' => [
'nyt-purr' => 'cfh' 'scriptTagRemove' => [
'gtm.js',
'ga.js',
'fbevents.js',
'pixel.js',
'chartbeat',
'analytics.js',
'cmp.js',
'wall.js',
'paywall.js',
'subscriber.js',
'piano.js',
'tiny.js',
'pywll.js',
'content-gate.js',
'signwall.js',
'pw.js',
'pw-',
'piano-',
'tinypass',
'tp.min.js',
'premium.js',
'amp-access-0.1.js',
'zephrBarriersScripts',
'leaky-paywall',
'cookie',
'gdpr',
'lgpd',
'push',
'sw.js',
'stats.js',
'piano.io',
'onesignal.com',
'getsitecontrol.com',
'navdmp.com',
'getblue.io',
'smartocto.com',
'cdn.pn.vg'
]
] ]
], ],
'correio24horas.com.br' => [ 'correio24horas.com.br' => [

22
app/inc/URLAnalyzer.php
View file

@ -134,15 +134,17 @@ class URLAnalyzer
// 4. Verifica se deve usar Selenium // 4. Verifica se deve usar Selenium
$domainRules = $this->getDomainRules($host); $domainRules = $this->getDomainRules($host);
if (isset($domainRules['useSelenium']) && $domainRules['useSelenium'] === true) { if (isset($domainRules['useSelenium']) && $domainRules['useSelenium'] === true) {
$content = $this->fetchFromSelenium($cleanUrl); try {
if (!empty($content)) { $content = $this->fetchFromSelenium($cleanUrl);
$processedContent = $this->processContent($content, $host, $cleanUrl); if (!empty($content)) {
$this->cache->set($cleanUrl, $processedContent); $processedContent = $this->processContent($content, $host, $cleanUrl);
return $processedContent; $this->cache->set($cleanUrl, $processedContent);
return $processedContent;
}
} catch (Exception $e) {
$this->logError($cleanUrl, "Selenium fetch error: " . $e->getMessage());
throw new Exception("Não foi possível obter o conteúdo via Selenium");
} }
$this->logError($cleanUrl, "Selenium fetch error: " . $e->getMessage());
throw new Exception("Não foi possível obter o conteúdo via Selenium");
} }
// 5. Tenta buscar conteúdo diretamente // 5. Tenta buscar conteúdo diretamente
@ -186,6 +188,10 @@ class URLAnalyzer
$profile = new FirefoxProfile(); $profile = new FirefoxProfile();
$profile->setPreference("permissions.default.image", 2); // Não carrega imagens $profile->setPreference("permissions.default.image", 2); // Não carrega imagens
$profile->setPreference("javascript.enabled", true); // Mantem habilitado javascripts $profile->setPreference("javascript.enabled", true); // Mantem habilitado javascripts
$profile->setPreference("network.http.referer.defaultPolicy", 0); // Sempre envia referer
$profile->setPreference("network.http.referer.defaultReferer", "https://www.google.com.br"); // Define referer padrão
$profile->setPreference("network.http.referer.spoofSource", true); // Permite spoofing do referer
$profile->setPreference("network.http.referer.trimmingPolicy", 0); // Não corta o referer
$options = new FirefoxOptions(); $options = new FirefoxOptions();
$options->setProfile($profile); $options->setProfile($profile);