rewrite proxy url

This commit is contained in:
Renan Bernordi 2026-07-18 00:02:36 -03:00
parent 2bfedbee90
commit 37bd93f201
7 changed files with 151 additions and 363 deletions

View file

@ -1,5 +1,5 @@
name: 🛠️ Main
run-name: 🚀 Version Deployment
run-name: 🚀 Deployment
on:
push:
@ -9,24 +9,81 @@ on:
env:
DOCKER_REGISTRY: ghcr.io
DOCKER_IMAGE_NAME: ${{ github.repository }}
DOCKERHUB_REPOSITORY: ${{ secrets.DOCKERHUB_USERNAME }}/marreta
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true
jobs:
docker-build-amd64:
name: 🐳 Build and Push (AMD64)
php-lint:
name: 🔍 PHP syntax check
runs-on: ubuntu-24.04
steps:
- name: 📥 Checkout code
uses: actions/checkout@v6
with:
fetch-depth: 0
- name: 🐘 Install PHP
uses: shivammathur/setup-php@v2
with:
php-version: '8.3'
tools: none
- name: 🔍 Validate PHP syntax
run: |
PREVIOUS_TAG=$(git describe --tags --abbrev=0 HEAD^ 2>/dev/null || echo "")
if [ -z "$PREVIOUS_TAG" ]; then
FILES=$(git ls-files | grep '\.php$' || true)
else
FILES=$(git diff --name-only --diff-filter=d "$PREVIOUS_TAG"..HEAD | grep '\.php$' || true)
fi
if [ -z "$FILES" ]; then
echo "No PHP files changed since last tag."
exit 0
fi
echo "$FILES" | xargs -P4 -I{} php -l {}
setup:
name: 🔍 Extract version
runs-on: ubuntu-24.04
needs: [php-lint]
outputs:
version: ${{ steps.get_version.outputs.VERSION }}
prerelease: ${{ steps.get_version.outputs.PRERELEASE }}
steps:
- name: 🏷️ Extract version from tag
id: get_version
run: |
VERSION=${GITHUB_REF#refs/tags/}
echo "VERSION=$VERSION" >> $GITHUB_OUTPUT
if [[ "$VERSION" == *"-"* ]]; then
echo "PRERELEASE=true" >> $GITHUB_OUTPUT
else
echo "PRERELEASE=false" >> $GITHUB_OUTPUT
fi
docker-build:
name: 🐳 Build and push (${{ matrix.cache_scope }})
runs-on: ${{ matrix.runner }}
needs: [setup]
permissions:
contents: read
packages: write
strategy:
matrix:
include:
- platform: linux/amd64
runner: ubuntu-24.04
suffix: -amd64
cache_scope: amd64
- platform: linux/arm64
runner: ubuntu-24.04-arm
suffix: -arm64
cache_scope: arm64
steps:
- name: 📥 Checkout code
uses: actions/checkout@v4
- name: 🏷️ Extract version from tag
id: get_version
run: echo "VERSION=${GITHUB_REF#refs/tags/}" >> $GITHUB_OUTPUT
- name: 🔧 Set up QEMU
uses: docker/setup-qemu-action@v3
@ -37,22 +94,14 @@ jobs:
id: meta
uses: docker/metadata-action@v5
with:
images: |
${{ env.DOCKER_REGISTRY }}/${{ env.DOCKER_IMAGE_NAME }}
${{ env.DOCKERHUB_REPOSITORY }}
images: ${{ env.DOCKER_REGISTRY }}/${{ env.DOCKER_IMAGE_NAME }}
tags: |
type=semver,pattern={{version}}
type=semver,pattern={{major}}.{{minor}}
type=sha
type=raw,value=latest
type=raw,value=latest,enable=${{ needs.setup.outputs.prerelease == 'false' }}
flavor: |
suffix=-amd64
- name: 🔐 Log in to Docker Hub
uses: docker/login-action@v3
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
suffix=${{ matrix.suffix }}
- name: 🔐 Log in to GitHub Registry
uses: docker/login-action@v3
@ -61,108 +110,39 @@ jobs:
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: 🏗️ Build and Push (AMD64)
uses: docker/build-push-action@v5
- name: 🏗️ Build and push (${{ matrix.cache_scope }})
uses: docker/build-push-action@v6
with:
context: .
platforms: linux/amd64
platforms: ${{ matrix.platform }}
push: true
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}
cache-from: type=gha,scope=amd64
cache-to: type=gha,mode=max,scope=amd64
docker-build-arm64:
name: 🐳 Build and Push (ARM64)
runs-on: ubuntu-24.04-arm
permissions:
contents: read
packages: write
steps:
- name: 📥 Checkout code
uses: actions/checkout@v4
- name: 🏷️ Extract version from tag
id: get_version
run: echo "VERSION=${GITHUB_REF#refs/tags/}" >> $GITHUB_OUTPUT
- name: 🛠️ Set up Docker Buildx
uses: docker/setup-buildx-action@v3
- name: 📋 Extract Docker metadata
id: meta
uses: docker/metadata-action@v5
with:
images: |
${{ env.DOCKER_REGISTRY }}/${{ env.DOCKER_IMAGE_NAME }}
${{ env.DOCKERHUB_REPOSITORY }}
tags: |
type=semver,pattern={{version}}
type=semver,pattern={{major}}.{{minor}}
type=sha
type=raw,value=latest
flavor: |
suffix=-arm64
- name: 🔐 Log in to Docker Hub
uses: docker/login-action@v3
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
- name: 🔐 Log in to GitHub Registry
uses: docker/login-action@v3
with:
registry: ${{ env.DOCKER_REGISTRY }}
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: 🏗️ Build and Push (ARM64)
uses: docker/build-push-action@v5
with:
context: .
platforms: linux/arm64
push: true
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}
cache-from: type=gha,scope=arm64
cache-to: type=gha,mode=max,scope=arm64
cache-from: type=gha,scope=${{ matrix.cache_scope }}
cache-to: type=gha,mode=max,scope=${{ matrix.cache_scope }}
docker-manifest:
name: 🔗 Create Multi-Platform Manifest
name: 🔗 Multi-platform manifest
runs-on: ubuntu-24.04
needs: [docker-build-amd64, docker-build-arm64]
needs: [setup, docker-build]
permissions:
contents: read
packages: write
steps:
- name: 📥 Checkout code
uses: actions/checkout@v4
- name: 🏷️ Extract version from tag
id: get_version
run: echo "VERSION=${GITHUB_REF#refs/tags/}" >> $GITHUB_OUTPUT
- name: 📋 Extract Docker metadata
id: meta
uses: docker/metadata-action@v5
with:
images: |
${{ env.DOCKER_REGISTRY }}/${{ env.DOCKER_IMAGE_NAME }}
${{ env.DOCKERHUB_REPOSITORY }}
images: ${{ env.DOCKER_REGISTRY }}/${{ env.DOCKER_IMAGE_NAME }}
tags: |
type=semver,pattern={{version}}
type=semver,pattern={{major}}.{{minor}}
type=sha
type=raw,value=latest
- name: 🔐 Log in to Docker Hub
uses: docker/login-action@v3
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
type=raw,value=latest,enable=${{ needs.setup.outputs.prerelease == 'false' }}
- name: 🔐 Log in to GitHub Registry
uses: docker/login-action@v3
@ -174,11 +154,9 @@ jobs:
- name: 🛠️ Set up Docker Buildx
uses: docker/setup-buildx-action@v3
- name: 🔗 Create and Push Multi-Platform Manifest
- name: 🔗 Create and push multi-platform manifest
run: |
# Extract the main tags (without suffixes)
TAGS="${{ steps.meta.outputs.tags }}"
for tag in $TAGS; do
echo "Creating manifest for $tag"
docker buildx imagetools create \
@ -188,25 +166,65 @@ jobs:
done
publish-release:
name: 📦 Publish Release
name: 📦 Publish release
runs-on: ubuntu-24.04
needs: [docker-build-amd64, docker-build-arm64, docker-manifest]
needs: [setup, docker-build, docker-manifest]
permissions:
contents: write
models: read
steps:
- name: 📥 Checkout code
uses: actions/checkout@v4
- name: 🏷️ Extract version from tag
id: get_version
run: echo "VERSION=${GITHUB_REF#refs/tags/}" >> $GITHUB_OUTPUT
- name: 📝 Create Release
uses: softprops/action-gh-release@v1
with:
name: "🎉 Release v${{ steps.get_version.outputs.VERSION }}"
tag_name: ${{ steps.get_version.outputs.VERSION }}
generate_release_notes: true
fetch-depth: 0
- name: 📋 Get commits since last tag
id: commits
run: |
PREVIOUS_TAG=$(git describe --tags --abbrev=0 HEAD^ 2>/dev/null || echo "")
if [ -z "$PREVIOUS_TAG" ]; then
LOG=$(git log --pretty=format:"%s (%h)" HEAD)
else
LOG=$(git log --pretty=format:"%s (%h)" "$PREVIOUS_TAG"..HEAD)
fi
echo "LOG<<EOF" >> $GITHUB_OUTPUT
echo "$LOG" >> $GITHUB_OUTPUT
echo "EOF" >> $GITHUB_OUTPUT
- name: 🤖 Generate release notes
id: ai_notes
run: |
RESPONSE=$(curl -s https://models.inference.ai.azure.com/chat/completions \
-H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" \
-H "Content-Type: application/json" \
-d "$(jq -n \
--arg version "${{ needs.setup.outputs.version }}" \
--arg log "${{ steps.commits.outputs.LOG }}" \
'{
model: "gpt-4o-mini",
messages: [
{
role: "system",
content: "You are a technical writer for open source projects. Generate polished, friendly release notes in English from a list of git commits. Group changes into sections using these emojis: 🚀 New Features, 🐛 Bug Fixes, ⚡ Improvements, 💥 Breaking Changes. Omit sections with no entries. Be concise and developer-friendly. Do not include a title or version header."
},
{
role: "user",
content: ("Generate release notes for version " + $version + " from these commits:\n" + $log)
}
]
}')"
)
NOTES=$(echo "$RESPONSE" | jq -r '.choices[0].message.content // "Release notes unavailable."')
echo "NOTES<<EOF" >> $GITHUB_OUTPUT
echo "$NOTES" >> $GITHUB_OUTPUT
echo "EOF" >> $GITHUB_OUTPUT
- name: 📝 Create release
uses: softprops/action-gh-release@v2
with:
name: "🎉 v${{ needs.setup.outputs.version }}"
tag_name: ${{ needs.setup.outputs.version }}
body: ${{ steps.ai_notes.outputs.NOTES }}
draft: false
prerelease: false
prerelease: ${{ needs.setup.outputs.prerelease }}

View file

@ -53,7 +53,6 @@ COPY default.conf /etc/nginx/sites-available/default
COPY docker-entrypoint.sh /usr/local/bin/
RUN chmod +x /usr/local/bin/docker-entrypoint.sh
RUN chmod +x /app/bin/cleanup
RUN chmod +x /app/bin/proxy
# Create cache, database, and logs folders
RUN mkdir -p /app/cache /app/cache/database /app/logs
@ -65,7 +64,6 @@ RUN chown -R www-data:www-data /app \
# Configure Cron
RUN touch /app/logs/cron.log
RUN echo '0 * * * * root php "/app/bin/cleanup" >> /app/logs/cleanup.log 2>&1' >> /etc/crontab
RUN echo '0 * * * * root php "/app/bin/proxy" >> /app/logs/proxy.log 2>&1' >> /etc/crontab
EXPOSE 80

View file

@ -48,10 +48,5 @@ DEBUG=false
# If not set, no files will be cleaned
CLEANUP_DAYS=7
# Proxy List Configuration
# URL to download proxy list from (used by bin/proxy script)
# The proxy list should contain proxies in one of these formats:
# 1. http://USER:PASSWORD@HOST:PORT
# 2. IP:PORT:USER:PASSWORD
# Example: PROXY_LIST=https://example.com/proxy-list.txt
PROXY_LIST=
# Proxy URL Configuration
# PROXY_URL=

View file

@ -1,196 +0,0 @@
#!/usr/bin/env php
<?php
/**
* Proxy List Cache Updater
*
* Downloads proxy list from the URL specified in the PROXY_LIST environment variable
* and stores it in the cache directory for reuse.
* This script should be run daily via cron to keep the proxy list updated.
*
* Supported proxy list formats:
* 1. http://USER:PASSWORD@HOST:PORT
* 2. IP:PORT:USER:PASSWORD
*/
require_once __DIR__ . '/../vendor/autoload.php';
use League\CLImate\CLImate;
use Dotenv\Dotenv;
use Curl\Curl;
$climate = new CLImate();
$climate->bold()->out('Proxy List Cache Updater');
$climate->br();
try {
$dotenv = Dotenv::createImmutable(__DIR__ . '/..');
$dotenv->load();
$climate->out('Environment variables loaded');
} catch (\Exception $e) {
$climate->yellow()->out('Warning: ' . $e->getMessage());
exit(0);
}
if (!defined('CACHE_DIR')) {
define('CACHE_DIR', __DIR__ . '/../cache');
}
if (!isset($_ENV['PROXY_LIST']) || empty($_ENV['PROXY_LIST'])) {
$climate->yellow()->out('PROXY_LIST environment variable not set. No proxies to cache.');
exit(0);
}
$proxyListUrl = $_ENV['PROXY_LIST'];
$proxyCachePath = CACHE_DIR . '/proxy_list.json';
// Download proxy list from URL
$climate->out('Downloading proxy list from: ' . $proxyListUrl);
$proxyList = downloadProxyList($proxyListUrl, $climate);
if ($proxyList === false) {
$climate->red()->out('Failed to download proxy list from URL: ' . $proxyListUrl);
exit(1);
}
$climate->green()->out('Proxy list downloaded successfully (' . strlen($proxyList) . ' bytes)');
if (!is_dir(CACHE_DIR)) {
if (!mkdir(CACHE_DIR, 0755, true)) {
$climate->red()->out('Failed to create cache directory: ' . CACHE_DIR);
exit(1);
}
}
$climate->out('Parsing proxy list from environment variable...');
$proxies = parseProxyList($proxyList);
if (empty($proxies)) {
$climate->red()->out('No valid proxies found in PROXY_LIST. Supported formats are:');
$climate->red()->out('1. http://USER:PASSWORD@HOST:PORT');
$climate->red()->out('2. IP:PORT:USER:PASSWORD');
exit(1);
}
$climate->out('Found ' . count($proxies) . ' valid proxies.');
if (file_put_contents($proxyCachePath, json_encode($proxies))) {
$climate->green()->out('Proxy list successfully cached to: ' . $proxyCachePath);
} else {
$climate->red()->out('Failed to write proxy list to cache file: ' . $proxyCachePath);
exit(1);
}
/**
* Parse proxy list from environment variable
*
* @param string $proxyListString Proxy list in format http://USER:PASSWORD@HOST:PORT or IP:PORT:USER:PASSWORD
* @return array Array of valid proxy URLs
*/
function parseProxyList($proxyListString) {
$proxies = [];
$lines = preg_split('/[\r\n,]+/', $proxyListString);
foreach ($lines as $line) {
$line = trim($line);
if (empty($line)) continue;
// Format 1: http://USER:PASSWORD@HOST:PORT
if (preg_match('/^https?:\/\/[^:]+:[^@]+@[^:]+:\d+$/i', $line)) {
$proxies[] = $line;
continue;
}
// Format 2: IP:PORT:USER:PASSWORD
if (preg_match('/^(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}):(\d+):([^:]+):(.+)$/', $line, $matches)) {
$ip = $matches[1];
$port = $matches[2];
$user = $matches[3];
$password = $matches[4];
// Convert to standard format
$proxies[] = "http://{$user}:{$password}@{$ip}:{$port}";
}
}
return $proxies;
}
/**
* Download proxy list from URL using php-curl-class
*
* @param string $url URL to download proxy list from
* @param CLImate $climate CLImate instance for output
* @return string|false Downloaded content or false on failure
*/
function downloadProxyList($url, $climate = null) {
$curl = new Curl();
// Configure cURL options
$curl->setTimeout(30);
$curl->setConnectTimeout(10);
$curl->setUserAgent('Marreta Proxy Updater/1.0');
$curl->setHeader('Accept', 'text/plain, text/html, */*');
$curl->setHeader('Accept-Encoding', 'gzip, deflate');
$curl->setOpt(CURLOPT_FOLLOWLOCATION, true);
$curl->setOpt(CURLOPT_MAXREDIRS, 3);
$curl->setOpt(CURLOPT_SSL_VERIFYPEER, false);
$curl->setOpt(CURLOPT_SSL_VERIFYHOST, false);
try {
if ($climate) {
$climate->out('Making HTTP request with php-curl-class...');
}
$curl->get($url);
if ($curl->error) {
$errorMsg = 'cURL request failed: ' . $curl->errorMessage . ' (Code: ' . $curl->errorCode . ')';
if ($climate) {
$climate->red()->out($errorMsg);
} else {
error_log($errorMsg);
}
return false;
}
$statusCode = $curl->httpStatusCode;
if ($climate) {
$climate->out('HTTP Status Code: ' . $statusCode);
}
if ($statusCode === 200) {
$content = $curl->response;
if ($climate) {
$contentType = $curl->responseHeaders['Content-Type'] ?? 'unknown';
$climate->out('Content-Type: ' . $contentType);
$climate->out('Content-Length: ' . strlen($content) . ' bytes');
}
return $content;
}
if ($climate) {
$climate->yellow()->out('Unexpected HTTP status code: ' . $statusCode);
}
return false;
} catch (\Exception $e) {
$errorMsg = 'Unexpected error during download: ' . $e->getMessage();
if ($climate) {
$climate->red()->out($errorMsg);
} else {
error_log($errorMsg);
}
return false;
} finally {
$curl->close();
}
}

View file

@ -21,44 +21,25 @@ class URLAnalyzerFetch extends URLAnalyzerBase
/**
* Sets up the fetch handler with error handling capability
*/
/** @var array List of available proxies */
private $proxyList = [];
/** @var string Path to proxy cache file */
private $proxyCachePath = '';
/** @var string|null Proxy URL from the PROXY_URL environment variable */
private $proxyUrl = null;
public function __construct()
{
parent::__construct();
$this->error = new URLAnalyzerError();
$this->proxyCachePath = __DIR__ . '/../../cache/proxy_list.json';
$this->loadProxyList();
}
/**
* Loads proxy list from cache if available
*/
private function loadProxyList()
{
if (isset($_ENV['PROXY_LIST']) && file_exists($this->proxyCachePath)) {
$cachedList = file_get_contents($this->proxyCachePath);
if (!empty($cachedList)) {
$this->proxyList = json_decode($cachedList, true);
}
if (isset($_ENV['PROXY_URL']) && !empty($_ENV['PROXY_URL'])) {
$this->proxyUrl = $_ENV['PROXY_URL'];
}
}
/**
* Gets a random proxy from the list
* @return string|null Random proxy URL or null if none available
* Gets the configured proxy URL
* @return string|null Proxy URL or null if not configured
*/
private function getRandomProxy()
private function getProxy()
{
if (empty($this->proxyList)) {
return null;
}
return $this->proxyList[array_rand($this->proxyList)];
return $this->proxyUrl;
}
/**
@ -153,7 +134,7 @@ class URLAnalyzerFetch extends URLAnalyzerBase
$curl->setOpt(CURLOPT_ENCODING, '');
if (isset($domainRules['proxy']) && $domainRules['proxy'] === true) {
$proxy = $this->getRandomProxy();
$proxy = $this->getProxy();
if ($proxy) {
$curl->setOpt(CURLOPT_PROXY, $proxy);
}
@ -224,7 +205,7 @@ class URLAnalyzerFetch extends URLAnalyzerBase
$curl->setUserAgent($this->getRandomUserAgent());
if (isset($domainRules['proxy']) && $domainRules['proxy'] === true) {
$proxy = $this->getRandomProxy();
$proxy = $this->getProxy();
if ($proxy) {
$curl->setOpt(CURLOPT_PROXY, $proxy);
}
@ -255,7 +236,7 @@ class URLAnalyzerFetch extends URLAnalyzerBase
$curl->setUserAgent($this->getRandomUserAgent());
if (isset($domainRules['proxy']) && $domainRules['proxy'] === true) {
$proxy = $this->getRandomProxy();
$proxy = $this->getProxy();
if ($proxy) {
$curl->setOpt(CURLOPT_PROXY, $proxy);
}
@ -291,7 +272,7 @@ class URLAnalyzerFetch extends URLAnalyzerBase
}
$useProxy = isset($domainRules['proxy']) && $domainRules['proxy'] === true;
$proxy = $useProxy ? $this->getRandomProxy() : null;
$proxy = $useProxy ? $this->getProxy() : null;
if ($browser === 'chrome') {
$options = new ChromeOptions();

View file

@ -17,7 +17,7 @@ services:
- LOG_LEVEL=${LOG_LEVEL:-WARNING}
- SELENIUM_HOST=${SELENIUM_HOST:-selenium-hub:4444}
- CLEANUP_DAYS=7 # Optional
- PROXY_LIST=url # Optional
- PROXY_URL=url # Optional
restart: unless-stopped
# Selenium
selenium-hub:

View file

@ -122,14 +122,6 @@ log_success "Cron started"
echo -e "\n${GREEN}=== Marreta initialized ===${NC}\n"
# Run proxy list updater
log_info "Running proxy list updater..."
if php /app/bin/proxy; then
log_success "Proxy list updater completed successfully"
else
log_info "Proxy list updater finished (may not have been configured)"
fi
# Wait for any process to exit
wait -n