Yuriy Yuriev
337 lines
12 KiB
Python
337 lines
12 KiB
Python
"""
|
||
Утилита проверки прокси.
|
||
|
||
- Удаляет дубликаты по ip:port
|
||
- Тестирует напрямую (HTTP и SOCKS5)
|
||
- Тестирует через локальный SOCKS5→HTTP туннель
|
||
- Сохраняет результат обратно в файл
|
||
|
||
Использование:
|
||
python tools/proxy_checker.py
|
||
python tools/proxy_checker.py --file proxies.txt --concurrency 10
|
||
python tools/proxy_checker.py --no-save # только показать, не писать файл
|
||
"""
|
||
|
||
import asyncio
|
||
import sys
|
||
import time
|
||
import argparse
|
||
from pathlib import Path
|
||
from dataclasses import dataclass, field
|
||
from typing import Optional
|
||
|
||
sys.path.insert(0, str(Path(__file__).parent.parent))
|
||
|
||
from managers.proxy_manager import Proxy, ProxyType
|
||
|
||
TEST_URL = "https://www.google.com/generate_204"
|
||
TIMEOUT = 10.0
|
||
|
||
|
||
@dataclass
|
||
class CheckResult:
|
||
proxy: Proxy
|
||
direct_ok: bool = False
|
||
direct_ms: Optional[float] = None
|
||
tunnel_ok: bool = False
|
||
tunnel_ms: Optional[float] = None
|
||
error: Optional[str] = None
|
||
|
||
@property
|
||
def ok(self) -> bool:
|
||
return self.direct_ok or self.tunnel_ok
|
||
|
||
def summary(self) -> str:
|
||
parts = []
|
||
if self.direct_ok:
|
||
parts.append(f"direct {self.direct_ms:.0f}ms")
|
||
if self.tunnel_ok:
|
||
parts.append(f"tunnel {self.tunnel_ms:.0f}ms")
|
||
if not parts:
|
||
return f"FAIL {self.error or ''}"
|
||
return "OK " + " | ".join(parts)
|
||
|
||
|
||
async def _check_direct(proxy: Proxy, timeout: float) -> tuple[bool, Optional[float]]:
|
||
"""Прямая проверка через requests в thread pool."""
|
||
import asyncio
|
||
loop = asyncio.get_running_loop()
|
||
|
||
def _sync():
|
||
import requests
|
||
t0 = time.perf_counter()
|
||
scheme = proxy.proxy_type.value
|
||
if proxy.login and proxy.password:
|
||
url = f"{scheme}://{proxy.login}:{proxy.password}@{proxy.ip}:{proxy.port}"
|
||
else:
|
||
url = f"{scheme}://{proxy.ip}:{proxy.port}"
|
||
proxies = {"http": url, "https": url}
|
||
try:
|
||
r = requests.get(TEST_URL, proxies=proxies, timeout=timeout)
|
||
ms = (time.perf_counter() - t0) * 1000
|
||
return r.status_code in (200, 204), ms
|
||
except Exception:
|
||
return False, None
|
||
|
||
return await loop.run_in_executor(None, _sync)
|
||
|
||
|
||
async def _check_tunnel(proxy: Proxy, timeout: float) -> tuple[bool, Optional[float]]:
|
||
"""Проверка через SOCKS5→HTTP туннель (только для SOCKS5/SOCKS4)."""
|
||
if proxy.proxy_type not in (ProxyType.SOCKS5, ProxyType.SOCKS4):
|
||
return False, None
|
||
|
||
from services.socks5_to_http_proxy import Socks5ToHttpProxy
|
||
import aiohttp
|
||
|
||
tunnel = Socks5ToHttpProxy(
|
||
socks5_host=proxy.ip,
|
||
socks5_port=int(proxy.port),
|
||
username=proxy.login,
|
||
password=proxy.password,
|
||
)
|
||
try:
|
||
await asyncio.wait_for(tunnel.start(), timeout=5)
|
||
t0 = time.perf_counter()
|
||
ok = await tunnel.check_connection(test_url=TEST_URL, timeout=timeout)
|
||
ms = (time.perf_counter() - t0) * 1000 if ok else None
|
||
return ok, ms
|
||
except Exception:
|
||
return False, None
|
||
finally:
|
||
try:
|
||
await asyncio.wait_for(tunnel.stop(), timeout=3)
|
||
except Exception:
|
||
pass
|
||
|
||
|
||
async def _check_direct_as_http(proxy: Proxy, timeout: float) -> tuple[bool, Optional[float]]:
|
||
"""Принудительная проверка как HTTP прокси (игнорирует настроенный тип)."""
|
||
loop = asyncio.get_running_loop()
|
||
|
||
def _sync():
|
||
import requests
|
||
t0 = time.perf_counter()
|
||
if proxy.login and proxy.password:
|
||
url = f"http://{proxy.login}:{proxy.password}@{proxy.ip}:{proxy.port}"
|
||
else:
|
||
url = f"http://{proxy.ip}:{proxy.port}"
|
||
proxies = {"http": url, "https": url}
|
||
try:
|
||
r = requests.get(TEST_URL, proxies=proxies, timeout=timeout)
|
||
ms = (time.perf_counter() - t0) * 1000
|
||
return r.status_code in (200, 204), ms
|
||
except Exception:
|
||
return False, None
|
||
|
||
return await loop.run_in_executor(None, _sync)
|
||
|
||
|
||
async def check_proxy(proxy: Proxy, timeout: float) -> CheckResult:
|
||
result = CheckResult(proxy=proxy)
|
||
try:
|
||
direct_ok, direct_ms = await asyncio.wait_for(
|
||
_check_direct(proxy, timeout), timeout=timeout + 2
|
||
)
|
||
result.direct_ok = direct_ok
|
||
result.direct_ms = direct_ms
|
||
|
||
if proxy.proxy_type in (ProxyType.SOCKS5, ProxyType.SOCKS4):
|
||
tunnel_ok, tunnel_ms = await asyncio.wait_for(
|
||
_check_tunnel(proxy, timeout), timeout=timeout + 5
|
||
)
|
||
result.tunnel_ok = tunnel_ok
|
||
result.tunnel_ms = tunnel_ms
|
||
|
||
except Exception as e:
|
||
result.error = str(e)[:60]
|
||
return result
|
||
|
||
|
||
async def retry_failed_as_http(results: list[CheckResult], timeout: float, sem: asyncio.Semaphore) -> dict[str, CheckResult]:
|
||
"""
|
||
Повторно проверяет упавшие прокси принудительно как HTTP.
|
||
Возвращает dict[ip:port -> CheckResult] только для тех что теперь прошли.
|
||
"""
|
||
failed = [r for r in results if not r.ok]
|
||
if not failed:
|
||
return {}
|
||
|
||
recovered: dict[str, CheckResult] = {}
|
||
|
||
async def _retry(r: CheckResult):
|
||
async with sem:
|
||
try:
|
||
ok, ms = await asyncio.wait_for(
|
||
_check_direct_as_http(r.proxy, timeout), timeout=timeout + 2
|
||
)
|
||
except (TimeoutError, asyncio.TimeoutError, asyncio.CancelledError, Exception):
|
||
ok, ms = False, None
|
||
if ok:
|
||
key = f"{r.proxy.ip}:{r.proxy.port}"
|
||
r.direct_ok = True
|
||
r.direct_ms = ms
|
||
r.error = None
|
||
r.proxy.proxy_type = ProxyType.HTTP
|
||
recovered[key] = r
|
||
|
||
await asyncio.gather(*[_retry(r) for r in failed], return_exceptions=True)
|
||
return recovered
|
||
|
||
|
||
def _load_proxies(file_path: str) -> tuple[list[Proxy], list[str]]:
|
||
"""Загружает прокси из файла. Возвращает (список прокси, сырые строки)."""
|
||
raw_lines = Path(file_path).read_text(encoding="utf-8").splitlines()
|
||
proxies = []
|
||
seen = set() # ip:port для дедупликации
|
||
|
||
for line in raw_lines:
|
||
stripped = line.strip()
|
||
if not stripped or stripped.startswith("#") or stripped.startswith("!"):
|
||
continue
|
||
p = Proxy.from_line(stripped)
|
||
if not p:
|
||
continue
|
||
key = f"{p.ip}:{p.port}"
|
||
if key in seen:
|
||
continue
|
||
seen.add(key)
|
||
proxies.append(p)
|
||
|
||
return proxies, raw_lines
|
||
|
||
|
||
def _save_results(file_path: str, results: list[CheckResult], raw_lines: list[str]):
|
||
"""Перезаписывает файл: рабочие без '!', нерабочие с '!'.
|
||
Использует актуальный тип прокси из results (может быть изменён retry-фазой).
|
||
"""
|
||
# ip:port → актуальный объект Proxy с возможно обновлённым типом
|
||
proxy_map: dict[str, Proxy] = {
|
||
f"{r.proxy.ip}:{r.proxy.port}": r.proxy for r in results
|
||
}
|
||
working = {f"{r.proxy.ip}:{r.proxy.port}" for r in results if r.ok}
|
||
|
||
new_lines = []
|
||
seen_keys: set[str] = set() # для удаления дубликатов
|
||
|
||
for line in raw_lines:
|
||
stripped = line.strip()
|
||
if not stripped or stripped.startswith("#"):
|
||
new_lines.append(line)
|
||
continue
|
||
|
||
clean = stripped.lstrip("!")
|
||
p = Proxy.from_line(clean)
|
||
if not p:
|
||
new_lines.append(line)
|
||
continue
|
||
|
||
key = f"{p.ip}:{p.port}"
|
||
if key in seen_keys:
|
||
continue # дубликат — пропускаем
|
||
seen_keys.add(key)
|
||
|
||
actual = proxy_map.get(key, p)
|
||
scheme = actual.proxy_type.value
|
||
if actual.login and actual.password:
|
||
formatted = f"{scheme}://{actual.ip}:{actual.port}:{actual.login}:{actual.password}"
|
||
else:
|
||
formatted = f"{scheme}://{actual.ip}:{actual.port}"
|
||
|
||
if key in working:
|
||
new_lines.append(formatted)
|
||
else:
|
||
new_lines.append(f"!{formatted}")
|
||
|
||
Path(file_path).write_text("\n".join(new_lines) + "\n", encoding="utf-8")
|
||
|
||
|
||
async def run_checker(file_path: str, concurrency: int, timeout: float, save: bool, retry_http: bool):
|
||
proxies, raw_lines = _load_proxies(file_path)
|
||
|
||
total_raw = sum(
|
||
1 for l in raw_lines
|
||
if l.strip() and not l.strip().startswith("#") and not l.strip().startswith("!")
|
||
)
|
||
duplicates = total_raw - len(proxies)
|
||
|
||
print(f"\n{'='*60}")
|
||
print(f" Файл: {file_path}")
|
||
print(f" Загружено: {total_raw} | Дубликатов: {duplicates} | К проверке: {len(proxies)}")
|
||
print(f" Параллельность: {concurrency} | Таймаут: {timeout}s")
|
||
if retry_http:
|
||
print(f" Повтор упавших через HTTP: ВКЛ")
|
||
print(f"{'='*60}\n")
|
||
|
||
sem = asyncio.Semaphore(concurrency)
|
||
results: list[CheckResult] = []
|
||
done = 0
|
||
|
||
async def _run(proxy: Proxy):
|
||
nonlocal done
|
||
async with sem:
|
||
r = await check_proxy(proxy, timeout)
|
||
results.append(r)
|
||
done += 1
|
||
status = "✅" if r.ok else "❌"
|
||
print(f" [{done:>3}/{len(proxies)}] {status} {proxy.id:<35} {r.summary()}")
|
||
|
||
await asyncio.gather(*[_run(p) for p in proxies])
|
||
|
||
# Фаза 2: повторная проверка упавших как HTTP
|
||
recovered_count = 0
|
||
if retry_http:
|
||
failed_count = sum(1 for r in results if not r.ok)
|
||
if failed_count:
|
||
print(f"\n ── Повтор {failed_count} упавших как HTTP прямую ──\n")
|
||
recovered = await retry_failed_as_http(results, timeout, sem)
|
||
recovered_count = len(recovered)
|
||
for key, r in recovered.items():
|
||
print(f" [retry] ✅ {r.proxy.ip}:{r.proxy.port} → HTTP {r.direct_ms:.0f}ms")
|
||
|
||
ok_count = sum(1 for r in results if r.ok)
|
||
fail_count = len(results) - ok_count
|
||
direct_ok = sum(1 for r in results if r.direct_ok)
|
||
tunnel_ok = sum(1 for r in results if r.tunnel_ok)
|
||
|
||
print(f"\n{'='*60}")
|
||
print(f" ИТОГО")
|
||
print(f"{'='*60}")
|
||
print(f" Рабочих : {ok_count} / {len(proxies)}")
|
||
print(f" Нерабочих : {fail_count}")
|
||
print(f" Direct OK : {direct_ok}")
|
||
print(f" Tunnel OK : {tunnel_ok}")
|
||
if recovered_count:
|
||
print(f" Спасено HTTP: {recovered_count}")
|
||
if duplicates:
|
||
print(f" Дубликатов : {duplicates} (удалены)")
|
||
|
||
if save:
|
||
_save_results(file_path, results, raw_lines)
|
||
print(f"\n Файл обновлён: {file_path}")
|
||
print(f" Нерабочие помечены '!'")
|
||
|
||
print(f"{'='*60}\n")
|
||
|
||
|
||
def main():
|
||
parser = argparse.ArgumentParser(description="Проверка прокси")
|
||
parser.add_argument("--file", default="proxies.txt", help="Файл с прокси")
|
||
parser.add_argument("--concurrency", type=int, default=5, help="Параллельность (по умолчанию: 5)")
|
||
parser.add_argument("--timeout", type=float, default=10.0, help="Таймаут секунд (по умолчанию: 10)")
|
||
parser.add_argument("--no-save", action="store_true", help="Не перезаписывать файл")
|
||
parser.add_argument("--retry-http", action="store_true", help="Повторить упавшие как HTTP напрямую")
|
||
args = parser.parse_args()
|
||
|
||
asyncio.run(run_checker(
|
||
file_path=args.file,
|
||
concurrency=args.concurrency,
|
||
timeout=args.timeout,
|
||
save=not args.no_save,
|
||
retry_http=args.retry_http,
|
||
))
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|