add restart storage
This commit is contained in:
Yuriy Yuriev
@@ -0,0 +1,336 @@
|
||||
"""
|
||||
Утилита проверки прокси.
|
||||
|
||||
- Удаляет дубликаты по ip:port
|
||||
- Тестирует напрямую (HTTP и SOCKS5)
|
||||
- Тестирует через локальный SOCKS5→HTTP туннель
|
||||
- Сохраняет результат обратно в файл
|
||||
|
||||
Использование:
|
||||
python tools/proxy_checker.py
|
||||
python tools/proxy_checker.py --file proxies.txt --concurrency 10
|
||||
python tools/proxy_checker.py --no-save # только показать, не писать файл
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import sys
|
||||
import time
|
||||
import argparse
|
||||
from pathlib import Path
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Optional
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).parent.parent))
|
||||
|
||||
from managers.proxy_manager import Proxy, ProxyType
|
||||
|
||||
TEST_URL = "https://www.google.com/generate_204"
|
||||
TIMEOUT = 10.0
|
||||
|
||||
|
||||
@dataclass
|
||||
class CheckResult:
|
||||
proxy: Proxy
|
||||
direct_ok: bool = False
|
||||
direct_ms: Optional[float] = None
|
||||
tunnel_ok: bool = False
|
||||
tunnel_ms: Optional[float] = None
|
||||
error: Optional[str] = None
|
||||
|
||||
@property
|
||||
def ok(self) -> bool:
|
||||
return self.direct_ok or self.tunnel_ok
|
||||
|
||||
def summary(self) -> str:
|
||||
parts = []
|
||||
if self.direct_ok:
|
||||
parts.append(f"direct {self.direct_ms:.0f}ms")
|
||||
if self.tunnel_ok:
|
||||
parts.append(f"tunnel {self.tunnel_ms:.0f}ms")
|
||||
if not parts:
|
||||
return f"FAIL {self.error or ''}"
|
||||
return "OK " + " | ".join(parts)
|
||||
|
||||
|
||||
async def _check_direct(proxy: Proxy, timeout: float) -> tuple[bool, Optional[float]]:
|
||||
"""Прямая проверка через requests в thread pool."""
|
||||
import asyncio
|
||||
loop = asyncio.get_running_loop()
|
||||
|
||||
def _sync():
|
||||
import requests
|
||||
t0 = time.perf_counter()
|
||||
scheme = proxy.proxy_type.value
|
||||
if proxy.login and proxy.password:
|
||||
url = f"{scheme}://{proxy.login}:{proxy.password}@{proxy.ip}:{proxy.port}"
|
||||
else:
|
||||
url = f"{scheme}://{proxy.ip}:{proxy.port}"
|
||||
proxies = {"http": url, "https": url}
|
||||
try:
|
||||
r = requests.get(TEST_URL, proxies=proxies, timeout=timeout)
|
||||
ms = (time.perf_counter() - t0) * 1000
|
||||
return r.status_code in (200, 204), ms
|
||||
except Exception:
|
||||
return False, None
|
||||
|
||||
return await loop.run_in_executor(None, _sync)
|
||||
|
||||
|
||||
async def _check_tunnel(proxy: Proxy, timeout: float) -> tuple[bool, Optional[float]]:
|
||||
"""Проверка через SOCKS5→HTTP туннель (только для SOCKS5/SOCKS4)."""
|
||||
if proxy.proxy_type not in (ProxyType.SOCKS5, ProxyType.SOCKS4):
|
||||
return False, None
|
||||
|
||||
from services.socks5_to_http_proxy import Socks5ToHttpProxy
|
||||
import aiohttp
|
||||
|
||||
tunnel = Socks5ToHttpProxy(
|
||||
socks5_host=proxy.ip,
|
||||
socks5_port=int(proxy.port),
|
||||
username=proxy.login,
|
||||
password=proxy.password,
|
||||
)
|
||||
try:
|
||||
await asyncio.wait_for(tunnel.start(), timeout=5)
|
||||
t0 = time.perf_counter()
|
||||
ok = await tunnel.check_connection(test_url=TEST_URL, timeout=timeout)
|
||||
ms = (time.perf_counter() - t0) * 1000 if ok else None
|
||||
return ok, ms
|
||||
except Exception:
|
||||
return False, None
|
||||
finally:
|
||||
try:
|
||||
await asyncio.wait_for(tunnel.stop(), timeout=3)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
async def _check_direct_as_http(proxy: Proxy, timeout: float) -> tuple[bool, Optional[float]]:
|
||||
"""Принудительная проверка как HTTP прокси (игнорирует настроенный тип)."""
|
||||
loop = asyncio.get_running_loop()
|
||||
|
||||
def _sync():
|
||||
import requests
|
||||
t0 = time.perf_counter()
|
||||
if proxy.login and proxy.password:
|
||||
url = f"http://{proxy.login}:{proxy.password}@{proxy.ip}:{proxy.port}"
|
||||
else:
|
||||
url = f"http://{proxy.ip}:{proxy.port}"
|
||||
proxies = {"http": url, "https": url}
|
||||
try:
|
||||
r = requests.get(TEST_URL, proxies=proxies, timeout=timeout)
|
||||
ms = (time.perf_counter() - t0) * 1000
|
||||
return r.status_code in (200, 204), ms
|
||||
except Exception:
|
||||
return False, None
|
||||
|
||||
return await loop.run_in_executor(None, _sync)
|
||||
|
||||
|
||||
async def check_proxy(proxy: Proxy, timeout: float) -> CheckResult:
|
||||
result = CheckResult(proxy=proxy)
|
||||
try:
|
||||
direct_ok, direct_ms = await asyncio.wait_for(
|
||||
_check_direct(proxy, timeout), timeout=timeout + 2
|
||||
)
|
||||
result.direct_ok = direct_ok
|
||||
result.direct_ms = direct_ms
|
||||
|
||||
if proxy.proxy_type in (ProxyType.SOCKS5, ProxyType.SOCKS4):
|
||||
tunnel_ok, tunnel_ms = await asyncio.wait_for(
|
||||
_check_tunnel(proxy, timeout), timeout=timeout + 5
|
||||
)
|
||||
result.tunnel_ok = tunnel_ok
|
||||
result.tunnel_ms = tunnel_ms
|
||||
|
||||
except Exception as e:
|
||||
result.error = str(e)[:60]
|
||||
return result
|
||||
|
||||
|
||||
async def retry_failed_as_http(results: list[CheckResult], timeout: float, sem: asyncio.Semaphore) -> dict[str, CheckResult]:
|
||||
"""
|
||||
Повторно проверяет упавшие прокси принудительно как HTTP.
|
||||
Возвращает dict[ip:port -> CheckResult] только для тех что теперь прошли.
|
||||
"""
|
||||
failed = [r for r in results if not r.ok]
|
||||
if not failed:
|
||||
return {}
|
||||
|
||||
recovered: dict[str, CheckResult] = {}
|
||||
|
||||
async def _retry(r: CheckResult):
|
||||
async with sem:
|
||||
try:
|
||||
ok, ms = await asyncio.wait_for(
|
||||
_check_direct_as_http(r.proxy, timeout), timeout=timeout + 2
|
||||
)
|
||||
except (TimeoutError, asyncio.TimeoutError, asyncio.CancelledError, Exception):
|
||||
ok, ms = False, None
|
||||
if ok:
|
||||
key = f"{r.proxy.ip}:{r.proxy.port}"
|
||||
r.direct_ok = True
|
||||
r.direct_ms = ms
|
||||
r.error = None
|
||||
r.proxy.proxy_type = ProxyType.HTTP
|
||||
recovered[key] = r
|
||||
|
||||
await asyncio.gather(*[_retry(r) for r in failed], return_exceptions=True)
|
||||
return recovered
|
||||
|
||||
|
||||
def _load_proxies(file_path: str) -> tuple[list[Proxy], list[str]]:
|
||||
"""Загружает прокси из файла. Возвращает (список прокси, сырые строки)."""
|
||||
raw_lines = Path(file_path).read_text(encoding="utf-8").splitlines()
|
||||
proxies = []
|
||||
seen = set() # ip:port для дедупликации
|
||||
|
||||
for line in raw_lines:
|
||||
stripped = line.strip()
|
||||
if not stripped or stripped.startswith("#") or stripped.startswith("!"):
|
||||
continue
|
||||
p = Proxy.from_line(stripped)
|
||||
if not p:
|
||||
continue
|
||||
key = f"{p.ip}:{p.port}"
|
||||
if key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
proxies.append(p)
|
||||
|
||||
return proxies, raw_lines
|
||||
|
||||
|
||||
def _save_results(file_path: str, results: list[CheckResult], raw_lines: list[str]):
|
||||
"""Перезаписывает файл: рабочие без '!', нерабочие с '!'.
|
||||
Использует актуальный тип прокси из results (может быть изменён retry-фазой).
|
||||
"""
|
||||
# ip:port → актуальный объект Proxy с возможно обновлённым типом
|
||||
proxy_map: dict[str, Proxy] = {
|
||||
f"{r.proxy.ip}:{r.proxy.port}": r.proxy for r in results
|
||||
}
|
||||
working = {f"{r.proxy.ip}:{r.proxy.port}" for r in results if r.ok}
|
||||
|
||||
new_lines = []
|
||||
seen_keys: set[str] = set() # для удаления дубликатов
|
||||
|
||||
for line in raw_lines:
|
||||
stripped = line.strip()
|
||||
if not stripped or stripped.startswith("#"):
|
||||
new_lines.append(line)
|
||||
continue
|
||||
|
||||
clean = stripped.lstrip("!")
|
||||
p = Proxy.from_line(clean)
|
||||
if not p:
|
||||
new_lines.append(line)
|
||||
continue
|
||||
|
||||
key = f"{p.ip}:{p.port}"
|
||||
if key in seen_keys:
|
||||
continue # дубликат — пропускаем
|
||||
seen_keys.add(key)
|
||||
|
||||
actual = proxy_map.get(key, p)
|
||||
scheme = actual.proxy_type.value
|
||||
if actual.login and actual.password:
|
||||
formatted = f"{scheme}://{actual.ip}:{actual.port}:{actual.login}:{actual.password}"
|
||||
else:
|
||||
formatted = f"{scheme}://{actual.ip}:{actual.port}"
|
||||
|
||||
if key in working:
|
||||
new_lines.append(formatted)
|
||||
else:
|
||||
new_lines.append(f"!{formatted}")
|
||||
|
||||
Path(file_path).write_text("\n".join(new_lines) + "\n", encoding="utf-8")
|
||||
|
||||
|
||||
async def run_checker(file_path: str, concurrency: int, timeout: float, save: bool, retry_http: bool):
|
||||
proxies, raw_lines = _load_proxies(file_path)
|
||||
|
||||
total_raw = sum(
|
||||
1 for l in raw_lines
|
||||
if l.strip() and not l.strip().startswith("#") and not l.strip().startswith("!")
|
||||
)
|
||||
duplicates = total_raw - len(proxies)
|
||||
|
||||
print(f"\n{'='*60}")
|
||||
print(f" Файл: {file_path}")
|
||||
print(f" Загружено: {total_raw} | Дубликатов: {duplicates} | К проверке: {len(proxies)}")
|
||||
print(f" Параллельность: {concurrency} | Таймаут: {timeout}s")
|
||||
if retry_http:
|
||||
print(f" Повтор упавших через HTTP: ВКЛ")
|
||||
print(f"{'='*60}\n")
|
||||
|
||||
sem = asyncio.Semaphore(concurrency)
|
||||
results: list[CheckResult] = []
|
||||
done = 0
|
||||
|
||||
async def _run(proxy: Proxy):
|
||||
nonlocal done
|
||||
async with sem:
|
||||
r = await check_proxy(proxy, timeout)
|
||||
results.append(r)
|
||||
done += 1
|
||||
status = "✅" if r.ok else "❌"
|
||||
print(f" [{done:>3}/{len(proxies)}] {status} {proxy.id:<35} {r.summary()}")
|
||||
|
||||
await asyncio.gather(*[_run(p) for p in proxies])
|
||||
|
||||
# Фаза 2: повторная проверка упавших как HTTP
|
||||
recovered_count = 0
|
||||
if retry_http:
|
||||
failed_count = sum(1 for r in results if not r.ok)
|
||||
if failed_count:
|
||||
print(f"\n ── Повтор {failed_count} упавших как HTTP прямую ──\n")
|
||||
recovered = await retry_failed_as_http(results, timeout, sem)
|
||||
recovered_count = len(recovered)
|
||||
for key, r in recovered.items():
|
||||
print(f" [retry] ✅ {r.proxy.ip}:{r.proxy.port} → HTTP {r.direct_ms:.0f}ms")
|
||||
|
||||
ok_count = sum(1 for r in results if r.ok)
|
||||
fail_count = len(results) - ok_count
|
||||
direct_ok = sum(1 for r in results if r.direct_ok)
|
||||
tunnel_ok = sum(1 for r in results if r.tunnel_ok)
|
||||
|
||||
print(f"\n{'='*60}")
|
||||
print(f" ИТОГО")
|
||||
print(f"{'='*60}")
|
||||
print(f" Рабочих : {ok_count} / {len(proxies)}")
|
||||
print(f" Нерабочих : {fail_count}")
|
||||
print(f" Direct OK : {direct_ok}")
|
||||
print(f" Tunnel OK : {tunnel_ok}")
|
||||
if recovered_count:
|
||||
print(f" Спасено HTTP: {recovered_count}")
|
||||
if duplicates:
|
||||
print(f" Дубликатов : {duplicates} (удалены)")
|
||||
|
||||
if save:
|
||||
_save_results(file_path, results, raw_lines)
|
||||
print(f"\n Файл обновлён: {file_path}")
|
||||
print(f" Нерабочие помечены '!'")
|
||||
|
||||
print(f"{'='*60}\n")
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Проверка прокси")
|
||||
parser.add_argument("--file", default="proxies.txt", help="Файл с прокси")
|
||||
parser.add_argument("--concurrency", type=int, default=5, help="Параллельность (по умолчанию: 5)")
|
||||
parser.add_argument("--timeout", type=float, default=10.0, help="Таймаут секунд (по умолчанию: 10)")
|
||||
parser.add_argument("--no-save", action="store_true", help="Не перезаписывать файл")
|
||||
parser.add_argument("--retry-http", action="store_true", help="Повторить упавшие как HTTP напрямую")
|
||||
args = parser.parse_args()
|
||||
|
||||
asyncio.run(run_checker(
|
||||
file_path=args.file,
|
||||
concurrency=args.concurrency,
|
||||
timeout=args.timeout,
|
||||
save=not args.no_save,
|
||||
retry_http=args.retry_http,
|
||||
))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user