add register
This commit is contained in:
@@ -3,7 +3,7 @@
|
||||
import asyncio
|
||||
import logging
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Dict, Optional, Callable, Coroutine, Any
|
||||
from typing import Dict, Coroutine
|
||||
|
||||
from core.constants import TaskStatus
|
||||
|
||||
@@ -77,21 +77,14 @@ class BackgroundTaskManager:
|
||||
return True
|
||||
|
||||
def _on_task_done(self, task_id: str, task: asyncio.Task) -> None:
|
||||
"""Callback когда задача завершилась."""
|
||||
try:
|
||||
exc = task.exception()
|
||||
if exc:
|
||||
if isinstance(exc, asyncio.CancelledError):
|
||||
logger.info(f"Task {task_id} cancelled (normal)")
|
||||
# Не считаем ошибкой
|
||||
else:
|
||||
logger.error(f"Task {task_id} failed: {exc}")
|
||||
else:
|
||||
logger.info(f"Task {task_id} completed successfully")
|
||||
except asyncio.CancelledError:
|
||||
logger.info(f"Task {task_id} was cancelled")
|
||||
except Exception as e:
|
||||
logger.error(f"Error in task callback: {e}")
|
||||
if task.cancelled():
|
||||
logger.info(f"Task {task_id} cancelled")
|
||||
return
|
||||
exc = task.exception()
|
||||
if exc:
|
||||
logger.error(f"Task {task_id} failed: {exc}")
|
||||
else:
|
||||
logger.info(f"Task {task_id} completed successfully")
|
||||
|
||||
async def cancel_task(self, task_id: str) -> bool:
|
||||
"""
|
||||
@@ -126,7 +119,7 @@ class BackgroundTaskManager:
|
||||
cancelled = 0
|
||||
|
||||
async with self._lock:
|
||||
for task_id, task in list(self._tasks.items()):
|
||||
for task in list(self._tasks.values()):
|
||||
if not task.done():
|
||||
task.cancel()
|
||||
try:
|
||||
|
||||
+30
-17
@@ -4,9 +4,10 @@
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
import random
|
||||
from collections import defaultdict
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Optional, List, Dict
|
||||
from typing import Optional, List, Dict, Set
|
||||
from enum import Enum
|
||||
import logging
|
||||
|
||||
@@ -119,7 +120,8 @@ class ProxyManager:
|
||||
self._usage_count: Dict[str, int] = defaultdict(int)
|
||||
self._cooldown_until: Dict[str, float] = {}
|
||||
self._working_status: Dict[str, bool] = {}
|
||||
|
||||
self._used_ids: Set[str] = set()
|
||||
|
||||
self._load_proxies()
|
||||
|
||||
def _load_proxies(self) -> None:
|
||||
@@ -159,59 +161,68 @@ class ProxyManager:
|
||||
def _is_available(self, proxy: Proxy) -> bool:
|
||||
"""Проверка доступности прокси."""
|
||||
proxy_id = proxy.id
|
||||
|
||||
|
||||
if proxy_id in self._in_use:
|
||||
return False
|
||||
|
||||
|
||||
if proxy_id in self._used_ids:
|
||||
return False
|
||||
|
||||
if proxy_id in self._cooldown_until:
|
||||
loop = asyncio.get_running_loop()
|
||||
if loop.time() < self._cooldown_until[proxy_id]:
|
||||
return False
|
||||
del self._cooldown_until[proxy_id]
|
||||
|
||||
|
||||
if self._usage_count[proxy_id] >= self.max_usage:
|
||||
return False
|
||||
|
||||
|
||||
return self._working_status.get(proxy_id, False)
|
||||
|
||||
async def acquire_proxy(self, timeout: float = 60) -> Optional[Proxy]:
|
||||
"""Получает доступный прокси."""
|
||||
if not self._proxies:
|
||||
return None
|
||||
|
||||
|
||||
loop = asyncio.get_running_loop()
|
||||
start_time = loop.time()
|
||||
attempt = 0
|
||||
|
||||
|
||||
while True:
|
||||
attempt += 1
|
||||
|
||||
|
||||
async with self._lock:
|
||||
available = [
|
||||
p for p in self._proxies
|
||||
if self._is_available(p)
|
||||
]
|
||||
|
||||
|
||||
if available:
|
||||
proxy = min(available, key=lambda p: self._usage_count[p.id])
|
||||
|
||||
proxy = random.choice(available)
|
||||
|
||||
self._in_use[proxy.id] = asyncio.Event()
|
||||
self._usage_count[proxy.id] += 1
|
||||
|
||||
self._used_ids.add(proxy.id)
|
||||
|
||||
logger.info(
|
||||
f"Proxy acquired: {proxy.id} "
|
||||
f"(type: {proxy.proxy_type.value}, "
|
||||
f"used: {self._usage_count[proxy.id]}x)"
|
||||
)
|
||||
return proxy
|
||||
|
||||
|
||||
all_used = len(self._used_ids) >= len(self._proxies)
|
||||
if all_used:
|
||||
logger.info("All proxies used once, resetting used list")
|
||||
self._used_ids.clear()
|
||||
|
||||
logger.debug(f"No proxies available (attempt {attempt})")
|
||||
|
||||
|
||||
elapsed = loop.time() - start_time
|
||||
if elapsed >= timeout:
|
||||
logger.warning(f"Proxy acquire timeout ({timeout}s)")
|
||||
return None
|
||||
|
||||
|
||||
await asyncio.sleep(1)
|
||||
|
||||
async def release_proxy(self, proxy: Proxy, success: bool = True) -> None:
|
||||
@@ -256,11 +267,13 @@ class ProxyManager:
|
||||
|
||||
@property
|
||||
def available_count(self) -> int:
|
||||
loop = asyncio.get_running_loop()
|
||||
now = loop.time()
|
||||
working = sum(1 for s in self._working_status.values() if s)
|
||||
in_use = len(self._in_use)
|
||||
in_cooldown = sum(
|
||||
1 for t in self._cooldown_until.values()
|
||||
if asyncio.get_event_loop().time() < t
|
||||
if now < t
|
||||
)
|
||||
return working - in_use - in_cooldown
|
||||
|
||||
|
||||
Reference in New Issue
Block a user