fix proxy ip
This commit is contained in:
+10
-21
@@ -7,56 +7,45 @@ from logging.handlers import RotatingFileHandler
|
|||||||
|
|
||||||
|
|
||||||
def setup_logger(
|
def setup_logger(
|
||||||
name: str = "bot",
|
name: str = None,
|
||||||
log_level: int = logging.INFO,
|
log_level: int = logging.INFO,
|
||||||
log_file: str = "logs/bot.log"
|
log_file: str = "logs/bot.log"
|
||||||
) -> logging.Logger:
|
) -> logging.Logger:
|
||||||
"""
|
root = logging.getLogger()
|
||||||
Configure and return logger instance.
|
|
||||||
|
|
||||||
Args:
|
# Не добавляем хендлеры повторно
|
||||||
name: Logger name
|
if root.handlers:
|
||||||
log_level: Logging level
|
return logging.getLogger(name) if name else root
|
||||||
log_file: Path to log file
|
|
||||||
|
|
||||||
Returns:
|
root.setLevel(log_level)
|
||||||
Configured logger instance
|
|
||||||
"""
|
|
||||||
logger = logging.getLogger(name)
|
|
||||||
logger.setLevel(log_level)
|
|
||||||
|
|
||||||
# Create formatters
|
|
||||||
console_formatter = logging.Formatter(
|
console_formatter = logging.Formatter(
|
||||||
'%(asctime)s | %(levelname)-8s | %(name)s | %(message)s',
|
'%(asctime)s | %(levelname)-8s | %(name)s | %(message)s',
|
||||||
datefmt='%Y-%m-%d %H:%M:%S'
|
datefmt='%Y-%m-%d %H:%M:%S'
|
||||||
)
|
)
|
||||||
|
|
||||||
file_formatter = logging.Formatter(
|
file_formatter = logging.Formatter(
|
||||||
'%(asctime)s | %(levelname)-8s | %(name)s | %(funcName)s:%(lineno)d | %(message)s',
|
'%(asctime)s | %(levelname)-8s | %(name)s | %(funcName)s:%(lineno)d | %(message)s',
|
||||||
datefmt='%Y-%m-%d %H:%M:%S'
|
datefmt='%Y-%m-%d %H:%M:%S'
|
||||||
)
|
)
|
||||||
|
|
||||||
# Console handler
|
|
||||||
console_handler = logging.StreamHandler(sys.stdout)
|
console_handler = logging.StreamHandler(sys.stdout)
|
||||||
console_handler.setLevel(log_level)
|
console_handler.setLevel(log_level)
|
||||||
console_handler.setFormatter(console_formatter)
|
console_handler.setFormatter(console_formatter)
|
||||||
|
root.addHandler(console_handler)
|
||||||
|
|
||||||
# File handler
|
|
||||||
if log_file:
|
if log_file:
|
||||||
Path(log_file).parent.mkdir(parents=True, exist_ok=True)
|
Path(log_file).parent.mkdir(parents=True, exist_ok=True)
|
||||||
file_handler = RotatingFileHandler(
|
file_handler = RotatingFileHandler(
|
||||||
log_file,
|
log_file,
|
||||||
maxBytes=10*1024*1024, # 10MB
|
maxBytes=10 * 1024 * 1024,
|
||||||
backupCount=5,
|
backupCount=5,
|
||||||
encoding='utf-8'
|
encoding='utf-8'
|
||||||
)
|
)
|
||||||
file_handler.setLevel(log_level)
|
file_handler.setLevel(log_level)
|
||||||
file_handler.setFormatter(file_formatter)
|
file_handler.setFormatter(file_formatter)
|
||||||
logger.addHandler(file_handler)
|
root.addHandler(file_handler)
|
||||||
|
|
||||||
logger.addHandler(console_handler)
|
return logging.getLogger(name) if name else root
|
||||||
|
|
||||||
return logger
|
|
||||||
|
|
||||||
|
|
||||||
logger = setup_logger()
|
logger = setup_logger()
|
||||||
+72
-47
@@ -20,6 +20,21 @@ from services.socks5_to_http_proxy import Socks5ProxyPool
|
|||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
def _get_proxy_locale(proxy_config: dict) -> str:
|
||||||
|
"""Возвращает locale без script-тега для IP прокси (ru-Cyrl-RU → ru-RU)."""
|
||||||
|
try:
|
||||||
|
from camoufox.utils import get_geolocation
|
||||||
|
ip = proxy_config.get("server", "").replace("http://", "").split(":")[0]
|
||||||
|
geo = get_geolocation(ip)
|
||||||
|
lang = geo.locale.language
|
||||||
|
region = geo.locale.region
|
||||||
|
if lang and region:
|
||||||
|
return f"{lang}-{region}"
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
return "en-US"
|
||||||
|
|
||||||
|
|
||||||
@dataclass
|
@dataclass
|
||||||
class VisitResult:
|
class VisitResult:
|
||||||
"""Результат посещения страницы."""
|
"""Результат посещения страницы."""
|
||||||
@@ -58,29 +73,27 @@ class BrowserService:
|
|||||||
settings.DEFAULT_MAX_READING
|
settings.DEFAULT_MAX_READING
|
||||||
)
|
)
|
||||||
|
|
||||||
for attempt in range(3):
|
proxy = None
|
||||||
proxy = None
|
use_socks5 = False
|
||||||
use_socks5 = False
|
|
||||||
|
|
||||||
try:
|
try:
|
||||||
# Получаем прокси
|
if self.proxy_manager.has_proxies():
|
||||||
if self.proxy_manager.has_proxies():
|
proxy = await self.proxy_manager.acquire_proxy(timeout=30)
|
||||||
proxy = await self.proxy_manager.acquire_proxy(timeout=30)
|
|
||||||
|
|
||||||
# Определяем конфигурацию
|
if proxy and proxy.proxy_type in [ProxyType.SOCKS5, ProxyType.SOCKS4]:
|
||||||
if proxy and proxy.proxy_type in [ProxyType.SOCKS5, ProxyType.SOCKS4]:
|
proxy_config = await self.socks5_pool.get_proxy_config(proxy)
|
||||||
proxy_config = await self.socks5_pool.get_proxy_config(proxy)
|
use_socks5 = True
|
||||||
use_socks5 = True
|
logger.info(f"Using SOCKS5 tunnel for: {proxy.id}")
|
||||||
logger.info(f"Using SOCKS5 tunnel for: {proxy.id}")
|
elif proxy:
|
||||||
elif proxy:
|
proxy_config = proxy.proxy_config
|
||||||
proxy_config = proxy.proxy_config
|
logger.info(f"Using HTTP proxy: {proxy.id}")
|
||||||
logger.info(f"Using HTTP proxy: {proxy.id}")
|
else:
|
||||||
else:
|
proxy_config = None
|
||||||
proxy_config = None
|
logger.info("Direct connection")
|
||||||
logger.info("Direct connection")
|
|
||||||
|
|
||||||
# Запускаем браузер
|
if proxy_config:
|
||||||
if proxy_config:
|
# Сначала пробуем с geoip — fingerprint соответствует региону прокси
|
||||||
|
try:
|
||||||
async with AsyncCamoufox(
|
async with AsyncCamoufox(
|
||||||
headless=True,
|
headless=True,
|
||||||
geoip=True,
|
geoip=True,
|
||||||
@@ -94,42 +107,54 @@ class BrowserService:
|
|||||||
proxy.id if proxy else "direct",
|
proxy.id if proxy else "direct",
|
||||||
reading_time
|
reading_time
|
||||||
)
|
)
|
||||||
else:
|
except ValueError:
|
||||||
|
# Camoufox на Linux формирует locale с script-тегом (ru-Cyrl-RU),
|
||||||
|
# которого нет в базе browserforge — передаём базовую locale без script
|
||||||
|
try:
|
||||||
|
geo = _get_proxy_locale(proxy_config)
|
||||||
|
except Exception:
|
||||||
|
geo = "ru-RU"
|
||||||
|
logger.warning(f"geoip script-locale unavailable, retrying with base locale: {geo}")
|
||||||
async with AsyncCamoufox(
|
async with AsyncCamoufox(
|
||||||
headless=True,
|
headless=True,
|
||||||
geoip=False,
|
geoip=True,
|
||||||
humanize=True,
|
humanize=True,
|
||||||
locale="ru-RU",
|
os="windows",
|
||||||
exclude_addons=[DefaultAddons.UBO]
|
locale=geo,
|
||||||
|
exclude_addons=[DefaultAddons.UBO],
|
||||||
|
proxy=proxy_config
|
||||||
) as browser:
|
) as browser:
|
||||||
result = await self._browse_page(
|
result = await self._browse_page(
|
||||||
browser, url, "direct", reading_time
|
browser, url,
|
||||||
|
proxy.id if proxy else "direct",
|
||||||
|
reading_time
|
||||||
)
|
)
|
||||||
|
else:
|
||||||
|
async with AsyncCamoufox(
|
||||||
|
headless=True,
|
||||||
|
geoip=False,
|
||||||
|
humanize=True,
|
||||||
|
locale="ru-RU",
|
||||||
|
exclude_addons=[DefaultAddons.UBO]
|
||||||
|
) as browser:
|
||||||
|
result = await self._browse_page(
|
||||||
|
browser, url, "direct", reading_time
|
||||||
|
)
|
||||||
|
|
||||||
# Освобождаем
|
if proxy:
|
||||||
if proxy:
|
await self.proxy_manager.release_proxy(proxy, result.success)
|
||||||
await self.proxy_manager.release_proxy(proxy, result.success)
|
if use_socks5:
|
||||||
if use_socks5:
|
await self.socks5_pool.release(proxy)
|
||||||
await self.socks5_pool.release(proxy)
|
|
||||||
|
|
||||||
return result
|
return result
|
||||||
|
|
||||||
except ValueError as e:
|
except Exception as e:
|
||||||
logger.warning(f"Visit attempt {attempt + 1} fingerprint error, retrying with new proxy: {e}")
|
logger.error(f"Visit error: {e}", exc_info=True)
|
||||||
if proxy:
|
if proxy:
|
||||||
await self.proxy_manager.release_proxy(proxy, success=False)
|
await self.proxy_manager.release_proxy(proxy, success=False)
|
||||||
if use_socks5:
|
if use_socks5:
|
||||||
await self.socks5_pool.release(proxy)
|
await self.socks5_pool.release(proxy)
|
||||||
|
return VisitResult(url=url, success=False, error=str(e))
|
||||||
except Exception as e:
|
|
||||||
logger.error(f"Visit error: {e}", exc_info=True)
|
|
||||||
if proxy:
|
|
||||||
await self.proxy_manager.release_proxy(proxy, success=False)
|
|
||||||
if use_socks5:
|
|
||||||
await self.socks5_pool.release(proxy)
|
|
||||||
return VisitResult(url=url, success=False, error=str(e))
|
|
||||||
|
|
||||||
return VisitResult(url=url, success=False, error="All attempts failed (geoip fingerprint error)")
|
|
||||||
|
|
||||||
async def _browse_page(
|
async def _browse_page(
|
||||||
self, browser, url: str, proxy_info: str, reading_time: int
|
self, browser, url: str, proxy_info: str, reading_time: int
|
||||||
|
|||||||
Reference in New Issue
Block a user