diff --git a/services/browser_service.py b/services/browser_service.py index 452f6cf..92bd67a 100644 --- a/services/browser_service.py +++ b/services/browser_service.py @@ -20,22 +20,18 @@ from services.socks5_to_http_proxy import Socks5ProxyPool logger = logging.getLogger(__name__) -def _get_proxy_geo(ip: str) -> dict: - """Возвращает geoip-данные для реального IP прокси в виде kwargs для AsyncCamoufox.""" +def _locale_for_ip(ip: str) -> str: + """Возвращает BCP47 locale без script-тега по реальному IP прокси.""" try: from camoufox.utils import get_geolocation geo = get_geolocation(ip) lang = geo.locale.language region = geo.locale.region - locale = f"{lang}-{region}" if lang and region else "en-US" - result = {"locale": locale} - if geo.timezone: - result["timezone"] = geo.timezone - if geo.longitude is not None and geo.latitude is not None: - result["geolocation"] = {"longitude": geo.longitude, "latitude": geo.latitude} - return result + if lang and region: + return f"{lang}-{region}" except Exception: - return {"locale": "en-US"} + pass + return "en-US" @dataclass @@ -57,13 +53,13 @@ class BrowserService: Сервис для посещения страниц через Camoufox. Поддерживает HTTP, SOCKS5 (через туннель) и прямое соединение. """ - + def __init__(self, proxy_manager: ProxyManager): self.proxy_manager = proxy_manager self.socks5_pool = Socks5ProxyPool(idle_timeout=300) self.screenshots_dir = Path(settings.SCREENSHOTS_DIR) self.screenshots_dir.mkdir(exist_ok=True) - + async def visit_page( self, url: str, @@ -75,85 +71,83 @@ class BrowserService: settings.DEFAULT_MIN_READING, settings.DEFAULT_MAX_READING ) - - proxy = None - use_socks5 = False - try: - if self.proxy_manager.has_proxies(): - proxy = await self.proxy_manager.acquire_proxy(timeout=30) + for attempt in range(3): + proxy = None + use_socks5 = False - real_proxy_ip = None - if proxy and proxy.proxy_type in [ProxyType.SOCKS5, ProxyType.SOCKS4]: - proxy_config = await self.socks5_pool.get_proxy_config(proxy) - use_socks5 = True - # Извлекаем реальный IP прокси до туннелирования - real_proxy_ip = proxy.id.split("://")[-1].split(":")[0] - logger.info(f"Using SOCKS5 tunnel for: {proxy.id}") - elif proxy: - proxy_config = proxy.proxy_config - logger.info(f"Using HTTP proxy: {proxy.id}") - else: - proxy_config = None - logger.info("Direct connection") + try: + if self.proxy_manager.has_proxies(): + proxy = await self.proxy_manager.acquire_proxy(timeout=30) - if proxy_config: - # Camoufox при SOCKS5 видит 127.0.0.1 вместо реального IP прокси — - # geoip lookup делаем сами и передаём timezone/locale/geolocation явно, - # это полный эквивалент geoip=True но без внутреннего constraint'а camoufox - geo_kwargs = _get_proxy_geo(real_proxy_ip) if real_proxy_ip else {"locale": "en-US"} - kwargs = dict( - headless=True, - geoip=True, - humanize=True, - exclude_addons=[DefaultAddons.UBO], - proxy=proxy_config, - **geo_kwargs, - ) - try: - async with AsyncCamoufox(**kwargs) as browser: - result = await self._browse_page( - browser, url, - proxy.id if proxy else "direct", - reading_time - ) - except ValueError: - logger.warning("geoip fingerprint failed, retrying with en-US locale") - kwargs["locale"] = "en-US" - kwargs.pop("geolocation", None) - async with AsyncCamoufox(**kwargs) as browser: - result = await self._browse_page( - browser, url, - proxy.id if proxy else "direct", - reading_time - ) - else: - async with AsyncCamoufox( - headless=True, - geoip=False, - humanize=True, - locale="ru-RU", - exclude_addons=[DefaultAddons.UBO] - ) as browser: - result = await self._browse_page( - browser, url, "direct", reading_time + socks5_locale = None + if proxy and proxy.proxy_type in [ProxyType.SOCKS5, ProxyType.SOCKS4]: + proxy_config = await self.socks5_pool.get_proxy_config(proxy) + use_socks5 = True + # При SOCKS5 camoufox видит 127.0.0.1 → geoip берёт IP сервера → ru-Cyrl-RU + # Делаем lookup по реальному IP прокси сами и передаём чистый locale + real_ip = proxy.id.split("://")[-1].split(":")[0] + socks5_locale = _locale_for_ip(real_ip) + logger.info(f"Using SOCKS5 tunnel for: {proxy.id} locale={socks5_locale}") + elif proxy: + proxy_config = proxy.proxy_config + logger.info(f"Using HTTP proxy: {proxy.id}") + else: + proxy_config = None + logger.info("Direct connection") + + if proxy_config: + camoufox_kwargs = dict( + headless=True, + geoip=True, + humanize=True, + exclude_addons=[DefaultAddons.UBO], + proxy=proxy_config, ) + if socks5_locale: + camoufox_kwargs["locale"] = socks5_locale + async with AsyncCamoufox(**camoufox_kwargs) as browser: + result = await self._browse_page( + browser, url, + proxy.id if proxy else "direct", + reading_time + ) + else: + async with AsyncCamoufox( + headless=True, + geoip=False, + humanize=True, + locale="ru-RU", + exclude_addons=[DefaultAddons.UBO] + ) as browser: + result = await self._browse_page( + browser, url, "direct", reading_time + ) - if proxy: - await self.proxy_manager.release_proxy(proxy, result.success) - if use_socks5: - await self.socks5_pool.release(proxy) + if proxy: + await self.proxy_manager.release_proxy(proxy, result.success) + if use_socks5: + await self.socks5_pool.release(proxy) - return result + return result + + except ValueError as e: + logger.warning(f"Visit attempt {attempt + 1} fingerprint error, retrying with new proxy: {e}") + if proxy: + await self.proxy_manager.release_proxy(proxy, success=False) + if use_socks5: + await self.socks5_pool.release(proxy) + + except Exception as e: + logger.error(f"Visit error: {e}", exc_info=True) + if proxy: + await self.proxy_manager.release_proxy(proxy, success=False) + if use_socks5: + await self.socks5_pool.release(proxy) + return VisitResult(url=url, success=False, error=str(e)) + + return VisitResult(url=url, success=False, error="All attempts failed (fingerprint error)") - except Exception as e: - logger.error(f"Visit error: {e}", exc_info=True) - if proxy: - await self.proxy_manager.release_proxy(proxy, success=False) - if use_socks5: - await self.socks5_pool.release(proxy) - return VisitResult(url=url, success=False, error=str(e)) - async def _browse_page( self, browser, url: str, proxy_info: str, reading_time: int ) -> VisitResult: @@ -219,31 +213,31 @@ class BrowserService: ) finally: await page.close() - + async def _move_mouse(self, page, task): """Двигает мышь.""" w, h = settings.VIEWPORT_WIDTH, settings.VIEWPORT_HEIGHT try: while not task.done(): await page.mouse.move( - random.randint(100, w-100), - random.randint(100, h-100) + random.randint(100, w - 100), + random.randint(100, h - 100) ) await asyncio.sleep(random.uniform(0.1, 0.3)) - except: + except Exception: pass - + async def _wait_redirect(self, page, initial_url): """Ждет редирект.""" loop = asyncio.get_running_loop() start = loop.time() redirect = False current = initial_url - + while loop.time() - start < settings.MAX_REDIRECT_WAIT: await asyncio.sleep(2) new_url = page.url - + if new_url != current: current = new_url redirect = True @@ -253,25 +247,25 @@ class BrowserService: await self._move_mouse(page, task) await task break - + await self._random_action(page) - + return page.url, redirect - + async def _simulate_reading(self, page, duration): """Симулирует чтение.""" loop = asyncio.get_running_loop() start = loop.time() while loop.time() - start < duration: await self._random_action(page) - + async def _random_action(self, page): """Случайное действие.""" try: action = random.choice(['move', 'scroll', 'pause']) if action == 'move': - x = random.randint(100, settings.VIEWPORT_WIDTH-100) - y = random.randint(100, settings.VIEWPORT_HEIGHT-100) + x = random.randint(100, settings.VIEWPORT_WIDTH - 100) + y = random.randint(100, settings.VIEWPORT_HEIGHT - 100) await page.mouse.move(x, y) await asyncio.sleep(random.uniform(0.3, 1.0)) elif action == 'scroll': @@ -279,9 +273,9 @@ class BrowserService: await asyncio.sleep(random.uniform(0.3, 0.8)) else: await asyncio.sleep(random.uniform(1, 3)) - except: + except Exception: await asyncio.sleep(0.5) - + async def _take_screenshot(self, page, url): """Скриншот.""" try: @@ -293,7 +287,7 @@ class BrowserService: except Exception as e: logger.error(f"Screenshot: {e}") return None - + async def cleanup(self): """Очистка с таймаутом.""" try: @@ -302,25 +296,25 @@ class BrowserService: logger.warning("SOCKS5 pool cleanup timeout") except Exception as e: logger.error(f"Cleanup error: {e}") - + async def visit_page_from_twitch(self, url: str, channel: str, reading_time: int = None) -> VisitResult: if reading_time is None: reading_time = random.randint(settings.DEFAULT_MIN_READING, settings.DEFAULT_MAX_READING) - + proxy = None - - for attempt in range(2): # Две попытки с разными прокси + + for attempt in range(2): try: if self.proxy_manager.has_proxies(): proxy = await self.proxy_manager.acquire_proxy(timeout=90) - + if proxy and proxy.proxy_type in [ProxyType.SOCKS5, ProxyType.SOCKS4]: proxy_config = await self.socks5_pool.get_proxy_config(proxy) elif proxy: proxy_config = proxy.proxy_config else: proxy_config = None - + browser_kwargs = { "headless": True, "geoip": True, @@ -338,164 +332,21 @@ class BrowserService: proxy.id if proxy else "direct", reading_time ) - + if proxy: await self.proxy_manager.release_proxy(proxy, result.success) - + if result.success: return result - + except Exception as e: logger.warning(f"Attempt {attempt + 1} failed: {e}") if proxy: await self.proxy_manager.release_proxy(proxy, success=False) await self.socks5_pool.release(proxy) proxy = None - + if attempt < 1: await asyncio.sleep(2) - + return VisitResult(url=url, success=False, error="All attempts failed") - - async def _browse_twitch_ref(self, browser, url: str, channel: str, proxy_info: str, reading_time: int) -> VisitResult: - """Посещает страницу как переход с Twitch канала.""" - page = await browser.new_page() - twitch_url = f"https://www.twitch.tv/{channel}" - - try: - await page.set_viewport_size({"width": settings.VIEWPORT_WIDTH, "height": settings.VIEWPORT_HEIGHT}) - - # Заголовки имитирующие переход с Twitch - await page.set_extra_http_headers({ - "Referer": twitch_url, - "Origin": "https://www.twitch.tv", - }) - - logger.info(f"📺 From Twitch: {channel} → {url}") - - # Переход с Referer - goto = asyncio.create_task(page.goto(url, referer=twitch_url, wait_until="commit", timeout=30000)) - await self._move_mouse(page, goto) - await goto - - initial_url = page.url - final_url, redirect = await self._wait_redirect(page, initial_url) - await self._simulate_reading(page, reading_time) - - return VisitResult( - url=url, initial_url=initial_url, final_url=final_url, - redirect_occurred=redirect, proxy_info=proxy_info, - reading_time=reading_time, success=True - ) - except Exception as e: - return VisitResult(url=url, success=False, error=str(e)) - finally: - await page.close() - - async def _browse_from_twitch( - self, - browser, - url: str, - channel: str, - proxy_info: str, - reading_time: int - ) -> VisitResult: - """ - Эмулирует переход с Twitch: - 1. Открывает страницу стримера - 2. Прокручивает чат - 3. Переходит по ссылке - """ - page = await browser.new_page() - twitch_url = f"https://www.twitch.tv/{channel}" - - try: - await page.set_viewport_size({ - "width": settings.VIEWPORT_WIDTH, - "height": settings.VIEWPORT_HEIGHT - }) - - # === Шаг 1: Заходим на Twitch === - logger.info(f"📺 Opening Twitch: {twitch_url}") - - goto_task = asyncio.create_task( - page.goto(twitch_url, wait_until="commit", timeout=30000) - ) - await self._move_mouse(page, goto_task) - await goto_task - - # Ждем загрузки - await asyncio.sleep(random.uniform(2, 4)) - - # === Шаг 2: Имитируем просмотр стрима === - logger.info("👀 Watching stream...") - - # Прокручиваем страницу как будто смотрим - for _ in range(random.randint(2, 4)): - scroll = random.randint(200, 500) - await page.evaluate(f"window.scrollBy(0, {scroll})") - await asyncio.sleep(random.uniform(0.5, 1.5)) - - # Двигаем мышь (как будто читаем чат) - w, h = settings.VIEWPORT_WIDTH, settings.VIEWPORT_HEIGHT - for _ in range(random.randint(3, 6)): - x = random.randint(100, w - 100) - y = random.randint(100, h - 100) - await page.mouse.move(x, y) - await asyncio.sleep(random.uniform(0.3, 0.8)) - - # === Шаг 3: Переходим по ссылке === - logger.info(f"🔗 Clicking link: {url}") - - # Создаем новую вкладку для перехода (как target="_blank") - # Или просто переходим с Referer - await page.evaluate(f""" - window.open('{url}', '_blank'); - """) - - # Ждем открытия новой вкладки - await asyncio.sleep(2) - - # Получаем новую вкладку - pages = await browser.pages() - if len(pages) > 1: - new_page = pages[-1] - await new_page.bring_to_front() - else: - # Если вкладка не открылась - переходим в этой же - new_page = page - await new_page.goto(url, referer=twitch_url, wait_until="commit") - - initial_url = new_page.url - logger.info(f"📍 Landed: {initial_url}") - - # === Шаг 4: Ждем редирект === - final_url, redirect = await self._wait_redirect(new_page, initial_url) - - # === Шаг 5: Читаем страницу === - await self._simulate_reading(new_page, reading_time) - - # Закрываем новую вкладку если она отдельная - if new_page != page: - await new_page.close() - - # Возвращаемся на Twitch и закрываем - await page.close() - - return VisitResult( - url=url, - initial_url=initial_url, - final_url=final_url, - redirect_occurred=redirect, - proxy_info=proxy_info, - reading_time=reading_time, - success=True - ) - - except Exception as e: - logger.error(f"Browse from Twitch error: {e}") - try: - await page.close() - except: - pass - return VisitResult(url=url, reading_time=reading_time, success=False, error=str(e)) \ No newline at end of file