fix real ip

services/browser_service.py

@@ -20,22 +20,18 @@ from services.socks5_to_http_proxy import Socks5ProxyPool
 logger = logging.getLogger(__name__)
 
 
-def _get_proxy_geo(ip: str) -> dict:
+def _locale_for_ip(ip: str) -> str:
-    """Возвращает geoip-данные для реального IP прокси в виде kwargs для AsyncCamoufox."""
+    """Возвращает BCP47 locale без script-тега по реальному IP прокси."""
     try:
         from camoufox.utils import get_geolocation
         geo = get_geolocation(ip)
         lang = geo.locale.language
         region = geo.locale.region
-        locale = f"{lang}-{region}" if lang and region else "en-US"
+        if lang and region:
-        result = {"locale": locale}
+            return f"{lang}-{region}"
-        if geo.timezone:
-            result["timezone"] = geo.timezone
-        if geo.longitude is not None and geo.latitude is not None:
-            result["geolocation"] = {"longitude": geo.longitude, "latitude": geo.latitude}
-        return result
     except Exception:
-        return {"locale": "en-US"}
+        pass
+    return "en-US"
 
 
 @dataclass
@@ -57,13 +53,13 @@ class BrowserService:
     Сервис для посещения страниц через Camoufox.
     Поддерживает HTTP, SOCKS5 (через туннель) и прямое соединение.
     """
-    
+
     def __init__(self, proxy_manager: ProxyManager):
         self.proxy_manager = proxy_manager
         self.socks5_pool = Socks5ProxyPool(idle_timeout=300)
         self.screenshots_dir = Path(settings.SCREENSHOTS_DIR)
         self.screenshots_dir.mkdir(exist_ok=True)
-    
+
     async def visit_page(
         self,
         url: str,
@@ -75,85 +71,83 @@ class BrowserService:
                 settings.DEFAULT_MIN_READING,
                 settings.DEFAULT_MAX_READING
             )
-        
-        proxy = None
-        use_socks5 = False
 
-        try:
+        for attempt in range(3):
-            if self.proxy_manager.has_proxies():
+            proxy = None
-                proxy = await self.proxy_manager.acquire_proxy(timeout=30)
+            use_socks5 = False
 
-            real_proxy_ip = None
+            try:
-            if proxy and proxy.proxy_type in [ProxyType.SOCKS5, ProxyType.SOCKS4]:
+                if self.proxy_manager.has_proxies():
-                proxy_config = await self.socks5_pool.get_proxy_config(proxy)
+                    proxy = await self.proxy_manager.acquire_proxy(timeout=30)
-                use_socks5 = True
-                # Извлекаем реальный IP прокси до туннелирования
-                real_proxy_ip = proxy.id.split("://")[-1].split(":")[0]
-                logger.info(f"Using SOCKS5 tunnel for: {proxy.id}")
-            elif proxy:
-                proxy_config = proxy.proxy_config
-                logger.info(f"Using HTTP proxy: {proxy.id}")
-            else:
-                proxy_config = None
-                logger.info("Direct connection")
 
-            if proxy_config:
+                socks5_locale = None
-                # Camoufox при SOCKS5 видит 127.0.0.1 вместо реального IP прокси —
+                if proxy and proxy.proxy_type in [ProxyType.SOCKS5, ProxyType.SOCKS4]:
-                # geoip lookup делаем сами и передаём timezone/locale/geolocation явно,
+                    proxy_config = await self.socks5_pool.get_proxy_config(proxy)
-                # это полный эквивалент geoip=True но без внутреннего constraint'а camoufox
+                    use_socks5 = True
-                geo_kwargs = _get_proxy_geo(real_proxy_ip) if real_proxy_ip else {"locale": "en-US"}
+                    # При SOCKS5 camoufox видит 127.0.0.1 → geoip берёт IP сервера → ru-Cyrl-RU
-                kwargs = dict(
+                    # Делаем lookup по реальному IP прокси сами и передаём чистый locale
-                    headless=True,
+                    real_ip = proxy.id.split("://")[-1].split(":")[0]
-                    geoip=True,
+                    socks5_locale = _locale_for_ip(real_ip)
-                    humanize=True,
+                    logger.info(f"Using SOCKS5 tunnel for: {proxy.id} locale={socks5_locale}")
-                    exclude_addons=[DefaultAddons.UBO],
+                elif proxy:
-                    proxy=proxy_config,
+                    proxy_config = proxy.proxy_config
-                    **geo_kwargs,
+                    logger.info(f"Using HTTP proxy: {proxy.id}")
-                )
+                else:
-                try:
+                    proxy_config = None
-                    async with AsyncCamoufox(**kwargs) as browser:
+                    logger.info("Direct connection")
-                        result = await self._browse_page(
+
-                            browser, url,
+                if proxy_config:
-                            proxy.id if proxy else "direct",
+                    camoufox_kwargs = dict(
-                            reading_time
+                        headless=True,
-                        )
+                        geoip=True,
-                except ValueError:
+                        humanize=True,
-                    logger.warning("geoip fingerprint failed, retrying with en-US locale")
+                        exclude_addons=[DefaultAddons.UBO],
-                    kwargs["locale"] = "en-US"
+                        proxy=proxy_config,
-                    kwargs.pop("geolocation", None)
-                    async with AsyncCamoufox(**kwargs) as browser:
-                        result = await self._browse_page(
-                            browser, url,
-                            proxy.id if proxy else "direct",
-                            reading_time
-                        )
-            else:
-                async with AsyncCamoufox(
-                    headless=True,
-                    geoip=False,
-                    humanize=True,
-                    locale="ru-RU",
-                    exclude_addons=[DefaultAddons.UBO]
-                ) as browser:
-                    result = await self._browse_page(
-                        browser, url, "direct", reading_time
                     )
+                    if socks5_locale:
+                        camoufox_kwargs["locale"] = socks5_locale
+                    async with AsyncCamoufox(**camoufox_kwargs) as browser:
+                        result = await self._browse_page(
+                            browser, url,
+                            proxy.id if proxy else "direct",
+                            reading_time
+                        )
+                else:
+                    async with AsyncCamoufox(
+                        headless=True,
+                        geoip=False,
+                        humanize=True,
+                        locale="ru-RU",
+                        exclude_addons=[DefaultAddons.UBO]
+                    ) as browser:
+                        result = await self._browse_page(
+                            browser, url, "direct", reading_time
+                        )
 
-            if proxy:
+                if proxy:
-                await self.proxy_manager.release_proxy(proxy, result.success)
+                    await self.proxy_manager.release_proxy(proxy, result.success)
-                if use_socks5:
+                    if use_socks5:
-                    await self.socks5_pool.release(proxy)
+                        await self.socks5_pool.release(proxy)
 
-            return result
+                return result
+
+            except ValueError as e:
+                logger.warning(f"Visit attempt {attempt + 1} fingerprint error, retrying with new proxy: {e}")
+                if proxy:
+                    await self.proxy_manager.release_proxy(proxy, success=False)
+                    if use_socks5:
+                        await self.socks5_pool.release(proxy)
+
+            except Exception as e:
+                logger.error(f"Visit error: {e}", exc_info=True)
+                if proxy:
+                    await self.proxy_manager.release_proxy(proxy, success=False)
+                    if use_socks5:
+                        await self.socks5_pool.release(proxy)
+                return VisitResult(url=url, success=False, error=str(e))
+
+        return VisitResult(url=url, success=False, error="All attempts failed (fingerprint error)")
 
-        except Exception as e:
-            logger.error(f"Visit error: {e}", exc_info=True)
-            if proxy:
-                await self.proxy_manager.release_proxy(proxy, success=False)
-                if use_socks5:
-                    await self.socks5_pool.release(proxy)
-            return VisitResult(url=url, success=False, error=str(e))
-    
     async def _browse_page(
         self, browser, url: str, proxy_info: str, reading_time: int
     ) -> VisitResult:
@@ -219,31 +213,31 @@ class BrowserService:
             )
         finally:
             await page.close()
-    
+
     async def _move_mouse(self, page, task):
         """Двигает мышь."""
         w, h = settings.VIEWPORT_WIDTH, settings.VIEWPORT_HEIGHT
         try:
             while not task.done():
                 await page.mouse.move(
-                    random.randint(100, w-100),
+                    random.randint(100, w - 100),
-                    random.randint(100, h-100)
+                    random.randint(100, h - 100)
                 )
                 await asyncio.sleep(random.uniform(0.1, 0.3))
-        except:
+        except Exception:
             pass
-    
+
     async def _wait_redirect(self, page, initial_url):
         """Ждет редирект."""
         loop = asyncio.get_running_loop()
         start = loop.time()
         redirect = False
         current = initial_url
-        
+
         while loop.time() - start < settings.MAX_REDIRECT_WAIT:
             await asyncio.sleep(2)
             new_url = page.url
-            
+
             if new_url != current:
                 current = new_url
                 redirect = True
@@ -253,25 +247,25 @@ class BrowserService:
                 await self._move_mouse(page, task)
                 await task
                 break
-            
+
             await self._random_action(page)
-        
+
         return page.url, redirect
-    
+
     async def _simulate_reading(self, page, duration):
         """Симулирует чтение."""
         loop = asyncio.get_running_loop()
         start = loop.time()
         while loop.time() - start < duration:
             await self._random_action(page)
-    
+
     async def _random_action(self, page):
         """Случайное действие."""
         try:
             action = random.choice(['move', 'scroll', 'pause'])
             if action == 'move':
-                x = random.randint(100, settings.VIEWPORT_WIDTH-100)
+                x = random.randint(100, settings.VIEWPORT_WIDTH - 100)
-                y = random.randint(100, settings.VIEWPORT_HEIGHT-100)
+                y = random.randint(100, settings.VIEWPORT_HEIGHT - 100)
                 await page.mouse.move(x, y)
                 await asyncio.sleep(random.uniform(0.3, 1.0))
             elif action == 'scroll':
@@ -279,9 +273,9 @@ class BrowserService:
                 await asyncio.sleep(random.uniform(0.3, 0.8))
             else:
                 await asyncio.sleep(random.uniform(1, 3))
-        except:
+        except Exception:
             await asyncio.sleep(0.5)
-    
+
     async def _take_screenshot(self, page, url):
         """Скриншот."""
         try:
@@ -293,7 +287,7 @@ class BrowserService:
         except Exception as e:
             logger.error(f"Screenshot: {e}")
             return None
-    
+
     async def cleanup(self):
         """Очистка с таймаутом."""
         try:
@@ -302,25 +296,25 @@ class BrowserService:
             logger.warning("SOCKS5 pool cleanup timeout")
         except Exception as e:
             logger.error(f"Cleanup error: {e}")
-            
+
     async def visit_page_from_twitch(self, url: str, channel: str, reading_time: int = None) -> VisitResult:
         if reading_time is None:
             reading_time = random.randint(settings.DEFAULT_MIN_READING, settings.DEFAULT_MAX_READING)
-        
+
         proxy = None
-        
+
-        for attempt in range(2):  # Две попытки с разными прокси
+        for attempt in range(2):
             try:
                 if self.proxy_manager.has_proxies():
                     proxy = await self.proxy_manager.acquire_proxy(timeout=90)
-                
+
                 if proxy and proxy.proxy_type in [ProxyType.SOCKS5, ProxyType.SOCKS4]:
                     proxy_config = await self.socks5_pool.get_proxy_config(proxy)
                 elif proxy:
                     proxy_config = proxy.proxy_config
                 else:
                     proxy_config = None
-                
+
                 browser_kwargs = {
                     "headless": True,
                     "geoip": True,
@@ -338,164 +332,21 @@ class BrowserService:
                         proxy.id if proxy else "direct",
                         reading_time
                     )
-                
+
                 if proxy:
                     await self.proxy_manager.release_proxy(proxy, result.success)
-                
+
                 if result.success:
                     return result
-                
+
             except Exception as e:
                 logger.warning(f"Attempt {attempt + 1} failed: {e}")
                 if proxy:
                     await self.proxy_manager.release_proxy(proxy, success=False)
                     await self.socks5_pool.release(proxy)
                     proxy = None
-            
+
             if attempt < 1:
                 await asyncio.sleep(2)
-        
+
         return VisitResult(url=url, success=False, error="All attempts failed")
-
-    async def _browse_twitch_ref(self, browser, url: str, channel: str, proxy_info: str, reading_time: int) -> VisitResult:
-        """Посещает страницу как переход с Twitch канала."""
-        page = await browser.new_page()
-        twitch_url = f"https://www.twitch.tv/{channel}"
-        
-        try:
-            await page.set_viewport_size({"width": settings.VIEWPORT_WIDTH, "height": settings.VIEWPORT_HEIGHT})
-            
-            # Заголовки имитирующие переход с Twitch
-            await page.set_extra_http_headers({
-                "Referer": twitch_url,
-                "Origin": "https://www.twitch.tv",
-            })
-            
-            logger.info(f"📺 From Twitch: {channel} → {url}")
-            
-            # Переход с Referer
-            goto = asyncio.create_task(page.goto(url, referer=twitch_url, wait_until="commit", timeout=30000))
-            await self._move_mouse(page, goto)
-            await goto
-            
-            initial_url = page.url
-            final_url, redirect = await self._wait_redirect(page, initial_url)
-            await self._simulate_reading(page, reading_time)
-            
-            return VisitResult(
-                url=url, initial_url=initial_url, final_url=final_url,
-                redirect_occurred=redirect, proxy_info=proxy_info,
-                reading_time=reading_time, success=True
-            )
-        except Exception as e:
-            return VisitResult(url=url, success=False, error=str(e))
-        finally:
-            await page.close()
-
-    async def _browse_from_twitch(
-        self,
-        browser,
-        url: str,
-        channel: str,
-        proxy_info: str,
-        reading_time: int
-    ) -> VisitResult:
-        """
-        Эмулирует переход с Twitch:
-        1. Открывает страницу стримера
-        2. Прокручивает чат
-        3. Переходит по ссылке
-        """
-        page = await browser.new_page()
-        twitch_url = f"https://www.twitch.tv/{channel}"
-        
-        try:
-            await page.set_viewport_size({
-                "width": settings.VIEWPORT_WIDTH,
-                "height": settings.VIEWPORT_HEIGHT
-            })
-            
-            # === Шаг 1: Заходим на Twitch ===
-            logger.info(f"📺 Opening Twitch: {twitch_url}")
-            
-            goto_task = asyncio.create_task(
-                page.goto(twitch_url, wait_until="commit", timeout=30000)
-            )
-            await self._move_mouse(page, goto_task)
-            await goto_task
-            
-            # Ждем загрузки
-            await asyncio.sleep(random.uniform(2, 4))
-            
-            # === Шаг 2: Имитируем просмотр стрима ===
-            logger.info("👀 Watching stream...")
-            
-            # Прокручиваем страницу как будто смотрим
-            for _ in range(random.randint(2, 4)):
-                scroll = random.randint(200, 500)
-                await page.evaluate(f"window.scrollBy(0, {scroll})")
-                await asyncio.sleep(random.uniform(0.5, 1.5))
-            
-            # Двигаем мышь (как будто читаем чат)
-            w, h = settings.VIEWPORT_WIDTH, settings.VIEWPORT_HEIGHT
-            for _ in range(random.randint(3, 6)):
-                x = random.randint(100, w - 100)
-                y = random.randint(100, h - 100)
-                await page.mouse.move(x, y)
-                await asyncio.sleep(random.uniform(0.3, 0.8))
-            
-            # === Шаг 3: Переходим по ссылке ===
-            logger.info(f"🔗 Clicking link: {url}")
-            
-            # Создаем новую вкладку для перехода (как target="_blank")
-            # Или просто переходим с Referer
-            await page.evaluate(f"""
-                window.open('{url}', '_blank');
-            """)
-            
-            # Ждем открытия новой вкладки
-            await asyncio.sleep(2)
-            
-            # Получаем новую вкладку
-            pages = await browser.pages()
-            if len(pages) > 1:
-                new_page = pages[-1]
-                await new_page.bring_to_front()
-            else:
-                # Если вкладка не открылась - переходим в этой же
-                new_page = page
-                await new_page.goto(url, referer=twitch_url, wait_until="commit")
-            
-            initial_url = new_page.url
-            logger.info(f"📍 Landed: {initial_url}")
-            
-            # === Шаг 4: Ждем редирект ===
-            final_url, redirect = await self._wait_redirect(new_page, initial_url)
-            
-            # === Шаг 5: Читаем страницу ===
-            await self._simulate_reading(new_page, reading_time)
-            
-            # Закрываем новую вкладку если она отдельная
-            if new_page != page:
-                await new_page.close()
-            
-            # Возвращаемся на Twitch и закрываем
-            await page.close()
-            
-            return VisitResult(
-                url=url,
-                initial_url=initial_url,
-                final_url=final_url,
-                redirect_occurred=redirect,
-                proxy_info=proxy_info,
-                reading_time=reading_time,
-                success=True
-            )
-            
-        except Exception as e:
-            logger.error(f"Browse from Twitch error: {e}")
-            try:
-                await page.close()
-            except:
-                pass
-            return VisitResult(url=url, reading_time=reading_time, success=False, error=str(e))