fix real ip
This commit is contained in:
Yuriy Yuriev
+81
-230
@@ -20,22 +20,18 @@ from services.socks5_to_http_proxy import Socks5ProxyPool
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _get_proxy_geo(ip: str) -> dict:
|
||||
"""Возвращает geoip-данные для реального IP прокси в виде kwargs для AsyncCamoufox."""
|
||||
def _locale_for_ip(ip: str) -> str:
|
||||
"""Возвращает BCP47 locale без script-тега по реальному IP прокси."""
|
||||
try:
|
||||
from camoufox.utils import get_geolocation
|
||||
geo = get_geolocation(ip)
|
||||
lang = geo.locale.language
|
||||
region = geo.locale.region
|
||||
locale = f"{lang}-{region}" if lang and region else "en-US"
|
||||
result = {"locale": locale}
|
||||
if geo.timezone:
|
||||
result["timezone"] = geo.timezone
|
||||
if geo.longitude is not None and geo.latitude is not None:
|
||||
result["geolocation"] = {"longitude": geo.longitude, "latitude": geo.latitude}
|
||||
return result
|
||||
if lang and region:
|
||||
return f"{lang}-{region}"
|
||||
except Exception:
|
||||
return {"locale": "en-US"}
|
||||
pass
|
||||
return "en-US"
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -76,83 +72,81 @@ class BrowserService:
|
||||
settings.DEFAULT_MAX_READING
|
||||
)
|
||||
|
||||
proxy = None
|
||||
use_socks5 = False
|
||||
for attempt in range(3):
|
||||
proxy = None
|
||||
use_socks5 = False
|
||||
|
||||
try:
|
||||
if self.proxy_manager.has_proxies():
|
||||
proxy = await self.proxy_manager.acquire_proxy(timeout=30)
|
||||
try:
|
||||
if self.proxy_manager.has_proxies():
|
||||
proxy = await self.proxy_manager.acquire_proxy(timeout=30)
|
||||
|
||||
real_proxy_ip = None
|
||||
if proxy and proxy.proxy_type in [ProxyType.SOCKS5, ProxyType.SOCKS4]:
|
||||
proxy_config = await self.socks5_pool.get_proxy_config(proxy)
|
||||
use_socks5 = True
|
||||
# Извлекаем реальный IP прокси до туннелирования
|
||||
real_proxy_ip = proxy.id.split("://")[-1].split(":")[0]
|
||||
logger.info(f"Using SOCKS5 tunnel for: {proxy.id}")
|
||||
elif proxy:
|
||||
proxy_config = proxy.proxy_config
|
||||
logger.info(f"Using HTTP proxy: {proxy.id}")
|
||||
else:
|
||||
proxy_config = None
|
||||
logger.info("Direct connection")
|
||||
socks5_locale = None
|
||||
if proxy and proxy.proxy_type in [ProxyType.SOCKS5, ProxyType.SOCKS4]:
|
||||
proxy_config = await self.socks5_pool.get_proxy_config(proxy)
|
||||
use_socks5 = True
|
||||
# При SOCKS5 camoufox видит 127.0.0.1 → geoip берёт IP сервера → ru-Cyrl-RU
|
||||
# Делаем lookup по реальному IP прокси сами и передаём чистый locale
|
||||
real_ip = proxy.id.split("://")[-1].split(":")[0]
|
||||
socks5_locale = _locale_for_ip(real_ip)
|
||||
logger.info(f"Using SOCKS5 tunnel for: {proxy.id} locale={socks5_locale}")
|
||||
elif proxy:
|
||||
proxy_config = proxy.proxy_config
|
||||
logger.info(f"Using HTTP proxy: {proxy.id}")
|
||||
else:
|
||||
proxy_config = None
|
||||
logger.info("Direct connection")
|
||||
|
||||
if proxy_config:
|
||||
# Camoufox при SOCKS5 видит 127.0.0.1 вместо реального IP прокси —
|
||||
# geoip lookup делаем сами и передаём timezone/locale/geolocation явно,
|
||||
# это полный эквивалент geoip=True но без внутреннего constraint'а camoufox
|
||||
geo_kwargs = _get_proxy_geo(real_proxy_ip) if real_proxy_ip else {"locale": "en-US"}
|
||||
kwargs = dict(
|
||||
headless=True,
|
||||
geoip=True,
|
||||
humanize=True,
|
||||
exclude_addons=[DefaultAddons.UBO],
|
||||
proxy=proxy_config,
|
||||
**geo_kwargs,
|
||||
)
|
||||
try:
|
||||
async with AsyncCamoufox(**kwargs) as browser:
|
||||
result = await self._browse_page(
|
||||
browser, url,
|
||||
proxy.id if proxy else "direct",
|
||||
reading_time
|
||||
)
|
||||
except ValueError:
|
||||
logger.warning("geoip fingerprint failed, retrying with en-US locale")
|
||||
kwargs["locale"] = "en-US"
|
||||
kwargs.pop("geolocation", None)
|
||||
async with AsyncCamoufox(**kwargs) as browser:
|
||||
result = await self._browse_page(
|
||||
browser, url,
|
||||
proxy.id if proxy else "direct",
|
||||
reading_time
|
||||
)
|
||||
else:
|
||||
async with AsyncCamoufox(
|
||||
headless=True,
|
||||
geoip=False,
|
||||
humanize=True,
|
||||
locale="ru-RU",
|
||||
exclude_addons=[DefaultAddons.UBO]
|
||||
) as browser:
|
||||
result = await self._browse_page(
|
||||
browser, url, "direct", reading_time
|
||||
if proxy_config:
|
||||
camoufox_kwargs = dict(
|
||||
headless=True,
|
||||
geoip=True,
|
||||
humanize=True,
|
||||
exclude_addons=[DefaultAddons.UBO],
|
||||
proxy=proxy_config,
|
||||
)
|
||||
if socks5_locale:
|
||||
camoufox_kwargs["locale"] = socks5_locale
|
||||
async with AsyncCamoufox(**camoufox_kwargs) as browser:
|
||||
result = await self._browse_page(
|
||||
browser, url,
|
||||
proxy.id if proxy else "direct",
|
||||
reading_time
|
||||
)
|
||||
else:
|
||||
async with AsyncCamoufox(
|
||||
headless=True,
|
||||
geoip=False,
|
||||
humanize=True,
|
||||
locale="ru-RU",
|
||||
exclude_addons=[DefaultAddons.UBO]
|
||||
) as browser:
|
||||
result = await self._browse_page(
|
||||
browser, url, "direct", reading_time
|
||||
)
|
||||
|
||||
if proxy:
|
||||
await self.proxy_manager.release_proxy(proxy, result.success)
|
||||
if use_socks5:
|
||||
await self.socks5_pool.release(proxy)
|
||||
if proxy:
|
||||
await self.proxy_manager.release_proxy(proxy, result.success)
|
||||
if use_socks5:
|
||||
await self.socks5_pool.release(proxy)
|
||||
|
||||
return result
|
||||
return result
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Visit error: {e}", exc_info=True)
|
||||
if proxy:
|
||||
await self.proxy_manager.release_proxy(proxy, success=False)
|
||||
if use_socks5:
|
||||
await self.socks5_pool.release(proxy)
|
||||
return VisitResult(url=url, success=False, error=str(e))
|
||||
except ValueError as e:
|
||||
logger.warning(f"Visit attempt {attempt + 1} fingerprint error, retrying with new proxy: {e}")
|
||||
if proxy:
|
||||
await self.proxy_manager.release_proxy(proxy, success=False)
|
||||
if use_socks5:
|
||||
await self.socks5_pool.release(proxy)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Visit error: {e}", exc_info=True)
|
||||
if proxy:
|
||||
await self.proxy_manager.release_proxy(proxy, success=False)
|
||||
if use_socks5:
|
||||
await self.socks5_pool.release(proxy)
|
||||
return VisitResult(url=url, success=False, error=str(e))
|
||||
|
||||
return VisitResult(url=url, success=False, error="All attempts failed (fingerprint error)")
|
||||
|
||||
async def _browse_page(
|
||||
self, browser, url: str, proxy_info: str, reading_time: int
|
||||
@@ -226,11 +220,11 @@ class BrowserService:
|
||||
try:
|
||||
while not task.done():
|
||||
await page.mouse.move(
|
||||
random.randint(100, w-100),
|
||||
random.randint(100, h-100)
|
||||
random.randint(100, w - 100),
|
||||
random.randint(100, h - 100)
|
||||
)
|
||||
await asyncio.sleep(random.uniform(0.1, 0.3))
|
||||
except:
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
async def _wait_redirect(self, page, initial_url):
|
||||
@@ -270,8 +264,8 @@ class BrowserService:
|
||||
try:
|
||||
action = random.choice(['move', 'scroll', 'pause'])
|
||||
if action == 'move':
|
||||
x = random.randint(100, settings.VIEWPORT_WIDTH-100)
|
||||
y = random.randint(100, settings.VIEWPORT_HEIGHT-100)
|
||||
x = random.randint(100, settings.VIEWPORT_WIDTH - 100)
|
||||
y = random.randint(100, settings.VIEWPORT_HEIGHT - 100)
|
||||
await page.mouse.move(x, y)
|
||||
await asyncio.sleep(random.uniform(0.3, 1.0))
|
||||
elif action == 'scroll':
|
||||
@@ -279,7 +273,7 @@ class BrowserService:
|
||||
await asyncio.sleep(random.uniform(0.3, 0.8))
|
||||
else:
|
||||
await asyncio.sleep(random.uniform(1, 3))
|
||||
except:
|
||||
except Exception:
|
||||
await asyncio.sleep(0.5)
|
||||
|
||||
async def _take_screenshot(self, page, url):
|
||||
@@ -309,7 +303,7 @@ class BrowserService:
|
||||
|
||||
proxy = None
|
||||
|
||||
for attempt in range(2): # Две попытки с разными прокси
|
||||
for attempt in range(2):
|
||||
try:
|
||||
if self.proxy_manager.has_proxies():
|
||||
proxy = await self.proxy_manager.acquire_proxy(timeout=90)
|
||||
@@ -356,146 +350,3 @@ class BrowserService:
|
||||
await asyncio.sleep(2)
|
||||
|
||||
return VisitResult(url=url, success=False, error="All attempts failed")
|
||||
|
||||
async def _browse_twitch_ref(self, browser, url: str, channel: str, proxy_info: str, reading_time: int) -> VisitResult:
|
||||
"""Посещает страницу как переход с Twitch канала."""
|
||||
page = await browser.new_page()
|
||||
twitch_url = f"https://www.twitch.tv/{channel}"
|
||||
|
||||
try:
|
||||
await page.set_viewport_size({"width": settings.VIEWPORT_WIDTH, "height": settings.VIEWPORT_HEIGHT})
|
||||
|
||||
# Заголовки имитирующие переход с Twitch
|
||||
await page.set_extra_http_headers({
|
||||
"Referer": twitch_url,
|
||||
"Origin": "https://www.twitch.tv",
|
||||
})
|
||||
|
||||
logger.info(f"📺 From Twitch: {channel} → {url}")
|
||||
|
||||
# Переход с Referer
|
||||
goto = asyncio.create_task(page.goto(url, referer=twitch_url, wait_until="commit", timeout=30000))
|
||||
await self._move_mouse(page, goto)
|
||||
await goto
|
||||
|
||||
initial_url = page.url
|
||||
final_url, redirect = await self._wait_redirect(page, initial_url)
|
||||
await self._simulate_reading(page, reading_time)
|
||||
|
||||
return VisitResult(
|
||||
url=url, initial_url=initial_url, final_url=final_url,
|
||||
redirect_occurred=redirect, proxy_info=proxy_info,
|
||||
reading_time=reading_time, success=True
|
||||
)
|
||||
except Exception as e:
|
||||
return VisitResult(url=url, success=False, error=str(e))
|
||||
finally:
|
||||
await page.close()
|
||||
|
||||
async def _browse_from_twitch(
|
||||
self,
|
||||
browser,
|
||||
url: str,
|
||||
channel: str,
|
||||
proxy_info: str,
|
||||
reading_time: int
|
||||
) -> VisitResult:
|
||||
"""
|
||||
Эмулирует переход с Twitch:
|
||||
1. Открывает страницу стримера
|
||||
2. Прокручивает чат
|
||||
3. Переходит по ссылке
|
||||
"""
|
||||
page = await browser.new_page()
|
||||
twitch_url = f"https://www.twitch.tv/{channel}"
|
||||
|
||||
try:
|
||||
await page.set_viewport_size({
|
||||
"width": settings.VIEWPORT_WIDTH,
|
||||
"height": settings.VIEWPORT_HEIGHT
|
||||
})
|
||||
|
||||
# === Шаг 1: Заходим на Twitch ===
|
||||
logger.info(f"📺 Opening Twitch: {twitch_url}")
|
||||
|
||||
goto_task = asyncio.create_task(
|
||||
page.goto(twitch_url, wait_until="commit", timeout=30000)
|
||||
)
|
||||
await self._move_mouse(page, goto_task)
|
||||
await goto_task
|
||||
|
||||
# Ждем загрузки
|
||||
await asyncio.sleep(random.uniform(2, 4))
|
||||
|
||||
# === Шаг 2: Имитируем просмотр стрима ===
|
||||
logger.info("👀 Watching stream...")
|
||||
|
||||
# Прокручиваем страницу как будто смотрим
|
||||
for _ in range(random.randint(2, 4)):
|
||||
scroll = random.randint(200, 500)
|
||||
await page.evaluate(f"window.scrollBy(0, {scroll})")
|
||||
await asyncio.sleep(random.uniform(0.5, 1.5))
|
||||
|
||||
# Двигаем мышь (как будто читаем чат)
|
||||
w, h = settings.VIEWPORT_WIDTH, settings.VIEWPORT_HEIGHT
|
||||
for _ in range(random.randint(3, 6)):
|
||||
x = random.randint(100, w - 100)
|
||||
y = random.randint(100, h - 100)
|
||||
await page.mouse.move(x, y)
|
||||
await asyncio.sleep(random.uniform(0.3, 0.8))
|
||||
|
||||
# === Шаг 3: Переходим по ссылке ===
|
||||
logger.info(f"🔗 Clicking link: {url}")
|
||||
|
||||
# Создаем новую вкладку для перехода (как target="_blank")
|
||||
# Или просто переходим с Referer
|
||||
await page.evaluate(f"""
|
||||
window.open('{url}', '_blank');
|
||||
""")
|
||||
|
||||
# Ждем открытия новой вкладки
|
||||
await asyncio.sleep(2)
|
||||
|
||||
# Получаем новую вкладку
|
||||
pages = await browser.pages()
|
||||
if len(pages) > 1:
|
||||
new_page = pages[-1]
|
||||
await new_page.bring_to_front()
|
||||
else:
|
||||
# Если вкладка не открылась - переходим в этой же
|
||||
new_page = page
|
||||
await new_page.goto(url, referer=twitch_url, wait_until="commit")
|
||||
|
||||
initial_url = new_page.url
|
||||
logger.info(f"📍 Landed: {initial_url}")
|
||||
|
||||
# === Шаг 4: Ждем редирект ===
|
||||
final_url, redirect = await self._wait_redirect(new_page, initial_url)
|
||||
|
||||
# === Шаг 5: Читаем страницу ===
|
||||
await self._simulate_reading(new_page, reading_time)
|
||||
|
||||
# Закрываем новую вкладку если она отдельная
|
||||
if new_page != page:
|
||||
await new_page.close()
|
||||
|
||||
# Возвращаемся на Twitch и закрываем
|
||||
await page.close()
|
||||
|
||||
return VisitResult(
|
||||
url=url,
|
||||
initial_url=initial_url,
|
||||
final_url=final_url,
|
||||
redirect_occurred=redirect,
|
||||
proxy_info=proxy_info,
|
||||
reading_time=reading_time,
|
||||
success=True
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Browse from Twitch error: {e}")
|
||||
try:
|
||||
await page.close()
|
||||
except:
|
||||
pass
|
||||
return VisitResult(url=url, reading_time=reading_time, success=False, error=str(e))
|
||||
Reference in New Issue
Block a user