commit 27c53e897e0f3ab1864f582160dfd6e2c3a99f2e Author: Yuriy Yuriev Date: Thu May 14 16:15:58 2026 +0700 init auth and main function diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..93665a1 --- /dev/null +++ b/.env.example @@ -0,0 +1,20 @@ +# FastAPI +APP_NAME="PythonBot Backend" +APP_VERSION="1.0.0" +HOST=0.0.0.0 +PORT=8000 +DEBUG=false + +# Telegram Bot (получите у @BotFather) +BOT_TOKEN=8628226925:AAELN1fS4AStuwzpGCNVeZuUE6kLdaIeuTc + +# Task Queue +MAX_WORKERS=3 +TASK_TIMEOUT=300 + +# Captcha Solver +TWO_CAPTCHA_API_KEY= +TEN_CAPTCHA_API_KEY= +CAPTCHA_SOLVER=click +CAPTCHA_MAX_ATTEMPTS=3 +CAPTCHA_ATTEMPT_DELAY=5 diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..b85f2b7 --- /dev/null +++ b/.gitignore @@ -0,0 +1,220 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[codz] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py.cover +*.lcov +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +# Pipfile.lock + +# UV +# Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# uv.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +# poetry.lock +# poetry.toml + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +# pdm recommends including project-wide configuration in pdm.toml, but excluding .pdm-python. +# https://pdm-project.org/en/latest/usage/project/#working-with-version-control +# pdm.lock +# pdm.toml +.pdm-python +.pdm-build/ + +# pixi +# Similar to Pipfile.lock, it is generally recommended to include pixi.lock in version control. +# pixi.lock +# Pixi creates a virtual environment in the .pixi directory, just like venv module creates one +# in the .venv directory. It is recommended not to include this directory in version control. +.pixi/* +!.pixi/config.toml + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule* +celerybeat.pid + +# Redis +*.rdb +*.aof +*.pid + +# RabbitMQ +mnesia/ +rabbitmq/ +rabbitmq-data/ + +# ActiveMQ +activemq-data/ + +# SageMath parsed files +*.sage.py + +# Environments +.env +.envrc +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +# .idea/ + +# Abstra +# Abstra is an AI-powered process automation framework. +# Ignore directories containing user credentials, local state, and settings. +# Learn more at https://abstra.io/docs +.abstra/ + +# Visual Studio Code +# Visual Studio Code specific template is maintained in a separate VisualStudioCode.gitignore +# that can be found at https://github.com/github/gitignore/blob/main/Global/VisualStudioCode.gitignore +# and can be added to the global gitignore or merged into this file. However, if you prefer, +# you could uncomment the following to ignore the entire vscode folder +# .vscode/ +# Temporary file for partial code execution +tempCodeRunnerFile.py + +# Ruff stuff: +.ruff_cache/ + +# PyPI configuration file +.pypirc + +# Marimo +marimo/_static/ +marimo/_lsp/ +__marimo__/ + +# Streamlit +.streamlit/secrets.toml \ No newline at end of file diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000..7e68766 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,3 @@ +{ + "python-envs.pythonProjects": [] +} \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..f2fe624 --- /dev/null +++ b/README.md @@ -0,0 +1,160 @@ +# PythonBot Backend + +Бэкенд сервис с **FastAPI** + **Telegram Bot** и асинхронной очередью задач. + +## Возможности + +- 🚀 FastAPI REST API с авто-документацией (Swagger) +- 🤖 Telegram бот для создания и мониторинга задач +- ⚡ Асинхронная очередь задач с воркерами +- 🔔 Уведомления о завершении задач в Telegram + +## Структура проекта + +``` +PythonBot/ +├── Bot.py # Точка входа — запускает FastAPI + Telegram бота +├── main.py # FastAPI приложение + эндпоинты +├── bot.py # Telegram бот (aiogram) +├── tasks.py # Очередь задач + воркеры +├── config.py # Настройки (pydantic-settings) +├── requirements.txt +├── .env.example +└── README.md +``` + +## Быстрый старт + +### 1. Установка зависимостей + +```bash +pip install -r requirements.txt +``` + +### 2. Настройка + +```bash +cp .env.example .env +# Отредактируйте .env — укажите BOT_TOKEN от @BotFather +``` + +### 3. Запуск + +```bash +python Bot.py +``` + +Сервис запустится на `http://localhost:8000` + +## API эндпоинты + +| Метод | Путь | Описание | +|-------|------|----------| +| `POST` | `/tasks` | Создать задачу | +| `GET` | `/tasks` | Список задач (фильтр: `?status=pending`) | +| `GET` | `/tasks/{id}` | Статус задачи | +| `POST` | `/tasks/{id}/cancel` | Отменить задачу | +| `GET` | `/health` | Проверка здоровья | + +### Примеры запросов + +```bash +# Создать echo-задачу +curl -X POST http://localhost:8000/tasks \ + -H "Content-Type: application/json" \ + -d '{"name": "My Echo", "task_type": "echo", "params": {"message": "Hello!"}}' + +# Создать задачу решения капчи +curl -X POST http://localhost:8000/tasks \ + -H "Content-Type: application/json" \ + -d '{ + "name": "Solve CF Turnstile", + "task_type": "captcha_solve", + "params": { + "url": "https://example.com/page-with-captcha", + "captcha_type": "cloudflare_turnstile" + } + }' + +# Список задач +curl http://localhost:8000/tasks + +# Статус задачи +curl http://localhost:8000/tasks/{task_id} + +# Swagger UI +open http://localhost:8000/docs +``` + +## Telegram бот + +Получите токен у **@BotFather** и укажите в `.env`: + +``` +BOT_TOKEN=123456:ABC-DEF... +``` + +### Команды бота + +| Команда | Описание | +|---------|----------| +| `/start` | Приветствие + список команд | +| `/echo <текст>` | Создать echo-задачу | +| `/delay <текст> [сек]` | Отложенное сообщение | +| `/sum ` | Сумма двух чисел | +| `/captcha [тип]` | Решение капчи на странице | +| `/tasks` | Список ваших задач | +| `/task ` | Статус задачи | +| `/help` | Справка | + +## Типы задач + +| Тип | Параметры | Описание | +|-----|-----------|----------| +| `echo` | `message: str` | Вернуть сообщение | +| `delayed_message` | `message: str, delay: int` | Ответить через N секунд | +| `sum` | `a: int, b: int` | Сложить два числа | +| `captcha_solve` | `url: str, captcha_type: str` | Решить капчу на странице | + +## Добавление своих задач + +1. Опишите асинхронную функцию в `main.py`: + +```python +async def my_task(arg1: str, arg2: int) -> str: + await asyncio.sleep(2) + return f"Result: {arg1} {arg2}" +``` + +2. Добавьте в реестр в `main.py`: + +```python +TASK_REGISTRY = { + "echo": echo_task, + "delayed_message": delayed_task, + "sum": sum_task, + "my_task": my_task, # ← +} +``` + +3. Обновите фабрику `create_task()` аналогично. + +## Конфигурация + +| Переменная | По умолчанию | Описание | +|------------|-------------|----------| +| `HOST` | `0.0.0.0` | Хост FastAPI | +| `PORT` | `8000` | Порт FastAPI | +| `DEBUG` | `false` | Режим разработки (reload) | +| `BOT_TOKEN` | `` | Токен Telegram бота | +| `MAX_WORKERS` | `3` | Количество воркеров | +| `TASK_TIMEOUT` | `300` | Таймаут задачи (сек) | +| `CAPTCHA_SOLVER` | `click` | Тип решателя: `click` \| `2captcha` \| `10captcha` | +| `CAPTCHA_MAX_ATTEMPTS` | `3` | Макс. попыток решения | +| `CAPTCHA_ATTEMPT_DELAY` | `5` | Задержка между попытками (сек) | +| `TWO_CAPTCHA_API_KEY` | `` | API ключ 2Captcha | +| `TEN_CAPTCHA_API_KEY` | `` | API ключ 10Captcha | + +## Лиценхия + +MIT diff --git a/auth/__init__.py b/auth/__init__.py new file mode 100644 index 0000000..c258189 --- /dev/null +++ b/auth/__init__.py @@ -0,0 +1,4 @@ +from .manager import AuthManager +from .storage import AuthStorage + +__all__ = ["AuthManager", "AuthStorage"] \ No newline at end of file diff --git a/auth/manager.py b/auth/manager.py new file mode 100644 index 0000000..a546bb6 --- /dev/null +++ b/auth/manager.py @@ -0,0 +1,74 @@ +import asyncio +import hashlib +import secrets +import logging +from datetime import datetime, timedelta +from typing import Dict, Optional + +logger = logging.getLogger(__name__) + + +class AuthManager: + """Управляет авторизацией: хеширование паролей, сессии, верификация.""" + + def __init__(self, session_timeout_minutes: int = 120): + self._sessions: Dict[int, datetime] = {} + self._failed_attempts: Dict[int, list] = {} + self._max_attempts = 5 + self._lockout_duration = 300 # 5 минут блокировки + self._session_timeout = timedelta(minutes=session_timeout_minutes) + + @staticmethod + def _hash_password(password: str, salt: str = None) -> tuple: + if salt is None: + salt = secrets.token_hex(16) + result = hashlib.sha256((salt + password).encode()).hexdigest() + return result, salt + + def verify_password(self, password: str, password_hash: str, salt: str) -> bool: + computed, _ = self._hash_password(password, salt) + return secrets.compare_digest(computed, password_hash) + + def is_authenticated(self, user_id: int) -> bool: + if user_id not in self._sessions: + return False + if datetime.now() - self._sessions[user_id] > self._session_timeout: + self.logout(user_id) + return False + return True + + def login(self, user_id: int): + self._sessions[user_id] = datetime.now() + self._failed_attempts.pop(user_id, None) + logger.info(f"User {user_id} logged in") + + def logout(self, user_id: int): + self._sessions.pop(user_id, None) + self._failed_attempts.pop(user_id, None) + logger.info(f"User {user_id} logged out") + + def is_locked_out(self, user_id: int) -> bool: + if user_id not in self._failed_attempts: + return False + attempts = self._failed_attempts[user_id] + recent = [t for t in attempts if datetime.now() - t < timedelta(minutes=5)] + self._failed_attempts[user_id] = recent + if len(recent) >= self._max_attempts: + return True + return False + + def record_failed_attempt(self, user_id: int): + if user_id not in self._failed_attempts: + self._failed_attempts[user_id] = [] + self._failed_attempts[user_id].append(datetime.now()) + + def get_session_info(self, user_id: int) -> Optional[dict]: + if user_id not in self._sessions: + return None + login_time = self._sessions[user_id] + elapsed = datetime.now() - login_time + return { + "user_id": user_id, + "login_time": login_time.isoformat(), + "elapsed_minutes": int(elapsed.total_seconds() // 60), + } \ No newline at end of file diff --git a/auth/storage.py b/auth/storage.py new file mode 100644 index 0000000..6e360a4 --- /dev/null +++ b/auth/storage.py @@ -0,0 +1,88 @@ +""" +Хранилище учётных данных пользователей. +""" + +import json +import asyncio +import logging +from pathlib import Path +from typing import Optional, Dict + +logger = logging.getLogger(__name__) + + +class AuthStorage: + """Хранилище паролей пользователей в JSON-файле.""" + + def __init__(self, file_path: str = "data/users.json"): + self.file_path = Path(file_path) + self.file_path.parent.mkdir(parents=True, exist_ok=True) + self._lock = asyncio.Lock() + + async def register(self, user_id: int, password_hash: str, salt: str) -> bool: + """Регистрирует нового пользователя. Возвращает True если успешно.""" + async with self._lock: + data = self._load() + if str(user_id) in data: + return False + data[str(user_id)] = { + "password_hash": password_hash, + "salt": salt, + } + self._save(data) + logger.info(f"User {user_id} registered") + return True + + async def get_user(self, user_id: int) -> Optional[Dict]: + """Получает данные пользователя по ID.""" + async with self._lock: + data = self._load() + return data.get(str(user_id)) + + async def change_password(self, user_id: int, new_hash: str, new_salt: str) -> bool: + """Сменить пароль пользователя.""" + async with self._lock: + data = self._load() + user_str = str(user_id) + if user_str not in data: + return False + data[user_str]["password_hash"] = new_hash + data[user_str]["salt"] = new_salt + self._save(data) + logger.info(f"User {user_id} changed password") + return True + + async def delete_user(self, user_id: int) -> bool: + """Удалить учётную запись пользователя.""" + async with self._lock: + data = self._load() + user_str = str(user_id) + if user_str not in data: + return False + del data[user_str] + self._save(data) + logger.info(f"User {user_id} deleted") + return True + + async def user_exists(self, user_id: int) -> bool: + """Проверяет, существует ли пользователь.""" + async with self._lock: + data = self._load() + return str(user_id) in data + + def _load(self) -> dict: + if not self.file_path.exists(): + return {} + try: + with open(self.file_path, "r", encoding="utf-8") as f: + return json.load(f) + except (json.JSONDecodeError, IOError) as e: + logger.warning(f"Failed to load auth storage: {e}") + return {} + + def _save(self, data: dict): + try: + with open(self.file_path, "w", encoding="utf-8") as f: + json.dump(data, f, ensure_ascii=False, indent=2) + except IOError as e: + logger.error(f"Failed to save auth storage: {e}") \ No newline at end of file diff --git a/config/__init__.py b/config/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/config/settings.py b/config/settings.py new file mode 100644 index 0000000..09c73fb --- /dev/null +++ b/config/settings.py @@ -0,0 +1,60 @@ +from pydantic_settings import BaseSettings +from typing import Optional +from pathlib import Path + + +class Settings(BaseSettings): + """Application settings.""" + + # Bot + BOT_TOKEN: str = "" + + # Telethon/MTProto settings + API_ID: int = 12345 + API_HASH: str = "abc123" + + # IRC Configuration + IRC_SERVER: str = "irc.chat.twitch.tv" + IRC_PORT: int = 6697 # SSL port + IRC_USERNAME: str = "justinfan12345" + IRC_OAUTH: str = "oauth:12345" + + # Browser Configuration + VIEWPORT_WIDTH: int = 1366 + VIEWPORT_HEIGHT: int = 768 + + # Default values + DEFAULT_MIN_DELAY: int = 10 + DEFAULT_MAX_DELAY: int = 30 + DEFAULT_MIN_READING: int = 10 + DEFAULT_MAX_READING: int = 30 + DEFAULT_MONITOR_MINUTES: int = 10 + DEFAULT_VISITS_PER_LINK: int = 5 + MTPROTO_SECRET: str = "00000000000000000000000000000000" + # Timing + MAX_REDIRECT_WAIT: int = 30 + MAX_RECONNECTS: int = 10 + PONG_TIMEOUT: int = 300 + SOCKET_TIMEOUT: int = 30 + RECV_TIMEOUT: float = 5.0 + + # Proxy + PROXY_FILE: str = "proxies.txt" + PROXY_COOLDOWN_TIME: int = 30 + PROXY_MAX_USAGE_BEFORE_COOLDOWN: int = 5 + + # Screenshots + SCREENSHOTS_DIR: str = "" + + # Logging + LOG_LEVEL: str = "INFO" + LOG_FILE: str = "logs/bot.log" + + class Config: + env_file = ".env" + env_file_encoding = "utf-8" + case_sensitive = True + extra = "ignore" # Игнорировать лишние поля из .env + + +settings = Settings() \ No newline at end of file diff --git a/core/__init__.py b/core/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/core/constants.py b/core/constants.py new file mode 100644 index 0000000..161285d --- /dev/null +++ b/core/constants.py @@ -0,0 +1,39 @@ +"""Application constants.""" + +from enum import Enum + + +class TaskStatus(str, Enum): + """Task status enum.""" + RUNNING = "running" + COMPLETED = "completed" + FAILED = "failed" + CANCELLED = "cancelled" + + +class TaskType(str, Enum): + """Type of background tasks.""" + VISIT = "visit" + TWITCH_IRC = "twitch_irc" + + +class ProxyStatus(str, Enum): + """Proxy status.""" + WORKING = "working" + NOT_WORKING = "not_working" + + +# Regex patterns +URL_PATTERN = r'https?://[^\s<>"]+' + +# Message templates +WELCOME_MESSAGE = ( + "👋 **Бот для посещения страниц и мониторинга Twitch**\n\n" + "**Команды:**\n" + "/visit — запустить периодическое посещение\n" + "/twitch_irc — мониторинг чата Twitch через IRC\n" + "/stop — остановить все задачи\n" + "/tasks — список активных задач\n" + "/proxies — статистика прокси\n" + "/help — подробная справка" +) \ No newline at end of file diff --git a/core/exceptions.py b/core/exceptions.py new file mode 100644 index 0000000..8c68681 --- /dev/null +++ b/core/exceptions.py @@ -0,0 +1,41 @@ +"""Custom exceptions for the application.""" + + +class BotError(Exception): + """Base exception for bot errors.""" + pass + + +class ProxyError(BotError): + """Proxy related errors.""" + pass + + +class ProxyTimeoutError(ProxyError): + """Raised when proxy acquisition times out.""" + pass + + +class NoProxiesAvailableError(ProxyError): + """Raised when no proxies are available.""" + pass + + +class BrowserError(BotError): + """Browser related errors.""" + pass + + +class IRCError(BotError): + """IRC related errors.""" + pass + + +class IRCConnectionError(IRCError): + """Raised when IRC connection fails.""" + pass + + +class TaskError(BotError): + """Task related errors.""" + pass \ No newline at end of file diff --git a/core/logger.py b/core/logger.py new file mode 100644 index 0000000..56b2a0a --- /dev/null +++ b/core/logger.py @@ -0,0 +1,62 @@ +"""Logging configuration.""" + +import logging +import sys +from pathlib import Path +from logging.handlers import RotatingFileHandler + + +def setup_logger( + name: str = "bot", + log_level: int = logging.INFO, + log_file: str = "logs/bot.log" +) -> logging.Logger: + """ + Configure and return logger instance. + + Args: + name: Logger name + log_level: Logging level + log_file: Path to log file + + Returns: + Configured logger instance + """ + logger = logging.getLogger(name) + logger.setLevel(log_level) + + # Create formatters + console_formatter = logging.Formatter( + '%(asctime)s | %(levelname)-8s | %(name)s | %(message)s', + datefmt='%Y-%m-%d %H:%M:%S' + ) + + file_formatter = logging.Formatter( + '%(asctime)s | %(levelname)-8s | %(name)s | %(funcName)s:%(lineno)d | %(message)s', + datefmt='%Y-%m-%d %H:%M:%S' + ) + + # Console handler + console_handler = logging.StreamHandler(sys.stdout) + console_handler.setLevel(log_level) + console_handler.setFormatter(console_formatter) + + # File handler + if log_file: + Path(log_file).parent.mkdir(parents=True, exist_ok=True) + file_handler = RotatingFileHandler( + log_file, + maxBytes=10*1024*1024, # 10MB + backupCount=5, + encoding='utf-8' + ) + file_handler.setLevel(log_level) + file_handler.setFormatter(file_formatter) + logger.addHandler(file_handler) + + logger.addHandler(console_handler) + + return logger + + +logger = setup_logger() \ No newline at end of file diff --git a/data/chats.json b/data/chats.json new file mode 100644 index 0000000..128e8d7 --- /dev/null +++ b/data/chats.json @@ -0,0 +1,5 @@ +{ + "1126721382": [ + "jentteno" + ] +} \ No newline at end of file diff --git a/data/tasks.json b/data/tasks.json new file mode 100644 index 0000000..2729acf --- /dev/null +++ b/data/tasks.json @@ -0,0 +1,25 @@ +{ + "twitch_jentteno_195246": { + "url": "", + "task_type": "twitch_irc", + "min_delay": 10, + "max_delay": 30, + "min_reading": 10, + "max_reading": 30, + "max_visits": null, + "current_visit": 0, + "channel": "jentteno", + "target_username": "*", + "visits_per_link": 5, + "monitor_minutes": 30, + "allowed_domains": null, + "paused": true, + "completed": true, + "stopped": false, + "total_visits": 0, + "successful_visits": 0, + "links_found": 0, + "chat_id": 1126721382, + "started_at": "2026-05-12T19:52:46.692107" + } +} \ No newline at end of file diff --git a/handlers/__init__.py b/handlers/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/handlers/commands.py b/handlers/commands.py new file mode 100644 index 0000000..e86c34d --- /dev/null +++ b/handlers/commands.py @@ -0,0 +1,924 @@ +""" +Обработчики команд с удобным интерфейсом управления задачами. +""" + +import asyncio +import logging +from datetime import datetime +from typing import Optional, Dict + +from aiogram import Dispatcher, F, Bot +from aiogram.filters import Command +from aiogram.types import ( + Message, CallbackQuery, InlineKeyboardMarkup, + InlineKeyboardButton, ReplyKeyboardMarkup, KeyboardButton +) +from aiogram.utils.keyboard import InlineKeyboardBuilder, ReplyKeyboardBuilder + +from config.settings import settings +from managers.background_tasks import BackgroundTaskManager +from managers.proxy_manager import ProxyManager +from managers.task_manager import TaskManager, TaskParams +from services.browser_service import BrowserService +from services.visit_service import VisitScheduler +from utils.helpers import parse_range, format_range, extract_domain +from utils.telegram import send_message_safe, send_visit_result +from managers.storage import TaskStorage, ChatStorage +from auth.manager import AuthManager +from auth.storage import AuthStorage + +logger = logging.getLogger(__name__) + + +class BotInterface: + """Интерфейс бота с управлением задачами.""" + + def __init__( + self, + background_tasks: BackgroundTaskManager, + proxy_manager: ProxyManager, + browser_service: BrowserService, + storage: TaskStorage = None, + chat_storage: ChatStorage = None, + ): + self.background_tasks = background_tasks + self.proxy_manager = proxy_manager + self.browser_service = browser_service + self.task_manager = TaskManager() + self.storage = storage + self.chat_storage = chat_storage + self.auth_manager = AuthManager() + self.auth_storage = AuthStorage() + self._user_input_state: Dict[int, dict] = {} + + def register(self, dp: Dispatcher): + """Регистрация всех обработчиков.""" + + # Сохраняем ссылку на self для использования в обработчиках + interface = self + + # --- АВТОРИЗАЦИЯ --- + + @dp.message(Command("register")) + async def cmd_register(message: Message): + """Регистрация: /register <пароль>""" + parts = message.text.split(maxsplit=1) + if len(parts) < 2 or len(parts[1]) < 4: + await message.answer( + "❌ Использование: /register <пароль от 4 символов>" + ) + return + password = parts[1] + user_id = message.from_user.id + exists = await interface.auth_storage.user_exists(user_id) + if exists: + await message.answer( + "⚠️ Вы уже зарегистрированы. Используйте /login <пароль>" + ) + return + password_hash, salt = interface.auth_manager._hash_password(password) + success = await interface.auth_storage.register( + user_id, password_hash, salt + ) + if success: + interface.auth_manager.login(user_id) + await message.answer( + "✅ Регистрация прошла успешно!\n" + "Теперь вы можете использовать бота." + ) + else: + await message.answer("❌ Ошибка регистрации") + + @dp.message(Command("login")) + async def cmd_login(message: Message): + """Вход: /login <пароль>""" + parts = message.text.split(maxsplit=1) + if len(parts) < 2: + await message.answer("❌ Использование: /login <пароль>") + return + password = parts[1] + user_id = message.from_user.id + user_data = await interface.auth_storage.get_user(user_id) + if not user_data: + await message.answer( + "❌ Вы не зарегистрированы.\n" + "Используйте /register <пароль>" + ) + return + locked = interface.auth_manager.is_locked_out(user_id) + if locked: + await message.answer("🔒 Слишком много попыток. Попробуйте позже.") + return + if interface.auth_manager.verify_password( + password, user_data["password_hash"], user_data["salt"] + ): + interface.auth_manager.login(user_id) + await message.answer( + "✅ Вы вошли в систему!\n" + "Используйте /start для главного меню." + ) + else: + interface.auth_manager.record_failed_attempt(user_id) + await message.answer("❌ Неверный пароль") + + @dp.message(Command("logout")) + async def cmd_logout(message: Message): + """Выйти из аккаунта.""" + interface.auth_manager.logout(message.from_user.id) + await message.answer("🔑 Вы вышли из системы.") + + # Декоратор: доступ только для авторизованных + async def require_auth(handler_fn, event): + user_id = event.from_user.id + if not interface.auth_manager.is_authenticated(user_id): + await event.answer( + "🔒 Требуется авторизация!\n\n" + "/register <пароль> — зарегистрироваться\n" + "/login <пароль> — войти" + ) + return False + return True + +# === Главное меню === + @dp.message(Command("start")) + @dp.message(F.text == "📋 Главное меню") + async def cmd_start(message: Message): + if not await require_auth(message): + return + await interface._show_main_menu(message) + + @dp.callback_query(F.data == "menu_main") + async def cb_main_menu(callback: CallbackQuery): + if not await require_auth(callback): + return + await interface._show_main_menu(callback.message) + await callback.answer() + + # === Стримеры === + @dp.message(F.text == "📺 Стримеры") + @dp.message(Command("streamers")) + async def btn_streamers(message: Message): + if not await require_auth(message): + return + await interface._show_streamers_list(message) + + @dp.callback_query(F.data == "menu_streamers") + async def cb_streamers(callback: CallbackQuery): + if not await require_auth(callback): + return + await interface._show_streamers_list(callback.message, edit=True) + await callback.answer() + + # === Задачи === + @dp.message(F.text == "📊 Задачи") + @dp.message(Command("tasks")) + async def btn_tasks(message: Message): + if not await require_auth(message): + return + await interface._show_tasks_list(message) + + @dp.callback_query(F.data == "menu_tasks") + async def cb_tasks(callback: CallbackQuery): + if not await require_auth(callback): + return + await interface._show_tasks_list(callback.message, edit=True) + await callback.answer() + + # === Статус === + @dp.message(F.text == "📈 Статус") + async def btn_status(message: Message): + if not await require_auth(message): + return + await interface._show_status(message) + + @dp.callback_query(F.data == "menu_status") + async def cb_status(callback: CallbackQuery): + if not await require_auth(callback): + return + await interface._show_status(callback.message, edit=True) + await callback.answer() + + # === Стоп всё === + @dp.message(F.text == "🛑 Остановить всё") + @dp.callback_query(F.data == "stop_all") + async def handle_stop_all(event): + if not await require_auth(event): + return + count = await interface.background_tasks.cancel_all() + for task_id in list(interface.task_manager._tasks.keys()): + await interface.task_manager.remove_task(task_id) + + text = f"🛑 Остановлено задач: {count}" + kb = InlineKeyboardBuilder() + kb.row(InlineKeyboardButton(text="🔙 В меню", callback_data="menu_main")) + + if isinstance(event, CallbackQuery): + await event.message.edit_text(text, reply_markup=kb.as_markup()) + await event.answer() + else: + await event.answer(text, reply_markup=kb.as_markup()) + + # === Быстрый старт === + @dp.message(F.text == "🚀 Быстрый старт") + async def btn_quick(message: Message): + if not await require_auth(message): + return + await message.answer( + "🚀 **БЫСТРЫЙ СТАРТ**\n\n" + "**Twitch мониторинг:**\n" + "`<канал> [посещений] [минут] [задержка] [чтение] [домены]`\n\n" + "**Пример:** `streamer user123 5 30 10-30 10-30`\n\n" + "**Посещение URL:**\n" + "` [задержка] [повторов] [чтение]`\n\n" + "**Пример:** `https://example.com 10-30 100 15-30`" + ) + + # === Управление задачей === + @dp.callback_query(F.data.startswith("tdetail_")) + async def cb_task_detail(callback: CallbackQuery): + if not await require_auth(callback): + return + task_id = callback.data.replace("tdetail_", "", 1) + await interface._show_task_detail(callback, task_id) + await callback.answer() + + @dp.callback_query(F.data.startswith("tpause_")) + async def cb_task_pause(callback: CallbackQuery): + if not await require_auth(callback): + return + task_id = callback.data.replace("tpause_", "", 1) + await interface.task_manager.pause_task(task_id) + await callback.answer("⏸️ Пауза") + await interface._show_task_detail(callback, task_id) + + @dp.callback_query(F.data.startswith("tresume_")) + async def cb_task_resume(callback: CallbackQuery): + if not await require_auth(callback): + return + task_id = callback.data.replace("tresume_", "", 1) + await interface.task_manager.resume_task(task_id) + await callback.answer("▶️ Возобновлено") + await interface._show_task_detail(callback, task_id) + + @dp.callback_query(F.data.startswith("tstop_")) + async def tstop(callback: CallbackQuery): + if not await require_auth(callback): + return + tid = callback.data.replace("tstop_", "", 1) + await interface.task_manager.stop_task(tid) + await interface.background_tasks.cancel_task(tid) + await callback.answer("🛑 Остановлена") + await interface._show_task_detail(callback, tid) + + @dp.callback_query(F.data.startswith("tskip_")) + async def cb_task_skip(callback: CallbackQuery): + if not await require_auth(callback): + return + task_id = callback.data.replace("tskip_", "", 1) + await interface.task_manager.update_task(task_id, skip_next=True) + await callback.answer("⏭️ Пропущено") + + @dp.callback_query(F.data.startswith("trestart_")) + async def cb_task_restart(callback: CallbackQuery): + data = callback.data + + # Проверяем, что это именно перезапуск задачи (trestart_task_) + if data.startswith("trestart_task_"): + tid = data.replace("trestart_task_", "", 1) + logger.info(f"Перезапуск задачи: {tid}") + + params = await interface.task_manager.get_task(tid) + if not params: + await callback.answer("❌ Не найдена") + return + + await interface.task_manager.update_task( + tid, + completed=False, stopped=False, paused=False, + current_visit=0, total_visits=0, + successful_visits=0, links_found=0, + started_at=datetime.now() + ) + + ok = await interface.background_tasks.start_task( + task_id=tid, + coro=interface._run_twitch(tid, params, callback.message), + task_type="twitch_irc", + metadata={'type': 'twitch_irc', 'channel': params.channel, 'chat_id': params.chat_id} + ) + + if ok: + await callback.answer("🔄 Перезапущено!") + else: + await callback.answer("❌ Ошибка запуска") + + await interface._show_task_detail(callback, tid) + else: + # Обычный сброс статистики (trestart_ без task) + task_id = data.replace("trestart_", "", 1) + await interface.task_manager.restart_task(task_id) + await callback.answer("🔄 Статистика сброшена") + await interface._show_task_detail(callback, task_id) + + @dp.callback_query(F.data.startswith("tset_")) + async def cb_task_set(callback: CallbackQuery): + if not await require_auth(callback): + return + """Установка параметра задачи.""" + data = callback.data + + logger.info(f"SET callback: {data}") + + # Формат: tset_TASKID_PARAM + # Убираем префикс + data = data.replace("tset_", "", 1) + + # Ищем последнее подчеркивание для разделения task_id и param + idx = data.rfind("_") + if idx == -1: + await callback.answer("❌ Неверный формат") + return + + task_id = data[:idx] + param = data[idx + 1:] + + logger.info(f"Parsed: task_id={task_id[:20]}, param={param}") + + # Сохраняем состояние ожидания ввода + interface._user_input_state[callback.from_user.id] = { + "task_id": task_id, + "param": param + } + + prompts = { + "delay": "⏱️ Задержка (сек): `10-30` или `15`", + "reading": "📖 Чтение (сек): `10-30` или `20`", + "perlink": "🔗 Посещений на ссылку: `5`", + "minutes": "⏰ Длительность (мин): `30`", + "domains": "🌐 Домены через запятую: `site1.com, site2.ru`\nИли `all` для всех", + } + + builder = InlineKeyboardBuilder() + builder.row(InlineKeyboardButton(text="❌ Отмена", callback_data=f"tdetail_{task_id}")) + + prompt = prompts.get(param, f"Введите значение для {param}:") + await callback.message.answer(prompt, reply_markup=builder.as_markup()) + await callback.answer(f"Ожидаю ввод: {param}") + + # === Ввод параметров === + @dp.message() + async def handle_all_text(message: Message): + """Обрабатывает ВСЕ текстовые сообщения.""" + text = message.text.strip() if message.text else "" + user_id = message.from_user.id + + if not interface.auth_manager.is_authenticated(user_id): + await message.answer( + "🔒 Требуется авторизация!\n\n" + "/register <пароль> — зарегистрироваться\n" + "/login <пароль> — войти" + ) + return + + logger.info(f"Handle text: '{text[:50]}' from user {user_id}, waiting: {user_id in interface._user_input_state}") + + # 1. Проверяем - ожидаем ввод параметра + if user_id in interface._user_input_state: + await interface._process_param_input(message) + return + + # 2. Проверяем - это URL + if text.startswith('http://') or text.startswith('https://'): + parts = text.split() + await interface._add_visit(message) + return + + # 3. Проверяем - это команда добавления стримера с параметрами + # Формат: channel [visits] [minutes] [delay] [reading] [domains] + parts = text.split() + if len(parts) >= 1 and not text.startswith('/'): + # Проверяем что первый параметр похож на канал (буквы/цифры/_) + import re + if re.match(r'^[a-zA-Z0-9_]+$', parts[0]): + await interface._add_streamer(message, parts) + return + + # 4. Если ничего не подошло + logger.info(f"Unhandled message: {text[:50]}") + + @dp.callback_query(F.data.startswith("tdelete_")) + async def tdelete(callback: CallbackQuery): + if not await require_auth(callback): + return + tid = callback.data.replace("tdelete_", "", 1) + await interface.task_manager.remove_task(tid) + await callback.answer("🗑️ Удалена") + await interface._show_streamers_list(callback.message, edit=True) + + + @dp.callback_query(F.data.startswith("treset_")) + async def treset(callback: CallbackQuery): + if not await require_auth(callback): + return + """Сброс статистики активной задачи.""" + tid = callback.data.replace("treset_", "", 1) + await interface.task_manager.update_task( + tid, + current_visit=0, + total_visits=0, + successful_visits=0, + links_found=0 + ) + await callback.answer("🔄 Статистика сброшена") + await interface._show_task_detail(callback, tid) + + + # ========================================================================= + # ДОБАВЛЕНИЕ ЗАДАЧ + # ========================================================================= + + async def _add_streamer(self, message: Message, parts: list): + channel = parts[0].lower() + + # Проверяем, что второй параметр - число (посещений), а не username + vl = settings.DEFAULT_VISITS_PER_LINK + mm = settings.DEFAULT_MONITOR_MINUTES + mind = settings.DEFAULT_MIN_DELAY + maxd = settings.DEFAULT_MAX_DELAY + minr = settings.DEFAULT_MIN_READING + maxr = settings.DEFAULT_MAX_READING + domains = None + + try: + if len(parts) > 1: vl = int(parts[1]) + if len(parts) > 2: mm = int(parts[2]) + if len(parts) > 3 and '-' in parts[3]: mind, maxd = parse_range(parts[3]) + if len(parts) > 4 and '-' in parts[4]: minr, maxr = parse_range(parts[4], default_min=1) + if len(parts) > 5: domains = [d.strip() for d in parts[5].split(',')] + except ValueError as e: + await message.answer(f"❌ {e}") + return + + tid = f"twitch_{channel}_{datetime.now().strftime('%H%M%S')}" + + params = TaskParams( + task_type="twitch_irc", + channel=channel, + target_username="*", # Все пользователи + visits_per_link=vl, + monitor_minutes=mm, + min_delay=mind, max_delay=maxd, + min_reading=minr, max_reading=maxr, + allowed_domains=domains, + started_at=datetime.now(), + chat_id=message.chat.id + ) + + await self.task_manager.add_task(tid, params) + + success = await self.background_tasks.start_task( + task_id=tid, + coro=self._run_twitch(tid, params, message), + task_type="twitch_irc", + metadata={'type': 'twitch_irc', 'channel': channel, 'chat_id': message.chat.id} + ) + + if success: + if self.storage: + await self.storage.save_task(tid, params) + if self.chat_storage: + await self.chat_storage.add_streamer(message.chat.id, channel) + domains_str = ", ".join(domains) if domains else "все" + text = ( + f"✅ **МОНИТОРИНГ ЗАПУЩЕН**\n\n" + f"📺 Канал: `{channel}`\n" + f"👤 Отслеживание: все пользователи\n" + f"🔗 На ссылку: {vl} визитов\n" + f"⏱️ Мониторинг: {mm} мин\n" + f"⏱️ Задержка: {format_range(mind, maxd)}с\n" + f"📖 Чтение: {format_range(minr, maxr)}с\n" + f"🌐 Домены: {domains_str}\n" + ) + text += f"\n🆔 `{tid[:12]}...`\n📺 /streamers — управление" + await message.answer(text) + + async def _add_visit(self, message: Message): + """Добавление URL для посещения.""" + parts = message.text.strip().split() + url = parts[0] + + min_delay = settings.DEFAULT_MIN_DELAY + max_delay = settings.DEFAULT_MAX_DELAY + min_reading = settings.DEFAULT_MIN_READING + max_reading = settings.DEFAULT_MAX_READING + max_visits = None + + try: + if len(parts) > 1 and '-' in parts[1]: + min_delay, max_delay = parse_range(parts[1]) + elif len(parts) > 1: + min_delay = max_delay = int(parts[1]) + if len(parts) > 2: + if '-' in parts[2]: + min_reading, max_reading = parse_range(parts[2], default_min=1) + else: + max_visits = int(parts[2]) + if len(parts) > 3 and '-' in parts[3]: + min_reading, max_reading = parse_range(parts[3], default_min=1) + except ValueError as e: + await message.answer(f"❌ Ошибка: {e}") + return + + domain = extract_domain(url)[:20] + task_id = f"visit_{domain}_{datetime.now().strftime('%H%M%S')}" + + params = TaskParams( + url=url, task_type="visit", + min_delay=min_delay, max_delay=max_delay, + min_reading=min_reading, max_reading=max_reading, + max_visits=max_visits, + started_at=datetime.now(), chat_id=message.chat.id + ) + + await self.task_manager.add_task(task_id, params) + + scheduler = VisitScheduler( + browser_service=self.browser_service, + url=url, + min_delay=min_delay, max_delay=max_delay, + min_reading=min_reading, max_reading=max_reading, + max_visits=max_visits, + ) + + async def on_complete(visit_num, result, delay, reading): + await send_visit_result(message.bot, message.chat.id, visit_num, result, delay, reading, scheduler.successful, scheduler.failed) + + scheduler.on_visit_complete = on_complete + + success = await self.background_tasks.start_task( + task_id=task_id, coro=scheduler.run(), task_type="visit", + metadata={'type': 'visit', 'url': url, 'chat_id': message.chat.id} + ) + + if success: + await message.answer( + f"✅ **ПОСЕЩЕНИЕ ЗАПУЩЕНО**\n\n" + f"🌐 `{url}`\n" + f"⏱️ Задержка: {format_range(min_delay, max_delay)}с\n" + f"📖 Чтение: {format_range(min_reading, max_reading)}с\n" + f"🔢 Повторов: {'∞' if max_visits is None else max_visits}\n" + f"🆔 `{task_id}`" + ) + + # ========================================================================= + # ОТОБРАЖЕНИЕ + # ========================================================================= + + async def _show_main_menu(self, message: Message): + """Главное меню.""" + stats = await self.task_manager.get_stats() + + text = ( + "👋 **ГЛАВНОЕ МЕНЮ**\n\n" + f"📺 Twitch: **{stats['twitch']}**\n" + f"🌐 Визитов: **{stats['visit']}**\n" + f"⏸️ Пауза: **{stats['paused']}**\n\n" + "Выберите раздел:" + ) + + builder = InlineKeyboardBuilder() + builder.row(InlineKeyboardButton(text=f"📺 Стримеры ({stats['twitch']})", callback_data="menu_streamers")) + builder.row(InlineKeyboardButton(text=f"📊 Задачи ({stats['total']})", callback_data="menu_tasks")) + builder.row(InlineKeyboardButton(text="📈 Статус", callback_data="menu_status")) + builder.row(InlineKeyboardButton(text="🛑 Остановить всё", callback_data="stop_all")) + + reply = ReplyKeyboardBuilder() + reply.row(KeyboardButton(text="📺 Стримеры"), KeyboardButton(text="📊 Задачи")) + reply.row(KeyboardButton(text="📈 Статус"), KeyboardButton(text="🚀 Быстрый старт")) + reply.row(KeyboardButton(text="🛑 Остановить всё"), KeyboardButton(text="📋 Главное меню")) + + await message.answer(text, reply_markup=reply.as_markup(resize_keyboard=True)) + await message.answer("💡 Кнопки управления:", reply_markup=builder.as_markup()) + + async def _show_streamers_list(self, message: Message, edit: bool = False): + active = await self.task_manager.get_active_tasks() + completed = await self.task_manager.get_completed_tasks() + + twitch_active = {tid: p for tid, p in active.items() if p.task_type == "twitch_irc"} + twitch_done = {tid: p for tid, p in completed.items() if p.task_type == "twitch_irc"} + + if not twitch_active and not twitch_done: + text = "📺 **СТРИМЕРЫ**\n\nНет задач.\nОтправьте `<канал>` для добавления." + builder = InlineKeyboardBuilder() + builder.row(InlineKeyboardButton(text="🔙 В меню", callback_data="menu_main")) + else: + text = "📺 **СТРИМЕРЫ**\n\n" + builder = InlineKeyboardBuilder() + + # Активные + if twitch_active: + text += "**🔄 Активные:**\n" + for tid, p in list(twitch_active.items())[:10]: + emoji = p.get_status_emoji() + domains = ", ".join(p.allowed_domains) if p.allowed_domains else "все" + + text += ( + f"{emoji} **{p.channel}**\n" + f" 🔗{p.links_found} | 📊{p.total_visits} | 🌐{domains}\n" + ) + builder.row(InlineKeyboardButton( + text=f"{emoji} {p.channel} (🔗{p.links_found})", + callback_data=f"tdetail_{tid}" + )) + + # Завершенные + if twitch_done: + text += "\n**📁 Завершенные:**\n" + for tid, p in list(twitch_done.items())[:5]: + emoji = p.get_status_emoji() + domains = ", ".join(p.allowed_domains) if p.allowed_domains else "все" + + text += ( + f"{emoji} **{p.channel}**\n" + f" 🔗{p.links_found} | 📊{p.total_visits} | 🌐{domains}\n" + ) + builder.row(InlineKeyboardButton( + text=f"{emoji} {p.channel} (завершена)", + callback_data=f"tdetail_{tid}" + )) + + builder.row(InlineKeyboardButton(text="🔙 В меню", callback_data="menu_main")) + + if edit: + try: + await message.edit_text(text, reply_markup=builder.as_markup()) + except Exception as e: + logger.debug(f"Edit message skipped: {e}") + else: + await message.answer(text, reply_markup=builder.as_markup()) + + async def _show_tasks_list(self, message: Message, edit: bool = False): + """Все задачи.""" + text = await self.task_manager.format_task_list() + builder = InlineKeyboardBuilder() + builder.row(InlineKeyboardButton(text="🔄 Обновить", callback_data="menu_tasks")) + builder.row(InlineKeyboardButton(text="🔙 В меню", callback_data="menu_main")) + + if edit: + try: + await message.edit_text(text, reply_markup=builder.as_markup()) + except Exception as e: + logger.debug(f"Edit message skipped: {e}") + else: + await message.answer(text, reply_markup=builder.as_markup()) + + async def _show_status(self, message: Message, edit: bool = False): + """Статус.""" + stats = await self.task_manager.get_stats() + proxy_stats = self.proxy_manager.get_stats() + + text = ( + "📈 **СТАТУС**\n\n" + f"📺 Twitch: **{stats['twitch']}**\n" + f"🌐 Визиты: **{stats['visit']}**\n" + f"⏸️ Пауза: **{stats['paused']}**\n" + f"🔄 Активно: **{stats['active']}**\n\n" + f"🔌 Прокси: {proxy_stats['total']} всего, {proxy_stats['available']} доступно" + ) + + builder = InlineKeyboardBuilder() + builder.row(InlineKeyboardButton(text="🔄 Обновить", callback_data="menu_status")) + builder.row(InlineKeyboardButton(text="🔙 В меню", callback_data="menu_main")) + + if edit: + try: + await message.edit_text(text, reply_markup=builder.as_markup()) + except Exception as e: + logger.debug(f"Edit message skipped: {e}") + else: + await message.answer(text, reply_markup=builder.as_markup()) + + # ========================================================================= + # УПРАВЛЕНИЕ ЗАДАЧЕЙ + # ========================================================================= + + async def _show_task_detail(self, callback: CallbackQuery, task_id: str): + params = await self.task_manager.get_task(task_id) + if not params: + await callback.answer("❌ Задача не найдена") + return + + text = f"📋 **УПРАВЛЕНИЕ**\n\n{params.get_stats_text()}" + + builder = InlineKeyboardBuilder() + + # Для активных задач + if not params.completed and not params.stopped: + if params.paused: + builder.row(InlineKeyboardButton(text="▶️ Возобновить", callback_data=f"tresume_{task_id}")) + else: + builder.row(InlineKeyboardButton(text="⏸️ Пауза", callback_data=f"tpause_{task_id}")) + + builder.row( + InlineKeyboardButton(text="⏱️ Задержка", callback_data=f"tset_{task_id}_delay"), + InlineKeyboardButton(text="📖 Чтение", callback_data=f"tset_{task_id}_reading") + ) + builder.row( + InlineKeyboardButton(text="🔗 На ссылку", callback_data=f"tset_{task_id}_perlink"), + InlineKeyboardButton(text="⏰ Минуты", callback_data=f"tset_{task_id}_minutes") + ) + builder.row( + InlineKeyboardButton(text="🌐 Домены", callback_data=f"tset_{task_id}_domains"), + InlineKeyboardButton(text="⏭️ Пропустить", callback_data=f"tskip_{task_id}") + ) + builder.row( + InlineKeyboardButton(text="🔄 Сброс", callback_data=f"treset_{task_id}"), + InlineKeyboardButton(text="🛑 Стоп", callback_data=f"tstop_{task_id}") + ) + else: + # Для завершенных/остановленных + builder.row( + InlineKeyboardButton(text="🔄 Перезапустить", callback_data=f"trestart_task_{task_id}"), + ) + builder.row( + InlineKeyboardButton(text="🗑️ Удалить", callback_data=f"tdelete_{task_id}") + ) + + builder.row(InlineKeyboardButton(text="🔙 К списку", callback_data="menu_streamers")) + + try: + await callback.message.edit_text(text, reply_markup=builder.as_markup()) + except Exception as e: + if "not modified" not in str(e): + logger.error(f"Edit error: {e}") + + + + async def _process_param_input(self, message: Message): + """Обработка ввода параметров.""" + user_id = message.from_user.id + state = self._user_input_state.pop(user_id) + + task_id = state["task_id"] + param = state["param"] + + # Получаем текст и очищаем от кавычек + value = message.text.strip() if message.text else "" + value = value.replace("'", "").replace('"', "").replace("`", "").strip() + + logger.info(f"Process param: {param} = {value}") + + if not value: + await message.answer("❌ Пустое значение. Попробуйте ещё раз.") + return + + try: + if param == "delay": + if "-" in value: + mn, mx = parse_range(value) + else: + mn = mx = int(value) + await self.task_manager.update_task(task_id, min_delay=mn, max_delay=mx) + await message.answer(f"✅ Задержка: {format_range(mn, mx)}с") + + elif param == "reading": + if "-" in value: + mn, mx = parse_range(value, default_min=1) + else: + mn = mx = int(value) + await self.task_manager.update_task(task_id, min_reading=mn, max_reading=mx) + await message.answer(f"✅ Чтение: {format_range(mn, mx)}с") + + elif param == "perlink": + await self.task_manager.update_task(task_id, visits_per_link=int(value)) + await message.answer(f"✅ На ссылку: {value}") + + elif param == "minutes": + await self.task_manager.update_task(task_id, monitor_minutes=int(value)) + await message.answer(f"✅ Мониторинг: {value} мин") + + elif param == "domains": + if value.lower() in ["all", "все"]: + await self.task_manager.update_task(task_id, allowed_domains=None) + await message.answer("✅ Домены: все (фильтр отключен)") + else: + domains = [d.strip().lower() for d in value.split(",") if d.strip()] + await self.task_manager.update_task(task_id, allowed_domains=domains) + await message.answer(f"✅ Домены: {', '.join(domains)}") + else: + await message.answer(f"❌ Неизвестный параметр: {param}") + + except ValueError as e: + await message.answer(f"❌ Ошибка: {e}") + + async def _run_twitch_monitor(self, task_id: str, params: TaskParams, message: Message): + """Запуск Twitch мониторинга.""" + from services.irc_service import TwitchIRCClient + + irc = TwitchIRCClient(params.channel, params.target_username) + + async def on_url(url: str, username: str): + params.links_found += 1 + try: + from urllib.parse import urlparse + domain = urlparse(url).netloc + await send_message_safe( + message.bot, message.chat.id, + f"🔗 @{username}: `{url[:60]}`\n🌐 Домен: `{domain}`" + ) + except Exception as e: + logger.warning(f"Failed to send URL with domain: {e}") + await send_message_safe(message.bot, message.chat.id, f"🔗 @{username}: `{url[:60]}`") + + for i in range(params.visits_per_link): + while params.paused: + await asyncio.sleep(1) + + reading = params.get_reading_time() + result = await self.browser_service.visit_page(url, reading) + + params.total_visits += 1 + if result.success: + params.successful_visits += 1 + + if i < params.visits_per_link - 1: + await asyncio.sleep(params.get_delay()) + + try: + await irc.listen_for_messages(on_url, params.monitor_duration, params.allowed_domains) + except asyncio.CancelledError: + raise + finally: + await irc.disconnect() + await self.task_manager.remove_task(task_id) + + async def _run_twitch(self, task_id: str, params: TaskParams, message: Message): + from services.irc_service import TwitchIRCClient + + irc = TwitchIRCClient(params.channel, params.target_username) + + async def on_url(url, username): + """Запускаем обработку ссылки в отдельной задаче.""" + task = asyncio.create_task(self._process_url(url, username, params, message)) + task.add_done_callback( + lambda t: logger.error(f"URL processing failed: {t.exception()}") if not t.done() and t.exception() else None + ) + + def is_active(): + return not params.paused and not params.stopped + + logger.info(f"🚀 IRC monitor: {params.channel}") + + try: + await irc.listen_for_messages(on_url, params.monitor_duration, params.allowed_domains, is_active=is_active) + await self.task_manager.complete_task(task_id) + except asyncio.CancelledError: + raise + finally: + await irc.disconnect() + if self.storage: + await self.storage.save_task(task_id, params) + + + + async def _process_url(self, url: str, username: str, params: TaskParams, message: Message): + """Обрабатывает найденную ссылку (выполняется параллельно).""" + try: + params.links_found += 1 + + from urllib.parse import urlparse + domain = urlparse(url).netloc + + logger.info(f"🔗 @{username}: {url}") + + await send_message_safe( + message.bot, message.chat.id, + f"🔗 @{username}: `{url[:60]}`" + ) + + for i in range(params.visits_per_link): + while params.paused: + await asyncio.sleep(1) + + reading = params.get_reading_time() + + try: + result = await self.browser_service.visit_page_from_twitch( + url=url, + channel=params.channel, + reading_time=reading + ) + params.total_visits += 1 + if result.success: + params.successful_visits += 1 + except Exception as e: + logger.error(f"Visit error: {e}") + + if i < params.visits_per_link - 1: + await asyncio.sleep(params.get_delay()) + + except Exception as e: + logger.error(f"Process URL error: {e}") + + + diff --git a/main.py b/main.py new file mode 100644 index 0000000..fd59598 --- /dev/null +++ b/main.py @@ -0,0 +1,137 @@ +""" +Главный файл бота с сохранением задач. +""" + +import asyncio +import logging +from aiogram import Bot, Dispatcher +from aiogram.types import BotCommand +from aiogram.client.session.aiohttp import AiohttpSession +from config.settings import settings +from core.logger import setup_logger +from managers.proxy_manager import ProxyManager +from managers.background_tasks import BackgroundTaskManager +from services.browser_service import BrowserService +from handlers.commands import BotInterface +from managers.storage import TaskStorage, ChatStorage + +logger = setup_logger(__name__) + + +async def set_bot_commands(bot: Bot): + """Устанавливает команды бота в меню.""" + commands = [ + BotCommand(command="start", description="🏠 Главное меню"), + BotCommand(command="register", description="🔐 Зарегистрироваться"), + BotCommand(command="login", description="🔑 Войти"), + BotCommand(command="logout", description="🔒 Выйти"), + BotCommand(command="streamers", description="📺 Стримеры"), + BotCommand(command="tasks", description="📊 Задачи"), + BotCommand(command="status", description="📈 Статус"), + ] + await bot.set_my_commands(commands) + + +class BotApplication: + + def __init__(self): + self.bot: Bot = None + self.dispatcher: Dispatcher = None + + self.proxy_manager = ProxyManager() + self.background_tasks = BackgroundTaskManager() + self.browser_service = BrowserService(self.proxy_manager) + self.storage = TaskStorage() + self.chat_storage = ChatStorage() + + self.interface = BotInterface( + background_tasks=self.background_tasks, + proxy_manager=self.proxy_manager, + browser_service=self.browser_service, + storage=self.storage, + chat_storage=self.chat_storage, + ) + + async def initialize(self): + """Инициализация бота.""" + logger.info("Initializing bot...") + + if not settings.BOT_TOKEN: + raise ValueError("BOT_TOKEN not set!") + + + self.bot = Bot(token=settings.BOT_TOKEN) + self.dispatcher = Dispatcher() + + self.interface.register(self.dispatcher) + await set_bot_commands(self.bot) + + # Восстанавливаем задачи + await self._restore_tasks() + + logger.info(f"Proxies: {self.proxy_manager.count}") + logger.info("Bot initialized!") + + async def _restore_tasks(self): + """Восстанавливает задачи после перезапуска.""" + saved_tasks = await self.storage.load_tasks() + + if not saved_tasks: + logger.info("No tasks to restore") + return + + restored = 0 + for task_id, params in saved_tasks.items(): + if params.completed or params.stopped: + continue + + await self.interface.task_manager.add_task(task_id, params) + + if params.task_type == "twitch_irc" and params.chat_id: + class FakeMsg: + chat = type('obj', (object,), {'id': params.chat_id}) + bot = self.bot + + await self.background_tasks.start_task( + task_id=task_id, + coro=self.interface._run_twitch(task_id, params, FakeMsg()), + task_type="twitch_irc", + metadata={'type': 'twitch_irc', 'channel': params.channel, 'chat_id': params.chat_id} + ) + restored += 1 + + logger.info(f"🔄 Restored {restored} tasks") + + async def start(self): + """Запуск бота.""" + await self.initialize() # ← ВОТ ЭТО БЫЛО ПРОПУЩЕНО + logger.info("Starting polling...") + await self.dispatcher.start_polling(self.bot) + + async def stop(self): + """Остановка с сохранением.""" + logger.info("Shutting down...") + + # Сохраняем задачи + tasks = self.interface.task_manager._tasks + await self.storage.save_tasks(tasks) + + await self.background_tasks.cancel_all() + await self.browser_service.cleanup() + if self.bot: + await self.bot.session.close() + logger.info("Bot stopped") + + +async def main(): + app = BotApplication() + try: + await app.start() + except KeyboardInterrupt: + logger.info("Interrupted") + finally: + await app.stop() + + +if __name__ == "__main__": + asyncio.run(main()) \ No newline at end of file diff --git a/managers/__init__.py b/managers/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/managers/background_tasks.py b/managers/background_tasks.py new file mode 100644 index 0000000..f2651ec --- /dev/null +++ b/managers/background_tasks.py @@ -0,0 +1,170 @@ +"""Background task manager for async task tracking.""" + +import asyncio +import logging +from dataclasses import dataclass, field +from typing import Dict, Optional, Callable, Coroutine, Any + +from core.constants import TaskStatus + +logger = logging.getLogger(__name__) + + +@dataclass +class TaskInfo: + """Information about a running task.""" + task_id: str + task_type: str + coroutine: Coroutine = None + status: TaskStatus = TaskStatus.RUNNING + created_at: float = field(default_factory=lambda: asyncio.get_running_loop().time()) + metadata: dict = field(default_factory=dict) + + +class BackgroundTaskManager: + """ + Manages background async tasks with tracking and cancellation support. + + Features: + - Start and track async tasks + - Cancel individual or all tasks + - Get task status and metadata + - Prevent duplicate tasks + """ + + def __init__(self): + self._tasks: Dict[str, asyncio.Task] = {} + self._task_info: Dict[str, TaskInfo] = {} + self._lock = asyncio.Lock() + + async def start_task( + self, + task_id: str, + coro: Coroutine, + task_type: str = "unknown", + metadata: dict = None + ) -> bool: + """ + Start a background task. + + Args: + task_id: Unique task identifier + coro: Coroutine to run + task_type: Type of task + metadata: Additional task metadata + + Returns: + True if started successfully, False if already running + """ + async with self._lock: + if task_id in self._tasks and not self._tasks[task_id].done(): + logger.warning(f"Task {task_id} is already running") + return False + + task = asyncio.create_task(coro) + self._tasks[task_id] = task + + self._task_info[task_id] = TaskInfo( + task_id=task_id, + task_type=task_type, + coroutine=coro, + metadata=metadata or {} + ) + + task.add_done_callback(lambda t, tid=task_id: self._on_task_done(tid, t)) + + logger.info(f"Started background task: {task_id} ({task_type})") + return True + + def _on_task_done(self, task_id: str, task: asyncio.Task) -> None: + """Callback когда задача завершилась.""" + try: + exc = task.exception() + if exc: + if isinstance(exc, asyncio.CancelledError): + logger.info(f"Task {task_id} cancelled (normal)") + # Не считаем ошибкой + else: + logger.error(f"Task {task_id} failed: {exc}") + else: + logger.info(f"Task {task_id} completed successfully") + except asyncio.CancelledError: + logger.info(f"Task {task_id} was cancelled") + except Exception as e: + logger.error(f"Error in task callback: {e}") + + async def cancel_task(self, task_id: str) -> bool: + """ + Cancel a specific task. + + Args: + task_id: Task identifier to cancel + + Returns: + True if task was cancelled, False if not found + """ + async with self._lock: + task = self._tasks.get(task_id) + if task and not task.done(): + task.cancel() + try: + await task + except asyncio.CancelledError: + pass + + logger.info(f"Task cancelled: {task_id}") + return True + return False + + async def cancel_all(self) -> int: + """ + Cancel all running tasks. + + Returns: + Number of tasks cancelled + """ + cancelled = 0 + + async with self._lock: + for task_id, task in list(self._tasks.items()): + if not task.done(): + task.cancel() + try: + await task + except asyncio.CancelledError: + pass + cancelled += 1 + + self._tasks.clear() + self._task_info.clear() + + logger.info(f"All tasks cancelled: {cancelled}") + return cancelled + + def get_active_tasks(self) -> Dict[str, dict]: + """ + Get all active tasks with their metadata. + + Returns: + Dictionary of active tasks + """ + active = {} + for task_id, task in self._tasks.items(): + if not task.done(): + info = self._task_info.get(task_id) + active[task_id] = { + "info": info.metadata if info else {}, + "status": TaskStatus.RUNNING.value, + "type": info.task_type if info else "unknown", + "created_at": info.created_at if info else 0 + } + return active + + @property + def active_count(self) -> int: + """Number of currently active tasks.""" + return sum(1 for t in self._tasks.values() if not t.done()) + + def has_active_tasks(self) -> bool: + """Check if there are any active tasks.""" + return self.active_count > 0 \ No newline at end of file diff --git a/managers/proxy_manager.py b/managers/proxy_manager.py new file mode 100644 index 0000000..82e220c --- /dev/null +++ b/managers/proxy_manager.py @@ -0,0 +1,276 @@ +""" +Менеджер прокси с поддержкой SOCKS5 и HTTP. +""" + +import asyncio +import os +from collections import defaultdict +from dataclasses import dataclass, field +from typing import Optional, List, Dict +from enum import Enum +import logging + +from config.settings import settings +from core.exceptions import ProxyTimeoutError, NoProxiesAvailableError + +logger = logging.getLogger(__name__) + + +class ProxyType(str, Enum): + """Типы прокси.""" + SOCKS5 = "socks5" + SOCKS4 = "socks4" + HTTP = "http" + HTTPS = "https" + + +@dataclass +class Proxy: + """Прокси с поддержкой разных типов.""" + ip: str + port: str + proxy_type: ProxyType = ProxyType.SOCKS5 + login: Optional[str] = None + password: Optional[str] = None + + @property + def id(self) -> str: + return f"{self.proxy_type.value}://{self.ip}:{self.port}" + + @property + def server(self) -> str: + """URL сервера для Camoufox/Playwright.""" + protocol = "socks5" if self.proxy_type in [ProxyType.SOCKS5, ProxyType.SOCKS4] else "http" + return f"{protocol}://{self.ip}:{self.port}" + + @property + def proxy_config(self) -> dict: + """Конфигурация для браузера.""" + config = { + 'server': self.server, + } + if self.login and self.password: + config.update({ + 'username': self.login, + 'password': self.password + }) + return config + + @classmethod + def from_line(cls, line: str, default_type: ProxyType = ProxyType.SOCKS5) -> Optional['Proxy']: + """ + Парсит строку прокси. + + Форматы: + - ip:port (SOCKS5 по умолчанию) + - ip:port:login:password + - socks5://ip:port:login:password + - http://ip:port:login:password + """ + line = line.strip() + + # Определяем тип прокси по префиксу + proxy_type = default_type + if "://" in line: + protocol, rest = line.split("://", 1) + protocol = protocol.lower() + + if protocol in ["socks5", "socks"]: + proxy_type = ProxyType.SOCKS5 + elif protocol == "socks4": + proxy_type = ProxyType.SOCKS4 + elif protocol == "http": + proxy_type = ProxyType.HTTP + elif protocol == "https": + proxy_type = ProxyType.HTTPS + + line = rest + + # Парсим части + parts = line.split(":") + + if len(parts) == 2: + return cls(ip=parts[0], port=parts[1], proxy_type=proxy_type) + elif len(parts) == 4: + return cls( + ip=parts[0], + port=parts[1], + login=parts[2], + password=parts[3], + proxy_type=proxy_type + ) + + return None + + +class ProxyManager: + """ + Менеджер прокси с поддержкой SOCKS5 и HTTP. + """ + + def __init__(self, proxy_file: Optional[str] = None): + self.proxy_file = proxy_file or settings.PROXY_FILE + self.cooldown_time = settings.PROXY_COOLDOWN_TIME + self.max_usage = settings.PROXY_MAX_USAGE_BEFORE_COOLDOWN + + self._proxies: List[Proxy] = [] + self._lock = asyncio.Lock() + self._in_use: Dict[str, asyncio.Event] = {} + self._usage_count: Dict[str, int] = defaultdict(int) + self._cooldown_until: Dict[str, float] = {} + self._working_status: Dict[str, bool] = {} + + self._load_proxies() + + def _load_proxies(self) -> None: + """Загружает прокси из файла.""" + if not os.path.exists(self.proxy_file): + logger.warning(f"Proxy file not found: {self.proxy_file}") + return + + try: + with open(self.proxy_file, 'r', encoding='utf-8') as f: + lines = [ + line.strip() + for line in f + if line.strip() and not line.startswith('#') + ] + + loaded = [] + for line in lines: + proxy = Proxy.from_line(line, default_type=ProxyType.SOCKS5) + if proxy: + self._proxies.append(proxy) + self._working_status[proxy.id] = True + loaded.append(proxy) + + logger.info(f"Loaded {len(loaded)} proxies from {self.proxy_file}") + + # Статистика по типам + type_counts = defaultdict(int) + for p in loaded: + type_counts[p.proxy_type.value] += 1 + for ptype, count in type_counts.items(): + logger.info(f" {ptype}: {count}") + + except Exception as e: + logger.error(f"Failed to load proxies: {e}", exc_info=True) + + def _is_available(self, proxy: Proxy) -> bool: + """Проверка доступности прокси.""" + proxy_id = proxy.id + + if proxy_id in self._in_use: + return False + + if proxy_id in self._cooldown_until: + loop = asyncio.get_running_loop() + if loop.time() < self._cooldown_until[proxy_id]: + return False + del self._cooldown_until[proxy_id] + + if self._usage_count[proxy_id] >= self.max_usage: + return False + + return self._working_status.get(proxy_id, False) + + async def acquire_proxy(self, timeout: float = 60) -> Optional[Proxy]: + """Получает доступный прокси.""" + if not self._proxies: + return None + + loop = asyncio.get_running_loop() + start_time = loop.time() + attempt = 0 + + while True: + attempt += 1 + + async with self._lock: + available = [ + p for p in self._proxies + if self._is_available(p) + ] + + if available: + proxy = min(available, key=lambda p: self._usage_count[p.id]) + + self._in_use[proxy.id] = asyncio.Event() + self._usage_count[proxy.id] += 1 + + logger.info( + f"Proxy acquired: {proxy.id} " + f"(type: {proxy.proxy_type.value}, " + f"used: {self._usage_count[proxy.id]}x)" + ) + return proxy + + logger.debug(f"No proxies available (attempt {attempt})") + + elapsed = loop.time() - start_time + if elapsed >= timeout: + logger.warning(f"Proxy acquire timeout ({timeout}s)") + return None + + await asyncio.sleep(1) + + async def release_proxy(self, proxy: Proxy, success: bool = True) -> None: + """Освобождает прокси.""" + if not proxy: + return + + proxy_id = proxy.id + + async with self._lock: + if proxy_id in self._in_use: + del self._in_use[proxy_id] + + if self._usage_count[proxy_id] >= self.max_usage: + loop = asyncio.get_running_loop() + self._cooldown_until[proxy_id] = ( + loop.time() + self.cooldown_time + ) + self._usage_count[proxy_id] = 0 + + if not success: + self._working_status[proxy_id] = False + + logger.debug(f"Proxy released: {proxy_id}") + + def get_stats(self) -> dict: + """Статистика прокси.""" + loop = asyncio.get_running_loop() + now = loop.time() + + return { + "total": len(self._proxies), + "working": sum(1 for s in self._working_status.values() if s), + "in_use": len(self._in_use), + "available": self.available_count, + "in_cooldown": sum(1 for t in self._cooldown_until.values() if now < t), + } + + @property + def count(self) -> int: + return len(self._proxies) + + @property + def available_count(self) -> int: + working = sum(1 for s in self._working_status.values() if s) + in_use = len(self._in_use) + in_cooldown = sum( + 1 for t in self._cooldown_until.values() + if asyncio.get_event_loop().time() < t + ) + return working - in_use - in_cooldown + + def has_proxies(self) -> bool: + return len(self._proxies) > 0 + + def get_socks5_proxies(self) -> List[Proxy]: + """Список SOCKS5 прокси.""" + return [p for p in self._proxies if p.proxy_type == ProxyType.SOCKS5] + + def get_http_proxies(self) -> List[Proxy]: + """Список HTTP прокси.""" + return [p for p in self._proxies if p.proxy_type in [ProxyType.HTTP, ProxyType.HTTPS]] \ No newline at end of file diff --git a/managers/storage.py b/managers/storage.py new file mode 100644 index 0000000..3e0d4c0 --- /dev/null +++ b/managers/storage.py @@ -0,0 +1,177 @@ +""" +Сохранение и загрузка задач между перезапусками. +""" + +import json +import asyncio +import logging +from pathlib import Path +from datetime import datetime +from typing import Dict, Optional + +from managers.task_manager import TaskParams + +logger = logging.getLogger(__name__) + + +class TaskStorage: + """Хранилище задач в JSON файле.""" + + def __init__(self, file_path: str = "data/tasks.json"): + self.file_path = Path(file_path) + self.file_path.parent.mkdir(parents=True, exist_ok=True) + self._lock = asyncio.Lock() + + def _params_to_dict(self, params: TaskParams) -> dict: + """Конвертирует TaskParams в словарь.""" + return { + "url": params.url, + "task_type": params.task_type, + "min_delay": params.min_delay, + "max_delay": params.max_delay, + "min_reading": params.min_reading, + "max_reading": params.max_reading, + "max_visits": params.max_visits, + "current_visit": params.current_visit, + "channel": params.channel, + "target_username": params.target_username, + "visits_per_link": params.visits_per_link, + "monitor_minutes": params.monitor_minutes, + "allowed_domains": params.allowed_domains, + "paused": params.paused, + "completed": params.completed, + "stopped": params.stopped, + "total_visits": params.total_visits, + "successful_visits": params.successful_visits, + "links_found": params.links_found, + "chat_id": params.chat_id, + "started_at": params.started_at.isoformat() if params.started_at else None, + } + + def _dict_to_params(self, data: dict) -> TaskParams: + """Конвертирует словарь в TaskParams.""" + params = TaskParams( + url=data.get("url", ""), + task_type=data.get("task_type", "visit"), + min_delay=data.get("min_delay", 10), + max_delay=data.get("max_delay", 30), + min_reading=data.get("min_reading", 5), + max_reading=data.get("max_reading", 15), + max_visits=data.get("max_visits"), + current_visit=data.get("current_visit", 0), + channel=data.get("channel", ""), + target_username=data.get("target_username", "*"), + visits_per_link=data.get("visits_per_link", 5), + monitor_minutes=data.get("monitor_minutes", 10), + allowed_domains=data.get("allowed_domains"), + paused=data.get("paused", False), + completed=data.get("completed", False), + stopped=data.get("stopped", False), + total_visits=data.get("total_visits", 0), + successful_visits=data.get("successful_visits", 0), + links_found=data.get("links_found", 0), + chat_id=data.get("chat_id"), + ) + + if data.get("started_at"): + try: + params.started_at = datetime.fromisoformat(data["started_at"]) + except (ValueError, TypeError) as e: + logger.warning(f"Failed to parse started_at: {e}") + + return params + + async def save_tasks(self, tasks: Dict[str, TaskParams]) -> None: + """Сохраняет все задачи в файл.""" + async with self._lock: + try: + data = {} + for task_id, params in tasks.items(): + data[task_id] = self._params_to_dict(params) + + with open(self.file_path, 'w', encoding='utf-8') as f: + json.dump(data, f, ensure_ascii=False, indent=2) + + logger.info(f"💾 Saved {len(data)} tasks to {self.file_path}") + except Exception as e: + logger.error(f"Save error: {e}") + + async def load_tasks(self) -> Dict[str, TaskParams]: + """Загружает задачи из файла.""" + if not self.file_path.exists(): + logger.info("No saved tasks found") + return {} + + try: + with open(self.file_path, 'r', encoding='utf-8') as f: + data = json.load(f) + + tasks = {} + for task_id, params_data in data.items(): + tasks[task_id] = self._dict_to_params(params_data) + + logger.info(f"📂 Loaded {len(tasks)} tasks from {self.file_path}") + return tasks + except Exception as e: + logger.error(f"Load error: {e}") + return {} + + async def save_task(self, task_id: str, params: TaskParams) -> None: + """Сохраняет одну задачу (добавляет к существующим).""" + tasks = await self.load_tasks() + tasks[task_id] = params + await self.save_tasks(tasks) + + async def delete_task(self, task_id: str) -> None: + """Удаляет задачу из файла.""" + tasks = await self.load_tasks() + if task_id in tasks: + del tasks[task_id] + await self.save_tasks(tasks) + + +class ChatStorage: + """Хранилище привязки стримеров к чатам.""" + + def __init__(self, file_path: str = "data/chats.json"): + self.file_path = Path(file_path) + self.file_path.parent.mkdir(parents=True, exist_ok=True) + self._lock = asyncio.Lock() + + async def get_chat_streamers(self, chat_id: int) -> list: + """Получает список стримеров для чата.""" + data = await self._load() + return data.get(str(chat_id), []) + + async def add_streamer(self, chat_id: int, channel: str) -> None: + """Добавляет стримера в чат.""" + data = await self._load() + key = str(chat_id) + if key not in data: + data[key] = [] + if channel not in data[key]: + data[key].append(channel) + await self._save(data) + + async def remove_streamer(self, chat_id: int, channel: str) -> None: + """Удаляет стримера из чата.""" + data = await self._load() + key = str(chat_id) + if key in data and channel in data[key]: + data[key].remove(channel) + await self._save(data) + + async def _load(self) -> dict: + if not self.file_path.exists(): + return {} + try: + with open(self.file_path, 'r', encoding='utf-8') as f: + return json.load(f) + except (json.JSONDecodeError, IOError) as e: + logger.warning(f"Failed to load chats: {e}") + return {} + + async def _save(self, data: dict) -> None: + async with self._lock: + with open(self.file_path, 'w', encoding='utf-8') as f: + json.dump(data, f, ensure_ascii=False, indent=2) \ No newline at end of file diff --git a/managers/task_manager.py b/managers/task_manager.py new file mode 100644 index 0000000..65ef786 --- /dev/null +++ b/managers/task_manager.py @@ -0,0 +1,303 @@ +""" +Менеджер задач по стримерам. +Простой список с возможностью изменять параметры на лету. +""" + +import asyncio +from dataclasses import dataclass, field +from typing import Optional, List, Dict +from datetime import datetime +import random +import logging + +logger = logging.getLogger(__name__) + + +@dataclass +class TaskParams: + """Динамические параметры задачи.""" + url: str = "" + task_type: str = "visit" + + # Задержки + min_delay: int = 10 + max_delay: int = 30 + min_reading: int = 5 + max_reading: int = 15 + + # Лимиты + max_visits: Optional[int] = None + current_visit: int = 0 + + # Twitch параметры + channel: str = "" + target_username: str = "" + visits_per_link: int = 5 + monitor_minutes: int = 10 + allowed_domains: Optional[List[str]] = None + + # Состояние + paused: bool = False + skip_next: bool = False + force_delay: Optional[int] = None + force_reading: Optional[int] = None + + # Статистика + started_at: Optional[datetime] = None + total_visits: int = 0 + successful_visits: int = 0 + links_found: int = 0 + chat_id: Optional[int] = None + + completed: bool = False # Задача завершена + stopped: bool = False # Задача остановлена пользователем + + @property + def monitor_duration(self) -> int: + return self.monitor_minutes * 60 + + def update(self, **kwargs): + for key, value in kwargs.items(): + if hasattr(self, key): + setattr(self, key, value) + + def get_delay(self) -> int: + if self.skip_next: + self.skip_next = False + return 0 + if self.force_delay: + delay = self.force_delay + self.force_delay = None + return delay + return random.randint(self.min_delay, self.max_delay) + + def get_reading_time(self) -> int: + if self.force_reading: + reading = self.force_reading + self.force_reading = None + return reading + return random.randint(self.min_reading, self.max_reading) + + def get_status_emoji(self) -> str: + if self.stopped: + return "🛑" + if self.completed: + return "✅" + if self.paused: + return "⏸️" + return "🔄" + + def get_status_text(self) -> str: + if self.stopped: + return "🛑 Остановлена" + if self.completed: + return "✅ Завершена" + if self.paused: + return "⏸️ Пауза" + return "🔄 Активна" + + + def get_stats_text(self) -> str: + """Текст статистики.""" + if self.task_type == "twitch_irc": + status = self.get_status_text() + duration = "" + if self.started_at and (self.completed or self.stopped): + delta = datetime.now() - self.started_at + duration = f"├ ⏱️ Длилась: {delta.seconds // 60} мин\n" + + # Домены + domains_text = "" + if self.allowed_domains: + domains_text = f"├ 🌐 Домены: {', '.join(self.allowed_domains)}\n" + else: + domains_text = "├ 🌐 Домены: все\n" + + return ( + f"📺 **{self.channel}**\n" + f"├ Статус: {status}\n" + f"├ 🔗 Ссылок найдено: {self.links_found}\n" + f"├ 📊 Всего визитов: {self.total_visits}\n" + f"├ ✅ Успешно: {self.successful_visits}\n" + f"├ ⏱️ Задержка: {self.min_delay}-{self.max_delay}с\n" + f"├ 📖 Чтение: {self.min_reading}-{self.max_reading}с\n" + f"├ 🔗 На ссылку: {self.visits_per_link} визитов\n" + f"├ ⏰ Мониторинг: {self.monitor_minutes} мин\n" + f"{domains_text}" + f"{duration}" + ) + else: + return ( + f"🌐 `{self.url[:50]}`\n" + f"├ Статус: {self.get_status_text()}\n" + f"├ 📊 Визитов: {self.total_visits}\n" + f"├ ✅ Успешно: {self.successful_visits}\n" + ) + + + +class TaskManager: + """ + Менеджер задач. + Хранит список стримеров и позволяет управлять каждым. + """ + + def __init__(self): + self._tasks: Dict[str, TaskParams] = {} # task_id -> params + self._lock = asyncio.Lock() + + async def add_task(self, task_id: str, params: TaskParams) -> None: + """Добавляет задачу.""" + async with self._lock: + self._tasks[task_id] = params + logger.info(f"➕ Задача: {task_id[:12]} ({params.channel or params.url})") + + async def remove_task(self, task_id: str) -> bool: + """Удаляет задачу.""" + async with self._lock: + if task_id in self._tasks: + del self._tasks[task_id] + logger.info(f"➖ Задача удалена: {task_id[:12]}") + return True + return False + + async def get_task(self, task_id: str) -> Optional[TaskParams]: + """Получает задачу.""" + async with self._lock: + return self._tasks.get(task_id) + + async def update_task(self, task_id: str, **kwargs) -> bool: + """Обновляет параметры задачи.""" + params = await self.get_task(task_id) + if params: + params.update(**kwargs) + return True + return False + + async def complete_task(self, task_id: str) -> bool: + """Пометить задачу как завершенную.""" + return await self.update_task(task_id, completed=True, paused=True) + + async def stop_task(self, task_id: str) -> bool: + """Пометить задачу как остановленную.""" + return await self.update_task(task_id, stopped=True, paused=True) + + async def get_active_tasks(self) -> Dict[str, TaskParams]: + """Только активные задачи (не завершенные и не остановленные).""" + return { + tid: p for tid, p in self._tasks.items() + if not p.completed and not p.stopped + } + + async def get_completed_tasks(self) -> Dict[str, TaskParams]: + """Завершенные задачи (для просмотра статистики).""" + return { + tid: p for tid, p in self._tasks.items() + if p.completed or p.stopped + } + + + async def pause_task(self, task_id: str) -> bool: + """Пауза задачи.""" + return await self.update_task(task_id, paused=True) + + async def resume_task(self, task_id: str) -> bool: + """Снять с паузы.""" + return await self.update_task(task_id, paused=False) + + async def skip_next_task(self, task_id: str) -> bool: + """Пропустить следующее действие.""" + return await self.update_task(task_id, skip_next=True) + + async def restart_task(self, task_id: str) -> bool: + """Сбросить счетчики для перезапуска.""" + return await self.update_task( + task_id, + current_visit=0, + total_visits=0, + successful_visits=0, + links_found=0 + ) + + async def get_all_tasks(self) -> Dict[str, TaskParams]: + """Все задачи.""" + async with self._lock: + return dict(self._tasks) + + async def get_tasks_by_type(self, task_type: str) -> Dict[str, TaskParams]: + """Задачи по типу.""" + return { + tid: p for tid, p in self._tasks.items() + if p.task_type == task_type + } + + async def get_twitch_tasks(self) -> Dict[str, TaskParams]: + """Только Twitch задачи.""" + return await self.get_tasks_by_type("twitch_irc") + + async def get_visit_tasks(self) -> Dict[str, TaskParams]: + """Только задачи посещения.""" + return await self.get_tasks_by_type("visit") + + async def get_stats(self) -> dict: + """Общая статистика.""" + twitch_tasks = await self.get_twitch_tasks() + visit_tasks = await self.get_visit_tasks() + + return { + "total": len(self._tasks), + "twitch": len(twitch_tasks), + "visit": len(visit_tasks), + "paused": sum(1 for p in self._tasks.values() if p.paused), + "active": sum(1 for p in self._tasks.values() if not p.paused), + } + + def _get_task_info(self, task_id: str, params: TaskParams) -> str: + """Информация о задаче для отображения.""" + status = params.get_status_emoji() + + if params.task_type == "twitch_irc": + return ( + f"{status} 📺 **{params.channel}**\n" + f" `{task_id[:12]}...`\n" + f" 👤 @{params.target_username} | " + f"🔗{params.links_found} | " + f"📊{params.total_visits}\n" + ) + else: + return ( + f"{status} 🌐 `{params.url[:40]}`\n" + f" `{task_id[:12]}...`\n" + f" 📊{params.total_visits} визитов\n" + ) + + async def format_task_list(self, max_items: int = 15) -> str: + """Форматирует список задач для сообщения.""" + async with self._lock: + if not self._tasks: + return "📊 Нет активных задач" + + # Сначала Twitch, потом визиты + twitch = await self.get_twitch_tasks() + visits = await self.get_visit_tasks() + + async with self._lock: + total_tasks = len(self._tasks) + text = f"📊 **ЗАДАЧИ: {total_tasks}**\n\n" + + if twitch: + text += "📺 **Twitch мониторинг:**\n" + for i, (task_id, params) in enumerate(list(twitch.items())[:5]): + text += self._get_task_info(task_id, params) + if len(twitch) > 5: + text += f" _...и еще {len(twitch) - 5}_\n" + text += "\n" + + if visits: + text += "🌐 **Посещение страниц:**\n" + for i, (task_id, params) in enumerate(list(visits.items())[:5]): + text += self._get_task_info(task_id, params) + if len(visits) > 5: + text += f" _...и еще {len(visits) - 5}_\n" + + return text \ No newline at end of file diff --git a/proxies.txt b/proxies.txt new file mode 100644 index 0000000..dd73bc6 --- /dev/null +++ b/proxies.txt @@ -0,0 +1,3 @@ +95.181.163.3:40351:E42uKM8b1N:ksRKwOeFqp +78.153.143.156:54392:8rcwbesBmR:dsiXZMHR5u +185.200.170.173:56648:0lehqA5awt:JxgMrHf5Xl \ No newline at end of file diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..45fe4c5 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,10 @@ +fastapi>=0.115.0 +uvicorn>=0.34.0 +aiogram>=3.17.0 +pydantic>=2.10.0 +pydantic-settings>=2.7.0 +python-dotenv>=1.0.1 +aiofiles>=24.1.0 +playwright-captcha>=0.1.0 +playwright>=1.49.0 +camoufox[geoip]>=0.4.0 diff --git a/screenshots/20260510_164209_uplify.pro.png b/screenshots/20260510_164209_uplify.pro.png new file mode 100644 index 0000000..43903b8 Binary files /dev/null and b/screenshots/20260510_164209_uplify.pro.png differ diff --git a/screenshots/20260510_164346_uplify.pro.png b/screenshots/20260510_164346_uplify.pro.png new file mode 100644 index 0000000..ebd7d14 Binary files /dev/null and b/screenshots/20260510_164346_uplify.pro.png differ diff --git a/screenshots/20260510_172138_httpbin.org.png b/screenshots/20260510_172138_httpbin.org.png new file mode 100644 index 0000000..b78ab3f Binary files /dev/null and b/screenshots/20260510_172138_httpbin.org.png differ diff --git a/screenshots/20260511_161652_uplify.pro.png b/screenshots/20260511_161652_uplify.pro.png new file mode 100644 index 0000000..c2fd669 Binary files /dev/null and b/screenshots/20260511_161652_uplify.pro.png differ diff --git a/services/__init__.py b/services/__init__.py new file mode 100644 index 0000000..d987660 --- /dev/null +++ b/services/__init__.py @@ -0,0 +1,2 @@ +from services.browser_service import BrowserService, VisitResult +from services.socks5_to_http_proxy import Socks5ToHttpProxy, Socks5ProxyPool diff --git a/services/browser_service.py b/services/browser_service.py new file mode 100644 index 0000000..c36953f --- /dev/null +++ b/services/browser_service.py @@ -0,0 +1,450 @@ +""" +Browser service с поддержкой SOCKS5 через локальный HTTP туннель. +""" + +import asyncio +import random +import logging +from pathlib import Path +from datetime import datetime +from typing import Optional +from dataclasses import dataclass + +from camoufox import DefaultAddons +from camoufox.async_api import AsyncCamoufox + +from config.settings import settings +from managers.proxy_manager import ProxyManager, Proxy, ProxyType +from services.socks5_to_http_proxy import Socks5ProxyPool + +logger = logging.getLogger(__name__) + + +@dataclass +class VisitResult: + """Результат посещения страницы.""" + url: str + initial_url: Optional[str] = None + final_url: Optional[str] = None + redirect_occurred: bool = False + proxy_info: str = "direct" + reading_time: int = 0 + screenshot_path: Optional[str] = None + success: bool = False + error: Optional[str] = None + + +class BrowserService: + """ + Сервис для посещения страниц через Camoufox. + Поддерживает HTTP, SOCKS5 (через туннель) и прямое соединение. + """ + + def __init__(self, proxy_manager: ProxyManager): + self.proxy_manager = proxy_manager + self.socks5_pool = Socks5ProxyPool(idle_timeout=300) + self.screenshots_dir = Path(settings.SCREENSHOTS_DIR) + self.screenshots_dir.mkdir(exist_ok=True) + + async def visit_page( + self, + url: str, + reading_time: Optional[int] = None + ) -> VisitResult: + """Посещает страницу.""" + if reading_time is None: + reading_time = random.randint( + settings.DEFAULT_MIN_READING, + settings.DEFAULT_MAX_READING + ) + + proxy = None + use_socks5 = False + + try: + # Получаем прокси + if self.proxy_manager.has_proxies(): + proxy = await self.proxy_manager.acquire_proxy(timeout=30) + + # Определяем конфигурацию + if proxy and proxy.proxy_type in [ProxyType.SOCKS5, ProxyType.SOCKS4]: + # SOCKS5 через локальный HTTP туннель + proxy_config = await self.socks5_pool.get_proxy_config(proxy) + use_socks5 = True + logger.info(f"Using SOCKS5 tunnel for: {proxy.id}") + elif proxy: + # HTTP прокси напрямую + proxy_config = proxy.proxy_config + logger.info(f"Using HTTP proxy: {proxy.id}") + else: + # Прямое соединение + proxy_config = None + logger.info("Direct connection") + + # Запускаем браузер + if proxy_config: + async with AsyncCamoufox( + headless=False, + geoip=True, + humanize=True, + exclude_addons=[DefaultAddons.UBO], + proxy=proxy_config + ) as browser: + result = await self._browse_page( + browser, url, + proxy.id if proxy else "direct", + reading_time + ) + else: + async with AsyncCamoufox( + headless=False, + geoip=False, + humanize=True, + locale="ru-RU", + exclude_addons=[DefaultAddons.UBO] + ) as browser: + result = await self._browse_page( + browser, url, "direct", reading_time + ) + + # Освобождаем + if proxy: + await self.proxy_manager.release_proxy(proxy, result.success) + if use_socks5: + await self.socks5_pool.release(proxy) + + return result + + except Exception as e: + logger.error(f"Visit error: {e}", exc_info=True) + + if proxy: + await self.proxy_manager.release_proxy(proxy, success=False) + if use_socks5: + await self.socks5_pool.release(proxy) + + return VisitResult(url=url, success=False, error=str(e)) + + async def _browse_page( + self, browser, url: str, proxy_info: str, reading_time: int + ) -> VisitResult: + """Выполняет просмотр страницы.""" + page = await browser.new_page() + + try: + await page.set_viewport_size({ + "width": settings.VIEWPORT_WIDTH, + "height": settings.VIEWPORT_HEIGHT + }) + + goto_task = asyncio.create_task( + page.goto(url, wait_until="commit", timeout=30000) + ) + await self._move_mouse(page, goto_task) + await goto_task + + initial_url = page.url + final_url, redirect = await self._wait_redirect(page, initial_url) + await self._simulate_reading(page, reading_time) + + screenshot_path = None + if settings.SCREENSHOTS_DIR: + screenshot_path = await self._take_screenshot(page, final_url) + + return VisitResult( + url=url, + initial_url=initial_url, + final_url=final_url, + redirect_occurred=redirect, + proxy_info=proxy_info, + reading_time=reading_time, + screenshot_path=screenshot_path, + success=True + ) + except Exception as e: + return VisitResult(url=url, reading_time=reading_time, + success=False, error=str(e)) + finally: + if page.url == url: + logger.error() + await page.close() + + async def _move_mouse(self, page, task): + """Двигает мышь.""" + w, h = settings.VIEWPORT_WIDTH, settings.VIEWPORT_HEIGHT + try: + while not task.done(): + await page.mouse.move( + random.randint(100, w-100), + random.randint(100, h-100) + ) + await asyncio.sleep(random.uniform(0.1, 0.3)) + except: + pass + + async def _wait_redirect(self, page, initial_url): + """Ждет редирект.""" + loop = asyncio.get_running_loop() + start = loop.time() + redirect = False + current = initial_url + + while loop.time() - start < settings.MAX_REDIRECT_WAIT: + await asyncio.sleep(2) + new_url = page.url + + if new_url != current: + current = new_url + redirect = True + task = asyncio.create_task( + page.wait_for_load_state("load", timeout=30000) + ) + await self._move_mouse(page, task) + await task + break + + await self._random_action(page) + + return page.url, redirect + + async def _simulate_reading(self, page, duration): + """Симулирует чтение.""" + loop = asyncio.get_running_loop() + start = loop.time() + while loop.time() - start < duration: + await self._random_action(page) + + async def _random_action(self, page): + """Случайное действие.""" + try: + action = random.choice(['move', 'scroll', 'pause']) + if action == 'move': + x = random.randint(100, settings.VIEWPORT_WIDTH-100) + y = random.randint(100, settings.VIEWPORT_HEIGHT-100) + await page.mouse.move(x, y) + await asyncio.sleep(random.uniform(0.3, 1.0)) + elif action == 'scroll': + await page.evaluate(f"window.scrollBy(0, {random.randint(100, 300)})") + await asyncio.sleep(random.uniform(0.3, 0.8)) + else: + await asyncio.sleep(random.uniform(1, 3)) + except: + await asyncio.sleep(0.5) + + async def _take_screenshot(self, page, url): + """Скриншот.""" + try: + ts = datetime.now().strftime("%Y%m%d_%H%M%S") + domain = url.replace("https://", "").replace("http://", "").split("/")[0][:30] + fp = self.screenshots_dir / f"{ts}_{domain}.png" + await page.screenshot(path=str(fp), full_page=True) + return str(fp) + except Exception as e: + logger.error(f"Screenshot: {e}") + return None + + async def cleanup(self): + """Очистка с таймаутом.""" + try: + await asyncio.wait_for(self.socks5_pool.stop_all(), timeout=10) + except asyncio.TimeoutError: + logger.warning("SOCKS5 pool cleanup timeout") + except Exception as e: + logger.error(f"Cleanup error: {e}") + + async def visit_page_from_twitch(self, url: str, channel: str, reading_time: int = None) -> VisitResult: + if reading_time is None: + reading_time = random.randint(settings.DEFAULT_MIN_READING, settings.DEFAULT_MAX_READING) + + proxy = None + + for attempt in range(2): # Две попытки с разными прокси + try: + if self.proxy_manager.has_proxies(): + proxy = await self.proxy_manager.acquire_proxy(timeout=90) + + if proxy and proxy.proxy_type in [ProxyType.SOCKS5, ProxyType.SOCKS4]: + proxy_config = await self.socks5_pool.get_proxy_config(proxy) + elif proxy: + proxy_config = proxy.proxy_config + else: + proxy_config = None + + browser_kwargs = { + "headless": False, + "geoip": True, + "humanize": True, + "exclude_addons": [DefaultAddons.UBO], + } + + if proxy_config: + browser_kwargs["proxy"] = proxy_config + + async with AsyncCamoufox(**browser_kwargs) as browser: + result = await self._browse_page( + browser, + url, + proxy.id if proxy else "direct", + reading_time + ) + + if proxy: + await self.proxy_manager.release_proxy(proxy, result.success) + + if result.success: + return result + + except Exception as e: + logger.warning(f"Attempt {attempt + 1} failed: {e}") + if proxy: + await self.proxy_manager.release_proxy(proxy, success=False) + await self.socks5_pool.release(proxy) + proxy = None + + if attempt < 1: + await asyncio.sleep(2) + + return VisitResult(url=url, success=False, error="All attempts failed") + + async def _browse_twitch_ref(self, browser, url: str, channel: str, proxy_info: str, reading_time: int) -> VisitResult: + """Посещает страницу как переход с Twitch канала.""" + page = await browser.new_page() + twitch_url = f"https://www.twitch.tv/{channel}" + + try: + await page.set_viewport_size({"width": settings.VIEWPORT_WIDTH, "height": settings.VIEWPORT_HEIGHT}) + + # Заголовки имитирующие переход с Twitch + await page.set_extra_http_headers({ + "Referer": twitch_url, + "Origin": "https://www.twitch.tv", + }) + + logger.info(f"📺 From Twitch: {channel} → {url}") + + # Переход с Referer + goto = asyncio.create_task(page.goto(url, referer=twitch_url, wait_until="commit", timeout=30000)) + await self._move_mouse(page, goto) + await goto + + initial_url = page.url + final_url, redirect = await self._wait_redirect(page, initial_url) + await self._simulate_reading(page, reading_time) + + return VisitResult( + url=url, initial_url=initial_url, final_url=final_url, + redirect_occurred=redirect, proxy_info=proxy_info, + reading_time=reading_time, success=True + ) + except Exception as e: + return VisitResult(url=url, success=False, error=str(e)) + finally: + await page.close() + + async def _browse_from_twitch( + self, + browser, + url: str, + channel: str, + proxy_info: str, + reading_time: int + ) -> VisitResult: + """ + Эмулирует переход с Twitch: + 1. Открывает страницу стримера + 2. Прокручивает чат + 3. Переходит по ссылке + """ + page = await browser.new_page() + twitch_url = f"https://www.twitch.tv/{channel}" + + try: + await page.set_viewport_size({ + "width": settings.VIEWPORT_WIDTH, + "height": settings.VIEWPORT_HEIGHT + }) + + # === Шаг 1: Заходим на Twitch === + logger.info(f"📺 Opening Twitch: {twitch_url}") + + goto_task = asyncio.create_task( + page.goto(twitch_url, wait_until="commit", timeout=30000) + ) + await self._move_mouse(page, goto_task) + await goto_task + + # Ждем загрузки + await asyncio.sleep(random.uniform(2, 4)) + + # === Шаг 2: Имитируем просмотр стрима === + logger.info("👀 Watching stream...") + + # Прокручиваем страницу как будто смотрим + for _ in range(random.randint(2, 4)): + scroll = random.randint(200, 500) + await page.evaluate(f"window.scrollBy(0, {scroll})") + await asyncio.sleep(random.uniform(0.5, 1.5)) + + # Двигаем мышь (как будто читаем чат) + w, h = settings.VIEWPORT_WIDTH, settings.VIEWPORT_HEIGHT + for _ in range(random.randint(3, 6)): + x = random.randint(100, w - 100) + y = random.randint(100, h - 100) + await page.mouse.move(x, y) + await asyncio.sleep(random.uniform(0.3, 0.8)) + + # === Шаг 3: Переходим по ссылке === + logger.info(f"🔗 Clicking link: {url}") + + # Создаем новую вкладку для перехода (как target="_blank") + # Или просто переходим с Referer + await page.evaluate(f""" + window.open('{url}', '_blank'); + """) + + # Ждем открытия новой вкладки + await asyncio.sleep(2) + + # Получаем новую вкладку + pages = await browser.pages() + if len(pages) > 1: + new_page = pages[-1] + await new_page.bring_to_front() + else: + # Если вкладка не открылась - переходим в этой же + new_page = page + await new_page.goto(url, referer=twitch_url, wait_until="commit") + + initial_url = new_page.url + logger.info(f"📍 Landed: {initial_url}") + + # === Шаг 4: Ждем редирект === + final_url, redirect = await self._wait_redirect(new_page, initial_url) + + # === Шаг 5: Читаем страницу === + await self._simulate_reading(new_page, reading_time) + + # Закрываем новую вкладку если она отдельная + if new_page != page: + await new_page.close() + + # Возвращаемся на Twitch и закрываем + await page.close() + + return VisitResult( + url=url, + initial_url=initial_url, + final_url=final_url, + redirect_occurred=redirect, + proxy_info=proxy_info, + reading_time=reading_time, + success=True + ) + + except Exception as e: + logger.error(f"Browse from Twitch error: {e}") + try: + await page.close() + except: + pass + return VisitResult(url=url, reading_time=reading_time, success=False, error=str(e)) \ No newline at end of file diff --git a/services/irc_service.py b/services/irc_service.py new file mode 100644 index 0000000..00b5f0b --- /dev/null +++ b/services/irc_service.py @@ -0,0 +1,264 @@ +""" +IRC сервис для мониторинга Twitch чата. +""" + +import asyncio +import re +import logging +from typing import Optional, Callable, Awaitable, List +from datetime import datetime +from urllib.parse import urlparse + +from config.settings import settings + +logger = logging.getLogger(__name__) + + +class TwitchIRCClient: + """ + Twitch IRC клиент для мониторинга чата. + """ + + def __init__( + self, + channel: str, + target_username: str, + irc_username: str = None, + irc_oauth: str = None + ): + self.channel = channel.lower() + self.target_username = target_username.lower() + self.irc_username = irc_username or settings.IRC_USERNAME + self.irc_oauth = irc_oauth or settings.IRC_OAUTH + + self._reader: Optional[asyncio.StreamReader] = None + self._writer: Optional[asyncio.StreamWriter] = None + self._connected = False + self._url_pattern = re.compile(r'https?://[^\s<>"]+') + + async def connect(self) -> bool: + """Подключение к IRC серверу.""" + try: + import ssl + ssl_context = ssl.create_default_context() + self._reader, self._writer = await asyncio.open_connection( + settings.IRC_SERVER, settings.IRC_PORT, + ssl=ssl_context + ) + + await self._send(f"PASS {self.irc_oauth}") + await self._send(f"NICK {self.irc_username}") + await self._send("CAP REQ :twitch.tv/tags") + await self._send("CAP REQ :twitch.tv/commands") + await self._send(f"JOIN #{self.channel}") + + await asyncio.sleep(2) + self._connected = True + logger.info(f"✅ Подключен к #{self.channel}") + return True + + except Exception as e: + logger.error(f"❌ Ошибка подключения: {e}") + return False + + async def _send(self, message: str) -> None: + """Отправка сообщения.""" + if self._writer: + self._writer.write(f"{message}\r\n".encode()) + await self._writer.drain() + + async def _read_line(self) -> Optional[str]: + """Чтение строки с обработкой PING.""" + if not self._reader: + return None + + try: + line = await self._reader.readline() + + if not line: + logger.warning("Empty line - connection closed") + self._connected = False + return None + + decoded = line.decode('utf-8', errors='ignore').strip() + + # PING - отвечаем СРАЗУ + if decoded.startswith('PING'): + pong_msg = f"PONG {decoded.split()[1]}" + self._writer.write(f"{pong_msg}\r\n".encode()) + try: + await self._writer.drain() + except (ConnectionResetError, BrokenPipeError, OSError) as e: + logger.warning(f"Failed to send PONG: {e}") + logger.debug(f"🏓 {pong_msg}") + return f"PONG {decoded.split()[1]}" # Возвращаем PONG для отслеживания + + return decoded + + except Exception as e: + logger.error(f"Read error: {e}") + self._connected = False + return None + + + def _parse_message(self, line: str) -> Optional[dict]: + if not line or 'PRIVMSG' not in line: + return None + + try: + # Парсим display-name из тегов + tags = {} + if line.startswith('@'): + tags_part = line.split(' ', 1)[0] + for tag in tags_part.lstrip('@').split(';'): + if '=' in tag: + k, v = tag.split('=', 1) + tags[k] = v + + display_name = tags.get('display-name', '') + + # Извлекаем сообщение + if ' :' in line: + message_text = line.split(' :', 1)[1] + else: + message_text = '' + + # Пропускаем пустые сообщения + if not message_text.strip(): + return None + + return { + 'display_name': display_name or 'unknown', + 'message': message_text, + 'tags': tags + } + except Exception as e: + logger.error(f"Parse error: {e}") + return None + + + def _extract_urls(self, text: str) -> List[str]: + """Извлечение URL из текста.""" + urls = [] + for raw_url in self._url_pattern.findall(text): + try: + parsed = urlparse(raw_url) + clean = f"{parsed.scheme}://{parsed.netloc}{parsed.path}" + if parsed.query: + clean += f"?{parsed.query}" + urls.append(clean) + except Exception: + urls.append(raw_url.strip()) + return urls + + def _is_domain_allowed(self, url: str, allowed_domains: List[str]) -> bool: + """Проверка домена.""" + if not allowed_domains: + return True + try: + domain = urlparse(url).netloc.lower() + return any(d.lower() in domain for d in allowed_domains) + except Exception: + return False + + async def listen_for_messages( + self, + on_url_found: Callable[[str, str], Awaitable[None]], + duration: int, + allowed_domains: Optional[List[str]] = None, + is_active: Callable[[], bool] = None, # Функция проверки активности +) -> dict: + stats = {"links_found": 0, "messages": 0, "reconnects": 0} + + start_time = datetime.now() + last_message_time = datetime.now() + + await self.connect() + + while (datetime.now() - start_time).seconds < duration: + # === ПРОВЕРКА АКТИВНОСТИ === + if is_active and not is_active(): + logger.info("⏸️ Task paused/stopped, disconnecting...") + break + # === ПЕРЕПОДКЛЮЧЕНИЕ КАЖДЫЕ 30 СЕКУНД === + if (datetime.now() - last_message_time).seconds > 30: + logger.info("🔄 Reconnecting (30s)...") + + # Сначала читаем всё что осталось + try: + while True: + line = await asyncio.wait_for(self._reader.readline(), timeout=1.0) + if not line: + break + decoded = line.decode('utf-8', errors='ignore').strip() + if 'PRIVMSG' in decoded: + msg = self._parse_message(decoded) + if msg: + stats["messages"] += 1 + for url in self._extract_urls(msg['message']): + if self._is_domain_allowed(url, allowed_domains): + stats["links_found"] += 1 + try: + await on_url_found(url, msg['display_name']) + except Exception as e: + logger.warning(f"URL callback error: {e}") + except Exception as e: + logger.warning(f"Error draining IRC data: {e}") + + # Переподключаемся + try: + await self.disconnect() + except Exception as e: + logger.warning(f"Disconnect error: {e}") + await asyncio.sleep(1) + await self.connect() + + stats["reconnects"] += 1 + last_message_time = datetime.now() + continue + + # Читаем строку + try: + line = await asyncio.wait_for(self._read_line(), timeout=5.0) + except (asyncio.TimeoutError, asyncio.CancelledError): + continue + except Exception as e: + logger.warning(f"IRC read error: {e}") + continue + + if not line: + continue + + if line.startswith('PONG'): + continue + + msg = self._parse_message(line) + if not msg: + continue + + last_message_time = datetime.now() # Обновляем таймер + stats["messages"] += 1 + logger.debug(f"IRC message from {msg['display_name']}: {msg['message'][:50]}...") + + for url in self._extract_urls(msg['message']): + if self._is_domain_allowed(url, allowed_domains): + stats["links_found"] += 1 + try: + await on_url_found(url, msg['display_name']) + except: + pass + + return stats + + + async def disconnect(self) -> None: + """Закрытие соединения.""" + self._connected = False + if self._writer: + try: + self._writer.close() + await self._writer.wait_closed() + except (ConnectionResetError, BrokenPipeError, OSError) as e: + logger.debug(f"Disconnect warning: {e}") + self._writer = None + self._reader = None \ No newline at end of file diff --git a/services/socks5_to_http_proxy.py b/services/socks5_to_http_proxy.py new file mode 100644 index 0000000..f0a2270 --- /dev/null +++ b/services/socks5_to_http_proxy.py @@ -0,0 +1,381 @@ +""" +SOCKS5 → HTTP/HTTPS прокси туннель с поддержкой CONNECT метода. +""" + +import asyncio +import socket +import logging +from typing import Optional, Dict +import traceback + +from aiohttp import ClientSession, ClientTimeout +from aiohttp_socks import ProxyConnector + +logger = logging.getLogger(__name__) + + +class Socks5ToHttpProxy: + """ + Локальный HTTP/HTTPS → SOCKS5 прокси с поддержкой CONNECT. + """ + + def __init__( + self, + socks5_host: str, + socks5_port: int, + username: Optional[str] = None, + password: Optional[str] = None + ): + self.socks5_host = socks5_host + self.socks5_port = int(socks5_port) + self.username = username + self.password = password + + self._session: Optional[ClientSession] = None + self._server = None + self._port: Optional[int] = None + self._active_connections = 0 + self._lock = asyncio.Lock() + + @property + def local_url(self) -> str: + return f"http://127.0.0.1:{self._port}" + + @property + def proxy_config_for_browser(self) -> dict: + return {'server': self.local_url} + + async def start(self) -> str: + """Запускает прокси сервер.""" + self._port = self._find_free_port() + + socks_url = self._build_socks5_url() + connector = ProxyConnector.from_url(socks_url) + timeout = ClientTimeout(total=30, connect=15) + self._session = ClientSession(connector=connector, timeout=timeout) + + self._server = await asyncio.start_server( + self._handle_client, + '127.0.0.1', + self._port + ) + + logger.info(f"🟢 SOCKS5: {self.local_url}") + return self.local_url + + async def stop(self): + """Останавливает прокси с таймаутом.""" + logger.debug(f"Stopping proxy 127.0.0.1:{self._port}") + + # Закрываем сервер (не принимать новые соединения) + if self._server: + self._server.close() + + # Ждем завершения активных соединений с таймаутом + try: + async with asyncio.timeout(5): # Максимум 5 секунд на завершение + while self._active_connections > 0: + await asyncio.sleep(0.1) + except asyncio.TimeoutError: + logger.warning(f"Force stopping proxy with {self._active_connections} active connections") + + # Закрываем сервер принудительно + if self._server: + try: + await asyncio.wait_for(self._server.wait_closed(), timeout=3) + except asyncio.TimeoutError: + logger.warning("Server close timeout") + self._server = None + + # Закрываем сессию + if self._session: + try: + await asyncio.wait_for(self._session.close(), timeout=3) + except asyncio.TimeoutError: + pass + self._session = None + + logger.debug(f"Proxy stopped: 127.0.0.1:{self._port}") + + def _find_free_port(self) -> int: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + s.bind(('127.0.0.1', 0)) + return s.getsockname()[1] + + def _build_socks5_url(self) -> str: + if self.username and self.password: + return f"socks5://{self.username}:{self.password}@{self.socks5_host}:{self.socks5_port}" + return f"socks5://{self.socks5_host}:{self.socks5_port}" + + async def _handle_client( + self, + reader: asyncio.StreamReader, + writer: asyncio.StreamWriter + ): + """Обрабатывает входящее подключение.""" + async with self._lock: + self._active_connections += 1 + + try: + # Таймаут на чтение первого запроса + request_line = await asyncio.wait_for( + reader.readline(), + timeout=10 + ) + + if not request_line: + return + + request_text = request_line.decode('utf-8', errors='ignore').strip() + + if request_text.upper().startswith('CONNECT'): + await self._handle_connect(request_text, reader, writer) + else: + await self._handle_http(request_line, reader, writer) + + except asyncio.TimeoutError: + logger.debug("Client timeout") + except ConnectionResetError: + logger.debug("Connection reset by client") + except Exception as e: + if "Server disconnected" not in str(e): + logger.error(f"Client error: {e}") + finally: + async with self._lock: + self._active_connections -= 1 + + try: + writer.close() + except Exception: + pass + + async def _handle_connect( + self, + request_line: str, + reader: asyncio.StreamReader, + writer: asyncio.StreamWriter + ): + """Обрабатывает CONNECT запрос.""" + import socks + remote_socket = None + + try: + parts = request_line.split() + if len(parts) < 2: + writer.write(b'HTTP/1.1 400 Bad Request\r\n\r\n') + return + + target = parts[1] + if ':' in target: + host, port = target.rsplit(':', 1) + port = int(port) + else: + host = target + port = 443 + + # Пропускаем заголовки (с таймаутом) + while True: + line = await asyncio.wait_for(reader.readline(), timeout=5) + if not line or line in [b'\r\n', b'\n']: + break + + # Создаем SOCKS5 соединение + remote_socket = socks.socksocket() + remote_socket.settimeout(15) + + if self.username: + remote_socket.set_proxy( + socks.SOCKS5, self.socks5_host, self.socks5_port, + username=self.username, password=self.password + ) + else: + remote_socket.set_proxy( + socks.SOCKS5, self.socks5_host, self.socks5_port + ) + + # Подключаемся (в отдельном потоке чтобы не блокировать) + loop = asyncio.get_running_loop() + await loop.run_in_executor(None, remote_socket.connect, (host, port)) + + # Отвечаем клиенту + writer.write(b'HTTP/1.1 200 Connection Established\r\n\r\n') + await writer.drain() + + # Пересылка данных с таймаутом + await asyncio.wait_for( + self._relay(reader, writer, remote_socket), + timeout=60 + ) + + except asyncio.TimeoutError: + logger.debug("CONNECT timeout") + except Exception as e: + logger.debug(f"CONNECT error: {e}") + finally: + if remote_socket: + try: + remote_socket.close() + except Exception: + pass + + async def _relay( + self, + client_reader: asyncio.StreamReader, + client_writer: asyncio.StreamWriter, + remote_socket + ): + """Двунаправленная пересылка данных.""" + loop = asyncio.get_running_loop() + running = True + + async def client_to_remote(): + try: + while running: + data = await asyncio.wait_for(client_reader.read(8192), timeout=30) + if not data: + break + await loop.sock_sendall(remote_socket, data) + except Exception: + pass + + async def remote_to_client(): + try: + while running: + data = await loop.sock_recv(remote_socket, 8192) + if not data: + break + client_writer.write(data) + await client_writer.drain() + except Exception: + pass + + try: + await asyncio.gather(client_to_remote(), remote_to_client()) + except Exception: + pass + finally: + running = False + + async def _handle_http( + self, + request_line: bytes, + reader: asyncio.StreamReader, + writer: asyncio.StreamWriter + ): + """Обрабатывает HTTP запрос.""" + try: + first_line = request_line.decode('utf-8', errors='ignore').strip() + parts = first_line.split() + + if len(parts) < 2: + return + + method = parts[0] + path = parts[1] + + # Читаем заголовки + headers = {} + while True: + line = await asyncio.wait_for(reader.readline(), timeout=5) + if not line or line in [b'\r\n', b'\n']: + break + + decoded = line.decode('utf-8', errors='ignore').strip() + if ':' in decoded: + key, value = decoded.split(':', 1) + headers[key.strip()] = value.strip() + + # Тело запроса + body = b'' + content_length = int(headers.get('Content-Length', 0)) + if content_length > 0: + body = await reader.readexactly(content_length) + + host = headers.get('Host', 'localhost') + url = f"http://{host}{path}" + + headers.pop('Proxy-Connection', None) + + # Проксируем + async with self._session.request( + method=method, url=url, + headers=headers, data=body if body else None, + allow_redirects=True + ) as response: + + response_body = await response.read() + + status_line = f"HTTP/1.1 {response.status} {response.reason}\r\n" + response_text = status_line + + for key, value in response.headers.items(): + if key.lower() != 'transfer-encoding': + response_text += f"{key}: {value}\r\n" + response_text += f"Content-Length: {len(response_body)}\r\n\r\n" + + writer.write(response_text.encode() + response_body) + await writer.drain() + + except asyncio.TimeoutError: + pass + except Exception as e: + if "Server disconnected" not in str(e): + logger.debug(f"HTTP error: {e}") + +class Socks5ProxyPool: + """Пул SOCKS5→HTTP прокси.""" + + def __init__(self, idle_timeout: int = 300): + self._proxies: Dict[str, Socks5ToHttpProxy] = {} + self._last_used: Dict[str, float] = {} + self._lock = asyncio.Lock() + self._idle_timeout = idle_timeout + self._cleanup_task: Optional[asyncio.Task] = None + + async def get_proxy_config(self, socks5_proxy) -> dict: + """Получает или создает конфигурацию HTTP прокси для SOCKS5.""" + key = socks5_proxy.id + + async with self._lock: + if key not in self._proxies: + proxy = Socks5ToHttpProxy( + socks5_host=socks5_proxy.ip, + socks5_port=int(socks5_proxy.port), + username=socks5_proxy.login, + password=socks5_proxy.password + ) + await proxy.start() + self._proxies[key] = proxy + + loop = asyncio.get_running_loop() + self._last_used[key] = loop.time() + return self._proxies[key].proxy_config_for_browser + + async def release(self, socks5_proxy): + """Помечает прокси как неиспользуемый.""" + key = socks5_proxy.id + async with self._lock: + if key in self._last_used: + self._last_used[key] = 0 + + async def stop_all(self): + """Останавливает все прокси с таймаутом.""" + if self._cleanup_task: + self._cleanup_task.cancel() + + async with self._lock: + proxies = list(self._proxies.values()) + self._proxies.clear() + self._last_used.clear() + + for proxy in proxies: + try: + await asyncio.wait_for(proxy.stop(), timeout=3) + except (asyncio.TimeoutError, Exception): + pass + + logger.info("All SOCKS5 proxies stopped") + + @property + def active_count(self) -> int: + return len(self._proxies) \ No newline at end of file diff --git a/services/visit_service.py b/services/visit_service.py new file mode 100644 index 0000000..50dfea3 --- /dev/null +++ b/services/visit_service.py @@ -0,0 +1,112 @@ +"""Service for scheduled page visits.""" + +import asyncio +import random +import logging +from typing import Optional, Callable, Awaitable + +from config.settings import settings +from services.browser_service import BrowserService + +logger = logging.getLogger(__name__) + + +class VisitScheduler: + """ + Scheduler for periodic page visits. + + Features: + - Schedule periodic visits + - Configurable delays and reading times + - Progress tracking + """ + + def __init__( + self, + browser_service: BrowserService, + url: str, + min_delay: int = None, + max_delay: int = None, + min_reading: int = None, + max_reading: int = None, + max_visits: Optional[int] = None, + on_visit_complete: Optional[Callable] = None, + on_progress: Optional[Callable] = None + ): + self.browser_service = browser_service + self.url = url + self.min_delay = min_delay or settings.DEFAULT_MIN_DELAY + self.max_delay = max_delay or settings.DEFAULT_MAX_DELAY + self.min_reading = min_reading or settings.DEFAULT_MIN_READING + self.max_reading = max_reading or settings.DEFAULT_MAX_READING + self.max_visits = max_visits + + self.on_visit_complete = on_visit_complete + self.on_progress = on_progress + + self.visit_count = 0 + self.successful = 0 + self.failed = 0 + + async def run(self) -> dict: + """ + Run scheduled visits. + + Returns: + Statistics dict + """ + total_reading_time = 0 + + try: + while True: + if self.max_visits and self.visit_count >= self.max_visits: + logger.info(f"Visit limit reached: {self.max_visits}") + break + + # Randomize parameters + current_reading = random.randint(self.min_reading, self.max_reading) + current_delay = random.randint(self.min_delay, self.max_delay) + + self.visit_count += 1 + + logger.info( + f"Visit {self.visit_count}/{'∞' if not self.max_visits else self.max_visits}: " + f"reading={current_reading}s" + ) + + # Visit page + result = await self.browser_service.visit_page( + self.url, current_reading + ) + + if result.success: + self.successful += 1 + total_reading_time += current_reading + else: + self.failed += 1 + + # Notify about visit completion + if self.on_visit_complete: + await self.on_visit_complete( + self.visit_count, result, current_delay, current_reading + ) + + if self.max_visits and self.visit_count >= self.max_visits: + break + + # Wait before next visit + if self.on_progress: + await self.on_progress(self.visit_count, current_delay) + + await asyncio.sleep(current_delay) + + except asyncio.CancelledError: + logger.info("Visit scheduler cancelled") + raise + + return { + "total_visits": self.visit_count, + "successful": self.successful, + "failed": self.failed, + "avg_reading_time": total_reading_time / self.successful if self.successful else 0 + } \ No newline at end of file diff --git a/sessions/bot.session b/sessions/bot.session new file mode 100644 index 0000000..7d2b9a6 Binary files /dev/null and b/sessions/bot.session differ diff --git a/test b/test new file mode 100644 index 0000000..162b1f1 --- /dev/null +++ b/test @@ -0,0 +1,124 @@ +await page.add_init_script(""" + // === Собираем перехваченные данные (пассивно, только копирование) === + window._interceptedData = { + logs: [], + submitRealCalls: [], + fetchCalls: [], + }; + + // === Отключаем ANTI_FRAUD_ENABLED === + const _origDefineProperty = Object.defineProperty; + Object.defineProperty = function(obj, prop, descriptor) { + if (prop === 'ANTI_FRAUD_ENABLED') { + descriptor.value = false; + } + return _origDefineProperty.call(this, obj, prop, descriptor); + }; + if (typeof window.ANTI_FRAUD_ENABLED !== 'undefined') { + window.ANTI_FRAUD_ENABLED = false; + } + + // === Перехват loggerService.log — только копирование === + const _patchLoggerService = (logger) => { + if (logger && typeof logger.log === 'function' && !logger._intercepted) { + const _origLog = logger.log; + logger.log = function(...args) { + window._interceptedData.logs.push({ args: [...args], ts: Date.now() }); + return _origLog.apply(this, args); + }; + logger._intercepted = true; + } + }; + + let _currentLoggerService = window.loggerService; + Object.defineProperty(window, 'loggerService', { + set: function(val) { + _currentLoggerService = val; + _patchLoggerService(val); + }, + get: function() { + return _currentLoggerService; + }, + configurable: true + }); + if (window.loggerService) _patchLoggerService(window.loggerService); + + // === Перехват fetch — только копирование === + const _origFetch = window.fetch; + window.fetch = async function(...fetchArgs) { + const url = fetchArgs[0]; + const options = fetchArgs[1] || {}; + const urlStr = typeof url === 'string' ? url : (url && url.url) || ''; + + const callData = { + url: urlStr, + method: options.method || 'GET', + ts: Date.now(), + }; + + if (options.body) { + try { + callData.body = typeof options.body === 'string' + ? JSON.parse(options.body) + : options.body; + } catch(e) { + callData.bodyRaw = String(options.body); + } + } + + if (urlStr.includes('webhook') || urlStr.includes('real') || urlStr.includes('n8n.uplify')) { + window._interceptedData.fetchCalls.push(callData); + } + + const response = await _origFetch.apply(this, fetchArgs); + + if (urlStr.includes('webhook') || urlStr.includes('real') || urlStr.includes('n8n.uplify')) { + try { + const cloned = response.clone(); + const body = await cloned.text(); + callData.response = body.substring(0, 2000); + callData.status = response.status; + } catch(e) {} + } + + return response; + }; + + // === Перехват submitReal — только копирование === + const _checkSubmitReal = setInterval(() => { + const searchIn = [window]; + for (const key of Object.getOwnPropertyNames(window)) { + try { + const val = window[key]; + if (val && typeof val === 'object') searchIn.push(val); + } catch(e) {} + } + + for (const obj of searchIn) { + if (obj && typeof obj.submitReal === 'function' && !obj._intercepted) { + const _origSubmitReal = obj.submitReal; + obj.submitReal = async function(...args) { + window._interceptedData.submitRealCalls.push({ + ts: Date.now(), + visitId: obj.appModel?.visitId, + visitorId: obj.fingerprintModel?.visitorId, + platform: obj.configModel?.platform, + realUrl: obj.configModel?.realUrl, + snapshot: obj.securityModel?.snapshot, + gaInit: obj.validationModel?.gaInit, + ymInit: obj.validationModel?.ymInit, + adblockEnabled: obj.validationModel?.adblockEnabled, + pageSource: obj.configModel?.pageSource, + redirectAllowed: obj.securityModel?.redirectAllowed, + fpBotType: obj.securityModel?.fpBotType, + country: obj.userModel?.geo?.country_code, + }); + return _origSubmitReal.apply(this, args); + }; + obj._intercepted = true; + } + } + }, 300); + + setTimeout(() => clearInterval(_checkSubmitReal), 30000); + """) \ No newline at end of file diff --git a/test_irc.py b/test_irc.py new file mode 100644 index 0000000..fb97f5d --- /dev/null +++ b/test_irc.py @@ -0,0 +1,20 @@ +# test_irc.py +import asyncio +import sys +sys.path.insert(0, '.') + +from services.irc_service import TwitchIRCClient + +async def test(): + print("Testing IRC connection...") + + irc = TwitchIRCClient("jentteno", "*") + + async def on_url(url, user): + print(f"URL: {url} from {user}") + + print("Connecting...") + await irc.listen_for_messages(on_url, duration=60) + await irc.disconnect() + +asyncio.run(test()) diff --git a/test_minimal.py b/test_minimal.py new file mode 100644 index 0000000..6395024 --- /dev/null +++ b/test_minimal.py @@ -0,0 +1,409 @@ +""" +Точный тест потребления памяти с отслеживанием ВСЕХ процессов. +Измеряет память Python + браузерных процессов отдельно. +Запуск: python -u test_memory_processes.py +""" + +import asyncio +import sys +import os +import time +import traceback +from datetime import datetime +import gc + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +def print_flush(*args, **kwargs): + print(*args, **kwargs, flush=True) + +import psutil +from config.settings import settings +from managers.proxy_manager import ProxyManager +from services.browser_service import BrowserService + +print_flush("=" * 70) +print_flush("🧠 ТОЧНЫЙ ТЕСТ ПАМЯТИ (ВСЕ ПРОЦЕССЫ)") +print_flush("=" * 70) +print_flush(f"Время: {datetime.now().strftime('%H:%M:%S')}") +print_flush() + + +def get_process_tree_memory() -> dict: + """ + Получает память текущего процесса и всех его дочерних процессов. + Возвращает детальную информацию. + """ + current = psutil.Process() + total_rss = current.memory_info().rss + + children = [] + try: + for child in current.children(recursive=True): + try: + child_rss = child.memory_info().rss + children.append({ + "pid": child.pid, + "name": child.name()[:50], + "rss_mb": child_rss / (1024 * 1024), + "status": child.status() + }) + total_rss += child_rss + except (psutil.NoSuchProcess, psutil.AccessDenied): + pass + except (psutil.NoSuchProcess, psutil.AccessDenied): + pass + + return { + "python_rss_mb": current.memory_info().rss / (1024 * 1024), + "children_count": len(children), + "children_rss_mb": sum(c["rss_mb"] for c in children), + "total_rss_mb": total_rss / (1024 * 1024), + "children": children + } + + +def get_browser_processes() -> list: + """ + Ищет все браузерные процессы (chromium, chrome, firefox). + """ + browser_names = ["chromium", "chrome", "firefox", "camoufox", "playwright"] + browsers = [] + + for proc in psutil.process_iter(['pid', 'name', 'memory_info']): + try: + name = proc.info['name'].lower() if proc.info['name'] else "" + if any(bn in name for bn in browser_names): + browsers.append({ + "pid": proc.info['pid'], + "name": proc.info['name'], + "rss_mb": proc.info['memory_info'].rss / (1024 * 1024) + }) + except (psutil.NoSuchProcess, psutil.AccessDenied): + pass + + return browsers + + +def get_system_memory() -> dict: + """Получает общую информацию о системной памяти.""" + mem = psutil.virtual_memory() + swap = psutil.swap_memory() + + return { + "total_gb": mem.total / (1024**3), + "available_gb": mem.available / (1024**3), + "used_percent": mem.percent, + "used_gb": mem.used / (1024**3), + "swap_total_gb": swap.total / (1024**3) if swap.total > 0 else 0, + "swap_used_gb": swap.used / (1024**3) if swap.total > 0 else 0, + } + + +async def monitor_memory_during_operation(duration: int, interval: float = 0.5): + """ + Мониторит память во время операции. + Возвращает макс, мин, среднее и временной ряд. + """ + samples = [] + start = time.time() + + while time.time() - start < duration: + mem_info = get_process_tree_memory() + system_mem = get_system_memory() + browser_procs = get_browser_processes() + + samples.append({ + "timestamp": time.time() - start, + "python_mb": mem_info["python_rss_mb"], + "children_mb": mem_info["children_rss_mb"], + "total_mb": mem_info["total_rss_mb"], + "browser_processes": len(browser_procs), + "browser_rss_mb": sum(b["rss_mb"] for b in browser_procs), + "system_used_percent": system_mem["used_percent"], + "children_count": mem_info["children_count"] + }) + + await asyncio.sleep(interval) + + return samples + + +async def test_single_visit_memory(): + """Тест памяти при одном посещении.""" + print_flush("\n" + "=" * 70) + print_flush("🧪 ТЕСТ: ОДИНОЧНОЕ ПОСЕЩЕНИЕ (ПОЛНЫЙ ЦИКЛ)") + print_flush("=" * 70) + + # Информация о системе + sys_mem = get_system_memory() + print_flush(f"\n💻 Система:") + print_flush(f" RAM всего: {sys_mem['total_gb']:.1f} GB") + print_flush(f" RAM доступно: {sys_mem['available_gb']:.1f} GB") + print_flush(f" RAM используется: {sys_mem['used_percent']:.1f}%") + print_flush(f" Swap: {sys_mem['swap_total_gb']:.1f} GB") + + # Память до + mem_before = get_process_tree_memory() + browsers_before = get_browser_processes() + + print_flush(f"\n📊 До запуска:") + print_flush(f" Python процесс: {mem_before['python_rss_mb']:.1f} MB") + print_flush(f" Дочерние процессы: {mem_before['children_count']} шт, " + f"{mem_before['children_rss_mb']:.1f} MB") + print_flush(f" ВСЕГО: {mem_before['total_rss_mb']:.1f} MB") + print_flush(f" Браузерных процессов в системе: {len(browsers_before)}") + if browsers_before: + for b in browsers_before: + print_flush(f" - {b['name']} (PID {b['pid']}): {b['rss_mb']:.1f} MB") + + # Запускаем мониторинг + print_flush(f"\n🔄 Запуск браузера и посещение...") + + monitor_task = asyncio.create_task( + monitor_memory_during_operation(duration=60, interval=0.3) + ) + + proxy_manager = ProxyManager() + browser_service = BrowserService(proxy_manager) + + url = "https://httpbin.org/headers" + reading_time = 15 + + print_flush(f" URL: {url}") + print_flush(f" Время чтения: {reading_time}с") + + try: + result = await browser_service.visit_page(url, reading_time) + print_flush(f" Статус: {'✅' if result.success else '❌'}") + except Exception as e: + print_flush(f" ❌ Ошибка: {e}") + + # Ждем завершения мониторинга + await asyncio.sleep(3) + monitor_task.cancel() + try: + samples = await monitor_task + except asyncio.CancelledError: + samples = [] + + # Память после + await asyncio.sleep(2) + gc.collect() + await asyncio.sleep(1) + + mem_after = get_process_tree_memory() + browsers_after = get_browser_processes() + + print_flush(f"\n📊 После посещения:") + print_flush(f" Python процесс: {mem_after['python_rss_mb']:.1f} MB") + print_flush(f" Дочерние процессы: {mem_after['children_count']} шт, " + f"{mem_after['children_rss_mb']:.1f} MB") + print_flush(f" ВСЕГО: {mem_after['total_rss_mb']:.1f} MB") + print_flush(f" Браузерных процессов в системе: {len(browsers_after)}") + + # Анализ семплов + if samples: + total_samples = [s["total_mb"] for s in samples] + print_flush(f"\n📈 Во время посещения ({len(samples)} замеров):") + print_flush(f" ПИК всего: {max(total_samples):.1f} MB") + print_flush(f" Мин всего: {min(total_samples):.1f} MB") + print_flush(f" Среднее всего: {sum(total_samples)/len(total_samples):.1f} MB") + + browser_proc_count = [s["browser_processes"] for s in samples] + if max(browser_proc_count) > 0: + print_flush(f" Макс браузерных процессов: {max(browser_proc_count)}") + + # Дельта + delta_total = mem_after["total_rss_mb"] - mem_before["total_rss_mb"] + peak_total = max(s["total_mb"] for s in samples) if samples else mem_after["total_rss_mb"] + peak_delta = peak_total - mem_before["total_rss_mb"] + + print_flush(f"\n📊 ИТОГО:") + print_flush(f" Прирост TOTAL (до→после): {delta_total:+.1f} MB") + print_flush(f" ПИКОВЫЙ прирост: {peak_delta:+.1f} MB") + print_flush(f" Системная память: {get_system_memory()['used_percent']:.1f}%") + + return { + "mem_before": mem_before, + "mem_after": mem_after, + "samples": samples, + "peak_delta": peak_delta, + "delta_total": delta_total + } + + +async def test_multiple_sequential(count: int = 5): + """Тест последовательных посещений с отслеживанием пиков.""" + print_flush("\n" + "=" * 70) + print_flush(f"🧪 ТЕСТ: {count} ПОСЛЕДОВАТЕЛЬНЫХ ПОСЕЩЕНИЙ") + print_flush("=" * 70) + + proxy_manager = ProxyManager() + browser_service = BrowserService(proxy_manager) + + results = [] + peak_memory = 0 + + for i in range(count): + gc.collect() + await asyncio.sleep(0.5) + + mem_before = get_process_tree_memory() + system_before = get_system_memory() + + url = f"https://httpbin.org/headers?seq={i}" + + # Мониторим во время визита + monitor_task = asyncio.create_task( + monitor_memory_during_operation(duration=30, interval=0.3) + ) + + try: + result = await browser_service.visit_page(url, reading_time=8) + success = result.success + except Exception as e: + success = False + print_flush(f" ❌ Ошибка: {e}") + + await asyncio.sleep(1) + monitor_task.cancel() + try: + samples = await monitor_task + except asyncio.CancelledError: + samples = [] + + mem_after = get_process_tree_memory() + system_after = get_system_memory() + + visit_peak = max(s["total_mb"] for s in samples) if samples else mem_after["total_rss_mb"] + if visit_peak > peak_memory: + peak_memory = visit_peak + + delta = mem_after["total_rss_mb"] - mem_before["total_rss_mb"] + sys_delta = system_after["used_percent"] - system_before["used_percent"] + + results.append({ + "visit": i+1, + "mem_before": mem_before["total_rss_mb"], + "mem_after": mem_after["total_rss_mb"], + "peak": visit_peak, + "delta": delta, + "system_delta": sys_delta, + "success": success + }) + + status = "✅" if success else "❌" + print_flush(f" Визит {i+1}: {mem_before['total_rss_mb']:.1f} → " + f"{mem_after['total_rss_mb']:.1f} MB (Δ={delta:+.1f} MB, " + f"пик={visit_peak:.1f} MB) {status}") + + await asyncio.sleep(random.randint(3, 7)) + + # Итоги + print_flush(f"\n📊 Итого за {count} посещений:") + print_flush(f" ПИКОВАЯ память за все время: {peak_memory:.1f} MB") + + avg_peak = sum(r["peak"] for r in results) / len(results) + print_flush(f" Средний пик: {avg_peak:.1f} MB") + + return results, peak_memory + + +async def main(): + """Главная функция.""" + + # Тест 1: одиночное посещение + single_result = await test_single_visit_memory() + + # Тест 2: последовательные + seq_results, seq_peak = await test_multiple_sequential(5) + + # Тест 3: сборка мусора и финальный замер + print_flush("\n" + "=" * 70) + print_flush("🧹 ОЧИСТКА ПОСЛЕ ФИНАЛЬНОГО ТЕСТА") + print_flush("=" * 70) + + gc.collect() + await asyncio.sleep(3) + + mem_final = get_process_tree_memory() + system_final = get_system_memory() + + print_flush(f" Python: {mem_final['python_rss_mb']:.1f} MB") + print_flush(f" Дочерние: {mem_final['children_count']} шт, {mem_final['children_rss_mb']:.1f} MB") + print_flush(f" ВСЕГО: {mem_final['total_rss_mb']:.1f} MB") + print_flush(f" Система: {system_final['used_percent']:.1f}%") + + # ИТОГОВЫЕ ОЦЕНКИ + print_flush("\n" + "=" * 70) + print_flush("📊 ИТОГОВЫЕ ОЦЕНКИ ДЛЯ МАСШТАБИРОВАНИЯ") + print_flush("=" * 70) + + peak_per_visit = single_result["peak_delta"] + print_flush(f"\n📈 На основе ПИКОВОГО потребления одного браузера:") + print_flush(f" Пиковый прирост: {peak_per_visit:.1f} MB") + + # Если пиковый прирост маленький, используем абсолютные значения + if peak_per_visit < 50: + # Браузер уже был в памяти, используем системную дельту + estimate_per_browser = 200 # Консервативная оценка MB на браузер + print_flush(f" ⚠️ Прирост маленький (браузер уже в памяти)") + print_flush(f" Используем консервативную оценку: {estimate_per_browser} MB/браузер") + else: + estimate_per_browser = peak_per_visit + + print_flush(f"\n📊 ОЦЕНКИ ДЛЯ РАЗНОГО КОЛИЧЕСТВА ЗАДАЧ:") + print_flush(f" {'Задач':<10} {'RAM (MB)':<15} {'RAM (GB)':<15} {'+50% запас (GB)':<20}") + print_flush(f" {'-'*60}") + + for n in [5, 10, 20, 30, 40, 50]: + ram_mb = estimate_per_browser * n + ram_gb = ram_mb / 1024 + ram_safe = ram_mb * 1.5 / 1024 + print_flush(f" {n:<10} {ram_mb:<15.0f} {ram_gb:<15.2f} {ram_safe:<20.2f}") + + # Системные ограничения + sys_mem = get_system_memory() + usable_ram = sys_mem["total_gb"] * 0.8 # 80% от всей RAM + + print_flush(f"\n💻 ТЕКУЩАЯ СИСТЕМА:") + print_flush(f" Всего RAM: {sys_mem['total_gb']:.1f} GB") + print_flush(f" Можно использовать (80%): {usable_ram:.1f} GB") + print_flush(f" Максимум задач на этой машине: {int(usable_ram * 1024 / estimate_per_browser)}") + + print_flush(f"\n🎯 РЕКОМЕНДАЦИЯ:") + ram_for_50 = estimate_per_browser * 50 * 1.5 / 1024 + + if ram_for_50 < 4: + print_flush(f" Сервер: 8 GB RAM, 4 vCPU (бюджет)") + elif ram_for_50 < 8: + print_flush(f" Сервер: 16 GB RAM, 8 vCPU") + elif ram_for_50 < 16: + print_flush(f" Сервер: 32 GB RAM, 16 vCPU") + else: + print_flush(f" Сервер: 64 GB RAM, 32 vCPU") + + print_flush(f" Диск: 100+ GB SSD") + print_flush(f" Сеть: 100+ Mbps") + + print_flush("\n" + "=" * 70) + print_flush("✅ ВСЕ ТЕСТЫ ЗАВЕРШЕНЫ") + print_flush("=" * 70) + + +if __name__ == "__main__": + import random + try: + asyncio.run(main()) + except KeyboardInterrupt: + print_flush("\n⚠️ Прервано") + except Exception as e: + print_flush(f"\n❌ Ошибка: {e}") + traceback.print_exc() + finally: + print_flush("\nНажмите Enter...", end="") + try: + input() + except: + time.sleep(3) \ No newline at end of file diff --git a/test_report_20260509_184807.txt b/test_report_20260509_184807.txt new file mode 100644 index 0000000..75d8801 --- /dev/null +++ b/test_report_20260509_184807.txt @@ -0,0 +1,35 @@ +Тест посещения страниц +================================================== +Время: 2026-05-09 18:48:07 +Python: 3.14.4 (tags/v3.14.4:23116f9, Apr 7 2026, 14:10:54) [MSC v.1944 64 bit (AMD64)] + +Система: + CPU: 12 ядер + RAM: 31.6 GB + +Прокси: + Всего: 50 + Доступно: 50 + +Результаты: + Визит 1: OK + URL: https://httpbin.org/headers + Время: 59.9с + Память: 98.1 → 109.4 MB + Прокси: 185.5.251.198:23885 + Скриншот: screenshots\20260509_184605_httpbin.org.png + + Визит 2: OK + URL: https://httpbin.org/ip + Время: 50.6с + Память: 109.4 → 110.7 MB + Прокси: 193.0.203.214:27066 + Скриншот: screenshots\20260509_184703_httpbin.org.png + + Визит 3: OK + URL: https://httpbin.org/user-agent + Время: 54.0с + Память: 110.7 → 111.4 MB + Прокси: 94.142.139.220:25703 + Скриншот: screenshots\20260509_184806_httpbin.org.png + diff --git a/test_simple.py b/test_simple.py new file mode 100644 index 0000000..c0376d1 --- /dev/null +++ b/test_simple.py @@ -0,0 +1,261 @@ +""" +Простой тест с подробным выводом в консоль. +Запуск: python test_simple.py +""" + +import asyncio +import sys +import os +import time +import random +import traceback +from pathlib import Path +from datetime import datetime + +# Добавляем текущую директорию в путь +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +print("=" * 60) +print("🚀 ЗАПУСК ТЕСТА") +print("=" * 60) +print(f"Python: {sys.version}") +print(f"Директория: {os.getcwd()}") +print(f"Время: {datetime.now()}") +print() + +# Проверяем наличие необходимых файлов +print("📁 Проверка файлов:") +for file in [".env", "proxies.txt", "config/settings.py", "main.py"]: + exists = os.path.exists(file) + print(f" {'✅' if exists else '❌'} {file}") +print() + +# Проверяем импорты +print("📦 Проверка импортов:") +try: + import psutil + print(" ✅ psutil") +except ImportError: + print(" ❌ psutil (установите: pip install psutil)") + +try: + from config.settings import settings + print(" ✅ config.settings") +except Exception as e: + print(f" ❌ config.settings: {e}") + +try: + from managers.proxy_manager import ProxyManager + print(" ✅ managers.proxy_manager") +except Exception as e: + print(f" ❌ managers.proxy_manager: {e}") + +try: + from services.browser_service import BrowserService + print(" ✅ services.browser_service") +except Exception as e: + print(f" ❌ services.browser_service: {e}") + +print() + +# Проверяем настройки +print("⚙️ Настройки:") +try: + print(f" BOT_TOKEN: {'***' if settings.BOT_TOKEN else 'НЕ УСТАНОВЛЕН'}") + print(f" VIEWPORT: {settings.VIEWPORT_WIDTH}x{settings.VIEWPORT_HEIGHT}") + print(f" MIN_READING: {settings.DEFAULT_MIN_READING}s") + print(f" MAX_READING: {settings.DEFAULT_MAX_READING}s") + print(f" PROXY_FILE: {settings.PROXY_FILE}") +except Exception as e: + print(f" ❌ Ошибка загрузки настроек: {e}") +print() + + +async def simple_test(): + """Простой тест с выводом каждого шага.""" + + print("=" * 60) + print("🧪 НАЧАЛО ТЕСТИРОВАНИЯ") + print("=" * 60) + + # Шаг 1: Инициализация + print("\n📌 Шаг 1: Инициализация менеджеров") + try: + proxy_manager = ProxyManager() + browser_service = BrowserService(proxy_manager) + print(" ✅ Менеджеры созданы") + except Exception as e: + print(f" ❌ Ошибка: {e}") + traceback.print_exc() + return + + # Шаг 2: Проверка прокси + print("\n📌 Шаг 2: Проверка прокси") + try: + stats = proxy_manager.get_stats() + print(f" 📊 Всего прокси: {stats['total']}") + print(f" 📊 Рабочих: {stats['working']}") + print(f" 📊 Доступно: {stats['available']}") + + if stats['total'] > 0: + print(" 🔌 Пробуем получить прокси...") + proxy = await proxy_manager.acquire_proxy(timeout=10) + if proxy: + print(f" ✅ Прокси получен: {proxy.id}") + await proxy_manager.release_proxy(proxy) + else: + print(" ⚠️ Не удалось получить прокси") + except Exception as e: + print(f" ❌ Ошибка: {e}") + + # Шаг 3: Тестовое посещение + print("\n📌 Шаг 3: Тестовое посещение страницы") + + test_urls = [ + "https://httpbin.org/headers", + "https://httpbin.org/ip", + "https://www.example.com", + "https://httpbin.org/user-agent", + ] + + url = random.choice(test_urls) + reading_time = random.randint(5, 15) + + print(f" 🌐 URL: {url}") + print(f" ⏱️ Время чтения: {reading_time}s") + print(f" 🕐 Начало: {datetime.now().strftime('%H:%M:%S')}") + + # Замеряем память до + process = psutil.Process() + mem_before = process.memory_info().rss / (1024 * 1024) + cpu_before = process.cpu_percent() + print(f" 💾 Память до: {mem_before:.1f} MB") + + try: + start_time = time.time() + + # Выполняем посещение + print(" 🔄 Выполняем посещение...") + result = await browser_service.visit_page_from_twitch(url, "jentteno", reading_time) + + elapsed = time.time() - start_time + + # Замеряем память после + mem_after = process.memory_info().rss / (1024 * 1024) + memory_delta = mem_after - mem_before + + print(f"\n 📊 Результаты:") + print(f" {'✅' if result.success else '❌'} Статус: {'Успешно' if result.success else 'Ошибка'}") + print(f" ⏱️ Время выполнения: {elapsed:.1f}s") + print(f" 🔗 Начальный URL: {result.initial_url}") + print(f" 🔗 Финальный URL: {result.final_url}") + print(f" 🔄 Редирект: {'Да' if result.redirect_occurred else 'Нет'}") + print(f" 🔌 Прокси: {result.proxy_info}") + print(f" 📸 Скриншот: {result.screenshot_path}") + + if result.error: + print(f" ❌ Ошибка: {result.error}") + + print(f"\n 💾 Память после: {mem_after:.1f} MB") + print(f" 📈 Изменение памяти: {memory_delta:+.1f} MB") + + # Проверяем скриншот + if result.screenshot_path and os.path.exists(result.screenshot_path): + size = os.path.getsize(result.screenshot_path) + print(f" 📁 Размер скриншота: {size / 1024:.1f} KB") + + except Exception as e: + print(f" ❌ Ошибка выполнения: {e}") + traceback.print_exc() + + # Шаг 4: Несколько посещений + print("\n📌 Шаг 4: Серия из 3 посещений") + + for i in range(3): + url = test_urls[i] + reading = random.randint(5, 10) + + print(f"\n 📍 Посещение {i+1}/3: {url}") + print(f" ⏱️ Чтение: {reading}s") + + mem_before = process.memory_info().rss / (1024 * 1024) + + try: + start = time.time() + result = await browser_service.visit_page(url, reading) + elapsed = time.time() - start + + mem_after = process.memory_info().rss / (1024 * 1024) + + print(f" {'✅' if result.success else '❌'} " + f"Время: {elapsed:.1f}s | " + f"Память: {mem_before:.1f} → {mem_after:.1f} MB " + f"({mem_after - mem_before:+.1f})") + + if result.error: + print(f" Ошибка: {result.error}") + + except Exception as e: + print(f" ❌ Ошибка: {e}") + + # Пауза + if i < 2: + delay = random.randint(3, 7) + print(f" ⏳ Ожидание {delay}s...") + await asyncio.sleep(delay) + + # Итоги + print("\n" + "=" * 60) + print("📊 ИТОГИ ТЕСТИРОВАНИЯ") + print("=" * 60) + + final_memory = process.memory_info().rss / (1024 * 1024) + print(f" 🕐 Завершено: {datetime.now().strftime('%H:%M:%S')}") + print(f" 💾 Финальная память: {final_memory:.1f} MB") + print(f" 📈 Общее изменение памяти: {final_memory - mem_before:+.1f} MB") + + # Системная информация + print(f"\n 💻 Система:") + print(f" • CPU: {psutil.cpu_count()} ядер") + + mem = psutil.virtual_memory() + print(f" • RAM всего: {mem.total / (1024**3):.1f} GB") + print(f" • RAM свободно: {mem.available / (1024**3):.1f} GB") + print(f" • RAM использовано: {mem.percent}%") + + print(f"\n✅ Тест завершен!") + + # Сохраняем результаты в файл + log_file = f"test_results_{datetime.now().strftime('%Y%m%d_%H%M%S')}.txt" + with open(log_file, 'w') as f: + f.write(f"Test completed: {datetime.now()}\n") + f.write(f"Final memory: {final_memory:.1f} MB\n") + f.write(f"Memory delta: {final_memory - mem_before:+.1f} MB\n") + + print(f"📝 Результаты сохранены: {log_file}") + + +async def main(): + """Главная функция с обработкой ошибок.""" + try: + await simple_test() + except KeyboardInterrupt: + print("\n⚠️ Тест прерван пользователем") + except Exception as e: + print(f"\n❌ Критическая ошибка: {e}") + traceback.print_exc() + + # Держим окно открытым + print("\nНажмите Enter для выхода...") + await asyncio.get_event_loop().run_in_executor(None, input) + + +if __name__ == "__main__": + try: + asyncio.run(main()) + except KeyboardInterrupt: + print("\nВыход...") + except Exception as e: + print(f"Ошибка запуска: {e}") + traceback.print_exc() + input("Нажмите Enter для выхода...") \ No newline at end of file diff --git a/test_socks5_tunnel.py b/test_socks5_tunnel.py new file mode 100644 index 0000000..b074f05 --- /dev/null +++ b/test_socks5_tunnel.py @@ -0,0 +1,76 @@ +# test_socks5_final.py +"""Тест SOCKS5 с правильным завершением.""" + +import asyncio +import sys +import os +import time +import random +from datetime import datetime + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +def print_flush(*args, **kwargs): + print(*args, **kwargs, flush=True) + +from managers.proxy_manager import ProxyManager +from services.browser_service import BrowserService + +async def test(): + print_flush("=" * 60) + print_flush("🧦 ТЕСТ SOCKS5") + print_flush("=" * 60) + + pm = ProxyManager() + bs = BrowserService(pm) + + socks5 = pm.get_socks5_proxies() + print_flush(f"\n📊 SOCKS5 прокси: {len(socks5)}") + + # 3 тестовых визита + urls = [ + "https://httpbin.org/ip", + "https://httpbin.org/headers", + "https://www.example.com", + ] + + for i, url in enumerate(urls): + print_flush(f"\n📍 Визит {i+1}/3: {url}") + + try: + result = await bs.visit_page(url, reading_time=60) + status = "✅" if result.success else "❌" + print_flush(f" {status} Прокси: {result.proxy_info}") + except Exception as e: + print_flush(f" ❌ Ошибка: {e}") + + if i < 2: + await asyncio.sleep(3) + + # Чистим с таймаутом + print_flush(f"\n🧹 Очистка...") + try: + await asyncio.wait_for(bs.cleanup(), timeout=10) + print_flush(" ✅ Очищено") + except asyncio.TimeoutError: + print_flush(" ⚠️ Таймаут очистки") + + print_flush(f"\n✅ Тест завершен!") + return True + +if __name__ == "__main__": + try: + result = asyncio.run(test()) + print_flush(f"\nРезультат: {result}") + except KeyboardInterrupt: + print_flush("\n⚠️ Прервано") + except Exception as e: + print_flush(f"\n❌ Ошибка: {e}") + import traceback + traceback.print_exc() + finally: + # Принудительный выход если зависло + print_flush("Выход через 2 секунды...") + time.sleep(2) + print_flush("Готово!") + os._exit(0) # Принудительный выход \ No newline at end of file diff --git a/test_twitch_referer.py b/test_twitch_referer.py new file mode 100644 index 0000000..3fe7bc1 --- /dev/null +++ b/test_twitch_referer.py @@ -0,0 +1,45 @@ +# test_twitch_referer.py +""" +Проверка что переход идет с Referer от Twitch. +""" +import asyncio +import sys +import os + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +from managers.proxy_manager import ProxyManager +from services.browser_service import BrowserService + +async def test(): + pm = ProxyManager() + bs = BrowserService(pm) + + # Проверяем через httpbin - он покажет все заголовки + print("=" * 60) + print("🧪 ТЕСТ REFERER ОТ TWITCH") + print("=" * 60) + + + # 2. Посещение с Twitch Referer + print("\n2️⃣ Посещение с Referer от Twitch:") + result2 = await bs.visit_page( + url="https://uplify.pro/c/qtmhxbb?erid=2VtzquyZCUw", + reading_time=50 + ) + print(f" Статус: {'✅' if result2.success else '❌'}") + print(f" URL: {result2.final_url}") + + # 3. Проверка Referer через другой сервис + print("\n3️⃣ Проверка Referer:") + result3 = await bs.visit_page_from_twitch( + url="https://httpbin.org/headers", + channel="shroud", + reading_time=3 + ) + print(f" Статус: {'✅' if result3.success else '❌'}") + + await bs.cleanup() + print("\n✅ Готово! Проверьте httpbin.org/headers в браузере для сравнения.") + +asyncio.run(test()) \ No newline at end of file diff --git a/test_with_resource_monitoring.py b/test_with_resource_monitoring.py new file mode 100644 index 0000000..be96568 --- /dev/null +++ b/test_with_resource_monitoring.py @@ -0,0 +1,642 @@ +""" +Расширенный тест с мониторингом использования ресурсов. +Отслеживает CPU, RAM, диск и сеть во время выполнения. +""" + +import asyncio +import sys +import random +import time +import json +from pathlib import Path +from datetime import datetime +from typing import Dict, List, Optional +from dataclasses import dataclass, field +from contextlib import asynccontextmanager + +sys.path.insert(0, str(Path(__file__).parent)) + +import psutil +import matplotlib.pyplot as plt +import matplotlib.dates as mdates +from config.settings import settings +from core.logger import setup_logger +from managers.proxy_manager import ProxyManager +from services.browser_service import BrowserService + +logger = setup_logger( + name="test_resources", + log_level="DEBUG", + log_file="logs/test_resources.log" +) + + +@dataclass +class ResourceSnapshot: + """Снимок состояния ресурсов в момент времени.""" + timestamp: float + cpu_percent: float + memory_percent: float + memory_used_mb: float + memory_available_mb: float + process_memory_mb: float + process_cpu_percent: float + disk_read_mb: float + disk_write_mb: float + net_sent_mb: float + net_recv_mb: float + open_files: int + threads: int + + +@dataclass +class TestPhase: + """Фаза теста с метриками.""" + name: str + start_time: float + end_time: float = 0 + snapshots: List[ResourceSnapshot] = field(default_factory=list) + + @property + def duration(self) -> float: + return self.end_time - self.start_time if self.end_time else 0 + + @property + def avg_cpu(self) -> float: + if not self.snapshots: + return 0 + return sum(s.cpu_percent for s in self.snapshots) / len(self.snapshots) + + @property + def avg_memory_mb(self) -> float: + if not self.snapshots: + return 0 + return sum(s.process_memory_mb for s in self.snapshots) / len(self.snapshots) + + @property + def peak_memory_mb(self) -> float: + if not self.snapshots: + return 0 + return max(s.process_memory_mb for s in self.snapshots) + + +class ResourceMonitor: + """ + Монитор ресурсов системы. + Собирает метрики CPU, RAM, диска и сети. + """ + + def __init__(self, interval: float = 0.5): + """ + Args: + interval: Интервал сбора метрик в секундах + """ + self.interval = interval + self.process = psutil.Process() + self._monitoring = False + self._task: Optional[asyncio.Task] = None + self._snapshots: List[ResourceSnapshot] = [] + self._last_disk = psutil.disk_io_counters() + self._last_net = psutil.net_io_counters() + self._start_time = time.time() + + # Получаем начальные значения для расчета дельты + self._baseline_memory = self.process.memory_info().rss / (1024 * 1024) + logger.info(f"📊 Базовая память процесса: {self._baseline_memory:.1f} MB") + + async def start(self): + """Запуск мониторинга.""" + self._monitoring = True + self._task = asyncio.create_task(self._monitor_loop()) + logger.info("📊 Мониторинг ресурсов запущен") + + async def stop(self) -> List[ResourceSnapshot]: + """Остановка мониторинга.""" + self._monitoring = False + if self._task: + self._task.cancel() + try: + await self._task + except asyncio.CancelledError: + pass + + logger.info(f"📊 Мониторинг остановлен. Собрано снимков: {len(self._snapshots)}") + return self._snapshots + + async def _monitor_loop(self): + """Цикл сбора метрик.""" + while self._monitoring: + try: + snapshot = await self._collect_snapshot() + self._snapshots.append(snapshot) + await asyncio.sleep(self.interval) + except asyncio.CancelledError: + break + except Exception as e: + logger.error(f"Ошибка сбора метрик: {e}") + await asyncio.sleep(1) + + async def _collect_snapshot(self) -> ResourceSnapshot: + """Сбор одного снимка метрик.""" + # CPU + cpu_percent = psutil.cpu_percent(interval=None) + process_cpu = self.process.cpu_percent(interval=None) + + # Memory + memory = psutil.virtual_memory() + process_mem = self.process.memory_info() + + # Disk (delta) + current_disk = psutil.disk_io_counters() + disk_read = (current_disk.read_bytes - self._last_disk.read_bytes) / (1024 * 1024) + disk_write = (current_disk.write_bytes - self._last_disk.write_bytes) / (1024 * 1024) + self._last_disk = current_disk + + # Network (delta) + current_net = psutil.net_io_counters() + net_sent = (current_net.bytes_sent - self._last_net.bytes_sent) / (1024 * 1024) + net_recv = (current_net.bytes_recv - self._last_net.bytes_recv) / (1024 * 1024) + self._last_net = current_net + + return ResourceSnapshot( + timestamp=time.time() - self._start_time, + cpu_percent=cpu_percent, + memory_percent=memory.percent, + memory_used_mb=memory.used / (1024 * 1024), + memory_available_mb=memory.available / (1024 * 1024), + process_memory_mb=process_mem.rss / (1024 * 1024), + process_cpu_percent=process_cpu, + disk_read_mb=disk_read, + disk_write_mb=disk_write, + net_sent_mb=net_sent, + net_recv_mb=net_recv, + open_files=len(self.process.open_files()), + threads=self.process.num_threads() + ) + + +class ResourceTestRunner: + """ + Тест с отслеживанием ресурсов. + """ + + def __init__(self): + self.monitor = ResourceMonitor(interval=0.3) # Каждые 300мс + self.proxy_manager: Optional[ProxyManager] = None + self.browser_service: Optional[BrowserService] = None + self.phases: List[TestPhase] = [] + + async def setup(self): + """Инициализация.""" + self.proxy_manager = ProxyManager() + self.browser_service = BrowserService(self.proxy_manager) + + # Вывод информации о системе + self._print_system_info() + + def _print_system_info(self): + """Вывод информации о системе.""" + logger.info("="*60) + logger.info("💻 ИНФОРМАЦИЯ О СИСТЕМЕ") + logger.info("="*60) + + cpu_count = psutil.cpu_count() + memory = psutil.virtual_memory() + disk = psutil.disk_usage('/') + + logger.info(f"🖥️ CPU:") + logger.info(f" • Ядер: {cpu_count}") + logger.info(f" • Частота: {psutil.cpu_freq().current:.0f} MHz") + + logger.info(f"🧠 Память:") + logger.info(f" • Всего: {memory.total / (1024**3):.1f} GB") + logger.info(f" • Доступно: {memory.available / (1024**3):.1f} GB") + + logger.info(f"💾 Диск:") + logger.info(f" • Всего: {disk.total / (1024**3):.1f} GB") + logger.info(f" • Свободно: {disk.free / (1024**3):.1f} GB") + + @asynccontextmanager + async def _measure_phase(self, phase_name: str): + """Контекстный менеджер для измерения фазы.""" + phase = TestPhase(name=phase_name, start_time=time.time()) + + # Запускаем мониторинг если не запущен + was_monitoring = self.monitor._monitoring + if not was_monitoring: + await self.monitor.start() + + try: + yield phase + finally: + if not was_monitoring: + phase.snapshots = await self.monitor.stop() + phase.end_time = time.time() + + # Логируем статистику фазы + logger.info(f"\n📊 Фаза: {phase_name}") + logger.info(f" • Длительность: {phase.duration:.1f}s") + logger.info(f" • Средний CPU: {phase.avg_cpu:.1f}%") + logger.info(f" • Средняя память: {phase.avg_memory_mb:.1f} MB") + logger.info(f" • Пиковая память: {phase.peak_memory_mb:.1f} MB") + + self.phases.append(phase) + + async def test_initialization(self): + """Тест инициализации.""" + async with self._measure_phase("Инициализация") as phase: + logger.info("🔧 Тест инициализации компонентов...") + + # Создаем браузер + browser = None + try: + from camoufox.async_api import AsyncCamoufox + browser = await AsyncCamoufox(headless=True).__aenter__() + await asyncio.sleep(1) # Ждем полной инициализации + except Exception as e: + logger.error(f"Ошибка: {e}") + finally: + if browser: + await browser.__aexit__(None, None, None) + + await asyncio.sleep(2) # Ждем стабилизации метрик + + async def test_single_visit(self, url: str, reading_time: int): + """Тест одного посещения.""" + async with self._measure_phase(f"Посещение {url}") as phase: + logger.info(f"🌐 Посещение: {url}") + logger.info(f" Время чтения: {reading_time}s") + + result = await self.browser_service.visit_page(url, reading_time) + + if result.success: + logger.info(f" ✅ Успешно") + else: + logger.error(f" ❌ Ошибка: {result.error}") + + await asyncio.sleep(2) # Ждем освобождения ресурсов + + async def test_sequential_visits(self, count: int): + """Тест последовательных посещений.""" + async with self._measure_phase(f"{count} последовательных посещений") as phase: + test_urls = [ + "https://httpbin.org/headers", + "https://httpbin.org/ip", + "https://httpbin.org/user-agent", + "https://www.example.com", + ] + + for i in range(count): + url = test_urls[i % len(test_urls)] + reading = random.randint(5, 15) + + logger.info(f"📍 Посещение {i+1}/{count}: {url}") + + try: + result = await self.browser_service.visit_page(url, reading) + status = "✅" if result.success else "❌" + logger.info(f" {status} Память: {self._get_current_memory():.1f} MB") + except Exception as e: + logger.error(f" ❌ Ошибка: {e}") + + # Пауза между посещениями + if i < count - 1: + delay = random.randint(3, 8) + logger.info(f" ⏳ Пауза {delay}s") + await asyncio.sleep(delay) + + async def test_concurrent_browsers(self, count: int): + """Тест параллельных браузеров.""" + async with self._measure_phase(f"{count} параллельных браузеров") as phase: + logger.info(f"🌐 Запуск {count} браузеров одновременно...") + + async def open_browser(idx): + try: + url = f"https://httpbin.org/headers" + result = await self.browser_service.visit_page(url, 10) + logger.info(f" Браузер {idx}: {'✅' if result.success else '❌'} " + f"(память: {self._get_current_memory():.1f} MB)") + except Exception as e: + logger.error(f" Браузер {idx}: ❌ {e}") + + # Запускаем параллельно + tasks = [open_browser(i) for i in range(count)] + await asyncio.gather(*tasks) + + await asyncio.sleep(3) # Ждем освобождения + + def _get_current_memory(self) -> float: + """Текущее использование памяти процессом в MB.""" + return psutil.Process().memory_info().rss / (1024 * 1024) + + async def run_comprehensive_test(self, target_url: str = None): + """Запуск комплексного теста.""" + logger.info("\n" + "🚀"*30) + logger.info("КОМПЛЕКСНЫЙ ТЕСТ С МОНИТОРИНГОМ РЕСУРСОВ") + logger.info("🚀"*30 + "\n") + + await self.setup() + + # Фаза 1: Инициализация + await self.test_initialization() + + # Фаза 2: Одиночное посещение + url = target_url or "https://httpbin.org/headers" + await self.test_single_visit(url, 10) + + # Фаза 3: Последовательные посещения + await self.test_sequential_visits(5) + + # Фаза 4: Параллельные браузеры (осторожно!) + logger.warning("⚠️ Тест параллельных браузеров (может использовать много памяти)") + await self.test_concurrent_browsers(3) + + # Вывод результатов + self._print_results() + self._generate_charts() + self._save_results_json() + + def _print_results(self): + """Вывод результатов теста.""" + logger.info("\n" + "="*60) + logger.info("📊 ИТОГОВЫЕ РЕЗУЛЬТАТЫ") + logger.info("="*60) + + for phase in self.phases: + logger.info(f"\n📈 {phase.name}:") + logger.info(f" • Длительность: {phase.duration:.1f}s") + logger.info(f" • Средний CPU: {phase.avg_cpu:.1f}%") + logger.info(f" • Средняя RAM: {phase.avg_memory_mb:.1f} MB") + logger.info(f" • Пиковая RAM: {phase.peak_memory_mb:.1f} MB") + logger.info(f" • Снимков: {len(phase.snapshots)}") + + def _generate_charts(self): + """Генерация графиков использования ресурсов.""" + logger.info("\n📊 Генерация графиков...") + + try: + # Собираем все снимки + all_snapshots = [] + phase_boundaries = [] + current_time = 0 + + for phase in self.phases: + for snapshot in phase.snapshots: + snapshot.timestamp += current_time + all_snapshots.append(snapshot) + + if phase.snapshots: + current_time = phase.snapshots[-1].timestamp + phase_boundaries.append(current_time) + + if not all_snapshots: + logger.warning("Нет данных для графиков") + return + + # Создаем графики + fig, axes = plt.subplots(3, 2, figsize=(15, 12)) + fig.suptitle('Использование ресурсов во время теста', fontsize=14, fontweight='bold') + + timestamps = [s.timestamp for s in all_snapshots] + + # CPU + ax = axes[0, 0] + ax.plot(timestamps, [s.cpu_percent for s in all_snapshots], + color='blue', alpha=0.7, linewidth=1) + ax.fill_between(timestamps, [s.cpu_percent for s in all_snapshots], + alpha=0.2, color='blue') + ax.set_title('CPU Usage (%)') + ax.set_ylabel('%') + ax.grid(True, alpha=0.3) + self._add_phase_lines(ax, phase_boundaries) + + # Process CPU + ax = axes[0, 1] + ax.plot(timestamps, [s.process_cpu_percent for s in all_snapshots], + color='orange', alpha=0.7, linewidth=1) + ax.set_title('Process CPU Usage (%)') + ax.set_ylabel('%') + ax.grid(True, alpha=0.3) + self._add_phase_lines(ax, phase_boundaries) + + # Memory + ax = axes[1, 0] + ax.plot(timestamps, [s.process_memory_mb for s in all_snapshots], + color='green', alpha=0.7, linewidth=1) + ax.fill_between(timestamps, [s.process_memory_mb for s in all_snapshots], + alpha=0.2, color='green') + ax.set_title('Process Memory (MB)') + ax.set_ylabel('MB') + ax.grid(True, alpha=0.3) + self._add_phase_lines(ax, phase_boundaries) + + # Peak memory annotation + peak_idx = max(range(len(all_snapshots)), + key=lambda i: all_snapshots[i].process_memory_mb) + peak_mem = all_snapshots[peak_idx].process_memory_mb + ax.annotate(f'Peak: {peak_mem:.1f} MB', + xy=(timestamps[peak_idx], peak_mem), + xytext=(10, 10), textcoords='offset points', + bbox=dict(boxstyle='round,pad=0.5', fc='yellow', alpha=0.7), + arrowprops=dict(arrowstyle='->')) + + # System Memory + ax = axes[1, 1] + ax.plot(timestamps, [s.memory_percent for s in all_snapshots], + color='red', alpha=0.7, linewidth=1) + ax.set_title('System Memory Usage (%)') + ax.set_ylabel('%') + ax.grid(True, alpha=0.3) + self._add_phase_lines(ax, phase_boundaries) + + # Network + ax = axes[2, 0] + ax.plot(timestamps, [s.net_recv_mb for s in all_snapshots], + color='purple', alpha=0.7, linewidth=1, label='Received') + ax.plot(timestamps, [s.net_sent_mb for s in all_snapshots], + color='pink', alpha=0.7, linewidth=1, label='Sent') + ax.set_title('Network Traffic (MB/s)') + ax.set_ylabel('MB/s') + ax.legend() + ax.grid(True, alpha=0.3) + self._add_phase_lines(ax, phase_boundaries) + + # Threads & Files + ax = axes[2, 1] + ax2 = ax.twinx() + + ax.plot(timestamps, [s.threads for s in all_snapshots], + color='brown', alpha=0.7, linewidth=1, label='Threads') + ax2.plot(timestamps, [s.open_files for s in all_snapshots], + color='gray', alpha=0.7, linewidth=1, label='Open Files') + + ax.set_title('Threads & Open Files') + ax.set_ylabel('Threads', color='brown') + ax2.set_ylabel('Open Files', color='gray') + ax.grid(True, alpha=0.3) + + lines1, labels1 = ax.get_legend_handles_labels() + lines2, labels2 = ax2.get_legend_handles_labels() + ax.legend(lines1 + lines2, labels1 + labels2) + + self._add_phase_lines(ax, phase_boundaries) + + plt.tight_layout() + + # Сохраняем + chart_file = f"test_results_{datetime.now().strftime('%Y%m%d_%H%M%S')}.png" + plt.savefig(chart_file, dpi=150, bbox_inches='tight') + plt.close() + + logger.info(f"✅ Графики сохранены: {chart_file}") + + except Exception as e: + logger.error(f"❌ Ошибка создания графиков: {e}") + + def _add_phase_lines(self, ax, boundaries): + """Добавление линий фаз на график.""" + for boundary in boundaries: + ax.axvline(x=boundary, color='black', linestyle='--', alpha=0.3) + + def _save_results_json(self): + """Сохранение результатов в JSON.""" + try: + results = { + "test_time": datetime.now().isoformat(), + "system_info": { + "cpu_count": psutil.cpu_count(), + "total_memory_gb": psutil.virtual_memory().total / (1024**3), + }, + "phases": [] + } + + for phase in self.phases: + phase_data = { + "name": phase.name, + "duration_seconds": phase.duration, + "avg_cpu_percent": phase.avg_cpu, + "avg_memory_mb": phase.avg_memory_mb, + "peak_memory_mb": phase.peak_memory_mb, + "snapshots_count": len(phase.snapshots), + "memory_timeline": [ + { + "time": s.timestamp, + "cpu": s.cpu_percent, + "memory_mb": s.process_memory_mb + } + for s in phase.snapshots[::5] # Каждый 5-й для компактности + ] + } + results["phases"].append(phase_data) + + json_file = f"test_results_{datetime.now().strftime('%Y%m%d_%H%M%S')}.json" + with open(json_file, 'w', encoding='utf-8') as f: + json.dump(results, f, indent=2, ensure_ascii=False) + + logger.info(f"✅ JSON результаты сохранены: {json_file}") + + except Exception as e: + logger.error(f"❌ Ошибка сохранения JSON: {e}") + + +async def run_memory_test(): + """ + Специальный тест для измерения потребления памяти. + Показывает сколько памяти используется в каждый момент. + """ + runner = ResourceTestRunner() + await runner.setup() + + print("\n" + "="*60) + print("🧠 ТЕСТ ПОТРЕБЛЕНИЯ ПАМЯТИ") + print("="*60) + + # Тест 1: Базовое потребление + print("\n1️⃣ Базовое потребление (без браузеров):") + for i in range(5): + mem = runner._get_current_memory() + print(f" {i+1}. {mem:.1f} MB") + await asyncio.sleep(1) + + # Тест 2: С одним браузером + print("\n2️⃣ Запуск браузера и посещение:") + for i in range(3): + mem_before = runner._get_current_memory() + print(f" До посещения: {mem_before:.1f} MB") + + result = await runner.browser_service.visit_page( + "https://httpbin.org/headers", 5 + ) + + mem_after = runner._get_current_memory() + print(f" После посещения: {mem_after:.1f} MB " + f"(Δ = {mem_after - mem_before:+.1f} MB)") + + await asyncio.sleep(3) + + # Тест 3: Параллельные браузеры + print("\n3️⃣ Параллельные браузеры:") + for count in [1, 2, 3, 5]: + mem_before = runner._get_current_memory() + + async def visit_page(idx): + return await runner.browser_service.visit_page( + "https://httpbin.org/headers", 3 + ) + + tasks = [visit_page(i) for i in range(count)] + await asyncio.gather(*tasks) + + await asyncio.sleep(2) + mem_during = runner._get_current_memory() + print(f" {count} браузеров: было {mem_before:.1f} MB, " + f"стало {mem_during:.1f} MB " + f"(Δ = {mem_during - mem_before:+.1f} MB)") + + await asyncio.sleep(5) # Ждем освобождения памяти + + print("\n✅ Тест памяти завершен") + + +async def main(): + """Точка входа.""" + import argparse + + parser = argparse.ArgumentParser(description="Тест с мониторингом ресурсов") + parser.add_argument("--url", type=str, help="URL для тестирования") + parser.add_argument("--quick", action="store_true", help="Быстрый тест") + parser.add_argument("--memory", action="store_true", help="Тест только памяти") + parser.add_argument("--full", action="store_true", help="Полный тест") + + args = parser.parse_args() + + try: + if args.memory: + await run_memory_test() + elif args.quick: + runner = ResourceTestRunner() + await runner.setup() + await runner.test_single_visit( + args.url or "https://httpbin.org/headers", 10 + ) + runner._print_results() + runner._generate_charts() + else: + # Полный тест по умолчанию + runner = ResourceTestRunner() + await runner.run_comprehensive_test(args.url) + + except KeyboardInterrupt: + print("\n⚠️ Тест прерван") + except Exception as e: + logger.error(f"Ошибка: {e}", exc_info=True) + + +if __name__ == "__main__": + print(""" +╔══════════════════════════════════════════════════════════════╗ +║ 🧪 ТЕСТ С МОНИТОРИНГОМ РЕСУРСОВ (CPU, RAM, DISK) ║ +╚══════════════════════════════════════════════════════════════╝ + """) + + asyncio.run(main()) \ No newline at end of file diff --git a/twitch_bot.session b/twitch_bot.session new file mode 100644 index 0000000..32b9419 Binary files /dev/null and b/twitch_bot.session differ diff --git a/utils/__init__.py b/utils/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/utils/helpers.py b/utils/helpers.py new file mode 100644 index 0000000..3e84b68 --- /dev/null +++ b/utils/helpers.py @@ -0,0 +1,45 @@ +"""Utility helper functions.""" + +from typing import Tuple +from config.settings import settings + + +def parse_range(value: str, default_min: int = 5) -> tuple: + """Парсит строку диапазона 'мин-макс' или одиночное число.""" + # Очищаем от кавычек и пробелов + value = str(value).replace("'", "").replace('"', "").replace("`", "").strip() + + if '-' in value: + parts = value.split('-') + if len(parts) != 2: + raise ValueError(f"Неверный формат: {value}. Пример: 10-30") + try: + min_val = int(parts[0].strip()) + max_val = int(parts[1].strip()) + except ValueError: + raise ValueError(f"Не числа: {value}. Пример: 10-30") + + if min_val < default_min: + raise ValueError(f"Минимум: {default_min}") + if min_val >= max_val: + raise ValueError("Мин должно быть меньше макс") + return min_val, max_val + else: + try: + val = int(value) + except ValueError: + raise ValueError(f"Не число: {value}") + if val < default_min: + raise ValueError(f"Минимум: {default_min}") + return val, val + + + +def format_range(min_val: int, max_val: int) -> str: + """Format range for display.""" + return f"{min_val}-{max_val}" if min_val != max_val else str(min_val) + + +def extract_domain(url: str) -> str: + """Extract domain from URL.""" + return url.replace("https://", "").replace("http://", "").split("/")[0] \ No newline at end of file diff --git a/utils/telegram.py b/utils/telegram.py new file mode 100644 index 0000000..885188b --- /dev/null +++ b/utils/telegram.py @@ -0,0 +1,83 @@ +"""Telegram message utilities.""" + +import logging +import os +from typing import Optional + +from aiogram import Bot +from aiogram.types import FSInputFile + +logger = logging.getLogger(__name__) + + +async def send_message_safe( + bot: Optional[Bot], + chat_id: Optional[int], + text: str, + **kwargs +) -> None: + """ + Safely send message to Telegram. + + Args: + bot: Bot instance + chat_id: Target chat ID + text: Message text + **kwargs: Additional arguments for send_message + """ + if not bot or not chat_id: + return + + try: + await bot.send_message(chat_id=chat_id, text=text, **kwargs) + except Exception as e: + logger.error(f"Failed to send message: {e}") + + +async def send_visit_result( + bot: Optional[Bot], + chat_id: int, + visit_num: int, + result: dict, + delay: int, + reading: int, + successful: int, + failed: int +) -> None: + """ + Send visit result with optional screenshot. + + Args: + bot: Bot instance + chat_id: Target chat ID + visit_num: Visit number + result: Visit result dictionary + delay: Next visit delay + reading: Reading time + successful: Count of successful visits + failed: Count of failed visits + """ + if not bot: + return + + try: + text = ( + f"{'✅' if result['success'] else '❌'} **Посещение {visit_num}**\n" + f"🌐 `{result.get('final_url', 'N/A')[:60]}`\n" + f"📖 Чтение: {reading}с | След. через: {delay}с\n" + f"📊 Успешно: {successful} | Ошибок: {failed}" + ) + + screenshot_path = result.get('screenshot_path') + if screenshot_path and os.path.exists(screenshot_path): + photo = FSInputFile(screenshot_path) + await bot.send_photo(chat_id=chat_id, photo=photo, caption=text) + else: + await bot.send_message(chat_id=chat_id, text=text) + + except Exception as e: + logger.error(f"Failed to send visit result: {e}") + try: + await bot.send_message(chat_id=chat_id, text=text) + except Exception as e2: + logger.warning(f"Fallback send failed: {e2}") \ No newline at end of file