From efce7bbc0bfae84f9f2b2713dcc2f7a863bdaf93 Mon Sep 17 00:00:00 2001 From: thsrite Date: Tue, 21 Nov 2023 20:39:28 +0800 Subject: [PATCH] =?UTF-8?q?feat=20=E7=AB=99=E7=82=B9=E6=95=B0=E6=8D=AE?= =?UTF-8?q?=E7=BB=9F=E8=AE=A1=E6=8F=92=E4=BB=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- plugins/sitestatisticnomsg/__init__.py | 1148 +++++++++++++++++ .../siteuserinfo/__init__.py | 338 +++++ 2 files changed, 1486 insertions(+) create mode 100644 plugins/sitestatisticnomsg/__init__.py create mode 100644 plugins/sitestatisticnomsg/siteuserinfo/__init__.py diff --git a/plugins/sitestatisticnomsg/__init__.py b/plugins/sitestatisticnomsg/__init__.py new file mode 100644 index 0000000..76b5db9 --- /dev/null +++ b/plugins/sitestatisticnomsg/__init__.py @@ -0,0 +1,1148 @@ +import re +import warnings +from datetime import datetime, timedelta +from multiprocessing.dummy import Pool as ThreadPool +from threading import Lock +from typing import Optional, Any, List, Dict, Tuple + +import pytz +import requests +from apscheduler.schedulers.background import BackgroundScheduler +from apscheduler.triggers.cron import CronTrigger +from ruamel.yaml import CommentedMap + +from app import schemas +from app.core.config import settings +from app.core.event import Event +from app.core.event import eventmanager +from app.db.site_oper import SiteOper +from app.helper.browser import PlaywrightHelper +from app.helper.module import ModuleHelper +from app.helper.sites import SitesHelper +from app.log import logger +from app.plugins import _PluginBase +from app.plugins.sitestatistic.siteuserinfo import ISiteUserInfo +from app.schemas.types import EventType, NotificationType +from app.utils.http import RequestUtils +from app.utils.string import StringUtils +from app.utils.timer import TimerUtils + +warnings.filterwarnings("ignore", category=FutureWarning) + +lock = Lock() + + +class SiteStatisticNoMsg(_PluginBase): + # 插件名称 + plugin_name = "站点数据统计" + # 插件描述 + plugin_desc = "自动统计和展示站点数据（无站点未读消息）。" + # 插件图标 + plugin_icon = "statistic.png" + # 主题色 + plugin_color = "#324A5E" + # 插件版本 + plugin_version = "1.0" + # 插件作者 + plugin_author = "lightolly" + # 作者主页 + author_url = "https://github.com/lightolly" + # 插件配置项ID前缀 + plugin_config_prefix = "sitestatisticnomsg_" + # 加载顺序 + plugin_order = 1 + # 可使用的用户级别 + auth_level = 2 + + # 私有属性 + sites = None + siteoper = None + _scheduler: Optional[BackgroundScheduler] = None + _last_update_time: Optional[datetime] = None + _sites_data: dict = {} + _site_schema: List[ISiteUserInfo] = None + + # 配置属性 + _enabled: bool = False + _onlyonce: bool = False + _cron: str = "" + _notify: bool = False + _queue_cnt: int = 5 + _statistic_type: str = None + _statistic_sites: list = [] + + def init_plugin(self, config: dict = None): + self.sites = SitesHelper() + self.siteoper = SiteOper() + # 停止现有任务 + self.stop_service() + + # 配置 + if config: + self._enabled = config.get("enabled") + self._onlyonce = config.get("onlyonce") + self._cron = config.get("cron") + self._notify = config.get("notify") + self._queue_cnt = config.get("queue_cnt") + self._statistic_type = config.get("statistic_type") or "all" + self._statistic_sites = config.get("statistic_sites") or [] + + # 过滤掉已删除的站点 + all_sites = [site for site in self.sites.get_indexers() if not site.get("public")] + self.__custom_sites() + self._statistic_sites = [site.get("id") for site in all_sites if + not site.get("public") and site.get("id") in self._statistic_sites] + self.__update_config() + + if self._enabled or self._onlyonce: + # 加载模块 + self._site_schema = ModuleHelper.load('app.plugins.sitestatisticnomsg.siteuserinfo', + filter_func=lambda _, obj: hasattr(obj, 'schema')) + + # 定时服务 + self._scheduler = BackgroundScheduler(timezone=settings.TZ) + + self._site_schema.sort(key=lambda x: x.order) + # 站点上一次更新时间 + self._last_update_time = None + # 站点数据 + self._sites_data = {} + + # 立即运行一次 + if self._onlyonce: + logger.info(f"站点数据统计服务启动，立即运行一次") + self._scheduler.add_job(self.refresh_all_site_data, 'date', + run_date=datetime.now( + tz=pytz.timezone(settings.TZ)) + timedelta(seconds=3) + ) + # 关闭一次性开关 + self._onlyonce = False + + # 保存配置 + self.__update_config() + + # 周期运行 + if self._enabled and self._cron: + try: + self._scheduler.add_job(func=self.refresh_all_site_data, + trigger=CronTrigger.from_crontab(self._cron), + name="站点数据统计") + except Exception as err: + logger.error(f"定时任务配置错误：{err}") + # 推送实时消息 + self.systemmessage.put(f"执行周期配置错误：{err}") + else: + triggers = TimerUtils.random_scheduler(num_executions=1, + begin_hour=0, + end_hour=1, + min_interval=1, + max_interval=60) + for trigger in triggers: + self._scheduler.add_job(self.refresh_all_site_data, "cron", + hour=trigger.hour, minute=trigger.minute, + name="站点数据统计") + + # 启动任务 + if self._scheduler.get_jobs(): + self._scheduler.print_jobs() + self._scheduler.start() + + def get_state(self) -> bool: + return self._enabled + + @staticmethod + def get_command() -> List[Dict[str, Any]]: + """ + 定义远程控制命令 + :return: 命令关键字、事件、描述、附带数据 + """ + return [{ + "cmd": "/site_statistic", + "event": EventType.SiteStatistic, + "desc": "站点数据统计", + "category": "站点", + "data": {} + }] + + def get_api(self) -> List[Dict[str, Any]]: + """ + 获取插件API + [{ + "path": "/xx", + "endpoint": self.xxx, + "methods": ["GET", "POST"], + "summary": "API说明" + }] + """ + return [{ + "path": "/refresh_by_domain", + "endpoint": self.refresh_by_domain, + "methods": ["GET"], + "summary": "刷新站点数据", + "description": "刷新对应域名的站点数据", + }] + + def get_form(self) -> Tuple[List[dict], Dict[str, Any]]: + """ + 拼装插件配置页面，需要返回两块数据：1、页面配置；2、数据结构 + """ + # 站点的可选项（内置站点 + 自定义站点） + customSites = self.__custom_sites() + + site_options = ([{"title": site.name, "value": site.id} + for site in self.siteoper.list_order_by_pri()] + + [{"title": site.get("name"), "value": site.get("id")} + for site in customSites]) + + return [ + { + 'component': 'VForm', + 'content': [ + { + 'component': 'VRow', + 'content': [ + { + 'component': 'VCol', + 'props': { + 'cols': 12, + 'md': 4 + }, + 'content': [ + { + 'component': 'VSwitch', + 'props': { + 'model': 'enabled', + 'label': '启用插件', + } + } + ] + }, + { + 'component': 'VCol', + 'props': { + 'cols': 12, + 'md': 4 + }, + 'content': [ + { + 'component': 'VSwitch', + 'props': { + 'model': 'notify', + 'label': '发送通知', + } + } + ] + }, + { + 'component': 'VCol', + 'props': { + 'cols': 12, + 'md': 4 + }, + 'content': [ + { + 'component': 'VSwitch', + 'props': { + 'model': 'onlyonce', + 'label': '立即运行一次', + } + } + ] + } + ] + }, + { + 'component': 'VRow', + 'content': [ + { + 'component': 'VCol', + 'props': { + 'cols': 12, + 'md': 4 + }, + 'content': [ + { + 'component': 'VTextField', + 'props': { + 'model': 'cron', + 'label': '执行周期', + 'placeholder': '5位cron表达式，留空自动' + } + } + ] + }, + { + 'component': 'VCol', + 'props': { + 'cols': 12, + 'md': 4 + }, + 'content': [ + { + 'component': 'VTextField', + 'props': { + 'model': 'queue_cnt', + 'label': '队列数量' + } + } + ] + }, + { + 'component': 'VCol', + 'props': { + 'cols': 12, + 'md': 4 + }, + 'content': [ + { + 'component': 'VSelect', + 'props': { + 'model': 'statistic_type', + 'label': '统计类型', + 'items': [ + {'title': '全量', 'value': 'all'}, + {'title': '增量', 'value': 'add'} + ] + } + } + ] + } + ] + }, + { + 'component': 'VRow', + 'content': [ + { + 'component': 'VCol', + 'content': [ + { + 'component': 'VSelect', + 'props': { + 'chips': True, + 'multiple': True, + 'model': 'statistic_sites', + 'label': '统计站点', + 'items': site_options + } + } + ] + } + ] + } + ] + } + ], { + "enabled": False, + "onlyonce": False, + "notify": True, + "cron": "5 1 * * *", + "queue_cnt": 5, + "statistic_type": "all", + "statistic_sites": [] + } + + def get_page(self) -> List[dict]: + """ + 拼装插件详情页面，需要返回页面配置，同时附带数据 + """ + # + # 最近两天的日期数组 + date_list = [(datetime.now() - timedelta(days=i)).date() for i in range(2)] + # 最近一天的签到数据 + stattistic_data: Dict[str, Dict[str, Any]] = {} + for day in date_list: + current_day = day.strftime("%Y-%m-%d") + stattistic_data = self.get_data(current_day) + if stattistic_data: + break + if not stattistic_data: + return [ + { + 'component': 'div', + 'text': '暂无数据', + 'props': { + 'class': 'text-center', + } + } + ] + # 数据按时间降序排序 + stattistic_data = dict(sorted(stattistic_data.items(), + key=lambda item: item[1].get('upload') or 0, + reverse=True)) + # 总上传量 + total_upload = sum([data.get("upload") + for data in stattistic_data.values() if data.get("upload")]) + # 总下载量 + total_download = sum([data.get("download") + for data in stattistic_data.values() if data.get("download")]) + # 总做种数 + total_seed = sum([data.get("seeding") + for data in stattistic_data.values() if data.get("seeding")]) + # 总做种体积 + total_seed_size = sum([data.get("seeding_size") + for data in stattistic_data.values() if data.get("seeding_size")]) + + # 站点数据明细 + site_trs = [ + { + 'component': 'tr', + 'props': { + 'class': 'text-sm' + }, + 'content': [ + { + 'component': 'td', + 'props': { + 'class': 'whitespace-nowrap break-keep text-high-emphasis' + }, + 'text': site + }, + { + 'component': 'td', + 'text': data.get("username") + }, + { + 'component': 'td', + 'text': data.get("user_level") + }, + { + 'component': 'td', + 'props': { + 'class': 'text-success' + }, + 'text': StringUtils.str_filesize(data.get("upload")) + }, + { + 'component': 'td', + 'props': { + 'class': 'text-error' + }, + 'text': StringUtils.str_filesize(data.get("download")) + }, + { + 'component': 'td', + 'text': data.get('ratio') + }, + { + 'component': 'td', + 'text': '{:,.1f}'.format(data.get('bonus') or 0) + }, + { + 'component': 'td', + 'text': data.get('seeding') + }, + { + 'component': 'td', + 'text': StringUtils.str_filesize(data.get('seeding_size')) + } + ] + } for site, data in stattistic_data.items() if not data.get("err_msg") + ] + + # 拼装页面 + return [ + { + 'component': 'VRow', + 'content': [ + # 总上传量 + { + 'component': 'VCol', + 'props': { + 'cols': 12, + 'md': 3, + 'sm': 6 + }, + 'content': [ + { + 'component': 'VCard', + 'props': { + 'variant': 'tonal', + }, + 'content': [ + { + 'component': 'VCardText', + 'props': { + 'class': 'd-flex align-center', + }, + 'content': [ + { + 'component': 'VAvatar', + 'props': { + 'rounded': True, + 'variant': 'text', + 'class': 'me-3' + }, + 'content': [ + { + 'component': 'VImg', + 'props': { + 'src': '/plugin_icon/upload.png' + } + } + ] + }, + { + 'component': 'div', + 'content': [ + { + 'component': 'span', + 'props': { + 'class': 'text-caption' + }, + 'text': '总上传量' + }, + { + 'component': 'div', + 'props': { + 'class': 'd-flex align-center flex-wrap' + }, + 'content': [ + { + 'component': 'span', + 'props': { + 'class': 'text-h6' + }, + 'text': StringUtils.str_filesize(total_upload) + } + ] + } + ] + } + ] + } + ] + }, + ] + }, + # 总下载量 + { + 'component': 'VCol', + 'props': { + 'cols': 12, + 'md': 3, + 'sm': 6 + }, + 'content': [ + { + 'component': 'VCard', + 'props': { + 'variant': 'tonal', + }, + 'content': [ + { + 'component': 'VCardText', + 'props': { + 'class': 'd-flex align-center', + }, + 'content': [ + { + 'component': 'VAvatar', + 'props': { + 'rounded': True, + 'variant': 'text', + 'class': 'me-3' + }, + 'content': [ + { + 'component': 'VImg', + 'props': { + 'src': '/plugin_icon/download.png' + } + } + ] + }, + { + 'component': 'div', + 'content': [ + { + 'component': 'span', + 'props': { + 'class': 'text-caption' + }, + 'text': '总下载量' + }, + { + 'component': 'div', + 'props': { + 'class': 'd-flex align-center flex-wrap' + }, + 'content': [ + { + 'component': 'span', + 'props': { + 'class': 'text-h6' + }, + 'text': StringUtils.str_filesize(total_download) + } + ] + } + ] + } + ] + } + ] + }, + ] + }, + # 总做种数 + { + 'component': 'VCol', + 'props': { + 'cols': 12, + 'md': 3, + 'sm': 6 + }, + 'content': [ + { + 'component': 'VCard', + 'props': { + 'variant': 'tonal', + }, + 'content': [ + { + 'component': 'VCardText', + 'props': { + 'class': 'd-flex align-center', + }, + 'content': [ + { + 'component': 'VAvatar', + 'props': { + 'rounded': True, + 'variant': 'text', + 'class': 'me-3' + }, + 'content': [ + { + 'component': 'VImg', + 'props': { + 'src': '/plugin_icon/seed.png' + } + } + ] + }, + { + 'component': 'div', + 'content': [ + { + 'component': 'span', + 'props': { + 'class': 'text-caption' + }, + 'text': '总做种数' + }, + { + 'component': 'div', + 'props': { + 'class': 'd-flex align-center flex-wrap' + }, + 'content': [ + { + 'component': 'span', + 'props': { + 'class': 'text-h6' + }, + 'text': f'{"{:,}".format(total_seed)}' + } + ] + } + ] + } + ] + } + ] + }, + ] + }, + # 总做种体积 + { + 'component': 'VCol', + 'props': { + 'cols': 12, + 'md': 3, + 'sm': 6 + }, + 'content': [ + { + 'component': 'VCard', + 'props': { + 'variant': 'tonal', + }, + 'content': [ + { + 'component': 'VCardText', + 'props': { + 'class': 'd-flex align-center', + }, + 'content': [ + { + 'component': 'VAvatar', + 'props': { + 'rounded': True, + 'variant': 'text', + 'class': 'me-3' + }, + 'content': [ + { + 'component': 'VImg', + 'props': { + 'src': '/plugin_icon/database.png' + } + } + ] + }, + { + 'component': 'div', + 'content': [ + { + 'component': 'span', + 'props': { + 'class': 'text-caption' + }, + 'text': '总做种体积' + }, + { + 'component': 'div', + 'props': { + 'class': 'd-flex align-center flex-wrap' + }, + 'content': [ + { + 'component': 'span', + 'props': { + 'class': 'text-h6' + }, + 'text': StringUtils.str_filesize(total_seed_size) + } + ] + } + ] + } + ] + } + ] + } + ] + }, + # 各站点数据明细 + { + 'component': 'VCol', + 'props': { + 'cols': 12, + }, + 'content': [ + { + 'component': 'VTable', + 'props': { + 'hover': True + }, + 'content': [ + { + 'component': 'thead', + 'content': [ + { + 'component': 'th', + 'props': { + 'class': 'text-start ps-4' + }, + 'text': '站点' + }, + { + 'component': 'th', + 'props': { + 'class': 'text-start ps-4' + }, + 'text': '用户名' + }, + { + 'component': 'th', + 'props': { + 'class': 'text-start ps-4' + }, + 'text': '用户等级' + }, + { + 'component': 'th', + 'props': { + 'class': 'text-start ps-4' + }, + 'text': '上传量' + }, + { + 'component': 'th', + 'props': { + 'class': 'text-start ps-4' + }, + 'text': '下载量' + }, + { + 'component': 'th', + 'props': { + 'class': 'text-start ps-4' + }, + 'text': '分享率' + }, + { + 'component': 'th', + 'props': { + 'class': 'text-start ps-4' + }, + 'text': '魔力值' + }, + { + 'component': 'th', + 'props': { + 'class': 'text-start ps-4' + }, + 'text': '做种数' + }, + { + 'component': 'th', + 'props': { + 'class': 'text-start ps-4' + }, + 'text': '做种体积' + } + ] + }, + { + 'component': 'tbody', + 'content': site_trs + } + ] + } + ] + } + ] + } + ] + + def stop_service(self): + """ + 退出插件 + """ + try: + if self._scheduler: + self._scheduler.remove_all_jobs() + if self._scheduler.running: + self._scheduler.shutdown() + self._scheduler = None + except Exception as e: + logger.error("退出插件失败：%s" % str(e)) + + def __build_class(self, html_text: str) -> Any: + for site_schema in self._site_schema: + try: + if site_schema.match(html_text): + return site_schema + except Exception as e: + logger.error(f"站点匹配失败 {e}") + return None + + def build(self, site_info: CommentedMap) -> Optional[ISiteUserInfo]: + """ + 构建站点信息 + """ + site_cookie = site_info.get("cookie") + if not site_cookie: + return None + site_name = site_info.get("name") + url = site_info.get("url") + proxy = site_info.get("proxy") + ua = site_info.get("ua") + # 会话管理 + with requests.Session() as session: + proxies = settings.PROXY if proxy else None + proxy_server = settings.PROXY_SERVER if proxy else None + render = site_info.get("render") + + logger.debug(f"站点 {site_name} url={url} site_cookie={site_cookie} ua={ua}") + if render: + # 演染模式 + html_text = PlaywrightHelper().get_page_source(url=url, + cookies=site_cookie, + ua=ua, + proxies=proxy_server) + else: + # 普通模式 + res = RequestUtils(cookies=site_cookie, + session=session, + ua=ua, + proxies=proxies + ).get_res(url=url) + if res and res.status_code == 200: + if re.search(r"charset=\"?utf-8\"?", res.text, re.IGNORECASE): + res.encoding = "utf-8" + else: + res.encoding = res.apparent_encoding + html_text = res.text + # 第一次登录反爬 + if html_text.find("title") == -1: + i = html_text.find("window.location") + if i == -1: + return None + tmp_url = url + html_text[i:html_text.find(";")] \ + .replace("\"", "") \ + .replace("+", "") \ + .replace(" ", "") \ + .replace("window.location=", "") + res = RequestUtils(cookies=site_cookie, + session=session, + ua=ua, + proxies=proxies + ).get_res(url=tmp_url) + if res and res.status_code == 200: + if "charset=utf-8" in res.text or "charset=UTF-8" in res.text: + res.encoding = "UTF-8" + else: + res.encoding = res.apparent_encoding + html_text = res.text + if not html_text: + return None + else: + logger.error("站点 %s 被反爬限制：%s, 状态码：%s" % (site_name, url, res.status_code)) + return None + + # 兼容假首页情况，假首页通常没有 schemas.Response: + """ + 刷新一个站点数据，可由API调用 + """ + site_info = self.sites.get_indexer(domain) + if site_info: + site_data = self.__refresh_site_data(site_info) + if site_data: + return schemas.Response( + success=True, + message=f"站点 {domain} 刷新成功", + data=site_data.to_dict() + ) + return schemas.Response( + success=False, + message=f"站点 {domain} 刷新数据失败，未获取到数据" + ) + return schemas.Response( + success=False, + message=f"站点 {domain} 不存在" + ) + + def __refresh_site_data(self, site_info: CommentedMap) -> Optional[ISiteUserInfo]: + """ + 更新单个site 数据信息 + :param site_info: + :return: + """ + site_name = site_info.get('name') + site_url = site_info.get('url') + if not site_url: + return None + try: + site_user_info: ISiteUserInfo = self.build(site_info=site_info) + if site_user_info: + logger.debug(f"站点 {site_name} 开始以 {site_user_info.site_schema()} 模型解析") + # 开始解析 + site_user_info.parse() + logger.debug(f"站点 {site_name} 解析完成") + + # 获取不到数据时，仅返回错误信息，不做历史数据更新 + if site_user_info.err_msg: + self._sites_data.update({site_name: {"err_msg": site_user_info.err_msg}}) + return None + + # 分享率接近1时，发送消息提醒 + if site_user_info.ratio and float(site_user_info.ratio) < 1: + self.post_message(mtype=NotificationType.SiteMessage, + title=f"【站点分享率低预警】", + text=f"站点 {site_user_info.site_name} 分享率 {site_user_info.ratio}，请注意！") + + self._sites_data.update( + { + site_name: { + "upload": site_user_info.upload, + "username": site_user_info.username, + "user_level": site_user_info.user_level, + "join_at": site_user_info.join_at, + "download": site_user_info.download, + "ratio": site_user_info.ratio, + "seeding": site_user_info.seeding, + "seeding_size": site_user_info.seeding_size, + "leeching": site_user_info.leeching, + "bonus": site_user_info.bonus, + "url": site_url, + "err_msg": site_user_info.err_msg, + "message_unread": site_user_info.message_unread + } + }) + return site_user_info + + except Exception as e: + logger.error(f"站点 {site_name} 获取流量数据失败：{str(e)}") + return None + + @eventmanager.register(EventType.SiteStatistic) + def refresh(self, event: Event): + """ + 刷新站点数据 + """ + if event: + logger.info("收到命令，开始刷新站点数据 ...") + self.post_message(channel=event.event_data.get("channel"), + title="开始刷新站点数据 ...", + userid=event.event_data.get("user")) + self.refresh_all_site_data() + if event: + self.post_message(channel=event.event_data.get("channel"), + title="站点数据刷新完成！", userid=event.event_data.get("user")) + + def refresh_all_site_data(self): + """ + 多线程刷新站点下载上传量，默认间隔6小时 + """ + if not self.sites.get_indexers(): + return + + logger.info("开始刷新站点数据 ...") + + with lock: + + all_sites = [site for site in self.sites.get_indexers() if not site.get("public")] + self.__custom_sites() + # 没有指定站点，默认使用全部站点 + if not self._statistic_sites: + refresh_sites = all_sites + else: + refresh_sites = [site for site in all_sites if + site.get("id") in self._statistic_sites] + if not refresh_sites: + return + + # 并发刷新 + with ThreadPool(min(len(refresh_sites), int(self._queue_cnt or 5))) as p: + p.map(self.__refresh_site_data, refresh_sites) + + # 通知刷新完成 + if self._notify: + yesterday_sites_data = {} + # 增量数据 + if self._statistic_type == "add": + last_update_time = self.get_data("last_update_time") + if last_update_time: + yesterday_sites_data = self.get_data(last_update_time) or {} + + messages = [] + # 按照上传降序排序 + sites = self._sites_data.keys() + uploads = [self._sites_data[site].get("upload") or 0 if not yesterday_sites_data.get(site) else + (self._sites_data[site].get("upload") or 0) - ( + yesterday_sites_data[site].get("upload") or 0) for site in sites] + downloads = [self._sites_data[site].get("download") or 0 if not yesterday_sites_data.get(site) else + (self._sites_data[site].get("download") or 0) - ( + yesterday_sites_data[site].get("download") or 0) for site in sites] + data_list = sorted(list(zip(sites, uploads, downloads)), + key=lambda x: x[1], + reverse=True) + # 总上传 + incUploads = 0 + # 总下载 + incDownloads = 0 + for data in data_list: + site = data[0] + upload = int(data[1]) + download = int(data[2]) + if upload > 0 or download > 0: + incUploads += int(upload) + incDownloads += int(download) + messages.append(f"【{site}】\n" + f"上传量：{StringUtils.str_filesize(upload)}\n" + f"下载量：{StringUtils.str_filesize(download)}\n" + f"————————————") + + if incDownloads or incUploads: + messages.insert(0, f"【汇总】\n" + f"总上传：{StringUtils.str_filesize(incUploads)}\n" + f"总下载：{StringUtils.str_filesize(incDownloads)}\n" + f"————————————") + self.post_message(mtype=NotificationType.SiteMessage, + title="站点数据统计", text="\n".join(messages)) + + # 获取今天的日期 + key = datetime.now().strftime('%Y-%m-%d') + # 保存数据 + self.save_data(key, self._sites_data) + + # 更新时间 + self.save_data("last_update_time", key) + logger.info("站点数据刷新完成") + + def __custom_sites(self) -> List[Any]: + custom_sites = [] + custom_sites_config = self.get_config("CustomSites") + if custom_sites_config and custom_sites_config.get("enabled"): + custom_sites = custom_sites_config.get("sites") + return custom_sites + + def __update_config(self): + self.update_config({ + "enabled": self._enabled, + "onlyonce": self._onlyonce, + "cron": self._cron, + "notify": self._notify, + "queue_cnt": self._queue_cnt, + "statistic_type": self._statistic_type, + "statistic_sites": self._statistic_sites, + }) + + @eventmanager.register(EventType.SiteDeleted) + def site_deleted(self, event): + """ + 删除对应站点选中 + """ + site_id = event.event_data.get("site_id") + config = self.get_config() + if config: + statistic_sites = config.get("statistic_sites") + if statistic_sites: + if isinstance(statistic_sites, str): + statistic_sites = [statistic_sites] + + # 删除对应站点 + if site_id: + statistic_sites = [site for site in statistic_sites if int(site) != int(site_id)] + else: + # 清空 + statistic_sites = [] + + # 若无站点，则停止 + if len(statistic_sites) == 0: + self._enabled = False + + self._statistic_sites = statistic_sites + # 保存配置 + self.__update_config() \ No newline at end of file diff --git a/plugins/sitestatisticnomsg/siteuserinfo/__init__.py b/plugins/sitestatisticnomsg/siteuserinfo/__init__.py new file mode 100644 index 0000000..e8e218f --- /dev/null +++ b/plugins/sitestatisticnomsg/siteuserinfo/__init__.py @@ -0,0 +1,338 @@ +# -*- coding: utf-8 -*- +import json +import re +from abc import ABCMeta, abstractmethod +from enum import Enum +from typing import Optional +from urllib.parse import urljoin, urlsplit + +from requests import Session + +from app.core.config import settings +from app.helper.cloudflare import under_challenge +from app.log import logger +from app.utils.http import RequestUtils +from app.utils.site import SiteUtils + +SITE_BASE_ORDER = 1000 + + +# 站点框架 +class SiteSchema(Enum): + DiscuzX = "Discuz!" + Gazelle = "Gazelle" + Ipt = "IPTorrents" + NexusPhp = "NexusPhp" + NexusProject = "NexusProject" + NexusRabbit = "NexusRabbit" + NexusHhanclub = "NexusHhanclub" + SmallHorse = "Small Horse" + Unit3d = "Unit3d" + TorrentLeech = "TorrentLeech" + FileList = "FileList" + TNode = "TNode" + + +class ISiteUserInfo(metaclass=ABCMeta): + # 站点模版 + schema = SiteSchema.NexusPhp + # 站点解析时判断顺序，值越小越先解析 + order = SITE_BASE_ORDER + + def __init__(self, site_name: str, + url: str, + site_cookie: str, + index_html: str, + session: Session = None, + ua: str = None, + emulate: bool = False, + proxy: bool = None): + super().__init__() + # 站点信息 + self.site_name = None + self.site_url = None + # 用户信息 + self.username = None + self.userid = None + # 未读消息 + self.message_unread = 0 + self.message_unread_contents = [] + + # 流量信息 + self.upload = 0 + self.download = 0 + self.ratio = 0 + + # 种子信息 + self.seeding = 0 + self.leeching = 0 + self.uploaded = 0 + self.completed = 0 + self.incomplete = 0 + self.seeding_size = 0 + self.leeching_size = 0 + self.uploaded_size = 0 + self.completed_size = 0 + self.incomplete_size = 0 + # 做种人数, 种子大小 + self.seeding_info = [] + + # 用户详细信息 + self.user_level = None + self.join_at = None + self.bonus = 0.0 + + # 错误信息 + self.err_msg = None + # 内部数据 + self._base_url = None + self._site_cookie = None + self._index_html = None + self._addition_headers = None + + # 站点页面 + self._brief_page = "index.php" + self._user_detail_page = "userdetails.php?id=" + self._user_traffic_page = "index.php" + self._torrent_seeding_page = "getusertorrentlistajax.php?userid=" + self._user_mail_unread_page = "messages.php?action=viewmailbox&box=1&unread=yes" + self._sys_mail_unread_page = "messages.php?action=viewmailbox&box=-2&unread=yes" + self._torrent_seeding_params = None + self._torrent_seeding_headers = None + + split_url = urlsplit(url) + self.site_name = site_name + self.site_url = url + self._base_url = f"{split_url.scheme}://{split_url.netloc}" + self._site_cookie = site_cookie + self._index_html = index_html + self._session = session if session else None + self._ua = ua + + self._emulate = emulate + self._proxy = proxy + + def site_schema(self) -> SiteSchema: + """ + 站点解析模型 + :return: 站点解析模型 + """ + return self.schema + + @classmethod + def match(cls, html_text: str) -> bool: + """ + 是否匹配当前解析模型 + :param html_text: 站点首页html + :return: 是否匹配 + """ + pass + + def parse(self): + """ + 解析站点信息 + :return: + """ + if not self._parse_logged_in(self._index_html): + return + + self._parse_site_page(self._index_html) + self._parse_user_base_info(self._index_html) + self._pase_unread_msgs() + if self._user_traffic_page: + self._parse_user_traffic_info(self._get_page_content(urljoin(self._base_url, self._user_traffic_page))) + if self._user_detail_page: + self._parse_user_detail_info(self._get_page_content(urljoin(self._base_url, self._user_detail_page))) + + self._parse_seeding_pages() + self.seeding_info = json.dumps(self.seeding_info) + + def _pase_unread_msgs(self): + """ + 解析所有未读消息标题和内容 + :return: + """ + unread_msg_links = [] + if self.message_unread > 0: + links = {self._user_mail_unread_page, self._sys_mail_unread_page} + for link in links: + if not link: + continue + + msg_links = [] + next_page = self._parse_message_unread_links( + self._get_page_content(urljoin(self._base_url, link)), msg_links) + while next_page: + next_page = self._parse_message_unread_links( + self._get_page_content(urljoin(self._base_url, next_page)), msg_links) + + unread_msg_links.extend(msg_links) + + for msg_link in unread_msg_links: + logger.debug(f"{self.site_name} 信息链接 {msg_link}") + head, date, content = self._parse_message_content(self._get_page_content(urljoin(self._base_url, msg_link))) + logger.debug(f"{self.site_name} 标题 {head} 时间 {date} 内容 {content}") + self.message_unread_contents.append((head, date, content)) + + def _parse_seeding_pages(self): + if self._torrent_seeding_page: + # 第一页 + next_page = self._parse_user_torrent_seeding_info( + self._get_page_content(urljoin(self._base_url, self._torrent_seeding_page), + self._torrent_seeding_params, + self._torrent_seeding_headers)) + + # 其他页处理 + while next_page: + next_page = self._parse_user_torrent_seeding_info( + self._get_page_content(urljoin(urljoin(self._base_url, self._torrent_seeding_page), next_page), + self._torrent_seeding_params, + self._torrent_seeding_headers), + multi_page=True) + + @staticmethod + def _prepare_html_text(html_text): + """ + 处理掉HTML中的干扰部分 + """ + return re.sub(r"#\d+", "", re.sub(r"\d+px", "", html_text)) + + @abstractmethod + def _parse_message_unread_links(self, html_text: str, msg_links: list) -> Optional[str]: + """ + 获取未阅读消息链接 + :param html_text: + :return: + """ + pass + + def _get_page_content(self, url: str, params: dict = None, headers: dict = None): + """ + :param url: 网页地址 + :param params: post参数 + :param headers: 额外的请求头 + :return: + """ + req_headers = None + proxies = settings.PROXY if self._proxy else None + if self._ua or headers or self._addition_headers: + req_headers = {} + if headers: + req_headers.update(headers) + + req_headers.update({ + "Content-Type": "application/x-www-form-urlencoded; charset=UTF-8", + "User-Agent": f"{self._ua}" + }) + + if self._addition_headers: + req_headers.update(self._addition_headers) + + if params: + res = RequestUtils(cookies=self._site_cookie, + session=self._session, + timeout=60, + proxies=proxies, + headers=req_headers).post_res(url=url, data=params) + else: + res = RequestUtils(cookies=self._site_cookie, + session=self._session, + timeout=60, + proxies=proxies, + headers=req_headers).get_res(url=url) + if res is not None and res.status_code in (200, 500, 403): + # 如果cloudflare 有防护，尝试使用浏览器仿真 + if under_challenge(res.text): + logger.warn( + f"{self.site_name} 检测到Cloudflare，请更新Cookie和UA") + return "" + if re.search(r"charset=\"?utf-8\"?", res.text, re.IGNORECASE): + res.encoding = "utf-8" + else: + res.encoding = res.apparent_encoding + return res.text + + return "" + + @abstractmethod + def _parse_site_page(self, html_text: str): + """ + 解析站点相关信息页面 + :param html_text: + :return: + """ + pass + + @abstractmethod + def _parse_user_base_info(self, html_text: str): + """ + 解析用户基础信息 + :param html_text: + :return: + """ + pass + + def _parse_logged_in(self, html_text): + """ + 解析用户是否已经登陆 + :param html_text: + :return: True/False + """ + logged_in = SiteUtils.is_logged_in(html_text) + if not logged_in: + self.err_msg = "未检测到已登陆，请检查cookies是否过期" + logger.warn(f"{self.site_name} 未登录，跳过后续操作") + + return logged_in + + @abstractmethod + def _parse_user_traffic_info(self, html_text: str): + """ + 解析用户的上传，下载，分享率等信息 + :param html_text: + :return: + """ + pass + + @abstractmethod + def _parse_user_torrent_seeding_info(self, html_text: str, multi_page: bool = False) -> Optional[str]: + """ + 解析用户的做种相关信息 + :param html_text: + :param multi_page: 是否多页数据 + :return: 下页地址 + """ + pass + + @abstractmethod + def _parse_user_detail_info(self, html_text: str): + """ + 解析用户的详细信息 + 加入时间/等级/魔力值等 + :param html_text: + :return: + """ + pass + + @abstractmethod + def _parse_message_content(self, html_text): + """ + 解析短消息内容 + :param html_text: + :return: head: message, date: time, content: message content + """ + pass + + def to_dict(self): + """ + 转化为字典 + """ + attributes = [ + attr for attr in dir(self) + if not callable(getattr(self, attr)) and not attr.startswith("_") + ] + return { + attr: getattr(self, attr).value + if isinstance(getattr(self, attr), SiteSchema) + else getattr(self, attr) for attr in attributes + }