From 89facd678d8ffd7a567da12b51d7fc48719da4d7 Mon Sep 17 00:00:00 2001 From: thsrite Date: Tue, 26 Mar 2024 09:22:33 +0800 Subject: [PATCH] =?UTF-8?q?fix=20=E7=AB=99=E7=82=B9=E6=9C=AA=E8=AF=BB?= =?UTF-8?q?=E6=B6=88=E6=81=AF=E6=94=AF=E6=8C=81mtorrent?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 2 +- docs/SiteUnreadMsg.md | 1 + package.json | 2 +- plugins/siteunreadmsg/__init__.py | 2 +- .../siteunreadmsg/siteuserinfo/__init__.py | 174 ++++++++++++---- .../siteunreadmsg/siteuserinfo/mtorrent.py | 196 ++++++++++++++++++ .../siteunreadmsg/siteuserinfo/nexus_php.py | 6 + 7 files changed, 335 insertions(+), 48 deletions(-) create mode 100644 plugins/siteunreadmsg/siteuserinfo/mtorrent.py diff --git a/README.md b/README.md index e665cb5..d213b5e 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ MoviePilot三方插件市场：https://github.com/thsrite/MoviePilot-Plugins/ ### 插件新增 - [站点数据统计 1.4](docs%2FSiteStatisticNoMsg.md) （无未读消息版本）(废弃) -- [站点未读消息 1.2](docs%2FSiteUnreadMsg.md) +- [站点未读消息 1.3](docs%2FSiteUnreadMsg.md) - [云盘Strm生成 3.6](docs%2FCloudStrm.md) - [Strm文件模式转换 1.0](docs%2FStrmConvert.md) - [清理订阅缓存 1.0](docs%2FSubscribeClear.md) diff --git a/docs/SiteUnreadMsg.md b/docs/SiteUnreadMsg.md index 3d28bb6..6cec2ad 100644 --- a/docs/SiteUnreadMsg.md +++ b/docs/SiteUnreadMsg.md @@ -2,6 +2,7 @@ ### 更新记录 +- 1.3 feat mtorrent - 1.2 站点消息历史存库 - 1.1 防止同一消息重复发送 - 1.0 定时获取站点消息 diff --git a/package.json b/package.json index 0d5cc3d..782f196 100644 --- a/package.json +++ b/package.json @@ -18,7 +18,7 @@ "SiteUnreadMsg": { "name": "站点未读消息", "description": "发送站点未读消息。", - "version": "1.2", + "version": "1.3", "icon": "Synomail_A.png", "author": "thsrite", "level": 2 diff --git a/plugins/siteunreadmsg/__init__.py b/plugins/siteunreadmsg/__init__.py index 6e4b665..c2417fb 100644 --- a/plugins/siteunreadmsg/__init__.py +++ b/plugins/siteunreadmsg/__init__.py @@ -37,7 +37,7 @@ class SiteUnreadMsg(_PluginBase): # 插件图标 plugin_icon = "Synomail_A.png" # 插件版本 - plugin_version = "1.2" + plugin_version = "1.3" # 插件作者 plugin_author = "thsrite" # 作者主页 diff --git a/plugins/siteunreadmsg/siteuserinfo/__init__.py b/plugins/siteunreadmsg/siteuserinfo/__init__.py index e8e218f..44fd276 100644 --- a/plugins/siteunreadmsg/siteuserinfo/__init__.py +++ b/plugins/siteunreadmsg/siteuserinfo/__init__.py @@ -31,6 +31,7 @@ class SiteSchema(Enum): TorrentLeech = "TorrentLeech" FileList = "FileList" TNode = "TNode" + MTorrent = "MTorrent" class ISiteUserInfo(metaclass=ABCMeta): @@ -38,6 +39,8 @@ class ISiteUserInfo(metaclass=ABCMeta): schema = SiteSchema.NexusPhp # 站点解析时判断顺序，值越小越先解析 order = SITE_BASE_ORDER + # 请求模式 cookie/apikey + request_mode = "cookie" def __init__(self, site_name: str, url: str, @@ -78,6 +81,9 @@ class ISiteUserInfo(metaclass=ABCMeta): self.seeding_info = [] # 用户详细信息 + self._user_basic_page = None + self._user_basic_params = None + self._user_basic_headers = None self.user_level = None self.join_at = None self.bonus = 0.0 @@ -93,16 +99,25 @@ class ISiteUserInfo(metaclass=ABCMeta): # 站点页面 self._brief_page = "index.php" self._user_detail_page = "userdetails.php?id=" + self._user_detail_params = None + self._user_detail_headers = None self._user_traffic_page = "index.php" - self._torrent_seeding_page = "getusertorrentlistajax.php?userid=" + self._user_traffic_params = None + self._user_traffic_headers = None self._user_mail_unread_page = "messages.php?action=viewmailbox&box=1&unread=yes" self._sys_mail_unread_page = "messages.php?action=viewmailbox&box=-2&unread=yes" + self._mail_unread_params = None + self._mail_unread_headers = None + self._mail_content_params = None + self._mail_content_headers = None + self._torrent_seeding_page = "getusertorrentlistajax.php?userid=" self._torrent_seeding_params = None self._torrent_seeding_headers = None split_url = urlsplit(url) self.site_name = site_name self.site_url = url + self.site_domain = split_url.netloc self._base_url = f"{split_url.scheme}://{split_url.netloc}" self._site_cookie = site_cookie self._index_html = index_html @@ -133,17 +148,43 @@ class ISiteUserInfo(metaclass=ABCMeta): 解析站点信息 :return: """ + # 检查是否已经登录 if not self._parse_logged_in(self._index_html): return - + # 解析站点页面 self._parse_site_page(self._index_html) - self._parse_user_base_info(self._index_html) - self._pase_unread_msgs() - if self._user_traffic_page: - self._parse_user_traffic_info(self._get_page_content(urljoin(self._base_url, self._user_traffic_page))) + # 解析用户基础信息 + if self._user_basic_page: + self._parse_user_base_info( + self._get_page_content( + url=urljoin(self._base_url, self._user_basic_page), + params=self._user_basic_params, + headers=self._user_basic_headers + ) + ) + else: + self._parse_user_base_info(self._index_html) + # 解析用户详细信息 if self._user_detail_page: - self._parse_user_detail_info(self._get_page_content(urljoin(self._base_url, self._user_detail_page))) - + self._parse_user_detail_info( + self._get_page_content( + url=urljoin(self._base_url, self._user_detail_page), + params=self._user_detail_params, + headers=self._user_detail_headers + ) + ) + # 解析用户未读消息 + self._pase_unread_msgs() + # 解析用户上传、下载、分享率等信息 + if self._user_traffic_page: + self._parse_user_traffic_info( + self._get_page_content( + url=urljoin(self._base_url, self._user_traffic_page), + params=self._user_traffic_params, + headers=self._user_traffic_headers + ) + ) + # 解析用户做种信息 self._parse_seeding_pages() self.seeding_info = json.dumps(self.seeding_info) @@ -158,36 +199,59 @@ class ISiteUserInfo(metaclass=ABCMeta): for link in links: if not link: continue - msg_links = [] next_page = self._parse_message_unread_links( - self._get_page_content(urljoin(self._base_url, link)), msg_links) + self._get_page_content( + url=urljoin(self._base_url, link), + params=self._mail_unread_params, + headers=self._mail_unread_headers + ), + msg_links) while next_page: next_page = self._parse_message_unread_links( - self._get_page_content(urljoin(self._base_url, next_page)), msg_links) - + self._get_page_content( + url=urljoin(self._base_url, next_page), + params=self._mail_unread_params, + headers=self._mail_unread_headers + ), + msg_links + ) unread_msg_links.extend(msg_links) - + # 解析未读消息内容 for msg_link in unread_msg_links: logger.debug(f"{self.site_name} 信息链接 {msg_link}") - head, date, content = self._parse_message_content(self._get_page_content(urljoin(self._base_url, msg_link))) + head, date, content = self._parse_message_content( + self._get_page_content( + urljoin(self._base_url, msg_link), + params=self._mail_content_params, + headers=self._mail_content_headers + ) + ) logger.debug(f"{self.site_name} 标题 {head} 时间 {date} 内容 {content}") self.message_unread_contents.append((head, date, content)) def _parse_seeding_pages(self): + """ + 解析做种页面 + """ if self._torrent_seeding_page: # 第一页 next_page = self._parse_user_torrent_seeding_info( - self._get_page_content(urljoin(self._base_url, self._torrent_seeding_page), - self._torrent_seeding_params, - self._torrent_seeding_headers)) + self._get_page_content( + url=urljoin(self._base_url, self._torrent_seeding_page), + params=self._torrent_seeding_params, + headers=self._torrent_seeding_headers + ) + ) # 其他页处理 - while next_page: + while next_page is not None and next_page is not False: next_page = self._parse_user_torrent_seeding_info( - self._get_page_content(urljoin(urljoin(self._base_url, self._torrent_seeding_page), next_page), - self._torrent_seeding_params, - self._torrent_seeding_headers), + self._get_page_content( + url=urljoin(urljoin(self._base_url, self._torrent_seeding_page), next_page), + params=self._torrent_seeding_params, + headers=self._torrent_seeding_headers + ), multi_page=True) @staticmethod @@ -216,41 +280,61 @@ class ISiteUserInfo(metaclass=ABCMeta): req_headers = None proxies = settings.PROXY if self._proxy else None if self._ua or headers or self._addition_headers: - req_headers = {} + req_headers = { + "User-Agent": f"{self._ua}" + } + if headers: req_headers.update(headers) + else: + req_headers.update({ + "Content-Type": "application/x-www-form-urlencoded; charset=UTF-8", + }) + if self._addition_headers: + req_headers.update(self._addition_headers) - req_headers.update({ - "Content-Type": "application/x-www-form-urlencoded; charset=UTF-8", - "User-Agent": f"{self._ua}" - }) - - if self._addition_headers: - req_headers.update(self._addition_headers) + if self.request_mode == "apikey": + # 使用apikey请求，通过请求头传递 + cookie = None + session = None + else: + # 使用cookie请求 + cookie = self._site_cookie + session = self._session if params: - res = RequestUtils(cookies=self._site_cookie, - session=self._session, - timeout=60, - proxies=proxies, - headers=req_headers).post_res(url=url, data=params) + if req_headers.get("Content-Type") == "application/json": + res = RequestUtils(cookies=cookie, + session=session, + timeout=60, + proxies=proxies, + headers=req_headers).post_res(url=url, json=params) + else: + res = RequestUtils(cookies=cookie, + session=session, + timeout=60, + proxies=proxies, + headers=req_headers).post_res(url=url, data=params) else: - res = RequestUtils(cookies=self._site_cookie, - session=self._session, + res = RequestUtils(cookies=cookie, + session=session, timeout=60, proxies=proxies, headers=req_headers).get_res(url=url) if res is not None and res.status_code in (200, 500, 403): - # 如果cloudflare 有防护，尝试使用浏览器仿真 - if under_challenge(res.text): - logger.warn( - f"{self.site_name} 检测到Cloudflare，请更新Cookie和UA") - return "" - if re.search(r"charset=\"?utf-8\"?", res.text, re.IGNORECASE): - res.encoding = "utf-8" + if req_headers and "application/json" in req_headers.get("Accept"): + return json.dumps(res.json()) else: - res.encoding = res.apparent_encoding - return res.text + # 如果cloudflare 有防护，尝试使用浏览器仿真 + if under_challenge(res.text): + logger.warn( + f"{self.site_name} 检测到Cloudflare，请更新Cookie和UA") + return "" + if re.search(r"charset=\"?utf-8\"?", res.text, re.IGNORECASE): + res.encoding = "utf-8" + else: + res.encoding = res.apparent_encoding + return res.text return "" diff --git a/plugins/siteunreadmsg/siteuserinfo/mtorrent.py b/plugins/siteunreadmsg/siteuserinfo/mtorrent.py new file mode 100644 index 0000000..07d273f --- /dev/null +++ b/plugins/siteunreadmsg/siteuserinfo/mtorrent.py @@ -0,0 +1,196 @@ +# -*- coding: utf-8 -*- +import json +from typing import Optional, Tuple +from urllib.parse import urljoin +from lxml import etree + +from app.log import logger +from app.db.systemconfig_oper import SystemConfigOper +from app.plugins.sitestatistic.siteuserinfo import ISiteUserInfo, SITE_BASE_ORDER, SiteSchema +from app.utils.string import StringUtils + + +class MTorrentSiteUserInfo(ISiteUserInfo): + schema = SiteSchema.MTorrent + order = SITE_BASE_ORDER + 60 + request_mode = "apikey" + + # 用户级别字典 + MTeam_sysRoleList = { + "1": "User", + "2": "Power User", + "3": "Elite User", + "4": "Crazy User", + "5": "Insane User", + "6": "Veteran User", + "7": "Extreme User", + "8": "Ultimate User", + "9": "Nexus Master", + "10": "VIP", + "11": "Retiree", + "12": "Uploader", + "13": "Moderator", + "14": "Administrator", + "15": "Sysop", + "16": "Staff", + "17": "Offer memberStaff", + "18": "Bet memberStaff", + } + + @classmethod + def match(cls, html_text: str) -> bool: + html = etree.HTML(html_text) + if not html: + return False + if html.xpath("//title/text()") and "M-Team" in html.xpath("//title/text()")[0]: + return True + return False + + def _parse_site_page(self, html_text: str): + """ + 获取站点页面地址 + """ + self._user_traffic_page = None + self._user_detail_page = None + self._user_basic_page = "api/member/profile" + self._user_basic_params = { + "uid": self.userid + } + self._sys_mail_unread_page = None + self._user_mail_unread_page = "api/msg/search" + self._mail_unread_params = { + "keyword": "", + "box": "-2", + "type": "pageNumber", + "pageSize": 100 + } + self._torrent_seeding_page = "api/member/getUserTorrentList" + domain = StringUtils.get_url_host(self.site_url) + self._torrent_seeding_headers = { + "Content-Type": "application/json", + "Accept": "application/json, text/plain, */*", + "x-api-key": SystemConfigOper().get(f"site.{domain}.apikey"), + } + + def _parse_logged_in(self, html_text): + """ + 判断是否登录成功, 通过判断是否存在用户信息 + 暂时跳过检测，待后续优化 + :param html_text: + :return: + """ + return True + + def _parse_user_base_info(self, html_text: str): + """ + 解析用户基本信息，这里把_parse_user_traffic_info和_parse_user_detail_info合并到这里 + """ + if not html_text: + return None + detail = json.loads(html_text) + if not detail or detail.get("code") != "0": + return + user_info = detail.get("data", {}) + self.userid = user_info.get("id") + self.username = user_info.get("username") + self.user_level = self.MTeam_sysRoleList.get(user_info.get("role") or "1") + self.join_at = user_info.get("memberStatus", {}).get("createdDate") + + self.upload = int(user_info.get("memberCount", {}).get("uploaded") or '0') + self.download = int(user_info.get("memberCount", {}).get("downloaded") or '0') + self.ratio = user_info.get("memberCount", {}).get("shareRate") or 0 + self.bonus = user_info.get("memberCount", {}).get("bonus") or 0 + self.message_unread = 1 + + self._torrent_seeding_params = { + "pageNumber": 1, + "pageSize": 200, + "type": "SEEDING", + "userid": self.userid + } + + def _parse_user_traffic_info(self, html_text: str): + """ + 解析用户流量信息 + """ + pass + + def _parse_user_detail_info(self, html_text: str): + """ + 解析用户详细信息 + """ + pass + + def _parse_user_torrent_seeding_info(self, html_text: str, multi_page: bool = False) -> Optional[str]: + """ + 解析用户做种信息 + """ + if not html_text: + return None + seeding_info = json.loads(html_text) + if not seeding_info or seeding_info.get("code") != "0": + return None + torrents = seeding_info.get("data", {}).get("data", []) + page_seeding_size = 0 + page_seeding_info = [] + for info in torrents: + torrent = info.get("torrent", {}) + size = int(torrent.get("size") or '0') + seeders = int(torrent.get("source") or '0') + page_seeding_size += size + page_seeding_info.append([seeders, size]) + self.seeding += len(torrents) + self.seeding_size += page_seeding_size + self.seeding_info.extend(page_seeding_info) + + # 查询总做种数 + seeder_count = 0 + try: + result = self._get_page_content( + url=urljoin(self.site_url, "api/tracker/myPeerStatus"), + params={"uid": self.userid}, + ) + if result: + seeder_info = json.loads(result) + seeder_count = int(seeder_info.get("data", {}).get("seeder") or 0) + except Exception as e: + logger.error(f"获取做种数失败: {str(e)}") + if not seeder_count: + return None + if self.seeding >= seeder_count: + return None + # 还有下一页 + self._torrent_seeding_params["pageNumber"] += 1 + return "" + + def _parse_message_unread_links(self, html_text: str, msg_links: list) -> Optional[str]: + """ + 解析未读消息链接，这里直接读出详情 + """ + if not html_text: + return None + messages_info = json.loads(html_text) + if not messages_info or messages_info.get("code") != "0": + return None + messages = messages_info.get("data", {}).get("data", []) + for message in messages: + if not message.get("unread"): + continue + head = message.get("title") + date = message.get("createdDate") + content = message.get("context") + if head and date and content: + self.message_unread_contents.append((head, date, content)) + # 设置已读 + self._get_page_content( + url=urljoin(self.site_url, f"api/msg/markRead"), + params={"msgId": message.get("id")} + ) + # 是否存在下页数据 + return None + + def _parse_message_content(self, html_text) -> Tuple[Optional[str], Optional[str], Optional[str]]: + """ + 解析消息内容 + """ + pass diff --git a/plugins/siteunreadmsg/siteuserinfo/nexus_php.py b/plugins/siteunreadmsg/siteuserinfo/nexus_php.py index 8d5b6c9..783f92b 100644 --- a/plugins/siteunreadmsg/siteuserinfo/nexus_php.py +++ b/plugins/siteunreadmsg/siteuserinfo/nexus_php.py @@ -61,6 +61,9 @@ class NexusPhpSiteUserInfo(ISiteUserInfo): self.message_unread = StringUtils.str_int(message_text) def _parse_user_base_info(self, html_text: str): + """ + 解析用户基本信息 + """ # 合并解析，减少额外请求调用 self._parse_user_traffic_info(html_text) self._user_traffic_page = None @@ -85,6 +88,9 @@ class NexusPhpSiteUserInfo(ISiteUserInfo): return def _parse_user_traffic_info(self, html_text): + """ + 解析用户流量信息 + """ html_text = self._prepare_html_text(html_text) upload_match = re.search(r"[^总]上[传傳]量?[:：_<>/a-zA-Z-=\"'\s#;]+([\d,.\s]+[KMGTPI]*B)", html_text, re.IGNORECASE)