mirror of
https://github.com/jxxghp/MoviePilot.git
synced 2026-06-14 07:26:50 +00:00
Compare commits
18 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
086b1f1403 | ||
|
|
19608fa98e | ||
|
|
b0d17deda1 | ||
|
|
4c979c458e | ||
|
|
c5e93169ad | ||
|
|
1e2ca294de | ||
|
|
7165c4a275 | ||
|
|
cbe81ba33c | ||
|
|
fdbfae953d | ||
|
|
c7ba274877 | ||
|
|
8b15a16ca1 | ||
|
|
9f2c8d3811 | ||
|
|
7343dfbed8 | ||
|
|
90f74d8d2b | ||
|
|
7e3e0e1178 | ||
|
|
d890e38a10 | ||
|
|
e505b5c85f | ||
|
|
6230f55116 |
@@ -106,7 +106,7 @@ def wechat_verify(echostr: str, msg_signature: str, timestamp: Union[str, int],
|
||||
return str(err)
|
||||
|
||||
|
||||
async def vocechat_verify() -> Any:
|
||||
def vocechat_verify() -> Any:
|
||||
"""
|
||||
VoceChat验证响应
|
||||
"""
|
||||
|
||||
@@ -78,10 +78,14 @@ async def create_subscribe(
|
||||
title = None
|
||||
# 订阅用户
|
||||
subscribe_in.username = current_user.name
|
||||
# 转化为字典
|
||||
subscribe_dict = subscribe_in.dict()
|
||||
if subscribe_in.id:
|
||||
subscribe_dict.pop("id", None)
|
||||
sid, message = await SubscribeChain().async_add(mtype=mtype,
|
||||
title=title,
|
||||
exist_ok=True,
|
||||
**subscribe_in.dict())
|
||||
**subscribe_dict)
|
||||
return schemas.Response(
|
||||
success=bool(sid), message=message, data={"id": sid}
|
||||
)
|
||||
|
||||
@@ -318,11 +318,17 @@ class MediaChain(ChainBase):
|
||||
if not event:
|
||||
return
|
||||
event_data = event.event_data or {}
|
||||
# 媒体根目录
|
||||
fileitem: FileItem = event_data.get("fileitem")
|
||||
# 媒体文件列表
|
||||
file_list: List[str] = event_data.get("file_list", [])
|
||||
# 媒体元数据
|
||||
meta: MetaBase = event_data.get("meta")
|
||||
# 媒体信息
|
||||
mediainfo: MediaInfo = event_data.get("mediainfo")
|
||||
# 是否覆盖
|
||||
overwrite = event_data.get("overwrite", False)
|
||||
# 检查媒体根目录
|
||||
if not fileitem:
|
||||
return
|
||||
|
||||
@@ -342,31 +348,62 @@ class MediaChain(ChainBase):
|
||||
parent=storagechain.get_parent_item(fileitem),
|
||||
overwrite=overwrite)
|
||||
else:
|
||||
# 检查目的目录下是否已经有nfo刮削文件
|
||||
has_nfo_file = storagechain.any_files(fileitem, extensions=['.nfo'])
|
||||
if has_nfo_file and file_list:
|
||||
logger.info(f"目录 {fileitem.path} 已有NFO文件,开始增量刮削...")
|
||||
for file_path in file_list:
|
||||
file_item = storagechain.get_file_item(storage=fileitem.storage,
|
||||
path=Path(file_path))
|
||||
if file_item:
|
||||
# 对于电视剧文件,应该保存到与视频文件相同的目录
|
||||
# 而不是电视剧根目录
|
||||
self.scrape_metadata(fileitem=file_item,
|
||||
if file_list:
|
||||
# 1. 收集fileitem和file_list中每个文件之间所有子目录
|
||||
all_dirs = set()
|
||||
root_path = Path(fileitem.path)
|
||||
|
||||
logger.debug(f"开始收集目录,根目录:{root_path}")
|
||||
# 收集根目录
|
||||
all_dirs.add(root_path)
|
||||
|
||||
# 收集所有目录(包括所有层级)
|
||||
for sub_file in file_list:
|
||||
sub_path = Path(sub_file)
|
||||
# 收集从根目录到文件的所有父目录
|
||||
current_path = sub_path.parent
|
||||
while current_path != root_path and current_path.is_relative_to(root_path):
|
||||
all_dirs.add(current_path)
|
||||
current_path = current_path.parent
|
||||
|
||||
logger.debug(f"共收集到 {len(all_dirs)} 个目录")
|
||||
|
||||
# 2. 初始化一遍子目录,但不处理文件
|
||||
for sub_dir in all_dirs:
|
||||
sub_dir_item = storagechain.get_file_item(storage=fileitem.storage, path=sub_dir)
|
||||
if sub_dir_item:
|
||||
logger.info(f"为目录生成海报和nfo:{sub_dir}")
|
||||
# 初始化目录元数据,但不处理文件
|
||||
self.scrape_metadata(fileitem=sub_dir_item,
|
||||
mediainfo=mediainfo,
|
||||
init_folder=True,
|
||||
recursive=False,
|
||||
overwrite=overwrite)
|
||||
else:
|
||||
logger.warn(f"无法获取目录项:{sub_dir}")
|
||||
|
||||
# 3. 刮削每个文件
|
||||
logger.info(f"开始刮削 {len(file_list)} 个文件")
|
||||
for sub_file_path in file_list:
|
||||
sub_file_item = storagechain.get_file_item(storage=fileitem.storage,
|
||||
path=Path(sub_file_path))
|
||||
if sub_file_item:
|
||||
self.scrape_metadata(fileitem=sub_file_item,
|
||||
mediainfo=mediainfo,
|
||||
init_folder=False,
|
||||
parent=None, # 让函数内部自动获取正确的父目录
|
||||
overwrite=overwrite)
|
||||
else:
|
||||
logger.warn(f"无法获取文件项:{sub_file_path}")
|
||||
else:
|
||||
# 执行全量刮削
|
||||
logger.info(f"开始全量刮削目录 {fileitem.path} ...")
|
||||
logger.info(f"开始刮削目录 {fileitem.path} ...")
|
||||
self.scrape_metadata(fileitem=fileitem, meta=meta, init_folder=True,
|
||||
mediainfo=mediainfo, overwrite=overwrite)
|
||||
|
||||
def scrape_metadata(self, fileitem: schemas.FileItem,
|
||||
meta: MetaBase = None, mediainfo: MediaInfo = None,
|
||||
init_folder: bool = True, parent: schemas.FileItem = None,
|
||||
overwrite: bool = False):
|
||||
overwrite: bool = False, recursive: bool = True):
|
||||
"""
|
||||
手动刮削媒体信息
|
||||
:param fileitem: 刮削目录或文件
|
||||
@@ -375,6 +412,7 @@ class MediaChain(ChainBase):
|
||||
:param init_folder: 是否刮削根目录
|
||||
:param parent: 上级目录
|
||||
:param overwrite: 是否覆盖已有文件
|
||||
:param recursive: 是否递归处理目录内文件
|
||||
"""
|
||||
|
||||
storagechain = StorageChain()
|
||||
@@ -481,31 +519,33 @@ class MediaChain(ChainBase):
|
||||
logger.info("电影NFO刮削已关闭,跳过")
|
||||
else:
|
||||
# 电影目录
|
||||
if is_bluray_folder(fileitem):
|
||||
# 原盘目录
|
||||
if scraping_switchs.get('movie_nfo', True):
|
||||
nfo_path = filepath / (filepath.name + ".nfo")
|
||||
if overwrite or not storagechain.get_file_item(storage=fileitem.storage, path=nfo_path):
|
||||
# 生成原盘nfo
|
||||
movie_nfo = self.metadata_nfo(meta=meta, mediainfo=mediainfo)
|
||||
if movie_nfo:
|
||||
# 保存或上传nfo文件到当前目录
|
||||
__save_file(_fileitem=fileitem, _path=nfo_path, _content=movie_nfo)
|
||||
if recursive:
|
||||
# 处理文件
|
||||
if is_bluray_folder(fileitem):
|
||||
# 原盘目录
|
||||
if scraping_switchs.get('movie_nfo', True):
|
||||
nfo_path = filepath / (filepath.name + ".nfo")
|
||||
if overwrite or not storagechain.get_file_item(storage=fileitem.storage, path=nfo_path):
|
||||
# 生成原盘nfo
|
||||
movie_nfo = self.metadata_nfo(meta=meta, mediainfo=mediainfo)
|
||||
if movie_nfo:
|
||||
# 保存或上传nfo文件到当前目录
|
||||
__save_file(_fileitem=fileitem, _path=nfo_path, _content=movie_nfo)
|
||||
else:
|
||||
logger.warn(f"{filepath.name} nfo文件生成失败!")
|
||||
else:
|
||||
logger.warn(f"{filepath.name} nfo文件生成失败!")
|
||||
logger.info(f"已存在nfo文件:{nfo_path}")
|
||||
else:
|
||||
logger.info(f"已存在nfo文件:{nfo_path}")
|
||||
logger.info("电影NFO刮削已关闭,跳过")
|
||||
else:
|
||||
logger.info("电影NFO刮削已关闭,跳过")
|
||||
else:
|
||||
# 处理目录内的文件
|
||||
files = __list_files(_fileitem=fileitem)
|
||||
for file in files:
|
||||
self.scrape_metadata(fileitem=file,
|
||||
mediainfo=mediainfo,
|
||||
init_folder=False,
|
||||
parent=fileitem,
|
||||
overwrite=overwrite)
|
||||
# 处理目录内的文件
|
||||
files = __list_files(_fileitem=fileitem)
|
||||
for file in files:
|
||||
self.scrape_metadata(fileitem=file,
|
||||
mediainfo=mediainfo,
|
||||
init_folder=False,
|
||||
parent=fileitem,
|
||||
overwrite=overwrite)
|
||||
# 生成目录内图片文件
|
||||
if init_folder:
|
||||
# 图片
|
||||
@@ -597,13 +637,14 @@ class MediaChain(ChainBase):
|
||||
logger.info("集缩略图刮削已关闭,跳过")
|
||||
else:
|
||||
# 当前为电视剧目录,处理目录内的文件
|
||||
files = __list_files(_fileitem=fileitem)
|
||||
for file in files:
|
||||
self.scrape_metadata(fileitem=file,
|
||||
mediainfo=mediainfo,
|
||||
parent=fileitem if file.type == "file" else None,
|
||||
init_folder=True if file.type == "dir" else False,
|
||||
overwrite=overwrite)
|
||||
if recursive:
|
||||
files = __list_files(_fileitem=fileitem)
|
||||
for file in files:
|
||||
self.scrape_metadata(fileitem=file,
|
||||
mediainfo=mediainfo,
|
||||
parent=fileitem if file.type == "file" else None,
|
||||
init_folder=True if file.type == "dir" else False,
|
||||
overwrite=overwrite)
|
||||
# 生成目录的nfo和图片
|
||||
if init_folder:
|
||||
# 识别文件夹名称
|
||||
|
||||
@@ -330,7 +330,8 @@ class SiteChain(ChainBase):
|
||||
url=site_info.url,
|
||||
cookie=cookie,
|
||||
ua=site_info.ua or settings.USER_AGENT,
|
||||
proxy=True if site_info.proxy else False
|
||||
proxy=True if site_info.proxy else False,
|
||||
timeout=site_info.timeout
|
||||
)
|
||||
if rss_url:
|
||||
logger.info(f"更新站点 {domain} RSS地址 ...")
|
||||
@@ -558,13 +559,15 @@ class SiteChain(ChainBase):
|
||||
public = site_info.public
|
||||
proxies = settings.PROXY if site_info.proxy else None
|
||||
proxy_server = settings.PROXY_SERVER if site_info.proxy else None
|
||||
timeout = site_info.timeout or 60
|
||||
|
||||
# 访问链接
|
||||
if render:
|
||||
page_source = PlaywrightHelper().get_page_source(url=site_url,
|
||||
cookies=site_cookie,
|
||||
ua=ua,
|
||||
proxies=proxy_server)
|
||||
proxies=proxy_server,
|
||||
timeout=timeout)
|
||||
if not public and not SiteUtils.is_logged_in(page_source):
|
||||
if under_challenge(page_source):
|
||||
return False, f"无法通过Cloudflare!"
|
||||
@@ -697,7 +700,8 @@ class SiteChain(ChainBase):
|
||||
username=username,
|
||||
password=password,
|
||||
two_step_code=two_step_code,
|
||||
proxies=settings.PROXY_HOST if site_info.proxy else None
|
||||
proxies=settings.PROXY_SERVER if site_info.proxy else None,
|
||||
timeout=site_info.timeout or 60
|
||||
)
|
||||
if result:
|
||||
cookie, ua, msg = result
|
||||
|
||||
@@ -340,7 +340,8 @@ class TorrentsChain(ChainBase):
|
||||
url=site.get("url"),
|
||||
cookie=site.get("cookie"),
|
||||
ua=site.get("ua") or settings.USER_AGENT,
|
||||
proxy=True if site.get("proxy") else False
|
||||
proxy=True if site.get("proxy") else False,
|
||||
timeout=site.get("timeout"),
|
||||
)
|
||||
if rss_url:
|
||||
# 获取新的日期的passkey
|
||||
|
||||
@@ -8,6 +8,7 @@ import sys
|
||||
import threading
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional, Tuple, Type
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from dotenv import set_key
|
||||
from pydantic import BaseModel, BaseSettings, validator, Field
|
||||
@@ -319,6 +320,10 @@ class ConfigModel(BaseModel):
|
||||
RCLONE_SNAPSHOT_CHECK_FOLDER_MODTIME = True
|
||||
# 对OpenList进行快照对比时,是否检查文件夹的修改时间
|
||||
OPENLIST_SNAPSHOT_CHECK_FOLDER_MODTIME = True
|
||||
# 仿真类型:playwright 或 flaresolverr
|
||||
BROWSER_EMULATION: str = "playwright"
|
||||
# FlareSolverr 服务地址,例如 http://127.0.0.1:8191
|
||||
FLARESOLVERR_URL: Optional[str] = None
|
||||
|
||||
|
||||
class Settings(BaseSettings, ConfigModel, LogConfigModel):
|
||||
@@ -615,9 +620,22 @@ class Settings(BaseSettings, ConfigModel, LogConfigModel):
|
||||
@property
|
||||
def PROXY_SERVER(self):
|
||||
if self.PROXY_HOST:
|
||||
return {
|
||||
"server": self.PROXY_HOST
|
||||
}
|
||||
try:
|
||||
parsed = urlparse(self.PROXY_HOST)
|
||||
if not parsed.scheme:
|
||||
return {"server": self.PROXY_HOST}
|
||||
host = parsed.hostname or ""
|
||||
port = f":{parsed.port}" if parsed.port else ""
|
||||
server = f"{parsed.scheme}://{host}{port}"
|
||||
proxy = {"server": server}
|
||||
if parsed.username:
|
||||
proxy["username"] = parsed.username
|
||||
if parsed.password:
|
||||
proxy["password"] = parsed.password
|
||||
return proxy
|
||||
except Exception as err:
|
||||
logger.error(f"解析代理服务器地址 '{self.PROXY_HOST}' 时出错: {err}")
|
||||
return {"server": self.PROXY_HOST}
|
||||
return None
|
||||
|
||||
@property
|
||||
|
||||
@@ -105,10 +105,11 @@ class ReleaseGroupsMatcher(metaclass=Singleton):
|
||||
else:
|
||||
groups = self.__release_groups
|
||||
title = f"{title} "
|
||||
groups_re = re.compile(r"(?<=[-@\[£【&])(?:%s)(?=[@.\s\S\]\[】&])" % groups, re.I)
|
||||
# 处理一个制作组识别多次的情况,保留顺序
|
||||
groups_re = re.compile(r"(?<=[-@\[£【&])(?:(?:%s))(?=[@.\s\S\]\[】&])" % groups, re.I)
|
||||
unique_groups = []
|
||||
for item in re.findall(groups_re, title):
|
||||
if item not in unique_groups:
|
||||
unique_groups.append(item)
|
||||
item_str = item[0] if isinstance(item, tuple) else item
|
||||
if item_str not in unique_groups:
|
||||
unique_groups.append(item_str)
|
||||
|
||||
return "@".join(unique_groups)
|
||||
|
||||
@@ -108,7 +108,7 @@ class SubscribeOper(DbOper):
|
||||
"""
|
||||
获取订阅
|
||||
"""
|
||||
return await Subscribe.async_get(self._db, id=sid)
|
||||
return await Subscribe.async_get(self._db, rid=sid)
|
||||
|
||||
def list(self, state: Optional[str] = None) -> List[Subscribe]:
|
||||
"""
|
||||
|
||||
@@ -1,9 +1,12 @@
|
||||
import uuid
|
||||
from typing import Callable, Any, Optional
|
||||
|
||||
from cf_clearance import sync_cf_retry, sync_stealth
|
||||
from playwright.sync_api import sync_playwright, Page
|
||||
|
||||
from app.core.config import settings
|
||||
from app.log import logger
|
||||
from app.utils.http import RequestUtils, cookie_parse
|
||||
|
||||
|
||||
class PlaywrightHelper:
|
||||
@@ -19,13 +22,120 @@ class PlaywrightHelper:
|
||||
page.goto(url)
|
||||
return sync_cf_retry(page)[0]
|
||||
|
||||
@staticmethod
|
||||
def __fs_cookie_str(cookies: list) -> str:
|
||||
if not cookies:
|
||||
return ""
|
||||
return "; ".join([f"{c.get('name')}={c.get('value')}" for c in cookies if c and c.get('name') is not None])
|
||||
|
||||
@staticmethod
|
||||
def __flaresolverr_request(url: str,
|
||||
cookies: Optional[str] = None,
|
||||
proxy_config: Optional[dict] = None,
|
||||
timeout: Optional[int] = 60) -> Optional[dict]:
|
||||
"""
|
||||
调用 FlareSolverr 解决 Cloudflare 并返回 solution 结果
|
||||
参考: https://github.com/FlareSolverr/FlareSolverr
|
||||
"""
|
||||
if not settings.FLARESOLVERR_URL:
|
||||
logger.warn("未配置 FLARESOLVERR_URL,无法使用 FlareSolverr")
|
||||
return None
|
||||
|
||||
fs_api = settings.FLARESOLVERR_URL.rstrip("/") + "/v1"
|
||||
session_id = None
|
||||
|
||||
try:
|
||||
# 检查是否需要代理认证
|
||||
need_proxy_auth = (proxy_config and proxy_config.get("server") and
|
||||
(proxy_config.get("username") or proxy_config.get("password")))
|
||||
|
||||
if need_proxy_auth:
|
||||
# 使用 session 模式支持代理认证
|
||||
logger.debug("检测到flaresolverr代理需要认证,使用 session 模式")
|
||||
|
||||
# 1. 创建会话
|
||||
session_id = str(uuid.uuid4())
|
||||
create_payload: dict = {
|
||||
"cmd": "sessions.create",
|
||||
"session": session_id
|
||||
}
|
||||
|
||||
# 添加代理配置到会话创建请求
|
||||
if proxy_config and proxy_config.get("server"):
|
||||
proxy_payload: dict = {"url": proxy_config["server"]}
|
||||
if proxy_config.get("username"):
|
||||
proxy_payload["username"] = proxy_config["username"]
|
||||
if proxy_config.get("password"):
|
||||
proxy_payload["password"] = proxy_config["password"]
|
||||
create_payload["proxy"] = proxy_payload
|
||||
|
||||
# 创建会话
|
||||
create_result = RequestUtils(content_type="application/json",
|
||||
timeout=timeout or 60).post_json(url=fs_api, json=create_payload)
|
||||
if not create_result or create_result.get("status") != "ok":
|
||||
logger.error(
|
||||
f"创建 FlareSolverr 会话失败: {create_result.get('message') if create_result else '无响应'}")
|
||||
return None
|
||||
|
||||
# 2. 使用会话发送请求
|
||||
request_payload = {
|
||||
"cmd": "request.get",
|
||||
"url": url,
|
||||
"session": session_id,
|
||||
"maxTimeout": int(timeout or 60) * 1000,
|
||||
}
|
||||
else:
|
||||
# 使用普通模式(无代理认证)
|
||||
request_payload = {
|
||||
"cmd": "request.get",
|
||||
"url": url,
|
||||
"maxTimeout": int(timeout or 60) * 1000,
|
||||
}
|
||||
# 添加代理配置(仅 URL,无认证)
|
||||
if proxy_config and proxy_config.get("server"):
|
||||
request_payload["proxy"] = {"url": proxy_config["server"]}
|
||||
|
||||
# 将 cookies 以数组形式传递给 FlareSolverr
|
||||
if cookies:
|
||||
try:
|
||||
request_payload["cookies"] = cookie_parse(cookies, array=True)
|
||||
except Exception as e:
|
||||
logger.debug(f"解析 cookies 失败,忽略: {str(e)}")
|
||||
|
||||
# 发送请求
|
||||
data = RequestUtils(content_type="application/json",
|
||||
timeout=timeout or 60).post_json(url=fs_api, json=request_payload)
|
||||
if not data:
|
||||
logger.error("FlareSolverr 返回空响应")
|
||||
return None
|
||||
if data.get("status") != "ok":
|
||||
logger.error(f"FlareSolverr 调用失败: {data.get('message')}")
|
||||
return None
|
||||
return data.get("solution")
|
||||
except Exception as e:
|
||||
logger.error(f"调用 FlareSolverr 失败: {str(e)}")
|
||||
return None
|
||||
finally:
|
||||
# 清理会话
|
||||
if session_id:
|
||||
try:
|
||||
destroy_payload = {
|
||||
"cmd": "sessions.destroy",
|
||||
"session": session_id
|
||||
}
|
||||
RequestUtils(content_type="application/json",
|
||||
timeout=10).post_json(url=fs_api, json=destroy_payload)
|
||||
logger.debug(f"已清理 FlareSolverr 会话: {session_id}")
|
||||
except Exception as e:
|
||||
logger.warning(f"清理 FlareSolverr 会话失败: {str(e)}")
|
||||
|
||||
def action(self, url: str,
|
||||
callback: Callable,
|
||||
cookies: Optional[str] = None,
|
||||
ua: Optional[str] = None,
|
||||
proxies: Optional[dict] = None,
|
||||
headless: Optional[bool] = False,
|
||||
timeout: Optional[int] = 30) -> Any:
|
||||
timeout: Optional[int] = 60) -> Any:
|
||||
"""
|
||||
访问网页,接收Page对象并执行操作
|
||||
:param url: 网页地址
|
||||
@@ -43,15 +153,30 @@ class PlaywrightHelper:
|
||||
context = None
|
||||
page = None
|
||||
try:
|
||||
# 如果配置使用 FlareSolverr,先通过其获取清除后的 cookies 与 UA
|
||||
fs_cookie_header = None
|
||||
fs_ua = None
|
||||
if settings.BROWSER_EMULATION == "flaresolverr":
|
||||
solution = self.__flaresolverr_request(url=url, cookies=cookies,
|
||||
proxy_config=proxies, timeout=timeout)
|
||||
if solution:
|
||||
fs_cookie_header = self.__fs_cookie_str(solution.get("cookies", []))
|
||||
fs_ua = solution.get("userAgent")
|
||||
|
||||
browser = playwright[self.browser_type].launch(headless=headless)
|
||||
context = browser.new_context(user_agent=ua, proxy=proxies)
|
||||
context = browser.new_context(user_agent=fs_ua or ua, proxy=proxies)
|
||||
page = context.new_page()
|
||||
|
||||
if cookies:
|
||||
page.set_extra_http_headers({"cookie": cookies})
|
||||
# 优先使用 FlareSolverr 返回,其次使用入参
|
||||
merged_cookie = fs_cookie_header or cookies
|
||||
if merged_cookie:
|
||||
page.set_extra_http_headers({"cookie": merged_cookie})
|
||||
|
||||
if not self.__pass_cloudflare(url, page):
|
||||
logger.warn("cloudflare challenge fail!")
|
||||
if settings.BROWSER_EMULATION == "playwright":
|
||||
if not self.__pass_cloudflare(url, page):
|
||||
logger.warn("cloudflare challenge fail!")
|
||||
else:
|
||||
page.goto(url)
|
||||
page.wait_for_load_state("networkidle", timeout=timeout * 1000)
|
||||
|
||||
# 回调函数
|
||||
@@ -76,7 +201,7 @@ class PlaywrightHelper:
|
||||
ua: Optional[str] = None,
|
||||
proxies: Optional[dict] = None,
|
||||
headless: Optional[bool] = False,
|
||||
timeout: Optional[int] = 20) -> Optional[str]:
|
||||
timeout: Optional[int] = 60) -> Optional[str]:
|
||||
"""
|
||||
获取网页源码
|
||||
:param url: 网页地址
|
||||
@@ -87,6 +212,15 @@ class PlaywrightHelper:
|
||||
:param timeout: 超时时间
|
||||
"""
|
||||
source = None
|
||||
# 如果配置为 FlareSolverr,则直接调用获取页面源码
|
||||
if settings.BROWSER_EMULATION == "flaresolverr":
|
||||
try:
|
||||
solution = self.__flaresolverr_request(url=url, cookies=cookies,
|
||||
proxy_config=proxies, timeout=timeout)
|
||||
if solution:
|
||||
return solution.get("response")
|
||||
except Exception as e:
|
||||
logger.error(f"FlareSolverr 获取源码失败: {str(e)}")
|
||||
try:
|
||||
with sync_playwright() as playwright:
|
||||
browser = None
|
||||
@@ -121,13 +255,3 @@ class PlaywrightHelper:
|
||||
logger.error(f"Playwright初始化失败: {str(e)}")
|
||||
|
||||
return source
|
||||
|
||||
|
||||
# 示例用法
|
||||
if __name__ == "__main__":
|
||||
utils = PlaywrightHelper()
|
||||
test_url = "https://piggo.me"
|
||||
test_cookies = ""
|
||||
test_user_agent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/93.0.4577.63 Safari/537.36"
|
||||
source_code = utils.get_page_source(test_url, cookies=test_cookies, ua=test_user_agent)
|
||||
print(source_code)
|
||||
|
||||
@@ -74,7 +74,8 @@ class CookieHelper:
|
||||
username: str,
|
||||
password: str,
|
||||
two_step_code: Optional[str] = None,
|
||||
proxies: Optional[dict] = None) -> Tuple[Optional[str], Optional[str], str]:
|
||||
proxies: Optional[dict] = None,
|
||||
timeout: int = None) -> Tuple[Optional[str], Optional[str], str]:
|
||||
"""
|
||||
获取站点cookie和ua
|
||||
:param url: 站点地址
|
||||
@@ -82,6 +83,7 @@ class CookieHelper:
|
||||
:param password: 密码
|
||||
:param two_step_code: 二步验证码或密钥
|
||||
:param proxies: 代理
|
||||
:param timeout: 超时时间
|
||||
:return: cookie、ua、message
|
||||
"""
|
||||
|
||||
@@ -230,7 +232,8 @@ class CookieHelper:
|
||||
|
||||
return PlaywrightHelper().action(url=url,
|
||||
callback=__page_handler,
|
||||
proxies=proxies)
|
||||
proxies=proxies,
|
||||
timeout=timeout)
|
||||
|
||||
@staticmethod
|
||||
def __get_captcha_text(cookie: str, ua: str, code_url: str) -> str:
|
||||
|
||||
@@ -429,13 +429,14 @@ class RssHelper:
|
||||
|
||||
return ret_array
|
||||
|
||||
def get_rss_link(self, url: str, cookie: str, ua: str, proxy: bool = False) -> Tuple[str, str]:
|
||||
def get_rss_link(self, url: str, cookie: str, ua: str, proxy: bool = False, timeout: int = None) -> Tuple[str, str]:
|
||||
"""
|
||||
获取站点rss地址
|
||||
:param url: 站点地址
|
||||
:param cookie: 站点cookie
|
||||
:param ua: 站点ua
|
||||
:param proxy: 是否使用代理
|
||||
:param timeout: 请求超时时间
|
||||
:return: rss地址、错误信息
|
||||
"""
|
||||
try:
|
||||
@@ -453,12 +454,13 @@ class RssHelper:
|
||||
url=rss_url,
|
||||
cookies=cookie,
|
||||
ua=ua,
|
||||
proxies=settings.PROXY if proxy else None
|
||||
proxies=settings.PROXY_SERVER if proxy else None,
|
||||
timeout=timeout or 60
|
||||
)
|
||||
else:
|
||||
res = RequestUtils(
|
||||
cookies=cookie,
|
||||
timeout=60,
|
||||
timeout=timeout or 30,
|
||||
ua=ua,
|
||||
proxies=settings.PROXY if proxy else None
|
||||
).post_res(url=rss_url, data=rss_params)
|
||||
|
||||
@@ -183,8 +183,11 @@ class HddolbySpider:
|
||||
timeout=self._timeout
|
||||
).post_res(url=self._searchurl, json=params)
|
||||
if res and res.status_code == 200:
|
||||
results = res.json().get('data', []) or []
|
||||
return False, self.__parse_result(results)
|
||||
result = res.json()
|
||||
if result.get("error"):
|
||||
logger.warn(f"{self._name} 搜索失败,错误信息:{result.get('error').get('message')}")
|
||||
return True, []
|
||||
return False, self.__parse_result(result.get('data'))
|
||||
elif res is not None:
|
||||
logger.warn(f"{self._name} 搜索失败,错误码:{res.status_code}")
|
||||
return True, []
|
||||
@@ -212,8 +215,11 @@ class HddolbySpider:
|
||||
timeout=self._timeout
|
||||
).post_res(url=self._searchurl, json=params)
|
||||
if res and res.status_code == 200:
|
||||
results = res.json().get('data', []) or []
|
||||
return False, self.__parse_result(results)
|
||||
result = res.json()
|
||||
if result.get("error"):
|
||||
logger.warn(f"{self._name} 搜索失败,错误信息:{result.get('error').get('message')}")
|
||||
return True, []
|
||||
return False, self.__parse_result(result.get('data'))
|
||||
elif res is not None:
|
||||
logger.warn(f"{self._name} 搜索失败,错误码:{res.status_code}")
|
||||
return True, []
|
||||
|
||||
@@ -5,13 +5,14 @@ from app.core.cache import cached
|
||||
from app.core.config import settings
|
||||
from app.log import logger
|
||||
from app.utils.http import RequestUtils, AsyncRequestUtils
|
||||
from app.utils.singleton import Singleton
|
||||
from app.utils.singleton import SingletonClass
|
||||
from app.utils.string import StringUtils
|
||||
|
||||
|
||||
class TNodeSpider(metaclass=Singleton):
|
||||
class TNodeSpider(metaclass=SingletonClass):
|
||||
_size = 100
|
||||
_timeout = 15
|
||||
_proxy = None
|
||||
_baseurl = "%sapi/torrent/advancedSearch"
|
||||
_downloadurl = "%sapi/torrent/download/%s"
|
||||
_pageurl = "%storrent/info/%s"
|
||||
@@ -53,7 +54,7 @@ class TNodeSpider(metaclass=Singleton):
|
||||
if res and res.status_code == 200:
|
||||
csrf_token = re.search(r'<meta name="x-csrf-token" content="(.+?)">', res.text)
|
||||
if csrf_token:
|
||||
_token = csrf_token.group(1)
|
||||
return csrf_token.group(1)
|
||||
return None
|
||||
|
||||
def __get_params(self, keyword: str = None, page: Optional[int] = 0) -> dict:
|
||||
@@ -154,7 +155,7 @@ class TNodeSpider(metaclass=Singleton):
|
||||
# 发送请求
|
||||
res = await AsyncRequestUtils(
|
||||
headers={
|
||||
'X-CSRF-TOKEN': _token,
|
||||
'x-csrf-token': _token,
|
||||
"Content-Type": "application/json; charset=utf-8",
|
||||
"User-Agent": f"{self._ua}"
|
||||
},
|
||||
|
||||
@@ -348,9 +348,13 @@ class TmdbApi:
|
||||
处理网站搜索得到的链接
|
||||
"""
|
||||
if len(tmdb_links) == 1:
|
||||
tmdbid = self._parse_tmdb_id_from_link(tmdb_links[0])
|
||||
if not tmdbid:
|
||||
logger.warn(f"无法从链接解析TMDBID:{tmdb_links[0]}")
|
||||
return {}
|
||||
tmdbinfo = get_info_func(
|
||||
mtype=MediaType.TV if tmdb_links[0].startswith("/tv") else MediaType.MOVIE,
|
||||
tmdbid=tmdb_links[0].split("/")[-1])
|
||||
tmdbid=tmdbid)
|
||||
if tmdbinfo:
|
||||
if mtype == MediaType.TV and tmdbinfo.get('media_type') != MediaType.TV:
|
||||
return {}
|
||||
@@ -368,9 +372,13 @@ class TmdbApi:
|
||||
处理网站搜索得到的链接(异步版本)
|
||||
"""
|
||||
if len(tmdb_links) == 1:
|
||||
tmdbid = self._parse_tmdb_id_from_link(tmdb_links[0])
|
||||
if not tmdbid:
|
||||
logger.warn(f"无法从链接解析TMDBID:{tmdb_links[0]}")
|
||||
return {}
|
||||
tmdbinfo = await self.async_get_info(
|
||||
mtype=MediaType.TV if tmdb_links[0].startswith("/tv") else MediaType.MOVIE,
|
||||
tmdbid=int(tmdb_links[0].split("/")[-1]))
|
||||
tmdbid=tmdbid)
|
||||
if tmdbinfo:
|
||||
if mtype == MediaType.TV and tmdbinfo.get('media_type') != MediaType.TV:
|
||||
return {}
|
||||
@@ -382,6 +390,22 @@ class TmdbApi:
|
||||
logger.info("%s TMDB网站未查询到媒体信息!" % name)
|
||||
return {}
|
||||
|
||||
@staticmethod
|
||||
def _parse_tmdb_id_from_link(link: str) -> Optional[int]:
|
||||
"""
|
||||
从 TMDB 相对链接中解析数值 ID。
|
||||
兼容格式:/movie/1195631-william-tell、/tv/65942-re、/tv/79744-the-rookie
|
||||
"""
|
||||
if not link:
|
||||
return None
|
||||
match = re.match(r"^/[^/]+/(\d+)", link)
|
||||
if match:
|
||||
try:
|
||||
return int(match.group(1))
|
||||
except Exception:
|
||||
return None
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def __get_names(tmdb_info: dict) -> List[str]:
|
||||
"""
|
||||
|
||||
@@ -108,7 +108,7 @@ class TransferInfo(BaseModel):
|
||||
success: bool = True
|
||||
# 整理⼁路径
|
||||
fileitem: Optional[FileItem] = None
|
||||
# 转移后的目录项
|
||||
# 转移后的目录项,媒体的根目录
|
||||
target_diritem: Optional[FileItem] = None
|
||||
# 转移后路径
|
||||
target_item: Optional[FileItem] = None
|
||||
|
||||
@@ -2,7 +2,7 @@ import re
|
||||
import sys
|
||||
from contextlib import contextmanager, asynccontextmanager
|
||||
from pathlib import Path
|
||||
from typing import Any, Optional, Union
|
||||
from typing import Any, Optional, Tuple, Union
|
||||
|
||||
import chardet
|
||||
import httpx
|
||||
@@ -395,7 +395,7 @@ class RequestUtils:
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def parse_cache_control(header: str) -> (str, int):
|
||||
def parse_cache_control(header: str) -> Tuple[str, Optional[int]]:
|
||||
"""
|
||||
解析 Cache-Control 头,返回 cache_directive 和 max_age
|
||||
:param header: Cache-Control 头部的字符串
|
||||
|
||||
@@ -1,2 +1,2 @@
|
||||
APP_VERSION = 'v2.7.0'
|
||||
FRONTEND_VERSION = 'v2.7.0'
|
||||
APP_VERSION = 'v2.7.1'
|
||||
FRONTEND_VERSION = 'v2.7.1'
|
||||
|
||||
Reference in New Issue
Block a user