Compare commits

..

24 Commits

Author SHA1 Message Date
jxxghp
4666b9051d 更新 version.py 2025-01-23 07:11:50 +08:00
jxxghp
56c524a822 Merge pull request #3792 from InfinityPacer/feature/cache 2025-01-23 06:55:31 +08:00
jxxghp
43e8df1b9f Merge pull request #3791 from InfinityPacer/feature/subscribe 2025-01-23 06:54:09 +08:00
InfinityPacer
dbc465f6e5 fix(cache): update tmdb match_web base_wait to 5 for finer control 2025-01-23 00:11:36 +08:00
InfinityPacer
bfbd3c527c fix(cache): ensure consistent parameter ordering in get_cache_key 2025-01-22 23:53:19 +08:00
InfinityPacer
412405f69b fix(subscribe): optimize site list retrieval in get_sub_sites 2025-01-22 23:22:15 +08:00
jxxghp
12b74eb04f 更新 subscribe.py 2025-01-22 22:50:51 +08:00
jxxghp
2305a6287a fix #3777 2025-01-22 22:23:15 +08:00
jxxghp
68245be081 fix meta 2025-01-22 22:20:17 +08:00
jxxghp
29e01294bd Merge pull request #3789 from InfinityPacer/feature/cache
fix(cache): enhance fanart image caching
2025-01-22 22:16:37 +08:00
InfinityPacer
d35bee54a6 fix(limit): log accurate wait time after triggering limit 2025-01-22 21:34:59 +08:00
InfinityPacer
bf63be18e4 fix(cache): enhance fanart image caching 2025-01-22 21:34:44 +08:00
jxxghp
3dc7adc61a 更新 scheduler.py 2025-01-22 19:39:02 +08:00
jxxghp
047d1e0afd Merge pull request #3788 from InfinityPacer/feature/cache
feat(cache): optimize serialization with type-based caching
2025-01-22 18:47:08 +08:00
InfinityPacer
7c017faf31 feat(cache): optimize serialization with type-based caching 2025-01-22 17:41:11 +08:00
jxxghp
7a59565761 fix 优化订阅匹配的识别量 2025-01-22 16:37:49 +08:00
jxxghp
9afb904d40 Merge pull request #3785 from InfinityPacer/feature/cache
fix(cache): enhance tmdb match_web rate-limiting and caching
2025-01-22 15:27:08 +08:00
jxxghp
8189de589a fix #3775 2025-01-22 15:21:10 +08:00
jxxghp
c458d7525d fix #3778 2025-01-22 15:01:24 +08:00
InfinityPacer
5c7bd95f6b fix(cache): enhance tmdb match_web rate-limiting and caching 2025-01-22 14:58:56 +08:00
InfinityPacer
70c4509682 feat(cache): add exists to check key presence in cache backends 2025-01-22 14:25:30 +08:00
jxxghp
f34e36c571 feat:Follow订阅分享人功能 2025-01-22 13:32:13 +08:00
jxxghp
5054ffe7e4 Merge pull request #3776 from kiri-to/patch-1 2025-01-21 19:38:30 +08:00
kiri-to
ed30933ca2 Update nexus_php.py
修复'站点数据刷新'时潜在429问题
2025-01-21 19:10:52 +08:00
16 changed files with 348 additions and 156 deletions

View File

@@ -15,10 +15,11 @@ from app.db import get_db
from app.db.models.subscribe import Subscribe
from app.db.models.subscribehistory import SubscribeHistory
from app.db.models.user import User
from app.db.systemconfig_oper import SystemConfigOper
from app.db.user_oper import get_current_active_user
from app.helper.subscribe import SubscribeHelper
from app.scheduler import Scheduler
from app.schemas.types import MediaType, EventType
from app.schemas.types import MediaType, EventType, SystemConfigKey
router = APIRouter()
@@ -497,6 +498,42 @@ def subscribe_fork(
return result
@router.get("/follow", summary="查询已Follow的订阅分享人", response_model=List[str])
def followed_subscribers(_: schemas.TokenPayload = Depends(verify_token)) -> Any:
"""
查询已Follow的订阅分享人
"""
return SystemConfigOper().get(SystemConfigKey.FollowSubscribers) or []
@router.post("/follow", summary="Follow订阅分享人", response_model=schemas.Response)
def follow_subscriber(
share_uid: str = None,
_: schemas.TokenPayload = Depends(verify_token)) -> Any:
"""
Follow订阅分享人
"""
subscribers = SystemConfigOper().get(SystemConfigKey.FollowSubscribers) or []
if share_uid and share_uid not in subscribers:
subscribers.append(share_uid)
SystemConfigOper().set(SystemConfigKey.FollowSubscribers, subscribers)
return schemas.Response(success=True)
@router.delete("/follow", summary="取消Follow订阅分享人", response_model=schemas.Response)
def unfollow_subscriber(
share_uid: str = None,
_: schemas.TokenPayload = Depends(verify_token)) -> Any:
"""
取消Follow订阅分享人
"""
subscribers = SystemConfigOper().get(SystemConfigKey.FollowSubscribers) or []
if share_uid and share_uid in subscribers:
subscribers.remove(share_uid)
SystemConfigOper().set(SystemConfigKey.FollowSubscribers, subscribers)
return schemas.Response(success=True)
@router.get("/shares", summary="查询分享的订阅", response_model=List[schemas.SubscribeShare])
def popular_subscribes(
name: str = None,

View File

@@ -6,6 +6,7 @@ import time
from datetime import datetime
from typing import Dict, List, Optional, Union, Tuple
from app import schemas
from app.chain import ChainBase
from app.chain.download import DownloadChain
from app.chain.media import MediaChain
@@ -27,8 +28,6 @@ from app.helper.message import MessageHelper
from app.helper.subscribe import SubscribeHelper
from app.helper.torrent import TorrentHelper
from app.log import logger
from app.schemas import NotExistMediaInfo, Notification, SubscrbieInfo, SubscribeEpisodeInfo, SubscribeDownloadFileInfo, \
SubscribeLibraryFileInfo
from app.schemas.types import MediaType, SystemConfigKey, MessageChannel, NotificationType, EventType
from app.utils.singleton import Singleton
@@ -173,14 +172,14 @@ class SubscribeChain(ChainBase, metaclass=Singleton):
logger.error(f'{mediainfo.title_year} {err_msg}')
if not exist_ok and message:
# 失败发回原用户
self.post_message(Notification(channel=channel,
source=source,
mtype=NotificationType.Subscribe,
title=f"{mediainfo.title_year} {metainfo.season} "
f"添加订阅失败!",
text=f"{err_msg}",
image=mediainfo.get_message_image(),
userid=userid))
self.post_message(schemas.Notification(channel=channel,
source=source,
mtype=NotificationType.Subscribe,
title=f"{mediainfo.title_year} {metainfo.season} "
f"添加订阅失败!",
text=f"{err_msg}",
image=mediainfo.get_message_image(),
userid=userid))
return None, err_msg
elif message:
logger.info(f'{mediainfo.title_year} {metainfo.season} 添加订阅成功')
@@ -193,12 +192,12 @@ class SubscribeChain(ChainBase, metaclass=Singleton):
else:
link = settings.MP_DOMAIN('#/subscribe/movie?tab=mysub')
# 订阅成功按规则发送消息
self.post_message(Notification(mtype=NotificationType.Subscribe,
title=f"{mediainfo.title_year} {metainfo.season} 已添加订阅",
text=text,
image=mediainfo.get_message_image(),
link=link,
username=username))
self.post_message(schemas.Notification(mtype=NotificationType.Subscribe,
title=f"{mediainfo.title_year} {metainfo.season} 已添加订阅",
text=text,
image=mediainfo.get_message_image(),
link=link,
username=username))
# 发送事件
EventManager().send_event(EventType.SubscribeAdded, {
"subscribe_id": sid,
@@ -409,7 +408,7 @@ class SubscribeChain(ChainBase, metaclass=Singleton):
def finish_subscribe_or_not(self, subscribe: Subscribe, meta: MetaBase, mediainfo: MediaInfo,
downloads: List[Context] = None,
lefts: Dict[Union[int | str], Dict[int, NotExistMediaInfo]] = None,
lefts: Dict[Union[int | str], Dict[int, schemas.NotExistMediaInfo]] = None,
force: bool = False):
"""
判断是否应完成订阅
@@ -464,18 +463,16 @@ class SubscribeChain(ChainBase, metaclass=Singleton):
"""
# 从系统配置获取默认订阅站点
default_sites = self.systemconfig.get(SystemConfigKey.RssSites) or []
# 如果订阅未指定站点信息,直接返回默认站点
# 如果订阅未指定站点,直接返回默认站点
if not subscribe.sites:
return default_sites
# 如果默认订阅站点未设置,直接返回订阅指定站点
if not default_sites:
return subscribe.sites or []
# 尝试解析订阅中的站点数据
user_sites = subscribe.sites
# 计算 user_sites 和 default_sites 的交集
intersection_sites = [site for site in user_sites if site in default_sites]
# 如果交集与原始订阅不一致,更新数据库
if set(intersection_sites) != set(user_sites):
self.subscribeoper.update(subscribe.id, {
"sites": intersection_sites
})
# 如果交集为空,返回默认站点
return intersection_sites if intersection_sites else default_sites
@@ -574,9 +571,9 @@ class SubscribeChain(ChainBase, metaclass=Singleton):
continue
# 有自定义识别词时,需要判断是否需要重新识别
apply_words = None
if custom_words_list:
_, apply_words = WordsMatcher().prepare(torrent_info.title,
# 使用org_string应用一次后理论上不能再次应用
_, apply_words = WordsMatcher().prepare(torrent_meta.org_string,
custom_words=custom_words_list)
if apply_words:
logger.info(
@@ -584,6 +581,8 @@ class SubscribeChain(ChainBase, metaclass=Singleton):
# 重新识别元数据
torrent_meta = MetaInfo(title=torrent_info.title, subtitle=torrent_info.description,
custom_words=custom_words_list)
# 更新元数据缓存
context.meta_info = torrent_meta
# 媒体信息需要重新识别
torrent_mediainfo = None
@@ -594,8 +593,7 @@ class SubscribeChain(ChainBase, metaclass=Singleton):
torrent_mediainfo = self.recognize_media(meta=torrent_meta)
if torrent_mediainfo:
# 更新种子缓存
if not apply_words:
context.media_info = torrent_mediainfo
context.media_info = torrent_mediainfo
else:
# 通过标题匹配兜底
logger.warn(
@@ -607,9 +605,7 @@ class SubscribeChain(ChainBase, metaclass=Singleton):
logger.info(
f'{mediainfo.title_year} 通过标题匹配到可选资源:{torrent_info.site_name} - {torrent_info.title}')
torrent_mediainfo = mediainfo
# 更新种子缓存
if not apply_words:
context.media_info = mediainfo
context.media_info = torrent_mediainfo
else:
continue
@@ -784,6 +780,63 @@ class SubscribeChain(ChainBase, metaclass=Singleton):
})
logger.info(f'{subscribe.name} 订阅元数据更新完成')
def follow(self):
"""
刷新follow的用户分享并自动添加订阅
"""
follow_users: List[str] = self.systemconfig.get(SystemConfigKey.FollowSubscribers)
if not follow_users:
return
share_subs = self.subscribehelper.get_shares()
logger.info(f'开始刷新follow用户分享订阅 ...')
success_count = 0
for share_sub in share_subs:
uid = share_sub.get("share_uid")
if uid and uid in follow_users:
# 订阅已存在则跳过
if self.subscribeoper.exists(tmdbid=share_sub.get("tmdbid"),
doubanid=share_sub.get("doubanid"),
season=share_sub.get("season")):
continue
# 去除无效属性
for key in list(share_sub.keys()):
if not hasattr(schemas.Subscribe(), key):
share_sub.pop(key)
# 类型转换
subscribe_in = schemas.Subscribe(**share_sub)
mtype = MediaType(subscribe_in.type)
# 豆瓣标题处理
if subscribe_in.doubanid or subscribe_in.bangumiid:
meta = MetaInfo(subscribe_in.name)
subscribe_in.name = meta.name
subscribe_in.season = meta.begin_season
# 标题转换
if subscribe_in.name:
title = subscribe_in.name
else:
title = None
sid, message = SubscribeChain().add(mtype=mtype,
title=title,
year=subscribe_in.year,
tmdbid=subscribe_in.tmdbid,
season=subscribe_in.season,
doubanid=subscribe_in.doubanid,
bangumiid=subscribe_in.bangumiid,
username="订阅分享",
best_version=subscribe_in.best_version,
save_path=subscribe_in.save_path,
search_imdbid=subscribe_in.search_imdbid,
custom_words=subscribe_in.custom_words,
media_category=subscribe_in.media_category,
filter_groups=subscribe_in.filter_groups,
exist_ok=True)
if sid:
success_count += 1
logger.info(f'follow用户分享订阅 {title} 添加成功')
else:
logger.error(f'follow用户分享订阅 {title} 添加失败:{message}')
logger.info(f'follow用户分享订阅刷新完成共添加 {success_count} 个订阅')
def __update_subscribe_note(self, subscribe: Subscribe, downloads: List[Context]):
"""
更新已下载信息到note字段
@@ -840,7 +893,7 @@ class SubscribeChain(ChainBase, metaclass=Singleton):
return note
return []
def __update_lack_episodes(self, lefts: Dict[Union[int, str], Dict[int, NotExistMediaInfo]],
def __update_lack_episodes(self, lefts: Dict[Union[int, str], Dict[int, schemas.NotExistMediaInfo]],
subscribe: Subscribe,
mediainfo: MediaInfo,
update_date: bool = False):
@@ -895,11 +948,11 @@ class SubscribeChain(ChainBase, metaclass=Singleton):
else:
link = settings.MP_DOMAIN('#/subscribe/movie?tab=mysub')
# 完成订阅按规则发送消息
self.post_message(Notification(mtype=NotificationType.Subscribe,
title=f'{mediainfo.title_year} {meta.season} 已完成{msgstr}',
image=mediainfo.get_message_image(),
link=link,
username=subscribe.username))
self.post_message(schemas.Notification(mtype=NotificationType.Subscribe,
title=f'{mediainfo.title_year} {meta.season} 已完成{msgstr}',
image=mediainfo.get_message_image(),
link=link,
username=subscribe.username))
# 发送事件
EventManager().send_event(EventType.SubscribeComplete, {
"subscribe_id": subscribe.id,
@@ -919,9 +972,9 @@ class SubscribeChain(ChainBase, metaclass=Singleton):
"""
subscribes = self.subscribeoper.list()
if not subscribes:
self.post_message(Notification(channel=channel,
source=source,
title='没有任何订阅!', userid=userid))
self.post_message(schemas.Notification(channel=channel,
source=source,
title='没有任何订阅!', userid=userid))
return
title = f"共有 {len(subscribes)} 个订阅,回复对应指令操作: " \
f"\n- 删除订阅:/subscribe_delete [id]" \
@@ -937,8 +990,8 @@ class SubscribeChain(ChainBase, metaclass=Singleton):
f"[{subscribe.total_episode - (subscribe.lack_episode or subscribe.total_episode)}"
f"/{subscribe.total_episode}]")
# 发送列表
self.post_message(Notification(channel=channel, source=source,
title=title, text='\n'.join(messages), userid=userid))
self.post_message(schemas.Notification(channel=channel, source=source,
title=title, text='\n'.join(messages), userid=userid))
def remote_delete(self, arg_str: str, channel: MessageChannel,
userid: Union[str, int] = None, source: str = None):
@@ -946,9 +999,9 @@ class SubscribeChain(ChainBase, metaclass=Singleton):
删除订阅
"""
if not arg_str:
self.post_message(Notification(channel=channel, source=source,
title="请输入正确的命令格式:/subscribe_delete [id]"
"[id]为订阅编号", userid=userid))
self.post_message(schemas.Notification(channel=channel, source=source,
title="请输入正确的命令格式:/subscribe_delete [id]"
"[id]为订阅编号", userid=userid))
return
arg_strs = str(arg_str).split()
for arg_str in arg_strs:
@@ -958,8 +1011,8 @@ class SubscribeChain(ChainBase, metaclass=Singleton):
subscribe_id = int(arg_str)
subscribe = self.subscribeoper.get(subscribe_id)
if not subscribe:
self.post_message(Notification(channel=channel, source=source,
title=f"订阅编号 {subscribe_id} 不存在!", userid=userid))
self.post_message(schemas.Notification(channel=channel, source=source,
title=f"订阅编号 {subscribe_id} 不存在!", userid=userid))
return
# 删除订阅
self.subscribeoper.delete(subscribe_id)
@@ -973,13 +1026,13 @@ class SubscribeChain(ChainBase, metaclass=Singleton):
@staticmethod
def __get_subscribe_no_exits(subscribe_name: str,
no_exists: Dict[Union[int, str], Dict[int, NotExistMediaInfo]],
no_exists: Dict[Union[int, str], Dict[int, schemas.NotExistMediaInfo]],
mediakey: Union[str, int],
begin_season: int,
total_episode: int,
start_episode: int,
downloaded_episodes: List[int] = None
) -> Tuple[bool, Dict[Union[int, str], Dict[int, NotExistMediaInfo]]]:
) -> Tuple[bool, Dict[Union[int, str], Dict[int, schemas.NotExistMediaInfo]]]:
"""
根据订阅开始集数和总集数结合TMDB信息计算当前订阅的缺失集数
:param subscribe_name: 订阅名称
@@ -1029,7 +1082,7 @@ class SubscribeChain(ChainBase, metaclass=Singleton):
# 与原集列表取交集
episodes = list(set(episode_list).intersection(set(new_episodes)))
# 更新集合
no_exists[mediakey][begin_season] = NotExistMediaInfo(
no_exists[mediakey][begin_season] = schemas.NotExistMediaInfo(
season=begin_season,
episodes=episodes,
total_episode=total_episode,
@@ -1056,7 +1109,7 @@ class SubscribeChain(ChainBase, metaclass=Singleton):
if not episodes:
return True, {}
# 更新集合
no_exists[mediakey][begin_season] = NotExistMediaInfo(
no_exists[mediakey][begin_season] = schemas.NotExistMediaInfo(
season=begin_season,
episodes=episodes,
total_episode=total,
@@ -1070,7 +1123,7 @@ class SubscribeChain(ChainBase, metaclass=Singleton):
# 如果存在已下载剧集,则差集为空时,说明所有均已存在
if not episodes:
return True, {}
no_exists[mediakey][begin_season] = NotExistMediaInfo(
no_exists[mediakey][begin_season] = schemas.NotExistMediaInfo(
season=begin_season,
episodes=episodes,
total_episode=total_episode,
@@ -1157,7 +1210,7 @@ class SubscribeChain(ChainBase, metaclass=Singleton):
"min_seeders_time": default_rule.get("min_seeders_time"),
}.items() if value is not None}
def subscribe_files_info(self, subscribe: Subscribe) -> Optional[SubscrbieInfo]:
def subscribe_files_info(self, subscribe: Subscribe) -> Optional[schemas.SubscrbieInfo]:
"""
订阅相关的下载和文件信息
"""
@@ -1165,10 +1218,10 @@ class SubscribeChain(ChainBase, metaclass=Singleton):
return
# 返回订阅数据
subscribe_info = SubscrbieInfo()
subscribe_info = schemas.SubscrbieInfo()
# 所有集的数据
episodes: Dict[int, SubscribeEpisodeInfo] = {}
episodes: Dict[int, schemas.SubscribeEpisodeInfo] = {}
if subscribe.tmdbid and subscribe.type == MediaType.TV.value:
# 查询TMDB中的集信息
tmdb_episodes = self.tmdbchain.tmdb_episodes(
@@ -1177,7 +1230,7 @@ class SubscribeChain(ChainBase, metaclass=Singleton):
)
if tmdb_episodes:
for episode in tmdb_episodes:
info = SubscribeEpisodeInfo()
info = schemas.SubscribeEpisodeInfo()
info.title = episode.name
info.description = episode.overview
info.backdrop = f"https://{settings.TMDB_IMAGE_DOMAIN}/t/p/w500${episode.still_path}"
@@ -1185,12 +1238,12 @@ class SubscribeChain(ChainBase, metaclass=Singleton):
elif subscribe.type == MediaType.TV.value:
# 根据开始结束集计算集信息
for i in range(subscribe.start_episode or 1, subscribe.total_episode + 1):
info = SubscribeEpisodeInfo()
info = schemas.SubscribeEpisodeInfo()
info.title = f'{i}'
episodes[i] = info
else:
# 电影
info = SubscribeEpisodeInfo()
info = schemas.SubscribeEpisodeInfo()
info.title = subscribe.name
episodes[0] = info
@@ -1205,7 +1258,7 @@ class SubscribeChain(ChainBase, metaclass=Singleton):
# 识别文件名
file_meta = MetaInfo(file.filepath)
# 下载文件信息
file_info = SubscribeDownloadFileInfo(
file_info = schemas.SubscribeDownloadFileInfo(
torrent_title=his.torrent_name,
site_name=his.torrent_site,
downloader=file.downloader,
@@ -1248,7 +1301,7 @@ class SubscribeChain(ChainBase, metaclass=Singleton):
# 识别文件名
file_meta = MetaInfo(fileitem.path)
# 媒体库文件信息
file_info = SubscribeLibraryFileInfo(
file_info = schemas.SubscribeLibraryFileInfo(
storage=fileitem.storage,
file_path=fileitem.path,
)
@@ -1308,7 +1361,7 @@ class SubscribeChain(ChainBase, metaclass=Singleton):
# 对于电视剧,构造缺失的媒体信息
no_exists = {
mediakey: {
subscribe.season: NotExistMediaInfo(
subscribe.season: schemas.NotExistMediaInfo(
season=subscribe.season,
episodes=[],
total_episode=subscribe.total_episode,

View File

@@ -453,9 +453,12 @@ class TransferChain(ChainBase, metaclass=Singleton):
if transferinfo.transfer_type in ["move"]:
# 所有成功的业务
tasks = self.jobview.success_tasks(task.mediainfo, task.meta.begin_season)
# 记录已处理的种子hash
processed_hashes = set()
for t in tasks:
# 下载器hash
if t.download_hash:
if t.download_hash and t.download_hash not in processed_hashes:
processed_hashes.add(t.download_hash)
if self.remove_torrents(t.download_hash, downloader=t.downloader):
logger.info(f"移动模式删除种子成功:{t.download_hash} ")
# 删除残留目录

View File

@@ -35,6 +35,17 @@ class CacheBackend(ABC):
"""
pass
@abstractmethod
def exists(self, key: str, region: str = DEFAULT_CACHE_REGION) -> bool:
"""
判断缓存键是否存在
:param key: 缓存的键
:param region: 缓存的区
:return: 存在返回 True否则返回 False
"""
pass
@abstractmethod
def get(self, key: str, region: str = DEFAULT_CACHE_REGION) -> Any:
"""
@@ -79,6 +90,30 @@ class CacheBackend(ABC):
"""
return f"region:{region}" if region else "region:default"
@staticmethod
def get_cache_key(func, args, kwargs):
"""
获取缓存的键,通过哈希函数对函数的参数进行处理
:param func: 被装饰的函数
:param args: 位置参数
:param kwargs: 关键字参数
:return: 缓存键
"""
signature = inspect.signature(func)
# 绑定传入的参数并应用默认值
bound = signature.bind(*args, **kwargs)
bound.apply_defaults()
# 忽略第一个参数,如果它是实例(self)或类(cls)
parameters = list(signature.parameters.keys())
if parameters and parameters[0] in ("self", "cls"):
bound.arguments.pop(parameters[0], None)
# 按照函数签名顺序提取参数值列表
keys = [
bound.arguments[param] for param in signature.parameters if param in bound.arguments
]
# 使用有序参数生成缓存键
return f"{func.__name__}_{hashkey(*keys)}"
class CacheToolsBackend(CacheBackend):
"""
@@ -130,6 +165,19 @@ class CacheToolsBackend(CacheBackend):
# 设置缓存值
region_cache[key] = value
def exists(self, key: str, region: str = DEFAULT_CACHE_REGION) -> bool:
"""
判断缓存键是否存在
:param key: 缓存的键
:param region: 缓存的区
:return: 存在返回 True否则返回 False
"""
region_cache = self.__get_region_cache(region)
if region_cache is None:
return False
return key in region_cache
def get(self, key: str, region: str = DEFAULT_CACHE_REGION) -> Any:
"""
获取缓存的值
@@ -194,6 +242,10 @@ class RedisBackend(CacheBackend):
- Pickle 反序列化可能存在安全风险,需进一步重构调用来源,避免复杂对象缓存
"""
# 类型缓存集合,针对非容器简单类型
_complex_serializable_types = set()
_simple_serializable_types = set()
def __init__(self, redis_url: str = "redis://localhost", ttl: int = 1800):
"""
初始化 Redis 缓存实例
@@ -234,19 +286,42 @@ class RedisBackend(CacheBackend):
logger.error(f"Failed to set Redis maxmemory or policy: {e}")
@staticmethod
def serialize(value: Any) -> bytes:
def is_container_type(t):
return t in (list, dict, tuple, set)
@classmethod
def serialize(cls, value: Any) -> bytes:
"""
将值序列化为二进制数据,根据序列化方式标识格式
"""
try:
# 尝试 JSON 序列化
return b"JSON" + b"\x00" + json.dumps(value).encode("utf-8")
except TypeError:
# 如果 JSON 序列化失败,使用 Pickle 序列化
return b"PICKLE" + b"\x00" + pickle.dumps(value)
vt = type(value)
# 针对非容器类型使用缓存策略
if not cls.is_container_type(vt):
# 如果已知需要复杂序列化
if vt in cls._complex_serializable_types:
return b"PICKLE" + b"\x00" + pickle.dumps(value)
# 如果已知可以简单序列化
if vt in cls._simple_serializable_types:
json_data = json.dumps(value).encode("utf-8")
return b"JSON" + b"\x00" + json_data
# 对于未知的非容器类型,尝试简单序列化,如抛出异常,再使用复杂序列化
try:
json_data = json.dumps(value).encode("utf-8")
cls._simple_serializable_types.add(vt)
return b"JSON" + b"\x00" + json_data
except TypeError:
cls._complex_serializable_types.add(vt)
return b"PICKLE" + b"\x00" + pickle.dumps(value)
# 针对容器类型,每次尝试简单序列化,不使用缓存
else:
try:
json_data = json.dumps(value).encode("utf-8")
return b"JSON" + b"\x00" + json_data
except TypeError:
return b"PICKLE" + b"\x00" + pickle.dumps(value)
@staticmethod
def deserialize(value: bytes) -> Any:
@classmethod
def deserialize(cls, value: bytes) -> Any:
"""
将二进制数据反序列化为原始值,根据格式标识区分序列化方式
"""
@@ -294,6 +369,21 @@ class RedisBackend(CacheBackend):
except Exception as e:
logger.error(f"Failed to set key: {key} in region: {region}, error: {e}")
def exists(self, key: str, region: str = DEFAULT_CACHE_REGION) -> bool:
"""
判断缓存键是否存在
:param key: 缓存的键
:param region: 缓存的区
:return: 存在返回 True否则返回 False
"""
try:
redis_key = self.get_redis_key(region, key)
return self.client.exists(redis_key) == 1
except Exception as e:
logger.error(f"Failed to exists key: {key} region: {region}, error: {e}")
return False
def get(self, key: str, region: str = DEFAULT_CACHE_REGION) -> Optional[Any]:
"""
获取缓存的值
@@ -392,7 +482,7 @@ def cached(region: Optional[str] = None, maxsize: int = 1000, ttl: int = 1800,
:param maxsize: 缓存的最大条目数,默认值为 1000
:param ttl: 缓存的存活时间,单位秒,默认值为 1800
:param skip_none: 跳过 None 缓存,默认为 True
:param skip_empty: 跳过空值缓存(如 [], {}, "", set()),默认为 False
:param skip_empty: 跳过空值缓存(如 None, [], {}, "", set()),默认为 False
:return: 装饰器函数
"""
@@ -405,34 +495,11 @@ def cached(region: Optional[str] = None, maxsize: int = 1000, ttl: int = 1800,
"""
if skip_none and value is None:
return False
# if disable_empty and value in [[], {}, "", set()]:
# if skip_empty and value in [None, [], {}, "", set()]:
if skip_empty and not value:
return False
return True
def get_cache_key(func, args, kwargs):
"""
获取缓存的键,通过哈希函数对函数的参数进行处理
:param func: 被装饰的函数
:param args: 位置参数
:param kwargs: 关键字参数
:return: 缓存键
"""
# 获取方法签名
signature = inspect.signature(func)
resolved_kwargs = {}
# 获取默认值并结合传递的参数(如果有)
for param, value in signature.parameters.items():
if param in kwargs:
# 使用显式传递的参数
resolved_kwargs[param] = kwargs[param]
elif value.default is not inspect.Parameter.empty:
# 没有传递参数时使用默认值
resolved_kwargs[param] = value.default
# 构造缓存键忽略实例self 或 cls
params_to_hash = args[1:] if len(args) > 1 else []
return f"{func.__name__}_{hashkey(*params_to_hash, **resolved_kwargs)}"
def decorator(func):
# 获取缓存区
@@ -441,7 +508,7 @@ def cached(region: Optional[str] = None, maxsize: int = 1000, ttl: int = 1800,
@wraps(func)
def wrapper(*args, **kwargs):
# 获取缓存键
cache_key = get_cache_key(func, args, kwargs)
cache_key = cache_backend.get_cache_key(func, args, kwargs)
# 尝试获取缓存
cached_value = cache_backend.get(cache_key, region=cache_region)
if should_cache(cached_value):

View File

@@ -259,9 +259,6 @@ class MetaBase(object):
except Exception as err:
logger.debug(f'识别集失败:{str(err)} - {traceback.format_exc()}')
return
if self.total_episode:
self.begin_episode = 1
self.end_episode = self.total_episode
self.type = MediaType.TV
self._subtitle_flag = True
return

View File

@@ -182,7 +182,7 @@ class SubscribeHelper(metaclass=Singleton):
return False, res.json().get("message")
@cached(region=_shares_cache_region)
def get_shares(self, name: str, page: int = 1, count: int = 30) -> List[dict]:
def get_shares(self, name: str = None, page: int = 1, count: int = 30) -> List[dict]:
"""
获取订阅分享数据
"""

View File

@@ -420,7 +420,7 @@ class FanartModule(_ModuleBase):
return result
@classmethod
@cached(maxsize=settings.CACHE_CONF["fanart"], ttl=settings.CACHE_CONF["meta"])
@cached(maxsize=settings.CACHE_CONF["fanart"], ttl=settings.CACHE_CONF["meta"], skip_none=False)
def __request_fanart(cls, media_type: MediaType, queryid: Union[str, int]) -> Optional[dict]:
if media_type == MediaType.MOVIE:
image_url = cls._movie_url % queryid

View File

@@ -368,7 +368,7 @@ class FileManagerModule(_ModuleBase):
# 覆盖模式
overwrite_mode = target_directory.overwrite_mode
# 是否需要刮削
need_scrape = scrape or target_directory.scraping
need_scrape = target_directory.scraping if scrape is None else scrape
# 目标存储类型
if not target_storage:
target_storage = target_directory.library_storage

View File

@@ -102,7 +102,7 @@ class StorageBase(metaclass=ABCMeta):
"""
获取父目录
"""
return self.get_folder(Path(fileitem.path).parent)
return self.get_item(Path(fileitem.path).parent)
@abstractmethod
def delete(self, fileitem: schemas.FileItem) -> bool:

View File

@@ -208,9 +208,16 @@ class NexusPhpSiteUserInfo(SiteParserBase):
# 是否存在下页数据
next_page = None
next_page_text = html.xpath('//a[contains(.//text(), "下一页") or contains(.//text(), "下一頁") or contains(.//text(), ">")]/@href')
if next_page_text:
next_page = next_page_text[-1].strip()
# fix up page url
#防止识别到详情页
while next_page_text:
next_page = next_page_text.pop().strip()
if not next_page.startswith('details.php'):
break;
next_page = None
# fix up page url
if next_page:
if self.userid not in next_page:
next_page = f'{next_page}&userid={self.userid}&type=seeding'

View File

@@ -340,7 +340,7 @@ class Slack:
return ""
conversation_id = ""
try:
for result in self._client.conversations_list():
for result in self._client.conversations_list(types="public_channel,private_channel"):
if conversation_id:
break
for channel in result["channels"]:

View File

@@ -8,8 +8,10 @@ from lxml import etree
from app.core.cache import cached
from app.core.config import settings
from app.log import logger
from app.schemas import APIRateLimitException
from app.schemas.types import MediaType
from app.utils.http import RequestUtils
from app.utils.limit import rate_limit_exponential
from app.utils.string import StringUtils
from .tmdbv3api import TMDb, Search, Movie, TV, Season, Episode, Discover, Trending, Person, Collection
from .tmdbv3api.exceptions import TMDbException
@@ -492,6 +494,7 @@ class TmdbApi:
return ret_info
@cached(maxsize=settings.CACHE_CONF["tmdb"], ttl=settings.CACHE_CONF["meta"])
@rate_limit_exponential(source="match_tmdb_web", base_wait=5, max_wait=1800, enable_logging=True)
def match_web(self, name: str, mtype: MediaType) -> Optional[dict]:
"""
搜索TMDB网站直接抓取结果结果只有一条时才返回
@@ -504,51 +507,56 @@ class TmdbApi:
return {}
logger.info("正在从TheDbMovie网站查询%s ..." % name)
tmdb_url = "https://www.themoviedb.org/search?query=%s" % quote(name)
res = RequestUtils(timeout=5, ua=settings.USER_AGENT).get_res(url=tmdb_url)
if res and res.status_code == 200:
html_text = res.text
if not html_text:
return None
try:
tmdb_links = []
html = etree.HTML(html_text)
if mtype == MediaType.TV:
links = html.xpath("//a[@data-id and @data-media-type='tv']/@href")
else:
links = html.xpath("//a[@data-id]/@href")
for link in links:
if not link or (not link.startswith("/tv") and not link.startswith("/movie")):
continue
if link not in tmdb_links:
tmdb_links.append(link)
if len(tmdb_links) == 1:
tmdbinfo = self.get_info(
mtype=MediaType.TV if tmdb_links[0].startswith("/tv") else MediaType.MOVIE,
tmdbid=tmdb_links[0].split("/")[-1])
if tmdbinfo:
if mtype == MediaType.TV and tmdbinfo.get('media_type') != MediaType.TV:
return {}
if tmdbinfo.get('media_type') == MediaType.MOVIE:
logger.info("%s 从WEB识别到 电影TMDBID=%s, 名称=%s, 上映日期=%s" % (
name,
tmdbinfo.get('id'),
tmdbinfo.get('title'),
tmdbinfo.get('release_date')))
else:
logger.info("%s 从WEB识别到 电视剧TMDBID=%s, 名称=%s, 首播日期=%s" % (
name,
tmdbinfo.get('id'),
tmdbinfo.get('name'),
tmdbinfo.get('first_air_date')))
return tmdbinfo
elif len(tmdb_links) > 1:
logger.info("%s TMDB网站返回数据过多%s" % (name, len(tmdb_links)))
else:
logger.info("%s TMDB网站未查询到媒体信息" % name)
except Exception as err:
logger.error(f"从TheDbMovie网站查询出错{str(err)}")
return None
return None
res = RequestUtils(timeout=5, ua=settings.USER_AGENT, proxies=settings.PROXY).get_res(url=tmdb_url)
if res is None:
return None
if res.status_code == 429:
raise APIRateLimitException("触发TheDbMovie网站限流获取媒体信息失败")
if res.status_code != 200:
return {}
html_text = res.text
if not html_text:
return {}
try:
tmdb_links = []
html = etree.HTML(html_text)
if mtype == MediaType.TV:
links = html.xpath("//a[@data-id and @data-media-type='tv']/@href")
else:
links = html.xpath("//a[@data-id]/@href")
for link in links:
if not link or (not link.startswith("/tv") and not link.startswith("/movie")):
continue
if link not in tmdb_links:
tmdb_links.append(link)
if len(tmdb_links) == 1:
tmdbinfo = self.get_info(
mtype=MediaType.TV if tmdb_links[0].startswith("/tv") else MediaType.MOVIE,
tmdbid=tmdb_links[0].split("/")[-1])
if tmdbinfo:
if mtype == MediaType.TV and tmdbinfo.get('media_type') != MediaType.TV:
return {}
if tmdbinfo.get('media_type') == MediaType.MOVIE:
logger.info("%s 从WEB识别到 电影TMDBID=%s, 名称=%s, 上映日期=%s" % (
name,
tmdbinfo.get('id'),
tmdbinfo.get('title'),
tmdbinfo.get('release_date')))
else:
logger.info("%s 从WEB识别到 电视剧TMDBID=%s, 名称=%s, 首播日期=%s" % (
name,
tmdbinfo.get('id'),
tmdbinfo.get('name'),
tmdbinfo.get('first_air_date')))
return tmdbinfo
elif len(tmdb_links) > 1:
logger.info("%s TMDB网站返回数据过多%s" % (name, len(tmdb_links)))
else:
logger.info("%s TMDB网站未查询到媒体信息" % name)
except Exception as err:
logger.error(f"从TheDbMovie网站查询出错{str(err)}")
return {}
return {}
def get_info(self,
mtype: MediaType,

View File

@@ -92,6 +92,11 @@ class Scheduler(metaclass=Singleton):
"func": SubscribeChain().refresh,
"running": False,
},
"subscribe_follow": {
"name": "关注的订阅分享",
"func": SubscribeChain().follow,
"running": False,
},
"transfer": {
"name": "下载文件整理",
"func": TransferChain().process,
@@ -241,6 +246,18 @@ class Scheduler(metaclass=Singleton):
}
)
# 关注订阅分享每1小时
self._scheduler.add_job(
self.start,
"interval",
id="subscribe_follow",
name="关注的订阅分享",
hours=1,
kwargs={
'job_id': 'subscribe_follow'
}
)
# 下载器文件转移每5分钟
self._scheduler.add_job(
self.start,

View File

@@ -135,6 +135,8 @@ class SystemConfigKey(Enum):
DefaultTvSubscribeConfig = "DefaultTvSubscribeConfig"
# 用户站点认证参数
UserSiteAuthParams = "UserSiteAuthParams"
# Follow订阅分享者
FollowSubscribers = "FollowSubscribers"
# 处理进度Key字典

View File

@@ -156,7 +156,8 @@ class ExponentialBackoffRateLimiter(BaseRateLimiter):
with self.lock:
self.next_allowed_time = current_time + self.current_wait
self.current_wait = min(self.current_wait * self.backoff_factor, self.max_wait)
self.log_warning(f"触发限流,将在 {self.current_wait} 秒后允许继续调用")
wait_time = self.next_allowed_time - current_time
self.log_warning(f"触发限流,将在 {wait_time:.2f} 秒后允许继续调用")
# 时间窗口限流器

View File

@@ -1,2 +1,2 @@
APP_VERSION = 'v2.2.3'
FRONTEND_VERSION = 'v2.2.3'
APP_VERSION = 'v2.2.4'
FRONTEND_VERSION = 'v2.2.4'