mirror of
https://github.com/jxxghp/MoviePilot.git
synced 2026-06-13 07:26:45 +00:00
Compare commits
374 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
48b1ac28de | ||
|
|
6e329b17a9 | ||
|
|
6a492198a8 | ||
|
|
8bf9b6e7cb | ||
|
|
42e23ef564 | ||
|
|
c6806ee648 | ||
|
|
076fae696c | ||
|
|
ed294d3ea4 | ||
|
|
043be409d0 | ||
|
|
a5e7483870 | ||
|
|
365335be46 | ||
|
|
62543dd171 | ||
|
|
e2eef8ff21 | ||
|
|
3acf937d56 | ||
|
|
d572e523ba | ||
|
|
82113abe88 | ||
|
|
b7d121c58f | ||
|
|
6d5a85b144 | ||
|
|
78121917c6 | ||
|
|
a0913f0e32 | ||
|
|
e96e284715 | ||
|
|
c572a1b607 | ||
|
|
1845311f98 | ||
|
|
4f806db8b7 | ||
|
|
22858cc1e9 | ||
|
|
a0329a3eb0 | ||
|
|
b3e92088ee | ||
|
|
46db1c20f1 | ||
|
|
9d182e53b2 | ||
|
|
1205fc7fdb | ||
|
|
ff2826a448 | ||
|
|
ee750115ec | ||
|
|
0e13d22c97 | ||
|
|
8e7d040ac4 | ||
|
|
6755202958 | ||
|
|
8b7374a687 | ||
|
|
c17cca2365 | ||
|
|
8016a9539a | ||
|
|
e885fb15a0 | ||
|
|
c7f098771b | ||
|
|
fcd0908032 | ||
|
|
7ff1285084 | ||
|
|
b45b603b97 | ||
|
|
247208b8a9 | ||
|
|
182c46037b | ||
|
|
438d3210bc | ||
|
|
d523c7c916 | ||
|
|
09a19e94d5 | ||
|
|
3971c145df | ||
|
|
055117d83d | ||
|
|
c6baf43986 | ||
|
|
4ff16af3a7 | ||
|
|
17a1bd352b | ||
|
|
7421ca09cc | ||
|
|
9797e696e5 | ||
|
|
c36d6d8b2d | ||
|
|
3873786b99 | ||
|
|
76fdba7f09 | ||
|
|
72799e9638 | ||
|
|
2e77d03fe9 | ||
|
|
0c58eae5e7 | ||
|
|
b609567c38 | ||
|
|
7ecfa44fa0 | ||
|
|
a685b1dc3b | ||
|
|
63ce49a17c | ||
|
|
820fbe4076 | ||
|
|
efa05b7775 | ||
|
|
003781e903 | ||
|
|
ee71bafc96 | ||
|
|
bdd5f1231e | ||
|
|
6fee532c96 | ||
|
|
78aaad7b59 | ||
|
|
b128b0ede2 | ||
|
|
737d2f3bc6 | ||
|
|
179be53a65 | ||
|
|
1867f5e7c2 | ||
|
|
6662d24565 | ||
|
|
5880566a99 | ||
|
|
5d05b32711 | ||
|
|
fa2b720e92 | ||
|
|
d381238f83 | ||
|
|
751d627ead | ||
|
|
3e66a8de9b | ||
|
|
266052b12b | ||
|
|
803f4328f4 | ||
|
|
8e95568e11 | ||
|
|
ab09ee4819 | ||
|
|
41f94a172f | ||
|
|
566e597994 | ||
|
|
765fb9c05f | ||
|
|
b6720a19f7 | ||
|
|
3b130651c4 | ||
|
|
3f6c35dabe | ||
|
|
db2a952bca | ||
|
|
0ea9770bc3 | ||
|
|
0b20956c90 | ||
|
|
9f73b47d54 | ||
|
|
ce9c99af71 | ||
|
|
784024fb5d | ||
|
|
1145b32299 | ||
|
|
ab71df0011 | ||
|
|
fb137252a9 | ||
|
|
f57a680306 | ||
|
|
8bb3eaa320 | ||
|
|
9489730a44 | ||
|
|
d4795bb897 | ||
|
|
63775872c7 | ||
|
|
beff508a1f | ||
|
|
deaae8a2c6 | ||
|
|
46a27bd50c | ||
|
|
24f2993433 | ||
|
|
c80bfbfac5 | ||
|
|
06abfc45c7 | ||
|
|
440a773081 | ||
|
|
0797bcb38b | ||
|
|
d463b5bf0d | ||
|
|
0733c8edcc | ||
|
|
86c7c05cb1 | ||
|
|
18ff7ce753 | ||
|
|
8f2ed1004d | ||
|
|
14961323c3 | ||
|
|
f8c682b183 | ||
|
|
dd92708f60 | ||
|
|
4d9eeccefa | ||
|
|
cd7b251031 | ||
|
|
db614180b9 | ||
|
|
b6e527e5f4 | ||
|
|
77c0f8f39e | ||
|
|
58816d73c8 | ||
|
|
3b194d282e | ||
|
|
397f66433d | ||
|
|
04a4ed1d0e | ||
|
|
625850d4e7 | ||
|
|
6c572baca5 | ||
|
|
ee0406a13f | ||
|
|
608a049ba3 | ||
|
|
4d9b5198e2 | ||
|
|
24b6c970aa | ||
|
|
239c47f469 | ||
|
|
f0fc64c517 | ||
|
|
8481fd38ce | ||
|
|
5f425129d5 | ||
|
|
92955b1315 | ||
|
|
a3872d5bb5 | ||
|
|
a123ff2c04 | ||
|
|
188de34306 | ||
|
|
3d43750e9b | ||
|
|
fea228c68d | ||
|
|
a71a28e563 | ||
|
|
3b5d4982b5 | ||
|
|
b201e9ab8c | ||
|
|
d30b9282fd | ||
|
|
4f304a70b7 | ||
|
|
59a54d4f04 | ||
|
|
1e94d794ed | ||
|
|
5bd210406b | ||
|
|
e00514d36d | ||
|
|
f013bf1931 | ||
|
|
107cbbad1d | ||
|
|
481f1f9d30 | ||
|
|
704364061c | ||
|
|
c1bd2d6cf1 | ||
|
|
a018e1228c | ||
|
|
d962d9c7f6 | ||
|
|
4ea28cbca5 | ||
|
|
1b48b8b4cc | ||
|
|
73df197e33 | ||
|
|
bdc66e55ca | ||
|
|
926343ee86 | ||
|
|
8e6021c5e7 | ||
|
|
ac2b6c76ce | ||
|
|
9e966d0a7f | ||
|
|
6c10defaa1 | ||
|
|
b6a76f6f7c | ||
|
|
84e5b77a5c | ||
|
|
89b0ea0bf1 | ||
|
|
48aeb98bf1 | ||
|
|
8a5d864812 | ||
|
|
ae79e645a6 | ||
|
|
0947deb372 | ||
|
|
69c92911a2 | ||
|
|
b16bb37b75 | ||
|
|
9c9ec8adf2 | ||
|
|
eb0e67fc42 | ||
|
|
9cc50bddab | ||
|
|
d3ba0fa487 | ||
|
|
39f6505a80 | ||
|
|
36a6802439 | ||
|
|
d7e2633a92 | ||
|
|
88049e741e | ||
|
|
ff7fb14087 | ||
|
|
816c64bd48 | ||
|
|
d2756e6f2d | ||
|
|
147e12acbb | ||
|
|
4098018ee9 | ||
|
|
133e7578b9 | ||
|
|
74a2bdbf09 | ||
|
|
f22bc68af4 | ||
|
|
26cc6da650 | ||
|
|
d21f1f1b87 | ||
|
|
7cdaafffe1 | ||
|
|
0265dca197 | ||
|
|
9d68366043 | ||
|
|
c8c671d915 | ||
|
|
142daa9d15 | ||
|
|
2552219991 | ||
|
|
a038b698d7 | ||
|
|
a3b222574e | ||
|
|
e0cd467293 | ||
|
|
9c056030d2 | ||
|
|
19efa9d4cc | ||
|
|
90633a6495 | ||
|
|
edc432fbd8 | ||
|
|
1b7bdbf516 | ||
|
|
8c1be70c85 | ||
|
|
b8e0c0db9e | ||
|
|
7b7fb6cc82 | ||
|
|
62512ba215 | ||
|
|
e1beb64c01 | ||
|
|
c81f26ddad | ||
|
|
340114c2a1 | ||
|
|
cd7767b331 | ||
|
|
25289dad8a | ||
|
|
47c6917129 | ||
|
|
6379cda148 | ||
|
|
91a124ab8f | ||
|
|
2357a7135e | ||
|
|
da0b3b3de9 | ||
|
|
6664fb1716 | ||
|
|
1206f24fa9 | ||
|
|
ffb5823e84 | ||
|
|
d45a7fb262 | ||
|
|
918d192c0f | ||
|
|
f7cd6eac50 | ||
|
|
88f4428ff0 | ||
|
|
069ea22ba2 | ||
|
|
8fac8c5307 | ||
|
|
2285befebb | ||
|
|
1cd0648e4e | ||
|
|
0b7ba285c6 | ||
|
|
30446c4526 | ||
|
|
9b843c9ed2 | ||
|
|
2ce1c3bef8 | ||
|
|
e463094dc7 | ||
|
|
71a9fe10f4 | ||
|
|
ba146e13ef | ||
|
|
c060d7e3e0 | ||
|
|
ba96678822 | ||
|
|
4f6354f383 | ||
|
|
2766e80346 | ||
|
|
7cc3777a60 | ||
|
|
cb1dd9f17d | ||
|
|
31f342fe4f | ||
|
|
e90359eb08 | ||
|
|
58b0768a30 | ||
|
|
3b04506893 | ||
|
|
354165aa0a | ||
|
|
343109836f | ||
|
|
fcadac2adb | ||
|
|
5e7dcdfe97 | ||
|
|
2ec9a57391 | ||
|
|
973c545723 | ||
|
|
fd62eecfef | ||
|
|
b5ca7058c2 | ||
|
|
57a48f099f | ||
|
|
4699f511bf | ||
|
|
cd8f7e72e0 | ||
|
|
78803fa284 | ||
|
|
2e8d75df16 | ||
|
|
7e3bbfd960 | ||
|
|
1734d53b3c | ||
|
|
f37540f4e5 | ||
|
|
addb9d836a | ||
|
|
4184d8c7ac | ||
|
|
724c15a68c | ||
|
|
499bdf9b48 | ||
|
|
41cd1ccda1 | ||
|
|
b9521cb3a9 | ||
|
|
1f40663b90 | ||
|
|
5261ed7c4c | ||
|
|
aa8768b18a | ||
|
|
aad07433f4 | ||
|
|
4a7630079b | ||
|
|
44a6ee1994 | ||
|
|
56bd6e69ed | ||
|
|
d1e04588d0 | ||
|
|
21cdaef6d5 | ||
|
|
a1723d18fb | ||
|
|
9e065138e9 | ||
|
|
1c73c92bfd | ||
|
|
bcd560d74e | ||
|
|
02339562ed | ||
|
|
e5804378c2 | ||
|
|
da1c8a162d | ||
|
|
d457a23a1f | ||
|
|
b6154e58b8 | ||
|
|
5f18776c61 | ||
|
|
68b0b9ec7a | ||
|
|
0f5036972e | ||
|
|
0b199b8421 | ||
|
|
a59730f6eb | ||
|
|
c6c84fe65b | ||
|
|
03c757bba6 | ||
|
|
bfeb8d238a | ||
|
|
daf0c08c4b | ||
|
|
d12c1b9ac4 | ||
|
|
bc242f4fd4 | ||
|
|
a240c1bca9 | ||
|
|
219aa6c574 | ||
|
|
abca1b481a | ||
|
|
db72fd2ef5 | ||
|
|
31cca58943 | ||
|
|
c06a4b759c | ||
|
|
f05a23a490 | ||
|
|
1e0f2ffde0 | ||
|
|
06df42ee3d | ||
|
|
65ee1638f7 | ||
|
|
87eefe7673 | ||
|
|
5c124d3988 | ||
|
|
8c69ce624f | ||
|
|
bb73acdde5 | ||
|
|
993bc3775b | ||
|
|
3d2ff28bcd | ||
|
|
9b78deb802 | ||
|
|
dadc525d0b | ||
|
|
22b2140c94 | ||
|
|
f07496a4a0 | ||
|
|
1b2938cbc8 | ||
|
|
d4d2f58830 | ||
|
|
b3113e13ec | ||
|
|
055c8e26f0 | ||
|
|
2a7a7239d7 | ||
|
|
2fa40dac3f | ||
|
|
6b4fbd7dc2 | ||
|
|
5b0bb19717 | ||
|
|
843dfc430a | ||
|
|
69cb07c527 | ||
|
|
89e8a64734 | ||
|
|
5eb2dec32d | ||
|
|
db0ea7d6c4 | ||
|
|
1eb85003de | ||
|
|
cca170f84a | ||
|
|
c8c016caa8 | ||
|
|
45d5874026 | ||
|
|
69b1ce60ff | ||
|
|
3ff3e4b106 | ||
|
|
dc50a68b01 | ||
|
|
968cfd8654 | ||
|
|
cf28d93be6 | ||
|
|
be08d6ebb5 | ||
|
|
4bc24f3b00 | ||
|
|
15833f94cf | ||
|
|
aeb297efcf | ||
|
|
d48c6b98e8 | ||
|
|
b79ccfafed | ||
|
|
c87ba59552 | ||
|
|
91fd71c858 | ||
|
|
6f64e67538 | ||
|
|
bd7a0b072f | ||
|
|
01ca001c97 | ||
|
|
324ad2a87c | ||
|
|
d9ad2630f0 | ||
|
|
83958a4a48 | ||
|
|
f6a6efdc42 | ||
|
|
1bbe7657b9 | ||
|
|
38189753b5 | ||
|
|
5b0e658617 | ||
|
|
b6cf54d57f | ||
|
|
e8058c8813 | ||
|
|
784868048d | ||
|
|
2bf9779f2f | ||
|
|
d98ceea381 | ||
|
|
1ab2da74b9 | ||
|
|
3723cf8ac2 |
@@ -1,3 +1,84 @@
|
||||
# Ignore git
|
||||
# Git
|
||||
.github
|
||||
.git
|
||||
.git
|
||||
.gitignore
|
||||
|
||||
# Documentation
|
||||
docs/
|
||||
README.md
|
||||
LICENSE
|
||||
|
||||
# Development files
|
||||
.pylintrc
|
||||
*.pyc
|
||||
__pycache__/
|
||||
*.pyo
|
||||
*.pyd
|
||||
.Python
|
||||
*.so
|
||||
.pytest_cache/
|
||||
.coverage
|
||||
htmlcov/
|
||||
.tox/
|
||||
.nox/
|
||||
.hypothesis/
|
||||
.mypy_cache/
|
||||
.dmypy.json
|
||||
dmypy.json
|
||||
|
||||
# Virtual environments
|
||||
venv/
|
||||
env/
|
||||
ENV/
|
||||
env.bak/
|
||||
venv.bak/
|
||||
|
||||
# IDE
|
||||
.vscode/
|
||||
.idea/
|
||||
*.swp
|
||||
*.swo
|
||||
*~
|
||||
|
||||
# OS
|
||||
.DS_Store
|
||||
.DS_Store?
|
||||
._*
|
||||
.Spotlight-V100
|
||||
.Trashes
|
||||
ehthumbs.db
|
||||
Thumbs.db
|
||||
|
||||
# Logs
|
||||
*.log
|
||||
logs/
|
||||
|
||||
# Temporary files
|
||||
*.tmp
|
||||
*.temp
|
||||
tmp/
|
||||
temp/
|
||||
|
||||
# Database
|
||||
*.db
|
||||
*.sqlite
|
||||
*.sqlite3
|
||||
|
||||
# Test files
|
||||
tests/
|
||||
test_*
|
||||
*_test.py
|
||||
|
||||
# Build artifacts
|
||||
build/
|
||||
dist/
|
||||
*.egg-info/
|
||||
|
||||
# Docker
|
||||
Dockerfile*
|
||||
docker-compose*
|
||||
.dockerignore
|
||||
|
||||
# Other
|
||||
app.ico
|
||||
frozen.spec
|
||||
60
.github/workflows/beta.yml
vendored
Normal file
60
.github/workflows/beta.yml
vendored
Normal file
@@ -0,0 +1,60 @@
|
||||
name: MoviePilot Builder Beta
|
||||
on:
|
||||
workflow_dispatch:
|
||||
|
||||
jobs:
|
||||
Docker-build:
|
||||
runs-on: ubuntu-latest
|
||||
name: Build Docker Image
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Release version
|
||||
id: release_version
|
||||
run: |
|
||||
app_version=$(cat version.py |sed -ne "s/APP_VERSION\s=\s'v\(.*\)'/\1/gp")
|
||||
echo "app_version=$app_version" >> $GITHUB_ENV
|
||||
|
||||
- name: Docker Meta
|
||||
id: meta
|
||||
uses: docker/metadata-action@v5
|
||||
with:
|
||||
images: |
|
||||
${{ secrets.DOCKER_USERNAME }}/moviepilot-v2
|
||||
ghcr.io/${{ github.repository }}
|
||||
tags: |
|
||||
type=raw,value=beta
|
||||
|
||||
- name: Set Up QEMU
|
||||
uses: docker/setup-qemu-action@v3
|
||||
|
||||
- name: Set Up Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
|
||||
- name: Login DockerHub
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
|
||||
- name: Login GitHub Container Registry
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ github.actor }}
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Build Image
|
||||
uses: docker/build-push-action@v5
|
||||
with:
|
||||
context: .
|
||||
file: docker/Dockerfile
|
||||
platforms: |
|
||||
linux/amd64
|
||||
linux/arm64/v8
|
||||
push: true
|
||||
tags: ${{ steps.meta.outputs.tags }}
|
||||
labels: ${{ steps.meta.outputs.labels }}
|
||||
cache-from: type=gha, scope=${{ github.workflow }}-docker
|
||||
cache-to: type=gha, scope=${{ github.workflow }}-docker
|
||||
1
.github/workflows/build.yml
vendored
1
.github/workflows/build.yml
vendored
@@ -27,6 +27,7 @@ jobs:
|
||||
with:
|
||||
images: |
|
||||
${{ secrets.DOCKER_USERNAME }}/moviepilot-v2
|
||||
${{ secrets.DOCKER_USERNAME }}/moviepilot
|
||||
ghcr.io/${{ github.repository }}
|
||||
tags: |
|
||||
type=raw,value=${{ env.app_version }}
|
||||
|
||||
@@ -40,10 +40,11 @@ git clone https://github.com/jxxghp/MoviePilot
|
||||
```shell
|
||||
git clone https://github.com/jxxghp/MoviePilot-Resources
|
||||
```
|
||||
- 安装后端依赖,设置`app`为源代码根目录,运行 `main.py` 启动后端服务,默认监听端口:`3001`,API文档地址:`http://localhost:3001/docs`
|
||||
- 安装后端依赖,运行 `main.py` 启动后端服务,默认监听端口:`3001`,API文档地址:`http://localhost:3001/docs`
|
||||
```shell
|
||||
cd MoviePilot
|
||||
pip install -r requirements.txt
|
||||
python3 main.py
|
||||
python3 -m app.main
|
||||
```
|
||||
- 克隆前端项目 [MoviePilot-Frontend](https://github.com/jxxghp/MoviePilot-Frontend)
|
||||
```shell
|
||||
|
||||
355
app/agent/__init__.py
Normal file
355
app/agent/__init__.py
Normal file
@@ -0,0 +1,355 @@
|
||||
"""MoviePilot AI智能体实现"""
|
||||
|
||||
import asyncio
|
||||
from typing import Dict, List, Any
|
||||
|
||||
from langchain.agents import AgentExecutor, create_openai_tools_agent
|
||||
from langchain.prompts import ChatPromptTemplate, MessagesPlaceholder
|
||||
from langchain_community.callbacks import get_openai_callback
|
||||
from langchain_core.chat_history import InMemoryChatMessageHistory
|
||||
from langchain_core.messages import HumanMessage, AIMessage, ToolCall
|
||||
from langchain_core.runnables.history import RunnableWithMessageHistory
|
||||
|
||||
from app.agent.callback import StreamingCallbackHandler
|
||||
from app.agent.memory import ConversationMemoryManager
|
||||
from app.agent.prompt import PromptManager
|
||||
from app.agent.tools import MoviePilotToolFactory
|
||||
from app.chain import ChainBase
|
||||
from app.core.config import settings
|
||||
from app.helper.message import MessageHelper
|
||||
from app.log import logger
|
||||
from app.schemas import Notification
|
||||
|
||||
|
||||
class AgentChain(ChainBase):
|
||||
pass
|
||||
|
||||
|
||||
class MoviePilotAgent:
|
||||
"""MoviePilot AI智能体"""
|
||||
|
||||
def __init__(self, session_id: str, user_id: str = None,
|
||||
channel: str = None, source: str = None, username: str = None):
|
||||
self.session_id = session_id
|
||||
self.user_id = user_id
|
||||
self.channel = channel # 消息渠道
|
||||
self.source = source # 消息来源
|
||||
self.username = username # 用户名
|
||||
|
||||
# 消息助手
|
||||
self.message_helper = MessageHelper()
|
||||
|
||||
# 记忆管理器
|
||||
self.memory_manager = ConversationMemoryManager()
|
||||
|
||||
# 提示词管理器
|
||||
self.prompt_manager = PromptManager()
|
||||
|
||||
# 回调处理器
|
||||
self.callback_handler = StreamingCallbackHandler(
|
||||
session_id=session_id
|
||||
)
|
||||
|
||||
# LLM模型
|
||||
self.llm = self._initialize_llm()
|
||||
|
||||
# 工具
|
||||
self.tools = self._initialize_tools()
|
||||
|
||||
# 会话存储
|
||||
self.session_store = self._initialize_session_store()
|
||||
|
||||
# 提示词模板
|
||||
self.prompt = self._initialize_prompt()
|
||||
|
||||
# Agent执行器
|
||||
self.agent_executor = self._create_agent_executor()
|
||||
|
||||
def _initialize_llm(self):
|
||||
"""初始化LLM模型"""
|
||||
provider = settings.LLM_PROVIDER.lower()
|
||||
api_key = settings.LLM_API_KEY
|
||||
if not api_key:
|
||||
raise ValueError("未配置 LLM_API_KEY")
|
||||
|
||||
if provider == "google":
|
||||
from langchain_google_genai import ChatGoogleGenerativeAI
|
||||
return ChatGoogleGenerativeAI(
|
||||
model=settings.LLM_MODEL,
|
||||
google_api_key=api_key,
|
||||
max_retries=3,
|
||||
temperature=settings.LLM_TEMPERATURE,
|
||||
streaming=True,
|
||||
callbacks=[self.callback_handler]
|
||||
)
|
||||
elif provider == "deepseek":
|
||||
from langchain_deepseek import ChatDeepSeek
|
||||
return ChatDeepSeek(
|
||||
model=settings.LLM_MODEL,
|
||||
api_key=api_key,
|
||||
max_retries=3,
|
||||
temperature=settings.LLM_TEMPERATURE,
|
||||
streaming=True,
|
||||
callbacks=[self.callback_handler],
|
||||
stream_usage=True
|
||||
)
|
||||
else:
|
||||
from langchain_openai import ChatOpenAI
|
||||
return ChatOpenAI(
|
||||
model=settings.LLM_MODEL,
|
||||
api_key=api_key,
|
||||
max_retries=3,
|
||||
base_url=settings.LLM_BASE_URL,
|
||||
temperature=settings.LLM_TEMPERATURE,
|
||||
streaming=True,
|
||||
callbacks=[self.callback_handler],
|
||||
stream_usage=True
|
||||
)
|
||||
|
||||
def _initialize_tools(self) -> List:
|
||||
"""初始化工具列表"""
|
||||
return MoviePilotToolFactory.create_tools(
|
||||
session_id=self.session_id,
|
||||
user_id=self.user_id,
|
||||
channel=self.channel,
|
||||
source=self.source,
|
||||
username=self.username,
|
||||
callback_handler=self.callback_handler
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _initialize_session_store() -> Dict[str, InMemoryChatMessageHistory]:
|
||||
"""初始化内存存储"""
|
||||
return {}
|
||||
|
||||
def get_session_history(self, session_id: str) -> InMemoryChatMessageHistory:
|
||||
"""获取会话历史"""
|
||||
if session_id not in self.session_store:
|
||||
chat_history = InMemoryChatMessageHistory()
|
||||
messages: List[dict] = self.memory_manager.get_recent_messages_for_agent(
|
||||
session_id=session_id,
|
||||
user_id=self.user_id
|
||||
)
|
||||
if messages:
|
||||
for msg in messages:
|
||||
if msg.get("role") == "user":
|
||||
chat_history.add_user_message(HumanMessage(content=msg.get("content", "")))
|
||||
elif msg.get("role") == "agent":
|
||||
chat_history.add_ai_message(AIMessage(content=msg.get("content", "")))
|
||||
elif msg.get("role") == "tool_call":
|
||||
metadata = msg.get("metadata", {})
|
||||
chat_history.add_ai_message(AIMessage(
|
||||
content=msg.get("content", ""),
|
||||
tool_calls=[ToolCall(
|
||||
id=metadata.get("call_id"),
|
||||
name=metadata.get("tool_name"),
|
||||
args=metadata.get("parameters"),
|
||||
)]
|
||||
))
|
||||
elif msg.get("role") == "tool_result":
|
||||
chat_history.add_ai_message(AIMessage(content=msg.get("content", "")))
|
||||
elif msg.get("role") == "system":
|
||||
chat_history.add_ai_message(AIMessage(content=msg.get("content", "")))
|
||||
self.session_store[session_id] = chat_history
|
||||
return self.session_store[session_id]
|
||||
|
||||
@staticmethod
|
||||
def _initialize_prompt() -> ChatPromptTemplate:
|
||||
"""初始化提示词模板"""
|
||||
try:
|
||||
prompt_template = ChatPromptTemplate.from_messages([
|
||||
("system", "{system_prompt}"),
|
||||
MessagesPlaceholder(variable_name="chat_history"),
|
||||
("user", "{input}"),
|
||||
MessagesPlaceholder(variable_name="agent_scratchpad"),
|
||||
])
|
||||
logger.info("LangChain提示词模板初始化成功")
|
||||
return prompt_template
|
||||
except Exception as e:
|
||||
logger.error(f"初始化提示词失败: {e}")
|
||||
raise e
|
||||
|
||||
def _create_agent_executor(self) -> RunnableWithMessageHistory:
|
||||
"""创建Agent执行器"""
|
||||
try:
|
||||
agent = create_openai_tools_agent(
|
||||
llm=self.llm,
|
||||
tools=self.tools,
|
||||
prompt=self.prompt
|
||||
)
|
||||
executor = AgentExecutor(
|
||||
agent=agent,
|
||||
tools=self.tools,
|
||||
verbose=settings.LLM_VERBOSE,
|
||||
max_iterations=settings.LLM_MAX_ITERATIONS,
|
||||
return_intermediate_steps=True,
|
||||
handle_parsing_errors=True,
|
||||
early_stopping_method="force"
|
||||
)
|
||||
return RunnableWithMessageHistory(
|
||||
executor,
|
||||
self.get_session_history,
|
||||
input_messages_key="input",
|
||||
history_messages_key="chat_history"
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(f"创建Agent执行器失败: {e}")
|
||||
raise e
|
||||
|
||||
async def process_message(self, message: str) -> str:
|
||||
"""处理用户消息"""
|
||||
try:
|
||||
# 添加用户消息到记忆
|
||||
await self.memory_manager.add_memory(
|
||||
self.session_id,
|
||||
user_id=self.user_id,
|
||||
role="user",
|
||||
content=message
|
||||
)
|
||||
|
||||
# 构建输入上下文
|
||||
input_context = {
|
||||
"system_prompt": self.prompt_manager.get_agent_prompt(channel=self.channel),
|
||||
"input": message
|
||||
}
|
||||
|
||||
# 执行Agent
|
||||
logger.info(f"Agent执行推理: session_id={self.session_id}, input={message}")
|
||||
await self._execute_agent(input_context)
|
||||
|
||||
# 获取Agent回复
|
||||
agent_message = await self.callback_handler.get_message()
|
||||
|
||||
# 发送Agent回复给用户(通过原渠道)
|
||||
await self.send_agent_message(agent_message)
|
||||
|
||||
# 添加Agent回复到记忆
|
||||
await self.memory_manager.add_memory(
|
||||
session_id=self.session_id,
|
||||
user_id=self.user_id,
|
||||
role="agent",
|
||||
content=agent_message
|
||||
)
|
||||
|
||||
return agent_message
|
||||
|
||||
except Exception as e:
|
||||
error_message = f"处理消息时发生错误: {str(e)}"
|
||||
logger.error(error_message)
|
||||
# 发送错误消息给用户(通过原渠道)
|
||||
await self.send_agent_message(error_message)
|
||||
return error_message
|
||||
|
||||
async def _execute_agent(self, input_context: Dict[str, Any]) -> Dict[str, Any]:
|
||||
"""执行LangChain Agent"""
|
||||
try:
|
||||
with get_openai_callback() as cb:
|
||||
result = await self.agent_executor.ainvoke(
|
||||
input_context,
|
||||
config={"configurable": {"session_id": self.session_id}},
|
||||
callbacks=[self.callback_handler]
|
||||
)
|
||||
logger.info(f"LLM调用消耗: \n{cb}")
|
||||
|
||||
if cb.total_tokens > 0:
|
||||
result["token_usage"] = {
|
||||
"prompt_tokens": cb.prompt_tokens,
|
||||
"completion_tokens": cb.completion_tokens,
|
||||
"total_tokens": cb.total_tokens
|
||||
}
|
||||
return result
|
||||
except asyncio.CancelledError:
|
||||
logger.info(f"Agent执行被取消: session_id={self.session_id}")
|
||||
return {
|
||||
"output": "任务已取消",
|
||||
"intermediate_steps": [],
|
||||
"token_usage": {}
|
||||
}
|
||||
except Exception as e:
|
||||
logger.error(f"Agent执行失败: {e}")
|
||||
return {
|
||||
"output": f"执行过程中发生错误: {str(e)}",
|
||||
"intermediate_steps": [],
|
||||
"token_usage": {}
|
||||
}
|
||||
|
||||
async def send_agent_message(self, message: str, title: str = "MoviePilot助手"):
|
||||
"""通过原渠道发送消息给用户"""
|
||||
await AgentChain().async_post_message(
|
||||
Notification(
|
||||
channel=self.channel,
|
||||
source=self.source,
|
||||
userid=self.user_id,
|
||||
username=self.username,
|
||||
title=title,
|
||||
text=message
|
||||
)
|
||||
)
|
||||
|
||||
async def cleanup(self):
|
||||
"""清理智能体资源"""
|
||||
if self.session_id in self.session_store:
|
||||
del self.session_store[self.session_id]
|
||||
logger.info(f"MoviePilot智能体已清理: session_id={self.session_id}")
|
||||
|
||||
|
||||
class AgentManager:
|
||||
"""AI智能体管理器"""
|
||||
|
||||
def __init__(self):
|
||||
self.active_agents: Dict[str, MoviePilotAgent] = {}
|
||||
self.memory_manager = ConversationMemoryManager()
|
||||
|
||||
async def initialize(self):
|
||||
"""初始化管理器"""
|
||||
await self.memory_manager.initialize()
|
||||
|
||||
async def close(self):
|
||||
"""关闭管理器"""
|
||||
await self.memory_manager.close()
|
||||
# 清理所有活跃的智能体
|
||||
for agent in self.active_agents.values():
|
||||
await agent.cleanup()
|
||||
self.active_agents.clear()
|
||||
|
||||
async def process_message(self, session_id: str, user_id: str, message: str,
|
||||
channel: str = None, source: str = None, username: str = None) -> str:
|
||||
"""处理用户消息"""
|
||||
# 获取或创建Agent实例
|
||||
if session_id not in self.active_agents:
|
||||
logger.info(f"创建新的AI智能体实例,session_id: {session_id}, user_id: {user_id}")
|
||||
agent = MoviePilotAgent(
|
||||
session_id=session_id,
|
||||
user_id=user_id,
|
||||
channel=channel,
|
||||
source=source,
|
||||
username=username
|
||||
)
|
||||
agent.memory_manager = self.memory_manager
|
||||
self.active_agents[session_id] = agent
|
||||
else:
|
||||
agent = self.active_agents[session_id]
|
||||
agent.user_id = user_id # 确保user_id是最新的
|
||||
# 更新渠道信息
|
||||
if channel:
|
||||
agent.channel = channel
|
||||
if source:
|
||||
agent.source = source
|
||||
if username:
|
||||
agent.username = username
|
||||
|
||||
# 处理消息
|
||||
return await agent.process_message(message)
|
||||
|
||||
async def clear_session(self, session_id: str, user_id: str):
|
||||
"""清空会话"""
|
||||
if session_id in self.active_agents:
|
||||
agent = self.active_agents[session_id]
|
||||
await agent.cleanup()
|
||||
del self.active_agents[session_id]
|
||||
await self.memory_manager.clear_memory(session_id, user_id)
|
||||
logger.info(f"会话 {session_id} 的记忆已清空")
|
||||
|
||||
|
||||
# 全局智能体管理器实例
|
||||
agent_manager = AgentManager()
|
||||
33
app/agent/callback/__init__.py
Normal file
33
app/agent/callback/__init__.py
Normal file
@@ -0,0 +1,33 @@
|
||||
import threading
|
||||
|
||||
from langchain_core.callbacks import AsyncCallbackHandler
|
||||
|
||||
from app.log import logger
|
||||
|
||||
|
||||
class StreamingCallbackHandler(AsyncCallbackHandler):
|
||||
"""流式输出回调处理器"""
|
||||
|
||||
def __init__(self, session_id: str):
|
||||
self._lock = threading.Lock()
|
||||
self.session_id = session_id
|
||||
self.current_message = ""
|
||||
|
||||
async def get_message(self):
|
||||
"""获取当前消息内容,获取后清空"""
|
||||
with self._lock:
|
||||
if not self.current_message:
|
||||
return ""
|
||||
msg = self.current_message
|
||||
logger.info(f"Agent消息: {msg}")
|
||||
self.current_message = ""
|
||||
return msg
|
||||
|
||||
async def on_llm_new_token(self, token: str, **kwargs):
|
||||
"""处理新的token"""
|
||||
if not token:
|
||||
return
|
||||
with self._lock:
|
||||
# 缓存当前消息
|
||||
self.current_message += token
|
||||
|
||||
280
app/agent/memory/__init__.py
Normal file
280
app/agent/memory/__init__.py
Normal file
@@ -0,0 +1,280 @@
|
||||
"""对话记忆管理器"""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
from datetime import datetime, timedelta
|
||||
from typing import Dict, List, Optional, Any
|
||||
|
||||
from app.core.config import settings
|
||||
from app.helper.redis import AsyncRedisHelper
|
||||
from app.log import logger
|
||||
from app.schemas.agent import ConversationMemory
|
||||
|
||||
|
||||
class ConversationMemoryManager:
|
||||
"""对话记忆管理器"""
|
||||
|
||||
def __init__(self):
|
||||
# 内存中的会话记忆缓存
|
||||
self.memory_cache: Dict[str, ConversationMemory] = {}
|
||||
# 使用现有的Redis助手
|
||||
self.redis_helper = AsyncRedisHelper()
|
||||
# 内存缓存清理任务(Redis通过TTL自动过期)
|
||||
self.cleanup_task: Optional[asyncio.Task] = None
|
||||
|
||||
async def initialize(self):
|
||||
"""初始化记忆管理器"""
|
||||
try:
|
||||
# 启动内存缓存清理任务(Redis通过TTL自动过期)
|
||||
self.cleanup_task = asyncio.create_task(self._cleanup_expired_memories())
|
||||
logger.info("对话记忆管理器初始化完成")
|
||||
|
||||
except Exception as e:
|
||||
logger.warning(f"Redis连接失败,将使用内存存储: {e}")
|
||||
|
||||
async def close(self):
|
||||
"""关闭记忆管理器"""
|
||||
if self.cleanup_task:
|
||||
self.cleanup_task.cancel()
|
||||
try:
|
||||
await self.cleanup_task
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
|
||||
await self.redis_helper.close()
|
||||
|
||||
logger.info("对话记忆管理器已关闭")
|
||||
|
||||
async def get_memory(self, session_id: str, user_id: str) -> ConversationMemory:
|
||||
"""获取会话记忆"""
|
||||
# 首先检查缓存
|
||||
cache_key = f"{user_id}:{session_id}" if user_id else session_id
|
||||
if cache_key in self.memory_cache:
|
||||
return self.memory_cache[cache_key]
|
||||
|
||||
# 尝试从Redis加载
|
||||
if settings.CACHE_BACKEND_TYPE == "redis":
|
||||
try:
|
||||
redis_key = f"agent_memory:{user_id}:{session_id}" if user_id else f"agent_memory:{session_id}"
|
||||
memory_data = await self.redis_helper.get(redis_key, region="AI_AGENT")
|
||||
if memory_data:
|
||||
memory_dict = json.loads(memory_data) if isinstance(memory_data, str) else memory_data
|
||||
memory = ConversationMemory(**memory_dict)
|
||||
self.memory_cache[cache_key] = memory
|
||||
return memory
|
||||
except Exception as e:
|
||||
logger.warning(f"从Redis加载记忆失败: {e}")
|
||||
|
||||
# 创建新的记忆
|
||||
memory = ConversationMemory(session_id=session_id, user_id=user_id)
|
||||
self.memory_cache[cache_key] = memory
|
||||
await self._save_memory(memory)
|
||||
|
||||
return memory
|
||||
|
||||
async def set_title(self, session_id: str, user_id: str, title: str):
|
||||
"""设置会话标题"""
|
||||
memory = await self.get_memory(session_id=session_id, user_id=user_id)
|
||||
memory.title = title
|
||||
memory.updated_at = datetime.now()
|
||||
await self._save_memory(memory)
|
||||
|
||||
async def get_title(self, session_id: str, user_id: str) -> Optional[str]:
|
||||
"""获取会话标题"""
|
||||
memory = await self.get_memory(session_id=session_id, user_id=user_id)
|
||||
return memory.title
|
||||
|
||||
async def list_sessions(self, user_id: str, limit: int = 100) -> List[Dict[str, Any]]:
|
||||
"""列出历史会话摘要(按更新时间倒序)
|
||||
|
||||
- 当启用Redis时:遍历 `agent_memory:*` 键并读取摘要
|
||||
- 当未启用Redis时:基于内存缓存返回
|
||||
"""
|
||||
sessions: List[ConversationMemory] = []
|
||||
# 从Redis遍历
|
||||
if settings.CACHE_BACKEND_TYPE == "redis":
|
||||
try:
|
||||
# 使用Redis助手的items方法遍历所有键
|
||||
async for key, value in self.redis_helper.items(region="AI_AGENT"):
|
||||
if key.startswith("agent_memory:"):
|
||||
try:
|
||||
# 解析键名获取user_id和session_id
|
||||
key_parts = key.split(":")
|
||||
if len(key_parts) >= 3:
|
||||
key_user_id = key_parts[2] if len(key_parts) > 3 else None
|
||||
if not user_id or key_user_id == user_id:
|
||||
data = value if isinstance(value, dict) else json.loads(value)
|
||||
memory = ConversationMemory(**data)
|
||||
sessions.append(memory)
|
||||
except Exception as err:
|
||||
logger.warning(f"解析Redis记忆数据失败: {err}")
|
||||
continue
|
||||
except Exception as e:
|
||||
logger.warning(f"遍历Redis会话失败: {e}")
|
||||
|
||||
# 合并内存缓存(确保包含近期的会话)
|
||||
for cache_key, memory in self.memory_cache.items():
|
||||
# 如果指定了user_id,只返回该用户的会话
|
||||
if not user_id or memory.user_id == user_id:
|
||||
sessions.append(memory)
|
||||
|
||||
# 去重(以 session_id 为键,取最近updated)
|
||||
uniq: Dict[str, ConversationMemory] = {}
|
||||
for mem in sessions:
|
||||
existed = uniq.get(mem.session_id)
|
||||
if (not existed) or (mem.updated_at > existed.updated_at):
|
||||
uniq[mem.session_id] = mem
|
||||
|
||||
# 排序并裁剪
|
||||
sorted_list = sorted(uniq.values(), key=lambda m: m.updated_at, reverse=True)[:limit]
|
||||
return [
|
||||
{
|
||||
"session_id": m.session_id,
|
||||
"title": m.title or "新会话",
|
||||
"message_count": len(m.messages),
|
||||
"created_at": m.created_at.isoformat(),
|
||||
"updated_at": m.updated_at.isoformat(),
|
||||
}
|
||||
for m in sorted_list
|
||||
]
|
||||
|
||||
async def add_memory(
|
||||
self,
|
||||
session_id: str,
|
||||
user_id: str,
|
||||
role: str,
|
||||
content: str,
|
||||
metadata: Optional[Dict[str, Any]] = None
|
||||
):
|
||||
"""添加消息到记忆"""
|
||||
memory = await self.get_memory(session_id=session_id, user_id=user_id)
|
||||
|
||||
message = {
|
||||
"role": role,
|
||||
"content": content,
|
||||
"timestamp": datetime.now().isoformat(),
|
||||
"metadata": metadata or {}
|
||||
}
|
||||
|
||||
memory.messages.append(message)
|
||||
memory.updated_at = datetime.now()
|
||||
|
||||
# 限制消息数量,避免记忆过大
|
||||
max_messages = settings.LLM_MAX_MEMORY_MESSAGES
|
||||
if len(memory.messages) > max_messages:
|
||||
# 保留最近的消息,但保留第一条系统消息
|
||||
system_messages = [msg for msg in memory.messages if msg["role"] == "system"]
|
||||
recent_messages = memory.messages[-(max_messages - len(system_messages)):]
|
||||
memory.messages = system_messages + recent_messages
|
||||
|
||||
await self._save_memory(memory)
|
||||
|
||||
logger.debug(f"消息已添加到记忆: session_id={session_id}, user_id={user_id}, role={role}")
|
||||
|
||||
def get_recent_messages_for_agent(
|
||||
self,
|
||||
session_id: str,
|
||||
user_id: str
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""为Agent获取最近的消息(仅内存缓存)
|
||||
|
||||
如果消息Token数量超过模型最大上下文长度的阀值,会自动进行摘要裁剪
|
||||
"""
|
||||
cache_key = f"{user_id}:{session_id}" if user_id else session_id
|
||||
memory = self.memory_cache.get(cache_key)
|
||||
if not memory:
|
||||
return []
|
||||
|
||||
# 获取所有消息
|
||||
messages = memory.messages
|
||||
|
||||
return messages
|
||||
|
||||
async def get_recent_messages(
|
||||
self,
|
||||
session_id: str,
|
||||
user_id: str,
|
||||
limit: int = 10,
|
||||
role_filter: Optional[list] = None
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""获取最近的消息"""
|
||||
memory = await self.get_memory(session_id=session_id, user_id=user_id)
|
||||
|
||||
messages = memory.messages
|
||||
if role_filter:
|
||||
messages = [msg for msg in messages if msg["role"] in role_filter]
|
||||
|
||||
return messages[-limit:] if messages else []
|
||||
|
||||
async def get_context(self, session_id: str, user_id: str) -> Dict[str, Any]:
|
||||
"""获取会话上下文"""
|
||||
memory = await self.get_memory(session_id=session_id, user_id=user_id)
|
||||
return memory.context
|
||||
|
||||
async def clear_memory(self, session_id: str, user_id: str):
|
||||
"""清空会话记忆"""
|
||||
cache_key = f"{user_id}:{session_id}" if user_id else session_id
|
||||
if cache_key in self.memory_cache:
|
||||
del self.memory_cache[cache_key]
|
||||
|
||||
if settings.CACHE_BACKEND_TYPE == "redis":
|
||||
redis_key = f"agent_memory:{user_id}:{session_id}" if user_id else f"agent_memory:{session_id}"
|
||||
await self.redis_helper.delete(redis_key, region="AI_AGENT")
|
||||
|
||||
logger.info(f"会话记忆已清空: session_id={session_id}, user_id={user_id}")
|
||||
|
||||
async def _save_memory(self, memory: ConversationMemory):
|
||||
"""保存记忆到存储
|
||||
|
||||
Redis中的记忆会自动通过TTL机制过期,无需手动清理
|
||||
"""
|
||||
# 更新内存缓存
|
||||
cache_key = f"{memory.user_id}:{memory.session_id}" if memory.user_id else memory.session_id
|
||||
self.memory_cache[cache_key] = memory
|
||||
|
||||
# 保存到Redis,设置TTL自动过期
|
||||
if settings.CACHE_BACKEND_TYPE == "redis":
|
||||
try:
|
||||
memory_dict = memory.model_dump()
|
||||
redis_key = f"agent_memory:{memory.user_id}:{memory.session_id}" if memory.user_id else f"agent_memory:{memory.session_id}"
|
||||
ttl = int(timedelta(days=settings.LLM_REDIS_MEMORY_RETENTION_DAYS).total_seconds())
|
||||
await self.redis_helper.set(
|
||||
redis_key,
|
||||
memory_dict,
|
||||
ttl=ttl,
|
||||
region="AI_AGENT"
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"保存记忆到Redis失败: {e}")
|
||||
|
||||
async def _cleanup_expired_memories(self):
|
||||
"""清理内存中过期记忆的后台任务
|
||||
|
||||
注意:Redis中的记忆通过TTL机制自动过期,这里只清理内存缓存
|
||||
"""
|
||||
while True:
|
||||
try:
|
||||
# 每小时清理一次
|
||||
await asyncio.sleep(3600)
|
||||
|
||||
current_time = datetime.now()
|
||||
expired_sessions = []
|
||||
|
||||
# 只检查内存缓存中的过期记忆
|
||||
# Redis中的记忆会通过TTL自动过期,无需手动处理
|
||||
for cache_key, memory in self.memory_cache.items():
|
||||
if (current_time - memory.updated_at).days > settings.LLM_MEMORY_RETENTION_DAYS:
|
||||
expired_sessions.append(cache_key)
|
||||
|
||||
# 只清理内存缓存,不删除Redis中的键(Redis会自动过期)
|
||||
for cache_key in expired_sessions:
|
||||
if cache_key in self.memory_cache:
|
||||
del self.memory_cache[cache_key]
|
||||
|
||||
if expired_sessions:
|
||||
logger.info(f"清理了{len(expired_sessions)}个过期内存会话记忆")
|
||||
|
||||
except asyncio.CancelledError:
|
||||
break
|
||||
except Exception as e:
|
||||
logger.error(f"清理记忆时发生错误: {e}")
|
||||
70
app/agent/prompt/Agent Prompt.txt
Normal file
70
app/agent/prompt/Agent Prompt.txt
Normal file
@@ -0,0 +1,70 @@
|
||||
You are MoviePilot's AI assistant, specialized in helping users manage media resources including subscriptions, searching, downloading, and organization.
|
||||
|
||||
## Your Identity and Capabilities
|
||||
|
||||
You are an AI agent for the MoviePilot media management system with the following core capabilities:
|
||||
|
||||
### Media Management Capabilities
|
||||
- **Search Media Resources**: Search for movies, TV shows, anime, and other media content based on user requirements
|
||||
- **Add Subscriptions**: Create subscription rules for media content that users are interested in
|
||||
- **Manage Downloads**: Search and add torrent resources to downloaders
|
||||
- **Query Status**: Check subscription status, download progress, and media library status
|
||||
|
||||
### Intelligent Interaction Capabilities
|
||||
- **Natural Language Understanding**: Understand user requests in natural language (Chinese/English)
|
||||
- **Context Memory**: Remember conversation history and user preferences
|
||||
- **Smart Recommendations**: Recommend related media content based on user preferences
|
||||
- **Task Execution**: Automatically execute complex media management tasks
|
||||
|
||||
## Working Principles
|
||||
|
||||
1. **Always respond in Chinese**: All responses must be in Chinese
|
||||
2. **Proactive Task Completion**: Understand user needs and proactively use tools to complete related operations
|
||||
3. **Provide Detailed Information**: Explain what you're doing when executing operations
|
||||
4. **Safety First**: Confirm user intent before performing download operations
|
||||
5. **Continuous Learning**: Remember user preferences and habits to provide personalized service
|
||||
|
||||
## Common Operation Workflows
|
||||
|
||||
### Add Subscription Workflow
|
||||
1. Understand the media content the user wants to subscribe to
|
||||
2. Search for related media information
|
||||
3. Create subscription rules
|
||||
4. Confirm successful subscription
|
||||
|
||||
### Search and Download Workflow
|
||||
1. Understand user requirements (movie names, TV show names, etc.)
|
||||
2. Search for related media information
|
||||
3. Search for related torrent resources by media info
|
||||
4. Filter suitable resources
|
||||
5. Add to downloader
|
||||
|
||||
### Query Status Workflow
|
||||
1. Understand what information the user wants to know
|
||||
2. Query related data
|
||||
3. Organize and present results
|
||||
|
||||
## Tool Usage Guidelines
|
||||
|
||||
### Tool Usage Principles
|
||||
- Use tools proactively to complete user requests
|
||||
- Always explain what you're doing when using tools
|
||||
- Provide detailed results and explanations
|
||||
- Handle errors gracefully and suggest alternatives
|
||||
- Confirm user intent before performing download operations
|
||||
|
||||
### Response Format
|
||||
- Always respond in Chinese
|
||||
- Use clear and friendly language
|
||||
- Provide structured information when appropriate
|
||||
- Include relevant details about media content (title, year, type, etc.)
|
||||
- Explain the results of tool operations clearly
|
||||
|
||||
## Important Notes
|
||||
|
||||
- Always confirm user intent before performing download operations
|
||||
- If search results are not ideal, proactively adjust search strategies
|
||||
- Maintain a friendly and professional tone
|
||||
- Seek solutions proactively when encountering problems
|
||||
- Remember user preferences and provide personalized recommendations
|
||||
- Handle errors gracefully and provide helpful suggestions
|
||||
118
app/agent/prompt/__init__.py
Normal file
118
app/agent/prompt/__init__.py
Normal file
@@ -0,0 +1,118 @@
|
||||
"""提示词管理器"""
|
||||
|
||||
from pathlib import Path
|
||||
from typing import Dict
|
||||
|
||||
from app.log import logger
|
||||
|
||||
|
||||
class PromptManager:
|
||||
"""提示词管理器"""
|
||||
|
||||
def __init__(self, prompts_dir: str = None):
|
||||
if prompts_dir is None:
|
||||
self.prompts_dir = Path(__file__).parent
|
||||
else:
|
||||
self.prompts_dir = Path(prompts_dir)
|
||||
self.prompts_cache: Dict[str, str] = {}
|
||||
|
||||
def load_prompt(self, prompt_name: str) -> str:
|
||||
"""加载指定的提示词"""
|
||||
if prompt_name in self.prompts_cache:
|
||||
return self.prompts_cache[prompt_name]
|
||||
|
||||
prompt_file = self.prompts_dir / prompt_name
|
||||
|
||||
try:
|
||||
with open(prompt_file, 'r', encoding='utf-8') as f:
|
||||
content = f.read().strip()
|
||||
|
||||
# 缓存提示词
|
||||
self.prompts_cache[prompt_name] = content
|
||||
|
||||
logger.info(f"提示词加载成功: {prompt_name},长度:{len(content)} 字符")
|
||||
return content
|
||||
|
||||
except FileNotFoundError:
|
||||
logger.error(f"提示词文件不存在: {prompt_file}")
|
||||
raise
|
||||
except Exception as e:
|
||||
logger.error(f"加载提示词失败: {prompt_name}, 错误: {e}")
|
||||
raise
|
||||
|
||||
def get_agent_prompt(self, channel: str = None) -> str:
|
||||
"""
|
||||
获取智能体提示词
|
||||
:param channel: 消息渠道(Telegram、微信、Slack等)
|
||||
:return: 提示词内容
|
||||
"""
|
||||
base_prompt = self.load_prompt("Agent Prompt.txt")
|
||||
|
||||
# 根据渠道添加特定的格式说明
|
||||
if channel:
|
||||
channel_format_info = self._get_channel_format_info(channel)
|
||||
if channel_format_info:
|
||||
base_prompt += f"\n\n## Current Message Channel Format Requirements\n\n{channel_format_info}"
|
||||
|
||||
return base_prompt
|
||||
|
||||
@staticmethod
|
||||
def _get_channel_format_info(channel: str) -> str:
|
||||
"""
|
||||
获取渠道特定的格式说明
|
||||
:param channel: 消息渠道
|
||||
:return: 格式说明文本
|
||||
"""
|
||||
channel_lower = channel.lower() if channel else ""
|
||||
|
||||
if "telegram" in channel_lower:
|
||||
return """Messages are being sent through the **Telegram** channel. You must follow these format requirements:
|
||||
|
||||
**Supported Formatting:**
|
||||
- **Bold text**: Use `*text*` (single asterisk, not double asterisks)
|
||||
- **Italic text**: Use `_text_` (underscore)
|
||||
- **Code**: Use `` `text` `` (backtick)
|
||||
- **Links**: Use `[text](url)` format
|
||||
- **Strikethrough**: Use `~text~` (tilde)
|
||||
|
||||
**IMPORTANT - Headings and Lists:**
|
||||
- **DO NOT use heading syntax** (`#`, `##`, `###`) - Telegram MarkdownV2 does NOT support it
|
||||
- **Instead, use bold text for headings**: `*Heading Text*` followed by a blank line
|
||||
- **DO NOT use list syntax** (`-`, `*`, `+` at line start) - these will be escaped and won't display as lists
|
||||
- **For lists**, use plain text with line breaks, or use bold for list item labels: `*Item 1:* description`
|
||||
|
||||
**Examples:**
|
||||
- ❌ Wrong heading: `# Main Title` or `## Subtitle`
|
||||
- ✅ Correct heading: `*Main Title*` (followed by blank line) or `*Subtitle*` (followed by blank line)
|
||||
- ❌ Wrong list: `- Item 1` or `* Item 2`
|
||||
- ✅ Correct list format: `*Item 1:* description` or use plain text with line breaks
|
||||
|
||||
**Special Characters:**
|
||||
- Avoid using special characters that need escaping in MarkdownV2: `_*[]()~`>#+-=|{}.!` unless they are part of the formatting syntax
|
||||
- Keep formatting simple, avoid nested formatting to ensure proper rendering in Telegram"""
|
||||
|
||||
elif "wechat" in channel_lower or "微信" in channel:
|
||||
return """Messages are being sent through the **WeChat** channel. Please follow these format requirements:
|
||||
|
||||
- WeChat does NOT support Markdown formatting. Use plain text format only.
|
||||
- Do NOT use any Markdown syntax (such as `**bold**`, `*italic*`, `` `code` `` etc.)
|
||||
- Use plain text descriptions. You can organize content using line breaks and punctuation
|
||||
- Links can be provided directly as URLs, no Markdown link format needed
|
||||
- Keep messages concise and clear, use natural Chinese expressions"""
|
||||
|
||||
elif "slack" in channel_lower:
|
||||
return """Messages are being sent through the **Slack** channel. Please follow these format requirements:
|
||||
|
||||
- Slack supports Markdown formatting
|
||||
- Use `*text*` for bold
|
||||
- Use `_text_` for italic
|
||||
- Use `` `text` `` for code
|
||||
- Link format: `<url|text>` or `[text](url)`"""
|
||||
|
||||
# 其他渠道使用标准Markdown
|
||||
return None
|
||||
|
||||
def clear_cache(self):
|
||||
"""清空缓存"""
|
||||
self.prompts_cache.clear()
|
||||
logger.info("提示词缓存已清空")
|
||||
31
app/agent/tools/__init__.py
Normal file
31
app/agent/tools/__init__.py
Normal file
@@ -0,0 +1,31 @@
|
||||
"""MoviePilot工具模块"""
|
||||
|
||||
from .base import MoviePilotTool
|
||||
from app.agent.tools.impl.search_media import SearchMediaTool
|
||||
from app.agent.tools.impl.add_subscribe import AddSubscribeTool
|
||||
from app.agent.tools.impl.search_torrents import SearchTorrentsTool
|
||||
from app.agent.tools.impl.add_download import AddDownloadTool
|
||||
from app.agent.tools.impl.query_subscribes import QuerySubscribesTool
|
||||
from app.agent.tools.impl.query_downloads import QueryDownloadsTool
|
||||
from app.agent.tools.impl.query_downloaders import QueryDownloadersTool
|
||||
from app.agent.tools.impl.query_sites import QuerySitesTool
|
||||
from app.agent.tools.impl.get_recommendations import GetRecommendationsTool
|
||||
from app.agent.tools.impl.query_media_library import QueryMediaLibraryTool
|
||||
from app.agent.tools.impl.send_message import SendMessageTool
|
||||
from .factory import MoviePilotToolFactory
|
||||
|
||||
__all__ = [
|
||||
"MoviePilotTool",
|
||||
"SearchMediaTool",
|
||||
"AddSubscribeTool",
|
||||
"SearchTorrentsTool",
|
||||
"AddDownloadTool",
|
||||
"QuerySubscribesTool",
|
||||
"QueryDownloadsTool",
|
||||
"QueryDownloadersTool",
|
||||
"QuerySitesTool",
|
||||
"GetRecommendationsTool",
|
||||
"QueryMediaLibraryTool",
|
||||
"SendMessageTool",
|
||||
"MoviePilotToolFactory"
|
||||
]
|
||||
73
app/agent/tools/base.py
Normal file
73
app/agent/tools/base.py
Normal file
@@ -0,0 +1,73 @@
|
||||
"""MoviePilot工具基类"""
|
||||
from abc import ABCMeta, abstractmethod
|
||||
from typing import Callable, Any
|
||||
|
||||
from langchain.tools import BaseTool
|
||||
from pydantic import PrivateAttr
|
||||
|
||||
from app.agent import StreamingCallbackHandler
|
||||
from app.chain import ChainBase
|
||||
from app.schemas import Notification
|
||||
|
||||
|
||||
class ToolChain(ChainBase):
|
||||
pass
|
||||
|
||||
|
||||
class MoviePilotTool(BaseTool, metaclass=ABCMeta):
|
||||
"""MoviePilot专用工具基类"""
|
||||
|
||||
_session_id: str = PrivateAttr()
|
||||
_user_id: str = PrivateAttr()
|
||||
_channel: str = PrivateAttr(default=None)
|
||||
_source: str = PrivateAttr(default=None)
|
||||
_username: str = PrivateAttr(default=None)
|
||||
_callback_handler: StreamingCallbackHandler = PrivateAttr(default=None)
|
||||
|
||||
def __init__(self, session_id: str, user_id: str, **kwargs):
|
||||
super().__init__(**kwargs)
|
||||
self._session_id = session_id
|
||||
self._user_id = user_id
|
||||
|
||||
def _run(self, *args: Any, **kwargs: Any) -> Any:
|
||||
pass
|
||||
|
||||
async def _arun(self, **kwargs) -> str:
|
||||
"""异步运行工具"""
|
||||
# 发送运行工具前的消息
|
||||
agent_message = await self._callback_handler.get_message()
|
||||
if agent_message:
|
||||
await self.send_tool_message(agent_message, title="MoviePilot助手")
|
||||
# 发送执行工具说明
|
||||
explanation = kwargs.get("explanation")
|
||||
if explanation:
|
||||
await self.send_tool_message(f"▶️️{explanation}")
|
||||
return await self.run(**kwargs)
|
||||
|
||||
@abstractmethod
|
||||
async def run(self, **kwargs) -> str:
|
||||
raise NotImplementedError
|
||||
|
||||
def set_message_attr(self, channel: str, source: str, username: str):
|
||||
"""设置消息属性"""
|
||||
self._channel = channel
|
||||
self._source = source
|
||||
self._username = username
|
||||
|
||||
def set_callback_handler(self, callback_handler: StreamingCallbackHandler):
|
||||
"""设置回调处理器"""
|
||||
self._callback_handler = callback_handler
|
||||
|
||||
async def send_tool_message(self, message: str, title: str = ""):
|
||||
"""发送工具消息"""
|
||||
await ToolChain().async_post_message(
|
||||
Notification(
|
||||
channel=self._channel,
|
||||
source=self._source,
|
||||
userid=self._user_id,
|
||||
username=self._username,
|
||||
title=title,
|
||||
text=message
|
||||
),
|
||||
escape_markdown=False
|
||||
)
|
||||
84
app/agent/tools/factory.py
Normal file
84
app/agent/tools/factory.py
Normal file
@@ -0,0 +1,84 @@
|
||||
"""MoviePilot工具工厂"""
|
||||
|
||||
from typing import List, Callable
|
||||
|
||||
from app.agent.tools.impl.add_download import AddDownloadTool
|
||||
from app.agent.tools.impl.add_subscribe import AddSubscribeTool
|
||||
from app.agent.tools.impl.get_recommendations import GetRecommendationsTool
|
||||
from app.agent.tools.impl.query_downloaders import QueryDownloadersTool
|
||||
from app.agent.tools.impl.query_downloads import QueryDownloadsTool
|
||||
from app.agent.tools.impl.query_media_library import QueryMediaLibraryTool
|
||||
from app.agent.tools.impl.query_sites import QuerySitesTool
|
||||
from app.agent.tools.impl.query_subscribes import QuerySubscribesTool
|
||||
from app.agent.tools.impl.search_media import SearchMediaTool
|
||||
from app.agent.tools.impl.search_torrents import SearchTorrentsTool
|
||||
from app.agent.tools.impl.send_message import SendMessageTool
|
||||
from app.core.plugin import PluginManager
|
||||
from app.log import logger
|
||||
from .base import MoviePilotTool
|
||||
|
||||
|
||||
class MoviePilotToolFactory:
|
||||
"""MoviePilot工具工厂"""
|
||||
|
||||
@staticmethod
|
||||
def create_tools(session_id: str, user_id: str,
|
||||
channel: str = None, source: str = None, username: str = None,
|
||||
callback_handler: Callable = None) -> List[MoviePilotTool]:
|
||||
"""创建MoviePilot工具列表"""
|
||||
tools = []
|
||||
tool_definitions = [
|
||||
SearchMediaTool,
|
||||
AddSubscribeTool,
|
||||
SearchTorrentsTool,
|
||||
AddDownloadTool,
|
||||
QuerySubscribesTool,
|
||||
QueryDownloadsTool,
|
||||
QueryDownloadersTool,
|
||||
QuerySitesTool,
|
||||
GetRecommendationsTool,
|
||||
QueryMediaLibraryTool,
|
||||
SendMessageTool
|
||||
]
|
||||
# 创建内置工具
|
||||
for ToolClass in tool_definitions:
|
||||
tool = ToolClass(
|
||||
session_id=session_id,
|
||||
user_id=user_id
|
||||
)
|
||||
tool.set_message_attr(channel=channel, source=source, username=username)
|
||||
tool.set_callback_handler(callback_handler=callback_handler)
|
||||
tools.append(tool)
|
||||
|
||||
# 加载插件提供的工具
|
||||
plugin_tools_count = 0
|
||||
plugin_tools_info = PluginManager().get_plugin_agent_tools()
|
||||
for plugin_info in plugin_tools_info:
|
||||
plugin_id = plugin_info.get("plugin_id")
|
||||
plugin_name = plugin_info.get("plugin_name")
|
||||
tool_classes = plugin_info.get("tools", [])
|
||||
for ToolClass in tool_classes:
|
||||
try:
|
||||
# 验证工具类是否继承自 MoviePilotTool
|
||||
if not issubclass(ToolClass, MoviePilotTool):
|
||||
logger.warning(f"插件 {plugin_name}({plugin_id}) 提供的工具类 {ToolClass.__name__} 未继承自 MoviePilotTool,已跳过")
|
||||
continue
|
||||
# 创建工具实例
|
||||
tool = ToolClass(
|
||||
session_id=session_id,
|
||||
user_id=user_id
|
||||
)
|
||||
tool.set_message_attr(channel=channel, source=source, username=username)
|
||||
tool.set_callback_handler(callback_handler=callback_handler)
|
||||
tools.append(tool)
|
||||
plugin_tools_count += 1
|
||||
logger.debug(f"成功加载插件 {plugin_name}({plugin_id}) 的工具: {ToolClass.__name__}")
|
||||
except Exception as e:
|
||||
logger.error(f"加载插件 {plugin_name}({plugin_id}) 的工具 {ToolClass.__name__} 失败: {str(e)}")
|
||||
|
||||
builtin_tools_count = len(tool_definitions)
|
||||
if plugin_tools_count > 0:
|
||||
logger.info(f"成功创建 {len(tools)} 个MoviePilot工具(内置工具: {builtin_tools_count} 个,插件工具: {plugin_tools_count} 个)")
|
||||
else:
|
||||
logger.info(f"成功创建 {len(tools)} 个MoviePilot工具")
|
||||
return tools
|
||||
0
app/agent/tools/impl/__init__.py
Normal file
0
app/agent/tools/impl/__init__.py
Normal file
92
app/agent/tools/impl/add_download.py
Normal file
92
app/agent/tools/impl/add_download.py
Normal file
@@ -0,0 +1,92 @@
|
||||
"""添加下载工具"""
|
||||
|
||||
from typing import Optional, Type
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from app.agent.tools.base import MoviePilotTool, ToolChain
|
||||
from app.chain.download import DownloadChain
|
||||
from app.core.context import Context
|
||||
from app.core.metainfo import MetaInfo
|
||||
from app.db.site_oper import SiteOper
|
||||
from app.log import logger
|
||||
from app.schemas import TorrentInfo
|
||||
|
||||
|
||||
class AddDownloadInput(BaseModel):
|
||||
"""添加下载工具的输入参数模型"""
|
||||
explanation: str = Field(..., description="Clear explanation of why this tool is being used in the current context")
|
||||
site_name: str = Field(..., description="Name of the torrent site/source (e.g., 'The Pirate Bay')")
|
||||
torrent_title: str = Field(...,
|
||||
description="The display name/title of the torrent (e.g., 'The.Matrix.1999.1080p.BluRay.x264')")
|
||||
torrent_url: str = Field(..., description="Direct URL to the torrent file (.torrent) or magnet link")
|
||||
torrent_description: Optional[str] = Field(None,
|
||||
description="Brief description of the torrent content (optional)")
|
||||
downloader: Optional[str] = Field(None,
|
||||
description="Name of the downloader to use (optional, uses default if not specified)")
|
||||
save_path: Optional[str] = Field(None,
|
||||
description="Directory path where the downloaded files should be saved (optional, uses default path if not specified)")
|
||||
labels: Optional[str] = Field(None,
|
||||
description="Comma-separated list of labels/tags to assign to the download (optional, e.g., 'movie,hd,bluray')")
|
||||
|
||||
|
||||
class AddDownloadTool(MoviePilotTool):
|
||||
name: str = "add_download"
|
||||
description: str = "Add torrent download task to the configured downloader (qBittorrent, Transmission, etc.). Downloads the torrent file and starts the download process with specified settings."
|
||||
args_schema: Type[BaseModel] = AddDownloadInput
|
||||
|
||||
async def run(self, site_name: str, torrent_title: str, torrent_url: str, torrent_description: Optional[str] = None,
|
||||
downloader: Optional[str] = None, save_path: Optional[str] = None,
|
||||
labels: Optional[str] = None, **kwargs) -> str:
|
||||
logger.info(
|
||||
f"执行工具: {self.name}, 参数: site_name={site_name}, torrent_title={torrent_title}, torrent_url={torrent_url}, downloader={downloader}, save_path={save_path}, labels={labels}")
|
||||
|
||||
try:
|
||||
if not torrent_title or not torrent_url:
|
||||
return "错误:必须提供种子标题和下载链接"
|
||||
|
||||
# 使用DownloadChain添加下载
|
||||
download_chain = DownloadChain()
|
||||
|
||||
# 根据站点名称查询站点cookie
|
||||
if not site_name:
|
||||
return "错误:必须提供站点名称,请从搜索资源结果信息中获取"
|
||||
siteinfo = await SiteOper().async_get_by_name(site_name)
|
||||
if not siteinfo:
|
||||
return f"错误:未找到站点信息:{site_name}"
|
||||
|
||||
# 创建下载上下文
|
||||
torrent_info = TorrentInfo(
|
||||
title=torrent_title,
|
||||
description=torrent_description,
|
||||
enclosure=torrent_url,
|
||||
site_name=site_name,
|
||||
site_ua=siteinfo.ua,
|
||||
site_cookie=siteinfo.cookie,
|
||||
site_proxy=siteinfo.proxy,
|
||||
site_order=siteinfo.pri,
|
||||
site_downloader=siteinfo.downloader
|
||||
)
|
||||
meta_info = MetaInfo(title=torrent_title, subtitle=torrent_description)
|
||||
media_info = await ToolChain().async_recognize_media(meta=meta_info)
|
||||
if not media_info:
|
||||
return "错误:无法识别媒体信息,无法添加下载任务"
|
||||
context = Context(
|
||||
torrent_info=torrent_info,
|
||||
meta_info=meta_info,
|
||||
media_info=media_info
|
||||
)
|
||||
|
||||
did = download_chain.download_single(
|
||||
context=context,
|
||||
downloader=downloader,
|
||||
save_path=save_path,
|
||||
label=labels
|
||||
)
|
||||
if did:
|
||||
return f"成功添加下载任务:{torrent_title}"
|
||||
else:
|
||||
return "添加下载任务失败"
|
||||
except Exception as e:
|
||||
logger.error(f"添加下载任务失败: {e}", exc_info=True)
|
||||
return f"添加下载任务时发生错误: {str(e)}"
|
||||
60
app/agent/tools/impl/add_subscribe.py
Normal file
60
app/agent/tools/impl/add_subscribe.py
Normal file
@@ -0,0 +1,60 @@
|
||||
"""添加订阅工具"""
|
||||
|
||||
from typing import Optional, Type
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from app.agent.tools.base import MoviePilotTool
|
||||
from app.chain.subscribe import SubscribeChain
|
||||
from app.log import logger
|
||||
from app.schemas.types import MediaType
|
||||
|
||||
|
||||
class AddSubscribeInput(BaseModel):
|
||||
"""添加订阅工具的输入参数模型"""
|
||||
explanation: str = Field(..., description="Clear explanation of why this tool is being used in the current context")
|
||||
title: str = Field(..., description="The title of the media to subscribe to (e.g., 'The Matrix', 'Breaking Bad')")
|
||||
year: str = Field(..., description="Release year of the media (required for accurate identification)")
|
||||
media_type: str = Field(...,
|
||||
description="Type of media content: '电影' for films, '电视剧' for television series or anime series")
|
||||
season: Optional[int] = Field(None,
|
||||
description="Season number for TV shows (optional, if not specified will subscribe to all seasons)")
|
||||
tmdb_id: Optional[str] = Field(None,
|
||||
description="TMDB database ID for precise media identification (optional but recommended for accuracy)")
|
||||
|
||||
|
||||
class AddSubscribeTool(MoviePilotTool):
|
||||
name: str = "add_subscribe"
|
||||
description: str = "Add media subscription to create automated download rules for movies and TV shows. The system will automatically search and download new episodes or releases based on the subscription criteria."
|
||||
args_schema: Type[BaseModel] = AddSubscribeInput
|
||||
|
||||
async def run(self, title: str, year: str, media_type: str,
|
||||
season: Optional[int] = None, tmdb_id: Optional[str] = None, **kwargs) -> str:
|
||||
logger.info(
|
||||
f"执行工具: {self.name}, 参数: title={title}, year={year}, media_type={media_type}, season={season}, tmdb_id={tmdb_id}")
|
||||
|
||||
try:
|
||||
subscribe_chain = SubscribeChain()
|
||||
# 转换 tmdb_id 为整数
|
||||
tmdbid_int = None
|
||||
if tmdb_id:
|
||||
try:
|
||||
tmdbid_int = int(tmdb_id)
|
||||
except (ValueError, TypeError):
|
||||
logger.warning(f"无效的 tmdb_id: {tmdb_id},将忽略")
|
||||
|
||||
sid, message = await subscribe_chain.async_add(
|
||||
mtype=MediaType(media_type),
|
||||
title=title,
|
||||
year=year,
|
||||
tmdbid=tmdbid_int,
|
||||
season=season,
|
||||
username=self._user_id
|
||||
)
|
||||
if sid:
|
||||
return f"成功添加订阅:{title} ({year})"
|
||||
else:
|
||||
return f"添加订阅失败:{message}"
|
||||
except Exception as e:
|
||||
logger.error(f"添加订阅失败: {e}", exc_info=True)
|
||||
return f"添加订阅时发生错误: {str(e)}"
|
||||
84
app/agent/tools/impl/get_recommendations.py
Normal file
84
app/agent/tools/impl/get_recommendations.py
Normal file
@@ -0,0 +1,84 @@
|
||||
"""获取推荐工具"""
|
||||
|
||||
import json
|
||||
from typing import Optional, Type
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from app.agent.tools.base import MoviePilotTool
|
||||
from app.chain.recommend import RecommendChain
|
||||
from app.log import logger
|
||||
|
||||
|
||||
class GetRecommendationsInput(BaseModel):
|
||||
"""获取推荐工具的输入参数模型"""
|
||||
explanation: str = Field(..., description="Clear explanation of why this tool is being used in the current context")
|
||||
source: Optional[str] = Field("tmdb_trending",
|
||||
description="Recommendation source: 'tmdb_trending' for TMDB trending content, 'douban_hot' for Douban popular content, 'bangumi_calendar' for Bangumi anime calendar")
|
||||
media_type: Optional[str] = Field("all",
|
||||
description="Type of media content: '电影' for films, '电视剧' for television series or anime series, 'all' for all types")
|
||||
limit: Optional[int] = Field(20,
|
||||
description="Maximum number of recommendations to return (default: 20, maximum: 100)")
|
||||
|
||||
|
||||
class GetRecommendationsTool(MoviePilotTool):
|
||||
name: str = "get_recommendations"
|
||||
description: str = "Get trending and popular media recommendations from various sources. Returns curated lists of popular movies, TV shows, and anime based on different criteria like trending, ratings, or calendar schedules."
|
||||
args_schema: Type[BaseModel] = GetRecommendationsInput
|
||||
|
||||
async def run(self, source: Optional[str] = "tmdb_trending",
|
||||
media_type: Optional[str] = "all", limit: Optional[int] = 20, **kwargs) -> str:
|
||||
logger.info(f"执行工具: {self.name}, 参数: source={source}, media_type={media_type}, limit={limit}")
|
||||
try:
|
||||
name_dicts = {
|
||||
"tmdb_trending": "TMDB 热门推荐",
|
||||
"douban_hot": "豆瓣热门推荐",
|
||||
"bangumi_calendar": "番组计划推荐"
|
||||
}
|
||||
recommend_chain = RecommendChain()
|
||||
results = []
|
||||
if source == "tmdb_trending":
|
||||
results = await recommend_chain.async_tmdb_trending(limit=limit)
|
||||
elif source == "douban_hot":
|
||||
if media_type == "movie":
|
||||
results = await recommend_chain.async_douban_movie_hot(limit=limit)
|
||||
elif media_type == "tv":
|
||||
results = await recommend_chain.async_douban_tv_hot(limit=limit)
|
||||
else: # all
|
||||
results.extend(await recommend_chain.async_douban_movie_hot(limit=limit))
|
||||
results.extend(await recommend_chain.async_douban_tv_hot(limit=limit))
|
||||
elif source == "bangumi_calendar":
|
||||
results = await recommend_chain.async_bangumi_calendar(limit=limit)
|
||||
|
||||
if results:
|
||||
# 限制最多20条结果
|
||||
total_count = len(results)
|
||||
limited_results = results[:20]
|
||||
# 精简字段,只保留关键信息
|
||||
simplified_results = []
|
||||
for r in limited_results:
|
||||
# r 已经是字典格式(to_dict的结果)
|
||||
simplified = {
|
||||
"title": r.get("title"),
|
||||
"en_title": r.get("en_title"),
|
||||
"year": r.get("year"),
|
||||
"type": r.get("type"),
|
||||
"season": r.get("season"),
|
||||
"tmdb_id": r.get("tmdb_id"),
|
||||
"imdb_id": r.get("imdb_id"),
|
||||
"douban_id": r.get("douban_id"),
|
||||
"overview": r.get("overview", "")[:200] + "..." if r.get("overview") and len(r.get("overview", "")) > 200 else r.get("overview"),
|
||||
"vote_average": r.get("vote_average"),
|
||||
"poster_path": r.get("poster_path"),
|
||||
"detail_link": r.get("detail_link")
|
||||
}
|
||||
simplified_results.append(simplified)
|
||||
result_json = json.dumps(simplified_results, ensure_ascii=False, indent=2)
|
||||
# 如果结果被裁剪,添加提示信息
|
||||
if total_count > 20:
|
||||
return f"注意:推荐结果共找到 {total_count} 条,为节省上下文空间,仅显示前 20 条结果。\n\n{result_json}"
|
||||
return result_json
|
||||
return "未找到推荐内容。"
|
||||
except Exception as e:
|
||||
logger.error(f"获取推荐失败: {e}", exc_info=True)
|
||||
return f"获取推荐时发生错误: {str(e)}"
|
||||
34
app/agent/tools/impl/query_downloaders.py
Normal file
34
app/agent/tools/impl/query_downloaders.py
Normal file
@@ -0,0 +1,34 @@
|
||||
"""查询下载器工具"""
|
||||
|
||||
import json
|
||||
from typing import Type
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from app.agent.tools.base import MoviePilotTool
|
||||
from app.db.systemconfig_oper import SystemConfigOper
|
||||
from app.log import logger
|
||||
from app.schemas.types import SystemConfigKey
|
||||
|
||||
|
||||
class QueryDownloadersInput(BaseModel):
|
||||
"""查询下载器工具的输入参数模型"""
|
||||
explanation: str = Field(..., description="Clear explanation of why this tool is being used in the current context")
|
||||
|
||||
|
||||
class QueryDownloadersTool(MoviePilotTool):
|
||||
name: str = "query_downloaders"
|
||||
description: str = "Query downloader configuration and list all available downloaders. Shows downloader status, connection details, and configuration settings."
|
||||
args_schema: Type[BaseModel] = QueryDownloadersInput
|
||||
|
||||
async def run(self, **kwargs) -> str:
|
||||
logger.info(f"执行工具: {self.name}")
|
||||
try:
|
||||
system_config_oper = SystemConfigOper()
|
||||
downloaders_config = system_config_oper.get(SystemConfigKey.Downloaders)
|
||||
if downloaders_config:
|
||||
return json.dumps(downloaders_config, ensure_ascii=False, indent=2)
|
||||
return "未配置下载器。"
|
||||
except Exception as e:
|
||||
logger.error(f"查询下载器失败: {e}")
|
||||
return f"查询下载器时发生错误: {str(e)}"
|
||||
80
app/agent/tools/impl/query_downloads.py
Normal file
80
app/agent/tools/impl/query_downloads.py
Normal file
@@ -0,0 +1,80 @@
|
||||
"""查询下载工具"""
|
||||
|
||||
import json
|
||||
from typing import Optional, Type
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from app.agent.tools.base import MoviePilotTool
|
||||
from app.chain.download import DownloadChain
|
||||
from app.log import logger
|
||||
|
||||
|
||||
class QueryDownloadsInput(BaseModel):
|
||||
"""查询下载工具的输入参数模型"""
|
||||
explanation: str = Field(..., description="Clear explanation of why this tool is being used in the current context")
|
||||
downloader: Optional[str] = Field(None,
|
||||
description="Name of specific downloader to query (optional, if not provided queries all configured downloaders)")
|
||||
status: Optional[str] = Field("all",
|
||||
description="Filter downloads by status: 'downloading' for active downloads, 'completed' for finished downloads, 'paused' for paused downloads, 'all' for all downloads")
|
||||
|
||||
|
||||
class QueryDownloadsTool(MoviePilotTool):
|
||||
name: str = "query_downloads"
|
||||
description: str = "Query download status and list all active download tasks. Shows download progress, completion status, and task details from configured downloaders."
|
||||
args_schema: Type[BaseModel] = QueryDownloadsInput
|
||||
|
||||
async def run(self, downloader: Optional[str] = None,
|
||||
status: Optional[str] = "all", **kwargs) -> str:
|
||||
logger.info(f"执行工具: {self.name}, 参数: downloader={downloader}, status={status}")
|
||||
try:
|
||||
download_chain = DownloadChain()
|
||||
# 使用 DownloadChain.downloading 方法获取正在下载的任务
|
||||
downloads = download_chain.downloading(name=downloader)
|
||||
filtered_downloads = []
|
||||
for dl in downloads:
|
||||
if downloader and dl.downloader != downloader:
|
||||
continue
|
||||
if status != "all" and dl.status != status:
|
||||
continue
|
||||
filtered_downloads.append(dl)
|
||||
if filtered_downloads:
|
||||
# 限制最多20条结果
|
||||
total_count = len(filtered_downloads)
|
||||
limited_downloads = filtered_downloads[:20]
|
||||
# 精简字段,只保留关键信息
|
||||
simplified_downloads = []
|
||||
for d in limited_downloads:
|
||||
simplified = {
|
||||
"downloader": d.downloader,
|
||||
"hash": d.hash,
|
||||
"title": d.title,
|
||||
"name": d.name,
|
||||
"year": d.year,
|
||||
"season_episode": d.season_episode,
|
||||
"size": d.size,
|
||||
"progress": d.progress,
|
||||
"state": d.state,
|
||||
"upspeed": d.upspeed,
|
||||
"dlspeed": d.dlspeed,
|
||||
"left_time": d.left_time
|
||||
}
|
||||
# 精简 media 字段
|
||||
if d.media:
|
||||
simplified["media"] = {
|
||||
"tmdbid": d.media.get("tmdbid"),
|
||||
"type": d.media.get("type"),
|
||||
"title": d.media.get("title"),
|
||||
"season": d.media.get("season"),
|
||||
"episode": d.media.get("episode")
|
||||
}
|
||||
simplified_downloads.append(simplified)
|
||||
result_json = json.dumps(simplified_downloads, ensure_ascii=False, indent=2)
|
||||
# 如果结果被裁剪,添加提示信息
|
||||
if total_count > 20:
|
||||
return f"注意:查询结果共找到 {total_count} 条,为节省上下文空间,仅显示前 20 条结果。\n\n{result_json}"
|
||||
return result_json
|
||||
return "未找到相关下载任务"
|
||||
except Exception as e:
|
||||
logger.error(f"查询下载失败: {e}", exc_info=True)
|
||||
return f"查询下载时发生错误: {str(e)}"
|
||||
41
app/agent/tools/impl/query_media_library.py
Normal file
41
app/agent/tools/impl/query_media_library.py
Normal file
@@ -0,0 +1,41 @@
|
||||
"""查询媒体库工具"""
|
||||
|
||||
import json
|
||||
from typing import Optional, List, Type
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from app.agent.tools.base import MoviePilotTool
|
||||
from app.db.mediaserver_oper import MediaServerOper
|
||||
from app.log import logger
|
||||
from app.schemas import MediaServerItem
|
||||
|
||||
|
||||
class QueryMediaLibraryInput(BaseModel):
|
||||
"""查询媒体库工具的输入参数模型"""
|
||||
explanation: str = Field(..., description="Clear explanation of why this tool is being used in the current context")
|
||||
media_type: Optional[str] = Field("all",
|
||||
description="Type of media content: '电影' for films, '电视剧' for television series or anime series, 'all' for all types")
|
||||
title: Optional[str] = Field(None,
|
||||
description="Specific media title to check if it exists in the media library (optional, if provided checks for that specific media)")
|
||||
year: Optional[str] = Field(None,
|
||||
description="Release year of the media (optional, helps narrow down search results)")
|
||||
|
||||
|
||||
class QueryMediaLibraryTool(MoviePilotTool):
|
||||
name: str = "query_media_library"
|
||||
description: str = "Check if a specific media resource already exists in the media library (Plex, Emby, Jellyfin). Use this tool to verify whether a movie or TV series has been successfully processed and added to the media server before performing operations like downloading or subscribing."
|
||||
args_schema: Type[BaseModel] = QueryMediaLibraryInput
|
||||
|
||||
async def run(self, media_type: Optional[str] = "all",
|
||||
title: Optional[str] = None, year: Optional[str] = None, **kwargs) -> str:
|
||||
logger.info(f"执行工具: {self.name}, 参数: media_type={media_type}, title={title}")
|
||||
try:
|
||||
media_server_oper = MediaServerOper()
|
||||
filtered_medias: List[MediaServerItem] = await media_server_oper.async_exists(title=title, year=year, mtype=media_type)
|
||||
if filtered_medias:
|
||||
return json.dumps([m.to_dict() for m in filtered_medias])
|
||||
return "媒体库中未找到相关媒体"
|
||||
except Exception as e:
|
||||
logger.error(f"查询媒体库失败: {e}", exc_info=True)
|
||||
return f"查询媒体库时发生错误: {str(e)}"
|
||||
66
app/agent/tools/impl/query_sites.py
Normal file
66
app/agent/tools/impl/query_sites.py
Normal file
@@ -0,0 +1,66 @@
|
||||
"""查询站点工具"""
|
||||
|
||||
import json
|
||||
from typing import Optional, Type
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from app.agent.tools.base import MoviePilotTool
|
||||
from app.db.site_oper import SiteOper
|
||||
from app.log import logger
|
||||
|
||||
|
||||
class QuerySitesInput(BaseModel):
|
||||
"""查询站点工具的输入参数模型"""
|
||||
explanation: str = Field(..., description="Clear explanation of why this tool is being used in the current context")
|
||||
status: Optional[str] = Field("all",
|
||||
description="Filter sites by status: 'active' for enabled sites, 'inactive' for disabled sites, 'all' for all sites")
|
||||
name: Optional[str] = Field(None,
|
||||
description="Filter sites by name (partial match, optional)")
|
||||
|
||||
|
||||
class QuerySitesTool(MoviePilotTool):
|
||||
name: str = "query_sites"
|
||||
description: str = "Query site status and list all configured sites. Shows site name, domain, status, priority, and basic configuration."
|
||||
args_schema: Type[BaseModel] = QuerySitesInput
|
||||
|
||||
async def run(self, status: Optional[str] = "all", name: Optional[str] = None, **kwargs) -> str:
|
||||
logger.info(f"执行工具: {self.name}, 参数: status={status}, name={name}")
|
||||
try:
|
||||
site_oper = SiteOper()
|
||||
# 获取所有站点(按优先级排序)
|
||||
sites = await site_oper.async_list()
|
||||
filtered_sites = []
|
||||
for site in sites:
|
||||
# 按状态过滤
|
||||
if status == "active" and not site.is_active:
|
||||
continue
|
||||
if status == "inactive" and site.is_active:
|
||||
continue
|
||||
# 按名称过滤(部分匹配)
|
||||
if name and name.lower() not in (site.name or "").lower():
|
||||
continue
|
||||
filtered_sites.append(site)
|
||||
if filtered_sites:
|
||||
# 精简字段,只保留关键信息
|
||||
simplified_sites = []
|
||||
for s in filtered_sites:
|
||||
simplified = {
|
||||
"id": s.id,
|
||||
"name": s.name,
|
||||
"domain": s.domain,
|
||||
"url": s.url,
|
||||
"pri": s.pri,
|
||||
"is_active": s.is_active,
|
||||
"downloader": s.downloader,
|
||||
"proxy": s.proxy,
|
||||
"timeout": s.timeout
|
||||
}
|
||||
simplified_sites.append(simplified)
|
||||
result_json = json.dumps(simplified_sites, ensure_ascii=False, indent=2)
|
||||
return result_json
|
||||
return "未找到相关站点"
|
||||
except Exception as e:
|
||||
logger.error(f"查询站点失败: {e}", exc_info=True)
|
||||
return f"查询站点时发生错误: {str(e)}"
|
||||
|
||||
73
app/agent/tools/impl/query_subscribes.py
Normal file
73
app/agent/tools/impl/query_subscribes.py
Normal file
@@ -0,0 +1,73 @@
|
||||
"""查询订阅工具"""
|
||||
|
||||
import json
|
||||
from typing import Optional, Type
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from app.agent.tools.base import MoviePilotTool
|
||||
from app.db.subscribe_oper import SubscribeOper
|
||||
from app.log import logger
|
||||
|
||||
|
||||
class QuerySubscribesInput(BaseModel):
|
||||
"""查询订阅工具的输入参数模型"""
|
||||
explanation: str = Field(..., description="Clear explanation of why this tool is being used in the current context")
|
||||
status: Optional[str] = Field("all",
|
||||
description="Filter subscriptions by status: 'R' for enabled subscriptions, 'P' for disabled ones, 'all' for all subscriptions")
|
||||
media_type: Optional[str] = Field("all",
|
||||
description="Filter by media type: 'movie' for films, 'tv' for television series, 'all' for all types")
|
||||
|
||||
|
||||
class QuerySubscribesTool(MoviePilotTool):
|
||||
name: str = "query_subscribes"
|
||||
description: str = "Query subscription status and list all user subscriptions. Shows active subscriptions, their download status, and configuration details."
|
||||
args_schema: Type[BaseModel] = QuerySubscribesInput
|
||||
|
||||
async def run(self, status: Optional[str] = "all", media_type: Optional[str] = "all", **kwargs) -> str:
|
||||
logger.info(f"执行工具: {self.name}, 参数: status={status}, media_type={media_type}")
|
||||
try:
|
||||
subscribe_oper = SubscribeOper()
|
||||
subscribes = await subscribe_oper.async_list()
|
||||
filtered_subscribes = []
|
||||
for sub in subscribes:
|
||||
if status != "all" and sub.state != status:
|
||||
continue
|
||||
if media_type != "all" and sub.type != media_type:
|
||||
continue
|
||||
filtered_subscribes.append(sub)
|
||||
if filtered_subscribes:
|
||||
# 限制最多20条结果
|
||||
total_count = len(filtered_subscribes)
|
||||
limited_subscribes = filtered_subscribes[:20]
|
||||
# 精简字段,只保留关键信息
|
||||
simplified_subscribes = []
|
||||
for s in limited_subscribes:
|
||||
simplified = {
|
||||
"id": s.id,
|
||||
"name": s.name,
|
||||
"year": s.year,
|
||||
"type": s.type,
|
||||
"season": s.season,
|
||||
"tmdbid": s.tmdbid,
|
||||
"doubanid": s.doubanid,
|
||||
"bangumiid": s.bangumiid,
|
||||
"poster": s.poster,
|
||||
"vote": s.vote,
|
||||
"description": s.description[:200] + "..." if s.description and len(s.description) > 200 else s.description,
|
||||
"state": s.state,
|
||||
"total_episode": s.total_episode,
|
||||
"lack_episode": s.lack_episode,
|
||||
"last_update": s.last_update,
|
||||
"username": s.username
|
||||
}
|
||||
simplified_subscribes.append(simplified)
|
||||
result_json = json.dumps(simplified_subscribes, ensure_ascii=False, indent=2)
|
||||
# 如果结果被裁剪,添加提示信息
|
||||
if total_count > 20:
|
||||
return f"注意:查询结果共找到 {total_count} 条,为节省上下文空间,仅显示前 20 条结果。\n\n{result_json}"
|
||||
return result_json
|
||||
return "未找到相关订阅"
|
||||
except Exception as e:
|
||||
logger.error(f"查询订阅失败: {e}", exc_info=True)
|
||||
return f"查询订阅时发生错误: {str(e)}"
|
||||
96
app/agent/tools/impl/search_media.py
Normal file
96
app/agent/tools/impl/search_media.py
Normal file
@@ -0,0 +1,96 @@
|
||||
"""搜索媒体工具"""
|
||||
|
||||
import json
|
||||
from typing import Optional, Type
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from app.agent.tools.base import MoviePilotTool
|
||||
from app.chain.media import MediaChain
|
||||
from app.log import logger
|
||||
from app.schemas.types import MediaType
|
||||
|
||||
|
||||
class SearchMediaInput(BaseModel):
|
||||
"""搜索媒体工具的输入参数模型"""
|
||||
explanation: str = Field(..., description="Clear explanation of why this tool is being used in the current context")
|
||||
title: str = Field(..., description="The title of the media to search for (e.g., 'The Matrix', 'Breaking Bad')")
|
||||
year: Optional[str] = Field(None, description="Release year of the media (optional, helps narrow down results)")
|
||||
media_type: Optional[str] = Field(None,
|
||||
description="Type of media content: '电影' for films, '电视剧' for television series or anime series")
|
||||
season: Optional[int] = Field(None,
|
||||
description="Season number for TV shows and anime (optional, only applicable for series)")
|
||||
|
||||
|
||||
class SearchMediaTool(MoviePilotTool):
|
||||
name: str = "search_media"
|
||||
description: str = "Search for media resources including movies, TV shows, anime, etc. Supports searching by title, year, type, and other criteria. Returns detailed media information from TMDB database."
|
||||
args_schema: Type[BaseModel] = SearchMediaInput
|
||||
|
||||
async def run(self, title: str, year: Optional[str] = None,
|
||||
media_type: Optional[str] = None, season: Optional[int] = None, **kwargs) -> str:
|
||||
logger.info(
|
||||
f"执行工具: {self.name}, 参数: title={title}, year={year}, media_type={media_type}, season={season}")
|
||||
|
||||
try:
|
||||
media_chain = MediaChain()
|
||||
# 构建搜索标题
|
||||
search_title = title
|
||||
if year:
|
||||
search_title = f"{title} {year}"
|
||||
if media_type:
|
||||
search_title = f"{search_title} {media_type}"
|
||||
if season:
|
||||
search_title = f"{search_title} S{season:02d}"
|
||||
|
||||
# 使用 MediaChain.search 方法
|
||||
meta, results = await media_chain.async_search(title=search_title)
|
||||
|
||||
# 过滤结果
|
||||
if results:
|
||||
filtered_results = []
|
||||
for result in results:
|
||||
if year and result.year != year:
|
||||
continue
|
||||
if media_type:
|
||||
if result.type != MediaType(media_type):
|
||||
continue
|
||||
if season and result.season != season:
|
||||
continue
|
||||
filtered_results.append(result)
|
||||
|
||||
if filtered_results:
|
||||
# 限制最多20条结果
|
||||
total_count = len(filtered_results)
|
||||
limited_results = filtered_results[:20]
|
||||
# 精简字段,只保留关键信息
|
||||
simplified_results = []
|
||||
for r in limited_results:
|
||||
simplified = {
|
||||
"title": r.title,
|
||||
"en_title": r.en_title,
|
||||
"year": r.year,
|
||||
"type": r.type.value if r.type else None,
|
||||
"season": r.season,
|
||||
"tmdb_id": r.tmdb_id,
|
||||
"imdb_id": r.imdb_id,
|
||||
"douban_id": r.douban_id,
|
||||
"overview": r.overview[:200] + "..." if r.overview and len(r.overview) > 200 else r.overview,
|
||||
"vote_average": r.vote_average,
|
||||
"poster_path": r.poster_path,
|
||||
"detail_link": r.detail_link
|
||||
}
|
||||
simplified_results.append(simplified)
|
||||
result_json = json.dumps(simplified_results, ensure_ascii=False, indent=2)
|
||||
# 如果结果被裁剪,添加提示信息
|
||||
if total_count > 20:
|
||||
return f"注意:搜索结果共找到 {total_count} 条,为节省上下文空间,仅显示前 20 条结果。\n\n{result_json}"
|
||||
return result_json
|
||||
else:
|
||||
return f"未找到符合条件的媒体资源: {title}"
|
||||
else:
|
||||
return f"未找到相关媒体资源: {title}"
|
||||
except Exception as e:
|
||||
error_message = f"搜索媒体失败: {str(e)}"
|
||||
logger.error(f"搜索媒体失败: {e}", exc_info=True)
|
||||
return error_message
|
||||
122
app/agent/tools/impl/search_torrents.py
Normal file
122
app/agent/tools/impl/search_torrents.py
Normal file
@@ -0,0 +1,122 @@
|
||||
"""搜索种子工具"""
|
||||
|
||||
import json
|
||||
import re
|
||||
from typing import List, Optional, Type
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from app.agent.tools.base import MoviePilotTool
|
||||
from app.chain.search import SearchChain
|
||||
from app.log import logger
|
||||
from app.schemas.types import MediaType
|
||||
|
||||
|
||||
class SearchTorrentsInput(BaseModel):
|
||||
"""搜索种子工具的输入参数模型"""
|
||||
explanation: str = Field(..., description="Clear explanation of why this tool is being used in the current context")
|
||||
title: str = Field(...,
|
||||
description="The title of the media resource to search for (e.g., 'The Matrix 1999', 'Breaking Bad S01E01')")
|
||||
year: Optional[str] = Field(None,
|
||||
description="Release year of the media (optional, helps narrow down search results)")
|
||||
media_type: Optional[str] = Field(None,
|
||||
description="Type of media content: '电影' for films, '电视剧' for television series or anime series")
|
||||
season: Optional[int] = Field(None, description="Season number for TV shows (optional, only applicable for series)")
|
||||
sites: Optional[List[int]] = Field(None,
|
||||
description="Array of specific site IDs to search on (optional, if not provided searches all configured sites)")
|
||||
filter_pattern: Optional[str] = Field(None,
|
||||
description="Regular expression pattern to filter torrent titles by resolution, quality, or other keywords (e.g., '4K|2160p|UHD' for 4K content, '1080p|BluRay' for 1080p BluRay)")
|
||||
|
||||
|
||||
class SearchTorrentsTool(MoviePilotTool):
|
||||
name: str = "search_torrents"
|
||||
description: str = "Search for torrent files across configured indexer sites based on media information. Returns available torrent downloads with details like file size, quality, and download links."
|
||||
args_schema: Type[BaseModel] = SearchTorrentsInput
|
||||
|
||||
async def run(self, title: str, year: Optional[str] = None,
|
||||
media_type: Optional[str] = None, season: Optional[int] = None,
|
||||
sites: Optional[List[int]] = None, filter_pattern: Optional[str] = None, **kwargs) -> str:
|
||||
logger.info(
|
||||
f"执行工具: {self.name}, 参数: title={title}, year={year}, media_type={media_type}, season={season}, sites={sites}, filter_pattern={filter_pattern}")
|
||||
|
||||
try:
|
||||
search_chain = SearchChain()
|
||||
torrents = await search_chain.async_search_by_title(title=title, sites=sites)
|
||||
filtered_torrents = []
|
||||
# 编译正则表达式(如果提供)
|
||||
regex_pattern = None
|
||||
if filter_pattern:
|
||||
try:
|
||||
regex_pattern = re.compile(filter_pattern, re.IGNORECASE)
|
||||
except re.error as e:
|
||||
logger.warning(f"正则表达式编译失败: {filter_pattern}, 错误: {e}")
|
||||
return f"正则表达式格式错误: {str(e)}"
|
||||
|
||||
for torrent in torrents:
|
||||
# torrent 是 Context 对象,需要通过 meta_info 和 media_info 访问属性
|
||||
if year and torrent.meta_info and torrent.meta_info.year != year:
|
||||
continue
|
||||
if media_type and torrent.media_info:
|
||||
if torrent.media_info.type != MediaType(media_type):
|
||||
continue
|
||||
if season and torrent.meta_info and torrent.meta_info.begin_season != season:
|
||||
continue
|
||||
# 使用正则表达式过滤标题(分辨率、质量等关键字)
|
||||
if regex_pattern and torrent.torrent_info and torrent.torrent_info.title:
|
||||
if not regex_pattern.search(torrent.torrent_info.title):
|
||||
continue
|
||||
filtered_torrents.append(torrent)
|
||||
|
||||
if filtered_torrents:
|
||||
# 限制最多50条结果
|
||||
total_count = len(filtered_torrents)
|
||||
limited_torrents = filtered_torrents[:50]
|
||||
# 精简字段,只保留关键信息
|
||||
simplified_torrents = []
|
||||
for t in limited_torrents:
|
||||
simplified = {}
|
||||
# 精简 torrent_info
|
||||
if t.torrent_info:
|
||||
simplified["torrent_info"] = {
|
||||
"title": t.torrent_info.title,
|
||||
"size": t.torrent_info.size,
|
||||
"seeders": t.torrent_info.seeders,
|
||||
"peers": t.torrent_info.peers,
|
||||
"site_name": t.torrent_info.site_name,
|
||||
"enclosure": t.torrent_info.enclosure,
|
||||
"page_url": t.torrent_info.page_url,
|
||||
"volume_factor": t.torrent_info.volume_factor,
|
||||
"pubdate": t.torrent_info.pubdate
|
||||
}
|
||||
# 精简 media_info
|
||||
if t.media_info:
|
||||
simplified["media_info"] = {
|
||||
"title": t.media_info.title,
|
||||
"en_title": t.media_info.en_title,
|
||||
"year": t.media_info.year,
|
||||
"type": t.media_info.type.value if t.media_info.type else None,
|
||||
"season": t.media_info.season,
|
||||
"tmdb_id": t.media_info.tmdb_id
|
||||
}
|
||||
# 精简 meta_info
|
||||
if t.meta_info:
|
||||
simplified["meta_info"] = {
|
||||
"name": t.meta_info.name,
|
||||
"cn_name": t.meta_info.cn_name,
|
||||
"en_name": t.meta_info.en_name,
|
||||
"year": t.meta_info.year,
|
||||
"type": t.meta_info.type.value if t.meta_info.type else None,
|
||||
"begin_season": t.meta_info.begin_season
|
||||
}
|
||||
simplified_torrents.append(simplified)
|
||||
result_json = json.dumps(simplified_torrents, ensure_ascii=False, indent=2)
|
||||
# 如果结果被裁剪,添加提示信息
|
||||
if total_count > 50:
|
||||
return f"注意:搜索结果共找到 {total_count} 条,为节省上下文空间,仅显示前 50 条结果。\n\n{result_json}"
|
||||
return result_json
|
||||
else:
|
||||
return f"未找到相关种子资源: {title}"
|
||||
except Exception as e:
|
||||
error_message = f"搜索种子时发生错误: {str(e)}"
|
||||
logger.error(f"搜索种子失败: {e}", exc_info=True)
|
||||
return error_message
|
||||
31
app/agent/tools/impl/send_message.py
Normal file
31
app/agent/tools/impl/send_message.py
Normal file
@@ -0,0 +1,31 @@
|
||||
"""发送消息工具"""
|
||||
|
||||
from typing import Optional, Type
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from app.agent.tools.base import MoviePilotTool
|
||||
from app.log import logger
|
||||
|
||||
|
||||
class SendMessageInput(BaseModel):
|
||||
"""发送消息工具的输入参数模型"""
|
||||
explanation: str = Field(..., description="Clear explanation of why this tool is being used in the current context")
|
||||
message: str = Field(..., description="The message content to send to the user (should be clear and informative)")
|
||||
message_type: Optional[str] = Field("info",
|
||||
description="Type of message: 'info' for general information, 'success' for successful operations, 'warning' for warnings, 'error' for error messages")
|
||||
|
||||
|
||||
class SendMessageTool(MoviePilotTool):
|
||||
name: str = "send_message"
|
||||
description: str = "Send notification message to the user through configured notification channels (Telegram, Slack, WeChat, etc.). Used to inform users about operation results, errors, or important updates."
|
||||
args_schema: Type[BaseModel] = SendMessageInput
|
||||
|
||||
async def run(self, message: str, message_type: Optional[str] = None, **kwargs) -> str:
|
||||
logger.info(f"执行工具: {self.name}, 参数: message={message}, message_type={message_type}")
|
||||
try:
|
||||
await self.send_tool_message(message, title=message_type)
|
||||
return "消息已发送"
|
||||
except Exception as e:
|
||||
logger.error(f"发送消息失败: {e}")
|
||||
return f"发送消息时发生错误: {str(e)}"
|
||||
@@ -2,7 +2,7 @@ from fastapi import APIRouter
|
||||
|
||||
from app.api.endpoints import login, user, webhook, message, site, subscribe, \
|
||||
media, douban, search, plugin, tmdb, history, system, download, dashboard, \
|
||||
transfer, mediaserver, bangumi, storage, discover, recommend, workflow, torrent, monitoring
|
||||
transfer, mediaserver, bangumi, storage, discover, recommend, workflow, torrent
|
||||
|
||||
api_router = APIRouter()
|
||||
api_router.include_router(login.router, prefix="/login", tags=["login"])
|
||||
@@ -28,4 +28,3 @@ api_router.include_router(discover.router, prefix="/discover", tags=["discover"]
|
||||
api_router.include_router(recommend.router, prefix="/recommend", tags=["recommend"])
|
||||
api_router.include_router(workflow.router, prefix="/workflow", tags=["workflow"])
|
||||
api_router.include_router(torrent.router, prefix="/torrent", tags=["torrent"])
|
||||
api_router.include_router(monitoring.router, prefix="/monitoring", tags=["monitoring"])
|
||||
|
||||
@@ -123,7 +123,7 @@ async def schedule2(_: Annotated[str, Depends(verify_apitoken)]) -> Any:
|
||||
"""
|
||||
查询下载器信息 API_TOKEN认证(?token=xxx)
|
||||
"""
|
||||
return schedule()
|
||||
return await schedule()
|
||||
|
||||
|
||||
@router.get("/transfer", summary="文件整理统计", response_model=List[int])
|
||||
@@ -137,7 +137,7 @@ async def transfer(days: Optional[int] = 7,
|
||||
return [stat[1] for stat in transfer_stat]
|
||||
|
||||
|
||||
@router.get("/cpu", summary="获取当前CPU使用率", response_model=int)
|
||||
@router.get("/cpu", summary="获取当前CPU使用率", response_model=float)
|
||||
def cpu(_: schemas.TokenPayload = Depends(verify_token)) -> Any:
|
||||
"""
|
||||
获取当前CPU使用率
|
||||
@@ -145,7 +145,7 @@ def cpu(_: schemas.TokenPayload = Depends(verify_token)) -> Any:
|
||||
return SystemUtils.cpu_usage()
|
||||
|
||||
|
||||
@router.get("/cpu2", summary="获取当前CPU使用率(API_TOKEN)", response_model=int)
|
||||
@router.get("/cpu2", summary="获取当前CPU使用率(API_TOKEN)", response_model=float)
|
||||
def cpu2(_: Annotated[str, Depends(verify_apitoken)]) -> Any:
|
||||
"""
|
||||
获取当前CPU使用率 API_TOKEN认证(?token=xxx)
|
||||
|
||||
@@ -40,10 +40,10 @@ def download(
|
||||
metainfo = MetaInfo(title=torrent_in.title, subtitle=torrent_in.description)
|
||||
# 媒体信息
|
||||
mediainfo = MediaInfo()
|
||||
mediainfo.from_dict(media_in.dict())
|
||||
mediainfo.from_dict(media_in.model_dump())
|
||||
# 种子信息
|
||||
torrentinfo = TorrentInfo()
|
||||
torrentinfo.from_dict(torrent_in.dict())
|
||||
torrentinfo.from_dict(torrent_in.model_dump())
|
||||
# 手动下载始终使用选择的下载器
|
||||
torrentinfo.site_downloader = downloader
|
||||
# 上下文
|
||||
@@ -64,6 +64,9 @@ def download(
|
||||
@router.post("/add", summary="添加下载(不含媒体信息)", response_model=schemas.Response)
|
||||
def add(
|
||||
torrent_in: schemas.TorrentInfo,
|
||||
tmdbid: Annotated[int | None, Body()] = None,
|
||||
doubanid: Annotated[str | None, Body()] = None,
|
||||
bangumiid: Annotated[int | None, Body()] = None,
|
||||
downloader: Annotated[str | None, Body()] = None,
|
||||
save_path: Annotated[str | None, Body()] = None,
|
||||
current_user: User = Depends(get_current_active_user)) -> Any:
|
||||
@@ -73,12 +76,12 @@ def add(
|
||||
# 元数据
|
||||
metainfo = MetaInfo(title=torrent_in.title, subtitle=torrent_in.description)
|
||||
# 媒体信息
|
||||
mediainfo = MediaChain().recognize_media(meta=metainfo)
|
||||
mediainfo = MediaChain().recognize_media(meta=metainfo, tmdbid=tmdbid, doubanid=doubanid, bangumiid=bangumiid)
|
||||
if not mediainfo:
|
||||
return schemas.Response(success=False, message="无法识别媒体信息")
|
||||
# 种子信息
|
||||
torrentinfo = TorrentInfo()
|
||||
torrentinfo.from_dict(torrent_in.dict())
|
||||
torrentinfo.from_dict(torrent_in.model_dump())
|
||||
# 上下文
|
||||
context = Context(
|
||||
meta_info=metainfo,
|
||||
|
||||
@@ -14,7 +14,7 @@ from app.db.models import User
|
||||
from app.db.models.downloadhistory import DownloadHistory
|
||||
from app.db.models.transferhistory import TransferHistory
|
||||
from app.db.user_oper import get_current_active_superuser_async, get_current_active_superuser
|
||||
from app.schemas.types import EventType, MediaType
|
||||
from app.schemas.types import EventType
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
@@ -70,7 +70,7 @@ async def transfer_history(title: Optional[str] = None,
|
||||
|
||||
return schemas.Response(success=True,
|
||||
data={
|
||||
"list": result,
|
||||
"list": [item.to_dict() for item in result],
|
||||
"total": total,
|
||||
})
|
||||
|
||||
@@ -90,7 +90,7 @@ def delete_transfer_history(history_in: schemas.TransferHistory,
|
||||
# 册除媒体库文件
|
||||
if deletedest and history.dest_fileitem:
|
||||
dest_fileitem = schemas.FileItem(**history.dest_fileitem)
|
||||
StorageChain().delete_media_file(fileitem=dest_fileitem, mtype=MediaType(history.type))
|
||||
StorageChain().delete_media_file(dest_fileitem)
|
||||
|
||||
# 删除源文件
|
||||
if deletesrc and history.src_fileitem:
|
||||
|
||||
@@ -8,8 +8,10 @@ from app import schemas
|
||||
from app.chain.user import UserChain
|
||||
from app.core import security
|
||||
from app.core.config import settings
|
||||
from app.db.systemconfig_oper import SystemConfigOper
|
||||
from app.helper.sites import SitesHelper # noqa
|
||||
from app.helper.wallpaper import WallpaperHelper
|
||||
from app.schemas.types import SystemConfigKey
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
@@ -29,7 +31,10 @@ def login_access_token(
|
||||
if not success:
|
||||
raise HTTPException(status_code=401, detail=user_or_message)
|
||||
|
||||
# 用户等级
|
||||
level = SitesHelper().auth_level
|
||||
# 是否显示配置向导
|
||||
show_wizard = not SystemConfigOper().get(SystemConfigKey.SetupWizardState) and not settings.ADVANCED_MODE
|
||||
return schemas.Token(
|
||||
access_token=security.create_access_token(
|
||||
userid=user_or_message.id,
|
||||
@@ -45,6 +50,7 @@ def login_access_token(
|
||||
avatar=user_or_message.avatar,
|
||||
level=level,
|
||||
permissions=user_or_message.permissions or {},
|
||||
widzard=show_wizard
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -79,7 +79,7 @@ def exists(media_in: schemas.MediaInfo,
|
||||
"""
|
||||
# 转化为媒体信息对象
|
||||
mediainfo = MediaInfo()
|
||||
mediainfo.from_dict(media_in.dict())
|
||||
mediainfo.from_dict(media_in.model_dump())
|
||||
existsinfo: schemas.ExistMediaInfo = MediaServerChain().media_exists(mediainfo=mediainfo)
|
||||
if not existsinfo:
|
||||
return []
|
||||
@@ -108,7 +108,7 @@ def not_exists(media_in: schemas.MediaInfo,
|
||||
meta.year = media_in.year
|
||||
# 转化为媒体信息对象
|
||||
mediainfo = MediaInfo()
|
||||
mediainfo.from_dict(media_in.dict())
|
||||
mediainfo.from_dict(media_in.model_dump())
|
||||
exist_flag, no_exists = DownloadChain().get_no_exists_info(meta=meta, mediainfo=mediainfo)
|
||||
mediakey = mediainfo.tmdb_id or mediainfo.douban_id
|
||||
if mediainfo.type == MediaType.MOVIE:
|
||||
|
||||
@@ -132,7 +132,7 @@ async def subscribe(subscription: schemas.Subscription, _: schemas.TokenPayload
|
||||
"""
|
||||
客户端webpush通知订阅
|
||||
"""
|
||||
subinfo = subscription.dict()
|
||||
subinfo = subscription.model_dump()
|
||||
if subinfo not in global_vars.get_subscriptions():
|
||||
global_vars.push_subscription(subinfo)
|
||||
logger.debug(f"通知订阅成功: {subinfo}")
|
||||
@@ -148,7 +148,7 @@ def send_notification(payload: schemas.SubscriptionMessage, _: schemas.TokenPayl
|
||||
try:
|
||||
webpush(
|
||||
subscription_info=sub,
|
||||
data=json.dumps(payload.dict()),
|
||||
data=json.dumps(payload.model_dump()),
|
||||
vapid_private_key=settings.VAPID.get("privateKey"),
|
||||
vapid_claims={
|
||||
"sub": settings.VAPID.get("subject")
|
||||
|
||||
@@ -1,409 +0,0 @@
|
||||
from typing import Any, List
|
||||
|
||||
from fastapi import APIRouter, Depends, Query
|
||||
from fastapi.responses import HTMLResponse
|
||||
|
||||
from app import schemas
|
||||
from app.core.security import verify_apitoken
|
||||
from app.monitoring import monitor, get_metrics_response
|
||||
from app.schemas.monitoring import (
|
||||
PerformanceSnapshot,
|
||||
EndpointStats,
|
||||
ErrorRequest,
|
||||
MonitoringOverview
|
||||
)
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
@router.get("/overview", summary="获取监控概览", response_model=schemas.MonitoringOverview)
|
||||
def get_overview(_: str = Depends(verify_apitoken)) -> Any:
|
||||
"""
|
||||
获取完整的监控概览信息
|
||||
"""
|
||||
# 获取性能快照
|
||||
performance = monitor.get_performance_snapshot()
|
||||
|
||||
# 获取最活跃端点
|
||||
top_endpoints = monitor.get_top_endpoints(limit=10)
|
||||
|
||||
# 获取最近错误
|
||||
recent_errors = monitor.get_recent_errors(limit=20)
|
||||
|
||||
# 检查告警
|
||||
alerts = monitor.check_alerts()
|
||||
|
||||
return MonitoringOverview(
|
||||
performance=PerformanceSnapshot(
|
||||
timestamp=performance.timestamp,
|
||||
cpu_usage=performance.cpu_usage,
|
||||
memory_usage=performance.memory_usage,
|
||||
active_requests=performance.active_requests,
|
||||
request_rate=performance.request_rate,
|
||||
avg_response_time=performance.avg_response_time,
|
||||
error_rate=performance.error_rate,
|
||||
slow_requests=performance.slow_requests
|
||||
),
|
||||
top_endpoints=[EndpointStats(**endpoint) for endpoint in top_endpoints],
|
||||
recent_errors=[ErrorRequest(**error) for error in recent_errors],
|
||||
alerts=alerts
|
||||
)
|
||||
|
||||
|
||||
@router.get("/performance", summary="获取性能快照", response_model=schemas.PerformanceSnapshot)
|
||||
def get_performance(_: str = Depends(verify_apitoken)) -> Any:
|
||||
"""
|
||||
获取当前性能快照
|
||||
"""
|
||||
snapshot = monitor.get_performance_snapshot()
|
||||
return PerformanceSnapshot(
|
||||
timestamp=snapshot.timestamp,
|
||||
cpu_usage=snapshot.cpu_usage,
|
||||
memory_usage=snapshot.memory_usage,
|
||||
active_requests=snapshot.active_requests,
|
||||
request_rate=snapshot.request_rate,
|
||||
avg_response_time=snapshot.avg_response_time,
|
||||
error_rate=snapshot.error_rate,
|
||||
slow_requests=snapshot.slow_requests
|
||||
)
|
||||
|
||||
|
||||
@router.get("/endpoints", summary="获取端点统计", response_model=List[schemas.EndpointStats])
|
||||
def get_endpoints(
|
||||
limit: int = Query(10, ge=1, le=50, description="返回的端点数量"),
|
||||
_: str = Depends(verify_apitoken)
|
||||
) -> Any:
|
||||
"""
|
||||
获取最活跃的API端点统计
|
||||
"""
|
||||
endpoints = monitor.get_top_endpoints(limit=limit)
|
||||
return [EndpointStats(**endpoint) for endpoint in endpoints]
|
||||
|
||||
|
||||
@router.get("/errors", summary="获取错误请求", response_model=List[schemas.ErrorRequest])
|
||||
def get_errors(
|
||||
limit: int = Query(20, ge=1, le=100, description="返回的错误数量"),
|
||||
_: str = Depends(verify_apitoken)
|
||||
) -> Any:
|
||||
"""
|
||||
获取最近的错误请求记录
|
||||
"""
|
||||
errors = monitor.get_recent_errors(limit=limit)
|
||||
return [ErrorRequest(**error) for error in errors]
|
||||
|
||||
|
||||
@router.get("/alerts", summary="获取告警信息", response_model=List[str])
|
||||
def get_alerts(_: str = Depends(verify_apitoken)) -> Any:
|
||||
"""
|
||||
获取当前告警信息
|
||||
"""
|
||||
return monitor.check_alerts()
|
||||
|
||||
|
||||
@router.get("/metrics", summary="Prometheus指标")
|
||||
def get_prometheus_metrics(_: str = Depends(verify_apitoken)) -> Any:
|
||||
"""
|
||||
获取Prometheus格式的监控指标
|
||||
"""
|
||||
return get_metrics_response()
|
||||
|
||||
|
||||
@router.get("/dashboard", summary="监控仪表板", response_class=HTMLResponse)
|
||||
def get_dashboard(_: str = Depends(verify_apitoken)) -> Any:
|
||||
"""
|
||||
获取实时监控仪表板HTML页面
|
||||
"""
|
||||
return HTMLResponse(content="""
|
||||
<!DOCTYPE html>
|
||||
<html lang="zh-CN">
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
<title>MoviePilot 性能监控仪表板</title>
|
||||
<script src="https://cdn.jsdelivr.net/npm/chart.js"></script>
|
||||
<style>
|
||||
body {
|
||||
font-family: 'Segoe UI', Tahoma, Geneva, Verdana, sans-serif;
|
||||
margin: 0;
|
||||
padding: 20px;
|
||||
background-color: #f5f5f5;
|
||||
}
|
||||
.container {
|
||||
max-width: 1200px;
|
||||
margin: 0 auto;
|
||||
}
|
||||
.header {
|
||||
text-align: center;
|
||||
margin-bottom: 30px;
|
||||
color: #333;
|
||||
}
|
||||
.metrics-grid {
|
||||
display: grid;
|
||||
grid-template-columns: repeat(auto-fit, minmax(250px, 1fr));
|
||||
gap: 20px;
|
||||
margin-bottom: 30px;
|
||||
}
|
||||
.metric-card {
|
||||
background: white;
|
||||
padding: 20px;
|
||||
border-radius: 10px;
|
||||
box-shadow: 0 2px 10px rgba(0,0,0,0.1);
|
||||
text-align: center;
|
||||
}
|
||||
.metric-value {
|
||||
font-size: 2em;
|
||||
font-weight: bold;
|
||||
color: #2196F3;
|
||||
}
|
||||
.metric-label {
|
||||
color: #666;
|
||||
margin-top: 5px;
|
||||
}
|
||||
.chart-container {
|
||||
background: white;
|
||||
padding: 20px;
|
||||
border-radius: 10px;
|
||||
box-shadow: 0 2px 10px rgba(0,0,0,0.1);
|
||||
margin-bottom: 20px;
|
||||
}
|
||||
.alerts {
|
||||
background: #fff3cd;
|
||||
border: 1px solid #ffeaa7;
|
||||
border-radius: 5px;
|
||||
padding: 15px;
|
||||
margin-bottom: 20px;
|
||||
}
|
||||
.alert-item {
|
||||
color: #856404;
|
||||
margin: 5px 0;
|
||||
}
|
||||
.refresh-btn {
|
||||
background: #2196F3;
|
||||
color: white;
|
||||
border: none;
|
||||
padding: 10px 20px;
|
||||
border-radius: 5px;
|
||||
cursor: pointer;
|
||||
margin-bottom: 20px;
|
||||
}
|
||||
.refresh-btn:hover {
|
||||
background: #1976D2;
|
||||
}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<div class="container">
|
||||
<div class="header">
|
||||
<h1>🎬 MoviePilot 性能监控仪表板</h1>
|
||||
<button class="refresh-btn" onclick="refreshData()">刷新数据</button>
|
||||
</div>
|
||||
|
||||
<div id="alerts" class="alerts" style="display: none;">
|
||||
<h3>⚠️ 告警信息</h3>
|
||||
<div id="alerts-list"></div>
|
||||
</div>
|
||||
|
||||
<div class="metrics-grid">
|
||||
<div class="metric-card">
|
||||
<div class="metric-value" id="cpu-usage">--</div>
|
||||
<div class="metric-label">CPU使用率 (%)</div>
|
||||
</div>
|
||||
<div class="metric-card">
|
||||
<div class="metric-value" id="memory-usage">--</div>
|
||||
<div class="metric-label">内存使用率 (%)</div>
|
||||
</div>
|
||||
<div class="metric-card">
|
||||
<div class="metric-value" id="active-requests">--</div>
|
||||
<div class="metric-label">活跃请求数</div>
|
||||
</div>
|
||||
<div class="metric-card">
|
||||
<div class="metric-value" id="request-rate">--</div>
|
||||
<div class="metric-label">请求率 (req/min)</div>
|
||||
</div>
|
||||
<div class="metric-card">
|
||||
<div class="metric-value" id="avg-response-time">--</div>
|
||||
<div class="metric-label">平均响应时间 (s)</div>
|
||||
</div>
|
||||
<div class="metric-card">
|
||||
<div class="metric-value" id="error-rate">--</div>
|
||||
<div class="metric-label">错误率 (%)</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="chart-container">
|
||||
<h3>📊 性能趋势</h3>
|
||||
<canvas id="performanceChart" width="400" height="200"></canvas>
|
||||
</div>
|
||||
|
||||
<div class="chart-container">
|
||||
<h3>🔥 最活跃端点</h3>
|
||||
<canvas id="endpointsChart" width="400" height="200"></canvas>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<script>
|
||||
let performanceChart, endpointsChart;
|
||||
let performanceData = {
|
||||
labels: [],
|
||||
cpu: [],
|
||||
memory: [],
|
||||
requests: []
|
||||
};
|
||||
|
||||
// 初始化图表
|
||||
function initCharts() {
|
||||
const ctx1 = document.getElementById('performanceChart').getContext('2d');
|
||||
performanceChart = new Chart(ctx1, {
|
||||
type: 'line',
|
||||
data: {
|
||||
labels: performanceData.labels,
|
||||
datasets: [{
|
||||
label: 'CPU使用率 (%)',
|
||||
data: performanceData.cpu,
|
||||
borderColor: '#2196F3',
|
||||
backgroundColor: 'rgba(33, 150, 243, 0.1)',
|
||||
tension: 0.4
|
||||
}, {
|
||||
label: '内存使用率 (%)',
|
||||
data: performanceData.memory,
|
||||
borderColor: '#4CAF50',
|
||||
backgroundColor: 'rgba(76, 175, 80, 0.1)',
|
||||
tension: 0.4
|
||||
}, {
|
||||
label: '活跃请求数',
|
||||
data: performanceData.requests,
|
||||
borderColor: '#FF9800',
|
||||
backgroundColor: 'rgba(255, 152, 0, 0.1)',
|
||||
tension: 0.4
|
||||
}]
|
||||
},
|
||||
options: {
|
||||
responsive: true,
|
||||
scales: {
|
||||
y: {
|
||||
beginAtZero: true
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
const ctx2 = document.getElementById('endpointsChart').getContext('2d');
|
||||
endpointsChart = new Chart(ctx2, {
|
||||
type: 'bar',
|
||||
data: {
|
||||
labels: [],
|
||||
datasets: [{
|
||||
label: '请求数',
|
||||
data: [],
|
||||
backgroundColor: 'rgba(33, 150, 243, 0.8)'
|
||||
}]
|
||||
},
|
||||
options: {
|
||||
responsive: true,
|
||||
scales: {
|
||||
y: {
|
||||
beginAtZero: true
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
// 更新性能数据
|
||||
function updatePerformanceData(data) {
|
||||
const now = new Date().toLocaleTimeString();
|
||||
|
||||
performanceData.labels.push(now);
|
||||
performanceData.cpu.push(data.performance.cpu_usage);
|
||||
performanceData.memory.push(data.performance.memory_usage);
|
||||
performanceData.requests.push(data.performance.active_requests);
|
||||
|
||||
// 保持最近20个数据点
|
||||
if (performanceData.labels.length > 20) {
|
||||
performanceData.labels.shift();
|
||||
performanceData.cpu.shift();
|
||||
performanceData.memory.shift();
|
||||
performanceData.requests.shift();
|
||||
}
|
||||
|
||||
// 更新图表
|
||||
performanceChart.data.labels = performanceData.labels;
|
||||
performanceChart.data.datasets[0].data = performanceData.cpu;
|
||||
performanceChart.data.datasets[1].data = performanceData.memory;
|
||||
performanceChart.data.datasets[2].data = performanceData.requests;
|
||||
performanceChart.update();
|
||||
|
||||
// 更新端点图表
|
||||
const endpointLabels = data.top_endpoints.map(e => e.endpoint.substring(0, 20));
|
||||
const endpointData = data.top_endpoints.map(e => e.count);
|
||||
|
||||
endpointsChart.data.labels = endpointLabels;
|
||||
endpointsChart.data.datasets[0].data = endpointData;
|
||||
endpointsChart.update();
|
||||
}
|
||||
|
||||
// 更新指标显示
|
||||
function updateMetrics(data) {
|
||||
document.getElementById('cpu-usage').textContent = data.performance.cpu_usage.toFixed(1);
|
||||
document.getElementById('memory-usage').textContent = data.performance.memory_usage.toFixed(1);
|
||||
document.getElementById('active-requests').textContent = data.performance.active_requests;
|
||||
document.getElementById('request-rate').textContent = data.performance.request_rate.toFixed(0);
|
||||
document.getElementById('avg-response-time').textContent = data.performance.avg_response_time.toFixed(3);
|
||||
document.getElementById('error-rate').textContent = (data.performance.error_rate * 100).toFixed(2);
|
||||
}
|
||||
|
||||
// 更新告警
|
||||
function updateAlerts(alerts) {
|
||||
const alertsDiv = document.getElementById('alerts');
|
||||
const alertsList = document.getElementById('alerts-list');
|
||||
|
||||
if (alerts.length > 0) {
|
||||
alertsDiv.style.display = 'block';
|
||||
alertsList.innerHTML = alerts.map(alert =>
|
||||
`<div class="alert-item">⚠️ ${alert}</div>`
|
||||
).join('');
|
||||
} else {
|
||||
alertsDiv.style.display = 'none';
|
||||
}
|
||||
}
|
||||
|
||||
// 获取URL中的token参数
|
||||
function getTokenFromUrl() {
|
||||
const urlParams = new URLSearchParams(window.location.search);
|
||||
return urlParams.get('token');
|
||||
}
|
||||
|
||||
// 刷新数据
|
||||
async function refreshData() {
|
||||
try {
|
||||
const token = getTokenFromUrl();
|
||||
if (!token) {
|
||||
console.error('未找到token参数');
|
||||
return;
|
||||
}
|
||||
|
||||
const response = await fetch(`/api/v1/monitoring/overview?token=${token}`);
|
||||
|
||||
if (response.ok) {
|
||||
const data = await response.json();
|
||||
updateMetrics(data);
|
||||
updatePerformanceData(data);
|
||||
updateAlerts(data.alerts);
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('获取监控数据失败:', error);
|
||||
}
|
||||
}
|
||||
|
||||
// 页面加载完成后初始化
|
||||
document.addEventListener('DOMContentLoaded', function() {
|
||||
initCharts();
|
||||
refreshData();
|
||||
|
||||
// 每5秒自动刷新
|
||||
setInterval(refreshData, 5000);
|
||||
});
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
""")
|
||||
@@ -3,7 +3,7 @@ import shutil
|
||||
from typing import Annotated, Any, List, Optional
|
||||
|
||||
import aiofiles
|
||||
from aiopath import AsyncPath
|
||||
from anyio import Path as AsyncPath
|
||||
from fastapi import APIRouter, Depends, Header, HTTPException
|
||||
from fastapi.concurrency import run_in_threadpool
|
||||
from starlette import status
|
||||
@@ -463,6 +463,36 @@ async def update_folder_plugins(folder_name: str, plugin_ids: List[str],
|
||||
return schemas.Response(success=True, message=f"文件夹 '{folder_name}' 中的插件已更新")
|
||||
|
||||
|
||||
@router.post("/clone/{plugin_id}", summary="创建插件分身", response_model=schemas.Response)
|
||||
def clone_plugin(plugin_id: str,
|
||||
clone_data: dict,
|
||||
_: User = Depends(get_current_active_superuser)) -> Any:
|
||||
"""
|
||||
创建插件分身
|
||||
"""
|
||||
try:
|
||||
success, message = PluginManager().clone_plugin(
|
||||
plugin_id=plugin_id,
|
||||
suffix=clone_data.get("suffix", ""),
|
||||
name=clone_data.get("name", ""),
|
||||
description=clone_data.get("description", ""),
|
||||
version=clone_data.get("version", ""),
|
||||
icon=clone_data.get("icon", "")
|
||||
)
|
||||
|
||||
if success:
|
||||
# 注册插件服务
|
||||
reload_plugin(message)
|
||||
# 将分身插件添加到原插件所在的文件夹中
|
||||
_add_clone_to_plugin_folder(plugin_id, message)
|
||||
return schemas.Response(success=True, message="插件分身创建成功")
|
||||
else:
|
||||
return schemas.Response(success=False, message=message)
|
||||
except Exception as e:
|
||||
logger.error(f"创建插件分身失败:{str(e)}")
|
||||
return schemas.Response(success=False, message=f"创建插件分身失败:{str(e)}")
|
||||
|
||||
|
||||
@router.get("/{plugin_id}", summary="获取插件配置")
|
||||
async def plugin_config(plugin_id: str,
|
||||
_: User = Depends(get_current_active_superuser_async)) -> dict:
|
||||
@@ -528,36 +558,6 @@ def uninstall_plugin(plugin_id: str,
|
||||
return schemas.Response(success=True)
|
||||
|
||||
|
||||
@router.post("/clone/{plugin_id}", summary="创建插件分身", response_model=schemas.Response)
|
||||
def clone_plugin(plugin_id: str,
|
||||
clone_data: dict,
|
||||
_: User = Depends(get_current_active_superuser)) -> Any:
|
||||
"""
|
||||
创建插件分身
|
||||
"""
|
||||
try:
|
||||
success, message = PluginManager().clone_plugin(
|
||||
plugin_id=plugin_id,
|
||||
suffix=clone_data.get("suffix", ""),
|
||||
name=clone_data.get("name", ""),
|
||||
description=clone_data.get("description", ""),
|
||||
version=clone_data.get("version", ""),
|
||||
icon=clone_data.get("icon", "")
|
||||
)
|
||||
|
||||
if success:
|
||||
# 注册插件服务
|
||||
reload_plugin(message)
|
||||
# 将分身插件添加到原插件所在的文件夹中
|
||||
_add_clone_to_plugin_folder(plugin_id, message)
|
||||
return schemas.Response(success=True, message="插件分身创建成功")
|
||||
else:
|
||||
return schemas.Response(success=False, message=message)
|
||||
except Exception as e:
|
||||
logger.error(f"创建插件分身失败:{str(e)}")
|
||||
return schemas.Response(success=False, message=f"创建插件分身失败:{str(e)}")
|
||||
|
||||
|
||||
def _add_clone_to_plugin_folder(original_plugin_id: str, clone_plugin_id: str):
|
||||
"""
|
||||
将分身插件添加到原插件所在的文件夹中
|
||||
|
||||
@@ -20,7 +20,7 @@ async def search_latest(_: schemas.TokenPayload = Depends(verify_token)) -> Any:
|
||||
"""
|
||||
查询搜索结果
|
||||
"""
|
||||
torrents = await SearchChain().async_last_search_results()
|
||||
torrents = await SearchChain().async_last_search_results() or []
|
||||
return [torrent.to_dict() for torrent in torrents]
|
||||
|
||||
|
||||
|
||||
@@ -67,7 +67,7 @@ async def add_site(
|
||||
site_in.name = site_info.get("name")
|
||||
site_in.id = None
|
||||
site_in.public = 1 if site_info.get("public") else 0
|
||||
site = Site(**site_in.dict())
|
||||
site = Site(**site_in.model_dump())
|
||||
site.create(db)
|
||||
# 通知站点更新
|
||||
await eventmanager.async_send_event(EventType.SiteUpdated, {
|
||||
@@ -92,7 +92,7 @@ async def update_site(
|
||||
# 校正地址格式
|
||||
_scheme, _netloc = StringUtils.get_url_netloc(site_in.url)
|
||||
site_in.url = f"{_scheme}://{_netloc}/"
|
||||
await site.async_update(db, site_in.dict())
|
||||
await site.async_update(db, site_in.model_dump())
|
||||
# 通知站点更新
|
||||
await eventmanager.async_send_event(EventType.SiteUpdated, {
|
||||
"domain": site_in.domain
|
||||
@@ -399,7 +399,7 @@ def auth_site(
|
||||
if not auth_info or not auth_info.site or not auth_info.params:
|
||||
return schemas.Response(success=False, message="请输入认证站点和认证参数")
|
||||
status, msg = SitesHelper().check_user(auth_info.site, auth_info.params)
|
||||
SystemConfigOper().set(SystemConfigKey.UserSiteAuthParams, auth_info.dict())
|
||||
SystemConfigOper().set(SystemConfigKey.UserSiteAuthParams, auth_info.model_dump())
|
||||
# 认证成功后,重新初始化插件
|
||||
PluginManager().init_config()
|
||||
Scheduler().init_plugin_jobs()
|
||||
|
||||
@@ -15,6 +15,7 @@ from app.db.models import User
|
||||
from app.db.user_oper import get_current_active_superuser, get_current_active_superuser_async
|
||||
from app.helper.progress import ProgressHelper
|
||||
from app.schemas.types import ProgressKey
|
||||
from app.utils.string import StringUtils
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
@@ -80,7 +81,7 @@ def list_files(fileitem: schemas.FileItem,
|
||||
file_list = StorageChain().list_files(fileitem)
|
||||
if file_list:
|
||||
if sort == "name":
|
||||
file_list.sort(key=lambda x: x.name or "")
|
||||
file_list.sort(key=lambda x: StringUtils.natural_sort_key(x.name or ""))
|
||||
else:
|
||||
file_list.sort(key=lambda x: x.modify_time or datetime.min, reverse=True)
|
||||
return file_list
|
||||
@@ -171,15 +172,14 @@ def rename(fileitem: schemas.FileItem,
|
||||
sub_files: List[schemas.FileItem] = StorageChain().list_files(fileitem)
|
||||
if sub_files:
|
||||
# 开始进度
|
||||
progress = ProgressHelper()
|
||||
progress.start(ProgressKey.BatchRename)
|
||||
progress = ProgressHelper(ProgressKey.BatchRename)
|
||||
progress.start()
|
||||
total = len(sub_files)
|
||||
handled = 0
|
||||
for sub_file in sub_files:
|
||||
handled += 1
|
||||
progress.update(value=handled / total * 100,
|
||||
text=f"正在处理 {sub_file.name} ...",
|
||||
key=ProgressKey.BatchRename)
|
||||
text=f"正在处理 {sub_file.name} ...")
|
||||
if sub_file.type == "dir":
|
||||
continue
|
||||
if not sub_file.extension:
|
||||
@@ -190,19 +190,19 @@ def rename(fileitem: schemas.FileItem,
|
||||
meta = MetaInfoPath(sub_path)
|
||||
mediainfo = transferchain.recognize_media(meta)
|
||||
if not mediainfo:
|
||||
progress.end(ProgressKey.BatchRename)
|
||||
progress.end()
|
||||
return schemas.Response(success=False, message=f"{sub_path.name} 未识别到媒体信息")
|
||||
new_path = transferchain.recommend_name(meta=meta, mediainfo=mediainfo)
|
||||
if not new_path:
|
||||
progress.end(ProgressKey.BatchRename)
|
||||
progress.end()
|
||||
return schemas.Response(success=False, message=f"{sub_path.name} 未识别到新名称")
|
||||
ret: schemas.Response = rename(fileitem=sub_file,
|
||||
new_name=Path(new_path).name,
|
||||
recursive=False)
|
||||
if not ret.success:
|
||||
progress.end(ProgressKey.BatchRename)
|
||||
progress.end()
|
||||
return schemas.Response(success=False, message=f"{sub_path.name} 重命名失败!")
|
||||
progress.end(ProgressKey.BatchRename)
|
||||
progress.end()
|
||||
# 重命名自己
|
||||
result = StorageChain().rename_file(fileitem, new_name)
|
||||
if result:
|
||||
|
||||
@@ -79,7 +79,7 @@ async def create_subscribe(
|
||||
# 订阅用户
|
||||
subscribe_in.username = current_user.name
|
||||
# 转化为字典
|
||||
subscribe_dict = subscribe_in.dict()
|
||||
subscribe_dict = subscribe_in.model_dump()
|
||||
if subscribe_in.id:
|
||||
subscribe_dict.pop("id", None)
|
||||
sid, message = await SubscribeChain().async_add(mtype=mtype,
|
||||
@@ -106,7 +106,7 @@ async def update_subscribe(
|
||||
return schemas.Response(success=False, message="订阅不存在")
|
||||
# 避免更新缺失集数
|
||||
old_subscribe_dict = subscribe.to_dict()
|
||||
subscribe_dict = subscribe_in.dict()
|
||||
subscribe_dict = subscribe_in.model_dump()
|
||||
if not subscribe_in.lack_episode:
|
||||
# 没有缺失集数时,缺失集数清空,避免更新为0
|
||||
subscribe_dict.pop("lack_episode")
|
||||
@@ -421,11 +421,23 @@ async def popular_subscribes(
|
||||
page: Optional[int] = 1,
|
||||
count: Optional[int] = 30,
|
||||
min_sub: Optional[int] = None,
|
||||
genre_id: Optional[int] = None,
|
||||
min_rating: Optional[float] = None,
|
||||
max_rating: Optional[float] = None,
|
||||
sort_type: Optional[str] = None,
|
||||
_: schemas.TokenPayload = Depends(verify_token)) -> Any:
|
||||
"""
|
||||
查询热门订阅
|
||||
"""
|
||||
subscribes = await SubscribeHelper().async_get_statistic(stype=stype, page=page, count=count)
|
||||
subscribes = await SubscribeHelper().async_get_statistic(
|
||||
stype=stype,
|
||||
page=page,
|
||||
count=count,
|
||||
genre_id=genre_id,
|
||||
min_rating=min_rating,
|
||||
max_rating=max_rating,
|
||||
sort_type=sort_type
|
||||
)
|
||||
if subscribes:
|
||||
ret_medias = []
|
||||
for sub in subscribes:
|
||||
@@ -517,7 +529,7 @@ async def subscribe_fork(
|
||||
"""
|
||||
复用订阅
|
||||
"""
|
||||
sub_dict = sub.dict()
|
||||
sub_dict = sub.model_dump()
|
||||
sub_dict.pop("id")
|
||||
for key in list(sub_dict.keys()):
|
||||
if not hasattr(schemas.Subscribe(), key):
|
||||
@@ -570,11 +582,23 @@ async def popular_subscribes(
|
||||
name: Optional[str] = None,
|
||||
page: Optional[int] = 1,
|
||||
count: Optional[int] = 30,
|
||||
genre_id: Optional[int] = None,
|
||||
min_rating: Optional[float] = None,
|
||||
max_rating: Optional[float] = None,
|
||||
sort_type: Optional[str] = None,
|
||||
_: schemas.TokenPayload = Depends(verify_token)) -> Any:
|
||||
"""
|
||||
查询分享的订阅
|
||||
"""
|
||||
return await SubscribeHelper().async_get_shares(name=name, page=page, count=count)
|
||||
return await SubscribeHelper().async_get_shares(
|
||||
name=name,
|
||||
page=page,
|
||||
count=count,
|
||||
genre_id=genre_id,
|
||||
min_rating=min_rating,
|
||||
max_rating=max_rating,
|
||||
sort_type=sort_type
|
||||
)
|
||||
|
||||
|
||||
@router.get("/share/statistics", summary="查询订阅分享统计", response_model=List[schemas.SubscribeShareStatistics])
|
||||
|
||||
@@ -4,19 +4,22 @@ import json
|
||||
import re
|
||||
from collections import deque
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Optional, Union, Annotated
|
||||
|
||||
import aiofiles
|
||||
import pillow_avif # noqa 用于自动注册AVIF支持
|
||||
from PIL import Image
|
||||
from aiopath import AsyncPath
|
||||
from anyio import Path as AsyncPath
|
||||
from app.helper.sites import SitesHelper # noqa # noqa
|
||||
from fastapi import APIRouter, Body, Depends, HTTPException, Header, Request, Response
|
||||
from fastapi.responses import StreamingResponse
|
||||
|
||||
from app import schemas
|
||||
from app.chain.mediaserver import MediaServerChain
|
||||
from app.chain.search import SearchChain
|
||||
from app.chain.system import SystemChain
|
||||
from app.core.cache import AsyncFileCache
|
||||
from app.core.config import global_vars, settings
|
||||
from app.core.event import eventmanager
|
||||
from app.core.metainfo import MetaInfo
|
||||
@@ -24,7 +27,8 @@ from app.core.module import ModuleManager
|
||||
from app.core.security import verify_apitoken, verify_resource_token, verify_token
|
||||
from app.db.models import User
|
||||
from app.db.systemconfig_oper import SystemConfigOper
|
||||
from app.db.user_oper import get_current_active_superuser, get_current_active_superuser_async
|
||||
from app.db.user_oper import get_current_active_superuser, get_current_active_superuser_async, \
|
||||
get_current_active_user_async
|
||||
from app.helper.mediaserver import MediaServerHelper
|
||||
from app.helper.message import MessageHelper
|
||||
from app.helper.progress import ProgressHelper
|
||||
@@ -47,86 +51,81 @@ router = APIRouter()
|
||||
async def fetch_image(
|
||||
url: str,
|
||||
proxy: bool = False,
|
||||
use_disk_cache: bool = False,
|
||||
use_cache: bool = False,
|
||||
if_none_match: Optional[str] = None,
|
||||
allowed_domains: Optional[set[str]] = None) -> Response:
|
||||
cookies: Optional[str | dict] = None,
|
||||
allowed_domains: Optional[set[str]] = None) -> Optional[Response]:
|
||||
"""
|
||||
处理图片缓存逻辑,支持HTTP缓存和磁盘缓存
|
||||
"""
|
||||
if not url:
|
||||
raise HTTPException(status_code=404, detail="URL not provided")
|
||||
return None
|
||||
|
||||
if allowed_domains is None:
|
||||
allowed_domains = set(settings.SECURITY_IMAGE_DOMAINS)
|
||||
|
||||
# 验证URL安全性
|
||||
if not SecurityUtils.is_safe_url(url, allowed_domains):
|
||||
raise HTTPException(status_code=404, detail="Unsafe URL")
|
||||
|
||||
# 后续观察系统性能表现,如果发现磁盘缓存和HTTP缓存无法满足高并发情况下的响应速度需求,可以考虑重新引入内存缓存
|
||||
cache_path: Optional[AsyncPath] = None
|
||||
if use_disk_cache:
|
||||
# 生成缓存路径
|
||||
base_path = AsyncPath(settings.CACHE_PATH)
|
||||
sanitized_path = SecurityUtils.sanitize_url_path(url)
|
||||
cache_path = base_path / "images" / sanitized_path
|
||||
logger.warn(f"Blocked unsafe image URL: {url}")
|
||||
return None
|
||||
|
||||
# 缓存路径
|
||||
sanitized_path = SecurityUtils.sanitize_url_path(url)
|
||||
cache_path = Path("images") / sanitized_path
|
||||
if not cache_path.suffix:
|
||||
# 没有文件类型,则添加后缀,在恶意文件类型和实际需求下的折衷选择
|
||||
if not cache_path.suffix:
|
||||
cache_path = cache_path.with_suffix(".jpg")
|
||||
cache_path = cache_path.with_suffix(".jpg")
|
||||
|
||||
# 确保缓存路径和文件类型合法
|
||||
if not await SecurityUtils.async_is_safe_path(base_path=base_path,
|
||||
user_path=cache_path,
|
||||
allowed_suffixes=settings.SECURITY_IMAGE_SUFFIXES):
|
||||
raise HTTPException(status_code=400, detail="Invalid cache path or file type")
|
||||
# 缓存对像,缓存过期时间为全局图片缓存天数
|
||||
cache_backend = AsyncFileCache(base=settings.CACHE_PATH,
|
||||
ttl=settings.GLOBAL_IMAGE_CACHE_DAYS * 24 * 3600)
|
||||
|
||||
# 目前暂不考虑磁盘缓存文件是否过期,后续通过缓存清理机制处理
|
||||
if cache_path and await cache_path.exists():
|
||||
try:
|
||||
async with cache_path.open('rb') as f:
|
||||
content = await f.read()
|
||||
etag = HashUtils.md5(content)
|
||||
headers = RequestUtils.generate_cache_headers(etag, max_age=86400 * 7)
|
||||
if if_none_match == etag:
|
||||
return Response(status_code=304, headers=headers)
|
||||
return Response(content=content, media_type="image/jpeg", headers=headers)
|
||||
except Exception as e:
|
||||
# 如果读取磁盘缓存发生异常,这里仅记录日志,尝试再次请求远端进行处理
|
||||
logger.debug(f"Failed to read cache file {cache_path}: {e}")
|
||||
if use_cache:
|
||||
content = await cache_backend.get(cache_path.as_posix(), region="images")
|
||||
if content:
|
||||
# 检查 If-None-Match
|
||||
etag = HashUtils.md5(content)
|
||||
headers = RequestUtils.generate_cache_headers(etag, max_age=86400 * 7)
|
||||
if if_none_match == etag:
|
||||
return Response(status_code=304, headers=headers)
|
||||
# 返回缓存图片
|
||||
return Response(
|
||||
content=content,
|
||||
media_type=UrlUtils.get_mime_type(url, "image/jpeg"),
|
||||
headers=headers
|
||||
)
|
||||
|
||||
# 请求远程图片
|
||||
referer = "https://movie.douban.com/" if "doubanio.com" in url else None
|
||||
proxies = settings.PROXY if proxy else None
|
||||
response = await AsyncRequestUtils(ua=settings.NORMAL_USER_AGENT, proxies=proxies, referer=referer,
|
||||
accept_type="image/avif,image/webp,image/apng,*/*").get_res(url=url)
|
||||
response = await AsyncRequestUtils(
|
||||
ua=settings.NORMAL_USER_AGENT,
|
||||
proxies=proxies,
|
||||
referer=referer,
|
||||
cookies=cookies,
|
||||
accept_type="image/avif,image/webp,image/apng,*/*",
|
||||
).get_res(url=url)
|
||||
if not response:
|
||||
raise HTTPException(status_code=502, detail="Failed to fetch the image from the remote server")
|
||||
logger.warn(f"Failed to fetch image from URL: {url}")
|
||||
return None
|
||||
|
||||
# 验证下载的内容是否为有效图片
|
||||
try:
|
||||
content = response.content
|
||||
Image.open(io.BytesIO(content)).verify()
|
||||
except Exception as e:
|
||||
logger.debug(f"Invalid image format for URL {url}: {e}")
|
||||
raise HTTPException(status_code=502, detail="Invalid image format")
|
||||
logger.warn(f"Invalid image format for URL {url}: {e}")
|
||||
return None
|
||||
|
||||
# 获取请求响应头
|
||||
response_headers = response.headers
|
||||
|
||||
cache_control_header = response_headers.get("Cache-Control", "")
|
||||
cache_directive, max_age = RequestUtils.parse_cache_control(cache_control_header)
|
||||
|
||||
# 如果需要使用磁盘缓存,则保存到磁盘
|
||||
if use_disk_cache and cache_path:
|
||||
try:
|
||||
if not await cache_path.parent.exists():
|
||||
await cache_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
async with aiofiles.tempfile.NamedTemporaryFile(dir=cache_path.parent, delete=False) as tmp_file:
|
||||
await tmp_file.write(content)
|
||||
temp_path = AsyncPath(tmp_file.name)
|
||||
await temp_path.replace(cache_path)
|
||||
except Exception as e:
|
||||
logger.debug(f"Failed to write cache file {cache_path}: {e}")
|
||||
# 保存缓存
|
||||
if use_cache:
|
||||
await cache_backend.set(cache_path.as_posix(), content, region="images")
|
||||
logger.debug(f"Image cached at {cache_path.as_posix()}")
|
||||
|
||||
# 检查 If-None-Match
|
||||
etag = HashUtils.md5(content)
|
||||
@@ -134,8 +133,8 @@ async def fetch_image(
|
||||
headers = RequestUtils.generate_cache_headers(etag, cache_directive, max_age)
|
||||
return Response(status_code=304, headers=headers)
|
||||
|
||||
# 响应
|
||||
headers = RequestUtils.generate_cache_headers(etag, cache_directive, max_age)
|
||||
|
||||
return Response(
|
||||
content=content,
|
||||
media_type=response_headers.get("Content-Type") or UrlUtils.get_mime_type(url, "image/jpeg"),
|
||||
@@ -148,6 +147,7 @@ async def proxy_img(
|
||||
imgurl: str,
|
||||
proxy: bool = False,
|
||||
cache: bool = False,
|
||||
use_cookies: bool = False,
|
||||
if_none_match: Annotated[str | None, Header()] = None,
|
||||
_: schemas.TokenPayload = Depends(verify_resource_token)
|
||||
) -> Response:
|
||||
@@ -158,7 +158,12 @@ async def proxy_img(
|
||||
hosts = [config.config.get("host") for config in MediaServerHelper().get_configs().values() if
|
||||
config and config.config and config.config.get("host")]
|
||||
allowed_domains = set(settings.SECURITY_IMAGE_DOMAINS) | set(hosts)
|
||||
return await fetch_image(url=imgurl, proxy=proxy, use_disk_cache=cache,
|
||||
cookies = (
|
||||
MediaServerChain().get_image_cookies(server=None, image_url=imgurl)
|
||||
if use_cookies
|
||||
else None
|
||||
)
|
||||
return await fetch_image(url=imgurl, proxy=proxy, use_cache=cache, cookies=cookies,
|
||||
if_none_match=if_none_match, allowed_domains=allowed_domains)
|
||||
|
||||
|
||||
@@ -173,7 +178,7 @@ async def cache_img(
|
||||
"""
|
||||
# 如果没有启用全局图片缓存,则不使用磁盘缓存
|
||||
proxy = "doubanio.com" not in url
|
||||
return await fetch_image(url=url, proxy=proxy, use_disk_cache=settings.GLOBAL_IMAGE_CACHE,
|
||||
return await fetch_image(url=url, proxy=proxy, use_cache=settings.GLOBAL_IMAGE_CACHE,
|
||||
if_none_match=if_none_match)
|
||||
|
||||
|
||||
@@ -186,7 +191,7 @@ def get_global_setting(token: str):
|
||||
raise HTTPException(status_code=403, detail="Forbidden")
|
||||
|
||||
# FIXME: 新增敏感配置项时要在此处添加排除项
|
||||
info = settings.dict(
|
||||
info = settings.model_dump(
|
||||
exclude={"SECRET_KEY", "RESOURCE_SECRET_KEY", "API_TOKEN", "TMDB_API_KEY", "TVDB_API_KEY", "FANART_API_KEY",
|
||||
"COOKIECLOUD_KEY", "COOKIECLOUD_PASSWORD", "GITHUB_TOKEN", "REPO_GITHUB_TOKEN", "U115_APP_ID",
|
||||
"ALIPAN_APP_ID", "TVDB_V4_API_KEY", "TVDB_V4_API_PIN"}
|
||||
@@ -203,11 +208,11 @@ def get_global_setting(token: str):
|
||||
|
||||
|
||||
@router.get("/env", summary="查询系统配置", response_model=schemas.Response)
|
||||
async def get_env_setting(_: User = Depends(get_current_active_superuser_async)):
|
||||
async def get_env_setting(_: User = Depends(get_current_active_user_async)):
|
||||
"""
|
||||
查询系统环境变量,包括当前版本号(仅管理员)
|
||||
"""
|
||||
info = settings.dict(
|
||||
info = settings.model_dump(
|
||||
exclude={"SECRET_KEY", "RESOURCE_SECRET_KEY"}
|
||||
)
|
||||
info.update({
|
||||
@@ -264,14 +269,14 @@ async def get_progress(request: Request, process_type: str, _: schemas.TokenPayl
|
||||
"""
|
||||
实时获取处理进度,返回格式为SSE
|
||||
"""
|
||||
progress = ProgressHelper()
|
||||
progress = ProgressHelper(process_type)
|
||||
|
||||
async def event_generator():
|
||||
try:
|
||||
while not global_vars.is_system_stopped:
|
||||
if await request.is_disconnected():
|
||||
break
|
||||
detail = progress.get(process_type)
|
||||
detail = progress.get()
|
||||
yield f"data: {json.dumps(detail)}\n\n"
|
||||
await asyncio.sleep(0.5)
|
||||
except asyncio.CancelledError:
|
||||
@@ -282,7 +287,7 @@ async def get_progress(request: Request, process_type: str, _: schemas.TokenPayl
|
||||
|
||||
@router.get("/setting/{key}", summary="查询系统设置", response_model=schemas.Response)
|
||||
async def get_setting(key: str,
|
||||
_: User = Depends(get_current_active_superuser_async)):
|
||||
_: User = Depends(get_current_active_user_async)):
|
||||
"""
|
||||
查询系统设置(仅管理员)
|
||||
"""
|
||||
@@ -381,7 +386,7 @@ async def get_logging(request: Request, length: Optional[int] = 50, logfile: Opt
|
||||
file_size = file_stat.st_size
|
||||
|
||||
# 读取历史日志
|
||||
async with log_path.open(mode="r", encoding="utf-8", errors="ignore") as f:
|
||||
async with aiofiles.open(log_path, mode="r", encoding="utf-8", errors="ignore") as f:
|
||||
# 优化大文件读取策略
|
||||
if file_size > 100 * 1024:
|
||||
# 只读取最后100KB的内容
|
||||
@@ -408,7 +413,7 @@ async def get_logging(request: Request, length: Optional[int] = 50, logfile: Opt
|
||||
yield f"data: {line}\n\n"
|
||||
|
||||
# 实时监听新日志
|
||||
async with log_path.open(mode="r", encoding="utf-8", errors="ignore") as f:
|
||||
async with aiofiles.open(log_path, mode="r", encoding="utf-8", errors="ignore") as f:
|
||||
# 移动文件指针到文件末尾,继续监听新增内容
|
||||
await f.seek(0, 2)
|
||||
# 记录初始文件大小
|
||||
@@ -445,7 +450,7 @@ async def get_logging(request: Request, length: Optional[int] = 50, logfile: Opt
|
||||
return Response(content="日志文件不存在!", media_type="text/plain")
|
||||
try:
|
||||
# 使用 aiofiles 异步读取文件
|
||||
async with log_path.open(mode="r", encoding="utf-8", errors="ignore") as file:
|
||||
async with aiofiles.open(log_path, mode="r", encoding="utf-8", errors="ignore") as file:
|
||||
text = await file.read()
|
||||
# 倒序输出
|
||||
text = "\n".join(text.split("\n")[::-1])
|
||||
|
||||
@@ -135,8 +135,8 @@ def refresh_cache(_: User = Depends(get_current_active_superuser)):
|
||||
|
||||
@router.post("/cache/reidentify/{domain}/{torrent_hash}", summary="重新识别种子", response_model=schemas.Response)
|
||||
async def reidentify_cache(domain: str, torrent_hash: str,
|
||||
tmdbid: Optional[int] = None, doubanid: Optional[str] = None,
|
||||
_: User = Depends(get_current_active_superuser_async)):
|
||||
tmdbid: Optional[int] = None, doubanid: Optional[str] = None,
|
||||
_: User = Depends(get_current_active_superuser_async)):
|
||||
"""
|
||||
重新识别指定的种子
|
||||
:param domain: 站点域名
|
||||
|
||||
@@ -8,7 +8,7 @@ from app import schemas
|
||||
from app.chain.media import MediaChain
|
||||
from app.chain.storage import StorageChain
|
||||
from app.chain.transfer import TransferChain
|
||||
from app.core.config import settings
|
||||
from app.core.config import settings, global_vars
|
||||
from app.core.metainfo import MetaInfoPath
|
||||
from app.core.security import verify_token, verify_apitoken
|
||||
from app.db import get_db
|
||||
@@ -75,6 +75,8 @@ async def remove_queue(fileitem: schemas.FileItem, _: schemas.TokenPayload = Dep
|
||||
:param _: Token校验
|
||||
"""
|
||||
TransferChain().remove_from_queue(fileitem)
|
||||
# 取消整理
|
||||
global_vars.stop_transfer(fileitem.path)
|
||||
return schemas.Response(success=True)
|
||||
|
||||
|
||||
@@ -109,7 +111,7 @@ def manual_transfer(transer_item: ManualTransferItem,
|
||||
if history.dest_fileitem:
|
||||
# 删除旧的已整理文件
|
||||
dest_fileitem = FileItem(**history.dest_fileitem)
|
||||
state = StorageChain().delete_media_file(dest_fileitem, mtype=MediaType(history.type))
|
||||
state = StorageChain().delete_media_file(dest_fileitem)
|
||||
if not state:
|
||||
return schemas.Response(success=False, message=f"{dest_fileitem.path} 删除失败")
|
||||
|
||||
|
||||
@@ -41,7 +41,7 @@ async def create_user(
|
||||
user = await current_user.async_get_by_name(db, name=user_in.name)
|
||||
if user:
|
||||
return schemas.Response(success=False, message="用户已存在")
|
||||
user_info = user_in.dict()
|
||||
user_info = user_in.model_dump()
|
||||
if user_info.get("password"):
|
||||
user_info["hashed_password"] = get_password_hash(user_info["password"])
|
||||
user_info.pop("password")
|
||||
@@ -59,7 +59,7 @@ async def update_user(
|
||||
"""
|
||||
更新用户
|
||||
"""
|
||||
user_info = user_in.dict()
|
||||
user_info = user_in.model_dump()
|
||||
if user_info.get("password"):
|
||||
# 正则表达式匹配密码包含字母、数字、特殊字符中的至少两项
|
||||
pattern = r'^(?![a-zA-Z]+$)(?!\d+$)(?![^\da-zA-Z\s]+$).{6,50}$'
|
||||
|
||||
@@ -11,7 +11,7 @@ from app.chain.workflow import WorkflowChain
|
||||
from app.core.config import global_vars
|
||||
from app.core.plugin import PluginManager
|
||||
from app.core.security import verify_token
|
||||
from app.core.workflow import WorkFlowManager
|
||||
from app.workflow import WorkFlowManager
|
||||
from app.db import get_async_db, get_db
|
||||
from app.db.models import Workflow
|
||||
from app.db.systemconfig_oper import SystemConfigOper
|
||||
@@ -47,7 +47,7 @@ async def create_workflow(workflow: schemas.Workflow,
|
||||
workflow.state = "P"
|
||||
if not workflow.trigger_type:
|
||||
workflow.trigger_type = "timer"
|
||||
workflow_obj = Workflow(**workflow.dict())
|
||||
workflow_obj = Workflow(**workflow.model_dump())
|
||||
await workflow_obj.async_create(db)
|
||||
return schemas.Response(success=True, message="创建工作流成功")
|
||||
|
||||
@@ -277,7 +277,7 @@ def update_workflow(workflow: schemas.Workflow,
|
||||
return schemas.Response(success=False, message="工作流不存在")
|
||||
if not wf.trigger_type:
|
||||
workflow.trigger_type = "timer"
|
||||
wf.update(db, workflow.dict())
|
||||
wf.update(db, workflow.model_dump())
|
||||
# 更新后的工作流对象
|
||||
updated_workflow = workflow_oper.get(workflow.id)
|
||||
# 更新定时任务
|
||||
|
||||
@@ -2,7 +2,8 @@ import gzip
|
||||
import json
|
||||
from typing import Annotated, Callable, Any, Dict, Optional
|
||||
|
||||
from aiopath import AsyncPath
|
||||
import aiofiles
|
||||
from anyio import Path as AsyncPath
|
||||
from fastapi import APIRouter, Depends, HTTPException, Path, Request, Response
|
||||
from fastapi.responses import PlainTextResponse
|
||||
from fastapi.routing import APIRoute
|
||||
@@ -67,9 +68,9 @@ async def update_cookie(req: schemas.CookieData):
|
||||
"""
|
||||
file_path = AsyncPath(settings.COOKIE_PATH) / f"{req.uuid}.json"
|
||||
content = json.dumps({"encrypted": req.encrypted})
|
||||
async with file_path.open(encoding="utf-8", mode="w") as file:
|
||||
async with aiofiles.open(file_path, encoding="utf-8", mode="w") as file:
|
||||
await file.write(content)
|
||||
async with file_path.open(encoding="utf-8", mode="r") as file:
|
||||
async with aiofiles.open(file_path, encoding="utf-8", mode="r") as file:
|
||||
read_content = await file.read()
|
||||
if read_content == content:
|
||||
return {"action": "done"}
|
||||
@@ -88,7 +89,7 @@ async def load_encrypt_data(uuid: str) -> Dict[str, Any]:
|
||||
raise HTTPException(status_code=404, detail="Item not found")
|
||||
|
||||
# 读取文件
|
||||
async with file_path.open(encoding="utf-8", mode="r") as file:
|
||||
async with aiofiles.open(file_path, encoding="utf-8", mode="r") as file:
|
||||
read_content = await file.read()
|
||||
data = json.loads(read_content.encode("utf-8"))
|
||||
return data
|
||||
|
||||
@@ -8,12 +8,10 @@ from pathlib import Path
|
||||
from typing import Optional, Any, Tuple, List, Set, Union, Dict
|
||||
|
||||
from fastapi.concurrency import run_in_threadpool
|
||||
|
||||
import aiofiles
|
||||
from aiopath import AsyncPath
|
||||
from qbittorrentapi import TorrentFilesList
|
||||
from transmission_rpc import File
|
||||
|
||||
from app.core.cache import FileCache, AsyncFileCache, fresh, async_fresh
|
||||
from app.core.config import settings
|
||||
from app.core.context import Context, MediaInfo, TorrentInfo
|
||||
from app.core.event import EventManager
|
||||
@@ -48,78 +46,66 @@ class ChainBase(metaclass=ABCMeta):
|
||||
send_callback=self.run_module
|
||||
)
|
||||
self.pluginmanager = PluginManager()
|
||||
self.filecache = FileCache()
|
||||
self.async_filecache = AsyncFileCache()
|
||||
|
||||
@staticmethod
|
||||
def load_cache(filename: str) -> Any:
|
||||
def load_cache(self, filename: str) -> Any:
|
||||
"""
|
||||
从本地加载缓存
|
||||
加载缓存
|
||||
"""
|
||||
cache_path = settings.TEMP_PATH / filename
|
||||
if cache_path.exists():
|
||||
try:
|
||||
with open(cache_path, 'rb') as f:
|
||||
return pickle.load(f)
|
||||
except Exception as err:
|
||||
logger.error(f"加载缓存 {filename} 出错:{str(err)}")
|
||||
return None
|
||||
content = self.filecache.get(filename)
|
||||
if not content:
|
||||
return None
|
||||
try:
|
||||
return pickle.loads(content)
|
||||
except Exception as err:
|
||||
logger.error(f"加载缓存 {filename} 出错:{str(err)}")
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
async def async_load_cache(filename: str) -> Any:
|
||||
async def async_load_cache(self, filename: str) -> Any:
|
||||
"""
|
||||
异步从本地加载缓存
|
||||
异步加载缓存
|
||||
"""
|
||||
cache_path = settings.TEMP_PATH / filename
|
||||
if cache_path.exists():
|
||||
try:
|
||||
async with aiofiles.open(cache_path, 'rb') as f:
|
||||
content = await f.read()
|
||||
return pickle.loads(content)
|
||||
except Exception as err:
|
||||
logger.error(f"加载缓存 {filename} 出错:{str(err)}")
|
||||
return None
|
||||
content = await self.async_filecache.get(filename)
|
||||
if not content:
|
||||
return None
|
||||
try:
|
||||
return pickle.loads(content)
|
||||
except Exception as err:
|
||||
logger.error(f"异步加载缓存 {filename} 出错:{str(err)}")
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
async def async_save_cache(cache: Any, filename: str) -> None:
|
||||
async def async_save_cache(self, cache: Any, filename: str) -> None:
|
||||
"""
|
||||
异步保存缓存到本地
|
||||
异步保存缓存
|
||||
"""
|
||||
try:
|
||||
async with aiofiles.open(settings.TEMP_PATH / filename, 'wb') as f:
|
||||
await f.write(pickle.dumps(cache))
|
||||
await self.async_filecache.set(filename, pickle.dumps(cache))
|
||||
except Exception as err:
|
||||
logger.error(f"保存缓存 {filename} 出错:{str(err)}")
|
||||
logger.error(f"异步保存缓存 {filename} 出错:{str(err)}")
|
||||
return
|
||||
|
||||
@staticmethod
|
||||
def save_cache(cache: Any, filename: str) -> None:
|
||||
def save_cache(self, cache: Any, filename: str) -> None:
|
||||
"""
|
||||
保存缓存到本地
|
||||
保存缓存
|
||||
"""
|
||||
try:
|
||||
with open(settings.TEMP_PATH / filename, 'wb') as f:
|
||||
pickle.dump(cache, f) # noqa
|
||||
self.filecache.set(filename, pickle.dumps(cache))
|
||||
except Exception as err:
|
||||
logger.error(f"保存缓存 {filename} 出错:{str(err)}")
|
||||
return
|
||||
|
||||
@staticmethod
|
||||
def remove_cache(filename: str) -> None:
|
||||
def remove_cache(self, filename: str) -> None:
|
||||
"""
|
||||
删除本地缓存
|
||||
删除缓存,同时删除Redis和本地缓存
|
||||
"""
|
||||
cache_path = settings.TEMP_PATH / filename
|
||||
if cache_path.exists():
|
||||
cache_path.unlink()
|
||||
self.filecache.delete(filename)
|
||||
|
||||
@staticmethod
|
||||
async def async_remove_cache(filename: str) -> None:
|
||||
async def async_remove_cache(self, filename: str) -> None:
|
||||
"""
|
||||
异步删除本地缓存
|
||||
异步删除缓存,同时删除Redis和本地缓存
|
||||
"""
|
||||
cache_path = AsyncPath(settings.TEMP_PATH) / filename
|
||||
if await cache_path.exists():
|
||||
try:
|
||||
await cache_path.unlink()
|
||||
except Exception as err:
|
||||
logger.error(f"异步删除缓存 {filename} 出错:{str(err)}")
|
||||
await self.async_filecache.delete(filename)
|
||||
|
||||
@staticmethod
|
||||
def __is_valid_empty(ret):
|
||||
@@ -372,9 +358,10 @@ class ChainBase(metaclass=ABCMeta):
|
||||
if tmdbid:
|
||||
doubanid = None
|
||||
bangumiid = None
|
||||
return self.run_module("recognize_media", meta=meta, mtype=mtype,
|
||||
tmdbid=tmdbid, doubanid=doubanid, bangumiid=bangumiid,
|
||||
episode_group=episode_group, cache=cache)
|
||||
with fresh(not cache):
|
||||
return self.run_module("recognize_media", meta=meta, mtype=mtype,
|
||||
tmdbid=tmdbid, doubanid=doubanid, bangumiid=bangumiid,
|
||||
episode_group=episode_group, cache=cache)
|
||||
|
||||
async def async_recognize_media(self, meta: MetaBase = None,
|
||||
mtype: Optional[MediaType] = None,
|
||||
@@ -405,9 +392,10 @@ class ChainBase(metaclass=ABCMeta):
|
||||
if tmdbid:
|
||||
doubanid = None
|
||||
bangumiid = None
|
||||
return await self.async_run_module("async_recognize_media", meta=meta, mtype=mtype,
|
||||
tmdbid=tmdbid, doubanid=doubanid, bangumiid=bangumiid,
|
||||
episode_group=episode_group, cache=cache)
|
||||
async with async_fresh(not cache):
|
||||
return await self.async_run_module("async_recognize_media", meta=meta, mtype=mtype,
|
||||
tmdbid=tmdbid, doubanid=doubanid, bangumiid=bangumiid,
|
||||
episode_group=episode_group, cache=cache)
|
||||
|
||||
def match_doubaninfo(self, name: str, imdbid: Optional[str] = None,
|
||||
mtype: Optional[MediaType] = None, year: Optional[str] = None, season: Optional[int] = None,
|
||||
@@ -700,13 +688,13 @@ class ChainBase(metaclass=ABCMeta):
|
||||
return self.run_module("filter_torrents", rule_groups=rule_groups,
|
||||
torrent_list=torrent_list, mediainfo=mediainfo)
|
||||
|
||||
def download(self, content: Union[Path, str], download_dir: Path, cookie: str,
|
||||
def download(self, content: Union[Path, str, bytes], download_dir: Path, cookie: str,
|
||||
episodes: Set[int] = None, category: Optional[str] = None, label: Optional[str] = None,
|
||||
downloader: Optional[str] = None
|
||||
) -> Optional[Tuple[Optional[str], Optional[str], Optional[str], str]]:
|
||||
"""
|
||||
根据种子文件,选择并添加下载任务
|
||||
:param content: 种子文件地址或者磁力链接
|
||||
:param content: 种子文件地址或者磁力链接或者种子内容
|
||||
:param download_dir: 下载目录
|
||||
:param cookie: cookie
|
||||
:param episodes: 需要下载的集数
|
||||
@@ -719,15 +707,16 @@ class ChainBase(metaclass=ABCMeta):
|
||||
cookie=cookie, episodes=episodes, category=category, label=label,
|
||||
downloader=downloader)
|
||||
|
||||
def download_added(self, context: Context, download_dir: Path, torrent_path: Path = None) -> None:
|
||||
def download_added(self, context: Context, download_dir: Path, torrent_content: Union[str, bytes] = None) -> None:
|
||||
"""
|
||||
添加下载任务成功后,从站点下载字幕,保存到下载目录
|
||||
:param context: 上下文,包括识别信息、媒体信息、种子信息
|
||||
:param download_dir: 下载目录
|
||||
:param torrent_path: 种子文件地址
|
||||
:param torrent_content: 种子内容,如果有则直接使用该内容,否则从context中获取种子文件路径
|
||||
:return: None,该方法可被多个模块同时处理
|
||||
"""
|
||||
return self.run_module("download_added", context=context, torrent_path=torrent_path,
|
||||
return self.run_module("download_added", context=context,
|
||||
torrent_content=torrent_content,
|
||||
download_dir=download_dir)
|
||||
|
||||
def list_torrents(self, status: TorrentStatus = None,
|
||||
@@ -863,9 +852,13 @@ class ChainBase(metaclass=ABCMeta):
|
||||
# 渲染消息
|
||||
message = MessageTemplateHelper.render(message=message, meta=meta, mediainfo=mediainfo,
|
||||
torrentinfo=torrentinfo, transferinfo=transferinfo, **kwargs)
|
||||
# 检查消息是否有效
|
||||
if not message:
|
||||
logger.warning("消息为空,跳过发送")
|
||||
return
|
||||
# 保存消息
|
||||
self.messagehelper.put(message, role="user", title=message.title)
|
||||
self.messageoper.add(**message.dict())
|
||||
self.messageoper.add(**message.model_dump())
|
||||
# 发送消息按设置隔离
|
||||
if not message.userid and message.mtype:
|
||||
# 消息隔离设置
|
||||
@@ -912,23 +905,23 @@ class ChainBase(metaclass=ABCMeta):
|
||||
break
|
||||
# 按设定发送
|
||||
self.eventmanager.send_event(etype=EventType.NoticeMessage,
|
||||
data={**send_message.dict(), "type": send_message.mtype})
|
||||
self.messagequeue.send_message("post_message", message=send_message)
|
||||
data={**send_message.model_dump(), "type": send_message.mtype})
|
||||
self.messagequeue.send_message("post_message", message=send_message, **kwargs)
|
||||
if not send_orignal:
|
||||
return
|
||||
# 发送消息事件
|
||||
self.eventmanager.send_event(etype=EventType.NoticeMessage, data={**message.dict(), "type": message.mtype})
|
||||
self.eventmanager.send_event(etype=EventType.NoticeMessage, data={**message.model_dump(), "type": message.mtype})
|
||||
# 按原消息发送
|
||||
self.messagequeue.send_message("post_message", message=message,
|
||||
immediately=True if message.userid else False)
|
||||
immediately=True if message.userid else False, **kwargs)
|
||||
|
||||
async def async_post_message(self,
|
||||
message: Optional[Notification] = None,
|
||||
meta: Optional[MetaBase] = None,
|
||||
mediainfo: Optional[MediaInfo] = None,
|
||||
torrentinfo: Optional[TorrentInfo] = None,
|
||||
transferinfo: Optional[TransferInfo] = None,
|
||||
**kwargs) -> None:
|
||||
message: Optional[Notification] = None,
|
||||
meta: Optional[MetaBase] = None,
|
||||
mediainfo: Optional[MediaInfo] = None,
|
||||
torrentinfo: Optional[TorrentInfo] = None,
|
||||
transferinfo: Optional[TransferInfo] = None,
|
||||
**kwargs) -> None:
|
||||
"""
|
||||
异步发送消息
|
||||
:param message: Notification实例
|
||||
@@ -942,9 +935,13 @@ class ChainBase(metaclass=ABCMeta):
|
||||
# 渲染消息
|
||||
message = MessageTemplateHelper.render(message=message, meta=meta, mediainfo=mediainfo,
|
||||
torrentinfo=torrentinfo, transferinfo=transferinfo, **kwargs)
|
||||
# 检查消息是否有效
|
||||
if not message:
|
||||
logger.warning("消息为空,跳过发送")
|
||||
return
|
||||
# 保存消息
|
||||
self.messagehelper.put(message, role="user", title=message.title)
|
||||
await self.messageoper.async_add(**message.dict())
|
||||
await self.messageoper.async_add(**message.model_dump())
|
||||
# 发送消息按设置隔离
|
||||
if not message.userid and message.mtype:
|
||||
# 消息隔离设置
|
||||
@@ -991,15 +988,16 @@ class ChainBase(metaclass=ABCMeta):
|
||||
break
|
||||
# 按设定发送
|
||||
await self.eventmanager.async_send_event(etype=EventType.NoticeMessage,
|
||||
data={**send_message.dict(), "type": send_message.mtype})
|
||||
await self.messagequeue.async_send_message("post_message", message=send_message)
|
||||
data={**send_message.model_dump(), "type": send_message.mtype})
|
||||
await self.messagequeue.async_send_message("post_message", message=send_message, **kwargs)
|
||||
if not send_orignal:
|
||||
return
|
||||
# 发送消息事件
|
||||
await self.eventmanager.async_send_event(etype=EventType.NoticeMessage, data={**message.dict(), "type": message.mtype})
|
||||
await self.eventmanager.async_send_event(etype=EventType.NoticeMessage,
|
||||
data={**message.model_dump(), "type": message.mtype})
|
||||
# 按原消息发送
|
||||
await self.messagequeue.async_send_message("post_message", message=message,
|
||||
immediately=True if message.userid else False)
|
||||
immediately=True if message.userid else False, **kwargs)
|
||||
|
||||
def post_medias_message(self, message: Notification, medias: List[MediaInfo]) -> None:
|
||||
"""
|
||||
@@ -1010,7 +1008,7 @@ class ChainBase(metaclass=ABCMeta):
|
||||
"""
|
||||
note_list = [media.to_dict() for media in medias]
|
||||
self.messagehelper.put(message, role="user", note=note_list, title=message.title)
|
||||
self.messageoper.add(**message.dict(), note=note_list)
|
||||
self.messageoper.add(**message.model_dump(), note=note_list)
|
||||
return self.messagequeue.send_message("post_medias_message", message=message, medias=medias,
|
||||
immediately=True if message.userid else False)
|
||||
|
||||
@@ -1023,7 +1021,7 @@ class ChainBase(metaclass=ABCMeta):
|
||||
"""
|
||||
note_list = [torrent.torrent_info.to_dict() for torrent in torrents]
|
||||
self.messagehelper.put(message, role="user", note=note_list, title=message.title)
|
||||
self.messageoper.add(**message.dict(), note=note_list)
|
||||
self.messageoper.add(**message.model_dump(), note=note_list)
|
||||
return self.messagequeue.send_message("post_torrents_message", message=message, torrents=torrents,
|
||||
immediately=True if message.userid else False)
|
||||
|
||||
|
||||
@@ -8,6 +8,7 @@ from typing import List, Optional, Tuple, Set, Dict, Union
|
||||
|
||||
from app import schemas
|
||||
from app.chain import ChainBase
|
||||
from app.core.cache import FileCache
|
||||
from app.core.config import settings, global_vars
|
||||
from app.core.context import MediaInfo, TorrentInfo, Context
|
||||
from app.core.event import eventmanager, Event
|
||||
@@ -35,10 +36,10 @@ class DownloadChain(ChainBase):
|
||||
channel: MessageChannel = None,
|
||||
source: Optional[str] = None,
|
||||
userid: Union[str, int] = None
|
||||
) -> Tuple[Optional[Union[Path, str]], str, list]:
|
||||
) -> Tuple[Optional[Union[str, bytes]], str, list]:
|
||||
"""
|
||||
下载种子文件,如果是磁力链,会返回磁力链接本身
|
||||
:return: 种子路径,种子目录名,种子文件清单
|
||||
:return: 种子内容,种子目录名,种子文件清单
|
||||
"""
|
||||
|
||||
def __get_redict_url(url: str, ua: Optional[str] = None, cookie: Optional[str] = None) -> Optional[str]:
|
||||
@@ -117,7 +118,7 @@ class DownloadChain(ChainBase):
|
||||
logger.error(f"{torrent.title} 无法获取下载地址:{torrent.enclosure}!")
|
||||
return None, "", []
|
||||
# 下载种子文件
|
||||
torrent_file, content, download_folder, files, error_msg = TorrentHelper().download_torrent(
|
||||
_, content, download_folder, files, error_msg = TorrentHelper().download_torrent(
|
||||
url=torrent_url,
|
||||
cookie=site_cookie,
|
||||
ua=torrent.site_ua or settings.USER_AGENT,
|
||||
@@ -127,7 +128,7 @@ class DownloadChain(ChainBase):
|
||||
# 磁力链
|
||||
return content, "", []
|
||||
|
||||
if not torrent_file:
|
||||
if not content:
|
||||
logger.error(f"下载种子文件失败:{torrent.title} - {torrent_url}")
|
||||
self.post_message(Notification(
|
||||
channel=channel,
|
||||
@@ -139,9 +140,11 @@ class DownloadChain(ChainBase):
|
||||
return None, "", []
|
||||
|
||||
# 返回 种子文件路径,种子目录名,种子文件清单
|
||||
return torrent_file, download_folder, files
|
||||
return content, download_folder, files
|
||||
|
||||
def download_single(self, context: Context, torrent_file: Path = None,
|
||||
def download_single(self, context: Context,
|
||||
torrent_file: Path = None,
|
||||
torrent_content: Optional[Union[str, bytes]] = None,
|
||||
episodes: Set[int] = None,
|
||||
channel: MessageChannel = None,
|
||||
source: Optional[str] = None,
|
||||
@@ -154,6 +157,7 @@ class DownloadChain(ChainBase):
|
||||
下载及发送通知
|
||||
:param context: 资源上下文
|
||||
:param torrent_file: 种子文件路径
|
||||
:param torrent_content: 种子内容(磁力链或种子文件内容)
|
||||
:param episodes: 需要下载的集数
|
||||
:param channel: 通知渠道
|
||||
:param source: 来源(消息通知、Subscribe、Manual等)
|
||||
@@ -207,18 +211,26 @@ class DownloadChain(ChainBase):
|
||||
# 实际下载的集数
|
||||
download_episodes = StringUtils.format_ep(list(episodes)) if episodes else None
|
||||
_folder_name = ""
|
||||
if not torrent_file:
|
||||
if not torrent_file and not torrent_content:
|
||||
# 下载种子文件,得到的可能是文件也可能是磁力链
|
||||
content, _folder_name, _file_list = self.download_torrent(_torrent,
|
||||
channel=channel,
|
||||
source=source,
|
||||
userid=userid)
|
||||
if not content:
|
||||
return None
|
||||
else:
|
||||
content = torrent_file
|
||||
# 获取种子文件的文件夹名和文件清单
|
||||
_folder_name, _file_list = TorrentHelper().get_torrent_info(torrent_file)
|
||||
torrent_content, _folder_name, _file_list = self.download_torrent(_torrent,
|
||||
channel=channel,
|
||||
source=source,
|
||||
userid=userid)
|
||||
elif torrent_file:
|
||||
if torrent_file.exists():
|
||||
torrent_content = torrent_file.read_bytes()
|
||||
else:
|
||||
# 缓存处理器
|
||||
cache_backend = FileCache()
|
||||
# 读取缓存的种子文件
|
||||
torrent_content = cache_backend.get(torrent_file.as_posix(), region="torrents")
|
||||
|
||||
if not torrent_content:
|
||||
return None
|
||||
|
||||
# 获取种子文件的文件夹名和文件清单
|
||||
_folder_name, _file_list = TorrentHelper().get_fileinfo_from_torrent_content(torrent_content)
|
||||
|
||||
# 下载目录
|
||||
if save_path:
|
||||
@@ -249,7 +261,7 @@ class DownloadChain(ChainBase):
|
||||
return None
|
||||
|
||||
# 添加下载
|
||||
result: Optional[tuple] = self.download(content=content,
|
||||
result: Optional[tuple] = self.download(content=torrent_content,
|
||||
cookie=_torrent.site_cookie,
|
||||
episodes=episodes,
|
||||
download_dir=download_dir,
|
||||
@@ -278,7 +290,7 @@ class DownloadChain(ChainBase):
|
||||
# 登记下载记录
|
||||
downloadhis = DownloadHistoryOper()
|
||||
downloadhis.add(
|
||||
path=str(download_path),
|
||||
path=download_path.as_posix(),
|
||||
type=_media.type.value,
|
||||
title=_media.title,
|
||||
year=_media.year,
|
||||
@@ -319,8 +331,8 @@ class DownloadChain(ChainBase):
|
||||
files_to_add.append({
|
||||
"download_hash": _hash,
|
||||
"downloader": _downloader,
|
||||
"fullpath": str(_save_path / file),
|
||||
"savepath": str(_save_path),
|
||||
"fullpath": (_save_path / file).as_posix(),
|
||||
"savepath": _save_path.as_posix(),
|
||||
"filepath": file,
|
||||
"torrentname": _meta.org_string,
|
||||
})
|
||||
@@ -346,7 +358,7 @@ class DownloadChain(ChainBase):
|
||||
username=username,
|
||||
)
|
||||
# 下载成功后处理
|
||||
self.download_added(context=context, download_dir=download_dir, torrent_path=torrent_file)
|
||||
self.download_added(context=context, download_dir=download_dir, torrent_content=torrent_content)
|
||||
# 广播事件
|
||||
self.eventmanager.send_event(EventType.DownloadAdded, {
|
||||
"hash": _hash,
|
||||
@@ -560,7 +572,7 @@ class DownloadChain(ChainBase):
|
||||
logger.info(f"开始下载 {torrent.title} ...")
|
||||
download_id = self.download_single(
|
||||
context=context,
|
||||
torrent_file=content if isinstance(content, Path) else None,
|
||||
torrent_content=content,
|
||||
save_path=save_path,
|
||||
channel=channel,
|
||||
source=source,
|
||||
@@ -727,7 +739,7 @@ class DownloadChain(ChainBase):
|
||||
logger.info(f"开始下载 {torrent.title} ...")
|
||||
download_id = self.download_single(
|
||||
context=context,
|
||||
torrent_file=content if isinstance(content, Path) else None,
|
||||
torrent_content=content,
|
||||
episodes=selected_episodes,
|
||||
save_path=save_path,
|
||||
channel=channel,
|
||||
@@ -982,7 +994,7 @@ class DownloadChain(ChainBase):
|
||||
# 发出下载任务删除事件,如需处理辅种,可监听该事件
|
||||
self.eventmanager.send_event(EventType.DownloadDeleted, {
|
||||
"hash": hash_str,
|
||||
"torrents": [torrent.dict() for torrent in torrents]
|
||||
"torrents": [torrent.model_dump() for torrent in torrents]
|
||||
})
|
||||
else:
|
||||
logger.info(f"没有在下载器中查询到 {hash_str} 对应的下载任务")
|
||||
|
||||
@@ -1,4 +1,6 @@
|
||||
import os
|
||||
from pathlib import Path
|
||||
from tempfile import NamedTemporaryFile
|
||||
from threading import Lock
|
||||
from typing import Optional, List, Tuple, Union
|
||||
|
||||
@@ -20,6 +22,9 @@ from app.utils.string import StringUtils
|
||||
recognize_lock = Lock()
|
||||
scraping_lock = Lock()
|
||||
|
||||
current_umask = os.umask(0)
|
||||
os.umask(current_umask)
|
||||
|
||||
|
||||
class MediaChain(ChainBase):
|
||||
"""
|
||||
@@ -310,6 +315,21 @@ class MediaChain(ChainBase):
|
||||
)
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def is_bluray_folder(fileitem: schemas.FileItem) -> bool:
|
||||
"""
|
||||
判断是否为原盘目录
|
||||
"""
|
||||
if not fileitem or fileitem.type != "dir":
|
||||
return False
|
||||
# 蓝光原盘目录必备的文件或文件夹
|
||||
required_files = ['BDMV', 'CERTIFICATE']
|
||||
# 检查目录下是否存在所需文件或文件夹
|
||||
for item in StorageChain().list_files(fileitem):
|
||||
if item.name in required_files:
|
||||
return True
|
||||
return False
|
||||
|
||||
@eventmanager.register(EventType.MetadataScrape)
|
||||
def scrape_metadata_event(self, event: Event):
|
||||
"""
|
||||
@@ -349,51 +369,60 @@ class MediaChain(ChainBase):
|
||||
overwrite=overwrite)
|
||||
else:
|
||||
if file_list:
|
||||
# 1. 收集fileitem和file_list中每个文件之间所有子目录
|
||||
all_dirs = set()
|
||||
root_path = Path(fileitem.path)
|
||||
# 如果是BDMV原盘目录,只对根目录进行刮削,不处理子目录
|
||||
if self.is_bluray_folder(fileitem):
|
||||
logger.info(f"检测到BDMV原盘目录,只对根目录进行刮削:{fileitem.path}")
|
||||
self.scrape_metadata(fileitem=fileitem,
|
||||
mediainfo=mediainfo,
|
||||
init_folder=True,
|
||||
recursive=False,
|
||||
overwrite=overwrite)
|
||||
else:
|
||||
# 1. 收集fileitem和file_list中每个文件之间所有子目录
|
||||
all_dirs = set()
|
||||
root_path = Path(fileitem.path)
|
||||
|
||||
logger.debug(f"开始收集目录,根目录:{root_path}")
|
||||
# 收集根目录
|
||||
all_dirs.add(root_path)
|
||||
logger.debug(f"开始收集目录,根目录:{root_path}")
|
||||
# 收集根目录
|
||||
all_dirs.add(root_path)
|
||||
|
||||
# 收集所有目录(包括所有层级)
|
||||
for sub_file in file_list:
|
||||
sub_path = Path(sub_file)
|
||||
# 收集从根目录到文件的所有父目录
|
||||
current_path = sub_path.parent
|
||||
while current_path != root_path and current_path.is_relative_to(root_path):
|
||||
all_dirs.add(current_path)
|
||||
current_path = current_path.parent
|
||||
# 收集所有目录(包括所有层级)
|
||||
for sub_file in file_list:
|
||||
sub_path = Path(sub_file)
|
||||
# 收集从根目录到文件的所有父目录
|
||||
current_path = sub_path.parent
|
||||
while current_path != root_path and current_path.is_relative_to(root_path):
|
||||
all_dirs.add(current_path)
|
||||
current_path = current_path.parent
|
||||
|
||||
logger.debug(f"共收集到 {len(all_dirs)} 个目录")
|
||||
logger.debug(f"共收集到 {len(all_dirs)} 个目录")
|
||||
|
||||
# 2. 初始化一遍子目录,但不处理文件
|
||||
for sub_dir in all_dirs:
|
||||
sub_dir_item = storagechain.get_file_item(storage=fileitem.storage, path=sub_dir)
|
||||
if sub_dir_item:
|
||||
logger.info(f"为目录生成海报和nfo:{sub_dir}")
|
||||
# 初始化目录元数据,但不处理文件
|
||||
self.scrape_metadata(fileitem=sub_dir_item,
|
||||
mediainfo=mediainfo,
|
||||
init_folder=True,
|
||||
recursive=False,
|
||||
overwrite=overwrite)
|
||||
else:
|
||||
logger.warn(f"无法获取目录项:{sub_dir}")
|
||||
# 2. 初始化一遍子目录,但不处理文件
|
||||
for sub_dir in all_dirs:
|
||||
sub_dir_item = storagechain.get_file_item(storage=fileitem.storage, path=sub_dir)
|
||||
if sub_dir_item:
|
||||
logger.info(f"为目录生成海报和nfo:{sub_dir}")
|
||||
# 初始化目录元数据,但不处理文件
|
||||
self.scrape_metadata(fileitem=sub_dir_item,
|
||||
mediainfo=mediainfo,
|
||||
init_folder=True,
|
||||
recursive=False,
|
||||
overwrite=overwrite)
|
||||
else:
|
||||
logger.warn(f"无法获取目录项:{sub_dir}")
|
||||
|
||||
# 3. 刮削每个文件
|
||||
logger.info(f"开始刮削 {len(file_list)} 个文件")
|
||||
for sub_file_path in file_list:
|
||||
sub_file_item = storagechain.get_file_item(storage=fileitem.storage,
|
||||
path=Path(sub_file_path))
|
||||
if sub_file_item:
|
||||
self.scrape_metadata(fileitem=sub_file_item,
|
||||
mediainfo=mediainfo,
|
||||
init_folder=False,
|
||||
overwrite=overwrite)
|
||||
else:
|
||||
logger.warn(f"无法获取文件项:{sub_file_path}")
|
||||
# 3. 刮削每个文件
|
||||
logger.info(f"开始刮削 {len(file_list)} 个文件")
|
||||
for sub_file_path in file_list:
|
||||
sub_file_item = storagechain.get_file_item(storage=fileitem.storage,
|
||||
path=Path(sub_file_path))
|
||||
if sub_file_item:
|
||||
self.scrape_metadata(fileitem=sub_file_item,
|
||||
mediainfo=mediainfo,
|
||||
init_folder=False,
|
||||
overwrite=overwrite)
|
||||
else:
|
||||
logger.warn(f"无法获取文件项:{sub_file_path}")
|
||||
else:
|
||||
# 执行全量刮削
|
||||
logger.info(f"开始刮削目录 {fileitem.path} ...")
|
||||
@@ -417,20 +446,6 @@ class MediaChain(ChainBase):
|
||||
|
||||
storagechain = StorageChain()
|
||||
|
||||
def is_bluray_folder(_fileitem: schemas.FileItem) -> bool:
|
||||
"""
|
||||
判断是否为原盘目录
|
||||
"""
|
||||
if not _fileitem or _fileitem.type != "dir":
|
||||
return False
|
||||
# 蓝光原盘目录必备的文件或文件夹
|
||||
required_files = ['BDMV', 'CERTIFICATE']
|
||||
# 检查目录下是否存在所需文件或文件夹
|
||||
for item in storagechain.list_files(_fileitem):
|
||||
if item.name in required_files:
|
||||
return True
|
||||
return False
|
||||
|
||||
def __list_files(_fileitem: schemas.FileItem):
|
||||
"""
|
||||
列出下级文件
|
||||
@@ -446,36 +461,65 @@ class MediaChain(ChainBase):
|
||||
"""
|
||||
if not _fileitem or not _content or not _path:
|
||||
return
|
||||
# 保存文件到临时目录
|
||||
tmp_dir = settings.TEMP_PATH / StringUtils.generate_random_str(10)
|
||||
tmp_dir.mkdir(parents=True, exist_ok=True)
|
||||
tmp_file = tmp_dir / _path.name
|
||||
tmp_file.write_bytes(_content)
|
||||
# 获取文件的父目录
|
||||
try:
|
||||
item = storagechain.upload_file(fileitem=_fileitem, path=tmp_file, new_name=_path.name)
|
||||
# 使用tempfile创建临时文件,自动删除
|
||||
with NamedTemporaryFile(delete=True, delete_on_close=False, suffix=_path.suffix) as tmp_file:
|
||||
tmp_file_path = Path(tmp_file.name)
|
||||
# 写入内容
|
||||
if isinstance(_content, bytes):
|
||||
tmp_file.write(_content)
|
||||
else:
|
||||
tmp_file.write(_content.encode('utf-8'))
|
||||
tmp_file.flush()
|
||||
tmp_file.close() # 关闭文件句柄
|
||||
|
||||
# 刮削文件只需要读写权限
|
||||
tmp_file_path.chmod(0o666 & ~current_umask)
|
||||
|
||||
# 上传文件
|
||||
item = storagechain.upload_file(fileitem=_fileitem, path=tmp_file_path, new_name=_path.name)
|
||||
if item:
|
||||
logger.info(f"已保存文件:{item.path}")
|
||||
else:
|
||||
logger.warn(f"文件保存失败:{_path}")
|
||||
finally:
|
||||
if tmp_file.exists():
|
||||
tmp_file.unlink()
|
||||
|
||||
def __download_image(_url: str) -> Optional[bytes]:
|
||||
def __download_and_save_image(_fileitem: schemas.FileItem, _path: Path, _url: str):
|
||||
"""
|
||||
下载图片并保存
|
||||
流式下载图片并直接保存到文件(减少内存占用)
|
||||
:param _fileitem: 关联的媒体文件项
|
||||
:param _path: 图片文件路径
|
||||
:param _url: 图片下载URL
|
||||
"""
|
||||
if not _fileitem or not _url or not _path:
|
||||
return
|
||||
try:
|
||||
logger.info(f"正在下载图片:{_url} ...")
|
||||
r = RequestUtils(proxies=settings.PROXY, ua=settings.NORMAL_USER_AGENT).get_res(url=_url)
|
||||
if r:
|
||||
return r.content
|
||||
else:
|
||||
logger.info(f"{_url} 图片下载失败,请检查网络连通性!")
|
||||
request_utils = RequestUtils(proxies=settings.PROXY, ua=settings.NORMAL_USER_AGENT)
|
||||
with request_utils.get_stream(url=_url) as r:
|
||||
if r and r.status_code == 200:
|
||||
# 使用tempfile创建临时文件,自动删除
|
||||
with NamedTemporaryFile(delete=True, delete_on_close=False, suffix=_path.suffix) as tmp_file:
|
||||
tmp_file_path = Path(tmp_file.name)
|
||||
# 流式写入文件
|
||||
for chunk in r.iter_content(chunk_size=8192):
|
||||
if chunk:
|
||||
tmp_file.write(chunk)
|
||||
tmp_file.flush()
|
||||
tmp_file.close() # 关闭文件句柄
|
||||
|
||||
# 刮削的图片只需要读写权限
|
||||
tmp_file_path.chmod(0o666 & ~current_umask)
|
||||
|
||||
# 上传文件
|
||||
item = storagechain.upload_file(fileitem=_fileitem, path=tmp_file_path,
|
||||
new_name=_path.name)
|
||||
if item:
|
||||
logger.info(f"已保存图片:{item.path}")
|
||||
else:
|
||||
logger.warn(f"图片保存失败:{_path}")
|
||||
else:
|
||||
logger.info(f"{_url} 图片下载失败")
|
||||
except Exception as err:
|
||||
logger.error(f"{_url} 图片下载失败:{str(err)}!")
|
||||
return None
|
||||
|
||||
if not fileitem:
|
||||
return
|
||||
@@ -521,7 +565,7 @@ class MediaChain(ChainBase):
|
||||
# 电影目录
|
||||
if recursive:
|
||||
# 处理文件
|
||||
if is_bluray_folder(fileitem):
|
||||
if self.is_bluray_folder(fileitem):
|
||||
# 原盘目录
|
||||
if scraping_switchs.get('movie_nfo', True):
|
||||
nfo_path = filepath / (filepath.name + ".nfo")
|
||||
@@ -541,6 +585,9 @@ class MediaChain(ChainBase):
|
||||
# 处理目录内的文件
|
||||
files = __list_files(_fileitem=fileitem)
|
||||
for file in files:
|
||||
if file.type == "dir":
|
||||
# 电影不处理子目录
|
||||
continue
|
||||
self.scrape_metadata(fileitem=file,
|
||||
mediainfo=mediainfo,
|
||||
init_folder=False,
|
||||
@@ -574,11 +621,8 @@ class MediaChain(ChainBase):
|
||||
image_path = filepath.with_name(image_name)
|
||||
if overwrite or not storagechain.get_file_item(storage=fileitem.storage,
|
||||
path=image_path):
|
||||
# 下载图片
|
||||
content = __download_image(image_url)
|
||||
# 写入图片到当前目录
|
||||
if content:
|
||||
__save_file(_fileitem=fileitem, _path=image_path, _content=content)
|
||||
# 流式下载图片并直接保存
|
||||
__download_and_save_image(_fileitem=fileitem, _path=image_path, _url=image_url)
|
||||
else:
|
||||
logger.info(f"已存在图片文件:{image_path}")
|
||||
else:
|
||||
@@ -624,13 +668,10 @@ class MediaChain(ChainBase):
|
||||
for episode, image_url in image_dict.items():
|
||||
image_path = filepath.with_suffix(Path(image_url).suffix)
|
||||
if overwrite or not storagechain.get_file_item(storage=fileitem.storage, path=image_path):
|
||||
# 下载图片
|
||||
content = __download_image(image_url)
|
||||
# 保存图片文件到当前目录
|
||||
if content:
|
||||
if not parent:
|
||||
parent = storagechain.get_parent_item(fileitem)
|
||||
__save_file(_fileitem=parent, _path=image_path, _content=content)
|
||||
# 流式下载图片并直接保存
|
||||
if not parent:
|
||||
parent = storagechain.get_parent_item(fileitem)
|
||||
__download_and_save_image(_fileitem=parent, _path=image_path, _url=image_url)
|
||||
else:
|
||||
logger.info(f"已存在图片文件:{image_path}")
|
||||
else:
|
||||
@@ -640,6 +681,9 @@ class MediaChain(ChainBase):
|
||||
if recursive:
|
||||
files = __list_files(_fileitem=fileitem)
|
||||
for file in files:
|
||||
if file.type == "dir" and not file.name.lower().startswith("season"):
|
||||
# 电视剧不处理非季子目录
|
||||
continue
|
||||
self.scrape_metadata(fileitem=file,
|
||||
mediainfo=mediainfo,
|
||||
parent=fileitem if file.type == "file" else None,
|
||||
@@ -678,13 +722,10 @@ class MediaChain(ChainBase):
|
||||
image_path = filepath.with_name(image_name)
|
||||
if overwrite or not storagechain.get_file_item(storage=fileitem.storage,
|
||||
path=image_path):
|
||||
# 下载图片
|
||||
content = __download_image(image_url)
|
||||
# 保存图片文件到剧集目录
|
||||
if content:
|
||||
if not parent:
|
||||
parent = storagechain.get_parent_item(fileitem)
|
||||
__save_file(_fileitem=parent, _path=image_path, _content=content)
|
||||
# 流式下载图片并直接保存
|
||||
if not parent:
|
||||
parent = storagechain.get_parent_item(fileitem)
|
||||
__download_and_save_image(_fileitem=parent, _path=image_path, _url=image_url)
|
||||
else:
|
||||
logger.info(f"已存在图片文件:{image_path}")
|
||||
else:
|
||||
@@ -714,13 +755,11 @@ class MediaChain(ChainBase):
|
||||
continue
|
||||
if overwrite or not storagechain.get_file_item(storage=fileitem.storage,
|
||||
path=image_path):
|
||||
# 下载图片
|
||||
content = __download_image(image_url)
|
||||
# 保存图片文件到当前目录
|
||||
if content:
|
||||
if not parent:
|
||||
parent = storagechain.get_parent_item(fileitem)
|
||||
__save_file(_fileitem=parent, _path=image_path, _content=content)
|
||||
# 流式下载图片并直接保存
|
||||
if not parent:
|
||||
parent = storagechain.get_parent_item(fileitem)
|
||||
__download_and_save_image(_fileitem=parent, _path=image_path,
|
||||
_url=image_url)
|
||||
else:
|
||||
logger.info(f"已存在图片文件:{image_path}")
|
||||
else:
|
||||
@@ -770,11 +809,8 @@ class MediaChain(ChainBase):
|
||||
image_path = filepath / image_name
|
||||
if overwrite or not storagechain.get_file_item(storage=fileitem.storage,
|
||||
path=image_path):
|
||||
# 下载图片
|
||||
content = __download_image(image_url)
|
||||
# 保存图片文件到当前目录
|
||||
if content:
|
||||
__save_file(_fileitem=fileitem, _path=image_path, _content=content)
|
||||
# 流式下载图片并直接保存
|
||||
__download_and_save_image(_fileitem=fileitem, _path=image_path, _url=image_url)
|
||||
else:
|
||||
logger.info(f"已存在图片文件:{image_path}")
|
||||
else:
|
||||
|
||||
@@ -113,6 +113,16 @@ class MediaServerChain(ChainBase):
|
||||
"""
|
||||
return self.run_module("mediaserver_play_url", server=server, item_id=item_id)
|
||||
|
||||
def get_image_cookies(
|
||||
self, server: Optional[str], image_url: str
|
||||
) -> Optional[str | dict]:
|
||||
"""
|
||||
获取图片的Cookies
|
||||
"""
|
||||
return self.run_module(
|
||||
"mediaserver_image_cookies", server=server, image_url=image_url
|
||||
)
|
||||
|
||||
def sync(self):
|
||||
"""
|
||||
同步媒体库所有数据到本地数据库
|
||||
@@ -167,7 +177,7 @@ class MediaServerChain(ChainBase):
|
||||
for episode in espisodes_info:
|
||||
seasoninfo[episode.season] = episode.episodes
|
||||
# 插入数据
|
||||
item_dict = item.dict()
|
||||
item_dict = item.model_dump()
|
||||
item_dict["seasoninfo"] = seasoninfo
|
||||
item_dict["item_type"] = item_type
|
||||
dboper.add(**item_dict)
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
import asyncio
|
||||
import re
|
||||
from typing import Any, Optional, Dict, Union, List
|
||||
|
||||
from app.agent import agent_manager
|
||||
from app.chain import ChainBase
|
||||
from app.chain.download import DownloadChain
|
||||
from app.chain.media import MediaChain
|
||||
@@ -163,6 +165,10 @@ class MessageChain(ChainBase):
|
||||
original_message_id=original_message_id, original_chat_id=original_chat_id)
|
||||
else:
|
||||
logger.warning(f"渠道 {channel.value} 不支持回调,但收到了回调消息:{text}")
|
||||
elif text.startswith('/ai') or text.startswith('/AI'):
|
||||
# AI智能体处理
|
||||
self._handle_ai_message(text=text, channel=channel, source=source,
|
||||
userid=userid, username=username)
|
||||
elif text.startswith('/'):
|
||||
# 执行命令
|
||||
self.eventmanager.send_event(
|
||||
@@ -815,3 +821,86 @@ class MessageChain(ChainBase):
|
||||
buttons.append(page_buttons)
|
||||
|
||||
return buttons
|
||||
|
||||
def _handle_ai_message(self, text: str, channel: MessageChannel, source: str,
|
||||
userid: Union[str, int], username: str) -> None:
|
||||
"""
|
||||
处理AI智能体消息
|
||||
"""
|
||||
try:
|
||||
# 检查AI智能体是否启用
|
||||
if not settings.AI_AGENT_ENABLE:
|
||||
self.post_message(Notification(
|
||||
channel=channel,
|
||||
source=source,
|
||||
userid=userid,
|
||||
username=username,
|
||||
title="MoviePilot智能助手未启用,请在系统设置中启用"
|
||||
))
|
||||
return
|
||||
|
||||
# 检查LLM配置
|
||||
if not settings.LLM_API_KEY:
|
||||
self.post_message(Notification(
|
||||
channel=channel,
|
||||
source=source,
|
||||
userid=userid,
|
||||
username=username,
|
||||
title="MoviePilot智能助未配置,请在系统设置中配置"
|
||||
))
|
||||
return
|
||||
|
||||
# 提取用户消息
|
||||
user_message = text[3:].strip() # 移除 "/ai" 前缀
|
||||
if not user_message:
|
||||
self.post_message(Notification(
|
||||
channel=channel,
|
||||
source=source,
|
||||
userid=userid,
|
||||
username=username,
|
||||
title="请输入您的问题或需求"
|
||||
))
|
||||
return
|
||||
|
||||
# 发送处理中消息
|
||||
self.post_message(Notification(
|
||||
channel=channel,
|
||||
source=source,
|
||||
userid=userid,
|
||||
username=username,
|
||||
title="MoviePilot助手已收到您的请求,请稍候..."
|
||||
))
|
||||
|
||||
# 生成会话ID
|
||||
session_id = f"user_{userid}_{hash(user_message) % 10000}"
|
||||
|
||||
# 在事件循环中处理
|
||||
try:
|
||||
loop = asyncio.get_event_loop()
|
||||
loop.run_until_complete(
|
||||
agent_manager.process_message(
|
||||
session_id=session_id,
|
||||
user_id=str(userid),
|
||||
message=user_message,
|
||||
channel=channel.value if channel else None,
|
||||
source=source,
|
||||
username=username
|
||||
)
|
||||
)
|
||||
except RuntimeError:
|
||||
# 如果没有事件循环,创建新的
|
||||
asyncio.run(
|
||||
agent_manager.process_message(
|
||||
session_id=session_id,
|
||||
user_id=str(userid),
|
||||
message=user_message,
|
||||
channel=channel.value if channel else None,
|
||||
source=source,
|
||||
username=username
|
||||
)
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"处理AI智能体消息失败: {e}")
|
||||
self.messagehelper.put(f"AI智能体处理失败: {str(e)}", role="system", title="MoviePilot助手")
|
||||
|
||||
|
||||
@@ -1,48 +1,142 @@
|
||||
import asyncio
|
||||
import io
|
||||
from pathlib import Path
|
||||
from typing import List, Optional
|
||||
|
||||
import aiofiles
|
||||
import pillow_avif # noqa 用于自动注册AVIF支持
|
||||
from PIL import Image
|
||||
from aiopath import AsyncPath
|
||||
|
||||
from app.chain import ChainBase
|
||||
from app.chain.bangumi import BangumiChain
|
||||
from app.chain.douban import DoubanChain
|
||||
from app.chain.tmdb import TmdbChain
|
||||
from app.core.cache import cache_backend, cached
|
||||
from app.core.cache import cached, FileCache
|
||||
from app.core.config import settings, global_vars
|
||||
from app.log import logger
|
||||
from app.schemas import MediaType
|
||||
from app.utils.asyncio import AsyncUtils
|
||||
from app.utils.common import log_execution_time
|
||||
from app.utils.http import AsyncRequestUtils
|
||||
from app.utils.http import RequestUtils
|
||||
from app.utils.security import SecurityUtils
|
||||
from app.utils.singleton import Singleton
|
||||
|
||||
# 推荐相关的专用缓存
|
||||
recommend_ttl = 24 * 3600
|
||||
recommend_cache_region = "recommend"
|
||||
|
||||
|
||||
class RecommendChain(ChainBase, metaclass=Singleton):
|
||||
"""
|
||||
推荐处理链,单例运行
|
||||
"""
|
||||
|
||||
# 推荐数据的缓存页数
|
||||
# 推荐缓存时间
|
||||
recommend_ttl = 24 * 3600
|
||||
# 推荐缓存页数
|
||||
cache_max_pages = 5
|
||||
# 推荐缓存区域
|
||||
recommend_cache_region = "recommend"
|
||||
|
||||
def refresh_recommend(self):
|
||||
"""
|
||||
刷新推荐数据 - 同步包装器
|
||||
刷新推荐
|
||||
"""
|
||||
logger.debug("Starting to refresh Recommend data.")
|
||||
|
||||
# 推荐来源方法
|
||||
recommend_methods = [
|
||||
self.tmdb_movies,
|
||||
self.tmdb_tvs,
|
||||
self.tmdb_trending,
|
||||
self.bangumi_calendar,
|
||||
self.douban_movie_showing,
|
||||
self.douban_movies,
|
||||
self.douban_tvs,
|
||||
self.douban_movie_top250,
|
||||
self.douban_tv_weekly_chinese,
|
||||
self.douban_tv_weekly_global,
|
||||
self.douban_tv_animation,
|
||||
self.douban_movie_hot,
|
||||
self.douban_tv_hot,
|
||||
]
|
||||
|
||||
# 缓存并刷新所有推荐数据
|
||||
recommends = []
|
||||
# 记录哪些方法已完成
|
||||
methods_finished = set()
|
||||
# 这里避免区间内连续调用相同来源,因此遍历方案为每页遍历所有推荐来源,再进行页数遍历
|
||||
for page in range(1, self.cache_max_pages + 1):
|
||||
for method in recommend_methods:
|
||||
if global_vars.is_system_stopped:
|
||||
return
|
||||
if method in methods_finished:
|
||||
continue
|
||||
logger.debug(f"Fetch {method.__name__} data for page {page}.")
|
||||
data = method(page=page)
|
||||
if not data:
|
||||
logger.debug("All recommendation methods have finished fetching data. Ending pagination early.")
|
||||
methods_finished.add(method)
|
||||
continue
|
||||
recommends.extend(data)
|
||||
# 如果所有方法都已经完成,提前结束循环
|
||||
if len(methods_finished) == len(recommend_methods):
|
||||
break
|
||||
|
||||
# 缓存收集到的海报
|
||||
self.__cache_posters(recommends)
|
||||
logger.debug("Recommend data refresh completed.")
|
||||
|
||||
def __cache_posters(self, datas: List[dict]):
|
||||
"""
|
||||
提取 poster_path 并缓存图片
|
||||
:param datas: 数据列表
|
||||
"""
|
||||
if not settings.GLOBAL_IMAGE_CACHE:
|
||||
return
|
||||
|
||||
for data in datas:
|
||||
if global_vars.is_system_stopped:
|
||||
return
|
||||
poster_path = data.get("poster_path")
|
||||
if poster_path:
|
||||
poster_url = poster_path.replace("original", "w500")
|
||||
logger.debug(f"Caching poster image: {poster_url}")
|
||||
self.__fetch_and_save_image(poster_url)
|
||||
|
||||
@staticmethod
|
||||
def __fetch_and_save_image(url: str):
|
||||
"""
|
||||
请求并保存图片
|
||||
:param url: 图片路径
|
||||
"""
|
||||
# 生成缓存路径
|
||||
sanitized_path = SecurityUtils.sanitize_url_path(url)
|
||||
cache_path = Path("images") / sanitized_path
|
||||
# 没有文件类型,则添加后缀,在恶意文件类型和实际需求下的折衷选择
|
||||
if not cache_path.suffix:
|
||||
cache_path = cache_path.with_suffix(".jpg")
|
||||
|
||||
# 获取缓存后端,并设置缓存时间为全局配置的缓存天数
|
||||
cache_backend = FileCache(base=settings.CACHE_PATH,
|
||||
ttl=settings.GLOBAL_IMAGE_CACHE_DAYS * 24 * 3600)
|
||||
|
||||
# 本地存在缓存图片,则直接跳过
|
||||
if cache_backend.get(cache_path.as_posix(), region="images"):
|
||||
logger.debug(f"Cache hit: Image already exists at {cache_path}")
|
||||
return
|
||||
|
||||
# 请求远程图片
|
||||
referer = "https://movie.douban.com/" if "doubanio.com" in url else None
|
||||
proxies = settings.PROXY if not referer else None
|
||||
response = RequestUtils(ua=settings.NORMAL_USER_AGENT, proxies=proxies, referer=referer).get_res(url=url)
|
||||
if not response:
|
||||
logger.debug(f"Empty response for URL: {url}")
|
||||
return
|
||||
|
||||
# 验证下载的内容是否为有效图片
|
||||
try:
|
||||
AsyncUtils.run_async(self.async_refresh_recommend())
|
||||
Image.open(io.BytesIO(response.content)).verify()
|
||||
except Exception as e:
|
||||
logger.error(f"刷新推荐数据失败:{str(e)}")
|
||||
raise
|
||||
logger.debug(f"Invalid image format for URL {url}: {e}")
|
||||
return
|
||||
|
||||
# 保存缓存
|
||||
cache_backend.set(cache_path.as_posix(), response.content, region="images")
|
||||
logger.debug(f"Successfully cached image at {cache_path} for URL: {url}")
|
||||
|
||||
@log_execution_time(logger=logger)
|
||||
@cached(ttl=recommend_ttl, region=recommend_cache_region)
|
||||
@@ -199,162 +293,6 @@ class RecommendChain(ChainBase, metaclass=Singleton):
|
||||
tvs = DoubanChain().tv_hot(page=page, count=count)
|
||||
return [media.to_dict() for media in tvs] if tvs else []
|
||||
|
||||
# 异步版本的方法
|
||||
async def async_refresh_recommend(self):
|
||||
"""
|
||||
异步刷新推荐
|
||||
"""
|
||||
logger.debug("Starting to async refresh Recommend data.")
|
||||
cache_backend.clear(region=recommend_cache_region)
|
||||
logger.debug("Recommend Cache has been cleared.")
|
||||
|
||||
# 推荐来源方法
|
||||
recommend_methods = [
|
||||
self.async_tmdb_movies,
|
||||
self.async_tmdb_tvs,
|
||||
self.async_tmdb_trending,
|
||||
self.async_bangumi_calendar,
|
||||
self.async_douban_movie_showing,
|
||||
self.async_douban_movies,
|
||||
self.async_douban_tvs,
|
||||
self.async_douban_movie_top250,
|
||||
self.async_douban_tv_weekly_chinese,
|
||||
self.async_douban_tv_weekly_global,
|
||||
self.async_douban_tv_animation,
|
||||
self.async_douban_movie_hot,
|
||||
self.async_douban_tv_hot,
|
||||
]
|
||||
|
||||
# 缓存并刷新所有推荐数据
|
||||
recommends = []
|
||||
# 记录哪些方法已完成
|
||||
methods_finished = set()
|
||||
# 这里避免区间内连续调用相同来源,因此遍历方案为每页遍历所有推荐来源,再进行页数遍历
|
||||
for page in range(1, self.cache_max_pages + 1):
|
||||
# 为每个页面并发执行所有方法
|
||||
tasks = []
|
||||
for method in recommend_methods:
|
||||
if global_vars.is_system_stopped:
|
||||
return
|
||||
if method in methods_finished:
|
||||
continue
|
||||
tasks.append(self._async_fetch_method_data(method, page, methods_finished))
|
||||
|
||||
# 并发执行所有任务
|
||||
if tasks:
|
||||
results = await asyncio.gather(*tasks, return_exceptions=True)
|
||||
for result in results:
|
||||
if isinstance(result, list) and result:
|
||||
recommends.extend(result)
|
||||
|
||||
# 如果所有方法都已经完成,提前结束循环
|
||||
if len(methods_finished) == len(recommend_methods):
|
||||
break
|
||||
|
||||
# 缓存收集到的海报
|
||||
await self.__async_cache_posters(recommends)
|
||||
logger.debug("Async recommend data refresh completed.")
|
||||
|
||||
@staticmethod
|
||||
async def _async_fetch_method_data(method, page: int, methods_finished: set):
|
||||
"""
|
||||
异步获取方法数据的辅助函数
|
||||
"""
|
||||
try:
|
||||
logger.debug(f"Async fetch {method.__name__} data for page {page}.")
|
||||
data = await method(page=page)
|
||||
if not data:
|
||||
logger.debug(f"Method {method.__name__} finished fetching data. Ending pagination early.")
|
||||
methods_finished.add(method)
|
||||
return []
|
||||
return data
|
||||
except Exception as e:
|
||||
logger.error(f"Error fetching data from {method.__name__}: {e}")
|
||||
methods_finished.add(method)
|
||||
return []
|
||||
|
||||
async def __async_cache_posters(self, datas: List[dict]):
|
||||
"""
|
||||
异步提取 poster_path 并缓存图片
|
||||
:param datas: 数据列表
|
||||
"""
|
||||
if not settings.GLOBAL_IMAGE_CACHE:
|
||||
return
|
||||
|
||||
tasks = []
|
||||
for data in datas:
|
||||
if global_vars.is_system_stopped:
|
||||
return
|
||||
poster_path = data.get("poster_path")
|
||||
if poster_path:
|
||||
poster_url = poster_path.replace("original", "w500")
|
||||
logger.debug(f"Async caching poster image: {poster_url}")
|
||||
tasks.append(self.__async_fetch_and_save_image(poster_url))
|
||||
|
||||
# 并发缓存图片
|
||||
if tasks:
|
||||
await asyncio.gather(*tasks, return_exceptions=True)
|
||||
|
||||
@staticmethod
|
||||
async def __async_fetch_and_save_image(url: str):
|
||||
"""
|
||||
异步请求并保存图片
|
||||
:param url: 图片路径
|
||||
"""
|
||||
if not settings.GLOBAL_IMAGE_CACHE or not url:
|
||||
return
|
||||
|
||||
# 生成缓存路径
|
||||
base_path = AsyncPath(settings.CACHE_PATH)
|
||||
sanitized_path = SecurityUtils.sanitize_url_path(url)
|
||||
cache_path = base_path / "images" / sanitized_path
|
||||
|
||||
# 没有文件类型,则添加后缀,在恶意文件类型和实际需求下的折衷选择
|
||||
if not cache_path.suffix:
|
||||
cache_path = cache_path.with_suffix(".jpg")
|
||||
|
||||
# 确保缓存路径和文件类型合法
|
||||
if not await SecurityUtils.async_is_safe_path(base_path=base_path,
|
||||
user_path=cache_path,
|
||||
allowed_suffixes=settings.SECURITY_IMAGE_SUFFIXES):
|
||||
logger.debug(f"Invalid cache path or file type for URL: {url}, sanitized path: {sanitized_path}")
|
||||
return
|
||||
|
||||
# 本地存在缓存图片,则直接跳过
|
||||
if await cache_path.exists():
|
||||
logger.debug(f"Cache hit: Image already exists at {cache_path}")
|
||||
return
|
||||
|
||||
# 请求远程图片
|
||||
referer = "https://movie.douban.com/" if "doubanio.com" in url else None
|
||||
proxies = settings.PROXY if not referer else None
|
||||
response = await AsyncRequestUtils(ua=settings.NORMAL_USER_AGENT,
|
||||
proxies=proxies, referer=referer).get_res(url=url)
|
||||
if not response:
|
||||
logger.debug(f"Empty response for URL: {url}")
|
||||
return
|
||||
|
||||
# 验证下载的内容是否为有效图片
|
||||
try:
|
||||
Image.open(io.BytesIO(response.content)).verify()
|
||||
except Exception as e:
|
||||
logger.debug(f"Invalid image format for URL {url}: {e}")
|
||||
return
|
||||
|
||||
if not cache_path:
|
||||
return
|
||||
|
||||
try:
|
||||
if not await cache_path.parent.exists():
|
||||
await cache_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
async with aiofiles.tempfile.NamedTemporaryFile(dir=cache_path.parent, delete=False) as tmp_file:
|
||||
await tmp_file.write(response.content)
|
||||
temp_path = AsyncPath(tmp_file.name)
|
||||
await temp_path.replace(cache_path)
|
||||
logger.debug(f"Successfully cached image at {cache_path} for URL: {url}")
|
||||
except Exception as e:
|
||||
logger.debug(f"Failed to write cache file {cache_path} for URL {url}: {e}")
|
||||
|
||||
@log_execution_time(logger=logger)
|
||||
@cached(ttl=recommend_ttl, region=recommend_cache_region)
|
||||
async def async_tmdb_movies(self, sort_by: Optional[str] = "popularity.desc",
|
||||
|
||||
@@ -1,13 +1,12 @@
|
||||
import asyncio
|
||||
import pickle
|
||||
import random
|
||||
import time
|
||||
import traceback
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
from datetime import datetime
|
||||
from typing import Dict, Tuple
|
||||
from typing import List, Optional
|
||||
|
||||
from app.helper.sites import SitesHelper # noqa
|
||||
from fastapi.concurrency import run_in_threadpool
|
||||
|
||||
from app.chain import ChainBase
|
||||
@@ -18,7 +17,6 @@ from app.core.event import eventmanager, Event
|
||||
from app.core.metainfo import MetaInfo
|
||||
from app.db.systemconfig_oper import SystemConfigOper
|
||||
from app.helper.progress import ProgressHelper
|
||||
from app.helper.sites import SitesHelper # noqa
|
||||
from app.helper.torrent import TorrentHelper
|
||||
from app.log import logger
|
||||
from app.schemas import NotExistMediaInfo
|
||||
@@ -59,7 +57,7 @@ class SearchChain(ChainBase):
|
||||
results = self.process(mediainfo=mediainfo, sites=sites, area=area, no_exists=no_exists)
|
||||
# 保存到本地文件
|
||||
if cache_local:
|
||||
self.save_cache(pickle.dumps(results), self.__result_temp_file)
|
||||
self.save_cache(results, self.__result_temp_file)
|
||||
return results
|
||||
|
||||
def search_by_title(self, title: str, page: Optional[int] = 0,
|
||||
@@ -85,36 +83,20 @@ class SearchChain(ChainBase):
|
||||
torrent_info=torrent) for torrent in torrents]
|
||||
# 保存到本地文件
|
||||
if cache_local:
|
||||
self.save_cache(pickle.dumps(contexts), self.__result_temp_file)
|
||||
self.save_cache(contexts, self.__result_temp_file)
|
||||
return contexts
|
||||
|
||||
def last_search_results(self) -> List[Context]:
|
||||
def last_search_results(self) -> Optional[List[Context]]:
|
||||
"""
|
||||
获取上次搜索结果
|
||||
"""
|
||||
# 读取本地文件缓存
|
||||
content = self.load_cache(self.__result_temp_file)
|
||||
if not content:
|
||||
return []
|
||||
try:
|
||||
return pickle.loads(content)
|
||||
except Exception as e:
|
||||
logger.error(f'加载搜索结果失败:{str(e)} - {traceback.format_exc()}')
|
||||
return []
|
||||
return self.load_cache(self.__result_temp_file)
|
||||
|
||||
async def async_last_search_results(self) -> List[Context]:
|
||||
async def async_last_search_results(self) -> Optional[List[Context]]:
|
||||
"""
|
||||
异步获取上次搜索结果
|
||||
"""
|
||||
# 读取本地文件缓存
|
||||
content = await self.async_load_cache(self.__result_temp_file)
|
||||
if not content:
|
||||
return []
|
||||
try:
|
||||
return pickle.loads(content)
|
||||
except Exception as e:
|
||||
logger.error(f'加载搜索结果失败:{str(e)} - {traceback.format_exc()}')
|
||||
return []
|
||||
return await self.async_load_cache(self.__result_temp_file)
|
||||
|
||||
async def async_search_by_id(self, tmdbid: Optional[int] = None, doubanid: Optional[str] = None,
|
||||
mtype: MediaType = None, area: Optional[str] = "title", season: Optional[int] = None,
|
||||
@@ -143,7 +125,7 @@ class SearchChain(ChainBase):
|
||||
results = await self.async_process(mediainfo=mediainfo, sites=sites, area=area, no_exists=no_exists)
|
||||
# 保存到本地文件
|
||||
if cache_local:
|
||||
await self.async_save_cache(pickle.dumps(results), self.__result_temp_file)
|
||||
await self.async_save_cache(results, self.__result_temp_file)
|
||||
return results
|
||||
|
||||
async def async_search_by_title(self, title: str, page: Optional[int] = 0,
|
||||
@@ -169,7 +151,7 @@ class SearchChain(ChainBase):
|
||||
torrent_info=torrent) for torrent in torrents]
|
||||
# 保存到本地文件
|
||||
if cache_local:
|
||||
await self.async_save_cache(pickle.dumps(contexts), self.__result_temp_file)
|
||||
await self.async_save_cache(contexts, self.__result_temp_file)
|
||||
return contexts
|
||||
|
||||
@staticmethod
|
||||
@@ -233,12 +215,11 @@ class SearchChain(ChainBase):
|
||||
return []
|
||||
|
||||
# 开始新进度
|
||||
progress = ProgressHelper()
|
||||
progress.start(ProgressKey.Search)
|
||||
progress = ProgressHelper(ProgressKey.Search)
|
||||
progress.start()
|
||||
|
||||
# 开始过滤
|
||||
progress.update(value=0, text=f'开始过滤,总 {len(torrents)} 个资源,请稍候...',
|
||||
key=ProgressKey.Search)
|
||||
progress.update(value=0, text=f'开始过滤,总 {len(torrents)} 个资源,请稍候...')
|
||||
# 匹配订阅附加参数
|
||||
if filter_params:
|
||||
logger.info(f'开始附加参数过滤,附加参数:{filter_params} ...')
|
||||
@@ -256,7 +237,7 @@ class SearchChain(ChainBase):
|
||||
logger.info(f"过滤规则/剧集过滤完成,剩余 {len(torrents)} 个资源")
|
||||
|
||||
# 过滤完成
|
||||
progress.update(value=50, text=f'过滤完成,剩余 {len(torrents)} 个资源', key=ProgressKey.Search)
|
||||
progress.update(value=50, text=f'过滤完成,剩余 {len(torrents)} 个资源')
|
||||
|
||||
# 总数
|
||||
_total = len(torrents)
|
||||
@@ -269,14 +250,13 @@ class SearchChain(ChainBase):
|
||||
try:
|
||||
# 英文标题应该在别名/原标题中,不需要再匹配
|
||||
logger.info(f"开始匹配结果 标题:{mediainfo.title},原标题:{mediainfo.original_title},别名:{mediainfo.names}")
|
||||
progress.update(value=51, text=f'开始匹配,总 {_total} 个资源 ...', key=ProgressKey.Search)
|
||||
progress.update(value=51, text=f'开始匹配,总 {_total} 个资源 ...')
|
||||
for torrent in torrents:
|
||||
if global_vars.is_system_stopped:
|
||||
break
|
||||
_count += 1
|
||||
progress.update(value=(_count / _total) * 96,
|
||||
text=f'正在匹配 {torrent.site_name},已完成 {_count} / {_total} ...',
|
||||
key=ProgressKey.Search)
|
||||
text=f'正在匹配 {torrent.site_name},已完成 {_count} / {_total} ...')
|
||||
if not torrent.title:
|
||||
continue
|
||||
|
||||
@@ -309,8 +289,7 @@ class SearchChain(ChainBase):
|
||||
# 匹配完成
|
||||
logger.info(f"匹配完成,共匹配到 {len(_match_torrents)} 个资源")
|
||||
progress.update(value=97,
|
||||
text=f'匹配完成,共匹配到 {len(_match_torrents)} 个资源',
|
||||
key=ProgressKey.Search)
|
||||
text=f'匹配完成,共匹配到 {len(_match_torrents)} 个资源')
|
||||
|
||||
# 去掉mediainfo中多余的数据
|
||||
mediainfo.clear()
|
||||
@@ -326,16 +305,14 @@ class SearchChain(ChainBase):
|
||||
|
||||
# 排序
|
||||
progress.update(value=99,
|
||||
text=f'正在对 {len(contexts)} 个资源进行排序,请稍候...',
|
||||
key=ProgressKey.Search)
|
||||
text=f'正在对 {len(contexts)} 个资源进行排序,请稍候...')
|
||||
contexts = torrenthelper.sort_torrents(contexts)
|
||||
|
||||
# 结束进度
|
||||
logger.info(f'搜索完成,共 {len(contexts)} 个资源')
|
||||
progress.update(value=100,
|
||||
text=f'搜索完成,共 {len(contexts)} 个资源',
|
||||
key=ProgressKey.Search)
|
||||
progress.end(ProgressKey.Search)
|
||||
text=f'搜索完成,共 {len(contexts)} 个资源')
|
||||
progress.end()
|
||||
|
||||
# 去重后返回
|
||||
return self.__remove_duplicate(contexts)
|
||||
@@ -347,9 +324,6 @@ class SearchChain(ChainBase):
|
||||
:param _torrents: 种子列表
|
||||
:return: 去重后的种子列表
|
||||
"""
|
||||
if not settings.SEARCH_MULTIPLE_NAME:
|
||||
return _torrents
|
||||
# 通过encosure去重
|
||||
return list({f"{t.torrent_info.site_name}_{t.torrent_info.title}_{t.torrent_info.description}": t
|
||||
for t in _torrents}.values())
|
||||
|
||||
@@ -407,16 +381,23 @@ class SearchChain(ChainBase):
|
||||
if search_count > 0:
|
||||
logger.info(f"已搜索 {search_count} 次,强制休眠 1-10 秒 ...")
|
||||
time.sleep(random.randint(1, 10))
|
||||
|
||||
# 搜索站点
|
||||
torrents.extend(
|
||||
self.__search_all_sites(
|
||||
mediainfo=mediainfo,
|
||||
keyword=search_word,
|
||||
sites=sites,
|
||||
area=area
|
||||
) or []
|
||||
)
|
||||
results = self.__search_all_sites(
|
||||
mediainfo=mediainfo,
|
||||
keyword=search_word,
|
||||
sites=sites,
|
||||
area=area
|
||||
) or []
|
||||
# 合并结果
|
||||
|
||||
search_count += 1
|
||||
torrents.extend(results)
|
||||
|
||||
# 有结果则停止
|
||||
if not settings.SEARCH_MULTIPLE_NAME and torrents:
|
||||
logger.info(f"共搜索到 {len(torrents)} 个资源,停止搜索")
|
||||
break
|
||||
|
||||
# 处理结果
|
||||
return self.__parse_result(
|
||||
@@ -539,8 +520,8 @@ class SearchChain(ChainBase):
|
||||
return []
|
||||
|
||||
# 开始进度
|
||||
progress = ProgressHelper()
|
||||
progress.start(ProgressKey.Search)
|
||||
progress = ProgressHelper(ProgressKey.Search)
|
||||
progress.start()
|
||||
# 开始计时
|
||||
start_time = datetime.now()
|
||||
# 总数
|
||||
@@ -549,8 +530,7 @@ class SearchChain(ChainBase):
|
||||
finish_count = 0
|
||||
# 更新进度
|
||||
progress.update(value=0,
|
||||
text=f"开始搜索,共 {total_num} 个站点 ...",
|
||||
key=ProgressKey.Search)
|
||||
text=f"开始搜索,共 {total_num} 个站点 ...")
|
||||
# 结果集
|
||||
results = []
|
||||
# 多线程
|
||||
@@ -579,17 +559,15 @@ class SearchChain(ChainBase):
|
||||
results.extend(result)
|
||||
logger.info(f"站点搜索进度:{finish_count} / {total_num}")
|
||||
progress.update(value=finish_count / total_num * 100,
|
||||
text=f"正在搜索{keyword or ''},已完成 {finish_count} / {total_num} 个站点 ...",
|
||||
key=ProgressKey.Search)
|
||||
text=f"正在搜索{keyword or ''},已完成 {finish_count} / {total_num} 个站点 ...")
|
||||
# 计算耗时
|
||||
end_time = datetime.now()
|
||||
# 更新进度
|
||||
progress.update(value=100,
|
||||
text=f"站点搜索完成,有效资源数:{len(results)},总耗时 {(end_time - start_time).seconds} 秒",
|
||||
key=ProgressKey.Search)
|
||||
text=f"站点搜索完成,有效资源数:{len(results)},总耗时 {(end_time - start_time).seconds} 秒")
|
||||
logger.info(f"站点搜索完成,有效资源数:{len(results)},总耗时 {(end_time - start_time).seconds} 秒")
|
||||
# 结束进度
|
||||
progress.end(ProgressKey.Search)
|
||||
progress.end()
|
||||
|
||||
# 返回
|
||||
return results
|
||||
@@ -624,8 +602,8 @@ class SearchChain(ChainBase):
|
||||
return []
|
||||
|
||||
# 开始进度
|
||||
progress = ProgressHelper()
|
||||
progress.start(ProgressKey.Search)
|
||||
progress = ProgressHelper(ProgressKey.Search)
|
||||
progress.start()
|
||||
# 开始计时
|
||||
start_time = datetime.now()
|
||||
# 总数
|
||||
@@ -634,8 +612,7 @@ class SearchChain(ChainBase):
|
||||
finish_count = 0
|
||||
# 更新进度
|
||||
progress.update(value=0,
|
||||
text=f"开始搜索,共 {total_num} 个站点 ...",
|
||||
key=ProgressKey.Search)
|
||||
text=f"开始搜索,共 {total_num} 个站点 ...")
|
||||
# 结果集
|
||||
results = []
|
||||
|
||||
@@ -666,18 +643,16 @@ class SearchChain(ChainBase):
|
||||
results.extend(result)
|
||||
logger.info(f"站点搜索进度:{finish_count} / {total_num}")
|
||||
progress.update(value=finish_count / total_num * 100,
|
||||
text=f"正在搜索{keyword or ''},已完成 {finish_count} / {total_num} 个站点 ...",
|
||||
key=ProgressKey.Search)
|
||||
text=f"正在搜索{keyword or ''},已完成 {finish_count} / {total_num} 个站点 ...")
|
||||
|
||||
# 计算耗时
|
||||
end_time = datetime.now()
|
||||
# 更新进度
|
||||
progress.update(value=100,
|
||||
text=f"站点搜索完成,有效资源数:{len(results)},总耗时 {(end_time - start_time).seconds} 秒",
|
||||
key=ProgressKey.Search)
|
||||
text=f"站点搜索完成,有效资源数:{len(results)},总耗时 {(end_time - start_time).seconds} 秒")
|
||||
logger.info(f"站点搜索完成,有效资源数:{len(results)},总耗时 {(end_time - start_time).seconds} 秒")
|
||||
# 结束进度
|
||||
progress.end(ProgressKey.Search)
|
||||
progress.end()
|
||||
|
||||
# 返回
|
||||
return results
|
||||
|
||||
@@ -56,7 +56,7 @@ class SiteChain(ChainBase):
|
||||
if userdata:
|
||||
SiteOper().update_userdata(domain=StringUtils.get_url_domain(site.get("domain")),
|
||||
name=site.get("name"),
|
||||
payload=userdata.dict())
|
||||
payload=userdata.model_dump())
|
||||
# 发送事件
|
||||
eventmanager.send_event(EventType.SiteRefreshed, {
|
||||
"site_id": site.get("id")
|
||||
@@ -313,11 +313,16 @@ class SiteChain(ChainBase):
|
||||
siteoper = SiteOper()
|
||||
rsshelper = RssHelper()
|
||||
for domain, cookie in cookies.items():
|
||||
# 检查系统是否停止
|
||||
if global_vars.is_system_stopped:
|
||||
logger.info("系统正在停止,中断CookieCloud同步")
|
||||
return False, "系统正在停止,同步被中断"
|
||||
|
||||
# 索引器信息
|
||||
indexer = siteshelper.get_indexer(domain)
|
||||
# 数据库的站点信息
|
||||
site_info = siteoper.get_by_domain(domain)
|
||||
if site_info and site_info.is_active == 1:
|
||||
if site_info and site_info.is_active:
|
||||
# 站点已存在,检查站点连通性
|
||||
status, msg = self.test(domain)
|
||||
# 更新站点Cookie
|
||||
@@ -331,7 +336,7 @@ class SiteChain(ChainBase):
|
||||
cookie=cookie,
|
||||
ua=site_info.ua or settings.USER_AGENT,
|
||||
proxy=True if site_info.proxy else False,
|
||||
timeout=site_info.timeout
|
||||
timeout=site_info.timeout or 15
|
||||
)
|
||||
if rss_url:
|
||||
logger.info(f"更新站点 {domain} RSS地址 ...")
|
||||
|
||||
@@ -6,7 +6,6 @@ from app.chain import ChainBase
|
||||
from app.core.config import settings
|
||||
from app.helper.directory import DirectoryHelper
|
||||
from app.log import logger
|
||||
from app.schemas import MediaType
|
||||
|
||||
|
||||
class StorageChain(ChainBase):
|
||||
@@ -134,8 +133,7 @@ class StorageChain(ChainBase):
|
||||
"""
|
||||
return self.run_module("support_transtype", storage=storage)
|
||||
|
||||
def delete_media_file(self, fileitem: schemas.FileItem,
|
||||
mtype: MediaType = None, delete_self: bool = True) -> bool:
|
||||
def delete_media_file(self, fileitem: schemas.FileItem, delete_self: bool = True) -> bool:
|
||||
"""
|
||||
删除媒体文件,以及不含媒体文件的目录
|
||||
"""
|
||||
@@ -152,7 +150,8 @@ class StorageChain(ChainBase):
|
||||
return False
|
||||
|
||||
media_exts = settings.RMT_MEDIAEXT + settings.DOWNLOAD_TMPEXT
|
||||
if fileitem.path == "/" or len(Path(fileitem.path).parts) <= 2:
|
||||
fileitem_path = Path(fileitem.path) if fileitem.path else Path("")
|
||||
if len(fileitem_path.parts) <= 2:
|
||||
logger.warn(f"【{fileitem.storage}】{fileitem.path} 根目录或一级目录不允许删除")
|
||||
return False
|
||||
if fileitem.type == "dir":
|
||||
@@ -162,13 +161,7 @@ class StorageChain(ChainBase):
|
||||
if not self.delete_file(fileitem):
|
||||
logger.warn(f"【{fileitem.storage}】{fileitem.path} 删除失败")
|
||||
return False
|
||||
elif self.any_files(fileitem, extensions=media_exts) is False:
|
||||
logger.warn(f"【{fileitem.storage}】{fileitem.path} 不存在其它媒体文件,正在删除空目录")
|
||||
if not self.delete_file(fileitem):
|
||||
logger.warn(f"【{fileitem.storage}】{fileitem.path} 删除失败")
|
||||
return False
|
||||
# 不处理父目录
|
||||
return True
|
||||
|
||||
elif delete_self:
|
||||
# 本身是文件,需要删除文件
|
||||
logger.warn(f"正在删除文件【{fileitem.storage}】{fileitem.path}")
|
||||
@@ -176,35 +169,43 @@ class StorageChain(ChainBase):
|
||||
logger.warn(f"【{fileitem.storage}】{fileitem.path} 删除失败")
|
||||
return False
|
||||
|
||||
if mtype:
|
||||
# 重命名格式
|
||||
rename_format = settings.RENAME_FORMAT(mtype)
|
||||
media_path = DirectoryHelper.get_media_root_path(
|
||||
rename_format, rename_path=Path(fileitem.path)
|
||||
)
|
||||
if not media_path:
|
||||
return True
|
||||
# 处理媒体文件根目录
|
||||
dir_item = self.get_file_item(storage=fileitem.storage, path=media_path)
|
||||
else:
|
||||
# 处理上级目录
|
||||
dir_item = self.get_parent_item(fileitem)
|
||||
# 检查和删除上级空目录
|
||||
dir_item = fileitem if fileitem.type == "dir" else self.get_parent_item(fileitem)
|
||||
if not dir_item:
|
||||
logger.warn(f"【{fileitem.storage}】{fileitem.path} 上级目录不存在")
|
||||
return True
|
||||
|
||||
# 检查和删除上级目录
|
||||
if dir_item and len(Path(dir_item.path).parts) > 2:
|
||||
# 如何目录是所有下载目录、媒体库目录的上级,则不处理
|
||||
for d in DirectoryHelper().get_dirs():
|
||||
if d.download_path and Path(d.download_path).is_relative_to(Path(dir_item.path)):
|
||||
logger.debug(f"【{dir_item.storage}】{dir_item.path} 是下载目录本级或上级目录,不删除")
|
||||
return True
|
||||
if d.library_path and Path(d.library_path).is_relative_to(Path(dir_item.path)):
|
||||
logger.debug(f"【{dir_item.storage}】{dir_item.path} 是媒体库目录本级或上级目录,不删除")
|
||||
return True
|
||||
# 不存在其他媒体文件,删除空目录
|
||||
if self.any_files(dir_item, extensions=media_exts) is False:
|
||||
logger.warn(f"【{dir_item.storage}】{dir_item.path} 不存在其它媒体文件,正在删除空目录")
|
||||
if not self.delete_file(dir_item):
|
||||
logger.warn(f"【{dir_item.storage}】{dir_item.path} 删除失败")
|
||||
return False
|
||||
# 查找操作文件项匹配的配置目录(资源目录、媒体库目录)
|
||||
associated_dir = max(
|
||||
(
|
||||
Path(p)
|
||||
for d in DirectoryHelper().get_dirs()
|
||||
for p in (d.download_path, d.library_path)
|
||||
if p and fileitem_path.is_relative_to(p)
|
||||
),
|
||||
key=lambda path: len(path.parts),
|
||||
default=None,
|
||||
)
|
||||
|
||||
while dir_item and len(Path(dir_item.path).parts) > 2:
|
||||
# 目录是资源目录、媒体库目录的上级,则不处理
|
||||
if associated_dir and associated_dir.is_relative_to(Path(dir_item.path)):
|
||||
logger.debug(f"【{dir_item.storage}】{dir_item.path} 位于资源或媒体库目录结构中,不删除")
|
||||
break
|
||||
|
||||
elif not associated_dir and self.list_files(dir_item, recursion=False):
|
||||
logger.debug(f"【{dir_item.storage}】{dir_item.path} 不是空目录,不删除")
|
||||
break
|
||||
|
||||
if self.any_files(dir_item, extensions=media_exts) is not False:
|
||||
logger.debug(f"【{dir_item.storage}】{dir_item.path} 存在媒体文件,不删除")
|
||||
break
|
||||
|
||||
# 删除空目录并继续处理父目录
|
||||
logger.warn(f"【{dir_item.storage}】{dir_item.path} 不存在其它媒体文件,正在删除空目录")
|
||||
if not self.delete_file(dir_item):
|
||||
logger.warn(f"【{dir_item.storage}】{dir_item.path} 删除失败")
|
||||
return False
|
||||
dir_item = self.get_parent_item(dir_item)
|
||||
|
||||
return True
|
||||
|
||||
@@ -1184,6 +1184,42 @@ class SubscribeChain(ChainBase):
|
||||
logger.error(f'follow用户分享订阅 {title} 添加失败:{message}')
|
||||
logger.info(f'follow用户分享订阅刷新完成,共添加 {success_count} 个订阅')
|
||||
|
||||
async def cache_calendar(self):
|
||||
"""
|
||||
预缓存订阅日历,实际上就是查询一遍所有订阅的媒体信息
|
||||
前端请示是异常的,所以需要使用异步缓存方法
|
||||
"""
|
||||
logger.info(f'开始预缓存订阅日历 ...')
|
||||
for subscribe in await SubscribeOper().async_list():
|
||||
if global_vars.is_system_stopped:
|
||||
break
|
||||
try:
|
||||
mtype = MediaType(subscribe.type)
|
||||
except ValueError:
|
||||
logger.error(f'订阅 {subscribe.name} 类型错误:{subscribe.type}')
|
||||
continue
|
||||
# 识别媒体信息
|
||||
if mtype == MediaType.MOVIE:
|
||||
mediainfo: MediaInfo = await self.async_recognize_media(mtype=mtype,
|
||||
tmdbid=subscribe.tmdbid,
|
||||
doubanid=subscribe.doubanid,
|
||||
bangumiid=subscribe.bangumiid,
|
||||
episode_group=subscribe.episode_group,
|
||||
cache=False)
|
||||
if not mediainfo:
|
||||
logger.warn(
|
||||
f'未识别到媒体信息,标题:{subscribe.name},tmdbid:{subscribe.tmdbid},doubanid:{subscribe.doubanid}')
|
||||
continue
|
||||
else:
|
||||
episodes = await TmdbChain().async_tmdb_episodes(tmdbid=subscribe.tmdbid,
|
||||
season=subscribe.season,
|
||||
episode_group=subscribe.episode_group)
|
||||
if not episodes:
|
||||
logger.warn(
|
||||
f'未识别到季集信息,标题:{subscribe.name},tmdbid:{subscribe.tmdbid},豆瓣ID:{subscribe.doubanid},季:{subscribe.season}')
|
||||
continue
|
||||
logger.info(f'订阅日历预缓存完成')
|
||||
|
||||
@staticmethod
|
||||
def __update_subscribe_note(subscribe: Subscribe, downloads: Optional[List[Context]]):
|
||||
"""
|
||||
|
||||
@@ -150,7 +150,7 @@ class TorrentsChain(ChainBase):
|
||||
return []
|
||||
# 解析RSS
|
||||
rss_items = RssHelper().parse(site.get("rss"), True if site.get("proxy") else False,
|
||||
timeout=int(site.get("timeout") or 30))
|
||||
timeout=int(site.get("timeout") or 30), ua=site.get("ua") if site.get("ua") else None)
|
||||
if rss_items is None:
|
||||
# rss过期,尝试保留原配置生成新的rss
|
||||
self.__renew_rss_url(domain=domain, site=site)
|
||||
|
||||
@@ -33,6 +33,7 @@ from app.schemas.types import TorrentStatus, EventType, MediaType, ProgressKey,
|
||||
SystemConfigKey, ChainEventType, ContentType
|
||||
from app.utils.singleton import Singleton
|
||||
from app.utils.string import StringUtils
|
||||
from app.utils.system import SystemUtils
|
||||
|
||||
downloader_lock = threading.Lock()
|
||||
job_lock = threading.Lock()
|
||||
@@ -329,8 +330,12 @@ class JobManager:
|
||||
# 计算状态为完成的任务数
|
||||
if __mediaid__ not in self._job_view:
|
||||
return 0
|
||||
return sum([task.fileitem.size for task in self._job_view[__mediaid__].tasks if
|
||||
task.state == "completed" and task.fileitem.size is not None])
|
||||
return sum([
|
||||
task.fileitem.size if task.fileitem.size is not None
|
||||
else (SystemUtils.get_directory_size(Path(task.fileitem.path)) if task.fileitem.storage == "local" else 0)
|
||||
for task in self._job_view[__mediaid__].tasks
|
||||
if task.state == "completed"
|
||||
])
|
||||
|
||||
def total(self) -> int:
|
||||
"""
|
||||
@@ -501,7 +506,8 @@ class TransferChain(ChainBase, metaclass=Singleton):
|
||||
# 获取整理屏蔽词
|
||||
transfer_exclude_words = SystemConfigOper().get(SystemConfigKey.TransferExcludeWords)
|
||||
for t in tasks:
|
||||
if t.download_hash and self._can_delete_torrent(t.download_hash, t.downloader, transfer_exclude_words):
|
||||
if t.download_hash and self._can_delete_torrent(t.download_hash, t.downloader,
|
||||
transfer_exclude_words):
|
||||
if self.remove_torrents(t.download_hash, downloader=t.downloader):
|
||||
logger.info(f"移动模式删除种子成功:{t.download_hash}")
|
||||
if t.fileitem:
|
||||
@@ -554,8 +560,10 @@ class TransferChain(ChainBase, metaclass=Singleton):
|
||||
processed_num = 0
|
||||
# 失败数量
|
||||
fail_num = 0
|
||||
# 已完成文件
|
||||
finished_files = []
|
||||
|
||||
progress = ProgressHelper()
|
||||
progress = ProgressHelper(ProgressKey.FileTransfer)
|
||||
|
||||
while not global_vars.is_system_stopped:
|
||||
try:
|
||||
@@ -570,7 +578,7 @@ class TransferChain(ChainBase, metaclass=Singleton):
|
||||
if __queue_start:
|
||||
logger.info("开始整理队列处理...")
|
||||
# 启动进度
|
||||
progress.start(ProgressKey.FileTransfer)
|
||||
progress.start()
|
||||
# 重置计数
|
||||
processed_num = 0
|
||||
fail_num = 0
|
||||
@@ -578,8 +586,7 @@ class TransferChain(ChainBase, metaclass=Singleton):
|
||||
__process_msg = f"开始整理队列处理,当前共 {total_num} 个文件 ..."
|
||||
logger.info(__process_msg)
|
||||
progress.update(value=0,
|
||||
text=__process_msg,
|
||||
key=ProgressKey.FileTransfer)
|
||||
text=__process_msg)
|
||||
# 队列已开始
|
||||
__queue_start = False
|
||||
# 更新进度
|
||||
@@ -587,7 +594,10 @@ class TransferChain(ChainBase, metaclass=Singleton):
|
||||
logger.info(__process_msg)
|
||||
progress.update(value=processed_num / total_num * 100,
|
||||
text=__process_msg,
|
||||
key=ProgressKey.FileTransfer)
|
||||
data={
|
||||
"current": Path(fileitem.path).as_posix(),
|
||||
"finished": finished_files
|
||||
})
|
||||
# 整理
|
||||
state, err_msg = self.__handle_transfer(task=task, callback=item.callback)
|
||||
if not state:
|
||||
@@ -595,20 +605,20 @@ class TransferChain(ChainBase, metaclass=Singleton):
|
||||
fail_num += 1
|
||||
# 更新进度
|
||||
processed_num += 1
|
||||
finished_files.append(Path(fileitem.path).as_posix())
|
||||
__process_msg = f"{fileitem.name} 整理完成"
|
||||
logger.info(__process_msg)
|
||||
progress.update(value=processed_num / total_num * 100,
|
||||
progress.update(value=(processed_num / total_num) * 100,
|
||||
text=__process_msg,
|
||||
key=ProgressKey.FileTransfer)
|
||||
data={})
|
||||
except queue.Empty:
|
||||
if not __queue_start:
|
||||
# 结束进度
|
||||
__end_msg = f"整理队列处理完成,共整理 {processed_num} 个文件,失败 {fail_num} 个"
|
||||
logger.info(__end_msg)
|
||||
progress.update(value=100,
|
||||
text=__end_msg,
|
||||
key=ProgressKey.FileTransfer)
|
||||
progress.end(ProgressKey.FileTransfer)
|
||||
text=__end_msg)
|
||||
progress.end()
|
||||
# 重置计数
|
||||
processed_num = 0
|
||||
fail_num = 0
|
||||
@@ -1106,6 +1116,7 @@ class TransferChain(ChainBase, metaclass=Singleton):
|
||||
file_meta=file_meta)
|
||||
if begin_ep is not None:
|
||||
file_meta.begin_episode = begin_ep
|
||||
if part is not None:
|
||||
file_meta.part = part
|
||||
if end_ep is not None:
|
||||
file_meta.end_episode = end_ep
|
||||
@@ -1115,10 +1126,10 @@ class TransferChain(ChainBase, metaclass=Singleton):
|
||||
downloadhis = DownloadHistoryOper()
|
||||
if bluray_dir:
|
||||
# 蓝光原盘,按目录名查询
|
||||
download_history = downloadhis.get_by_path(str(file_path))
|
||||
download_history = downloadhis.get_by_path(file_path.as_posix())
|
||||
else:
|
||||
# 按文件全路径查询
|
||||
download_file = downloadhis.get_file_by_fullpath(str(file_path))
|
||||
download_file = downloadhis.get_file_by_fullpath(file_path.as_posix())
|
||||
if download_file:
|
||||
download_history = downloadhis.get_by_hash(download_file.download_hash)
|
||||
|
||||
@@ -1164,15 +1175,16 @@ class TransferChain(ChainBase, metaclass=Singleton):
|
||||
processed_num = 0
|
||||
# 失败数量
|
||||
fail_num = 0
|
||||
# 已完成文件
|
||||
finished_files = []
|
||||
|
||||
# 启动进度
|
||||
progress = ProgressHelper()
|
||||
progress.start(ProgressKey.FileTransfer)
|
||||
progress = ProgressHelper(ProgressKey.FileTransfer)
|
||||
progress.start()
|
||||
__process_msg = f"开始整理,共 {total_num} 个文件 ..."
|
||||
logger.info(__process_msg)
|
||||
progress.update(value=0,
|
||||
text=__process_msg,
|
||||
key=ProgressKey.FileTransfer)
|
||||
text=__process_msg)
|
||||
try:
|
||||
for transfer_task in transfer_tasks:
|
||||
if global_vars.is_system_stopped:
|
||||
@@ -1184,7 +1196,10 @@ class TransferChain(ChainBase, metaclass=Singleton):
|
||||
logger.info(__process_msg)
|
||||
progress.update(value=(processed_num + fail_num) / total_num * 100,
|
||||
text=__process_msg,
|
||||
key=ProgressKey.FileTransfer)
|
||||
data={
|
||||
"current": Path(transfer_task.fileitem.path).as_posix(),
|
||||
"finished": finished_files,
|
||||
})
|
||||
state, err_msg = self.__handle_transfer(
|
||||
task=transfer_task,
|
||||
callback=self.__default_callback
|
||||
@@ -1196,6 +1211,8 @@ class TransferChain(ChainBase, metaclass=Singleton):
|
||||
fail_num += 1
|
||||
else:
|
||||
processed_num += 1
|
||||
# 记录已完成
|
||||
finished_files.append(Path(transfer_task.fileitem.path).as_posix())
|
||||
finally:
|
||||
transfer_tasks.clear()
|
||||
del transfer_tasks
|
||||
@@ -1205,8 +1222,8 @@ class TransferChain(ChainBase, metaclass=Singleton):
|
||||
logger.info(__end_msg)
|
||||
progress.update(value=100,
|
||||
text=__end_msg,
|
||||
key=ProgressKey.FileTransfer)
|
||||
progress.end(ProgressKey.FileTransfer)
|
||||
data={})
|
||||
progress.end()
|
||||
|
||||
error_msg = "、".join(err_msgs[:2]) + (f",等{len(err_msgs)}个文件错误!" if len(err_msgs) > 2 else "")
|
||||
return all_success, error_msg
|
||||
@@ -1351,12 +1368,7 @@ class TransferChain(ChainBase, metaclass=Singleton):
|
||||
else:
|
||||
# 更新媒体图片
|
||||
self.obtain_images(mediainfo=mediainfo)
|
||||
# 开始进度
|
||||
progress = ProgressHelper()
|
||||
progress.start(ProgressKey.FileTransfer)
|
||||
progress.update(value=0,
|
||||
text=f"开始整理 {fileitem.path} ...",
|
||||
key=ProgressKey.FileTransfer)
|
||||
|
||||
# 开始整理
|
||||
state, errmsg = self.do_transfer(
|
||||
fileitem=fileitem,
|
||||
@@ -1377,7 +1389,6 @@ class TransferChain(ChainBase, metaclass=Singleton):
|
||||
if not state:
|
||||
return False, errmsg
|
||||
|
||||
progress.end(ProgressKey.FileTransfer)
|
||||
logger.info(f"{fileitem.path} 整理完成")
|
||||
return True, ""
|
||||
else:
|
||||
@@ -1431,11 +1442,10 @@ class TransferChain(ChainBase, metaclass=Singleton):
|
||||
|
||||
for keyword in exclude_words:
|
||||
if keyword and re.search(r"%s" % keyword, file_path, re.IGNORECASE):
|
||||
logger.debug(f"{file_path} 命中屏蔽词 {keyword}")
|
||||
logger.warn(f"{file_path} 命中屏蔽词 {keyword}")
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def _can_delete_torrent(self, download_hash: str, downloader: str, transfer_exclude_words) -> bool:
|
||||
"""
|
||||
检查是否可以删除种子文件
|
||||
@@ -1467,13 +1477,9 @@ class TransferChain(ChainBase, metaclass=Singleton):
|
||||
for file in torrent_files:
|
||||
file_path = save_path / file.name
|
||||
# 如果存在未被屏蔽的媒体文件,则不删除种子
|
||||
if (
|
||||
file_path.suffix in self.all_exts
|
||||
and not self._is_blocked_by_exclude_words(
|
||||
str(file_path), transfer_exclude_words
|
||||
)
|
||||
and file_path.exists()
|
||||
):
|
||||
if (file_path.suffix in self.all_exts
|
||||
and not self._is_blocked_by_exclude_words(file_path.as_posix(), transfer_exclude_words)
|
||||
and file_path.exists()):
|
||||
return False
|
||||
|
||||
# 所有媒体文件都被屏蔽或不存在,可以删除种子
|
||||
|
||||
@@ -11,7 +11,7 @@ from pydantic.fields import Callable
|
||||
from app.chain import ChainBase
|
||||
from app.core.config import global_vars
|
||||
from app.core.event import Event, eventmanager
|
||||
from app.core.workflow import WorkFlowManager
|
||||
from app.workflow import WorkFlowManager
|
||||
from app.db.models import Workflow
|
||||
from app.db.workflow_oper import WorkflowOper
|
||||
from app.log import logger
|
||||
@@ -180,7 +180,7 @@ class WorkflowExecutor:
|
||||
"""
|
||||
合并上下文
|
||||
"""
|
||||
for key, value in context.dict().items():
|
||||
for key, value in context.model_dump().items():
|
||||
if not getattr(self.context, key, None):
|
||||
setattr(self.context, key, value)
|
||||
|
||||
|
||||
@@ -215,7 +215,7 @@ class Command(metaclass=Singleton):
|
||||
except Exception as e:
|
||||
logger.error(f"Error occurred during command initialization in background: {e}", exc_info=True)
|
||||
|
||||
def __trigger_register_commands_event(self) -> (Optional[Event], dict):
|
||||
def __trigger_register_commands_event(self) -> tuple[Optional[Event], dict]:
|
||||
"""
|
||||
触发事件,允许调整命令数据
|
||||
"""
|
||||
|
||||
1433
app/core/cache.py
1433
app/core/cache.py
File diff suppressed because it is too large
Load Diff
@@ -11,7 +11,8 @@ from typing import Any, Dict, List, Optional, Tuple, Type
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from dotenv import set_key
|
||||
from pydantic import BaseModel, BaseSettings, validator, Field
|
||||
from pydantic import BaseModel, Field, ConfigDict, model_validator
|
||||
from pydantic_settings import BaseSettings, SettingsConfigDict
|
||||
|
||||
from app.log import logger, log_settings, LogConfigModel
|
||||
from app.schemas import MediaType
|
||||
@@ -42,10 +43,6 @@ class SystemConfModel(BaseModel):
|
||||
scheduler: int = 0
|
||||
# 线程池大小
|
||||
threadpool: int = 0
|
||||
# 数据库连接池大小
|
||||
dbpool: int = 0
|
||||
# 数据库连接池溢出数量
|
||||
dbpooloverflow: int = 0
|
||||
|
||||
|
||||
class ConfigModel(BaseModel):
|
||||
@@ -53,9 +50,9 @@ class ConfigModel(BaseModel):
|
||||
Pydantic 配置模型,描述所有配置项及其类型和默认值
|
||||
"""
|
||||
|
||||
class Config:
|
||||
extra = "ignore" # 忽略未定义的配置项
|
||||
model_config = ConfigDict(extra="ignore") # 忽略未定义的配置项
|
||||
|
||||
# ==================== 基础应用配置 ====================
|
||||
# 项目名称
|
||||
PROJECT_NAME: str = "MoviePilot"
|
||||
# 域名 格式;https://movie-pilot.org
|
||||
@@ -64,6 +61,24 @@ class ConfigModel(BaseModel):
|
||||
API_V1_STR: str = "/api/v1"
|
||||
# 前端资源路径
|
||||
FRONTEND_PATH: str = "/public"
|
||||
# 时区
|
||||
TZ: str = "Asia/Shanghai"
|
||||
# API监听地址
|
||||
HOST: str = "0.0.0.0"
|
||||
# API监听端口
|
||||
PORT: int = 3001
|
||||
# 前端监听端口
|
||||
NGINX_PORT: int = 3000
|
||||
# 配置文件目录
|
||||
CONFIG_DIR: Optional[str] = None
|
||||
# 是否调试模式
|
||||
DEBUG: bool = False
|
||||
# 是否开发模式
|
||||
DEV: bool = False
|
||||
# 高级设置模式
|
||||
ADVANCED_MODE: bool = True
|
||||
|
||||
# ==================== 安全认证配置 ====================
|
||||
# 密钥
|
||||
SECRET_KEY: str = secrets.token_urlsafe(32)
|
||||
# RESOURCE密钥
|
||||
@@ -74,20 +89,26 @@ class ConfigModel(BaseModel):
|
||||
ACCESS_TOKEN_EXPIRE_MINUTES: int = 60 * 24 * 8
|
||||
# RESOURCE_TOKEN过期时间
|
||||
RESOURCE_ACCESS_TOKEN_EXPIRE_SECONDS: int = 60 * 30
|
||||
# 时区
|
||||
TZ: str = "Asia/Shanghai"
|
||||
# API监听地址
|
||||
HOST: str = "0.0.0.0"
|
||||
# API监听端口
|
||||
PORT: int = 3001
|
||||
# 前端监听端口
|
||||
NGINX_PORT: int = 3000
|
||||
# 是否调试模式
|
||||
DEBUG: bool = False
|
||||
# 是否开发模式
|
||||
DEV: bool = False
|
||||
# 超级管理员初始用户名
|
||||
SUPERUSER: str = "admin"
|
||||
# 超级管理员初始密码
|
||||
SUPERUSER_PASSWORD: Optional[str] = None
|
||||
# 辅助认证,允许通过外部服务进行认证、单点登录以及自动创建用户
|
||||
AUXILIARY_AUTH_ENABLE: bool = False
|
||||
# API密钥,需要更换
|
||||
API_TOKEN: Optional[str] = None
|
||||
# 用户认证站点
|
||||
AUTH_SITE: str = ""
|
||||
|
||||
# ==================== 数据库配置 ====================
|
||||
# 数据库类型,支持 sqlite 和 postgresql,默认使用 sqlite
|
||||
DB_TYPE: str = "sqlite"
|
||||
# 是否在控制台输出 SQL 语句,默认关闭
|
||||
DB_ECHO: bool = False
|
||||
# 数据库连接超时时间(秒),默认为 60 秒
|
||||
DB_TIMEOUT: int = 60
|
||||
# 是否启用 WAL 模式,仅适用于SQLite,默认开启
|
||||
DB_WAL_ENABLE: bool = True
|
||||
# 数据库连接池类型,QueuePool, NullPool
|
||||
DB_POOL_TYPE: str = "QueuePool"
|
||||
# 是否在获取连接时进行预先 ping 操作
|
||||
@@ -96,71 +117,44 @@ class ConfigModel(BaseModel):
|
||||
DB_POOL_RECYCLE: int = 300
|
||||
# 数据库连接池获取连接的超时时间(秒)
|
||||
DB_POOL_TIMEOUT: int = 30
|
||||
# SQLite 的 busy_timeout 参数,默认为 60 秒
|
||||
DB_TIMEOUT: int = 60
|
||||
# SQLite 是否启用 WAL 模式,默认开启
|
||||
DB_WAL_ENABLE: bool = True
|
||||
# SQLite 连接池大小
|
||||
DB_SQLITE_POOL_SIZE: int = 10
|
||||
# SQLite 连接池溢出数量
|
||||
DB_SQLITE_MAX_OVERFLOW: int = 50
|
||||
# PostgreSQL 主机地址
|
||||
DB_POSTGRESQL_HOST: str = "localhost"
|
||||
# PostgreSQL 端口
|
||||
DB_POSTGRESQL_PORT: int = 5432
|
||||
# PostgreSQL 数据库名
|
||||
DB_POSTGRESQL_DATABASE: str = "moviepilot"
|
||||
# PostgreSQL 用户名
|
||||
DB_POSTGRESQL_USERNAME: str = "moviepilot"
|
||||
# PostgreSQL 密码
|
||||
DB_POSTGRESQL_PASSWORD: str = "moviepilot"
|
||||
# PostgreSQL 连接池大小
|
||||
DB_POSTGRESQL_POOL_SIZE: int = 10
|
||||
# PostgreSQL 连接池溢出数量
|
||||
DB_POSTGRESQL_MAX_OVERFLOW: int = 50
|
||||
|
||||
# ==================== 缓存配置 ====================
|
||||
# 缓存类型,支持 cachetools 和 redis,默认使用 cachetools
|
||||
CACHE_BACKEND_TYPE: str = "cachetools"
|
||||
# 缓存连接字符串,仅外部缓存(如 Redis、Memcached)需要
|
||||
CACHE_BACKEND_URL: Optional[str] = None
|
||||
CACHE_BACKEND_URL: Optional[str] = "redis://localhost:6379"
|
||||
# Redis 缓存最大内存限制,未配置时,如开启大内存模式时为 "1024mb",未开启时为 "256mb"
|
||||
CACHE_REDIS_MAXMEMORY: Optional[str] = None
|
||||
# 配置文件目录
|
||||
CONFIG_DIR: Optional[str] = None
|
||||
# 超级管理员
|
||||
SUPERUSER: str = "admin"
|
||||
# 辅助认证,允许通过外部服务进行认证、单点登录以及自动创建用户
|
||||
AUXILIARY_AUTH_ENABLE: bool = False
|
||||
# API密钥,需要更换
|
||||
API_TOKEN: Optional[str] = None
|
||||
# 全局图片缓存,将媒体图片缓存到本地
|
||||
GLOBAL_IMAGE_CACHE: bool = False
|
||||
# 全局图片缓存保留天数
|
||||
GLOBAL_IMAGE_CACHE_DAYS: int = 7
|
||||
# 临时文件保留天数
|
||||
TEMP_FILE_DAYS: int = 3
|
||||
# 元数据识别缓存过期时间(小时),0为自动
|
||||
META_CACHE_EXPIRE: int = 0
|
||||
|
||||
# ==================== 网络代理配置 ====================
|
||||
# 网络代理服务器地址
|
||||
PROXY_HOST: Optional[str] = None
|
||||
# 登录页面电影海报,tmdb/bing/mediaserver
|
||||
WALLPAPER: str = "tmdb"
|
||||
# 自定义壁纸api地址
|
||||
CUSTOMIZE_WALLPAPER_API_URL: Optional[str] = None
|
||||
# 媒体搜索来源 themoviedb/douban/bangumi,多个用,分隔
|
||||
SEARCH_SOURCE: str = "themoviedb,douban,bangumi"
|
||||
# 媒体识别来源 themoviedb/douban
|
||||
RECOGNIZE_SOURCE: str = "themoviedb"
|
||||
# 刮削来源 themoviedb/douban
|
||||
SCRAP_SOURCE: str = "themoviedb"
|
||||
# 新增已入库媒体是否跟随TMDB信息变化
|
||||
SCRAP_FOLLOW_TMDB: bool = True
|
||||
# TMDB图片地址
|
||||
TMDB_IMAGE_DOMAIN: str = "image.tmdb.org"
|
||||
# TMDB API地址
|
||||
TMDB_API_DOMAIN: str = "api.themoviedb.org"
|
||||
# TMDB元数据语言
|
||||
TMDB_LOCALE: str = "zh"
|
||||
# 刮削使用TMDB原始语种图片
|
||||
TMDB_SCRAP_ORIGINAL_IMAGE: bool = False
|
||||
# TMDB API Key
|
||||
TMDB_API_KEY: str = "db55323b8d3e4154498498a75642b381"
|
||||
# TVDB API Key
|
||||
TVDB_V4_API_KEY: str = "ed2aa66b-7899-4677-92a7-67bc9ce3d93a"
|
||||
TVDB_V4_API_PIN: str = ""
|
||||
# Fanart开关
|
||||
FANART_ENABLE: bool = True
|
||||
# Fanart语言
|
||||
FANART_LANG: str = "zh,en"
|
||||
# Fanart API Key
|
||||
FANART_API_KEY: str = "d2d31f9ecabea050fc7d68aa3146015f"
|
||||
# 115 AppId
|
||||
U115_APP_ID: str = "100196807"
|
||||
# Alipan AppId
|
||||
ALIPAN_APP_ID: str = "ac1bf04dc9fd4d9aaabb65b4a668d403"
|
||||
# 元数据识别缓存过期时间(小时)
|
||||
META_CACHE_EXPIRE: int = 0
|
||||
# 电视剧动漫的分类genre_ids
|
||||
ANIME_GENREIDS: List[int] = Field(default=[16])
|
||||
# 用户认证站点
|
||||
AUTH_SITE: str = ""
|
||||
# 重启自动升级
|
||||
MOVIEPILOT_AUTO_UPDATE: str = 'release'
|
||||
# 自动检查和更新站点资源包(站点索引、认证等)
|
||||
AUTO_UPDATE_RESOURCE: bool = True
|
||||
# 是否启用DOH解析域名
|
||||
DOH_ENABLE: bool = False
|
||||
# 使用 DOH 解析的域名列表
|
||||
@@ -174,6 +168,55 @@ class ConfigModel(BaseModel):
|
||||
"api.telegram.org")
|
||||
# DOH 解析服务器列表
|
||||
DOH_RESOLVERS: str = "1.0.0.1,1.1.1.1,9.9.9.9,149.112.112.112"
|
||||
|
||||
# ==================== 媒体元数据配置 ====================
|
||||
# 媒体搜索来源 themoviedb/douban/bangumi,多个用,分隔
|
||||
SEARCH_SOURCE: str = "themoviedb"
|
||||
# 媒体识别来源 themoviedb/douban
|
||||
RECOGNIZE_SOURCE: str = "themoviedb"
|
||||
# 刮削来源 themoviedb/douban
|
||||
SCRAP_SOURCE: str = "themoviedb"
|
||||
# 电视剧动漫的分类genre_ids
|
||||
ANIME_GENREIDS: List[int] = Field(default=[16])
|
||||
|
||||
# ==================== TMDB配置 ====================
|
||||
# TMDB图片地址
|
||||
TMDB_IMAGE_DOMAIN: str = "image.tmdb.org"
|
||||
# TMDB API地址
|
||||
TMDB_API_DOMAIN: str = "api.themoviedb.org"
|
||||
# TMDB元数据语言
|
||||
TMDB_LOCALE: str = "zh"
|
||||
# 刮削使用TMDB原始语种图片
|
||||
TMDB_SCRAP_ORIGINAL_IMAGE: bool = False
|
||||
# TMDB API Key
|
||||
TMDB_API_KEY: str = "db55323b8d3e4154498498a75642b381"
|
||||
|
||||
# ==================== TVDB配置 ====================
|
||||
# TVDB API Key
|
||||
TVDB_V4_API_KEY: str = "ed2aa66b-7899-4677-92a7-67bc9ce3d93a"
|
||||
TVDB_V4_API_PIN: str = ""
|
||||
|
||||
# ==================== Fanart配置 ====================
|
||||
# Fanart开关
|
||||
FANART_ENABLE: bool = True
|
||||
# Fanart语言
|
||||
FANART_LANG: str = "zh,en"
|
||||
# Fanart API Key
|
||||
FANART_API_KEY: str = "d2d31f9ecabea050fc7d68aa3146015f"
|
||||
|
||||
# ==================== 云盘配置 ====================
|
||||
# 115 AppId
|
||||
U115_APP_ID: str = "100196807"
|
||||
# Alipan AppId
|
||||
ALIPAN_APP_ID: str = "ac1bf04dc9fd4d9aaabb65b4a668d403"
|
||||
|
||||
# ==================== 系统升级配置 ====================
|
||||
# 重启自动升级
|
||||
MOVIEPILOT_AUTO_UPDATE: str = 'release'
|
||||
# 自动检查和更新站点资源包(站点索引、认证等)
|
||||
AUTO_UPDATE_RESOURCE: bool = True
|
||||
|
||||
# ==================== 媒体文件格式配置 ====================
|
||||
# 支持的后缀格式
|
||||
RMT_MEDIAEXT: list = Field(
|
||||
default_factory=lambda: ['.mp4', '.mkv', '.ts', '.iso',
|
||||
@@ -196,10 +239,12 @@ class ConfigModel(BaseModel):
|
||||
'.aifc', '.aiff', '.alac', '.adif', '.adts',
|
||||
'.flac', '.midi', '.opus', '.sfalc']
|
||||
)
|
||||
# 下载器临时文件后缀
|
||||
DOWNLOAD_TMPEXT: list = Field(default_factory=lambda: ['.!qb', '.part'])
|
||||
|
||||
# ==================== 媒体服务器配置 ====================
|
||||
# 媒体服务器同步间隔(小时)
|
||||
MEDIASERVER_SYNC_INTERVAL: int = 6
|
||||
|
||||
# ==================== 订阅配置 ====================
|
||||
# 订阅模式
|
||||
SUBSCRIBE_MODE: str = "spider"
|
||||
# RSS订阅模式刷新时间间隔(分钟)
|
||||
@@ -208,24 +253,42 @@ class ConfigModel(BaseModel):
|
||||
SUBSCRIBE_STATISTIC_SHARE: bool = True
|
||||
# 订阅搜索开关
|
||||
SUBSCRIBE_SEARCH: bool = False
|
||||
# 订阅搜索时间间隔(小时)
|
||||
SUBSCRIBE_SEARCH_INTERVAL: int = 24
|
||||
# 检查本地媒体库是否存在资源开关
|
||||
LOCAL_EXISTS_SEARCH: bool = False
|
||||
# 搜索多个名称
|
||||
SEARCH_MULTIPLE_NAME: bool = False
|
||||
# 最大搜索名称数量
|
||||
MAX_SEARCH_NAME_LIMIT: int = 2
|
||||
LOCAL_EXISTS_SEARCH: bool = True
|
||||
|
||||
# ==================== 站点配置 ====================
|
||||
# 站点数据刷新间隔(小时)
|
||||
SITEDATA_REFRESH_INTERVAL: int = 6
|
||||
# 读取和发送站点消息
|
||||
SITE_MESSAGE: bool = True
|
||||
# 不能缓存站点资源的站点域名,多个使用,分隔
|
||||
NO_CACHE_SITE_KEY: str = "m-team"
|
||||
# OCR服务器地址,用于识别站点验证码
|
||||
OCR_HOST: str = "https://movie-pilot.org"
|
||||
# 仿真类型:playwright 或 flaresolverr
|
||||
BROWSER_EMULATION: str = "playwright"
|
||||
# FlareSolverr 服务地址,例如 http://127.0.0.1:8191
|
||||
FLARESOLVERR_URL: Optional[str] = None
|
||||
|
||||
# ==================== 搜索配置 ====================
|
||||
# 搜索多个名称
|
||||
SEARCH_MULTIPLE_NAME: bool = False
|
||||
# 最大搜索名称数量
|
||||
MAX_SEARCH_NAME_LIMIT: int = 2
|
||||
|
||||
# ==================== 下载配置 ====================
|
||||
# 种子标签
|
||||
TORRENT_TAG: str = "MOVIEPILOT"
|
||||
# 下载站点字幕
|
||||
DOWNLOAD_SUBTITLE: bool = True
|
||||
# 交互搜索自动下载用户ID,使用,分割
|
||||
AUTO_DOWNLOAD_USER: Optional[str] = None
|
||||
# 下载器临时文件后缀
|
||||
DOWNLOAD_TMPEXT: list = Field(default_factory=lambda: ['.!qb', '.part'])
|
||||
|
||||
# ==================== CookieCloud配置 ====================
|
||||
# CookieCloud是否启动本地服务
|
||||
COOKIECLOUD_ENABLE_LOCAL: Optional[bool] = False
|
||||
# CookieCloud服务器地址
|
||||
@@ -238,6 +301,8 @@ class ConfigModel(BaseModel):
|
||||
COOKIECLOUD_INTERVAL: Optional[int] = 60 * 24
|
||||
# CookieCloud同步黑名单,多个域名,分割
|
||||
COOKIECLOUD_BLACKLIST: Optional[str] = None
|
||||
|
||||
# ==================== 整理配置 ====================
|
||||
# 电影重命名格式
|
||||
MOVIE_RENAME_FORMAT: str = "{{title}}{% if year %} ({{year}}){% endif %}" \
|
||||
"/{{title}}{% if year %} ({{year}}){% endif %}{% if part %}-{{part}}{% endif %}{% if videoFormat %} - {{videoFormat}}{% endif %}" \
|
||||
@@ -247,10 +312,24 @@ class ConfigModel(BaseModel):
|
||||
"/Season {{season}}" \
|
||||
"/{{title}} - {{season_episode}}{% if part %}-{{part}}{% endif %}{% if episode %} - 第 {{episode}} 集{% endif %}" \
|
||||
"{{fileExt}}"
|
||||
# OCR服务器地址
|
||||
OCR_HOST: str = "https://movie-pilot.org"
|
||||
# 重命名时支持的S0别名
|
||||
RENAME_FORMAT_S0_NAMES: list = Field(default=["Specials", "SPs"])
|
||||
# 为指定默认字幕添加.default后缀
|
||||
DEFAULT_SUB: Optional[str] = "zh-cn"
|
||||
# 新增已入库媒体是否跟随TMDB信息变化
|
||||
SCRAP_FOLLOW_TMDB: bool = True
|
||||
|
||||
# ==================== 服务地址配置 ====================
|
||||
# 服务器地址,对应 https://github.com/jxxghp/MoviePilot-Server 项目
|
||||
MP_SERVER_HOST: str = "https://movie-pilot.org"
|
||||
|
||||
# ==================== 个性化 ====================
|
||||
# 登录页面电影海报,tmdb/bing/mediaserver
|
||||
WALLPAPER: str = "tmdb"
|
||||
# 自定义壁纸api地址
|
||||
CUSTOMIZE_WALLPAPER_API_URL: Optional[str] = None
|
||||
|
||||
# ==================== 插件配置 ====================
|
||||
# 插件市场仓库地址,多个地址使用,分隔,地址以/结尾
|
||||
PLUGIN_MARKET: str = ("https://github.com/jxxghp/MoviePilot-Plugins,"
|
||||
"https://github.com/thsrite/MoviePilot-Plugins,"
|
||||
@@ -271,6 +350,8 @@ class ConfigModel(BaseModel):
|
||||
PLUGIN_STATISTIC_SHARE: bool = True
|
||||
# 是否开启插件热加载
|
||||
PLUGIN_AUTO_RELOAD: bool = False
|
||||
|
||||
# ==================== Github & PIP ====================
|
||||
# Github token,提高请求api限流阈值 ghp_****
|
||||
GITHUB_TOKEN: Optional[str] = None
|
||||
# Github代理服务器,格式:https://mirror.ghproxy.com/
|
||||
@@ -279,16 +360,18 @@ class ConfigModel(BaseModel):
|
||||
PIP_PROXY: Optional[str] = ''
|
||||
# 指定的仓库Github token,多个仓库使用,分隔,格式:{user1}/{repo1}:ghp_****,{user2}/{repo2}:github_pat_****
|
||||
REPO_GITHUB_TOKEN: Optional[str] = None
|
||||
|
||||
# ==================== 性能配置 ====================
|
||||
# 大内存模式
|
||||
BIG_MEMORY_MODE: bool = False
|
||||
# FastApi性能监控
|
||||
PERFORMANCE_MONITOR_ENABLE: bool = False
|
||||
# 全局图片缓存,将媒体图片缓存到本地
|
||||
GLOBAL_IMAGE_CACHE: bool = False
|
||||
# 是否启用编码探测的性能模式
|
||||
ENCODING_DETECTION_PERFORMANCE_MODE: bool = True
|
||||
# 编码探测的最低置信度阈值
|
||||
ENCODING_DETECTION_MIN_CONFIDENCE: float = 0.8
|
||||
# 主动内存回收时间间隔(分钟),0为不启用
|
||||
MEMORY_GC_INTERVAL: int = 30
|
||||
|
||||
# ==================== 安全配置 ====================
|
||||
# 允许的图片缓存域名
|
||||
SECURITY_IMAGE_DOMAINS: list = Field(default=[
|
||||
"image.tmdb.org",
|
||||
@@ -308,22 +391,46 @@ class ConfigModel(BaseModel):
|
||||
])
|
||||
# 允许的图片文件后缀格式
|
||||
SECURITY_IMAGE_SUFFIXES: list = Field(default=[".jpg", ".jpeg", ".png", ".webp", ".gif", ".svg", ".avif"])
|
||||
# 重命名时支持的S0别名
|
||||
RENAME_FORMAT_S0_NAMES: list = Field(default=["Specials", "SPs"])
|
||||
# 为指定默认字幕添加.default后缀
|
||||
DEFAULT_SUB: Optional[str] = "zh-cn"
|
||||
# Docker Client API地址
|
||||
DOCKER_CLIENT_API: Optional[str] = "tcp://127.0.0.1:38379"
|
||||
|
||||
# ==================== 工作流配置 ====================
|
||||
# 工作流数据共享
|
||||
WORKFLOW_STATISTIC_SHARE: bool = True
|
||||
|
||||
# ==================== 存储配置 ====================
|
||||
# 对rclone进行快照对比时,是否检查文件夹的修改时间
|
||||
RCLONE_SNAPSHOT_CHECK_FOLDER_MODTIME = True
|
||||
RCLONE_SNAPSHOT_CHECK_FOLDER_MODTIME: bool = True
|
||||
# 对OpenList进行快照对比时,是否检查文件夹的修改时间
|
||||
OPENLIST_SNAPSHOT_CHECK_FOLDER_MODTIME = True
|
||||
# 仿真类型:playwright 或 flaresolverr
|
||||
BROWSER_EMULATION: str = "playwright"
|
||||
# FlareSolverr 服务地址,例如 http://127.0.0.1:8191
|
||||
FLARESOLVERR_URL: Optional[str] = None
|
||||
OPENLIST_SNAPSHOT_CHECK_FOLDER_MODTIME: bool = True
|
||||
|
||||
# ==================== Docker配置 ====================
|
||||
# Docker Client API地址
|
||||
DOCKER_CLIENT_API: Optional[str] = "tcp://127.0.0.1:38379"
|
||||
|
||||
# ==================== AI智能体配置 ====================
|
||||
# AI智能体开关
|
||||
AI_AGENT_ENABLE: bool = False
|
||||
# LLM提供商 (openai/google/deepseek)
|
||||
LLM_PROVIDER: str = "deepseek"
|
||||
# LLM模型名称
|
||||
LLM_MODEL: str = "deepseek-chat"
|
||||
# LLM API密钥
|
||||
LLM_API_KEY: Optional[str] = None
|
||||
# LLM基础URL(用于自定义API端点)
|
||||
LLM_BASE_URL: Optional[str] = "https://api.deepseek.com"
|
||||
# LLM温度参数
|
||||
LLM_TEMPERATURE: float = 0.1
|
||||
# LLM最大迭代次数
|
||||
LLM_MAX_ITERATIONS: int = 15
|
||||
# LLM工具调用超时时间(秒)
|
||||
LLM_TOOL_TIMEOUT: int = 300
|
||||
# 是否启用详细日志
|
||||
LLM_VERBOSE: bool = False
|
||||
# 最大记忆消息数量
|
||||
LLM_MAX_MEMORY_MESSAGES: int = 50
|
||||
# 记忆保留天数
|
||||
LLM_MEMORY_RETENTION_DAYS: int = 30
|
||||
# Redis记忆保留天数(如果使用Redis)
|
||||
LLM_REDIS_MEMORY_RETENTION_DAYS: int = 7
|
||||
|
||||
|
||||
class Settings(BaseSettings, ConfigModel, LogConfigModel):
|
||||
@@ -331,10 +438,11 @@ class Settings(BaseSettings, ConfigModel, LogConfigModel):
|
||||
系统配置类
|
||||
"""
|
||||
|
||||
class Config:
|
||||
case_sensitive = True
|
||||
env_file = SystemUtils.get_env_path()
|
||||
env_file_encoding = "utf-8"
|
||||
model_config = SettingsConfigDict(
|
||||
case_sensitive=True,
|
||||
env_file=SystemUtils.get_env_path(),
|
||||
env_file_encoding="utf-8",
|
||||
)
|
||||
|
||||
def __init__(self, **kwargs):
|
||||
super().__init__(**kwargs)
|
||||
@@ -431,33 +539,54 @@ class Settings(BaseSettings, ConfigModel, LogConfigModel):
|
||||
f"配置项 '{field_name}' 的值 '{value}' 无法转换成正确的类型,使用默认值 '{default}',错误信息: {e}")
|
||||
return default, True
|
||||
|
||||
@validator('*', pre=True, always=True)
|
||||
def generic_type_validator(cls, value: Any, field): # noqa
|
||||
@model_validator(mode='before')
|
||||
@classmethod
|
||||
def generic_type_validator(cls, data: Any): # noqa
|
||||
"""
|
||||
通用校验器,尝试将配置值转换为期望的类型
|
||||
"""
|
||||
if field.name == "API_TOKEN":
|
||||
converted_value, needs_update = cls.validate_api_token(value, value)
|
||||
else:
|
||||
converted_value, needs_update = cls.generic_type_converter(value, value, field.type_, field.default,
|
||||
field.name)
|
||||
if needs_update:
|
||||
cls.update_env_config(field, value, converted_value)
|
||||
return converted_value
|
||||
if not isinstance(data, dict):
|
||||
return data
|
||||
|
||||
# 处理 API_TOKEN 特殊验证
|
||||
if 'API_TOKEN' in data:
|
||||
converted_value, needs_update = cls.validate_api_token(data['API_TOKEN'], data['API_TOKEN'])
|
||||
if needs_update:
|
||||
cls.update_env_config("API_TOKEN", data["API_TOKEN"], converted_value)
|
||||
data['API_TOKEN'] = converted_value
|
||||
|
||||
# 对其他字段进行类型转换
|
||||
for field_name, field_info in cls.model_fields.items():
|
||||
if field_name not in data:
|
||||
continue
|
||||
value = data[field_name]
|
||||
if value is None:
|
||||
continue
|
||||
|
||||
field = cls.model_fields.get(field_name)
|
||||
if field:
|
||||
converted_value, needs_update = cls.generic_type_converter(
|
||||
value, value, field.annotation, field.default, field_name
|
||||
)
|
||||
if needs_update:
|
||||
cls.update_env_config(field_name, value, converted_value)
|
||||
data[field_name] = converted_value
|
||||
|
||||
return data
|
||||
|
||||
@staticmethod
|
||||
def update_env_config(field: Any, original_value: Any, converted_value: Any) -> Tuple[bool, str]:
|
||||
def update_env_config(field_name: str, original_value: Any, converted_value: Any) -> Tuple[bool, str]:
|
||||
"""
|
||||
更新 env 配置
|
||||
"""
|
||||
message = None
|
||||
is_converted = original_value is not None and str(original_value) != str(converted_value)
|
||||
if is_converted:
|
||||
message = f"配置项 '{field.name}' 的值 '{original_value}' 无效,已替换为 '{converted_value}'"
|
||||
message = f"配置项 '{field_name}' 的值 '{original_value}' 无效,已替换为 '{converted_value}'"
|
||||
logger.warning(message)
|
||||
|
||||
if field.name in os.environ:
|
||||
message = f"配置项 '{field.name}' 已在环境变量中设置,请手动更新以保持一致性"
|
||||
if field_name in os.environ:
|
||||
message = f"配置项 '{field_name}' 已在环境变量中设置,请手动更新以保持一致性"
|
||||
logger.warning(message)
|
||||
return False, message
|
||||
else:
|
||||
@@ -467,10 +596,10 @@ class Settings(BaseSettings, ConfigModel, LogConfigModel):
|
||||
else:
|
||||
value_to_write = str(converted_value) if converted_value is not None else ""
|
||||
|
||||
set_key(dotenv_path=SystemUtils.get_env_path(), key_to_set=field.name, value_to_set=value_to_write,
|
||||
set_key(dotenv_path=SystemUtils.get_env_path(), key_to_set=field_name, value_to_set=value_to_write,
|
||||
quote_mode="always")
|
||||
if is_converted:
|
||||
logger.info(f"配置项 '{field.name}' 已自动修正并写入到 'app.env' 文件")
|
||||
logger.info(f"配置项 '{field_name}' 已自动修正并写入到 'app.env' 文件")
|
||||
return True, message
|
||||
|
||||
def update_setting(self, key: str, value: Any) -> Tuple[Optional[bool], str]:
|
||||
@@ -484,19 +613,17 @@ class Settings(BaseSettings, ConfigModel, LogConfigModel):
|
||||
return False, f"配置项 '{key}' 不存在"
|
||||
|
||||
try:
|
||||
field = self.__fields__[key]
|
||||
field = Settings.model_fields[key]
|
||||
original_value = getattr(self, key)
|
||||
if field.name == "API_TOKEN":
|
||||
if key == "API_TOKEN":
|
||||
converted_value, needs_update = self.validate_api_token(value, original_value)
|
||||
else:
|
||||
converted_value, needs_update = self.generic_type_converter(value,
|
||||
original_value,
|
||||
field.type_,
|
||||
field.default,
|
||||
key)
|
||||
converted_value, needs_update = self.generic_type_converter(
|
||||
value, original_value, field.annotation, field.default, key
|
||||
)
|
||||
# 如果没有抛出异常,则统一使用 converted_value 进行更新
|
||||
if needs_update or str(value) != str(converted_value):
|
||||
success, message = self.update_env_config(field, value, converted_value)
|
||||
success, message = self.update_env_config(key, value, converted_value)
|
||||
# 仅成功更新配置时,才更新内存
|
||||
if success:
|
||||
setattr(self, key, converted_value)
|
||||
@@ -588,11 +715,9 @@ class Settings(BaseSettings, ConfigModel, LogConfigModel):
|
||||
douban=512,
|
||||
bangumi=512,
|
||||
fanart=512,
|
||||
meta=(self.META_CACHE_EXPIRE or 24) * 3600,
|
||||
meta=(self.META_CACHE_EXPIRE or 72) * 3600,
|
||||
scheduler=100,
|
||||
threadpool=100,
|
||||
dbpool=100,
|
||||
dbpooloverflow=50
|
||||
threadpool=100
|
||||
)
|
||||
return SystemConfModel(
|
||||
torrents=100,
|
||||
@@ -601,11 +726,9 @@ class Settings(BaseSettings, ConfigModel, LogConfigModel):
|
||||
douban=256,
|
||||
bangumi=256,
|
||||
fanart=128,
|
||||
meta=(self.META_CACHE_EXPIRE or 2) * 3600,
|
||||
meta=(self.META_CACHE_EXPIRE or 24) * 3600,
|
||||
scheduler=50,
|
||||
threadpool=50,
|
||||
dbpool=50,
|
||||
dbpooloverflow=20
|
||||
threadpool=50
|
||||
)
|
||||
|
||||
@property
|
||||
@@ -727,6 +850,8 @@ class GlobalVar(object):
|
||||
SUBSCRIPTIONS: List[dict] = []
|
||||
# 需应急停止的工作流
|
||||
EMERGENCY_STOP_WORKFLOWS: List[int] = []
|
||||
# 需应急停止文件整理
|
||||
EMERGENCY_STOP_TRANSFER: List[str] = []
|
||||
|
||||
def stop_system(self):
|
||||
"""
|
||||
@@ -767,12 +892,30 @@ class GlobalVar(object):
|
||||
if workflow_id in self.EMERGENCY_STOP_WORKFLOWS:
|
||||
self.EMERGENCY_STOP_WORKFLOWS.remove(workflow_id)
|
||||
|
||||
def is_workflow_stopped(self, workflow_id: int):
|
||||
def is_workflow_stopped(self, workflow_id: int) -> bool:
|
||||
"""
|
||||
是否停止工作流
|
||||
"""
|
||||
return self.is_system_stopped or workflow_id in self.EMERGENCY_STOP_WORKFLOWS
|
||||
|
||||
def stop_transfer(self, path: str):
|
||||
"""
|
||||
停止文件整理
|
||||
"""
|
||||
if path not in self.EMERGENCY_STOP_TRANSFER:
|
||||
self.EMERGENCY_STOP_TRANSFER.append(path)
|
||||
|
||||
def is_transfer_stopped(self, path: str) -> bool:
|
||||
"""
|
||||
是否停止文件整理
|
||||
"""
|
||||
if self.is_system_stopped:
|
||||
return True
|
||||
if path in self.EMERGENCY_STOP_TRANSFER:
|
||||
self.EMERGENCY_STOP_TRANSFER.remove(path)
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
# 全局标识
|
||||
global_vars = GlobalVar()
|
||||
|
||||
@@ -250,6 +250,8 @@ class MediaInfo:
|
||||
production_countries: list = field(default_factory=list)
|
||||
# 语种
|
||||
spoken_languages: list = field(default_factory=list)
|
||||
# 所有发行日期
|
||||
release_dates: list = field(default_factory=list)
|
||||
# 状态
|
||||
status: str = None
|
||||
# 标签
|
||||
@@ -257,7 +259,7 @@ class MediaInfo:
|
||||
# 评价数量
|
||||
vote_count: int = None
|
||||
# 流行度
|
||||
popularity: int = None
|
||||
popularity: float = None
|
||||
# 时长
|
||||
runtime: int = None
|
||||
# 下一集
|
||||
@@ -433,6 +435,18 @@ class MediaInfo:
|
||||
if self.release_date:
|
||||
# 年份
|
||||
self.year = self.release_date[:4]
|
||||
# 所有发行日期
|
||||
self.release_dates = [
|
||||
{
|
||||
"date": release_date.get("release_date"),
|
||||
"iso_code": result.get("iso_3166_1"),
|
||||
"note": release_date.get("note"),
|
||||
"type": release_date.get("type"),
|
||||
}
|
||||
for result in info.get("release_dates", {}).get("results", [])
|
||||
for release_date in result.get("release_dates", [])
|
||||
if release_date.get("release_date")
|
||||
]
|
||||
else:
|
||||
# 电视剧
|
||||
self.title = info.get('name')
|
||||
@@ -483,7 +497,7 @@ class MediaInfo:
|
||||
continue
|
||||
if current_value is None:
|
||||
setattr(self, key, value)
|
||||
elif type(current_value) == type(value):
|
||||
elif type(current_value) is type(value):
|
||||
setattr(self, key, value)
|
||||
|
||||
def set_douban_info(self, info: dict):
|
||||
@@ -624,7 +638,7 @@ class MediaInfo:
|
||||
continue
|
||||
if current_value is None:
|
||||
setattr(self, key, value)
|
||||
elif type(current_value) == type(value):
|
||||
elif type(current_value) is type(value):
|
||||
setattr(self, key, value)
|
||||
|
||||
def set_bangumi_info(self, info: dict):
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import asyncio
|
||||
import importlib
|
||||
import inspect
|
||||
import random
|
||||
@@ -71,15 +72,26 @@ class EventManager(metaclass=Singleton):
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
self.__executor = ThreadHelper() # 动态线程池,用于消费事件
|
||||
self.__consumer_threads = [] # 用于保存启动的事件消费者线程
|
||||
self.__event_queue = PriorityQueue() # 优先级队列
|
||||
self.__broadcast_subscribers: Dict[EventType, Dict[str, Callable]] = {} # 广播事件的订阅者
|
||||
self.__chain_subscribers: Dict[ChainEventType, Dict[str, tuple[int, Callable]]] = {} # 链式事件的订阅者
|
||||
self.__disabled_handlers = set() # 禁用的事件处理器集合
|
||||
self.__disabled_classes = set() # 禁用的事件处理器类集合
|
||||
self.__lock = threading.Lock() # 线程锁
|
||||
self.__event = threading.Event() # 退出事件
|
||||
# 动态线程池,用于消费事件
|
||||
self.__executor = ThreadHelper()
|
||||
# 用于保存启动的事件消费者线程
|
||||
self.__consumer_threads = []
|
||||
# 优先级队列
|
||||
self.__event_queue = PriorityQueue()
|
||||
# 广播事件的订阅者
|
||||
self.__broadcast_subscribers: Dict[EventType, Dict[str, Callable]] = {}
|
||||
# 链式事件的订阅者
|
||||
self.__chain_subscribers: Dict[ChainEventType, Dict[str, tuple[int, Callable]]] = {}
|
||||
# 禁用的事件处理器集合
|
||||
self.__disabled_handlers = set()
|
||||
# 禁用的事件处理器类集合
|
||||
self.__disabled_classes = set()
|
||||
# 线程锁
|
||||
self.__lock = threading.Lock()
|
||||
# 退出事件
|
||||
self.__event = threading.Event()
|
||||
# 当前事件循环
|
||||
self.loop = asyncio.get_event_loop()
|
||||
|
||||
def start(self):
|
||||
"""
|
||||
@@ -438,7 +450,15 @@ class EventManager(metaclass=Singleton):
|
||||
isolated_event = Event(event_type=event.event_type,
|
||||
event_data=event_data_copy,
|
||||
priority=event.priority)
|
||||
self.__executor.submit(self.__safe_invoke_handler, handler, isolated_event)
|
||||
if inspect.iscoroutinefunction(handler):
|
||||
# 对于异步函数,直接在事件循环中运行
|
||||
asyncio.run_coroutine_threadsafe(
|
||||
self.__safe_invoke_handler_async(handler, isolated_event),
|
||||
self.loop
|
||||
)
|
||||
else:
|
||||
# 对于同步函数,在线程池中运行
|
||||
self.__executor.submit(self.__safe_invoke_handler, handler, isolated_event)
|
||||
|
||||
def __safe_invoke_handler(self, handler: Callable, event: Event):
|
||||
"""
|
||||
@@ -450,10 +470,7 @@ class EventManager(metaclass=Singleton):
|
||||
logger.debug(f"Handler {self.__get_handler_identifier(handler)} is disabled. Skipping execution")
|
||||
return
|
||||
|
||||
try:
|
||||
self.__invoke_handler_by_type_sync(handler, event)
|
||||
except Exception as e:
|
||||
self.__handle_event_error(event, handler, e)
|
||||
self.__invoke_handler_by_type_sync(handler, event)
|
||||
|
||||
async def __safe_invoke_handler_async(self, handler: Callable, event: Event):
|
||||
"""
|
||||
@@ -465,10 +482,7 @@ class EventManager(metaclass=Singleton):
|
||||
logger.debug(f"Handler {self.__get_handler_identifier(handler)} is disabled. Skipping execution")
|
||||
return
|
||||
|
||||
try:
|
||||
await self.__invoke_handler_by_type_async(handler, event)
|
||||
except Exception as e:
|
||||
self.__handle_event_error(event, handler, e)
|
||||
await self.__invoke_handler_by_type_async(handler, event)
|
||||
|
||||
def __invoke_handler_by_type_sync(self, handler: Callable, event: Event):
|
||||
"""
|
||||
@@ -486,7 +500,17 @@ class EventManager(metaclass=Singleton):
|
||||
|
||||
if class_name in plugin_manager.get_plugin_ids():
|
||||
# 插件处理器
|
||||
plugin_manager.run_plugin_method(class_name, method_name, event)
|
||||
plugin = plugin_manager.running_plugins.get(class_name)
|
||||
if not plugin:
|
||||
return
|
||||
method = getattr(plugin, method_name, None)
|
||||
if not method:
|
||||
return
|
||||
try:
|
||||
method(event)
|
||||
except Exception as e:
|
||||
self.__handle_event_error(event=event, module_name=plugin.name,
|
||||
class_name=class_name, method_name=method_name, e=e)
|
||||
elif class_name in module_manager.get_module_ids():
|
||||
# 模块处理器
|
||||
module = module_manager.get_running_module(class_name)
|
||||
@@ -495,16 +519,24 @@ class EventManager(metaclass=Singleton):
|
||||
method = getattr(module, method_name, None)
|
||||
if not method:
|
||||
return
|
||||
method(event)
|
||||
try:
|
||||
method(event)
|
||||
except Exception as e:
|
||||
self.__handle_event_error(event=event, module_name=module.get_name(),
|
||||
class_name=class_name, method_name=method_name, e=e)
|
||||
else:
|
||||
# 全局处理器
|
||||
class_obj = self.__get_class_instance(class_name)
|
||||
if not class_obj or not hasattr(class_obj, method_name):
|
||||
return
|
||||
method = getattr(class_obj, method_name)
|
||||
method = getattr(class_obj, method_name, None)
|
||||
if not method:
|
||||
return
|
||||
method(event)
|
||||
try:
|
||||
method(event)
|
||||
except Exception as e:
|
||||
self.__handle_event_error(event=event, module_name=class_name,
|
||||
class_name=class_name, method_name=method_name, e=e)
|
||||
|
||||
async def __invoke_handler_by_type_async(self, handler: Callable, event: Event):
|
||||
"""
|
||||
@@ -537,52 +569,63 @@ class EventManager(metaclass=Singleton):
|
||||
names = handler.__qualname__.split(".")
|
||||
return names[0], names[1]
|
||||
|
||||
@staticmethod
|
||||
async def __invoke_plugin_method_async(handler: Any, class_name: str, method_name: str, event: Event):
|
||||
async def __invoke_plugin_method_async(self, handler: Any, class_name: str, method_name: str, event: Event):
|
||||
"""
|
||||
异步调用插件方法
|
||||
"""
|
||||
plugin = handler.running_plugins.get(class_name)
|
||||
if plugin and hasattr(plugin, method_name):
|
||||
method = getattr(plugin, method_name)
|
||||
if not plugin:
|
||||
return
|
||||
method = getattr(plugin, method_name, None)
|
||||
if not method:
|
||||
return
|
||||
try:
|
||||
if inspect.iscoroutinefunction(method):
|
||||
await method(event)
|
||||
else:
|
||||
# 插件同步函数在异步环境中运行,避免阻塞
|
||||
await run_in_threadpool(method, event)
|
||||
except Exception as e:
|
||||
self.__handle_event_error(event=event, module_name=plugin.name,
|
||||
class_name=class_name, method_name=method_name, e=e)
|
||||
|
||||
@staticmethod
|
||||
async def __invoke_module_method_async(handler: Any, class_name: str, method_name: str, event: Event):
|
||||
async def __invoke_module_method_async(self, handler: Any, class_name: str, method_name: str, event: Event):
|
||||
"""
|
||||
异步调用模块方法
|
||||
"""
|
||||
module = handler.get_running_module(class_name)
|
||||
if not module:
|
||||
return
|
||||
|
||||
method = getattr(module, method_name, None)
|
||||
if not method:
|
||||
return
|
||||
|
||||
if inspect.iscoroutinefunction(method):
|
||||
await method(event)
|
||||
else:
|
||||
method(event)
|
||||
try:
|
||||
if inspect.iscoroutinefunction(method):
|
||||
await method(event)
|
||||
else:
|
||||
method(event)
|
||||
except Exception as e:
|
||||
self.__handle_event_error(event=event, module_name=module.get_name(),
|
||||
class_name=class_name, method_name=method_name, e=e)
|
||||
|
||||
async def __invoke_global_method_async(self, class_name: str, method_name: str, event: Event):
|
||||
"""
|
||||
异步调用全局对象方法
|
||||
"""
|
||||
class_obj = self.__get_class_instance(class_name)
|
||||
if not class_obj or not hasattr(class_obj, method_name):
|
||||
if not class_obj:
|
||||
return
|
||||
|
||||
method = getattr(class_obj, method_name)
|
||||
|
||||
if inspect.iscoroutinefunction(method):
|
||||
await method(event)
|
||||
else:
|
||||
method(event)
|
||||
method = getattr(class_obj, method_name, None)
|
||||
if not method:
|
||||
return
|
||||
try:
|
||||
if inspect.iscoroutinefunction(method):
|
||||
await method(event)
|
||||
else:
|
||||
method(event)
|
||||
except Exception as e:
|
||||
self.__handle_event_error(event=event, module_name=class_name,
|
||||
class_name=class_name, method_name=method_name, e=e)
|
||||
|
||||
@staticmethod
|
||||
def __get_class_instance(class_name: str):
|
||||
@@ -609,7 +652,11 @@ class EventManager(metaclass=Singleton):
|
||||
module_name = f"app.chain.{class_name[:-5].lower()}"
|
||||
module = importlib.import_module(module_name)
|
||||
elif class_name.endswith("Helper"):
|
||||
module_name = f"app.helper.{class_name[:-6].lower()}"
|
||||
# 特殊处理 Async 类
|
||||
if class_name.startswith("Async"):
|
||||
module_name = f"app.helper.{class_name[5:-6].lower()}"
|
||||
else:
|
||||
module_name = f"app.helper.{class_name[:-6].lower()}"
|
||||
module = importlib.import_module(module_name)
|
||||
else:
|
||||
module_name = f"app.{class_name.lower()}"
|
||||
@@ -649,18 +696,16 @@ class EventManager(metaclass=Singleton):
|
||||
"""
|
||||
logger.debug(f"{stage} - {event}")
|
||||
|
||||
def __handle_event_error(self, event: Event, handler: Callable, e: Exception):
|
||||
def __handle_event_error(self, event: Event, module_name: str,
|
||||
class_name: str, method_name: str, e: Exception):
|
||||
"""
|
||||
全局错误处理器,用于处理事件处理中的异常
|
||||
"""
|
||||
logger.error(f"事件处理出错:{str(e)} - {traceback.format_exc()}")
|
||||
|
||||
names = handler.__qualname__.split(".")
|
||||
class_name, method_name = names[0], names[1]
|
||||
logger.error(f"{module_name} 事件处理出错:{str(e)} - {traceback.format_exc()}")
|
||||
|
||||
# 发送系统错误通知
|
||||
from app.helper.message import MessageHelper
|
||||
MessageHelper().put(title=f"{event.event_type} 事件处理出错",
|
||||
MessageHelper().put(title=f"{module_name} 处理事件 {event.event_type} 时出错",
|
||||
message=f"{class_name}.{method_name}:{str(e)}",
|
||||
role="system")
|
||||
self.send_event(
|
||||
|
||||
@@ -94,7 +94,6 @@ class MetaVideo(MetaBase):
|
||||
title = re.sub(r'\d{4}[\s._-]\d{1,2}[\s._-]\d{1,2}', "", title)
|
||||
# 拆分tokens
|
||||
tokens = Tokens(title)
|
||||
self.tokens = tokens
|
||||
# 实例化StreamingPlatforms对象
|
||||
streaming_platforms = StreamingPlatforms()
|
||||
# 解析名称、年份、季、集、资源类型、分辨率等
|
||||
@@ -102,7 +101,7 @@ class MetaVideo(MetaBase):
|
||||
while token:
|
||||
self._index += 1 # 更新当前处理的token索引
|
||||
# Part
|
||||
self.__init_part(token)
|
||||
self.__init_part(token, tokens)
|
||||
# 标题
|
||||
if self._continue_flag:
|
||||
self.__init_name(token)
|
||||
@@ -123,7 +122,7 @@ class MetaVideo(MetaBase):
|
||||
self.__init_resource_type(token)
|
||||
# 流媒体平台
|
||||
if self._continue_flag:
|
||||
self.__init_web_source(token, streaming_platforms)
|
||||
self.__init_web_source(token, tokens, streaming_platforms)
|
||||
# 视频编码
|
||||
if self._continue_flag:
|
||||
self.__init_video_encode(token)
|
||||
@@ -311,7 +310,7 @@ class MetaVideo(MetaBase):
|
||||
self.en_name = token
|
||||
self._last_token_type = "enname"
|
||||
|
||||
def __init_part(self, token: str):
|
||||
def __init_part(self, token: str, tokens: Tokens):
|
||||
"""
|
||||
识别Part
|
||||
"""
|
||||
@@ -327,12 +326,12 @@ class MetaVideo(MetaBase):
|
||||
if re_res:
|
||||
if not self.part:
|
||||
self.part = re_res.group(1)
|
||||
nextv = self.tokens.cur()
|
||||
nextv = tokens.cur()
|
||||
if nextv \
|
||||
and ((nextv.isdigit() and (len(nextv) == 1 or len(nextv) == 2 and nextv.startswith('0')))
|
||||
or nextv.upper() in ['A', 'B', 'C', 'I', 'II', 'III']):
|
||||
self.part = "%s%s" % (self.part, nextv)
|
||||
self.tokens.get_next()
|
||||
tokens.get_next()
|
||||
self._last_token_type = "part"
|
||||
self._continue_flag = False
|
||||
# self._stop_name_flag = False
|
||||
@@ -582,7 +581,7 @@ class MetaVideo(MetaBase):
|
||||
self._effect.append(effect)
|
||||
self._last_token = effect.upper()
|
||||
|
||||
def __init_web_source(self, token: str, streaming_platforms: StreamingPlatforms):
|
||||
def __init_web_source(self, token: str, tokens: Tokens, streaming_platforms: StreamingPlatforms):
|
||||
"""
|
||||
识别流媒体平台
|
||||
"""
|
||||
@@ -594,10 +593,10 @@ class MetaVideo(MetaBase):
|
||||
|
||||
prev_token = None
|
||||
prev_idx = self._index - 2
|
||||
if 0 <= prev_idx < len(self.tokens.tokens):
|
||||
prev_token = self.tokens.tokens[prev_idx]
|
||||
if 0 <= prev_idx < len(tokens.tokens):
|
||||
prev_token = tokens.tokens[prev_idx]
|
||||
|
||||
next_token = self.tokens.peek()
|
||||
next_token = tokens.peek()
|
||||
|
||||
if streaming_platforms.is_streaming_platform(token):
|
||||
platform_name = streaming_platforms.get_streaming_platform_name(token)
|
||||
@@ -616,7 +615,7 @@ class MetaVideo(MetaBase):
|
||||
platform_name = streaming_platforms.get_streaming_platform_name(combined_token)
|
||||
query_range = 2
|
||||
if is_next:
|
||||
self.tokens.get_next()
|
||||
tokens.get_next()
|
||||
break
|
||||
|
||||
if not platform_name:
|
||||
@@ -626,8 +625,8 @@ class MetaVideo(MetaBase):
|
||||
match_start_idx = self._index - query_range
|
||||
match_end_idx = self._index - 1
|
||||
start_index = max(0, match_start_idx - query_range)
|
||||
end_index = min(len(self.tokens.tokens), match_end_idx + 1 + query_range)
|
||||
tokens_to_check = self.tokens.tokens[start_index:end_index]
|
||||
end_index = min(len(tokens.tokens), match_end_idx + 1 + query_range)
|
||||
tokens_to_check = tokens.tokens[start_index:end_index]
|
||||
|
||||
if any(tok and tok.upper() in web_tokens for tok in tokens_to_check):
|
||||
self.web_source = platform_name
|
||||
|
||||
@@ -48,7 +48,7 @@ class ModuleManager(metaclass=Singleton):
|
||||
# 通过模板开关控制加载
|
||||
_module.init_module()
|
||||
self._running_modules[module_id] = _module
|
||||
logger.info(f"Moudle Loaded:{module_id}")
|
||||
logger.debug(f"Moudle Loaded:{module_id}")
|
||||
except Exception as err:
|
||||
logger.error(f"Load Moudle Error:{module_id},{str(err)} - {traceback.format_exc()}", exc_info=True)
|
||||
|
||||
@@ -61,7 +61,7 @@ class ModuleManager(metaclass=Singleton):
|
||||
if hasattr(module, "stop"):
|
||||
try:
|
||||
module.stop()
|
||||
logger.info(f"Moudle Stoped:{module_id}")
|
||||
logger.debug(f"Moudle Stoped:{module_id}")
|
||||
except Exception as err:
|
||||
logger.error(f"Stop Moudle Error:{module_id},{str(err)} - {traceback.format_exc()}", exc_info=True)
|
||||
logger.info("所有模块停止完成")
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import ast
|
||||
import asyncio
|
||||
import concurrent
|
||||
import concurrent.futures
|
||||
@@ -9,14 +10,15 @@ import time
|
||||
import traceback
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
from pathlib import Path
|
||||
import threading
|
||||
from typing import Any, Dict, List, Optional, Type, Union, Callable, Tuple
|
||||
|
||||
from fastapi import HTTPException
|
||||
from starlette import status
|
||||
from watchdog.events import FileSystemEventHandler
|
||||
from watchdog.observers import Observer
|
||||
from watchfiles import watch
|
||||
|
||||
from app import schemas
|
||||
from app.core.cache import fresh, async_fresh
|
||||
from app.core.config import settings
|
||||
from app.core.event import eventmanager, Event
|
||||
from app.db.plugindata_oper import PluginDataOper
|
||||
@@ -26,64 +28,12 @@ from app.helper.sites import SitesHelper # noqa
|
||||
from app.log import logger
|
||||
from app.schemas.types import EventType, SystemConfigKey
|
||||
from app.utils.crypto import RSAUtils
|
||||
from app.utils.limit import rate_limit_window
|
||||
from app.utils.object import ObjectUtils
|
||||
from app.utils.singleton import Singleton
|
||||
from app.utils.string import StringUtils
|
||||
from app.utils.system import SystemUtils
|
||||
|
||||
|
||||
class PluginMonitorHandler(FileSystemEventHandler):
|
||||
|
||||
def on_modified(self, event):
|
||||
"""
|
||||
插件文件修改后重载
|
||||
"""
|
||||
if event.is_directory:
|
||||
return
|
||||
# 使用 pathlib 处理文件路径,跳过非 .py 文件以及 pycache 目录中的文件
|
||||
event_path = Path(event.src_path)
|
||||
if not event_path.name.endswith(".py") or "pycache" in event_path.parts:
|
||||
return
|
||||
|
||||
# 读取插件根目录下的__init__.py文件,读取class XXXX(_PluginBase)的类名
|
||||
try:
|
||||
plugins_root = settings.ROOT_PATH / "app" / "plugins"
|
||||
# 确保修改的文件在 plugins 目录下
|
||||
if plugins_root not in event_path.parents:
|
||||
return
|
||||
# 获取插件目录路径,没有找到__init__.py时,说明不是有效包,跳过插件重载
|
||||
# 插件重载目前没有支持app/plugins/plugin/package/__init__.py的场景,这里也不做支持
|
||||
plugin_dir = event_path.parent
|
||||
init_file = plugin_dir / "__init__.py"
|
||||
if not init_file.exists():
|
||||
logger.debug(f"{plugin_dir} 下没有找到 __init__.py,跳过插件重载")
|
||||
return
|
||||
|
||||
with open(init_file, "r", encoding="utf-8") as f:
|
||||
lines = f.readlines()
|
||||
pid = None
|
||||
for line in lines:
|
||||
if line.startswith("class") and "(_PluginBase)" in line:
|
||||
pid = line.split("class ")[1].split("(_PluginBase)")[0].strip()
|
||||
if pid:
|
||||
self.__reload_plugin(pid)
|
||||
except Exception as e:
|
||||
logger.error(f"插件文件修改后重载出错:{str(e)}")
|
||||
|
||||
@staticmethod
|
||||
@rate_limit_window(max_calls=1, window_seconds=2, source="PluginMonitor", enable_logging=False)
|
||||
def __reload_plugin(pid):
|
||||
"""
|
||||
重新加载插件
|
||||
"""
|
||||
try:
|
||||
logger.info(f"插件 {pid} 文件修改,重新加载...")
|
||||
PluginManager().reload_plugin(pid)
|
||||
except Exception as e:
|
||||
logger.error(f"插件文件修改后重载出错:{str(e)}")
|
||||
|
||||
|
||||
class PluginManager(metaclass=Singleton):
|
||||
"""
|
||||
插件管理器
|
||||
@@ -96,8 +46,10 @@ class PluginManager(metaclass=Singleton):
|
||||
self._running_plugins: dict = {}
|
||||
# 配置Key
|
||||
self._config_key: str = "plugin.%s"
|
||||
# 监听器
|
||||
self._observer: Observer = None
|
||||
# 监控线程
|
||||
self._monitor_thread: Optional[threading.Thread] = None
|
||||
# 监控停止事件
|
||||
self._stop_monitor_event = threading.Event()
|
||||
# 开发者模式监测插件修改
|
||||
if settings.DEV or settings.PLUGIN_AUTO_RELOAD:
|
||||
self.__start_monitor()
|
||||
@@ -264,7 +216,6 @@ class PluginManager(metaclass=Singleton):
|
||||
|
||||
# 导入模块
|
||||
module = importlib.import_module(module_name)
|
||||
importlib.reload(module)
|
||||
|
||||
# 检查模块中的类
|
||||
for name, obj in module.__dict__.items():
|
||||
@@ -318,10 +269,9 @@ class PluginManager(metaclass=Singleton):
|
||||
重新加载插件文件修改监测
|
||||
"""
|
||||
if settings.DEV or settings.PLUGIN_AUTO_RELOAD:
|
||||
if self._observer and self._observer.is_alive():
|
||||
logger.info("插件文件修改监测已经在运行中...")
|
||||
else:
|
||||
self.__start_monitor()
|
||||
# 先关闭已有监测,再重新启动
|
||||
self.stop_monitor()
|
||||
self.__start_monitor()
|
||||
else:
|
||||
self.stop_monitor()
|
||||
|
||||
@@ -329,25 +279,123 @@ class PluginManager(metaclass=Singleton):
|
||||
"""
|
||||
启用监测插件文件修改监测
|
||||
"""
|
||||
if self._monitor_thread and self._monitor_thread.is_alive():
|
||||
logger.info("插件文件修改监测已经在运行中...")
|
||||
return
|
||||
|
||||
logger.info("开始监测插件文件修改...")
|
||||
monitor_handler = PluginMonitorHandler()
|
||||
self._observer = Observer()
|
||||
self._observer.schedule(monitor_handler, str(settings.ROOT_PATH / "app" / "plugins"), recursive=True)
|
||||
self._observer.start()
|
||||
|
||||
# 在启动新线程之前,确保停止事件是清除状态
|
||||
self._stop_monitor_event.clear()
|
||||
|
||||
# 创建并启动监控线程
|
||||
self._monitor_thread = threading.Thread(
|
||||
target=self._run_file_watcher,
|
||||
daemon=True
|
||||
)
|
||||
self._monitor_thread.start()
|
||||
|
||||
def stop_monitor(self):
|
||||
"""
|
||||
停止监测插件文件修改监测
|
||||
"""
|
||||
# 停止监测
|
||||
if self._observer and self._observer.is_alive():
|
||||
if self._monitor_thread and self._monitor_thread.is_alive():
|
||||
logger.info("正在停止插件文件修改监测...")
|
||||
self._observer.stop()
|
||||
self._observer.join()
|
||||
self._stop_monitor_event.set()
|
||||
self._monitor_thread.join(timeout=5)
|
||||
if self._monitor_thread.is_alive():
|
||||
logger.warning("插件文件修改监测线程在5秒内未能正常停止。")
|
||||
self._monitor_thread = None
|
||||
logger.info("插件文件修改监测停止完成")
|
||||
else:
|
||||
logger.info("未启用插件文件修改监测,无需停止")
|
||||
|
||||
def _run_file_watcher(self):
|
||||
"""
|
||||
运行 watchfiles 监视器的主循环。
|
||||
"""
|
||||
# 监视插件目录
|
||||
plugins_path = str(settings.ROOT_PATH / "app" / "plugins")
|
||||
logger.info(">>> 监控线程已启动,准备进入watch循环...")
|
||||
# 使用 watchfiles 监视目录变化,并响应变化事件
|
||||
# Todo: yield_on_timeout = True 时,每秒检查停止事件,会返回空集合;后续可以考虑用来做心跳之类的功能?
|
||||
for changes in watch(plugins_path, stop_event=self._stop_monitor_event, rust_timeout=1000,
|
||||
yield_on_timeout=True):
|
||||
# 如果收到停止事件,退出循环
|
||||
if not changes:
|
||||
continue
|
||||
|
||||
# 处理变化事件
|
||||
plugins_to_reload = set()
|
||||
for _change_type, path_str in changes:
|
||||
event_path = Path(path_str)
|
||||
|
||||
# 跳过非 .py 文件以及 pycache 目录中的文件
|
||||
if not event_path.name.endswith(".py") or "__pycache__" in event_path.parts:
|
||||
continue
|
||||
|
||||
# 解析插件ID
|
||||
pid = self._get_plugin_id_from_path(event_path)
|
||||
# 跳过无效插件文件
|
||||
if pid:
|
||||
# 收集需要重载的插件ID,自动去重,避免重复重载
|
||||
plugins_to_reload.add(pid)
|
||||
|
||||
# 触发重载
|
||||
if plugins_to_reload:
|
||||
logger.info(f"检测到插件文件变化,准备重载: {list(plugins_to_reload)}")
|
||||
for pid in plugins_to_reload:
|
||||
try:
|
||||
self.reload_plugin(pid)
|
||||
except Exception as e:
|
||||
logger.error(f"插件 {pid} 热重载失败: {e}", exc_info=True)
|
||||
|
||||
@staticmethod
|
||||
def _get_plugin_id_from_path(event_path: Path) -> Optional[str]:
|
||||
"""
|
||||
根据文件路径解析出插件的ID。
|
||||
:param event_path: 被修改文件的 Path 对象。
|
||||
:return: 插件ID字符串,如果不是有效插件文件则返回 None。
|
||||
"""
|
||||
try:
|
||||
plugins_root = settings.ROOT_PATH / "app" / "plugins"
|
||||
# 确保修改的文件在 plugins 目录下
|
||||
if not event_path.is_relative_to(plugins_root):
|
||||
return None
|
||||
|
||||
try:
|
||||
plugin_dir_name = event_path.relative_to(plugins_root).parts[0]
|
||||
plugin_dir = plugins_root / plugin_dir_name
|
||||
except (ValueError, IndexError):
|
||||
return None
|
||||
|
||||
init_file = plugin_dir / "__init__.py"
|
||||
if not init_file.exists():
|
||||
return None
|
||||
|
||||
# 读取 __init__.py 文件,查找插件主类名
|
||||
with open(init_file, "r", encoding="utf-8") as f:
|
||||
source_code = f.read()
|
||||
|
||||
tree = ast.parse(source_code)
|
||||
|
||||
# 遍历AST,查找继承自 _PluginBase 的类
|
||||
for node in ast.walk(tree):
|
||||
# 检查节点是否为类定义
|
||||
if isinstance(node, ast.ClassDef):
|
||||
# 遍历该类的所有基类
|
||||
for base in node.bases:
|
||||
# 检查基类是否是我们寻找的 _PluginBase
|
||||
# ast.Name 用于处理简单的基类名
|
||||
if isinstance(base, ast.Name) and base.id == '_PluginBase':
|
||||
# 返回这个类的名字
|
||||
return node.name
|
||||
|
||||
return None
|
||||
except Exception as e:
|
||||
logger.error(f"从路径解析插件ID时出错: {e}")
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def __stop_plugin(plugin: Any):
|
||||
"""
|
||||
@@ -410,6 +458,10 @@ class PluginManager(metaclass=Singleton):
|
||||
except KeyError:
|
||||
# 模块可能已经被删除
|
||||
pass
|
||||
|
||||
importlib.invalidate_caches()
|
||||
logger.debug("已清除查找器的缓存")
|
||||
|
||||
if plugin_id:
|
||||
if modules_to_remove:
|
||||
logger.info(f"插件 {plugin_id} 共清除 {len(modules_to_remove)} 个模块缓存:{modules_to_remove}")
|
||||
@@ -693,6 +745,36 @@ class PluginManager(metaclass=Singleton):
|
||||
logger.error(f"获取插件 {plugin_id} 动作出错:{str(e)}")
|
||||
return ret_actions
|
||||
|
||||
def get_plugin_agent_tools(self, pid: Optional[str] = None) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
获取插件智能体工具
|
||||
[{
|
||||
"plugin_id": "插件ID",
|
||||
"plugin_name": "插件名称",
|
||||
"tools": [ToolClass1, ToolClass2, ...]
|
||||
}]
|
||||
"""
|
||||
ret_tools = []
|
||||
# 创建字典快照避免并发修改
|
||||
running_plugins_snapshot = dict(self._running_plugins)
|
||||
for plugin_id, plugin in running_plugins_snapshot.items():
|
||||
if pid and pid != plugin_id:
|
||||
continue
|
||||
if hasattr(plugin, "get_agent_tools") and ObjectUtils.check_method(plugin.get_agent_tools):
|
||||
try:
|
||||
if not plugin.get_state():
|
||||
continue
|
||||
tools = plugin.get_agent_tools()
|
||||
if tools:
|
||||
ret_tools.append({
|
||||
"plugin_id": plugin_id,
|
||||
"plugin_name": plugin.plugin_name,
|
||||
"tools": tools
|
||||
})
|
||||
except Exception as e:
|
||||
logger.error(f"获取插件 {plugin_id} 智能体工具出错:{str(e)}")
|
||||
return ret_tools
|
||||
|
||||
@staticmethod
|
||||
def get_plugin_remote_entry(plugin_id: str, dist_path: str) -> str:
|
||||
"""
|
||||
@@ -1024,7 +1106,8 @@ class PluginManager(metaclass=Singleton):
|
||||
# 已安装插件
|
||||
installed_apps = SystemConfigOper().get(SystemConfigKey.UserInstalledPlugins) or []
|
||||
# 获取在线插件
|
||||
online_plugins = PluginHelper().get_plugins(market, package_version, force)
|
||||
with fresh(force):
|
||||
online_plugins = PluginHelper().get_plugins(market, package_version)
|
||||
if online_plugins is None:
|
||||
logger.warning(
|
||||
f"获取{package_version if package_version else ''}插件库失败:{market},请检查 GitHub 网络连接")
|
||||
@@ -1165,6 +1248,7 @@ class PluginManager(metaclass=Singleton):
|
||||
async def async_get_online_plugins(self, force: bool = False) -> List[schemas.Plugin]:
|
||||
"""
|
||||
异步获取所有在线插件信息
|
||||
:param force: 是否强制刷新(忽略缓存)
|
||||
"""
|
||||
if not settings.PLUGIN_MARKET:
|
||||
return []
|
||||
@@ -1230,7 +1314,8 @@ class PluginManager(metaclass=Singleton):
|
||||
# 已安装插件
|
||||
installed_apps = SystemConfigOper().get(SystemConfigKey.UserInstalledPlugins) or []
|
||||
# 获取在线插件
|
||||
online_plugins = await PluginHelper().async_get_plugins(market, package_version, force)
|
||||
async with async_fresh(force):
|
||||
online_plugins = await PluginHelper().async_get_plugins(market, package_version)
|
||||
if online_plugins is None:
|
||||
logger.warning(
|
||||
f"获取{package_version if package_version else ''}插件库失败:{market},请检查 GitHub 网络连接")
|
||||
|
||||
@@ -252,19 +252,19 @@ def __verify_key(key: str, expected_key: str, key_type: str) -> str:
|
||||
def verify_apitoken(token: Annotated[str, Security(__get_api_token)]) -> str:
|
||||
"""
|
||||
使用 API Token 进行身份认证
|
||||
:param token: API Token,从 URL 查询参数中获取
|
||||
:param token: API Token,从 URL 查询参数中获取 token=xxx
|
||||
:return: 返回校验通过的 API Token
|
||||
"""
|
||||
return __verify_key(token, settings.API_TOKEN, "API_TOKEN")
|
||||
return __verify_key(token, settings.API_TOKEN, "token")
|
||||
|
||||
|
||||
def verify_apikey(apikey: Annotated[str, Security(__get_api_key)]) -> str:
|
||||
"""
|
||||
使用 API Key 进行身份认证
|
||||
:param apikey: API Key,从 URL 查询参数或请求头中获取
|
||||
:param apikey: API Key,从 URL 查询参数中获取 apikey=xxx
|
||||
:return: 返回校验通过的 API Key
|
||||
"""
|
||||
return __verify_key(apikey, settings.API_TOKEN, "API_KEY")
|
||||
return __verify_key(apikey, settings.API_TOKEN, "apikey")
|
||||
|
||||
|
||||
def verify_password(plain_password: str, hashed_password: str) -> bool:
|
||||
|
||||
@@ -1,19 +1,43 @@
|
||||
import asyncio
|
||||
from typing import Any, Generator, List, Optional, Self, Tuple, AsyncGenerator, Sequence, Union
|
||||
from typing import Any, Generator, List, Optional, Self, Tuple, AsyncGenerator, Union
|
||||
|
||||
from sqlalchemy import NullPool, QueuePool, and_, create_engine, inspect, text, select, delete
|
||||
from sqlalchemy import NullPool, QueuePool, and_, create_engine, inspect, text, select, delete, Column, Integer, \
|
||||
Sequence, Identity
|
||||
from sqlalchemy.ext.asyncio import create_async_engine, AsyncSession, async_sessionmaker
|
||||
from sqlalchemy.orm import Session, as_declarative, declared_attr, scoped_session, sessionmaker
|
||||
|
||||
from app.core.config import settings
|
||||
|
||||
|
||||
def get_id_column():
|
||||
"""
|
||||
根据数据库类型返回合适的ID列定义
|
||||
"""
|
||||
if settings.DB_TYPE.lower() == "postgresql":
|
||||
# PostgreSQL使用SERIAL类型,让数据库自动处理序列
|
||||
return Column(Integer, Identity(start=1, cycle=True), primary_key=True, index=True)
|
||||
else:
|
||||
# SQLite使用Sequence
|
||||
return Column(Integer, Sequence('id'), primary_key=True, index=True)
|
||||
|
||||
|
||||
def _get_database_engine(is_async: bool = False):
|
||||
"""
|
||||
获取数据库连接参数并设置WAL模式
|
||||
:param is_async: 是否创建异步引擎,True - 异步引擎, False - 同步引擎
|
||||
:return: 返回对应的数据库引擎
|
||||
"""
|
||||
# 根据数据库类型选择连接方式
|
||||
if settings.DB_TYPE.lower() == "postgresql":
|
||||
return _get_postgresql_engine(is_async)
|
||||
else:
|
||||
return _get_sqlite_engine(is_async)
|
||||
|
||||
|
||||
def _get_sqlite_engine(is_async: bool = False):
|
||||
"""
|
||||
获取SQLite数据库引擎
|
||||
"""
|
||||
# 连接参数
|
||||
_connect_args = {
|
||||
"timeout": settings.DB_TIMEOUT,
|
||||
@@ -40,9 +64,9 @@ def _get_database_engine(is_async: bool = False):
|
||||
# 当使用 QueuePool 时,添加 QueuePool 特有的参数
|
||||
if _pool_class == QueuePool:
|
||||
_db_kwargs.update({
|
||||
"pool_size": settings.CONF.dbpool,
|
||||
"pool_size": settings.DB_SQLITE_POOL_SIZE,
|
||||
"pool_timeout": settings.DB_POOL_TIMEOUT,
|
||||
"max_overflow": settings.CONF.dbpooloverflow
|
||||
"max_overflow": settings.DB_SQLITE_MAX_OVERFLOW
|
||||
})
|
||||
|
||||
# 创建数据库引擎
|
||||
@@ -52,7 +76,7 @@ def _get_database_engine(is_async: bool = False):
|
||||
_journal_mode = "WAL" if settings.DB_WAL_ENABLE else "DELETE"
|
||||
with engine.connect() as connection:
|
||||
current_mode = connection.execute(text(f"PRAGMA journal_mode={_journal_mode};")).scalar()
|
||||
print(f"Database journal mode set to: {current_mode}")
|
||||
print(f"SQLite database journal mode set to: {current_mode}")
|
||||
|
||||
return engine
|
||||
else:
|
||||
@@ -78,12 +102,73 @@ def _get_database_engine(is_async: bool = False):
|
||||
async with async_engine.connect() as _connection:
|
||||
result = await _connection.execute(text(f"PRAGMA journal_mode={_journal_mode};"))
|
||||
_current_mode = result.scalar()
|
||||
print(f"Async database journal mode set to: {_current_mode}")
|
||||
print(f"Async SQLite database journal mode set to: {_current_mode}")
|
||||
|
||||
try:
|
||||
asyncio.run(set_async_wal_mode())
|
||||
except Exception as e:
|
||||
print(f"Failed to set async WAL mode: {e}")
|
||||
print(f"Failed to set async SQLite WAL mode: {e}")
|
||||
|
||||
return async_engine
|
||||
|
||||
|
||||
def _get_postgresql_engine(is_async: bool = False):
|
||||
"""
|
||||
获取PostgreSQL数据库引擎
|
||||
"""
|
||||
# 构建PostgreSQL连接URL
|
||||
if settings.DB_POSTGRESQL_PASSWORD:
|
||||
db_url = f"postgresql://{settings.DB_POSTGRESQL_USERNAME}:{settings.DB_POSTGRESQL_PASSWORD}@{settings.DB_POSTGRESQL_HOST}:{settings.DB_POSTGRESQL_PORT}/{settings.DB_POSTGRESQL_DATABASE}"
|
||||
else:
|
||||
db_url = f"postgresql://{settings.DB_POSTGRESQL_USERNAME}@{settings.DB_POSTGRESQL_HOST}:{settings.DB_POSTGRESQL_PORT}/{settings.DB_POSTGRESQL_DATABASE}"
|
||||
|
||||
# PostgreSQL连接参数
|
||||
_connect_args = {}
|
||||
|
||||
# 创建同步引擎
|
||||
if not is_async:
|
||||
# 根据池类型设置 poolclass 和相关参数
|
||||
_pool_class = NullPool if settings.DB_POOL_TYPE == "NullPool" else QueuePool
|
||||
|
||||
# 数据库参数
|
||||
_db_kwargs = {
|
||||
"url": db_url,
|
||||
"pool_pre_ping": settings.DB_POOL_PRE_PING,
|
||||
"echo": settings.DB_ECHO,
|
||||
"poolclass": _pool_class,
|
||||
"pool_recycle": settings.DB_POOL_RECYCLE,
|
||||
"connect_args": _connect_args
|
||||
}
|
||||
|
||||
# 当使用 QueuePool 时,添加 QueuePool 特有的参数
|
||||
if _pool_class == QueuePool:
|
||||
_db_kwargs.update({
|
||||
"pool_size": settings.DB_POSTGRESQL_POOL_SIZE,
|
||||
"pool_timeout": settings.DB_POOL_TIMEOUT,
|
||||
"max_overflow": settings.DB_POSTGRESQL_MAX_OVERFLOW
|
||||
})
|
||||
|
||||
# 创建数据库引擎
|
||||
engine = create_engine(**_db_kwargs)
|
||||
print(f"PostgreSQL database connected to {settings.DB_POSTGRESQL_HOST}:{settings.DB_POSTGRESQL_PORT}/{settings.DB_POSTGRESQL_DATABASE}")
|
||||
|
||||
return engine
|
||||
else:
|
||||
# 构建异步PostgreSQL连接URL
|
||||
async_db_url = f"postgresql+asyncpg://{settings.DB_POSTGRESQL_USERNAME}:{settings.DB_POSTGRESQL_PASSWORD}@{settings.DB_POSTGRESQL_HOST}:{settings.DB_POSTGRESQL_PORT}/{settings.DB_POSTGRESQL_DATABASE}"
|
||||
|
||||
# 数据库参数,只能使用 NullPool
|
||||
_db_kwargs = {
|
||||
"url": async_db_url,
|
||||
"pool_pre_ping": settings.DB_POOL_PRE_PING,
|
||||
"echo": settings.DB_ECHO,
|
||||
"poolclass": NullPool,
|
||||
"pool_recycle": settings.DB_POOL_RECYCLE,
|
||||
"connect_args": _connect_args
|
||||
}
|
||||
# 创建异步数据库引擎
|
||||
async_engine = create_async_engine(**_db_kwargs)
|
||||
print(f"Async PostgreSQL database connected to {settings.DB_POSTGRESQL_HOST}:{settings.DB_POSTGRESQL_PORT}/{settings.DB_POSTGRESQL_DATABASE}")
|
||||
|
||||
return async_engine
|
||||
|
||||
|
||||
@@ -18,12 +18,22 @@ def update_db():
|
||||
"""
|
||||
更新数据库
|
||||
"""
|
||||
db_location = settings.CONFIG_PATH / 'user.db'
|
||||
script_location = settings.ROOT_PATH / 'database'
|
||||
try:
|
||||
alembic_cfg = Config()
|
||||
alembic_cfg.set_main_option('script_location', str(script_location))
|
||||
alembic_cfg.set_main_option('sqlalchemy.url', f"sqlite:///{db_location}")
|
||||
|
||||
# 根据数据库类型设置不同的URL
|
||||
if settings.DB_TYPE.lower() == "postgresql":
|
||||
if settings.DB_POSTGRESQL_PASSWORD:
|
||||
db_url = f"postgresql://{settings.DB_POSTGRESQL_USERNAME}:{settings.DB_POSTGRESQL_PASSWORD}@{settings.DB_POSTGRESQL_HOST}:{settings.DB_POSTGRESQL_PORT}/{settings.DB_POSTGRESQL_DATABASE}"
|
||||
else:
|
||||
db_url = f"postgresql://{settings.DB_POSTGRESQL_USERNAME}@{settings.DB_POSTGRESQL_HOST}:{settings.DB_POSTGRESQL_PORT}/{settings.DB_POSTGRESQL_DATABASE}"
|
||||
else:
|
||||
db_location = settings.CONFIG_PATH / 'user.db'
|
||||
db_url = f"sqlite:///{db_location}"
|
||||
|
||||
alembic_cfg.set_main_option('sqlalchemy.url', db_url)
|
||||
upgrade(alembic_cfg, 'head')
|
||||
except Exception as e:
|
||||
logger.error(f'数据库更新失败:{str(e)}')
|
||||
|
||||
@@ -1,18 +1,18 @@
|
||||
import time
|
||||
from typing import Optional
|
||||
|
||||
from sqlalchemy import Column, Integer, String, Sequence, JSON, select
|
||||
from sqlalchemy import Column, Integer, String, JSON, select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.db import db_query, db_update, Base, async_db_query
|
||||
from app.db import db_query, db_update, get_id_column, Base, async_db_query
|
||||
|
||||
|
||||
class DownloadHistory(Base):
|
||||
"""
|
||||
下载历史记录
|
||||
"""
|
||||
id = Column(Integer, Sequence('id'), primary_key=True, index=True)
|
||||
id = get_id_column()
|
||||
# 保存路径
|
||||
path = Column(String, nullable=False, index=True)
|
||||
# 类型 电影/电视剧
|
||||
@@ -188,7 +188,7 @@ class DownloadFiles(Base):
|
||||
"""
|
||||
下载文件记录
|
||||
"""
|
||||
id = Column(Integer, Sequence('id'), primary_key=True, index=True)
|
||||
id = get_id_column()
|
||||
# 下载器
|
||||
downloader = Column(String)
|
||||
# 下载任务Hash
|
||||
|
||||
@@ -1,19 +1,19 @@
|
||||
from datetime import datetime
|
||||
from typing import Optional
|
||||
|
||||
from sqlalchemy import Column, Integer, String, Sequence, JSON
|
||||
from sqlalchemy import Column, Integer, String, JSON
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.db import db_query, db_update, async_db_query, Base
|
||||
from app.db import db_query, db_update, get_id_column, async_db_query, Base
|
||||
|
||||
|
||||
class MediaServerItem(Base):
|
||||
"""
|
||||
媒体服务器媒体条目表
|
||||
"""
|
||||
id = Column(Integer, Sequence('id'), primary_key=True, index=True)
|
||||
id = get_id_column()
|
||||
# 服务器类型
|
||||
server = Column(String)
|
||||
# 媒体库ID
|
||||
|
||||
@@ -1,17 +1,17 @@
|
||||
from typing import Optional
|
||||
|
||||
from sqlalchemy import Column, Integer, String, Sequence, JSON, select
|
||||
from sqlalchemy import Column, Integer, String, JSON, select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.db import db_query, Base, async_db_query
|
||||
from app.db import db_query, Base, get_id_column, async_db_query
|
||||
|
||||
|
||||
class Message(Base):
|
||||
"""
|
||||
消息表
|
||||
"""
|
||||
id = Column(Integer, Sequence('id'), primary_key=True, index=True)
|
||||
id = get_id_column()
|
||||
# 消息渠道
|
||||
channel = Column(String)
|
||||
# 消息来源
|
||||
|
||||
@@ -1,14 +1,14 @@
|
||||
from sqlalchemy import Column, Integer, String, Sequence, JSON
|
||||
from sqlalchemy import Column, String, JSON
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.db import db_query, db_update, Base
|
||||
from app.db import db_query, db_update, get_id_column, Base
|
||||
|
||||
|
||||
class PluginData(Base):
|
||||
"""
|
||||
插件数据表
|
||||
"""
|
||||
id = Column(Integer, Sequence('id'), primary_key=True, index=True)
|
||||
id = get_id_column()
|
||||
plugin_id = Column(String, nullable=False, index=True)
|
||||
key = Column(String, index=True, nullable=False)
|
||||
value = Column(JSON)
|
||||
|
||||
@@ -1,17 +1,17 @@
|
||||
from datetime import datetime
|
||||
|
||||
from sqlalchemy import Boolean, Column, Integer, String, Sequence, JSON, select, delete
|
||||
from sqlalchemy import Boolean, Column, Integer, String, JSON, select, delete
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.db import db_query, db_update, Base, async_db_query, async_db_update
|
||||
from app.db import db_query, db_update, Base, async_db_query, async_db_update, get_id_column
|
||||
|
||||
|
||||
class Site(Base):
|
||||
"""
|
||||
站点表
|
||||
"""
|
||||
id = Column(Integer, Sequence('id'), primary_key=True, index=True)
|
||||
id = get_id_column()
|
||||
# 站点名
|
||||
name = Column(String, nullable=False)
|
||||
# 域名Key
|
||||
@@ -66,15 +66,21 @@ class Site(Base):
|
||||
result = await db.execute(select(cls).where(cls.domain == domain))
|
||||
return result.scalar_one_or_none()
|
||||
|
||||
@classmethod
|
||||
@async_db_query
|
||||
async def async_get_by_name(cls, db: AsyncSession, name: str):
|
||||
result = await db.execute(select(cls).where(cls.name == name))
|
||||
return result.scalar_one_or_none()
|
||||
|
||||
@classmethod
|
||||
@db_query
|
||||
def get_actives(cls, db: Session):
|
||||
return db.query(cls).filter(cls.is_active == 1).all()
|
||||
return db.query(cls).filter(cls.is_active).all()
|
||||
|
||||
@classmethod
|
||||
@async_db_query
|
||||
async def async_get_actives(cls, db: AsyncSession):
|
||||
result = await db.execute(select(cls).where(cls.is_active == 1))
|
||||
result = await db.execute(select(cls).where(cls.is_active))
|
||||
return result.scalars().all()
|
||||
|
||||
@classmethod
|
||||
|
||||
@@ -1,15 +1,15 @@
|
||||
from sqlalchemy import Column, Integer, String, Sequence, select
|
||||
from sqlalchemy import Column, String, select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.db import db_query, Base, async_db_query
|
||||
from app.db import db_query, Base, get_id_column, async_db_query
|
||||
|
||||
|
||||
class SiteIcon(Base):
|
||||
"""
|
||||
站点图标表
|
||||
"""
|
||||
id = Column(Integer, Sequence('id'), primary_key=True, index=True)
|
||||
id = get_id_column()
|
||||
# 站点名称
|
||||
name = Column(String, nullable=False)
|
||||
# 域名Key
|
||||
|
||||
@@ -1,17 +1,17 @@
|
||||
from datetime import datetime
|
||||
|
||||
from sqlalchemy import Column, Integer, String, Sequence, JSON, select
|
||||
from sqlalchemy import Column, Integer, String, JSON, select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.db import db_query, db_update, Base, async_db_query
|
||||
from app.db import db_query, db_update, get_id_column, Base, async_db_query
|
||||
|
||||
|
||||
class SiteStatistic(Base):
|
||||
"""
|
||||
站点统计表
|
||||
"""
|
||||
id = Column(Integer, Sequence('id'), primary_key=True, index=True)
|
||||
id = get_id_column()
|
||||
# 域名Key
|
||||
domain = Column(String, index=True)
|
||||
# 成功次数
|
||||
|
||||
@@ -1,18 +1,18 @@
|
||||
from datetime import datetime
|
||||
from typing import Optional
|
||||
|
||||
from sqlalchemy import Column, Integer, String, Sequence, Float, JSON, func, or_, select
|
||||
from sqlalchemy import Column, Integer, String, Float, JSON, func, or_, select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.db import db_query, Base, async_db_query
|
||||
from app.db import db_query, Base, get_id_column, async_db_query
|
||||
|
||||
|
||||
class SiteUserData(Base):
|
||||
"""
|
||||
站点数据表
|
||||
"""
|
||||
id = Column(Integer, Sequence('id'), primary_key=True, index=True)
|
||||
id = get_id_column()
|
||||
# 站点域名
|
||||
domain = Column(String, index=True)
|
||||
# 站点名称
|
||||
@@ -20,7 +20,7 @@ class SiteUserData(Base):
|
||||
# 用户名
|
||||
username = Column(String)
|
||||
# 用户ID
|
||||
userid = Column(Integer)
|
||||
userid = Column(String)
|
||||
# 用户等级
|
||||
user_level = Column(String)
|
||||
# 加入时间
|
||||
|
||||
@@ -1,18 +1,18 @@
|
||||
import time
|
||||
from typing import Optional
|
||||
|
||||
from sqlalchemy import Column, Integer, String, Sequence, Float, JSON, select
|
||||
from sqlalchemy import Column, Integer, String, Float, JSON, select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.db import db_query, db_update, Base, async_db_query, async_db_update
|
||||
from app.db import db_query, db_update, get_id_column, Base, async_db_query, async_db_update
|
||||
|
||||
|
||||
class Subscribe(Base):
|
||||
"""
|
||||
订阅表
|
||||
"""
|
||||
id = Column(Integer, Sequence('id'), primary_key=True, index=True)
|
||||
id = get_id_column()
|
||||
# 标题
|
||||
name = Column(String, nullable=False, index=True)
|
||||
# 年份
|
||||
|
||||
@@ -1,17 +1,17 @@
|
||||
from typing import Optional
|
||||
|
||||
from sqlalchemy import Column, Integer, String, Sequence, Float, JSON, select
|
||||
from sqlalchemy import Column, Integer, String, Float, JSON, select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.db import db_query, Base, async_db_query
|
||||
from app.db import db_query, Base, get_id_column, async_db_query
|
||||
|
||||
|
||||
class SubscribeHistory(Base):
|
||||
"""
|
||||
订阅历史表
|
||||
"""
|
||||
id = Column(Integer, Sequence('id'), primary_key=True, index=True)
|
||||
id = get_id_column()
|
||||
# 标题
|
||||
name = Column(String, nullable=False, index=True)
|
||||
# 年份
|
||||
|
||||
@@ -1,15 +1,15 @@
|
||||
from sqlalchemy import Column, Integer, String, Sequence, JSON, select
|
||||
from sqlalchemy import Column, String, JSON, select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.db import db_query, db_update, Base, async_db_query
|
||||
from app.db import db_query, db_update, Base, async_db_query, get_id_column
|
||||
|
||||
|
||||
class SystemConfig(Base):
|
||||
"""
|
||||
配置表
|
||||
"""
|
||||
id = Column(Integer, Sequence('id'), primary_key=True, index=True)
|
||||
id = get_id_column()
|
||||
# 主键
|
||||
key = Column(String, index=True)
|
||||
# 值
|
||||
|
||||
@@ -1,18 +1,18 @@
|
||||
import time
|
||||
from typing import Optional
|
||||
|
||||
from sqlalchemy import Column, Integer, String, Sequence, Boolean, func, or_, JSON, select
|
||||
from sqlalchemy import Column, Integer, String, Boolean, func, or_, JSON, select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.db import db_query, db_update, Base, async_db_query
|
||||
from app.db import db_query, db_update, get_id_column, Base, async_db_query
|
||||
|
||||
|
||||
class TransferHistory(Base):
|
||||
"""
|
||||
整理记录
|
||||
"""
|
||||
id = Column(Integer, Sequence('id'), primary_key=True, index=True)
|
||||
id = get_id_column()
|
||||
# 源路径
|
||||
src = Column(String, index=True)
|
||||
# 源存储
|
||||
@@ -65,76 +65,92 @@ class TransferHistory(Base):
|
||||
def list_by_title(cls, db: Session, title: str, page: Optional[int] = 1, count: Optional[int] = 30,
|
||||
status: bool = None):
|
||||
if status is not None:
|
||||
return db.query(cls).filter(
|
||||
query = db.query(cls).filter(
|
||||
cls.status == status
|
||||
).order_by(
|
||||
cls.date.desc()
|
||||
).offset((page - 1) * count).limit(count).all()
|
||||
)
|
||||
else:
|
||||
return db.query(cls).filter(or_(
|
||||
query = db.query(cls).filter(or_(
|
||||
cls.title.like(f'%{title}%'),
|
||||
cls.src.like(f'%{title}%'),
|
||||
cls.dest.like(f'%{title}%'),
|
||||
)).order_by(
|
||||
cls.date.desc()
|
||||
).offset((page - 1) * count).limit(count).all()
|
||||
)
|
||||
|
||||
# 当count为负数时,不限制页数查询所有
|
||||
if count >= 0:
|
||||
query = query.offset((page - 1) * count).limit(count)
|
||||
|
||||
return query.all()
|
||||
|
||||
@classmethod
|
||||
@async_db_query
|
||||
async def async_list_by_title(cls, db: AsyncSession, title: str, page: Optional[int] = 1, count: Optional[int] = 30,
|
||||
status: bool = None):
|
||||
if status is not None:
|
||||
result = await db.execute(
|
||||
select(cls).filter(
|
||||
cls.status == status
|
||||
).order_by(
|
||||
cls.date.desc()
|
||||
).offset((page - 1) * count).limit(count)
|
||||
query = select(cls).filter(
|
||||
cls.status == status
|
||||
).order_by(
|
||||
cls.date.desc()
|
||||
)
|
||||
else:
|
||||
result = await db.execute(
|
||||
select(cls).filter(or_(
|
||||
cls.title.like(f'%{title}%'),
|
||||
cls.src.like(f'%{title}%'),
|
||||
cls.dest.like(f'%{title}%'),
|
||||
)).order_by(
|
||||
cls.date.desc()
|
||||
).offset((page - 1) * count).limit(count)
|
||||
query = select(cls).filter(or_(
|
||||
cls.title.like(f'%{title}%'),
|
||||
cls.src.like(f'%{title}%'),
|
||||
cls.dest.like(f'%{title}%'),
|
||||
)).order_by(
|
||||
cls.date.desc()
|
||||
)
|
||||
|
||||
# 当count为负数时,不限制页数查询所有
|
||||
if count >= 0:
|
||||
query = query.offset((page - 1) * count).limit(count)
|
||||
|
||||
result = await db.execute(query)
|
||||
return result.scalars().all()
|
||||
|
||||
@classmethod
|
||||
@db_query
|
||||
def list_by_page(cls, db: Session, page: Optional[int] = 1, count: Optional[int] = 30, status: bool = None):
|
||||
if status is not None:
|
||||
return db.query(cls).filter(
|
||||
query = db.query(cls).filter(
|
||||
cls.status == status
|
||||
).order_by(
|
||||
cls.date.desc()
|
||||
).offset((page - 1) * count).limit(count).all()
|
||||
)
|
||||
else:
|
||||
return db.query(cls).order_by(
|
||||
query = db.query(cls).order_by(
|
||||
cls.date.desc()
|
||||
).offset((page - 1) * count).limit(count).all()
|
||||
)
|
||||
|
||||
# 当count为负数时,不限制页数查询所有
|
||||
if count >= 0:
|
||||
query = query.offset((page - 1) * count).limit(count)
|
||||
|
||||
return query.all()
|
||||
|
||||
@classmethod
|
||||
@async_db_query
|
||||
async def async_list_by_page(cls, db: AsyncSession, page: Optional[int] = 1, count: Optional[int] = 30,
|
||||
status: bool = None):
|
||||
if status is not None:
|
||||
result = await db.execute(
|
||||
select(cls).filter(
|
||||
cls.status == status
|
||||
).order_by(
|
||||
cls.date.desc()
|
||||
).offset((page - 1) * count).limit(count)
|
||||
query = select(cls).filter(
|
||||
cls.status == status
|
||||
).order_by(
|
||||
cls.date.desc()
|
||||
)
|
||||
else:
|
||||
result = await db.execute(
|
||||
select(cls).order_by(
|
||||
cls.date.desc()
|
||||
).offset((page - 1) * count).limit(count)
|
||||
query = select(cls).order_by(
|
||||
cls.date.desc()
|
||||
)
|
||||
|
||||
# 当count为负数时,不限制页数查询所有
|
||||
if count >= 0:
|
||||
query = query.offset((page - 1) * count).limit(count)
|
||||
|
||||
result = await db.execute(query)
|
||||
return result.scalars().all()
|
||||
|
||||
@classmethod
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
from sqlalchemy import Boolean, Column, Integer, JSON, Sequence, String, select
|
||||
from sqlalchemy import Boolean, Column, JSON, String, select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.db import Base, db_query, db_update, async_db_query, async_db_update
|
||||
from app.db import Base, db_query, db_update, async_db_query, async_db_update, get_id_column
|
||||
|
||||
|
||||
class User(Base):
|
||||
@@ -10,7 +10,7 @@ class User(Base):
|
||||
用户表
|
||||
"""
|
||||
# ID
|
||||
id = Column(Integer, Sequence('id'), primary_key=True, index=True)
|
||||
id = get_id_column()
|
||||
# 用户名,唯一值
|
||||
name = Column(String, index=True, nullable=False)
|
||||
# 邮箱
|
||||
|
||||
@@ -1,14 +1,14 @@
|
||||
from sqlalchemy import Column, Integer, String, Sequence, UniqueConstraint, Index, JSON
|
||||
from sqlalchemy import Column, String, UniqueConstraint, Index, JSON
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.db import db_query, db_update, Base
|
||||
from app.db import db_query, db_update, get_id_column, Base
|
||||
|
||||
|
||||
class UserConfig(Base):
|
||||
"""
|
||||
用户配置表
|
||||
"""
|
||||
id = Column(Integer, Sequence('id'), primary_key=True, index=True)
|
||||
id = get_id_column()
|
||||
# 用户名
|
||||
username = Column(String, index=True)
|
||||
# 配置键
|
||||
|
||||
@@ -1,10 +1,10 @@
|
||||
from datetime import datetime
|
||||
from typing import Optional
|
||||
|
||||
from sqlalchemy import Column, Integer, JSON, Sequence, String, and_, or_, select
|
||||
from sqlalchemy import Column, Integer, JSON, String, and_, or_, select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.db import Base, db_query, db_update, async_db_query, async_db_update
|
||||
from app.db import Base, db_query, get_id_column, db_update, async_db_query, async_db_update
|
||||
|
||||
|
||||
class Workflow(Base):
|
||||
@@ -12,7 +12,7 @@ class Workflow(Base):
|
||||
工作流表
|
||||
"""
|
||||
# ID
|
||||
id = Column(Integer, Sequence('id'), primary_key=True, index=True)
|
||||
id = get_id_column()
|
||||
# 名称
|
||||
name = Column(String, index=True, nullable=False)
|
||||
# 描述
|
||||
|
||||
@@ -85,6 +85,12 @@ class SiteOper(DbOper):
|
||||
"""
|
||||
return await Site.async_get_by_domain(self._db, domain)
|
||||
|
||||
async def async_get_by_name(self, name: str) -> Site:
|
||||
"""
|
||||
异步按名称获取站点
|
||||
"""
|
||||
return await Site.async_get_by_name(self._db, name)
|
||||
|
||||
def get_domains_by_ids(self, ids: List[int]) -> List[str]:
|
||||
"""
|
||||
按ID获取站点域名
|
||||
|
||||
@@ -34,6 +34,7 @@ class SubscribeOper(DbOper):
|
||||
"backdrop": mediainfo.get_backdrop_image(),
|
||||
"vote": mediainfo.vote_average,
|
||||
"description": mediainfo.overview,
|
||||
"search_imdbid": 1 if kwargs.get('search_imdbid') else 0,
|
||||
"date": time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
|
||||
})
|
||||
if not subscribe:
|
||||
@@ -118,6 +119,14 @@ class SubscribeOper(DbOper):
|
||||
return Subscribe.get_by_state(self._db, state)
|
||||
return Subscribe.list(self._db)
|
||||
|
||||
async def async_list(self, state: Optional[str] = None) -> List[Subscribe]:
|
||||
"""
|
||||
异步获取订阅列表
|
||||
"""
|
||||
if state:
|
||||
return await Subscribe.async_get_by_state(self._db, state)
|
||||
return await Subscribe.async_list(self._db)
|
||||
|
||||
def delete(self, sid: int):
|
||||
"""
|
||||
删除订阅
|
||||
|
||||
@@ -128,10 +128,10 @@ class TransferHistoryOper(DbOper):
|
||||
self.add_force(
|
||||
src=fileitem.path,
|
||||
src_storage=fileitem.storage,
|
||||
src_fileitem=fileitem.dict(),
|
||||
src_fileitem=fileitem.model_dump(),
|
||||
dest=transferinfo.target_item.path if transferinfo.target_item else None,
|
||||
dest_storage=transferinfo.target_item.storage if transferinfo.target_item else None,
|
||||
dest_fileitem=transferinfo.target_item.dict() if transferinfo.target_item else None,
|
||||
dest_fileitem=transferinfo.target_item.model_dump() if transferinfo.target_item else None,
|
||||
mode=mode,
|
||||
type=mediainfo.type.value,
|
||||
category=mediainfo.category,
|
||||
@@ -159,10 +159,10 @@ class TransferHistoryOper(DbOper):
|
||||
his = self.add_force(
|
||||
src=fileitem.path,
|
||||
src_storage=fileitem.storage,
|
||||
src_fileitem=fileitem.dict(),
|
||||
src_fileitem=fileitem.model_dump(),
|
||||
dest=transferinfo.target_item.path if transferinfo.target_item else None,
|
||||
dest_storage=transferinfo.target_item.storage if transferinfo.target_item else None,
|
||||
dest_fileitem=transferinfo.target_item.dict() if transferinfo.target_item else None,
|
||||
dest_fileitem=transferinfo.target_item.model_dump() if transferinfo.target_item else None,
|
||||
mode=mode,
|
||||
type=mediainfo.type.value,
|
||||
category=mediainfo.category,
|
||||
@@ -188,7 +188,7 @@ class TransferHistoryOper(DbOper):
|
||||
year=meta.year,
|
||||
src=fileitem.path,
|
||||
src_storage=fileitem.storage,
|
||||
src_fileitem=fileitem.dict(),
|
||||
src_fileitem=fileitem.model_dump(),
|
||||
mode=mode,
|
||||
seasons=meta.season,
|
||||
episodes=meta.episode,
|
||||
|
||||
@@ -2,7 +2,6 @@ from fastapi import FastAPI
|
||||
from fastapi.middleware.cors import CORSMiddleware
|
||||
|
||||
from app.core.config import settings
|
||||
from app.monitoring import setup_prometheus_metrics
|
||||
from app.startup.lifecycle import lifespan
|
||||
|
||||
|
||||
@@ -25,9 +24,6 @@ def create_app() -> FastAPI:
|
||||
allow_headers=["*"],
|
||||
)
|
||||
|
||||
# 设置性能监控
|
||||
setup_prometheus_metrics(_app)
|
||||
|
||||
return _app
|
||||
|
||||
|
||||
|
||||
@@ -10,9 +10,9 @@ from datetime import datetime
|
||||
from typing import Any, Literal, Optional, List, Dict, Union
|
||||
from typing import Callable
|
||||
|
||||
from cachetools import TTLCache
|
||||
from jinja2 import Template
|
||||
|
||||
from app.core.cache import TTLCache
|
||||
from app.core.config import global_vars
|
||||
from app.core.context import MediaInfo, TorrentInfo
|
||||
from app.core.meta import MetaBase
|
||||
@@ -307,7 +307,7 @@ class TemplateHelper(metaclass=SingletonClass):
|
||||
|
||||
def __init__(self):
|
||||
self.builder = TemplateContextBuilder()
|
||||
self.cache = TTLCache(maxsize=100, ttl=600)
|
||||
self.cache = TTLCache(region="notification", maxsize=100, ttl=600)
|
||||
|
||||
@staticmethod
|
||||
def _generate_cache_key(cuntent: Union[str, dict]) -> str:
|
||||
@@ -367,7 +367,6 @@ class TemplateHelper(metaclass=SingletonClass):
|
||||
return rendered
|
||||
return None
|
||||
except Exception as e:
|
||||
logger.error(f"模板处理失败: {str(e)}")
|
||||
raise ValueError(f"模板处理失败: {str(e)}") from e
|
||||
|
||||
@staticmethod
|
||||
@@ -471,6 +470,13 @@ class TemplateHelper(metaclass=SingletonClass):
|
||||
except json.JSONDecodeError:
|
||||
return rendered
|
||||
|
||||
def close(self):
|
||||
"""
|
||||
清理资源
|
||||
"""
|
||||
if self.cache:
|
||||
self.cache.close()
|
||||
|
||||
|
||||
class MessageTemplateHelper:
|
||||
"""
|
||||
@@ -704,7 +710,9 @@ class MessageQueueManager(metaclass=SingletonClass):
|
||||
停止队列管理器
|
||||
"""
|
||||
self._running = False
|
||||
logger.info("正在停止消息队列...")
|
||||
self.thread.join()
|
||||
logger.info("消息队列已停止")
|
||||
|
||||
|
||||
class MessageHelper(metaclass=Singleton):
|
||||
@@ -765,3 +773,13 @@ class MessageHelper(metaclass=Singleton):
|
||||
if not self.user_queue.empty():
|
||||
return self.user_queue.get(block=False)
|
||||
return None
|
||||
|
||||
|
||||
def stop_message():
|
||||
"""
|
||||
停止消息服务
|
||||
"""
|
||||
# 停止消息队列
|
||||
MessageQueueManager().stop()
|
||||
# 关闭消息演染器
|
||||
TemplateHelper().close()
|
||||
|
||||
@@ -1,18 +1,18 @@
|
||||
import importlib
|
||||
import io
|
||||
import json
|
||||
import shutil
|
||||
import site
|
||||
import sys
|
||||
import traceback
|
||||
import zipfile
|
||||
from pathlib import Path
|
||||
from typing import Dict, List, Optional, Tuple, Set, Callable, Awaitable
|
||||
import zipfile
|
||||
import io
|
||||
|
||||
import aiofiles
|
||||
import aioshutil
|
||||
import httpx
|
||||
from aiopath import AsyncPath
|
||||
from anyio import Path as AsyncPath
|
||||
from packaging.specifiers import SpecifierSet, InvalidSpecifier
|
||||
from packaging.version import Version, InvalidVersion
|
||||
from pkg_resources import Requirement, working_set
|
||||
@@ -48,35 +48,13 @@ class PluginHelper(metaclass=WeakSingleton):
|
||||
if self.install_report():
|
||||
self.systemconfig.set(SystemConfigKey.PluginInstallReport, "1")
|
||||
|
||||
def get_plugins(self, repo_url: str, package_version: Optional[str] = None,
|
||||
force: bool = False) -> Optional[Dict[str, dict]]:
|
||||
@cached(maxsize=128, ttl=1800)
|
||||
def get_plugins(self, repo_url: str,
|
||||
package_version: Optional[str] = None) -> Optional[Dict[str, dict]]:
|
||||
"""
|
||||
获取Github所有最新插件列表
|
||||
:param repo_url: Github仓库地址
|
||||
:param package_version: 首选插件版本 (如 "v2", "v3"),如果不指定则获取 v1 版本
|
||||
:param force: 是否强制刷新,忽略缓存
|
||||
"""
|
||||
# 如果强制刷新,直接调用不带缓存的版本
|
||||
if force:
|
||||
return self._get_plugins_uncached(repo_url, package_version)
|
||||
|
||||
# 正常情况下调用带缓存的版本
|
||||
return self._get_plugins_cached(repo_url, package_version)
|
||||
|
||||
@cached(maxsize=64, ttl=1800)
|
||||
def _get_plugins_cached(self, repo_url: str, package_version: Optional[str] = None) -> Optional[Dict[str, dict]]:
|
||||
"""
|
||||
获取Github所有最新插件列表(使用缓存)
|
||||
:param repo_url: Github仓库地址
|
||||
:param package_version: 首选插件版本 (如 "v2", "v3"),如果不指定则获取 v1 版本
|
||||
"""
|
||||
return self._get_plugins_uncached(repo_url, package_version)
|
||||
|
||||
def _get_plugins_uncached(self, repo_url: str, package_version: Optional[str] = None) -> Optional[Dict[str, dict]]:
|
||||
"""
|
||||
获取Github所有最新插件列表(不使用缓存)
|
||||
:param repo_url: Github仓库地址
|
||||
:param package_version: 首选插件版本 (如 "v2", "v3"),如果不指定则获取 v1 版本
|
||||
"""
|
||||
if not repo_url:
|
||||
return None
|
||||
@@ -161,7 +139,7 @@ class PluginHelper(metaclass=WeakSingleton):
|
||||
return res.json()
|
||||
return {}
|
||||
|
||||
def install_reg(self, pid: str) -> bool:
|
||||
def install_reg(self, pid: str, repo_url: Optional[str] = None) -> bool:
|
||||
"""
|
||||
安装插件统计
|
||||
"""
|
||||
@@ -170,24 +148,39 @@ class PluginHelper(metaclass=WeakSingleton):
|
||||
if not pid:
|
||||
return False
|
||||
install_reg_url = self._install_reg.format(pid=pid)
|
||||
res = RequestUtils(proxies=settings.PROXY, timeout=5).get_res(install_reg_url)
|
||||
res = RequestUtils(
|
||||
proxies=settings.PROXY,
|
||||
content_type="application/json",
|
||||
timeout=5
|
||||
).post(install_reg_url, json={
|
||||
"plugin_id": pid,
|
||||
"repo_url": repo_url
|
||||
})
|
||||
if res and res.status_code == 200:
|
||||
return True
|
||||
return False
|
||||
|
||||
def install_report(self) -> bool:
|
||||
def install_report(self, items: Optional[List[Tuple[str, Optional[str]]]] = None) -> bool:
|
||||
"""
|
||||
上报存量插件安装统计
|
||||
上报存量插件安装统计(批量)。支持上送 repo_url。
|
||||
:param items: 可选,形如 [(plugin_id, repo_url), ...];不传则回落到历史配置,仅上送 plugin_id。
|
||||
"""
|
||||
if not settings.PLUGIN_STATISTIC_SHARE:
|
||||
return False
|
||||
plugins = self.systemconfig.get(SystemConfigKey.UserInstalledPlugins)
|
||||
if not plugins:
|
||||
return False
|
||||
payload_plugins = []
|
||||
if items:
|
||||
for pid, repo_url in items:
|
||||
if pid:
|
||||
payload_plugins.append({"plugin_id": pid, "repo_url": repo_url})
|
||||
else:
|
||||
plugins = self.systemconfig.get(SystemConfigKey.UserInstalledPlugins)
|
||||
if not plugins:
|
||||
return False
|
||||
payload_plugins = [{"plugin_id": plugin, "repo_url": None} for plugin in plugins]
|
||||
res = RequestUtils(proxies=settings.PROXY,
|
||||
content_type="application/json",
|
||||
timeout=5).post(self._install_report,
|
||||
json={"plugins": [{"plugin_id": plugin} for plugin in plugins]})
|
||||
json={"plugins": payload_plugins})
|
||||
return True if res else False
|
||||
|
||||
def install(self, pid: str, repo_url: str, package_version: Optional[str] = None, force_install: bool = False) \
|
||||
@@ -248,19 +241,20 @@ class PluginHelper(metaclass=WeakSingleton):
|
||||
return False, f"未在插件清单中找到 {pid} 的版本号,无法进行 Release 安装"
|
||||
# 拼接 release_tag
|
||||
release_tag = f"{pid}_v{plugin_version}"
|
||||
|
||||
# 使用 release 进行安装
|
||||
def prepare_release() -> Tuple[bool, str]:
|
||||
return self.__install_from_release(
|
||||
pid.lower(), user_repo, release_tag
|
||||
pid, user_repo, release_tag
|
||||
)
|
||||
|
||||
return self.__install_flow_sync(pid.lower(), force_install, prepare_release)
|
||||
return self.__install_flow_sync(pid, force_install, prepare_release, repo_url)
|
||||
else:
|
||||
# 如果 release_tag 不存在,说明插件没有发布版本,使用文件列表方式安装
|
||||
def prepare_filelist() -> Tuple[bool, str]:
|
||||
return self.__prepare_content_via_filelist_sync(pid.lower(), user_repo, package_version)
|
||||
|
||||
return self.__install_flow_sync(pid.lower(), force_install, prepare_filelist)
|
||||
return self.__install_flow_sync(pid, force_install, prepare_filelist, repo_url)
|
||||
|
||||
def __get_file_list(self, pid: str, user_repo: str, package_version: Optional[str] = None) -> \
|
||||
Tuple[Optional[list], Optional[str]]:
|
||||
@@ -274,7 +268,7 @@ class PluginHelper(metaclass=WeakSingleton):
|
||||
# 如果 package_version 存在(如 "v2"),则加上版本号
|
||||
if package_version:
|
||||
file_api += f".{package_version}"
|
||||
file_api += f"/{pid}"
|
||||
file_api += f"/{pid.lower()}"
|
||||
|
||||
res = self.__request_with_fallback(file_api,
|
||||
headers=settings.REPO_GITHUB_HEADERS(repo=user_repo),
|
||||
@@ -407,8 +401,8 @@ class PluginHelper(metaclass=WeakSingleton):
|
||||
:param pid: 插件 ID
|
||||
:return: 备份目录路径
|
||||
"""
|
||||
plugin_dir = PLUGIN_DIR / pid
|
||||
backup_dir = Path(settings.TEMP_PATH) / "plugin_backup" / pid
|
||||
plugin_dir = PLUGIN_DIR / pid.lower()
|
||||
backup_dir = Path(settings.TEMP_PATH) / "plugin_backup" / pid.lower()
|
||||
|
||||
if plugin_dir.exists():
|
||||
# 备份时清理已有的备份目录,防止残留文件影响
|
||||
@@ -428,7 +422,7 @@ class PluginHelper(metaclass=WeakSingleton):
|
||||
:param pid: 插件 ID
|
||||
:param backup_dir: 备份目录路径
|
||||
"""
|
||||
plugin_dir = PLUGIN_DIR / pid
|
||||
plugin_dir = PLUGIN_DIR / pid.lower()
|
||||
if plugin_dir.exists():
|
||||
shutil.rmtree(plugin_dir, ignore_errors=True)
|
||||
logger.debug(f"{pid} 已清理插件目录 {plugin_dir}")
|
||||
@@ -445,7 +439,7 @@ class PluginHelper(metaclass=WeakSingleton):
|
||||
删除旧插件
|
||||
:param pid: 插件 ID
|
||||
"""
|
||||
plugin_dir = PLUGIN_DIR / pid
|
||||
plugin_dir = PLUGIN_DIR / pid.lower()
|
||||
if plugin_dir.exists():
|
||||
shutil.rmtree(plugin_dir, ignore_errors=True)
|
||||
|
||||
@@ -456,7 +450,18 @@ class PluginHelper(metaclass=WeakSingleton):
|
||||
:param requirements_file: 依赖的 requirements.txt 文件路径
|
||||
:return: (是否成功, 错误信息)
|
||||
"""
|
||||
base_cmd = [sys.executable, "-m", "pip", "install", "-r", str(requirements_file)]
|
||||
wheels_dir = requirements_file.parent / "wheels"
|
||||
|
||||
find_links_option = []
|
||||
if wheels_dir.is_dir():
|
||||
# 如果目录存在,增加 --find-links 选项
|
||||
logger.debug(f"[PIP] 发现插件内嵌的 wheels 目录: {wheels_dir},将优先从本地安装。")
|
||||
find_links_option = ["--find-links", str(wheels_dir)]
|
||||
else:
|
||||
# 如果不存在,选项为空列表,对后续命令无影响
|
||||
logger.debug(f"[PIP] 未发现插件内嵌的 wheels 目录,将仅使用在线源。")
|
||||
|
||||
base_cmd = [sys.executable, "-m", "pip", "install"] + find_links_option + ["-r", str(requirements_file)]
|
||||
strategies = []
|
||||
|
||||
# 添加策略到列表中
|
||||
@@ -533,53 +538,54 @@ class PluginHelper(metaclass=WeakSingleton):
|
||||
return None
|
||||
|
||||
def __get_plugin_meta(self, pid: str, repo_url: str,
|
||||
package_version: Optional[str]) -> dict:
|
||||
package_version: Optional[str]) -> dict:
|
||||
try:
|
||||
plugins = (
|
||||
self.get_plugins(repo_url) if not package_version
|
||||
else self.get_plugins(repo_url, package_version)
|
||||
) or {}
|
||||
self.get_plugins(repo_url) if not package_version
|
||||
else self.get_plugins(repo_url, package_version)
|
||||
) or {}
|
||||
meta = plugins.get(pid)
|
||||
return meta if isinstance(meta, dict) else {}
|
||||
except Exception as e:
|
||||
logger.error(f"获取插件 {pid} 元数据失败:{e}")
|
||||
return {}
|
||||
|
||||
def __install_flow_sync(self, pid_lower: str, force_install: bool,
|
||||
prepare_content: Callable[[], Tuple[bool, str]]) -> Tuple[bool, str]:
|
||||
def __install_flow_sync(self, pid: str, force_install: bool,
|
||||
prepare_content: Callable[[], Tuple[bool, str]],
|
||||
repo_url: Optional[str] = None) -> Tuple[bool, str]:
|
||||
"""
|
||||
同步安装统一流程:备份→清理→准备内容→安装依赖→上报
|
||||
prepare_content 负责把插件文件放到 app/plugins/{pid}
|
||||
"""
|
||||
backup_dir = None
|
||||
if not force_install:
|
||||
backup_dir = self.__backup_plugin(pid_lower)
|
||||
backup_dir = self.__backup_plugin(pid)
|
||||
|
||||
self.__remove_old_plugin(pid_lower)
|
||||
self.__remove_old_plugin(pid)
|
||||
|
||||
success, message = prepare_content()
|
||||
if not success:
|
||||
logger.error(f"{pid_lower} 准备插件内容失败:{message}")
|
||||
logger.error(f"{pid} 准备插件内容失败:{message}")
|
||||
if backup_dir:
|
||||
self.__restore_plugin(pid_lower, backup_dir)
|
||||
logger.warning(f"{pid_lower} 插件安装失败,已还原备份插件")
|
||||
self.__restore_plugin(pid, backup_dir)
|
||||
logger.warning(f"{pid} 插件安装失败,已还原备份插件")
|
||||
else:
|
||||
self.__remove_old_plugin(pid_lower)
|
||||
logger.warning(f"{pid_lower} 已清理对应插件目录,请尝试重新安装")
|
||||
self.__remove_old_plugin(pid)
|
||||
logger.warning(f"{pid} 已清理对应插件目录,请尝试重新安装")
|
||||
return False, message
|
||||
|
||||
dependencies_exist, dep_ok, dep_msg = self.__install_dependencies_if_required(pid_lower)
|
||||
dependencies_exist, dep_ok, dep_msg = self.__install_dependencies_if_required(pid)
|
||||
if dependencies_exist and not dep_ok:
|
||||
logger.error(f"{pid_lower} 依赖安装失败:{dep_msg}")
|
||||
logger.error(f"{pid} 依赖安装失败:{dep_msg}")
|
||||
if backup_dir:
|
||||
self.__restore_plugin(pid_lower, backup_dir)
|
||||
logger.warning(f"{pid_lower} 插件安装失败,已还原备份插件")
|
||||
self.__restore_plugin(pid, backup_dir)
|
||||
logger.warning(f"{pid} 插件安装失败,已还原备份插件")
|
||||
else:
|
||||
self.__remove_old_plugin(pid_lower)
|
||||
logger.warning(f"{pid_lower} 已清理对应插件目录,请尝试重新安装")
|
||||
self.__remove_old_plugin(pid)
|
||||
logger.warning(f"{pid} 已清理对应插件目录,请尝试重新安装")
|
||||
return False, dep_msg
|
||||
|
||||
self.install_reg(pid_lower)
|
||||
self.install_reg(pid, repo_url)
|
||||
return True, ""
|
||||
|
||||
def __install_from_release(self, pid: str, user_repo: str, release_tag: str) -> Tuple[bool, str]:
|
||||
@@ -607,14 +613,19 @@ class PluginHelper(metaclass=WeakSingleton):
|
||||
asset = next((a for a in assets if a.get("name") == asset_name), None)
|
||||
if not asset:
|
||||
return False, f"未找到资产文件:{asset_name}"
|
||||
download_url = asset.get("browser_download_url")
|
||||
if not download_url:
|
||||
return False, "资产缺少下载地址"
|
||||
asset_id = asset.get("id")
|
||||
if not asset_id:
|
||||
return False, "资产缺少ID信息"
|
||||
# 构建资产的API下载URL
|
||||
download_url = f"https://api.github.com/repos/{user_repo}/releases/assets/{asset_id}"
|
||||
except Exception as e:
|
||||
logger.error(f"解析 Release 信息失败:{e}")
|
||||
return False, f"解析 Release 信息失败:{e}"
|
||||
|
||||
res = self.__request_with_fallback(download_url, headers=settings.REPO_GITHUB_HEADERS(repo=user_repo))
|
||||
# 使用资产的API端点下载,需要设置Accept头为application/octet-stream
|
||||
headers = settings.REPO_GITHUB_HEADERS(repo=user_repo).copy()
|
||||
headers["Accept"] = "application/octet-stream"
|
||||
res = self.__request_with_fallback(download_url, headers=headers, is_api=True)
|
||||
if res is None or res.status_code != 200:
|
||||
return False, f"下载资产失败:{res.status_code if res else '连接失败'}"
|
||||
|
||||
@@ -898,35 +909,13 @@ class PluginHelper(metaclass=WeakSingleton):
|
||||
logger.error(f"[GitHub] 所有策略均请求失败,URL: {url},请检查网络连接或 GitHub 配置")
|
||||
return None
|
||||
|
||||
async def async_get_plugins(self, repo_url: str, package_version: Optional[str] = None,
|
||||
force: bool = False) -> Optional[Dict[str, dict]]:
|
||||
@cached(maxsize=128, ttl=1800)
|
||||
async def async_get_plugins(self, repo_url: str,
|
||||
package_version: Optional[str] = None) -> Optional[Dict[str, dict]]:
|
||||
"""
|
||||
异步获取Github所有最新插件列表
|
||||
:param repo_url: Github仓库地址
|
||||
:param package_version: 首选插件版本 (如 "v2", "v3"),如果不指定则获取 v1 版本
|
||||
:param force: 是否强制刷新,忽略缓存
|
||||
"""
|
||||
# 异步版本直接调用不带缓存的版本(缓存在异步环境下可能有并发问题)
|
||||
if force:
|
||||
return await self._async_get_plugins_uncached(repo_url, package_version)
|
||||
return await self._async_get_plugins_cached(repo_url, package_version)
|
||||
|
||||
@cached(maxsize=64, ttl=1800)
|
||||
async def _async_get_plugins_cached(self, repo_url: str,
|
||||
package_version: Optional[str] = None) -> Optional[Dict[str, dict]]:
|
||||
"""
|
||||
获取Github所有最新插件列表(使用缓存)
|
||||
:param repo_url: Github仓库地址
|
||||
:param package_version: 首选插件版本 (如 "v2", "v3"),如果不指定则获取 v1 版本
|
||||
"""
|
||||
return await self._async_get_plugins_uncached(repo_url, package_version)
|
||||
|
||||
async def _async_get_plugins_uncached(self, repo_url: str,
|
||||
package_version: Optional[str] = None) -> Optional[Dict[str, dict]]:
|
||||
"""
|
||||
异步获取Github所有最新插件列表(不使用缓存)
|
||||
:param repo_url: Github仓库地址
|
||||
:param package_version: 首选插件版本 (如 "v2", "v3"),如果不指定则获取 v1 版本
|
||||
"""
|
||||
if not repo_url:
|
||||
return None
|
||||
@@ -963,7 +952,7 @@ class PluginHelper(metaclass=WeakSingleton):
|
||||
return res.json()
|
||||
return {}
|
||||
|
||||
async def async_install_reg(self, pid: str) -> bool:
|
||||
async def async_install_reg(self, pid: str, repo_url: Optional[str] = None) -> bool:
|
||||
"""
|
||||
异步安装插件统计
|
||||
"""
|
||||
@@ -972,24 +961,39 @@ class PluginHelper(metaclass=WeakSingleton):
|
||||
if not pid:
|
||||
return False
|
||||
install_reg_url = self._install_reg.format(pid=pid)
|
||||
res = await AsyncRequestUtils(proxies=settings.PROXY, timeout=5).get_res(install_reg_url)
|
||||
res = await AsyncRequestUtils(
|
||||
proxies=settings.PROXY,
|
||||
content_type="application/json",
|
||||
timeout=5
|
||||
).post(install_reg_url, json={
|
||||
"plugin_id": pid,
|
||||
"repo_url": repo_url
|
||||
})
|
||||
if res and res.status_code == 200:
|
||||
return True
|
||||
return False
|
||||
|
||||
async def async_install_report(self) -> bool:
|
||||
async def async_install_report(self, items: Optional[List[Tuple[str, Optional[str]]]] = None) -> bool:
|
||||
"""
|
||||
异步上报存量插件安装统计
|
||||
异步上报存量插件安装统计(批量)。支持上送 repo_url。
|
||||
:param items: 可选,形如 [(plugin_id, repo_url), ...];不传则回落到历史配置,仅上送 plugin_id。
|
||||
"""
|
||||
if not settings.PLUGIN_STATISTIC_SHARE:
|
||||
return False
|
||||
plugins = self.systemconfig.get(SystemConfigKey.UserInstalledPlugins)
|
||||
if not plugins:
|
||||
return False
|
||||
payload_plugins = []
|
||||
if items:
|
||||
for pid, repo_url in items:
|
||||
if pid:
|
||||
payload_plugins.append({"plugin_id": pid, "repo_url": repo_url})
|
||||
else:
|
||||
plugins = self.systemconfig.get(SystemConfigKey.UserInstalledPlugins)
|
||||
if not plugins:
|
||||
return False
|
||||
payload_plugins = [{"plugin_id": plugin, "repo_url": None} for plugin in plugins]
|
||||
res = await AsyncRequestUtils(proxies=settings.PROXY,
|
||||
content_type="application/json",
|
||||
timeout=5).post(self._install_report,
|
||||
json={"plugins": [{"plugin_id": plugin} for plugin in plugins]})
|
||||
json={"plugins": payload_plugins})
|
||||
return True if res else False
|
||||
|
||||
async def __async_get_file_list(self, pid: str, user_repo: str, package_version: Optional[str] = None) -> \
|
||||
@@ -1004,7 +1008,7 @@ class PluginHelper(metaclass=WeakSingleton):
|
||||
# 如果 package_version 存在(如 "v2"),则加上版本号
|
||||
if package_version:
|
||||
file_api += f".{package_version}"
|
||||
file_api += f"/{pid}"
|
||||
file_api += f"/{pid.lower()}"
|
||||
|
||||
res = await self.__async_request_with_fallback(file_api,
|
||||
headers=settings.REPO_GITHUB_HEADERS(repo=user_repo),
|
||||
@@ -1116,8 +1120,8 @@ class PluginHelper(metaclass=WeakSingleton):
|
||||
:param pid: 插件 ID
|
||||
:return: 备份目录路径
|
||||
"""
|
||||
plugin_dir = AsyncPath(PLUGIN_DIR) / pid
|
||||
backup_dir = AsyncPath(settings.TEMP_PATH) / "plugin_backup" / pid
|
||||
plugin_dir = AsyncPath(PLUGIN_DIR) / pid.lower()
|
||||
backup_dir = AsyncPath(settings.TEMP_PATH) / "plugin_backup" / pid.lower()
|
||||
|
||||
if await plugin_dir.exists():
|
||||
# 备份时清理已有的备份目录,防止残留文件影响
|
||||
@@ -1137,7 +1141,7 @@ class PluginHelper(metaclass=WeakSingleton):
|
||||
:param pid: 插件 ID
|
||||
:param backup_dir: 备份目录路径
|
||||
"""
|
||||
plugin_dir = AsyncPath(PLUGIN_DIR) / pid
|
||||
plugin_dir = AsyncPath(PLUGIN_DIR) / pid.lower()
|
||||
if await plugin_dir.exists():
|
||||
await aioshutil.rmtree(plugin_dir, ignore_errors=True)
|
||||
logger.debug(f"{pid} 已清理插件目录 {plugin_dir}")
|
||||
@@ -1155,7 +1159,7 @@ class PluginHelper(metaclass=WeakSingleton):
|
||||
异步删除旧插件
|
||||
:param pid: 插件 ID
|
||||
"""
|
||||
plugin_dir = AsyncPath(PLUGIN_DIR) / pid
|
||||
plugin_dir = AsyncPath(PLUGIN_DIR) / pid.lower()
|
||||
if await plugin_dir.exists():
|
||||
await aioshutil.rmtree(plugin_dir, ignore_errors=True)
|
||||
|
||||
@@ -1393,105 +1397,107 @@ class PluginHelper(metaclass=WeakSingleton):
|
||||
return False, f"未在插件清单中找到 {pid} 的版本号,无法进行 Release 安装"
|
||||
# 拼接 release_tag
|
||||
release_tag = f"{pid}_v{plugin_version}"
|
||||
|
||||
# 使用 release 进行安装
|
||||
async def prepare_release() -> Tuple[bool, str]:
|
||||
return await self.__async_install_from_release(
|
||||
pid.lower(), user_repo, release_tag
|
||||
pid, user_repo, release_tag
|
||||
)
|
||||
|
||||
return await self.__install_flow_async(pid.lower(), force_install, prepare_release)
|
||||
return await self.__install_flow_async(pid, force_install, prepare_release, repo_url)
|
||||
else:
|
||||
# 如果没有 release_tag,则使用文件列表安装方式
|
||||
async def prepare_filelist() -> Tuple[bool, str]:
|
||||
return await self.__prepare_content_via_filelist_async(pid.lower(), user_repo, package_version)
|
||||
return await self.__prepare_content_via_filelist_async(pid, user_repo, package_version)
|
||||
|
||||
return await self.__install_flow_async(pid.lower(), force_install, prepare_filelist)
|
||||
return await self.__install_flow_async(pid, force_install, prepare_filelist, repo_url)
|
||||
|
||||
async def __async_get_plugin_meta(self, pid: str, repo_url: str,
|
||||
package_version: Optional[str]) -> dict:
|
||||
try:
|
||||
plugins = (
|
||||
await self.async_get_plugins(repo_url) if not package_version
|
||||
else await self.async_get_plugins(repo_url, package_version)
|
||||
) or {}
|
||||
await self.async_get_plugins(repo_url) if not package_version
|
||||
else await self.async_get_plugins(repo_url, package_version)
|
||||
) or {}
|
||||
meta = plugins.get(pid)
|
||||
return meta if isinstance(meta, dict) else {}
|
||||
except Exception as e:
|
||||
logger.warn(f"获取插件 {pid} 元数据失败:{e}")
|
||||
return {}
|
||||
|
||||
async def __install_flow_async(self, pid_lower: str, force_install: bool,
|
||||
prepare_content: Callable[[], Awaitable[Tuple[bool, str]]]) -> Tuple[bool, str]:
|
||||
async def __install_flow_async(self, pid: str, force_install: bool,
|
||||
prepare_content: Callable[[], Awaitable[Tuple[bool, str]]],
|
||||
repo_url: Optional[str] = None) -> Tuple[bool, str]:
|
||||
"""
|
||||
异步安装流程,处理插件内容准备、依赖安装和注册
|
||||
"""
|
||||
backup_dir = None
|
||||
if not force_install:
|
||||
backup_dir = await self.__async_backup_plugin(pid_lower)
|
||||
backup_dir = await self.__async_backup_plugin(pid)
|
||||
|
||||
await self.__async_remove_old_plugin(pid_lower)
|
||||
await self.__async_remove_old_plugin(pid)
|
||||
|
||||
success, message = await prepare_content()
|
||||
if not success:
|
||||
logger.error(f"{pid_lower} 准备插件内容失败:{message}")
|
||||
logger.error(f"{pid} 准备插件内容失败:{message}")
|
||||
if backup_dir:
|
||||
await self.__async_restore_plugin(pid_lower, backup_dir)
|
||||
logger.warning(f"{pid_lower} 插件安装失败,已还原备份插件")
|
||||
await self.__async_restore_plugin(pid, backup_dir)
|
||||
logger.warning(f"{pid} 插件安装失败,已还原备份插件")
|
||||
else:
|
||||
await self.__async_remove_old_plugin(pid_lower)
|
||||
logger.warning(f"{pid_lower} 已清理对应插件目录,请尝试重新安装")
|
||||
await self.__async_remove_old_plugin(pid)
|
||||
logger.warning(f"{pid} 已清理对应插件目录,请尝试重新安装")
|
||||
return False, message
|
||||
|
||||
dependencies_exist, dep_ok, dep_msg = await self.__async_install_dependencies_if_required(pid_lower)
|
||||
dependencies_exist, dep_ok, dep_msg = await self.__async_install_dependencies_if_required(pid)
|
||||
if dependencies_exist and not dep_ok:
|
||||
logger.error(f"{pid_lower} 依赖安装失败:{dep_msg}")
|
||||
logger.error(f"{pid} 依赖安装失败:{dep_msg}")
|
||||
if backup_dir:
|
||||
await self.__async_restore_plugin(pid_lower, backup_dir)
|
||||
logger.warning(f"{pid_lower} 插件安装失败,已还原备份插件")
|
||||
await self.__async_restore_plugin(pid, backup_dir)
|
||||
logger.warning(f"{pid} 插件安装失败,已还原备份插件")
|
||||
else:
|
||||
await self.__async_remove_old_plugin(pid_lower)
|
||||
logger.warning(f"{pid_lower} 已清理对应插件目录,请尝试重新安装")
|
||||
await self.__async_remove_old_plugin(pid)
|
||||
logger.warning(f"{pid} 已清理对应插件目录,请尝试重新安装")
|
||||
return False, dep_msg
|
||||
|
||||
await self.async_install_reg(pid_lower)
|
||||
await self.async_install_reg(pid, repo_url)
|
||||
return True, ""
|
||||
|
||||
def __prepare_content_via_filelist_sync(self, pid_lower: str, user_repo: str,
|
||||
def __prepare_content_via_filelist_sync(self, pid: str, user_repo: str,
|
||||
package_version: Optional[str]) -> Tuple[bool, str]:
|
||||
"""
|
||||
同步准备插件内容,通过文件列表获取插件文件和依赖
|
||||
"""
|
||||
file_list, msg = self.__get_file_list(pid_lower, user_repo, package_version)
|
||||
file_list, msg = self.__get_file_list(pid, user_repo, package_version)
|
||||
if not file_list:
|
||||
return False, msg
|
||||
requirements_file_info = next((f for f in file_list if f.get("name") == "requirements.txt"), None)
|
||||
if requirements_file_info:
|
||||
ok, m = self.__download_and_install_requirements(requirements_file_info, pid_lower, user_repo)
|
||||
ok, m = self.__download_and_install_requirements(requirements_file_info, pid, user_repo)
|
||||
if not ok:
|
||||
logger.debug(f"{pid_lower} 依赖预安装失败:{m}")
|
||||
logger.debug(f"{pid} 依赖预安装失败:{m}")
|
||||
else:
|
||||
logger.debug(f"{pid_lower} 依赖预安装成功")
|
||||
ok, m = self.__download_files(pid_lower, file_list, user_repo, package_version, True)
|
||||
logger.debug(f"{pid} 依赖预安装成功")
|
||||
ok, m = self.__download_files(pid, file_list, user_repo, package_version, True)
|
||||
if not ok:
|
||||
return False, m
|
||||
return True, ""
|
||||
|
||||
async def __prepare_content_via_filelist_async(self, pid_lower: str, user_repo: str,
|
||||
async def __prepare_content_via_filelist_async(self, pid: str, user_repo: str,
|
||||
package_version: Optional[str]) -> Tuple[bool, str]:
|
||||
"""
|
||||
异步准备插件内容,通过文件列表获取插件文件和依赖
|
||||
"""
|
||||
file_list, msg = await self.__async_get_file_list(pid_lower, user_repo, package_version)
|
||||
file_list, msg = await self.__async_get_file_list(pid, user_repo, package_version)
|
||||
if not file_list:
|
||||
return False, msg
|
||||
requirements_file_info = next((f for f in file_list if f.get("name") == "requirements.txt"), None)
|
||||
if requirements_file_info:
|
||||
ok, m = await self.__async_download_and_install_requirements(requirements_file_info, pid_lower, user_repo)
|
||||
ok, m = await self.__async_download_and_install_requirements(requirements_file_info, pid, user_repo)
|
||||
if not ok:
|
||||
logger.debug(f"{pid_lower} 依赖预安装失败:{m}")
|
||||
logger.debug(f"{pid} 依赖预安装失败:{m}")
|
||||
else:
|
||||
logger.debug(f"{pid_lower} 依赖预安装成功")
|
||||
ok, m = await self.__async_download_files(pid_lower, file_list, user_repo, package_version, True)
|
||||
logger.debug(f"{pid} 依赖预安装成功")
|
||||
ok, m = await self.__async_download_files(pid, file_list, user_repo, package_version, True)
|
||||
if not ok:
|
||||
return False, m
|
||||
return True, ""
|
||||
@@ -1521,14 +1527,21 @@ class PluginHelper(metaclass=WeakSingleton):
|
||||
asset = next((a for a in assets if a.get("name") == asset_name), None)
|
||||
if not asset:
|
||||
return False, f"未找到资产文件:{asset_name}"
|
||||
download_url = asset.get("browser_download_url")
|
||||
if not download_url:
|
||||
return False, "资产缺少下载地址"
|
||||
asset_id = asset.get("id")
|
||||
if not asset_id:
|
||||
return False, "资产缺少ID信息"
|
||||
# 构建资产的API下载URL
|
||||
download_url = f"https://api.github.com/repos/{user_repo}/releases/assets/{asset_id}"
|
||||
except Exception as e:
|
||||
logger.error(f"解析 Release 信息失败:{e}")
|
||||
return False, f"解析 Release 信息失败:{e}"
|
||||
|
||||
res = await self.__async_request_with_fallback(download_url, headers=settings.REPO_GITHUB_HEADERS(repo=user_repo))
|
||||
# 使用资产的API端点下载,需要设置Accept头为application/octet-stream
|
||||
headers = settings.REPO_GITHUB_HEADERS(repo=user_repo).copy()
|
||||
headers["Accept"] = "application/octet-stream"
|
||||
res = await self.__async_request_with_fallback(download_url,
|
||||
headers=headers,
|
||||
is_api=True)
|
||||
if res is None or res.status_code != 200:
|
||||
return False, f"下载资产失败:{res.status_code if res else '连接失败'}"
|
||||
|
||||
|
||||
@@ -1,55 +1,76 @@
|
||||
from enum import Enum
|
||||
from typing import Union, Optional
|
||||
|
||||
from app.core.cache import TTLCache
|
||||
from app.schemas.types import ProgressKey
|
||||
from app.utils.singleton import WeakSingleton
|
||||
|
||||
|
||||
class ProgressHelper(metaclass=WeakSingleton):
|
||||
"""
|
||||
处理进度辅助类
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
self._process_detail = {}
|
||||
|
||||
def init_config(self):
|
||||
pass
|
||||
|
||||
def __reset(self, key: Union[ProgressKey, str]):
|
||||
def __init__(self, key: Union[ProgressKey, str]):
|
||||
if isinstance(key, Enum):
|
||||
key = key.value
|
||||
self._process_detail[key] = {
|
||||
self._key = key
|
||||
self._progress = TTLCache(region="progress", maxsize=1024, ttl=24 * 60 * 60)
|
||||
|
||||
def __reset(self):
|
||||
"""
|
||||
重置进度
|
||||
"""
|
||||
self._progress[self._key] = {
|
||||
"enable": False,
|
||||
"value": 0,
|
||||
"text": "请稍候..."
|
||||
"text": "请稍候...",
|
||||
"data": {}
|
||||
}
|
||||
|
||||
def start(self, key: Union[ProgressKey, str]):
|
||||
self.__reset(key)
|
||||
if isinstance(key, Enum):
|
||||
key = key.value
|
||||
self._process_detail[key]['enable'] = True
|
||||
|
||||
def end(self, key: Union[ProgressKey, str]):
|
||||
if isinstance(key, Enum):
|
||||
key = key.value
|
||||
if not self._process_detail.get(key):
|
||||
def start(self):
|
||||
"""
|
||||
开始进度
|
||||
"""
|
||||
self.__reset()
|
||||
current = self._progress.get(self._key)
|
||||
if not current:
|
||||
return
|
||||
self._process_detail[key] = {
|
||||
"enable": False,
|
||||
"value": 100,
|
||||
"text": "正在处理..."
|
||||
}
|
||||
current['enable'] = True
|
||||
self._progress[self._key] = current
|
||||
|
||||
def update(self, key: Union[ProgressKey, str], value: Union[float, int] = None, text: Optional[str] = None):
|
||||
if isinstance(key, Enum):
|
||||
key = key.value
|
||||
if not self._process_detail.get(key, {}).get('enable'):
|
||||
def end(self):
|
||||
"""
|
||||
结束进度
|
||||
"""
|
||||
current = self._progress.get(self._key)
|
||||
if not current:
|
||||
return
|
||||
current.update(
|
||||
{
|
||||
"enable": False,
|
||||
"value": 100,
|
||||
"text": ""
|
||||
}
|
||||
)
|
||||
self._progress[self._key] = current
|
||||
|
||||
def update(self, value: Union[float, int] = None, text: Optional[str] = None, data: dict = None):
|
||||
"""
|
||||
更新进度
|
||||
"""
|
||||
current = self._progress.get(self._key)
|
||||
if not current or not current.get('enable'):
|
||||
return
|
||||
if value:
|
||||
self._process_detail[key]['value'] = value
|
||||
current['value'] = value
|
||||
if text:
|
||||
self._process_detail[key]['text'] = text
|
||||
current['text'] = text
|
||||
if data:
|
||||
if not current.get('data'):
|
||||
current['data'] = {}
|
||||
current['data'].update(data)
|
||||
self._progress[self._key] = current
|
||||
|
||||
def get(self, key: Union[ProgressKey, str]) -> dict:
|
||||
if isinstance(key, Enum):
|
||||
key = key.value
|
||||
return self._process_detail.get(key)
|
||||
def get(self) -> dict:
|
||||
return self._progress.get(self._key)
|
||||
|
||||
548
app/helper/redis.py
Normal file
548
app/helper/redis.py
Normal file
@@ -0,0 +1,548 @@
|
||||
import json
|
||||
import pickle
|
||||
from typing import Any, Optional, Generator, Tuple, AsyncGenerator, Union
|
||||
from urllib.parse import quote
|
||||
|
||||
import redis
|
||||
from redis.asyncio import Redis
|
||||
|
||||
from app.core.config import settings
|
||||
from app.core.event import eventmanager, Event
|
||||
from app.log import logger
|
||||
from app.schemas import ConfigChangeEventData
|
||||
from app.schemas.types import EventType
|
||||
from app.utils.singleton import Singleton
|
||||
|
||||
# 类型缓存集合,针对非容器简单类型
|
||||
_complex_serializable_types = set()
|
||||
_simple_serializable_types = set()
|
||||
|
||||
# 默认连接参数
|
||||
_socket_timeout = 30
|
||||
_socket_connect_timeout = 5
|
||||
_health_check_interval = 60
|
||||
|
||||
|
||||
def serialize(value: Any) -> bytes:
|
||||
"""
|
||||
将值序列化为二进制数据,根据序列化方式标识格式
|
||||
"""
|
||||
|
||||
def _is_container_type(t):
|
||||
"""
|
||||
判断是否为容器类型
|
||||
"""
|
||||
return t in (list, dict, tuple, set)
|
||||
|
||||
vt = type(value)
|
||||
# 针对非容器类型使用缓存策略
|
||||
if not _is_container_type(vt):
|
||||
# 如果已知需要复杂序列化
|
||||
if vt in _complex_serializable_types:
|
||||
return b"PICKLE" + b"\x00" + pickle.dumps(value)
|
||||
# 如果已知可以简单序列化
|
||||
if vt in _simple_serializable_types:
|
||||
json_data = json.dumps(value).encode("utf-8")
|
||||
return b"JSON" + b"\x00" + json_data
|
||||
# 对于未知的非容器类型,尝试简单序列化,如抛出异常,再使用复杂序列化
|
||||
try:
|
||||
json_data = json.dumps(value).encode("utf-8")
|
||||
_simple_serializable_types.add(vt)
|
||||
return b"JSON" + b"\x00" + json_data
|
||||
except TypeError:
|
||||
_complex_serializable_types.add(vt)
|
||||
return b"PICKLE" + b"\x00" + pickle.dumps(value)
|
||||
else:
|
||||
# 针对容器类型,每次尝试简单序列化,不使用缓存
|
||||
try:
|
||||
json_data = json.dumps(value).encode("utf-8")
|
||||
return b"JSON" + b"\x00" + json_data
|
||||
except TypeError:
|
||||
return b"PICKLE" + b"\x00" + pickle.dumps(value)
|
||||
|
||||
|
||||
def deserialize(value: bytes) -> Any:
|
||||
"""
|
||||
将二进制数据反序列化为原始值,根据格式标识区分序列化方式
|
||||
"""
|
||||
format_marker, data = value.split(b"\x00", 1)
|
||||
if format_marker == b"JSON":
|
||||
return json.loads(data.decode("utf-8"))
|
||||
elif format_marker == b"PICKLE":
|
||||
return pickle.loads(data)
|
||||
else:
|
||||
raise ValueError("Unknown serialization format")
|
||||
|
||||
|
||||
class RedisHelper(metaclass=Singleton):
|
||||
"""
|
||||
Redis连接和操作助手类,单例模式
|
||||
|
||||
特性:
|
||||
- 管理Redis连接池和客户端
|
||||
- 提供序列化和反序列化功能
|
||||
- 支持内存限制和淘汰策略设置
|
||||
- 提供键名生成和区域管理功能
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
"""
|
||||
初始化Redis助手实例
|
||||
"""
|
||||
self.redis_url = settings.CACHE_BACKEND_URL
|
||||
self.client = None
|
||||
|
||||
def _connect(self):
|
||||
"""
|
||||
建立Redis连接
|
||||
"""
|
||||
try:
|
||||
if self.client is None:
|
||||
self.client = redis.Redis.from_url(
|
||||
self.redis_url,
|
||||
decode_responses=False,
|
||||
socket_timeout=_socket_timeout,
|
||||
socket_connect_timeout=_socket_connect_timeout,
|
||||
health_check_interval=_health_check_interval,
|
||||
)
|
||||
# 测试连接,确保Redis可用
|
||||
self.client.ping()
|
||||
logger.info(f"Successfully connected to Redis:{self.redis_url}")
|
||||
self.set_memory_limit()
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to connect to Redis: {e}")
|
||||
self.client = None
|
||||
raise RuntimeError("Redis connection failed") from e
|
||||
|
||||
@eventmanager.register(EventType.ConfigChanged)
|
||||
def handle_config_changed(self, event: Event):
|
||||
"""
|
||||
处理配置变更事件,更新Redis设置
|
||||
:param event: 事件对象
|
||||
"""
|
||||
if not event:
|
||||
return
|
||||
event_data: ConfigChangeEventData = event.event_data
|
||||
if event_data.key not in ['CACHE_BACKEND_TYPE', 'CACHE_BACKEND_URL', 'CACHE_REDIS_MAXMEMORY']:
|
||||
return
|
||||
logger.info("配置变更,重连Redis...")
|
||||
self.close()
|
||||
self._connect()
|
||||
|
||||
def set_memory_limit(self, policy: Optional[str] = "allkeys-lru"):
|
||||
"""
|
||||
动态设置Redis最大内存和内存淘汰策略
|
||||
|
||||
:param policy: 淘汰策略(如'allkeys-lru')
|
||||
"""
|
||||
try:
|
||||
# 如果有显式值,则直接使用,为0时说明不限制,如果未配置,开启BIG_MEMORY_MODE时为"1024mb",未开启时为"256mb"
|
||||
maxmemory = settings.CACHE_REDIS_MAXMEMORY or ("1024mb" if settings.BIG_MEMORY_MODE else "256mb")
|
||||
self.client.config_set("maxmemory", maxmemory)
|
||||
self.client.config_set("maxmemory-policy", policy)
|
||||
logger.debug(f"Redis maxmemory set to {maxmemory}, policy: {policy}")
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to set Redis maxmemory or policy: {e}")
|
||||
|
||||
@staticmethod
|
||||
def __get_region(region: Optional[str] = None):
|
||||
"""
|
||||
获取缓存的区
|
||||
"""
|
||||
return f"region:{quote(region)}" if region else "region:DEFAULT"
|
||||
|
||||
def __make_redis_key(self, region: str, key: str) -> str:
|
||||
"""
|
||||
获取缓存Key
|
||||
"""
|
||||
# 使用region作为缓存键的一部分
|
||||
region = self.__get_region(region)
|
||||
return f"{region}:key:{quote(key)}"
|
||||
|
||||
@staticmethod
|
||||
def __get_original_key(redis_key: Union[str, bytes]) -> str:
|
||||
"""
|
||||
从Redis键中提取原始key
|
||||
"""
|
||||
try:
|
||||
if isinstance(redis_key, bytes):
|
||||
redis_key = redis_key.decode("utf-8")
|
||||
parts = redis_key.split(":key:")
|
||||
return parts[-1]
|
||||
except Exception as e:
|
||||
logger.warn(f"Failed to parse redis key: {redis_key}, error: {e}")
|
||||
return redis_key
|
||||
|
||||
def set(self, key: str, value: Any, ttl: Optional[int] = None,
|
||||
region: Optional[str] = "DEFAULT", **kwargs) -> None:
|
||||
"""
|
||||
设置缓存
|
||||
|
||||
:param key: 缓存的键
|
||||
:param value: 缓存的值
|
||||
:param ttl: 缓存的存活时间,单位秒
|
||||
:param region: 缓存的区
|
||||
:param kwargs: 其他参数
|
||||
"""
|
||||
try:
|
||||
self._connect()
|
||||
redis_key = self.__make_redis_key(region, key)
|
||||
# 对值进行序列化
|
||||
serialized_value = serialize(value)
|
||||
kwargs.pop("maxsize", None)
|
||||
self.client.set(redis_key, serialized_value, ex=ttl, **kwargs)
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to set key: {key} in region: {region}, error: {e}")
|
||||
|
||||
def exists(self, key: str, region: Optional[str] = "DEFAULT") -> bool:
|
||||
"""
|
||||
判断缓存键是否存在
|
||||
|
||||
:param key: 缓存的键
|
||||
:param region: 缓存的区
|
||||
:return: 存在返回True,否则返回False
|
||||
"""
|
||||
try:
|
||||
self._connect()
|
||||
redis_key = self.__make_redis_key(region, key)
|
||||
return self.client.exists(redis_key) == 1
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to exists key: {key} region: {region}, error: {e}")
|
||||
return False
|
||||
|
||||
def get(self, key: str, region: Optional[str] = "DEFAULT") -> Optional[Any]:
|
||||
"""
|
||||
获取缓存的值
|
||||
|
||||
:param key: 缓存的键
|
||||
:param region: 缓存的区
|
||||
:return: 返回缓存的值,如果缓存不存在返回None
|
||||
"""
|
||||
try:
|
||||
self._connect()
|
||||
redis_key = self.__make_redis_key(region, key)
|
||||
value = self.client.get(redis_key)
|
||||
if value is not None:
|
||||
return deserialize(value)
|
||||
return None
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to get key: {key} in region: {region}, error: {e}")
|
||||
return None
|
||||
|
||||
def delete(self, key: str, region: Optional[str] = "DEFAULT") -> None:
|
||||
"""
|
||||
删除缓存
|
||||
|
||||
:param key: 缓存的键
|
||||
:param region: 缓存的区
|
||||
"""
|
||||
try:
|
||||
self._connect()
|
||||
redis_key = self.__make_redis_key(region, key)
|
||||
self.client.delete(redis_key)
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to delete key: {key} in region: {region}, error: {e}")
|
||||
|
||||
def clear(self, region: Optional[str] = None) -> None:
|
||||
"""
|
||||
清除指定区域的缓存或全部缓存
|
||||
|
||||
:param region: 缓存的区
|
||||
"""
|
||||
try:
|
||||
self._connect()
|
||||
if region:
|
||||
cache_region = self.__get_region(region)
|
||||
redis_key = f"{cache_region}:key:*"
|
||||
with self.client.pipeline() as pipe:
|
||||
for key in self.client.scan_iter(redis_key):
|
||||
pipe.delete(key)
|
||||
pipe.execute()
|
||||
logger.debug(f"Cleared Redis cache for region: {region}")
|
||||
else:
|
||||
self.client.flushdb()
|
||||
logger.info("All Redis cache Cleared!")
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to clear cache, region: {region}, error: {e}")
|
||||
|
||||
def items(self, region: Optional[str] = None) -> Generator[Tuple[str, Any], None, None]:
|
||||
"""
|
||||
获取指定区域的所有缓存键值对
|
||||
|
||||
:param region: 缓存的区
|
||||
:return: 返回键值对生成器
|
||||
"""
|
||||
try:
|
||||
self._connect()
|
||||
if region:
|
||||
cache_region = self.__get_region(region)
|
||||
redis_key = f"{cache_region}:key:*"
|
||||
for key in self.client.scan_iter(redis_key):
|
||||
value = self.client.get(key)
|
||||
if value is not None:
|
||||
yield self.__get_original_key(key), deserialize(value)
|
||||
else:
|
||||
for key in self.client.scan_iter("*"):
|
||||
value = self.client.get(key)
|
||||
if value is not None:
|
||||
yield self.__get_original_key(key), deserialize(value)
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to get items from Redis, region: {region}, error: {e}")
|
||||
|
||||
def test(self) -> bool:
|
||||
"""
|
||||
测试Redis连接性
|
||||
"""
|
||||
try:
|
||||
self._connect()
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.error(f"Redis connection test failed: {e}")
|
||||
return False
|
||||
|
||||
def close(self) -> None:
|
||||
"""
|
||||
关闭Redis客户端的连接池
|
||||
"""
|
||||
if self.client:
|
||||
self.client.close()
|
||||
self.client = None
|
||||
logger.debug("Redis connection closed")
|
||||
|
||||
|
||||
class AsyncRedisHelper(metaclass=Singleton):
|
||||
"""
|
||||
异步Redis连接和操作助手类,单例模式
|
||||
|
||||
特性:
|
||||
- 管理异步Redis连接池和客户端
|
||||
- 提供序列化和反序列化功能
|
||||
- 支持内存限制和淘汰策略设置
|
||||
- 提供键名生成和区域管理功能
|
||||
- 所有操作都是异步的
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
"""
|
||||
初始化异步Redis助手实例
|
||||
"""
|
||||
self.redis_url = settings.CACHE_BACKEND_URL
|
||||
self.client: Optional[Redis] = None
|
||||
|
||||
async def _connect(self):
|
||||
"""
|
||||
建立异步Redis连接
|
||||
"""
|
||||
try:
|
||||
if self.client is None:
|
||||
self.client = Redis.from_url(
|
||||
self.redis_url,
|
||||
decode_responses=False,
|
||||
socket_timeout=_socket_timeout,
|
||||
socket_connect_timeout=_socket_connect_timeout,
|
||||
health_check_interval=_health_check_interval,
|
||||
)
|
||||
# 测试连接,确保Redis可用
|
||||
await self.client.ping()
|
||||
logger.info(f"Successfully connected to Redis (async):{self.redis_url}")
|
||||
await self.set_memory_limit()
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to connect to Redis (async): {e}")
|
||||
self.client = None
|
||||
raise RuntimeError("Redis async connection failed") from e
|
||||
|
||||
@eventmanager.register(EventType.ConfigChanged)
|
||||
async def handle_config_changed(self, event: Event):
|
||||
"""
|
||||
处理配置变更事件,更新Redis设置
|
||||
:param event: 事件对象
|
||||
"""
|
||||
if not event:
|
||||
return
|
||||
event_data: ConfigChangeEventData = event.event_data
|
||||
if event_data.key not in ['CACHE_BACKEND_TYPE', 'CACHE_BACKEND_URL', 'CACHE_REDIS_MAXMEMORY']:
|
||||
return
|
||||
logger.info("配置变更,重连Redis (async)...")
|
||||
await self.close()
|
||||
await self._connect()
|
||||
|
||||
async def set_memory_limit(self, policy: Optional[str] = "allkeys-lru"):
|
||||
"""
|
||||
动态设置Redis最大内存和内存淘汰策略
|
||||
|
||||
:param policy: 淘汰策略(如'allkeys-lru')
|
||||
"""
|
||||
try:
|
||||
# 如果有显式值,则直接使用,为0时说明不限制,如果未配置,开启BIG_MEMORY_MODE时为"1024mb",未开启时为"256mb"
|
||||
maxmemory = settings.CACHE_REDIS_MAXMEMORY or ("1024mb" if settings.BIG_MEMORY_MODE else "256mb")
|
||||
await self.client.config_set("maxmemory", maxmemory)
|
||||
await self.client.config_set("maxmemory-policy", policy)
|
||||
logger.debug(f"Redis maxmemory set to {maxmemory}, policy: {policy} (async)")
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to set Redis maxmemory or policy (async): {e}")
|
||||
|
||||
@staticmethod
|
||||
def __get_region(region: Optional[str] = "DEFAULT"):
|
||||
"""
|
||||
获取缓存的区
|
||||
"""
|
||||
return f"region:{region}" if region else "region:default"
|
||||
|
||||
def __make_redis_key(self, region: str, key: str) -> str:
|
||||
"""
|
||||
获取缓存Key
|
||||
"""
|
||||
# 使用region作为缓存键的一部分
|
||||
region = self.__get_region(region)
|
||||
return f"{region}:key:{quote(key)}"
|
||||
|
||||
@staticmethod
|
||||
def __get_original_key(redis_key: Union[str, bytes]) -> str:
|
||||
"""
|
||||
从Redis键中提取原始key
|
||||
"""
|
||||
try:
|
||||
if isinstance(redis_key, bytes):
|
||||
redis_key = redis_key.decode("utf-8")
|
||||
parts = redis_key.split(":key:")
|
||||
return parts[-1]
|
||||
except Exception as e:
|
||||
logger.warn(f"Failed to parse redis key: {redis_key}, error: {e}")
|
||||
return redis_key
|
||||
|
||||
async def set(self, key: str, value: Any, ttl: Optional[int] = None,
|
||||
region: Optional[str] = "DEFAULT", **kwargs) -> None:
|
||||
"""
|
||||
异步设置缓存
|
||||
|
||||
:param key: 缓存的键
|
||||
:param value: 缓存的值
|
||||
:param ttl: 缓存的存活时间,单位秒
|
||||
:param region: 缓存的区
|
||||
:param kwargs: 其他参数
|
||||
"""
|
||||
try:
|
||||
await self._connect()
|
||||
redis_key = self.__make_redis_key(region, key)
|
||||
# 对值进行序列化
|
||||
serialized_value = serialize(value)
|
||||
kwargs.pop("maxsize", None)
|
||||
await self.client.set(redis_key, serialized_value, ex=ttl, **kwargs)
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to set key (async): {key} in region: {region}, error: {e}")
|
||||
|
||||
async def exists(self, key: str, region: Optional[str] = "DEFAULT") -> bool:
|
||||
"""
|
||||
异步判断缓存键是否存在
|
||||
|
||||
:param key: 缓存的键
|
||||
:param region: 缓存的区
|
||||
:return: 存在返回True,否则返回False
|
||||
"""
|
||||
try:
|
||||
await self._connect()
|
||||
redis_key = self.__make_redis_key(region, key)
|
||||
result = await self.client.exists(redis_key)
|
||||
return result == 1
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to exists key (async): {key} region: {region}, error: {e}")
|
||||
return False
|
||||
|
||||
async def get(self, key: str, region: Optional[str] = "DEFAULT") -> Optional[Any]:
|
||||
"""
|
||||
异步获取缓存的值
|
||||
|
||||
:param key: 缓存的键
|
||||
:param region: 缓存的区
|
||||
:return: 返回缓存的值,如果缓存不存在返回None
|
||||
"""
|
||||
try:
|
||||
await self._connect()
|
||||
redis_key = self.__make_redis_key(region, key)
|
||||
value = await self.client.get(redis_key)
|
||||
if value is not None:
|
||||
return deserialize(value)
|
||||
return None
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to get key (async): {key} in region: {region}, error: {e}")
|
||||
return None
|
||||
|
||||
async def delete(self, key: str, region: Optional[str] = "DEFAULT") -> None:
|
||||
"""
|
||||
异步删除缓存
|
||||
|
||||
:param key: 缓存的键
|
||||
:param region: 缓存的区
|
||||
"""
|
||||
try:
|
||||
await self._connect()
|
||||
redis_key = self.__make_redis_key(region, key)
|
||||
await self.client.delete(redis_key)
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to delete key (async): {key} in region: {region}, error: {e}")
|
||||
|
||||
async def clear(self, region: Optional[str] = None) -> None:
|
||||
"""
|
||||
异步清除指定区域的缓存或全部缓存
|
||||
|
||||
:param region: 缓存的区
|
||||
"""
|
||||
try:
|
||||
await self._connect()
|
||||
if region:
|
||||
cache_region = self.__get_region(region)
|
||||
redis_key = f"{cache_region}:key:*"
|
||||
async with self.client.pipeline() as pipe:
|
||||
async for key in self.client.scan_iter(redis_key):
|
||||
await pipe.delete(key)
|
||||
await pipe.execute()
|
||||
logger.debug(f"Cleared Redis cache for region (async): {region}")
|
||||
else:
|
||||
await self.client.flushdb()
|
||||
logger.info("Cleared all Redis cache (async)")
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to clear cache (async), region: {region}, error: {e}")
|
||||
|
||||
async def items(self, region: Optional[str] = None) -> AsyncGenerator[Tuple[str, Any], None]:
|
||||
"""
|
||||
获取指定区域的所有缓存键值对
|
||||
|
||||
:param region: 缓存的区
|
||||
:return: 返回键值对生成器
|
||||
"""
|
||||
try:
|
||||
await self._connect()
|
||||
if region:
|
||||
cache_region = self.__get_region(region)
|
||||
redis_key = f"{cache_region}:key:*"
|
||||
async for key in self.client.scan_iter(redis_key):
|
||||
value = await self.client.get(key)
|
||||
if value is not None:
|
||||
yield self.__get_original_key(key), deserialize(value)
|
||||
else:
|
||||
async for key in self.client.scan_iter("*"):
|
||||
value = await self.client.get(key)
|
||||
if value is not None:
|
||||
yield self.__get_original_key(key), deserialize(value)
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to get items from Redis, region: {region}, error: {e}")
|
||||
|
||||
async def test(self) -> bool:
|
||||
"""
|
||||
异步测试Redis连接性
|
||||
"""
|
||||
try:
|
||||
await self._connect()
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.error(f"Redis async connection test failed: {e}")
|
||||
return False
|
||||
|
||||
async def close(self) -> None:
|
||||
"""
|
||||
关闭异步Redis客户端的连接池
|
||||
"""
|
||||
if self.client:
|
||||
await self.client.close()
|
||||
self.client = None
|
||||
logger.debug("Redis async connection closed")
|
||||
@@ -8,7 +8,6 @@ from app.log import logger
|
||||
from app.utils.http import RequestUtils
|
||||
from app.utils.string import StringUtils
|
||||
from app.utils.system import SystemUtils
|
||||
from version import APP_VERSION
|
||||
|
||||
|
||||
class ResourceHelper:
|
||||
@@ -59,12 +58,6 @@ class ResourceHelper:
|
||||
if rtype == "auth":
|
||||
# 站点认证资源
|
||||
local_version = SitesHelper().auth_version
|
||||
# 阻断站点认证资源v2.3.0以下的版本直接更新,避免无限重启
|
||||
if StringUtils.compare_version(local_version, "<", "2.3.0"):
|
||||
continue
|
||||
# 阻断主程序版本v2.6.3以下的版本直接更新,避免搜索异常
|
||||
if StringUtils.compare_version(APP_VERSION, "<", "2.6.3"):
|
||||
continue
|
||||
elif rtype == "sites":
|
||||
# 站点索引资源
|
||||
local_version = SitesHelper().indexer_version
|
||||
|
||||
@@ -228,13 +228,14 @@ class RssHelper:
|
||||
}
|
||||
|
||||
def parse(self, url, proxy: bool = False,
|
||||
timeout: Optional[int] = 15, headers: dict = None) -> Union[List[dict], None, bool]:
|
||||
timeout: Optional[int] = 15, headers: dict = None, ua: str = None) -> Union[List[dict], None, bool]:
|
||||
"""
|
||||
解析RSS订阅URL,获取RSS中的种子信息
|
||||
:param url: RSS地址
|
||||
:param proxy: 是否使用代理
|
||||
:param timeout: 请求超时
|
||||
:param headers: 自定义请求头
|
||||
:param ua: 自定义User-Agent
|
||||
:return: 种子信息列表,如为None代表Rss过期,如果为False则为错误
|
||||
"""
|
||||
# 开始处理
|
||||
@@ -243,8 +244,9 @@ class RssHelper:
|
||||
return False
|
||||
|
||||
try:
|
||||
ret = RequestUtils(proxies=settings.PROXY if proxy else None,
|
||||
timeout=timeout, headers=headers).get_res(url)
|
||||
ret = RequestUtils(ua=ua,
|
||||
proxies=settings.PROXY if proxy else None,
|
||||
timeout=timeout or 30, headers=headers).get_res(url)
|
||||
if not ret:
|
||||
logger.error(f"获取RSS失败:请求返回空值,URL: {url}")
|
||||
return False
|
||||
@@ -384,6 +386,9 @@ class RssHelper:
|
||||
pubdate = ""
|
||||
if pubdate_nodes and pubdate_nodes[0].text:
|
||||
pubdate = StringUtils.get_time(pubdate_nodes[0].text)
|
||||
if pubdate is not None:
|
||||
# 转为本地时区
|
||||
pubdate = pubdate.astimezone(tz=None)
|
||||
|
||||
# 获取豆瓣昵称
|
||||
nickname_nodes = item.xpath('.//*[local-name()="creator"]')
|
||||
|
||||
@@ -47,7 +47,7 @@ class StorageHelper:
|
||||
if s.type == storage:
|
||||
s.config = conf
|
||||
break
|
||||
SystemConfigOper().set(SystemConfigKey.Storages, [s.dict() for s in storagies])
|
||||
SystemConfigOper().set(SystemConfigKey.Storages, [s.model_dump() for s in storagies])
|
||||
|
||||
def add_storage(self, storage: str, name: str, conf: dict):
|
||||
"""
|
||||
@@ -68,7 +68,7 @@ class StorageHelper:
|
||||
name=name,
|
||||
config=conf
|
||||
))
|
||||
SystemConfigOper().set(SystemConfigKey.Storages, [s.dict() for s in storagies])
|
||||
SystemConfigOper().set(SystemConfigKey.Storages, [s.model_dump() for s in storagies])
|
||||
|
||||
def reset_storage(self, storage: str):
|
||||
"""
|
||||
@@ -79,4 +79,4 @@ class StorageHelper:
|
||||
if s.type == storage:
|
||||
s.config = {}
|
||||
break
|
||||
SystemConfigOper().set(SystemConfigKey.Storages, [s.dict() for s in storagies])
|
||||
SystemConfigOper().set(SystemConfigKey.Storages, [s.model_dump() for s in storagies])
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
from threading import Thread
|
||||
from typing import List, Tuple, Optional
|
||||
|
||||
from app.core.cache import cached, cache_backend
|
||||
from app.core.cache import cached
|
||||
from app.core.config import settings
|
||||
from app.db.subscribe_oper import SubscribeOper
|
||||
from app.db.systemconfig_oper import SystemConfigOper
|
||||
@@ -111,7 +111,12 @@ class SubscribeHelper(metaclass=WeakSingleton):
|
||||
if res and res.status_code == 200:
|
||||
# 清除缓存
|
||||
if clear_cache:
|
||||
cache_backend.clear(region=self._shares_cache_region)
|
||||
self.get_shares.cache_clear()
|
||||
self.get_statistic.cache_clear()
|
||||
self.get_share_statistics.cache_clear()
|
||||
self.async_get_shares.cache_clear()
|
||||
self.async_get_statistic.cache_clear()
|
||||
self.async_get_share_statistics.cache_clear()
|
||||
return True, ""
|
||||
else:
|
||||
return False, res.json().get("message")
|
||||
@@ -126,7 +131,9 @@ class SubscribeHelper(metaclass=WeakSingleton):
|
||||
return []
|
||||
|
||||
@cached(region=_shares_cache_region, maxsize=5, ttl=1800, skip_empty=True)
|
||||
def get_statistic(self, stype: str, page: Optional[int] = 1, count: Optional[int] = 30) -> List[dict]:
|
||||
def get_statistic(self, stype: str, page: Optional[int] = 1, count: Optional[int] = 30,
|
||||
genre_id: Optional[int] = None, min_rating: Optional[float] = None,
|
||||
max_rating: Optional[float] = None, sort_type: Optional[str] = None) -> List[dict]:
|
||||
"""
|
||||
获取订阅统计数据
|
||||
"""
|
||||
@@ -134,16 +141,30 @@ class SubscribeHelper(metaclass=WeakSingleton):
|
||||
if not enabled:
|
||||
return []
|
||||
|
||||
res = RequestUtils(proxies=settings.PROXY, timeout=15).get_res(self._sub_statistic, params={
|
||||
params = {
|
||||
"stype": stype,
|
||||
"page": page,
|
||||
"count": count
|
||||
})
|
||||
}
|
||||
|
||||
# 添加可选参数
|
||||
if genre_id is not None:
|
||||
params["genre_id"] = genre_id
|
||||
if min_rating is not None:
|
||||
params["min_rating"] = min_rating
|
||||
if max_rating is not None:
|
||||
params["max_rating"] = max_rating
|
||||
if sort_type is not None:
|
||||
params["sort_type"] = sort_type
|
||||
|
||||
res = RequestUtils(proxies=settings.PROXY, timeout=15).get_res(self._sub_statistic, params=params)
|
||||
|
||||
return self._handle_list_response(res)
|
||||
|
||||
@cached(region=_shares_cache_region, maxsize=5, ttl=1800, skip_empty=True)
|
||||
async def async_get_statistic(self, stype: str, page: Optional[int] = 1, count: Optional[int] = 30) -> List[dict]:
|
||||
async def async_get_statistic(self, stype: str, page: Optional[int] = 1, count: Optional[int] = 30,
|
||||
genre_id: Optional[int] = None, min_rating: Optional[float] = None,
|
||||
max_rating: Optional[float] = None, sort_type: Optional[str] = None) -> List[dict]:
|
||||
"""
|
||||
异步获取订阅统计数据
|
||||
"""
|
||||
@@ -151,11 +172,23 @@ class SubscribeHelper(metaclass=WeakSingleton):
|
||||
if not enabled:
|
||||
return []
|
||||
|
||||
res = await AsyncRequestUtils(proxies=settings.PROXY, timeout=15).get_res(self._sub_statistic, params={
|
||||
params = {
|
||||
"stype": stype,
|
||||
"page": page,
|
||||
"count": count
|
||||
})
|
||||
}
|
||||
|
||||
# 添加可选参数
|
||||
if genre_id is not None:
|
||||
params["genre_id"] = genre_id
|
||||
if min_rating is not None:
|
||||
params["min_rating"] = min_rating
|
||||
if max_rating is not None:
|
||||
params["max_rating"] = max_rating
|
||||
if sort_type is not None:
|
||||
params["sort_type"] = sort_type
|
||||
|
||||
res = await AsyncRequestUtils(proxies=settings.PROXY, timeout=15).get_res(self._sub_statistic, params=params)
|
||||
|
||||
return self._handle_list_response(res)
|
||||
|
||||
@@ -353,7 +386,9 @@ class SubscribeHelper(metaclass=WeakSingleton):
|
||||
return self._handle_response(res, clear_cache=False)
|
||||
|
||||
@cached(region=_shares_cache_region, maxsize=1, ttl=1800, skip_empty=True)
|
||||
def get_shares(self, name: Optional[str] = None, page: Optional[int] = 1, count: Optional[int] = 30) -> List[dict]:
|
||||
def get_shares(self, name: Optional[str] = None, page: Optional[int] = 1, count: Optional[int] = 30,
|
||||
genre_id: Optional[int] = None, min_rating: Optional[float] = None,
|
||||
max_rating: Optional[float] = None, sort_type: Optional[str] = None) -> List[dict]:
|
||||
"""
|
||||
获取订阅分享数据
|
||||
"""
|
||||
@@ -361,17 +396,30 @@ class SubscribeHelper(metaclass=WeakSingleton):
|
||||
if not enabled:
|
||||
return []
|
||||
|
||||
res = RequestUtils(proxies=settings.PROXY, timeout=15).get_res(self._sub_shares, params={
|
||||
params = {
|
||||
"name": name,
|
||||
"page": page,
|
||||
"count": count
|
||||
})
|
||||
}
|
||||
|
||||
# 添加可选参数
|
||||
if genre_id is not None:
|
||||
params["genre_id"] = genre_id
|
||||
if min_rating is not None:
|
||||
params["min_rating"] = min_rating
|
||||
if max_rating is not None:
|
||||
params["max_rating"] = max_rating
|
||||
if sort_type is not None:
|
||||
params["sort_type"] = sort_type
|
||||
|
||||
res = RequestUtils(proxies=settings.PROXY, timeout=15).get_res(self._sub_shares, params=params)
|
||||
|
||||
return self._handle_list_response(res)
|
||||
|
||||
@cached(region=_shares_cache_region, maxsize=1, ttl=1800, skip_empty=True)
|
||||
async def async_get_shares(self, name: Optional[str] = None, page: Optional[int] = 1, count: Optional[int] = 30) -> \
|
||||
List[dict]:
|
||||
async def async_get_shares(self, name: Optional[str] = None, page: Optional[int] = 1, count: Optional[int] = 30,
|
||||
genre_id: Optional[int] = None, min_rating: Optional[float] = None,
|
||||
max_rating: Optional[float] = None, sort_type: Optional[str] = None) -> List[dict]:
|
||||
"""
|
||||
异步获取订阅分享数据
|
||||
"""
|
||||
@@ -379,11 +427,23 @@ class SubscribeHelper(metaclass=WeakSingleton):
|
||||
if not enabled:
|
||||
return []
|
||||
|
||||
res = await AsyncRequestUtils(proxies=settings.PROXY, timeout=15).get_res(self._sub_shares, params={
|
||||
params = {
|
||||
"name": name,
|
||||
"page": page,
|
||||
"count": count
|
||||
})
|
||||
}
|
||||
|
||||
# 添加可选参数
|
||||
if genre_id is not None:
|
||||
params["genre_id"] = genre_id
|
||||
if min_rating is not None:
|
||||
params["min_rating"] = min_rating
|
||||
if max_rating is not None:
|
||||
params["max_rating"] = max_rating
|
||||
if sort_type is not None:
|
||||
params["sort_type"] = sort_type
|
||||
|
||||
res = await AsyncRequestUtils(proxies=settings.PROXY, timeout=15).get_res(self._sub_shares, params=params)
|
||||
|
||||
return self._handle_list_response(res)
|
||||
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
import os
|
||||
import signal
|
||||
import threading
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Tuple
|
||||
|
||||
@@ -41,8 +43,8 @@ class SystemHelper:
|
||||
判断是否可以内部重启
|
||||
"""
|
||||
return (
|
||||
Path("/var/run/docker.sock").exists()
|
||||
or settings.DOCKER_CLIENT_API != "tcp://127.0.0.1:38379"
|
||||
Path("/var/run/docker.sock").exists()
|
||||
or settings.DOCKER_CLIENT_API != "tcp://127.0.0.1:38379"
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
@@ -64,7 +66,7 @@ class SystemHelper:
|
||||
if index_resolv_conf != -1:
|
||||
index_second_slash = data.rfind(" ", 0, index_resolv_conf)
|
||||
index_first_slash = (
|
||||
data.rfind("/", 0, index_second_slash) + 1
|
||||
data.rfind("/", 0, index_second_slash) + 1
|
||||
)
|
||||
container_id = data[index_first_slash:index_second_slash]
|
||||
except Exception as e:
|
||||
@@ -113,6 +115,8 @@ class SystemHelper:
|
||||
if has_restart_policy:
|
||||
# 有重启策略,使用优雅退出方式
|
||||
logger.info("检测到容器配置了自动重启策略,使用优雅重启方式...")
|
||||
# 启动优雅退出超时监控
|
||||
SystemHelper._start_graceful_shutdown_monitor()
|
||||
# 发送SIGTERM信号给当前进程,触发优雅停止
|
||||
os.kill(os.getpid(), signal.SIGTERM)
|
||||
return True, ""
|
||||
@@ -126,6 +130,25 @@ class SystemHelper:
|
||||
logger.warning("降级为Docker API重启...")
|
||||
return SystemHelper._docker_api_restart()
|
||||
|
||||
@staticmethod
|
||||
def _start_graceful_shutdown_monitor():
|
||||
"""
|
||||
启动优雅退出超时监控
|
||||
如果30秒内进程没有退出,则使用Docker API强制重启
|
||||
"""
|
||||
|
||||
def monitor_thread():
|
||||
time.sleep(30) # 等待30秒
|
||||
logger.warning("优雅退出超时30秒,使用Docker API强制重启...")
|
||||
try:
|
||||
SystemHelper._docker_api_restart()
|
||||
except Exception as e:
|
||||
logger.error(f"强制重启失败: {str(e)}")
|
||||
|
||||
# 在后台线程中启动监控
|
||||
thread = threading.Thread(target=monitor_thread, daemon=True)
|
||||
thread.start()
|
||||
|
||||
@staticmethod
|
||||
def _docker_api_restart() -> Tuple[bool, str]:
|
||||
"""
|
||||
|
||||
@@ -6,6 +6,8 @@ from urllib.parse import unquote
|
||||
|
||||
from torrentool.api import Torrent
|
||||
|
||||
from app.core.cache import FileCache
|
||||
from app.core.cache import TTLCache
|
||||
from app.core.config import settings
|
||||
from app.core.context import Context, TorrentInfo, MediaInfo
|
||||
from app.core.meta import MetaBase
|
||||
@@ -15,17 +17,16 @@ from app.db.systemconfig_oper import SystemConfigOper
|
||||
from app.log import logger
|
||||
from app.schemas.types import MediaType, SystemConfigKey
|
||||
from app.utils.http import RequestUtils
|
||||
from app.utils.singleton import WeakSingleton
|
||||
from app.utils.string import StringUtils
|
||||
|
||||
|
||||
class TorrentHelper(metaclass=WeakSingleton):
|
||||
class TorrentHelper:
|
||||
"""
|
||||
种子帮助类
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
self._invalid_torrents = []
|
||||
self._invalid_torrents = TTLCache(maxsize=128, ttl=3600 * 24)
|
||||
|
||||
def download_torrent(self, url: str,
|
||||
cookie: Optional[str] = None,
|
||||
@@ -35,27 +36,29 @@ class TorrentHelper(metaclass=WeakSingleton):
|
||||
-> Tuple[Optional[Path], Optional[Union[str, bytes]], Optional[str], Optional[list], Optional[str]]:
|
||||
"""
|
||||
把种子下载到本地
|
||||
:return: 种子保存路径、种子内容、种子主目录、种子文件清单、错误信息
|
||||
:return: 种子缓存相对路径【用于索引缓存】, 种子内容、种子主目录、种子文件清单、错误信息
|
||||
"""
|
||||
if url.startswith("magnet:"):
|
||||
return None, url, "", [], f"磁力链接"
|
||||
# 构建 torrent 种子文件的存储路径
|
||||
file_path = (Path(settings.TEMP_PATH) / StringUtils.md5_hash(url)).with_suffix(".torrent")
|
||||
if file_path.exists():
|
||||
# 构建 torrent 种子文件的缓存路径
|
||||
cache_path = Path(StringUtils.md5_hash(url)).with_suffix(".torrent")
|
||||
# 缓存处理器
|
||||
cache_backend = FileCache()
|
||||
# 读取缓存的种子文件
|
||||
torrent_content = cache_backend.get(cache_path.as_posix(), region="torrents")
|
||||
if torrent_content:
|
||||
# 缓存已存在
|
||||
try:
|
||||
# 获取种子目录和文件清单
|
||||
folder_name, file_list = self.get_torrent_info(file_path)
|
||||
folder_name, file_list = self.get_fileinfo_from_torrent_content(torrent_content)
|
||||
# 无法获取信息,则认为缓存文件无效
|
||||
if not folder_name and not file_list:
|
||||
raise ValueError("无效的缓存种子文件")
|
||||
# 获取种子数据
|
||||
content = file_path.read_bytes()
|
||||
# 成功拿到种子数据
|
||||
return file_path, content, folder_name, file_list, ""
|
||||
return cache_path, torrent_content, folder_name, file_list, ""
|
||||
except Exception as err:
|
||||
logger.error(f"处理缓存的种子文件 {file_path} 时出错: {err},将重新下载")
|
||||
file_path.unlink(missing_ok=True)
|
||||
# 请求种子文件
|
||||
logger.error(f"处理缓存的种子文件 {cache_path} 时出错: {err},将重新下载")
|
||||
# 下载种子文件
|
||||
req = RequestUtils(
|
||||
ua=ua,
|
||||
cookies=cookie,
|
||||
@@ -74,11 +77,11 @@ class TorrentHelper(metaclass=WeakSingleton):
|
||||
).get_res(url=url, allow_redirects=False)
|
||||
if req and req.status_code == 200:
|
||||
if not req.content:
|
||||
return None, None, "", [], "未下载到种子数据"
|
||||
return cache_path, None, "", [], "未下载到种子数据"
|
||||
# 解析内容格式
|
||||
if req.content.startswith(b"magnet:"):
|
||||
# 磁力链接
|
||||
return None, req.text, "", [], f"获取到磁力链接"
|
||||
return cache_path, req.text, "", [], f"获取到磁力链接"
|
||||
if "下载种子文件".encode("utf-8") in req.content:
|
||||
# 首次下载提示页面
|
||||
skip_flag = False
|
||||
@@ -116,34 +119,34 @@ class TorrentHelper(metaclass=WeakSingleton):
|
||||
except Exception as err:
|
||||
logger.warn(f"触发了站点首次种子下载,尝试自动跳过时出现错误:{str(err)},链接:{url}")
|
||||
if not skip_flag:
|
||||
return None, None, "", [], "种子数据有误,请确认链接是否正确,如为PT站点则需手工在站点下载一次种子"
|
||||
return cache_path, None, "", [], "种子数据有误,请确认链接是否正确,如为PT站点则需手工在站点下载一次种子"
|
||||
# 种子内容
|
||||
if req.content:
|
||||
# 检查是不是种子文件,如果不是仍然抛出异常
|
||||
try:
|
||||
# 保存到文件
|
||||
file_path.write_bytes(req.content)
|
||||
# 获取种子目录和文件清单
|
||||
folder_name, file_list = self.get_torrent_info(file_path)
|
||||
folder_name, file_list = self.get_fileinfo_from_torrent_content(req.content)
|
||||
if file_list:
|
||||
# 保存到缓存
|
||||
cache_backend.set(cache_path.as_posix(), req.content, region="torrents")
|
||||
# 成功拿到种子数据
|
||||
return file_path, req.content, folder_name, file_list, ""
|
||||
return cache_path, req.content, folder_name, file_list, ""
|
||||
except Exception as err:
|
||||
logger.error(f"种子文件解析失败:{str(err)}")
|
||||
# 种子数据仍然错误
|
||||
return None, None, "", [], "种子数据有误,请确认链接是否正确"
|
||||
return cache_path, None, "", [], "种子数据有误,请确认链接是否正确"
|
||||
# 返回失败
|
||||
return None, None, "", [], ""
|
||||
return cache_path, None, "", [], ""
|
||||
elif req is None:
|
||||
return None, None, "", [], "无法打开链接"
|
||||
return cache_path, None, "", [], "无法打开链接"
|
||||
elif req.status_code == 429:
|
||||
return None, None, "", [], "触发站点流控,请稍后重试"
|
||||
return cache_path, None, "", [], "触发站点流控,请稍后重试"
|
||||
else:
|
||||
# 把错误的种子记下来,避免重复使用
|
||||
self.add_invalid(url)
|
||||
return None, None, "", [], f"下载种子出错,状态码:{req.status_code}"
|
||||
return cache_path, None, "", [], f"下载种子出错,状态码:{req.status_code}"
|
||||
|
||||
@staticmethod
|
||||
def get_torrent_info(torrent_path: Path) -> Tuple[str, List[str]]:
|
||||
def get_torrent_info(self, torrent_path: Path) -> Tuple[str, List[str]]:
|
||||
"""
|
||||
获取种子文件的文件夹名和文件清单
|
||||
:param torrent_path: 种子文件路径
|
||||
@@ -154,32 +157,65 @@ class TorrentHelper(metaclass=WeakSingleton):
|
||||
try:
|
||||
torrentinfo = Torrent.from_file(torrent_path)
|
||||
# 获取文件清单
|
||||
if (not torrentinfo.files
|
||||
or (len(torrentinfo.files) == 1
|
||||
and torrentinfo.files[0].name == torrentinfo.name)):
|
||||
# 单文件种子目录名返回空
|
||||
folder_name = ""
|
||||
# 单文件种子
|
||||
file_list = [torrentinfo.name]
|
||||
else:
|
||||
# 目录名
|
||||
folder_name = torrentinfo.name
|
||||
# 文件清单,如果一级目录与种子名相同则去掉
|
||||
file_list = []
|
||||
for fileinfo in torrentinfo.files:
|
||||
file_path = Path(fileinfo.name)
|
||||
# 根路径
|
||||
root_path = file_path.parts[0]
|
||||
if root_path == folder_name:
|
||||
file_list.append(str(file_path.relative_to(root_path)))
|
||||
else:
|
||||
file_list.append(fileinfo.name)
|
||||
logger.debug(f"解析种子:{torrent_path.name} => 目录:{folder_name},文件清单:{file_list}")
|
||||
return folder_name, file_list
|
||||
return self.get_fileinfo_from_torrent(torrentinfo)
|
||||
except Exception as err:
|
||||
logger.error(f"种子文件解析失败:{str(err)}")
|
||||
return "", []
|
||||
|
||||
@staticmethod
|
||||
def get_fileinfo_from_torrent(torrent: Torrent) -> Tuple[str, List[str]]:
|
||||
"""
|
||||
从种子文件中获取文件清单
|
||||
:param torrent: 种子文件对象
|
||||
:return: 文件夹名、文件清单,单文件种子返回空文件夹名
|
||||
"""
|
||||
if not torrent or not torrent.files:
|
||||
return "", []
|
||||
# 获取文件清单
|
||||
if len(torrent.files) == 1 and torrent.files[0].name == torrent.name:
|
||||
# 单文件种子目录名返回空
|
||||
folder_name = ""
|
||||
# 单文件种子
|
||||
file_list = [torrent.name]
|
||||
else:
|
||||
# 目录名
|
||||
folder_name = torrent.name
|
||||
# 文件清单,如果一级目录与种子名相同则去掉
|
||||
file_list = []
|
||||
for fileinfo in torrent.files:
|
||||
file_path = Path(fileinfo.name)
|
||||
# 根路径
|
||||
root_path = file_path.parts[0]
|
||||
if root_path == folder_name:
|
||||
file_list.append(str(file_path.relative_to(root_path)))
|
||||
else:
|
||||
file_list.append(fileinfo.name)
|
||||
logger.debug(f"解析种子:{torrent.name} => 目录:{folder_name},文件清单:{file_list}")
|
||||
return folder_name, file_list
|
||||
|
||||
def get_fileinfo_from_torrent_content(self, torrent_content: Union[str, bytes]) -> Tuple[str, List[str]]:
|
||||
"""
|
||||
从种子内容中获取文件夹名和文件清单
|
||||
:param torrent_content: 种子内容
|
||||
:return: 文件夹名、文件清单,单文件种子返回空文件夹名
|
||||
"""
|
||||
|
||||
if not torrent_content:
|
||||
return "", []
|
||||
|
||||
# 检查是否为磁力链接
|
||||
if StringUtils.is_magnet_link(torrent_content):
|
||||
return "", []
|
||||
|
||||
try:
|
||||
# 解析种子内容
|
||||
torrentinfo = Torrent.from_string(torrent_content)
|
||||
# 获取文件清单
|
||||
return self.get_fileinfo_from_torrent(torrentinfo)
|
||||
except Exception as err:
|
||||
logger.error(f"种子内容解析失败:{str(err)}")
|
||||
return "", []
|
||||
|
||||
@staticmethod
|
||||
def get_url_filename(req: Any, url: str) -> str:
|
||||
"""
|
||||
@@ -316,7 +352,7 @@ class TorrentHelper(metaclass=WeakSingleton):
|
||||
添加无效种子
|
||||
"""
|
||||
if url not in self._invalid_torrents:
|
||||
self._invalid_torrents.append(url)
|
||||
self._invalid_torrents[url] = True
|
||||
|
||||
@staticmethod
|
||||
def match_torrent(mediainfo: MediaInfo, torrent_meta: MetaBase, torrent: TorrentInfo) -> bool:
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import json
|
||||
from typing import List, Tuple, Optional
|
||||
|
||||
from app.core.cache import cached, cache_backend
|
||||
from app.core.cache import cached
|
||||
from app.core.config import settings
|
||||
from app.db.models import Workflow
|
||||
from app.db.workflow_oper import WorkflowOper
|
||||
@@ -89,7 +89,8 @@ class WorkflowHelper(metaclass=WeakSingleton):
|
||||
if success:
|
||||
# 清除缓存
|
||||
if clear_cache:
|
||||
cache_backend.clear(region=self._shares_cache_region)
|
||||
self.get_shares.cache_clear()
|
||||
self.async_get_shares.cache_clear()
|
||||
return True, ""
|
||||
else:
|
||||
try:
|
||||
|
||||
15
app/log.py
15
app/log.py
@@ -11,7 +11,8 @@ from pathlib import Path
|
||||
from typing import Dict, Any, Optional
|
||||
|
||||
import click
|
||||
from pydantic import BaseSettings, BaseModel
|
||||
from pydantic import BaseModel, ConfigDict
|
||||
from pydantic_settings import BaseSettings
|
||||
|
||||
from app.utils.system import SystemUtils
|
||||
|
||||
@@ -21,8 +22,7 @@ class LogConfigModel(BaseModel):
|
||||
Pydantic 配置模型,描述所有配置项及其类型和默认值
|
||||
"""
|
||||
|
||||
class Config:
|
||||
extra = "ignore" # 忽略未定义的配置项
|
||||
model_config = ConfigDict(extra="ignore") # 忽略未定义的配置项
|
||||
|
||||
# 配置文件目录
|
||||
CONFIG_DIR: Optional[str] = None
|
||||
@@ -71,10 +71,11 @@ class LogSettings(BaseSettings, LogConfigModel):
|
||||
"""
|
||||
return self.LOG_MAX_FILE_SIZE * 1024 * 1024
|
||||
|
||||
class Config:
|
||||
case_sensitive = True
|
||||
env_file = SystemUtils.get_env_path()
|
||||
env_file_encoding = "utf-8"
|
||||
model_config = ConfigDict(
|
||||
case_sensitive=True,
|
||||
env_file=SystemUtils.get_env_path(),
|
||||
env_file_encoding="utf-8"
|
||||
)
|
||||
|
||||
|
||||
# 实例化日志设置
|
||||
|
||||
@@ -95,4 +95,4 @@ if __name__ == '__main__':
|
||||
# 更新数据库
|
||||
update_db()
|
||||
# 启动API服务
|
||||
Server.run()
|
||||
Server.run()
|
||||
@@ -232,6 +232,19 @@ class _DownloaderBase(ServiceBase[TService, DownloaderConf]):
|
||||
super().__init__()
|
||||
self._default_config_name: Optional[str] = None
|
||||
|
||||
def init_service(self, service_name: str,
|
||||
service_type: Optional[Union[Type[TService], Callable[..., TService]]] = None):
|
||||
"""
|
||||
初始化服务,获取配置并实例化对应服务
|
||||
|
||||
:param service_name: 服务名称,作为配置匹配的依据
|
||||
:param service_type: 服务的类型,可以是类类型(Type[TService])、工厂函数(Callable)或 None 来跳过实例化
|
||||
"""
|
||||
# 重置默认配置名称
|
||||
self.reset_default_config_name()
|
||||
# 初始化服务
|
||||
super().init_service(service_name, service_type)
|
||||
|
||||
def get_default_config_name(self) -> Optional[str]:
|
||||
"""
|
||||
获取默认服务配置的名称
|
||||
@@ -263,6 +276,12 @@ class _DownloaderBase(ServiceBase[TService, DownloaderConf]):
|
||||
return {}
|
||||
return {conf.name: conf for conf in configs if conf.type == self._service_name and conf.enabled}
|
||||
|
||||
def reset_default_config_name(self):
|
||||
"""
|
||||
重置默认配置名称
|
||||
"""
|
||||
self._default_config_name = None
|
||||
|
||||
|
||||
class _MediaServerBase(ServiceBase[TService, MediaServerConf]):
|
||||
"""
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user