mtfyun/OpenList
1
0 Fork 0
mirror of https://github.com/OpenListTeam/OpenList.git synced 2026-11-13 22:20:47 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge branch 'main' into feat/139-app-client-image-suffix-upload

Browse Source
This commit is contained in:
Pikachu Ren authored and GitHub committed 2026-11-02 21:41:54 +08:00
2 parents 880523245e e72822f539
commit 421bfb60a4
161 files changed
+27811 -756
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

.github/workflows/beta_release.yml
+4 -4
View File
@@ -24,7 +24,7 @@ jobs:
- name: Create or update ref
id: create-or-update-ref
uses: ovsds/create-or-update-ref-action@1157b2991820188d5a8e6f22656775d1d171f762 # v1
uses: ovsds/create-or-update-ref-action@f66e3f24fa250e50da7d3e599bae67bc3f749a5e # v1
with:
ref: tags/beta
sha: ${{ github.sha }}
@@ -51,7 +51,7 @@ jobs:
gh release upload beta "CHANGELOG.md" --clobber
- name: Upload assets to github artifact
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7
with:
name: beta changelog
path: ${{ github.workspace }}/CHANGELOG.md
@@ -115,7 +115,7 @@ jobs:
- name: Setup Go
uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7
with:
go-version: "1.27.1"
go-version: "1.27.3"
- name: Setup web
run: bash build.sh dev web
@@ -185,7 +185,7 @@ jobs:
echo "cleaned_target=$CLEANED_TARGET" >> $GITHUB_ENV
- name: Upload assets to github artifact
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7
with:
name: beta builds for ${{ env.cleaned_target }}
path: ${{ github.workspace }}/build/compress/*
.github/workflows/build.yml
+18 -2
View File
@@ -10,6 +10,22 @@ concurrency:
cancel-in-progress: true
jobs:
# 媒体库相关的安全/配额/迁移测试单独跑一次，不必跟随 7 个构建目标重复执行
test:
name: Unit tests
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
- name: Setup Go
uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7
with:
go-version: "1.27.3"
- name: Go test
run: go test ./internal/db/... ./internal/transcode/... ./internal/media/...
build:
strategy:
matrix:
@@ -33,7 +49,7 @@ jobs:
- name: Setup Go
uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7
with:
go-version: "1.27.1"
go-version: "1.27.3"
- name: Setup web
run: bash build.sh dev web
@@ -69,7 +85,7 @@ jobs:
fi
- name: Upload artifact
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7
with:
name: openlist_${{ steps.short-sha.outputs.sha }}_${{ matrix.target }}
path: build/*
.github/workflows/release.yml
+2 -2
View File
@@ -33,7 +33,7 @@ jobs:
- name: Free Disk Space (Ubuntu)
if: matrix.target-platform == ''
uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be # main
uses: jlumbroso/free-disk-space@36e9a5af4eddb91fc11aec4c6f6f79b3cc0cb523 # main
with:
tool-cache: false
android: true
@@ -46,7 +46,7 @@ jobs:
- name: Setup Go
uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7
with:
go-version: '1.27.1'
go-version: '1.27.3'
- name: Checkout
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
.github/workflows/release_docker.yml
+16 -16
View File
@@ -47,7 +47,7 @@ jobs:
- uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7
with:
go-version: '1.27'
go-version: '1.27.3'
- name: Cache Musl
id: cache-musl
@@ -69,7 +69,7 @@ jobs:
FRONTEND_REPO: ${{ vars.FRONTEND_REPO }}
- name: Upload artifacts
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7
with:
name: ${{ env.ARTIFACT_NAME }}
overwrite: true
@@ -87,7 +87,7 @@ jobs:
- uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7
with:
go-version: '1.27.1'
go-version: '1.27.3'
- name: Cache Musl
id: cache-musl
@@ -109,7 +109,7 @@ jobs:
FRONTEND_REPO: ${{ vars.FRONTEND_REPO }}
- name: Upload artifacts
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7
with:
name: ${{ env.ARTIFACT_NAME_LITE }}
overwrite: true
@@ -147,20 +147,20 @@ jobs:
steps:
- name: Checkout
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
- uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
- uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8
with:
name: ${{ env.ARTIFACT_NAME }}
path: 'build/'
- name: Set up QEMU
uses: docker/setup-qemu-action@1f40c72289eff860ee54a304f1438e3cff362e0a # v4
uses: docker/setup-qemu-action@99012661954931238ded8c8b007157a8430204e1 # v4
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4
uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4
- name: Login to GitHub Container Registry
if: env.IMAGE_PUSH == 'true'
uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4
uses: docker/login-action@81c6f93d4bc991e510b342a00b419b3bb4efc5c6 # v4
with:
registry: ${{ env.REGISTRY }}
username: ${{ github.actor }}
@@ -168,7 +168,7 @@ jobs:
- name: Login to DockerHub Container Registry
if: env.IMAGE_PUSH == 'true'
uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4
uses: docker/login-action@81c6f93d4bc991e510b342a00b419b3bb4efc5c6 # v4
with:
username: ${{ vars.DOCKERHUB_ORG_NAME_BACKUP || env.DOCKERHUB_ORG_NAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
@@ -190,7 +190,7 @@ jobs:
- name: Build and push
id: docker_build
uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7
uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7
with:
context: .
file: Dockerfile.ci
@@ -231,20 +231,20 @@ jobs:
steps:
- name: Checkout
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
- uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
- uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8
with:
name: ${{ env.ARTIFACT_NAME_LITE }}
path: 'build/'
- name: Set up QEMU
uses: docker/setup-qemu-action@1f40c72289eff860ee54a304f1438e3cff362e0a # v4
uses: docker/setup-qemu-action@99012661954931238ded8c8b007157a8430204e1 # v4
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4
uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4
- name: Login to GitHub Container Registry
if: env.IMAGE_PUSH == 'true'
uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4
uses: docker/login-action@81c6f93d4bc991e510b342a00b419b3bb4efc5c6 # v4
with:
registry: ${{ env.REGISTRY }}
username: ${{ github.actor }}
@@ -252,7 +252,7 @@ jobs:
- name: Login to DockerHub Container Registry
if: env.IMAGE_PUSH == 'true'
uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4
uses: docker/login-action@81c6f93d4bc991e510b342a00b419b3bb4efc5c6 # v4
with:
username: ${{ vars.DOCKERHUB_ORG_NAME_BACKUP || env.DOCKERHUB_ORG_NAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
@@ -274,7 +274,7 @@ jobs:
- name: Build and push
id: docker_build
uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7
uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7
with:
context: .
file: Dockerfile.ci
.github/workflows/test_docker.yml
+8 -8
View File
@@ -36,7 +36,7 @@ jobs:
- uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7
with:
go-version: '1.27.1'
go-version: '1.27.3'
- name: Cache Musl
id: cache-musl
@@ -58,7 +58,7 @@ jobs:
FRONTEND_REPO: ${{ vars.FRONTEND_REPO }}
- name: Upload artifacts
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7
with:
name: ${{ env.ARTIFACT_NAME }}
overwrite: true
@@ -98,20 +98,20 @@ jobs:
steps:
- name: Checkout
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
- uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
- uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8
with:
name: ${{ env.ARTIFACT_NAME }}
path: 'build/'
- name: Set up QEMU
uses: docker/setup-qemu-action@1f40c72289eff860ee54a304f1438e3cff362e0a # v4
uses: docker/setup-qemu-action@99012661954931238ded8c8b007157a8430204e1 # v4
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4
uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4
- name: Login to GitHub Container Registry
if: env.IMAGE_PUSH == 'true'
uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4
uses: docker/login-action@81c6f93d4bc991e510b342a00b419b3bb4efc5c6 # v4
with:
registry: ${{ env.REGISTRY }}
username: ${{ github.actor }}
@@ -119,7 +119,7 @@ jobs:
- name: Login to DockerHub Container Registry
if: env.IMAGE_PUSH == 'true'
uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4
uses: docker/login-action@81c6f93d4bc991e510b342a00b419b3bb4efc5c6 # v4
with:
username: ${{ vars.DOCKERHUB_ORG_NAME_BACKUP || env.DOCKERHUB_ORG_NAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
@@ -137,7 +137,7 @@ jobs:
- name: Build and push
id: docker_build
uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7
uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7
with:
context: .
file: Dockerfile.ci
.gitignore
+1
View File
@@ -28,6 +28,7 @@ output/
/tmp/
/log/
/lang/
/imgs/
/daemon/
/public/dist/*
!/public/dist/README.md
SECURITY.md
+32
View File
@@ -38,6 +38,22 @@ To keep your OpenList instance secure:
- Regularly review mounted storage permissions and revoke unused API tokens.
- When using Docker, avoid running the container as root if possible.
## Security Response Scope (Applicable to 4.x)
This policy applies only to version 4.x.
In 4.x, privilege isolation between authenticated users (including administrators and regular users) is not currently considered a security boundary that this project commits to maintaining.
In principle, only security vulnerabilities that can be exploited without valid authentication (including authentication bypass vulnerabilities) fall within the scope of this project's dedicated security response process.
Privilege isolation issues that require valid authentication to exploit should preferably be submitted and fixed through regular Pull Requests rather than handled through Security Advisories.
This policy only defines the scope of the project's security response and does not imply that such issues are not security vulnerabilities. For exceptional cases that may have severe security implications, the maintenance team reserves the right to initiate the security response process at its discretion.
This policy does not retroactively affect previously accepted security vulnerability reports.
This project is currently preparing version 5.x. In the future, we may redefine the security boundaries between authenticated users by refactoring the user and permission systems and adjust the security response scope accordingly.
## Acknowledgments
We sincerely thank all security researchers and community members who responsibly disclose vulnerabilities and help make OpenList safer for everyone.
@@ -84,6 +100,22 @@ We sincerely thank all security researchers and community members who responsibl
- 定期检查已挂载存储的权限，并撤销未使用的 API 令牌。
- 使用 Docker 部署时，尽可能避免以 root 用户运行容器。
## 安全响应范围 (适用于 4.x)
本政策仅适用于 4.x 版本。
在 4.x 中，已认证用户 (包括管理员和普通用户) 之间的权限隔离暂不属于本项目承诺维护的安全边界。
只有无需有效身份认证即可利用的安全漏洞 (包括身份认证绕过漏洞)，原则上才纳入本项目的专门安全响应流程。
需要有效身份认证才能利用的权限隔离问题，应优先通过普通 Pull Request 提交和修复，而非通过安全公告 (Security Advisory) 处理。
本政策仅限定项目维护团队的安全响应范围，不代表此类问题不构成安全漏洞。对于可能造成严重安全影响的特殊情况，维护团队保留酌情启动安全响应流程的权利。
本政策不追溯影响此前已正式接受的安全漏洞报告。
本项目正在准备 5.x 版本，未来可能通过重构用户及权限系统，重新定义已认证用户之间的安全边界，并相应调整安全响应范围。
## 致谢
我们衷心感谢所有负责任地披露漏洞、帮助 OpenList 变得更加安全的安全研究人员和社区成员。
cmd/worker/main.go
+580
View File
@@ -0,0 +1,580 @@
// Command openlist-transcode-worker 是独立的远程转码 Worker 节点。
// 它通过 OpenList 自研协议（HTTP/JSON）注册到 Master，长轮询领任务，调用
// 本机 ffmpeg 转码，并把生成的 HLS 切片实时上传回 Master。
package main
import (
"bytes"
"context"
"encoding/json"
"flag"
"fmt"
"io"
"net/http"
"os"
"os/exec"
"os/signal"
"path/filepath"
"runtime"
"strconv"
"strings"
"sync"
"syscall"
"time"
)
var (
masterURL = flag.String("master", "http://127.0.0.1:5244", "OpenList Master URL")
secret = flag.String("secret", "", "Worker 共享密钥（与 Master 配置一致）")
name = flag.String("name", hostname(), "Worker 名称")
capacity = flag.Int("capacity", 2, "并发任务数")
hwaccel = flag.String("hwaccel", "none", "硬件加速：none/auto/nvenc/qsv/vaapi/amf/videotoolbox")
workdir = flag.String("workdir", "./worker_tmp", "临时工作目录")
ffmpeg = flag.String("ffmpeg", "ffmpeg", "ffmpeg 路径")
)
func hostname() string {
h, _ := os.Hostname()
if h == "" {
h = "worker"
}
return h
}
// ---- 协议结构 ----（与 Master 端 internal/transcode/protocol.go 一致）
type RegisterReq struct {
Name string `json:"name"`
Version string `json:"version"`
Capacity int `json:"capacity"`
HWAccel []string `json:"hwaccel"`
CodecsDecode []string `json:"codecs_decode"`
CodecsEncode []string `json:"codecs_encode"`
MaxResolution string `json:"max_resolution"`
Tags []string `json:"tags"`
}
type RegisterResp struct {
Data struct {
WorkerID string `json:"worker_id"`
HeartbeatInterval int `json:"heartbeat_interval"`
} `json:"data"`
}
type HeartbeatReq struct {
WorkerID string `json:"worker_id"`
Load float64 `json:"load"`
Running []string `json:"running"`
FreeSlots int `json:"free_slots"`
}
type HeartbeatResp struct {
Data struct {
OK bool `json:"ok"`
Kick bool `json:"kick"`
Cancel []string `json:"cancel"`
} `json:"data"`
}
type ClaimReq struct {
WorkerID string `json:"worker_id"`
Slots int `json:"slots"`
Wait int `json:"wait"`
}
type Profile struct {
Name string `json:"name"`
VideoCodec string `json:"video_codec"`
VideoBitrate string `json:"video_bitrate"`
Scale string `json:"scale"`
AudioCodec string `json:"audio_codec"`
AudioBitrate string `json:"audio_bitrate"`
HWAccel string `json:"hwaccel"`
}
type OutputSpec struct {
Format string `json:"format"`
SegmentDuration int `json:"segment_duration"`
}
type Job struct {
ID string `json:"id"`
Path string `json:"path"`
SourceURL string `json:"source_url"`
Profiles []Profile `json:"profiles"`
Output OutputSpec `json:"output"`
CallbackToken string `json:"callback_token"`
Deadline int64 `json:"deadline"`
}
type ClaimResp struct {
Data struct {
Jobs []Job `json:"jobs"`
} `json:"data"`
}
type FinishReq struct {
JobID string `json:"job_id"`
Status string `json:"status"`
Error string `json:"error,omitempty"`
Stats struct {
Elapsed float64 `json:"elapsed"`
BytesOut int64 `json:"bytes_out"`
} `json:"stats"`
}
// ---- HTTP helper ----
var httpClient = &http.Client{Timeout: 60 * time.Second}
func postJSON(path string, body any, out any, bearer string) error {
buf, _ := json.Marshal(body)
req, _ := http.NewRequest(http.MethodPost, *masterURL+path, bytes.NewReader(buf))
req.Header.Set("Content-Type", "application/json")
if bearer != "" {
req.Header.Set("Authorization", "Bearer "+bearer)
}
resp, err := httpClient.Do(req)
if err != nil {
return err
}
defer resp.Body.Close()
if resp.StatusCode >= 400 {
raw, _ := io.ReadAll(resp.Body)
return fmt.Errorf("HTTP %d: %s", resp.StatusCode, string(raw))
}
if out != nil {
return json.NewDecoder(resp.Body).Decode(out)
}
return nil
}
// ---- main ----
func main() {
flag.Parse()
if *secret == "" {
fmt.Fprintln(os.Stderr, "missing -secret")
os.Exit(1)
}
if err := os.MkdirAll(*workdir, 0o755); err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
workerID, hbInterval, err := registerOnce()
if err != nil {
fmt.Fprintln(os.Stderr, "register failed:", err)
os.Exit(1)
}
fmt.Printf("[worker] registered: id=%s heartbeat=%ds\n", workerID, hbInterval)
ctx, cancel := context.WithCancel(context.Background())
sigCh := make(chan os.Signal, 1)
signal.Notify(sigCh, syscall.SIGINT, syscall.SIGTERM)
go func() {
<-sigCh
fmt.Println("[worker] shutting down")
cancel()
}()
state := &workerState{
id: workerID,
runningMu: sync.Mutex{},
running: map[string]context.CancelFunc{},
}
// 心跳协程
go func() {
t := time.NewTicker(time.Duration(hbInterval) * time.Second)
defer t.Stop()
for {
select {
case <-ctx.Done():
return
case <-t.C:
state.heartbeat()
}
}
}()
// 工作协程
for i := 0; i < *capacity; i++ {
go state.workLoop(ctx)
}
<-ctx.Done()
time.Sleep(500 * time.Millisecond)
}
func registerOnce() (string, int, error) {
req := RegisterReq{
Name: *name,
Version: "1.0.0",
Capacity: *capacity,
HWAccel: []string{*hwaccel},
CodecsDecode: []string{"h264", "hevc", "av1", "vp9", "mpeg4", "mpeg2video"},
CodecsEncode: []string{"h264", "hevc"},
MaxResolution: "3840x2160",
Tags: []string{runtime.GOOS, runtime.GOARCH},
}
var resp RegisterResp
if err := postJSON("/api/transcode/worker/register", req, &resp, *secret); err != nil {
return "", 0, err
}
if resp.Data.WorkerID == "" {
return "", 0, fmt.Errorf("master returned empty worker_id")
}
if resp.Data.HeartbeatInterval == 0 {
resp.Data.HeartbeatInterval = 10
}
return resp.Data.WorkerID, resp.Data.HeartbeatInterval, nil
}
// ---- 状态机 ----
type workerState struct {
id string
runningMu sync.Mutex
running map[string]context.CancelFunc
}
func (s *workerState) heartbeat() {
s.runningMu.Lock()
ids := make([]string, 0, len(s.running))
for id := range s.running {
ids = append(ids, id)
}
free := *capacity - len(ids)
s.runningMu.Unlock()
var resp HeartbeatResp
if err := postJSON("/api/transcode/worker/heartbeat", HeartbeatReq{
WorkerID: s.id,
Running: ids,
FreeSlots: free,
}, &resp, *secret); err != nil {
fmt.Println("[worker] heartbeat error:", err)
return
}
for _, jid := range resp.Data.Cancel {
s.runningMu.Lock()
if cancel, ok := s.running[jid]; ok {
cancel()
}
s.runningMu.Unlock()
}
}
func (s *workerState) workLoop(ctx context.Context) {
for {
if ctx.Err() != nil {
return
}
var resp ClaimResp
err := postJSON("/api/transcode/worker/claim", ClaimReq{
WorkerID: s.id, Slots: 1, Wait: 20,
}, &resp, *secret)
if err != nil {
fmt.Println("[worker] claim error:", err)
time.Sleep(5 * time.Second)
continue
}
if len(resp.Data.Jobs) == 0 {
continue
}
for _, j := range resp.Data.Jobs {
s.runJob(ctx, j)
}
}
}
func (s *workerState) runJob(parent context.Context, job Job) {
jobCtx, cancel := context.WithCancel(parent)
s.runningMu.Lock()
s.running[job.ID] = cancel
s.runningMu.Unlock()
defer func() {
s.runningMu.Lock()
delete(s.running, job.ID)
s.runningMu.Unlock()
cancel()
}()
startedAt := time.Now()
tmpDir := filepath.Join(*workdir, job.ID)
_ = os.MkdirAll(tmpDir, 0o755)
defer os.RemoveAll(tmpDir)
if len(job.Profiles) == 0 {
s.finish(job, "failed", "no profiles", 0, 0)
return
}
profile := job.Profiles[0]
playlist := filepath.Join(tmpDir, "playlist.m3u8")
segPattern := filepath.Join(tmpDir, "seg-%d.ts")
args := buildFFmpegArgs(job, profile, segPattern, playlist)
cmd := exec.CommandContext(jobCtx, *ffmpeg, args...)
stderr := &bytes.Buffer{}
cmd.Stderr = stderr
fmt.Printf("[worker] running job %s, ffmpeg %s\n", job.ID, strings.Join(args, " "))
if err := cmd.Start(); err != nil {
s.finish(job, "failed", err.Error(), 0, 0)
return
}
// 上传协程
stopUpload := make(chan struct{})
uploadDone := make(chan struct{})
go func() {
defer close(uploadDone)
s.watchAndUpload(jobCtx, job, profile.Name, tmpDir, stopUpload)
}()
err := cmd.Wait()
close(stopUpload)
<-uploadDone
// 最后一次扫描：把还未上传的尾片传上去
s.uploadOnce(job, profile.Name, tmpDir, true)
if err != nil {
if jobCtx.Err() == context.Canceled {
s.finish(job, "cancelled", "cancelled by master", 0, 0)
return
}
s.finish(job, "failed", fmt.Sprintf("%v: %s", err, truncate(stderr.String(), 1024)), 0, 0)
return
}
s.finish(job, "finished", "", time.Since(startedAt).Seconds(), 0)
}
func (s *workerState) finish(job Job, status, errStr string, elapsed float64, bytesOut int64) {
req := FinishReq{
JobID: job.ID, Status: status, Error: errStr,
}
req.Stats.Elapsed = elapsed
req.Stats.BytesOut = bytesOut
if err := postJSON("/api/transcode/worker/job/finish", req, nil, job.CallbackToken); err != nil {
fmt.Printf("[worker] finish report failed: %v\n", err)
}
}
func (s *workerState) watchAndUpload(_ context.Context, job Job, profile, dir string, stop chan struct{}) {
t := time.NewTicker(800 * time.Millisecond)
defer t.Stop()
uploaded := map[int]bool{}
flush := func(final bool) {
s.uploadDelta(job, profile, dir, uploaded, final)
}
for {
select {
case <-stop:
flush(false)
return
case <-t.C:
flush(false)
}
}
}
func (s *workerState) uploadOnce(job Job, profile, dir string, final bool) {
uploaded := map[int]bool{}
s.uploadDelta(job, profile, dir, uploaded, final)
}
// uploadDelta 扫描目录，上传新切片（uploaded 跟踪已传过的 seq；最后一片到 final 时才上传）
func (s *workerState) uploadDelta(job Job, profile, dir string, uploaded map[int]bool, final bool) {
entries, err := os.ReadDir(dir)
if err != nil {
return
}
type segFile struct {
seq int
path string
}
var segs []segFile
for _, e := range entries {
if e.IsDir() {
continue
}
n := e.Name()
if !strings.HasPrefix(n, "seg-") || !strings.HasSuffix(n, ".ts") {
continue
}
num, err := strconv.Atoi(strings.TrimSuffix(strings.TrimPrefix(n, "seg-"), ".ts"))
if err != nil {
continue
}
segs = append(segs, segFile{num, filepath.Join(dir, n)})
}
// 排序
for i := 0; i < len(segs); i++ {
for j := i + 1; j < len(segs); j++ {
if segs[j].seq < segs[i].seq {
segs[i], segs[j] = segs[j], segs[i]
}
}
}
limit := len(segs)
if !final && limit > 0 {
limit-- // 最后一片可能还在写，留到下一轮
}
segDur := job.Output.SegmentDuration
if segDur == 0 {
segDur = 6
}
for i := 0; i < limit; i++ {
seg := segs[i]
if uploaded[seg.seq] {
continue
}
isLast := final && i == limit-1
if err := s.uploadSegment(job, profile, seg.seq, float64(segDur), seg.path, isLast); err != nil {
fmt.Printf("[worker] upload seg-%d failed: %v\n", seg.seq, err)
return
}
uploaded[seg.seq] = true
}
}
func (s *workerState) uploadSegment(job Job, profile string, seq int, dur float64, file string, final bool) error {
f, err := os.Open(file)
if err != nil {
return err
}
defer f.Close()
q := fmt.Sprintf("?job=%s&profile=%s&seq=%d&duration=%f&final=%t",
job.ID, profile, seq, dur, final)
req, _ := http.NewRequest(http.MethodPut, *masterURL+"/api/transcode/worker/segment"+q, f)
req.Header.Set("Content-Type", "video/mp2t")
req.Header.Set("Authorization", "Bearer "+job.CallbackToken)
resp, err := httpClient.Do(req)
if err != nil {
return err
}
defer resp.Body.Close()
if resp.StatusCode >= 400 {
raw, _ := io.ReadAll(resp.Body)
return fmt.Errorf("HTTP %d: %s", resp.StatusCode, string(raw))
}
return nil
}
// ---- ffmpeg args（与 master 端 local_runner 保持一致）----
// protocolWhitelistFor 与 internal/transcode.ProtocolWhitelistFor 保持一致：
// 远程输入只放行 HTTP(S) 及其传输层，本地路径只放行 file。
func protocolWhitelistFor(source string) string {
s := strings.ToLower(strings.TrimSpace(source))
if strings.HasPrefix(s, "http://") || strings.HasPrefix(s, "https://") {
// 与 master 的 protocolWhitelistRemote 保持一致：HTTP 源用不到 file/crypto，
// 留着等于给被构造过的播放列表留一条读本地文件的通道。
return "http,https,tcp,tls"
}
return "file"
}
func buildFFmpegArgs(job Job, profile Profile, segPattern, playlist string) []string {
hw := strings.ToLower(profile.HWAccel)
args := []string{"-y", "-loglevel", "warning"}
switch hw {
case "nvenc":
args = append(args, "-hwaccel", "cuda")
case "qsv":
args = append(args, "-hwaccel", "qsv")
case "vaapi":
args = append(args, "-hwaccel", "vaapi", "-hwaccel_output_format", "vaapi", "-vaapi_device", "/dev/dri/renderD128")
case "amf":
args = append(args, "-hwaccel", "d3d11va")
case "videotoolbox":
args = append(args, "-hwaccel", "videotoolbox")
case "auto":
args = append(args, "-hwaccel", "auto")
}
// 限制 ffmpeg 可用协议：Master 下发的 source_url 不应被解释成 concat:/subfile: 等特殊协议。
// -protocol_whitelist 是输入选项，必须位于 -i 之前。
args = append(args, "-protocol_whitelist", protocolWhitelistFor(job.SourceURL))
args = append(args, "-i", job.SourceURL)
args = append(args, "-c:v", pickEncoder(profile.VideoCodec, hw))
if profile.VideoBitrate != "" {
args = append(args, "-b:v", profile.VideoBitrate, "-maxrate", profile.VideoBitrate, "-bufsize", profile.VideoBitrate)
}
if profile.Scale != "" {
switch hw {
case "vaapi":
args = append(args, "-vf", "scale_vaapi="+profile.Scale)
case "qsv":
args = append(args, "-vf", "scale_qsv="+profile.Scale)
default:
args = append(args, "-vf", "scale="+profile.Scale)
}
}
if profile.AudioCodec == "copy" {
args = append(args, "-c:a", "copy")
} else {
ac := profile.AudioCodec
if ac == "" {
ac = "aac"
}
args = append(args, "-c:a", ac)
if profile.AudioBitrate != "" {
args = append(args, "-b:a", profile.AudioBitrate)
}
}
segDur := job.Output.SegmentDuration
if segDur == 0 {
segDur = 6
}
args = append(args,
"-f", "hls",
"-hls_time", strconv.Itoa(segDur),
"-hls_playlist_type", "vod",
"-hls_segment_filename", segPattern,
"-hls_flags", "independent_segments",
playlist,
)
return args
}
func pickEncoder(codec, hw string) string {
codec = strings.ToLower(codec)
switch codec {
case "h264", "":
switch hw {
case "nvenc":
return "h264_nvenc"
case "qsv":
return "h264_qsv"
case "vaapi":
return "h264_vaapi"
case "amf":
return "h264_amf"
case "videotoolbox":
return "h264_videotoolbox"
}
return "libx264"
case "hevc", "h265":
switch hw {
case "nvenc":
return "hevc_nvenc"
case "qsv":
return "hevc_qsv"
case "vaapi":
return "hevc_vaapi"
case "amf":
return "hevc_amf"
case "videotoolbox":
return "hevc_videotoolbox"
}
return "libx265"
case "av1":
switch hw {
case "nvenc":
return "av1_nvenc"
case "qsv":
return "av1_qsv"
}
return "libsvtav1"
}
return "libx264"
}
func truncate(s string, n int) string {
if len(s) <= n {
return s
}
return s[:n]
}
drivers/115/seed_rapid.go
+74
View File
@@ -0,0 +1,74 @@
package _115
import (
"context"
"strings"
"github.com/OpenListTeam/OpenList/v4/internal/driver"
"github.com/OpenListTeam/OpenList/v4/internal/errs"
"github.com/OpenListTeam/OpenList/v4/internal/model"
"github.com/OpenListTeam/OpenList/v4/pkg/http_range"
"github.com/OpenListTeam/OpenList/v4/pkg/utils"
)
// RapidHashAlgos 返回 115 支持的秒传哈希算法（SHA1）
func (d *Pan115) RapidHashAlgos() []utils.HashType {
return []utils.HashType{*utils.SHA1}
}
// RapidHashNeedsPieces 115 不需要分片哈希
func (d *Pan115) RapidHashNeedsPieces() bool {
return false
}
// RapidUploadByHashes 使用种子中的 SHA1 哈希尝试秒传。
//
// 115 的秒传协议除了整文件 SHA1，还需要文件头部 128KB 的 SHA1（pre_hash），
// 因此当内容源可用时会打开它来计算前置哈希；内容不可用时无法秒传。
func (d *Pan115) RapidUploadByHashes(ctx context.Context, dstDir model.Obj, req *driver.SeedRapidUploadRequest, overwrite bool) (model.Obj, error) {
fullHash := strings.ToUpper(req.Whole.GetHash(utils.SHA1))
if len(fullHash) != utils.SHA1.Width {
return nil, errs.ErrUnavailableHash
}
if req.Open == nil {
return nil, errs.ErrUnavailableHash
}
src, err := req.Open()
if err != nil {
return nil, err
}
defer src.Close()
const PreHashSize int64 = 128 * utils.KB
hashSize := PreHashSize
if req.Size < PreHashSize {
hashSize = req.Size
}
reader, err := src.RangeRead(http_range.Range{Start: 0, Length: hashSize})
if err != nil {
return nil, err
}
preHash, err := utils.HashReader(utils.SHA1, reader)
if err != nil {
return nil, err
}
preHash = strings.ToUpper(preHash)
fastInfo, err := d.rapidUpload(req.Size, req.Name, dstDir.GetID(), preHash, fullHash, src)
if err != nil {
return nil, err
}
matched, err := fastInfo.Ok()
if err != nil {
return nil, err
}
if !matched {
return nil, errs.ErrRapidUploadFailed
}
f, err := d.getNewFileByPickCode(fastInfo.PickCode)
if err != nil {
return nil, err
}
return f, nil
}
drivers/115/util.go
+7 -1
View File
@@ -141,6 +141,7 @@ func (d *Pan115) rapidUpload(fileSize int64, fileName, dirID, preID, fileID stri
form.Set("sig", d.client.GenerateSignature(fileID, target))
signKey, signVal := "", ""
decryptRetryCount := 0
for retry := true; retry; {
t := driver115.NowMilli()
@@ -172,11 +173,16 @@ func (d *Pan115) rapidUpload(fileSize int64, fileName, dirID, preID, fileID stri
return nil, err
}
data := resp.RawBody()
defer data.Close()
if bodyBytes, err = io.ReadAll(data); err != nil {
_ = data.Close()
return nil, err
}
_ = data.Close()
if decrypted, err = ecdhCipher.Decrypt(bodyBytes); err != nil {
decryptRetryCount++
if decryptRetryCount < 3 {
continue
}
return nil, err
}
if err = driver115.CheckErr(json.Unmarshal(decrypted, &result), &result, resp); err != nil {
drivers/115_open/driver.go
+11
View File
@@ -171,11 +171,22 @@ func (d *Open115) Get(ctx context.Context, path string) (model.Obj, error) {
path = stdpath.Join(d.parentPath, path)
resp, err := d.client.GetFolderInfoByPath(ctx, path)
if err != nil {
// SDK-level "object not found" (empty array response from API)
if errors.Is(err, sdk.ErrObjectNotFound) {
return d.getFromParent(ctx, path, "")
}
// API-level error response (State=false), treat as not found
// since this is a path-lookup that can't resolve the target
var apiErr *sdk.Error
if errors.As(err, &apiErr) {
return nil, errs.ObjectNotFound
}
return nil, err
}
// Handle null/empty response (e.g., API returns null data for non-existent path)
if resp.FileID == "" {
return nil, errs.ObjectNotFound
}
obj := &Obj{
Fid: resp.FileID,
Fn: resp.FileName,
drivers/123/seed_rapid.go
+73
View File
@@ -0,0 +1,73 @@
package _123
import (
"context"
"net/http"
"strconv"
"strings"
"time"
"github.com/OpenListTeam/OpenList/v4/drivers/base"
"github.com/OpenListTeam/OpenList/v4/internal/driver"
"github.com/OpenListTeam/OpenList/v4/internal/errs"
"github.com/OpenListTeam/OpenList/v4/internal/model"
"github.com/OpenListTeam/OpenList/v4/pkg/utils"
"github.com/go-resty/resty/v2"
)
// RapidHashAlgos 返回 123 云盘支持的秒传哈希算法（MD5）
func (d *Pan123) RapidHashAlgos() []utils.HashType {
return []utils.HashType{*utils.MD5}
}
// RapidHashNeedsPieces 123 云盘不需要分片哈希
func (d *Pan123) RapidHashNeedsPieces() bool {
return false
}
// RapidUploadByHashes 使用种子中的 MD5（Etag）哈希尝试秒传。
//
// 123 的秒传即「上传请求返回 reuse=true」，无需真正传输内容。
func (d *Pan123) RapidUploadByHashes(ctx context.Context, dstDir model.Obj, req *driver.SeedRapidUploadRequest, overwrite bool) (model.Obj, error) {
etag := req.Whole.GetHash(utils.MD5)
if len(etag) < utils.MD5.Width {
return nil, errs.ErrUnavailableHash
}
duplicate := 0
if overwrite {
duplicate = 2
}
data := base.Json{
"driveId": 0,
"duplicate": duplicate,
"etag": strings.ToLower(etag),
"fileName": req.Name,
"parentFileId": dstDir.GetID(),
"size": req.Size,
"type": 0,
}
var resp UploadResp
_, err := d.Request(UploadRequest, http.MethodPost, func(r *resty.Request) {
r.SetBody(data).SetContext(ctx)
}, &resp)
if err != nil {
return nil, err
}
// reuse=true 或未返回上传 Key 均视为秒传成功
if !resp.Data.Reuse && resp.Data.Key != "" {
return nil, errs.ErrHashMismatch
}
return &model.ObjThumb{
Object: model.Object{
ID: strconv.FormatInt(resp.Data.FileId, 10),
Name: req.Name,
Size: req.Size,
Modified: time.Now(),
IsFolder: false,
HashInfo: utils.NewHashInfo(utils.MD5, strings.ToLower(etag)),
},
}, nil
}
drivers/123/util.go
+1 -1
View File
@@ -181,7 +181,7 @@ func (d *Pan123) login() error {
return err
}
if utils.Json.Get(res.Body(), "code").ToInt() != 200 {
err = fmt.Errorf(utils.Json.Get(res.Body(), "message").ToString())
err = fmt.Errorf("%s", utils.Json.Get(res.Body(), "message").ToString())
} else {
d.AccessToken = utils.Json.Get(res.Body(), "data", "token").ToString()
}
drivers/123_open/seed_rapid.go
+55
View File
@@ -0,0 +1,55 @@
package _123_open
import (
"context"
"path"
"strconv"
"time"
"github.com/OpenListTeam/OpenList/v4/internal/driver"
"github.com/OpenListTeam/OpenList/v4/internal/errs"
"github.com/OpenListTeam/OpenList/v4/internal/model"
"github.com/OpenListTeam/OpenList/v4/pkg/utils"
)
// RapidHashAlgos 返回 123 开放平台支持的秒传哈希算法（SHA1）
func (d *Open123) RapidHashAlgos() []utils.HashType {
return []utils.HashType{*utils.SHA1}
}
// RapidHashNeedsPieces 123 开放平台不需要分片哈希
func (d *Open123) RapidHashNeedsPieces() bool {
return false
}
// RapidUploadByHashes 使用种子中的 SHA1 哈希尝试秒传
func (d *Open123) RapidUploadByHashes(ctx context.Context, dstDir model.Obj, req *driver.SeedRapidUploadRequest, overwrite bool) (model.Obj, error) {
sha1Hash := req.Whole.GetHash(utils.SHA1)
if len(sha1Hash) < utils.SHA1.Width {
return nil, errs.ErrUnavailableHash
}
parentID, err := strconv.ParseInt(dstDir.GetID(), 10, 64)
if err != nil {
return nil, err
}
resp, err := d.sha1Reuse(parentID, req.Name, sha1Hash, req.Size, 1)
if err != nil {
return nil, err
}
if !resp.Data.Reuse {
return nil, errs.ErrHashMismatch
}
return &model.ObjThumb{
Object: model.Object{
ID: strconv.FormatInt(resp.Data.FileID, 10),
Name: req.Name,
Size: req.Size,
IsFolder: false,
Path: path.Join(dstDir.GetPath(), req.Name),
Modified: time.Now(),
},
}, nil
}
drivers/189/torrent.go
+4 -9
View File
@@ -6,7 +6,6 @@ import (
"encoding/hex"
"fmt"
"io"
"strings"
"github.com/OpenListTeam/OpenList/v4/internal/model"
"github.com/OpenListTeam/OpenList/v4/pkg/torrent"
@@ -15,12 +14,8 @@ import (
// GenerateTorrent 根据上传过程中收集的哈希信息生成包含 CAS 扩展的 torrent 文件
func GenerateTorrent(fileName string, fileSize int64, fileMD5 string, sliceMD5s []string, sliceSize int64, pieceHashes []byte) ([]byte, error) {
// 计算 sliceMD5
sliceMD5 := fileMD5
if len(sliceMD5s) > 1 {
joined := strings.Join(sliceMD5s, "\n")
sliceMD5 = strings.ToUpper(torrent.GetMD5Str(joined))
}
// 计算 sliceMD5（统一走规范实现）
sliceMD5 := torrent.SliceMD5FromPieces(sliceMD5s, fileMD5)
t := torrent.NewTorrent(fileName, fileSize, fileMD5)
t.Info.PieceLength = sliceSize
@@ -30,7 +25,7 @@ func GenerateTorrent(fileName string, fileSize int64, fileMD5 string, sliceMD5s
SliceMD5: sliceMD5,
SliceMD5s: sliceMD5s,
SliceSize: sliceSize,
Cloud: "189",
Cloud: torrent.Cloud189,
})
return t.Encode()
@@ -95,7 +90,7 @@ func ComputeTorrentFromReader(reader io.Reader, fileName string, fileSize int64,
sliceSize = torrent.DefaultPieceSize
}
hw := torrent.NewHashWriter(sliceSize, sliceSize)
hw := torrent.NewHashWriter(sliceSize, sliceSize, fileSize)
buf := make([]byte, 32*1024)
for {
drivers/189/util.go
+1 -1
View File
@@ -238,7 +238,7 @@ func (d *Cloud189) oldUpload(dstDir model.Obj, file model.FileStreamer) error {
if utils.Json.Get(res.Body(), "MD5").ToString() != "" {
return nil
}
log.Debugf(res.String())
log.Debug(res.String())
return errors.New(res.String())
}
drivers/189_tv/seed_rapid.go
+35
View File
@@ -0,0 +1,35 @@
package _189_tv
import (
"context"
"github.com/OpenListTeam/OpenList/v4/internal/driver"
"github.com/OpenListTeam/OpenList/v4/internal/errs"
"github.com/OpenListTeam/OpenList/v4/internal/model"
"github.com/OpenListTeam/OpenList/v4/pkg/utils"
)
// RapidHashAlgos 返回 189 电视支持的秒传哈希算法（MD5）
func (d *Cloud189TV) RapidHashAlgos() []utils.HashType {
return []utils.HashType{*utils.MD5}
}
// RapidHashNeedsPieces 189 电视不需要分片哈希
func (d *Cloud189TV) RapidHashNeedsPieces() bool {
return false
}
// RapidUploadByHashes 使用种子中的 MD5 哈希尝试秒传
func (d *Cloud189TV) RapidUploadByHashes(ctx context.Context, dstDir model.Obj, req *driver.SeedRapidUploadRequest, overwrite bool) (model.Obj, error) {
md5Hash := req.Whole.GetHash(utils.MD5)
if len(md5Hash) < utils.MD5.Width {
return nil, errs.ErrUnavailableHash
}
stream := driver.NewSeedHashStream(req)
obj, err := d.RapidUpload(ctx, dstDir, stream, d.isFamily(), overwrite)
if err != nil {
return nil, err
}
return obj, nil
}
drivers/189pc/seed_rapid.go
+35
View File
@@ -0,0 +1,35 @@
package _189pc
import (
"context"
"github.com/OpenListTeam/OpenList/v4/internal/driver"
"github.com/OpenListTeam/OpenList/v4/internal/errs"
"github.com/OpenListTeam/OpenList/v4/internal/model"
"github.com/OpenListTeam/OpenList/v4/pkg/utils"
)
// RapidHashAlgos 返回 189pc 支持的秒传哈希算法（MD5）
func (d *Cloud189PC) RapidHashAlgos() []utils.HashType {
return []utils.HashType{*utils.MD5}
}
// RapidHashNeedsPieces 189pc 的 CAS 秒传依赖分片 MD5
func (d *Cloud189PC) RapidHashNeedsPieces() bool {
return true
}
// RapidUploadByHashes 使用种子中的 MD5 哈希尝试秒传
func (d *Cloud189PC) RapidUploadByHashes(ctx context.Context, dstDir model.Obj, req *driver.SeedRapidUploadRequest, overwrite bool) (model.Obj, error) {
md5Hash := req.Whole.GetHash(utils.MD5)
if len(md5Hash) < utils.MD5.Width {
return nil, errs.ErrUnavailableHash
}
stream := driver.NewSeedHashStream(req)
obj, err := d.RapidUpload(ctx, dstDir, stream, d.isFamily(), overwrite)
if err != nil {
return nil, err
}
return obj, nil
}
drivers/189pc/torrent.go
+17 -109
View File
@@ -7,7 +7,6 @@ import (
"encoding/hex"
"fmt"
"io"
"net/url"
"strings"
"time"
@@ -28,12 +27,8 @@ import (
// fileName: 文件名
// fileSize: 文件大小
func GenerateTorrent(fileName string, fileSize int64, fileMD5 string, sliceMD5s []string, sliceSize int64, pieceHashes []byte) ([]byte, error) {
// 计算 sliceMD5
sliceMD5 := fileMD5
if len(sliceMD5s) > 1 {
joined := strings.Join(sliceMD5s, "\n")
sliceMD5 = strings.ToUpper(torrent.GetMD5Str(joined))
}
// 计算 sliceMD5（统一走规范实现）
sliceMD5 := torrent.SliceMD5FromPieces(sliceMD5s, fileMD5)
t := torrent.NewTorrent(fileName, fileSize, fileMD5)
t.Info.PieceLength = sliceSize
@@ -43,7 +38,7 @@ func GenerateTorrent(fileName string, fileSize int64, fileMD5 string, sliceMD5s
SliceMD5: sliceMD5,
SliceMD5s: sliceMD5s,
SliceSize: sliceSize,
Cloud: "189",
Cloud: torrent.Cloud189,
})
return t.Encode()
@@ -69,101 +64,18 @@ func (y *Cloud189PC) RapidUploadFromTorrent(ctx context.Context, dstDir model.Ob
fileName := t.Info.Name
fileSize := t.GetTotalSize()
// 统一 MD5 为大写（与正常上传保持一致，天翼云盘要求大写）
fileMD5Upper := strings.ToUpper(cas.FileMD5)
// 优先使用 torrent 中嵌入的分片大小，与生成时保持一致
sliceSize := cas.SliceSize
if sliceSize <= 0 {
sliceSize = partSize(fileSize)
// 优先使用 torrent 中嵌入的分片 MD5 与大写整文件 MD5
fileMD5 := strings.ToUpper(cas.FileMD5)
sliceMD5s := make([]string, len(cas.SliceMD5s))
for i, s := range cas.SliceMD5s {
sliceMD5s[i] = strings.ToUpper(s)
}
// 计算 sliceMd5（与上传时一致的算法）
// 优先使用 torrent 中已有的 SliceMD5；仅当有多分片列表时才重新计算
sliceMd5Hex := strings.ToUpper(cas.SliceMD5)
if sliceMd5Hex == "" {
sliceMd5Hex = fileMD5Upper
}
if len(cas.SliceMD5s) > 1 {
// 分片 MD5 也需要统一大写后再拼接计算
upperSliceMD5s := make([]string, len(cas.SliceMD5s))
for i, s := range cas.SliceMD5s {
upperSliceMD5s[i] = strings.ToUpper(s)
}
sliceMd5Hex = strings.ToUpper(utils.GetMD5EncodeStr(strings.Join(upperSliceMD5s, "\n")))
}
// 使用与 Web 端一致的三步秒传流程
fullUrl := "https://upload.cloud.189.cn"
if isFamily {
fullUrl += "/family"
} else {
fullUrl += "/person"
}
// Step 1: initMultiUpload（不传 fileMd5/sliceMd5，只传 lazyCheck）
initParams := Params{
"parentFolderId": dstDir.GetID(),
"fileName": url.QueryEscape(fileName),
"fileSize": fmt.Sprint(fileSize),
"sliceSize": fmt.Sprint(sliceSize),
"lazyCheck": "1",
}
if isFamily {
initParams.Set("familyId", y.FamilyID)
}
var uploadInfo InitMultiUploadResp
_, err = y.request(fullUrl+"/initMultiUpload", "GET", func(req *resty.Request) {
req.SetContext(ctx)
}, initParams, &uploadInfo, isFamily)
// 复用统一的 CAS 秒传核心实现
respObj, err := y.rapidUploadByCAS(ctx, dstDir, fileName, fileSize, fileMD5, sliceMD5s, cas.SliceSize, overwrite)
if err != nil {
return nil, fmt.Errorf("initMultiUpload 失败: %w", err)
}
uploadFileId := uploadInfo.Data.UploadFileID
// Step 2: checkTransSecond（用 fileMd5 + sliceMd5 + uploadFileId 检查秒传）
checkParams := Params{
"fileMd5": fileMD5Upper,
"sliceMd5": sliceMd5Hex,
"uploadFileId": uploadFileId,
}
var checkResp struct {
Data struct {
FileDataExists int `json:"fileDataExists"`
} `json:"data"`
}
_, err = y.request(fullUrl+"/checkTransSecond", "GET", func(req *resty.Request) {
req.SetContext(ctx)
}, checkParams, &checkResp, isFamily)
if err != nil {
utils.Log.Errorf("[RapidUpload] checkTransSecond 失败: uploadFileId=%s, err=%v", uploadFileId, err)
return nil, fmt.Errorf("秒传检查失败: %w", err)
}
if checkResp.Data.FileDataExists != 1 {
return nil, fmt.Errorf("秒传失败：云端不存在该文件（fileMD5=%s, sliceMD5=%s, size=%d）", fileMD5Upper, sliceMd5Hex, fileSize)
}
// Step 3: commitMultiUploadFile（传 fileMd5 + sliceMd5）
var resp CommitMultiUploadFileResp
commitParams := Params{
"uploadFileId": uploadFileId,
"fileMd5": fileMD5Upper,
"sliceMd5": sliceMd5Hex,
"lazyCheck": "1",
"opertype": IF(overwrite, "3", "1"),
}
_, err = y.request(fullUrl+"/commitMultiUploadFile", "GET", func(req *resty.Request) {
req.SetContext(ctx)
}, commitParams, &resp, isFamily)
if err != nil {
utils.Log.Errorf("[RapidUpload] commitMultiUploadFile 失败: uploadFileId=%s, err=%v", uploadFileId, err)
return nil, fmt.Errorf("提交上传失败: %w", err)
utils.Log.Errorf("[RapidUpload] 秒传失败: fileMD5=%s, err=%v", fileMD5, err)
return nil, err
}
// 秒传成功后，将 torrent 文件上传到目标目录（异步，不影响秒传结果）
@@ -196,7 +108,7 @@ func (y *Cloud189PC) RapidUploadFromTorrent(ctx context.Context, dstDir model.Ob
}()
}
return resp.toFile(), nil
return respObj, nil
}
// ComputeTorrentFromReader 从 io.Reader 计算并生成 torrent 文件
@@ -206,7 +118,7 @@ func ComputeTorrentFromReader(reader io.Reader, fileName string, fileSize int64,
sliceSize = torrent.DefaultPieceSize
}
hw := torrent.NewHashWriter(sliceSize, sliceSize)
hw := torrent.NewHashWriter(sliceSize, sliceSize, fileSize)
buf := make([]byte, 32*1024)
for {
@@ -259,12 +171,8 @@ func InjectCASIntoTorrent(torrentData []byte, fileMD5 string, sliceMD5s []string
return nil, fmt.Errorf("解析 torrent 失败: %w", err)
}
// 计算 sliceMD5
sliceMD5 := fileMD5
if len(sliceMD5s) > 1 {
joined := strings.Join(sliceMD5s, "\n")
sliceMD5 = strings.ToUpper(torrent.GetMD5Str(joined))
}
// 计算 sliceMD5（统一走规范实现）
sliceMD5 := torrent.SliceMD5FromPieces(sliceMD5s, fileMD5)
// 注入 CAS 信息
t.SetCASInfo(&torrent.CASInfo{
@@ -272,7 +180,7 @@ func InjectCASIntoTorrent(torrentData []byte, fileMD5 string, sliceMD5s []string
SliceMD5: sliceMD5,
SliceMD5s: sliceMD5s,
SliceSize: sliceSize,
Cloud: "189",
Cloud: torrent.Cloud189,
})
// 同时更新 info 中的 md5sum 字段
drivers/189pc/utils.go
+96
View File
@@ -1613,6 +1613,102 @@ func (y *Cloud189PC) OldUploadCommit(ctx context.Context, fileCommitUrl string,
return resp.toFile(), nil
}
// rapidUploadByCAS 使用 MD5 + 分片 MD5（CAS）执行天翼云盘秒传。
//
// 流程与 Web 端一致：
// 1. initMultiUpload（仅传 lazyCheck=1）
// 2. checkTransSecond（用 fileMd5 + sliceMd5 检查云端是否已存在文件数据）
// 3. commitMultiUploadFile（提交并返回文件对象）
func (y *Cloud189PC) rapidUploadByCAS(ctx context.Context, dstDir model.Obj, fileName string, fileSize int64, fileMD5 string, sliceMD5s []string, sliceSize int64, overwrite bool) (model.Obj, error) {
isFamily := y.isFamily()
// 统一 MD5 为大写（天翼云盘要求大写）
fileMD5Upper := strings.ToUpper(fileMD5)
// 优先使用传入的分片大小，否则按文件大小推导
if sliceSize <= 0 {
sliceSize = partSize(fileSize)
}
// 计算 sliceMd5（与上传时一致的算法）
sliceMd5Hex := fileMD5Upper
if len(sliceMD5s) > 1 {
upperSliceMD5s := make([]string, len(sliceMD5s))
for i, s := range sliceMD5s {
upperSliceMD5s[i] = strings.ToUpper(s)
}
sliceMd5Hex = strings.ToUpper(utils.GetMD5EncodeStr(strings.Join(upperSliceMD5s, "\n")))
} else if len(sliceMD5s) == 1 {
sliceMd5Hex = strings.ToUpper(sliceMD5s[0])
}
fullUrl := "https://upload.cloud.189.cn"
if isFamily {
fullUrl += "/family"
} else {
fullUrl += "/person"
}
// Step 1: initMultiUpload（不传 fileMd5/sliceMd5，只传 lazyCheck）
initParams := Params{
"parentFolderId": dstDir.GetID(),
"fileName": url.QueryEscape(fileName),
"fileSize": fmt.Sprint(fileSize),
"sliceSize": fmt.Sprint(sliceSize),
"lazyCheck": "1",
}
if isFamily {
initParams.Set("familyId", y.FamilyID)
}
var uploadInfo InitMultiUploadResp
if _, err := y.request(fullUrl+"/initMultiUpload", "GET", func(req *resty.Request) {
req.SetContext(ctx)
}, initParams, &uploadInfo, isFamily); err != nil {
return nil, fmt.Errorf("initMultiUpload 失败: %w", err)
}
uploadFileId := uploadInfo.Data.UploadFileID
// Step 2: checkTransSecond（用 fileMd5 + sliceMd5 + uploadFileId 检查秒传）
checkParams := Params{
"fileMd5": fileMD5Upper,
"sliceMd5": sliceMd5Hex,
"uploadFileId": uploadFileId,
}
var checkResp struct {
Data struct {
FileDataExists int `json:"fileDataExists"`
} `json:"data"`
}
if _, err := y.request(fullUrl+"/checkTransSecond", "GET", func(req *resty.Request) {
req.SetContext(ctx)
}, checkParams, &checkResp, isFamily); err != nil {
return nil, fmt.Errorf("秒传检查失败: %w", err)
}
if checkResp.Data.FileDataExists != 1 {
return nil, fmt.Errorf("秒传失败：云端不存在该文件（fileMD5=%s, sliceMD5=%s, size=%d）", fileMD5Upper, sliceMd5Hex, fileSize)
}
// Step 3: commitMultiUploadFile（传 fileMd5 + sliceMd5）
commitParams := Params{
"uploadFileId": uploadFileId,
"fileMd5": fileMD5Upper,
"sliceMd5": sliceMd5Hex,
"lazyCheck": "1",
"opertype": IF(overwrite, "3", "1"),
}
var resp CommitMultiUploadFileResp
if _, err := y.request(fullUrl+"/commitMultiUploadFile", "GET", func(req *resty.Request) {
req.SetContext(ctx)
}, commitParams, &resp, isFamily); err != nil {
return nil, fmt.Errorf("提交上传失败: %w", err)
}
return resp.toFile(), nil
}
func (y *Cloud189PC) isFamily() bool {
return y.Type == "family"
}
drivers/aliyundrive_open/seed_rapid.go
+88
View File
@@ -0,0 +1,88 @@
package aliyundrive_open
import (
"context"
"net/http"
"time"
"github.com/OpenListTeam/OpenList/v4/drivers/base"
"github.com/OpenListTeam/OpenList/v4/internal/driver"
"github.com/OpenListTeam/OpenList/v4/internal/errs"
"github.com/OpenListTeam/OpenList/v4/internal/model"
"github.com/OpenListTeam/OpenList/v4/pkg/utils"
"github.com/go-resty/resty/v2"
)
// RapidHashAlgos 返回阿里云盘支持的秒传哈希算法（SHA1）
func (d *AliyundriveOpen) RapidHashAlgos() []utils.HashType {
return []utils.HashType{*utils.SHA1}
}
// RapidHashNeedsPieces 阿里云盘不需要分片哈希
func (d *AliyundriveOpen) RapidHashNeedsPieces() bool {
return false
}
// RapidUploadByHashes 使用种子中的 SHA1 哈希尝试秒传。
//
// 阿里云盘的秒传还需要 proof_code（按 proof range 读取的一段内容），
// 因此当内容源不可用时无法完成秒传。
func (d *AliyundriveOpen) RapidUploadByHashes(ctx context.Context, dstDir model.Obj, req *driver.SeedRapidUploadRequest, overwrite bool) (model.Obj, error) {
sha1Hash := req.Whole.GetHash(utils.SHA1)
if len(sha1Hash) < utils.SHA1.Width {
return nil, errs.ErrUnavailableHash
}
if req.Open == nil {
return nil, errs.ErrUnavailableHash
}
stream, err := req.Open()
if err != nil {
return nil, err
}
defer stream.Close()
proofCode, err := d.calProofCode(stream)
if err != nil {
return nil, err
}
var resp CreateResp
_, err = d.request(ctx, limiterOther, "/adrive/v1.0/openFile/create", http.MethodPost, func(r *resty.Request) {
r.SetBody(base.Json{
"drive_id": d.DriveId,
"parent_file_id": dstDir.GetID(),
"name": req.Name,
"type": "file",
"check_name_mode": "auto_rename",
"size": req.Size,
"content_hash": sha1Hash,
"content_hash_name": "sha1",
"proof_version": "v1",
"proof_code": proofCode,
}).SetResult(&resp)
})
if err != nil {
return nil, err
}
if !resp.RapidUpload {
return nil, errs.ErrHashMismatch
}
if resp.FileId != "" {
obj, err := d.completeUpload(ctx, resp.FileId, resp.UploadId)
if err != nil {
return nil, err
}
return obj, nil
}
return &model.ObjThumb{
Object: model.Object{
Name: req.Name,
Size: req.Size,
Modified: time.Now(),
IsFolder: false,
},
}, nil
}
drivers/all.go
+1
View File
@@ -30,6 +30,7 @@ import (
_ "github.com/OpenListTeam/OpenList/v4/drivers/cloudreve_v4"
_ "github.com/OpenListTeam/OpenList/v4/drivers/cnb_releases"
_ "github.com/OpenListTeam/OpenList/v4/drivers/crypt"
_ "github.com/OpenListTeam/OpenList/v4/drivers/ctfile"
_ "github.com/OpenListTeam/OpenList/v4/drivers/degoo"
_ "github.com/OpenListTeam/OpenList/v4/drivers/doubao"
_ "github.com/OpenListTeam/OpenList/v4/drivers/doubao_new"
drivers/baidu_netdisk/seed_rapid.go
+35
View File
@@ -0,0 +1,35 @@
package baidu_netdisk
import (
"context"
"github.com/OpenListTeam/OpenList/v4/internal/driver"
"github.com/OpenListTeam/OpenList/v4/internal/errs"
"github.com/OpenListTeam/OpenList/v4/internal/model"
"github.com/OpenListTeam/OpenList/v4/pkg/utils"
)
// RapidHashAlgos 返回百度网盘支持的秒传哈希算法（MD5）
func (d *BaiduNetdisk) RapidHashAlgos() []utils.HashType {
return []utils.HashType{*utils.MD5}
}
// RapidHashNeedsPieces 百度网盘不需要分片哈希
func (d *BaiduNetdisk) RapidHashNeedsPieces() bool {
return false
}
// RapidUploadByHashes 使用种子中的 MD5 哈希尝试秒传
func (d *BaiduNetdisk) RapidUploadByHashes(ctx context.Context, dstDir model.Obj, req *driver.SeedRapidUploadRequest, overwrite bool) (model.Obj, error) {
md5Hash := req.Whole.GetHash(utils.MD5)
if len(md5Hash) < utils.MD5.Width {
return nil, errs.ErrUnavailableHash
}
stream := driver.NewSeedHashStream(req)
obj, err := d.PutRapid(ctx, dstDir, stream)
if err != nil {
return nil, err
}
return obj, nil
}
drivers/baidu_netdisk/util.go
+1 -1
View File
@@ -112,7 +112,7 @@ func (d *BaiduNetdisk) request(furl string, method string, callback base.ReqCall
log.Debugf("[baidu_netdisk] req: %s, resp: %s", furl, res.String())
errno := utils.Json.Get(res.Body(), "errno").ToInt()
if errno != 0 {
if utils.SliceContains([]int{111, -6}, errno) {
if utils.SliceContains([]int{111, -6, 20016, 20017}, errno) {
log.Info("[baidu_netdisk] refreshing baidu_netdisk token.")
err2 := d.refreshToken()
if err2 != nil {
drivers/chaoxing/driver.go
+2 -3
View File
@@ -21,7 +21,6 @@ import (
"github.com/OpenListTeam/OpenList/v4/pkg/cron"
"github.com/OpenListTeam/OpenList/v4/pkg/utils"
"github.com/go-resty/resty/v2"
"google.golang.org/appengine/log"
)
type ChaoXing struct {
@@ -55,13 +54,13 @@ func (d *ChaoXing) refreshCookie() error {
func (d *ChaoXing) Init(ctx context.Context) error {
err := d.refreshCookie()
if err != nil {
log.Errorf(ctx, err.Error())
utils.Log.Errorf("%v", err)
}
d.cron = cron.NewCron(time.Hour * 12)
d.cron.Do(func() {
err = d.refreshCookie()
if err != nil {
log.Errorf(ctx, err.Error())
utils.Log.Errorf("%v", err)
}
})
return nil
drivers/ctfile/driver.go
+252
View File
@@ -0,0 +1,252 @@
package ctfile
import (
"context"
"fmt"
"io"
"strings"
"time"
"github.com/OpenListTeam/OpenList/v4/internal/driver"
"github.com/OpenListTeam/OpenList/v4/internal/model"
"github.com/OpenListTeam/OpenList/v4/internal/stream"
"github.com/OpenListTeam/OpenList/v4/pkg/utils"
)
// CtFile implements driver.Driver for 城通网盘 (CtFile) via the OpenAPI
// session-based authentication. No email/password/SMS login is used.
type CtFile struct {
model.Storage
Addition
}
func (d *CtFile) Config() driver.Config {
return config
}
func (d *CtFile) GetAddition() driver.Additional {
return &d.Addition
}
func (d *CtFile) Init(ctx context.Context) error {
if strings.TrimSpace(d.Session) == "" {
return fmt.Errorf("[CtFile] session is required; generate one in the CtFile backend under 「开放接口登录密钥管理」")
}
return d.profile(ctx)
}
func (d *CtFile) Drop(ctx context.Context) error {
return nil
}
// List returns the contents of dir (both sub-folders and files).
func (d *CtFile) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) {
folderID := dir.GetID()
if folderID == "" {
folderID = rootFolderID
}
folders, err := d.listFolders(ctx, folderID)
if err != nil {
return nil, err
}
files, err := d.listFiles(ctx, folderID)
if err != nil {
return nil, err
}
var objs []model.Obj
for _, f := range folders {
objs = append(objs, &model.Object{
ID: f.Key,
Name: f.Name,
IsFolder: true,
Modified: parseDate(f.Date),
Ctime: parseDate(f.Date),
})
}
for _, f := range files {
objs = append(objs, &model.Object{
ID: f.Key,
Name: f.Name,
Size: f.Size,
IsFolder: false,
Modified: parseDate(f.Date),
Ctime: parseDate(f.Date),
})
}
return objs, nil
}
// Link fetches the CDN download URL for file via fetch_url.
func (d *CtFile) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) {
url, err := d.fetchURL(ctx, file.GetID())
if err != nil {
return nil, err
}
return &model.Link{
URL: url,
Header: map[string][]string{
"User-Agent": {d.ua()},
},
}, nil
}
// MakeDir creates a sub-folder under parentDir.
func (d *CtFile) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) (model.Obj, error) {
parentID := parentDir.GetID()
id, err := d.mkdir(ctx, parentID, dirName)
if err != nil {
return nil, err
}
return &model.Object{
ID: id,
Name: dirName,
IsFolder: true,
Modified: time.Now(),
Ctime: time.Now(),
}, nil
}
// Move relocates srcObj into dstDir.
func (d *CtFile) Move(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) {
err := d.move(ctx, []string{srcObj.GetID()}, dstDir.GetID())
if err != nil {
return nil, err
}
return srcObj, nil
}
// Rename changes the name of srcObj to newName.
func (d *CtFile) Rename(ctx context.Context, srcObj model.Obj, newName string) (model.Obj, error) {
var err error
if srcObj.IsDir() {
err = d.renameFolder(ctx, srcObj.GetID(), newName)
} else {
err = d.renameFile(ctx, srcObj.GetID(), newName)
}
if err != nil {
return nil, err
}
return &model.Object{
ID: srcObj.GetID(),
Name: newName,
Size: srcObj.GetSize(),
IsFolder: srcObj.IsDir(),
Modified: srcObj.ModTime(),
Ctime: srcObj.CreateTime(),
}, nil
}
// Copy copies srcObj into dstDir.
// The spec explicitly requires implementation; the Worker's "unimplemented"
// note is overridden here. The API endpoint is POST /{ns}/file/copy.
func (d *CtFile) Copy(ctx context.Context, srcObj, dstDir model.Obj) error {
return d.copyFiles(ctx, []string{srcObj.GetID()}, dstDir.GetID())
}
// Remove deletes obj (moves to trash).
func (d *CtFile) Remove(ctx context.Context, obj model.Obj) error {
return d.remove(ctx, []string{obj.GetID()})
}
// Put uploads a file to dstDir. The stream is cached so the checksum and the
// subsequent multipart upload can reuse it without keeping the entire file in memory.
func (d *CtFile) Put(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress) error {
md5hex := file.GetHash().GetHash(utils.MD5)
tmpFile := file.GetFile()
var err error
var cacheProgress *model.UpdateProgress
if up != nil {
cacheProgress = &up
}
if len(md5hex) != utils.MD5.Width {
// Cache the full stream and compute MD5 simultaneously.
tmpFile, md5hex, err = stream.CacheFullAndHash(file, cacheProgress, utils.MD5)
} else if tmpFile == nil {
// A pre-computed MD5 still needs a seekable source for the upload.
tmpFile, err = file.CacheFullAndWriter(cacheProgress, nil)
}
if err != nil {
return err
}
if tmpFile == nil {
return fmt.Errorf("[CtFile] failed to cache upload stream")
}
if _, err := tmpFile.Seek(0, io.SeekStart); err != nil {
return err
}
size := file.GetSize()
if size < 0 {
size, err = tmpFile.Seek(0, io.SeekEnd)
if err != nil {
return err
}
if _, err := tmpFile.Seek(0, io.SeekStart); err != nil {
return err
}
}
if up != nil {
up(50)
}
parentID := dstDir.GetID()
err = d.uploadFile(ctx, parentID, file.GetName(), tmpFile, size, md5hex)
if err != nil {
return err
}
if up != nil {
up(100)
}
return nil
}
// parseDate converts a CtFile date value (unix seconds or "YYYY-MM-DD HH:mm:ss")
// to time.Time, falling back to time.Now() on failure.
func parseDate(v any) time.Time {
if v == nil {
return time.Now()
}
switch tv := v.(type) {
case float64:
if tv > 0 {
return time.Unix(int64(tv), 0)
}
case string:
if tv == "" {
return time.Now()
}
// Try unix seconds as string
if n, err := parseInt64(tv); err == nil && n > 0 {
return time.Unix(n, 0)
}
// Try "YYYY-MM-DD HH:mm:ss"
layouts := []string{"2006-01-02 15:04:05", "2006-01-02T15:04:05Z"}
for _, layout := range layouts {
if t, err := time.Parse(layout, tv); err == nil {
return t
}
}
}
return time.Now()
}
func parseInt64(s string) (int64, error) {
var n int64
_, err := fmt.Sscanf(s, "%d", &n)
return n, err
}
var (
_ driver.Driver = (*CtFile)(nil)
_ driver.MkdirResult = (*CtFile)(nil)
_ driver.MoveResult = (*CtFile)(nil)
_ driver.RenameResult = (*CtFile)(nil)
_ driver.Copy = (*CtFile)(nil)
_ driver.Remove = (*CtFile)(nil)
_ driver.Put = (*CtFile)(nil)
)
drivers/ctfile/meta.go
+44
View File
@@ -0,0 +1,44 @@
package ctfile
import (
"github.com/OpenListTeam/OpenList/v4/internal/driver"
"github.com/OpenListTeam/OpenList/v4/internal/op"
)
// Addition holds all user-configurable fields for the CtFile driver.
type Addition struct {
// Session is the OpenAPI token generated in the CtFile backend under
// 「开放接口登录密钥管理」. Required.
Session string `json:"session" required:"true" secret:"true" help:"OpenAPI session token from 开放接口登录密钥管理"`
// CloudType selects namespace: "public" (default) or "private".
CloudType string `json:"cloud_type" type:"select" options:"public,private" default:"public"`
// BaseURL is the API root, defaulting to https://rest.ctfile.com/v1.
BaseURL string `json:"base_url" default:"https://rest.ctfile.com/v1" help:"API base URL"`
// OrderBy controls server-side sort field.
OrderBy string `json:"order_by" type:"select" options:"name,size,time" default:"name"`
// OrderDirection is the sort direction.
OrderDirection string `json:"order_direction" type:"select" options:"asc,desc" default:"asc"`
// UserAgent overrides the default Chrome-like UA sent with every request.
UserAgent string `json:"user_agent" default:"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/142.0.0.0 Safari/537.36"`
driver.RootID
}
var config = driver.Config{
Name: "CtFile",
DefaultRoot: "0",
LocalSort: true,
NoUpload: false,
OnlyProxy: false,
}
func init() {
op.RegisterDriver(func() driver.Driver {
return &CtFile{}
})
}
drivers/ctfile/types.go
+51
View File
@@ -0,0 +1,51 @@
package ctfile
// CtFile (城通网盘) OpenAPI response structs.
// API doc: https://openapi.ctfile.com/ (REST API v1, unified POST JSON + session auth).
// All business endpoints return a unified wrapper { code, message?, ...business fields }.
// `code == 200` means success; `401` means session expired; `403` with folder_id on mkdir
// means the folder already exists.
// Resp is the unified response wrapper. Business fields are decoded via
// utils.Json to stay tolerant of the server's loosely-typed payloads (numbers
// may arrive as strings and vice-versa).
type Resp struct {
Code int `json:"code"`
Message string `json:"message"`
// folder_id / file_id / download_url / upload_url / exists / results / ...
// are accessed through utils.Json, so we keep the raw bytes here.
}
// profileResp holds the fields returned by POST /user/info/profile.
type profileResp struct {
UserID int64 `json:"userid"`
UserName string `json:"username"`
NickName string `json:"nick_name"`
GroupName string `json:"group_name"`
MaxFileSz int64 `json:"max_filesize"`
}
// folderEntry is a single folder in folder/list results.
type folderEntry struct {
Key string `json:"key"`
Name string `json:"name"`
Icon string `json:"icon"`
Date any `json:"date"` // unix seconds or "YYYY-MM-DD HH:mm:ss"
}
// fileEntry is a single file in file/list results.
type fileEntry struct {
Key string `json:"key"`
Name string `json:"name"`
Icon string `json:"icon"`
ImgSrc string `json:"imgsrc"`
Size int64 `json:"size"`
Date any `json:"date"`
Status int `json:"status"` // 1=complete, 2=incomplete (not downloadable)
}
// uploadTicket is the POST /{ns}/file/upload response.
type uploadTicket struct {
UploadURL string `json:"upload_url"`
Exists int `json:"exists"` // >0 => instant upload hit
}
drivers/ctfile/util.go
+598
View File
@@ -0,0 +1,598 @@
package ctfile
import (
"bytes"
"context"
"encoding/json"
"fmt"
"io"
"mime/multipart"
"net/http"
"regexp"
"strconv"
"strings"
"time"
)
const (
defaultBaseURL = "https://rest.ctfile.com/v1"
defaultUserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/142.0.0.0 Safari/537.36"
rootFolderID = "d0"
maxListPages = 500
)
// sessionExpiredMsg is the human-readable error for 401 responses.
const sessionExpiredMsg = "[CtFile] Session expired (401). Please regenerate your session token in the CtFile backend under 「开放接口登录密钥管理」"
// stripIDPrefix removes a leading d or f prefix from an ID when the next
// character is a digit. E.g. "d123" → "123", "f456" → "456", "abc" → "abc".
var stripPrefixRe = regexp.MustCompile(`^([df])(\d)`)
func stripIDPrefix(id string) string {
if m := stripPrefixRe.FindStringSubmatch(id); m != nil {
return id[1:] // drop the leading letter
}
return id
}
func (d *CtFile) ns() string {
if d.CloudType == "private" {
return "private"
}
return "public"
}
func (d *CtFile) baseURL() string {
if u := strings.TrimRight(d.BaseURL, "/"); u != "" {
return u
}
return defaultBaseURL
}
func (d *CtFile) ua() string {
if d.UserAgent != "" {
return d.UserAgent
}
return defaultUserAgent
}
// orderby maps the user fields to CtFile's server-side sort tokens.
func (d *CtFile) orderby() string {
keys := map[string]string{"name": "az", "size": "big", "time": "new"}
rev := map[string]string{"az": "za", "big": "small", "new": "old"}
base, ok := keys[d.OrderBy]
if !ok {
base = "az"
}
if d.OrderDirection == "desc" {
return rev[base]
}
return base
}
// post is the raw transport layer: POST JSON, include session, retry on
// 429/5xx, map 401 to a clear error. Returns decoded JSON map or error.
func (d *CtFile) post(ctx context.Context, path string, body map[string]any) (map[string]any, error) {
session := strings.TrimSpace(d.Session)
if session == "" {
return nil, fmt.Errorf("[CtFile] session is not configured; please fill in the session token from 开放接口登录密钥管理")
}
payload := make(map[string]any, len(body)+1)
for k, v := range body {
payload[k] = v
}
payload["session"] = session
data, err := json.Marshal(payload)
if err != nil {
return nil, fmt.Errorf("[CtFile] json marshal: %w", err)
}
url := d.baseURL() + path
var lastErr error
for attempt := 0; attempt < 3; attempt++ {
if attempt > 0 {
select {
case <-ctx.Done():
return nil, ctx.Err()
case <-time.After(time.Duration(300*attempt) * time.Millisecond):
}
}
req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(data))
if err != nil {
return nil, err
}
req.Header.Set("Content-Type", "application/json")
req.Header.Set("Accept", "application/json")
req.Header.Set("User-Agent", d.ua())
resp, err := http.DefaultClient.Do(req)
if err != nil {
lastErr = err
continue
}
// Retry on 429 / 5xx
if resp.StatusCode == 429 || resp.StatusCode >= 500 {
_ = resp.Body.Close()
lastErr = fmt.Errorf("HTTP %d", resp.StatusCode)
continue
}
body2, err := io.ReadAll(resp.Body)
_ = resp.Body.Close()
if err != nil {
return nil, err
}
var result map[string]any
if err := json.Unmarshal(body2, &result); err != nil {
return nil, fmt.Errorf("[CtFile] %s: non-JSON response (HTTP %d): %.200s", path, resp.StatusCode, body2)
}
code := int(toInt64(result["code"]))
if code == 401 {
return nil, fmt.Errorf("%s", sessionExpiredMsg)
}
if code == 429 {
lastErr = fmt.Errorf("[CtFile] rate limited (code 429)")
continue
}
return result, nil
}
if lastErr != nil {
return nil, fmt.Errorf("[CtFile] %s failed after retries: %w", path, lastErr)
}
return nil, fmt.Errorf("[CtFile] %s failed after retries", path)
}
// call wraps post and checks code == 200.
func (d *CtFile) call(ctx context.Context, path string, body map[string]any) (map[string]any, error) {
result, err := d.post(ctx, path, body)
if err != nil {
return nil, err
}
// Missing / zero code is treated as success (Worker: code === undefined → ok).
if _, hasCode := result["code"]; hasCode {
code := int(toInt64(result["code"]))
if code != 200 {
return result, &apiError{code: code, message: toString(result["message"]), raw: result}
}
}
return result, nil
}
// callWithId calls an endpoint that takes a single id parameter, implementing
// the dual-try strategy (prefer one form; on 400/404 retry the other).
//
// prefer = "raw" → try id as-is first, fallback to stripped
// prefer = "stripped" → try stripped first, fallback to raw
func (d *CtFile) callWithId(ctx context.Context, path, idKey, id string, extra map[string]any, prefer string) (map[string]any, error) {
stripped := stripIDPrefix(id)
var primary, secondary string
if prefer == "stripped" {
primary, secondary = stripped, id
} else {
primary, secondary = id, stripped
}
body := make(map[string]any, len(extra)+1)
for k, v := range extra {
body[k] = v
}
body[idKey] = primary
result, err := d.call(ctx, path, body)
if err != nil {
var ae *apiError
if asAPIError(err, &ae) && (ae.code == 400 || ae.code == 404) && primary != secondary {
body[idKey] = secondary
return d.call(ctx, path, body)
}
return nil, err
}
return result, nil
}
// callWithIds calls an endpoint that takes a list of ids; tries comma-string
// first, on 400/404/405 retries JSON array.
func (d *CtFile) callWithIds(ctx context.Context, path string, ids []string, extra map[string]any) (map[string]any, error) {
body := make(map[string]any, len(extra)+2)
for k, v := range extra {
body[k] = v
}
// First attempt: comma-separated string
body["ids"] = strings.Join(ids, ",")
result, err := d.call(ctx, path, body)
if err != nil {
var ae *apiError
if asAPIError(err, &ae) && (ae.code == 400 || ae.code == 404 || ae.code == 405) {
// Second attempt: JSON array
body["ids"] = ids
return d.call(ctx, path, body)
}
return nil, err
}
return result, nil
}
// --------------------------------------------------------------------------
// High-level API helpers
// --------------------------------------------------------------------------
// profile fetches the user's profile; used to validate the session in Init.
func (d *CtFile) profile(ctx context.Context) error {
_, err := d.call(ctx, "/user/info/profile", nil)
return err
}
// listFolders returns all sub-folders for a given parent folder ID.
func (d *CtFile) listFolders(ctx context.Context, folderID string) ([]folderEntry, error) {
if folderID == "" || folderID == "0" {
folderID = rootFolderID
}
result, err := d.call(ctx, "/"+d.ns()+"/folder/list", map[string]any{
"folder_id": folderID,
"orderby": d.orderby(),
})
if err != nil {
return nil, err
}
items := toSlice(result["results"])
out := make([]folderEntry, 0, len(items))
for _, raw := range items {
m, ok := raw.(map[string]any)
if !ok {
continue
}
out = append(out, folderEntry{
Key: toString(m["key"]),
Name: toString(m["name"]),
Icon: toString(m["icon"]),
Date: m["date"],
})
}
return out, nil
}
// listFiles returns all files in a folder, paginating until empty or duplicate.
func (d *CtFile) listFiles(ctx context.Context, folderID string) ([]fileEntry, error) {
if folderID == "" || folderID == "0" {
folderID = rootFolderID
}
var all []fileEntry
start := 0
prevFirstKey := ""
for page := 0; page < maxListPages; page++ {
result, err := d.call(ctx, "/"+d.ns()+"/file/list", map[string]any{
"folder_id": folderID,
"start": start,
"orderby": d.orderby(),
})
if err != nil {
return nil, err
}
items := toSlice(result["results"])
if len(items) == 0 {
break
}
// Guard against server ignoring `start` and looping forever
firstKey := ""
if m, ok := items[0].(map[string]any); ok {
firstKey = toString(m["key"])
}
if page > 0 && firstKey == prevFirstKey {
break
}
prevFirstKey = firstKey
for _, raw := range items {
m, ok := raw.(map[string]any)
if !ok {
continue
}
all = append(all, fileEntry{
Key: toString(m["key"]),
Name: toString(m["name"]),
Icon: toString(m["icon"]),
ImgSrc: toString(m["imgsrc"]),
Size: toInt64(m["size"]),
Date: m["date"],
Status: int(toInt64(m["status"])),
})
}
start += len(items)
}
return all, nil
}
// mkdir creates a folder and returns its ID. 403 with an existing folder_id is
// treated as success (folder already exists).
func (d *CtFile) mkdir(ctx context.Context, parentID, name string) (string, error) {
if parentID == "" || parentID == "0" {
parentID = rootFolderID
}
result, err := d.call(ctx, "/"+d.ns()+"/folder/create", map[string]any{
"folder_id": parentID,
"name": name,
"description": "",
"is_hidden": false,
})
if err != nil {
var ae *apiError
if asAPIError(err, &ae) && ae.code == 403 {
// Folder already exists; the response still carries folder_id
if id := toString(ae.raw["folder_id"]); id != "" {
return id, nil
}
}
return "", err
}
id := toString(result["folder_id"])
if id == "" {
id = toString(result["path"])
}
return id, nil
}
// fetchURL returns the CDN download URL for a file.
func (d *CtFile) fetchURL(ctx context.Context, fileID string) (string, error) {
result, err := d.callWithId(ctx, "/"+d.ns()+"/file/fetch_url", "file_id", fileID, nil, "stripped")
if err != nil {
return "", err
}
url := toString(result["download_url"])
if url == "" {
return "", fmt.Errorf("[CtFile] fetch_url returned empty download_url")
}
return url, nil
}
// renameFile renames a file.
func (d *CtFile) renameFile(ctx context.Context, fileID, newName string) error {
_, err := d.callWithId(ctx, "/"+d.ns()+"/file/modify_meta", "file_id", fileID, map[string]any{"name": newName}, "stripped")
return err
}
// renameFolder renames a folder.
func (d *CtFile) renameFolder(ctx context.Context, folderID, newName string) error {
_, err := d.callWithId(ctx, "/"+d.ns()+"/folder/modify_meta", "folder_id", folderID, map[string]any{
"name": newName,
"is_rename": true,
}, "raw")
return err
}
// remove moves files/folders to trash.
func (d *CtFile) remove(ctx context.Context, ids []string) error {
if len(ids) == 0 {
return nil
}
_, err := d.callWithIds(ctx, "/"+d.ns()+"/file/delete", ids, nil)
return err
}
// move moves files/folders to another folder.
func (d *CtFile) move(ctx context.Context, ids []string, targetFolderID string) error {
if len(ids) == 0 {
return nil
}
if targetFolderID == "" || targetFolderID == "0" {
targetFolderID = rootFolderID
}
_, err := d.callWithIds(ctx, "/"+d.ns()+"/file/move", ids, map[string]any{"folder_id": targetFolderID})
return err
}
// copyFiles copies files to a target folder.
func (d *CtFile) copyFiles(ctx context.Context, ids []string, targetFolderID string) error {
if len(ids) == 0 {
return nil
}
if targetFolderID == "" || targetFolderID == "0" {
targetFolderID = rootFolderID
}
_, err := d.callWithIds(ctx, "/"+d.ns()+"/file/copy", ids, map[string]any{"folder_id": targetFolderID})
return err
}
// uploadFile uploads a stream to CtFile using the two-step ticket+multipart flow.
// checksum format: md5hex + "-" + size
func (d *CtFile) uploadFile(ctx context.Context, folderID, fileName string, content io.Reader, size int64, md5hex string) error {
if folderID == "" || folderID == "0" {
folderID = rootFolderID
}
checksum := fmt.Sprintf("%s-%d", md5hex, size)
// Step 1: get upload ticket
result, err := d.call(ctx, "/"+d.ns()+"/file/upload", map[string]any{
"folder_id": folderID,
"checksum": checksum,
"size": size,
"name": fileName,
})
if err != nil {
return err
}
exists := toInt64(result["exists"])
if exists > 0 {
// Instant upload / deduplication
return nil
}
uploadURL := toString(result["upload_url"])
if uploadURL == "" {
return fmt.Errorf("[CtFile] upload ticket returned no upload_url")
}
// Step 2: multipart upload to the presigned URL
return d.multipartUpload(ctx, uploadURL, fileName, content, size)
}
// countingWriter counts bytes without retaining the written data.
type countingWriter int64
func (w *countingWriter) Write(p []byte) (int, error) {
*w += countingWriter(len(p))
return len(p), nil
}
func writeMultipartFields(mw *multipart.Writer, fileName string, size int64) (io.Writer, error) {
if err := mw.WriteField("name", fileName); err != nil {
return nil, err
}
if err := mw.WriteField("filesize", strconv.FormatInt(size, 10)); err != nil {
return nil, err
}
return mw.CreateFormFile("file", fileName)
}
func multipartLayout(fileName string, size int64) (int64, string, error) {
var count countingWriter
mw := multipart.NewWriter(&count)
if _, err := writeMultipartFields(mw, fileName, size); err != nil {
return 0, "", err
}
if err := mw.Close(); err != nil {
return 0, "", err
}
return int64(count) + size, mw.Boundary(), nil
}
// multipartUpload POSTs name+filesize+file to a presigned upload URL without
// retaining the complete multipart body in memory.
func (d *CtFile) multipartUpload(ctx context.Context, uploadURL, fileName string, content io.Reader, size int64) error {
contentLength, boundary, err := multipartLayout(fileName, size)
if err != nil {
return err
}
pr, pw := io.Pipe()
req, err := http.NewRequestWithContext(ctx, http.MethodPost, uploadURL, pr)
if err != nil {
return err
}
req.ContentLength = contentLength
req.Header.Set("Content-Type", "multipart/form-data; boundary="+boundary)
req.Header.Set("User-Agent", d.ua())
go func() {
mw := multipart.NewWriter(pw)
if err := mw.SetBoundary(boundary); err != nil {
_ = pw.CloseWithError(err)
return
}
fw, err := writeMultipartFields(mw, fileName, size)
if err != nil {
_ = pw.CloseWithError(err)
return
}
if _, err := io.Copy(fw, content); err != nil {
_ = pw.CloseWithError(err)
return
}
if err := mw.Close(); err != nil {
_ = pw.CloseWithError(err)
return
}
_ = pw.Close()
}()
resp, err := http.DefaultClient.Do(req)
if err != nil {
_ = pr.CloseWithError(err)
return err
}
defer resp.Body.Close()
body, err := io.ReadAll(resp.Body)
if err != nil {
return err
}
if resp.StatusCode != 200 {
return fmt.Errorf("[CtFile] multipart upload HTTP %d: %.200s", resp.StatusCode, body)
}
var result map[string]any
if err := json.Unmarshal(body, &result); err != nil {
return fmt.Errorf("[CtFile] multipart upload non-JSON: %.200s", body)
}
code := int(toInt64(result["code"]))
if code != 200 {
return fmt.Errorf("[CtFile] multipart upload failed (code=%d): %s", code, toString(result["message"]))
}
return nil
}
// --------------------------------------------------------------------------
// apiError carries a structured error with code + raw body for retry checks.
// --------------------------------------------------------------------------
type apiError struct {
code int
message string
raw map[string]any
}
func (e *apiError) Error() string {
return fmt.Sprintf("[CtFile] API error (code=%d): %s", e.code, e.message)
}
// asAPIError sets dst and returns true when err is an *apiError.
func asAPIError(err error, dst **apiError) bool {
if ae, ok := err.(*apiError); ok {
*dst = ae
return true
}
return false
}
// --------------------------------------------------------------------------
// Type-coercion helpers for loosely-typed JSON payloads.
// --------------------------------------------------------------------------
func toInt64(v any) int64 {
if v == nil {
return 0
}
switch n := v.(type) {
case float64:
return int64(n)
case int64:
return n
case int:
return int64(n)
case string:
i, _ := strconv.ParseInt(n, 10, 64)
return i
default:
return 0
}
}
func toString(v any) string {
if v == nil {
return ""
}
switch s := v.(type) {
case string:
return s
default:
return fmt.Sprintf("%v", v)
}
}
func toSlice(v any) []any {
if s, ok := v.([]any); ok {
return s
}
return nil
}
drivers/ctfile/util_test.go
+352
View File
@@ -0,0 +1,352 @@
package ctfile
import (
"context"
"crypto/md5"
"fmt"
"io"
"net/http"
"net/http/httptest"
"strings"
"testing"
)
// ---------------------------------------------------------------------------
// Helpers / mock
// ---------------------------------------------------------------------------
// buildServer creates a test HTTP server that returns pre-scripted responses in
// sequence. Each response is a JSON string (already complete). The variadic
// responses are consumed one at a time; the last one is reused for all
// subsequent requests.
func buildServer(t *testing.T, responses ...string) (*httptest.Server, *[]string) {
t.Helper()
calls := new([]string)
idx := 0
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
body, _ := io.ReadAll(r.Body)
*calls = append(*calls, string(body))
resp := responses[idx]
if idx < len(responses)-1 {
idx++
}
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(200)
_, _ = w.Write([]byte(resp))
}))
return srv, calls
}
// driverWithBase returns a minimal CtFile driver pointing at the test server.
func driverWithBase(baseURL string) *CtFile {
d := &CtFile{}
d.Session = "test-session"
d.BaseURL = baseURL
d.CloudType = "public"
d.OrderBy = "name"
d.OrderDirection = "asc"
return d
}
// ---------------------------------------------------------------------------
// StripIDPrefix
// ---------------------------------------------------------------------------
func TestStripIDPrefix(t *testing.T) {
cases := []struct {
in string
out string
}{
{"d123", "123"},
{"f456", "456"},
{"abc", "abc"},
{"d", "d"}, // no digit follows → keep as-is
{"f", "f"}, // no digit follows → keep as-is
{"d0", "0"},
{"", ""},
{"123", "123"},
}
for _, tc := range cases {
got := stripIDPrefix(tc.in)
if got != tc.out {
t.Errorf("stripIDPrefix(%q) = %q, want %q", tc.in, got, tc.out)
}
}
}
// ---------------------------------------------------------------------------
// callWithId dual-try: 400 → retry with alternate id form
// ---------------------------------------------------------------------------
func TestCallWithId_DualTry(t *testing.T) {
// First call returns code 400; second returns code 200 with a name field.
srv, calls := buildServer(t,
`{"code":400,"message":"not found"}`,
`{"code":200,"name":"Photos"}`,
)
defer srv.Close()
d := driverWithBase(srv.URL)
ctx := context.Background()
// prefer = "raw" → first attempt with "d123", fallback with "123"
result, err := d.callWithId(ctx, "/public/folder/meta", "folder_id", "d123", nil, "raw")
if err != nil {
t.Fatalf("callWithId returned error: %v", err)
}
if toString(result["name"]) != "Photos" {
t.Errorf("expected name=Photos, got %v", result["name"])
}
if len(*calls) != 2 {
t.Fatalf("expected 2 HTTP calls, got %d", len(*calls))
}
if !strings.Contains((*calls)[0], `"d123"`) {
t.Errorf("first call should use raw id d123, got: %s", (*calls)[0])
}
if !strings.Contains((*calls)[1], `"123"`) {
t.Errorf("second call should use stripped id 123, got: %s", (*calls)[1])
}
}
// ---------------------------------------------------------------------------
// callWithId: prefer stripped → file/fetch_url pattern
// ---------------------------------------------------------------------------
func TestCallWithId_PreferStripped(t *testing.T) {
// First call (stripped "456") returns 400; second (raw "f456") returns 200.
srv, calls := buildServer(t,
`{"code":400,"message":"param error"}`,
`{"code":200,"download_url":"https://cdn.example.com/file"}`,
)
defer srv.Close()
d := driverWithBase(srv.URL)
ctx := context.Background()
result, err := d.callWithId(ctx, "/public/file/fetch_url", "file_id", "f456", nil, "stripped")
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if toString(result["download_url"]) != "https://cdn.example.com/file" {
t.Errorf("unexpected download_url: %v", result["download_url"])
}
if len(*calls) != 2 {
t.Fatalf("expected 2 calls, got %d", len(*calls))
}
if !strings.Contains((*calls)[0], `"456"`) {
t.Errorf("first call should use stripped 456: %s", (*calls)[0])
}
if !strings.Contains((*calls)[1], `"f456"`) {
t.Errorf("second call should use raw f456: %s", (*calls)[1])
}
}
// ---------------------------------------------------------------------------
// callWithIds: comma-string → JSON array fallback on 400
// ---------------------------------------------------------------------------
func TestCallWithIds_DualTry(t *testing.T) {
srv, calls := buildServer(t,
`{"code":400,"message":"param error"}`,
`{"code":200,"message":"deleted"}`,
)
defer srv.Close()
d := driverWithBase(srv.URL)
ctx := context.Background()
err := d.remove(ctx, []string{"f1", "f2"})
if err != nil {
t.Fatalf("remove returned error: %v", err)
}
if len(*calls) != 2 {
t.Fatalf("expected 2 calls, got %d", len(*calls))
}
// First attempt: ids is comma-string
if !strings.Contains((*calls)[0], `"f1,f2"`) {
t.Errorf("first call should have comma-string ids: %s", (*calls)[0])
}
// Second attempt: ids is JSON array
if !strings.Contains((*calls)[1], `["f1","f2"]`) {
t.Errorf("second call should have JSON array ids: %s", (*calls)[1])
}
}
// ---------------------------------------------------------------------------
// Checksum format: md5hex-size
// ---------------------------------------------------------------------------
func TestChecksumFormat(t *testing.T) {
content := []byte("hello")
sum := md5.Sum(content)
checksum := fmt.Sprintf("%x-%d", sum, len(content))
// md5("hello") = 5d41402abc4b2a76b9719d911017c592
want := "5d41402abc4b2a76b9719d911017c592-5"
if checksum != want {
t.Errorf("checksum = %q, want %q", checksum, want)
}
}
// ---------------------------------------------------------------------------
// 401 → session expired message
// ---------------------------------------------------------------------------
func TestSession401Error(t *testing.T) {
srv, _ := buildServer(t, `{"code":401,"message":"not logged in"}`)
defer srv.Close()
d := driverWithBase(srv.URL)
ctx := context.Background()
err := d.profile(ctx)
if err == nil {
t.Fatal("expected error for 401, got nil")
}
msg := err.Error()
if !strings.Contains(msg, "401") && !strings.Contains(msg, "expired") && !strings.Contains(msg, "Session") {
t.Errorf("error message should mention session/expired/401; got: %s", msg)
}
if !strings.Contains(msg, "开放接口登录密钥管理") {
t.Errorf("error message should mention 开放接口登录密钥管理; got: %s", msg)
}
}
// ---------------------------------------------------------------------------
// mkdir: 403 with folder_id → treat as success
// ---------------------------------------------------------------------------
func TestMkdir403Success(t *testing.T) {
// Server returns 403 with an existing folder_id
srv, _ := buildServer(t, `{"code":403,"message":"folder exists","folder_id":"d999"}`)
defer srv.Close()
d := driverWithBase(srv.URL)
ctx := context.Background()
id, err := d.mkdir(ctx, "d0", "MyFolder")
if err != nil {
t.Fatalf("mkdir should succeed on 403 with folder_id, got error: %v", err)
}
if id != "d999" {
t.Errorf("expected id=d999, got %q", id)
}
}
// ---------------------------------------------------------------------------
// list files: pagination stops on empty page
// ---------------------------------------------------------------------------
func TestListFiles_Pagination(t *testing.T) {
// Page 1: two files; page 2: empty
srv, calls := buildServer(t,
`{"code":200,"results":[{"key":"f1","name":"a.txt","size":100},{"key":"f2","name":"b.txt","size":200}]}`,
`{"code":200,"results":[]}`,
)
defer srv.Close()
d := driverWithBase(srv.URL)
ctx := context.Background()
files, err := d.listFiles(ctx, "d0")
if err != nil {
t.Fatalf("listFiles: %v", err)
}
if len(files) != 2 {
t.Errorf("expected 2 files, got %d", len(files))
}
if len(*calls) != 2 {
t.Errorf("expected 2 HTTP calls (2 pages), got %d", len(*calls))
}
}
// ---------------------------------------------------------------------------
// list files: duplicate first key terminates pagination (anti-loop guard)
// ---------------------------------------------------------------------------
func TestListFiles_DuplicateKeyStops(t *testing.T) {
sameResp := `{"code":200,"results":[{"key":"f1","name":"a.txt","size":1},{"key":"f2","name":"b.txt","size":2}]}`
srv, calls := buildServer(t, sameResp, sameResp, sameResp)
defer srv.Close()
d := driverWithBase(srv.URL)
ctx := context.Background()
files, err := d.listFiles(ctx, "d0")
if err != nil {
t.Fatalf("listFiles: %v", err)
}
// Should stop after 2 calls (first page + duplicate detection on second)
if len(*calls) != 2 {
t.Errorf("expected 2 calls (duplicate guard), got %d", len(*calls))
}
if len(files) != 2 {
t.Errorf("expected 2 files, got %d", len(files))
}
}
// ---------------------------------------------------------------------------
// namespace: private cloud uses /private/ prefix
// ---------------------------------------------------------------------------
func TestNamespace_Private(t *testing.T) {
srv, calls := buildServer(t, `{"code":200,"results":[]}`)
defer srv.Close()
d := driverWithBase(srv.URL)
d.CloudType = "private"
ctx := context.Background()
_, _ = d.listFolders(ctx, "d0")
if len(*calls) < 1 {
t.Fatal("expected at least 1 call")
}
// The request URL path should contain /private/
// We can't directly read the URL from body, but we can check the server
// received a body referencing private — instead check via a separate handler.
// We verify indirectly by checking the test server was hit (calls recorded).
}
func TestNamespace_PrivateURL(t *testing.T) {
var gotPath string
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
gotPath = r.URL.Path
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(200)
_, _ = w.Write([]byte(`{"code":200,"results":[]}`))
}))
defer srv.Close()
d := driverWithBase(srv.URL)
d.CloudType = "private"
ctx := context.Background()
_, _ = d.listFolders(ctx, "d0")
if !strings.Contains(gotPath, "/private/") {
t.Errorf("expected /private/ in path, got %q", gotPath)
}
}
// ---------------------------------------------------------------------------
// session missing → clear error
// ---------------------------------------------------------------------------
func TestSessionMissing(t *testing.T) {
d := &CtFile{}
d.Session = ""
d.BaseURL = "http://localhost:9"
ctx := context.Background()
err := d.profile(ctx)
if err == nil {
t.Fatal("expected error for missing session")
}
if !strings.Contains(err.Error(), "session") {
t.Errorf("error should mention session, got: %v", err)
}
}
// ---------------------------------------------------------------------------
// helper: background context for tests (unused now — kept as reference)
// ---------------------------------------------------------------------------
drivers/google_drive/util.go
+2 -2
View File
@@ -170,7 +170,7 @@ func (d *GoogleDrive) refreshToken() error {
}
log.Debug(res.String())
if e.Error != "" {
return fmt.Errorf(e.Error)
return fmt.Errorf("%s", e.Error)
}
d.AccessToken = resp.AccessToken
return nil
@@ -192,7 +192,7 @@ func (d *GoogleDrive) refreshToken() error {
}
log.Debug(res.String())
if e.Error != "" {
return fmt.Errorf(e.Error)
return fmt.Errorf("%s", e.Error)
}
d.AccessToken = resp.AccessToken
return nil
drivers/google_photo/util.go
+1 -1
View File
@@ -32,7 +32,7 @@ func (d *GooglePhoto) refreshToken() error {
return err
}
if e.Error != "" {
return fmt.Errorf(e.Error)
return fmt.Errorf("%s", e.Error)
}
d.AccessToken = resp.AccessToken
return nil
drivers/lanzou/util.go
+1 -1
View File
@@ -90,7 +90,7 @@ func (d *LanZou) _post(url string, callback base.ReqCallback, resp interface{},
if info == "" {
info = utils.Json.Get(data, "info").ToString()
}
return data, fmt.Errorf(info)
return data, fmt.Errorf("%s", info)
}
}
drivers/local/driver.go
+10 -5
View File
@@ -7,6 +7,7 @@ import (
"fmt"
"io/fs"
"net/http"
"net/url"
"os"
stdpath "path"
"path/filepath"
@@ -14,6 +15,7 @@ import (
"strings"
"time"
"github.com/OpenListTeam/OpenList/v4/internal/conf"
"github.com/OpenListTeam/OpenList/v4/internal/driver"
"github.com/OpenListTeam/OpenList/v4/internal/errs"
"github.com/OpenListTeam/OpenList/v4/internal/model"
@@ -151,12 +153,15 @@ func (d *Local) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([
func (d *Local) FileInfoToObj(ctx context.Context, f fs.FileInfo, reqPath string, fullPath string) model.Obj {
thumb := ""
if d.Thumbnail {
if d.supportsThumbnail(f.Name()) {
thumb = common.GetApiUrl(ctx) + stdpath.Join("/d", reqPath, f.Name())
thumb = utils.EncodePath(thumb, true)
thumb += "?type=thumb&sign=" + sign.Sign(stdpath.Join(reqPath, f.Name()))
if d.Thumbnail && d.supportsThumbnail(f.Name()) {
base, ok := ctx.Value(conf.ThumbnailBaseURLKey).(*url.URL)
if !ok {
base = &url.URL{
Path: stdpath.Join("/d", reqPath),
RawQuery: url.Values{"type": {"thumb"}, "sign": {sign.Sign(stdpath.Join(reqPath, f.Name()))}}.Encode(),
}
}
thumb = common.GetApiUrl(ctx) + base.JoinPath(url.PathEscape(f.Name())).String()
}
isFolder := f.IsDir() || isSymlinkDir(f, fullPath)
var size int64
drivers/misskey/util.go
+1
View File
@@ -245,6 +245,7 @@ func mFile2Object(file MFile) *model.ObjThumbURL {
Ctime: ctime,
IsFolder: false,
Size: file.Size,
HashInfo: utils.NewHashInfo(utils.MD5, file.MD5),
},
Thumbnail: model.Thumbnail{
Thumbnail: file.ThumbnailURL,
drivers/pikpak/seed_rapid.go
+58
View File
@@ -0,0 +1,58 @@
package pikpak
import (
"context"
"net/http"
"strings"
"github.com/OpenListTeam/OpenList/v4/drivers/base"
"github.com/OpenListTeam/OpenList/v4/internal/driver"
"github.com/OpenListTeam/OpenList/v4/internal/errs"
"github.com/OpenListTeam/OpenList/v4/internal/model"
"github.com/OpenListTeam/OpenList/v4/pkg/utils"
hash_extend "github.com/OpenListTeam/OpenList/v4/pkg/utils/hash"
"github.com/go-resty/resty/v2"
)
// RapidHashAlgos 返回 PikPak 支持的秒传哈希算法（GCID）
func (d *PikPak) RapidHashAlgos() []utils.HashType {
return []utils.HashType{*hash_extend.GCID}
}
// RapidHashNeedsPieces PikPak 不需要分片哈希
func (d *PikPak) RapidHashNeedsPieces() bool {
return false
}
// RapidUploadByHashes 使用种子中的 GCID 哈希尝试秒传
func (d *PikPak) RapidUploadByHashes(ctx context.Context, dstDir model.Obj, req *driver.SeedRapidUploadRequest, overwrite bool) (model.Obj, error) {
gcid := req.Whole.GetHash(hash_extend.GCID)
if len(gcid) < hash_extend.GCID.Width {
return nil, errs.ErrUnavailableHash
}
var resp UploadTaskData
_, err := d.request("https://api-drive.mypikpak.net/drive/v1/files", http.MethodPost, func(r *resty.Request) {
r.SetContext(ctx).SetBody(base.Json{
"kind": "drive#file",
"name": req.Name,
"size": req.Size,
"hash": strings.ToUpper(gcid),
"upload_type": "UPLOAD_TYPE_RESUMABLE",
"objProvider": base.Json{"provider": "UPLOAD_TYPE_UNKNOWN"},
"parent_id": dstDir.GetID(),
"folder_type": "NORMAL",
})
}, &resp)
if err != nil {
return nil, err
}
// 秒传成功时不会返回 Resumable
if resp.Resumable == nil {
file := fileToObj(resp.File)
return file, nil
}
return nil, errs.ErrHashMismatch
}
drivers/quark_open/seed_rapid.go
+60
View File
@@ -0,0 +1,60 @@
package quark_open
import (
"context"
"time"
"github.com/OpenListTeam/OpenList/v4/internal/driver"
"github.com/OpenListTeam/OpenList/v4/internal/errs"
"github.com/OpenListTeam/OpenList/v4/internal/model"
"github.com/OpenListTeam/OpenList/v4/pkg/utils"
)
// RapidHashAlgos 返回夸克网盘支持的秒传哈希算法（MD5 + SHA1）
func (d *QuarkOpen) RapidHashAlgos() []utils.HashType {
return []utils.HashType{*utils.MD5, *utils.SHA1}
}
// RapidHashNeedsPieces 夸克网盘不需要分片哈希
func (d *QuarkOpen) RapidHashNeedsPieces() bool {
return false
}
// RapidUploadByHashes 使用种子中的 MD5/SHA1 哈希尝试秒传。
//
// 夸克网盘的预上传需要 proof_code（按 proof range 读取的一段内容），
// 因此当内容源不可用时无法完成秒传。
func (d *QuarkOpen) RapidUploadByHashes(ctx context.Context, dstDir model.Obj, req *driver.SeedRapidUploadRequest, overwrite bool) (model.Obj, error) {
md5Hash := req.Whole.GetHash(utils.MD5)
sha1Hash := req.Whole.GetHash(utils.SHA1)
if len(md5Hash) < utils.MD5.Width || len(sha1Hash) < utils.SHA1.Width {
return nil, errs.ErrUnavailableHash
}
if req.Open == nil {
return nil, errs.ErrUnavailableHash
}
stream, err := req.Open()
if err != nil {
return nil, err
}
defer stream.Close()
pre, err := d.upPre(ctx, stream, dstDir.GetID(), md5Hash, sha1Hash)
if err != nil {
return nil, err
}
if !pre.Data.Finish {
return nil, errs.ErrHashMismatch
}
return &model.ObjThumb{
Object: model.Object{
ID: pre.Data.Fid,
Name: req.Name,
Size: req.Size,
IsFolder: false,
Modified: time.Now(),
},
}, nil
}
drivers/quark_open/types.go
+2
View File
@@ -2,6 +2,7 @@ package quark_open
import (
"github.com/OpenListTeam/OpenList/v4/internal/model"
"github.com/OpenListTeam/OpenList/v4/pkg/utils"
"time"
)
@@ -57,6 +58,7 @@ func fileToObj(f File) *model.ObjThumb {
Modified: time.UnixMilli(f.UpdatedAt),
IsFolder: f.FileType == "0",
Ctime: time.UnixMilli(f.CreatedAt),
HashInfo: utils.NewHashInfo(utils.SHA1, f.ContentHash),
},
Thumbnail: model.Thumbnail{Thumbnail: f.ThumbnailURL},
}
drivers/thunder/seed_rapid.go
+56
View File
@@ -0,0 +1,56 @@
package thunder
import (
"context"
"net/http"
"github.com/OpenListTeam/OpenList/v4/drivers/base"
"github.com/OpenListTeam/OpenList/v4/internal/driver"
"github.com/OpenListTeam/OpenList/v4/internal/errs"
"github.com/OpenListTeam/OpenList/v4/internal/model"
"github.com/OpenListTeam/OpenList/v4/pkg/utils"
hash_extend "github.com/OpenListTeam/OpenList/v4/pkg/utils/hash"
"github.com/go-resty/resty/v2"
)
// RapidHashAlgos 返回迅雷支持的秒传哈希算法（GCID）
func (xc *XunLeiCommon) RapidHashAlgos() []utils.HashType {
return []utils.HashType{*hash_extend.GCID}
}
// RapidHashNeedsPieces 迅雷不需要分片哈希
func (xc *XunLeiCommon) RapidHashNeedsPieces() bool {
return false
}
// RapidUploadByHashes 使用种子中的 GCID 哈希尝试秒传
func (xc *XunLeiCommon) RapidUploadByHashes(ctx context.Context, dstDir model.Obj, req *driver.SeedRapidUploadRequest, overwrite bool) (model.Obj, error) {
gcid := req.Whole.GetHash(hash_extend.GCID)
if len(gcid) < hash_extend.GCID.Width {
return nil, errs.ErrUnavailableHash
}
var resp UploadTaskResponse
_, err := xc.Request(FILE_API_URL, http.MethodPost, func(r *resty.Request) {
r.SetContext(ctx)
r.SetBody(&base.Json{
"kind": FILE,
"parent_id": dstDir.GetID(),
"name": req.Name,
"size": req.Size,
"hash": gcid,
"upload_type": UPLOAD_TYPE_RESUMABLE,
"space": xc.Space,
})
}, &resp)
if err != nil {
return nil, err
}
// 秒传成功（UploadType != UPLOAD_TYPE_RESUMABLE）
if resp.UploadType != UPLOAD_TYPE_RESUMABLE {
return &resp.File, nil
}
return nil, errs.ErrHashMismatch
}
drivers/thunder_browser/seed_rapid.go
+55
View File
@@ -0,0 +1,55 @@
package thunder_browser
import (
"context"
"net/http"
"github.com/OpenListTeam/OpenList/v4/drivers/base"
"github.com/OpenListTeam/OpenList/v4/internal/driver"
"github.com/OpenListTeam/OpenList/v4/internal/errs"
"github.com/OpenListTeam/OpenList/v4/internal/model"
"github.com/OpenListTeam/OpenList/v4/pkg/utils"
hash_extend "github.com/OpenListTeam/OpenList/v4/pkg/utils/hash"
"github.com/go-resty/resty/v2"
)
// RapidHashAlgos 返回迅雷浏览器支持的秒传哈希算法（GCID）
func (xc *XunLeiBrowserCommon) RapidHashAlgos() []utils.HashType {
return []utils.HashType{*hash_extend.GCID}
}
// RapidHashNeedsPieces 迅雷浏览器不需要分片哈希
func (xc *XunLeiBrowserCommon) RapidHashNeedsPieces() bool {
return false
}
// RapidUploadByHashes 使用种子中的 GCID 哈希尝试秒传
func (xc *XunLeiBrowserCommon) RapidUploadByHashes(ctx context.Context, dstDir model.Obj, req *driver.SeedRapidUploadRequest, overwrite bool) (model.Obj, error) {
gcid := req.Whole.GetHash(hash_extend.GCID)
if len(gcid) < hash_extend.GCID.Width {
return nil, errs.ErrUnavailableHash
}
var resp UploadTaskResponse
_, err := xc.Request(FILE_API_URL, http.MethodPost, func(r *resty.Request) {
r.SetContext(ctx)
r.SetBody(&base.Json{
"kind": FILE,
"parent_id": dstDir.GetID(),
"name": req.Name,
"size": req.Size,
"hash": gcid,
"upload_type": UPLOAD_TYPE_RESUMABLE,
})
}, &resp)
if err != nil {
return nil, err
}
// 秒传成功（UploadType != UPLOAD_TYPE_RESUMABLE）
if resp.UploadType != UPLOAD_TYPE_RESUMABLE {
return &resp.File, nil
}
return nil, errs.ErrHashMismatch
}
drivers/thunderx/seed_rapid.go
+55
View File
@@ -0,0 +1,55 @@
package thunderx
import (
"context"
"net/http"
"github.com/OpenListTeam/OpenList/v4/drivers/base"
"github.com/OpenListTeam/OpenList/v4/internal/driver"
"github.com/OpenListTeam/OpenList/v4/internal/errs"
"github.com/OpenListTeam/OpenList/v4/internal/model"
"github.com/OpenListTeam/OpenList/v4/pkg/utils"
hash_extend "github.com/OpenListTeam/OpenList/v4/pkg/utils/hash"
"github.com/go-resty/resty/v2"
)
// RapidHashAlgos 返回迅雷X支持的秒传哈希算法（GCID）
func (xc *XunLeiXCommon) RapidHashAlgos() []utils.HashType {
return []utils.HashType{*hash_extend.GCID}
}
// RapidHashNeedsPieces 迅雷X不需要分片哈希
func (xc *XunLeiXCommon) RapidHashNeedsPieces() bool {
return false
}
// RapidUploadByHashes 使用种子中的 GCID 哈希尝试秒传
func (xc *XunLeiXCommon) RapidUploadByHashes(ctx context.Context, dstDir model.Obj, req *driver.SeedRapidUploadRequest, overwrite bool) (model.Obj, error) {
gcid := req.Whole.GetHash(hash_extend.GCID)
if len(gcid) < hash_extend.GCID.Width {
return nil, errs.ErrUnavailableHash
}
var resp UploadTaskResponse
_, err := xc.Request(FILE_API_URL, http.MethodPost, func(r *resty.Request) {
r.SetContext(ctx)
r.SetBody(&base.Json{
"kind": FILE,
"parent_id": dstDir.GetID(),
"name": req.Name,
"size": req.Size,
"hash": gcid,
"upload_type": UPLOAD_TYPE_RESUMABLE,
})
}, &resp)
if err != nil {
return nil, err
}
// 秒传成功（UploadType != UPLOAD_TYPE_RESUMABLE）
if resp.UploadType != UPLOAD_TYPE_RESUMABLE {
return &resp.File, nil
}
return nil, errs.ErrHashMismatch
}
drivers/webdav/driver.go
+6
View File
@@ -6,6 +6,7 @@ import (
"net/http"
"os"
"path"
"strings"
"time"
"github.com/OpenListTeam/OpenList/v4/drivers/base"
@@ -130,6 +131,11 @@ func (d *WebDav) Put(ctx context.Context, dstDir model.Obj, s model.FileStreamer
func (d *WebDav) Get(ctx context.Context, _path string) (model.Obj, error) {
_path = path.Join(d.GetRootPath(), _path)
info, err := d.client.Stat(_path)
if err != nil && !gowebdav.IsErrNotFound(err) && !strings.HasSuffix(_path, "/") {
// As in RFC 4918 Section 5.2, retry failed slash-less collection PROPFINDs
// once with the canonical trailing-slash URL
info, err = d.client.Stat(_path + "/")
}
if err != nil {
if gowebdav.IsErrNotFound(err) {
return nil, errs.ObjectNotFound
drivers/webdav/driver_test.go
+49
View File
@@ -42,6 +42,55 @@ func TestMakeDirAfterMissingWebDAVStat(t *testing.T) {
}
}
// TestGetRetriesCollectionWithTrailingSlash covers servers that return
// 200 HTML for a slash-less collection PROPFIND. RFC 4918 Section 5.2
// recommends using the trailing-slash for all collection URLs.
func TestGetRetriesCollectionWithTrailingSlash(t *testing.T) {
collectionXML := func(href string) string {
return `<?xml version="1.0" encoding="utf-8"?>
<d:multistatus xmlns:d="DAV:">
<d:response>
<d:href>` + href + `</d:href>
<d:propstat>
<d:prop>
<d:displayname>share</d:displayname>
<d:resourcetype><d:collection/></d:resourcetype>
</d:prop>
<d:status>HTTP/1.1 200 OK</d:status>
</d:propstat>
</d:response>
</d:multistatus>`
}
d, cleanup := newTestDriver(t, func(w http.ResponseWriter, r *http.Request) bool {
if r.Method != "PROPFIND" {
return false
}
switch r.URL.Path {
case "/share":
// The quirky server: no 207 without the trailing slash.
w.Header().Set("Content-Type", "text/html; charset=utf-8")
w.WriteHeader(http.StatusOK)
_, _ = w.Write([]byte("<html>not a multistatus</html>"))
return true
case "/share/":
w.Header().Set("Content-Type", "application/xml; charset=utf-8")
w.WriteHeader(http.StatusMultiStatus)
_, _ = w.Write([]byte(collectionXML("/share/")))
return true
}
return false
})
defer cleanup()
obj, err := d.Get(context.Background(), "/share")
if err != nil {
t.Fatalf("Get on a collection whose slash-less PROPFIND 200s must succeed via the trailing-slash retry: %v", err)
}
if !obj.IsDir() {
t.Fatalf("expected a folder, got file %q", obj.GetName())
}
}
func newTestDriver(t *testing.T, extra func(http.ResponseWriter, *http.Request) bool) (*WebDav, func()) {
t.Helper()
drivers/wopan/driver.go
+58 -7
View File
@@ -2,6 +2,7 @@ package template
import (
"context"
"errors"
"fmt"
"strconv"
@@ -13,6 +14,11 @@ import (
"github.com/go-resty/resty/v2"
)
// minAccessTokenLen mirrors the check inside the SDK: the first 16 bytes of the
// access token are used as the AES key of every dispatcher request, so a token
// shorter than that can never be used and must be refreshed first.
const minAccessTokenLen = 16
type Wopan struct {
model.Storage
Addition
@@ -20,6 +26,18 @@ type Wopan struct {
defaultFamilyID string
}
// onTokenRefreshed keeps the driver storage in sync with the tokens issued by
// the SDK, so a refresh survives restarts.
func (d *Wopan) onTokenRefreshed(accessToken, refreshToken string) {
d.setTokens(accessToken, refreshToken)
op.MustSaveDriverStorage(d)
}
func (d *Wopan) setTokens(accessToken, refreshToken string) {
d.AccessToken = accessToken
d.RefreshToken = refreshToken
}
func (d *Wopan) Config() driver.Config {
return config
}
@@ -31,17 +49,50 @@ func (d *Wopan) GetAddition() driver.Additional {
func (d *Wopan) Init(ctx context.Context) error {
d.client = wopan.DefaultWithRefreshToken(d.RefreshToken)
d.client.SetAccessToken(d.AccessToken)
d.client.OnRefreshToken(func(accessToken, refreshToken string) {
d.AccessToken = accessToken
d.RefreshToken = refreshToken
op.MustSaveDriverStorage(d)
d.client.OnRefreshToken(d.onTokenRefreshed)
return d.initClient(ctx)
}
// initClient prepares the client before any dispatcher request is served.
//
// Two things have to happen here, in this order:
//
// 1. The browser User-Agent must be applied to the underlying resty client.
// The SDK only records it in WithUA/SetUA (and uses w.ua for upload2C), so
// dispatcher requests would otherwise be sent with resty's default
// "go-resty/..." UA, which the WoPan edge rejects with HTTP 403
// ("failed to list objs: request failed with status: 403 Forbidden").
// 2. A missing or too short access token must be refreshed before the first
// request. Every wohome/api-user request body is AES encrypted with
// accessToken[:16]; with an empty token the SDK fails with
// "crypto/aes: invalid key size 0", which used to break refresh-token-only
// configurations during FamilyUserCurrentEncode.
func (d *Wopan) initClient(ctx context.Context) error {
d.client.SetUserAgent(wopan.DefaultUA)
if len(d.AccessToken) < minAccessTokenLen {
if d.RefreshToken == "" {
return fmt.Errorf("an access token of at least %d characters or a refresh token is required", minAccessTokenLen)
}
if err := d.client.RefreshToken(); err != nil {
return fmt.Errorf("refresh access token: %w", err)
}
if accessToken, _ := d.client.GetToken(); len(accessToken) < minAccessTokenLen {
return errors.New("refresh access token: the server returned an invalid access token")
}
}
if err := d.client.InitData(); err != nil {
return fmt.Errorf("initialize user data: %w", err)
}
fml, err := d.client.FamilyUserCurrentEncode(func(req *resty.Request) {
req.SetContext(ctx)
})
fml, err := d.client.FamilyUserCurrentEncode()
if err != nil {
return err
return fmt.Errorf("query default family: %w", err)
}
d.defaultFamilyID = strconv.Itoa(fml.DefaultHomeId)
return d.client.InitData()
return nil
}
func (d *Wopan) Drop(ctx context.Context) error {
drivers/wopan/driver_test.go
+189
View File
@@ -0,0 +1,189 @@
package template
import (
"context"
"encoding/json"
"io"
"net/http"
"strings"
"testing"
"github.com/OpenListTeam/wopan-sdk-go"
)
type roundTripFunc func(*http.Request) (*http.Response, error)
func (f roundTripFunc) RoundTrip(r *http.Request) (*http.Response, error) {
return f(r)
}
// woPanMock mimics the WoPan dispatcher: it answers with the documented
// envelope and rejects requests whose User-Agent is not a browser UA, exactly
// like the edge did before the driver applied wopan.DefaultUA.
type woPanMock struct {
calls []string
refreshed bool
badUA int
refreshURL string
}
func (m *woPanMock) transport(t *testing.T) http.RoundTripper {
t.Helper()
return roundTripFunc(func(r *http.Request) (*http.Response, error) {
ua := r.Header.Get("User-Agent")
if ua != wopan.DefaultUA {
m.badUA++
return &http.Response{
StatusCode: http.StatusForbidden,
Status: "403 Forbidden",
Body: io.NopCloser(strings.NewReader("forbidden")),
Header: make(http.Header),
Request: r,
}, nil
}
raw, err := io.ReadAll(r.Body)
if err != nil {
t.Fatalf("read request body: %v", err)
}
var req struct {
Header struct {
Key string `json:"key"`
} `json:"header"`
}
if err := json.Unmarshal(raw, &req); err != nil {
t.Fatalf("decode request body: %v", err)
}
m.calls = append(m.calls, req.Header.Key)
data := "{}"
switch req.Header.Key {
case wopan.KeyAppRefreshToken:
m.refreshed = true
data = `{"access_token":"fedcba9876543210","refresh_token":"new-refresh"}`
case wopan.KeyAppQueryUser:
data = `{"userId":"test-user"}`
case wopan.KeyClassifyRule:
data = `{"fileTypes":{}}`
case wopan.KeyGetZoneInfo:
data = `{"url":"https://example.invalid"}`
case wopan.KeyFamilyUserCurrentEncode:
data = `{"defaultHomeId":42}`
}
body := `{"STATUS":"200","RSP":{"RSP_CODE":"0000","RSP_DESC":"","DATA":` + data + `}}`
return &http.Response{
StatusCode: http.StatusOK,
Status: "200 OK",
Body: io.NopCloser(strings.NewReader(body)),
Header: http.Header{"Content-Type": []string{"application/json"}},
Request: r,
}, nil
})
}
// newTestClient mirrors Wopan.Init, but injects a mock transport and a token
// callback that does not touch the database.
func newTestClient(d *Wopan, rt http.RoundTripper) {
d.client = wopan.DefaultWithRefreshToken(d.RefreshToken)
d.client.SetAccessToken(d.AccessToken)
d.client.SetHttpClient(&http.Client{Transport: rt})
d.client.OnRefreshToken(d.setTokens)
}
func TestInitClientCredentials(t *testing.T) {
const validAccessToken = "0123456789abcdef"
tests := []struct {
name string
accessToken string
refreshToken string
wantRefresh bool
wantErr bool
wantAccessToken string
wantFamilyID string
}{
{
name: "refresh only",
refreshToken: "refresh",
wantRefresh: true,
wantAccessToken: "fedcba9876543210",
wantFamilyID: "42",
},
{
name: "short access token with refresh",
accessToken: "short",
refreshToken: "refresh",
wantRefresh: true,
wantAccessToken: "fedcba9876543210",
wantFamilyID: "42",
},
{
name: "both tokens",
accessToken: validAccessToken,
refreshToken: "refresh",
wantAccessToken: validAccessToken,
wantFamilyID: "42",
},
{
name: "access token only",
accessToken: validAccessToken,
wantAccessToken: validAccessToken,
wantFamilyID: "42",
},
{
name: "no credentials",
wantErr: true,
},
{
name: "short access token without refresh",
accessToken: "short",
wantErr: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
mock := &woPanMock{}
d := &Wopan{Addition: Addition{
AccessToken: tt.accessToken,
RefreshToken: tt.refreshToken,
}}
newTestClient(d, mock.transport(t))
err := d.initClient(context.Background())
if tt.wantErr {
if err == nil {
t.Fatal("expected an error, got nil")
}
if len(mock.calls) != 0 {
t.Fatalf("expected no dispatcher call on configuration error, got %v", mock.calls)
}
return
}
if err != nil {
t.Fatalf("initClient returned error: %v", err)
}
if mock.badUA != 0 {
t.Fatalf("%d request(s) were sent without the browser user agent", mock.badUA)
}
if mock.refreshed != tt.wantRefresh {
t.Fatalf("refresh called = %v, want %v (calls: %v)", mock.refreshed, tt.wantRefresh, mock.calls)
}
if len(mock.calls) == 0 {
t.Fatal("expected dispatcher calls, got none")
}
if tt.wantRefresh && mock.calls[0] != wopan.KeyAppRefreshToken {
t.Fatalf("first call = %q, want %q", mock.calls[0], wopan.KeyAppRefreshToken)
}
if d.AccessToken != tt.wantAccessToken {
t.Fatalf("AccessToken = %q, want %q", d.AccessToken, tt.wantAccessToken)
}
if tt.wantRefresh && d.RefreshToken != "new-refresh" {
t.Fatalf("RefreshToken = %q, want new-refresh", d.RefreshToken)
}
if d.defaultFamilyID != tt.wantFamilyID {
t.Fatalf("defaultFamilyID = %q, want %q", d.defaultFamilyID, tt.wantFamilyID)
}
})
}
}
drivers/wopan/meta.go
+2 -2
View File
@@ -9,11 +9,11 @@ type Addition struct {
// Usually one of two
driver.RootID
// define other
RefreshToken string `json:"refresh_token" required:"true"`
RefreshToken string `json:"refresh_token" help:"Provide a refresh token or an access token (at least 16 characters)"`
FamilyID string `json:"family_id" help:"Keep it empty if you want to use your personal drive"`
SortRule string `json:"sort_rule" type:"select" options:"name_asc,name_desc,time_asc,time_desc,size_asc,size_desc" default:"name_asc"`
AccessToken string `json:"access_token"`
AccessToken string `json:"access_token" help:"Optional when a refresh token is provided; it is refreshed automatically when missing or too short"`
}
var config = driver.Config{
go.mod
+26 -23
View File
@@ -2,15 +2,15 @@ module github.com/OpenListTeam/OpenList/v4
go 1.26.0
toolchain go1.27.1
toolchain go1.27.3
require (
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.23.1
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.8.0
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.8.1
github.com/KarpelesLab/reflink v1.0.2
github.com/KirCute/zip v1.0.1
github.com/OpenListTeam/go-cache v0.1.0
github.com/OpenListTeam/gofakes3 v0.8.2-0.20260911142347-cd3c030a83b4
github.com/OpenListTeam/gofakes3 v0.8.2
github.com/OpenListTeam/sftpd-openlist v1.0.1
github.com/OpenListTeam/tache v0.2.2
github.com/OpenListTeam/times v0.1.0
@@ -64,7 +64,8 @@ require (
github.com/pkg/sftp v1.13.11
github.com/pquerna/otp v1.5.0
github.com/quic-go/quic-go v0.61.0
github.com/rclone/rclone v1.75.1
github.com/rclone/rclone v1.75.2
github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd
github.com/shirou/gopsutil/v4 v4.26.8
github.com/sirupsen/logrus v1.10.2
github.com/spf13/afero v1.15.0
@@ -75,15 +76,14 @@ require (
github.com/u2takey/ffmpeg-go v0.5.0
github.com/upyun/go-sdk/v3 v3.0.4
github.com/zzzhr1990/go-common-entity v0.0.0-20250202070650-1a200048f0d3
golang.org/x/crypto v0.56.0
golang.org/x/crypto v0.57.0
golang.org/x/image v0.45.0
golang.org/x/net v0.58.0
golang.org/x/net v0.60.0
golang.org/x/oauth2 v0.36.0
golang.org/x/time v0.15.0
google.golang.org/appengine v1.6.8
gopkg.in/ldap.v3 v3.1.0
gorm.io/driver/mysql v1.6.0
gorm.io/driver/postgres v1.6.2
gorm.io/driver/postgres v1.6.3
gorm.io/driver/sqlite v1.6.0
gorm.io/gorm v1.31.2
)
@@ -100,6 +100,7 @@ require (
github.com/RoaringBitmap/roaring/v2 v2.14.5 // indirect
github.com/adrg/xdg v0.5.3 // indirect
github.com/andybalholm/cascadia v1.3.4 // indirect
github.com/apache/arrow-go/v18 v18.7.0 // indirect
github.com/blevesearch/zapx/v17 v17.2.3 // indirect
github.com/bradenaw/juniper v0.15.3 // indirect
github.com/bytedance/gopkg v0.1.3 // indirect
@@ -120,6 +121,7 @@ require (
github.com/go-viper/mapstructure/v2 v2.5.0 // indirect
github.com/goccy/go-yaml v1.19.2 // indirect
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
github.com/google/flatbuffers v25.12.19+incompatible // indirect
github.com/hashicorp/go-uuid v1.0.3 // indirect
github.com/jcmturner/aescts/v2 v2.0.0 // indirect
github.com/jcmturner/dnsutils/v2 v2.0.0 // indirect
@@ -132,6 +134,7 @@ require (
github.com/minio/minlz v1.2.0 // indirect
github.com/minio/xxml v0.0.3 // indirect
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
github.com/ncruces/go-strftime v1.0.0 // indirect
github.com/philhofer/fwd v1.2.0 // indirect
github.com/quic-go/qpack v0.6.0 // indirect
github.com/relvacode/iso8601 v1.7.0 // indirect
@@ -145,12 +148,12 @@ require (
go.mongodb.org/mongo-driver/v2 v2.5.0 // indirect
go.yaml.in/yaml/v2 v2.4.4 // indirect
go.yaml.in/yaml/v3 v3.0.5 // indirect
golang.org/x/exp v0.0.0-20260709172345-9ea1abe57597 // indirect
golang.org/x/exp v0.0.0-20260813180055-c1d0aacb2297 // indirect
gopkg.in/go-jose/go-jose.v2 v2.6.3 // indirect
modernc.org/libc v1.22.5 // indirect
modernc.org/mathutil v1.5.0 // indirect
modernc.org/memory v1.5.0 // indirect
modernc.org/sqlite v1.23.1 // indirect
modernc.org/libc v1.73.4 // indirect
modernc.org/mathutil v1.7.1 // indirect
modernc.org/memory v1.11.0 // indirect
modernc.org/sqlite v1.53.0 // indirect
)
require (
@@ -251,7 +254,7 @@ require (
github.com/jmespath/go-jmespath v0.4.0 // indirect
github.com/josharian/intern v1.0.0 // indirect
github.com/jzelinskie/whirlpool v0.0.0-20201016144138-0675e54bb004 // indirect
github.com/klauspost/compress v1.19.0 // indirect
github.com/klauspost/compress v1.19.2 // indirect
github.com/klauspost/cpuid/v2 v2.4.0 // indirect
github.com/kr/fs v0.1.0 // indirect
github.com/leodido/go-urn v1.4.0 // indirect
@@ -285,7 +288,7 @@ require (
github.com/multiformats/go-multistream v0.4.1 // indirect
github.com/multiformats/go-varint v0.1.0 // indirect
github.com/pelletier/go-toml/v2 v2.2.4 // indirect
github.com/pierrec/lz4/v4 v4.1.27 // indirect
github.com/pierrec/lz4/v4 v4.1.28 // indirect
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect
github.com/pquerna/cachecontrol v0.1.0 // indirect
@@ -309,14 +312,14 @@ require (
github.com/yusufpapurcu/wmi v1.2.4 // indirect
go.etcd.io/bbolt v1.5.0 // indirect
golang.org/x/arch v0.23.0 // indirect
golang.org/x/sync v0.22.0
golang.org/x/sys v0.47.0
golang.org/x/term v0.45.0 // indirect
golang.org/x/text v0.41.0
golang.org/x/tools v0.48.0 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20260715232425-e75dac1f907d // indirect
google.golang.org/grpc v1.85.0-dev
google.golang.org/protobuf v1.36.11 // indirect
golang.org/x/sync v0.23.0
golang.org/x/sys v0.48.0
golang.org/x/term v0.46.0 // indirect
golang.org/x/text v0.42.0
golang.org/x/tools v0.49.0 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20260817212433-ac3dfec99bb1 // indirect
google.golang.org/grpc v1.85.0-dev.0.20260825072537-93e31b48545e
google.golang.org/protobuf v1.36.12 // indirect
gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
lukechampine.com/blake3 v1.1.7 // indirect
go.sum
+50 -2
View File
@@ -31,6 +31,8 @@ github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.4 h1:jWQK1GI+LeGGUKBAD
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.4/go.mod h1:8mwH4klAm9DUgR2EEHyEEAQlRDvLPyg5fQry3y+cDew=
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.8.0 h1:irsmOWwkp0KCTTNS5e2hdFeIvSQClQo2No3IaNmL3Vw=
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.8.0/go.mod h1:GWcBkQj3MqN7ozHKLaCCAuNLiXoIGv2RtanfAwSjY/Y=
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.8.1 h1:gkBLVmB3Z/HnGP/Jo4o12/RDpi0agnKav6sCKsX5Vu0=
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.8.1/go.mod h1:e3/1P5K+jIUi9JevDRklq/tFeTvbBb75bNAjU4xd31w=
github.com/AzureAD/microsoft-authentication-library-for-go v1.4.2 h1:oygO0locgZJe7PpYPXT5A29ZkwJaPqcva7BVeemZOZs=
github.com/AzureAD/microsoft-authentication-library-for-go v1.4.2/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI=
github.com/BurntSushi/toml v1.6.0 h1:dRaEfpa2VI55EwlIW72hMRHdWouJeRF7TPYhI+AUQjk=
@@ -53,6 +55,8 @@ github.com/OpenListTeam/go-cache v0.1.0 h1:eV2+FCP+rt+E4OCJqLUW7wGccWZNJMV0NNkh+
github.com/OpenListTeam/go-cache v0.1.0/go.mod h1:AHWjKhNK3LE4rorVdKyEALDHoeMnP8SjiNyfVlB+Pz4=
github.com/OpenListTeam/gofakes3 v0.8.2-0.20260911142347-cd3c030a83b4 h1:Zy7/qg6aCS0OF/FPIoJh9/d0IgcIxpWRvn79ACm2R/Y=
github.com/OpenListTeam/gofakes3 v0.8.2-0.20260911142347-cd3c030a83b4/go.mod h1:mS9Ywbo6aId6BrRzeYjOIOpK0QDVnMoKOIb0hpaQZ3U=
github.com/OpenListTeam/gofakes3 v0.8.2 h1:iR4B8WH0qWqWkzVTNSj2TgWw6kovTh2bV8TGMOFSnVI=
github.com/OpenListTeam/gofakes3 v0.8.2/go.mod h1:mS9Ywbo6aId6BrRzeYjOIOpK0QDVnMoKOIb0hpaQZ3U=
github.com/OpenListTeam/gsync v0.1.0 h1:ywzGybOvA3lW8K1BUjKZ2IUlT2FSlzPO4DOazfYXjcs=
github.com/OpenListTeam/gsync v0.1.0/go.mod h1:h/Rvv9aX/6CdW/7B8di3xK3xNV8dUg45Fehrd/ksZ9s=
github.com/OpenListTeam/reflink v0.0.0-20260701021214-78760eaeafef h1:67uGHancMF/abMrnkc8abVUWQiG73Wk5d8CKt3RzkFo=
@@ -118,6 +122,8 @@ github.com/antchfx/xpath v1.3.6 h1:s0y+ElRRtTQdfHP609qFu0+c6bglDv20pqOViQjjdPI=
github.com/antchfx/xpath v1.3.6/go.mod h1:i54GszH55fYfBmoZXapTHN8T8tkcHfRgLyVwwqzXNcs=
github.com/antchfx/xpath v1.3.8 h1:RQlkLaJDKk1Ew1H6CUPUTKM+IQxm+6HTyOgcrfqOU9c=
github.com/antchfx/xpath v1.3.8/go.mod h1:i54GszH55fYfBmoZXapTHN8T8tkcHfRgLyVwwqzXNcs=
github.com/apache/arrow-go/v18 v18.7.0 h1:Vw/i+cJyebUofT7JlqFpe65LrmwxULn166jjwStM4HY=
github.com/apache/arrow-go/v18 v18.7.0/go.mod h1:PM6IigLJkdMwIpeHXnymo+xZ52f42a9EYiLtRel4p/A=
github.com/avast/retry-go v3.0.0+incompatible h1:4SOWQ7Qs+oroOTQOYnAHqelpCO0biHSxpiH9JdtuBj0=
github.com/avast/retry-go v3.0.0+incompatible/go.mod h1:XtSnn+n/sHqQIpZ10K1qAevBhOOCWBLXXy3hyiqqBrY=
github.com/aws/aws-sdk-go v1.38.20/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro=
@@ -462,6 +468,8 @@ github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM=
github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
github.com/golang/snappy v1.0.0 h1:Oy607GVXHs7RtbggtPBnr2RmDArIsAefDwvrdWvRhGs=
github.com/golang/snappy v1.0.0/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
github.com/google/flatbuffers v25.12.19+incompatible h1:haMV2JRRJCe1998HeW/p0X9UaMTK6SDo0ffLn2+DbLs=
github.com/google/flatbuffers v25.12.19+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8=
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
@@ -575,6 +583,8 @@ github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+o
github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
github.com/klauspost/compress v1.19.0 h1:sXLILfc9jV2QYWkzFOPWStmcUVH2RHEB1JCdY2oVvCQ=
github.com/klauspost/compress v1.19.0/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=
github.com/klauspost/compress v1.19.2 h1:hMRETovs/pu/dVWN7zIT1PGG8t509MwT6bO7XSi26R8=
github.com/klauspost/compress v1.19.2/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=
github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y=
@@ -704,6 +714,8 @@ github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
github.com/natefinch/lumberjack v2.0.0+incompatible h1:4QJd3OLAMgj7ph+yZTuX13Ld4UpgHp07nNdFX7mqFfM=
github.com/natefinch/lumberjack v2.0.0+incompatible/go.mod h1:Wi9p2TTF5DG5oU+6YfsmYQpsTIOm0B1VNzQg9Mw6nPk=
github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w=
github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls=
github.com/ncw/swift/v2 v2.0.5 h1:9o5Gsd7bInAFEqsGPcaUdsboMbqf8lnNtxqWKFT9iz8=
github.com/ncw/swift/v2 v2.0.5/go.mod h1:cbAO76/ZwcFrFlHdXPjaqWZ9R7Hdar7HpjRXBfbjigk=
github.com/nwaples/rardecode/v2 v2.2.5 h1:L5doqgGfQwI7qADJMqnkrSB86rpPsqQDrHeO0HWa5JY=
@@ -719,6 +731,8 @@ github.com/philhofer/fwd v1.2.0 h1:e6DnBTl7vGY+Gz322/ASL4Gyp1FspeMvx1RNDoToZuM=
github.com/philhofer/fwd v1.2.0/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM=
github.com/pierrec/lz4/v4 v4.1.27 h1:+PhzhWDrjRj89TH2sw43nE3+4+W8lSxIuQadEHZyjUk=
github.com/pierrec/lz4/v4 v4.1.27/go.mod h1:EoQMVJgeeEOMsCqCzqFm2O0cJvljX2nGZjcRIPL34O4=
github.com/pierrec/lz4/v4 v4.1.28 h1:pPEPwRJ4kybBTfGt28q7lQsRJQHhC08axprdLD5Ppio=
github.com/pierrec/lz4/v4 v4.1.28/go.mod h1:EoQMVJgeeEOMsCqCzqFm2O0cJvljX2nGZjcRIPL34O4=
github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ=
github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU=
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
@@ -773,6 +787,8 @@ github.com/rclone/rclone v1.75.0 h1:3ARHem4jXWltvl+b0PvDAG8s6J/inHd5BRzfwMRb3W8=
github.com/rclone/rclone v1.75.0/go.mod h1:PGLJUW/WSIJCysALqUcxmaCFyfMXUevf8CbuoOwsAdU=
github.com/rclone/rclone v1.75.1 h1:kIxQcoDLj2Gke/gMSHK7OnxhX1Gu1cJBLP1kJZoaFp0=
github.com/rclone/rclone v1.75.1/go.mod h1:4zmMjGatCkSJPRZDpo+7y3xOl8S29EMUyKvZop5mHr4=
github.com/rclone/rclone v1.75.2 h1:0cqa6P3WmhGnu2NeOdZSetJnacpwCRtpsNbHOICqPk8=
github.com/rclone/rclone v1.75.2/go.mod h1:YmKNkhnFIoIwLsEZl7FSN756Iq5ZXy3LPDuypnuFocA=
github.com/relvacode/iso8601 v1.6.0 h1:eFXUhMJN3Gz8Rcq82f9DTMW0svjtAVuIEULglM7QHTU=
github.com/relvacode/iso8601 v1.6.0/go.mod h1:FlNp+jz+TXpyRqgmM7tnzHHzBnz776kmAH2h3sZCn0I=
github.com/relvacode/iso8601 v1.7.0 h1:BXy+V60stMP6cpswc+a93Mq3e65PfXCgDFfhvNNGrdo=
@@ -789,6 +805,8 @@ github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUc
github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ=
github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc=
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd h1:CmH9+J6ZSsIjUK3dcGsnCnO41eRBOnY12zwkn5qVwgc=
github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd/go.mod h1:hPqNNc0+uJM6H+SuU8sEs5K5IQeKccPqeSjfgcKGgPk=
github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46 h1:GHRpF1pTW19a8tTFrMLUcfWwyC0pnifVo2ClaLq+hP8=
github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46/go.mod h1:uAQ5PCi+MFsC7HjREoAz1BU+Mq60+05gifQSsHSDG/8=
github.com/secsy/goftp v0.0.0-20200609142545-aa2de14babf4 h1:PT+ElG/UUFMfqy5HrxJxNzj3QBOf7dZwupeVC+mG1Lo=
@@ -950,12 +968,16 @@ golang.org/x/crypto v0.55.0 h1:+KWHjbgOaAQ66dh/YlkZKHlz9ZUlq61AFirAR9ntP8M=
golang.org/x/crypto v0.55.0/go.mod h1:uq0V9dE/fzQuJtbnL+2EhWOE63vo164FY8xqEnV9xis=
golang.org/x/crypto v0.56.0 h1:GUh5Ii4J5jtcseSMiRqr1jXCNHoxjeV9Fmekc2oLy6Y=
golang.org/x/crypto v0.56.0/go.mod h1:OMW5y6CY9l38uPLmxU6l6pwcXp1obtLo3e6gT7gQR2I=
golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M=
golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA=
golang.org/x/exp v0.0.0-20250606033433-dcc06ee1d476 h1:bsqhLWFR6G6xiQcb+JoGqdKdRU6WzPWmK8E0jxTjzo4=
golang.org/x/exp v0.0.0-20250606033433-dcc06ee1d476/go.mod h1:3//PLf8L/X+8b4vuAfHzxeRUl04Adcb341+IGKfnqS8=
golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f h1:W3F4c+6OLc6H2lb//N1q4WpJkhzJCK5J6kUi1NTVXfM=
golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f/go.mod h1:J1xhfL/vlindoeF/aINzNzt2Bket5bjo9sdOYzOsU80=
golang.org/x/exp v0.0.0-20260709172345-9ea1abe57597 h1:qLvzZeaANDgyVOA8pyHCOStGlXn0rseXma+GQjeuv2g=
golang.org/x/exp v0.0.0-20260709172345-9ea1abe57597/go.mod h1:EdfpwwqSu+0Li0mzskwHU6FWDV3t9Q+RZDo3QMUtL3Q=
golang.org/x/exp v0.0.0-20260813180055-c1d0aacb2297 h1:YXnL44eJ77R+ji4/ooy8UsXIhz+lbi2Qgdlc8iRN0gY=
golang.org/x/exp v0.0.0-20260813180055-c1d0aacb2297/go.mod h1:Mkmymgv+uMpSQ/XxJ/7GpdrdYoqm3u72jEbpCLiJmNk=
golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
golang.org/x/image v0.29.0 h1:HcdsyR4Gsuys/Axh0rDEmlBmB68rW1U9BUdB3UVHsas=
golang.org/x/image v0.29.0/go.mod h1:RVJROnf3SLK8d26OW91j4FrIHGbsJ8QnbEocVTOWQDA=
@@ -990,6 +1012,8 @@ golang.org/x/net v0.57.0 h1:K5+3DljvIuDG9/Jv9rvyMywYNFCQ9RSUY6OOTTkT+tE=
golang.org/x/net v0.57.0/go.mod h1:KpXc8iv+r3XplLAG/f7Jsf9RPszJzdR0f58q9vGOuEU=
golang.org/x/net v0.58.0 h1:ynWG7rqYi4ccpTEuPZ2QGWHktVEM9DMCj9yzDE0Q7To=
golang.org/x/net v0.58.0/go.mod h1:YwCddHnFlT7eLQqVprV19OnhLGtc5xOKgE0RyqgfWAU=
golang.org/x/net v0.60.0 h1:79p50tfZlm0J9YfoDsSi639qSXNGVwEzOPLCxM2FsYU=
golang.org/x/net v0.60.0/go.mod h1:2DA/G1UfVbCpQPeWTmMPGY7Cs2PkBkwu743bVX5PIVg=
golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs=
golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -1001,6 +1025,8 @@ golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek=
golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
golang.org/x/sync v0.23.0 h1:KameEIfc1IkluZyXWLn39Wd4tURc6GbCiISGiZm2bQk=
golang.org/x/sync v0.23.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -1026,6 +1052,8 @@ golang.org/x/sys v0.46.0 h1:noSf2Fq6F8DBgS+LysIkx7rIExoNHJsxOAtPp4rthXw=
golang.org/x/sys v0.46.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs=
golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo=
golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
@@ -1040,6 +1068,8 @@ golang.org/x/term v0.44.0 h1:0rLvDRCtNj0gZkyIXhCyOb2OAzEhLVqc4B+hrsBhrmc=
golang.org/x/term v0.44.0/go.mod h1:7ze4MdzUzLXpSAoFP1H0bOI9aXDqveSvatT5vKcFh2Y=
golang.org/x/term v0.45.0 h1:NwWyBmoJCbfTHpxrWoZ9C6/VxOf7ic219I8xZZFdrf0=
golang.org/x/term v0.45.0/go.mod h1:9aqxs0blBcrm/n0L9QW0aRVD+ktan8ssZromtqJC43w=
golang.org/x/term v0.46.0 h1:3+OXuTbaKDgwk8jTi3aSLHRlmWqHEUDUtxnbFigO4YE=
golang.org/x/term v0.46.0/go.mod h1:+K02xbkittuwc0Am4abfA3Fc+XRGXkvBXNO88NCXPoc=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
@@ -1056,6 +1086,8 @@ golang.org/x/text v0.40.0 h1:Ub2Z6/xjgF1WrYQz2nuITOEegKFtiIy+rieRJ5lHZKs=
golang.org/x/text v0.40.0/go.mod h1:hpnzDAfGV753zIKo+wk3u1bVKCGPbrnF7+7LBF/UHVY=
golang.org/x/text v0.41.0 h1:vz/seA0lnX87Othu2f/0L24RcgrXD9/YFTSuGjj3rH8=
golang.org/x/text v0.41.0/go.mod h1:jvf1O8ajNzZqhSrQBPbutR/EB83Cc0CFrezNQIwbb5M=
golang.org/x/text v0.42.0 h1:JbOZXgfeCPU9gacVtYliJqOhD+zhrEqK4LfdpmlUZqI=
golang.org/x/text v0.42.0/go.mod h1:ojzP1Z+2QtioaF8DTtO8K5q7JWVVYwZKenzujK0Zd0E=
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI=
golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4=
@@ -1073,28 +1105,34 @@ golang.org/x/tools v0.47.0 h1:7Kn5x/d1svx/PzryTsqeoZN4TZwqeH5pGWjefhLi/1Q=
golang.org/x/tools v0.47.0/go.mod h1:dFHnyTvFWY212G+h7ZY4Vsp/K3U4/7W9TyVaAul8uCA=
golang.org/x/tools v0.48.0 h1:3+hClM1aLL5mjMKm5ovokw9epgRXPuu2tILgismM6RE=
golang.org/x/tools v0.48.0/go.mod h1:08xX0orndb/F7jJxGDicx061tyd5pcMto75YMAXr6lk=
golang.org/x/tools v0.49.0 h1:3NI7VXzL9+1WZD52Dx2ttoPwD5DWrFGpl9mFZDlmisI=
golang.org/x/tools v0.49.0/go.mod h1:SJNXV9DBKT0UbdttsQjbfJlAE/q+y36++zo3uL3N0Oo=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4=
gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E=
google.golang.org/api v0.259.0 h1:90TaGVIxScrh1Vn/XI2426kRpBqHwWIzVBzJsVZ5XrQ=
google.golang.org/api v0.259.0/go.mod h1:LC2ISWGWbRoyQVpxGntWwLWN/vLNxxKBK9KuJRI8Te4=
google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM=
google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds=
google.golang.org/genproto/googleapis/rpc v0.0.0-20260414002931-afd174a4e478 h1:RmoJA1ujG+/lRGNfUnOMfhCy5EipVMyvUE+KNbPbTlw=
google.golang.org/genproto/googleapis/rpc v0.0.0-20260414002931-afd174a4e478/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8=
google.golang.org/genproto/googleapis/rpc v0.0.0-20260715232425-e75dac1f907d h1:Jkpk39hlTZOIp3RbfvNX9R8Hv+Sw0X89nlU/xFOErsc=
google.golang.org/genproto/googleapis/rpc v0.0.0-20260715232425-e75dac1f907d/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8=
google.golang.org/genproto/googleapis/rpc v0.0.0-20260817212433-ac3dfec99bb1 h1:utmqiUzZAEINbLmRyAt1QjVkMyLQ+lvIwCpLaB1rZfw=
google.golang.org/genproto/googleapis/rpc v0.0.0-20260817212433-ac3dfec99bb1/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8=
google.golang.org/grpc v1.82.0 h1:vguDnZUPjE26w09A63VoxZPnvPjB5Riyc0mkXPFmAIU=
google.golang.org/grpc v1.82.0/go.mod h1:yzTZ1TB1Z3SG+LIYaI+WiE8D5+PZ3ArnrSp8zF3+/ZA=
google.golang.org/grpc v1.84.0-dev.0.20260723093437-b6eac429d7b6 h1:HfjjkdGIa8u9sP9EW5WCygy0kQDuTI/Tax4j//t24Fo=
google.golang.org/grpc v1.84.0-dev.0.20260723093437-b6eac429d7b6/go.mod h1:ljCht0DrxQrXBDRTZp52Qxh3Ffk8CdYm2sj4O2QN2C0=
google.golang.org/grpc v1.85.0-dev h1:HxkDyKIIZPpFnroC56tQv5gNuKTmVvi0t7TzOf5zt7g=
google.golang.org/grpc v1.85.0-dev/go.mod h1:ljCht0DrxQrXBDRTZp52Qxh3Ffk8CdYm2sj4O2QN2C0=
google.golang.org/grpc v1.85.0-dev.0.20260825072537-93e31b48545e h1:ar+feAPij1Znzug45m6AoShUCc5AaR8FLA3I6LZd94k=
google.golang.org/grpc v1.85.0-dev.0.20260825072537-93e31b48545e/go.mod h1:7aY/KlAHanMAVIWHnk8wloQsdlfwzDz/PP9vCbJG6CU=
google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE=
google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
google.golang.org/protobuf v1.36.12 h1:pJOKDDOyeXErUroCihFAd5LQuwXBSpVnKGrj5o/fwxc=
google.golang.org/protobuf v1.36.12/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d h1:TxyelI5cVkbREznMhfzycHdkp5cLA7DpE+GKjSslYhM=
gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d/go.mod h1:cuepJuh7vyXfUyUwEgHQXw849cJrilpS5NeIjOWESAw=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
@@ -1124,6 +1162,8 @@ gorm.io/driver/postgres v1.6.0 h1:2dxzU8xJ+ivvqTRph34QX+WrRaJlmfyPqXmoGVjMBa4=
gorm.io/driver/postgres v1.6.0/go.mod h1:vUw0mrGgrTK+uPHEhAdV4sfFELrByKVGnaVRkXDhtWo=
gorm.io/driver/postgres v1.6.2 h1:BvXQ/cNUg63q5TFNg672DmDcowZSFrNLkkA3Xe6GXq4=
gorm.io/driver/postgres v1.6.2/go.mod h1:0c4fQA44XhOklXDkgtuKqysHCycTa5i9e3EIpDGCwXk=
gorm.io/driver/postgres v1.6.3 h1:bAn6O2pUa8LtpWEvL5NFU4+52Tfx8Ut7IVaIacCLcI0=
gorm.io/driver/postgres v1.6.3/go.mod h1:0c4fQA44XhOklXDkgtuKqysHCycTa5i9e3EIpDGCwXk=
gorm.io/driver/sqlite v1.6.0 h1:WHRRrIiulaPiPFmDcod6prc4l2VGVWHz80KspNsxSfQ=
gorm.io/driver/sqlite v1.6.0/go.mod h1:AO9V1qIQddBESngQUKWL9yoH93HIeA1X6V633rBwyT8=
gorm.io/gorm v1.25.7/go.mod h1:hbnx/Oo0ChWMn1BIhpy1oYozzpM15i4YPuHDmfYtwg8=
@@ -1133,12 +1173,20 @@ lukechampine.com/blake3 v1.1.7 h1:GgRMhmdsuK8+ii6UZFDL8Nb+VyMwadAgcJyfYHxG6n0=
lukechampine.com/blake3 v1.1.7/go.mod h1:tkKEOtDkNtklkXtLNEOGNq5tcV90tJiA1vAA12R78LA=
modernc.org/libc v1.22.5 h1:91BNch/e5B0uPbJFgqbxXuOnxBQjlS//icfQEGmvyjE=
modernc.org/libc v1.22.5/go.mod h1:jj+Z7dTNX8fBScMVNRAYZ/jF91K8fdT2hYMThc3YjBY=
modernc.org/libc v1.73.4 h1:+ra4Ui8ngyt8HDcO1FTDPWlkAh6yOdaO2yAoh8MddQA=
modernc.org/libc v1.73.4/go.mod h1:DXZ3eO8qMCNn2SnmTNCiC71nJ9Rcq3PsnpU6Vc4rWK8=
modernc.org/mathutil v1.5.0 h1:rV0Ko/6SfM+8G+yKiyI830l3Wuz1zRutdslNoQ0kfiQ=
modernc.org/mathutil v1.5.0/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E=
modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU=
modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg=
modernc.org/memory v1.5.0 h1:N+/8c5rE6EqugZwHii4IFsaJ7MUhoWX07J5tC/iI5Ds=
modernc.org/memory v1.5.0/go.mod h1:PkUhL0Mugw21sHPeskwZW4D6VscE/GQJOnIpCnW6pSU=
modernc.org/memory v1.11.0 h1:o4QC8aMQzmcwCK3t3Ux/ZHmwFPzE6hf2Y5LbkRs+hbI=
modernc.org/memory v1.11.0/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw=
modernc.org/sqlite v1.23.1 h1:nrSBg4aRQQwq59JpvGEQ15tNxoO5pX/kUjcRNwSAGQM=
modernc.org/sqlite v1.23.1/go.mod h1:OrDj17Mggn6MhE+iPbBNf7RGKODDE9NFT0f3EwDzJqk=
modernc.org/sqlite v1.53.0 h1:20WG8N9q4ji/dEqGk4uiI0c6OPjSeLTNYGFCc3+7c1M=
modernc.org/sqlite v1.53.0/go.mod h1:xoEpOIpGrgT48H5iiyt/YXPCZPEzlfmfFwtk8Lklw8s=
resty.dev/v3 v3.0.0-beta.2 h1:xu4mGAdbCLuc3kbk7eddWfWm4JfhwDtdapwss5nCjnQ=
resty.dev/v3 v3.0.0-beta.2/go.mod h1:OgkqiPvTDtOuV4MGZuUDhwOpkY8enjOsjjMzeOHefy4=
sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc=
internal/bootstrap/config.go
+1
View File
@@ -40,6 +40,7 @@ func InitConfig() {
if !filepath.IsAbs(dataDir) {
flags.DataDir = filepath.Join(pwd, flags.DataDir)
}
conf.DataDir = flags.DataDir
// Determine config file path: use flags.ConfigPath if provided, otherwise default to <dataDir>/config.json
configPath := flags.ConfigPath
if configPath == "" {
internal/bootstrap/data/setting.go
+116
View File
@@ -187,6 +187,14 @@ func InitialSettings() []model.SettingItem {
{Key: conf.HandleHookAfterWriting, Value: "false", Type: conf.TypeBool, Group: model.GLOBAL, Flag: model.PRIVATE},
{Key: conf.HandleHookRateLimit, Value: "0", Type: conf.TypeNumber, Group: model.GLOBAL, Flag: model.PRIVATE},
{Key: conf.IgnoreSystemFiles, Value: "false", Type: conf.TypeBool, Group: model.GLOBAL, Flag: model.PRIVATE, Help: `When enabled, ignores common system files during upload (.DS_Store, desktop.ini, Thumbs.db, and files starting with ._)`},
{Key: conf.SeedSiteURL, Value: "", Type: conf.TypeString, Group: model.GLOBAL, Flag: model.PRIVATE, Help: `Public base URL embedded in generated transfer seed sources when configured`},
{Key: conf.SeedDefaultMatrix, Value: `{"md5":{"whole":true,"pieces":false},"sha1":{"whole":true,"pieces":false},"sha256":{"whole":true,"pieces":false}}`, Type: conf.TypeText, Group: model.GLOBAL, Flag: model.PRIVATE, Help: `Default right-click hash matrix for transfer seed generation`},
{Key: conf.SeedFormatPolicies, Value: `{"oss":"off","torrent":"off","cas":"off"}`, Type: conf.TypeText, Group: model.GLOBAL, Flag: model.PRIVATE},
{Key: conf.SeedDefaultFormat, Value: "oss", Type: conf.TypeSelect, Options: "oss,torrent,cas", Group: model.GLOBAL, Flag: model.PRIVATE},
{Key: conf.SeedSingleDirectPreview, Value: "false", Type: conf.TypeBool, Group: model.GLOBAL, Flag: model.PUBLIC},
{Key: conf.SeedCASDirectAccess, Value: "false", Type: conf.TypeBool, Group: model.GLOBAL, Flag: model.PUBLIC, Help: `When opening a single-file CAS seed, immediately rapid-upload it into the same folder and preview the restored file`},
{Key: conf.SeedAutoGeneratePolicy, Value: "off", Type: conf.TypeSelect, Options: "off,on", Group: model.GLOBAL, Flag: model.PRIVATE, Help: `Global upload sidecar policy; storage-specific inheritance can be layered without changing the safe default`},
{Key: conf.SeedDefaultTrackers, Value: "", Type: conf.TypeText, Group: model.GLOBAL, Flag: model.PRIVATE, Help: `Default tracker list offered when generating torrent seeds (one tracker per line)`},
// single settings
{Key: conf.Token, Value: token, Type: conf.TypeString, Group: model.SINGLE, Flag: model.PRIVATE},
@@ -251,6 +259,114 @@ func InitialSettings() []model.SettingItem {
{Key: conf.StreamMaxClientUploadSpeed, Value: "-1", Type: conf.TypeNumber, Group: model.TRAFFIC, Flag: model.PRIVATE},
{Key: conf.StreamMaxServerDownloadSpeed, Value: "-1", Type: conf.TypeNumber, Group: model.TRAFFIC, Flag: model.PRIVATE},
{Key: conf.StreamMaxServerUploadSpeed, Value: "-1", Type: conf.TypeNumber, Group: model.TRAFFIC, Flag: model.PRIVATE},
// media settings
{Key: conf.MediaTMDBKey, Value: "", Type: conf.TypeString, Group: model.MEDIA, Flag: model.PRIVATE},
{Key: conf.MediaTMDBAPIURL, Value: "api.themoviedb.org", Type: conf.TypeString, Group: model.MEDIA, Flag: model.PRIVATE},
{Key: conf.MediaDiscogsToken, Value: "", Type: conf.TypeString, Group: model.MEDIA, Flag: model.PRIVATE},
{Key: conf.MediaDiscogsAPIURL, Value: "api.discogs.com", Type: conf.TypeString, Group: model.MEDIA, Flag: model.PRIVATE},
{Key: conf.MediaStoreThumbnail, Value: "false", Type: conf.TypeBool, Group: model.MEDIA, Flag: model.PRIVATE},
{Key: conf.MediaThumbnailMode, Value: "runtime", Type: conf.TypeSelect, Options: "base64,runtime,local", Group: model.MEDIA, Flag: model.PRIVATE,
Help: `缩略图存储方式：base64 存入数据库；runtime 存入程序数据目录（data/imgs）；local 写入挂载目录下的 VFS 路径`},
{Key: conf.MediaThumbnailPath, Value: "/imgs", Type: conf.TypeString, Group: model.MEDIA, Flag: model.PRIVATE},
{Key: conf.MediaScrapeConcurrency, Value: "5", Type: conf.TypeNumber, Group: model.MEDIA, Flag: model.PRIVATE},
{Key: conf.MediaScrapeLanguage, Value: "zh-CN", Type: conf.TypeSelect, Options: "zh-CN,zh-TW,en-US,ja-JP,ko-KR,fr-FR,de-DE,es-ES,ru-RU,pt-BR", Group: model.MEDIA, Flag: model.PRIVATE,
Help: `首选刮削语言：影响 TMDB 的标题/简介/海报文字与 Google Books 的检索语言；豆瓣固定为中文`},
{Key: conf.MediaBookSource, Value: "douban", Type: conf.TypeSelect, Options: "douban,google,openlibrary", Group: model.MEDIA, Flag: model.PRIVATE,
Help: `书籍刮削数据源：douban（豆瓣）、google（Google Books）、openlibrary（OpenLibrary）`},
{Key: conf.MediaGoogleBooksKey, Value: "", Type: conf.TypeString, Group: model.MEDIA, Flag: model.PRIVATE,
Help: `Google Books API Key，可留空（无 Key 时配额较低）；申请地址 https://developers.google.com/books/docs/v1/using#APIKey`},
{Key: conf.MediaDownloadCover, Value: "false", Type: conf.TypeBool, Group: model.MEDIA, Flag: model.PRIVATE,
Help: `开启后把 TMDB / Google Books / OpenLibrary 等返回的封面链接下载到下方缩略图存储路径，而不是直接保存图片链接`},
{Key: conf.MediaEnabled, Value: "false", Type: conf.TypeBool, Group: model.MEDIA, Flag: model.PUBLIC,
Help: `媒体库总开关。关闭时媒体库相关页面、API 与协议端点全部不可用（返回 404），与未安装媒体库时表现一致`},
{Key: conf.MediaEmbyEnabled, Value: "false", Type: conf.TypeBool, Group: model.MEDIA, Flag: model.PRIVATE,
Help: `开启后提供 Emby 兼容 API（/@emby），可供 Infuse、VidHub、MoviePilot 等客户端直接接入`},
{Key: conf.MediaSubsonicEnabled, Value: "false", Type: conf.TypeBool, Group: model.MEDIA, Flag: model.PRIVATE,
Help: `开启后提供 OpenSubsonic API（/@rest），可供 Symfonium、Feishin、Amperfy 等音乐客户端接入`},
{Key: conf.MediaSubsonicAllowPwd, Value: "false", Type: conf.TypeBool, Group: model.MEDIA, Flag: model.PRIVATE,
Help: `允许 Subsonic 客户端用 OpenList 登录密码（u+p 参数）认证。该方式会把主账号密码暴露给第三方客户端，默认关闭，推荐使用应用密钥（apiKey 或 t+s）`},
{Key: conf.MediaSubsonicProxyStream, Value: "false", Type: conf.TypeBool, Group: model.MEDIA, Flag: model.PRIVATE,
Help: `Subsonic 播放默认返回 302 直链；若客户端不跟随重定向，可开启此项改为服务端代理转发（占用服务器带宽）`},
{Key: conf.MediaProxyCover, Value: "false", Type: conf.TypeBool, Group: model.MEDIA, Flag: model.PRIVATE,
Help: `开启后由服务端代理 TMDB/豆瓣等外部封面图片，适合直连图片域名受限的网络；默认关闭，直接使用原始图片地址`},
{Key: conf.MediaCoverGateway, Value: "false", Type: conf.TypeBool, Group: model.MEDIA, Flag: model.PRIVATE,
Help: `封面代理直连失败时，回退到第三方图片网关 wsrv.nl 重试。开启后封面地址会发送给该第三方服务；默认关闭`},
{Key: conf.MediaStrmOnScan, Value: "false", Type: conf.TypeBool, Group: model.MEDIA, Flag: model.PRIVATE,
Help: `扫描完成后自动把媒体条目生成为 .strm 文件并写入下方目录`},
{Key: conf.MediaStrmOverwrite, Value: "true", Type: conf.TypeBool, Group: model.MEDIA, Flag: model.PRIVATE,
Help: `生成 STRM 时覆盖已存在的同名文件`},
{Key: conf.MediaStrmSameDir, Value: "true", Type: conf.TypeBool, Group: model.MEDIA, Flag: model.PRIVATE,
Help: `开启时 .strm 与媒体文件放在同一目录且同名（仅扩展名不同）；关闭时统一输出到下方指定目录`},
{Key: conf.MediaStrmPath, Value: "/strm", Type: conf.TypeString, Group: model.MEDIA, Flag: model.PRIVATE,
Help: `STRM 输出目录（VFS 路径，需位于可写存储中），仅在关闭「与媒体文件同目录」时使用`},
{Key: conf.MediaStrmTemplate, Value: "{site_url}/d{path}?sign={sign}", Type: conf.TypeString, Group: model.MEDIA, Flag: model.PRIVATE,
Help: `STRM 内容模板，可用变量：{site_url} 站点地址、{path} 文件虚拟路径、{sign} 签名、{name} 文件名`},
// transcode settings (FFmpeg 云端/本地转码) - 默认全部关闭
{Key: conf.TranscodeEnabled, Value: "false", Type: conf.TypeBool, Group: model.TRANSCODE, Flag: model.PRIVATE,
Help: `开启后，超过阈值的媒体文件将通过 FFmpeg 转码后再播放；默认关闭`},
{Key: conf.TranscodeRunMode, Value: "local", Type: conf.TypeSelect, Options: "local,remote,hybrid", Group: model.TRANSCODE, Flag: model.PRIVATE,
Help: `local=仅使用本机内置 worker；remote=只使用远程 Worker 节点；hybrid=本地优先，超载后派发到远程`},
{Key: conf.TranscodeMinSizeGB, Value: "5", Type: conf.TypeNumber, Group: model.TRANSCODE, Flag: model.PRIVATE,
Help: `文件大于该 GB 数才走转码（小于则直链播放），0=任意大小都转码`},
{Key: conf.TranscodeMinBitrateMbps, Value: "20", Type: conf.TypeNumber, Group: model.TRANSCODE, Flag: model.PRIVATE,
Help: `视频码率超过该 Mbps 才转码，0=不限制`},
{Key: conf.TranscodeSourceCodecs, Value: "hevc,h265,av1,vvc,vp9", Type: conf.TypeString, Group: model.TRANSCODE, Flag: model.PRIVATE,
Help: `仅对这些源视频编码进行转码（逗号分隔）。常见高码率/兼容性差的编码：hevc,av1,vvc,vp9`},
{Key: conf.TranscodeSourceExtensions, Value: "mkv,ts,m2ts,mov,avi,wmv,flv,rmvb,webm", Type: conf.TypeString, Group: model.TRANSCODE, Flag: model.PRIVATE,
Help: `仅对这些后缀进行转码（逗号分隔，不带点），mp4 默认不转码可直接播放`},
{Key: conf.TranscodeOutputFormat, Value: "hls", Type: conf.TypeSelect, Options: "hls,dash,mp4", Group: model.TRANSCODE, Flag: model.PRIVATE,
Help: `输出封装格式，HLS 兼容性最好`},
{Key: conf.TranscodeOutputCodec, Value: "h264", Type: conf.TypeSelect, Options: "h264,hevc,av1", Group: model.TRANSCODE, Flag: model.PRIVATE,
Help: `重新编码后的视频编码，推荐 h264（最广兼容、互联网分发免授权费）`},
{Key: conf.TranscodeOutputBitrate, Value: "4000k", Type: conf.TypeString, Group: model.TRANSCODE, Flag: model.PRIVATE,
Help: `输出视频码率，例如 4000k / 6M。建议 1080p:4000k、720p:2500k`},
{Key: conf.TranscodeOutputAudioCodec, Value: "aac", Type: conf.TypeSelect, Options: "aac,mp3,opus,copy", Group: model.TRANSCODE, Flag: model.PRIVATE,
Help: `输出音频编码，aac 兼容性最好；copy=直接复制源音频流`},
{Key: conf.TranscodeOutputAudioBitrate, Value: "160k", Type: conf.TypeString, Group: model.TRANSCODE, Flag: model.PRIVATE,
Help: `输出音频码率`},
{Key: conf.TranscodeOutputResolution, Value: "1920x1080", Type: conf.TypeSelect, Options: "source,3840x2160,2560x1440,1920x1080,1280x720,854x480", Group: model.TRANSCODE, Flag: model.PRIVATE,
Help: `输出分辨率上限，超过该分辨率会下采样；source=保持源分辨率`},
{Key: conf.TranscodeSegmentDuration, Value: "6", Type: conf.TypeNumber, Group: model.TRANSCODE, Flag: model.PRIVATE,
Help: `HLS/DASH 切片时长(秒)，越小首帧越快但请求数变多，推荐 4-10`},
{Key: conf.TranscodeHWAccel, Value: "none", Type: conf.TypeSelect,
Options: "none,auto,nvenc,qsv,vaapi,amf,videotoolbox",
Group: model.TRANSCODE, Flag: model.PRIVATE,
Help: `GPU 硬件加速：
none = 纯 CPU(libx264)
auto = 自动探测可用加速器
nvenc = NVIDIA GPU(GeForce/Tesla/Quadro/RTX,需 NVIDIA 驱动)
qsv = Intel 集显/独显 QuickSync(免费,功耗低)
vaapi = Linux 通用 VA-API(支持 Intel/AMD)
amf = AMD GPU(Windows AMF)
videotoolbox = macOS 硬件加速`},
{Key: conf.TranscodeFFmpegPath, Value: "ffmpeg", Type: conf.TypeString, Group: model.TRANSCODE, Flag: model.PRIVATE,
Help: `FFmpeg 可执行路径，留空则使用 PATH 中的 ffmpeg`},
{Key: conf.TranscodeFFprobePath, Value: "ffprobe", Type: conf.TypeString, Group: model.TRANSCODE, Flag: model.PRIVATE,
Help: `FFprobe 可执行路径`},
{Key: conf.TranscodeWorkerSecret, Value: "", Type: conf.TypeString, Group: model.TRANSCODE, Flag: model.PRIVATE,
Help: `远程 Worker 注册时使用的共享密钥，留空则禁用远程 Worker 注册`},
{Key: conf.TranscodeCachePath, Value: "data/ctmp", Type: conf.TypeString, Group: model.TRANSCODE, Flag: model.PRIVATE,
Help: `转码切片缓存目录`},
{Key: conf.TranscodeCacheMaxGB, Value: "20", Type: conf.TypeNumber, Group: model.TRANSCODE, Flag: model.PRIVATE,
Help: `切片缓存最大容量(GB)，超过后按 LRU 清理`},
{Key: conf.TranscodeJobTimeoutMin, Value: "120", Type: conf.TypeNumber, Group: model.TRANSCODE, Flag: model.PRIVATE,
Help: `单个转码任务超时分钟数，超时自动失败`},
{Key: conf.TranscodeLocalConcurrency, Value: "1", Type: conf.TypeNumber, Group: model.TRANSCODE, Flag: model.PRIVATE,
Help: `本机内置 worker 同时执行的转码任务数(local/hybrid 模式生效)`},
{Key: conf.TranscodeIdleTimeoutSec, Value: "90", Type: conf.TypeNumber, Group: model.TRANSCODE, Flag: model.PRIVATE,
Help: `播放端无请求多少秒后自动停止转码（默认90秒，0=禁用）`},
{Key: conf.TranscodeChunkDurationSec, Value: "60", Type: conf.TypeNumber, Group: model.TRANSCODE, Flag: model.PRIVATE,
Help: `每个 chunk 时长(秒)，越大每段缓冲越长但 ffmpeg 进程切换越少（默认60秒）`},
{Key: conf.TranscodeMaxChunkConcurrency, Value: "2", Type: conf.TypeNumber, Group: model.TRANSCODE, Flag: model.PRIVATE,
Help: `单个视频任务最多同时运行的 chunk 数（默认2，越大并发越高但 CPU/GPU 占用越多）`},
{Key: conf.TranscodeChunkIdleSec, Value: "60", Type: conf.TypeNumber, Group: model.TRANSCODE, Flag: model.PRIVATE,
Help: `chunk 多久没被访问就 kill ffmpeg 释放资源（默认60秒）`},
{Key: conf.TranscodeChunkPrefetch, Value: "true", Type: conf.TypeBool, Group: model.TRANSCODE, Flag: model.PRIVATE,
Help: `当前 chunk 转码进度过半时，是否预启动下一个 chunk（默认开启，可让顺序播放无缝衔接）`},
// multipart upload settings (from main)
{Key: conf.MultipartEnabled, Value: "true", Type: conf.TypeBool, Group: model.TRAFFIC, Flag: model.PUBLIC},
{Key: conf.MultipartChunkSize, Value: "10", Type: conf.TypeNumber, Group: model.TRAFFIC, Flag: model.PUBLIC, Help: `chunk size of multipart upload in MB (positive integer), keep it under your CDN's request body limit; each active session buffers up to 8 chunks on the server's disk`},
}
internal/bootstrap/run.go
+5
View File
@@ -15,6 +15,7 @@ import (
"github.com/OpenListTeam/OpenList/v4/internal/conf"
"github.com/OpenListTeam/OpenList/v4/internal/db"
"github.com/OpenListTeam/OpenList/v4/internal/fs"
"github.com/OpenListTeam/OpenList/v4/internal/transcode"
"github.com/OpenListTeam/OpenList/v4/pkg/utils"
"github.com/OpenListTeam/OpenList/v4/server"
"github.com/OpenListTeam/OpenList/v4/server/middlewares"
@@ -92,6 +93,10 @@ func Start() {
InitOfflineDownloadTools()
LoadStorages()
InitTaskManager()
// 初始化转码模块（总开关关时也会启动 manager，但本地 worker 仅在 enabled 时由 Manager.Start 内部按 run_mode 启动）
if transcode.IsEnabled() {
transcode.Default().Start()
}
if !flags.Debug && !flags.Dev {
gin.SetMode(gin.ReleaseMode)
}
internal/bootstrap/task.go
+1
View File
@@ -49,4 +49,5 @@ func InitTaskManager() {
op.RegisterSettingChangingCallback(func() {
fs.ArchiveContentUploadTaskManager.SetWorkersNumActive(taskFilterNegative(setting.GetInt(conf.TaskDecompressUploadThreadsNum, conf.Conf.Tasks.DecompressUpload.Workers)))
})
fs.SeedGenerateTaskManager = tache.NewManager[*fs.SeedGenerateTask](tache.WithWorks(setting.GetInt(conf.TaskUploadThreadsNum, conf.Conf.Tasks.Upload.Workers)), tache.WithMaxRetry(conf.Conf.Tasks.Upload.MaxRetry)) //seed generation will not support persist
}
internal/conf/const.go
+73 -2
View File
@@ -60,6 +60,16 @@ const (
HandleHookRateLimit = "handle_hook_rate_limit"
IgnoreSystemFiles = "ignore_system_files"
// transfer seeds
SeedSiteURL = "seed_site_url"
SeedDefaultMatrix = "seed_default_matrix"
SeedFormatPolicies = "seed_format_policies"
SeedDefaultFormat = "seed_default_format"
SeedSingleDirectPreview = "seed_single_direct_preview"
SeedCASDirectAccess = "seed_cas_direct_access"
SeedAutoGeneratePolicy = "seed_auto_generate_policy"
SeedDefaultTrackers = "seed_default_trackers"
// index
SearchIndex = "search_index"
AutoUpdateIndex = "auto_update_index"
@@ -165,8 +175,67 @@ const (
StreamMaxClientUploadSpeed = "max_client_upload_speed"
StreamMaxServerDownloadSpeed = "max_server_download_speed"
StreamMaxServerUploadSpeed = "max_server_upload_speed"
MultipartEnabled = "multipart_enabled"
MultipartChunkSize = "multipart_chunk_size"
// media
MediaTMDBKey = "media_tmdb_key"
MediaTMDBAPIURL = "media_tmdb_api_url"
MediaDiscogsToken = "media_discogs_token"
MediaDiscogsAPIURL = "media_discogs_api_url"
MediaThumbnailMode = "media_thumbnail_mode"
MediaThumbnailPath = "media_thumbnail_path"
MediaStoreThumbnail = "media_store_thumbnail"
MediaScrapeConcurrency = "media_scrape_concurrency"
MediaScrapeLanguage = "media_scrape_language" // 首选刮削语言（TMDB / Google Books）
MediaBookSource = "media_book_source" // 书籍刮削源：douban/google/openlibrary
MediaGoogleBooksKey = "media_google_books_key" // Google Books API Key（可选）
MediaDownloadCover = "media_download_cover" // 是否把远程封面下载到本地存储
// media library master switch & protocol switches
MediaEnabled = "media_enabled" // 媒体库总开关（PUBLIC，前端据此显示/隐藏）
MediaEmbyEnabled = "media_emby_enabled" // Emby 兼容 API 开关
MediaSubsonicEnabled = "media_subsonic_enabled" // OpenSubsonic API 开关
MediaSubsonicProxyStream = "media_subsonic_proxy_stream" // Subsonic 播放是否强制代理（兼容不跟随 302 的客户端）
MediaSubsonicAllowPwd = "media_subsonic_allow_password_auth" // Subsonic 是否允许 u+p 使用 OpenList 登录密码认证（默认关闭）
MediaProxyCover = "media_proxy_cover" // 是否通过服务端代理外部封面（TMDB/豆瓣等）
MediaCoverGateway = "media_cover_gateway" // 封面代理直连失败时是否回退第三方图片网关（wsrv.nl，默认关闭）
MediaStrmOnScan = "media_strm_on_scan" // 扫描完成后自动生成 STRM
MediaStrmSameDir = "media_strm_same_dir" // STRM 是否与媒体文件同目录（同名 .strm）
MediaStrmPath = "media_strm_path" // STRM 输出目录（VFS 路径，仅在指定目录模式下使用）
MediaStrmOverwrite = "media_strm_overwrite" // 生成 STRM 时是否覆盖已存在文件
MediaStrmTemplate = "media_strm_template" // STRM 内容模板
// transcode (云端/本地 FFmpeg 转码)
TranscodeEnabled = "transcode_enabled" // 总开关，默认关
TranscodeMinSizeGB = "transcode_min_size_gb" // 文件大于多少 GB 才转码
TranscodeMinBitrateMbps = "transcode_min_bitrate_mbps" // 视频码率高于多少 Mbps 才转码（0=不限）
TranscodeSourceCodecs = "transcode_source_codecs" // 仅对这些源编码进行转码
TranscodeSourceExtensions = "transcode_source_extensions" // 仅对这些后缀进行转码
TranscodeOutputFormat = "transcode_output_format" // 输出封装格式 hls/dash/mp4
TranscodeOutputCodec = "transcode_output_codec" // 重新编码的视频编码
TranscodeOutputBitrate = "transcode_output_bitrate" // 输出视频码率，例如 4000k
TranscodeOutputAudioCodec = "transcode_output_audio_codec" // 输出音频编码
TranscodeOutputAudioBitrate = "transcode_output_audio_bitrate" // 输出音频码率，例如 160k
TranscodeOutputResolution = "transcode_output_resolution" // 输出分辨率上限
TranscodeSegmentDuration = "transcode_segment_duration" // HLS 切片时长(秒)
TranscodeHWAccel = "transcode_hwaccel" // 硬件加速类型 none/auto/nvenc/qsv/vaapi/amf/videotoolbox
TranscodeFFmpegPath = "transcode_ffmpeg_path" // 自定义 ffmpeg 可执行路径
TranscodeFFprobePath = "transcode_ffprobe_path" // 自定义 ffprobe 可执行路径
TranscodeRunMode = "transcode_run_mode" // local / remote / hybrid
TranscodeWorkerSecret = "transcode_worker_secret" // 远程 Worker 共享密钥
TranscodeCachePath = "transcode_cache_path" // 切片缓存目录
TranscodeCacheMaxGB = "transcode_cache_max_gb" // 缓存最大占用，超出 LRU 淘汰
TranscodeJobTimeoutMin = "transcode_job_timeout_min" // 单个任务超时分钟
TranscodeLocalConcurrency = "transcode_local_concurrency" // 本地内置 worker 并发数(run_mode=local/hybrid 时生效)
TranscodeIdleTimeoutSec = "transcode_idle_timeout_sec" // 播放端无请求多少秒后自动停止转码（默认90秒）
// === 智能 chunk 调度相关 ===
TranscodeChunkDurationSec = "transcode_chunk_duration_sec" // 每个 chunk 时长(秒)，将整个视频分成多个 chunk 按需转码（默认60秒）
TranscodeMaxChunkConcurrency = "transcode_max_chunk_concurrency" // 单个 Job 最多同时运行的 chunk 数（默认2）
TranscodeChunkIdleSec = "transcode_chunk_idle_sec" // chunk 多久没被访问就 kill ffmpeg 释放资源（默认60秒）
TranscodeChunkPrefetch = "transcode_chunk_prefetch" // 当前 chunk 转码进度过半时，是否预启动下一个 chunk（默认true）
// multipart upload (from main)
MultipartEnabled = "multipart_enabled"
MultipartChunkSize = "multipart_chunk_size"
)
const (
@@ -198,4 +267,6 @@ const (
SharingIDKey
SkipHookKey
DirectUploadContentTypeKey
SkipNoOverwriteKey
ThumbnailBaseURLKey
)
internal/conf/var.go
+2
View File
@@ -18,6 +18,8 @@ var (
Conf *Config
URL *url.URL
ConfigPath string
// DataDir 程序数据目录（绝对路径），用于运行时文件（如本地缩略图）
DataDir string = "data"
)
var SlicesMap = make(map[string][]string)
internal/db/db.go
+98 -3
View File
@@ -1,6 +1,9 @@
package db
import (
"os"
"strings"
log "github.com/sirupsen/logrus"
"github.com/OpenListTeam/OpenList/v4/internal/conf"
@@ -10,12 +13,104 @@ import (
var db *gorm.DB
// MediaMigrateWipeEnv 设为 1/true 时，允许升级流程清空旧版 media_items 数据后重建。
// 旧版本的 folder_path 语义不同，无法原地迁移；但清库是破坏性操作，必须由管理员显式确认。
const MediaMigrateWipeEnv = "OPENLIST_MEDIA_MIGRATE_WIPE"
func Init(d *gorm.DB) {
db = d
err := AutoMigrate(new(model.Storage), new(model.User), new(model.Meta), new(model.SettingItem), new(model.SearchNode), new(model.TaskItem), new(model.SSHPublicKey), new(model.SharingDB))
if err != nil {
log.Fatalf("failed migrate database: %s", err.Error())
if err := migrateMediaItems(); err != nil {
log.Fatalf("failed migrate media_items: %s (如果旧数据无法原地迁移，请先备份数据库，再设置 %s=1 重建媒体表)", err.Error(), MediaMigrateWipeEnv)
}
if err := migrateWebDAVProperties(); err != nil {
log.Fatalf("failed migrate webdav_properties: %s", err.Error())
}
err := AutoMigrate(new(model.Storage), new(model.User), new(model.Meta), new(model.SettingItem), new(model.SearchNode), new(model.TaskItem), new(model.SSHPublicKey), new(model.SharingDB), new(model.WebDAVProperty), new(model.MediaItem), new(model.MediaConfig), new(model.MediaScanPath), new(model.MediaAppKey), new(model.MediaUserData), new(model.MediaPlaylist), new(model.MediaPlaylistItem), new(model.EmbyDevice), new(model.MediaAnnotation))
if err != nil {
// 典型场景：未确认清库时，旧 media_items 数据违反新建的唯一索引。
// 这里把解决办法直接写进日志，避免用户只看到一句 failed migrate database。
log.Fatalf(
"failed migrate database: %s (若由媒体表唯一索引与旧数据冲突导致，请先备份数据库，再设置 %s=1 重建媒体表)",
err.Error(),
MediaMigrateWipeEnv,
)
}
}
func mediaWipeConfirmed() bool {
switch os.Getenv(MediaMigrateWipeEnv) {
case "1", "true", "TRUE", "True", "yes":
return true
}
return false
}
// mediaItemsTable 返回 MediaItem 的真实表名。
// 表名带有 conf.Database.TablePrefix（可配置，默认 x_），不能硬编码。
func mediaItemsTable() (string, error) {
stmt := &gorm.Statement{DB: db}
if err := stmt.Parse(&model.MediaItem{}); err != nil {
return "", err
}
return stmt.Schema.Table, nil
}
// migrateMediaItems 处理 media_items 表的迁移兼容性
// 存储语义历史变更：
//
// v1: folder_path 为文件完整路径
// v2: folder_path 恒定为扫描根路径
// v3（当前）: folder_path 为文件/文件夹的实际父目录，供前端拼接播放地址
//
// 唯一性由 folder_path + file_name + album_name 组合索引保证。
//
// 检测到 v1/v2 的旧表（缺少新组合索引）时：
// - 默认不动数据，只输出醒目的告警，并让 AutoMigrate 在旧表上补建新索引；
// - 管理员设置环境变量 OPENLIST_MEDIA_MIGRATE_WIPE=1 后才会清空旧数据并重扫。
func migrateMediaItems() error {
table, err := mediaItemsTable()
if err != nil {
return err
}
m := db.Migrator()
if !m.HasTable(table) {
return nil
}
const idx = "idx_media_folder_file_album"
if m.HasIndex(table, idx) {
// 已是新索引：仅清理软删除残留数据，避免与唯一索引冲突
if err := db.Exec("DELETE FROM "+quoteIdent(table)+" WHERE deleted_at IS NOT NULL").Error; err != nil {
log.Warnf("media_items: 清理软删除残留数据失败: %v", err)
}
return nil
}
// 旧表存在但没有新组合索引：旧数据的 folder_path 语义已变更，无法复用。
if !mediaWipeConfirmed() {
log.Warnf("media_items: 检测到旧版本数据（表 %s 缺少索引 %s）。", table, idx)
log.Warnf("media_items: 旧数据的路径语义已变更，建议清空后重新扫描；为避免误删，本次启动不会自动清库。")
log.Warnf("media_items: 如确认清空，请设置环境变量 %s=1 后重启；也可在管理后台对各媒体库点击「清空数据库」再重扫。", MediaMigrateWipeEnv)
return nil
}
log.Warnf("media_items: 已确认 %s，清空旧版本数据（表 %s）后重新迁移", MediaMigrateWipeEnv, table)
// 先尝试删除旧的单字段唯一索引（如果存在），避免 AutoMigrate 冲突
oldIdx := "idx_" + table + "_folder_path"
if m.HasIndex(table, oldIdx) {
if err := m.DropIndex(table, oldIdx); err != nil {
log.Warnf("media_items: 删除旧唯一索引失败: %v", err)
}
}
if err := db.Exec("DELETE FROM " + quoteIdent(table)).Error; err != nil {
return err
}
return nil
}
// quoteIdent 按当前数据库方言给标识符加引号，表名来自 gorm schema 解析而非用户输入。
func quoteIdent(name string) string {
var sb strings.Builder
db.Dialector.QuoteTo(&sb, name)
return sb.String()
}
func AutoMigrate(dst ...interface{}) error {
internal/db/db_media_migrate_test.go
+123
View File
@@ -0,0 +1,123 @@
package db
import (
"testing"
"github.com/glebarez/sqlite"
"gorm.io/gorm"
"gorm.io/gorm/schema"
"github.com/OpenListTeam/OpenList/v4/internal/model"
)
func openTestDB(t *testing.T, prefix string) *gorm.DB {
t.Helper()
d, err := gorm.Open(sqlite.Open("file::memory:"), &gorm.Config{
NamingStrategy: schema.NamingStrategy{TablePrefix: prefix},
})
if err != nil {
t.Fatal(err)
}
return d
}
// 旧版表：只有 media_items 的基础结构，没有新的组合唯一索引。
func createLegacyMediaTable(t *testing.T, d *gorm.DB, table string) {
t.Helper()
if err := d.Exec("CREATE TABLE " + table + " (id integer primary key autoincrement, deleted_at datetime, folder_path text, file_name text)").Error; err != nil {
t.Fatal(err)
}
if err := d.Exec("INSERT INTO " + table + " (folder_path, file_name) VALUES ('/a/b.mp3', 'b.mp3')").Error; err != nil {
t.Fatal(err)
}
}
func countRows(t *testing.T, d *gorm.DB, table string) int64 {
t.Helper()
var n int64
if err := d.Table(table).Count(&n).Error; err != nil {
t.Fatal(err)
}
return n
}
func TestMediaItemsTableHonorsPrefix(t *testing.T) {
for _, prefix := range []string{"x_", "ol_", ""} {
db = openTestDB(t, prefix)
got, err := mediaItemsTable()
if err != nil {
t.Fatal(err)
}
if want := prefix + "media_items"; got != want {
t.Errorf("prefix %q: table = %q, want %q", prefix, got, want)
}
}
}
func TestMigrateKeepsLegacyDataWithoutConfirmation(t *testing.T) {
t.Setenv(MediaMigrateWipeEnv, "")
db = openTestDB(t, "ol_") // 非默认前缀：旧实现硬编码 x_ 会直接跳过迁移
createLegacyMediaTable(t, db, "ol_media_items")
if err := migrateMediaItems(); err != nil {
t.Fatal(err)
}
if n := countRows(t, db, "ol_media_items"); n != 1 {
t.Fatalf("legacy rows must be preserved without confirmation, got %d", n)
}
}
func TestMigrateWipesOnlyWhenConfirmed(t *testing.T) {
t.Setenv(MediaMigrateWipeEnv, "1")
db = openTestDB(t, "ol_")
createLegacyMediaTable(t, db, "ol_media_items")
if err := migrateMediaItems(); err != nil {
t.Fatal(err)
}
if n := countRows(t, db, "ol_media_items"); n != 0 {
t.Fatalf("legacy rows should be wiped after confirmation, got %d", n)
}
}
func TestAutoMigrateCreatesCompositeUniqueIndex(t *testing.T) {
db = openTestDB(t, "ol_")
if err := db.AutoMigrate(&model.MediaItem{}); err != nil {
t.Fatal(err)
}
if !db.Migrator().HasIndex("ol_media_items", "idx_media_folder_file_album") {
t.Fatal("composite unique index missing after AutoMigrate")
}
// 已有新索引时迁移不应触碰有效数据
if err := db.Create(&model.MediaItem{FolderPath: "/m", FileName: "a.mp3"}).Error; err != nil {
t.Fatal(err)
}
if err := migrateMediaItems(); err != nil {
t.Fatal(err)
}
if n := countRows(t, db, "ol_media_items"); n != 1 {
t.Fatalf("valid rows must survive migration, got %d", n)
}
}
func TestCreateOrUpdateRejectsOverlongIndexColumns(t *testing.T) {
db = openTestDB(t, "ol_")
if err := db.AutoMigrate(&model.MediaItem{}); err != nil {
t.Fatal(err)
}
long := make([]rune, model.MediaIndexColumnMax+1)
for i := range long {
long[i] = '文'
}
err := CreateOrUpdateMediaItem(&model.MediaItem{FolderPath: "/m", FileName: string(long)})
if err == nil {
t.Fatal("overlong file_name must be rejected")
}
ok := make([]rune, model.MediaIndexColumnMax)
for i := range ok {
ok[i] = '文'
}
if err := CreateOrUpdateMediaItem(&model.MediaItem{FolderPath: "/m", FileName: string(ok)}); err != nil {
t.Fatalf("boundary length must be accepted: %v", err)
}
}
internal/db/media.go
+626
View File
@@ -0,0 +1,626 @@
package db
import (
"encoding/json"
"fmt"
"strings"
"time"
"unicode/utf8"
"github.com/OpenListTeam/OpenList/v4/internal/model"
"gorm.io/gorm"
)
// ==================== MediaConfig ====================
// GetMediaConfig 获取指定类型的媒体库配置，不存在则返回默认值
func GetMediaConfig(mediaType model.MediaType) (*model.MediaConfig, error) {
var cfg model.MediaConfig
result := db.Where("media_type = ?", mediaType).First(&cfg)
if result.Error == gorm.ErrRecordNotFound {
return &model.MediaConfig{
MediaType: mediaType,
Enabled: false,
}, nil
}
return &cfg, result.Error
}
// GetAllMediaConfigs 获取所有媒体库配置
func GetAllMediaConfigs() ([]model.MediaConfig, error) {
var cfgs []model.MediaConfig
err := db.Find(&cfgs).Error
return cfgs, err
}
// SaveMediaConfig 保存媒体库配置（upsert）
func SaveMediaConfig(cfg *model.MediaConfig) error {
var existing model.MediaConfig
result := db.Where("media_type = ?", cfg.MediaType).First(&existing)
if result.Error == gorm.ErrRecordNotFound {
return db.Create(cfg).Error
}
cfg.ID = existing.ID
return db.Save(cfg).Error
}
// ==================== MediaScanPath ====================
// ListMediaScanPaths 获取指定媒体类型的所有扫描路径
func ListMediaScanPaths(mediaType model.MediaType) ([]model.MediaScanPath, error) {
var paths []model.MediaScanPath
tx := db.Model(&model.MediaScanPath{})
if mediaType != "" {
tx = tx.Where("media_type = ?", mediaType)
}
err := tx.Order("id asc").Find(&paths).Error
return paths, err
}
// GetMediaScanPath 按ID获取扫描路径
func GetMediaScanPath(id uint) (*model.MediaScanPath, error) {
var p model.MediaScanPath
err := db.First(&p, id).Error
return &p, err
}
// CreateMediaScanPath 创建扫描路径
func CreateMediaScanPath(p *model.MediaScanPath) error {
return db.Create(p).Error
}
// UpdateMediaScanPath 更新扫描路径
func UpdateMediaScanPath(p *model.MediaScanPath) error {
return db.Save(p).Error
}
// DeleteMediaScanPath 删除扫描路径（硬删除）
func DeleteMediaScanPath(id uint) error {
return db.Unscoped().Delete(&model.MediaScanPath{}, id).Error
}
// ==================== MediaItem ====================
// escapeLike 转义 LIKE 通配符，避免路径中的 % _ 影响前缀匹配
func escapeLike(s string) string {
s = strings.ReplaceAll(s, `\`, `\\`)
s = strings.ReplaceAll(s, "%", `\%`)
s = strings.ReplaceAll(s, "_", `\_`)
return s
}
// MediaItemQuery 媒体条目查询参数
type MediaItemQuery struct {
MediaType model.MediaType
ScanPathID uint // 按扫描路径ID筛选
FolderPath string // 按文件夹路径筛选
TypeTag string // 按类型标签筛选（电影、电视剧等）
ContentTag string // 按内容标签筛选（喜剧、惊悚等）
PathPrefix string // 按文件夹路径前缀筛选（用于用户 base_path 隔离）
Hidden *bool
Keyword string
OrderBy string // "name", "date", "size"
OrderDir string // "asc", "desc"
Page int
PageSize int
}
// ListMediaItems 分页查询媒体条目
func ListMediaItems(q MediaItemQuery) ([]model.MediaItem, int64, error) {
tx := db.Model(&model.MediaItem{})
if q.MediaType != "" {
tx = tx.Where("media_type = ?", q.MediaType)
}
if q.ScanPathID > 0 {
tx = tx.Where("scan_path_id = ?", q.ScanPathID)
}
if q.FolderPath != "" {
tx = tx.Where("folder_path = ?", q.FolderPath)
}
if q.PathPrefix != "" {
prefix := strings.TrimSuffix(q.PathPrefix, "/") + "/"
tx = tx.Where(`folder_path LIKE ? ESCAPE '\'`, escapeLike(prefix)+"%")
}
if q.Hidden != nil {
tx = tx.Where("hidden = ?", *q.Hidden)
}
if q.Keyword != "" {
like := "%" + q.Keyword + "%"
tx = tx.Where("file_name LIKE ? OR scraped_name LIKE ?", like, like)
}
// 按类型标签筛选（通过关联扫描路径的type_tag）
if q.TypeTag != "" {
tx = tx.Joins("JOIN media_scan_paths ON media_scan_paths.id = media_items.scan_path_id").
Where("media_scan_paths.type_tag = ?", q.TypeTag)
}
// 按内容标签筛选（通过关联扫描路径的content_tags）
if q.ContentTag != "" {
tx = tx.Joins("JOIN media_scan_paths ON media_scan_paths.id = media_items.scan_path_id").
Where("media_scan_paths.content_tags LIKE ?", "%"+q.ContentTag+"%")
}
var total int64
if err := tx.Count(&total).Error; err != nil {
return nil, 0, err
}
// 排序
orderCol := "created_at"
switch q.OrderBy {
case "name":
orderCol = "COALESCE(NULLIF(scraped_name,''), file_name)"
case "date":
orderCol = "release_date"
case "size":
orderCol = "file_size"
}
dir := "asc"
if q.OrderDir == "desc" {
dir = "desc"
}
tx = tx.Order(orderCol + " " + dir)
// 分页
if q.PageSize <= 0 {
q.PageSize = 20
}
if q.Page <= 0 {
q.Page = 1
}
tx = tx.Offset((q.Page - 1) * q.PageSize).Limit(q.PageSize)
var items []model.MediaItem
err := tx.Find(&items).Error
return items, total, err
}
// GetMediaItemByID 按ID获取媒体条目
func GetMediaItemByID(id uint) (*model.MediaItem, error) {
var item model.MediaItem
err := db.First(&item, id).Error
return &item, err
}
// GetMediaItemByFolderPath 按文件夹路径获取媒体条目（用于合并模式）
func GetMediaItemByFolderPath(folderPath string) (*model.MediaItem, error) {
var item model.MediaItem
result := db.Where("folder_path = ?", folderPath).First(&item)
return &item, result.Error
}
// CreateOrUpdateMediaItem 创建或更新媒体条目（按 folder_path + file_name + album_name 组合唯一）
// 更新时保留已有的刮削数据，避免重新扫描时把已刮削的字段清空
//
// 注意：使用 Unscoped() 查询是为了把软删除残留记录也包含进来，
// 否则唯一索引 idx_media_folder_file_album 会与软删除行冲突，
// 导致新建条目时报 UNIQUE constraint failed。
func CreateOrUpdateMediaItem(item *model.MediaItem) error {
// 复合唯一索引三列在 MySQL 下限定为 191 字符（见 model.MediaItem）。
// 超长值在严格模式下会写入失败、非严格模式下会被静默截断并造成不同文件撞同一条目，
// 因此显式拒绝，由调用方记录告警并跳过。
if utf8.RuneCountInString(item.FolderPath) > model.MediaIndexColumnMax ||
utf8.RuneCountInString(item.FileName) > model.MediaIndexColumnMax ||
utf8.RuneCountInString(item.AlbumName) > model.MediaIndexColumnMax {
return fmt.Errorf("media item path/name exceeds %d characters, skipped", model.MediaIndexColumnMax)
}
var existing model.MediaItem
result := db.Unscoped().Where("folder_path = ? AND file_name = ? AND album_name = ?", item.FolderPath, item.FileName, item.AlbumName).First(&existing)
if result.Error == gorm.ErrRecordNotFound {
return db.Create(item).Error
}
if result.Error != nil {
return result.Error
}
item.ID = existing.ID
item.CreatedAt = existing.CreatedAt
// 复用已有记录时清除软删除标记，确保该条目“恢复”为正常记录
item.DeletedAt = gorm.DeletedAt{}
// 如果已有刮削数据，保留刮削字段，防止重新扫描时覆盖刮削结果
if existing.ScrapedAt != nil {
item.ScrapedAt = existing.ScrapedAt
item.ScrapedName = existing.ScrapedName
item.Cover = existing.Cover
item.AlbumName = existing.AlbumName
item.AlbumArtist = existing.AlbumArtist
item.TrackNumber = existing.TrackNumber
item.Duration = existing.Duration
item.Genre = existing.Genre
item.ReleaseDate = existing.ReleaseDate
item.Rating = existing.Rating
item.Plot = existing.Plot
item.Authors = existing.Authors
item.Description = existing.Description
item.Publisher = existing.Publisher
item.ISBN = existing.ISBN
item.ExternalID = existing.ExternalID
}
// 扫描不覆盖用户手动隐藏的状态
item.Hidden = existing.Hidden
return db.Unscoped().Save(item).Error
}
// UpdateMediaItem 更新媒体条目
// 使用 Unscoped 确保即使存在软删除标记也能正确更新（避免命中唯一索引但行不可见的诡异情况）。
func UpdateMediaItem(item *model.MediaItem) error {
return db.Unscoped().Save(item).Error
}
// DeleteMediaItem 硬删除媒体条目（真正从数据库删除），并清理该条目的批注
func DeleteMediaItem(id uint) error {
return db.Transaction(func(tx *gorm.DB) error {
if err := tx.Unscoped().Delete(&model.MediaItem{}, id).Error; err != nil {
return err
}
return tx.Where("item_id = ?", id).Delete(&model.MediaAnnotation{}).Error
})
}
// ClearMediaItems 硬删除指定类型的所有媒体条目（真正从数据库删除），并清理对应批注
func ClearMediaItems(mediaType model.MediaType) error {
return db.Transaction(func(tx *gorm.DB) error {
ids := []uint{}
if err := tx.Unscoped().Model(&model.MediaItem{}).
Where("media_type = ?", mediaType).Pluck("id", &ids).Error; err != nil {
return err
}
if err := tx.Unscoped().Where("media_type = ?", mediaType).
Delete(&model.MediaItem{}).Error; err != nil {
return err
}
if len(ids) == 0 {
return nil
}
return tx.Where("item_id IN ?", ids).Delete(&model.MediaAnnotation{}).Error
})
}
// ClearMediaItemsByScanPath 硬删除指定扫描路径的所有媒体条目，并清理对应批注
func ClearMediaItemsByScanPath(scanPathID uint) error {
return db.Transaction(func(tx *gorm.DB) error {
ids := []uint{}
if err := tx.Unscoped().Model(&model.MediaItem{}).
Where("scan_path_id = ?", scanPathID).Pluck("id", &ids).Error; err != nil {
return err
}
if err := tx.Unscoped().Where("scan_path_id = ?", scanPathID).
Delete(&model.MediaItem{}).Error; err != nil {
return err
}
if len(ids) == 0 {
return nil
}
return tx.Where("item_id IN ?", ids).Delete(&model.MediaAnnotation{}).Error
})
}
// ClearMediaScrapedData 清空指定类型的所有刮削数据（保留扫描出的文件记录本身）
// 仅清空刮削结果相关字段，不删除条目，便于重新刮削。
// mediaType 为空时表示对所有类型生效。
func ClearMediaScrapedData(mediaType model.MediaType) (int64, error) {
tx := db.Unscoped().Model(&model.MediaItem{})
if mediaType != "" {
tx = tx.Where("media_type = ?", mediaType)
}
updates := map[string]interface{}{
"scraped_name": "",
"description": "",
"cover": "",
"release_date": "",
"rating": 0,
"genre": "",
"authors": "",
"plot": "",
"reviews": "",
"external_id": "",
"album_artist": "",
"publisher": "",
"isbn": "",
"lyrics": "",
"scraped_at": nil,
}
result := tx.Updates(updates)
return result.RowsAffected, result.Error
}
// ListAllValidMediaItemsForCheck 列出指定类型下所有未删除的媒体条目（仅取必要字段，用于失效检查）
// mediaType 为空时返回所有类型条目。
func ListAllValidMediaItemsForCheck(mediaType model.MediaType) ([]model.MediaItem, error) {
var items []model.MediaItem
tx := db.Model(&model.MediaItem{}).
Select("id, media_type, scan_path_id, file_name, folder_path, is_folder")
if mediaType != "" {
tx = tx.Where("media_type = ?", mediaType)
}
err := tx.Find(&items).Error
return items, err
}
// DeleteMediaItemsByIDs 按ID列表硬删除媒体条目，并清理对应批注
func DeleteMediaItemsByIDs(ids []uint) error {
if len(ids) == 0 {
return nil
}
return db.Transaction(func(tx *gorm.DB) error {
if err := tx.Unscoped().Where("id IN ?", ids).Delete(&model.MediaItem{}).Error; err != nil {
return err
}
return tx.Where("item_id IN ?", ids).Delete(&model.MediaAnnotation{}).Error
})
}
// ListAlbums 列出所有专辑（音乐专用）
func ListAlbums(q MediaItemQuery) ([]AlbumInfo, int64, error) {
type albumRow struct {
AlbumName string
AlbumArtist string
Cover string
ReleaseDate string
TrackCount int
ScanPathID uint
}
// 构建基础查询
baseQuery := db.Model(&model.MediaItem{}).
Where("media_type = ?", model.MediaTypeMusic)
if q.Hidden != nil {
baseQuery = baseQuery.Where("hidden = ?", *q.Hidden)
}
if q.ScanPathID > 0 {
baseQuery = baseQuery.Where("scan_path_id = ?", q.ScanPathID)
}
if q.Keyword != "" {
like := "%" + q.Keyword + "%"
baseQuery = baseQuery.Where("album_name LIKE ? OR album_artist LIKE ? OR scraped_name LIKE ?", like, like, like)
}
// 统计分组数（用子查询）
var total int64
if err := db.Table("(?) as sub", baseQuery.
Select("album_name, album_artist").
Group("album_name, album_artist")).
Count(&total).Error; err != nil {
return nil, 0, err
}
if q.PageSize <= 0 {
q.PageSize = 20
}
if q.Page <= 0 {
q.Page = 1
}
tx := baseQuery.
Select("album_name, album_artist, MAX(cover) as cover, MAX(release_date) as release_date, COUNT(*) as track_count, MAX(scan_path_id) as scan_path_id").
Group("album_name, album_artist").
Offset((q.Page - 1) * q.PageSize).Limit(q.PageSize)
var rows []albumRow
if err := tx.Scan(&rows).Error; err != nil {
return nil, 0, err
}
albums := make([]AlbumInfo, len(rows))
for i, r := range rows {
albums[i] = AlbumInfo{
AlbumName: r.AlbumName,
AlbumArtist: r.AlbumArtist,
Cover: r.Cover,
ReleaseDate: r.ReleaseDate,
TrackCount: r.TrackCount,
ScanPathID: r.ScanPathID,
}
}
return albums, total, nil
}
// AlbumInfo 专辑信息
type AlbumInfo struct {
AlbumName string `json:"album_name"`
AlbumArtist string `json:"album_artist"`
Cover string `json:"cover"`
ReleaseDate string `json:"release_date"`
TrackCount int `json:"track_count"`
ScanPathID uint `json:"scan_path_id"`
}
// GetAlbumTracks 获取专辑曲目列表
// 支持两种模式：
// 1. 普通模式（is_folder=false）：直接返回独立文件记录
// 2. 合并文件夹模式（is_folder=true）：把 episodes 展开成虚拟 MediaItem 列表
// 展开后每条记录的 folder_path = 原folder_path/file_name（文件夹实际路径），file_name = episode.FileName
func GetAlbumTracks(albumName, albumArtist string) ([]model.MediaItem, error) {
var items []model.MediaItem
tx := db.Where("media_type = ?", model.MediaTypeMusic)
if albumName != "" {
tx = tx.Where("album_name = ?", albumName)
} else {
tx = tx.Where("(album_name = '' OR album_name IS NULL)")
}
if albumArtist != "" {
tx = tx.Where("album_artist = ?", albumArtist)
}
err := tx.Order("track_number asc").Find(&items).Error
if err != nil {
return nil, err
}
// 展开合并文件夹条目的 episodes
var result []model.MediaItem
for _, item := range items {
if !item.IsFolder || item.Episodes == "" {
result = append(result, item)
continue
}
// 解析 episodes
type EpisodeInfo struct {
FileName string `json:"file_name"`
Index int `json:"index"`
Title string `json:"title"`
}
var eps []EpisodeInfo
if err := json.Unmarshal([]byte(item.Episodes), &eps); err != nil || len(eps) == 0 {
// 解析失败则跳过该条目（不返回文件夹本身，避免播放路径错误）
continue
}
// 文件夹实际路径 = folder_path + "/" + file_name
actualDir := strings.TrimRight(item.FolderPath, "/") + "/" + item.FileName
for _, ep := range eps {
track := item // 复制基础信息（封面、专辑名、艺术家等）
track.ID = 0
track.IsFolder = false
track.FolderPath = actualDir
track.FileName = ep.FileName
track.TrackNumber = ep.Index
track.ScrapedName = ep.Title
track.Episodes = ""
result = append(result, track)
}
}
return result, nil
}
// ListFolderPaths 列出指定媒体类型下的所有文件夹路径（目录浏览模式）
func ListFolderPaths(mediaType model.MediaType) ([]string, error) {
var paths []string
err := db.Model(&model.MediaItem{}).
Where("media_type = ?", mediaType).
Distinct("folder_path").
Pluck("folder_path", &paths).Error
return paths, err
}
// GetUnscrappedItems 获取未刮削或刮削不完整的媒体条目
// 只要 scraped_at 为空，或 cover/scraped_name/plot 任一为空，就需要重新刮削
// limit <= 0 表示不限制数量，返回所有未刮削条目
func GetUnscrappedItems(mediaType model.MediaType, limit int) ([]model.MediaItem, error) {
var items []model.MediaItem
tx := db.Where(
"media_type = ? AND (scraped_at IS NULL OR cover = '' OR cover IS NULL OR scraped_name = '' OR scraped_name IS NULL OR plot = '' OR plot IS NULL)",
mediaType,
)
if limit > 0 {
tx = tx.Limit(limit)
}
err := tx.Find(&items).Error
return items, err
}
// ==================== 导入/导出 ====================
// ListAllMediaItems 列出指定媒体类型下的所有条目（不分页，用于导出）
// mediaType 为空时返回全部类型的所有条目
func ListAllMediaItems(mediaType model.MediaType) ([]model.MediaItem, error) {
var items []model.MediaItem
tx := db.Model(&model.MediaItem{})
if mediaType != "" {
tx = tx.Where("media_type = ?", mediaType)
}
err := tx.Order("id asc").Find(&items).Error
return items, err
}
// ListMediaItemsByScanPath 列出指定扫描路径下的所有条目（不分页，用于导出）
func ListMediaItemsByScanPath(scanPathID uint) ([]model.MediaItem, error) {
var items []model.MediaItem
err := db.Where("scan_path_id = ?", scanPathID).Order("id asc").Find(&items).Error
return items, err
}
// ListAllMediaScanPaths 列出所有扫描路径（不区分类型，用于全量导出）
func ListAllMediaScanPaths() ([]model.MediaScanPath, error) {
var paths []model.MediaScanPath
err := db.Order("id asc").Find(&paths).Error
return paths, err
}
// ImportMediaItems 批量导入媒体条目
// 策略：按 (folder_path, file_name, album_name) 唯一键 upsert：
// - 已存在则覆盖刮削字段（导入是用户主动行为，覆盖优先于保留）
// - 不存在则新建
//
// 注意：导入数据中的 ID/CreatedAt/UpdatedAt 会被忽略，由数据库重新分配，
// 避免和现有记录的主键冲突。
func ImportMediaItems(items []model.MediaItem, scanPathIDOverride *uint) (created, updated int, err error) {
for i := range items {
it := items[i]
// 清空主键和时间戳，让数据库重新生成
it.ID = 0
it.CreatedAt = time.Time{}
it.UpdatedAt = time.Time{}
it.DeletedAt = gorm.DeletedAt{}
// 如果指定了覆盖的 scan_path_id（按扫描路径导入场景），则强制覆盖
if scanPathIDOverride != nil {
it.ScanPathID = *scanPathIDOverride
}
var existing model.MediaItem
result := db.Unscoped().Where(
"folder_path = ? AND file_name = ? AND album_name = ?",
it.FolderPath, it.FileName, it.AlbumName,
).First(&existing)
if result.Error == gorm.ErrRecordNotFound {
if e := db.Create(&it).Error; e != nil {
return created, updated, e
}
created++
continue
}
if result.Error != nil {
return created, updated, result.Error
}
// 覆盖更新
it.ID = existing.ID
it.CreatedAt = existing.CreatedAt
it.DeletedAt = gorm.DeletedAt{}
if e := db.Unscoped().Save(&it).Error; e != nil {
return created, updated, e
}
updated++
}
return created, updated, nil
}
// ImportMediaScanPaths 批量导入扫描路径
// 策略：按 (media_type, path) 唯一对去重：
// - 已存在则更新名称/标签等可编辑字段
// - 不存在则新建并返回新ID
//
// 返回值 idMap：导入数据中的原始ID -> 数据库实际ID 的映射，
// 供随后的 MediaItem 导入用于把 scan_path_id 重新指向新ID。
func ImportMediaScanPaths(paths []model.MediaScanPath) (idMap map[uint]uint, created, updated int, err error) {
idMap = make(map[uint]uint, len(paths))
for i := range paths {
p := paths[i]
originalID := p.ID
p.ID = 0
p.CreatedAt = time.Time{}
p.UpdatedAt = time.Time{}
p.DeletedAt = gorm.DeletedAt{}
var existing model.MediaScanPath
result := db.Where("media_type = ? AND path = ?", p.MediaType, p.Path).First(&existing)
if result.Error == gorm.ErrRecordNotFound {
if e := db.Create(&p).Error; e != nil {
return idMap, created, updated, e
}
idMap[originalID] = p.ID
created++
continue
}
if result.Error != nil {
return idMap, created, updated, result.Error
}
// 已存在：更新可编辑字段
existing.Name = p.Name
existing.PathMerge = p.PathMerge
existing.TypeTag = p.TypeTag
existing.ContentTags = p.ContentTags
existing.EnableScrape = p.EnableScrape
if e := db.Save(&existing).Error; e != nil {
return idMap, created, updated, e
}
idMap[originalID] = existing.ID
updated++
}
return idMap, created, updated, nil
}
internal/db/media_extra.go
+223
View File
@@ -0,0 +1,223 @@
package db
import (
"time"
"github.com/OpenListTeam/OpenList/v4/internal/model"
"gorm.io/gorm"
"gorm.io/gorm/clause"
)
// ==================== MediaAppKey ====================
func CreateMediaAppKey(k *model.MediaAppKey) error {
return db.Create(k).Error
}
func ListMediaAppKeys(userID uint) ([]model.MediaAppKey, error) {
var keys []model.MediaAppKey
err := db.Where("user_id = ?", userID).Order("id asc").Find(&keys).Error
return keys, err
}
func GetMediaAppKey(id uint) (*model.MediaAppKey, error) {
var k model.MediaAppKey
err := db.First(&k, id).Error
return &k, err
}
func DeleteMediaAppKey(userID, id uint) error {
return db.Where("id = ? AND user_id = ?", id, userID).Delete(&model.MediaAppKey{}).Error
}
func TouchMediaAppKey(id uint) {
now := time.Now()
_ = db.Model(&model.MediaAppKey{}).Where("id = ?", id).Update("last_used_at", now).Error
}
// ListMediaAppKeysByPrefix 认证时按明文前缀预筛选候选密钥
func ListMediaAppKeysByPrefix(prefix string) ([]model.MediaAppKey, error) {
var keys []model.MediaAppKey
err := db.Where("prefix = ?", prefix).Find(&keys).Error
return keys, err
}
// ==================== MediaUserData ====================
func GetMediaUserData(userID, itemID uint, episodeIndex int) (*model.MediaUserData, error) {
var d model.MediaUserData
err := db.Where("user_id = ? AND item_id = ? AND episode_index = ?", userID, itemID, episodeIndex).First(&d).Error
if err != nil {
return nil, err
}
return &d, nil
}
func SaveMediaUserData(d *model.MediaUserData) error {
var existing model.MediaUserData
err := db.Where("user_id = ? AND item_id = ? AND episode_index = ?", d.UserID, d.ItemID, d.EpisodeIndex).First(&existing).Error
if err == gorm.ErrRecordNotFound {
return db.Create(d).Error
}
if err != nil {
return err
}
d.ID = existing.ID
d.CreatedAt = existing.CreatedAt
return db.Save(d).Error
}
// ListMediaUserDataByItems 批量取用户对一组条目的播放状态
func ListMediaUserDataByItems(userID uint, itemIDs []uint) ([]model.MediaUserData, error) {
var list []model.MediaUserData
if len(itemIDs) == 0 {
return list, nil
}
err := db.Where("user_id = ? AND item_id IN ?", userID, itemIDs).Find(&list).Error
return list, err
}
// ListMediaFavorites 用户收藏的条目
func ListMediaFavorites(userID uint) ([]model.MediaUserData, error) {
var list []model.MediaUserData
err := db.Where("user_id = ? AND is_favorite = ?", userID, true).Order("updated_at desc").Find(&list).Error
return list, err
}
// ListMediaResume 继续观看：未看完且有进度
func ListMediaResume(userID uint, limit int) ([]model.MediaUserData, error) {
var list []model.MediaUserData
if limit <= 0 {
limit = 50
}
err := db.Where("user_id = ? AND played = ? AND position_ticks > 0", userID, false).
Order("last_played_at desc").Limit(limit).Find(&list).Error
return list, err
}
// ClearMediaUserDataByUser 删除用户全部播放状态
func ClearMediaUserDataByUser(userID uint) error {
return db.Where("user_id = ?", userID).Delete(&model.MediaUserData{}).Error
}
// ==================== MediaPlaylist ====================
func CreateMediaPlaylist(p *model.MediaPlaylist) error {
return db.Create(p).Error
}
func GetMediaPlaylist(id uint) (*model.MediaPlaylist, error) {
var p model.MediaPlaylist
err := db.First(&p, id).Error
return &p, err
}
func ListMediaPlaylists(userID uint) ([]model.MediaPlaylist, error) {
var list []model.MediaPlaylist
err := db.Where("user_id = ?", userID).Order("id asc").Find(&list).Error
return list, err
}
func UpdateMediaPlaylist(p *model.MediaPlaylist) error {
return db.Save(p).Error
}
func DeleteMediaPlaylist(id uint) error {
if err := db.Where("playlist_id = ?", id).Delete(&model.MediaPlaylistItem{}).Error; err != nil {
return err
}
return db.Delete(&model.MediaPlaylist{}, id).Error
}
// ReplaceMediaPlaylistItems 覆盖式写入播放列表条目
func ReplaceMediaPlaylistItems(playlistID uint, items []model.MediaPlaylistItem) error {
if err := db.Where("playlist_id = ?", playlistID).Delete(&model.MediaPlaylistItem{}).Error; err != nil {
return err
}
if len(items) == 0 {
return nil
}
for i := range items {
items[i].ID = 0
items[i].PlaylistID = playlistID
items[i].Order = i
}
return db.Create(&items).Error
}
func ListMediaPlaylistItems(playlistID uint) ([]model.MediaPlaylistItem, error) {
var items []model.MediaPlaylistItem
err := db.Where("playlist_id = ?", playlistID).Order("sort_order asc").Find(&items).Error
return items, err
}
// CountMediaPlaylistItems 统计条目数
func CountMediaPlaylistItems(playlistID uint) (int64, error) {
var n int64
err := db.Model(&model.MediaPlaylistItem{}).Where("playlist_id = ?", playlistID).Count(&n).Error
return n, err
}
// ==================== EmbyDevice ====================
func CreateEmbyDevice(d *model.EmbyDevice) error {
return db.Create(d).Error
}
func GetEmbyDeviceByTokenHash(hash string) (*model.EmbyDevice, error) {
var d model.EmbyDevice
err := db.Where("token_hash = ?", hash).First(&d).Error
if err != nil {
return nil, err
}
return &d, nil
}
func DeleteEmbyDevice(userID uint, deviceID string) error {
tx := db.Where("user_id = ?", userID)
if deviceID != "" {
tx = tx.Where("device_id = ?", deviceID)
}
return tx.Delete(&model.EmbyDevice{}).Error
}
func TouchEmbyDevice(id uint) {
_ = db.Model(&model.EmbyDevice{}).Where("id = ?", id).Update("last_seen_at", time.Now()).Error
}
// ==================== MediaAnnotation ====================
// ListMediaAnnotations 列出某用户对某个条目的全部批注
func ListMediaAnnotations(userID, itemID uint) ([]model.MediaAnnotation, error) {
list := []model.MediaAnnotation{}
err := db.Where("user_id = ? AND item_id = ?", userID, itemID).
Order("id asc").Find(&list).Error
return list, err
}
// SyncMediaAnnotations 增量保存批注：upserts 覆盖同一 remote_id 的记录，deletes 按 remote_id 删除。
// 放在一个事务里，避免多设备同时保存时出现半新半旧的状态。
func SyncMediaAnnotations(userID, itemID uint, upserts []model.MediaAnnotation, deletes []string) error {
return db.Transaction(func(tx *gorm.DB) error {
for i := range upserts {
row := upserts[i]
row.ID = 0
row.UserID, row.ItemID = userID, itemID
if err := tx.Clauses(clause.OnConflict{
Columns: []clause.Column{
{Name: "user_id"}, {Name: "item_id"}, {Name: "remote_id"},
},
DoUpdates: clause.AssignmentColumns([]string{"anchor", "updated_at"}),
}).Create(&row).Error; err != nil {
return err
}
}
if len(deletes) > 0 {
if err := tx.Where("user_id = ? AND item_id = ? AND remote_id IN ?", userID, itemID, deletes).
Delete(&model.MediaAnnotation{}).Error; err != nil {
return err
}
}
return nil
})
}
internal/db/sharing.go
+8
View File
@@ -4,6 +4,7 @@ import (
"github.com/OpenListTeam/OpenList/v4/internal/model"
"github.com/OpenListTeam/OpenList/v4/pkg/utils/random"
"github.com/pkg/errors"
"gorm.io/gorm"
)
func GetSharingById(id string) (*model.SharingDB, error) {
@@ -64,6 +65,13 @@ func UpdateSharing(s *model.SharingDB) error {
return errors.WithStack(db.Save(s).Error)
}
func TakeSharingSlot(id string) (bool, error) {
res := db.Model(&model.SharingDB{}).
Where("id = ? AND (max_accessed <= 0 OR accessed < max_accessed)", id).
UpdateColumn("accessed", gorm.Expr("accessed + 1"))
return res.RowsAffected > 0, errors.WithStack(res.Error)
}
func UpdateSharingId(oldId, newId string) error {
// Check if new ID already exists
if err := db.Where("id = ?", newId).First(&model.SharingDB{}).Error; err == nil {
internal/db/webdav_property_migrate.go
+71
View File
@@ -0,0 +1,71 @@
package db
import (
"database/sql"
"github.com/OpenListTeam/OpenList/v4/internal/model"
"gorm.io/gorm"
)
// migrateWebDAVProperties prepares existing dead properties for the exact
// hash-based unique index used by WebDAVProperty.
func migrateWebDAVProperties() error {
table, err := webdavPropertiesTable()
if err != nil {
return err
}
m := db.Migrator()
if !m.HasTable(table) {
return nil
}
if !m.HasColumn(table, "key_hash") {
if err := m.AddColumn(&model.WebDAVProperty{}, "KeyHash"); err != nil {
return err
}
}
var rows []struct {
ID uint
Path string
Namespace string
Name string
KeyHash sql.NullString
}
if err := db.Model(&model.WebDAVProperty{}).Select("id, path, namespace, name, key_hash").Find(&rows).Error; err != nil {
return err
}
for _, row := range rows {
hash := model.HashWebDAVPropertyKey(row.Path, row.Namespace, row.Name)
if row.KeyHash.Valid && row.KeyHash.String == hash {
continue
}
if err := db.Model(&model.WebDAVProperty{}).Where("id = ?", row.ID).UpdateColumn("key_hash", hash).Error; err != nil {
return err
}
}
if m.HasIndex(table, "idx_webdav_property") {
indexes, err := m.GetIndexes(&model.WebDAVProperty{})
if err != nil {
return err
}
for _, index := range indexes {
if index.Name() == "idx_webdav_property" && (len(index.Columns()) != 1 || index.Columns()[0] != "key_hash") {
if err := m.DropIndex(table, index.Name()); err != nil {
return err
}
break
}
}
}
return nil
}
func webdavPropertiesTable() (string, error) {
stmt := &gorm.Statement{DB: db}
if err := stmt.Parse(&model.WebDAVProperty{}); err != nil {
return "", err
}
return stmt.Schema.Table, nil
}
internal/db/webdav_property_migrate_test.go
+135
View File
@@ -0,0 +1,135 @@
package db
import (
"database/sql"
"fmt"
"strings"
"sync"
"testing"
"github.com/OpenListTeam/OpenList/v4/internal/model"
"github.com/glebarez/sqlite"
"gorm.io/driver/mysql"
"gorm.io/gorm"
"gorm.io/gorm/clause"
"gorm.io/gorm/migrator"
"gorm.io/gorm/schema"
)
// MySQL rejects indexes on TEXT columns without a key prefix length
// (Error 1170), see #3258. The unique index must therefore use the fixed-size
// hash column rather than a prefix of the full WebDAV property key.
func TestWebDAVPropertyMySQLIndexUsesKeyHash(t *testing.T) {
conn, err := sql.Open("mysql", "user:pass@tcp(127.0.0.1:1)/db")
if err != nil {
t.Fatal(err)
}
defer conn.Close()
d, err := gorm.Open(mysql.New(mysql.Config{Conn: conn, SkipInitializeWithVersion: true}), &gorm.Config{
DisableAutomaticPing: true,
})
if err != nil {
t.Fatal(err)
}
s, err := schema.Parse(&model.WebDAVProperty{}, &sync.Map{}, d.NamingStrategy)
if err != nil {
t.Fatal(err)
}
for _, name := range []string{"Path", "Namespace", "Name"} {
if got := strings.ToLower(d.Dialector.DataTypeOf(s.LookUpField(name))); got != "text" {
t.Fatalf("%s column type = %q, want text", name, got)
}
}
if got := strings.ToLower(d.Dialector.DataTypeOf(s.LookUpField("KeyHash"))); got != "varchar(64)" {
t.Fatalf("KeyHash column type = %q, want varchar(64)", got)
}
idx := s.LookIndex("idx_webdav_property")
if idx == nil || idx.Class != "UNIQUE" {
t.Fatalf("unique index idx_webdav_property not found: %+v", idx)
}
stmt := &gorm.Statement{DB: d, Schema: s}
opts := d.Migrator().(migrator.BuildIndexOptionsInterface).BuildIndexOptions(idx.Fields, stmt)
if len(opts) != 1 || opts[0].(clause.Expr).SQL != "`key_hash`" {
t.Fatalf("index parts = %v, want `key_hash`", opts)
}
}
func TestWebDAVPropertyLongPathSQLite(t *testing.T) {
d, err := gorm.Open(sqlite.Open("file::memory:"), &gorm.Config{})
if err != nil {
t.Fatal(err)
}
if err := d.AutoMigrate(&model.WebDAVProperty{}); err != nil {
t.Fatal(err)
}
if !d.Migrator().HasIndex(&model.WebDAVProperty{}, "idx_webdav_property") {
t.Fatal("idx_webdav_property not created")
}
// Re-running AutoMigrate on an existing table must be a no-op.
if err := d.AutoMigrate(&model.WebDAVProperty{}); err != nil {
t.Fatal(err)
}
long := "/" + strings.Repeat("目录/", 400) + "file.txt"
row := model.WebDAVProperty{Path: long, Namespace: "DAV:", Name: "displayname", InnerXML: []byte("x")}
if err := d.Create(&row).Error; err != nil {
t.Fatalf("insert long path: %v", err)
}
var got model.WebDAVProperty
if err := d.Where("path = ?", long).First(&got).Error; err != nil {
t.Fatal(err)
}
if got.Path != long {
t.Fatalf("path truncated: len %d, want %d", len(got.Path), len(long))
}
if got.KeyHash != model.HashWebDAVPropertyKey(long, "DAV:", "displayname") {
t.Fatalf("key hash = %q, want hash of complete property key", got.KeyHash)
}
dup := model.WebDAVProperty{Path: long, Namespace: "DAV:", Name: "displayname"}
if err := d.Create(&dup).Error; err == nil {
t.Fatal("duplicate (path, namespace, name) was accepted")
}
other := model.WebDAVProperty{Path: long + "2", Namespace: "DAV:", Name: "displayname"}
if err := d.Create(&other).Error; err != nil {
t.Fatal(fmt.Errorf("distinct path rejected: %w", err))
}
}
func TestMigrateWebDAVPropertiesBackfillsKeyHash(t *testing.T) {
d := openTestDB(t, "ol_")
db = d
if err := d.Exec(`CREATE TABLE ol_web_dav_properties (
id integer primary key autoincrement,
path text,
namespace text,
name text,
lang text,
inner_xml blob,
UNIQUE (path, namespace, name)
)`).Error; err != nil {
t.Fatal(err)
}
path := "/" + strings.Repeat("目录/", 400) + "file.txt"
if err := d.Exec("INSERT INTO ol_web_dav_properties (path, namespace, name) VALUES (?, ?, ?)", path, "DAV:", "displayname").Error; err != nil {
t.Fatal(err)
}
if err := migrateWebDAVProperties(); err != nil {
t.Fatal(err)
}
if err := d.AutoMigrate(&model.WebDAVProperty{}); err != nil {
t.Fatal(err)
}
var got model.WebDAVProperty
if err := d.Where("path = ?", path).First(&got).Error; err != nil {
t.Fatal(err)
}
if got.KeyHash != model.HashWebDAVPropertyKey(path, "DAV:", "displayname") {
t.Fatalf("migrated key hash = %q, want hash of complete property key", got.KeyHash)
}
if !d.Migrator().HasIndex("ol_web_dav_properties", "idx_webdav_property") {
t.Fatal("idx_webdav_property not recreated")
}
}
internal/driver/driver.go
+52
View File
@@ -4,6 +4,7 @@ import (
"context"
"github.com/OpenListTeam/OpenList/v4/internal/model"
"github.com/OpenListTeam/OpenList/v4/pkg/utils"
)
type Driver interface {
@@ -218,3 +219,54 @@ type DirectUploader interface {
// return errs.NotImplement if the driver does not support the given direct upload tool
GetDirectUploadInfo(ctx context.Context, tool string, dstDir model.Obj, fileName string, fileSize int64) (any, error)
}
// SeedRapidUploadRequest carries everything a driver needs to attempt a
// hash-driven rapid upload (秒传/CAS) without transferring the full content.
//
// It is populated from a parsed transfer seed, so that any cloud drive
// supporting hash-based rapid upload can be used as a seed save target.
type SeedRapidUploadRequest struct {
// Name is the target file name.
Name string
// Size is the total file size in bytes.
Size int64
// Whole holds whole-file hashes indexed by algorithm (e.g. utils.MD5,
// utils.SHA1). It must never be nil; individual entries may be empty.
Whole *utils.HashInfo
// SliceSize is the per-slice/piece size in bytes (0 when unknown).
SliceSize int64
// SliceMD5s is the ordered per-slice MD5 list (used by 189pc-style CAS).
SliceMD5s []string
// SliceSHA1s is the ordered per-slice SHA1 list (used by SHA1-piece drives).
SliceSHA1s []string
// Open lazily yields the file content as a model.FileStreamer. Drivers whose
// rapid-upload protocol needs partial or full content (e.g. a leading
// pre-hash or a proof-code) may call it; hash-only drivers may ignore it.
// Open may be nil when no content source is available, in which case
// drivers that strictly require content must fail gracefully.
Open func() (model.FileStreamer, error)
}
// SeedRapidUploader is an optional capability interface implemented by drivers
// that can perform a "rapid upload" (秒传/CAS) driven by precomputed hashes
// instead of a full content transfer.
//
// It generalizes the previous hard-coded 189pc-specific CAS path so that any
// cloud drive supporting hash-based rapid upload (e.g. 189pc via MD5+slice MD5,
// 115/aliyundrive_open via SHA1) can be used as a transfer-seed save target.
type SeedRapidUploader interface {
// RapidUploadByHashes attempts a rapid upload of a file into dstDir.
//
// Implementations should return errs.NotImplement (or a descriptive error)
// when the required hash is missing, the file does not exist remotely, or
// the rapid upload cannot be confirmed.
RapidUploadByHashes(ctx context.Context, dstDir model.Obj, req *SeedRapidUploadRequest, overwrite bool) (model.Obj, error)
// RapidHashAlgos reports the whole-file hash algorithms accepted by
// RapidUploadByHashes (e.g. utils.MD5, utils.SHA1).
RapidHashAlgos() []utils.HashType
// RapidHashNeedsPieces reports whether RapidUploadByHashes relies on
// per-slice hashes (CAS slice MD5s / SHA1 pieces) for this driver.
RapidHashNeedsPieces() bool
}
internal/driver/seed_stream.go
+72
View File
@@ -0,0 +1,72 @@
package driver
import (
"io"
"time"
"github.com/OpenListTeam/OpenList/v4/internal/model"
"github.com/OpenListTeam/OpenList/v4/pkg/http_range"
"github.com/OpenListTeam/OpenList/v4/pkg/utils"
)
// SeedHashStream is a model.FileStreamer that only carries file metadata and
// precomputed hashes; it never yields real content.
//
// It exists so that hash-driven rapid upload (秒传/CAS) implementations can reuse
// the drivers' existing Put/RapidUpload code paths, which expect a
// model.FileStreamer but only read the name/size/hash. Drivers that additionally
// need the real content (e.g. to compute a leading proof hash) should supply
// Source, which is exposed via GetReadCloser-like lazy opening.
type SeedHashStream struct {
name string
size int64
hashInfo utils.HashInfo
// Source lazily opens the underlying content streamer. May be nil.
Source func() (model.FileStreamer, error)
}
var (
_ model.FileStreamer = (*SeedHashStream)(nil)
_ utils.ClosersIF = (*SeedHashStream)(nil)
)
// NewSeedHashStream builds a hash-only streamer from a rapid-upload request.
func NewSeedHashStream(req *SeedRapidUploadRequest) *SeedHashStream {
s := &SeedHashStream{name: req.Name, size: req.Size, Source: req.Open}
if req.Whole != nil {
s.hashInfo = *req.Whole
}
return s
}
func (s *SeedHashStream) GetName() string { return s.name }
func (s *SeedHashStream) GetSize() int64 { return s.size }
func (s *SeedHashStream) GetHash() utils.HashInfo { return s.hashInfo }
func (s *SeedHashStream) GetMimetype() string { return "" }
func (s *SeedHashStream) ModTime() time.Time { return time.Now() }
func (s *SeedHashStream) CreateTime() time.Time { return time.Now() }
func (s *SeedHashStream) IsDir() bool { return false }
func (s *SeedHashStream) GetID() string { return "" }
func (s *SeedHashStream) GetPath() string { return "" }
func (s *SeedHashStream) NeedStore() bool { return false }
func (s *SeedHashStream) IsForceStreamUpload() bool { return true }
func (s *SeedHashStream) GetExist() model.Obj { return nil }
func (s *SeedHashStream) SetExist(model.Obj) {}
func (s *SeedHashStream) GetFile() model.File { return nil }
func (s *SeedHashStream) Add(io.Closer) {}
func (s *SeedHashStream) AddIfCloser(any) {}
func (s *SeedHashStream) Close() error { return nil }
// Read returns EOF: the stream carries hashes only, no content.
func (s *SeedHashStream) Read([]byte) (int, error) { return 0, io.EOF }
// RangeRead returns an empty reader, since no content is available.
func (s *SeedHashStream) RangeRead(http_range.Range) (io.Reader, error) {
return nil, io.EOF
}
// CacheFullAndWriter reports that the content cannot be materialized.
func (s *SeedHashStream) CacheFullAndWriter(*model.UpdateProgress, io.Writer) (model.File, error) {
return nil, io.EOF
}
internal/driver/seed_stream_test.go
+87
View File
@@ -0,0 +1,87 @@
package driver
import (
"errors"
"io"
"strings"
"testing"
"github.com/OpenListTeam/OpenList/v4/internal/model"
"github.com/OpenListTeam/OpenList/v4/pkg/http_range"
"github.com/OpenListTeam/OpenList/v4/pkg/utils"
)
func TestSeedHashStreamCarriesMetadataAndHashes(t *testing.T) {
md5Hash := strings.Repeat("a", utils.MD5.Width)
sha1Hash := strings.Repeat("b", utils.SHA1.Width)
whole := utils.NewHashInfoByMap(map[*utils.HashType]string{
utils.MD5: md5Hash,
utils.SHA1: sha1Hash,
})
stream := NewSeedHashStream(&SeedRapidUploadRequest{
Name: "hello.txt",
Size: 5,
Whole: &whole,
})
if got := stream.GetName(); got != "hello.txt" {
t.Fatalf("GetName() = %q", got)
}
if got := stream.GetSize(); got != 5 {
t.Fatalf("GetSize() = %d", got)
}
if got := stream.GetHash().GetHash(utils.MD5); got != md5Hash {
t.Fatalf("MD5 = %q", got)
}
if got := stream.GetHash().GetHash(utils.SHA1); got != sha1Hash {
t.Fatalf("SHA1 = %q", got)
}
if !stream.IsForceStreamUpload() {
t.Fatal("hash-only streams must force stream upload")
}
if stream.NeedStore() {
t.Fatal("hash-only streams must not be stored")
}
}
func TestSeedHashStreamNeverExposesContent(t *testing.T) {
whole := utils.NewHashInfo(utils.MD5, strings.Repeat("c", utils.MD5.Width))
stream := NewSeedHashStream(&SeedRapidUploadRequest{
Name: "hash-only.bin",
Size: 1,
Whole: &whole,
})
if n, err := stream.Read(make([]byte, 8)); n != 0 || !errors.Is(err, io.EOF) {
t.Fatalf("Read() = (%d, %v)", n, err)
}
if _, err := stream.RangeRead(http_range.Range{Start: 0, Length: 1}); !errors.Is(err, io.EOF) {
t.Fatalf("RangeRead() error = %v", err)
}
if _, err := stream.CacheFullAndWriter(nil, nil); !errors.Is(err, io.EOF) {
t.Fatalf("CacheFullAndWriter() error = %v", err)
}
}
func TestSeedHashStreamKeepsLazyContentSource(t *testing.T) {
called := false
stream := NewSeedHashStream(&SeedRapidUploadRequest{
Name: "with-source.bin",
Size: 8,
Whole: &utils.HashInfo{},
Open: func() (model.FileStreamer, error) {
called = true
return nil, errors.New("content unavailable")
},
})
if stream.Source == nil {
t.Fatal("Source must be preserved for drivers that need a proof hash")
}
if _, err := stream.Source(); err == nil {
t.Fatal("expected the content source error to propagate")
}
if !called {
t.Fatal("Source was not invoked")
}
}
internal/errs/driver.go
+12
View File
@@ -4,4 +4,16 @@ import "errors"
var (
EmptyToken = errors.New("empty token")
// ErrUnavailableHash indicates the seed does not carry the hash algorithm
// required by the destination driver, so rapid upload cannot be attempted.
ErrUnavailableHash = errors.New("required hash is unavailable")
// ErrEmptyHash indicates a required hash exists but is empty/too short.
ErrEmptyHash = errors.New("empty hash")
// ErrHashMismatch indicates the remote side rejected the provided hash, so
// a full content transfer is required instead of a rapid upload.
ErrHashMismatch = errors.New("hash mismatch")
// ErrRapidUploadFailed indicates the driver attempted a rapid upload but
// could not confirm success.
ErrRapidUploadFailed = errors.New("rapid upload failed")
)
internal/fs/copy_move.go
+104 -9
View File
@@ -7,6 +7,7 @@ import (
"time"
"github.com/OpenListTeam/OpenList/v4/internal/conf"
"github.com/OpenListTeam/OpenList/v4/internal/driver"
"github.com/OpenListTeam/OpenList/v4/internal/errs"
"github.com/OpenListTeam/OpenList/v4/internal/model"
"github.com/OpenListTeam/OpenList/v4/internal/op"
@@ -15,7 +16,9 @@ import (
"github.com/OpenListTeam/OpenList/v4/internal/task_group"
"github.com/OpenListTeam/OpenList/v4/pkg/utils"
"github.com/OpenListTeam/tache"
"github.com/google/uuid"
"github.com/pkg/errors"
log "github.com/sirupsen/logrus"
)
type taskType uint8
@@ -99,7 +102,79 @@ func (t *FileTransferTask) SetRetry(retry int, maxRetry int) {
}
}
func transfer(ctx context.Context, taskType taskType, srcObjPath, dstDirPath string, skipHook ...bool) (task.TaskExtensionInfo, error) {
func supportsNamedNativeCopy(storage driver.Driver) bool {
_, hasMkdir := storage.(driver.Mkdir)
if !hasMkdir {
_, hasMkdir = storage.(driver.MkdirResult)
}
_, hasCopy := storage.(driver.Copy)
if !hasCopy {
_, hasCopy = storage.(driver.CopyResult)
}
_, hasMove := storage.(driver.Move)
if !hasMove {
_, hasMove = storage.(driver.MoveResult)
}
_, hasRename := storage.(driver.Rename)
if !hasRename {
_, hasRename = storage.(driver.RenameResult)
}
_, hasRemove := storage.(driver.Remove)
return hasMkdir && hasCopy && hasMove && hasRename && hasRemove
}
func namedCopyStageObjectPath(stageDirPath, srcPath, dstName string) string {
name := stdpath.Base(srcPath)
if dstName != "" {
name = dstName
}
return stdpath.Join(stageDirPath, name)
}
func copyNamedInStorage(ctx context.Context, storage driver.Driver, srcPath, dstDirPath, dstName string) error {
stageDirPath := stdpath.Join(dstDirPath, ".openlist-copy-"+uuid.NewString())
stageCtx := context.WithValue(ctx, conf.SkipHookKey, struct{}{})
if err := op.MakeDir(stageCtx, storage, stageDirPath); err != nil {
return errors.WithMessage(err, "failed create copy staging directory")
}
stageCreated := true
removeStage := func() error {
return op.Remove(stageCtx, storage, stageDirPath)
}
defer func() {
if !stageCreated {
return
}
if err := removeStage(); err != nil {
log.WithError(err).Warnf("copy staging directory remains and requires cleanup: %s", stageDirPath)
}
}()
if err := op.Copy(stageCtx, storage, srcPath, stageDirPath); err != nil {
return errors.WithMessage(err, "failed copy object into staging directory")
}
srcName := stdpath.Base(srcPath)
stageObjPath := namedCopyStageObjectPath(stageDirPath, srcPath, "")
if srcName != dstName {
if err := op.Rename(stageCtx, storage, stageObjPath, dstName); err != nil {
return errors.WithMessage(err, "failed rename object in staging directory")
}
stageObjPath = namedCopyStageObjectPath(stageDirPath, srcPath, dstName)
}
// Staging operations suppress hooks so intermediate paths stay invisible.
// The final move intentionally uses the caller context to publish the target
// and trigger the normal cache/update hooks exactly once.
if err := op.Move(ctx, storage, stageObjPath, dstDirPath); err != nil {
return errors.WithMessage(err, "failed move copied object from staging directory")
}
if err := removeStage(); err != nil {
return errors.WithMessagef(err, "failed remove copy staging directory %s; cleanup will be retried", stageDirPath)
}
stageCreated = false
return nil
}
func transfer(ctx context.Context, taskType taskType, srcObjPath, dstDirPath, dstName string, skipHook ...bool) (task.TaskExtensionInfo, error) {
srcStorage, srcObjActualPath, err := op.GetStorageAndActualPath(srcObjPath)
if err != nil {
return nil, errors.WithMessage(err, "failed get src storage")
@@ -114,9 +189,18 @@ func transfer(ctx context.Context, taskType taskType, srcObjPath, dstDirPath str
ctx = context.WithValue(ctx, conf.SkipHookKey, struct{}{})
}
if taskType == copy || taskType == merge {
err = op.Copy(ctx, srcStorage, srcObjActualPath, dstDirActualPath)
if !errors.Is(err, errs.NotImplement) && !errors.Is(err, errs.NotSupport) {
return nil, err
if dstName != "" {
if supportsNamedNativeCopy(srcStorage) {
err = copyNamedInStorage(ctx, srcStorage, srcObjActualPath, dstDirActualPath, dstName)
if !errors.Is(err, errs.NotImplement) && !errors.Is(err, errs.NotSupport) {
return nil, err
}
}
} else {
err = op.Copy(ctx, srcStorage, srcObjActualPath, dstDirActualPath)
if !errors.Is(err, errs.NotImplement) && !errors.Is(err, errs.NotSupport) {
return nil, err
}
}
} else {
err = op.Move(ctx, srcStorage, srcObjActualPath, dstDirActualPath)
@@ -126,13 +210,15 @@ func transfer(ctx context.Context, taskType taskType, srcObjPath, dstDirPath str
}
}
// not in the same storage
// Use the transfer path when native copy cannot preserve the destination
// name or the source and destination storages differ.
t := &FileTransferTask{
TaskData: TaskData{
SrcStorage: srcStorage,
DstStorage: dstStorage,
SrcActualPath: srcObjActualPath,
DstActualPath: dstDirActualPath,
DstName: dstName,
SrcStorageMp: srcStorage.GetStorage().MountPath,
DstStorageMp: dstStorage.GetStorage().MountPath,
},
@@ -188,9 +274,14 @@ func (t *FileTransferTask) RunWithNextTaskCallback(f func(nextTask *FileTransfer
if err != nil {
return errors.WithMessagef(err, "failed list src [%s] objs", t.SrcActualPath)
}
dstActualPath := stdpath.Join(t.DstActualPath, srcObj.GetName())
task_group.TransferCoordinator.AppendPayload(t.groupID, task_group.DstPathToHook(dstActualPath))
dstName := srcObj.GetName()
if t.DstName != "" {
dstName = t.DstName
}
dstActualPath := stdpath.Join(t.DstActualPath, dstName)
if err := op.MakeDir(t.Ctx(), t.DstStorage, dstActualPath); err != nil {
return errors.WithMessagef(err, "failed create dst dir [%s]", dstActualPath)
}
existedObjs := make(map[string]bool)
if t.TaskType == merge {
dstObjs, err := op.List(t.Ctx(), t.DstStorage, dstActualPath, model.ListArgs{})
@@ -249,8 +340,12 @@ func (t *FileTransferTask) RunWithNextTaskCallback(f func(nextTask *FileTransfer
return errors.WithMessagef(err, "failed get [%s] link", t.SrcActualPath)
}
// any link provided is seekable
streamObj := srcObj
if t.DstName != "" {
streamObj = &model.ObjWrapName{Name: t.DstName, Obj: srcObj}
}
ss, err := stream.NewSeekableStream(&stream.FileStream{
Obj: srcObj,
Obj: streamObj,
Ctx: t.Ctx(),
}, link)
if err != nil {
internal/fs/copy_move_test.go
+23
View File
@@ -0,0 +1,23 @@
package fs
import "testing"
func TestNamedCopyStageObjectPath(t *testing.T) {
tests := []struct {
name string
stageDir string
srcPath string
dstName string
want string
}{
{name: "source basename", stageDir: "/dst/.stage", srcPath: "/src/foo", want: "/dst/.stage/foo"},
{name: "requested name", stageDir: "/dst/.stage", srcPath: "/src/foo", dstName: "bar", want: "/dst/.stage/bar"},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := namedCopyStageObjectPath(tt.stageDir, tt.srcPath, tt.dstName); got != tt.want {
t.Fatalf("namedCopyStageObjectPath(%q, %q, %q) = %q, want %q", tt.stageDir, tt.srcPath, tt.dstName, got, tt.want)
}
})
}
}
internal/fs/fs.go
+12 -3
View File
@@ -69,7 +69,7 @@ func MakeDir(ctx context.Context, path string) error {
}
func Move(ctx context.Context, srcPath, dstDirPath string, skipHook ...bool) (task.TaskExtensionInfo, error) {
req, err := transfer(ctx, move, srcPath, dstDirPath, skipHook...)
req, err := transfer(ctx, move, srcPath, dstDirPath, "", skipHook...)
if err != nil {
log.Errorf("failed move %s to %s: %+v", srcPath, dstDirPath, err)
}
@@ -77,15 +77,24 @@ func Move(ctx context.Context, srcPath, dstDirPath string, skipHook ...bool) (ta
}
func Copy(ctx context.Context, srcObjPath, dstDirPath string, skipHook ...bool) (task.TaskExtensionInfo, error) {
res, err := transfer(ctx, copy, srcObjPath, dstDirPath, skipHook...)
res, err := transfer(ctx, copy, srcObjPath, dstDirPath, "", skipHook...)
if err != nil {
log.Errorf("failed copy %s to %s: %+v", srcObjPath, dstDirPath, err)
}
return res, err
}
// CopyTo copies a file or directory to dstDirPath using dstName as its name.
func CopyTo(ctx context.Context, srcObjPath, dstDirPath, dstName string, skipHook ...bool) (task.TaskExtensionInfo, error) {
res, err := transfer(ctx, copy, srcObjPath, dstDirPath, dstName, skipHook...)
if err != nil {
log.Errorf("failed copy %s to %s as %s: %+v", srcObjPath, dstDirPath, dstName, err)
}
return res, err
}
func Merge(ctx context.Context, srcObjPath, dstDirPath string, skipHook ...bool) (task.TaskExtensionInfo, error) {
res, err := transfer(ctx, merge, srcObjPath, dstDirPath, skipHook...)
res, err := transfer(ctx, merge, srcObjPath, dstDirPath, "", skipHook...)
if err != nil {
log.Errorf("failed merge %s to %s: %+v", srcObjPath, dstDirPath, err)
}
internal/fs/other.go
+1
View File
@@ -53,6 +53,7 @@ type TaskData struct {
Status string `json:"-"` //don't save status to save space
SrcActualPath string `json:"src_path"`
DstActualPath string `json:"dst_path"`
DstName string `json:"dst_name,omitempty"`
SrcStorage driver.Driver `json:"-"`
DstStorage driver.Driver `json:"-"`
SrcStorageMp string `json:"src_storage_mp"`
internal/fs/seed_generate.go
+610
View File
@@ -0,0 +1,610 @@
package fs
import (
"bytes"
"context"
"encoding/base64"
"encoding/json"
"fmt"
"io"
stdpath "path"
"slices"
"strings"
"time"
"github.com/OpenListTeam/OpenList/v4/internal/conf"
"github.com/OpenListTeam/OpenList/v4/internal/errs"
"github.com/OpenListTeam/OpenList/v4/internal/model"
"github.com/OpenListTeam/OpenList/v4/internal/op"
"github.com/OpenListTeam/OpenList/v4/internal/setting"
"github.com/OpenListTeam/OpenList/v4/internal/stream"
"github.com/OpenListTeam/OpenList/v4/internal/task"
"github.com/OpenListTeam/OpenList/v4/pkg/http_range"
"github.com/OpenListTeam/OpenList/v4/pkg/torrent"
"github.com/OpenListTeam/OpenList/v4/pkg/utils"
"github.com/OpenListTeam/OpenList/v4/server/common"
"github.com/OpenListTeam/tache"
"github.com/pkg/errors"
)
// MaxSeedGenerateSyncSize bounds synchronous seed generation (1GB). Larger
// requests are turned into an asynchronous task.
const MaxSeedGenerateSyncSize = 1 * 1024 * 1024 * 1024
// SeedGenerateNeedsAsync reports whether the given files must be generated
// asynchronously because they exceed the synchronous size limit. It resolves
// each path and sums the file sizes, returning the first error encountered.
func SeedGenerateNeedsAsync(ctx context.Context, user *model.User, paths []string) (bool, error) {
var total int64
for _, requestedPath := range paths {
fullPath, err := user.JoinPath(requestedPath)
if err != nil {
return false, err
}
meta, metaErr := op.GetNearestMeta(fullPath)
if metaErr != nil && !errors.Is(errors.Cause(metaErr), errs.MetaNotFound) {
return false, metaErr
}
if !common.CanRead(user, meta, fullPath) {
return false, errs.PermissionDenied
}
storage, actualPath, err := op.GetStorageAndActualPath(fullPath)
if err != nil {
return false, err
}
obj, err := op.Get(ctx, storage, actualPath)
if err != nil {
return false, fmt.Errorf("seed path must be a readable file: %s", requestedPath)
}
if obj.IsDir() {
return false, fmt.Errorf("seed path must be a readable file: %s", requestedPath)
}
total += obj.GetSize()
if total > MaxSeedGenerateSyncSize {
return true, nil
}
}
return false, nil
}
// SeedHashSelection controls whole-file and piece hash inclusion.
type SeedHashSelection struct {
Whole bool `json:"whole"`
Pieces bool `json:"pieces"`
}
// SeedHashMatrix controls the optional hash metadata stored in a seed.
type SeedHashMatrix struct {
MD5 SeedHashSelection `json:"md5"`
SHA1 SeedHashSelection `json:"sha1"`
SHA256 SeedHashSelection `json:"sha256"`
}
// SeedGenerateParams carries a fully-resolved seed generation request, free of
// any HTTP transport concerns so it can run synchronously or as a task.
type SeedGenerateParams struct {
Paths []string
Formats []string
Name string
Comment string
FileComments map[string]string
HashMatrix SeedHashMatrix
PieceSize int64
Trackers []string
Channels []torrent.SeedChannel
OutputPath string
IncludeShare bool
IncludeDirectSource bool
ShareFiles []string
DirectFiles []string
}
// SeedArtifact is one generated seed container.
type SeedArtifact struct {
Format string `json:"format"`
Name string `json:"name"`
FileName string `json:"file_name"`
SeedData string `json:"seed_data"`
Size int `json:"size"`
Path string `json:"path,omitempty"`
}
// DeriveSeedName derives a sensible default seed name from the source paths:
// single selection uses the file name, multi selection uses the common base
// name (ignoring extensions) when all files share one, otherwise the folder name.
func DeriveSeedName(paths []string) string {
if len(paths) == 0 {
return "OpenList Seed"
}
if len(paths) == 1 {
return seedBaseName(paths[0])
}
// Common base name ignoring extensions (e.g. a.docx + a.exe -> "a").
commonBase := seedBaseName(paths[0])
for _, p := range paths[1:] {
if base := seedBaseName(p); base != commonBase {
commonBase = ""
break
}
}
if commonBase != "" {
return commonBase
}
// Fall back to the common parent directory name.
dir := commonParentDir(paths)
if base := stdpath.Base(dir); base != "" && base != "/" && base != "." {
return base
}
return "OpenList Seed"
}
// seedBaseName returns the file name without its extension.
func seedBaseName(p string) string {
base := stdpath.Base(p)
return strings.TrimSuffix(base, stdpath.Ext(base))
}
// commonParentDir returns the longest common parent directory of the given paths.
func commonParentDir(paths []string) string {
if len(paths) == 0 {
return "/"
}
parts := strings.Split(strings.Trim(stdpath.Dir(paths[0]), "/"), "/")
for _, p := range paths[1:] {
cur := strings.Split(strings.Trim(stdpath.Dir(p), "/"), "/")
n := 0
for n < len(parts) && n < len(cur) && parts[n] == cur[n] {
n++
}
parts = parts[:n]
}
if len(parts) == 0 {
return "/"
}
return "/" + strings.Join(parts, "/")
}
// NormalizeSeedFormats validates and deduplicates a list of seed format names.
func NormalizeSeedFormats(rawFormats []string) ([]string, error) {
formats := append([]string(nil), rawFormats...)
if len(formats) == 1 && strings.TrimSpace(formats[0]) == "" {
formats[0] = setting.GetStr(conf.SeedDefaultFormat, "oss")
}
if len(formats) > 3 {
return nil, fmt.Errorf("at most three seed formats may be generated")
}
result := make([]string, 0, len(formats))
seen := make(map[string]struct{}, len(formats))
for _, rawFormat := range formats {
format := strings.ToLower(strings.TrimPrefix(strings.TrimSpace(rawFormat), "."))
if format == "bt" {
format = "torrent"
}
if format != "oss" && format != "torrent" && format != "cas" {
return nil, fmt.Errorf("unsupported seed format %q", rawFormat)
}
if _, exists := seen[format]; exists {
continue
}
seen[format] = struct{}{}
result = append(result, format)
}
return result, nil
}
func seedFormats(params SeedGenerateParams) ([]string, error) {
formats, err := NormalizeSeedFormats(params.Formats)
if err != nil {
return nil, err
}
if len(formats) == 0 {
formats = []string{setting.GetStr(conf.SeedDefaultFormat, "oss")}
}
return formats, nil
}
func seedMatrixEmpty(matrix SeedHashMatrix) bool {
return !matrix.MD5.Whole && !matrix.MD5.Pieces && !matrix.SHA1.Whole && !matrix.SHA1.Pieces && !matrix.SHA256.Whole && !matrix.SHA256.Pieces
}
func loadSeedDefaultMatrix() SeedHashMatrix {
var matrix SeedHashMatrix
if raw := strings.TrimSpace(setting.GetStr(conf.SeedDefaultMatrix)); raw != "" {
_ = json.Unmarshal([]byte(raw), &matrix)
}
return matrix
}
func normalizedSeedMatrix(matrix SeedHashMatrix, formats []string) SeedHashMatrix {
if seedMatrixEmpty(matrix) {
matrix = loadSeedDefaultMatrix()
}
if seedMatrixEmpty(matrix) {
matrix = SeedHashMatrix{
MD5: SeedHashSelection{Whole: true, Pieces: true}, SHA1: SeedHashSelection{Whole: true, Pieces: true},
SHA256: SeedHashSelection{Whole: true, Pieces: true},
}
}
for _, format := range formats {
switch format {
case "torrent":
matrix.SHA1 = SeedHashSelection{Whole: true, Pieces: true}
case "cas":
matrix.MD5 = SeedHashSelection{Whole: true, Pieces: true}
}
}
return matrix
}
func canReuseListedHashes(hashInfo utils.HashInfo, matrix SeedHashMatrix) bool {
if matrix.MD5.Pieces || matrix.SHA1.Pieces || matrix.SHA256.Pieces {
return false
}
if matrix.MD5.Whole && hashInfo.GetHash(utils.MD5) == "" {
return false
}
if matrix.SHA1.Whole && hashInfo.GetHash(utils.SHA1) == "" {
return false
}
if matrix.SHA256.Whole && hashInfo.GetHash(utils.SHA256) == "" {
return false
}
return true
}
func applySeedMatrix(file *torrent.SeedFile, matrix SeedHashMatrix) {
if !matrix.MD5.Whole {
file.Hashes.MD5 = ""
}
if !matrix.SHA1.Whole {
file.Hashes.SHA1 = ""
}
if !matrix.SHA256.Whole {
file.Hashes.SHA256 = ""
}
if file.Hashes.Pieces == nil {
return
}
if !matrix.MD5.Pieces {
file.Hashes.Pieces.MD5 = nil
}
if !matrix.SHA1.Pieces {
file.Hashes.Pieces.SHA1 = nil
}
if !matrix.SHA256.Pieces {
file.Hashes.Pieces.SHA256 = nil
}
if len(file.Hashes.Pieces.MD5) == 0 && len(file.Hashes.Pieces.SHA1) == 0 && len(file.Hashes.Pieces.SHA256) == 0 {
file.Hashes.Pieces = nil
}
}
// EncodeGeneratedSeed serializes a seed in the requested container format.
func EncodeGeneratedSeed(seed *torrent.Seed, format string, standardPieces []byte) ([]byte, error) {
if format != "torrent" {
return torrent.EncodeSeed(seed, format)
}
t := &torrent.Torrent{
Info: torrent.TorrentInfo{Name: seed.Name, PieceLength: seed.PieceSize, Pieces: standardPieces},
Comment: seed.Comment,
CreatedBy: seed.CreatedBy,
CreationDate: time.Now().Unix(),
OpenList: seed,
}
if len(seed.Trackers) > 0 {
t.Announce = seed.Trackers[0]
for _, tracker := range seed.Trackers {
t.AnnounceList = append(t.AnnounceList, []string{tracker})
}
}
if len(seed.Files) == 1 {
file := seed.Files[0]
t.Info.Name = stdpath.Base(file.Path)
t.Info.Length = file.Size
t.Info.MD5Sum = file.Hashes.MD5
casCloud := file.CASCloud
if casCloud == "" {
casCloud = torrent.Cloud189
}
if file.CASSliceMD5 != "" {
t.SetCASInfo(&torrent.CASInfo{
FileMD5: strings.ToUpper(file.Hashes.MD5), SliceMD5: strings.ToUpper(file.CASSliceMD5),
SliceSize: torrent.DefaultPieceSize, Cloud: casCloud,
})
} else if seed.PieceSize == torrent.DefaultPieceSize && file.Hashes.Pieces != nil && len(file.Hashes.Pieces.MD5) > 0 {
t.SetCASInfo(torrent.BuildCASInfoFromMD5sWithCloud(file.Hashes.MD5, file.Hashes.Pieces.MD5, torrent.DefaultPieceSize, casCloud))
}
} else {
for _, file := range seed.Files {
t.Info.Files = append(t.Info.Files, torrent.TorrentFile{Length: file.Size, Path: strings.Split(file.Path, "/"), MD5Sum: file.Hashes.MD5})
}
}
return t.Encode()
}
// GenerateSeedArtifacts reads each file once while computing the requested
// hashes, then emits one or more seed containers. It has no HTTP dependency and
// can be driven synchronously or from a background task.
func GenerateSeedArtifacts(ctx context.Context, user *model.User, params SeedGenerateParams) ([]SeedArtifact, *torrent.Seed, error) {
if len(params.Paths) == 0 || len(params.Paths) > torrent.DefaultMaxSeedFiles {
return nil, nil, fmt.Errorf("invalid seed file count")
}
formats, err := seedFormats(params)
if err != nil {
return nil, nil, err
}
matrix := normalizedSeedMatrix(params.HashMatrix, formats)
pieceSize := params.PieceSize
if pieceSize <= 0 {
pieceSize = torrent.DefaultPieceSize
}
if slices.Contains(formats, "cas") {
pieceSize = torrent.DefaultPieceSize
}
seedName := strings.TrimSpace(params.Name)
if seedName == "" {
seedName = DeriveSeedName(params.Paths)
}
seed := torrent.NewSeed(seedName, "OpenList", pieceSize)
seed.Comment = params.Comment
seed.Trackers = params.Trackers
seed.Channels = params.Channels
shareSet := make(map[string]bool, len(params.ShareFiles))
for _, p := range params.ShareFiles {
if strings.TrimSpace(p) != "" {
shareSet[p] = true
}
}
directSet := make(map[string]bool, len(params.DirectFiles))
for _, p := range params.DirectFiles {
if strings.TrimSpace(p) != "" {
directSet[p] = true
}
}
useGlobalShare := len(shareSet) == 0 && params.IncludeShare
useGlobalDirect := len(directSet) == 0 && params.IncludeDirectSource
hasShare := useGlobalShare || len(shareSet) > 0
hasDirect := useGlobalDirect || len(directSet) > 0
if hasShare && !user.CanShare() {
return nil, nil, errs.PermissionDenied
}
if hasDirect && setting.GetBool(conf.SignAll) && !hasShare {
return nil, nil, fmt.Errorf("direct sources require an automatic share when global signing is enabled")
}
if (hasShare || hasDirect) && strings.TrimSpace(setting.GetStr(conf.SeedSiteURL)) == "" {
return nil, nil, fmt.Errorf("seed_site_url must be configured before embedding download sources")
}
globalHasher := torrent.NewHashWriter(pieceSize, pieceSize, 0)
fullPaths := make([]string, 0, len(params.Paths))
var total int64
for _, requestedPath := range params.Paths {
fullPath, err := user.JoinPath(requestedPath)
if err != nil {
return nil, nil, err
}
meta, err := op.GetNearestMeta(fullPath)
if err != nil && !errors.Is(errors.Cause(err), errs.MetaNotFound) {
return nil, nil, err
}
if !common.CanRead(user, meta, fullPath) {
return nil, nil, errs.PermissionDenied
}
storage, actualPath, err := op.GetStorageAndActualPath(fullPath)
if err != nil {
return nil, nil, err
}
obj, err := op.Get(ctx, storage, actualPath)
if err != nil || obj.IsDir() {
return nil, nil, fmt.Errorf("seed path must be a readable file: %s", requestedPath)
}
total += obj.GetSize()
modified := ""
if !obj.ModTime().IsZero() {
modified = obj.ModTime().UTC().Format(time.RFC3339)
}
seedPath := stdpath.Base(requestedPath)
if len(params.Paths) > 1 {
seedPath = strings.TrimPrefix(stdpath.Clean(requestedPath), "/")
}
hashInfo := obj.GetHash()
if canReuseListedHashes(hashInfo, matrix) {
seedFile := torrent.SeedFile{
Path: seedPath,
Size: obj.GetSize(),
Modified: modified,
Hashes: torrent.SeedHashes{
MD5: strings.ToLower(hashInfo.GetHash(utils.MD5)),
SHA1: strings.ToLower(hashInfo.GetHash(utils.SHA1)),
SHA256: strings.ToLower(hashInfo.GetHash(utils.SHA256)),
},
}
applySeedMatrix(&seedFile, matrix)
if comment := strings.TrimSpace(params.FileComments[requestedPath]); comment != "" {
seedFile.Comment = comment
} else if comment := strings.TrimSpace(params.FileComments[obj.GetName()]); comment != "" {
seedFile.Comment = comment
}
if useGlobalDirect || directSet[requestedPath] {
baseURL := strings.TrimRight(setting.GetStr(conf.SeedSiteURL), "/")
seedFile.Sources = []torrent.SeedSource{{Type: "openlist-direct", URL: baseURL + utils.EncodePath("/d"+fullPath)}}
}
seed.Files = append(seed.Files, seedFile)
fullPaths = append(fullPaths, fullPath)
continue
}
link, _, err := op.Link(ctx, storage, actualPath, model.LinkArgs{})
if err != nil {
return nil, nil, fmt.Errorf("storage cannot stream %s: %v", requestedPath, err)
}
rangeReader, err := stream.GetRangeReaderFromLink(obj.GetSize(), link)
if err != nil {
return nil, nil, fmt.Errorf("storage cannot stream %s", requestedPath)
}
rc, err := rangeReader.RangeRead(ctx, http_range.Range{Length: obj.GetSize()})
if err != nil {
return nil, nil, err
}
fileHasher := torrent.NewHashWriter(pieceSize, pieceSize, 0)
n, copyErr := io.Copy(io.MultiWriter(globalHasher, fileHasher), rc)
_ = rc.Close()
if copyErr != nil {
return nil, nil, fmt.Errorf("read %s: %w", requestedPath, copyErr)
}
if n != obj.GetSize() {
return nil, nil, fmt.Errorf("read %s: got %d of %d bytes", requestedPath, n, obj.GetSize())
}
fileHasher.Finish()
seedFile := fileHasher.BuildSeedFile(seedPath, modified)
applySeedMatrix(&seedFile, matrix)
if comment := strings.TrimSpace(params.FileComments[requestedPath]); comment != "" {
seedFile.Comment = comment
} else if comment := strings.TrimSpace(params.FileComments[obj.GetName()]); comment != "" {
seedFile.Comment = comment
}
if useGlobalDirect || directSet[requestedPath] {
baseURL := strings.TrimRight(setting.GetStr(conf.SeedSiteURL), "/")
seedFile.Sources = []torrent.SeedSource{{Type: "openlist-direct", URL: baseURL + utils.EncodePath("/d"+fullPath)}}
}
seed.Files = append(seed.Files, seedFile)
fullPaths = append(fullPaths, fullPath)
}
globalHasher.Finish()
createdShares := make([]string, 0, len(seed.Files))
keepCreatedShares := false
defer func() {
if !keepCreatedShares {
for _, createdID := range createdShares {
_ = op.DeleteSharing(createdID)
}
}
}()
if hasShare {
for index, fullPath := range fullPaths {
if !useGlobalShare && !shareSet[params.Paths[index]] {
continue
}
sharing := &model.Sharing{
SharingDB: &model.SharingDB{Remark: "Transfer seed source"},
Files: []string{fullPath}, Creator: user,
}
shareID, createErr := op.CreateSharing(sharing)
if createErr != nil {
return nil, nil, fmt.Errorf("create seed share: %w", createErr)
}
createdShares = append(createdShares, shareID)
seed.Files[index].Sources = []torrent.SeedSource{{
Type: "openlist-share", URL: strings.TrimRight(setting.GetStr(conf.SeedSiteURL), "/") + "/sd/" + shareID,
ShareID: shareID,
}}
}
}
if err := torrent.ValidateSeed(seed, torrent.DefaultParseLimits()); err != nil {
return nil, nil, err
}
outputPath := strings.TrimSpace(params.OutputPath)
artifacts := make([]SeedArtifact, 0, len(formats))
seenFormats := make(map[string]struct{}, len(formats))
// writeArtifact persists the encoded container into the destination folder
// when an output path is configured, returning the fully-populated artifact.
writeArtifact := func(format, fileName string, data []byte) (SeedArtifact, error) {
artifact := SeedArtifact{
Format: format,
Name: fileName,
FileName: fileName,
SeedData: base64.StdEncoding.EncodeToString(data),
Size: len(data),
}
if outputPath == "" {
return artifact, nil
}
dstDir, err := user.JoinPath(outputPath)
if err != nil {
return artifact, err
}
meta, metaErr := op.GetNearestMeta(dstDir)
if metaErr != nil && !errors.Is(errors.Cause(metaErr), errs.MetaNotFound) {
return artifact, metaErr
}
if (!user.CanWriteContent() && !common.CanWriteContentBypassUserPerms(meta, dstDir)) || !common.CanWrite(user, meta, dstDir) {
return artifact, errs.PermissionDenied
}
fileStream := &stream.FileStream{
Ctx: ctx,
Obj: &model.Object{Name: fileName, Size: int64(len(data)), Modified: time.Now()},
Reader: bytes.NewReader(data), Mimetype: "application/octet-stream",
}
if err = PutDirectly(ctx, dstDir, fileStream); err != nil {
return artifact, err
}
artifact.Path = stdpath.Join(outputPath, fileName)
return artifact, nil
}
for _, requestedFormat := range formats {
format := strings.ToLower(strings.TrimPrefix(strings.TrimSpace(requestedFormat), "."))
if format == "bt" {
format = "torrent"
}
if _, exists := seenFormats[format]; exists {
continue
}
seenFormats[format] = struct{}{}
data, err := EncodeGeneratedSeed(seed, format, globalHasher.GetPieceHashes())
if err != nil {
return nil, nil, fmt.Errorf("generate %s seed: %w", format, err)
}
fileName := stdpath.Base(seed.Name) + "." + format
artifact, err := writeArtifact(format, fileName, data)
if err != nil {
return nil, nil, err
}
artifacts = append(artifacts, artifact)
}
keepCreatedShares = true
return artifacts, seed, nil
}
// SeedGenerateTask generates seed containers asynchronously, writing them into
// params.OutputPath so they appear in the target folder once done.
type SeedGenerateTask struct {
task.TaskExtension
params SeedGenerateParams
}
func (t *SeedGenerateTask) GetName() string {
if len(t.params.Paths) == 1 {
return fmt.Sprintf("generate seed for %s", stdpath.Base(t.params.Paths[0]))
}
return fmt.Sprintf("generate seed for %d files", len(t.params.Paths))
}
func (t *SeedGenerateTask) GetStatus() string {
return "generating seed"
}
func (t *SeedGenerateTask) Run() error {
t.ClearEndTime()
t.SetStartTime(time.Now())
defer func() { t.SetEndTime(time.Now()) }()
_, _, err := GenerateSeedArtifacts(t.Ctx(), t.Creator, t.params)
return err
}
var SeedGenerateTaskManager *tache.Manager[*SeedGenerateTask]
// AddSeedGenerateTask schedules an asynchronous seed generation task.
func AddSeedGenerateTask(ctx context.Context, user *model.User, params SeedGenerateParams) (task.TaskExtensionInfo, error) {
t := &SeedGenerateTask{
TaskExtension: task.TaskExtension{
Creator: user,
ApiUrl: common.GetApiUrl(ctx),
},
params: params,
}
SeedGenerateTaskManager.Add(t)
return t, nil
}
internal/media/appkey.go
+120
View File
@@ -0,0 +1,120 @@
package media
import (
"crypto/aes"
"crypto/cipher"
"crypto/md5"
"crypto/rand"
"crypto/sha256"
"encoding/base64"
"encoding/hex"
"errors"
"strings"
"github.com/OpenListTeam/OpenList/v4/internal/conf"
"github.com/OpenListTeam/OpenList/v4/internal/setting"
)
const appKeyPrefixLen = 8
// AppKeySecretLen 明文密钥长度（不含前缀分隔符）
const appKeySecretLen = 32
// NewAppKeyPlain 生成一个新的应用密钥明文，格式：olmp_<40 位随机串>
func NewAppKeyPlain() (string, error) {
buf := make([]byte, appKeySecretLen)
if _, err := rand.Read(buf); err != nil {
return "", err
}
body := strings.TrimRight(base64.RawURLEncoding.EncodeToString(buf), "=")
if len(body) > 40 {
body = body[:40]
}
return "olmp_" + body, nil
}
// AppKeyPrefix 返回用于检索/展示的明文前缀
func AppKeyPrefix(plain string) string {
if len(plain) <= appKeyPrefixLen {
return plain
}
return plain[:appKeyPrefixLen]
}
func appKeyCipherKey() []byte {
seed := conf.Conf.JwtSecret
if seed == "" {
seed = setting.GetStr(conf.Token)
}
sum := sha256.Sum256([]byte("openlist-media-appkey:" + seed))
return sum[:]
}
func newGCM() (cipher.AEAD, error) {
block, err := aes.NewCipher(appKeyCipherKey())
if err != nil {
return nil, err
}
return cipher.NewGCM(block)
}
// EncryptAppKey 加密应用密钥明文
func EncryptAppKey(plain string) (string, error) {
gcm, err := newGCM()
if err != nil {
return "", err
}
nonce := make([]byte, gcm.NonceSize())
if _, err := rand.Read(nonce); err != nil {
return "", err
}
out := gcm.Seal(nonce, nonce, []byte(plain), nil)
return base64.RawURLEncoding.EncodeToString(out), nil
}
// DecryptAppKey 解密应用密钥明文
func DecryptAppKey(secret string) (string, error) {
raw, err := base64.RawURLEncoding.DecodeString(secret)
if err != nil {
return "", err
}
gcm, err := newGCM()
if err != nil {
return "", err
}
if len(raw) < gcm.NonceSize() {
return "", errors.New("invalid app key secret")
}
nonce, data := raw[:gcm.NonceSize()], raw[gcm.NonceSize():]
plain, err := gcm.Open(nil, nonce, data, nil)
if err != nil {
return "", err
}
return string(plain), nil
}
// HashToken 计算设备令牌的存储哈希
func HashToken(token string) string {
sum := sha256.Sum256([]byte("openlist-emby-token:" + token))
return hex.EncodeToString(sum[:])
}
// NewDeviceToken 生成 Emby 设备令牌
func NewDeviceToken() (string, error) {
buf := make([]byte, 24)
if _, err := rand.Read(buf); err != nil {
return "", err
}
return hex.EncodeToString(buf), nil
}
// SubsonicToken 计算 Subsonic 的 t = md5(password + salt)
func SubsonicToken(password, salt string) string {
sum := md5.Sum([]byte(password + salt))
return hex.EncodeToString(sum[:])
}
// MatchSubsonicToken 校验客户端提交的 t 是否匹配
func MatchSubsonicToken(password, salt, token string) bool {
return strings.EqualFold(SubsonicToken(password, salt), token)
}
internal/media/id3.go
+684
View File
@@ -0,0 +1,684 @@
package media
import (
"encoding/binary"
"io"
"strconv"
"strings"
"github.com/OpenListTeam/OpenList/v4/internal/media/imgmime"
)
// maxEmbeddedPictureBytes 内嵌封面图片最大字节数，防止畸形文件触发超大内存分配
const maxEmbeddedPictureBytes = 16 << 20
// MusicTag 音频文件标签信息
type MusicTag struct {
Title string // TIT2
Artist string // TPE1
Album string // TALB
AlbumArtist string // TPE2
TrackNumber int // TRCK
Year string // TYER / TDRC
Genre string // TCON
CoverData []byte // 封面图片原始字节（APIC / PICTURE）
CoverMIME string // 封面图片 MIME 类型（如 image/jpeg）
Lyrics string // 歌词内容（LRC 文本或纯文本）：来自 USLT / SYLT / Vorbis LYRICS
}
// ParseID3v2 从 io.Reader 中解析 ID3v2 标签（只读取文件头部，不需要 Seek）
// 支持 ID3v2.3 和 ID3v2.4
func ParseID3v2(r io.Reader) (*MusicTag, error) {
// 读取 ID3v2 头部（10 字节）
header := make([]byte, 10)
if _, err := io.ReadFull(r, header); err != nil {
return nil, err
}
// 检查魔数
if string(header[0:3]) != "ID3" {
return nil, nil // 不是 ID3v2 文件，不报错
}
version := header[3] // 主版本号：3 = ID3v2.3, 4 = ID3v2.4
if version < 3 || version > 4 {
return nil, nil // 只支持 v2.3 和 v2.4
}
// 解析标签总大小（syncsafe integer，4 字节，每字节最高位为 0）
tagSize := syncsafeToInt(header[6:10])
if tagSize <= 0 || tagSize > 10*1024*1024 { // 最大 10MB
return nil, nil
}
// 读取所有帧数据
data := make([]byte, tagSize)
if _, err := io.ReadFull(r, data); err != nil {
return nil, nil // 读取失败时静默跳过
}
tag := &MusicTag{}
pos := 0
// 跳过扩展头部（如果有）
flags := header[5]
if flags&0x40 != 0 { // 有扩展头部
if pos+4 > len(data) {
return tag, nil
}
var extSize int
if version == 4 {
extSize = syncsafeToInt(data[pos : pos+4])
} else {
extSize = int(binary.BigEndian.Uint32(data[pos : pos+4]))
}
pos += extSize
}
// 解析帧
for pos+10 <= len(data) {
frameID := string(data[pos : pos+4])
if frameID == "\x00\x00\x00\x00" {
break // 填充区域，结束
}
// ID3v2.4 帧大小使用 syncsafe integer（与标签头一致），
// ID3v2.3 使用普通 32 位大端整数。
var frameSize int
if version == 4 {
frameSize = syncsafeToInt(data[pos+4 : pos+8])
} else {
frameSize = int(binary.BigEndian.Uint32(data[pos+4 : pos+8]))
}
pos += 10 // 跳过帧头（4+4+2）
if frameSize <= 0 || pos+frameSize > len(data) {
break
}
frameData := data[pos : pos+frameSize]
pos += frameSize
// 解析文本帧（T 开头的帧）
if len(frameID) == 4 && frameID[0] == 'T' && len(frameData) > 0 {
text := decodeID3Text(frameData)
switch frameID {
case "TIT2":
tag.Title = text
case "TPE1":
tag.Artist = text
case "TALB":
tag.Album = text
case "TPE2":
tag.AlbumArtist = text
case "TRCK":
// 格式可能是 "1" 或 "1/12"
parts := strings.SplitN(text, "/", 2)
if n, err := strconv.Atoi(strings.TrimSpace(parts[0])); err == nil {
tag.TrackNumber = n
}
case "TYER", "TDRC":
if tag.Year == "" {
tag.Year = text
}
case "TCON":
tag.Genre = parseID3Genre(text)
}
}
// 解析 APIC 帧（内嵌封面图片），只取第一张
if frameID == "APIC" && tag.CoverData == nil && len(frameData) > 1 {
tag.CoverData, tag.CoverMIME = parseAPICFrame(frameData)
}
// 解析 USLT 帧（非同步歌词）
// 格式：编码字节(1) + 语言(3字节 ASCII) + 描述字符串(null 结尾) + 歌词文本
if frameID == "USLT" && tag.Lyrics == "" && len(frameData) > 5 {
tag.Lyrics = parseUSLTFrame(frameData)
}
// 解析 SYLT 帧（同步歌词，二进制时间戳格式），仅在没有 USLT 时尝试转 LRC
if frameID == "SYLT" && tag.Lyrics == "" && len(frameData) > 6 {
if lrc := parseSYLTFrame(frameData); lrc != "" {
tag.Lyrics = lrc
}
}
}
return tag, nil
}
// parseUSLTFrame 解析 ID3v2 USLT 帧（非同步歌词）
// 格式：编码字节(1) + 语言(3字节 ASCII) + 描述字符串(null 结尾) + 歌词文本
func parseUSLTFrame(data []byte) string {
if len(data) < 5 {
return ""
}
encoding := data[0]
// 跳过编码(1) + 语言(3) = 4 字节
pos := 4
// 跳过描述字符串
if encoding == 0x01 || encoding == 0x02 {
// UTF-16：找 \x00\x00 结尾
for pos+1 < len(data) {
if data[pos] == 0 && data[pos+1] == 0 {
pos += 2
break
}
pos += 2
}
} else {
// ISO-8859-1 / UTF-8：找单个 \x00 结尾
for pos < len(data) {
if data[pos] == 0 {
pos++
break
}
pos++
}
}
if pos >= len(data) {
return ""
}
// 剩余字节就是歌词文本，按编码解码（复用 decodeID3Text 但需要把 encoding 字节加回头部）
lyricsRaw := make([]byte, 1+len(data)-pos)
lyricsRaw[0] = encoding
copy(lyricsRaw[1:], data[pos:])
return strings.TrimSpace(decodeID3Text(lyricsRaw))
}
// parseSYLTFrame 解析 ID3v2 SYLT 帧（同步歌词），转换为 LRC 格式
// 格式：编码(1) + 语言(3) + 时间戳格式(1) + 内容类型(1) + 描述(null 结尾) + 多组 (歌词文本 + null + 4字节时间戳)
// 时间戳格式：1=MPEG帧数（不支持），2=毫秒
func parseSYLTFrame(data []byte) string {
if len(data) < 6 {
return ""
}
encoding := data[0]
timestampFormat := data[4] // 1=MPEG帧, 2=ms
if timestampFormat != 2 {
return "" // 只支持毫秒时间戳
}
pos := 6 // 跳过 encoding(1) + lang(3) + timestamp_fmt(1) + content_type(1)
// 跳过描述字符串
if encoding == 0x01 || encoding == 0x02 {
for pos+1 < len(data) {
if data[pos] == 0 && data[pos+1] == 0 {
pos += 2
break
}
pos += 2
}
} else {
for pos < len(data) {
if data[pos] == 0 {
pos++
break
}
pos++
}
}
var sb strings.Builder
wide := encoding == 0x01 || encoding == 0x02
for pos < len(data) {
// 找文本终止符
textStart := pos
textEnd := -1
if wide {
for i := pos; i+1 < len(data); i += 2 {
if data[i] == 0 && data[i+1] == 0 {
textEnd = i
break
}
}
} else {
for i := pos; i < len(data); i++ {
if data[i] == 0 {
textEnd = i
break
}
}
}
if textEnd < 0 {
break
}
// 解码歌词文本
lineRaw := make([]byte, 1+textEnd-textStart)
lineRaw[0] = encoding
copy(lineRaw[1:], data[textStart:textEnd])
lineText := decodeID3Text(lineRaw)
// 跳过文本终止符
if wide {
pos = textEnd + 2
} else {
pos = textEnd + 1
}
// 4 字节时间戳（毫秒）
if pos+4 > len(data) {
break
}
ms := int(binary.BigEndian.Uint32(data[pos : pos+4]))
pos += 4
minutes := ms / 60000
seconds := (ms % 60000) / 1000
cs := (ms % 1000) / 10
// LRC 标准格式 [mm:ss.xx]
sb.WriteString("[")
if minutes < 10 {
sb.WriteString("0")
}
sb.WriteString(strconv.Itoa(minutes))
sb.WriteString(":")
if seconds < 10 {
sb.WriteString("0")
}
sb.WriteString(strconv.Itoa(seconds))
sb.WriteString(".")
if cs < 10 {
sb.WriteString("0")
}
sb.WriteString(strconv.Itoa(cs))
sb.WriteString("]")
sb.WriteString(lineText)
sb.WriteString("\n")
}
return sb.String()
}
// parseAPICFrame 解析 ID3v2 APIC 帧，返回图片字节和 MIME 类型
// APIC 格式：编码字节(1) + MIME字符串(null结尾) + 图片类型(1) + 描述字符串(null结尾) + 图片数据
func parseAPICFrame(data []byte) ([]byte, string) {
if len(data) < 4 {
return nil, ""
}
// encoding := data[0] // 文本编码（只影响描述字段，MIME 始终是 ASCII）
pos := 1
// 读取 MIME 类型（null 结尾的 ASCII 字符串）
nullIdx := -1
for i := pos; i < len(data); i++ {
if data[i] == 0 {
nullIdx = i
break
}
}
if nullIdx < 0 {
return nil, ""
}
mimeType := string(data[pos:nullIdx])
pos = nullIdx + 1
if pos >= len(data) {
return nil, ""
}
// 图片类型（1字节，3 = Cover (front)，但我们取任意第一张）
pos++ // 跳过图片类型字节
// 跳过描述字符串（null 结尾，编码由第一字节决定）
encoding := data[0]
if encoding == 0x01 || encoding == 0x02 {
// UTF-16：找 \x00\x00 结尾
for pos+1 < len(data) {
if data[pos] == 0 && data[pos+1] == 0 {
pos += 2
break
}
pos += 2
}
} else {
// ISO-8859-1 / UTF-8：找单个 \x00 结尾
for pos < len(data) {
if data[pos] == 0 {
pos++
break
}
pos++
}
}
if pos >= len(data) {
return nil, ""
}
imgData := make([]byte, len(data)-pos)
copy(imgData, data[pos:])
// 声明的 MIME 来自被扫描文件本身，不可信（可能是 image/svg+xml）。
// 只按魔数判定并限定为四种栅格格式，无法识别则丢弃封面。
_ = mimeType
safe := imgmime.Sniff(imgData)
if safe == "" {
return nil, ""
}
return imgData, safe
}
// syncsafeToInt 将 syncsafe integer 转换为普通整数
func syncsafeToInt(b []byte) int {
result := 0
for _, v := range b {
result = (result << 7) | int(v&0x7F)
}
return result
}
// decodeID3Text 解码 ID3 文本帧（第一字节是编码标识）
func decodeID3Text(data []byte) string {
if len(data) == 0 {
return ""
}
encoding := data[0]
content := data[1:]
// 去掉末尾的 null 字节
switch encoding {
case 0x00: // ISO-8859-1
// 去掉末尾 null
content = trimNull(content, false)
// 尝试 UTF-8 解码，如果失败则按 Latin-1 处理
return latin1ToUTF8(content)
case 0x01: // UTF-16 with BOM
content = trimNull(content, true)
return utf16ToUTF8(content)
case 0x02: // UTF-16 BE without BOM
content = trimNull(content, true)
return utf16BEToUTF8(content)
case 0x03: // UTF-8
content = trimNull(content, false)
return string(content)
default:
return string(content)
}
}
// trimNull 去掉末尾的 null 字节
func trimNull(b []byte, wide bool) []byte {
if wide {
// UTF-16: 去掉末尾的 \x00\x00
for len(b) >= 2 && b[len(b)-2] == 0 && b[len(b)-1] == 0 {
b = b[:len(b)-2]
}
} else {
for len(b) > 0 && b[len(b)-1] == 0 {
b = b[:len(b)-1]
}
}
return b
}
// latin1ToUTF8 将 Latin-1 编码转换为 UTF-8
func latin1ToUTF8(b []byte) string {
runes := make([]rune, len(b))
for i, v := range b {
runes[i] = rune(v)
}
return string(runes)
}
// utf16ToUTF8 将 UTF-16（带 BOM）转换为 UTF-8
func utf16ToUTF8(b []byte) string {
if len(b) < 2 {
return ""
}
var bigEndian bool
if b[0] == 0xFF && b[1] == 0xFE {
bigEndian = false
b = b[2:]
} else if b[0] == 0xFE && b[1] == 0xFF {
bigEndian = true
b = b[2:]
}
return decodeUTF16(b, bigEndian)
}
// utf16BEToUTF8 将 UTF-16 BE（无 BOM）转换为 UTF-8
func utf16BEToUTF8(b []byte) string {
return decodeUTF16(b, true)
}
// decodeUTF16 解码 UTF-16 字节序列
func decodeUTF16(b []byte, bigEndian bool) string {
if len(b)%2 != 0 {
b = b[:len(b)-1]
}
runes := make([]rune, 0, len(b)/2)
for i := 0; i+1 < len(b); i += 2 {
var code uint16
if bigEndian {
code = uint16(b[i])<<8 | uint16(b[i+1])
} else {
code = uint16(b[i+1])<<8 | uint16(b[i])
}
runes = append(runes, rune(code))
}
return string(runes)
}
// parseID3Genre 解析 ID3 流派字段（可能是 "(17)" 格式的数字引用）
func parseID3Genre(s string) string {
s = strings.TrimSpace(s)
if len(s) > 2 && s[0] == '(' && s[len(s)-1] == ')' {
// 数字引用格式，直接返回原始字符串
return s
}
return s
}
// ─── FLAC Vorbis Comment 解析 ────────────────────────────────────────────────
// ParseFLACVorbisComment 从 io.Reader 中解析 FLAC 文件的 Vorbis Comment 元数据
// FLAC 格式：4字节魔数 "fLaC" + 若干 METADATA_BLOCK
// METADATA_BLOCK：1字节(最高位=是否最后块, 低7位=块类型) + 3字节长度 + 数据
// 块类型 4 = VORBIS_COMMENT
func ParseFLACVorbisComment(r io.Reader) (*MusicTag, error) {
// 读取魔数（4字节）
magic := make([]byte, 4)
if _, err := io.ReadFull(r, magic); err != nil {
return nil, err
}
if string(magic) != "fLaC" {
return nil, nil // 不是 FLAC 文件
}
// 遍历 METADATA_BLOCK，找到 VORBIS_COMMENT（类型4）和 PICTURE（类型6）
var tag *MusicTag
for {
// 读取块头（4字节：1字节标志+类型 + 3字节长度）
blockHeader := make([]byte, 4)
if _, err := io.ReadFull(r, blockHeader); err != nil {
return nil, nil // 读取失败，静默跳过
}
isLast := blockHeader[0]&0x80 != 0
blockType := blockHeader[0] & 0x7F
blockLen := int(blockHeader[1])<<16 | int(blockHeader[2])<<8 | int(blockHeader[3])
if blockLen < 0 || blockLen > 16*1024*1024 { // 最大 16MB
return nil, nil
}
if blockType == 4 {
// VORBIS_COMMENT 块
data := make([]byte, blockLen)
if _, err := io.ReadFull(r, data); err != nil {
return nil, nil
}
tag = parseVorbisCommentData(data)
if isLast {
return tag, nil
}
// 继续读取后续块，寻找 PICTURE 块（类型6）
continue
}
if blockType == 6 {
// PICTURE 块（FLAC 内嵌封面）
data := make([]byte, blockLen)
if _, err := io.ReadFull(r, data); err == nil && tag != nil && tag.CoverData == nil {
tag.CoverData, tag.CoverMIME = parseFLACPictureBlock(data)
} else if err != nil {
// 读取失败，跳过
_ = err
}
if isLast {
return tag, nil
}
continue
}
// 跳过此块
if _, err := io.CopyN(io.Discard, r, int64(blockLen)); err != nil {
return nil, nil
}
if isLast {
break
}
}
return tag, nil
}
// parseFLACPictureBlock 解析 FLAC PICTURE 元数据块，返回图片字节和 MIME 类型
// 格式（大端序）：
// 4字节 picture_type
// 4字节 mime_length + mime_string
// 4字节 description_length + description_string
// 4字节 width, 4字节 height, 4字节 color_depth, 4字节 color_count
// 4字节 data_length + data
func parseFLACPictureBlock(data []byte) ([]byte, string) {
if len(data) < 8 {
return nil, ""
}
pos := 4 // 跳过 picture_type
// 读取 MIME 类型
if pos+4 > len(data) {
return nil, ""
}
mimeLen := int(binary.BigEndian.Uint32(data[pos : pos+4]))
pos += 4
if pos+mimeLen > len(data) {
return nil, ""
}
mimeType := string(data[pos : pos+mimeLen])
pos += mimeLen
// 跳过描述字符串
if pos+4 > len(data) {
return nil, ""
}
descLen := int(binary.BigEndian.Uint32(data[pos : pos+4]))
pos += 4 + descLen
// 跳过 width(4) + height(4) + color_depth(4) + color_count(4)
pos += 16
// 读取图片数据
if pos+4 > len(data) {
return nil, ""
}
dataLen := int(binary.BigEndian.Uint32(data[pos : pos+4]))
pos += 4
if dataLen <= 0 || dataLen > maxEmbeddedPictureBytes || pos+dataLen > len(data) {
return nil, ""
}
imgData := make([]byte, dataLen)
copy(imgData, data[pos:pos+dataLen])
// 同 parseAPICFrame：不信任声明的 MIME，只按魔数判定
_ = mimeType
safe := imgmime.Sniff(imgData)
if safe == "" {
return nil, ""
}
return imgData, safe
}
// parseVorbisCommentData 解析 Vorbis Comment 数据块
// 格式（小端序）：
// 4字节 vendor_length + vendor_string
// 4字节 comment_count
// 每条注释：4字节长度 + UTF-8字符串（格式 "KEY=VALUE"）
func parseVorbisCommentData(data []byte) *MusicTag {
tag := &MusicTag{}
pos := 0
// 跳过 vendor string
if pos+4 > len(data) {
return tag
}
vendorLen := int(binary.LittleEndian.Uint32(data[pos : pos+4]))
pos += 4 + vendorLen
// 读取注释数量
if pos+4 > len(data) {
return tag
}
commentCount := int(binary.LittleEndian.Uint32(data[pos : pos+4]))
pos += 4
for i := 0; i < commentCount; i++ {
if pos+4 > len(data) {
break
}
commentLen := int(binary.LittleEndian.Uint32(data[pos : pos+4]))
pos += 4
if commentLen <= 0 || pos+commentLen > len(data) {
break
}
comment := string(data[pos : pos+commentLen])
pos += commentLen
// 解析 "KEY=VALUE" 格式（大小写不敏感）
eqIdx := strings.IndexByte(comment, '=')
if eqIdx < 0 {
continue
}
key := strings.ToUpper(comment[:eqIdx])
value := strings.TrimSpace(comment[eqIdx+1:])
if value == "" {
continue
}
switch key {
case "TITLE":
tag.Title = value
case "ARTIST":
if tag.Artist == "" {
tag.Artist = value
}
case "ALBUM":
tag.Album = value
case "ALBUMARTIST", "ALBUM ARTIST":
tag.AlbumArtist = value
case "TRACKNUMBER", "TRACK":
// 格式可能是 "1" 或 "1/12"
parts := strings.SplitN(value, "/", 2)
if n, err := strconv.Atoi(strings.TrimSpace(parts[0])); err == nil {
tag.TrackNumber = n
}
case "DATE", "YEAR":
if tag.Year == "" && len(value) >= 4 {
tag.Year = value[:4]
}
case "GENRE":
tag.Genre = value
case "LYRICS", "UNSYNCEDLYRICS", "UNSYNCED LYRICS", "SYNCEDLYRICS", "SYNCED LYRICS":
// FLAC/Vorbis 内嵌歌词字段（不同打标软件使用不同 key，全部兼容）
if tag.Lyrics == "" {
tag.Lyrics = value
}
}
}
return tag
}
internal/media/imgmime/imgmime.go
+36
View File
@@ -0,0 +1,36 @@
// Package imgmime 提供封面图片 MIME 的安全判定。
//
// 封面字节与 MIME 往往来自不可信来源（音频文件内嵌 APIC/PICTURE、远端站点的
// Content-Type）。如果把声明的 MIME 原样写进响应头，就可能把 image/svg+xml 或
// text/html 以站点同源方式交付给浏览器，造成存储型 XSS。
// 这里只信任文件头魔数，且只允许四种栅格格式。
package imgmime
import "bytes"
// Sniff 根据文件头魔数返回 image/jpeg、image/png、image/gif、image/webp 之一；
// 无法识别（包括 SVG、HTML、未知格式）时返回空串，调用方应丢弃该封面。
func Sniff(data []byte) string {
switch {
case len(data) >= 3 && data[0] == 0xFF && data[1] == 0xD8 && data[2] == 0xFF:
return "image/jpeg"
case len(data) >= 8 && bytes.Equal(data[:8], []byte{0x89, 'P', 'N', 'G', 0x0D, 0x0A, 0x1A, 0x0A}):
return "image/png"
case len(data) >= 6 && (bytes.Equal(data[:6], []byte("GIF87a")) || bytes.Equal(data[:6], []byte("GIF89a"))):
return "image/gif"
case len(data) >= 12 && bytes.Equal(data[:4], []byte("RIFF")) && bytes.Equal(data[8:12], []byte("WEBP")):
return "image/webp"
default:
return ""
}
}
// Allowed 判断 MIME 是否属于允许对外输出的栅格图片类型。
func Allowed(mime string) bool {
switch mime {
case "image/jpeg", "image/png", "image/gif", "image/webp":
return true
default:
return false
}
}
internal/media/imgmime/imgmime_test.go
+41
View File
@@ -0,0 +1,41 @@
package imgmime
import "testing"
func TestSniff(t *testing.T) {
cases := []struct {
name string
data []byte
want string
}{
{"jpeg", []byte{0xFF, 0xD8, 0xFF, 0xE0, 0, 0}, "image/jpeg"},
{"png", []byte{0x89, 'P', 'N', 'G', 0x0D, 0x0A, 0x1A, 0x0A, 0}, "image/png"},
{"gif89", []byte("GIF89a....."), "image/gif"},
{"webp", append(append([]byte("RIFF\x00\x00\x00\x00"), []byte("WEBP")...), 0), "image/webp"},
{"svg", []byte(`<svg xmlns="http://www.w3.org/2000/svg" onload="alert(1)"></svg>`), ""},
{"svg with xml prolog", []byte(`<?xml version="1.0"?><svg/>`), ""},
{"html", []byte("<html><script>alert(1)</script></html>"), ""},
{"empty", nil, ""},
{"short", []byte{0xFF, 0xD8}, ""},
// RIFF 但不是 WEBP（例如 WAV）不能被当作图片
{"riff wav", append(append([]byte("RIFF\x00\x00\x00\x00"), []byte("WAVE")...), 0), ""},
}
for _, tc := range cases {
if got := Sniff(tc.data); got != tc.want {
t.Errorf("%s: Sniff = %q, want %q", tc.name, got, tc.want)
}
}
}
func TestAllowed(t *testing.T) {
for _, m := range []string{"image/jpeg", "image/png", "image/gif", "image/webp"} {
if !Allowed(m) {
t.Errorf("%s should be allowed", m)
}
}
for _, m := range []string{"image/svg+xml", "text/html", "", "image/"} {
if Allowed(m) {
t.Errorf("%q should not be allowed", m)
}
}
}
internal/media/local_fill.go
+130
View File
@@ -0,0 +1,130 @@
package media
import (
"context"
"encoding/base64"
"encoding/json"
stdpath "path"
"strings"
"time"
"github.com/OpenListTeam/OpenList/v4/internal/fs"
"github.com/OpenListTeam/OpenList/v4/internal/model"
)
// FillMusicFromLocal 在刮削前/失败时，使用音乐文件本身的元数据（ID3 / Vorbis Comment）
// 以及同目录 .lrc 歌词，**只填充 item 中为空的字段**。
//
// 适用场景：
// 1. 用户清空刮削后重新刮削，原本由扫描阶段填入的本地数据（封面/歌词等）已丢失
// 2. 在线刮削失败，退而求其次用文件本地信息补全
//
// 行为：
// - is_folder=true 的合并文件夹模式：使用 episodes 中第一首音乐文件读取
// - is_folder=false：使用 folder_path/file_name 直接读取
//
// 错误以静默忽略为主——本地兜底失败不应阻塞主流程。
func FillMusicFromLocal(ctx context.Context, item *model.MediaItem) {
if item == nil || item.MediaType != model.MediaTypeMusic {
return
}
musicPath := pickMusicFilePath(item)
if musicPath == "" {
return
}
// 解析 tag（mp3 / flac / 其它）
ext := strings.ToLower(stdpath.Ext(musicPath))
var tag *MusicTag
readCtx, readCancel := context.WithTimeout(ctx, 15*time.Second)
if reader := FetchFileReader(readCtx, musicPath); reader != nil {
switch ext {
case ".flac":
tag, _ = ParseFLACVorbisComment(reader)
default:
tag, _ = ParseID3v2(reader)
}
_ = reader.Close()
}
readCancel()
// 应用 tag 到空字段
if tag != nil {
if item.AlbumName == "" && tag.Album != "" {
item.AlbumName = tag.Album
}
if item.AlbumArtist == "" {
if tag.AlbumArtist != "" {
item.AlbumArtist = tag.AlbumArtist
} else if tag.Artist != "" {
item.AlbumArtist = tag.Artist
}
}
if item.ScrapedName == "" {
if tag.Title != "" {
item.ScrapedName = tag.Title
} else if item.AlbumName != "" {
item.ScrapedName = item.AlbumName
}
}
if item.TrackNumber == 0 && tag.TrackNumber > 0 {
item.TrackNumber = tag.TrackNumber
}
if item.ReleaseDate == "" && len(tag.Year) >= 4 {
item.ReleaseDate = tag.Year[:4] + "-01-01"
}
if item.Genre == "" && tag.Genre != "" {
item.Genre = tag.Genre
}
if item.Authors == "" && tag.Artist != "" {
if b, err := json.Marshal([]string{tag.Artist}); err == nil {
item.Authors = string(b)
}
}
if item.Cover == "" && len(tag.CoverData) > 0 {
mimeType := tag.CoverMIME
if mimeType == "" {
mimeType = "image/jpeg"
}
item.Cover = "data:" + mimeType + ";base64," + base64.StdEncoding.EncodeToString(tag.CoverData)
}
}
// 歌词：若为空则尝试 .lrc 同名文件 / 内嵌歌词
if item.Lyrics == "" {
if lrc := loadLyricsForMusic(ctx, musicPath, tag); lrc != "" {
item.Lyrics = lrc
}
}
}
// pickMusicFilePath 从 MediaItem 推断出可读取的音乐文件 VFS 路径
// 合并模式 (is_folder=true): 取 episodes 中第一首；解析失败回退到 folder_path/file_name
// 普通模式: folder_path/file_name
func pickMusicFilePath(item *model.MediaItem) string {
if item == nil {
return ""
}
if item.IsFolder {
// episodes 是 JSON 数组：[{file_name,index,title,...}]
if item.Episodes != "" {
var eps []EpisodeInfo
if err := json.Unmarshal([]byte(item.Episodes), &eps); err == nil && len(eps) > 0 {
// 文件夹模式下 folder_path 是扫描根，file_name 是文件夹名
return stdpath.Join(item.FolderPath, item.FileName, eps[0].FileName)
}
}
// 回退：尝试列出文件夹内第一首音乐
folder := stdpath.Join(item.FolderPath, item.FileName)
if entries, err := fs.List(context.Background(), folder, &fs.ListArgs{NoLog: true}); err == nil {
for _, e := range entries {
if !e.IsDir() && isMediaFile(e.GetName(), model.MediaTypeMusic) {
return stdpath.Join(folder, e.GetName())
}
}
}
return ""
}
return stdpath.Join(item.FolderPath, item.FileName)
}
internal/media/scanner.go
+758
View File
@@ -0,0 +1,758 @@
package media
import (
"context"
"encoding/base64"
"encoding/json"
"fmt"
"io"
"mime"
stdpath "path"
"regexp"
"strings"
"sync"
"time"
"unicode"
log "github.com/sirupsen/logrus"
"github.com/OpenListTeam/OpenList/v4/internal/db"
"github.com/OpenListTeam/OpenList/v4/internal/fs"
"github.com/OpenListTeam/OpenList/v4/internal/model"
"github.com/OpenListTeam/OpenList/v4/internal/stream"
"github.com/OpenListTeam/OpenList/v4/pkg/http_range"
"github.com/OpenListTeam/OpenList/v4/pkg/utils"
)
// 支持的文件扩展名
var (
videoExts = map[string]bool{
".mp4": true, ".mkv": true, ".avi": true, ".mov": true,
".wmv": true, ".flv": true, ".webm": true, ".m4v": true,
".ts": true, ".rmvb": true, ".rm": true, ".3gp": true,
}
musicExts = map[string]bool{
".mp3": true, ".flac": true, ".aac": true, ".ogg": true,
".wav": true, ".wma": true, ".m4a": true, ".ape": true,
".opus": true, ".aiff": true,
}
imageExts = map[string]bool{
".jpg": true, ".jpeg": true, ".png": true, ".gif": true,
".webp": true, ".bmp": true, ".tiff": true, ".svg": true,
".heic": true, ".avif": true,
}
bookExts = map[string]bool{
".epub": true, ".pdf": true, ".mobi": true, ".azw3": true,
".txt": true, ".djvu": true, ".cbz": true, ".cbr": true,
}
)
// ScanProgress 扫描进度（全局，按媒体类型维护）
type ScanProgress struct {
mu sync.RWMutex
Running bool
Total int
Done int
// Skipped 入库被跳过的条目数（典型原因：目录/文件名超过唯一索引列长度上限）。
// 以前只在日志里打一条 Warn，用户完全看不到漏扫了多少条。
Skipped int
Message string
Error string
}
var progressMap = map[model.MediaType]*ScanProgress{
model.MediaTypeVideo: {},
model.MediaTypeMusic: {},
model.MediaTypeImage: {},
model.MediaTypeBook: {},
}
// GetProgress 获取扫描进度
func GetProgress(mediaType model.MediaType) model.MediaScanProgress {
p, ok := progressMap[mediaType]
if !ok {
return model.MediaScanProgress{MediaType: mediaType}
}
p.mu.RLock()
defer p.mu.RUnlock()
return model.MediaScanProgress{
MediaType: mediaType,
Running: p.Running,
Total: p.Total,
Done: p.Done,
Message: p.Message,
Error: p.Error,
}
}
// ScanMedia 扫描媒体文件（异步），扫描指定媒体类型下的所有启用扫描路径
func ScanMedia(cfg *model.MediaConfig) {
p, ok := progressMap[cfg.MediaType]
if !ok {
return
}
p.mu.Lock()
if p.Running {
p.mu.Unlock()
return
}
p.Running = true
p.Total = 0
p.Done = 0
p.Skipped = 0
p.Error = ""
p.Message = "正在扫描..."
p.mu.Unlock()
go func() {
defer func() {
p.mu.Lock()
p.Running = false
p.mu.Unlock()
}()
// 获取该媒体类型下所有扫描路径
scanPaths, err := db.ListMediaScanPaths(cfg.MediaType)
if err != nil {
p.mu.Lock()
p.Error = err.Error()
p.Message = "获取扫描路径失败"
p.mu.Unlock()
return
}
if len(scanPaths) == 0 {
p.mu.Lock()
p.Message = "没有配置扫描路径"
p.mu.Unlock()
return
}
for i := range scanPaths {
sp := &scanPaths[i]
p.mu.Lock()
p.Message = "正在扫描路径: " + sp.Name
p.mu.Unlock()
if err := doScanPath(sp, p); err != nil {
log.Errorf("media scan error [%s] path[%s]: %v", cfg.MediaType, sp.Path, err)
} else {
// 更新最后扫描时间
now := time.Now()
sp.LastScanAt = &now
_ = db.UpdateMediaScanPath(sp)
}
}
p.mu.Lock()
if p.Skipped > 0 {
// 让管理端能直接看到「有 N 条因为路径过长没入库」
p.Message = fmt.Sprintf("扫描完成（跳过 %d 条）", p.Skipped)
log.Warnf("media scan skipped %d items (path/name too long for unique index)", p.Skipped)
} else {
p.Message = "扫描完成"
}
p.mu.Unlock()
// 更新媒体库最后扫描时间
now := time.Now()
cfg.LastScanAt = &now
_ = db.SaveMediaConfig(cfg)
// 按需生成 STRM
TriggerStrmOnScan(cfg.MediaType, GetAPIRoot())
}()
}
// ScanMediaPath 扫描单个扫描路径（异步）
func ScanMediaPath(sp *model.MediaScanPath) {
p, ok := progressMap[sp.MediaType]
if !ok {
return
}
p.mu.Lock()
if p.Running {
p.mu.Unlock()
return
}
p.Running = true
p.Total = 0
p.Done = 0
p.Skipped = 0
p.Error = ""
p.Message = "正在扫描路径: " + sp.Name
p.mu.Unlock()
go func() {
defer func() {
p.mu.Lock()
p.Running = false
p.mu.Unlock()
}()
if err := doScanPath(sp, p); err != nil {
p.mu.Lock()
p.Error = err.Error()
p.Message = "扫描失败"
p.mu.Unlock()
log.Errorf("media scan error [%s] path[%s]: %v", sp.MediaType, sp.Path, err)
} else {
p.mu.Lock()
if p.Skipped > 0 {
p.Message = fmt.Sprintf("扫描完成（跳过 %d 条）", p.Skipped)
log.Warnf("media scan skipped %d items (path/name too long for unique index)", p.Skipped)
} else {
p.Message = "扫描完成"
}
p.mu.Unlock()
now := time.Now()
sp.LastScanAt = &now
_ = db.UpdateMediaScanPath(sp)
}
}()
}
func doScanPath(sp *model.MediaScanPath, p *ScanProgress) error {
scanRoot := sp.Path
if scanRoot == "" {
scanRoot = "/"
}
// 收集所有待处理路径（VFS 虚拟路径）
var targets []string
ctx := context.Background()
if sp.PathMerge {
// 路径合并模式：
// - 子文件夹 → 作为一个合并条目（带 Episodes 选集信息）
// - 直接放在根目录下的单个媒体文件 → 正常作为独立条目扫描
entries, err := fs.List(ctx, scanRoot, &fs.ListArgs{NoLog: true, Refresh: true})
if err != nil {
return err
}
for _, e := range entries {
childPath := stdpath.Join(scanRoot, e.GetName())
if e.IsDir() {
targets = append(targets, childPath)
} else if isMediaFile(e.GetName(), sp.MediaType) {
targets = append(targets, childPath)
}
}
} else {
// 普通模式：递归扫描所有匹配文件（每个目录都刷新缓存）
if err := walkVFS(ctx, scanRoot, sp.MediaType, &targets); err != nil {
return err
}
}
p.mu.Lock()
p.Total += len(targets)
p.mu.Unlock()
// 音乐类型：按专辑合并处理
if sp.MediaType == model.MediaTypeMusic {
return doScanMusicByAlbum(ctx, targets, sp, p)
}
for _, target := range targets {
item, err := buildMediaItemFromVFS(ctx, target, sp)
if err != nil {
log.Warnf("build media item error [%s]: %v", target, err)
continue
}
// 路径合并模式下，扫描文件夹内的文件，填充选集信息
if sp.PathMerge && item.IsFolder {
if episodes, err := buildEpisodesFromFolder(ctx, target, sp.MediaType); err == nil {
item.Episodes = episodes
} else {
log.Warnf("build episodes error [%s]: %v", target, err)
}
}
if err := db.CreateOrUpdateMediaItem(item); err != nil {
log.Warnf("save media item error [%s]: %v", target, err)
p.mu.Lock()
p.Skipped++
p.mu.Unlock()
}
p.mu.Lock()
p.Done++
p.Message = "已扫描: " + stdpath.Base(target)
p.mu.Unlock()
}
return nil
}
// doScanMusicByAlbum 音乐扫描
// 存储规则：
// - 普通文件（is_folder=false）：
// folder_path=文件所在真实目录，file_name=音乐文件名，episodes=空
// - 合并文件夹（is_folder=true）：
// folder_path=文件夹所在真实父目录，file_name=文件夹名，
// episodes=文件夹内所有音乐文件列表，封面/标签取第一首
func doScanMusicByAlbum(ctx context.Context, targets []string, sp *model.MediaScanPath, p *ScanProgress) error {
for _, target := range targets {
obj, err := fs.Get(ctx, target, &fs.GetArgs{NoLog: true})
if err != nil {
log.Warnf("get vfs object error [%s]: %v", target, err)
p.mu.Lock()
p.Done++
p.mu.Unlock()
continue
}
name := obj.GetName()
if obj.IsDir() {
// ---- 合并文件夹模式：文件夹条目 ----
// 列出文件夹内所有音乐文件
entries, err := fs.List(ctx, target, &fs.ListArgs{NoLog: true})
if err != nil {
log.Warnf("list music folder error [%s]: %v", target, err)
p.mu.Lock()
p.Done++
p.mu.Unlock()
continue
}
var musicFiles []string
var episodes []EpisodeInfo
for idx, e := range entries {
if e.IsDir() || !isMediaFile(e.GetName(), sp.MediaType) {
continue
}
musicFiles = append(musicFiles, stdpath.Join(target, e.GetName()))
ep := EpisodeInfo{
FileName: e.GetName(),
Index: idx + 1,
Title: strings.TrimSuffix(e.GetName(), stdpath.Ext(e.GetName())),
}
episodes = append(episodes, ep)
}
if len(musicFiles) == 0 {
p.mu.Lock()
p.Done++
p.mu.Unlock()
continue
}
// 取第一首音乐文件读取标签
firstFile := musicFiles[0]
firstExt := strings.ToLower(stdpath.Ext(firstFile))
var tag *MusicTag
readCtx, readCancel := context.WithTimeout(ctx, 15*time.Second)
if reader := FetchFileReader(readCtx, firstFile); reader != nil {
switch firstExt {
case ".flac":
tag, _ = ParseFLACVorbisComment(reader)
default:
tag, _ = ParseID3v2(reader)
}
_ = reader.Close()
}
readCancel()
// 解析标签（用文件夹名作为默认专辑名）
albumName := name
albumArtist := ""
cover := ""
releaseDate := ""
genre := ""
authors := ""
if tag != nil {
if tag.Album != "" {
albumName = tag.Album
}
albumArtist = tag.AlbumArtist
if albumArtist == "" {
albumArtist = tag.Artist
}
if tag.Year != "" && len(tag.Year) >= 4 {
releaseDate = tag.Year[:4] + "-01-01"
}
if tag.Genre != "" {
genre = tag.Genre
}
if tag.Artist != "" {
if authorsJSON, err := json.Marshal([]string{tag.Artist}); err == nil {
authors = string(authorsJSON)
}
}
if len(tag.CoverData) > 0 {
mimeType := tag.CoverMIME
if mimeType == "" {
mimeType = "image/jpeg"
}
cover = "data:" + mimeType + ";base64," + base64.StdEncoding.EncodeToString(tag.CoverData)
}
}
episodesJSON := ""
if b, err := json.Marshal(episodes); err == nil {
episodesJSON = string(b)
}
// 加载歌词：同目录同名 .lrc 优先，其次内嵌歌词
lyrics := loadLyricsForMusic(ctx, firstFile, tag)
item := &model.MediaItem{
MediaType: sp.MediaType,
ScanPathID: sp.ID,
FileName: name, // 文件夹名
FolderPath: stdpath.Dir(target), // 文件夹所在的真实父目录
IsFolder: true,
AlbumName: albumName,
AlbumArtist: albumArtist,
ScrapedName: albumName,
Cover: cover,
ReleaseDate: releaseDate,
Genre: genre,
Authors: authors,
Lyrics: lyrics,
Episodes: episodesJSON,
}
if err := db.CreateOrUpdateMediaItem(item); err != nil {
log.Warnf("save music folder item error [%s]: %v", target, err)
}
} else {
// ---- 普通文件：每首歌独立一条记录 ----
folderPath := stdpath.Dir(target)
ext := strings.ToLower(stdpath.Ext(name))
var tag *MusicTag
readCtx, readCancel := context.WithTimeout(ctx, 15*time.Second)
if reader := FetchFileReader(readCtx, target); reader != nil {
switch ext {
case ".flac":
tag, _ = ParseFLACVorbisComment(reader)
default:
tag, _ = ParseID3v2(reader)
}
_ = reader.Close()
}
readCancel()
albumName := ""
albumArtist := ""
trackTitle := strings.TrimSuffix(name, stdpath.Ext(name))
trackNumber := 0
cover := ""
releaseDate := ""
genre := ""
authors := ""
if tag != nil {
albumName = tag.Album
albumArtist = tag.AlbumArtist
if albumArtist == "" {
albumArtist = tag.Artist
}
if tag.Title != "" {
trackTitle = tag.Title
}
trackNumber = tag.TrackNumber
if tag.Year != "" && len(tag.Year) >= 4 {
releaseDate = tag.Year[:4] + "-01-01"
}
if tag.Genre != "" {
genre = tag.Genre
}
if tag.Artist != "" {
if authorsJSON, err := json.Marshal([]string{tag.Artist}); err == nil {
authors = string(authorsJSON)
}
}
if len(tag.CoverData) > 0 {
mimeType := tag.CoverMIME
if mimeType == "" {
mimeType = "image/jpeg"
}
cover = "data:" + mimeType + ";base64," + base64.StdEncoding.EncodeToString(tag.CoverData)
}
}
// 加载歌词：同目录同名 .lrc 优先，其次内嵌歌词
lyrics := loadLyricsForMusic(ctx, target, tag)
item := &model.MediaItem{
MediaType: sp.MediaType,
ScanPathID: sp.ID,
FileName: name,
FolderPath: folderPath,
IsFolder: false,
FileSize: obj.GetSize(),
MimeType: mime.TypeByExtension(ext),
AlbumName: albumName,
AlbumArtist: albumArtist,
ScrapedName: trackTitle,
Cover: cover,
ReleaseDate: releaseDate,
Genre: genre,
Authors: authors,
TrackNumber: trackNumber,
Lyrics: lyrics,
}
if err := db.CreateOrUpdateMediaItem(item); err != nil {
log.Warnf("save music item error [%s]: %v", target, err)
p.mu.Lock()
p.Skipped++
p.mu.Unlock()
}
}
p.mu.Lock()
p.Done++
p.Message = "已扫描: " + name
p.mu.Unlock()
}
return nil
}
// loadLyricsForMusic 为音乐文件加载歌词：
// 1. 优先读取同目录同名 .lrc 文件（如 song.mp3 → song.lrc）
// 2. 如果不存在，使用 tag 中的内嵌歌词（USLT / Vorbis LYRICS）
//
// musicVfsPath 为音乐文件的 VFS 全路径；tag 可为 nil
func loadLyricsForMusic(ctx context.Context, musicVfsPath string, tag *MusicTag) string {
// 尝试同目录同名 .lrc
ext := stdpath.Ext(musicVfsPath)
if ext != "" {
lrcPath := strings.TrimSuffix(musicVfsPath, ext) + ".lrc"
lrcCtx, lrcCancel := context.WithTimeout(ctx, 8*time.Second)
reader := FetchFileReader(lrcCtx, lrcPath)
if reader != nil {
// 限制读取上限 1MB，避免异常文件压垮内存
data, err := io.ReadAll(io.LimitReader(reader, 1024*1024))
_ = reader.Close()
lrcCancel()
if err == nil && len(data) > 0 {
// 去掉可能的 UTF-8 BOM
if len(data) >= 3 && data[0] == 0xEF && data[1] == 0xBB && data[2] == 0xBF {
data = data[3:]
}
s := strings.TrimSpace(string(data))
if s != "" {
return s
}
}
} else {
lrcCancel()
}
}
// 回退到内嵌歌词
if tag != nil && tag.Lyrics != "" {
return tag.Lyrics
}
return ""
}
// FetchFileReader 通过 VFS 路径获取文件内容流（用于刮削器读取文件内容）
// 优先使用 RangeReader 直接读取（本地存储无需 HTTP），失败时回退到 HTTP URL
// 返回 nil 表示无法获取（不影响主流程）
func FetchFileReader(ctx context.Context, vfsPath string) io.ReadCloser {
link, file, err := fs.Link(ctx, vfsPath, model.LinkArgs{})
if err != nil || link == nil {
return nil
}
size := link.ContentLength
if size <= 0 {
size = file.GetSize()
}
rr, err := stream.GetRangeReaderFromLink(size, link)
if err != nil {
_ = link.Close()
return nil
}
rc, err := rr.RangeRead(ctx, http_range.Range{Length: size})
if err != nil {
_ = link.Close()
return nil
}
link.SyncClosers.Add(rc)
return utils.ReadCloser{Reader: rc, Closer: link}
}
// walkVFS 递归遍历 VFS 路径，收集匹配的媒体文件路径（每个目录都刷新缓存）
func walkVFS(ctx context.Context, dirPath string, mediaType model.MediaType, targets *[]string) error {
entries, err := fs.List(ctx, dirPath, &fs.ListArgs{NoLog: true, Refresh: true})
if err != nil {
log.Warnf("media scan: list vfs path [%s] error: %v", dirPath, err)
return nil // 跳过无权限目录，不中断整体扫描
}
for _, e := range entries {
childPath := stdpath.Join(dirPath, e.GetName())
if e.IsDir() {
if err := walkVFS(ctx, childPath, mediaType, targets); err != nil {
return err
}
} else if isMediaFile(e.GetName(), mediaType) {
*targets = append(*targets, childPath)
}
}
return nil
}
// buildMediaItemFromVFS 根据 VFS 路径构建 MediaItem（非音乐类型使用）
// 存储规则：
// - folder_path：文件/文件夹所在的真实父目录（VFS 父路径），用于前端按 folder_path + file_name 反查真实文件
// - file_name：文件夹就是文件夹名，文件就是文件名
// - episodes：文件夹时存里面每个文件的信息，文件时为空
func buildMediaItemFromVFS(ctx context.Context, vfsPath string, sp *model.MediaScanPath) (*model.MediaItem, error) {
obj, err := fs.Get(ctx, vfsPath, &fs.GetArgs{NoLog: true})
if err != nil {
return nil, err
}
name := obj.GetName()
ext := strings.ToLower(stdpath.Ext(name))
item := &model.MediaItem{
MediaType: sp.MediaType,
ScanPathID: sp.ID,
FileName: name, // 文件夹名 或 文件名
FolderPath: stdpath.Dir(vfsPath), // 真实父目录
IsFolder: obj.IsDir(),
ScrapedName: strings.TrimSuffix(name, stdpath.Ext(name)), // 去掉扩展名作为默认名称
}
if !obj.IsDir() {
// 普通文件：记录大小和 MIME 类型，episodes 为空
item.FileSize = obj.GetSize()
item.MimeType = mime.TypeByExtension(ext)
}
return item, nil
}
// episodeNumRe 匹配文件名开头的数字序号，支持 "1、" "2." "3-" "4 " 等分隔符
var episodeNumRe = regexp.MustCompile(`^(\d+)[、.\-\s_]+(.*)`)
// EpisodeInfo 选集信息（定义在 model 包中，供协议层复用）
type EpisodeInfo = model.EpisodeInfo
// buildEpisodesFromFolder 扫描文件夹内的媒体文件，构建选集信息 JSON 字符串
func buildEpisodesFromFolder(ctx context.Context, folderPath string, mediaType model.MediaType) (string, error) {
entries, err := fs.List(ctx, folderPath, &fs.ListArgs{NoLog: true})
if err != nil {
return "", err
}
var episodes []EpisodeInfo
for _, e := range entries {
if e.IsDir() {
continue
}
name := e.GetName()
if !isMediaFile(name, mediaType) {
continue
}
// 去掉扩展名得到裸文件名
ext := stdpath.Ext(name)
baseName := strings.TrimSuffix(name, ext)
ep := EpisodeInfo{
FileName: name,
Index: 0,
Title: baseName,
}
// 尝试从文件名开头提取数字序号
if m := episodeNumRe.FindStringSubmatch(baseName); len(m) == 3 {
if idx := parseLeadingInt(m[1]); idx > 0 {
ep.Index = idx
ep.Title = strings.TrimSpace(m[2])
}
} else {
// 文件名直接以纯数字开头（无分隔符），也尝试提取
if idx, rest := splitLeadingNumber(baseName); idx > 0 {
ep.Index = idx
ep.Title = strings.TrimSpace(rest)
}
}
episodes = append(episodes, ep)
}
if len(episodes) == 0 {
return "", nil
}
b, err := json.Marshal(episodes)
if err != nil {
return "", err
}
return string(b), nil
}
// parseLeadingInt 将纯数字字符串解析为 int，失败返回 0
func parseLeadingInt(s string) int {
v := 0
for _, c := range s {
if c < '0' || c > '9' {
return 0
}
v = v*10 + int(c-'0')
}
return v
}
// splitLeadingNumber 从字符串开头提取连续数字，返回 (数字值, 剩余字符串)
// 仅当开头确实有数字时才返回非零值
func splitLeadingNumber(s string) (int, string) {
i := 0
for i < len(s) && s[i] >= '0' && s[i] <= '9' {
i++
}
if i == 0 || i == len(s) {
// 没有数字，或者整个字符串都是数字（没有标题部分）
return 0, s
}
// 剩余部分必须以非字母数字字符开头，避免把 "1080p" 之类的误识别
if unicode.IsLetter(rune(s[i])) || unicode.IsDigit(rune(s[i])) {
return 0, s
}
v := parseLeadingInt(s[:i])
return v, s[i:]
}
// isMediaFile 判断文件名是否为指定媒体类型（按扩展名判断）
func isMediaFile(name string, mediaType model.MediaType) bool {
ext := strings.ToLower(stdpath.Ext(name))
switch mediaType {
case model.MediaTypeVideo:
return videoExts[ext]
case model.MediaTypeMusic:
return musicExts[ext]
case model.MediaTypeImage:
return imageExts[ext]
case model.MediaTypeBook:
return bookExts[ext]
}
return false
}
// GetSupportedExts 获取指定媒体类型支持的扩展名列表
func GetSupportedExts(mediaType model.MediaType) []string {
var extMap map[string]bool
switch mediaType {
case model.MediaTypeVideo:
extMap = videoExts
case model.MediaTypeMusic:
extMap = musicExts
case model.MediaTypeImage:
extMap = imageExts
case model.MediaTypeBook:
extMap = bookExts
default:
return nil
}
exts := make([]string, 0, len(extMap))
for ext := range extMap {
exts = append(exts, ext)
}
return exts
}
internal/media/scraper/book_local.go
+389
View File
@@ -0,0 +1,389 @@
package scraper
import (
"archive/zip"
"bytes"
"context"
"encoding/base64"
"encoding/xml"
"errors"
"image/jpeg"
"io"
"strings"
"time"
"github.com/disintegration/imaging"
"github.com/OpenListTeam/OpenList/v4/internal/media/imgmime"
"github.com/OpenListTeam/OpenList/v4/internal/model"
)
// BookLocalScraper 书籍本地刮削器
// 尝试从书籍文件中提取封面图片并以 base64 存入 Cover 字段，或保存到本地文件。
// - EPUB：解压 zip，提取 OPF 声明的封面或 cover.jpg/cover.png
// - PDF：尝试提取内嵌 JPEG 图片流作为封面
// - 其他/失败：降级为文件路径
type BookLocalScraper struct {
// ThumbnailMode 缩略图存储方式："base64"（默认，存入数据库）或 "local"（存到本地文件）
ThumbnailMode string
// ThumbnailPath 本地存储路径，ThumbnailMode 为 "local" 时有效，默认 "/.thumbnail"
ThumbnailPath string
}
// NewBookLocalScraper 创建书籍本地刮削器
func NewBookLocalScraper() *BookLocalScraper {
return &BookLocalScraper{
ThumbnailMode: "base64",
ThumbnailPath: "/.thumbnail",
}
}
// NewBookLocalScraperWithConfig 创建带完整配置的书籍本地刮削器
func NewBookLocalScraperWithConfig(thumbnailMode, thumbnailPath string) *BookLocalScraper {
if thumbnailMode == "" {
thumbnailMode = "base64"
}
if thumbnailPath == "" {
thumbnailPath = "/.thumbnail"
}
return &BookLocalScraper{
ThumbnailMode: thumbnailMode,
ThumbnailPath: thumbnailPath,
}
}
// ScrapeBookLocal 从书籍文件流中提取封面图片，填充基本信息。
// reader 为书籍文件内容流，不能为 nil。
// 返回提取到的封面 base64 字符串（data URI），提取失败返回空字符串。
// 注意：本函数不会将文件路径作为 cover 降级，cover 为空表示提取失败。
func (s *BookLocalScraper) ScrapeBookLocal(item *model.MediaItem, reader io.Reader) error {
ext := strings.ToLower(getFileExt(item.FileName))
// 根据文件类型补充 MimeType
if item.MimeType == "" {
item.MimeType = mimeTypeByBookExt(ext)
}
// 去掉扩展名作为展示名
if item.ScrapedName == "" {
item.ScrapedName = trimExt(item.FileName)
}
// 从文件流提取封面，提取失败则 cover 保持为空
if reader != nil {
data, err := readAllLimited(reader, MaxBookFileBytes)
if err == nil && len(data) > 0 {
var coverData []byte
switch ext {
case ".epub":
coverData = extractEpubCoverData(data)
case ".pdf":
coverData = extractPDFCoverData(data)
}
if len(coverData) > 0 {
cover := s.storeCover(strings.TrimSuffix(item.FolderPath, "/")+"/"+item.FileName, coverData)
if cover != "" {
item.Cover = cover
}
}
}
}
now := time.Now()
item.ScrapedAt = &now
return nil
}
// ExtractLocalCover 从书籍文件流中提取封面并根据配置存储（供外部调用）。
// 提取失败返回空字符串，不降级为文件路径。
func (s *BookLocalScraper) ExtractLocalCover(fileName string, filePath string, reader io.Reader) string {
if reader == nil {
return ""
}
ext := strings.ToLower(getFileExt(fileName))
data, err := readAllLimited(reader, MaxBookFileBytes)
if err != nil || len(data) == 0 {
return ""
}
var coverData []byte
switch ext {
case ".epub":
coverData = extractEpubCoverData(data)
case ".pdf":
coverData = extractPDFCoverData(data)
}
if len(coverData) == 0 {
return ""
}
return s.storeCover(filePath, coverData)
}
// storeCover 根据 ThumbnailMode 存储封面数据，返回 Cover 字段值
func (s *BookLocalScraper) storeCover(filePath string, coverData []byte) string {
if NormalizeThumbMode(s.ThumbnailMode) == ThumbModeBase64 {
return imageDataToBase64Thumb(coverData)
}
thumb := resizeCoverBytes(coverData)
if len(thumb) == 0 {
thumb = coverData
}
return SaveThumb(context.Background(), s.ThumbnailMode, s.ThumbnailPath, filePath, "image/jpeg", thumb)
}
// resizeCoverBytes 将封面缩放为 300px 宽，返回 JPEG 字节
func resizeCoverBytes(data []byte) []byte {
img, err := decodeBounded(data)
if err != nil {
return nil
}
thumb := imaging.Resize(img, 300, 0, imaging.Lanczos)
var buf bytes.Buffer
if err := jpeg.Encode(&buf, thumb, &jpeg.Options{Quality: 80}); err != nil {
return nil
}
return buf.Bytes()
}
// ── EPUB 封面提取 ─────────────────────────────────────────────────────────────
// epubContainer 解析 META-INF/container.xml
type epubContainer struct {
Rootfiles []struct {
FullPath string `xml:"full-path,attr"`
} `xml:"rootfiles>rootfile"`
}
// epubOPF 解析 OPF 文件，找封面 item
type epubOPF struct {
Metadata struct {
Metas []struct {
Name string `xml:"name,attr"`
Content string `xml:"content,attr"`
} `xml:"meta"`
} `xml:"metadata"`
Manifest struct {
Items []struct {
ID string `xml:"id,attr"`
Href string `xml:"href,attr"`
MediaType string `xml:"media-type,attr"`
} `xml:"item"`
} `xml:"manifest"`
}
// extractEpubCoverData 从 EPUB（zip）数据中提取封面图片原始字节
func extractEpubCoverData(data []byte) []byte {
r, err := zip.NewReader(bytes.NewReader(data), int64(len(data)))
if err != nil {
return nil
}
// 建立文件名→ZipFile 映射（忽略大小写）
fileMap := make(map[string]*zip.File, len(r.File))
for _, f := range r.File {
fileMap[strings.ToLower(f.Name)] = f
}
// 1. 尝试读取 META-INF/container.xml 找 OPF 路径
opfPath := ""
if cf, ok := fileMap["meta-inf/container.xml"]; ok {
if rc, err := cf.Open(); err == nil {
var container epubContainer
if xml.NewDecoder(rc).Decode(&container) == nil && len(container.Rootfiles) > 0 {
opfPath = container.Rootfiles[0].FullPath
}
rc.Close()
}
}
// OPF 所在目录（用于拼接相对路径）
opfDir := ""
if idx := strings.LastIndex(opfPath, "/"); idx >= 0 {
opfDir = opfPath[:idx+1]
}
// 2. 从 OPF 中找封面 item ID
coverHref := ""
if opfPath != "" {
if of, ok := fileMap[strings.ToLower(opfPath)]; ok {
if rc, err := of.Open(); err == nil {
var opf epubOPF
if xml.NewDecoder(rc).Decode(&opf) == nil {
// 先找 <meta name="cover" content="..."/> 指向的 item id
coverItemID := ""
for _, m := range opf.Metadata.Metas {
if strings.EqualFold(m.Name, "cover") && m.Content != "" {
coverItemID = m.Content
break
}
}
// 在 manifest 中找对应 item
for _, item := range opf.Manifest.Items {
if coverItemID != "" && item.ID == coverItemID {
coverHref = opfDir + item.Href
break
}
// 如果没有指定 cover id，找第一个图片类型的 item
if coverItemID == "" && strings.Contains(strings.ToLower(item.MediaType), "image") {
// 优先选择 id 或 href 中包含 "cover" 的
if strings.Contains(strings.ToLower(item.ID), "cover") ||
strings.Contains(strings.ToLower(item.Href), "cover") {
coverHref = opfDir + item.Href
break
}
}
}
// 如果还没找到，取第一个图片
if coverHref == "" && coverItemID == "" {
for _, item := range opf.Manifest.Items {
if strings.Contains(strings.ToLower(item.MediaType), "image") {
coverHref = opfDir + item.Href
break
}
}
}
}
rc.Close()
}
}
}
// 3. 常见封面文件名候选（按优先级）
candidates := []string{}
if coverHref != "" {
candidates = append(candidates, strings.ToLower(coverHref))
}
candidates = append(candidates,
"cover.jpg", "cover.jpeg", "cover.png",
"images/cover.jpg", "images/cover.jpeg", "images/cover.png",
"oebps/cover.jpg", "oebps/images/cover.jpg",
"oebps/cover.jpeg", "oebps/images/cover.jpeg",
)
for _, name := range candidates {
if f, ok := fileMap[name]; ok {
if rc, err := f.Open(); err == nil {
imgData, readErr := readAllLimited(rc, MaxEmbeddedEntryBytes)
rc.Close()
if readErr == nil && len(imgData) > 0 {
return imgData
}
}
}
}
return nil
}
// extractEpubCoverBase64 从 EPUB（zip）数据中提取封面图片，返回 data URI base64（向后兼容）
func extractEpubCoverBase64(data []byte) string {
coverData := extractEpubCoverData(data)
if len(coverData) == 0 {
return ""
}
return imageDataToBase64Thumb(coverData)
}
// ── PDF 封面提取 ──────────────────────────────────────────────────────────────
// extractPDFCoverData 从 PDF 数据中提取第一个内嵌 JPEG 图片流，返回原始字节
// PDF 内嵌图片通常以 JPEG 流存储（SOI marker: 0xFF 0xD8），直接扫描字节流提取
func extractPDFCoverData(data []byte) []byte {
// 扫描 JPEG SOI (0xFF 0xD8 0xFF) 标记
for i := 0; i < len(data)-2; i++ {
if data[i] == 0xFF && data[i+1] == 0xD8 && data[i+2] == 0xFF {
// 找对应的 EOI (0xFF 0xD9)
for j := i + 2; j < len(data)-1; j++ {
if data[j] == 0xFF && data[j+1] == 0xD9 {
jpegData := data[i : j+2]
// 验证是否为有效图片（至少 1KB，避免误匹配小缩略图）
if len(jpegData) > 1024 {
return jpegData
}
break
}
}
}
}
return nil
}
// extractPDFCoverBase64 从 PDF 数据中提取第一个内嵌 JPEG 图片流，返回 data URI base64（向后兼容）
func extractPDFCoverBase64(data []byte) string {
coverData := extractPDFCoverData(data)
if len(coverData) == 0 {
return ""
}
return imageDataToBase64Thumb(coverData)
}
// ── 公共辅助 ──────────────────────────────────────────────────────────────────
// imageDataToBase64Thumb 将图片字节解码并缩放为 300px 宽缩略图，返回 data URI base64
func imageDataToBase64Thumb(data []byte) string {
img, err := decodeBounded(data)
if err != nil {
// 超限图片不做任何保留；解码失败但魔数是合法栅格格式时，原样 base64（可能是解码器不支持的变体）
if errors.Is(err, ErrTooLarge) || len(data) > MaxRawCoverFallbackBytes {
return ""
}
mimeType := imgmime.Sniff(data)
if mimeType == "" {
return ""
}
return "data:" + mimeType + ";base64," + base64.StdEncoding.EncodeToString(data)
}
thumb := imaging.Resize(img, 300, 0, imaging.Lanczos)
var buf bytes.Buffer
if err := jpeg.Encode(&buf, thumb, &jpeg.Options{Quality: 80}); err != nil {
return ""
}
return "data:image/jpeg;base64," + base64.StdEncoding.EncodeToString(buf.Bytes())
}
// detectImageMime 根据文件头检测图片 MIME 类型
func detectImageMime(data []byte) string {
if len(data) < 4 {
return "image/jpeg"
}
switch {
case data[0] == 0xFF && data[1] == 0xD8:
return "image/jpeg"
case data[0] == 0x89 && data[1] == 0x50 && data[2] == 0x4E && data[3] == 0x47:
return "image/png"
case data[0] == 0x47 && data[1] == 0x49 && data[2] == 0x46:
return "image/gif"
case data[0] == 0x52 && data[1] == 0x49 && data[2] == 0x46 && data[3] == 0x46:
return "image/webp"
default:
return "image/jpeg"
}
}
// getFileExt 获取文件扩展名（含点号，小写）
func getFileExt(fileName string) string {
if idx := strings.LastIndex(fileName, "."); idx >= 0 {
return fileName[idx:]
}
return ""
}
// mimeTypeByBookExt 根据书籍文件扩展名返回 MIME 类型
func mimeTypeByBookExt(ext string) string {
switch ext {
case ".pdf":
return "application/pdf"
case ".epub":
return "application/epub+zip"
case ".mobi":
return "application/x-mobipocket-ebook"
case ".azw3":
return "application/vnd.amazon.ebook"
case ".txt":
return "text/plain"
case ".djvu":
return "image/vnd.djvu"
case ".cbz":
return "application/vnd.comicbook+zip"
case ".cbr":
return "application/vnd.comicbook-rar"
default:
return "application/octet-stream"
}
}
internal/media/scraper/book_online.go
+278
View File
@@ -0,0 +1,278 @@
package scraper
import (
"encoding/json"
"fmt"
"net/http"
"net/url"
"strings"
"time"
"github.com/OpenListTeam/OpenList/v4/internal/model"
)
// ==================== Google Books ====================
// GoogleBooksScraper 通过 Google Books API 刮削书籍
type GoogleBooksScraper struct {
APIKey string
ThumbnailMode string
ThumbnailPath string
client *http.Client
}
func NewGoogleBooksScraper(apiKey, thumbnailMode, thumbnailPath string) *GoogleBooksScraper {
if thumbnailMode == "" {
thumbnailMode = "base64"
}
if thumbnailPath == "" {
thumbnailPath = "imgs"
}
return &GoogleBooksScraper{
APIKey: apiKey,
ThumbnailMode: thumbnailMode,
ThumbnailPath: thumbnailPath,
client: &http.Client{Timeout: 20 * time.Second},
}
}
type googleBooksResponse struct {
Items []struct {
ID string `json:"id"`
VolumeInfo struct {
Title string `json:"title"`
Authors []string `json:"authors"`
Description string `json:"description"`
PublishedDate string `json:"publishedDate"`
Publisher string `json:"publisher"`
Categories []string `json:"categories"`
AverageRating float64 `json:"averageRating"`
IndustryIDs []struct {
Type string `json:"type"`
Identifier string `json:"identifier"`
} `json:"industryIdentifiers"`
ImageLinks struct {
Thumbnail string `json:"thumbnail"`
SmallThumbnail string `json:"smallThumbnail"`
} `json:"imageLinks"`
} `json:"volumeInfo"`
} `json:"items"`
}
// ScrapeBook 用 Google Books 填充书籍信息，封面保存为远程链接（由调用方决定是否落盘）
func (s *GoogleBooksScraper) ScrapeBook(item *model.MediaItem) error {
query := strings.TrimSpace(item.ISBN)
if query == "" {
query = strings.TrimSpace(item.ScrapedName)
}
if query == "" && item.FileName != "" {
query = trimExt(item.FileName)
}
if query == "" {
return fmt.Errorf("empty query")
}
api := "https://www.googleapis.com/books/v1/volumes?maxResults=1&printType=books&langRestrict=" +
url.QueryEscape(ScrapeLanguagePrimary()) + "&q=" + url.QueryEscape(query)
if s.APIKey != "" {
api += "&key=" + url.QueryEscape(s.APIKey)
}
var resp googleBooksResponse
if err := s.getJSON(api, &resp); err != nil {
return err
}
if len(resp.Items) == 0 {
return fmt.Errorf("google books: no result for %q", query)
}
it := resp.Items[0]
v := it.VolumeInfo
if item.ScrapedName == "" {
item.ScrapedName = v.Title
}
if item.Plot == "" {
item.Plot = v.Description
}
if item.Description == "" {
item.Description = v.Description
}
if item.Publisher == "" {
item.Publisher = v.Publisher
}
if item.ReleaseDate == "" {
item.ReleaseDate = v.PublishedDate
}
if item.Authors == "" && len(v.Authors) > 0 {
item.Authors = toAuthorsJSON(v.Authors)
}
if item.Genre == "" && len(v.Categories) > 0 {
item.Genre = strings.Join(v.Categories, ",")
}
if item.ISBN == "" {
for _, id := range v.IndustryIDs {
if id.Type == "ISBN_13" || id.Type == "ISBN_10" {
item.ISBN = id.Identifier
break
}
}
}
if item.Rating == 0 && v.AverageRating > 0 {
item.Rating = float32(v.AverageRating * 2)
}
if item.ExternalID == "" {
item.ExternalID = "google:" + it.ID
}
if item.Cover == "" {
img := v.ImageLinks.Thumbnail
if img == "" {
img = v.ImageLinks.SmallThumbnail
}
item.Cover = normalizeGoogleImage(img)
}
return nil
}
func (s *GoogleBooksScraper) getJSON(api string, out any) error {
req, err := http.NewRequest(http.MethodGet, api, nil)
if err != nil {
return err
}
req.Header.Set("User-Agent", "Mozilla/5.0 (compatible; OpenList-Media/1.0)")
resp, err := s.client.Do(req)
if err != nil {
return err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return fmt.Errorf("google books: unexpected status %d", resp.StatusCode)
}
return json.NewDecoder(resp.Body).Decode(out)
}
func normalizeGoogleImage(img string) string {
if img == "" {
return ""
}
img = strings.Replace(img, "http://", "https://", 1)
if idx := strings.Index(img, "&edge="); idx > 0 {
img = img[:idx]
}
return img
}
// ==================== OpenLibrary ====================
// OpenLibraryScraper 通过 OpenLibrary Search API 刮削书籍（无需凭据）
type OpenLibraryScraper struct {
ThumbnailMode string
ThumbnailPath string
client *http.Client
}
func NewOpenLibraryScraper(thumbnailMode, thumbnailPath string) *OpenLibraryScraper {
if thumbnailMode == "" {
thumbnailMode = "base64"
}
if thumbnailPath == "" {
thumbnailPath = "imgs"
}
return &OpenLibraryScraper{
ThumbnailMode: thumbnailMode,
ThumbnailPath: thumbnailPath,
client: &http.Client{Timeout: 20 * time.Second},
}
}
type openLibraryResponse struct {
Docs []struct {
Key string `json:"key"`
Title string `json:"title"`
AuthorName []string `json:"author_name"`
FirstPublish int `json:"first_publish_year"`
Publisher []string `json:"publisher"`
ISBN []string `json:"isbn"`
CoverID int `json:"cover_i"`
Subject []string `json:"subject"`
RatingsAverage float64 `json:"ratings_average"`
} `json:"docs"`
}
// ScrapeBook 用 OpenLibrary 填充书籍信息，封面保存为远程链接（由调用方决定是否落盘）
func (s *OpenLibraryScraper) ScrapeBook(item *model.MediaItem) error {
query := strings.TrimSpace(item.ISBN)
if query == "" {
query = strings.TrimSpace(item.ScrapedName)
}
if query == "" && item.FileName != "" {
query = trimExt(item.FileName)
}
if query == "" {
return fmt.Errorf("empty query")
}
api := "https://openlibrary.org/search.json?limit=1&fields=key,title,author_name,first_publish_year,publisher,isbn,cover_i,subject,ratings_average&q=" + url.QueryEscape(query)
var resp openLibraryResponse
if err := s.getJSON(api, &resp); err != nil {
return err
}
if len(resp.Docs) == 0 {
return fmt.Errorf("openlibrary: no result for %q", query)
}
doc := resp.Docs[0]
if item.ScrapedName == "" {
item.ScrapedName = doc.Title
}
if item.ReleaseDate == "" && doc.FirstPublish > 0 {
item.ReleaseDate = fmt.Sprintf("%d", doc.FirstPublish)
}
if item.Publisher == "" && len(doc.Publisher) > 0 {
item.Publisher = doc.Publisher[0]
}
if item.Authors == "" && len(doc.AuthorName) > 0 {
item.Authors = toAuthorsJSON(doc.AuthorName)
}
if item.Genre == "" && len(doc.Subject) > 0 {
limit := len(doc.Subject)
if limit > 8 {
limit = 8
}
item.Genre = strings.Join(doc.Subject[:limit], ",")
}
if item.ISBN == "" && len(doc.ISBN) > 0 {
item.ISBN = doc.ISBN[0]
}
if item.Rating == 0 && doc.RatingsAverage > 0 {
item.Rating = float32(doc.RatingsAverage * 2)
}
if item.ExternalID == "" {
item.ExternalID = "openlibrary:" + doc.Key
}
if item.Cover == "" && doc.CoverID > 0 {
item.Cover = fmt.Sprintf("https://covers.openlibrary.org/b/id/%d-L.jpg", doc.CoverID)
}
return nil
}
func (s *OpenLibraryScraper) getJSON(api string, out any) error {
req, err := http.NewRequest(http.MethodGet, api, nil)
if err != nil {
return err
}
req.Header.Set("User-Agent", "Mozilla/5.0 (compatible; OpenList-Media/1.0)")
resp, err := s.client.Do(req)
if err != nil {
return err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return fmt.Errorf("openlibrary: unexpected status %d", resp.StatusCode)
}
return json.NewDecoder(resp.Body).Decode(out)
}
func toAuthorsJSON(authors []string) string {
b, err := json.Marshal(authors)
if err != nil {
return ""
}
return string(b)
}
internal/media/scraper/discogs.go
+317
View File
@@ -0,0 +1,317 @@
package scraper
import (
"encoding/json"
"fmt"
"io"
"net/http"
"net/url"
"strings"
"time"
"github.com/OpenListTeam/OpenList/v4/internal/model"
)
const discogsDefaultBaseURL = "https://api.discogs.com"
// DiscogsScraper Discogs音乐刮削器
type DiscogsScraper struct {
Token string
BaseURL string // 可自定义的 Discogs API 地址
client *http.Client
}
// NewDiscogsScraper 创建Discogs刮削器
//
// 用户只需填写「域名 + API 前缀」，例如：
// 1. 留空：使用默认 https://api.discogs.com
// 2. 官方：api.discogs.com -> https://api.discogs.com
// 3. 反代：example.edgeone.run/api/discogs -> https://example.edgeone.run/api/discogs
// 4. 反代：example.edgeone.run/api/discogs/ -> https://example.edgeone.run/api/discogs（自动去尾斜杠）
//
// 后续会基于 BaseURL 直接拼接 /database/search、/releases/{id} 等具体路径。
func NewDiscogsScraper(token string, baseURL string) *DiscogsScraper {
base := strings.TrimRight(strings.TrimSpace(baseURL), "/")
if base == "" {
base = discogsDefaultBaseURL
}
// 协议头容错：用户没填 http/https 时自动补 https://
if !strings.HasPrefix(base, "http://") && !strings.HasPrefix(base, "https://") {
base = "https://" + base
}
// 再次去掉末尾多余的 /，避免后续拼接出 //database/search 这类双斜杠
base = strings.TrimRight(base, "/")
return &DiscogsScraper{
Token: token,
BaseURL: base,
client: &http.Client{Timeout: 15 * time.Second},
}
}
type discogsSearchResult struct {
Results []struct {
ID int `json:"id"`
Title string `json:"title"`
Type string `json:"type"`
Year string `json:"year"`
Thumb string `json:"thumb"`
CoverImage string `json:"cover_image"`
Genre []string `json:"genre"`
Style []string `json:"style"`
ResourceURL string `json:"resource_url"`
} `json:"results"`
}
type discogsReleaseDetail struct {
ID int `json:"id"`
Title string `json:"title"`
Year int `json:"year"`
CoverImage string `json:"cover_image"`
Notes string `json:"notes"`
Artists []struct {
Name string `json:"name"`
} `json:"artists"`
Genres []string `json:"genres"`
Styles []string `json:"styles"`
Tracklist []struct {
Position string `json:"position"`
Title string `json:"title"`
Duration string `json:"duration"`
} `json:"tracklist"`
Community struct {
Rating struct {
Average float32 `json:"average"`
} `json:"rating"`
} `json:"community"`
}
// parseMusicFileName 从音乐文件名中提取搜索关键词列表
// 规则：去掉扩展名后，按" - "或"-"分割，返回各部分作为候选搜索词
// 例如：
//
// "周杰伦 - 七里香.mp3" -> ["周杰伦 - 七里香", "周杰伦", "七里香"]
// "Coldplay-Yellow.flac" -> ["Coldplay-Yellow", "Coldplay", "Yellow"]
// "七里香.mp3" -> ["七里香"]
func parseMusicFileName(fileName string) []string {
// 去掉扩展名
if idx := strings.LastIndex(fileName, "."); idx > 0 {
ext := strings.ToLower(fileName[idx:])
if len(ext) <= 5 {
fileName = fileName[:idx]
}
}
fileName = strings.TrimSpace(fileName)
if fileName == "" {
return nil
}
var queries []string
// 完整文件名作为第一候选
queries = append(queries, fileName)
// 按" - "（带空格）分割
parts := strings.SplitN(fileName, " - ", 2)
if len(parts) == 2 {
before := strings.TrimSpace(parts[0])
after := strings.TrimSpace(parts[1])
if before != "" {
queries = append(queries, before)
}
if after != "" {
queries = append(queries, after)
}
} else {
// 按"-"（不带空格）分割
parts = strings.SplitN(fileName, "-", 2)
if len(parts) == 2 {
before := strings.TrimSpace(parts[0])
after := strings.TrimSpace(parts[1])
if before != "" {
queries = append(queries, before)
}
if after != "" {
queries = append(queries, after)
}
}
}
return queries
}
// doDiscogsSearch 执行单次Discogs搜索
func (s *DiscogsScraper) doDiscogsSearch(query string) (*discogsSearchResult, error) {
searchURL := fmt.Sprintf("%s/database/search?q=%s&type=release&token=%s",
s.BaseURL, url.QueryEscape(query), s.Token)
req, _ := http.NewRequest("GET", searchURL, nil)
req.Header.Set("User-Agent", "OpenList/4.0 +https://github.com/OpenListTeam/OpenList")
resp, err := s.client.Do(req)
if err != nil {
return nil, fmt.Errorf("Discogs搜索请求失败: %w", err)
}
defer resp.Body.Close()
body, _ := io.ReadAll(resp.Body)
var result discogsSearchResult
if err := json.Unmarshal(body, &result); err != nil {
return nil, fmt.Errorf("Discogs搜索结果解析失败: %w", err)
}
return &result, nil
}
// buildMusicQuery 根据 MediaItem 已有字段拼接 Discogs 搜索词
// 优先级：专辑名 > ScrapedName > 文件名解析
// 若有歌手或年份信息，则附加到搜索词中以提高精准度
func buildMusicQuery(base string, item *model.MediaItem) string {
q := base
// 附加歌手
if item.AlbumArtist != "" {
q = item.AlbumArtist + " " + q
}
// 附加年份（取 ReleaseDate 前4位）
if len(item.ReleaseDate) >= 4 {
q = q + " " + item.ReleaseDate[:4]
}
return strings.TrimSpace(q)
}
// ScrapeMusic 刮削音乐/专辑信息
func (s *DiscogsScraper) ScrapeMusic(item *model.MediaItem) error {
if s.Token == "" {
return fmt.Errorf("Discogs Token 未配置")
}
// 构建候选搜索词列表
var queries []string
if item.AlbumName != "" {
queries = append(queries, buildMusicQuery(item.AlbumName, item))
// 同时保留不带附加信息的纯专辑名作为降级候选
if item.AlbumArtist != "" || len(item.ReleaseDate) >= 4 {
queries = append(queries, item.AlbumName)
}
} else if item.ScrapedName != "" {
queries = append(queries, buildMusicQuery(item.ScrapedName, item))
if item.AlbumArtist != "" || len(item.ReleaseDate) >= 4 {
queries = append(queries, item.ScrapedName)
}
} else {
// 从文件名解析，按"-"分割分别搜索
baseQueries := parseMusicFileName(item.FileName)
for _, bq := range baseQueries {
enhanced := buildMusicQuery(bq, item)
if enhanced != bq {
// 先放带附加信息的精准词，再放原始词作为降级
queries = append(queries, enhanced)
}
queries = append(queries, bq)
}
}
if len(queries) == 0 {
return fmt.Errorf("无法从文件名中提取有效搜索词")
}
// 依次尝试各候选词，找到结果即停止（模糊匹配降级）
var searchResult *discogsSearchResult
var lastQuery string
for _, q := range queries {
result, err := s.doDiscogsSearch(q)
if err != nil {
return err
}
lastQuery = q
if len(result.Results) > 0 {
searchResult = result
break
}
}
if searchResult == nil || len(searchResult.Results) == 0 {
return fmt.Errorf("Discogs未找到匹配结果: %s", lastQuery)
}
first := searchResult.Results[0]
// 获取详情
detailURL := fmt.Sprintf("%s/releases/%d?token=%s", s.BaseURL, first.ID, s.Token)
detailReq, _ := http.NewRequest("GET", detailURL, nil)
detailReq.Header.Set("User-Agent", "OpenList/4.0 +https://github.com/OpenListTeam/OpenList")
detailResp, err := s.client.Do(detailReq)
if err != nil {
return fmt.Errorf("Discogs详情请求失败: %w", err)
}
defer detailResp.Body.Close()
detailBody, _ := io.ReadAll(detailResp.Body)
var detail discogsReleaseDetail
if err := json.Unmarshal(detailBody, &detail); err != nil {
return fmt.Errorf("Discogs详情解析失败: %w", err)
}
// 填充字段：已有值的字段不覆盖，仅补充空缺
// Discogs 的 title 就是专辑名（不是 "Artist - Album" 格式）
if detail.Title != "" {
if item.AlbumName == "" {
item.AlbumName = detail.Title
}
if item.ScrapedName == "" {
item.ScrapedName = detail.Title
}
}
if item.ReleaseDate == "" {
if detail.Year > 0 {
item.ReleaseDate = fmt.Sprintf("%d-01-01", detail.Year)
} else if first.Year != "" {
item.ReleaseDate = first.Year + "-01-01"
}
}
if item.Rating == 0 {
item.Rating = detail.Community.Rating.Average
}
if item.Plot == "" {
item.Plot = detail.Notes
}
if item.ExternalID == "" {
item.ExternalID = fmt.Sprintf("discogs:%d", detail.ID)
}
// 封面（存储 URL，前端直接展示）
if item.Cover == "" {
if detail.CoverImage != "" {
item.Cover = detail.CoverImage
} else if first.CoverImage != "" {
item.Cover = first.CoverImage
} else if first.Thumb != "" {
item.Cover = first.Thumb
}
}
// 类型
if item.Genre == "" {
genres := append(detail.Genres, detail.Styles...)
item.Genre = strings.Join(genres, ",")
}
// 艺术家：仅在 Authors 和 AlbumArtist 均为空时才填充（ID3已有值则保留）
artists := make([]string, 0, len(detail.Artists))
for _, a := range detail.Artists {
artists = append(artists, a.Name)
}
if len(artists) > 0 {
if item.Authors == "" {
authorsJSON, _ := json.Marshal(artists)
item.Authors = string(authorsJSON)
}
if item.AlbumArtist == "" {
item.AlbumArtist = artists[0]
}
}
now := time.Now()
item.ScrapedAt = &now
return nil
}
internal/media/scraper/douban.go
+497
View File
@@ -0,0 +1,497 @@
package scraper
import (
"bytes"
"encoding/base64"
"encoding/json"
"errors"
"fmt"
"image/jpeg"
"io"
"net/http"
"net/url"
"regexp"
"strings"
"time"
"github.com/disintegration/imaging"
"github.com/OpenListTeam/OpenList/v4/internal/media/imgmime"
"github.com/OpenListTeam/OpenList/v4/internal/model"
"golang.org/x/net/html"
)
const (
doubanSearchURL = "https://www.douban.com/search"
doubanBookCat = "1001"
doubanBase = "https://book.douban.com/"
// maxCoverDownloadBytes 单张远端封面最大读取字节数
maxCoverDownloadBytes int64 = 16 << 20
)
var (
doubanBookURLPattern = regexp.MustCompile(`.*/subject/(\d+)/?`)
doubanDatePattern = regexp.MustCompile(`(\d{4})-(\d+)`)
doubanDefaultHeaders = map[string]string{
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
"Referer": doubanBase,
}
)
// DoubanScraper 豆瓣书籍刮削器（移植自 NewDouban.py）
type DoubanScraper struct {
client *http.Client
// ThumbnailMode 缩略图存储方式："base64"（默认，存入数据库）或 "local"（存到本地文件）
ThumbnailMode string
// ThumbnailPath 本地存储路径，ThumbnailMode 为 "local" 时有效，默认 "/.thumbnail"
ThumbnailPath string
}
// NewDoubanScraper 创建豆瓣刮削器
func NewDoubanScraper() *DoubanScraper {
return &DoubanScraper{
client: &http.Client{Timeout: 20 * time.Second},
ThumbnailMode: "base64",
ThumbnailPath: "/.thumbnail",
}
}
// NewDoubanScraperWithConfig 创建带完整配置的豆瓣刮削器
func NewDoubanScraperWithConfig(thumbnailMode, thumbnailPath string) *DoubanScraper {
if thumbnailMode == "" {
thumbnailMode = "base64"
}
if thumbnailPath == "" {
thumbnailPath = "/.thumbnail"
}
return &DoubanScraper{
client: &http.Client{Timeout: 20 * time.Second},
ThumbnailMode: thumbnailMode,
ThumbnailPath: thumbnailPath,
}
}
// ScrapeBook 刮削书籍信息
func (s *DoubanScraper) ScrapeBook(item *model.MediaItem) error {
query := item.ScrapedName
if query == "" {
query = strings.TrimSuffix(item.FileName, strings.ToLower(item.FileName[strings.LastIndex(item.FileName, "."):]))
}
// 搜索书籍URL列表
bookURLs, err := s.searchBookURLs(query)
if err != nil {
return fmt.Errorf("豆瓣搜索失败: %w", err)
}
if len(bookURLs) == 0 {
return fmt.Errorf("豆瓣未找到匹配书籍: %s", query)
}
// 取第一个结果
bookDetail, err := s.loadBookDetail(bookURLs[0])
if err != nil {
return fmt.Errorf("豆瓣获取书籍详情失败: %w", err)
}
// 填充字段（仅填空字段，已有值不覆盖）
if item.ScrapedName == "" && bookDetail.Title != "" {
item.ScrapedName = bookDetail.Title
}
if item.Plot == "" {
item.Plot = bookDetail.Description
}
// 仅在豆瓣返回了有效封面图片URL且当前 item 还没有封面时才下载/覆盖
if item.Cover == "" && bookDetail.Cover != "" {
if cover := s.downloadAndStoreCover(item.FolderPath, bookDetail.Cover); cover != "" {
item.Cover = cover
} else {
// 下载失败时保留原 URL（降级）
item.Cover = bookDetail.Cover
}
}
if item.Rating == 0 {
item.Rating = bookDetail.Rating
}
if item.ReleaseDate == "" {
item.ReleaseDate = bookDetail.PublishedDate
}
if item.Publisher == "" {
item.Publisher = bookDetail.Publisher
}
if item.ISBN == "" {
item.ISBN = bookDetail.ISBN
}
if item.ExternalID == "" {
item.ExternalID = "douban:" + bookDetail.ID
}
if item.Authors == "" && len(bookDetail.Authors) > 0 {
authorsJSON, _ := json.Marshal(bookDetail.Authors)
item.Authors = string(authorsJSON)
}
if item.Genre == "" && len(bookDetail.Tags) > 0 {
item.Genre = strings.Join(bookDetail.Tags, ",")
}
now := time.Now()
item.ScrapedAt = &now
return nil
}
type doubanBookDetail struct {
ID string
Title string
Authors []string
Publisher string
PublishedDate string
Cover string
Rating float32
Description string
Tags []string
ISBN string
}
// searchBookURLs 搜索豆瓣书籍URL列表
func (s *DoubanScraper) searchBookURLs(query string) ([]string, error) {
searchURL := fmt.Sprintf("%s?cat=%s&q=%s", doubanSearchURL, doubanBookCat, url.QueryEscape(query))
body, err := s.doGet(searchURL)
if err != nil {
return nil, err
}
doc, err := html.Parse(strings.NewReader(string(body)))
if err != nil {
return nil, err
}
var bookURLs []string
var traverse func(*html.Node)
traverse = func(n *html.Node) {
if n.Type == html.ElementNode && n.Data == "a" {
for _, attr := range n.Attr {
if attr.Key == "class" && attr.Val == "nbg" {
for _, a := range n.Attr {
if a.Key == "href" {
parsed := s.calcURL(a.Val)
if parsed != "" && len(bookURLs) < 5 {
bookURLs = append(bookURLs, parsed)
}
}
}
}
}
}
for c := n.FirstChild; c != nil; c = c.NextSibling {
traverse(c)
}
}
traverse(doc)
return bookURLs, nil
}
// calcURL 解析豆瓣搜索结果中的真实URL
func (s *DoubanScraper) calcURL(href string) string {
parsed, err := url.Parse(href)
if err != nil {
return ""
}
query := parsed.Query()
rawURL := query.Get("url")
if rawURL == "" {
return ""
}
decoded, err := url.QueryUnescape(rawURL)
if err != nil {
return ""
}
if doubanBookURLPattern.MatchString(decoded) {
return decoded
}
return ""
}
// loadBookDetail 加载书籍详情页
func (s *DoubanScraper) loadBookDetail(bookURL string) (*doubanBookDetail, error) {
body, err := s.doGet(bookURL)
if err != nil {
return nil, err
}
return s.parseBookHTML(bookURL, string(body))
}
// parseBookHTML 解析书籍HTML（移植自 DoubanBookHtmlParser.parse_book）
func (s *DoubanScraper) parseBookHTML(bookURL, content string) (*doubanBookDetail, error) {
doc, err := html.Parse(strings.NewReader(content))
if err != nil {
return nil, err
}
detail := &doubanBookDetail{}
// 提取豆瓣ID
if m := doubanBookURLPattern.FindStringSubmatch(bookURL); len(m) > 1 {
detail.ID = m[1]
}
var traverse func(*html.Node)
traverse = func(n *html.Node) {
if n.Type == html.ElementNode {
switch n.Data {
case "span":
// 标题
if getAttr(n, "property") == "v:itemreviewed" {
detail.Title = getTextContent(n)
}
// 元数据字段
if getAttr(n, "class") == "pl" {
text := getTextContent(n)
tail := getNextText(n)
switch {
case strings.HasPrefix(text, "作者") || strings.HasPrefix(text, "译者"):
// 作者通过链接提取
collectAuthors(n, &detail.Authors)
case strings.HasPrefix(text, "出版社"):
detail.Publisher = strings.TrimSpace(tail)
case strings.HasPrefix(text, "出版年"):
detail.PublishedDate = parseDoubanDate(strings.TrimSpace(tail))
case strings.HasPrefix(text, "ISBN"):
detail.ISBN = strings.TrimSpace(tail)
case strings.HasPrefix(text, "副标题"):
if detail.Title != "" {
detail.Title += ":" + strings.TrimSpace(tail)
}
}
}
// 评分
if getAttr(n, "property") == "v:average" {
ratingStr := getTextContent(n)
var rating float32
fmt.Sscanf(ratingStr, "%f", &rating)
detail.Rating = rating / 2
}
case "img":
// 封面图片：<img class="nbg" src="..." />
if getAttr(n, "class") == "nbg" {
src := getAttr(n, "src")
if src != "" && !strings.HasSuffix(src, "update_image") {
detail.Cover = src
}
}
// 备用：#mainpic 下的 img
if getAttr(n, "id") == "mainpic" || (n.Parent != nil && getAttr(n.Parent, "id") == "mainpic") {
src := getAttr(n, "src")
if src != "" && detail.Cover == "" {
detail.Cover = src
}
}
case "a":
// 标签（原封面逻辑移除，改为从img获取）
cls := getAttr(n, "class")
if strings.Contains(cls, "tag") {
tag := getTextContent(n)
if tag != "" {
detail.Tags = append(detail.Tags, tag)
}
}
case "div":
// 简介
if getAttr(n, "id") == "link-report" {
detail.Description = extractIntroText(n)
}
}
}
for c := n.FirstChild; c != nil; c = c.NextSibling {
traverse(c)
}
}
traverse(doc)
return detail, nil
}
// HTML辅助函数
func getAttr(n *html.Node, key string) string {
for _, attr := range n.Attr {
if attr.Key == key {
return attr.Val
}
}
return ""
}
func getTextContent(n *html.Node) string {
var sb strings.Builder
var walk func(*html.Node)
walk = func(node *html.Node) {
if node.Type == html.TextNode {
sb.WriteString(node.Data)
}
for c := node.FirstChild; c != nil; c = c.NextSibling {
walk(c)
}
}
walk(n)
return strings.TrimSpace(sb.String())
}
func getNextText(n *html.Node) string {
if n.NextSibling != nil {
if n.NextSibling.Type == html.TextNode {
return strings.TrimSpace(n.NextSibling.Data)
}
return getTextContent(n.NextSibling)
}
return ""
}
func collectAuthors(n *html.Node, authors *[]string) {
var walk func(*html.Node)
walk = func(node *html.Node) {
if node.Type == html.ElementNode && node.Data == "a" {
href := getAttr(node, "href")
if strings.Contains(href, "/author") || strings.Contains(href, "/search") {
name := getTextContent(node)
if name != "" {
*authors = append(*authors, name)
}
}
}
for c := node.FirstChild; c != nil; c = c.NextSibling {
walk(c)
}
}
// 从父节点开始找
if n.Parent != nil {
walk(n.Parent)
}
}
func extractIntroText(n *html.Node) string {
var sb strings.Builder
var walk func(*html.Node)
walk = func(node *html.Node) {
if node.Type == html.ElementNode && node.Data == "div" {
if getAttr(node, "class") == "intro" {
sb.WriteString(getTextContent(node))
return
}
}
for c := node.FirstChild; c != nil; c = c.NextSibling {
walk(c)
}
}
walk(n)
return strings.TrimSpace(sb.String())
}
func parseDoubanDate(dateStr string) string {
if dateStr == "" {
return ""
}
if m := doubanDatePattern.FindStringSubmatch(dateStr); len(m) > 2 {
return fmt.Sprintf("%s-%s-01", m[1], m[2])
}
return dateStr
}
// downloadAndStoreCover 下载封面图片并根据 ThumbnailMode 存储
// 失败时返回空字符串
func (s *DoubanScraper) downloadAndStoreCover(filePath, imgURL string) string {
data, mimeType := s.downloadImage(imgURL)
if len(data) == 0 {
return ""
}
if NormalizeThumbMode(s.ThumbnailMode) != ThumbModeBase64 {
// 运行目录 / 挂载目录模式
localScraper := NewBookLocalScraperWithConfig(s.ThumbnailMode, s.ThumbnailPath)
return localScraper.storeCover(filePath, data)
}
// Base64 模式（默认）
img, err := decodeBounded(data)
if err != nil {
// 解码失败时退回原始字节；超过阈值直接放弃，避免十几 MB 的图被 base64 写进 cover 列
if errors.Is(err, ErrTooLarge) || len(data) > MaxRawCoverFallbackBytes {
return ""
}
return "data:" + mimeType + ";base64," + base64.StdEncoding.EncodeToString(data)
}
thumb := imaging.Resize(img, 300, 0, imaging.Lanczos)
var buf bytes.Buffer
if err := jpeg.Encode(&buf, thumb, &jpeg.Options{Quality: 80}); err != nil {
return ""
}
return "data:image/jpeg;base64," + base64.StdEncoding.EncodeToString(buf.Bytes())
}
// downloadCoverAsBase64 下载封面图片并转为 data URI Base64 字符串（向后兼容）
// 失败时返回空字符串
func (s *DoubanScraper) downloadCoverAsBase64(imgURL string) string {
data, mimeType := s.downloadImage(imgURL)
if len(data) == 0 {
return ""
}
img, err := decodeBounded(data)
if err != nil {
// 解码失败时退回原始字节；超过阈值直接放弃，避免十几 MB 的图被 base64 写进 cover 列
if errors.Is(err, ErrTooLarge) || len(data) > MaxRawCoverFallbackBytes {
return ""
}
return "data:" + mimeType + ";base64," + base64.StdEncoding.EncodeToString(data)
}
thumb := imaging.Resize(img, 300, 0, imaging.Lanczos)
var buf bytes.Buffer
if err := jpeg.Encode(&buf, thumb, &jpeg.Options{Quality: 80}); err != nil {
return ""
}
return "data:image/jpeg;base64," + base64.StdEncoding.EncodeToString(buf.Bytes())
}
// downloadImage 下载图片，返回原始字节和 MIME 类型
func (s *DoubanScraper) downloadImage(imgURL string) ([]byte, string) {
req, err := http.NewRequest("GET", imgURL, nil)
if err != nil {
return nil, ""
}
for k, v := range doubanDefaultHeaders {
req.Header.Set(k, v)
}
resp, err := s.client.Do(req)
if err != nil {
return nil, ""
}
defer resp.Body.Close()
if resp.StatusCode != 200 {
return nil, ""
}
// 远端图片大小不可信，限制读取上限
data, err := io.ReadAll(io.LimitReader(resp.Body, maxCoverDownloadBytes))
if err != nil || len(data) == 0 {
return nil, ""
}
// 不信任远端 Content-Type：只接受魔数可识别的栅格图片，防止 SVG/HTML 被当作封面
mimeType := imgmime.Sniff(data)
if mimeType == "" {
return nil, ""
}
return data, mimeType
}
// doGet 发送GET请求
func (s *DoubanScraper) doGet(reqURL string) ([]byte, error) {
req, err := http.NewRequest("GET", reqURL, nil)
if err != nil {
return nil, err
}
for k, v := range doubanDefaultHeaders {
req.Header.Set(k, v)
}
resp, err := s.client.Do(req)
if err != nil {
return nil, err
}
defer resp.Body.Close()
if resp.StatusCode != 200 && resp.StatusCode != 201 {
return nil, fmt.Errorf("HTTP %d: %s", resp.StatusCode, reqURL)
}
return io.ReadAll(resp.Body)
}
internal/media/scraper/image.go
+196
View File
@@ -0,0 +1,196 @@
package scraper
import (
"bytes"
"context"
"fmt"
"image/jpeg"
"io"
"strings"
"time"
"github.com/disintegration/imaging"
"github.com/rwcarlsen/goexif/exif"
"github.com/OpenListTeam/OpenList/v4/internal/model"
)
// ImageScraper 图片刮削器
// 从图片 EXIF 中提取元数据，并生成缩略图写入 Cover 字段。
type ImageScraper struct {
// StoreThumbnail 为 true 时生成缩略图写入 Cover；
// 为 false 时直接用文件路径作为 Cover，节省数据库空间。
StoreThumbnail bool
// ThumbnailMode 缩略图存储方式："base64"（默认，存入数据库）或 "local"（存到本地文件）
ThumbnailMode string
// ThumbnailPath 本地存储路径，ThumbnailMode 为 "local" 时有效，默认 "/.thumbnail"
ThumbnailPath string
}
// NewImageScraper 创建图片刮削器
// storeThumbnail 为 true 时生成缩略图写入 Cover。
func NewImageScraper(storeThumbnail bool) *ImageScraper {
return &ImageScraper{
StoreThumbnail: storeThumbnail,
ThumbnailMode: "base64",
ThumbnailPath: "/.thumbnail",
}
}
// NewImageScraperWithConfig 创建带完整配置的图片刮削器
func NewImageScraperWithConfig(storeThumbnail bool, thumbnailMode, thumbnailPath string) *ImageScraper {
if thumbnailMode == "" {
thumbnailMode = "base64"
}
if thumbnailPath == "" {
thumbnailPath = "/.thumbnail"
}
return &ImageScraper{
StoreThumbnail: storeThumbnail,
ThumbnailMode: thumbnailMode,
ThumbnailPath: thumbnailPath,
}
}
// ScrapeImage 刮削图片信息
// reader 为图片文件内容流，必须传入有效流才能生成缩略图。
// 当 reader 不为 nil 时：
// - 从 EXIF 读取拍摄时间、GPS 地点、相机型号+参数、评分
// - 生成 300px 宽缩略图，根据 ThumbnailMode 存储为 Base64 或本地文件
//
// 注意：绝不将文件路径作为 cover，reader 为 nil 或缩略图生成失败时 cover 保持为空。
func (s *ImageScraper) ScrapeImage(item *model.MediaItem, reader io.Reader) error {
// 去掉扩展名作为展示名（如果尚未设置）
if item.ScrapedName == "" {
item.ScrapedName = trimExt(item.FileName)
}
if reader == nil {
// 无文件流时无法生成缩略图，cover 保持为空，等待下次重试
now := time.Now()
item.ScrapedAt = &now
return nil
}
// 将流读入内存（EXIF 解析和图像解码都需要完整数据）
data, err := readAllLimited(reader, MaxImageFileBytes)
if err != nil || len(data) == 0 {
// 读取失败时 cover 保持为空
now := time.Now()
item.ScrapedAt = &now
return nil
}
// ── 1. 解析 EXIF ──────────────────────────────────────────────
s.parseEXIF(item, bytes.NewReader(data))
// ── 2. 生成缩略图并按配置存储 ────────────────────────────────
if thumb := s.generateThumbBytes(data); len(thumb) > 0 {
fullPath := strings.TrimSuffix(item.FolderPath, "/") + "/" + item.FileName
item.Cover = SaveThumb(context.Background(), s.ThumbnailMode, s.ThumbnailPath, fullPath, "image/jpeg", thumb)
}
now := time.Now()
item.ScrapedAt = &now
return nil
}
// parseEXIF 从图片数据中解析 EXIF 元数据并写入 item
func (s *ImageScraper) parseEXIF(item *model.MediaItem, r io.Reader) {
x, err := exif.Decode(r)
if err != nil {
return
}
// ── 拍摄时间 → ReleaseDate ────────────────────────────────────
if item.ReleaseDate == "" {
if t, err := x.DateTime(); err == nil {
item.ReleaseDate = t.Format("2006-01-02")
}
}
// ── GPS 地点 → Authors（借用作者字段存储地点信息）────────────
if item.Authors == "" {
if lat, lon, err := x.LatLong(); err == nil {
item.Authors = fmt.Sprintf(`["GPS: %.6f, %.6f"]`, lat, lon)
}
}
// ── 相机型号 + 参数 → Genre ───────────────────────────────────
if item.Genre == "" {
var parts []string
if make, err := x.Get(exif.Make); err == nil {
if v, err := make.StringVal(); err == nil && v != "" {
parts = append(parts, strings.TrimSpace(v))
}
}
if model_, err := x.Get(exif.Model); err == nil {
if v, err := model_.StringVal(); err == nil && v != "" {
parts = append(parts, strings.TrimSpace(v))
}
}
if fnum, err := x.Get(exif.FNumber); err == nil {
if num, den, err := fnum.Rat2(0); err == nil && den != 0 {
parts = append(parts, fmt.Sprintf("f/%.1f", float64(num)/float64(den)))
}
}
if exp, err := x.Get(exif.ExposureTime); err == nil {
if num, den, err := exp.Rat2(0); err == nil && den != 0 {
if num == 1 {
parts = append(parts, fmt.Sprintf("1/%ds", den))
} else {
parts = append(parts, fmt.Sprintf("%d/%ds", num, den))
}
}
}
if iso, err := x.Get(exif.ISOSpeedRatings); err == nil {
if v, err := iso.Int(0); err == nil {
parts = append(parts, fmt.Sprintf("ISO%d", v))
}
}
if focal, err := x.Get(exif.FocalLength); err == nil {
if num, den, err := focal.Rat2(0); err == nil && den != 0 {
parts = append(parts, fmt.Sprintf("%.0fmm", float64(num)/float64(den)))
}
}
if len(parts) > 0 {
item.Genre = strings.Join(parts, ",")
}
}
// ── EXIF Rating（0-5 星）→ Rating（0-10 分）──────────────────
// Windows 资源管理器、Lightroom、Darktable 等软件会写入此标签
if item.Rating == 0 {
if ratingTag, err := x.Get(exif.FieldName("Rating")); err == nil {
if v, err := ratingTag.Int(0); err == nil && v > 0 {
// 1-5 星映射到 2-10 分
item.Rating = float32(v) * 2
}
}
}
}
// generateThumbBytes 将图片数据缩放为 300px 宽缩略图，返回 JPEG 字节
func (s *ImageScraper) generateThumbBytes(data []byte) []byte {
img, err := decodeBounded(data)
if err != nil {
return nil
}
// 缩放到宽度 300px，高度等比
thumb := imaging.Resize(img, 300, 0, imaging.Lanczos)
var buf bytes.Buffer
if err := jpeg.Encode(&buf, thumb, &jpeg.Options{Quality: 75}); err != nil {
return nil
}
return buf.Bytes()
}
// trimExt 去掉文件名的扩展名
func trimExt(fileName string) string {
if idx := strings.LastIndex(fileName, "."); idx > 0 {
return fileName[:idx]
}
return fileName
}
internal/media/scraper/language.go
+27
View File
@@ -0,0 +1,27 @@
package scraper
import (
"strings"
"github.com/OpenListTeam/OpenList/v4/internal/conf"
"github.com/OpenListTeam/OpenList/v4/internal/setting"
)
// ScrapeLanguage 返回首选刮削语言（BCP-47，如 zh-CN / en-US），未配置时默认中文。
func ScrapeLanguage() string {
lang := strings.TrimSpace(setting.GetStr(conf.MediaScrapeLanguage, "zh-CN"))
if lang == "" {
return "zh-CN"
}
return lang
}
// ScrapeLanguagePrimary 返回 ISO 639-1 主语言代码（如 zh / en），
// 供只接受两位语言码的接口使用（Google Books 的 langRestrict）。
func ScrapeLanguagePrimary() string {
lang := ScrapeLanguage()
if idx := strings.IndexAny(lang, "-_"); idx > 0 {
lang = lang[:idx]
}
return strings.ToLower(lang)
}
internal/media/scraper/limits.go
+65
View File
@@ -0,0 +1,65 @@
package scraper
import (
"bytes"
"errors"
"image"
"io"
)
// 刮削阶段会把文件整体读入内存再解码，必须限制单文件大小与像素总量，
// 否则几路并发刮削一张超大图片/压缩炸弹就能把进程内存打满。
const (
// MaxImageFileBytes 单张待刮削图片的最大字节数
MaxImageFileBytes int64 = 32 << 20
// MaxBookFileBytes 单本待提取封面的书籍（epub/pdf）最大字节数
MaxBookFileBytes int64 = 128 << 20
// MaxEmbeddedEntryBytes epub(zip) 内单个封面条目的最大解压字节数，防止 zip 炸弹
MaxEmbeddedEntryBytes int64 = 16 << 20
// MaxImagePixels 解码前允许的最大像素总量（约 5657x5657）。
// 解码后按 RGBA 计算约 128MB，默认 5 路并发刮削最坏约 640MB；
// 原来的 64MP 单张就要 256MB，并发时会接近 1.3GB。
MaxImagePixels int64 = 32_000_000
// MaxRawCoverFallbackBytes 解码失败时退回「原始字节 base64 写库」的上限。
// 超过就直接放弃封面——否则单条记录可能写入 16MB 以上的 base64。
MaxRawCoverFallbackBytes = 512 << 10
)
// ErrTooLarge 内容超过读取上限
var ErrTooLarge = errors.New("scraper: content exceeds size limit")
// decodeSem 限制同时解码的图片数：解码阶段的内存峰值 ≈ 像素数 × 4，
// 只靠单张上限无法约束多路并发叠加后的总量。
// 容量必须小于默认刮削并发（5），否则信号量等于不起作用：
// 每路最坏 32MP × 4B ≈ 128MB，12 路就是 1.5GB。
// 解码是 CPU 密集型操作，并发超过核数也不会更快，取 3 兼顾内存与吞吐。
var decodeSem = make(chan struct{}, 3)
// readAllLimited 最多读取 max 字节；超过上限返回 ErrTooLarge（不会继续读取剩余内容）。
func readAllLimited(r io.Reader, max int64) ([]byte, error) {
data, err := io.ReadAll(io.LimitReader(r, max+1))
if err != nil {
return nil, err
}
if int64(len(data)) > max {
return nil, ErrTooLarge
}
return data, nil
}
// decodeBounded 先只读取图片头部获取尺寸，超过像素上限直接拒绝，再做完整解码。
func decodeBounded(data []byte) (image.Image, error) {
cfg, _, err := image.DecodeConfig(bytes.NewReader(data))
if err != nil {
return nil, err
}
// 用 int64 相乘：32 位平台上 int 溢出会让超限图片绕过检查
if cfg.Width <= 0 || cfg.Height <= 0 ||
int64(cfg.Width)*int64(cfg.Height) > MaxImagePixels {
return nil, ErrTooLarge
}
decodeSem <- struct{}{}
defer func() { <-decodeSem }()
img, _, err := image.Decode(bytes.NewReader(data))
return img, err
}
internal/media/scraper/thumb_store.go
+160
View File
@@ -0,0 +1,160 @@
package scraper
import (
"bytes"
"context"
"crypto/sha1"
"encoding/base64"
"encoding/hex"
"fmt"
"io"
"net/http"
"net/url"
"os"
stdpath "path"
"path/filepath"
"strings"
"time"
"github.com/OpenListTeam/OpenList/v4/internal/conf"
"github.com/OpenListTeam/OpenList/v4/internal/fs"
"github.com/OpenListTeam/OpenList/v4/internal/model"
"github.com/OpenListTeam/OpenList/v4/internal/sign"
"github.com/OpenListTeam/OpenList/v4/internal/stream"
)
// 缩略图存储模式
const (
// ThumbModeBase64 缩略图以 base64 存入数据库
ThumbModeBase64 = "base64"
// ThumbModeRuntime 缩略图存入程序数据目录（<data>/<thumbnail_path>）
ThumbModeRuntime = "runtime"
// ThumbModeVFS 缩略图写入挂载目录（VFS 路径）
ThumbModeVFS = "local"
)
// NormalizeThumbMode 归一化缩略图模式，未知/空值按运行目录处理
func NormalizeThumbMode(mode string) string {
switch strings.ToLower(strings.TrimSpace(mode)) {
case ThumbModeBase64:
return ThumbModeBase64
case ThumbModeVFS, "vfs":
return ThumbModeVFS
default:
return ThumbModeRuntime
}
}
// RuntimeThumbRoot 返回运行目录模式下缩略图的本地根目录
func RuntimeThumbRoot(thumbPath string) string {
dir := strings.Trim(strings.TrimSpace(thumbPath), "/")
if dir == "" {
dir = "imgs"
}
return filepath.Join(conf.DataDir, filepath.FromSlash(dir))
}
// SaveThumb 按模式保存缩略图数据，返回 cover 字段值。
// rawPath 为媒体文件的完整 VFS 路径，用于生成稳定且唯一的文件名。
func SaveThumb(ctx context.Context, mode, thumbPath, rawPath, mimeType string, data []byte) string {
if len(data) == 0 {
return ""
}
mode = NormalizeThumbMode(mode)
if mimeType == "" {
mimeType = "image/jpeg"
}
if mode == ThumbModeBase64 {
return "data:" + mimeType + ";base64," + base64.StdEncoding.EncodeToString(data)
}
dir := strings.Trim(strings.TrimSpace(thumbPath), "/")
if dir == "" {
dir = "imgs"
}
name := thumbName(rawPath)
if mode == ThumbModeVFS {
vfsDir := "/" + dir
vfsFile := stdpath.Join(vfsDir, name)
file := &stream.FileStream{
Ctx: ctx,
Obj: &model.Object{
Name: name,
Size: int64(len(data)),
Modified: time.Now(),
},
Reader: bytes.NewReader(data),
Mimetype: mimeType,
}
if err := fs.PutDirectly(ctx, vfsDir, file, true); err != nil {
return ""
}
// 带上签名，兼容开启「签名所有」的站点
return "/d" + vfsFile + "?sign=" + sign.Sign(vfsFile)
}
// 运行目录模式：<data>/<dir>/<name>
root := RuntimeThumbRoot(dir)
if err := os.MkdirAll(root, 0755); err != nil {
return ""
}
if err := os.WriteFile(filepath.Join(root, name), data, 0644); err != nil {
return ""
}
return "thumb:/" + stdpath.Join(dir, name)
}
// thumbName 根据媒体文件路径生成缩略图文件名
func thumbName(rawPath string) string {
sum := sha1.Sum([]byte(rawPath))
return hex.EncodeToString(sum[:]) + ".jpg"
}
// DownloadCoverToStorage 下载远程封面并按缩略图配置保存，失败返回空字符串
func DownloadCoverToStorage(ctx context.Context, imgURL, rawPath, thumbMode, thumbPath string) string {
if imgURL == "" ||
!(strings.HasPrefix(imgURL, "http://") || strings.HasPrefix(imgURL, "https://")) {
return ""
}
data, contentType, err := fetchImageBytes(ctx, imgURL)
if err != nil {
// 部分网络环境无法直连图片域名，回退到公共图片网关
if u, parseErr := url.Parse(imgURL); parseErr == nil {
gateway := "https://wsrv.nl/?url=" + url.QueryEscape(
strings.TrimPrefix(strings.TrimPrefix(u.String(), "https://"), "http://"),
)
data, contentType, err = fetchImageBytes(ctx, gateway)
}
}
if err != nil || len(data) == 0 {
return ""
}
return SaveThumb(ctx, thumbMode, thumbPath, rawPath, contentType, data)
}
func fetchImageBytes(ctx context.Context, raw string) ([]byte, string, error) {
client := &http.Client{Timeout: 20 * time.Second}
req, err := http.NewRequestWithContext(ctx, http.MethodGet, raw, nil)
if err != nil {
return nil, "", err
}
req.Header.Set("User-Agent", "Mozilla/5.0 (compatible; OpenList-Media/1.0)")
resp, err := client.Do(req)
if err != nil {
return nil, "", err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return nil, "", fmt.Errorf("unexpected status %d", resp.StatusCode)
}
contentType := resp.Header.Get("Content-Type")
if contentType == "" {
contentType = "image/jpeg"
}
if !strings.HasPrefix(contentType, "image/") {
return nil, "", fmt.Errorf("unexpected content type %s", contentType)
}
data, err := io.ReadAll(io.LimitReader(resp.Body, 10<<20))
return data, contentType, err
}
internal/media/scraper/tmdb.go
+1448
View File
File diff suppressed because it is too large. Load diff
internal/media/strm.go
+336
View File
@@ -0,0 +1,336 @@
package media
import (
"bytes"
"context"
"encoding/json"
"fmt"
stdpath "path"
"strings"
"sync/atomic"
"time"
"github.com/OpenListTeam/OpenList/v4/internal/conf"
"github.com/OpenListTeam/OpenList/v4/internal/db"
"github.com/OpenListTeam/OpenList/v4/internal/fs"
"github.com/OpenListTeam/OpenList/v4/internal/model"
"github.com/OpenListTeam/OpenList/v4/internal/setting"
"github.com/OpenListTeam/OpenList/v4/internal/sign"
"github.com/OpenListTeam/OpenList/v4/internal/stream"
"github.com/OpenListTeam/OpenList/v4/pkg/utils"
log "github.com/sirupsen/logrus"
)
var apiRootHolder atomic.Value
// SetAPIRoot 记录本次运行使用的站点地址（由 HTTP 层注入，供异步任务复用）
func SetAPIRoot(u string) {
if u == "" {
return
}
apiRootHolder.Store(strings.TrimSuffix(u, "/"))
}
// GetAPIRoot 返回当前站点地址
func GetAPIRoot() string {
if v, ok := apiRootHolder.Load().(string); ok && v != "" {
return v
}
return strings.TrimSuffix(conf.Conf.SiteURL, "/")
}
// StrmOptions STRM 生成参数
type StrmOptions struct {
APIRoot string // 站点地址前缀，例如 https://example.com
Template string // 内容模板
Overwrite bool // 是否覆盖已存在文件
SameDir bool // 是否与媒体文件同目录（同名 .strm）
OutputPath string // 输出目录（VFS 路径，仅 SameDir=false 时使用）
}
// StrmTarget 一个待生成的 .strm 目标
type StrmTarget struct {
RelPath string // 相对输出目录的路径
RawPath string // 对应媒体文件的 VFS 路径
SameDirPath string // 与媒体文件同目录时的 .strm 路径
}
// strmSibling 计算与媒体文件同目录、同名的 .strm 路径
func strmSibling(rawPath string, isDir bool) string {
if isDir {
return stdpath.Join(rawPath, stdpath.Base(rawPath)+".strm")
}
ext := stdpath.Ext(rawPath)
if ext == "" {
return rawPath + ".strm"
}
return strings.TrimSuffix(rawPath, ext) + ".strm"
}
// DefaultStrmOptions 从设置中读取 STRM 参数
func DefaultStrmOptions(apiRoot string) StrmOptions {
return StrmOptions{
APIRoot: strings.TrimSuffix(apiRoot, "/"),
Template: setting.GetStr(conf.MediaStrmTemplate, "{site_url}/d{path}?sign={sign}"),
Overwrite: setting.GetBool(conf.MediaStrmOverwrite),
SameDir: setting.GetBool(conf.MediaStrmSameDir),
OutputPath: setting.GetStr(conf.MediaStrmPath, "/strm"),
}
}
// RenderStrmContent 渲染 .strm 内容（单行 URL）
func RenderStrmContent(tpl, apiRoot, rawPath string) string {
if tpl == "" {
tpl = "{site_url}/d{path}?sign={sign}"
}
apiRoot = strings.TrimSuffix(apiRoot, "/")
p := rawPath
if !strings.HasPrefix(p, "/") {
p = "/" + p
}
replaced := strings.NewReplacer(
"{site_url}", apiRoot,
"{path}", utils.EncodePath(p),
"{sign}", sign.Sign(rawPath),
"{raw_path}", p,
"{name}", stdpath.Base(p),
).Replace(tpl)
return strings.TrimSpace(replaced)
}
// sanitizeName 清洗文件/目录名中的非法字符
func sanitizeName(s string) string {
s = strings.TrimSpace(s)
if s == "" {
return "Unknown"
}
replacer := strings.NewReplacer(
"/", "_", "\\", "_", ":", "_", "*", "_", "?", "_",
"\"", "_", "<", "_", ">", "_", "|", "_",
"\n", " ", "\r", " ",
)
s = replacer.Replace(s)
if runes := []rune(s); len(runes) > 120 {
s = string(runes[:120])
}
return strings.TrimSpace(s)
}
func itemTitle(item *model.MediaItem) string {
if item.ScrapedName != "" {
return item.ScrapedName
}
name := item.FileName
if !item.IsFolder {
name = strings.TrimSuffix(name, stdpath.Ext(name))
}
return name
}
func itemYear(item *model.MediaItem) string {
if len(item.ReleaseDate) >= 4 {
return item.ReleaseDate[:4]
}
return ""
}
func parseEpisodes(item *model.MediaItem) []model.EpisodeInfo {
if item.Episodes == "" {
return nil
}
var eps []model.EpisodeInfo
if err := json.Unmarshal([]byte(item.Episodes), &eps); err != nil {
return nil
}
return eps
}
// MediaItemFolder 条目所在目录
func MediaItemFolder(item *model.MediaItem) string {
if item == nil {
return "/"
}
folder := strings.TrimSuffix(item.FolderPath, "/")
if folder == "" {
folder = "/"
}
return folder
}
// StrmTargets 计算条目对应的 STRM 目标列表。
// 图片/书籍返回空列表（不生成 STRM）。
func StrmTargets(item *model.MediaItem) []StrmTarget {
if item == nil {
return nil
}
title := sanitizeName(itemTitle(item))
year := itemYear(item)
folder := MediaItemFolder(item)
switch item.MediaType {
case model.MediaTypeVideo:
if item.VideoType == "tv" {
series := title
season := item.Season
if season <= 0 {
season = 1
}
var out []StrmTarget
if item.IsFolder {
for i, ep := range parseEpisodes(item) {
num := ep.Index
if num <= 0 {
num = i + 1
}
epTitle := sanitizeName(ep.Title)
epRaw := stdpath.Join(folder, item.FileName, ep.FileName)
out = append(out, StrmTarget{
RelPath: stdpath.Join(series, fmt.Sprintf("Season %02d", season),
fmt.Sprintf("%s S%02dE%02d - %s.strm", series, season, num, epTitle)),
RawPath: epRaw,
SameDirPath: strmSibling(epRaw, false),
})
}
return out
}
num := item.Episode
if num <= 0 {
num = 1
}
raw := stdpath.Join(folder, item.FileName)
return []StrmTarget{{
RelPath: stdpath.Join(series, fmt.Sprintf("Season %02d", season),
fmt.Sprintf("%s S%02dE%02d.strm", series, season, num)),
RawPath: raw,
SameDirPath: strmSibling(raw, false),
}}
}
name := title
if year != "" {
name = fmt.Sprintf("%s (%s)", title, year)
}
raw := stdpath.Join(folder, item.FileName)
return []StrmTarget{{
RelPath: stdpath.Join(name, name+".strm"),
RawPath: raw,
SameDirPath: strmSibling(raw, item.IsFolder),
}}
case model.MediaTypeMusic:
artist := sanitizeName(item.AlbumArtist)
if artist == "" {
artist = "Unknown Artist"
}
album := sanitizeName(item.AlbumName)
if album == "" {
album = "Unknown Album"
}
track := item.TrackNumber
if track <= 0 {
track = 1
}
raw := stdpath.Join(folder, item.FileName)
return []StrmTarget{{
RelPath: stdpath.Join(artist, album, fmt.Sprintf("%02d - %s.strm", track, title)),
RawPath: raw,
SameDirPath: strmSibling(raw, false),
}}
default:
return nil
}
}
// GenerateStrmForItems 为一组条目生成 STRM 文件，返回写入数量
func GenerateStrmForItems(ctx context.Context, items []model.MediaItem, opts StrmOptions) (int, error) {
if !opts.SameDir && opts.OutputPath == "" {
return 0, fmt.Errorf("strm output path is empty")
}
// 归一输出根目录，后续用它校验每个目标文件都在根目录之内
outRoot := ""
if !opts.SameDir {
outRoot = stdpath.Clean("/" + strings.TrimSpace(opts.OutputPath))
}
written := 0
for i := range items {
item := &items[i]
if item.Hidden {
continue
}
for _, target := range StrmTargets(item) {
dst := target.SameDirPath
if !opts.SameDir {
dst = stdpath.Join(outRoot, target.RelPath)
// RelPath 含刮削得到的艺术家/专辑名等外部数据，必须保证不会借 ../ 逃出输出目录
if dst != outRoot && !strings.HasPrefix(dst, strings.TrimSuffix(outRoot, "/")+"/") {
continue
}
}
if dst == "" {
continue
}
if !opts.Overwrite {
if _, err := fs.Get(ctx, dst, &fs.GetArgs{NoLog: true}); err == nil {
continue
}
}
data := []byte(RenderStrmContent(opts.Template, opts.APIRoot, target.RawPath))
file := &stream.FileStream{
Ctx: ctx,
Obj: &model.Object{
Name: stdpath.Base(dst),
Size: int64(len(data)),
Modified: time.Now(),
},
Reader: bytes.NewReader(data),
Mimetype: "text/plain",
}
if err := fs.PutDirectly(ctx, stdpath.Dir(dst), file, true); err != nil {
return written, fmt.Errorf("write %s: %w", dst, err)
}
written++
}
}
return written, nil
}
// GenerateStrmForType 按媒体类型（可选扫描路径）批量生成 STRM
func GenerateStrmForType(ctx context.Context, mediaType model.MediaType, scanPathID uint, opts StrmOptions) (int, error) {
var items []model.MediaItem
var err error
if scanPathID > 0 {
items, err = db.ListMediaItemsByScanPath(scanPathID)
} else {
items, err = db.ListAllMediaItems(mediaType)
}
if err != nil {
return 0, err
}
if mediaType != "" {
filtered := make([]model.MediaItem, 0, len(items))
for _, it := range items {
if it.MediaType == mediaType {
filtered = append(filtered, it)
}
}
items = filtered
}
return GenerateStrmForItems(ctx, items, opts)
}
// TriggerStrmOnScan 扫描结束后按需异步生成 STRM
func TriggerStrmOnScan(mediaType model.MediaType, apiRoot string) {
if !setting.GetBool(conf.MediaStrmOnScan) {
return
}
opts := DefaultStrmOptions(apiRoot)
if !opts.SameDir && opts.OutputPath == "" {
return
}
go func() {
n, err := GenerateStrmForType(context.Background(), mediaType, 0, opts)
if err != nil {
log.Errorf("[media] generate strm for %s failed: %+v", mediaType, err)
return
}
log.Infof("[media] generated %d strm files for %s", n, mediaType)
}()
}
internal/model/args.go
+7 -4
View File
@@ -96,18 +96,21 @@ type ArchiveDecompressArgs struct {
}
type SharingListArgs struct {
Refresh bool
Pwd string
Refresh bool
Pwd string
ClientIP string
}
type SharingArchiveMetaArgs struct {
ArchiveMetaArgs
Pwd string
Pwd string
ClientIP string
}
type SharingArchiveListArgs struct {
ArchiveListArgs
Pwd string
Pwd string
ClientIP string
}
type SharingLinkArgs struct {
internal/model/media.go
+129
View File
@@ -0,0 +1,129 @@
package model
import (
"time"
"gorm.io/gorm"
)
// MediaType 媒体类型，使用字符串便于后期扩展
type MediaType string
const (
MediaTypeVideo MediaType = "video"
MediaTypeMusic MediaType = "music"
MediaTypeImage MediaType = "image"
MediaTypeBook MediaType = "book"
)
// MediaScanPath 媒体库扫描路径（每个媒体库可配置多个扫描路径）
type MediaScanPath struct {
ID uint `gorm:"primarykey" json:"id"`
CreatedAt time.Time `json:"created_at"`
UpdatedAt time.Time `json:"updated_at"`
DeletedAt gorm.DeletedAt `gorm:"index" json:"-"`
MediaType MediaType `gorm:"size:32;index;not null" json:"media_type"` // 所属媒体类型
Name string `json:"name"` // 路径名称（用于前端筛选显示）
Path string `gorm:"not null" json:"path"` // VFS 扫描路径
PathMerge bool `gorm:"default:false" json:"path_merge"` // 路径合并模式（子文件夹作为一个条目）
TypeTag string `json:"type_tag"` // 类型标签：电影、电视剧等
ContentTags string `json:"content_tags"` // 内容标签：喜剧、惊悚等（逗号分隔）
EnableScrape bool `gorm:"default:true" json:"enable_scrape"` // 是否启用刮削
LastScanAt *time.Time `json:"last_scan_at"` // 最后扫描时间
}
// MediaIndexColumnMax 复合唯一索引列（folder_path / file_name / album_name）的最大字符数
const MediaIndexColumnMax = 191
// MediaItem 媒体条目（统一表，通过 media_type 区分类型，便于后期扩展新类型）
type MediaItem struct {
// 显式声明基础字段，避免同时嵌入 gorm.Model 后重复生成 deleted_at 索引列；
// JSON 标签继续保持媒体 API 的字段命名约定。
ID uint `gorm:"primarykey" json:"id"`
CreatedAt time.Time `json:"created_at"`
UpdatedAt time.Time `json:"updated_at"`
DeletedAt gorm.DeletedAt `gorm:"index" json:"-"`
// 基础信息
MediaType MediaType `gorm:"size:32;index;not null" json:"media_type"`
ScanPathID uint `gorm:"index" json:"scan_path_id"` // 关联的扫描路径ID
// 三个字段共同组成复合唯一索引。MySQL 对「有索引且未指定长度」的 string 会生成 longtext，
// 而 TEXT 列不能直接建索引（AutoMigrate 会失败）；因此必须显式限定 size。
// utf8mb4 下 InnoDB 索引总长上限 3072 字节：191*3*4 = 2292，留足余量。
FileName string `gorm:"size:191;uniqueIndex:idx_media_folder_file_album" json:"file_name"`
FileSize int64 `json:"file_size"`
MimeType string `json:"mime_type"`
Hidden bool `gorm:"default:false" json:"hidden"`
// 刮削/编辑信息
ScrapedName string `json:"scraped_name"`
Description string `gorm:"type:text" json:"description"`
Cover string `json:"cover"` // 封面URL或本地路径
ReleaseDate string `json:"release_date"` // 发布时间，格式 YYYY-MM-DD
Rating float32 `json:"rating"` // 评分 0-10
Genre string `json:"genre"` // 类型，逗号分隔，如 "动作,科幻"
Authors string `gorm:"type:text" json:"authors"` // 作者/演员，JSON数组字符串
Plot string `gorm:"type:text" json:"plot"` // 剧情/内容介绍
Reviews string `gorm:"type:text" json:"reviews"` // 用户评价，JSON数组字符串
// 外部ID（用于刮削关联）
ExternalID string `json:"external_id"` // TMDB ID / Discogs ID / 豆瓣ID
// 音乐专属字段
AlbumName string `gorm:"size:191;uniqueIndex:idx_media_folder_file_album" json:"album_name"` // 所属专辑名
AlbumArtist string `json:"album_artist"` // 专辑艺术家
TrackNumber int `json:"track_number"` // 曲目编号
Duration int `json:"duration"` // 时长（秒）
Lyrics string `gorm:"type:text" json:"lyrics"` // LRC格式歌词
// 视频专属字段
VideoType string `json:"video_type"` // "movie" 或 "tv"
Season int `json:"season"` // 季（电视剧）
Episode int `json:"episode"` // 集（电视剧）
// 书籍专属字段
Publisher string `json:"publisher"` // 出版社
ISBN string `json:"isbn"` // ISBN
// 目录合并模式
IsFolder bool `gorm:"default:false" json:"is_folder"` // 是否为文件夹模式条目
FolderPath string `gorm:"size:191;index;uniqueIndex:idx_media_folder_file_album" json:"folder_path"` // 扫描根路径（恒定为扫描路径，与file_name+album_name组合唯一）
Episodes string `gorm:"type:text" json:"episodes"` // 选集信息，JSON数组，格式：[{file_name,index,title},...]
ScrapedAt *time.Time `json:"scraped_at"`
}
// MediaConfig 媒体库配置（每种类型一条记录）
type MediaConfig struct {
ID uint `gorm:"primarykey" json:"id"`
CreatedAt time.Time `json:"created_at"`
UpdatedAt time.Time `json:"updated_at"`
DeletedAt gorm.DeletedAt `gorm:"index" json:"-"`
// 媒体类型只允许使用 MediaType 定义的短枚举值；显式长度使 MySQL 生成
// varchar 而不是无法建立索引的 longtext，满足该字段的唯一约束。
MediaType MediaType `gorm:"size:32;uniqueIndex;not null" json:"media_type"`
Enabled bool `gorm:"default:false" json:"enabled"`
LastScanAt *time.Time `json:"last_scan_at"`
LastScrapeAt *time.Time `json:"last_scrape_at"`
// ScraperConfig 该媒体类型独立的刮削配置（JSON）：
// store_thumbnail / thumbnail_mode / thumbnail_path，未设置时回落到全局设置
ScraperConfig string `gorm:"type:text" json:"scraper_config"`
}
// EpisodeInfo 选集信息（文件夹型媒体的单集描述）
type EpisodeInfo struct {
FileName string `json:"file_name"` // 原始文件名（含扩展名）
Index int `json:"index"` // 序号，默认0，文件名开头有数字则取该数字
Title string `json:"title"` // 选集标题（去掉序号后的文件名，不含扩展名）
}
// MediaScanProgress 扫描进度（内存中维护，不持久化）
type MediaScanProgress struct {
MediaType MediaType `json:"media_type"`
ScanPathID uint `json:"scan_path_id,omitempty"` // 0 表示全部路径
Running bool `json:"running"`
Total int `json:"total"`
Done int `json:"done"`
Message string `json:"message"`
Error string `json:"error,omitempty"`
}
internal/model/media_extra.go
+94
View File
@@ -0,0 +1,94 @@
package model
import "time"
// MediaAppKey 媒体应用密钥。
// 用于无法走交互式登录的客户端（Subsonic 的 token 认证、Emby 的 api_key、
// MoviePilot 对接等）。密钥本身以 AES-GCM 加密存储，数据库中不保留明文。
type MediaAppKey struct {
ID uint `gorm:"primarykey" json:"id"`
CreatedAt time.Time `json:"created_at"`
UpdatedAt time.Time `json:"updated_at"`
UserID uint `gorm:"index;not null" json:"user_id"`
Name string `json:"name"` // 备注名，例如 "Symfonium"、"MoviePilot"
// 前缀由 AppKeyPrefix 固定为最多 8 个 ASCII 字符；显式长度使 MySQL
// 将索引列建为 varchar，而不是无法建立索引的 longtext。
Prefix string `gorm:"size:32;index" json:"prefix"` // 明文前缀（便于识别/排查）
Secret string `gorm:"not null" json:"-"` // AES-GCM 密文（base64）
LastUsedAt *time.Time `json:"last_used_at"`
}
// MediaUserData 用户对媒体条目的播放状态，Emby 与 Subsonic 共用。
type MediaUserData struct {
ID uint `gorm:"primarykey" json:"id"`
CreatedAt time.Time `json:"created_at"`
UpdatedAt time.Time `json:"updated_at"`
UserID uint `gorm:"uniqueIndex:idx_media_user_item_ep;not null" json:"user_id"`
ItemID uint `gorm:"uniqueIndex:idx_media_user_item_ep;not null" json:"item_id"`
EpisodeIndex int `gorm:"uniqueIndex:idx_media_user_item_ep;not null" json:"episode_index"` // 0 表示条目本身，>0 表示选集序号
Played bool `json:"played"`
PlayCount int `json:"play_count"`
IsFavorite bool `gorm:"index" json:"is_favorite"`
Rating int `json:"rating"` // 0-5，Subsonic 语义
PositionTicks int64 `json:"position_ticks"` // 100ns tick，与 Emby 一致
LastPlayedAt *time.Time `json:"last_played_at"`
}
// MediaPlaylist 用户播放列表（Subsonic 需要，Emby 的 Playlist 复用同一份数据）
type MediaPlaylist struct {
ID uint `gorm:"primarykey" json:"id"`
CreatedAt time.Time `json:"created_at"`
UpdatedAt time.Time `json:"updated_at"`
UserID uint `gorm:"index;not null" json:"user_id"`
Name string `json:"name"`
Comment string `json:"comment"`
Public bool `json:"public"`
ItemCount int `json:"item_count"`
}
// MediaPlaylistItem 播放列表条目（按 Order 排序）
type MediaPlaylistItem struct {
ID uint `gorm:"primarykey" json:"id"`
PlaylistID uint `gorm:"index;not null" json:"playlist_id"`
ItemID uint `gorm:"index;not null" json:"item_id"`
EpisodeIndex int `json:"episode_index"`
Order int `gorm:"column:sort_order" json:"order"`
}
// MediaAnnotation 用户在 PDF 阅读器里添加的批注。
//
// 只存查看器原生的批注对象（Anchor，JSON）：页码、rect/quadPoints、颜色、
// 文字、绘制点集都在里面，拆成独立列会出现两份可能不一致的数据。
// 重新打开时原样回灌给 PDF 引擎，因此这里不重复建模。
type MediaAnnotation struct {
ID uint `gorm:"primarykey" json:"id"`
CreatedAt time.Time `json:"created_at"`
UpdatedAt time.Time `json:"updated_at"`
UserID uint `gorm:"uniqueIndex:idx_media_annotation_key;not null" json:"-"`
ItemID uint `gorm:"uniqueIndex:idx_media_annotation_key;not null;index" json:"item_id"`
// RemoteID 是查看器分配的批注 id：多设备打开同一份 PDF 时靠它做去重与更新
RemoteID string `gorm:"uniqueIndex:idx_media_annotation_key;size:128;not null" json:"remote_id"`
Anchor string `gorm:"type:text;not null" json:"anchor"`
}
// EmbyDevice Emby 客户端会话（由 AuthenticateByName 签发）
type EmbyDevice struct {
ID uint `gorm:"primarykey" json:"id"`
CreatedAt time.Time `json:"created_at"`
UserID uint `gorm:"index;not null" json:"user_id"`
DeviceID string `json:"device_id"` // 客户端上报的 DeviceId
DeviceName string `json:"device_name"`
Client string `json:"client"`
Version string `json:"version"`
// HashToken 始终返回 SHA-256 的 64 个十六进制字符，长度固定且适合唯一索引。
TokenHash string `gorm:"size:64;uniqueIndex;not null" json:"-"`
AppKeyID uint `json:"app_key_id"` // 通过应用密钥认证时为对应 key，否则为 0
LastSeenAt time.Time `json:"last_seen_at"`
}
internal/model/setting.go
+2
View File
@@ -13,6 +13,8 @@ const (
S3
FTP
TRAFFIC
MEDIA
TRANSCODE
)
const (
internal/model/sharing.go
+8 -3
View File
@@ -24,12 +24,17 @@ type Sharing struct {
}
func (s *Sharing) Valid() bool {
return s.Usable() && !s.LimitReached()
}
func (s *SharingDB) LimitReached() bool {
return s.MaxAccessed > 0 && s.Accessed >= s.MaxAccessed
}
func (s *Sharing) Usable() bool {
if s.Disabled {
return false
}
if s.MaxAccessed > 0 && s.Accessed >= s.MaxAccessed {
return false
}
if len(s.Files) == 0 {
return false
}
internal/model/storage.go
+1
View File
@@ -19,6 +19,7 @@ type Storage struct {
Disabled bool `json:"disabled"` // if disabled
DisableIndex bool `json:"disable_index"`
EnableSign bool `json:"enable_sign"`
SeedPolicy string `json:"seed_policy" gorm:"default:inherit"`
Sort
Proxy
}
internal/model/webdav_property.go
+35
View File
@@ -0,0 +1,35 @@
package model
import (
"crypto/sha256"
"encoding/hex"
"gorm.io/gorm"
)
// WebDAVProperty stores a dead WebDAV property for a resource.
//
// Path, Namespace and Name are TEXT columns so that long WebDAV paths are
// never truncated or rejected. MySQL cannot index TEXT columns without a key
// prefix length, so the unique index uses a SHA-256 hash of the complete
// property key instead.
type WebDAVProperty struct {
ID uint `json:"id" gorm:"primaryKey"`
Path string `json:"path" gorm:"type:text"`
Namespace string `json:"namespace" gorm:"type:text"`
Name string `json:"name" gorm:"type:text"`
KeyHash string `json:"-" gorm:"size:64;uniqueIndex:idx_webdav_property"`
Lang string `json:"lang"`
InnerXML []byte `json:"inner_xml"`
}
func (p *WebDAVProperty) BeforeSave(*gorm.DB) error {
p.KeyHash = HashWebDAVPropertyKey(p.Path, p.Namespace, p.Name)
return nil
}
// HashWebDAVPropertyKey returns a stable hash for the complete property key.
func HashWebDAVPropertyKey(path, namespace, name string) string {
sum := sha256.Sum256([]byte(path + "\x00" + namespace + "\x00" + name))
return hex.EncodeToString(sum[:])
}
internal/multipart/session_test.go
+19
View File
@@ -364,6 +364,19 @@ func TestInitResume(t *testing.T) {
if err != nil {
t.Fatalf("Init: %v", err)
}
m.mu.Lock()
firstSession := m.byID[snap.ID]
m.mu.Unlock()
sessions := []*Session{firstSession}
t.Cleanup(func() {
for _, session := range sessions {
if session == nil {
continue
}
m.terminate(session, context.Canceled)
<-session.done
}
})
sendChunk(t, m, user, snap.ID, data, 2, chunkSize)
// same hashes prove the same file: resume with buffered chunks skippable
@@ -383,6 +396,9 @@ func TestInitResume(t *testing.T) {
if err != nil || resumed || other.ID == snap.ID {
t.Fatalf("different-size Init = (id=%s, resumed=%v, err=%v)", other.ID, resumed, err)
}
m.mu.Lock()
sessions = append(sessions, m.byID[other.ID])
m.mu.Unlock()
if _, err := m.Find(user, "/local/test.bin", totalSize); err != nil {
t.Fatalf("Find: %v", err)
@@ -399,6 +415,9 @@ func TestInitResume(t *testing.T) {
if err != nil || resumed || fresh.ID == snap.ID {
t.Fatalf("bare re-Init = (id=%s, resumed=%v, err=%v), want a fresh session", fresh.ID, resumed, err)
}
m.mu.Lock()
sessions = append(sessions, m.byID[fresh.ID])
m.mu.Unlock()
if _, err := m.Status(user, snap.ID); !errors.Is(err, ErrSessionNotFound) {
t.Fatalf("superseded session err = %v, want ErrSessionNotFound", err)
}
internal/net/oss_test.go
+7 -3
View File
@@ -18,7 +18,7 @@ func TestNewOSSClientUsesEnvironmentHTTPSProxy(t *testing.T) {
t.Setenv("HTTP_PROXY", "")
t.Setenv("http_proxy", "")
t.Setenv("HTTPS_PROXY", "http://127.0.0.1:7890")
t.Setenv("https_proxy", "")
t.Setenv("https_proxy", "http://127.0.0.1:7890")
t.Setenv("NO_PROXY", "")
t.Setenv("no_proxy", "")
@@ -31,9 +31,13 @@ func TestNewOSSClientUsesEnvironmentHTTPSProxy(t *testing.T) {
t.Fatal("expected OSS client to use a custom HTTP client")
}
transport, ok := client.HTTPClient.Transport.(*http.Transport)
safe, ok := client.HTTPClient.Transport.(*safeTransport)
if !ok {
t.Fatalf("expected *http.Transport, got %T", client.HTTPClient.Transport)
t.Fatalf("expected *safeTransport, got %T", client.HTTPClient.Transport)
}
transport, ok := safe.base.(*http.Transport)
if !ok {
t.Fatalf("expected safe transport to wrap *http.Transport, got %T", safe.base)
}
if transport.Proxy == nil {
internal/net/serve.go
+5 -2
View File
@@ -268,7 +268,6 @@ var unsafeProxyHeaders = map[string]struct{}{
"x-real-ip": {},
}
func ProcessHeader(origin, override http.Header) http.Header {
result := http.Header{}
// client header
@@ -345,9 +344,13 @@ func HttpClient() *http.Client {
}
func NewHttpClient() *http.Client {
insecureSkipVerify := false
if conf.Conf != nil {
insecureSkipVerify = conf.Conf.TlsInsecureSkipVerify
}
transport := &http.Transport{
Proxy: http.ProxyFromEnvironment,
TLSClientConfig: &tls.Config{InsecureSkipVerify: conf.Conf.TlsInsecureSkipVerify},
TLSClientConfig: &tls.Config{InsecureSkipVerify: insecureSkipVerify},
}
SetProxyIfConfigured(transport)
internal/net/util.go
+2 -2
View File
@@ -69,7 +69,7 @@ func GetRangedHttpReader(readCloser io.ReadCloser, offset, length int64) (io.Rea
// SetProxyIfConfigured sets proxy for HTTP Transport if configured
func SetProxyIfConfigured(transport *http.Transport) {
// If proxy address is configured, override environment variable settings
if conf.Conf.ProxyAddress != "" {
if conf.Conf != nil && conf.Conf.ProxyAddress != "" {
if proxyURL, err := url.Parse(conf.Conf.ProxyAddress); err == nil {
transport.Proxy = http.ProxyURL(proxyURL)
}
@@ -79,7 +79,7 @@ func SetProxyIfConfigured(transport *http.Transport) {
// SetRestyProxyIfConfigured sets proxy for Resty client if configured
func SetRestyProxyIfConfigured(client *resty.Client) {
// If proxy address is configured, override environment variable settings
if conf.Conf.ProxyAddress != "" {
if conf.Conf != nil && conf.Conf.ProxyAddress != "" {
if proxyURL, err := url.Parse(conf.Conf.ProxyAddress); err == nil {
client.SetProxy(proxyURL.String())
}
internal/offline_download/115/client.go
+1 -1
View File
@@ -125,7 +125,7 @@ func (p *Cloud115) Status(task *tool.DownloadTask) (*tool.Status, error) {
s.Completed = t.IsDone()
s.TotalBytes = t.Size
if t.IsFailed() {
s.Err = fmt.Errorf(t.GetStatus())
s.Err = fmt.Errorf("%s", t.GetStatus())
}
return s, nil
}
internal/offline_download/115_open/client.go
+1 -1
View File
@@ -122,7 +122,7 @@ func (o *Open115) Status(task *tool.DownloadTask) (*tool.Status, error) {
s.Completed = t.IsDone()
s.TotalBytes = t.Size
if t.IsFailed() {
s.Err = fmt.Errorf(t.GetStatus())
s.Err = fmt.Errorf("%s", t.GetStatus())
}
return s, nil
}
internal/offline_download/pikpak/pikpak.go
+1 -1
View File
@@ -127,7 +127,7 @@ func (p *PikPak) Status(task *tool.DownloadTask) (*tool.Status, error) {
s.TotalBytes = 0
}
if t.Phase == "PHASE_TYPE_ERROR" {
s.Err = fmt.Errorf(t.Message)
s.Err = fmt.Errorf("%s", t.Message)
}
return s, nil
}
Loaded 100 of 161 files, more files were not shown because too many files have changed in this diff. Show more