diff --git a/.github/workflows/beta_release.yml b/.github/workflows/beta_release.yml index dc9c5e1b..10133feb 100644 --- a/.github/workflows/beta_release.yml +++ b/.github/workflows/beta_release.yml @@ -24,7 +24,7 @@ jobs: - name: Create or update ref id: create-or-update-ref - uses: ovsds/create-or-update-ref-action@1157b2991820188d5a8e6f22656775d1d171f762 # v1 + uses: ovsds/create-or-update-ref-action@f66e3f24fa250e50da7d3e599bae67bc3f749a5e # v1 with: ref: tags/beta sha: ${{ github.sha }} @@ -51,7 +51,7 @@ jobs: gh release upload beta "CHANGELOG.md" --clobber - name: Upload assets to github artifact - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 with: name: beta changelog path: ${{ github.workspace }}/CHANGELOG.md @@ -115,7 +115,7 @@ jobs: - name: Setup Go uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7 with: - go-version: "1.27.1" + go-version: "1.27.3" - name: Setup web run: bash build.sh dev web @@ -185,7 +185,7 @@ jobs: echo "cleaned_target=$CLEANED_TARGET" >> $GITHUB_ENV - name: Upload assets to github artifact - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 with: name: beta builds for ${{ env.cleaned_target }} path: ${{ github.workspace }}/build/compress/* diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index caa833ce..37556792 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -10,6 +10,22 @@ concurrency: cancel-in-progress: true jobs: + # 媒体库相关的安全/配额/迁移测试单独跑一次,不必跟随 7 个构建目标重复执行 + test: + name: Unit tests + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + + - name: Setup Go + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7 + with: + go-version: "1.27.3" + + - name: Go test + run: go test ./internal/db/... ./internal/transcode/... ./internal/media/... + build: strategy: matrix: @@ -33,7 +49,7 @@ jobs: - name: Setup Go uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7 with: - go-version: "1.27.1" + go-version: "1.27.3" - name: Setup web run: bash build.sh dev web @@ -69,7 +85,7 @@ jobs: fi - name: Upload artifact - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 with: name: openlist_${{ steps.short-sha.outputs.sha }}_${{ matrix.target }} path: build/* diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 99b721bc..8f847bc0 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -33,7 +33,7 @@ jobs: - name: Free Disk Space (Ubuntu) if: matrix.target-platform == '' - uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be # main + uses: jlumbroso/free-disk-space@36e9a5af4eddb91fc11aec4c6f6f79b3cc0cb523 # main with: tool-cache: false android: true @@ -46,7 +46,7 @@ jobs: - name: Setup Go uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7 with: - go-version: '1.27.1' + go-version: '1.27.3' - name: Checkout uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 diff --git a/.github/workflows/release_docker.yml b/.github/workflows/release_docker.yml index e8af8ebf..69f34715 100644 --- a/.github/workflows/release_docker.yml +++ b/.github/workflows/release_docker.yml @@ -47,7 +47,7 @@ jobs: - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7 with: - go-version: '1.27' + go-version: '1.27.3' - name: Cache Musl id: cache-musl @@ -69,7 +69,7 @@ jobs: FRONTEND_REPO: ${{ vars.FRONTEND_REPO }} - name: Upload artifacts - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 with: name: ${{ env.ARTIFACT_NAME }} overwrite: true @@ -87,7 +87,7 @@ jobs: - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7 with: - go-version: '1.27.1' + go-version: '1.27.3' - name: Cache Musl id: cache-musl @@ -109,7 +109,7 @@ jobs: FRONTEND_REPO: ${{ vars.FRONTEND_REPO }} - name: Upload artifacts - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 with: name: ${{ env.ARTIFACT_NAME_LITE }} overwrite: true @@ -147,20 +147,20 @@ jobs: steps: - name: Checkout uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8 with: name: ${{ env.ARTIFACT_NAME }} path: 'build/' - name: Set up QEMU - uses: docker/setup-qemu-action@1f40c72289eff860ee54a304f1438e3cff362e0a # v4 + uses: docker/setup-qemu-action@99012661954931238ded8c8b007157a8430204e1 # v4 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4 + uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4 - name: Login to GitHub Container Registry if: env.IMAGE_PUSH == 'true' - uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4 + uses: docker/login-action@81c6f93d4bc991e510b342a00b419b3bb4efc5c6 # v4 with: registry: ${{ env.REGISTRY }} username: ${{ github.actor }} @@ -168,7 +168,7 @@ jobs: - name: Login to DockerHub Container Registry if: env.IMAGE_PUSH == 'true' - uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4 + uses: docker/login-action@81c6f93d4bc991e510b342a00b419b3bb4efc5c6 # v4 with: username: ${{ vars.DOCKERHUB_ORG_NAME_BACKUP || env.DOCKERHUB_ORG_NAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} @@ -190,7 +190,7 @@ jobs: - name: Build and push id: docker_build - uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7 + uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7 with: context: . file: Dockerfile.ci @@ -231,20 +231,20 @@ jobs: steps: - name: Checkout uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8 with: name: ${{ env.ARTIFACT_NAME_LITE }} path: 'build/' - name: Set up QEMU - uses: docker/setup-qemu-action@1f40c72289eff860ee54a304f1438e3cff362e0a # v4 + uses: docker/setup-qemu-action@99012661954931238ded8c8b007157a8430204e1 # v4 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4 + uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4 - name: Login to GitHub Container Registry if: env.IMAGE_PUSH == 'true' - uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4 + uses: docker/login-action@81c6f93d4bc991e510b342a00b419b3bb4efc5c6 # v4 with: registry: ${{ env.REGISTRY }} username: ${{ github.actor }} @@ -252,7 +252,7 @@ jobs: - name: Login to DockerHub Container Registry if: env.IMAGE_PUSH == 'true' - uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4 + uses: docker/login-action@81c6f93d4bc991e510b342a00b419b3bb4efc5c6 # v4 with: username: ${{ vars.DOCKERHUB_ORG_NAME_BACKUP || env.DOCKERHUB_ORG_NAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} @@ -274,7 +274,7 @@ jobs: - name: Build and push id: docker_build - uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7 + uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7 with: context: . file: Dockerfile.ci diff --git a/.github/workflows/test_docker.yml b/.github/workflows/test_docker.yml index 84556617..d5fe6c68 100644 --- a/.github/workflows/test_docker.yml +++ b/.github/workflows/test_docker.yml @@ -36,7 +36,7 @@ jobs: - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7 with: - go-version: '1.27.1' + go-version: '1.27.3' - name: Cache Musl id: cache-musl @@ -58,7 +58,7 @@ jobs: FRONTEND_REPO: ${{ vars.FRONTEND_REPO }} - name: Upload artifacts - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 with: name: ${{ env.ARTIFACT_NAME }} overwrite: true @@ -98,20 +98,20 @@ jobs: steps: - name: Checkout uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8 with: name: ${{ env.ARTIFACT_NAME }} path: 'build/' - name: Set up QEMU - uses: docker/setup-qemu-action@1f40c72289eff860ee54a304f1438e3cff362e0a # v4 + uses: docker/setup-qemu-action@99012661954931238ded8c8b007157a8430204e1 # v4 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4 + uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4 - name: Login to GitHub Container Registry if: env.IMAGE_PUSH == 'true' - uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4 + uses: docker/login-action@81c6f93d4bc991e510b342a00b419b3bb4efc5c6 # v4 with: registry: ${{ env.REGISTRY }} username: ${{ github.actor }} @@ -119,7 +119,7 @@ jobs: - name: Login to DockerHub Container Registry if: env.IMAGE_PUSH == 'true' - uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4 + uses: docker/login-action@81c6f93d4bc991e510b342a00b419b3bb4efc5c6 # v4 with: username: ${{ vars.DOCKERHUB_ORG_NAME_BACKUP || env.DOCKERHUB_ORG_NAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} @@ -137,7 +137,7 @@ jobs: - name: Build and push id: docker_build - uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7 + uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7 with: context: . file: Dockerfile.ci diff --git a/.gitignore b/.gitignore index 06129e0a..6f951cd4 100644 --- a/.gitignore +++ b/.gitignore @@ -28,6 +28,7 @@ output/ /tmp/ /log/ /lang/ +/imgs/ /daemon/ /public/dist/* !/public/dist/README.md diff --git a/SECURITY.md b/SECURITY.md index ff558d64..65446a56 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -38,6 +38,22 @@ To keep your OpenList instance secure: - Regularly review mounted storage permissions and revoke unused API tokens. - When using Docker, avoid running the container as root if possible. +## Security Response Scope (Applicable to 4.x) + +This policy applies only to version 4.x. + +In 4.x, privilege isolation between authenticated users (including administrators and regular users) is not currently considered a security boundary that this project commits to maintaining. + +In principle, only security vulnerabilities that can be exploited without valid authentication (including authentication bypass vulnerabilities) fall within the scope of this project's dedicated security response process. + +Privilege isolation issues that require valid authentication to exploit should preferably be submitted and fixed through regular Pull Requests rather than handled through Security Advisories. + +This policy only defines the scope of the project's security response and does not imply that such issues are not security vulnerabilities. For exceptional cases that may have severe security implications, the maintenance team reserves the right to initiate the security response process at its discretion. + +This policy does not retroactively affect previously accepted security vulnerability reports. + +This project is currently preparing version 5.x. In the future, we may redefine the security boundaries between authenticated users by refactoring the user and permission systems and adjust the security response scope accordingly. + ## Acknowledgments We sincerely thank all security researchers and community members who responsibly disclose vulnerabilities and help make OpenList safer for everyone. @@ -84,6 +100,22 @@ We sincerely thank all security researchers and community members who responsibl - 定期检查已挂载存储的权限,并撤销未使用的 API 令牌。 - 使用 Docker 部署时,尽可能避免以 root 用户运行容器。 +## 安全响应范围 (适用于 4.x) + +本政策仅适用于 4.x 版本。 + +在 4.x 中,已认证用户 (包括管理员和普通用户) 之间的权限隔离暂不属于本项目承诺维护的安全边界。 + +只有无需有效身份认证即可利用的安全漏洞 (包括身份认证绕过漏洞),原则上才纳入本项目的专门安全响应流程。 + +需要有效身份认证才能利用的权限隔离问题,应优先通过普通 Pull Request 提交和修复,而非通过安全公告 (Security Advisory) 处理。 + +本政策仅限定项目维护团队的安全响应范围,不代表此类问题不构成安全漏洞。对于可能造成严重安全影响的特殊情况,维护团队保留酌情启动安全响应流程的权利。 + +本政策不追溯影响此前已正式接受的安全漏洞报告。 + +本项目正在准备 5.x 版本,未来可能通过重构用户及权限系统,重新定义已认证用户之间的安全边界,并相应调整安全响应范围。 + ## 致谢 我们衷心感谢所有负责任地披露漏洞、帮助 OpenList 变得更加安全的安全研究人员和社区成员。 diff --git a/cmd/worker/main.go b/cmd/worker/main.go new file mode 100644 index 00000000..e9686da6 --- /dev/null +++ b/cmd/worker/main.go @@ -0,0 +1,580 @@ +// Command openlist-transcode-worker 是独立的远程转码 Worker 节点。 +// 它通过 OpenList 自研协议(HTTP/JSON)注册到 Master,长轮询领任务,调用 +// 本机 ffmpeg 转码,并把生成的 HLS 切片实时上传回 Master。 +package main + +import ( + "bytes" + "context" + "encoding/json" + "flag" + "fmt" + "io" + "net/http" + "os" + "os/exec" + "os/signal" + "path/filepath" + "runtime" + "strconv" + "strings" + "sync" + "syscall" + "time" +) + +var ( + masterURL = flag.String("master", "http://127.0.0.1:5244", "OpenList Master URL") + secret = flag.String("secret", "", "Worker 共享密钥(与 Master 配置一致)") + name = flag.String("name", hostname(), "Worker 名称") + capacity = flag.Int("capacity", 2, "并发任务数") + hwaccel = flag.String("hwaccel", "none", "硬件加速:none/auto/nvenc/qsv/vaapi/amf/videotoolbox") + workdir = flag.String("workdir", "./worker_tmp", "临时工作目录") + ffmpeg = flag.String("ffmpeg", "ffmpeg", "ffmpeg 路径") +) + +func hostname() string { + h, _ := os.Hostname() + if h == "" { + h = "worker" + } + return h +} + +// ---- 协议结构 ----(与 Master 端 internal/transcode/protocol.go 一致) + +type RegisterReq struct { + Name string `json:"name"` + Version string `json:"version"` + Capacity int `json:"capacity"` + HWAccel []string `json:"hwaccel"` + CodecsDecode []string `json:"codecs_decode"` + CodecsEncode []string `json:"codecs_encode"` + MaxResolution string `json:"max_resolution"` + Tags []string `json:"tags"` +} +type RegisterResp struct { + Data struct { + WorkerID string `json:"worker_id"` + HeartbeatInterval int `json:"heartbeat_interval"` + } `json:"data"` +} + +type HeartbeatReq struct { + WorkerID string `json:"worker_id"` + Load float64 `json:"load"` + Running []string `json:"running"` + FreeSlots int `json:"free_slots"` +} +type HeartbeatResp struct { + Data struct { + OK bool `json:"ok"` + Kick bool `json:"kick"` + Cancel []string `json:"cancel"` + } `json:"data"` +} + +type ClaimReq struct { + WorkerID string `json:"worker_id"` + Slots int `json:"slots"` + Wait int `json:"wait"` +} + +type Profile struct { + Name string `json:"name"` + VideoCodec string `json:"video_codec"` + VideoBitrate string `json:"video_bitrate"` + Scale string `json:"scale"` + AudioCodec string `json:"audio_codec"` + AudioBitrate string `json:"audio_bitrate"` + HWAccel string `json:"hwaccel"` +} +type OutputSpec struct { + Format string `json:"format"` + SegmentDuration int `json:"segment_duration"` +} +type Job struct { + ID string `json:"id"` + Path string `json:"path"` + SourceURL string `json:"source_url"` + Profiles []Profile `json:"profiles"` + Output OutputSpec `json:"output"` + CallbackToken string `json:"callback_token"` + Deadline int64 `json:"deadline"` +} +type ClaimResp struct { + Data struct { + Jobs []Job `json:"jobs"` + } `json:"data"` +} + +type FinishReq struct { + JobID string `json:"job_id"` + Status string `json:"status"` + Error string `json:"error,omitempty"` + Stats struct { + Elapsed float64 `json:"elapsed"` + BytesOut int64 `json:"bytes_out"` + } `json:"stats"` +} + +// ---- HTTP helper ---- + +var httpClient = &http.Client{Timeout: 60 * time.Second} + +func postJSON(path string, body any, out any, bearer string) error { + buf, _ := json.Marshal(body) + req, _ := http.NewRequest(http.MethodPost, *masterURL+path, bytes.NewReader(buf)) + req.Header.Set("Content-Type", "application/json") + if bearer != "" { + req.Header.Set("Authorization", "Bearer "+bearer) + } + resp, err := httpClient.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + if resp.StatusCode >= 400 { + raw, _ := io.ReadAll(resp.Body) + return fmt.Errorf("HTTP %d: %s", resp.StatusCode, string(raw)) + } + if out != nil { + return json.NewDecoder(resp.Body).Decode(out) + } + return nil +} + +// ---- main ---- + +func main() { + flag.Parse() + if *secret == "" { + fmt.Fprintln(os.Stderr, "missing -secret") + os.Exit(1) + } + if err := os.MkdirAll(*workdir, 0o755); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + + workerID, hbInterval, err := registerOnce() + if err != nil { + fmt.Fprintln(os.Stderr, "register failed:", err) + os.Exit(1) + } + fmt.Printf("[worker] registered: id=%s heartbeat=%ds\n", workerID, hbInterval) + + ctx, cancel := context.WithCancel(context.Background()) + sigCh := make(chan os.Signal, 1) + signal.Notify(sigCh, syscall.SIGINT, syscall.SIGTERM) + go func() { + <-sigCh + fmt.Println("[worker] shutting down") + cancel() + }() + + state := &workerState{ + id: workerID, + runningMu: sync.Mutex{}, + running: map[string]context.CancelFunc{}, + } + + // 心跳协程 + go func() { + t := time.NewTicker(time.Duration(hbInterval) * time.Second) + defer t.Stop() + for { + select { + case <-ctx.Done(): + return + case <-t.C: + state.heartbeat() + } + } + }() + // 工作协程 + for i := 0; i < *capacity; i++ { + go state.workLoop(ctx) + } + <-ctx.Done() + time.Sleep(500 * time.Millisecond) +} + +func registerOnce() (string, int, error) { + req := RegisterReq{ + Name: *name, + Version: "1.0.0", + Capacity: *capacity, + HWAccel: []string{*hwaccel}, + CodecsDecode: []string{"h264", "hevc", "av1", "vp9", "mpeg4", "mpeg2video"}, + CodecsEncode: []string{"h264", "hevc"}, + MaxResolution: "3840x2160", + Tags: []string{runtime.GOOS, runtime.GOARCH}, + } + var resp RegisterResp + if err := postJSON("/api/transcode/worker/register", req, &resp, *secret); err != nil { + return "", 0, err + } + if resp.Data.WorkerID == "" { + return "", 0, fmt.Errorf("master returned empty worker_id") + } + if resp.Data.HeartbeatInterval == 0 { + resp.Data.HeartbeatInterval = 10 + } + return resp.Data.WorkerID, resp.Data.HeartbeatInterval, nil +} + +// ---- 状态机 ---- + +type workerState struct { + id string + runningMu sync.Mutex + running map[string]context.CancelFunc +} + +func (s *workerState) heartbeat() { + s.runningMu.Lock() + ids := make([]string, 0, len(s.running)) + for id := range s.running { + ids = append(ids, id) + } + free := *capacity - len(ids) + s.runningMu.Unlock() + var resp HeartbeatResp + if err := postJSON("/api/transcode/worker/heartbeat", HeartbeatReq{ + WorkerID: s.id, + Running: ids, + FreeSlots: free, + }, &resp, *secret); err != nil { + fmt.Println("[worker] heartbeat error:", err) + return + } + for _, jid := range resp.Data.Cancel { + s.runningMu.Lock() + if cancel, ok := s.running[jid]; ok { + cancel() + } + s.runningMu.Unlock() + } +} + +func (s *workerState) workLoop(ctx context.Context) { + for { + if ctx.Err() != nil { + return + } + var resp ClaimResp + err := postJSON("/api/transcode/worker/claim", ClaimReq{ + WorkerID: s.id, Slots: 1, Wait: 20, + }, &resp, *secret) + if err != nil { + fmt.Println("[worker] claim error:", err) + time.Sleep(5 * time.Second) + continue + } + if len(resp.Data.Jobs) == 0 { + continue + } + for _, j := range resp.Data.Jobs { + s.runJob(ctx, j) + } + } +} + +func (s *workerState) runJob(parent context.Context, job Job) { + jobCtx, cancel := context.WithCancel(parent) + s.runningMu.Lock() + s.running[job.ID] = cancel + s.runningMu.Unlock() + defer func() { + s.runningMu.Lock() + delete(s.running, job.ID) + s.runningMu.Unlock() + cancel() + }() + + startedAt := time.Now() + tmpDir := filepath.Join(*workdir, job.ID) + _ = os.MkdirAll(tmpDir, 0o755) + defer os.RemoveAll(tmpDir) + + if len(job.Profiles) == 0 { + s.finish(job, "failed", "no profiles", 0, 0) + return + } + profile := job.Profiles[0] + playlist := filepath.Join(tmpDir, "playlist.m3u8") + segPattern := filepath.Join(tmpDir, "seg-%d.ts") + args := buildFFmpegArgs(job, profile, segPattern, playlist) + + cmd := exec.CommandContext(jobCtx, *ffmpeg, args...) + stderr := &bytes.Buffer{} + cmd.Stderr = stderr + fmt.Printf("[worker] running job %s, ffmpeg %s\n", job.ID, strings.Join(args, " ")) + if err := cmd.Start(); err != nil { + s.finish(job, "failed", err.Error(), 0, 0) + return + } + + // 上传协程 + stopUpload := make(chan struct{}) + uploadDone := make(chan struct{}) + go func() { + defer close(uploadDone) + s.watchAndUpload(jobCtx, job, profile.Name, tmpDir, stopUpload) + }() + + err := cmd.Wait() + close(stopUpload) + <-uploadDone + // 最后一次扫描:把还未上传的尾片传上去 + s.uploadOnce(job, profile.Name, tmpDir, true) + + if err != nil { + if jobCtx.Err() == context.Canceled { + s.finish(job, "cancelled", "cancelled by master", 0, 0) + return + } + s.finish(job, "failed", fmt.Sprintf("%v: %s", err, truncate(stderr.String(), 1024)), 0, 0) + return + } + s.finish(job, "finished", "", time.Since(startedAt).Seconds(), 0) +} + +func (s *workerState) finish(job Job, status, errStr string, elapsed float64, bytesOut int64) { + req := FinishReq{ + JobID: job.ID, Status: status, Error: errStr, + } + req.Stats.Elapsed = elapsed + req.Stats.BytesOut = bytesOut + if err := postJSON("/api/transcode/worker/job/finish", req, nil, job.CallbackToken); err != nil { + fmt.Printf("[worker] finish report failed: %v\n", err) + } +} + +func (s *workerState) watchAndUpload(_ context.Context, job Job, profile, dir string, stop chan struct{}) { + t := time.NewTicker(800 * time.Millisecond) + defer t.Stop() + uploaded := map[int]bool{} + flush := func(final bool) { + s.uploadDelta(job, profile, dir, uploaded, final) + } + for { + select { + case <-stop: + flush(false) + return + case <-t.C: + flush(false) + } + } +} + +func (s *workerState) uploadOnce(job Job, profile, dir string, final bool) { + uploaded := map[int]bool{} + s.uploadDelta(job, profile, dir, uploaded, final) +} + +// uploadDelta 扫描目录,上传新切片(uploaded 跟踪已传过的 seq;最后一片到 final 时才上传) +func (s *workerState) uploadDelta(job Job, profile, dir string, uploaded map[int]bool, final bool) { + entries, err := os.ReadDir(dir) + if err != nil { + return + } + type segFile struct { + seq int + path string + } + var segs []segFile + for _, e := range entries { + if e.IsDir() { + continue + } + n := e.Name() + if !strings.HasPrefix(n, "seg-") || !strings.HasSuffix(n, ".ts") { + continue + } + num, err := strconv.Atoi(strings.TrimSuffix(strings.TrimPrefix(n, "seg-"), ".ts")) + if err != nil { + continue + } + segs = append(segs, segFile{num, filepath.Join(dir, n)}) + } + // 排序 + for i := 0; i < len(segs); i++ { + for j := i + 1; j < len(segs); j++ { + if segs[j].seq < segs[i].seq { + segs[i], segs[j] = segs[j], segs[i] + } + } + } + limit := len(segs) + if !final && limit > 0 { + limit-- // 最后一片可能还在写,留到下一轮 + } + segDur := job.Output.SegmentDuration + if segDur == 0 { + segDur = 6 + } + for i := 0; i < limit; i++ { + seg := segs[i] + if uploaded[seg.seq] { + continue + } + isLast := final && i == limit-1 + if err := s.uploadSegment(job, profile, seg.seq, float64(segDur), seg.path, isLast); err != nil { + fmt.Printf("[worker] upload seg-%d failed: %v\n", seg.seq, err) + return + } + uploaded[seg.seq] = true + } +} + +func (s *workerState) uploadSegment(job Job, profile string, seq int, dur float64, file string, final bool) error { + f, err := os.Open(file) + if err != nil { + return err + } + defer f.Close() + q := fmt.Sprintf("?job=%s&profile=%s&seq=%d&duration=%f&final=%t", + job.ID, profile, seq, dur, final) + req, _ := http.NewRequest(http.MethodPut, *masterURL+"/api/transcode/worker/segment"+q, f) + req.Header.Set("Content-Type", "video/mp2t") + req.Header.Set("Authorization", "Bearer "+job.CallbackToken) + resp, err := httpClient.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + if resp.StatusCode >= 400 { + raw, _ := io.ReadAll(resp.Body) + return fmt.Errorf("HTTP %d: %s", resp.StatusCode, string(raw)) + } + return nil +} + +// ---- ffmpeg args(与 master 端 local_runner 保持一致)---- + +// protocolWhitelistFor 与 internal/transcode.ProtocolWhitelistFor 保持一致: +// 远程输入只放行 HTTP(S) 及其传输层,本地路径只放行 file。 +func protocolWhitelistFor(source string) string { + s := strings.ToLower(strings.TrimSpace(source)) + if strings.HasPrefix(s, "http://") || strings.HasPrefix(s, "https://") { + // 与 master 的 protocolWhitelistRemote 保持一致:HTTP 源用不到 file/crypto, + // 留着等于给被构造过的播放列表留一条读本地文件的通道。 + return "http,https,tcp,tls" + } + return "file" +} + +func buildFFmpegArgs(job Job, profile Profile, segPattern, playlist string) []string { + hw := strings.ToLower(profile.HWAccel) + args := []string{"-y", "-loglevel", "warning"} + switch hw { + case "nvenc": + args = append(args, "-hwaccel", "cuda") + case "qsv": + args = append(args, "-hwaccel", "qsv") + case "vaapi": + args = append(args, "-hwaccel", "vaapi", "-hwaccel_output_format", "vaapi", "-vaapi_device", "/dev/dri/renderD128") + case "amf": + args = append(args, "-hwaccel", "d3d11va") + case "videotoolbox": + args = append(args, "-hwaccel", "videotoolbox") + case "auto": + args = append(args, "-hwaccel", "auto") + } + // 限制 ffmpeg 可用协议:Master 下发的 source_url 不应被解释成 concat:/subfile: 等特殊协议。 + // -protocol_whitelist 是输入选项,必须位于 -i 之前。 + args = append(args, "-protocol_whitelist", protocolWhitelistFor(job.SourceURL)) + args = append(args, "-i", job.SourceURL) + args = append(args, "-c:v", pickEncoder(profile.VideoCodec, hw)) + if profile.VideoBitrate != "" { + args = append(args, "-b:v", profile.VideoBitrate, "-maxrate", profile.VideoBitrate, "-bufsize", profile.VideoBitrate) + } + if profile.Scale != "" { + switch hw { + case "vaapi": + args = append(args, "-vf", "scale_vaapi="+profile.Scale) + case "qsv": + args = append(args, "-vf", "scale_qsv="+profile.Scale) + default: + args = append(args, "-vf", "scale="+profile.Scale) + } + } + if profile.AudioCodec == "copy" { + args = append(args, "-c:a", "copy") + } else { + ac := profile.AudioCodec + if ac == "" { + ac = "aac" + } + args = append(args, "-c:a", ac) + if profile.AudioBitrate != "" { + args = append(args, "-b:a", profile.AudioBitrate) + } + } + segDur := job.Output.SegmentDuration + if segDur == 0 { + segDur = 6 + } + args = append(args, + "-f", "hls", + "-hls_time", strconv.Itoa(segDur), + "-hls_playlist_type", "vod", + "-hls_segment_filename", segPattern, + "-hls_flags", "independent_segments", + playlist, + ) + return args +} + +func pickEncoder(codec, hw string) string { + codec = strings.ToLower(codec) + switch codec { + case "h264", "": + switch hw { + case "nvenc": + return "h264_nvenc" + case "qsv": + return "h264_qsv" + case "vaapi": + return "h264_vaapi" + case "amf": + return "h264_amf" + case "videotoolbox": + return "h264_videotoolbox" + } + return "libx264" + case "hevc", "h265": + switch hw { + case "nvenc": + return "hevc_nvenc" + case "qsv": + return "hevc_qsv" + case "vaapi": + return "hevc_vaapi" + case "amf": + return "hevc_amf" + case "videotoolbox": + return "hevc_videotoolbox" + } + return "libx265" + case "av1": + switch hw { + case "nvenc": + return "av1_nvenc" + case "qsv": + return "av1_qsv" + } + return "libsvtav1" + } + return "libx264" +} + +func truncate(s string, n int) string { + if len(s) <= n { + return s + } + return s[:n] +} diff --git a/drivers/115/seed_rapid.go b/drivers/115/seed_rapid.go new file mode 100644 index 00000000..2e674d96 --- /dev/null +++ b/drivers/115/seed_rapid.go @@ -0,0 +1,74 @@ +package _115 + +import ( + "context" + "strings" + + "github.com/OpenListTeam/OpenList/v4/internal/driver" + "github.com/OpenListTeam/OpenList/v4/internal/errs" + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/pkg/http_range" + "github.com/OpenListTeam/OpenList/v4/pkg/utils" +) + +// RapidHashAlgos 返回 115 支持的秒传哈希算法(SHA1) +func (d *Pan115) RapidHashAlgos() []utils.HashType { + return []utils.HashType{*utils.SHA1} +} + +// RapidHashNeedsPieces 115 不需要分片哈希 +func (d *Pan115) RapidHashNeedsPieces() bool { + return false +} + +// RapidUploadByHashes 使用种子中的 SHA1 哈希尝试秒传。 +// +// 115 的秒传协议除了整文件 SHA1,还需要文件头部 128KB 的 SHA1(pre_hash), +// 因此当内容源可用时会打开它来计算前置哈希;内容不可用时无法秒传。 +func (d *Pan115) RapidUploadByHashes(ctx context.Context, dstDir model.Obj, req *driver.SeedRapidUploadRequest, overwrite bool) (model.Obj, error) { + fullHash := strings.ToUpper(req.Whole.GetHash(utils.SHA1)) + if len(fullHash) != utils.SHA1.Width { + return nil, errs.ErrUnavailableHash + } + if req.Open == nil { + return nil, errs.ErrUnavailableHash + } + + src, err := req.Open() + if err != nil { + return nil, err + } + defer src.Close() + + const PreHashSize int64 = 128 * utils.KB + hashSize := PreHashSize + if req.Size < PreHashSize { + hashSize = req.Size + } + reader, err := src.RangeRead(http_range.Range{Start: 0, Length: hashSize}) + if err != nil { + return nil, err + } + preHash, err := utils.HashReader(utils.SHA1, reader) + if err != nil { + return nil, err + } + preHash = strings.ToUpper(preHash) + + fastInfo, err := d.rapidUpload(req.Size, req.Name, dstDir.GetID(), preHash, fullHash, src) + if err != nil { + return nil, err + } + matched, err := fastInfo.Ok() + if err != nil { + return nil, err + } + if !matched { + return nil, errs.ErrRapidUploadFailed + } + f, err := d.getNewFileByPickCode(fastInfo.PickCode) + if err != nil { + return nil, err + } + return f, nil +} diff --git a/drivers/115/util.go b/drivers/115/util.go index 104ebe8e..45bb2ac9 100644 --- a/drivers/115/util.go +++ b/drivers/115/util.go @@ -141,6 +141,7 @@ func (d *Pan115) rapidUpload(fileSize int64, fileName, dirID, preID, fileID stri form.Set("sig", d.client.GenerateSignature(fileID, target)) signKey, signVal := "", "" + decryptRetryCount := 0 for retry := true; retry; { t := driver115.NowMilli() @@ -172,11 +173,16 @@ func (d *Pan115) rapidUpload(fileSize int64, fileName, dirID, preID, fileID stri return nil, err } data := resp.RawBody() - defer data.Close() if bodyBytes, err = io.ReadAll(data); err != nil { + _ = data.Close() return nil, err } + _ = data.Close() if decrypted, err = ecdhCipher.Decrypt(bodyBytes); err != nil { + decryptRetryCount++ + if decryptRetryCount < 3 { + continue + } return nil, err } if err = driver115.CheckErr(json.Unmarshal(decrypted, &result), &result, resp); err != nil { diff --git a/drivers/115_open/driver.go b/drivers/115_open/driver.go index 65fc5eef..13673eb9 100644 --- a/drivers/115_open/driver.go +++ b/drivers/115_open/driver.go @@ -171,11 +171,22 @@ func (d *Open115) Get(ctx context.Context, path string) (model.Obj, error) { path = stdpath.Join(d.parentPath, path) resp, err := d.client.GetFolderInfoByPath(ctx, path) if err != nil { + // SDK-level "object not found" (empty array response from API) if errors.Is(err, sdk.ErrObjectNotFound) { return d.getFromParent(ctx, path, "") } + // API-level error response (State=false), treat as not found + // since this is a path-lookup that can't resolve the target + var apiErr *sdk.Error + if errors.As(err, &apiErr) { + return nil, errs.ObjectNotFound + } return nil, err } + // Handle null/empty response (e.g., API returns null data for non-existent path) + if resp.FileID == "" { + return nil, errs.ObjectNotFound + } obj := &Obj{ Fid: resp.FileID, Fn: resp.FileName, diff --git a/drivers/123/seed_rapid.go b/drivers/123/seed_rapid.go new file mode 100644 index 00000000..91232dac --- /dev/null +++ b/drivers/123/seed_rapid.go @@ -0,0 +1,73 @@ +package _123 + +import ( + "context" + "net/http" + "strconv" + "strings" + "time" + + "github.com/OpenListTeam/OpenList/v4/drivers/base" + "github.com/OpenListTeam/OpenList/v4/internal/driver" + "github.com/OpenListTeam/OpenList/v4/internal/errs" + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/pkg/utils" + "github.com/go-resty/resty/v2" +) + +// RapidHashAlgos 返回 123 云盘支持的秒传哈希算法(MD5) +func (d *Pan123) RapidHashAlgos() []utils.HashType { + return []utils.HashType{*utils.MD5} +} + +// RapidHashNeedsPieces 123 云盘不需要分片哈希 +func (d *Pan123) RapidHashNeedsPieces() bool { + return false +} + +// RapidUploadByHashes 使用种子中的 MD5(Etag)哈希尝试秒传。 +// +// 123 的秒传即「上传请求返回 reuse=true」,无需真正传输内容。 +func (d *Pan123) RapidUploadByHashes(ctx context.Context, dstDir model.Obj, req *driver.SeedRapidUploadRequest, overwrite bool) (model.Obj, error) { + etag := req.Whole.GetHash(utils.MD5) + if len(etag) < utils.MD5.Width { + return nil, errs.ErrUnavailableHash + } + + duplicate := 0 + if overwrite { + duplicate = 2 + } + data := base.Json{ + "driveId": 0, + "duplicate": duplicate, + "etag": strings.ToLower(etag), + "fileName": req.Name, + "parentFileId": dstDir.GetID(), + "size": req.Size, + "type": 0, + } + + var resp UploadResp + _, err := d.Request(UploadRequest, http.MethodPost, func(r *resty.Request) { + r.SetBody(data).SetContext(ctx) + }, &resp) + if err != nil { + return nil, err + } + // reuse=true 或未返回上传 Key 均视为秒传成功 + if !resp.Data.Reuse && resp.Data.Key != "" { + return nil, errs.ErrHashMismatch + } + + return &model.ObjThumb{ + Object: model.Object{ + ID: strconv.FormatInt(resp.Data.FileId, 10), + Name: req.Name, + Size: req.Size, + Modified: time.Now(), + IsFolder: false, + HashInfo: utils.NewHashInfo(utils.MD5, strings.ToLower(etag)), + }, + }, nil +} diff --git a/drivers/123/util.go b/drivers/123/util.go index 4b10b719..fef7483b 100644 --- a/drivers/123/util.go +++ b/drivers/123/util.go @@ -181,7 +181,7 @@ func (d *Pan123) login() error { return err } if utils.Json.Get(res.Body(), "code").ToInt() != 200 { - err = fmt.Errorf(utils.Json.Get(res.Body(), "message").ToString()) + err = fmt.Errorf("%s", utils.Json.Get(res.Body(), "message").ToString()) } else { d.AccessToken = utils.Json.Get(res.Body(), "data", "token").ToString() } diff --git a/drivers/123_open/seed_rapid.go b/drivers/123_open/seed_rapid.go new file mode 100644 index 00000000..7258e5a3 --- /dev/null +++ b/drivers/123_open/seed_rapid.go @@ -0,0 +1,55 @@ +package _123_open + +import ( + "context" + "path" + "strconv" + "time" + + "github.com/OpenListTeam/OpenList/v4/internal/driver" + "github.com/OpenListTeam/OpenList/v4/internal/errs" + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/pkg/utils" +) + +// RapidHashAlgos 返回 123 开放平台支持的秒传哈希算法(SHA1) +func (d *Open123) RapidHashAlgos() []utils.HashType { + return []utils.HashType{*utils.SHA1} +} + +// RapidHashNeedsPieces 123 开放平台不需要分片哈希 +func (d *Open123) RapidHashNeedsPieces() bool { + return false +} + +// RapidUploadByHashes 使用种子中的 SHA1 哈希尝试秒传 +func (d *Open123) RapidUploadByHashes(ctx context.Context, dstDir model.Obj, req *driver.SeedRapidUploadRequest, overwrite bool) (model.Obj, error) { + sha1Hash := req.Whole.GetHash(utils.SHA1) + if len(sha1Hash) < utils.SHA1.Width { + return nil, errs.ErrUnavailableHash + } + + parentID, err := strconv.ParseInt(dstDir.GetID(), 10, 64) + if err != nil { + return nil, err + } + + resp, err := d.sha1Reuse(parentID, req.Name, sha1Hash, req.Size, 1) + if err != nil { + return nil, err + } + if !resp.Data.Reuse { + return nil, errs.ErrHashMismatch + } + + return &model.ObjThumb{ + Object: model.Object{ + ID: strconv.FormatInt(resp.Data.FileID, 10), + Name: req.Name, + Size: req.Size, + IsFolder: false, + Path: path.Join(dstDir.GetPath(), req.Name), + Modified: time.Now(), + }, + }, nil +} diff --git a/drivers/189/torrent.go b/drivers/189/torrent.go index 4e41bf49..3d303b43 100644 --- a/drivers/189/torrent.go +++ b/drivers/189/torrent.go @@ -6,7 +6,6 @@ import ( "encoding/hex" "fmt" "io" - "strings" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/pkg/torrent" @@ -15,12 +14,8 @@ import ( // GenerateTorrent 根据上传过程中收集的哈希信息生成包含 CAS 扩展的 torrent 文件 func GenerateTorrent(fileName string, fileSize int64, fileMD5 string, sliceMD5s []string, sliceSize int64, pieceHashes []byte) ([]byte, error) { - // 计算 sliceMD5 - sliceMD5 := fileMD5 - if len(sliceMD5s) > 1 { - joined := strings.Join(sliceMD5s, "\n") - sliceMD5 = strings.ToUpper(torrent.GetMD5Str(joined)) - } + // 计算 sliceMD5(统一走规范实现) + sliceMD5 := torrent.SliceMD5FromPieces(sliceMD5s, fileMD5) t := torrent.NewTorrent(fileName, fileSize, fileMD5) t.Info.PieceLength = sliceSize @@ -30,7 +25,7 @@ func GenerateTorrent(fileName string, fileSize int64, fileMD5 string, sliceMD5s SliceMD5: sliceMD5, SliceMD5s: sliceMD5s, SliceSize: sliceSize, - Cloud: "189", + Cloud: torrent.Cloud189, }) return t.Encode() @@ -95,7 +90,7 @@ func ComputeTorrentFromReader(reader io.Reader, fileName string, fileSize int64, sliceSize = torrent.DefaultPieceSize } - hw := torrent.NewHashWriter(sliceSize, sliceSize) + hw := torrent.NewHashWriter(sliceSize, sliceSize, fileSize) buf := make([]byte, 32*1024) for { diff --git a/drivers/189/util.go b/drivers/189/util.go index a98b0d01..ae168137 100644 --- a/drivers/189/util.go +++ b/drivers/189/util.go @@ -238,7 +238,7 @@ func (d *Cloud189) oldUpload(dstDir model.Obj, file model.FileStreamer) error { if utils.Json.Get(res.Body(), "MD5").ToString() != "" { return nil } - log.Debugf(res.String()) + log.Debug(res.String()) return errors.New(res.String()) } diff --git a/drivers/189_tv/seed_rapid.go b/drivers/189_tv/seed_rapid.go new file mode 100644 index 00000000..87667a7f --- /dev/null +++ b/drivers/189_tv/seed_rapid.go @@ -0,0 +1,35 @@ +package _189_tv + +import ( + "context" + + "github.com/OpenListTeam/OpenList/v4/internal/driver" + "github.com/OpenListTeam/OpenList/v4/internal/errs" + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/pkg/utils" +) + +// RapidHashAlgos 返回 189 电视支持的秒传哈希算法(MD5) +func (d *Cloud189TV) RapidHashAlgos() []utils.HashType { + return []utils.HashType{*utils.MD5} +} + +// RapidHashNeedsPieces 189 电视不需要分片哈希 +func (d *Cloud189TV) RapidHashNeedsPieces() bool { + return false +} + +// RapidUploadByHashes 使用种子中的 MD5 哈希尝试秒传 +func (d *Cloud189TV) RapidUploadByHashes(ctx context.Context, dstDir model.Obj, req *driver.SeedRapidUploadRequest, overwrite bool) (model.Obj, error) { + md5Hash := req.Whole.GetHash(utils.MD5) + if len(md5Hash) < utils.MD5.Width { + return nil, errs.ErrUnavailableHash + } + + stream := driver.NewSeedHashStream(req) + obj, err := d.RapidUpload(ctx, dstDir, stream, d.isFamily(), overwrite) + if err != nil { + return nil, err + } + return obj, nil +} diff --git a/drivers/189pc/seed_rapid.go b/drivers/189pc/seed_rapid.go new file mode 100644 index 00000000..79eef6b2 --- /dev/null +++ b/drivers/189pc/seed_rapid.go @@ -0,0 +1,35 @@ +package _189pc + +import ( + "context" + + "github.com/OpenListTeam/OpenList/v4/internal/driver" + "github.com/OpenListTeam/OpenList/v4/internal/errs" + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/pkg/utils" +) + +// RapidHashAlgos 返回 189pc 支持的秒传哈希算法(MD5) +func (d *Cloud189PC) RapidHashAlgos() []utils.HashType { + return []utils.HashType{*utils.MD5} +} + +// RapidHashNeedsPieces 189pc 的 CAS 秒传依赖分片 MD5 +func (d *Cloud189PC) RapidHashNeedsPieces() bool { + return true +} + +// RapidUploadByHashes 使用种子中的 MD5 哈希尝试秒传 +func (d *Cloud189PC) RapidUploadByHashes(ctx context.Context, dstDir model.Obj, req *driver.SeedRapidUploadRequest, overwrite bool) (model.Obj, error) { + md5Hash := req.Whole.GetHash(utils.MD5) + if len(md5Hash) < utils.MD5.Width { + return nil, errs.ErrUnavailableHash + } + + stream := driver.NewSeedHashStream(req) + obj, err := d.RapidUpload(ctx, dstDir, stream, d.isFamily(), overwrite) + if err != nil { + return nil, err + } + return obj, nil +} diff --git a/drivers/189pc/torrent.go b/drivers/189pc/torrent.go index 30c195e8..7ef2e466 100644 --- a/drivers/189pc/torrent.go +++ b/drivers/189pc/torrent.go @@ -7,7 +7,6 @@ import ( "encoding/hex" "fmt" "io" - "net/url" "strings" "time" @@ -28,12 +27,8 @@ import ( // fileName: 文件名 // fileSize: 文件大小 func GenerateTorrent(fileName string, fileSize int64, fileMD5 string, sliceMD5s []string, sliceSize int64, pieceHashes []byte) ([]byte, error) { - // 计算 sliceMD5 - sliceMD5 := fileMD5 - if len(sliceMD5s) > 1 { - joined := strings.Join(sliceMD5s, "\n") - sliceMD5 = strings.ToUpper(torrent.GetMD5Str(joined)) - } + // 计算 sliceMD5(统一走规范实现) + sliceMD5 := torrent.SliceMD5FromPieces(sliceMD5s, fileMD5) t := torrent.NewTorrent(fileName, fileSize, fileMD5) t.Info.PieceLength = sliceSize @@ -43,7 +38,7 @@ func GenerateTorrent(fileName string, fileSize int64, fileMD5 string, sliceMD5s SliceMD5: sliceMD5, SliceMD5s: sliceMD5s, SliceSize: sliceSize, - Cloud: "189", + Cloud: torrent.Cloud189, }) return t.Encode() @@ -69,101 +64,18 @@ func (y *Cloud189PC) RapidUploadFromTorrent(ctx context.Context, dstDir model.Ob fileName := t.Info.Name fileSize := t.GetTotalSize() - // 统一 MD5 为大写(与正常上传保持一致,天翼云盘要求大写) - fileMD5Upper := strings.ToUpper(cas.FileMD5) - - // 优先使用 torrent 中嵌入的分片大小,与生成时保持一致 - sliceSize := cas.SliceSize - if sliceSize <= 0 { - sliceSize = partSize(fileSize) + // 优先使用 torrent 中嵌入的分片 MD5 与大写整文件 MD5 + fileMD5 := strings.ToUpper(cas.FileMD5) + sliceMD5s := make([]string, len(cas.SliceMD5s)) + for i, s := range cas.SliceMD5s { + sliceMD5s[i] = strings.ToUpper(s) } - // 计算 sliceMd5(与上传时一致的算法) - // 优先使用 torrent 中已有的 SliceMD5;仅当有多分片列表时才重新计算 - sliceMd5Hex := strings.ToUpper(cas.SliceMD5) - if sliceMd5Hex == "" { - sliceMd5Hex = fileMD5Upper - } - if len(cas.SliceMD5s) > 1 { - // 分片 MD5 也需要统一大写后再拼接计算 - upperSliceMD5s := make([]string, len(cas.SliceMD5s)) - for i, s := range cas.SliceMD5s { - upperSliceMD5s[i] = strings.ToUpper(s) - } - sliceMd5Hex = strings.ToUpper(utils.GetMD5EncodeStr(strings.Join(upperSliceMD5s, "\n"))) - } - - // 使用与 Web 端一致的三步秒传流程 - fullUrl := "https://upload.cloud.189.cn" - if isFamily { - fullUrl += "/family" - } else { - fullUrl += "/person" - } - - // Step 1: initMultiUpload(不传 fileMd5/sliceMd5,只传 lazyCheck) - initParams := Params{ - "parentFolderId": dstDir.GetID(), - "fileName": url.QueryEscape(fileName), - "fileSize": fmt.Sprint(fileSize), - "sliceSize": fmt.Sprint(sliceSize), - "lazyCheck": "1", - } - if isFamily { - initParams.Set("familyId", y.FamilyID) - } - - var uploadInfo InitMultiUploadResp - _, err = y.request(fullUrl+"/initMultiUpload", "GET", func(req *resty.Request) { - req.SetContext(ctx) - }, initParams, &uploadInfo, isFamily) + // 复用统一的 CAS 秒传核心实现 + respObj, err := y.rapidUploadByCAS(ctx, dstDir, fileName, fileSize, fileMD5, sliceMD5s, cas.SliceSize, overwrite) if err != nil { - return nil, fmt.Errorf("initMultiUpload 失败: %w", err) - } - - uploadFileId := uploadInfo.Data.UploadFileID - - // Step 2: checkTransSecond(用 fileMd5 + sliceMd5 + uploadFileId 检查秒传) - checkParams := Params{ - "fileMd5": fileMD5Upper, - "sliceMd5": sliceMd5Hex, - "uploadFileId": uploadFileId, - } - - var checkResp struct { - Data struct { - FileDataExists int `json:"fileDataExists"` - } `json:"data"` - } - _, err = y.request(fullUrl+"/checkTransSecond", "GET", func(req *resty.Request) { - req.SetContext(ctx) - }, checkParams, &checkResp, isFamily) - if err != nil { - utils.Log.Errorf("[RapidUpload] checkTransSecond 失败: uploadFileId=%s, err=%v", uploadFileId, err) - return nil, fmt.Errorf("秒传检查失败: %w", err) - } - - if checkResp.Data.FileDataExists != 1 { - return nil, fmt.Errorf("秒传失败:云端不存在该文件(fileMD5=%s, sliceMD5=%s, size=%d)", fileMD5Upper, sliceMd5Hex, fileSize) - } - - // Step 3: commitMultiUploadFile(传 fileMd5 + sliceMd5) - - var resp CommitMultiUploadFileResp - commitParams := Params{ - "uploadFileId": uploadFileId, - "fileMd5": fileMD5Upper, - "sliceMd5": sliceMd5Hex, - "lazyCheck": "1", - "opertype": IF(overwrite, "3", "1"), - } - - _, err = y.request(fullUrl+"/commitMultiUploadFile", "GET", func(req *resty.Request) { - req.SetContext(ctx) - }, commitParams, &resp, isFamily) - if err != nil { - utils.Log.Errorf("[RapidUpload] commitMultiUploadFile 失败: uploadFileId=%s, err=%v", uploadFileId, err) - return nil, fmt.Errorf("提交上传失败: %w", err) + utils.Log.Errorf("[RapidUpload] 秒传失败: fileMD5=%s, err=%v", fileMD5, err) + return nil, err } // 秒传成功后,将 torrent 文件上传到目标目录(异步,不影响秒传结果) @@ -196,7 +108,7 @@ func (y *Cloud189PC) RapidUploadFromTorrent(ctx context.Context, dstDir model.Ob }() } - return resp.toFile(), nil + return respObj, nil } // ComputeTorrentFromReader 从 io.Reader 计算并生成 torrent 文件 @@ -206,7 +118,7 @@ func ComputeTorrentFromReader(reader io.Reader, fileName string, fileSize int64, sliceSize = torrent.DefaultPieceSize } - hw := torrent.NewHashWriter(sliceSize, sliceSize) + hw := torrent.NewHashWriter(sliceSize, sliceSize, fileSize) buf := make([]byte, 32*1024) for { @@ -259,12 +171,8 @@ func InjectCASIntoTorrent(torrentData []byte, fileMD5 string, sliceMD5s []string return nil, fmt.Errorf("解析 torrent 失败: %w", err) } - // 计算 sliceMD5 - sliceMD5 := fileMD5 - if len(sliceMD5s) > 1 { - joined := strings.Join(sliceMD5s, "\n") - sliceMD5 = strings.ToUpper(torrent.GetMD5Str(joined)) - } + // 计算 sliceMD5(统一走规范实现) + sliceMD5 := torrent.SliceMD5FromPieces(sliceMD5s, fileMD5) // 注入 CAS 信息 t.SetCASInfo(&torrent.CASInfo{ @@ -272,7 +180,7 @@ func InjectCASIntoTorrent(torrentData []byte, fileMD5 string, sliceMD5s []string SliceMD5: sliceMD5, SliceMD5s: sliceMD5s, SliceSize: sliceSize, - Cloud: "189", + Cloud: torrent.Cloud189, }) // 同时更新 info 中的 md5sum 字段 diff --git a/drivers/189pc/utils.go b/drivers/189pc/utils.go index 713fa78b..454e1158 100644 --- a/drivers/189pc/utils.go +++ b/drivers/189pc/utils.go @@ -1613,6 +1613,102 @@ func (y *Cloud189PC) OldUploadCommit(ctx context.Context, fileCommitUrl string, return resp.toFile(), nil } +// rapidUploadByCAS 使用 MD5 + 分片 MD5(CAS)执行天翼云盘秒传。 +// +// 流程与 Web 端一致: +// 1. initMultiUpload(仅传 lazyCheck=1) +// 2. checkTransSecond(用 fileMd5 + sliceMd5 检查云端是否已存在文件数据) +// 3. commitMultiUploadFile(提交并返回文件对象) +func (y *Cloud189PC) rapidUploadByCAS(ctx context.Context, dstDir model.Obj, fileName string, fileSize int64, fileMD5 string, sliceMD5s []string, sliceSize int64, overwrite bool) (model.Obj, error) { + isFamily := y.isFamily() + + // 统一 MD5 为大写(天翼云盘要求大写) + fileMD5Upper := strings.ToUpper(fileMD5) + + // 优先使用传入的分片大小,否则按文件大小推导 + if sliceSize <= 0 { + sliceSize = partSize(fileSize) + } + + // 计算 sliceMd5(与上传时一致的算法) + sliceMd5Hex := fileMD5Upper + if len(sliceMD5s) > 1 { + upperSliceMD5s := make([]string, len(sliceMD5s)) + for i, s := range sliceMD5s { + upperSliceMD5s[i] = strings.ToUpper(s) + } + sliceMd5Hex = strings.ToUpper(utils.GetMD5EncodeStr(strings.Join(upperSliceMD5s, "\n"))) + } else if len(sliceMD5s) == 1 { + sliceMd5Hex = strings.ToUpper(sliceMD5s[0]) + } + + fullUrl := "https://upload.cloud.189.cn" + if isFamily { + fullUrl += "/family" + } else { + fullUrl += "/person" + } + + // Step 1: initMultiUpload(不传 fileMd5/sliceMd5,只传 lazyCheck) + initParams := Params{ + "parentFolderId": dstDir.GetID(), + "fileName": url.QueryEscape(fileName), + "fileSize": fmt.Sprint(fileSize), + "sliceSize": fmt.Sprint(sliceSize), + "lazyCheck": "1", + } + if isFamily { + initParams.Set("familyId", y.FamilyID) + } + + var uploadInfo InitMultiUploadResp + if _, err := y.request(fullUrl+"/initMultiUpload", "GET", func(req *resty.Request) { + req.SetContext(ctx) + }, initParams, &uploadInfo, isFamily); err != nil { + return nil, fmt.Errorf("initMultiUpload 失败: %w", err) + } + uploadFileId := uploadInfo.Data.UploadFileID + + // Step 2: checkTransSecond(用 fileMd5 + sliceMd5 + uploadFileId 检查秒传) + checkParams := Params{ + "fileMd5": fileMD5Upper, + "sliceMd5": sliceMd5Hex, + "uploadFileId": uploadFileId, + } + + var checkResp struct { + Data struct { + FileDataExists int `json:"fileDataExists"` + } `json:"data"` + } + if _, err := y.request(fullUrl+"/checkTransSecond", "GET", func(req *resty.Request) { + req.SetContext(ctx) + }, checkParams, &checkResp, isFamily); err != nil { + return nil, fmt.Errorf("秒传检查失败: %w", err) + } + if checkResp.Data.FileDataExists != 1 { + return nil, fmt.Errorf("秒传失败:云端不存在该文件(fileMD5=%s, sliceMD5=%s, size=%d)", fileMD5Upper, sliceMd5Hex, fileSize) + } + + // Step 3: commitMultiUploadFile(传 fileMd5 + sliceMd5) + commitParams := Params{ + "uploadFileId": uploadFileId, + "fileMd5": fileMD5Upper, + "sliceMd5": sliceMd5Hex, + "lazyCheck": "1", + "opertype": IF(overwrite, "3", "1"), + } + + var resp CommitMultiUploadFileResp + if _, err := y.request(fullUrl+"/commitMultiUploadFile", "GET", func(req *resty.Request) { + req.SetContext(ctx) + }, commitParams, &resp, isFamily); err != nil { + return nil, fmt.Errorf("提交上传失败: %w", err) + } + + return resp.toFile(), nil +} + func (y *Cloud189PC) isFamily() bool { return y.Type == "family" } diff --git a/drivers/aliyundrive_open/seed_rapid.go b/drivers/aliyundrive_open/seed_rapid.go new file mode 100644 index 00000000..575f1a3e --- /dev/null +++ b/drivers/aliyundrive_open/seed_rapid.go @@ -0,0 +1,88 @@ +package aliyundrive_open + +import ( + "context" + "net/http" + "time" + + "github.com/OpenListTeam/OpenList/v4/drivers/base" + "github.com/OpenListTeam/OpenList/v4/internal/driver" + "github.com/OpenListTeam/OpenList/v4/internal/errs" + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/pkg/utils" + "github.com/go-resty/resty/v2" +) + +// RapidHashAlgos 返回阿里云盘支持的秒传哈希算法(SHA1) +func (d *AliyundriveOpen) RapidHashAlgos() []utils.HashType { + return []utils.HashType{*utils.SHA1} +} + +// RapidHashNeedsPieces 阿里云盘不需要分片哈希 +func (d *AliyundriveOpen) RapidHashNeedsPieces() bool { + return false +} + +// RapidUploadByHashes 使用种子中的 SHA1 哈希尝试秒传。 +// +// 阿里云盘的秒传还需要 proof_code(按 proof range 读取的一段内容), +// 因此当内容源不可用时无法完成秒传。 +func (d *AliyundriveOpen) RapidUploadByHashes(ctx context.Context, dstDir model.Obj, req *driver.SeedRapidUploadRequest, overwrite bool) (model.Obj, error) { + sha1Hash := req.Whole.GetHash(utils.SHA1) + if len(sha1Hash) < utils.SHA1.Width { + return nil, errs.ErrUnavailableHash + } + if req.Open == nil { + return nil, errs.ErrUnavailableHash + } + + stream, err := req.Open() + if err != nil { + return nil, err + } + defer stream.Close() + + proofCode, err := d.calProofCode(stream) + if err != nil { + return nil, err + } + + var resp CreateResp + _, err = d.request(ctx, limiterOther, "/adrive/v1.0/openFile/create", http.MethodPost, func(r *resty.Request) { + r.SetBody(base.Json{ + "drive_id": d.DriveId, + "parent_file_id": dstDir.GetID(), + "name": req.Name, + "type": "file", + "check_name_mode": "auto_rename", + "size": req.Size, + "content_hash": sha1Hash, + "content_hash_name": "sha1", + "proof_version": "v1", + "proof_code": proofCode, + }).SetResult(&resp) + }) + if err != nil { + return nil, err + } + if !resp.RapidUpload { + return nil, errs.ErrHashMismatch + } + + if resp.FileId != "" { + obj, err := d.completeUpload(ctx, resp.FileId, resp.UploadId) + if err != nil { + return nil, err + } + return obj, nil + } + + return &model.ObjThumb{ + Object: model.Object{ + Name: req.Name, + Size: req.Size, + Modified: time.Now(), + IsFolder: false, + }, + }, nil +} diff --git a/drivers/all.go b/drivers/all.go index 601c7bfc..ea4fbd77 100644 --- a/drivers/all.go +++ b/drivers/all.go @@ -30,6 +30,7 @@ import ( _ "github.com/OpenListTeam/OpenList/v4/drivers/cloudreve_v4" _ "github.com/OpenListTeam/OpenList/v4/drivers/cnb_releases" _ "github.com/OpenListTeam/OpenList/v4/drivers/crypt" + _ "github.com/OpenListTeam/OpenList/v4/drivers/ctfile" _ "github.com/OpenListTeam/OpenList/v4/drivers/degoo" _ "github.com/OpenListTeam/OpenList/v4/drivers/doubao" _ "github.com/OpenListTeam/OpenList/v4/drivers/doubao_new" diff --git a/drivers/baidu_netdisk/seed_rapid.go b/drivers/baidu_netdisk/seed_rapid.go new file mode 100644 index 00000000..5d90012f --- /dev/null +++ b/drivers/baidu_netdisk/seed_rapid.go @@ -0,0 +1,35 @@ +package baidu_netdisk + +import ( + "context" + + "github.com/OpenListTeam/OpenList/v4/internal/driver" + "github.com/OpenListTeam/OpenList/v4/internal/errs" + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/pkg/utils" +) + +// RapidHashAlgos 返回百度网盘支持的秒传哈希算法(MD5) +func (d *BaiduNetdisk) RapidHashAlgos() []utils.HashType { + return []utils.HashType{*utils.MD5} +} + +// RapidHashNeedsPieces 百度网盘不需要分片哈希 +func (d *BaiduNetdisk) RapidHashNeedsPieces() bool { + return false +} + +// RapidUploadByHashes 使用种子中的 MD5 哈希尝试秒传 +func (d *BaiduNetdisk) RapidUploadByHashes(ctx context.Context, dstDir model.Obj, req *driver.SeedRapidUploadRequest, overwrite bool) (model.Obj, error) { + md5Hash := req.Whole.GetHash(utils.MD5) + if len(md5Hash) < utils.MD5.Width { + return nil, errs.ErrUnavailableHash + } + + stream := driver.NewSeedHashStream(req) + obj, err := d.PutRapid(ctx, dstDir, stream) + if err != nil { + return nil, err + } + return obj, nil +} diff --git a/drivers/baidu_netdisk/util.go b/drivers/baidu_netdisk/util.go index 7916f9b5..31f31257 100644 --- a/drivers/baidu_netdisk/util.go +++ b/drivers/baidu_netdisk/util.go @@ -112,7 +112,7 @@ func (d *BaiduNetdisk) request(furl string, method string, callback base.ReqCall log.Debugf("[baidu_netdisk] req: %s, resp: %s", furl, res.String()) errno := utils.Json.Get(res.Body(), "errno").ToInt() if errno != 0 { - if utils.SliceContains([]int{111, -6}, errno) { + if utils.SliceContains([]int{111, -6, 20016, 20017}, errno) { log.Info("[baidu_netdisk] refreshing baidu_netdisk token.") err2 := d.refreshToken() if err2 != nil { diff --git a/drivers/chaoxing/driver.go b/drivers/chaoxing/driver.go index dfd25d19..5365c919 100644 --- a/drivers/chaoxing/driver.go +++ b/drivers/chaoxing/driver.go @@ -21,7 +21,6 @@ import ( "github.com/OpenListTeam/OpenList/v4/pkg/cron" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/go-resty/resty/v2" - "google.golang.org/appengine/log" ) type ChaoXing struct { @@ -55,13 +54,13 @@ func (d *ChaoXing) refreshCookie() error { func (d *ChaoXing) Init(ctx context.Context) error { err := d.refreshCookie() if err != nil { - log.Errorf(ctx, err.Error()) + utils.Log.Errorf("%v", err) } d.cron = cron.NewCron(time.Hour * 12) d.cron.Do(func() { err = d.refreshCookie() if err != nil { - log.Errorf(ctx, err.Error()) + utils.Log.Errorf("%v", err) } }) return nil diff --git a/drivers/ctfile/driver.go b/drivers/ctfile/driver.go new file mode 100644 index 00000000..92464354 --- /dev/null +++ b/drivers/ctfile/driver.go @@ -0,0 +1,252 @@ +package ctfile + +import ( + "context" + "fmt" + "io" + "strings" + "time" + + "github.com/OpenListTeam/OpenList/v4/internal/driver" + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/internal/stream" + "github.com/OpenListTeam/OpenList/v4/pkg/utils" +) + +// CtFile implements driver.Driver for 城通网盘 (CtFile) via the OpenAPI +// session-based authentication. No email/password/SMS login is used. +type CtFile struct { + model.Storage + Addition +} + +func (d *CtFile) Config() driver.Config { + return config +} + +func (d *CtFile) GetAddition() driver.Additional { + return &d.Addition +} + +func (d *CtFile) Init(ctx context.Context) error { + if strings.TrimSpace(d.Session) == "" { + return fmt.Errorf("[CtFile] session is required; generate one in the CtFile backend under 「开放接口登录密钥管理」") + } + return d.profile(ctx) +} + +func (d *CtFile) Drop(ctx context.Context) error { + return nil +} + +// List returns the contents of dir (both sub-folders and files). +func (d *CtFile) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { + folderID := dir.GetID() + if folderID == "" { + folderID = rootFolderID + } + + folders, err := d.listFolders(ctx, folderID) + if err != nil { + return nil, err + } + files, err := d.listFiles(ctx, folderID) + if err != nil { + return nil, err + } + + var objs []model.Obj + for _, f := range folders { + objs = append(objs, &model.Object{ + ID: f.Key, + Name: f.Name, + IsFolder: true, + Modified: parseDate(f.Date), + Ctime: parseDate(f.Date), + }) + } + for _, f := range files { + objs = append(objs, &model.Object{ + ID: f.Key, + Name: f.Name, + Size: f.Size, + IsFolder: false, + Modified: parseDate(f.Date), + Ctime: parseDate(f.Date), + }) + } + return objs, nil +} + +// Link fetches the CDN download URL for file via fetch_url. +func (d *CtFile) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { + url, err := d.fetchURL(ctx, file.GetID()) + if err != nil { + return nil, err + } + return &model.Link{ + URL: url, + Header: map[string][]string{ + "User-Agent": {d.ua()}, + }, + }, nil +} + +// MakeDir creates a sub-folder under parentDir. +func (d *CtFile) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) (model.Obj, error) { + parentID := parentDir.GetID() + id, err := d.mkdir(ctx, parentID, dirName) + if err != nil { + return nil, err + } + return &model.Object{ + ID: id, + Name: dirName, + IsFolder: true, + Modified: time.Now(), + Ctime: time.Now(), + }, nil +} + +// Move relocates srcObj into dstDir. +func (d *CtFile) Move(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { + err := d.move(ctx, []string{srcObj.GetID()}, dstDir.GetID()) + if err != nil { + return nil, err + } + return srcObj, nil +} + +// Rename changes the name of srcObj to newName. +func (d *CtFile) Rename(ctx context.Context, srcObj model.Obj, newName string) (model.Obj, error) { + var err error + if srcObj.IsDir() { + err = d.renameFolder(ctx, srcObj.GetID(), newName) + } else { + err = d.renameFile(ctx, srcObj.GetID(), newName) + } + if err != nil { + return nil, err + } + return &model.Object{ + ID: srcObj.GetID(), + Name: newName, + Size: srcObj.GetSize(), + IsFolder: srcObj.IsDir(), + Modified: srcObj.ModTime(), + Ctime: srcObj.CreateTime(), + }, nil +} + +// Copy copies srcObj into dstDir. +// The spec explicitly requires implementation; the Worker's "unimplemented" +// note is overridden here. The API endpoint is POST /{ns}/file/copy. +func (d *CtFile) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { + return d.copyFiles(ctx, []string{srcObj.GetID()}, dstDir.GetID()) +} + +// Remove deletes obj (moves to trash). +func (d *CtFile) Remove(ctx context.Context, obj model.Obj) error { + return d.remove(ctx, []string{obj.GetID()}) +} + +// Put uploads a file to dstDir. The stream is cached so the checksum and the +// subsequent multipart upload can reuse it without keeping the entire file in memory. +func (d *CtFile) Put(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress) error { + md5hex := file.GetHash().GetHash(utils.MD5) + tmpFile := file.GetFile() + var err error + var cacheProgress *model.UpdateProgress + if up != nil { + cacheProgress = &up + } + + if len(md5hex) != utils.MD5.Width { + // Cache the full stream and compute MD5 simultaneously. + tmpFile, md5hex, err = stream.CacheFullAndHash(file, cacheProgress, utils.MD5) + } else if tmpFile == nil { + // A pre-computed MD5 still needs a seekable source for the upload. + tmpFile, err = file.CacheFullAndWriter(cacheProgress, nil) + } + if err != nil { + return err + } + if tmpFile == nil { + return fmt.Errorf("[CtFile] failed to cache upload stream") + } + if _, err := tmpFile.Seek(0, io.SeekStart); err != nil { + return err + } + + size := file.GetSize() + if size < 0 { + size, err = tmpFile.Seek(0, io.SeekEnd) + if err != nil { + return err + } + if _, err := tmpFile.Seek(0, io.SeekStart); err != nil { + return err + } + } + + if up != nil { + up(50) + } + + parentID := dstDir.GetID() + err = d.uploadFile(ctx, parentID, file.GetName(), tmpFile, size, md5hex) + if err != nil { + return err + } + + if up != nil { + up(100) + } + return nil +} + +// parseDate converts a CtFile date value (unix seconds or "YYYY-MM-DD HH:mm:ss") +// to time.Time, falling back to time.Now() on failure. +func parseDate(v any) time.Time { + if v == nil { + return time.Now() + } + switch tv := v.(type) { + case float64: + if tv > 0 { + return time.Unix(int64(tv), 0) + } + case string: + if tv == "" { + return time.Now() + } + // Try unix seconds as string + if n, err := parseInt64(tv); err == nil && n > 0 { + return time.Unix(n, 0) + } + // Try "YYYY-MM-DD HH:mm:ss" + layouts := []string{"2006-01-02 15:04:05", "2006-01-02T15:04:05Z"} + for _, layout := range layouts { + if t, err := time.Parse(layout, tv); err == nil { + return t + } + } + } + return time.Now() +} + +func parseInt64(s string) (int64, error) { + var n int64 + _, err := fmt.Sscanf(s, "%d", &n) + return n, err +} + +var ( + _ driver.Driver = (*CtFile)(nil) + _ driver.MkdirResult = (*CtFile)(nil) + _ driver.MoveResult = (*CtFile)(nil) + _ driver.RenameResult = (*CtFile)(nil) + _ driver.Copy = (*CtFile)(nil) + _ driver.Remove = (*CtFile)(nil) + _ driver.Put = (*CtFile)(nil) +) diff --git a/drivers/ctfile/meta.go b/drivers/ctfile/meta.go new file mode 100644 index 00000000..39bdd6ba --- /dev/null +++ b/drivers/ctfile/meta.go @@ -0,0 +1,44 @@ +package ctfile + +import ( + "github.com/OpenListTeam/OpenList/v4/internal/driver" + "github.com/OpenListTeam/OpenList/v4/internal/op" +) + +// Addition holds all user-configurable fields for the CtFile driver. +type Addition struct { + // Session is the OpenAPI token generated in the CtFile backend under + // 「开放接口登录密钥管理」. Required. + Session string `json:"session" required:"true" secret:"true" help:"OpenAPI session token from 开放接口登录密钥管理"` + + // CloudType selects namespace: "public" (default) or "private". + CloudType string `json:"cloud_type" type:"select" options:"public,private" default:"public"` + + // BaseURL is the API root, defaulting to https://rest.ctfile.com/v1. + BaseURL string `json:"base_url" default:"https://rest.ctfile.com/v1" help:"API base URL"` + + // OrderBy controls server-side sort field. + OrderBy string `json:"order_by" type:"select" options:"name,size,time" default:"name"` + + // OrderDirection is the sort direction. + OrderDirection string `json:"order_direction" type:"select" options:"asc,desc" default:"asc"` + + // UserAgent overrides the default Chrome-like UA sent with every request. + UserAgent string `json:"user_agent" default:"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/142.0.0.0 Safari/537.36"` + + driver.RootID +} + +var config = driver.Config{ + Name: "CtFile", + DefaultRoot: "0", + LocalSort: true, + NoUpload: false, + OnlyProxy: false, +} + +func init() { + op.RegisterDriver(func() driver.Driver { + return &CtFile{} + }) +} diff --git a/drivers/ctfile/types.go b/drivers/ctfile/types.go new file mode 100644 index 00000000..c65a2641 --- /dev/null +++ b/drivers/ctfile/types.go @@ -0,0 +1,51 @@ +package ctfile + +// CtFile (城通网盘) OpenAPI response structs. +// API doc: https://openapi.ctfile.com/ (REST API v1, unified POST JSON + session auth). +// All business endpoints return a unified wrapper { code, message?, ...business fields }. +// `code == 200` means success; `401` means session expired; `403` with folder_id on mkdir +// means the folder already exists. + +// Resp is the unified response wrapper. Business fields are decoded via +// utils.Json to stay tolerant of the server's loosely-typed payloads (numbers +// may arrive as strings and vice-versa). +type Resp struct { + Code int `json:"code"` + Message string `json:"message"` + // folder_id / file_id / download_url / upload_url / exists / results / ... + // are accessed through utils.Json, so we keep the raw bytes here. +} + +// profileResp holds the fields returned by POST /user/info/profile. +type profileResp struct { + UserID int64 `json:"userid"` + UserName string `json:"username"` + NickName string `json:"nick_name"` + GroupName string `json:"group_name"` + MaxFileSz int64 `json:"max_filesize"` +} + +// folderEntry is a single folder in folder/list results. +type folderEntry struct { + Key string `json:"key"` + Name string `json:"name"` + Icon string `json:"icon"` + Date any `json:"date"` // unix seconds or "YYYY-MM-DD HH:mm:ss" +} + +// fileEntry is a single file in file/list results. +type fileEntry struct { + Key string `json:"key"` + Name string `json:"name"` + Icon string `json:"icon"` + ImgSrc string `json:"imgsrc"` + Size int64 `json:"size"` + Date any `json:"date"` + Status int `json:"status"` // 1=complete, 2=incomplete (not downloadable) +} + +// uploadTicket is the POST /{ns}/file/upload response. +type uploadTicket struct { + UploadURL string `json:"upload_url"` + Exists int `json:"exists"` // >0 => instant upload hit +} diff --git a/drivers/ctfile/util.go b/drivers/ctfile/util.go new file mode 100644 index 00000000..1a7164b1 --- /dev/null +++ b/drivers/ctfile/util.go @@ -0,0 +1,598 @@ +package ctfile + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "mime/multipart" + "net/http" + "regexp" + "strconv" + "strings" + "time" +) + +const ( + defaultBaseURL = "https://rest.ctfile.com/v1" + defaultUserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/142.0.0.0 Safari/537.36" + rootFolderID = "d0" + maxListPages = 500 +) + +// sessionExpiredMsg is the human-readable error for 401 responses. +const sessionExpiredMsg = "[CtFile] Session expired (401). Please regenerate your session token in the CtFile backend under 「开放接口登录密钥管理」" + +// stripIDPrefix removes a leading d or f prefix from an ID when the next +// character is a digit. E.g. "d123" → "123", "f456" → "456", "abc" → "abc". +var stripPrefixRe = regexp.MustCompile(`^([df])(\d)`) + +func stripIDPrefix(id string) string { + if m := stripPrefixRe.FindStringSubmatch(id); m != nil { + return id[1:] // drop the leading letter + } + return id +} + +func (d *CtFile) ns() string { + if d.CloudType == "private" { + return "private" + } + return "public" +} + +func (d *CtFile) baseURL() string { + if u := strings.TrimRight(d.BaseURL, "/"); u != "" { + return u + } + return defaultBaseURL +} + +func (d *CtFile) ua() string { + if d.UserAgent != "" { + return d.UserAgent + } + return defaultUserAgent +} + +// orderby maps the user fields to CtFile's server-side sort tokens. +func (d *CtFile) orderby() string { + keys := map[string]string{"name": "az", "size": "big", "time": "new"} + rev := map[string]string{"az": "za", "big": "small", "new": "old"} + base, ok := keys[d.OrderBy] + if !ok { + base = "az" + } + if d.OrderDirection == "desc" { + return rev[base] + } + return base +} + +// post is the raw transport layer: POST JSON, include session, retry on +// 429/5xx, map 401 to a clear error. Returns decoded JSON map or error. +func (d *CtFile) post(ctx context.Context, path string, body map[string]any) (map[string]any, error) { + session := strings.TrimSpace(d.Session) + if session == "" { + return nil, fmt.Errorf("[CtFile] session is not configured; please fill in the session token from 开放接口登录密钥管理") + } + + payload := make(map[string]any, len(body)+1) + for k, v := range body { + payload[k] = v + } + payload["session"] = session + + data, err := json.Marshal(payload) + if err != nil { + return nil, fmt.Errorf("[CtFile] json marshal: %w", err) + } + + url := d.baseURL() + path + var lastErr error + for attempt := 0; attempt < 3; attempt++ { + if attempt > 0 { + select { + case <-ctx.Done(): + return nil, ctx.Err() + case <-time.After(time.Duration(300*attempt) * time.Millisecond): + } + } + + req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(data)) + if err != nil { + return nil, err + } + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Accept", "application/json") + req.Header.Set("User-Agent", d.ua()) + + resp, err := http.DefaultClient.Do(req) + if err != nil { + lastErr = err + continue + } + + // Retry on 429 / 5xx + if resp.StatusCode == 429 || resp.StatusCode >= 500 { + _ = resp.Body.Close() + lastErr = fmt.Errorf("HTTP %d", resp.StatusCode) + continue + } + + body2, err := io.ReadAll(resp.Body) + _ = resp.Body.Close() + if err != nil { + return nil, err + } + + var result map[string]any + if err := json.Unmarshal(body2, &result); err != nil { + return nil, fmt.Errorf("[CtFile] %s: non-JSON response (HTTP %d): %.200s", path, resp.StatusCode, body2) + } + + code := int(toInt64(result["code"])) + if code == 401 { + return nil, fmt.Errorf("%s", sessionExpiredMsg) + } + if code == 429 { + lastErr = fmt.Errorf("[CtFile] rate limited (code 429)") + continue + } + return result, nil + } + if lastErr != nil { + return nil, fmt.Errorf("[CtFile] %s failed after retries: %w", path, lastErr) + } + return nil, fmt.Errorf("[CtFile] %s failed after retries", path) +} + +// call wraps post and checks code == 200. +func (d *CtFile) call(ctx context.Context, path string, body map[string]any) (map[string]any, error) { + result, err := d.post(ctx, path, body) + if err != nil { + return nil, err + } + // Missing / zero code is treated as success (Worker: code === undefined → ok). + if _, hasCode := result["code"]; hasCode { + code := int(toInt64(result["code"])) + if code != 200 { + return result, &apiError{code: code, message: toString(result["message"]), raw: result} + } + } + return result, nil +} + +// callWithId calls an endpoint that takes a single id parameter, implementing +// the dual-try strategy (prefer one form; on 400/404 retry the other). +// +// prefer = "raw" → try id as-is first, fallback to stripped +// prefer = "stripped" → try stripped first, fallback to raw +func (d *CtFile) callWithId(ctx context.Context, path, idKey, id string, extra map[string]any, prefer string) (map[string]any, error) { + stripped := stripIDPrefix(id) + var primary, secondary string + if prefer == "stripped" { + primary, secondary = stripped, id + } else { + primary, secondary = id, stripped + } + + body := make(map[string]any, len(extra)+1) + for k, v := range extra { + body[k] = v + } + + body[idKey] = primary + result, err := d.call(ctx, path, body) + if err != nil { + var ae *apiError + if asAPIError(err, &ae) && (ae.code == 400 || ae.code == 404) && primary != secondary { + body[idKey] = secondary + return d.call(ctx, path, body) + } + return nil, err + } + return result, nil +} + +// callWithIds calls an endpoint that takes a list of ids; tries comma-string +// first, on 400/404/405 retries JSON array. +func (d *CtFile) callWithIds(ctx context.Context, path string, ids []string, extra map[string]any) (map[string]any, error) { + body := make(map[string]any, len(extra)+2) + for k, v := range extra { + body[k] = v + } + + // First attempt: comma-separated string + body["ids"] = strings.Join(ids, ",") + result, err := d.call(ctx, path, body) + if err != nil { + var ae *apiError + if asAPIError(err, &ae) && (ae.code == 400 || ae.code == 404 || ae.code == 405) { + // Second attempt: JSON array + body["ids"] = ids + return d.call(ctx, path, body) + } + return nil, err + } + return result, nil +} + +// -------------------------------------------------------------------------- +// High-level API helpers +// -------------------------------------------------------------------------- + +// profile fetches the user's profile; used to validate the session in Init. +func (d *CtFile) profile(ctx context.Context) error { + _, err := d.call(ctx, "/user/info/profile", nil) + return err +} + +// listFolders returns all sub-folders for a given parent folder ID. +func (d *CtFile) listFolders(ctx context.Context, folderID string) ([]folderEntry, error) { + if folderID == "" || folderID == "0" { + folderID = rootFolderID + } + result, err := d.call(ctx, "/"+d.ns()+"/folder/list", map[string]any{ + "folder_id": folderID, + "orderby": d.orderby(), + }) + if err != nil { + return nil, err + } + items := toSlice(result["results"]) + out := make([]folderEntry, 0, len(items)) + for _, raw := range items { + m, ok := raw.(map[string]any) + if !ok { + continue + } + out = append(out, folderEntry{ + Key: toString(m["key"]), + Name: toString(m["name"]), + Icon: toString(m["icon"]), + Date: m["date"], + }) + } + return out, nil +} + +// listFiles returns all files in a folder, paginating until empty or duplicate. +func (d *CtFile) listFiles(ctx context.Context, folderID string) ([]fileEntry, error) { + if folderID == "" || folderID == "0" { + folderID = rootFolderID + } + var all []fileEntry + start := 0 + prevFirstKey := "" + + for page := 0; page < maxListPages; page++ { + result, err := d.call(ctx, "/"+d.ns()+"/file/list", map[string]any{ + "folder_id": folderID, + "start": start, + "orderby": d.orderby(), + }) + if err != nil { + return nil, err + } + items := toSlice(result["results"]) + if len(items) == 0 { + break + } + + // Guard against server ignoring `start` and looping forever + firstKey := "" + if m, ok := items[0].(map[string]any); ok { + firstKey = toString(m["key"]) + } + if page > 0 && firstKey == prevFirstKey { + break + } + prevFirstKey = firstKey + + for _, raw := range items { + m, ok := raw.(map[string]any) + if !ok { + continue + } + all = append(all, fileEntry{ + Key: toString(m["key"]), + Name: toString(m["name"]), + Icon: toString(m["icon"]), + ImgSrc: toString(m["imgsrc"]), + Size: toInt64(m["size"]), + Date: m["date"], + Status: int(toInt64(m["status"])), + }) + } + start += len(items) + } + return all, nil +} + +// mkdir creates a folder and returns its ID. 403 with an existing folder_id is +// treated as success (folder already exists). +func (d *CtFile) mkdir(ctx context.Context, parentID, name string) (string, error) { + if parentID == "" || parentID == "0" { + parentID = rootFolderID + } + result, err := d.call(ctx, "/"+d.ns()+"/folder/create", map[string]any{ + "folder_id": parentID, + "name": name, + "description": "", + "is_hidden": false, + }) + if err != nil { + var ae *apiError + if asAPIError(err, &ae) && ae.code == 403 { + // Folder already exists; the response still carries folder_id + if id := toString(ae.raw["folder_id"]); id != "" { + return id, nil + } + } + return "", err + } + id := toString(result["folder_id"]) + if id == "" { + id = toString(result["path"]) + } + return id, nil +} + +// fetchURL returns the CDN download URL for a file. +func (d *CtFile) fetchURL(ctx context.Context, fileID string) (string, error) { + result, err := d.callWithId(ctx, "/"+d.ns()+"/file/fetch_url", "file_id", fileID, nil, "stripped") + if err != nil { + return "", err + } + url := toString(result["download_url"]) + if url == "" { + return "", fmt.Errorf("[CtFile] fetch_url returned empty download_url") + } + return url, nil +} + +// renameFile renames a file. +func (d *CtFile) renameFile(ctx context.Context, fileID, newName string) error { + _, err := d.callWithId(ctx, "/"+d.ns()+"/file/modify_meta", "file_id", fileID, map[string]any{"name": newName}, "stripped") + return err +} + +// renameFolder renames a folder. +func (d *CtFile) renameFolder(ctx context.Context, folderID, newName string) error { + _, err := d.callWithId(ctx, "/"+d.ns()+"/folder/modify_meta", "folder_id", folderID, map[string]any{ + "name": newName, + "is_rename": true, + }, "raw") + return err +} + +// remove moves files/folders to trash. +func (d *CtFile) remove(ctx context.Context, ids []string) error { + if len(ids) == 0 { + return nil + } + _, err := d.callWithIds(ctx, "/"+d.ns()+"/file/delete", ids, nil) + return err +} + +// move moves files/folders to another folder. +func (d *CtFile) move(ctx context.Context, ids []string, targetFolderID string) error { + if len(ids) == 0 { + return nil + } + if targetFolderID == "" || targetFolderID == "0" { + targetFolderID = rootFolderID + } + _, err := d.callWithIds(ctx, "/"+d.ns()+"/file/move", ids, map[string]any{"folder_id": targetFolderID}) + return err +} + +// copyFiles copies files to a target folder. +func (d *CtFile) copyFiles(ctx context.Context, ids []string, targetFolderID string) error { + if len(ids) == 0 { + return nil + } + if targetFolderID == "" || targetFolderID == "0" { + targetFolderID = rootFolderID + } + _, err := d.callWithIds(ctx, "/"+d.ns()+"/file/copy", ids, map[string]any{"folder_id": targetFolderID}) + return err +} + +// uploadFile uploads a stream to CtFile using the two-step ticket+multipart flow. +// checksum format: md5hex + "-" + size +func (d *CtFile) uploadFile(ctx context.Context, folderID, fileName string, content io.Reader, size int64, md5hex string) error { + if folderID == "" || folderID == "0" { + folderID = rootFolderID + } + + checksum := fmt.Sprintf("%s-%d", md5hex, size) + + // Step 1: get upload ticket + result, err := d.call(ctx, "/"+d.ns()+"/file/upload", map[string]any{ + "folder_id": folderID, + "checksum": checksum, + "size": size, + "name": fileName, + }) + if err != nil { + return err + } + + exists := toInt64(result["exists"]) + if exists > 0 { + // Instant upload / deduplication + return nil + } + + uploadURL := toString(result["upload_url"]) + if uploadURL == "" { + return fmt.Errorf("[CtFile] upload ticket returned no upload_url") + } + + // Step 2: multipart upload to the presigned URL + return d.multipartUpload(ctx, uploadURL, fileName, content, size) +} + +// countingWriter counts bytes without retaining the written data. +type countingWriter int64 + +func (w *countingWriter) Write(p []byte) (int, error) { + *w += countingWriter(len(p)) + return len(p), nil +} + +func writeMultipartFields(mw *multipart.Writer, fileName string, size int64) (io.Writer, error) { + if err := mw.WriteField("name", fileName); err != nil { + return nil, err + } + if err := mw.WriteField("filesize", strconv.FormatInt(size, 10)); err != nil { + return nil, err + } + return mw.CreateFormFile("file", fileName) +} + +func multipartLayout(fileName string, size int64) (int64, string, error) { + var count countingWriter + mw := multipart.NewWriter(&count) + if _, err := writeMultipartFields(mw, fileName, size); err != nil { + return 0, "", err + } + if err := mw.Close(); err != nil { + return 0, "", err + } + return int64(count) + size, mw.Boundary(), nil +} + +// multipartUpload POSTs name+filesize+file to a presigned upload URL without +// retaining the complete multipart body in memory. +func (d *CtFile) multipartUpload(ctx context.Context, uploadURL, fileName string, content io.Reader, size int64) error { + contentLength, boundary, err := multipartLayout(fileName, size) + if err != nil { + return err + } + + pr, pw := io.Pipe() + req, err := http.NewRequestWithContext(ctx, http.MethodPost, uploadURL, pr) + if err != nil { + return err + } + req.ContentLength = contentLength + req.Header.Set("Content-Type", "multipart/form-data; boundary="+boundary) + req.Header.Set("User-Agent", d.ua()) + + go func() { + mw := multipart.NewWriter(pw) + if err := mw.SetBoundary(boundary); err != nil { + _ = pw.CloseWithError(err) + return + } + fw, err := writeMultipartFields(mw, fileName, size) + if err != nil { + _ = pw.CloseWithError(err) + return + } + if _, err := io.Copy(fw, content); err != nil { + _ = pw.CloseWithError(err) + return + } + if err := mw.Close(); err != nil { + _ = pw.CloseWithError(err) + return + } + _ = pw.Close() + }() + + resp, err := http.DefaultClient.Do(req) + if err != nil { + _ = pr.CloseWithError(err) + return err + } + defer resp.Body.Close() + + body, err := io.ReadAll(resp.Body) + if err != nil { + return err + } + + if resp.StatusCode != 200 { + return fmt.Errorf("[CtFile] multipart upload HTTP %d: %.200s", resp.StatusCode, body) + } + + var result map[string]any + if err := json.Unmarshal(body, &result); err != nil { + return fmt.Errorf("[CtFile] multipart upload non-JSON: %.200s", body) + } + code := int(toInt64(result["code"])) + if code != 200 { + return fmt.Errorf("[CtFile] multipart upload failed (code=%d): %s", code, toString(result["message"])) + } + return nil +} + +// -------------------------------------------------------------------------- +// apiError carries a structured error with code + raw body for retry checks. +// -------------------------------------------------------------------------- + +type apiError struct { + code int + message string + raw map[string]any +} + +func (e *apiError) Error() string { + return fmt.Sprintf("[CtFile] API error (code=%d): %s", e.code, e.message) +} + +// asAPIError sets dst and returns true when err is an *apiError. +func asAPIError(err error, dst **apiError) bool { + if ae, ok := err.(*apiError); ok { + *dst = ae + return true + } + return false +} + +// -------------------------------------------------------------------------- +// Type-coercion helpers for loosely-typed JSON payloads. +// -------------------------------------------------------------------------- + +func toInt64(v any) int64 { + if v == nil { + return 0 + } + switch n := v.(type) { + case float64: + return int64(n) + case int64: + return n + case int: + return int64(n) + case string: + i, _ := strconv.ParseInt(n, 10, 64) + return i + default: + return 0 + } +} + +func toString(v any) string { + if v == nil { + return "" + } + switch s := v.(type) { + case string: + return s + default: + return fmt.Sprintf("%v", v) + } +} + +func toSlice(v any) []any { + if s, ok := v.([]any); ok { + return s + } + return nil +} diff --git a/drivers/ctfile/util_test.go b/drivers/ctfile/util_test.go new file mode 100644 index 00000000..c8e03c51 --- /dev/null +++ b/drivers/ctfile/util_test.go @@ -0,0 +1,352 @@ +package ctfile + +import ( + "context" + "crypto/md5" + "fmt" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" +) + +// --------------------------------------------------------------------------- +// Helpers / mock +// --------------------------------------------------------------------------- + +// buildServer creates a test HTTP server that returns pre-scripted responses in +// sequence. Each response is a JSON string (already complete). The variadic +// responses are consumed one at a time; the last one is reused for all +// subsequent requests. +func buildServer(t *testing.T, responses ...string) (*httptest.Server, *[]string) { + t.Helper() + calls := new([]string) + idx := 0 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body, _ := io.ReadAll(r.Body) + *calls = append(*calls, string(body)) + + resp := responses[idx] + if idx < len(responses)-1 { + idx++ + } + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(200) + _, _ = w.Write([]byte(resp)) + })) + return srv, calls +} + +// driverWithBase returns a minimal CtFile driver pointing at the test server. +func driverWithBase(baseURL string) *CtFile { + d := &CtFile{} + d.Session = "test-session" + d.BaseURL = baseURL + d.CloudType = "public" + d.OrderBy = "name" + d.OrderDirection = "asc" + return d +} + +// --------------------------------------------------------------------------- +// StripIDPrefix +// --------------------------------------------------------------------------- + +func TestStripIDPrefix(t *testing.T) { + cases := []struct { + in string + out string + }{ + {"d123", "123"}, + {"f456", "456"}, + {"abc", "abc"}, + {"d", "d"}, // no digit follows → keep as-is + {"f", "f"}, // no digit follows → keep as-is + {"d0", "0"}, + {"", ""}, + {"123", "123"}, + } + for _, tc := range cases { + got := stripIDPrefix(tc.in) + if got != tc.out { + t.Errorf("stripIDPrefix(%q) = %q, want %q", tc.in, got, tc.out) + } + } +} + +// --------------------------------------------------------------------------- +// callWithId dual-try: 400 → retry with alternate id form +// --------------------------------------------------------------------------- + +func TestCallWithId_DualTry(t *testing.T) { + // First call returns code 400; second returns code 200 with a name field. + srv, calls := buildServer(t, + `{"code":400,"message":"not found"}`, + `{"code":200,"name":"Photos"}`, + ) + defer srv.Close() + + d := driverWithBase(srv.URL) + ctx := context.Background() + + // prefer = "raw" → first attempt with "d123", fallback with "123" + result, err := d.callWithId(ctx, "/public/folder/meta", "folder_id", "d123", nil, "raw") + if err != nil { + t.Fatalf("callWithId returned error: %v", err) + } + if toString(result["name"]) != "Photos" { + t.Errorf("expected name=Photos, got %v", result["name"]) + } + if len(*calls) != 2 { + t.Fatalf("expected 2 HTTP calls, got %d", len(*calls)) + } + if !strings.Contains((*calls)[0], `"d123"`) { + t.Errorf("first call should use raw id d123, got: %s", (*calls)[0]) + } + if !strings.Contains((*calls)[1], `"123"`) { + t.Errorf("second call should use stripped id 123, got: %s", (*calls)[1]) + } +} + +// --------------------------------------------------------------------------- +// callWithId: prefer stripped → file/fetch_url pattern +// --------------------------------------------------------------------------- + +func TestCallWithId_PreferStripped(t *testing.T) { + // First call (stripped "456") returns 400; second (raw "f456") returns 200. + srv, calls := buildServer(t, + `{"code":400,"message":"param error"}`, + `{"code":200,"download_url":"https://cdn.example.com/file"}`, + ) + defer srv.Close() + + d := driverWithBase(srv.URL) + ctx := context.Background() + + result, err := d.callWithId(ctx, "/public/file/fetch_url", "file_id", "f456", nil, "stripped") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if toString(result["download_url"]) != "https://cdn.example.com/file" { + t.Errorf("unexpected download_url: %v", result["download_url"]) + } + if len(*calls) != 2 { + t.Fatalf("expected 2 calls, got %d", len(*calls)) + } + if !strings.Contains((*calls)[0], `"456"`) { + t.Errorf("first call should use stripped 456: %s", (*calls)[0]) + } + if !strings.Contains((*calls)[1], `"f456"`) { + t.Errorf("second call should use raw f456: %s", (*calls)[1]) + } +} + +// --------------------------------------------------------------------------- +// callWithIds: comma-string → JSON array fallback on 400 +// --------------------------------------------------------------------------- + +func TestCallWithIds_DualTry(t *testing.T) { + srv, calls := buildServer(t, + `{"code":400,"message":"param error"}`, + `{"code":200,"message":"deleted"}`, + ) + defer srv.Close() + + d := driverWithBase(srv.URL) + ctx := context.Background() + + err := d.remove(ctx, []string{"f1", "f2"}) + if err != nil { + t.Fatalf("remove returned error: %v", err) + } + if len(*calls) != 2 { + t.Fatalf("expected 2 calls, got %d", len(*calls)) + } + // First attempt: ids is comma-string + if !strings.Contains((*calls)[0], `"f1,f2"`) { + t.Errorf("first call should have comma-string ids: %s", (*calls)[0]) + } + // Second attempt: ids is JSON array + if !strings.Contains((*calls)[1], `["f1","f2"]`) { + t.Errorf("second call should have JSON array ids: %s", (*calls)[1]) + } +} + +// --------------------------------------------------------------------------- +// Checksum format: md5hex-size +// --------------------------------------------------------------------------- + +func TestChecksumFormat(t *testing.T) { + content := []byte("hello") + sum := md5.Sum(content) + checksum := fmt.Sprintf("%x-%d", sum, len(content)) + // md5("hello") = 5d41402abc4b2a76b9719d911017c592 + want := "5d41402abc4b2a76b9719d911017c592-5" + if checksum != want { + t.Errorf("checksum = %q, want %q", checksum, want) + } +} + +// --------------------------------------------------------------------------- +// 401 → session expired message +// --------------------------------------------------------------------------- + +func TestSession401Error(t *testing.T) { + srv, _ := buildServer(t, `{"code":401,"message":"not logged in"}`) + defer srv.Close() + + d := driverWithBase(srv.URL) + ctx := context.Background() + + err := d.profile(ctx) + if err == nil { + t.Fatal("expected error for 401, got nil") + } + msg := err.Error() + if !strings.Contains(msg, "401") && !strings.Contains(msg, "expired") && !strings.Contains(msg, "Session") { + t.Errorf("error message should mention session/expired/401; got: %s", msg) + } + if !strings.Contains(msg, "开放接口登录密钥管理") { + t.Errorf("error message should mention 开放接口登录密钥管理; got: %s", msg) + } +} + +// --------------------------------------------------------------------------- +// mkdir: 403 with folder_id → treat as success +// --------------------------------------------------------------------------- + +func TestMkdir403Success(t *testing.T) { + // Server returns 403 with an existing folder_id + srv, _ := buildServer(t, `{"code":403,"message":"folder exists","folder_id":"d999"}`) + defer srv.Close() + + d := driverWithBase(srv.URL) + ctx := context.Background() + + id, err := d.mkdir(ctx, "d0", "MyFolder") + if err != nil { + t.Fatalf("mkdir should succeed on 403 with folder_id, got error: %v", err) + } + if id != "d999" { + t.Errorf("expected id=d999, got %q", id) + } +} + +// --------------------------------------------------------------------------- +// list files: pagination stops on empty page +// --------------------------------------------------------------------------- + +func TestListFiles_Pagination(t *testing.T) { + // Page 1: two files; page 2: empty + srv, calls := buildServer(t, + `{"code":200,"results":[{"key":"f1","name":"a.txt","size":100},{"key":"f2","name":"b.txt","size":200}]}`, + `{"code":200,"results":[]}`, + ) + defer srv.Close() + + d := driverWithBase(srv.URL) + ctx := context.Background() + + files, err := d.listFiles(ctx, "d0") + if err != nil { + t.Fatalf("listFiles: %v", err) + } + if len(files) != 2 { + t.Errorf("expected 2 files, got %d", len(files)) + } + if len(*calls) != 2 { + t.Errorf("expected 2 HTTP calls (2 pages), got %d", len(*calls)) + } +} + +// --------------------------------------------------------------------------- +// list files: duplicate first key terminates pagination (anti-loop guard) +// --------------------------------------------------------------------------- + +func TestListFiles_DuplicateKeyStops(t *testing.T) { + sameResp := `{"code":200,"results":[{"key":"f1","name":"a.txt","size":1},{"key":"f2","name":"b.txt","size":2}]}` + srv, calls := buildServer(t, sameResp, sameResp, sameResp) + defer srv.Close() + + d := driverWithBase(srv.URL) + ctx := context.Background() + + files, err := d.listFiles(ctx, "d0") + if err != nil { + t.Fatalf("listFiles: %v", err) + } + // Should stop after 2 calls (first page + duplicate detection on second) + if len(*calls) != 2 { + t.Errorf("expected 2 calls (duplicate guard), got %d", len(*calls)) + } + if len(files) != 2 { + t.Errorf("expected 2 files, got %d", len(files)) + } +} + +// --------------------------------------------------------------------------- +// namespace: private cloud uses /private/ prefix +// --------------------------------------------------------------------------- + +func TestNamespace_Private(t *testing.T) { + srv, calls := buildServer(t, `{"code":200,"results":[]}`) + defer srv.Close() + + d := driverWithBase(srv.URL) + d.CloudType = "private" + ctx := context.Background() + + _, _ = d.listFolders(ctx, "d0") + if len(*calls) < 1 { + t.Fatal("expected at least 1 call") + } + // The request URL path should contain /private/ + // We can't directly read the URL from body, but we can check the server + // received a body referencing private — instead check via a separate handler. + // We verify indirectly by checking the test server was hit (calls recorded). +} + +func TestNamespace_PrivateURL(t *testing.T) { + var gotPath string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotPath = r.URL.Path + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(200) + _, _ = w.Write([]byte(`{"code":200,"results":[]}`)) + })) + defer srv.Close() + + d := driverWithBase(srv.URL) + d.CloudType = "private" + ctx := context.Background() + + _, _ = d.listFolders(ctx, "d0") + if !strings.Contains(gotPath, "/private/") { + t.Errorf("expected /private/ in path, got %q", gotPath) + } +} + +// --------------------------------------------------------------------------- +// session missing → clear error +// --------------------------------------------------------------------------- + +func TestSessionMissing(t *testing.T) { + d := &CtFile{} + d.Session = "" + d.BaseURL = "http://localhost:9" + ctx := context.Background() + + err := d.profile(ctx) + if err == nil { + t.Fatal("expected error for missing session") + } + if !strings.Contains(err.Error(), "session") { + t.Errorf("error should mention session, got: %v", err) + } +} + +// --------------------------------------------------------------------------- +// helper: background context for tests (unused now — kept as reference) +// --------------------------------------------------------------------------- + diff --git a/drivers/google_drive/util.go b/drivers/google_drive/util.go index 042abafa..e4f5f1a0 100644 --- a/drivers/google_drive/util.go +++ b/drivers/google_drive/util.go @@ -170,7 +170,7 @@ func (d *GoogleDrive) refreshToken() error { } log.Debug(res.String()) if e.Error != "" { - return fmt.Errorf(e.Error) + return fmt.Errorf("%s", e.Error) } d.AccessToken = resp.AccessToken return nil @@ -192,7 +192,7 @@ func (d *GoogleDrive) refreshToken() error { } log.Debug(res.String()) if e.Error != "" { - return fmt.Errorf(e.Error) + return fmt.Errorf("%s", e.Error) } d.AccessToken = resp.AccessToken return nil diff --git a/drivers/google_photo/util.go b/drivers/google_photo/util.go index 3a9b66ab..81d05b49 100644 --- a/drivers/google_photo/util.go +++ b/drivers/google_photo/util.go @@ -32,7 +32,7 @@ func (d *GooglePhoto) refreshToken() error { return err } if e.Error != "" { - return fmt.Errorf(e.Error) + return fmt.Errorf("%s", e.Error) } d.AccessToken = resp.AccessToken return nil diff --git a/drivers/lanzou/util.go b/drivers/lanzou/util.go index be922a6b..c02df953 100644 --- a/drivers/lanzou/util.go +++ b/drivers/lanzou/util.go @@ -90,7 +90,7 @@ func (d *LanZou) _post(url string, callback base.ReqCallback, resp interface{}, if info == "" { info = utils.Json.Get(data, "info").ToString() } - return data, fmt.Errorf(info) + return data, fmt.Errorf("%s", info) } } diff --git a/drivers/local/driver.go b/drivers/local/driver.go index 696e2480..3c851d98 100644 --- a/drivers/local/driver.go +++ b/drivers/local/driver.go @@ -7,6 +7,7 @@ import ( "fmt" "io/fs" "net/http" + "net/url" "os" stdpath "path" "path/filepath" @@ -14,6 +15,7 @@ import ( "strings" "time" + "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" @@ -151,12 +153,15 @@ func (d *Local) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([ func (d *Local) FileInfoToObj(ctx context.Context, f fs.FileInfo, reqPath string, fullPath string) model.Obj { thumb := "" - if d.Thumbnail { - if d.supportsThumbnail(f.Name()) { - thumb = common.GetApiUrl(ctx) + stdpath.Join("/d", reqPath, f.Name()) - thumb = utils.EncodePath(thumb, true) - thumb += "?type=thumb&sign=" + sign.Sign(stdpath.Join(reqPath, f.Name())) + if d.Thumbnail && d.supportsThumbnail(f.Name()) { + base, ok := ctx.Value(conf.ThumbnailBaseURLKey).(*url.URL) + if !ok { + base = &url.URL{ + Path: stdpath.Join("/d", reqPath), + RawQuery: url.Values{"type": {"thumb"}, "sign": {sign.Sign(stdpath.Join(reqPath, f.Name()))}}.Encode(), + } } + thumb = common.GetApiUrl(ctx) + base.JoinPath(url.PathEscape(f.Name())).String() } isFolder := f.IsDir() || isSymlinkDir(f, fullPath) var size int64 diff --git a/drivers/misskey/util.go b/drivers/misskey/util.go index 35ebe96c..d45c1f7c 100644 --- a/drivers/misskey/util.go +++ b/drivers/misskey/util.go @@ -245,6 +245,7 @@ func mFile2Object(file MFile) *model.ObjThumbURL { Ctime: ctime, IsFolder: false, Size: file.Size, + HashInfo: utils.NewHashInfo(utils.MD5, file.MD5), }, Thumbnail: model.Thumbnail{ Thumbnail: file.ThumbnailURL, diff --git a/drivers/pikpak/seed_rapid.go b/drivers/pikpak/seed_rapid.go new file mode 100644 index 00000000..05bf30c4 --- /dev/null +++ b/drivers/pikpak/seed_rapid.go @@ -0,0 +1,58 @@ +package pikpak + +import ( + "context" + "net/http" + "strings" + + "github.com/OpenListTeam/OpenList/v4/drivers/base" + "github.com/OpenListTeam/OpenList/v4/internal/driver" + "github.com/OpenListTeam/OpenList/v4/internal/errs" + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/pkg/utils" + hash_extend "github.com/OpenListTeam/OpenList/v4/pkg/utils/hash" + "github.com/go-resty/resty/v2" +) + +// RapidHashAlgos 返回 PikPak 支持的秒传哈希算法(GCID) +func (d *PikPak) RapidHashAlgos() []utils.HashType { + return []utils.HashType{*hash_extend.GCID} +} + +// RapidHashNeedsPieces PikPak 不需要分片哈希 +func (d *PikPak) RapidHashNeedsPieces() bool { + return false +} + +// RapidUploadByHashes 使用种子中的 GCID 哈希尝试秒传 +func (d *PikPak) RapidUploadByHashes(ctx context.Context, dstDir model.Obj, req *driver.SeedRapidUploadRequest, overwrite bool) (model.Obj, error) { + gcid := req.Whole.GetHash(hash_extend.GCID) + if len(gcid) < hash_extend.GCID.Width { + return nil, errs.ErrUnavailableHash + } + + var resp UploadTaskData + _, err := d.request("https://api-drive.mypikpak.net/drive/v1/files", http.MethodPost, func(r *resty.Request) { + r.SetContext(ctx).SetBody(base.Json{ + "kind": "drive#file", + "name": req.Name, + "size": req.Size, + "hash": strings.ToUpper(gcid), + "upload_type": "UPLOAD_TYPE_RESUMABLE", + "objProvider": base.Json{"provider": "UPLOAD_TYPE_UNKNOWN"}, + "parent_id": dstDir.GetID(), + "folder_type": "NORMAL", + }) + }, &resp) + if err != nil { + return nil, err + } + + // 秒传成功时不会返回 Resumable + if resp.Resumable == nil { + file := fileToObj(resp.File) + return file, nil + } + + return nil, errs.ErrHashMismatch +} diff --git a/drivers/quark_open/seed_rapid.go b/drivers/quark_open/seed_rapid.go new file mode 100644 index 00000000..87439e30 --- /dev/null +++ b/drivers/quark_open/seed_rapid.go @@ -0,0 +1,60 @@ +package quark_open + +import ( + "context" + "time" + + "github.com/OpenListTeam/OpenList/v4/internal/driver" + "github.com/OpenListTeam/OpenList/v4/internal/errs" + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/pkg/utils" +) + +// RapidHashAlgos 返回夸克网盘支持的秒传哈希算法(MD5 + SHA1) +func (d *QuarkOpen) RapidHashAlgos() []utils.HashType { + return []utils.HashType{*utils.MD5, *utils.SHA1} +} + +// RapidHashNeedsPieces 夸克网盘不需要分片哈希 +func (d *QuarkOpen) RapidHashNeedsPieces() bool { + return false +} + +// RapidUploadByHashes 使用种子中的 MD5/SHA1 哈希尝试秒传。 +// +// 夸克网盘的预上传需要 proof_code(按 proof range 读取的一段内容), +// 因此当内容源不可用时无法完成秒传。 +func (d *QuarkOpen) RapidUploadByHashes(ctx context.Context, dstDir model.Obj, req *driver.SeedRapidUploadRequest, overwrite bool) (model.Obj, error) { + md5Hash := req.Whole.GetHash(utils.MD5) + sha1Hash := req.Whole.GetHash(utils.SHA1) + if len(md5Hash) < utils.MD5.Width || len(sha1Hash) < utils.SHA1.Width { + return nil, errs.ErrUnavailableHash + } + if req.Open == nil { + return nil, errs.ErrUnavailableHash + } + + stream, err := req.Open() + if err != nil { + return nil, err + } + defer stream.Close() + + pre, err := d.upPre(ctx, stream, dstDir.GetID(), md5Hash, sha1Hash) + if err != nil { + return nil, err + } + if !pre.Data.Finish { + return nil, errs.ErrHashMismatch + } + + return &model.ObjThumb{ + Object: model.Object{ + ID: pre.Data.Fid, + Name: req.Name, + Size: req.Size, + IsFolder: false, + Modified: time.Now(), + }, + }, nil +} diff --git a/drivers/quark_open/types.go b/drivers/quark_open/types.go index 652b0a5a..c54cb618 100644 --- a/drivers/quark_open/types.go +++ b/drivers/quark_open/types.go @@ -2,6 +2,7 @@ package quark_open import ( "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/pkg/utils" "time" ) @@ -57,6 +58,7 @@ func fileToObj(f File) *model.ObjThumb { Modified: time.UnixMilli(f.UpdatedAt), IsFolder: f.FileType == "0", Ctime: time.UnixMilli(f.CreatedAt), + HashInfo: utils.NewHashInfo(utils.SHA1, f.ContentHash), }, Thumbnail: model.Thumbnail{Thumbnail: f.ThumbnailURL}, } diff --git a/drivers/thunder/seed_rapid.go b/drivers/thunder/seed_rapid.go new file mode 100644 index 00000000..165a5634 --- /dev/null +++ b/drivers/thunder/seed_rapid.go @@ -0,0 +1,56 @@ +package thunder + +import ( + "context" + "net/http" + + "github.com/OpenListTeam/OpenList/v4/drivers/base" + "github.com/OpenListTeam/OpenList/v4/internal/driver" + "github.com/OpenListTeam/OpenList/v4/internal/errs" + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/pkg/utils" + hash_extend "github.com/OpenListTeam/OpenList/v4/pkg/utils/hash" + "github.com/go-resty/resty/v2" +) + +// RapidHashAlgos 返回迅雷支持的秒传哈希算法(GCID) +func (xc *XunLeiCommon) RapidHashAlgos() []utils.HashType { + return []utils.HashType{*hash_extend.GCID} +} + +// RapidHashNeedsPieces 迅雷不需要分片哈希 +func (xc *XunLeiCommon) RapidHashNeedsPieces() bool { + return false +} + +// RapidUploadByHashes 使用种子中的 GCID 哈希尝试秒传 +func (xc *XunLeiCommon) RapidUploadByHashes(ctx context.Context, dstDir model.Obj, req *driver.SeedRapidUploadRequest, overwrite bool) (model.Obj, error) { + gcid := req.Whole.GetHash(hash_extend.GCID) + if len(gcid) < hash_extend.GCID.Width { + return nil, errs.ErrUnavailableHash + } + + var resp UploadTaskResponse + _, err := xc.Request(FILE_API_URL, http.MethodPost, func(r *resty.Request) { + r.SetContext(ctx) + r.SetBody(&base.Json{ + "kind": FILE, + "parent_id": dstDir.GetID(), + "name": req.Name, + "size": req.Size, + "hash": gcid, + "upload_type": UPLOAD_TYPE_RESUMABLE, + "space": xc.Space, + }) + }, &resp) + if err != nil { + return nil, err + } + + // 秒传成功(UploadType != UPLOAD_TYPE_RESUMABLE) + if resp.UploadType != UPLOAD_TYPE_RESUMABLE { + return &resp.File, nil + } + + return nil, errs.ErrHashMismatch +} diff --git a/drivers/thunder_browser/seed_rapid.go b/drivers/thunder_browser/seed_rapid.go new file mode 100644 index 00000000..c9b456bc --- /dev/null +++ b/drivers/thunder_browser/seed_rapid.go @@ -0,0 +1,55 @@ +package thunder_browser + +import ( + "context" + "net/http" + + "github.com/OpenListTeam/OpenList/v4/drivers/base" + "github.com/OpenListTeam/OpenList/v4/internal/driver" + "github.com/OpenListTeam/OpenList/v4/internal/errs" + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/pkg/utils" + hash_extend "github.com/OpenListTeam/OpenList/v4/pkg/utils/hash" + "github.com/go-resty/resty/v2" +) + +// RapidHashAlgos 返回迅雷浏览器支持的秒传哈希算法(GCID) +func (xc *XunLeiBrowserCommon) RapidHashAlgos() []utils.HashType { + return []utils.HashType{*hash_extend.GCID} +} + +// RapidHashNeedsPieces 迅雷浏览器不需要分片哈希 +func (xc *XunLeiBrowserCommon) RapidHashNeedsPieces() bool { + return false +} + +// RapidUploadByHashes 使用种子中的 GCID 哈希尝试秒传 +func (xc *XunLeiBrowserCommon) RapidUploadByHashes(ctx context.Context, dstDir model.Obj, req *driver.SeedRapidUploadRequest, overwrite bool) (model.Obj, error) { + gcid := req.Whole.GetHash(hash_extend.GCID) + if len(gcid) < hash_extend.GCID.Width { + return nil, errs.ErrUnavailableHash + } + + var resp UploadTaskResponse + _, err := xc.Request(FILE_API_URL, http.MethodPost, func(r *resty.Request) { + r.SetContext(ctx) + r.SetBody(&base.Json{ + "kind": FILE, + "parent_id": dstDir.GetID(), + "name": req.Name, + "size": req.Size, + "hash": gcid, + "upload_type": UPLOAD_TYPE_RESUMABLE, + }) + }, &resp) + if err != nil { + return nil, err + } + + // 秒传成功(UploadType != UPLOAD_TYPE_RESUMABLE) + if resp.UploadType != UPLOAD_TYPE_RESUMABLE { + return &resp.File, nil + } + + return nil, errs.ErrHashMismatch +} diff --git a/drivers/thunderx/seed_rapid.go b/drivers/thunderx/seed_rapid.go new file mode 100644 index 00000000..eda68c73 --- /dev/null +++ b/drivers/thunderx/seed_rapid.go @@ -0,0 +1,55 @@ +package thunderx + +import ( + "context" + "net/http" + + "github.com/OpenListTeam/OpenList/v4/drivers/base" + "github.com/OpenListTeam/OpenList/v4/internal/driver" + "github.com/OpenListTeam/OpenList/v4/internal/errs" + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/pkg/utils" + hash_extend "github.com/OpenListTeam/OpenList/v4/pkg/utils/hash" + "github.com/go-resty/resty/v2" +) + +// RapidHashAlgos 返回迅雷X支持的秒传哈希算法(GCID) +func (xc *XunLeiXCommon) RapidHashAlgos() []utils.HashType { + return []utils.HashType{*hash_extend.GCID} +} + +// RapidHashNeedsPieces 迅雷X不需要分片哈希 +func (xc *XunLeiXCommon) RapidHashNeedsPieces() bool { + return false +} + +// RapidUploadByHashes 使用种子中的 GCID 哈希尝试秒传 +func (xc *XunLeiXCommon) RapidUploadByHashes(ctx context.Context, dstDir model.Obj, req *driver.SeedRapidUploadRequest, overwrite bool) (model.Obj, error) { + gcid := req.Whole.GetHash(hash_extend.GCID) + if len(gcid) < hash_extend.GCID.Width { + return nil, errs.ErrUnavailableHash + } + + var resp UploadTaskResponse + _, err := xc.Request(FILE_API_URL, http.MethodPost, func(r *resty.Request) { + r.SetContext(ctx) + r.SetBody(&base.Json{ + "kind": FILE, + "parent_id": dstDir.GetID(), + "name": req.Name, + "size": req.Size, + "hash": gcid, + "upload_type": UPLOAD_TYPE_RESUMABLE, + }) + }, &resp) + if err != nil { + return nil, err + } + + // 秒传成功(UploadType != UPLOAD_TYPE_RESUMABLE) + if resp.UploadType != UPLOAD_TYPE_RESUMABLE { + return &resp.File, nil + } + + return nil, errs.ErrHashMismatch +} diff --git a/drivers/webdav/driver.go b/drivers/webdav/driver.go index 778d810b..e060c9ab 100644 --- a/drivers/webdav/driver.go +++ b/drivers/webdav/driver.go @@ -6,6 +6,7 @@ import ( "net/http" "os" "path" + "strings" "time" "github.com/OpenListTeam/OpenList/v4/drivers/base" @@ -130,6 +131,11 @@ func (d *WebDav) Put(ctx context.Context, dstDir model.Obj, s model.FileStreamer func (d *WebDav) Get(ctx context.Context, _path string) (model.Obj, error) { _path = path.Join(d.GetRootPath(), _path) info, err := d.client.Stat(_path) + if err != nil && !gowebdav.IsErrNotFound(err) && !strings.HasSuffix(_path, "/") { + // As in RFC 4918 Section 5.2, retry failed slash-less collection PROPFINDs + // once with the canonical trailing-slash URL + info, err = d.client.Stat(_path + "/") + } if err != nil { if gowebdav.IsErrNotFound(err) { return nil, errs.ObjectNotFound diff --git a/drivers/webdav/driver_test.go b/drivers/webdav/driver_test.go index cccd16d5..dd59ca9e 100644 --- a/drivers/webdav/driver_test.go +++ b/drivers/webdav/driver_test.go @@ -42,6 +42,55 @@ func TestMakeDirAfterMissingWebDAVStat(t *testing.T) { } } +// TestGetRetriesCollectionWithTrailingSlash covers servers that return +// 200 HTML for a slash-less collection PROPFIND. RFC 4918 Section 5.2 +// recommends using the trailing-slash for all collection URLs. +func TestGetRetriesCollectionWithTrailingSlash(t *testing.T) { + collectionXML := func(href string) string { + return ` + + + ` + href + ` + + + share + + + HTTP/1.1 200 OK + + +` + } + d, cleanup := newTestDriver(t, func(w http.ResponseWriter, r *http.Request) bool { + if r.Method != "PROPFIND" { + return false + } + switch r.URL.Path { + case "/share": + // The quirky server: no 207 without the trailing slash. + w.Header().Set("Content-Type", "text/html; charset=utf-8") + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("not a multistatus")) + return true + case "/share/": + w.Header().Set("Content-Type", "application/xml; charset=utf-8") + w.WriteHeader(http.StatusMultiStatus) + _, _ = w.Write([]byte(collectionXML("/share/"))) + return true + } + return false + }) + defer cleanup() + + obj, err := d.Get(context.Background(), "/share") + if err != nil { + t.Fatalf("Get on a collection whose slash-less PROPFIND 200s must succeed via the trailing-slash retry: %v", err) + } + if !obj.IsDir() { + t.Fatalf("expected a folder, got file %q", obj.GetName()) + } +} + func newTestDriver(t *testing.T, extra func(http.ResponseWriter, *http.Request) bool) (*WebDav, func()) { t.Helper() diff --git a/drivers/wopan/driver.go b/drivers/wopan/driver.go index 3ff39c0c..c88c4cc7 100644 --- a/drivers/wopan/driver.go +++ b/drivers/wopan/driver.go @@ -2,6 +2,7 @@ package template import ( "context" + "errors" "fmt" "strconv" @@ -13,6 +14,11 @@ import ( "github.com/go-resty/resty/v2" ) +// minAccessTokenLen mirrors the check inside the SDK: the first 16 bytes of the +// access token are used as the AES key of every dispatcher request, so a token +// shorter than that can never be used and must be refreshed first. +const minAccessTokenLen = 16 + type Wopan struct { model.Storage Addition @@ -20,6 +26,18 @@ type Wopan struct { defaultFamilyID string } +// onTokenRefreshed keeps the driver storage in sync with the tokens issued by +// the SDK, so a refresh survives restarts. +func (d *Wopan) onTokenRefreshed(accessToken, refreshToken string) { + d.setTokens(accessToken, refreshToken) + op.MustSaveDriverStorage(d) +} + +func (d *Wopan) setTokens(accessToken, refreshToken string) { + d.AccessToken = accessToken + d.RefreshToken = refreshToken +} + func (d *Wopan) Config() driver.Config { return config } @@ -31,17 +49,50 @@ func (d *Wopan) GetAddition() driver.Additional { func (d *Wopan) Init(ctx context.Context) error { d.client = wopan.DefaultWithRefreshToken(d.RefreshToken) d.client.SetAccessToken(d.AccessToken) - d.client.OnRefreshToken(func(accessToken, refreshToken string) { - d.AccessToken = accessToken - d.RefreshToken = refreshToken - op.MustSaveDriverStorage(d) + d.client.OnRefreshToken(d.onTokenRefreshed) + return d.initClient(ctx) +} + +// initClient prepares the client before any dispatcher request is served. +// +// Two things have to happen here, in this order: +// +// 1. The browser User-Agent must be applied to the underlying resty client. +// The SDK only records it in WithUA/SetUA (and uses w.ua for upload2C), so +// dispatcher requests would otherwise be sent with resty's default +// "go-resty/..." UA, which the WoPan edge rejects with HTTP 403 +// ("failed to list objs: request failed with status: 403 Forbidden"). +// 2. A missing or too short access token must be refreshed before the first +// request. Every wohome/api-user request body is AES encrypted with +// accessToken[:16]; with an empty token the SDK fails with +// "crypto/aes: invalid key size 0", which used to break refresh-token-only +// configurations during FamilyUserCurrentEncode. +func (d *Wopan) initClient(ctx context.Context) error { + d.client.SetUserAgent(wopan.DefaultUA) + + if len(d.AccessToken) < minAccessTokenLen { + if d.RefreshToken == "" { + return fmt.Errorf("an access token of at least %d characters or a refresh token is required", minAccessTokenLen) + } + if err := d.client.RefreshToken(); err != nil { + return fmt.Errorf("refresh access token: %w", err) + } + if accessToken, _ := d.client.GetToken(); len(accessToken) < minAccessTokenLen { + return errors.New("refresh access token: the server returned an invalid access token") + } + } + + if err := d.client.InitData(); err != nil { + return fmt.Errorf("initialize user data: %w", err) + } + fml, err := d.client.FamilyUserCurrentEncode(func(req *resty.Request) { + req.SetContext(ctx) }) - fml, err := d.client.FamilyUserCurrentEncode() if err != nil { - return err + return fmt.Errorf("query default family: %w", err) } d.defaultFamilyID = strconv.Itoa(fml.DefaultHomeId) - return d.client.InitData() + return nil } func (d *Wopan) Drop(ctx context.Context) error { diff --git a/drivers/wopan/driver_test.go b/drivers/wopan/driver_test.go new file mode 100644 index 00000000..bd541aa7 --- /dev/null +++ b/drivers/wopan/driver_test.go @@ -0,0 +1,189 @@ +package template + +import ( + "context" + "encoding/json" + "io" + "net/http" + "strings" + "testing" + + "github.com/OpenListTeam/wopan-sdk-go" +) + +type roundTripFunc func(*http.Request) (*http.Response, error) + +func (f roundTripFunc) RoundTrip(r *http.Request) (*http.Response, error) { + return f(r) +} + +// woPanMock mimics the WoPan dispatcher: it answers with the documented +// envelope and rejects requests whose User-Agent is not a browser UA, exactly +// like the edge did before the driver applied wopan.DefaultUA. +type woPanMock struct { + calls []string + refreshed bool + badUA int + refreshURL string +} + +func (m *woPanMock) transport(t *testing.T) http.RoundTripper { + t.Helper() + return roundTripFunc(func(r *http.Request) (*http.Response, error) { + ua := r.Header.Get("User-Agent") + if ua != wopan.DefaultUA { + m.badUA++ + return &http.Response{ + StatusCode: http.StatusForbidden, + Status: "403 Forbidden", + Body: io.NopCloser(strings.NewReader("forbidden")), + Header: make(http.Header), + Request: r, + }, nil + } + + raw, err := io.ReadAll(r.Body) + if err != nil { + t.Fatalf("read request body: %v", err) + } + var req struct { + Header struct { + Key string `json:"key"` + } `json:"header"` + } + if err := json.Unmarshal(raw, &req); err != nil { + t.Fatalf("decode request body: %v", err) + } + m.calls = append(m.calls, req.Header.Key) + + data := "{}" + switch req.Header.Key { + case wopan.KeyAppRefreshToken: + m.refreshed = true + data = `{"access_token":"fedcba9876543210","refresh_token":"new-refresh"}` + case wopan.KeyAppQueryUser: + data = `{"userId":"test-user"}` + case wopan.KeyClassifyRule: + data = `{"fileTypes":{}}` + case wopan.KeyGetZoneInfo: + data = `{"url":"https://example.invalid"}` + case wopan.KeyFamilyUserCurrentEncode: + data = `{"defaultHomeId":42}` + } + body := `{"STATUS":"200","RSP":{"RSP_CODE":"0000","RSP_DESC":"","DATA":` + data + `}}` + return &http.Response{ + StatusCode: http.StatusOK, + Status: "200 OK", + Body: io.NopCloser(strings.NewReader(body)), + Header: http.Header{"Content-Type": []string{"application/json"}}, + Request: r, + }, nil + }) +} + +// newTestClient mirrors Wopan.Init, but injects a mock transport and a token +// callback that does not touch the database. +func newTestClient(d *Wopan, rt http.RoundTripper) { + d.client = wopan.DefaultWithRefreshToken(d.RefreshToken) + d.client.SetAccessToken(d.AccessToken) + d.client.SetHttpClient(&http.Client{Transport: rt}) + d.client.OnRefreshToken(d.setTokens) +} + +func TestInitClientCredentials(t *testing.T) { + const validAccessToken = "0123456789abcdef" + + tests := []struct { + name string + accessToken string + refreshToken string + wantRefresh bool + wantErr bool + wantAccessToken string + wantFamilyID string + }{ + { + name: "refresh only", + refreshToken: "refresh", + wantRefresh: true, + wantAccessToken: "fedcba9876543210", + wantFamilyID: "42", + }, + { + name: "short access token with refresh", + accessToken: "short", + refreshToken: "refresh", + wantRefresh: true, + wantAccessToken: "fedcba9876543210", + wantFamilyID: "42", + }, + { + name: "both tokens", + accessToken: validAccessToken, + refreshToken: "refresh", + wantAccessToken: validAccessToken, + wantFamilyID: "42", + }, + { + name: "access token only", + accessToken: validAccessToken, + wantAccessToken: validAccessToken, + wantFamilyID: "42", + }, + { + name: "no credentials", + wantErr: true, + }, + { + name: "short access token without refresh", + accessToken: "short", + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + mock := &woPanMock{} + d := &Wopan{Addition: Addition{ + AccessToken: tt.accessToken, + RefreshToken: tt.refreshToken, + }} + newTestClient(d, mock.transport(t)) + + err := d.initClient(context.Background()) + if tt.wantErr { + if err == nil { + t.Fatal("expected an error, got nil") + } + if len(mock.calls) != 0 { + t.Fatalf("expected no dispatcher call on configuration error, got %v", mock.calls) + } + return + } + if err != nil { + t.Fatalf("initClient returned error: %v", err) + } + if mock.badUA != 0 { + t.Fatalf("%d request(s) were sent without the browser user agent", mock.badUA) + } + if mock.refreshed != tt.wantRefresh { + t.Fatalf("refresh called = %v, want %v (calls: %v)", mock.refreshed, tt.wantRefresh, mock.calls) + } + if len(mock.calls) == 0 { + t.Fatal("expected dispatcher calls, got none") + } + if tt.wantRefresh && mock.calls[0] != wopan.KeyAppRefreshToken { + t.Fatalf("first call = %q, want %q", mock.calls[0], wopan.KeyAppRefreshToken) + } + if d.AccessToken != tt.wantAccessToken { + t.Fatalf("AccessToken = %q, want %q", d.AccessToken, tt.wantAccessToken) + } + if tt.wantRefresh && d.RefreshToken != "new-refresh" { + t.Fatalf("RefreshToken = %q, want new-refresh", d.RefreshToken) + } + if d.defaultFamilyID != tt.wantFamilyID { + t.Fatalf("defaultFamilyID = %q, want %q", d.defaultFamilyID, tt.wantFamilyID) + } + }) + } +} diff --git a/drivers/wopan/meta.go b/drivers/wopan/meta.go index 55dade1c..655968fa 100644 --- a/drivers/wopan/meta.go +++ b/drivers/wopan/meta.go @@ -9,11 +9,11 @@ type Addition struct { // Usually one of two driver.RootID // define other - RefreshToken string `json:"refresh_token" required:"true"` + RefreshToken string `json:"refresh_token" help:"Provide a refresh token or an access token (at least 16 characters)"` FamilyID string `json:"family_id" help:"Keep it empty if you want to use your personal drive"` SortRule string `json:"sort_rule" type:"select" options:"name_asc,name_desc,time_asc,time_desc,size_asc,size_desc" default:"name_asc"` - AccessToken string `json:"access_token"` + AccessToken string `json:"access_token" help:"Optional when a refresh token is provided; it is refreshed automatically when missing or too short"` } var config = driver.Config{ diff --git a/go.mod b/go.mod index c8b089eb..0470237f 100644 --- a/go.mod +++ b/go.mod @@ -2,15 +2,15 @@ module github.com/OpenListTeam/OpenList/v4 go 1.26.0 -toolchain go1.27.1 +toolchain go1.27.3 require ( github.com/Azure/azure-sdk-for-go/sdk/azcore v1.23.1 - github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.8.0 + github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.8.1 github.com/KarpelesLab/reflink v1.0.2 github.com/KirCute/zip v1.0.1 github.com/OpenListTeam/go-cache v0.1.0 - github.com/OpenListTeam/gofakes3 v0.8.2-0.20260911142347-cd3c030a83b4 + github.com/OpenListTeam/gofakes3 v0.8.2 github.com/OpenListTeam/sftpd-openlist v1.0.1 github.com/OpenListTeam/tache v0.2.2 github.com/OpenListTeam/times v0.1.0 @@ -64,7 +64,8 @@ require ( github.com/pkg/sftp v1.13.11 github.com/pquerna/otp v1.5.0 github.com/quic-go/quic-go v0.61.0 - github.com/rclone/rclone v1.75.1 + github.com/rclone/rclone v1.75.2 + github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd github.com/shirou/gopsutil/v4 v4.26.8 github.com/sirupsen/logrus v1.10.2 github.com/spf13/afero v1.15.0 @@ -75,15 +76,14 @@ require ( github.com/u2takey/ffmpeg-go v0.5.0 github.com/upyun/go-sdk/v3 v3.0.4 github.com/zzzhr1990/go-common-entity v0.0.0-20250202070650-1a200048f0d3 - golang.org/x/crypto v0.56.0 + golang.org/x/crypto v0.57.0 golang.org/x/image v0.45.0 - golang.org/x/net v0.58.0 + golang.org/x/net v0.60.0 golang.org/x/oauth2 v0.36.0 golang.org/x/time v0.15.0 - google.golang.org/appengine v1.6.8 gopkg.in/ldap.v3 v3.1.0 gorm.io/driver/mysql v1.6.0 - gorm.io/driver/postgres v1.6.2 + gorm.io/driver/postgres v1.6.3 gorm.io/driver/sqlite v1.6.0 gorm.io/gorm v1.31.2 ) @@ -100,6 +100,7 @@ require ( github.com/RoaringBitmap/roaring/v2 v2.14.5 // indirect github.com/adrg/xdg v0.5.3 // indirect github.com/andybalholm/cascadia v1.3.4 // indirect + github.com/apache/arrow-go/v18 v18.7.0 // indirect github.com/blevesearch/zapx/v17 v17.2.3 // indirect github.com/bradenaw/juniper v0.15.3 // indirect github.com/bytedance/gopkg v0.1.3 // indirect @@ -120,6 +121,7 @@ require ( github.com/go-viper/mapstructure/v2 v2.5.0 // indirect github.com/goccy/go-yaml v1.19.2 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect + github.com/google/flatbuffers v25.12.19+incompatible // indirect github.com/hashicorp/go-uuid v1.0.3 // indirect github.com/jcmturner/aescts/v2 v2.0.0 // indirect github.com/jcmturner/dnsutils/v2 v2.0.0 // indirect @@ -132,6 +134,7 @@ require ( github.com/minio/minlz v1.2.0 // indirect github.com/minio/xxml v0.0.3 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/ncruces/go-strftime v1.0.0 // indirect github.com/philhofer/fwd v1.2.0 // indirect github.com/quic-go/qpack v0.6.0 // indirect github.com/relvacode/iso8601 v1.7.0 // indirect @@ -145,12 +148,12 @@ require ( go.mongodb.org/mongo-driver/v2 v2.5.0 // indirect go.yaml.in/yaml/v2 v2.4.4 // indirect go.yaml.in/yaml/v3 v3.0.5 // indirect - golang.org/x/exp v0.0.0-20260709172345-9ea1abe57597 // indirect + golang.org/x/exp v0.0.0-20260813180055-c1d0aacb2297 // indirect gopkg.in/go-jose/go-jose.v2 v2.6.3 // indirect - modernc.org/libc v1.22.5 // indirect - modernc.org/mathutil v1.5.0 // indirect - modernc.org/memory v1.5.0 // indirect - modernc.org/sqlite v1.23.1 // indirect + modernc.org/libc v1.73.4 // indirect + modernc.org/mathutil v1.7.1 // indirect + modernc.org/memory v1.11.0 // indirect + modernc.org/sqlite v1.53.0 // indirect ) require ( @@ -251,7 +254,7 @@ require ( github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/jzelinskie/whirlpool v0.0.0-20201016144138-0675e54bb004 // indirect - github.com/klauspost/compress v1.19.0 // indirect + github.com/klauspost/compress v1.19.2 // indirect github.com/klauspost/cpuid/v2 v2.4.0 // indirect github.com/kr/fs v0.1.0 // indirect github.com/leodido/go-urn v1.4.0 // indirect @@ -285,7 +288,7 @@ require ( github.com/multiformats/go-multistream v0.4.1 // indirect github.com/multiformats/go-varint v0.1.0 // indirect github.com/pelletier/go-toml/v2 v2.2.4 // indirect - github.com/pierrec/lz4/v4 v4.1.27 // indirect + github.com/pierrec/lz4/v4 v4.1.28 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect github.com/pquerna/cachecontrol v0.1.0 // indirect @@ -309,14 +312,14 @@ require ( github.com/yusufpapurcu/wmi v1.2.4 // indirect go.etcd.io/bbolt v1.5.0 // indirect golang.org/x/arch v0.23.0 // indirect - golang.org/x/sync v0.22.0 - golang.org/x/sys v0.47.0 - golang.org/x/term v0.45.0 // indirect - golang.org/x/text v0.41.0 - golang.org/x/tools v0.48.0 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20260715232425-e75dac1f907d // indirect - google.golang.org/grpc v1.85.0-dev - google.golang.org/protobuf v1.36.11 // indirect + golang.org/x/sync v0.23.0 + golang.org/x/sys v0.48.0 + golang.org/x/term v0.46.0 // indirect + golang.org/x/text v0.42.0 + golang.org/x/tools v0.49.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260817212433-ac3dfec99bb1 // indirect + google.golang.org/grpc v1.85.0-dev.0.20260825072537-93e31b48545e + google.golang.org/protobuf v1.36.12 // indirect gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d // indirect gopkg.in/yaml.v3 v3.0.1 // indirect lukechampine.com/blake3 v1.1.7 // indirect diff --git a/go.sum b/go.sum index 2f3b75f1..149f10f9 100644 --- a/go.sum +++ b/go.sum @@ -31,6 +31,8 @@ github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.4 h1:jWQK1GI+LeGGUKBAD github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.4/go.mod h1:8mwH4klAm9DUgR2EEHyEEAQlRDvLPyg5fQry3y+cDew= github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.8.0 h1:irsmOWwkp0KCTTNS5e2hdFeIvSQClQo2No3IaNmL3Vw= github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.8.0/go.mod h1:GWcBkQj3MqN7ozHKLaCCAuNLiXoIGv2RtanfAwSjY/Y= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.8.1 h1:gkBLVmB3Z/HnGP/Jo4o12/RDpi0agnKav6sCKsX5Vu0= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.8.1/go.mod h1:e3/1P5K+jIUi9JevDRklq/tFeTvbBb75bNAjU4xd31w= github.com/AzureAD/microsoft-authentication-library-for-go v1.4.2 h1:oygO0locgZJe7PpYPXT5A29ZkwJaPqcva7BVeemZOZs= github.com/AzureAD/microsoft-authentication-library-for-go v1.4.2/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI= github.com/BurntSushi/toml v1.6.0 h1:dRaEfpa2VI55EwlIW72hMRHdWouJeRF7TPYhI+AUQjk= @@ -53,6 +55,8 @@ github.com/OpenListTeam/go-cache v0.1.0 h1:eV2+FCP+rt+E4OCJqLUW7wGccWZNJMV0NNkh+ github.com/OpenListTeam/go-cache v0.1.0/go.mod h1:AHWjKhNK3LE4rorVdKyEALDHoeMnP8SjiNyfVlB+Pz4= github.com/OpenListTeam/gofakes3 v0.8.2-0.20260911142347-cd3c030a83b4 h1:Zy7/qg6aCS0OF/FPIoJh9/d0IgcIxpWRvn79ACm2R/Y= github.com/OpenListTeam/gofakes3 v0.8.2-0.20260911142347-cd3c030a83b4/go.mod h1:mS9Ywbo6aId6BrRzeYjOIOpK0QDVnMoKOIb0hpaQZ3U= +github.com/OpenListTeam/gofakes3 v0.8.2 h1:iR4B8WH0qWqWkzVTNSj2TgWw6kovTh2bV8TGMOFSnVI= +github.com/OpenListTeam/gofakes3 v0.8.2/go.mod h1:mS9Ywbo6aId6BrRzeYjOIOpK0QDVnMoKOIb0hpaQZ3U= github.com/OpenListTeam/gsync v0.1.0 h1:ywzGybOvA3lW8K1BUjKZ2IUlT2FSlzPO4DOazfYXjcs= github.com/OpenListTeam/gsync v0.1.0/go.mod h1:h/Rvv9aX/6CdW/7B8di3xK3xNV8dUg45Fehrd/ksZ9s= github.com/OpenListTeam/reflink v0.0.0-20260701021214-78760eaeafef h1:67uGHancMF/abMrnkc8abVUWQiG73Wk5d8CKt3RzkFo= @@ -118,6 +122,8 @@ github.com/antchfx/xpath v1.3.6 h1:s0y+ElRRtTQdfHP609qFu0+c6bglDv20pqOViQjjdPI= github.com/antchfx/xpath v1.3.6/go.mod h1:i54GszH55fYfBmoZXapTHN8T8tkcHfRgLyVwwqzXNcs= github.com/antchfx/xpath v1.3.8 h1:RQlkLaJDKk1Ew1H6CUPUTKM+IQxm+6HTyOgcrfqOU9c= github.com/antchfx/xpath v1.3.8/go.mod h1:i54GszH55fYfBmoZXapTHN8T8tkcHfRgLyVwwqzXNcs= +github.com/apache/arrow-go/v18 v18.7.0 h1:Vw/i+cJyebUofT7JlqFpe65LrmwxULn166jjwStM4HY= +github.com/apache/arrow-go/v18 v18.7.0/go.mod h1:PM6IigLJkdMwIpeHXnymo+xZ52f42a9EYiLtRel4p/A= github.com/avast/retry-go v3.0.0+incompatible h1:4SOWQ7Qs+oroOTQOYnAHqelpCO0biHSxpiH9JdtuBj0= github.com/avast/retry-go v3.0.0+incompatible/go.mod h1:XtSnn+n/sHqQIpZ10K1qAevBhOOCWBLXXy3hyiqqBrY= github.com/aws/aws-sdk-go v1.38.20/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= @@ -462,6 +468,8 @@ github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v1.0.0 h1:Oy607GVXHs7RtbggtPBnr2RmDArIsAefDwvrdWvRhGs= github.com/golang/snappy v1.0.0/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/flatbuffers v25.12.19+incompatible h1:haMV2JRRJCe1998HeW/p0X9UaMTK6SDo0ffLn2+DbLs= +github.com/google/flatbuffers v25.12.19+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= @@ -575,6 +583,8 @@ github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+o github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/klauspost/compress v1.19.0 h1:sXLILfc9jV2QYWkzFOPWStmcUVH2RHEB1JCdY2oVvCQ= github.com/klauspost/compress v1.19.0/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= +github.com/klauspost/compress v1.19.2 h1:hMRETovs/pu/dVWN7zIT1PGG8t509MwT6bO7XSi26R8= +github.com/klauspost/compress v1.19.2/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y= @@ -704,6 +714,8 @@ github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/natefinch/lumberjack v2.0.0+incompatible h1:4QJd3OLAMgj7ph+yZTuX13Ld4UpgHp07nNdFX7mqFfM= github.com/natefinch/lumberjack v2.0.0+incompatible/go.mod h1:Wi9p2TTF5DG5oU+6YfsmYQpsTIOm0B1VNzQg9Mw6nPk= +github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w= +github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= github.com/ncw/swift/v2 v2.0.5 h1:9o5Gsd7bInAFEqsGPcaUdsboMbqf8lnNtxqWKFT9iz8= github.com/ncw/swift/v2 v2.0.5/go.mod h1:cbAO76/ZwcFrFlHdXPjaqWZ9R7Hdar7HpjRXBfbjigk= github.com/nwaples/rardecode/v2 v2.2.5 h1:L5doqgGfQwI7qADJMqnkrSB86rpPsqQDrHeO0HWa5JY= @@ -719,6 +731,8 @@ github.com/philhofer/fwd v1.2.0 h1:e6DnBTl7vGY+Gz322/ASL4Gyp1FspeMvx1RNDoToZuM= github.com/philhofer/fwd v1.2.0/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM= github.com/pierrec/lz4/v4 v4.1.27 h1:+PhzhWDrjRj89TH2sw43nE3+4+W8lSxIuQadEHZyjUk= github.com/pierrec/lz4/v4 v4.1.27/go.mod h1:EoQMVJgeeEOMsCqCzqFm2O0cJvljX2nGZjcRIPL34O4= +github.com/pierrec/lz4/v4 v4.1.28 h1:pPEPwRJ4kybBTfGt28q7lQsRJQHhC08axprdLD5Ppio= +github.com/pierrec/lz4/v4 v4.1.28/go.mod h1:EoQMVJgeeEOMsCqCzqFm2O0cJvljX2nGZjcRIPL34O4= github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ= github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -773,6 +787,8 @@ github.com/rclone/rclone v1.75.0 h1:3ARHem4jXWltvl+b0PvDAG8s6J/inHd5BRzfwMRb3W8= github.com/rclone/rclone v1.75.0/go.mod h1:PGLJUW/WSIJCysALqUcxmaCFyfMXUevf8CbuoOwsAdU= github.com/rclone/rclone v1.75.1 h1:kIxQcoDLj2Gke/gMSHK7OnxhX1Gu1cJBLP1kJZoaFp0= github.com/rclone/rclone v1.75.1/go.mod h1:4zmMjGatCkSJPRZDpo+7y3xOl8S29EMUyKvZop5mHr4= +github.com/rclone/rclone v1.75.2 h1:0cqa6P3WmhGnu2NeOdZSetJnacpwCRtpsNbHOICqPk8= +github.com/rclone/rclone v1.75.2/go.mod h1:YmKNkhnFIoIwLsEZl7FSN756Iq5ZXy3LPDuypnuFocA= github.com/relvacode/iso8601 v1.6.0 h1:eFXUhMJN3Gz8Rcq82f9DTMW0svjtAVuIEULglM7QHTU= github.com/relvacode/iso8601 v1.6.0/go.mod h1:FlNp+jz+TXpyRqgmM7tnzHHzBnz776kmAH2h3sZCn0I= github.com/relvacode/iso8601 v1.7.0 h1:BXy+V60stMP6cpswc+a93Mq3e65PfXCgDFfhvNNGrdo= @@ -789,6 +805,8 @@ github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUc github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd h1:CmH9+J6ZSsIjUK3dcGsnCnO41eRBOnY12zwkn5qVwgc= +github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd/go.mod h1:hPqNNc0+uJM6H+SuU8sEs5K5IQeKccPqeSjfgcKGgPk= github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46 h1:GHRpF1pTW19a8tTFrMLUcfWwyC0pnifVo2ClaLq+hP8= github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46/go.mod h1:uAQ5PCi+MFsC7HjREoAz1BU+Mq60+05gifQSsHSDG/8= github.com/secsy/goftp v0.0.0-20200609142545-aa2de14babf4 h1:PT+ElG/UUFMfqy5HrxJxNzj3QBOf7dZwupeVC+mG1Lo= @@ -950,12 +968,16 @@ golang.org/x/crypto v0.55.0 h1:+KWHjbgOaAQ66dh/YlkZKHlz9ZUlq61AFirAR9ntP8M= golang.org/x/crypto v0.55.0/go.mod h1:uq0V9dE/fzQuJtbnL+2EhWOE63vo164FY8xqEnV9xis= golang.org/x/crypto v0.56.0 h1:GUh5Ii4J5jtcseSMiRqr1jXCNHoxjeV9Fmekc2oLy6Y= golang.org/x/crypto v0.56.0/go.mod h1:OMW5y6CY9l38uPLmxU6l6pwcXp1obtLo3e6gT7gQR2I= +golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M= +golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA= golang.org/x/exp v0.0.0-20250606033433-dcc06ee1d476 h1:bsqhLWFR6G6xiQcb+JoGqdKdRU6WzPWmK8E0jxTjzo4= golang.org/x/exp v0.0.0-20250606033433-dcc06ee1d476/go.mod h1:3//PLf8L/X+8b4vuAfHzxeRUl04Adcb341+IGKfnqS8= golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f h1:W3F4c+6OLc6H2lb//N1q4WpJkhzJCK5J6kUi1NTVXfM= golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f/go.mod h1:J1xhfL/vlindoeF/aINzNzt2Bket5bjo9sdOYzOsU80= golang.org/x/exp v0.0.0-20260709172345-9ea1abe57597 h1:qLvzZeaANDgyVOA8pyHCOStGlXn0rseXma+GQjeuv2g= golang.org/x/exp v0.0.0-20260709172345-9ea1abe57597/go.mod h1:EdfpwwqSu+0Li0mzskwHU6FWDV3t9Q+RZDo3QMUtL3Q= +golang.org/x/exp v0.0.0-20260813180055-c1d0aacb2297 h1:YXnL44eJ77R+ji4/ooy8UsXIhz+lbi2Qgdlc8iRN0gY= +golang.org/x/exp v0.0.0-20260813180055-c1d0aacb2297/go.mod h1:Mkmymgv+uMpSQ/XxJ/7GpdrdYoqm3u72jEbpCLiJmNk= golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/image v0.29.0 h1:HcdsyR4Gsuys/Axh0rDEmlBmB68rW1U9BUdB3UVHsas= golang.org/x/image v0.29.0/go.mod h1:RVJROnf3SLK8d26OW91j4FrIHGbsJ8QnbEocVTOWQDA= @@ -990,6 +1012,8 @@ golang.org/x/net v0.57.0 h1:K5+3DljvIuDG9/Jv9rvyMywYNFCQ9RSUY6OOTTkT+tE= golang.org/x/net v0.57.0/go.mod h1:KpXc8iv+r3XplLAG/f7Jsf9RPszJzdR0f58q9vGOuEU= golang.org/x/net v0.58.0 h1:ynWG7rqYi4ccpTEuPZ2QGWHktVEM9DMCj9yzDE0Q7To= golang.org/x/net v0.58.0/go.mod h1:YwCddHnFlT7eLQqVprV19OnhLGtc5xOKgE0RyqgfWAU= +golang.org/x/net v0.60.0 h1:79p50tfZlm0J9YfoDsSi639qSXNGVwEzOPLCxM2FsYU= +golang.org/x/net v0.60.0/go.mod h1:2DA/G1UfVbCpQPeWTmMPGY7Cs2PkBkwu743bVX5PIVg= golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs= golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -1001,6 +1025,8 @@ golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek= golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sync v0.23.0 h1:KameEIfc1IkluZyXWLn39Wd4tURc6GbCiISGiZm2bQk= +golang.org/x/sync v0.23.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1026,6 +1052,8 @@ golang.org/x/sys v0.46.0 h1:noSf2Fq6F8DBgS+LysIkx7rIExoNHJsxOAtPp4rthXw= golang.org/x/sys v0.46.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo= +golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -1040,6 +1068,8 @@ golang.org/x/term v0.44.0 h1:0rLvDRCtNj0gZkyIXhCyOb2OAzEhLVqc4B+hrsBhrmc= golang.org/x/term v0.44.0/go.mod h1:7ze4MdzUzLXpSAoFP1H0bOI9aXDqveSvatT5vKcFh2Y= golang.org/x/term v0.45.0 h1:NwWyBmoJCbfTHpxrWoZ9C6/VxOf7ic219I8xZZFdrf0= golang.org/x/term v0.45.0/go.mod h1:9aqxs0blBcrm/n0L9QW0aRVD+ktan8ssZromtqJC43w= +golang.org/x/term v0.46.0 h1:3+OXuTbaKDgwk8jTi3aSLHRlmWqHEUDUtxnbFigO4YE= +golang.org/x/term v0.46.0/go.mod h1:+K02xbkittuwc0Am4abfA3Fc+XRGXkvBXNO88NCXPoc= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= @@ -1056,6 +1086,8 @@ golang.org/x/text v0.40.0 h1:Ub2Z6/xjgF1WrYQz2nuITOEegKFtiIy+rieRJ5lHZKs= golang.org/x/text v0.40.0/go.mod h1:hpnzDAfGV753zIKo+wk3u1bVKCGPbrnF7+7LBF/UHVY= golang.org/x/text v0.41.0 h1:vz/seA0lnX87Othu2f/0L24RcgrXD9/YFTSuGjj3rH8= golang.org/x/text v0.41.0/go.mod h1:jvf1O8ajNzZqhSrQBPbutR/EB83Cc0CFrezNQIwbb5M= +golang.org/x/text v0.42.0 h1:JbOZXgfeCPU9gacVtYliJqOhD+zhrEqK4LfdpmlUZqI= +golang.org/x/text v0.42.0/go.mod h1:ojzP1Z+2QtioaF8DTtO8K5q7JWVVYwZKenzujK0Zd0E= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI= golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4= @@ -1073,28 +1105,34 @@ golang.org/x/tools v0.47.0 h1:7Kn5x/d1svx/PzryTsqeoZN4TZwqeH5pGWjefhLi/1Q= golang.org/x/tools v0.47.0/go.mod h1:dFHnyTvFWY212G+h7ZY4Vsp/K3U4/7W9TyVaAul8uCA= golang.org/x/tools v0.48.0 h1:3+hClM1aLL5mjMKm5ovokw9epgRXPuu2tILgismM6RE= golang.org/x/tools v0.48.0/go.mod h1:08xX0orndb/F7jJxGDicx061tyd5pcMto75YMAXr6lk= +golang.org/x/tools v0.49.0 h1:3NI7VXzL9+1WZD52Dx2ttoPwD5DWrFGpl9mFZDlmisI= +golang.org/x/tools v0.49.0/go.mod h1:SJNXV9DBKT0UbdttsQjbfJlAE/q+y36++zo3uL3N0Oo= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= google.golang.org/api v0.259.0 h1:90TaGVIxScrh1Vn/XI2426kRpBqHwWIzVBzJsVZ5XrQ= google.golang.org/api v0.259.0/go.mod h1:LC2ISWGWbRoyQVpxGntWwLWN/vLNxxKBK9KuJRI8Te4= -google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM= -google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds= google.golang.org/genproto/googleapis/rpc v0.0.0-20260414002931-afd174a4e478 h1:RmoJA1ujG+/lRGNfUnOMfhCy5EipVMyvUE+KNbPbTlw= google.golang.org/genproto/googleapis/rpc v0.0.0-20260414002931-afd174a4e478/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= google.golang.org/genproto/googleapis/rpc v0.0.0-20260715232425-e75dac1f907d h1:Jkpk39hlTZOIp3RbfvNX9R8Hv+Sw0X89nlU/xFOErsc= google.golang.org/genproto/googleapis/rpc v0.0.0-20260715232425-e75dac1f907d/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260817212433-ac3dfec99bb1 h1:utmqiUzZAEINbLmRyAt1QjVkMyLQ+lvIwCpLaB1rZfw= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260817212433-ac3dfec99bb1/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= google.golang.org/grpc v1.82.0 h1:vguDnZUPjE26w09A63VoxZPnvPjB5Riyc0mkXPFmAIU= google.golang.org/grpc v1.82.0/go.mod h1:yzTZ1TB1Z3SG+LIYaI+WiE8D5+PZ3ArnrSp8zF3+/ZA= google.golang.org/grpc v1.84.0-dev.0.20260723093437-b6eac429d7b6 h1:HfjjkdGIa8u9sP9EW5WCygy0kQDuTI/Tax4j//t24Fo= google.golang.org/grpc v1.84.0-dev.0.20260723093437-b6eac429d7b6/go.mod h1:ljCht0DrxQrXBDRTZp52Qxh3Ffk8CdYm2sj4O2QN2C0= google.golang.org/grpc v1.85.0-dev h1:HxkDyKIIZPpFnroC56tQv5gNuKTmVvi0t7TzOf5zt7g= google.golang.org/grpc v1.85.0-dev/go.mod h1:ljCht0DrxQrXBDRTZp52Qxh3Ffk8CdYm2sj4O2QN2C0= +google.golang.org/grpc v1.85.0-dev.0.20260825072537-93e31b48545e h1:ar+feAPij1Znzug45m6AoShUCc5AaR8FLA3I6LZd94k= +google.golang.org/grpc v1.85.0-dev.0.20260825072537-93e31b48545e/go.mod h1:7aY/KlAHanMAVIWHnk8wloQsdlfwzDz/PP9vCbJG6CU= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= +google.golang.org/protobuf v1.36.12 h1:pJOKDDOyeXErUroCihFAd5LQuwXBSpVnKGrj5o/fwxc= +google.golang.org/protobuf v1.36.12/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d h1:TxyelI5cVkbREznMhfzycHdkp5cLA7DpE+GKjSslYhM= gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d/go.mod h1:cuepJuh7vyXfUyUwEgHQXw849cJrilpS5NeIjOWESAw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -1124,6 +1162,8 @@ gorm.io/driver/postgres v1.6.0 h1:2dxzU8xJ+ivvqTRph34QX+WrRaJlmfyPqXmoGVjMBa4= gorm.io/driver/postgres v1.6.0/go.mod h1:vUw0mrGgrTK+uPHEhAdV4sfFELrByKVGnaVRkXDhtWo= gorm.io/driver/postgres v1.6.2 h1:BvXQ/cNUg63q5TFNg672DmDcowZSFrNLkkA3Xe6GXq4= gorm.io/driver/postgres v1.6.2/go.mod h1:0c4fQA44XhOklXDkgtuKqysHCycTa5i9e3EIpDGCwXk= +gorm.io/driver/postgres v1.6.3 h1:bAn6O2pUa8LtpWEvL5NFU4+52Tfx8Ut7IVaIacCLcI0= +gorm.io/driver/postgres v1.6.3/go.mod h1:0c4fQA44XhOklXDkgtuKqysHCycTa5i9e3EIpDGCwXk= gorm.io/driver/sqlite v1.6.0 h1:WHRRrIiulaPiPFmDcod6prc4l2VGVWHz80KspNsxSfQ= gorm.io/driver/sqlite v1.6.0/go.mod h1:AO9V1qIQddBESngQUKWL9yoH93HIeA1X6V633rBwyT8= gorm.io/gorm v1.25.7/go.mod h1:hbnx/Oo0ChWMn1BIhpy1oYozzpM15i4YPuHDmfYtwg8= @@ -1133,12 +1173,20 @@ lukechampine.com/blake3 v1.1.7 h1:GgRMhmdsuK8+ii6UZFDL8Nb+VyMwadAgcJyfYHxG6n0= lukechampine.com/blake3 v1.1.7/go.mod h1:tkKEOtDkNtklkXtLNEOGNq5tcV90tJiA1vAA12R78LA= modernc.org/libc v1.22.5 h1:91BNch/e5B0uPbJFgqbxXuOnxBQjlS//icfQEGmvyjE= modernc.org/libc v1.22.5/go.mod h1:jj+Z7dTNX8fBScMVNRAYZ/jF91K8fdT2hYMThc3YjBY= +modernc.org/libc v1.73.4 h1:+ra4Ui8ngyt8HDcO1FTDPWlkAh6yOdaO2yAoh8MddQA= +modernc.org/libc v1.73.4/go.mod h1:DXZ3eO8qMCNn2SnmTNCiC71nJ9Rcq3PsnpU6Vc4rWK8= modernc.org/mathutil v1.5.0 h1:rV0Ko/6SfM+8G+yKiyI830l3Wuz1zRutdslNoQ0kfiQ= modernc.org/mathutil v1.5.0/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E= +modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU= +modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg= modernc.org/memory v1.5.0 h1:N+/8c5rE6EqugZwHii4IFsaJ7MUhoWX07J5tC/iI5Ds= modernc.org/memory v1.5.0/go.mod h1:PkUhL0Mugw21sHPeskwZW4D6VscE/GQJOnIpCnW6pSU= +modernc.org/memory v1.11.0 h1:o4QC8aMQzmcwCK3t3Ux/ZHmwFPzE6hf2Y5LbkRs+hbI= +modernc.org/memory v1.11.0/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw= modernc.org/sqlite v1.23.1 h1:nrSBg4aRQQwq59JpvGEQ15tNxoO5pX/kUjcRNwSAGQM= modernc.org/sqlite v1.23.1/go.mod h1:OrDj17Mggn6MhE+iPbBNf7RGKODDE9NFT0f3EwDzJqk= +modernc.org/sqlite v1.53.0 h1:20WG8N9q4ji/dEqGk4uiI0c6OPjSeLTNYGFCc3+7c1M= +modernc.org/sqlite v1.53.0/go.mod h1:xoEpOIpGrgT48H5iiyt/YXPCZPEzlfmfFwtk8Lklw8s= resty.dev/v3 v3.0.0-beta.2 h1:xu4mGAdbCLuc3kbk7eddWfWm4JfhwDtdapwss5nCjnQ= resty.dev/v3 v3.0.0-beta.2/go.mod h1:OgkqiPvTDtOuV4MGZuUDhwOpkY8enjOsjjMzeOHefy4= sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= diff --git a/internal/bootstrap/config.go b/internal/bootstrap/config.go index 83044680..80d513bb 100644 --- a/internal/bootstrap/config.go +++ b/internal/bootstrap/config.go @@ -40,6 +40,7 @@ func InitConfig() { if !filepath.IsAbs(dataDir) { flags.DataDir = filepath.Join(pwd, flags.DataDir) } + conf.DataDir = flags.DataDir // Determine config file path: use flags.ConfigPath if provided, otherwise default to /config.json configPath := flags.ConfigPath if configPath == "" { diff --git a/internal/bootstrap/data/setting.go b/internal/bootstrap/data/setting.go index 4d87f9db..b34df84c 100644 --- a/internal/bootstrap/data/setting.go +++ b/internal/bootstrap/data/setting.go @@ -187,6 +187,14 @@ func InitialSettings() []model.SettingItem { {Key: conf.HandleHookAfterWriting, Value: "false", Type: conf.TypeBool, Group: model.GLOBAL, Flag: model.PRIVATE}, {Key: conf.HandleHookRateLimit, Value: "0", Type: conf.TypeNumber, Group: model.GLOBAL, Flag: model.PRIVATE}, {Key: conf.IgnoreSystemFiles, Value: "false", Type: conf.TypeBool, Group: model.GLOBAL, Flag: model.PRIVATE, Help: `When enabled, ignores common system files during upload (.DS_Store, desktop.ini, Thumbs.db, and files starting with ._)`}, + {Key: conf.SeedSiteURL, Value: "", Type: conf.TypeString, Group: model.GLOBAL, Flag: model.PRIVATE, Help: `Public base URL embedded in generated transfer seed sources when configured`}, + {Key: conf.SeedDefaultMatrix, Value: `{"md5":{"whole":true,"pieces":false},"sha1":{"whole":true,"pieces":false},"sha256":{"whole":true,"pieces":false}}`, Type: conf.TypeText, Group: model.GLOBAL, Flag: model.PRIVATE, Help: `Default right-click hash matrix for transfer seed generation`}, + {Key: conf.SeedFormatPolicies, Value: `{"oss":"off","torrent":"off","cas":"off"}`, Type: conf.TypeText, Group: model.GLOBAL, Flag: model.PRIVATE}, + {Key: conf.SeedDefaultFormat, Value: "oss", Type: conf.TypeSelect, Options: "oss,torrent,cas", Group: model.GLOBAL, Flag: model.PRIVATE}, + {Key: conf.SeedSingleDirectPreview, Value: "false", Type: conf.TypeBool, Group: model.GLOBAL, Flag: model.PUBLIC}, + {Key: conf.SeedCASDirectAccess, Value: "false", Type: conf.TypeBool, Group: model.GLOBAL, Flag: model.PUBLIC, Help: `When opening a single-file CAS seed, immediately rapid-upload it into the same folder and preview the restored file`}, + {Key: conf.SeedAutoGeneratePolicy, Value: "off", Type: conf.TypeSelect, Options: "off,on", Group: model.GLOBAL, Flag: model.PRIVATE, Help: `Global upload sidecar policy; storage-specific inheritance can be layered without changing the safe default`}, + {Key: conf.SeedDefaultTrackers, Value: "", Type: conf.TypeText, Group: model.GLOBAL, Flag: model.PRIVATE, Help: `Default tracker list offered when generating torrent seeds (one tracker per line)`}, // single settings {Key: conf.Token, Value: token, Type: conf.TypeString, Group: model.SINGLE, Flag: model.PRIVATE}, @@ -251,6 +259,114 @@ func InitialSettings() []model.SettingItem { {Key: conf.StreamMaxClientUploadSpeed, Value: "-1", Type: conf.TypeNumber, Group: model.TRAFFIC, Flag: model.PRIVATE}, {Key: conf.StreamMaxServerDownloadSpeed, Value: "-1", Type: conf.TypeNumber, Group: model.TRAFFIC, Flag: model.PRIVATE}, {Key: conf.StreamMaxServerUploadSpeed, Value: "-1", Type: conf.TypeNumber, Group: model.TRAFFIC, Flag: model.PRIVATE}, + + // media settings + {Key: conf.MediaTMDBKey, Value: "", Type: conf.TypeString, Group: model.MEDIA, Flag: model.PRIVATE}, + {Key: conf.MediaTMDBAPIURL, Value: "api.themoviedb.org", Type: conf.TypeString, Group: model.MEDIA, Flag: model.PRIVATE}, + {Key: conf.MediaDiscogsToken, Value: "", Type: conf.TypeString, Group: model.MEDIA, Flag: model.PRIVATE}, + {Key: conf.MediaDiscogsAPIURL, Value: "api.discogs.com", Type: conf.TypeString, Group: model.MEDIA, Flag: model.PRIVATE}, + {Key: conf.MediaStoreThumbnail, Value: "false", Type: conf.TypeBool, Group: model.MEDIA, Flag: model.PRIVATE}, + {Key: conf.MediaThumbnailMode, Value: "runtime", Type: conf.TypeSelect, Options: "base64,runtime,local", Group: model.MEDIA, Flag: model.PRIVATE, + Help: `缩略图存储方式:base64 存入数据库;runtime 存入程序数据目录(data/imgs);local 写入挂载目录下的 VFS 路径`}, + {Key: conf.MediaThumbnailPath, Value: "/imgs", Type: conf.TypeString, Group: model.MEDIA, Flag: model.PRIVATE}, + {Key: conf.MediaScrapeConcurrency, Value: "5", Type: conf.TypeNumber, Group: model.MEDIA, Flag: model.PRIVATE}, + {Key: conf.MediaScrapeLanguage, Value: "zh-CN", Type: conf.TypeSelect, Options: "zh-CN,zh-TW,en-US,ja-JP,ko-KR,fr-FR,de-DE,es-ES,ru-RU,pt-BR", Group: model.MEDIA, Flag: model.PRIVATE, + Help: `首选刮削语言:影响 TMDB 的标题/简介/海报文字与 Google Books 的检索语言;豆瓣固定为中文`}, + {Key: conf.MediaBookSource, Value: "douban", Type: conf.TypeSelect, Options: "douban,google,openlibrary", Group: model.MEDIA, Flag: model.PRIVATE, + Help: `书籍刮削数据源:douban(豆瓣)、google(Google Books)、openlibrary(OpenLibrary)`}, + {Key: conf.MediaGoogleBooksKey, Value: "", Type: conf.TypeString, Group: model.MEDIA, Flag: model.PRIVATE, + Help: `Google Books API Key,可留空(无 Key 时配额较低);申请地址 https://developers.google.com/books/docs/v1/using#APIKey`}, + {Key: conf.MediaDownloadCover, Value: "false", Type: conf.TypeBool, Group: model.MEDIA, Flag: model.PRIVATE, + Help: `开启后把 TMDB / Google Books / OpenLibrary 等返回的封面链接下载到下方缩略图存储路径,而不是直接保存图片链接`}, + {Key: conf.MediaEnabled, Value: "false", Type: conf.TypeBool, Group: model.MEDIA, Flag: model.PUBLIC, + Help: `媒体库总开关。关闭时媒体库相关页面、API 与协议端点全部不可用(返回 404),与未安装媒体库时表现一致`}, + {Key: conf.MediaEmbyEnabled, Value: "false", Type: conf.TypeBool, Group: model.MEDIA, Flag: model.PRIVATE, + Help: `开启后提供 Emby 兼容 API(/@emby),可供 Infuse、VidHub、MoviePilot 等客户端直接接入`}, + {Key: conf.MediaSubsonicEnabled, Value: "false", Type: conf.TypeBool, Group: model.MEDIA, Flag: model.PRIVATE, + Help: `开启后提供 OpenSubsonic API(/@rest),可供 Symfonium、Feishin、Amperfy 等音乐客户端接入`}, + {Key: conf.MediaSubsonicAllowPwd, Value: "false", Type: conf.TypeBool, Group: model.MEDIA, Flag: model.PRIVATE, + Help: `允许 Subsonic 客户端用 OpenList 登录密码(u+p 参数)认证。该方式会把主账号密码暴露给第三方客户端,默认关闭,推荐使用应用密钥(apiKey 或 t+s)`}, + {Key: conf.MediaSubsonicProxyStream, Value: "false", Type: conf.TypeBool, Group: model.MEDIA, Flag: model.PRIVATE, + Help: `Subsonic 播放默认返回 302 直链;若客户端不跟随重定向,可开启此项改为服务端代理转发(占用服务器带宽)`}, + {Key: conf.MediaProxyCover, Value: "false", Type: conf.TypeBool, Group: model.MEDIA, Flag: model.PRIVATE, + Help: `开启后由服务端代理 TMDB/豆瓣等外部封面图片,适合直连图片域名受限的网络;默认关闭,直接使用原始图片地址`}, + {Key: conf.MediaCoverGateway, Value: "false", Type: conf.TypeBool, Group: model.MEDIA, Flag: model.PRIVATE, + Help: `封面代理直连失败时,回退到第三方图片网关 wsrv.nl 重试。开启后封面地址会发送给该第三方服务;默认关闭`}, + {Key: conf.MediaStrmOnScan, Value: "false", Type: conf.TypeBool, Group: model.MEDIA, Flag: model.PRIVATE, + Help: `扫描完成后自动把媒体条目生成为 .strm 文件并写入下方目录`}, + {Key: conf.MediaStrmOverwrite, Value: "true", Type: conf.TypeBool, Group: model.MEDIA, Flag: model.PRIVATE, + Help: `生成 STRM 时覆盖已存在的同名文件`}, + {Key: conf.MediaStrmSameDir, Value: "true", Type: conf.TypeBool, Group: model.MEDIA, Flag: model.PRIVATE, + Help: `开启时 .strm 与媒体文件放在同一目录且同名(仅扩展名不同);关闭时统一输出到下方指定目录`}, + {Key: conf.MediaStrmPath, Value: "/strm", Type: conf.TypeString, Group: model.MEDIA, Flag: model.PRIVATE, + Help: `STRM 输出目录(VFS 路径,需位于可写存储中),仅在关闭「与媒体文件同目录」时使用`}, + {Key: conf.MediaStrmTemplate, Value: "{site_url}/d{path}?sign={sign}", Type: conf.TypeString, Group: model.MEDIA, Flag: model.PRIVATE, + Help: `STRM 内容模板,可用变量:{site_url} 站点地址、{path} 文件虚拟路径、{sign} 签名、{name} 文件名`}, + + // transcode settings (FFmpeg 云端/本地转码) - 默认全部关闭 + {Key: conf.TranscodeEnabled, Value: "false", Type: conf.TypeBool, Group: model.TRANSCODE, Flag: model.PRIVATE, + Help: `开启后,超过阈值的媒体文件将通过 FFmpeg 转码后再播放;默认关闭`}, + {Key: conf.TranscodeRunMode, Value: "local", Type: conf.TypeSelect, Options: "local,remote,hybrid", Group: model.TRANSCODE, Flag: model.PRIVATE, + Help: `local=仅使用本机内置 worker;remote=只使用远程 Worker 节点;hybrid=本地优先,超载后派发到远程`}, + {Key: conf.TranscodeMinSizeGB, Value: "5", Type: conf.TypeNumber, Group: model.TRANSCODE, Flag: model.PRIVATE, + Help: `文件大于该 GB 数才走转码(小于则直链播放),0=任意大小都转码`}, + {Key: conf.TranscodeMinBitrateMbps, Value: "20", Type: conf.TypeNumber, Group: model.TRANSCODE, Flag: model.PRIVATE, + Help: `视频码率超过该 Mbps 才转码,0=不限制`}, + {Key: conf.TranscodeSourceCodecs, Value: "hevc,h265,av1,vvc,vp9", Type: conf.TypeString, Group: model.TRANSCODE, Flag: model.PRIVATE, + Help: `仅对这些源视频编码进行转码(逗号分隔)。常见高码率/兼容性差的编码:hevc,av1,vvc,vp9`}, + {Key: conf.TranscodeSourceExtensions, Value: "mkv,ts,m2ts,mov,avi,wmv,flv,rmvb,webm", Type: conf.TypeString, Group: model.TRANSCODE, Flag: model.PRIVATE, + Help: `仅对这些后缀进行转码(逗号分隔,不带点),mp4 默认不转码可直接播放`}, + {Key: conf.TranscodeOutputFormat, Value: "hls", Type: conf.TypeSelect, Options: "hls,dash,mp4", Group: model.TRANSCODE, Flag: model.PRIVATE, + Help: `输出封装格式,HLS 兼容性最好`}, + {Key: conf.TranscodeOutputCodec, Value: "h264", Type: conf.TypeSelect, Options: "h264,hevc,av1", Group: model.TRANSCODE, Flag: model.PRIVATE, + Help: `重新编码后的视频编码,推荐 h264(最广兼容、互联网分发免授权费)`}, + {Key: conf.TranscodeOutputBitrate, Value: "4000k", Type: conf.TypeString, Group: model.TRANSCODE, Flag: model.PRIVATE, + Help: `输出视频码率,例如 4000k / 6M。建议 1080p:4000k、720p:2500k`}, + {Key: conf.TranscodeOutputAudioCodec, Value: "aac", Type: conf.TypeSelect, Options: "aac,mp3,opus,copy", Group: model.TRANSCODE, Flag: model.PRIVATE, + Help: `输出音频编码,aac 兼容性最好;copy=直接复制源音频流`}, + {Key: conf.TranscodeOutputAudioBitrate, Value: "160k", Type: conf.TypeString, Group: model.TRANSCODE, Flag: model.PRIVATE, + Help: `输出音频码率`}, + {Key: conf.TranscodeOutputResolution, Value: "1920x1080", Type: conf.TypeSelect, Options: "source,3840x2160,2560x1440,1920x1080,1280x720,854x480", Group: model.TRANSCODE, Flag: model.PRIVATE, + Help: `输出分辨率上限,超过该分辨率会下采样;source=保持源分辨率`}, + {Key: conf.TranscodeSegmentDuration, Value: "6", Type: conf.TypeNumber, Group: model.TRANSCODE, Flag: model.PRIVATE, + Help: `HLS/DASH 切片时长(秒),越小首帧越快但请求数变多,推荐 4-10`}, + {Key: conf.TranscodeHWAccel, Value: "none", Type: conf.TypeSelect, + Options: "none,auto,nvenc,qsv,vaapi,amf,videotoolbox", + Group: model.TRANSCODE, Flag: model.PRIVATE, + Help: `GPU 硬件加速: +none = 纯 CPU(libx264) +auto = 自动探测可用加速器 +nvenc = NVIDIA GPU(GeForce/Tesla/Quadro/RTX,需 NVIDIA 驱动) +qsv = Intel 集显/独显 QuickSync(免费,功耗低) +vaapi = Linux 通用 VA-API(支持 Intel/AMD) +amf = AMD GPU(Windows AMF) +videotoolbox = macOS 硬件加速`}, + {Key: conf.TranscodeFFmpegPath, Value: "ffmpeg", Type: conf.TypeString, Group: model.TRANSCODE, Flag: model.PRIVATE, + Help: `FFmpeg 可执行路径,留空则使用 PATH 中的 ffmpeg`}, + {Key: conf.TranscodeFFprobePath, Value: "ffprobe", Type: conf.TypeString, Group: model.TRANSCODE, Flag: model.PRIVATE, + Help: `FFprobe 可执行路径`}, + {Key: conf.TranscodeWorkerSecret, Value: "", Type: conf.TypeString, Group: model.TRANSCODE, Flag: model.PRIVATE, + Help: `远程 Worker 注册时使用的共享密钥,留空则禁用远程 Worker 注册`}, + {Key: conf.TranscodeCachePath, Value: "data/ctmp", Type: conf.TypeString, Group: model.TRANSCODE, Flag: model.PRIVATE, + Help: `转码切片缓存目录`}, + {Key: conf.TranscodeCacheMaxGB, Value: "20", Type: conf.TypeNumber, Group: model.TRANSCODE, Flag: model.PRIVATE, + Help: `切片缓存最大容量(GB),超过后按 LRU 清理`}, + {Key: conf.TranscodeJobTimeoutMin, Value: "120", Type: conf.TypeNumber, Group: model.TRANSCODE, Flag: model.PRIVATE, + Help: `单个转码任务超时分钟数,超时自动失败`}, + {Key: conf.TranscodeLocalConcurrency, Value: "1", Type: conf.TypeNumber, Group: model.TRANSCODE, Flag: model.PRIVATE, + Help: `本机内置 worker 同时执行的转码任务数(local/hybrid 模式生效)`}, + {Key: conf.TranscodeIdleTimeoutSec, Value: "90", Type: conf.TypeNumber, Group: model.TRANSCODE, Flag: model.PRIVATE, + Help: `播放端无请求多少秒后自动停止转码(默认90秒,0=禁用)`}, + {Key: conf.TranscodeChunkDurationSec, Value: "60", Type: conf.TypeNumber, Group: model.TRANSCODE, Flag: model.PRIVATE, + Help: `每个 chunk 时长(秒),越大每段缓冲越长但 ffmpeg 进程切换越少(默认60秒)`}, + {Key: conf.TranscodeMaxChunkConcurrency, Value: "2", Type: conf.TypeNumber, Group: model.TRANSCODE, Flag: model.PRIVATE, + Help: `单个视频任务最多同时运行的 chunk 数(默认2,越大并发越高但 CPU/GPU 占用越多)`}, + {Key: conf.TranscodeChunkIdleSec, Value: "60", Type: conf.TypeNumber, Group: model.TRANSCODE, Flag: model.PRIVATE, + Help: `chunk 多久没被访问就 kill ffmpeg 释放资源(默认60秒)`}, + {Key: conf.TranscodeChunkPrefetch, Value: "true", Type: conf.TypeBool, Group: model.TRANSCODE, Flag: model.PRIVATE, + Help: `当前 chunk 转码进度过半时,是否预启动下一个 chunk(默认开启,可让顺序播放无缝衔接)`}, + + // multipart upload settings (from main) {Key: conf.MultipartEnabled, Value: "true", Type: conf.TypeBool, Group: model.TRAFFIC, Flag: model.PUBLIC}, {Key: conf.MultipartChunkSize, Value: "10", Type: conf.TypeNumber, Group: model.TRAFFIC, Flag: model.PUBLIC, Help: `chunk size of multipart upload in MB (positive integer), keep it under your CDN's request body limit; each active session buffers up to 8 chunks on the server's disk`}, } diff --git a/internal/bootstrap/run.go b/internal/bootstrap/run.go index 6740dba6..558e297c 100644 --- a/internal/bootstrap/run.go +++ b/internal/bootstrap/run.go @@ -15,6 +15,7 @@ import ( "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/db" "github.com/OpenListTeam/OpenList/v4/internal/fs" + "github.com/OpenListTeam/OpenList/v4/internal/transcode" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/OpenListTeam/OpenList/v4/server" "github.com/OpenListTeam/OpenList/v4/server/middlewares" @@ -92,6 +93,10 @@ func Start() { InitOfflineDownloadTools() LoadStorages() InitTaskManager() + // 初始化转码模块(总开关关时也会启动 manager,但本地 worker 仅在 enabled 时由 Manager.Start 内部按 run_mode 启动) + if transcode.IsEnabled() { + transcode.Default().Start() + } if !flags.Debug && !flags.Dev { gin.SetMode(gin.ReleaseMode) } diff --git a/internal/bootstrap/task.go b/internal/bootstrap/task.go index 47e0b59e..a9ea494f 100644 --- a/internal/bootstrap/task.go +++ b/internal/bootstrap/task.go @@ -49,4 +49,5 @@ func InitTaskManager() { op.RegisterSettingChangingCallback(func() { fs.ArchiveContentUploadTaskManager.SetWorkersNumActive(taskFilterNegative(setting.GetInt(conf.TaskDecompressUploadThreadsNum, conf.Conf.Tasks.DecompressUpload.Workers))) }) + fs.SeedGenerateTaskManager = tache.NewManager[*fs.SeedGenerateTask](tache.WithWorks(setting.GetInt(conf.TaskUploadThreadsNum, conf.Conf.Tasks.Upload.Workers)), tache.WithMaxRetry(conf.Conf.Tasks.Upload.MaxRetry)) //seed generation will not support persist } diff --git a/internal/conf/const.go b/internal/conf/const.go index 9a0f11a4..436c3805 100644 --- a/internal/conf/const.go +++ b/internal/conf/const.go @@ -60,6 +60,16 @@ const ( HandleHookRateLimit = "handle_hook_rate_limit" IgnoreSystemFiles = "ignore_system_files" + // transfer seeds + SeedSiteURL = "seed_site_url" + SeedDefaultMatrix = "seed_default_matrix" + SeedFormatPolicies = "seed_format_policies" + SeedDefaultFormat = "seed_default_format" + SeedSingleDirectPreview = "seed_single_direct_preview" + SeedCASDirectAccess = "seed_cas_direct_access" + SeedAutoGeneratePolicy = "seed_auto_generate_policy" + SeedDefaultTrackers = "seed_default_trackers" + // index SearchIndex = "search_index" AutoUpdateIndex = "auto_update_index" @@ -165,8 +175,67 @@ const ( StreamMaxClientUploadSpeed = "max_client_upload_speed" StreamMaxServerDownloadSpeed = "max_server_download_speed" StreamMaxServerUploadSpeed = "max_server_upload_speed" - MultipartEnabled = "multipart_enabled" - MultipartChunkSize = "multipart_chunk_size" + + // media + MediaTMDBKey = "media_tmdb_key" + MediaTMDBAPIURL = "media_tmdb_api_url" + MediaDiscogsToken = "media_discogs_token" + MediaDiscogsAPIURL = "media_discogs_api_url" + MediaThumbnailMode = "media_thumbnail_mode" + MediaThumbnailPath = "media_thumbnail_path" + MediaStoreThumbnail = "media_store_thumbnail" + MediaScrapeConcurrency = "media_scrape_concurrency" + MediaScrapeLanguage = "media_scrape_language" // 首选刮削语言(TMDB / Google Books) + MediaBookSource = "media_book_source" // 书籍刮削源:douban/google/openlibrary + MediaGoogleBooksKey = "media_google_books_key" // Google Books API Key(可选) + MediaDownloadCover = "media_download_cover" // 是否把远程封面下载到本地存储 + + // media library master switch & protocol switches + MediaEnabled = "media_enabled" // 媒体库总开关(PUBLIC,前端据此显示/隐藏) + MediaEmbyEnabled = "media_emby_enabled" // Emby 兼容 API 开关 + MediaSubsonicEnabled = "media_subsonic_enabled" // OpenSubsonic API 开关 + MediaSubsonicProxyStream = "media_subsonic_proxy_stream" // Subsonic 播放是否强制代理(兼容不跟随 302 的客户端) + MediaSubsonicAllowPwd = "media_subsonic_allow_password_auth" // Subsonic 是否允许 u+p 使用 OpenList 登录密码认证(默认关闭) + MediaProxyCover = "media_proxy_cover" // 是否通过服务端代理外部封面(TMDB/豆瓣等) + MediaCoverGateway = "media_cover_gateway" // 封面代理直连失败时是否回退第三方图片网关(wsrv.nl,默认关闭) + MediaStrmOnScan = "media_strm_on_scan" // 扫描完成后自动生成 STRM + MediaStrmSameDir = "media_strm_same_dir" // STRM 是否与媒体文件同目录(同名 .strm) + MediaStrmPath = "media_strm_path" // STRM 输出目录(VFS 路径,仅在指定目录模式下使用) + MediaStrmOverwrite = "media_strm_overwrite" // 生成 STRM 时是否覆盖已存在文件 + MediaStrmTemplate = "media_strm_template" // STRM 内容模板 + + // transcode (云端/本地 FFmpeg 转码) + TranscodeEnabled = "transcode_enabled" // 总开关,默认关 + TranscodeMinSizeGB = "transcode_min_size_gb" // 文件大于多少 GB 才转码 + TranscodeMinBitrateMbps = "transcode_min_bitrate_mbps" // 视频码率高于多少 Mbps 才转码(0=不限) + TranscodeSourceCodecs = "transcode_source_codecs" // 仅对这些源编码进行转码 + TranscodeSourceExtensions = "transcode_source_extensions" // 仅对这些后缀进行转码 + TranscodeOutputFormat = "transcode_output_format" // 输出封装格式 hls/dash/mp4 + TranscodeOutputCodec = "transcode_output_codec" // 重新编码的视频编码 + TranscodeOutputBitrate = "transcode_output_bitrate" // 输出视频码率,例如 4000k + TranscodeOutputAudioCodec = "transcode_output_audio_codec" // 输出音频编码 + TranscodeOutputAudioBitrate = "transcode_output_audio_bitrate" // 输出音频码率,例如 160k + TranscodeOutputResolution = "transcode_output_resolution" // 输出分辨率上限 + TranscodeSegmentDuration = "transcode_segment_duration" // HLS 切片时长(秒) + TranscodeHWAccel = "transcode_hwaccel" // 硬件加速类型 none/auto/nvenc/qsv/vaapi/amf/videotoolbox + TranscodeFFmpegPath = "transcode_ffmpeg_path" // 自定义 ffmpeg 可执行路径 + TranscodeFFprobePath = "transcode_ffprobe_path" // 自定义 ffprobe 可执行路径 + TranscodeRunMode = "transcode_run_mode" // local / remote / hybrid + TranscodeWorkerSecret = "transcode_worker_secret" // 远程 Worker 共享密钥 + TranscodeCachePath = "transcode_cache_path" // 切片缓存目录 + TranscodeCacheMaxGB = "transcode_cache_max_gb" // 缓存最大占用,超出 LRU 淘汰 + TranscodeJobTimeoutMin = "transcode_job_timeout_min" // 单个任务超时分钟 + TranscodeLocalConcurrency = "transcode_local_concurrency" // 本地内置 worker 并发数(run_mode=local/hybrid 时生效) + TranscodeIdleTimeoutSec = "transcode_idle_timeout_sec" // 播放端无请求多少秒后自动停止转码(默认90秒) + // === 智能 chunk 调度相关 === + TranscodeChunkDurationSec = "transcode_chunk_duration_sec" // 每个 chunk 时长(秒),将整个视频分成多个 chunk 按需转码(默认60秒) + TranscodeMaxChunkConcurrency = "transcode_max_chunk_concurrency" // 单个 Job 最多同时运行的 chunk 数(默认2) + TranscodeChunkIdleSec = "transcode_chunk_idle_sec" // chunk 多久没被访问就 kill ffmpeg 释放资源(默认60秒) + TranscodeChunkPrefetch = "transcode_chunk_prefetch" // 当前 chunk 转码进度过半时,是否预启动下一个 chunk(默认true) + + // multipart upload (from main) + MultipartEnabled = "multipart_enabled" + MultipartChunkSize = "multipart_chunk_size" ) const ( @@ -198,4 +267,6 @@ const ( SharingIDKey SkipHookKey DirectUploadContentTypeKey + SkipNoOverwriteKey + ThumbnailBaseURLKey ) diff --git a/internal/conf/var.go b/internal/conf/var.go index 6b25bcfb..c7fed954 100644 --- a/internal/conf/var.go +++ b/internal/conf/var.go @@ -18,6 +18,8 @@ var ( Conf *Config URL *url.URL ConfigPath string + // DataDir 程序数据目录(绝对路径),用于运行时文件(如本地缩略图) + DataDir string = "data" ) var SlicesMap = make(map[string][]string) diff --git a/internal/db/db.go b/internal/db/db.go index 96529c15..eb0c2f68 100644 --- a/internal/db/db.go +++ b/internal/db/db.go @@ -1,6 +1,9 @@ package db import ( + "os" + "strings" + log "github.com/sirupsen/logrus" "github.com/OpenListTeam/OpenList/v4/internal/conf" @@ -10,12 +13,104 @@ import ( var db *gorm.DB +// MediaMigrateWipeEnv 设为 1/true 时,允许升级流程清空旧版 media_items 数据后重建。 +// 旧版本的 folder_path 语义不同,无法原地迁移;但清库是破坏性操作,必须由管理员显式确认。 +const MediaMigrateWipeEnv = "OPENLIST_MEDIA_MIGRATE_WIPE" + func Init(d *gorm.DB) { db = d - err := AutoMigrate(new(model.Storage), new(model.User), new(model.Meta), new(model.SettingItem), new(model.SearchNode), new(model.TaskItem), new(model.SSHPublicKey), new(model.SharingDB)) - if err != nil { - log.Fatalf("failed migrate database: %s", err.Error()) + if err := migrateMediaItems(); err != nil { + log.Fatalf("failed migrate media_items: %s (如果旧数据无法原地迁移,请先备份数据库,再设置 %s=1 重建媒体表)", err.Error(), MediaMigrateWipeEnv) } + if err := migrateWebDAVProperties(); err != nil { + log.Fatalf("failed migrate webdav_properties: %s", err.Error()) + } + err := AutoMigrate(new(model.Storage), new(model.User), new(model.Meta), new(model.SettingItem), new(model.SearchNode), new(model.TaskItem), new(model.SSHPublicKey), new(model.SharingDB), new(model.WebDAVProperty), new(model.MediaItem), new(model.MediaConfig), new(model.MediaScanPath), new(model.MediaAppKey), new(model.MediaUserData), new(model.MediaPlaylist), new(model.MediaPlaylistItem), new(model.EmbyDevice), new(model.MediaAnnotation)) + if err != nil { + // 典型场景:未确认清库时,旧 media_items 数据违反新建的唯一索引。 + // 这里把解决办法直接写进日志,避免用户只看到一句 failed migrate database。 + log.Fatalf( + "failed migrate database: %s (若由媒体表唯一索引与旧数据冲突导致,请先备份数据库,再设置 %s=1 重建媒体表)", + err.Error(), + MediaMigrateWipeEnv, + ) + } +} + +func mediaWipeConfirmed() bool { + switch os.Getenv(MediaMigrateWipeEnv) { + case "1", "true", "TRUE", "True", "yes": + return true + } + return false +} + +// mediaItemsTable 返回 MediaItem 的真实表名。 +// 表名带有 conf.Database.TablePrefix(可配置,默认 x_),不能硬编码。 +func mediaItemsTable() (string, error) { + stmt := &gorm.Statement{DB: db} + if err := stmt.Parse(&model.MediaItem{}); err != nil { + return "", err + } + return stmt.Schema.Table, nil +} + +// migrateMediaItems 处理 media_items 表的迁移兼容性 +// 存储语义历史变更: +// +// v1: folder_path 为文件完整路径 +// v2: folder_path 恒定为扫描根路径 +// v3(当前): folder_path 为文件/文件夹的实际父目录,供前端拼接播放地址 +// +// 唯一性由 folder_path + file_name + album_name 组合索引保证。 +// +// 检测到 v1/v2 的旧表(缺少新组合索引)时: +// - 默认不动数据,只输出醒目的告警,并让 AutoMigrate 在旧表上补建新索引; +// - 管理员设置环境变量 OPENLIST_MEDIA_MIGRATE_WIPE=1 后才会清空旧数据并重扫。 +func migrateMediaItems() error { + table, err := mediaItemsTable() + if err != nil { + return err + } + m := db.Migrator() + if !m.HasTable(table) { + return nil + } + const idx = "idx_media_folder_file_album" + if m.HasIndex(table, idx) { + // 已是新索引:仅清理软删除残留数据,避免与唯一索引冲突 + if err := db.Exec("DELETE FROM "+quoteIdent(table)+" WHERE deleted_at IS NOT NULL").Error; err != nil { + log.Warnf("media_items: 清理软删除残留数据失败: %v", err) + } + return nil + } + + // 旧表存在但没有新组合索引:旧数据的 folder_path 语义已变更,无法复用。 + if !mediaWipeConfirmed() { + log.Warnf("media_items: 检测到旧版本数据(表 %s 缺少索引 %s)。", table, idx) + log.Warnf("media_items: 旧数据的路径语义已变更,建议清空后重新扫描;为避免误删,本次启动不会自动清库。") + log.Warnf("media_items: 如确认清空,请设置环境变量 %s=1 后重启;也可在管理后台对各媒体库点击「清空数据库」再重扫。", MediaMigrateWipeEnv) + return nil + } + log.Warnf("media_items: 已确认 %s,清空旧版本数据(表 %s)后重新迁移", MediaMigrateWipeEnv, table) + // 先尝试删除旧的单字段唯一索引(如果存在),避免 AutoMigrate 冲突 + oldIdx := "idx_" + table + "_folder_path" + if m.HasIndex(table, oldIdx) { + if err := m.DropIndex(table, oldIdx); err != nil { + log.Warnf("media_items: 删除旧唯一索引失败: %v", err) + } + } + if err := db.Exec("DELETE FROM " + quoteIdent(table)).Error; err != nil { + return err + } + return nil +} + +// quoteIdent 按当前数据库方言给标识符加引号,表名来自 gorm schema 解析而非用户输入。 +func quoteIdent(name string) string { + var sb strings.Builder + db.Dialector.QuoteTo(&sb, name) + return sb.String() } func AutoMigrate(dst ...interface{}) error { diff --git a/internal/db/db_media_migrate_test.go b/internal/db/db_media_migrate_test.go new file mode 100644 index 00000000..a270f344 --- /dev/null +++ b/internal/db/db_media_migrate_test.go @@ -0,0 +1,123 @@ +package db + +import ( + "testing" + + "github.com/glebarez/sqlite" + "gorm.io/gorm" + "gorm.io/gorm/schema" + + "github.com/OpenListTeam/OpenList/v4/internal/model" +) + +func openTestDB(t *testing.T, prefix string) *gorm.DB { + t.Helper() + d, err := gorm.Open(sqlite.Open("file::memory:"), &gorm.Config{ + NamingStrategy: schema.NamingStrategy{TablePrefix: prefix}, + }) + if err != nil { + t.Fatal(err) + } + return d +} + +// 旧版表:只有 media_items 的基础结构,没有新的组合唯一索引。 +func createLegacyMediaTable(t *testing.T, d *gorm.DB, table string) { + t.Helper() + if err := d.Exec("CREATE TABLE " + table + " (id integer primary key autoincrement, deleted_at datetime, folder_path text, file_name text)").Error; err != nil { + t.Fatal(err) + } + if err := d.Exec("INSERT INTO " + table + " (folder_path, file_name) VALUES ('/a/b.mp3', 'b.mp3')").Error; err != nil { + t.Fatal(err) + } +} + +func countRows(t *testing.T, d *gorm.DB, table string) int64 { + t.Helper() + var n int64 + if err := d.Table(table).Count(&n).Error; err != nil { + t.Fatal(err) + } + return n +} + +func TestMediaItemsTableHonorsPrefix(t *testing.T) { + for _, prefix := range []string{"x_", "ol_", ""} { + db = openTestDB(t, prefix) + got, err := mediaItemsTable() + if err != nil { + t.Fatal(err) + } + if want := prefix + "media_items"; got != want { + t.Errorf("prefix %q: table = %q, want %q", prefix, got, want) + } + } +} + +func TestMigrateKeepsLegacyDataWithoutConfirmation(t *testing.T) { + t.Setenv(MediaMigrateWipeEnv, "") + db = openTestDB(t, "ol_") // 非默认前缀:旧实现硬编码 x_ 会直接跳过迁移 + createLegacyMediaTable(t, db, "ol_media_items") + + if err := migrateMediaItems(); err != nil { + t.Fatal(err) + } + if n := countRows(t, db, "ol_media_items"); n != 1 { + t.Fatalf("legacy rows must be preserved without confirmation, got %d", n) + } +} + +func TestMigrateWipesOnlyWhenConfirmed(t *testing.T) { + t.Setenv(MediaMigrateWipeEnv, "1") + db = openTestDB(t, "ol_") + createLegacyMediaTable(t, db, "ol_media_items") + + if err := migrateMediaItems(); err != nil { + t.Fatal(err) + } + if n := countRows(t, db, "ol_media_items"); n != 0 { + t.Fatalf("legacy rows should be wiped after confirmation, got %d", n) + } +} + +func TestAutoMigrateCreatesCompositeUniqueIndex(t *testing.T) { + db = openTestDB(t, "ol_") + if err := db.AutoMigrate(&model.MediaItem{}); err != nil { + t.Fatal(err) + } + if !db.Migrator().HasIndex("ol_media_items", "idx_media_folder_file_album") { + t.Fatal("composite unique index missing after AutoMigrate") + } + // 已有新索引时迁移不应触碰有效数据 + if err := db.Create(&model.MediaItem{FolderPath: "/m", FileName: "a.mp3"}).Error; err != nil { + t.Fatal(err) + } + if err := migrateMediaItems(); err != nil { + t.Fatal(err) + } + if n := countRows(t, db, "ol_media_items"); n != 1 { + t.Fatalf("valid rows must survive migration, got %d", n) + } +} + +func TestCreateOrUpdateRejectsOverlongIndexColumns(t *testing.T) { + db = openTestDB(t, "ol_") + if err := db.AutoMigrate(&model.MediaItem{}); err != nil { + t.Fatal(err) + } + long := make([]rune, model.MediaIndexColumnMax+1) + for i := range long { + long[i] = '文' + } + err := CreateOrUpdateMediaItem(&model.MediaItem{FolderPath: "/m", FileName: string(long)}) + if err == nil { + t.Fatal("overlong file_name must be rejected") + } + ok := make([]rune, model.MediaIndexColumnMax) + for i := range ok { + ok[i] = '文' + } + if err := CreateOrUpdateMediaItem(&model.MediaItem{FolderPath: "/m", FileName: string(ok)}); err != nil { + t.Fatalf("boundary length must be accepted: %v", err) + } +} diff --git a/internal/db/media.go b/internal/db/media.go new file mode 100644 index 00000000..9f7e92dc --- /dev/null +++ b/internal/db/media.go @@ -0,0 +1,626 @@ +package db + +import ( + "encoding/json" + "fmt" + "strings" + "time" + "unicode/utf8" + + "github.com/OpenListTeam/OpenList/v4/internal/model" + "gorm.io/gorm" +) + +// ==================== MediaConfig ==================== + +// GetMediaConfig 获取指定类型的媒体库配置,不存在则返回默认值 +func GetMediaConfig(mediaType model.MediaType) (*model.MediaConfig, error) { + var cfg model.MediaConfig + result := db.Where("media_type = ?", mediaType).First(&cfg) + if result.Error == gorm.ErrRecordNotFound { + return &model.MediaConfig{ + MediaType: mediaType, + Enabled: false, + }, nil + } + return &cfg, result.Error +} + +// GetAllMediaConfigs 获取所有媒体库配置 +func GetAllMediaConfigs() ([]model.MediaConfig, error) { + var cfgs []model.MediaConfig + err := db.Find(&cfgs).Error + return cfgs, err +} + +// SaveMediaConfig 保存媒体库配置(upsert) +func SaveMediaConfig(cfg *model.MediaConfig) error { + var existing model.MediaConfig + result := db.Where("media_type = ?", cfg.MediaType).First(&existing) + if result.Error == gorm.ErrRecordNotFound { + return db.Create(cfg).Error + } + cfg.ID = existing.ID + return db.Save(cfg).Error +} + +// ==================== MediaScanPath ==================== + +// ListMediaScanPaths 获取指定媒体类型的所有扫描路径 +func ListMediaScanPaths(mediaType model.MediaType) ([]model.MediaScanPath, error) { + var paths []model.MediaScanPath + tx := db.Model(&model.MediaScanPath{}) + if mediaType != "" { + tx = tx.Where("media_type = ?", mediaType) + } + err := tx.Order("id asc").Find(&paths).Error + return paths, err +} + +// GetMediaScanPath 按ID获取扫描路径 +func GetMediaScanPath(id uint) (*model.MediaScanPath, error) { + var p model.MediaScanPath + err := db.First(&p, id).Error + return &p, err +} + +// CreateMediaScanPath 创建扫描路径 +func CreateMediaScanPath(p *model.MediaScanPath) error { + return db.Create(p).Error +} + +// UpdateMediaScanPath 更新扫描路径 +func UpdateMediaScanPath(p *model.MediaScanPath) error { + return db.Save(p).Error +} + +// DeleteMediaScanPath 删除扫描路径(硬删除) +func DeleteMediaScanPath(id uint) error { + return db.Unscoped().Delete(&model.MediaScanPath{}, id).Error +} + +// ==================== MediaItem ==================== + +// escapeLike 转义 LIKE 通配符,避免路径中的 % _ 影响前缀匹配 +func escapeLike(s string) string { + s = strings.ReplaceAll(s, `\`, `\\`) + s = strings.ReplaceAll(s, "%", `\%`) + s = strings.ReplaceAll(s, "_", `\_`) + return s +} + +// MediaItemQuery 媒体条目查询参数 +type MediaItemQuery struct { + MediaType model.MediaType + ScanPathID uint // 按扫描路径ID筛选 + FolderPath string // 按文件夹路径筛选 + TypeTag string // 按类型标签筛选(电影、电视剧等) + ContentTag string // 按内容标签筛选(喜剧、惊悚等) + PathPrefix string // 按文件夹路径前缀筛选(用于用户 base_path 隔离) + Hidden *bool + Keyword string + OrderBy string // "name", "date", "size" + OrderDir string // "asc", "desc" + Page int + PageSize int +} + +// ListMediaItems 分页查询媒体条目 +func ListMediaItems(q MediaItemQuery) ([]model.MediaItem, int64, error) { + tx := db.Model(&model.MediaItem{}) + if q.MediaType != "" { + tx = tx.Where("media_type = ?", q.MediaType) + } + if q.ScanPathID > 0 { + tx = tx.Where("scan_path_id = ?", q.ScanPathID) + } + if q.FolderPath != "" { + tx = tx.Where("folder_path = ?", q.FolderPath) + } + if q.PathPrefix != "" { + prefix := strings.TrimSuffix(q.PathPrefix, "/") + "/" + tx = tx.Where(`folder_path LIKE ? ESCAPE '\'`, escapeLike(prefix)+"%") + } + if q.Hidden != nil { + tx = tx.Where("hidden = ?", *q.Hidden) + } + if q.Keyword != "" { + like := "%" + q.Keyword + "%" + tx = tx.Where("file_name LIKE ? OR scraped_name LIKE ?", like, like) + } + // 按类型标签筛选(通过关联扫描路径的type_tag) + if q.TypeTag != "" { + tx = tx.Joins("JOIN media_scan_paths ON media_scan_paths.id = media_items.scan_path_id"). + Where("media_scan_paths.type_tag = ?", q.TypeTag) + } + // 按内容标签筛选(通过关联扫描路径的content_tags) + if q.ContentTag != "" { + tx = tx.Joins("JOIN media_scan_paths ON media_scan_paths.id = media_items.scan_path_id"). + Where("media_scan_paths.content_tags LIKE ?", "%"+q.ContentTag+"%") + } + + var total int64 + if err := tx.Count(&total).Error; err != nil { + return nil, 0, err + } + + // 排序 + orderCol := "created_at" + switch q.OrderBy { + case "name": + orderCol = "COALESCE(NULLIF(scraped_name,''), file_name)" + case "date": + orderCol = "release_date" + case "size": + orderCol = "file_size" + } + dir := "asc" + if q.OrderDir == "desc" { + dir = "desc" + } + tx = tx.Order(orderCol + " " + dir) + + // 分页 + if q.PageSize <= 0 { + q.PageSize = 20 + } + if q.Page <= 0 { + q.Page = 1 + } + tx = tx.Offset((q.Page - 1) * q.PageSize).Limit(q.PageSize) + + var items []model.MediaItem + err := tx.Find(&items).Error + return items, total, err +} + +// GetMediaItemByID 按ID获取媒体条目 +func GetMediaItemByID(id uint) (*model.MediaItem, error) { + var item model.MediaItem + err := db.First(&item, id).Error + return &item, err +} + +// GetMediaItemByFolderPath 按文件夹路径获取媒体条目(用于合并模式) +func GetMediaItemByFolderPath(folderPath string) (*model.MediaItem, error) { + var item model.MediaItem + result := db.Where("folder_path = ?", folderPath).First(&item) + return &item, result.Error +} + +// CreateOrUpdateMediaItem 创建或更新媒体条目(按 folder_path + file_name + album_name 组合唯一) +// 更新时保留已有的刮削数据,避免重新扫描时把已刮削的字段清空 +// +// 注意:使用 Unscoped() 查询是为了把软删除残留记录也包含进来, +// 否则唯一索引 idx_media_folder_file_album 会与软删除行冲突, +// 导致新建条目时报 UNIQUE constraint failed。 +func CreateOrUpdateMediaItem(item *model.MediaItem) error { + // 复合唯一索引三列在 MySQL 下限定为 191 字符(见 model.MediaItem)。 + // 超长值在严格模式下会写入失败、非严格模式下会被静默截断并造成不同文件撞同一条目, + // 因此显式拒绝,由调用方记录告警并跳过。 + if utf8.RuneCountInString(item.FolderPath) > model.MediaIndexColumnMax || + utf8.RuneCountInString(item.FileName) > model.MediaIndexColumnMax || + utf8.RuneCountInString(item.AlbumName) > model.MediaIndexColumnMax { + return fmt.Errorf("media item path/name exceeds %d characters, skipped", model.MediaIndexColumnMax) + } + var existing model.MediaItem + result := db.Unscoped().Where("folder_path = ? AND file_name = ? AND album_name = ?", item.FolderPath, item.FileName, item.AlbumName).First(&existing) + if result.Error == gorm.ErrRecordNotFound { + return db.Create(item).Error + } + if result.Error != nil { + return result.Error + } + item.ID = existing.ID + item.CreatedAt = existing.CreatedAt + // 复用已有记录时清除软删除标记,确保该条目“恢复”为正常记录 + item.DeletedAt = gorm.DeletedAt{} + // 如果已有刮削数据,保留刮削字段,防止重新扫描时覆盖刮削结果 + if existing.ScrapedAt != nil { + item.ScrapedAt = existing.ScrapedAt + item.ScrapedName = existing.ScrapedName + item.Cover = existing.Cover + item.AlbumName = existing.AlbumName + item.AlbumArtist = existing.AlbumArtist + item.TrackNumber = existing.TrackNumber + item.Duration = existing.Duration + item.Genre = existing.Genre + item.ReleaseDate = existing.ReleaseDate + item.Rating = existing.Rating + item.Plot = existing.Plot + item.Authors = existing.Authors + item.Description = existing.Description + item.Publisher = existing.Publisher + item.ISBN = existing.ISBN + item.ExternalID = existing.ExternalID + } + // 扫描不覆盖用户手动隐藏的状态 + item.Hidden = existing.Hidden + return db.Unscoped().Save(item).Error +} + +// UpdateMediaItem 更新媒体条目 +// 使用 Unscoped 确保即使存在软删除标记也能正确更新(避免命中唯一索引但行不可见的诡异情况)。 +func UpdateMediaItem(item *model.MediaItem) error { + return db.Unscoped().Save(item).Error +} + +// DeleteMediaItem 硬删除媒体条目(真正从数据库删除),并清理该条目的批注 +func DeleteMediaItem(id uint) error { + return db.Transaction(func(tx *gorm.DB) error { + if err := tx.Unscoped().Delete(&model.MediaItem{}, id).Error; err != nil { + return err + } + return tx.Where("item_id = ?", id).Delete(&model.MediaAnnotation{}).Error + }) +} + +// ClearMediaItems 硬删除指定类型的所有媒体条目(真正从数据库删除),并清理对应批注 +func ClearMediaItems(mediaType model.MediaType) error { + return db.Transaction(func(tx *gorm.DB) error { + ids := []uint{} + if err := tx.Unscoped().Model(&model.MediaItem{}). + Where("media_type = ?", mediaType).Pluck("id", &ids).Error; err != nil { + return err + } + if err := tx.Unscoped().Where("media_type = ?", mediaType). + Delete(&model.MediaItem{}).Error; err != nil { + return err + } + if len(ids) == 0 { + return nil + } + return tx.Where("item_id IN ?", ids).Delete(&model.MediaAnnotation{}).Error + }) +} + +// ClearMediaItemsByScanPath 硬删除指定扫描路径的所有媒体条目,并清理对应批注 +func ClearMediaItemsByScanPath(scanPathID uint) error { + return db.Transaction(func(tx *gorm.DB) error { + ids := []uint{} + if err := tx.Unscoped().Model(&model.MediaItem{}). + Where("scan_path_id = ?", scanPathID).Pluck("id", &ids).Error; err != nil { + return err + } + if err := tx.Unscoped().Where("scan_path_id = ?", scanPathID). + Delete(&model.MediaItem{}).Error; err != nil { + return err + } + if len(ids) == 0 { + return nil + } + return tx.Where("item_id IN ?", ids).Delete(&model.MediaAnnotation{}).Error + }) +} + +// ClearMediaScrapedData 清空指定类型的所有刮削数据(保留扫描出的文件记录本身) +// 仅清空刮削结果相关字段,不删除条目,便于重新刮削。 +// mediaType 为空时表示对所有类型生效。 +func ClearMediaScrapedData(mediaType model.MediaType) (int64, error) { + tx := db.Unscoped().Model(&model.MediaItem{}) + if mediaType != "" { + tx = tx.Where("media_type = ?", mediaType) + } + updates := map[string]interface{}{ + "scraped_name": "", + "description": "", + "cover": "", + "release_date": "", + "rating": 0, + "genre": "", + "authors": "", + "plot": "", + "reviews": "", + "external_id": "", + "album_artist": "", + "publisher": "", + "isbn": "", + "lyrics": "", + "scraped_at": nil, + } + result := tx.Updates(updates) + return result.RowsAffected, result.Error +} + +// ListAllValidMediaItemsForCheck 列出指定类型下所有未删除的媒体条目(仅取必要字段,用于失效检查) +// mediaType 为空时返回所有类型条目。 +func ListAllValidMediaItemsForCheck(mediaType model.MediaType) ([]model.MediaItem, error) { + var items []model.MediaItem + tx := db.Model(&model.MediaItem{}). + Select("id, media_type, scan_path_id, file_name, folder_path, is_folder") + if mediaType != "" { + tx = tx.Where("media_type = ?", mediaType) + } + err := tx.Find(&items).Error + return items, err +} + +// DeleteMediaItemsByIDs 按ID列表硬删除媒体条目,并清理对应批注 +func DeleteMediaItemsByIDs(ids []uint) error { + if len(ids) == 0 { + return nil + } + return db.Transaction(func(tx *gorm.DB) error { + if err := tx.Unscoped().Where("id IN ?", ids).Delete(&model.MediaItem{}).Error; err != nil { + return err + } + return tx.Where("item_id IN ?", ids).Delete(&model.MediaAnnotation{}).Error + }) +} + +// ListAlbums 列出所有专辑(音乐专用) +func ListAlbums(q MediaItemQuery) ([]AlbumInfo, int64, error) { + type albumRow struct { + AlbumName string + AlbumArtist string + Cover string + ReleaseDate string + TrackCount int + ScanPathID uint + } + + // 构建基础查询 + baseQuery := db.Model(&model.MediaItem{}). + Where("media_type = ?", model.MediaTypeMusic) + if q.Hidden != nil { + baseQuery = baseQuery.Where("hidden = ?", *q.Hidden) + } + if q.ScanPathID > 0 { + baseQuery = baseQuery.Where("scan_path_id = ?", q.ScanPathID) + } + if q.Keyword != "" { + like := "%" + q.Keyword + "%" + baseQuery = baseQuery.Where("album_name LIKE ? OR album_artist LIKE ? OR scraped_name LIKE ?", like, like, like) + } + + // 统计分组数(用子查询) + var total int64 + if err := db.Table("(?) as sub", baseQuery. + Select("album_name, album_artist"). + Group("album_name, album_artist")). + Count(&total).Error; err != nil { + return nil, 0, err + } + + if q.PageSize <= 0 { + q.PageSize = 20 + } + if q.Page <= 0 { + q.Page = 1 + } + + tx := baseQuery. + Select("album_name, album_artist, MAX(cover) as cover, MAX(release_date) as release_date, COUNT(*) as track_count, MAX(scan_path_id) as scan_path_id"). + Group("album_name, album_artist"). + Offset((q.Page - 1) * q.PageSize).Limit(q.PageSize) + + var rows []albumRow + if err := tx.Scan(&rows).Error; err != nil { + return nil, 0, err + } + + albums := make([]AlbumInfo, len(rows)) + for i, r := range rows { + albums[i] = AlbumInfo{ + AlbumName: r.AlbumName, + AlbumArtist: r.AlbumArtist, + Cover: r.Cover, + ReleaseDate: r.ReleaseDate, + TrackCount: r.TrackCount, + ScanPathID: r.ScanPathID, + } + } + return albums, total, nil +} + +// AlbumInfo 专辑信息 +type AlbumInfo struct { + AlbumName string `json:"album_name"` + AlbumArtist string `json:"album_artist"` + Cover string `json:"cover"` + ReleaseDate string `json:"release_date"` + TrackCount int `json:"track_count"` + ScanPathID uint `json:"scan_path_id"` +} + +// GetAlbumTracks 获取专辑曲目列表 +// 支持两种模式: +// 1. 普通模式(is_folder=false):直接返回独立文件记录 +// 2. 合并文件夹模式(is_folder=true):把 episodes 展开成虚拟 MediaItem 列表 +// 展开后每条记录的 folder_path = 原folder_path/file_name(文件夹实际路径),file_name = episode.FileName +func GetAlbumTracks(albumName, albumArtist string) ([]model.MediaItem, error) { + var items []model.MediaItem + tx := db.Where("media_type = ?", model.MediaTypeMusic) + if albumName != "" { + tx = tx.Where("album_name = ?", albumName) + } else { + tx = tx.Where("(album_name = '' OR album_name IS NULL)") + } + if albumArtist != "" { + tx = tx.Where("album_artist = ?", albumArtist) + } + err := tx.Order("track_number asc").Find(&items).Error + if err != nil { + return nil, err + } + + // 展开合并文件夹条目的 episodes + var result []model.MediaItem + for _, item := range items { + if !item.IsFolder || item.Episodes == "" { + result = append(result, item) + continue + } + // 解析 episodes + type EpisodeInfo struct { + FileName string `json:"file_name"` + Index int `json:"index"` + Title string `json:"title"` + } + var eps []EpisodeInfo + if err := json.Unmarshal([]byte(item.Episodes), &eps); err != nil || len(eps) == 0 { + // 解析失败则跳过该条目(不返回文件夹本身,避免播放路径错误) + continue + } + // 文件夹实际路径 = folder_path + "/" + file_name + actualDir := strings.TrimRight(item.FolderPath, "/") + "/" + item.FileName + for _, ep := range eps { + track := item // 复制基础信息(封面、专辑名、艺术家等) + track.ID = 0 + track.IsFolder = false + track.FolderPath = actualDir + track.FileName = ep.FileName + track.TrackNumber = ep.Index + track.ScrapedName = ep.Title + track.Episodes = "" + result = append(result, track) + } + } + return result, nil +} + +// ListFolderPaths 列出指定媒体类型下的所有文件夹路径(目录浏览模式) +func ListFolderPaths(mediaType model.MediaType) ([]string, error) { + var paths []string + err := db.Model(&model.MediaItem{}). + Where("media_type = ?", mediaType). + Distinct("folder_path"). + Pluck("folder_path", &paths).Error + return paths, err +} + +// GetUnscrappedItems 获取未刮削或刮削不完整的媒体条目 +// 只要 scraped_at 为空,或 cover/scraped_name/plot 任一为空,就需要重新刮削 +// limit <= 0 表示不限制数量,返回所有未刮削条目 +func GetUnscrappedItems(mediaType model.MediaType, limit int) ([]model.MediaItem, error) { + var items []model.MediaItem + tx := db.Where( + "media_type = ? AND (scraped_at IS NULL OR cover = '' OR cover IS NULL OR scraped_name = '' OR scraped_name IS NULL OR plot = '' OR plot IS NULL)", + mediaType, + ) + if limit > 0 { + tx = tx.Limit(limit) + } + err := tx.Find(&items).Error + return items, err +} + +// ==================== 导入/导出 ==================== + +// ListAllMediaItems 列出指定媒体类型下的所有条目(不分页,用于导出) +// mediaType 为空时返回全部类型的所有条目 +func ListAllMediaItems(mediaType model.MediaType) ([]model.MediaItem, error) { + var items []model.MediaItem + tx := db.Model(&model.MediaItem{}) + if mediaType != "" { + tx = tx.Where("media_type = ?", mediaType) + } + err := tx.Order("id asc").Find(&items).Error + return items, err +} + +// ListMediaItemsByScanPath 列出指定扫描路径下的所有条目(不分页,用于导出) +func ListMediaItemsByScanPath(scanPathID uint) ([]model.MediaItem, error) { + var items []model.MediaItem + err := db.Where("scan_path_id = ?", scanPathID).Order("id asc").Find(&items).Error + return items, err +} + +// ListAllMediaScanPaths 列出所有扫描路径(不区分类型,用于全量导出) +func ListAllMediaScanPaths() ([]model.MediaScanPath, error) { + var paths []model.MediaScanPath + err := db.Order("id asc").Find(&paths).Error + return paths, err +} + +// ImportMediaItems 批量导入媒体条目 +// 策略:按 (folder_path, file_name, album_name) 唯一键 upsert: +// - 已存在则覆盖刮削字段(导入是用户主动行为,覆盖优先于保留) +// - 不存在则新建 +// +// 注意:导入数据中的 ID/CreatedAt/UpdatedAt 会被忽略,由数据库重新分配, +// 避免和现有记录的主键冲突。 +func ImportMediaItems(items []model.MediaItem, scanPathIDOverride *uint) (created, updated int, err error) { + for i := range items { + it := items[i] + // 清空主键和时间戳,让数据库重新生成 + it.ID = 0 + it.CreatedAt = time.Time{} + it.UpdatedAt = time.Time{} + it.DeletedAt = gorm.DeletedAt{} + // 如果指定了覆盖的 scan_path_id(按扫描路径导入场景),则强制覆盖 + if scanPathIDOverride != nil { + it.ScanPathID = *scanPathIDOverride + } + + var existing model.MediaItem + result := db.Unscoped().Where( + "folder_path = ? AND file_name = ? AND album_name = ?", + it.FolderPath, it.FileName, it.AlbumName, + ).First(&existing) + if result.Error == gorm.ErrRecordNotFound { + if e := db.Create(&it).Error; e != nil { + return created, updated, e + } + created++ + continue + } + if result.Error != nil { + return created, updated, result.Error + } + // 覆盖更新 + it.ID = existing.ID + it.CreatedAt = existing.CreatedAt + it.DeletedAt = gorm.DeletedAt{} + if e := db.Unscoped().Save(&it).Error; e != nil { + return created, updated, e + } + updated++ + } + return created, updated, nil +} + +// ImportMediaScanPaths 批量导入扫描路径 +// 策略:按 (media_type, path) 唯一对去重: +// - 已存在则更新名称/标签等可编辑字段 +// - 不存在则新建并返回新ID +// +// 返回值 idMap:导入数据中的原始ID -> 数据库实际ID 的映射, +// 供随后的 MediaItem 导入用于把 scan_path_id 重新指向新ID。 +func ImportMediaScanPaths(paths []model.MediaScanPath) (idMap map[uint]uint, created, updated int, err error) { + idMap = make(map[uint]uint, len(paths)) + for i := range paths { + p := paths[i] + originalID := p.ID + p.ID = 0 + p.CreatedAt = time.Time{} + p.UpdatedAt = time.Time{} + p.DeletedAt = gorm.DeletedAt{} + + var existing model.MediaScanPath + result := db.Where("media_type = ? AND path = ?", p.MediaType, p.Path).First(&existing) + if result.Error == gorm.ErrRecordNotFound { + if e := db.Create(&p).Error; e != nil { + return idMap, created, updated, e + } + idMap[originalID] = p.ID + created++ + continue + } + if result.Error != nil { + return idMap, created, updated, result.Error + } + // 已存在:更新可编辑字段 + existing.Name = p.Name + existing.PathMerge = p.PathMerge + existing.TypeTag = p.TypeTag + existing.ContentTags = p.ContentTags + existing.EnableScrape = p.EnableScrape + if e := db.Save(&existing).Error; e != nil { + return idMap, created, updated, e + } + idMap[originalID] = existing.ID + updated++ + } + return idMap, created, updated, nil +} diff --git a/internal/db/media_extra.go b/internal/db/media_extra.go new file mode 100644 index 00000000..b9e43497 --- /dev/null +++ b/internal/db/media_extra.go @@ -0,0 +1,223 @@ +package db + +import ( + "time" + + "github.com/OpenListTeam/OpenList/v4/internal/model" + "gorm.io/gorm" + "gorm.io/gorm/clause" +) + +// ==================== MediaAppKey ==================== + +func CreateMediaAppKey(k *model.MediaAppKey) error { + return db.Create(k).Error +} + +func ListMediaAppKeys(userID uint) ([]model.MediaAppKey, error) { + var keys []model.MediaAppKey + err := db.Where("user_id = ?", userID).Order("id asc").Find(&keys).Error + return keys, err +} + +func GetMediaAppKey(id uint) (*model.MediaAppKey, error) { + var k model.MediaAppKey + err := db.First(&k, id).Error + return &k, err +} + +func DeleteMediaAppKey(userID, id uint) error { + return db.Where("id = ? AND user_id = ?", id, userID).Delete(&model.MediaAppKey{}).Error +} + +func TouchMediaAppKey(id uint) { + now := time.Now() + _ = db.Model(&model.MediaAppKey{}).Where("id = ?", id).Update("last_used_at", now).Error +} + +// ListMediaAppKeysByPrefix 认证时按明文前缀预筛选候选密钥 +func ListMediaAppKeysByPrefix(prefix string) ([]model.MediaAppKey, error) { + var keys []model.MediaAppKey + err := db.Where("prefix = ?", prefix).Find(&keys).Error + return keys, err +} + +// ==================== MediaUserData ==================== + +func GetMediaUserData(userID, itemID uint, episodeIndex int) (*model.MediaUserData, error) { + var d model.MediaUserData + err := db.Where("user_id = ? AND item_id = ? AND episode_index = ?", userID, itemID, episodeIndex).First(&d).Error + if err != nil { + return nil, err + } + return &d, nil +} + +func SaveMediaUserData(d *model.MediaUserData) error { + var existing model.MediaUserData + err := db.Where("user_id = ? AND item_id = ? AND episode_index = ?", d.UserID, d.ItemID, d.EpisodeIndex).First(&existing).Error + if err == gorm.ErrRecordNotFound { + return db.Create(d).Error + } + if err != nil { + return err + } + d.ID = existing.ID + d.CreatedAt = existing.CreatedAt + return db.Save(d).Error +} + +// ListMediaUserDataByItems 批量取用户对一组条目的播放状态 +func ListMediaUserDataByItems(userID uint, itemIDs []uint) ([]model.MediaUserData, error) { + var list []model.MediaUserData + if len(itemIDs) == 0 { + return list, nil + } + err := db.Where("user_id = ? AND item_id IN ?", userID, itemIDs).Find(&list).Error + return list, err +} + +// ListMediaFavorites 用户收藏的条目 +func ListMediaFavorites(userID uint) ([]model.MediaUserData, error) { + var list []model.MediaUserData + err := db.Where("user_id = ? AND is_favorite = ?", userID, true).Order("updated_at desc").Find(&list).Error + return list, err +} + +// ListMediaResume 继续观看:未看完且有进度 +func ListMediaResume(userID uint, limit int) ([]model.MediaUserData, error) { + var list []model.MediaUserData + if limit <= 0 { + limit = 50 + } + err := db.Where("user_id = ? AND played = ? AND position_ticks > 0", userID, false). + Order("last_played_at desc").Limit(limit).Find(&list).Error + return list, err +} + +// ClearMediaUserDataByUser 删除用户全部播放状态 +func ClearMediaUserDataByUser(userID uint) error { + return db.Where("user_id = ?", userID).Delete(&model.MediaUserData{}).Error +} + +// ==================== MediaPlaylist ==================== + +func CreateMediaPlaylist(p *model.MediaPlaylist) error { + return db.Create(p).Error +} + +func GetMediaPlaylist(id uint) (*model.MediaPlaylist, error) { + var p model.MediaPlaylist + err := db.First(&p, id).Error + return &p, err +} + +func ListMediaPlaylists(userID uint) ([]model.MediaPlaylist, error) { + var list []model.MediaPlaylist + err := db.Where("user_id = ?", userID).Order("id asc").Find(&list).Error + return list, err +} + +func UpdateMediaPlaylist(p *model.MediaPlaylist) error { + return db.Save(p).Error +} + +func DeleteMediaPlaylist(id uint) error { + if err := db.Where("playlist_id = ?", id).Delete(&model.MediaPlaylistItem{}).Error; err != nil { + return err + } + return db.Delete(&model.MediaPlaylist{}, id).Error +} + +// ReplaceMediaPlaylistItems 覆盖式写入播放列表条目 +func ReplaceMediaPlaylistItems(playlistID uint, items []model.MediaPlaylistItem) error { + if err := db.Where("playlist_id = ?", playlistID).Delete(&model.MediaPlaylistItem{}).Error; err != nil { + return err + } + if len(items) == 0 { + return nil + } + for i := range items { + items[i].ID = 0 + items[i].PlaylistID = playlistID + items[i].Order = i + } + return db.Create(&items).Error +} + +func ListMediaPlaylistItems(playlistID uint) ([]model.MediaPlaylistItem, error) { + var items []model.MediaPlaylistItem + err := db.Where("playlist_id = ?", playlistID).Order("sort_order asc").Find(&items).Error + return items, err +} + +// CountMediaPlaylistItems 统计条目数 +func CountMediaPlaylistItems(playlistID uint) (int64, error) { + var n int64 + err := db.Model(&model.MediaPlaylistItem{}).Where("playlist_id = ?", playlistID).Count(&n).Error + return n, err +} + +// ==================== EmbyDevice ==================== + +func CreateEmbyDevice(d *model.EmbyDevice) error { + return db.Create(d).Error +} + +func GetEmbyDeviceByTokenHash(hash string) (*model.EmbyDevice, error) { + var d model.EmbyDevice + err := db.Where("token_hash = ?", hash).First(&d).Error + if err != nil { + return nil, err + } + return &d, nil +} + +func DeleteEmbyDevice(userID uint, deviceID string) error { + tx := db.Where("user_id = ?", userID) + if deviceID != "" { + tx = tx.Where("device_id = ?", deviceID) + } + return tx.Delete(&model.EmbyDevice{}).Error +} + +func TouchEmbyDevice(id uint) { + _ = db.Model(&model.EmbyDevice{}).Where("id = ?", id).Update("last_seen_at", time.Now()).Error +} + +// ==================== MediaAnnotation ==================== + +// ListMediaAnnotations 列出某用户对某个条目的全部批注 +func ListMediaAnnotations(userID, itemID uint) ([]model.MediaAnnotation, error) { + list := []model.MediaAnnotation{} + err := db.Where("user_id = ? AND item_id = ?", userID, itemID). + Order("id asc").Find(&list).Error + return list, err +} + +// SyncMediaAnnotations 增量保存批注:upserts 覆盖同一 remote_id 的记录,deletes 按 remote_id 删除。 +// 放在一个事务里,避免多设备同时保存时出现半新半旧的状态。 +func SyncMediaAnnotations(userID, itemID uint, upserts []model.MediaAnnotation, deletes []string) error { + return db.Transaction(func(tx *gorm.DB) error { + for i := range upserts { + row := upserts[i] + row.ID = 0 + row.UserID, row.ItemID = userID, itemID + if err := tx.Clauses(clause.OnConflict{ + Columns: []clause.Column{ + {Name: "user_id"}, {Name: "item_id"}, {Name: "remote_id"}, + }, + DoUpdates: clause.AssignmentColumns([]string{"anchor", "updated_at"}), + }).Create(&row).Error; err != nil { + return err + } + } + if len(deletes) > 0 { + if err := tx.Where("user_id = ? AND item_id = ? AND remote_id IN ?", userID, itemID, deletes). + Delete(&model.MediaAnnotation{}).Error; err != nil { + return err + } + } + return nil + }) +} diff --git a/internal/db/sharing.go b/internal/db/sharing.go index 9c356440..7887f71f 100644 --- a/internal/db/sharing.go +++ b/internal/db/sharing.go @@ -4,6 +4,7 @@ import ( "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/pkg/utils/random" "github.com/pkg/errors" + "gorm.io/gorm" ) func GetSharingById(id string) (*model.SharingDB, error) { @@ -64,6 +65,13 @@ func UpdateSharing(s *model.SharingDB) error { return errors.WithStack(db.Save(s).Error) } +func TakeSharingSlot(id string) (bool, error) { + res := db.Model(&model.SharingDB{}). + Where("id = ? AND (max_accessed <= 0 OR accessed < max_accessed)", id). + UpdateColumn("accessed", gorm.Expr("accessed + 1")) + return res.RowsAffected > 0, errors.WithStack(res.Error) +} + func UpdateSharingId(oldId, newId string) error { // Check if new ID already exists if err := db.Where("id = ?", newId).First(&model.SharingDB{}).Error; err == nil { diff --git a/internal/db/webdav_property_migrate.go b/internal/db/webdav_property_migrate.go new file mode 100644 index 00000000..345a5c67 --- /dev/null +++ b/internal/db/webdav_property_migrate.go @@ -0,0 +1,71 @@ +package db + +import ( + "database/sql" + + "github.com/OpenListTeam/OpenList/v4/internal/model" + "gorm.io/gorm" +) + +// migrateWebDAVProperties prepares existing dead properties for the exact +// hash-based unique index used by WebDAVProperty. +func migrateWebDAVProperties() error { + table, err := webdavPropertiesTable() + if err != nil { + return err + } + m := db.Migrator() + if !m.HasTable(table) { + return nil + } + + if !m.HasColumn(table, "key_hash") { + if err := m.AddColumn(&model.WebDAVProperty{}, "KeyHash"); err != nil { + return err + } + } + + var rows []struct { + ID uint + Path string + Namespace string + Name string + KeyHash sql.NullString + } + if err := db.Model(&model.WebDAVProperty{}).Select("id, path, namespace, name, key_hash").Find(&rows).Error; err != nil { + return err + } + for _, row := range rows { + hash := model.HashWebDAVPropertyKey(row.Path, row.Namespace, row.Name) + if row.KeyHash.Valid && row.KeyHash.String == hash { + continue + } + if err := db.Model(&model.WebDAVProperty{}).Where("id = ?", row.ID).UpdateColumn("key_hash", hash).Error; err != nil { + return err + } + } + + if m.HasIndex(table, "idx_webdav_property") { + indexes, err := m.GetIndexes(&model.WebDAVProperty{}) + if err != nil { + return err + } + for _, index := range indexes { + if index.Name() == "idx_webdav_property" && (len(index.Columns()) != 1 || index.Columns()[0] != "key_hash") { + if err := m.DropIndex(table, index.Name()); err != nil { + return err + } + break + } + } + } + return nil +} + +func webdavPropertiesTable() (string, error) { + stmt := &gorm.Statement{DB: db} + if err := stmt.Parse(&model.WebDAVProperty{}); err != nil { + return "", err + } + return stmt.Schema.Table, nil +} diff --git a/internal/db/webdav_property_migrate_test.go b/internal/db/webdav_property_migrate_test.go new file mode 100644 index 00000000..7b37a2c4 --- /dev/null +++ b/internal/db/webdav_property_migrate_test.go @@ -0,0 +1,135 @@ +package db + +import ( + "database/sql" + "fmt" + "strings" + "sync" + "testing" + + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/glebarez/sqlite" + "gorm.io/driver/mysql" + "gorm.io/gorm" + "gorm.io/gorm/clause" + "gorm.io/gorm/migrator" + "gorm.io/gorm/schema" +) + +// MySQL rejects indexes on TEXT columns without a key prefix length +// (Error 1170), see #3258. The unique index must therefore use the fixed-size +// hash column rather than a prefix of the full WebDAV property key. +func TestWebDAVPropertyMySQLIndexUsesKeyHash(t *testing.T) { + conn, err := sql.Open("mysql", "user:pass@tcp(127.0.0.1:1)/db") + if err != nil { + t.Fatal(err) + } + defer conn.Close() + d, err := gorm.Open(mysql.New(mysql.Config{Conn: conn, SkipInitializeWithVersion: true}), &gorm.Config{ + DisableAutomaticPing: true, + }) + if err != nil { + t.Fatal(err) + } + + s, err := schema.Parse(&model.WebDAVProperty{}, &sync.Map{}, d.NamingStrategy) + if err != nil { + t.Fatal(err) + } + for _, name := range []string{"Path", "Namespace", "Name"} { + if got := strings.ToLower(d.Dialector.DataTypeOf(s.LookUpField(name))); got != "text" { + t.Fatalf("%s column type = %q, want text", name, got) + } + } + if got := strings.ToLower(d.Dialector.DataTypeOf(s.LookUpField("KeyHash"))); got != "varchar(64)" { + t.Fatalf("KeyHash column type = %q, want varchar(64)", got) + } + + idx := s.LookIndex("idx_webdav_property") + if idx == nil || idx.Class != "UNIQUE" { + t.Fatalf("unique index idx_webdav_property not found: %+v", idx) + } + stmt := &gorm.Statement{DB: d, Schema: s} + opts := d.Migrator().(migrator.BuildIndexOptionsInterface).BuildIndexOptions(idx.Fields, stmt) + if len(opts) != 1 || opts[0].(clause.Expr).SQL != "`key_hash`" { + t.Fatalf("index parts = %v, want `key_hash`", opts) + } +} + +func TestWebDAVPropertyLongPathSQLite(t *testing.T) { + d, err := gorm.Open(sqlite.Open("file::memory:"), &gorm.Config{}) + if err != nil { + t.Fatal(err) + } + if err := d.AutoMigrate(&model.WebDAVProperty{}); err != nil { + t.Fatal(err) + } + if !d.Migrator().HasIndex(&model.WebDAVProperty{}, "idx_webdav_property") { + t.Fatal("idx_webdav_property not created") + } + // Re-running AutoMigrate on an existing table must be a no-op. + if err := d.AutoMigrate(&model.WebDAVProperty{}); err != nil { + t.Fatal(err) + } + + long := "/" + strings.Repeat("目录/", 400) + "file.txt" + row := model.WebDAVProperty{Path: long, Namespace: "DAV:", Name: "displayname", InnerXML: []byte("x")} + if err := d.Create(&row).Error; err != nil { + t.Fatalf("insert long path: %v", err) + } + var got model.WebDAVProperty + if err := d.Where("path = ?", long).First(&got).Error; err != nil { + t.Fatal(err) + } + if got.Path != long { + t.Fatalf("path truncated: len %d, want %d", len(got.Path), len(long)) + } + if got.KeyHash != model.HashWebDAVPropertyKey(long, "DAV:", "displayname") { + t.Fatalf("key hash = %q, want hash of complete property key", got.KeyHash) + } + dup := model.WebDAVProperty{Path: long, Namespace: "DAV:", Name: "displayname"} + if err := d.Create(&dup).Error; err == nil { + t.Fatal("duplicate (path, namespace, name) was accepted") + } + other := model.WebDAVProperty{Path: long + "2", Namespace: "DAV:", Name: "displayname"} + if err := d.Create(&other).Error; err != nil { + t.Fatal(fmt.Errorf("distinct path rejected: %w", err)) + } +} + +func TestMigrateWebDAVPropertiesBackfillsKeyHash(t *testing.T) { + d := openTestDB(t, "ol_") + db = d + if err := d.Exec(`CREATE TABLE ol_web_dav_properties ( + id integer primary key autoincrement, + path text, + namespace text, + name text, + lang text, + inner_xml blob, + UNIQUE (path, namespace, name) + )`).Error; err != nil { + t.Fatal(err) + } + path := "/" + strings.Repeat("目录/", 400) + "file.txt" + if err := d.Exec("INSERT INTO ol_web_dav_properties (path, namespace, name) VALUES (?, ?, ?)", path, "DAV:", "displayname").Error; err != nil { + t.Fatal(err) + } + + if err := migrateWebDAVProperties(); err != nil { + t.Fatal(err) + } + if err := d.AutoMigrate(&model.WebDAVProperty{}); err != nil { + t.Fatal(err) + } + var got model.WebDAVProperty + if err := d.Where("path = ?", path).First(&got).Error; err != nil { + t.Fatal(err) + } + if got.KeyHash != model.HashWebDAVPropertyKey(path, "DAV:", "displayname") { + t.Fatalf("migrated key hash = %q, want hash of complete property key", got.KeyHash) + } + if !d.Migrator().HasIndex("ol_web_dav_properties", "idx_webdav_property") { + t.Fatal("idx_webdav_property not recreated") + } +} diff --git a/internal/driver/driver.go b/internal/driver/driver.go index 373bb565..bf58d256 100644 --- a/internal/driver/driver.go +++ b/internal/driver/driver.go @@ -4,6 +4,7 @@ import ( "context" "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/pkg/utils" ) type Driver interface { @@ -218,3 +219,54 @@ type DirectUploader interface { // return errs.NotImplement if the driver does not support the given direct upload tool GetDirectUploadInfo(ctx context.Context, tool string, dstDir model.Obj, fileName string, fileSize int64) (any, error) } + +// SeedRapidUploadRequest carries everything a driver needs to attempt a +// hash-driven rapid upload (秒传/CAS) without transferring the full content. +// +// It is populated from a parsed transfer seed, so that any cloud drive +// supporting hash-based rapid upload can be used as a seed save target. +type SeedRapidUploadRequest struct { + // Name is the target file name. + Name string + // Size is the total file size in bytes. + Size int64 + // Whole holds whole-file hashes indexed by algorithm (e.g. utils.MD5, + // utils.SHA1). It must never be nil; individual entries may be empty. + Whole *utils.HashInfo + // SliceSize is the per-slice/piece size in bytes (0 when unknown). + SliceSize int64 + // SliceMD5s is the ordered per-slice MD5 list (used by 189pc-style CAS). + SliceMD5s []string + // SliceSHA1s is the ordered per-slice SHA1 list (used by SHA1-piece drives). + SliceSHA1s []string + // Open lazily yields the file content as a model.FileStreamer. Drivers whose + // rapid-upload protocol needs partial or full content (e.g. a leading + // pre-hash or a proof-code) may call it; hash-only drivers may ignore it. + // Open may be nil when no content source is available, in which case + // drivers that strictly require content must fail gracefully. + Open func() (model.FileStreamer, error) +} + +// SeedRapidUploader is an optional capability interface implemented by drivers +// that can perform a "rapid upload" (秒传/CAS) driven by precomputed hashes +// instead of a full content transfer. +// +// It generalizes the previous hard-coded 189pc-specific CAS path so that any +// cloud drive supporting hash-based rapid upload (e.g. 189pc via MD5+slice MD5, +// 115/aliyundrive_open via SHA1) can be used as a transfer-seed save target. +type SeedRapidUploader interface { + // RapidUploadByHashes attempts a rapid upload of a file into dstDir. + // + // Implementations should return errs.NotImplement (or a descriptive error) + // when the required hash is missing, the file does not exist remotely, or + // the rapid upload cannot be confirmed. + RapidUploadByHashes(ctx context.Context, dstDir model.Obj, req *SeedRapidUploadRequest, overwrite bool) (model.Obj, error) + + // RapidHashAlgos reports the whole-file hash algorithms accepted by + // RapidUploadByHashes (e.g. utils.MD5, utils.SHA1). + RapidHashAlgos() []utils.HashType + + // RapidHashNeedsPieces reports whether RapidUploadByHashes relies on + // per-slice hashes (CAS slice MD5s / SHA1 pieces) for this driver. + RapidHashNeedsPieces() bool +} diff --git a/internal/driver/seed_stream.go b/internal/driver/seed_stream.go new file mode 100644 index 00000000..23161b89 --- /dev/null +++ b/internal/driver/seed_stream.go @@ -0,0 +1,72 @@ +package driver + +import ( + "io" + "time" + + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/pkg/http_range" + "github.com/OpenListTeam/OpenList/v4/pkg/utils" +) + +// SeedHashStream is a model.FileStreamer that only carries file metadata and +// precomputed hashes; it never yields real content. +// +// It exists so that hash-driven rapid upload (秒传/CAS) implementations can reuse +// the drivers' existing Put/RapidUpload code paths, which expect a +// model.FileStreamer but only read the name/size/hash. Drivers that additionally +// need the real content (e.g. to compute a leading proof hash) should supply +// Source, which is exposed via GetReadCloser-like lazy opening. +type SeedHashStream struct { + name string + size int64 + hashInfo utils.HashInfo + // Source lazily opens the underlying content streamer. May be nil. + Source func() (model.FileStreamer, error) +} + +var ( + _ model.FileStreamer = (*SeedHashStream)(nil) + _ utils.ClosersIF = (*SeedHashStream)(nil) +) + +// NewSeedHashStream builds a hash-only streamer from a rapid-upload request. +func NewSeedHashStream(req *SeedRapidUploadRequest) *SeedHashStream { + s := &SeedHashStream{name: req.Name, size: req.Size, Source: req.Open} + if req.Whole != nil { + s.hashInfo = *req.Whole + } + return s +} + +func (s *SeedHashStream) GetName() string { return s.name } +func (s *SeedHashStream) GetSize() int64 { return s.size } +func (s *SeedHashStream) GetHash() utils.HashInfo { return s.hashInfo } +func (s *SeedHashStream) GetMimetype() string { return "" } +func (s *SeedHashStream) ModTime() time.Time { return time.Now() } +func (s *SeedHashStream) CreateTime() time.Time { return time.Now() } +func (s *SeedHashStream) IsDir() bool { return false } +func (s *SeedHashStream) GetID() string { return "" } +func (s *SeedHashStream) GetPath() string { return "" } + +func (s *SeedHashStream) NeedStore() bool { return false } +func (s *SeedHashStream) IsForceStreamUpload() bool { return true } +func (s *SeedHashStream) GetExist() model.Obj { return nil } +func (s *SeedHashStream) SetExist(model.Obj) {} +func (s *SeedHashStream) GetFile() model.File { return nil } +func (s *SeedHashStream) Add(io.Closer) {} +func (s *SeedHashStream) AddIfCloser(any) {} +func (s *SeedHashStream) Close() error { return nil } + +// Read returns EOF: the stream carries hashes only, no content. +func (s *SeedHashStream) Read([]byte) (int, error) { return 0, io.EOF } + +// RangeRead returns an empty reader, since no content is available. +func (s *SeedHashStream) RangeRead(http_range.Range) (io.Reader, error) { + return nil, io.EOF +} + +// CacheFullAndWriter reports that the content cannot be materialized. +func (s *SeedHashStream) CacheFullAndWriter(*model.UpdateProgress, io.Writer) (model.File, error) { + return nil, io.EOF +} diff --git a/internal/driver/seed_stream_test.go b/internal/driver/seed_stream_test.go new file mode 100644 index 00000000..1ca54d13 --- /dev/null +++ b/internal/driver/seed_stream_test.go @@ -0,0 +1,87 @@ +package driver + +import ( + "errors" + "io" + "strings" + "testing" + + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/pkg/http_range" + "github.com/OpenListTeam/OpenList/v4/pkg/utils" +) + +func TestSeedHashStreamCarriesMetadataAndHashes(t *testing.T) { + md5Hash := strings.Repeat("a", utils.MD5.Width) + sha1Hash := strings.Repeat("b", utils.SHA1.Width) + whole := utils.NewHashInfoByMap(map[*utils.HashType]string{ + utils.MD5: md5Hash, + utils.SHA1: sha1Hash, + }) + stream := NewSeedHashStream(&SeedRapidUploadRequest{ + Name: "hello.txt", + Size: 5, + Whole: &whole, + }) + + if got := stream.GetName(); got != "hello.txt" { + t.Fatalf("GetName() = %q", got) + } + if got := stream.GetSize(); got != 5 { + t.Fatalf("GetSize() = %d", got) + } + if got := stream.GetHash().GetHash(utils.MD5); got != md5Hash { + t.Fatalf("MD5 = %q", got) + } + if got := stream.GetHash().GetHash(utils.SHA1); got != sha1Hash { + t.Fatalf("SHA1 = %q", got) + } + if !stream.IsForceStreamUpload() { + t.Fatal("hash-only streams must force stream upload") + } + if stream.NeedStore() { + t.Fatal("hash-only streams must not be stored") + } +} + +func TestSeedHashStreamNeverExposesContent(t *testing.T) { + whole := utils.NewHashInfo(utils.MD5, strings.Repeat("c", utils.MD5.Width)) + stream := NewSeedHashStream(&SeedRapidUploadRequest{ + Name: "hash-only.bin", + Size: 1, + Whole: &whole, + }) + + if n, err := stream.Read(make([]byte, 8)); n != 0 || !errors.Is(err, io.EOF) { + t.Fatalf("Read() = (%d, %v)", n, err) + } + if _, err := stream.RangeRead(http_range.Range{Start: 0, Length: 1}); !errors.Is(err, io.EOF) { + t.Fatalf("RangeRead() error = %v", err) + } + if _, err := stream.CacheFullAndWriter(nil, nil); !errors.Is(err, io.EOF) { + t.Fatalf("CacheFullAndWriter() error = %v", err) + } +} + +func TestSeedHashStreamKeepsLazyContentSource(t *testing.T) { + called := false + stream := NewSeedHashStream(&SeedRapidUploadRequest{ + Name: "with-source.bin", + Size: 8, + Whole: &utils.HashInfo{}, + Open: func() (model.FileStreamer, error) { + called = true + return nil, errors.New("content unavailable") + }, + }) + + if stream.Source == nil { + t.Fatal("Source must be preserved for drivers that need a proof hash") + } + if _, err := stream.Source(); err == nil { + t.Fatal("expected the content source error to propagate") + } + if !called { + t.Fatal("Source was not invoked") + } +} diff --git a/internal/errs/driver.go b/internal/errs/driver.go index 4b6b5cac..b24799b4 100644 --- a/internal/errs/driver.go +++ b/internal/errs/driver.go @@ -4,4 +4,16 @@ import "errors" var ( EmptyToken = errors.New("empty token") + + // ErrUnavailableHash indicates the seed does not carry the hash algorithm + // required by the destination driver, so rapid upload cannot be attempted. + ErrUnavailableHash = errors.New("required hash is unavailable") + // ErrEmptyHash indicates a required hash exists but is empty/too short. + ErrEmptyHash = errors.New("empty hash") + // ErrHashMismatch indicates the remote side rejected the provided hash, so + // a full content transfer is required instead of a rapid upload. + ErrHashMismatch = errors.New("hash mismatch") + // ErrRapidUploadFailed indicates the driver attempted a rapid upload but + // could not confirm success. + ErrRapidUploadFailed = errors.New("rapid upload failed") ) diff --git a/internal/fs/copy_move.go b/internal/fs/copy_move.go index 1d171a9b..a1b34ebf 100644 --- a/internal/fs/copy_move.go +++ b/internal/fs/copy_move.go @@ -7,6 +7,7 @@ import ( "time" "github.com/OpenListTeam/OpenList/v4/internal/conf" + "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/op" @@ -15,7 +16,9 @@ import ( "github.com/OpenListTeam/OpenList/v4/internal/task_group" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/OpenListTeam/tache" + "github.com/google/uuid" "github.com/pkg/errors" + log "github.com/sirupsen/logrus" ) type taskType uint8 @@ -99,7 +102,79 @@ func (t *FileTransferTask) SetRetry(retry int, maxRetry int) { } } -func transfer(ctx context.Context, taskType taskType, srcObjPath, dstDirPath string, skipHook ...bool) (task.TaskExtensionInfo, error) { +func supportsNamedNativeCopy(storage driver.Driver) bool { + _, hasMkdir := storage.(driver.Mkdir) + if !hasMkdir { + _, hasMkdir = storage.(driver.MkdirResult) + } + _, hasCopy := storage.(driver.Copy) + if !hasCopy { + _, hasCopy = storage.(driver.CopyResult) + } + _, hasMove := storage.(driver.Move) + if !hasMove { + _, hasMove = storage.(driver.MoveResult) + } + _, hasRename := storage.(driver.Rename) + if !hasRename { + _, hasRename = storage.(driver.RenameResult) + } + _, hasRemove := storage.(driver.Remove) + return hasMkdir && hasCopy && hasMove && hasRename && hasRemove +} + +func namedCopyStageObjectPath(stageDirPath, srcPath, dstName string) string { + name := stdpath.Base(srcPath) + if dstName != "" { + name = dstName + } + return stdpath.Join(stageDirPath, name) +} + +func copyNamedInStorage(ctx context.Context, storage driver.Driver, srcPath, dstDirPath, dstName string) error { + stageDirPath := stdpath.Join(dstDirPath, ".openlist-copy-"+uuid.NewString()) + stageCtx := context.WithValue(ctx, conf.SkipHookKey, struct{}{}) + if err := op.MakeDir(stageCtx, storage, stageDirPath); err != nil { + return errors.WithMessage(err, "failed create copy staging directory") + } + stageCreated := true + removeStage := func() error { + return op.Remove(stageCtx, storage, stageDirPath) + } + defer func() { + if !stageCreated { + return + } + if err := removeStage(); err != nil { + log.WithError(err).Warnf("copy staging directory remains and requires cleanup: %s", stageDirPath) + } + }() + + if err := op.Copy(stageCtx, storage, srcPath, stageDirPath); err != nil { + return errors.WithMessage(err, "failed copy object into staging directory") + } + srcName := stdpath.Base(srcPath) + stageObjPath := namedCopyStageObjectPath(stageDirPath, srcPath, "") + if srcName != dstName { + if err := op.Rename(stageCtx, storage, stageObjPath, dstName); err != nil { + return errors.WithMessage(err, "failed rename object in staging directory") + } + stageObjPath = namedCopyStageObjectPath(stageDirPath, srcPath, dstName) + } + // Staging operations suppress hooks so intermediate paths stay invisible. + // The final move intentionally uses the caller context to publish the target + // and trigger the normal cache/update hooks exactly once. + if err := op.Move(ctx, storage, stageObjPath, dstDirPath); err != nil { + return errors.WithMessage(err, "failed move copied object from staging directory") + } + if err := removeStage(); err != nil { + return errors.WithMessagef(err, "failed remove copy staging directory %s; cleanup will be retried", stageDirPath) + } + stageCreated = false + return nil +} + +func transfer(ctx context.Context, taskType taskType, srcObjPath, dstDirPath, dstName string, skipHook ...bool) (task.TaskExtensionInfo, error) { srcStorage, srcObjActualPath, err := op.GetStorageAndActualPath(srcObjPath) if err != nil { return nil, errors.WithMessage(err, "failed get src storage") @@ -114,9 +189,18 @@ func transfer(ctx context.Context, taskType taskType, srcObjPath, dstDirPath str ctx = context.WithValue(ctx, conf.SkipHookKey, struct{}{}) } if taskType == copy || taskType == merge { - err = op.Copy(ctx, srcStorage, srcObjActualPath, dstDirActualPath) - if !errors.Is(err, errs.NotImplement) && !errors.Is(err, errs.NotSupport) { - return nil, err + if dstName != "" { + if supportsNamedNativeCopy(srcStorage) { + err = copyNamedInStorage(ctx, srcStorage, srcObjActualPath, dstDirActualPath, dstName) + if !errors.Is(err, errs.NotImplement) && !errors.Is(err, errs.NotSupport) { + return nil, err + } + } + } else { + err = op.Copy(ctx, srcStorage, srcObjActualPath, dstDirActualPath) + if !errors.Is(err, errs.NotImplement) && !errors.Is(err, errs.NotSupport) { + return nil, err + } } } else { err = op.Move(ctx, srcStorage, srcObjActualPath, dstDirActualPath) @@ -126,13 +210,15 @@ func transfer(ctx context.Context, taskType taskType, srcObjPath, dstDirPath str } } - // not in the same storage + // Use the transfer path when native copy cannot preserve the destination + // name or the source and destination storages differ. t := &FileTransferTask{ TaskData: TaskData{ SrcStorage: srcStorage, DstStorage: dstStorage, SrcActualPath: srcObjActualPath, DstActualPath: dstDirActualPath, + DstName: dstName, SrcStorageMp: srcStorage.GetStorage().MountPath, DstStorageMp: dstStorage.GetStorage().MountPath, }, @@ -188,9 +274,14 @@ func (t *FileTransferTask) RunWithNextTaskCallback(f func(nextTask *FileTransfer if err != nil { return errors.WithMessagef(err, "failed list src [%s] objs", t.SrcActualPath) } - dstActualPath := stdpath.Join(t.DstActualPath, srcObj.GetName()) - task_group.TransferCoordinator.AppendPayload(t.groupID, task_group.DstPathToHook(dstActualPath)) - + dstName := srcObj.GetName() + if t.DstName != "" { + dstName = t.DstName + } + dstActualPath := stdpath.Join(t.DstActualPath, dstName) + if err := op.MakeDir(t.Ctx(), t.DstStorage, dstActualPath); err != nil { + return errors.WithMessagef(err, "failed create dst dir [%s]", dstActualPath) + } existedObjs := make(map[string]bool) if t.TaskType == merge { dstObjs, err := op.List(t.Ctx(), t.DstStorage, dstActualPath, model.ListArgs{}) @@ -249,8 +340,12 @@ func (t *FileTransferTask) RunWithNextTaskCallback(f func(nextTask *FileTransfer return errors.WithMessagef(err, "failed get [%s] link", t.SrcActualPath) } // any link provided is seekable + streamObj := srcObj + if t.DstName != "" { + streamObj = &model.ObjWrapName{Name: t.DstName, Obj: srcObj} + } ss, err := stream.NewSeekableStream(&stream.FileStream{ - Obj: srcObj, + Obj: streamObj, Ctx: t.Ctx(), }, link) if err != nil { diff --git a/internal/fs/copy_move_test.go b/internal/fs/copy_move_test.go new file mode 100644 index 00000000..95f65751 --- /dev/null +++ b/internal/fs/copy_move_test.go @@ -0,0 +1,23 @@ +package fs + +import "testing" + +func TestNamedCopyStageObjectPath(t *testing.T) { + tests := []struct { + name string + stageDir string + srcPath string + dstName string + want string + }{ + {name: "source basename", stageDir: "/dst/.stage", srcPath: "/src/foo", want: "/dst/.stage/foo"}, + {name: "requested name", stageDir: "/dst/.stage", srcPath: "/src/foo", dstName: "bar", want: "/dst/.stage/bar"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := namedCopyStageObjectPath(tt.stageDir, tt.srcPath, tt.dstName); got != tt.want { + t.Fatalf("namedCopyStageObjectPath(%q, %q, %q) = %q, want %q", tt.stageDir, tt.srcPath, tt.dstName, got, tt.want) + } + }) + } +} diff --git a/internal/fs/fs.go b/internal/fs/fs.go index 67a1ac06..b7f064ac 100644 --- a/internal/fs/fs.go +++ b/internal/fs/fs.go @@ -69,7 +69,7 @@ func MakeDir(ctx context.Context, path string) error { } func Move(ctx context.Context, srcPath, dstDirPath string, skipHook ...bool) (task.TaskExtensionInfo, error) { - req, err := transfer(ctx, move, srcPath, dstDirPath, skipHook...) + req, err := transfer(ctx, move, srcPath, dstDirPath, "", skipHook...) if err != nil { log.Errorf("failed move %s to %s: %+v", srcPath, dstDirPath, err) } @@ -77,15 +77,24 @@ func Move(ctx context.Context, srcPath, dstDirPath string, skipHook ...bool) (ta } func Copy(ctx context.Context, srcObjPath, dstDirPath string, skipHook ...bool) (task.TaskExtensionInfo, error) { - res, err := transfer(ctx, copy, srcObjPath, dstDirPath, skipHook...) + res, err := transfer(ctx, copy, srcObjPath, dstDirPath, "", skipHook...) if err != nil { log.Errorf("failed copy %s to %s: %+v", srcObjPath, dstDirPath, err) } return res, err } +// CopyTo copies a file or directory to dstDirPath using dstName as its name. +func CopyTo(ctx context.Context, srcObjPath, dstDirPath, dstName string, skipHook ...bool) (task.TaskExtensionInfo, error) { + res, err := transfer(ctx, copy, srcObjPath, dstDirPath, dstName, skipHook...) + if err != nil { + log.Errorf("failed copy %s to %s as %s: %+v", srcObjPath, dstDirPath, dstName, err) + } + return res, err +} + func Merge(ctx context.Context, srcObjPath, dstDirPath string, skipHook ...bool) (task.TaskExtensionInfo, error) { - res, err := transfer(ctx, merge, srcObjPath, dstDirPath, skipHook...) + res, err := transfer(ctx, merge, srcObjPath, dstDirPath, "", skipHook...) if err != nil { log.Errorf("failed merge %s to %s: %+v", srcObjPath, dstDirPath, err) } diff --git a/internal/fs/other.go b/internal/fs/other.go index a23beb73..a74eff41 100644 --- a/internal/fs/other.go +++ b/internal/fs/other.go @@ -53,6 +53,7 @@ type TaskData struct { Status string `json:"-"` //don't save status to save space SrcActualPath string `json:"src_path"` DstActualPath string `json:"dst_path"` + DstName string `json:"dst_name,omitempty"` SrcStorage driver.Driver `json:"-"` DstStorage driver.Driver `json:"-"` SrcStorageMp string `json:"src_storage_mp"` diff --git a/internal/fs/seed_generate.go b/internal/fs/seed_generate.go new file mode 100644 index 00000000..217bc6ff --- /dev/null +++ b/internal/fs/seed_generate.go @@ -0,0 +1,610 @@ +package fs + +import ( + "bytes" + "context" + "encoding/base64" + "encoding/json" + "fmt" + "io" + stdpath "path" + "slices" + "strings" + "time" + + "github.com/OpenListTeam/OpenList/v4/internal/conf" + "github.com/OpenListTeam/OpenList/v4/internal/errs" + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/internal/op" + "github.com/OpenListTeam/OpenList/v4/internal/setting" + "github.com/OpenListTeam/OpenList/v4/internal/stream" + "github.com/OpenListTeam/OpenList/v4/internal/task" + "github.com/OpenListTeam/OpenList/v4/pkg/http_range" + "github.com/OpenListTeam/OpenList/v4/pkg/torrent" + "github.com/OpenListTeam/OpenList/v4/pkg/utils" + "github.com/OpenListTeam/OpenList/v4/server/common" + "github.com/OpenListTeam/tache" + "github.com/pkg/errors" +) + +// MaxSeedGenerateSyncSize bounds synchronous seed generation (1GB). Larger +// requests are turned into an asynchronous task. +const MaxSeedGenerateSyncSize = 1 * 1024 * 1024 * 1024 + +// SeedGenerateNeedsAsync reports whether the given files must be generated +// asynchronously because they exceed the synchronous size limit. It resolves +// each path and sums the file sizes, returning the first error encountered. +func SeedGenerateNeedsAsync(ctx context.Context, user *model.User, paths []string) (bool, error) { + var total int64 + for _, requestedPath := range paths { + fullPath, err := user.JoinPath(requestedPath) + if err != nil { + return false, err + } + meta, metaErr := op.GetNearestMeta(fullPath) + if metaErr != nil && !errors.Is(errors.Cause(metaErr), errs.MetaNotFound) { + return false, metaErr + } + if !common.CanRead(user, meta, fullPath) { + return false, errs.PermissionDenied + } + storage, actualPath, err := op.GetStorageAndActualPath(fullPath) + if err != nil { + return false, err + } + obj, err := op.Get(ctx, storage, actualPath) + if err != nil { + return false, fmt.Errorf("seed path must be a readable file: %s", requestedPath) + } + if obj.IsDir() { + return false, fmt.Errorf("seed path must be a readable file: %s", requestedPath) + } + total += obj.GetSize() + if total > MaxSeedGenerateSyncSize { + return true, nil + } + } + return false, nil +} + +// SeedHashSelection controls whole-file and piece hash inclusion. +type SeedHashSelection struct { + Whole bool `json:"whole"` + Pieces bool `json:"pieces"` +} + +// SeedHashMatrix controls the optional hash metadata stored in a seed. +type SeedHashMatrix struct { + MD5 SeedHashSelection `json:"md5"` + SHA1 SeedHashSelection `json:"sha1"` + SHA256 SeedHashSelection `json:"sha256"` +} + +// SeedGenerateParams carries a fully-resolved seed generation request, free of +// any HTTP transport concerns so it can run synchronously or as a task. +type SeedGenerateParams struct { + Paths []string + Formats []string + Name string + Comment string + FileComments map[string]string + HashMatrix SeedHashMatrix + PieceSize int64 + Trackers []string + Channels []torrent.SeedChannel + OutputPath string + IncludeShare bool + IncludeDirectSource bool + ShareFiles []string + DirectFiles []string +} + +// SeedArtifact is one generated seed container. +type SeedArtifact struct { + Format string `json:"format"` + Name string `json:"name"` + FileName string `json:"file_name"` + SeedData string `json:"seed_data"` + Size int `json:"size"` + Path string `json:"path,omitempty"` +} + +// DeriveSeedName derives a sensible default seed name from the source paths: +// single selection uses the file name, multi selection uses the common base +// name (ignoring extensions) when all files share one, otherwise the folder name. +func DeriveSeedName(paths []string) string { + if len(paths) == 0 { + return "OpenList Seed" + } + if len(paths) == 1 { + return seedBaseName(paths[0]) + } + // Common base name ignoring extensions (e.g. a.docx + a.exe -> "a"). + commonBase := seedBaseName(paths[0]) + for _, p := range paths[1:] { + if base := seedBaseName(p); base != commonBase { + commonBase = "" + break + } + } + if commonBase != "" { + return commonBase + } + // Fall back to the common parent directory name. + dir := commonParentDir(paths) + if base := stdpath.Base(dir); base != "" && base != "/" && base != "." { + return base + } + return "OpenList Seed" +} + +// seedBaseName returns the file name without its extension. +func seedBaseName(p string) string { + base := stdpath.Base(p) + return strings.TrimSuffix(base, stdpath.Ext(base)) +} + +// commonParentDir returns the longest common parent directory of the given paths. +func commonParentDir(paths []string) string { + if len(paths) == 0 { + return "/" + } + parts := strings.Split(strings.Trim(stdpath.Dir(paths[0]), "/"), "/") + for _, p := range paths[1:] { + cur := strings.Split(strings.Trim(stdpath.Dir(p), "/"), "/") + n := 0 + for n < len(parts) && n < len(cur) && parts[n] == cur[n] { + n++ + } + parts = parts[:n] + } + if len(parts) == 0 { + return "/" + } + return "/" + strings.Join(parts, "/") +} + +// NormalizeSeedFormats validates and deduplicates a list of seed format names. +func NormalizeSeedFormats(rawFormats []string) ([]string, error) { + formats := append([]string(nil), rawFormats...) + if len(formats) == 1 && strings.TrimSpace(formats[0]) == "" { + formats[0] = setting.GetStr(conf.SeedDefaultFormat, "oss") + } + if len(formats) > 3 { + return nil, fmt.Errorf("at most three seed formats may be generated") + } + result := make([]string, 0, len(formats)) + seen := make(map[string]struct{}, len(formats)) + for _, rawFormat := range formats { + format := strings.ToLower(strings.TrimPrefix(strings.TrimSpace(rawFormat), ".")) + if format == "bt" { + format = "torrent" + } + if format != "oss" && format != "torrent" && format != "cas" { + return nil, fmt.Errorf("unsupported seed format %q", rawFormat) + } + if _, exists := seen[format]; exists { + continue + } + seen[format] = struct{}{} + result = append(result, format) + } + return result, nil +} + +func seedFormats(params SeedGenerateParams) ([]string, error) { + formats, err := NormalizeSeedFormats(params.Formats) + if err != nil { + return nil, err + } + if len(formats) == 0 { + formats = []string{setting.GetStr(conf.SeedDefaultFormat, "oss")} + } + return formats, nil +} + +func seedMatrixEmpty(matrix SeedHashMatrix) bool { + return !matrix.MD5.Whole && !matrix.MD5.Pieces && !matrix.SHA1.Whole && !matrix.SHA1.Pieces && !matrix.SHA256.Whole && !matrix.SHA256.Pieces +} + +func loadSeedDefaultMatrix() SeedHashMatrix { + var matrix SeedHashMatrix + if raw := strings.TrimSpace(setting.GetStr(conf.SeedDefaultMatrix)); raw != "" { + _ = json.Unmarshal([]byte(raw), &matrix) + } + return matrix +} + +func normalizedSeedMatrix(matrix SeedHashMatrix, formats []string) SeedHashMatrix { + if seedMatrixEmpty(matrix) { + matrix = loadSeedDefaultMatrix() + } + if seedMatrixEmpty(matrix) { + matrix = SeedHashMatrix{ + MD5: SeedHashSelection{Whole: true, Pieces: true}, SHA1: SeedHashSelection{Whole: true, Pieces: true}, + SHA256: SeedHashSelection{Whole: true, Pieces: true}, + } + } + for _, format := range formats { + switch format { + case "torrent": + matrix.SHA1 = SeedHashSelection{Whole: true, Pieces: true} + case "cas": + matrix.MD5 = SeedHashSelection{Whole: true, Pieces: true} + } + } + return matrix +} + +func canReuseListedHashes(hashInfo utils.HashInfo, matrix SeedHashMatrix) bool { + if matrix.MD5.Pieces || matrix.SHA1.Pieces || matrix.SHA256.Pieces { + return false + } + if matrix.MD5.Whole && hashInfo.GetHash(utils.MD5) == "" { + return false + } + if matrix.SHA1.Whole && hashInfo.GetHash(utils.SHA1) == "" { + return false + } + if matrix.SHA256.Whole && hashInfo.GetHash(utils.SHA256) == "" { + return false + } + return true +} + +func applySeedMatrix(file *torrent.SeedFile, matrix SeedHashMatrix) { + if !matrix.MD5.Whole { + file.Hashes.MD5 = "" + } + if !matrix.SHA1.Whole { + file.Hashes.SHA1 = "" + } + if !matrix.SHA256.Whole { + file.Hashes.SHA256 = "" + } + if file.Hashes.Pieces == nil { + return + } + if !matrix.MD5.Pieces { + file.Hashes.Pieces.MD5 = nil + } + if !matrix.SHA1.Pieces { + file.Hashes.Pieces.SHA1 = nil + } + if !matrix.SHA256.Pieces { + file.Hashes.Pieces.SHA256 = nil + } + if len(file.Hashes.Pieces.MD5) == 0 && len(file.Hashes.Pieces.SHA1) == 0 && len(file.Hashes.Pieces.SHA256) == 0 { + file.Hashes.Pieces = nil + } +} + +// EncodeGeneratedSeed serializes a seed in the requested container format. +func EncodeGeneratedSeed(seed *torrent.Seed, format string, standardPieces []byte) ([]byte, error) { + if format != "torrent" { + return torrent.EncodeSeed(seed, format) + } + t := &torrent.Torrent{ + Info: torrent.TorrentInfo{Name: seed.Name, PieceLength: seed.PieceSize, Pieces: standardPieces}, + Comment: seed.Comment, + CreatedBy: seed.CreatedBy, + CreationDate: time.Now().Unix(), + OpenList: seed, + } + if len(seed.Trackers) > 0 { + t.Announce = seed.Trackers[0] + for _, tracker := range seed.Trackers { + t.AnnounceList = append(t.AnnounceList, []string{tracker}) + } + } + if len(seed.Files) == 1 { + file := seed.Files[0] + t.Info.Name = stdpath.Base(file.Path) + t.Info.Length = file.Size + t.Info.MD5Sum = file.Hashes.MD5 + casCloud := file.CASCloud + if casCloud == "" { + casCloud = torrent.Cloud189 + } + if file.CASSliceMD5 != "" { + t.SetCASInfo(&torrent.CASInfo{ + FileMD5: strings.ToUpper(file.Hashes.MD5), SliceMD5: strings.ToUpper(file.CASSliceMD5), + SliceSize: torrent.DefaultPieceSize, Cloud: casCloud, + }) + } else if seed.PieceSize == torrent.DefaultPieceSize && file.Hashes.Pieces != nil && len(file.Hashes.Pieces.MD5) > 0 { + t.SetCASInfo(torrent.BuildCASInfoFromMD5sWithCloud(file.Hashes.MD5, file.Hashes.Pieces.MD5, torrent.DefaultPieceSize, casCloud)) + } + } else { + for _, file := range seed.Files { + t.Info.Files = append(t.Info.Files, torrent.TorrentFile{Length: file.Size, Path: strings.Split(file.Path, "/"), MD5Sum: file.Hashes.MD5}) + } + } + return t.Encode() +} + +// GenerateSeedArtifacts reads each file once while computing the requested +// hashes, then emits one or more seed containers. It has no HTTP dependency and +// can be driven synchronously or from a background task. +func GenerateSeedArtifacts(ctx context.Context, user *model.User, params SeedGenerateParams) ([]SeedArtifact, *torrent.Seed, error) { + if len(params.Paths) == 0 || len(params.Paths) > torrent.DefaultMaxSeedFiles { + return nil, nil, fmt.Errorf("invalid seed file count") + } + formats, err := seedFormats(params) + if err != nil { + return nil, nil, err + } + matrix := normalizedSeedMatrix(params.HashMatrix, formats) + pieceSize := params.PieceSize + if pieceSize <= 0 { + pieceSize = torrent.DefaultPieceSize + } + if slices.Contains(formats, "cas") { + pieceSize = torrent.DefaultPieceSize + } + seedName := strings.TrimSpace(params.Name) + if seedName == "" { + seedName = DeriveSeedName(params.Paths) + } + seed := torrent.NewSeed(seedName, "OpenList", pieceSize) + seed.Comment = params.Comment + seed.Trackers = params.Trackers + seed.Channels = params.Channels + shareSet := make(map[string]bool, len(params.ShareFiles)) + for _, p := range params.ShareFiles { + if strings.TrimSpace(p) != "" { + shareSet[p] = true + } + } + directSet := make(map[string]bool, len(params.DirectFiles)) + for _, p := range params.DirectFiles { + if strings.TrimSpace(p) != "" { + directSet[p] = true + } + } + useGlobalShare := len(shareSet) == 0 && params.IncludeShare + useGlobalDirect := len(directSet) == 0 && params.IncludeDirectSource + hasShare := useGlobalShare || len(shareSet) > 0 + hasDirect := useGlobalDirect || len(directSet) > 0 + if hasShare && !user.CanShare() { + return nil, nil, errs.PermissionDenied + } + if hasDirect && setting.GetBool(conf.SignAll) && !hasShare { + return nil, nil, fmt.Errorf("direct sources require an automatic share when global signing is enabled") + } + if (hasShare || hasDirect) && strings.TrimSpace(setting.GetStr(conf.SeedSiteURL)) == "" { + return nil, nil, fmt.Errorf("seed_site_url must be configured before embedding download sources") + } + globalHasher := torrent.NewHashWriter(pieceSize, pieceSize, 0) + fullPaths := make([]string, 0, len(params.Paths)) + var total int64 + for _, requestedPath := range params.Paths { + fullPath, err := user.JoinPath(requestedPath) + if err != nil { + return nil, nil, err + } + meta, err := op.GetNearestMeta(fullPath) + if err != nil && !errors.Is(errors.Cause(err), errs.MetaNotFound) { + return nil, nil, err + } + if !common.CanRead(user, meta, fullPath) { + return nil, nil, errs.PermissionDenied + } + storage, actualPath, err := op.GetStorageAndActualPath(fullPath) + if err != nil { + return nil, nil, err + } + obj, err := op.Get(ctx, storage, actualPath) + if err != nil || obj.IsDir() { + return nil, nil, fmt.Errorf("seed path must be a readable file: %s", requestedPath) + } + total += obj.GetSize() + modified := "" + if !obj.ModTime().IsZero() { + modified = obj.ModTime().UTC().Format(time.RFC3339) + } + seedPath := stdpath.Base(requestedPath) + if len(params.Paths) > 1 { + seedPath = strings.TrimPrefix(stdpath.Clean(requestedPath), "/") + } + + hashInfo := obj.GetHash() + if canReuseListedHashes(hashInfo, matrix) { + seedFile := torrent.SeedFile{ + Path: seedPath, + Size: obj.GetSize(), + Modified: modified, + Hashes: torrent.SeedHashes{ + MD5: strings.ToLower(hashInfo.GetHash(utils.MD5)), + SHA1: strings.ToLower(hashInfo.GetHash(utils.SHA1)), + SHA256: strings.ToLower(hashInfo.GetHash(utils.SHA256)), + }, + } + applySeedMatrix(&seedFile, matrix) + if comment := strings.TrimSpace(params.FileComments[requestedPath]); comment != "" { + seedFile.Comment = comment + } else if comment := strings.TrimSpace(params.FileComments[obj.GetName()]); comment != "" { + seedFile.Comment = comment + } + if useGlobalDirect || directSet[requestedPath] { + baseURL := strings.TrimRight(setting.GetStr(conf.SeedSiteURL), "/") + seedFile.Sources = []torrent.SeedSource{{Type: "openlist-direct", URL: baseURL + utils.EncodePath("/d"+fullPath)}} + } + seed.Files = append(seed.Files, seedFile) + fullPaths = append(fullPaths, fullPath) + continue + } + + link, _, err := op.Link(ctx, storage, actualPath, model.LinkArgs{}) + if err != nil { + return nil, nil, fmt.Errorf("storage cannot stream %s: %v", requestedPath, err) + } + rangeReader, err := stream.GetRangeReaderFromLink(obj.GetSize(), link) + if err != nil { + return nil, nil, fmt.Errorf("storage cannot stream %s", requestedPath) + } + rc, err := rangeReader.RangeRead(ctx, http_range.Range{Length: obj.GetSize()}) + if err != nil { + return nil, nil, err + } + fileHasher := torrent.NewHashWriter(pieceSize, pieceSize, 0) + n, copyErr := io.Copy(io.MultiWriter(globalHasher, fileHasher), rc) + _ = rc.Close() + if copyErr != nil { + return nil, nil, fmt.Errorf("read %s: %w", requestedPath, copyErr) + } + if n != obj.GetSize() { + return nil, nil, fmt.Errorf("read %s: got %d of %d bytes", requestedPath, n, obj.GetSize()) + } + fileHasher.Finish() + seedFile := fileHasher.BuildSeedFile(seedPath, modified) + applySeedMatrix(&seedFile, matrix) + if comment := strings.TrimSpace(params.FileComments[requestedPath]); comment != "" { + seedFile.Comment = comment + } else if comment := strings.TrimSpace(params.FileComments[obj.GetName()]); comment != "" { + seedFile.Comment = comment + } + if useGlobalDirect || directSet[requestedPath] { + baseURL := strings.TrimRight(setting.GetStr(conf.SeedSiteURL), "/") + seedFile.Sources = []torrent.SeedSource{{Type: "openlist-direct", URL: baseURL + utils.EncodePath("/d"+fullPath)}} + } + seed.Files = append(seed.Files, seedFile) + fullPaths = append(fullPaths, fullPath) + } + globalHasher.Finish() + createdShares := make([]string, 0, len(seed.Files)) + keepCreatedShares := false + defer func() { + if !keepCreatedShares { + for _, createdID := range createdShares { + _ = op.DeleteSharing(createdID) + } + } + }() + if hasShare { + for index, fullPath := range fullPaths { + if !useGlobalShare && !shareSet[params.Paths[index]] { + continue + } + sharing := &model.Sharing{ + SharingDB: &model.SharingDB{Remark: "Transfer seed source"}, + Files: []string{fullPath}, Creator: user, + } + shareID, createErr := op.CreateSharing(sharing) + if createErr != nil { + return nil, nil, fmt.Errorf("create seed share: %w", createErr) + } + createdShares = append(createdShares, shareID) + seed.Files[index].Sources = []torrent.SeedSource{{ + Type: "openlist-share", URL: strings.TrimRight(setting.GetStr(conf.SeedSiteURL), "/") + "/sd/" + shareID, + ShareID: shareID, + }} + } + } + if err := torrent.ValidateSeed(seed, torrent.DefaultParseLimits()); err != nil { + return nil, nil, err + } + outputPath := strings.TrimSpace(params.OutputPath) + artifacts := make([]SeedArtifact, 0, len(formats)) + seenFormats := make(map[string]struct{}, len(formats)) + + // writeArtifact persists the encoded container into the destination folder + // when an output path is configured, returning the fully-populated artifact. + writeArtifact := func(format, fileName string, data []byte) (SeedArtifact, error) { + artifact := SeedArtifact{ + Format: format, + Name: fileName, + FileName: fileName, + SeedData: base64.StdEncoding.EncodeToString(data), + Size: len(data), + } + if outputPath == "" { + return artifact, nil + } + dstDir, err := user.JoinPath(outputPath) + if err != nil { + return artifact, err + } + meta, metaErr := op.GetNearestMeta(dstDir) + if metaErr != nil && !errors.Is(errors.Cause(metaErr), errs.MetaNotFound) { + return artifact, metaErr + } + if (!user.CanWriteContent() && !common.CanWriteContentBypassUserPerms(meta, dstDir)) || !common.CanWrite(user, meta, dstDir) { + return artifact, errs.PermissionDenied + } + fileStream := &stream.FileStream{ + Ctx: ctx, + Obj: &model.Object{Name: fileName, Size: int64(len(data)), Modified: time.Now()}, + Reader: bytes.NewReader(data), Mimetype: "application/octet-stream", + } + if err = PutDirectly(ctx, dstDir, fileStream); err != nil { + return artifact, err + } + artifact.Path = stdpath.Join(outputPath, fileName) + return artifact, nil + } + + for _, requestedFormat := range formats { + format := strings.ToLower(strings.TrimPrefix(strings.TrimSpace(requestedFormat), ".")) + if format == "bt" { + format = "torrent" + } + if _, exists := seenFormats[format]; exists { + continue + } + seenFormats[format] = struct{}{} + + data, err := EncodeGeneratedSeed(seed, format, globalHasher.GetPieceHashes()) + if err != nil { + return nil, nil, fmt.Errorf("generate %s seed: %w", format, err) + } + fileName := stdpath.Base(seed.Name) + "." + format + artifact, err := writeArtifact(format, fileName, data) + if err != nil { + return nil, nil, err + } + artifacts = append(artifacts, artifact) + } + keepCreatedShares = true + return artifacts, seed, nil +} + +// SeedGenerateTask generates seed containers asynchronously, writing them into +// params.OutputPath so they appear in the target folder once done. +type SeedGenerateTask struct { + task.TaskExtension + params SeedGenerateParams +} + +func (t *SeedGenerateTask) GetName() string { + if len(t.params.Paths) == 1 { + return fmt.Sprintf("generate seed for %s", stdpath.Base(t.params.Paths[0])) + } + return fmt.Sprintf("generate seed for %d files", len(t.params.Paths)) +} + +func (t *SeedGenerateTask) GetStatus() string { + return "generating seed" +} + +func (t *SeedGenerateTask) Run() error { + t.ClearEndTime() + t.SetStartTime(time.Now()) + defer func() { t.SetEndTime(time.Now()) }() + _, _, err := GenerateSeedArtifacts(t.Ctx(), t.Creator, t.params) + return err +} + +var SeedGenerateTaskManager *tache.Manager[*SeedGenerateTask] + +// AddSeedGenerateTask schedules an asynchronous seed generation task. +func AddSeedGenerateTask(ctx context.Context, user *model.User, params SeedGenerateParams) (task.TaskExtensionInfo, error) { + t := &SeedGenerateTask{ + TaskExtension: task.TaskExtension{ + Creator: user, + ApiUrl: common.GetApiUrl(ctx), + }, + params: params, + } + SeedGenerateTaskManager.Add(t) + return t, nil +} diff --git a/internal/media/appkey.go b/internal/media/appkey.go new file mode 100644 index 00000000..6e4edad4 --- /dev/null +++ b/internal/media/appkey.go @@ -0,0 +1,120 @@ +package media + +import ( + "crypto/aes" + "crypto/cipher" + "crypto/md5" + "crypto/rand" + "crypto/sha256" + "encoding/base64" + "encoding/hex" + "errors" + "strings" + + "github.com/OpenListTeam/OpenList/v4/internal/conf" + "github.com/OpenListTeam/OpenList/v4/internal/setting" +) + +const appKeyPrefixLen = 8 + +// AppKeySecretLen 明文密钥长度(不含前缀分隔符) +const appKeySecretLen = 32 + +// NewAppKeyPlain 生成一个新的应用密钥明文,格式:olmp_<40 位随机串> +func NewAppKeyPlain() (string, error) { + buf := make([]byte, appKeySecretLen) + if _, err := rand.Read(buf); err != nil { + return "", err + } + body := strings.TrimRight(base64.RawURLEncoding.EncodeToString(buf), "=") + if len(body) > 40 { + body = body[:40] + } + return "olmp_" + body, nil +} + +// AppKeyPrefix 返回用于检索/展示的明文前缀 +func AppKeyPrefix(plain string) string { + if len(plain) <= appKeyPrefixLen { + return plain + } + return plain[:appKeyPrefixLen] +} + +func appKeyCipherKey() []byte { + seed := conf.Conf.JwtSecret + if seed == "" { + seed = setting.GetStr(conf.Token) + } + sum := sha256.Sum256([]byte("openlist-media-appkey:" + seed)) + return sum[:] +} + +func newGCM() (cipher.AEAD, error) { + block, err := aes.NewCipher(appKeyCipherKey()) + if err != nil { + return nil, err + } + return cipher.NewGCM(block) +} + +// EncryptAppKey 加密应用密钥明文 +func EncryptAppKey(plain string) (string, error) { + gcm, err := newGCM() + if err != nil { + return "", err + } + nonce := make([]byte, gcm.NonceSize()) + if _, err := rand.Read(nonce); err != nil { + return "", err + } + out := gcm.Seal(nonce, nonce, []byte(plain), nil) + return base64.RawURLEncoding.EncodeToString(out), nil +} + +// DecryptAppKey 解密应用密钥明文 +func DecryptAppKey(secret string) (string, error) { + raw, err := base64.RawURLEncoding.DecodeString(secret) + if err != nil { + return "", err + } + gcm, err := newGCM() + if err != nil { + return "", err + } + if len(raw) < gcm.NonceSize() { + return "", errors.New("invalid app key secret") + } + nonce, data := raw[:gcm.NonceSize()], raw[gcm.NonceSize():] + plain, err := gcm.Open(nil, nonce, data, nil) + if err != nil { + return "", err + } + return string(plain), nil +} + +// HashToken 计算设备令牌的存储哈希 +func HashToken(token string) string { + sum := sha256.Sum256([]byte("openlist-emby-token:" + token)) + return hex.EncodeToString(sum[:]) +} + +// NewDeviceToken 生成 Emby 设备令牌 +func NewDeviceToken() (string, error) { + buf := make([]byte, 24) + if _, err := rand.Read(buf); err != nil { + return "", err + } + return hex.EncodeToString(buf), nil +} + +// SubsonicToken 计算 Subsonic 的 t = md5(password + salt) +func SubsonicToken(password, salt string) string { + sum := md5.Sum([]byte(password + salt)) + return hex.EncodeToString(sum[:]) +} + +// MatchSubsonicToken 校验客户端提交的 t 是否匹配 +func MatchSubsonicToken(password, salt, token string) bool { + return strings.EqualFold(SubsonicToken(password, salt), token) +} diff --git a/internal/media/id3.go b/internal/media/id3.go new file mode 100644 index 00000000..08acd080 --- /dev/null +++ b/internal/media/id3.go @@ -0,0 +1,684 @@ +package media + +import ( + "encoding/binary" + "io" + "strconv" + "strings" + + "github.com/OpenListTeam/OpenList/v4/internal/media/imgmime" +) + +// maxEmbeddedPictureBytes 内嵌封面图片最大字节数,防止畸形文件触发超大内存分配 +const maxEmbeddedPictureBytes = 16 << 20 + +// MusicTag 音频文件标签信息 +type MusicTag struct { + Title string // TIT2 + Artist string // TPE1 + Album string // TALB + AlbumArtist string // TPE2 + TrackNumber int // TRCK + Year string // TYER / TDRC + Genre string // TCON + CoverData []byte // 封面图片原始字节(APIC / PICTURE) + CoverMIME string // 封面图片 MIME 类型(如 image/jpeg) + Lyrics string // 歌词内容(LRC 文本或纯文本):来自 USLT / SYLT / Vorbis LYRICS +} + +// ParseID3v2 从 io.Reader 中解析 ID3v2 标签(只读取文件头部,不需要 Seek) +// 支持 ID3v2.3 和 ID3v2.4 +func ParseID3v2(r io.Reader) (*MusicTag, error) { + // 读取 ID3v2 头部(10 字节) + header := make([]byte, 10) + if _, err := io.ReadFull(r, header); err != nil { + return nil, err + } + + // 检查魔数 + if string(header[0:3]) != "ID3" { + return nil, nil // 不是 ID3v2 文件,不报错 + } + + version := header[3] // 主版本号:3 = ID3v2.3, 4 = ID3v2.4 + if version < 3 || version > 4 { + return nil, nil // 只支持 v2.3 和 v2.4 + } + + // 解析标签总大小(syncsafe integer,4 字节,每字节最高位为 0) + tagSize := syncsafeToInt(header[6:10]) + if tagSize <= 0 || tagSize > 10*1024*1024 { // 最大 10MB + return nil, nil + } + + // 读取所有帧数据 + data := make([]byte, tagSize) + if _, err := io.ReadFull(r, data); err != nil { + return nil, nil // 读取失败时静默跳过 + } + + tag := &MusicTag{} + pos := 0 + + // 跳过扩展头部(如果有) + flags := header[5] + if flags&0x40 != 0 { // 有扩展头部 + if pos+4 > len(data) { + return tag, nil + } + var extSize int + if version == 4 { + extSize = syncsafeToInt(data[pos : pos+4]) + } else { + extSize = int(binary.BigEndian.Uint32(data[pos : pos+4])) + } + pos += extSize + } + + // 解析帧 + for pos+10 <= len(data) { + frameID := string(data[pos : pos+4]) + if frameID == "\x00\x00\x00\x00" { + break // 填充区域,结束 + } + + // ID3v2.4 帧大小使用 syncsafe integer(与标签头一致), + // ID3v2.3 使用普通 32 位大端整数。 + var frameSize int + if version == 4 { + frameSize = syncsafeToInt(data[pos+4 : pos+8]) + } else { + frameSize = int(binary.BigEndian.Uint32(data[pos+4 : pos+8])) + } + pos += 10 // 跳过帧头(4+4+2) + + if frameSize <= 0 || pos+frameSize > len(data) { + break + } + + frameData := data[pos : pos+frameSize] + pos += frameSize + + // 解析文本帧(T 开头的帧) + if len(frameID) == 4 && frameID[0] == 'T' && len(frameData) > 0 { + text := decodeID3Text(frameData) + switch frameID { + case "TIT2": + tag.Title = text + case "TPE1": + tag.Artist = text + case "TALB": + tag.Album = text + case "TPE2": + tag.AlbumArtist = text + case "TRCK": + // 格式可能是 "1" 或 "1/12" + parts := strings.SplitN(text, "/", 2) + if n, err := strconv.Atoi(strings.TrimSpace(parts[0])); err == nil { + tag.TrackNumber = n + } + case "TYER", "TDRC": + if tag.Year == "" { + tag.Year = text + } + case "TCON": + tag.Genre = parseID3Genre(text) + } + } + + // 解析 APIC 帧(内嵌封面图片),只取第一张 + if frameID == "APIC" && tag.CoverData == nil && len(frameData) > 1 { + tag.CoverData, tag.CoverMIME = parseAPICFrame(frameData) + } + + // 解析 USLT 帧(非同步歌词) + // 格式:编码字节(1) + 语言(3字节 ASCII) + 描述字符串(null 结尾) + 歌词文本 + if frameID == "USLT" && tag.Lyrics == "" && len(frameData) > 5 { + tag.Lyrics = parseUSLTFrame(frameData) + } + + // 解析 SYLT 帧(同步歌词,二进制时间戳格式),仅在没有 USLT 时尝试转 LRC + if frameID == "SYLT" && tag.Lyrics == "" && len(frameData) > 6 { + if lrc := parseSYLTFrame(frameData); lrc != "" { + tag.Lyrics = lrc + } + } + } + + return tag, nil +} + +// parseUSLTFrame 解析 ID3v2 USLT 帧(非同步歌词) +// 格式:编码字节(1) + 语言(3字节 ASCII) + 描述字符串(null 结尾) + 歌词文本 +func parseUSLTFrame(data []byte) string { + if len(data) < 5 { + return "" + } + encoding := data[0] + // 跳过编码(1) + 语言(3) = 4 字节 + pos := 4 + + // 跳过描述字符串 + if encoding == 0x01 || encoding == 0x02 { + // UTF-16:找 \x00\x00 结尾 + for pos+1 < len(data) { + if data[pos] == 0 && data[pos+1] == 0 { + pos += 2 + break + } + pos += 2 + } + } else { + // ISO-8859-1 / UTF-8:找单个 \x00 结尾 + for pos < len(data) { + if data[pos] == 0 { + pos++ + break + } + pos++ + } + } + + if pos >= len(data) { + return "" + } + + // 剩余字节就是歌词文本,按编码解码(复用 decodeID3Text 但需要把 encoding 字节加回头部) + lyricsRaw := make([]byte, 1+len(data)-pos) + lyricsRaw[0] = encoding + copy(lyricsRaw[1:], data[pos:]) + return strings.TrimSpace(decodeID3Text(lyricsRaw)) +} + +// parseSYLTFrame 解析 ID3v2 SYLT 帧(同步歌词),转换为 LRC 格式 +// 格式:编码(1) + 语言(3) + 时间戳格式(1) + 内容类型(1) + 描述(null 结尾) + 多组 (歌词文本 + null + 4字节时间戳) +// 时间戳格式:1=MPEG帧数(不支持),2=毫秒 +func parseSYLTFrame(data []byte) string { + if len(data) < 6 { + return "" + } + encoding := data[0] + timestampFormat := data[4] // 1=MPEG帧, 2=ms + if timestampFormat != 2 { + return "" // 只支持毫秒时间戳 + } + pos := 6 // 跳过 encoding(1) + lang(3) + timestamp_fmt(1) + content_type(1) + + // 跳过描述字符串 + if encoding == 0x01 || encoding == 0x02 { + for pos+1 < len(data) { + if data[pos] == 0 && data[pos+1] == 0 { + pos += 2 + break + } + pos += 2 + } + } else { + for pos < len(data) { + if data[pos] == 0 { + pos++ + break + } + pos++ + } + } + + var sb strings.Builder + wide := encoding == 0x01 || encoding == 0x02 + for pos < len(data) { + // 找文本终止符 + textStart := pos + textEnd := -1 + if wide { + for i := pos; i+1 < len(data); i += 2 { + if data[i] == 0 && data[i+1] == 0 { + textEnd = i + break + } + } + } else { + for i := pos; i < len(data); i++ { + if data[i] == 0 { + textEnd = i + break + } + } + } + if textEnd < 0 { + break + } + // 解码歌词文本 + lineRaw := make([]byte, 1+textEnd-textStart) + lineRaw[0] = encoding + copy(lineRaw[1:], data[textStart:textEnd]) + lineText := decodeID3Text(lineRaw) + + // 跳过文本终止符 + if wide { + pos = textEnd + 2 + } else { + pos = textEnd + 1 + } + // 4 字节时间戳(毫秒) + if pos+4 > len(data) { + break + } + ms := int(binary.BigEndian.Uint32(data[pos : pos+4])) + pos += 4 + + minutes := ms / 60000 + seconds := (ms % 60000) / 1000 + cs := (ms % 1000) / 10 + // LRC 标准格式 [mm:ss.xx] + sb.WriteString("[") + if minutes < 10 { + sb.WriteString("0") + } + sb.WriteString(strconv.Itoa(minutes)) + sb.WriteString(":") + if seconds < 10 { + sb.WriteString("0") + } + sb.WriteString(strconv.Itoa(seconds)) + sb.WriteString(".") + if cs < 10 { + sb.WriteString("0") + } + sb.WriteString(strconv.Itoa(cs)) + sb.WriteString("]") + sb.WriteString(lineText) + sb.WriteString("\n") + } + return sb.String() +} + +// parseAPICFrame 解析 ID3v2 APIC 帧,返回图片字节和 MIME 类型 +// APIC 格式:编码字节(1) + MIME字符串(null结尾) + 图片类型(1) + 描述字符串(null结尾) + 图片数据 +func parseAPICFrame(data []byte) ([]byte, string) { + if len(data) < 4 { + return nil, "" + } + // encoding := data[0] // 文本编码(只影响描述字段,MIME 始终是 ASCII) + pos := 1 + + // 读取 MIME 类型(null 结尾的 ASCII 字符串) + nullIdx := -1 + for i := pos; i < len(data); i++ { + if data[i] == 0 { + nullIdx = i + break + } + } + if nullIdx < 0 { + return nil, "" + } + mimeType := string(data[pos:nullIdx]) + pos = nullIdx + 1 + + if pos >= len(data) { + return nil, "" + } + // 图片类型(1字节,3 = Cover (front),但我们取任意第一张) + pos++ // 跳过图片类型字节 + + // 跳过描述字符串(null 结尾,编码由第一字节决定) + encoding := data[0] + if encoding == 0x01 || encoding == 0x02 { + // UTF-16:找 \x00\x00 结尾 + for pos+1 < len(data) { + if data[pos] == 0 && data[pos+1] == 0 { + pos += 2 + break + } + pos += 2 + } + } else { + // ISO-8859-1 / UTF-8:找单个 \x00 结尾 + for pos < len(data) { + if data[pos] == 0 { + pos++ + break + } + pos++ + } + } + + if pos >= len(data) { + return nil, "" + } + + imgData := make([]byte, len(data)-pos) + copy(imgData, data[pos:]) + // 声明的 MIME 来自被扫描文件本身,不可信(可能是 image/svg+xml)。 + // 只按魔数判定并限定为四种栅格格式,无法识别则丢弃封面。 + _ = mimeType + safe := imgmime.Sniff(imgData) + if safe == "" { + return nil, "" + } + return imgData, safe +} + +// syncsafeToInt 将 syncsafe integer 转换为普通整数 +func syncsafeToInt(b []byte) int { + result := 0 + for _, v := range b { + result = (result << 7) | int(v&0x7F) + } + return result +} + +// decodeID3Text 解码 ID3 文本帧(第一字节是编码标识) +func decodeID3Text(data []byte) string { + if len(data) == 0 { + return "" + } + encoding := data[0] + content := data[1:] + + // 去掉末尾的 null 字节 + switch encoding { + case 0x00: // ISO-8859-1 + // 去掉末尾 null + content = trimNull(content, false) + // 尝试 UTF-8 解码,如果失败则按 Latin-1 处理 + return latin1ToUTF8(content) + case 0x01: // UTF-16 with BOM + content = trimNull(content, true) + return utf16ToUTF8(content) + case 0x02: // UTF-16 BE without BOM + content = trimNull(content, true) + return utf16BEToUTF8(content) + case 0x03: // UTF-8 + content = trimNull(content, false) + return string(content) + default: + return string(content) + } +} + +// trimNull 去掉末尾的 null 字节 +func trimNull(b []byte, wide bool) []byte { + if wide { + // UTF-16: 去掉末尾的 \x00\x00 + for len(b) >= 2 && b[len(b)-2] == 0 && b[len(b)-1] == 0 { + b = b[:len(b)-2] + } + } else { + for len(b) > 0 && b[len(b)-1] == 0 { + b = b[:len(b)-1] + } + } + return b +} + +// latin1ToUTF8 将 Latin-1 编码转换为 UTF-8 +func latin1ToUTF8(b []byte) string { + runes := make([]rune, len(b)) + for i, v := range b { + runes[i] = rune(v) + } + return string(runes) +} + +// utf16ToUTF8 将 UTF-16(带 BOM)转换为 UTF-8 +func utf16ToUTF8(b []byte) string { + if len(b) < 2 { + return "" + } + var bigEndian bool + if b[0] == 0xFF && b[1] == 0xFE { + bigEndian = false + b = b[2:] + } else if b[0] == 0xFE && b[1] == 0xFF { + bigEndian = true + b = b[2:] + } + return decodeUTF16(b, bigEndian) +} + +// utf16BEToUTF8 将 UTF-16 BE(无 BOM)转换为 UTF-8 +func utf16BEToUTF8(b []byte) string { + return decodeUTF16(b, true) +} + +// decodeUTF16 解码 UTF-16 字节序列 +func decodeUTF16(b []byte, bigEndian bool) string { + if len(b)%2 != 0 { + b = b[:len(b)-1] + } + runes := make([]rune, 0, len(b)/2) + for i := 0; i+1 < len(b); i += 2 { + var code uint16 + if bigEndian { + code = uint16(b[i])<<8 | uint16(b[i+1]) + } else { + code = uint16(b[i+1])<<8 | uint16(b[i]) + } + runes = append(runes, rune(code)) + } + return string(runes) +} + +// parseID3Genre 解析 ID3 流派字段(可能是 "(17)" 格式的数字引用) +func parseID3Genre(s string) string { + s = strings.TrimSpace(s) + if len(s) > 2 && s[0] == '(' && s[len(s)-1] == ')' { + // 数字引用格式,直接返回原始字符串 + return s + } + return s +} + +// ─── FLAC Vorbis Comment 解析 ──────────────────────────────────────────────── + +// ParseFLACVorbisComment 从 io.Reader 中解析 FLAC 文件的 Vorbis Comment 元数据 +// FLAC 格式:4字节魔数 "fLaC" + 若干 METADATA_BLOCK +// METADATA_BLOCK:1字节(最高位=是否最后块, 低7位=块类型) + 3字节长度 + 数据 +// 块类型 4 = VORBIS_COMMENT +func ParseFLACVorbisComment(r io.Reader) (*MusicTag, error) { + // 读取魔数(4字节) + magic := make([]byte, 4) + if _, err := io.ReadFull(r, magic); err != nil { + return nil, err + } + if string(magic) != "fLaC" { + return nil, nil // 不是 FLAC 文件 + } + + // 遍历 METADATA_BLOCK,找到 VORBIS_COMMENT(类型4)和 PICTURE(类型6) + var tag *MusicTag + for { + // 读取块头(4字节:1字节标志+类型 + 3字节长度) + blockHeader := make([]byte, 4) + if _, err := io.ReadFull(r, blockHeader); err != nil { + return nil, nil // 读取失败,静默跳过 + } + + isLast := blockHeader[0]&0x80 != 0 + blockType := blockHeader[0] & 0x7F + blockLen := int(blockHeader[1])<<16 | int(blockHeader[2])<<8 | int(blockHeader[3]) + + if blockLen < 0 || blockLen > 16*1024*1024 { // 最大 16MB + return nil, nil + } + + if blockType == 4 { + // VORBIS_COMMENT 块 + data := make([]byte, blockLen) + if _, err := io.ReadFull(r, data); err != nil { + return nil, nil + } + tag = parseVorbisCommentData(data) + if isLast { + return tag, nil + } + // 继续读取后续块,寻找 PICTURE 块(类型6) + continue + } + + if blockType == 6 { + // PICTURE 块(FLAC 内嵌封面) + data := make([]byte, blockLen) + if _, err := io.ReadFull(r, data); err == nil && tag != nil && tag.CoverData == nil { + tag.CoverData, tag.CoverMIME = parseFLACPictureBlock(data) + } else if err != nil { + // 读取失败,跳过 + _ = err + } + if isLast { + return tag, nil + } + continue + } + + // 跳过此块 + if _, err := io.CopyN(io.Discard, r, int64(blockLen)); err != nil { + return nil, nil + } + + if isLast { + break + } + } + + return tag, nil +} + +// parseFLACPictureBlock 解析 FLAC PICTURE 元数据块,返回图片字节和 MIME 类型 +// 格式(大端序): +// 4字节 picture_type +// 4字节 mime_length + mime_string +// 4字节 description_length + description_string +// 4字节 width, 4字节 height, 4字节 color_depth, 4字节 color_count +// 4字节 data_length + data +func parseFLACPictureBlock(data []byte) ([]byte, string) { + if len(data) < 8 { + return nil, "" + } + pos := 4 // 跳过 picture_type + + // 读取 MIME 类型 + if pos+4 > len(data) { + return nil, "" + } + mimeLen := int(binary.BigEndian.Uint32(data[pos : pos+4])) + pos += 4 + if pos+mimeLen > len(data) { + return nil, "" + } + mimeType := string(data[pos : pos+mimeLen]) + pos += mimeLen + + // 跳过描述字符串 + if pos+4 > len(data) { + return nil, "" + } + descLen := int(binary.BigEndian.Uint32(data[pos : pos+4])) + pos += 4 + descLen + + // 跳过 width(4) + height(4) + color_depth(4) + color_count(4) + pos += 16 + + // 读取图片数据 + if pos+4 > len(data) { + return nil, "" + } + dataLen := int(binary.BigEndian.Uint32(data[pos : pos+4])) + pos += 4 + if dataLen <= 0 || dataLen > maxEmbeddedPictureBytes || pos+dataLen > len(data) { + return nil, "" + } + + imgData := make([]byte, dataLen) + copy(imgData, data[pos:pos+dataLen]) + // 同 parseAPICFrame:不信任声明的 MIME,只按魔数判定 + _ = mimeType + safe := imgmime.Sniff(imgData) + if safe == "" { + return nil, "" + } + return imgData, safe +} + +// parseVorbisCommentData 解析 Vorbis Comment 数据块 +// 格式(小端序): +// 4字节 vendor_length + vendor_string +// 4字节 comment_count +// 每条注释:4字节长度 + UTF-8字符串(格式 "KEY=VALUE") +func parseVorbisCommentData(data []byte) *MusicTag { + tag := &MusicTag{} + pos := 0 + + // 跳过 vendor string + if pos+4 > len(data) { + return tag + } + vendorLen := int(binary.LittleEndian.Uint32(data[pos : pos+4])) + pos += 4 + vendorLen + + // 读取注释数量 + if pos+4 > len(data) { + return tag + } + commentCount := int(binary.LittleEndian.Uint32(data[pos : pos+4])) + pos += 4 + + for i := 0; i < commentCount; i++ { + if pos+4 > len(data) { + break + } + commentLen := int(binary.LittleEndian.Uint32(data[pos : pos+4])) + pos += 4 + + if commentLen <= 0 || pos+commentLen > len(data) { + break + } + + comment := string(data[pos : pos+commentLen]) + pos += commentLen + + // 解析 "KEY=VALUE" 格式(大小写不敏感) + eqIdx := strings.IndexByte(comment, '=') + if eqIdx < 0 { + continue + } + key := strings.ToUpper(comment[:eqIdx]) + value := strings.TrimSpace(comment[eqIdx+1:]) + if value == "" { + continue + } + + switch key { + case "TITLE": + tag.Title = value + case "ARTIST": + if tag.Artist == "" { + tag.Artist = value + } + case "ALBUM": + tag.Album = value + case "ALBUMARTIST", "ALBUM ARTIST": + tag.AlbumArtist = value + case "TRACKNUMBER", "TRACK": + // 格式可能是 "1" 或 "1/12" + parts := strings.SplitN(value, "/", 2) + if n, err := strconv.Atoi(strings.TrimSpace(parts[0])); err == nil { + tag.TrackNumber = n + } + case "DATE", "YEAR": + if tag.Year == "" && len(value) >= 4 { + tag.Year = value[:4] + } + case "GENRE": + tag.Genre = value + case "LYRICS", "UNSYNCEDLYRICS", "UNSYNCED LYRICS", "SYNCEDLYRICS", "SYNCED LYRICS": + // FLAC/Vorbis 内嵌歌词字段(不同打标软件使用不同 key,全部兼容) + if tag.Lyrics == "" { + tag.Lyrics = value + } + } + } + + return tag +} diff --git a/internal/media/imgmime/imgmime.go b/internal/media/imgmime/imgmime.go new file mode 100644 index 00000000..a2534b84 --- /dev/null +++ b/internal/media/imgmime/imgmime.go @@ -0,0 +1,36 @@ +// Package imgmime 提供封面图片 MIME 的安全判定。 +// +// 封面字节与 MIME 往往来自不可信来源(音频文件内嵌 APIC/PICTURE、远端站点的 +// Content-Type)。如果把声明的 MIME 原样写进响应头,就可能把 image/svg+xml 或 +// text/html 以站点同源方式交付给浏览器,造成存储型 XSS。 +// 这里只信任文件头魔数,且只允许四种栅格格式。 +package imgmime + +import "bytes" + +// Sniff 根据文件头魔数返回 image/jpeg、image/png、image/gif、image/webp 之一; +// 无法识别(包括 SVG、HTML、未知格式)时返回空串,调用方应丢弃该封面。 +func Sniff(data []byte) string { + switch { + case len(data) >= 3 && data[0] == 0xFF && data[1] == 0xD8 && data[2] == 0xFF: + return "image/jpeg" + case len(data) >= 8 && bytes.Equal(data[:8], []byte{0x89, 'P', 'N', 'G', 0x0D, 0x0A, 0x1A, 0x0A}): + return "image/png" + case len(data) >= 6 && (bytes.Equal(data[:6], []byte("GIF87a")) || bytes.Equal(data[:6], []byte("GIF89a"))): + return "image/gif" + case len(data) >= 12 && bytes.Equal(data[:4], []byte("RIFF")) && bytes.Equal(data[8:12], []byte("WEBP")): + return "image/webp" + default: + return "" + } +} + +// Allowed 判断 MIME 是否属于允许对外输出的栅格图片类型。 +func Allowed(mime string) bool { + switch mime { + case "image/jpeg", "image/png", "image/gif", "image/webp": + return true + default: + return false + } +} diff --git a/internal/media/imgmime/imgmime_test.go b/internal/media/imgmime/imgmime_test.go new file mode 100644 index 00000000..70b36d02 --- /dev/null +++ b/internal/media/imgmime/imgmime_test.go @@ -0,0 +1,41 @@ +package imgmime + +import "testing" + +func TestSniff(t *testing.T) { + cases := []struct { + name string + data []byte + want string + }{ + {"jpeg", []byte{0xFF, 0xD8, 0xFF, 0xE0, 0, 0}, "image/jpeg"}, + {"png", []byte{0x89, 'P', 'N', 'G', 0x0D, 0x0A, 0x1A, 0x0A, 0}, "image/png"}, + {"gif89", []byte("GIF89a....."), "image/gif"}, + {"webp", append(append([]byte("RIFF\x00\x00\x00\x00"), []byte("WEBP")...), 0), "image/webp"}, + {"svg", []byte(``), ""}, + {"svg with xml prolog", []byte(``), ""}, + {"html", []byte(""), ""}, + {"empty", nil, ""}, + {"short", []byte{0xFF, 0xD8}, ""}, + // RIFF 但不是 WEBP(例如 WAV)不能被当作图片 + {"riff wav", append(append([]byte("RIFF\x00\x00\x00\x00"), []byte("WAVE")...), 0), ""}, + } + for _, tc := range cases { + if got := Sniff(tc.data); got != tc.want { + t.Errorf("%s: Sniff = %q, want %q", tc.name, got, tc.want) + } + } +} + +func TestAllowed(t *testing.T) { + for _, m := range []string{"image/jpeg", "image/png", "image/gif", "image/webp"} { + if !Allowed(m) { + t.Errorf("%s should be allowed", m) + } + } + for _, m := range []string{"image/svg+xml", "text/html", "", "image/"} { + if Allowed(m) { + t.Errorf("%q should not be allowed", m) + } + } +} diff --git a/internal/media/local_fill.go b/internal/media/local_fill.go new file mode 100644 index 00000000..9c9a4737 --- /dev/null +++ b/internal/media/local_fill.go @@ -0,0 +1,130 @@ +package media + +import ( + "context" + "encoding/base64" + "encoding/json" + stdpath "path" + "strings" + "time" + + "github.com/OpenListTeam/OpenList/v4/internal/fs" + "github.com/OpenListTeam/OpenList/v4/internal/model" +) + +// FillMusicFromLocal 在刮削前/失败时,使用音乐文件本身的元数据(ID3 / Vorbis Comment) +// 以及同目录 .lrc 歌词,**只填充 item 中为空的字段**。 +// +// 适用场景: +// 1. 用户清空刮削后重新刮削,原本由扫描阶段填入的本地数据(封面/歌词等)已丢失 +// 2. 在线刮削失败,退而求其次用文件本地信息补全 +// +// 行为: +// - is_folder=true 的合并文件夹模式:使用 episodes 中第一首音乐文件读取 +// - is_folder=false:使用 folder_path/file_name 直接读取 +// +// 错误以静默忽略为主——本地兜底失败不应阻塞主流程。 +func FillMusicFromLocal(ctx context.Context, item *model.MediaItem) { + if item == nil || item.MediaType != model.MediaTypeMusic { + return + } + + musicPath := pickMusicFilePath(item) + if musicPath == "" { + return + } + + // 解析 tag(mp3 / flac / 其它) + ext := strings.ToLower(stdpath.Ext(musicPath)) + var tag *MusicTag + readCtx, readCancel := context.WithTimeout(ctx, 15*time.Second) + if reader := FetchFileReader(readCtx, musicPath); reader != nil { + switch ext { + case ".flac": + tag, _ = ParseFLACVorbisComment(reader) + default: + tag, _ = ParseID3v2(reader) + } + _ = reader.Close() + } + readCancel() + + // 应用 tag 到空字段 + if tag != nil { + if item.AlbumName == "" && tag.Album != "" { + item.AlbumName = tag.Album + } + if item.AlbumArtist == "" { + if tag.AlbumArtist != "" { + item.AlbumArtist = tag.AlbumArtist + } else if tag.Artist != "" { + item.AlbumArtist = tag.Artist + } + } + if item.ScrapedName == "" { + if tag.Title != "" { + item.ScrapedName = tag.Title + } else if item.AlbumName != "" { + item.ScrapedName = item.AlbumName + } + } + if item.TrackNumber == 0 && tag.TrackNumber > 0 { + item.TrackNumber = tag.TrackNumber + } + if item.ReleaseDate == "" && len(tag.Year) >= 4 { + item.ReleaseDate = tag.Year[:4] + "-01-01" + } + if item.Genre == "" && tag.Genre != "" { + item.Genre = tag.Genre + } + if item.Authors == "" && tag.Artist != "" { + if b, err := json.Marshal([]string{tag.Artist}); err == nil { + item.Authors = string(b) + } + } + if item.Cover == "" && len(tag.CoverData) > 0 { + mimeType := tag.CoverMIME + if mimeType == "" { + mimeType = "image/jpeg" + } + item.Cover = "data:" + mimeType + ";base64," + base64.StdEncoding.EncodeToString(tag.CoverData) + } + } + + // 歌词:若为空则尝试 .lrc 同名文件 / 内嵌歌词 + if item.Lyrics == "" { + if lrc := loadLyricsForMusic(ctx, musicPath, tag); lrc != "" { + item.Lyrics = lrc + } + } +} + +// pickMusicFilePath 从 MediaItem 推断出可读取的音乐文件 VFS 路径 +// 合并模式 (is_folder=true): 取 episodes 中第一首;解析失败回退到 folder_path/file_name +// 普通模式: folder_path/file_name +func pickMusicFilePath(item *model.MediaItem) string { + if item == nil { + return "" + } + if item.IsFolder { + // episodes 是 JSON 数组:[{file_name,index,title,...}] + if item.Episodes != "" { + var eps []EpisodeInfo + if err := json.Unmarshal([]byte(item.Episodes), &eps); err == nil && len(eps) > 0 { + // 文件夹模式下 folder_path 是扫描根,file_name 是文件夹名 + return stdpath.Join(item.FolderPath, item.FileName, eps[0].FileName) + } + } + // 回退:尝试列出文件夹内第一首音乐 + folder := stdpath.Join(item.FolderPath, item.FileName) + if entries, err := fs.List(context.Background(), folder, &fs.ListArgs{NoLog: true}); err == nil { + for _, e := range entries { + if !e.IsDir() && isMediaFile(e.GetName(), model.MediaTypeMusic) { + return stdpath.Join(folder, e.GetName()) + } + } + } + return "" + } + return stdpath.Join(item.FolderPath, item.FileName) +} diff --git a/internal/media/scanner.go b/internal/media/scanner.go new file mode 100644 index 00000000..2baa7a95 --- /dev/null +++ b/internal/media/scanner.go @@ -0,0 +1,758 @@ +package media + +import ( + "context" + "encoding/base64" + "encoding/json" + "fmt" + "io" + "mime" + stdpath "path" + "regexp" + "strings" + "sync" + "time" + "unicode" + + log "github.com/sirupsen/logrus" + + "github.com/OpenListTeam/OpenList/v4/internal/db" + "github.com/OpenListTeam/OpenList/v4/internal/fs" + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/internal/stream" + "github.com/OpenListTeam/OpenList/v4/pkg/http_range" + "github.com/OpenListTeam/OpenList/v4/pkg/utils" +) + +// 支持的文件扩展名 +var ( + videoExts = map[string]bool{ + ".mp4": true, ".mkv": true, ".avi": true, ".mov": true, + ".wmv": true, ".flv": true, ".webm": true, ".m4v": true, + ".ts": true, ".rmvb": true, ".rm": true, ".3gp": true, + } + musicExts = map[string]bool{ + ".mp3": true, ".flac": true, ".aac": true, ".ogg": true, + ".wav": true, ".wma": true, ".m4a": true, ".ape": true, + ".opus": true, ".aiff": true, + } + imageExts = map[string]bool{ + ".jpg": true, ".jpeg": true, ".png": true, ".gif": true, + ".webp": true, ".bmp": true, ".tiff": true, ".svg": true, + ".heic": true, ".avif": true, + } + bookExts = map[string]bool{ + ".epub": true, ".pdf": true, ".mobi": true, ".azw3": true, + ".txt": true, ".djvu": true, ".cbz": true, ".cbr": true, + } +) + +// ScanProgress 扫描进度(全局,按媒体类型维护) +type ScanProgress struct { + mu sync.RWMutex + Running bool + Total int + Done int + // Skipped 入库被跳过的条目数(典型原因:目录/文件名超过唯一索引列长度上限)。 + // 以前只在日志里打一条 Warn,用户完全看不到漏扫了多少条。 + Skipped int + Message string + Error string +} + +var progressMap = map[model.MediaType]*ScanProgress{ + model.MediaTypeVideo: {}, + model.MediaTypeMusic: {}, + model.MediaTypeImage: {}, + model.MediaTypeBook: {}, +} + +// GetProgress 获取扫描进度 +func GetProgress(mediaType model.MediaType) model.MediaScanProgress { + p, ok := progressMap[mediaType] + if !ok { + return model.MediaScanProgress{MediaType: mediaType} + } + p.mu.RLock() + defer p.mu.RUnlock() + return model.MediaScanProgress{ + MediaType: mediaType, + Running: p.Running, + Total: p.Total, + Done: p.Done, + Message: p.Message, + Error: p.Error, + } +} + +// ScanMedia 扫描媒体文件(异步),扫描指定媒体类型下的所有启用扫描路径 +func ScanMedia(cfg *model.MediaConfig) { + p, ok := progressMap[cfg.MediaType] + if !ok { + return + } + p.mu.Lock() + if p.Running { + p.mu.Unlock() + return + } + p.Running = true + p.Total = 0 + p.Done = 0 + p.Skipped = 0 + p.Error = "" + p.Message = "正在扫描..." + p.mu.Unlock() + + go func() { + defer func() { + p.mu.Lock() + p.Running = false + p.mu.Unlock() + }() + + // 获取该媒体类型下所有扫描路径 + scanPaths, err := db.ListMediaScanPaths(cfg.MediaType) + if err != nil { + p.mu.Lock() + p.Error = err.Error() + p.Message = "获取扫描路径失败" + p.mu.Unlock() + return + } + if len(scanPaths) == 0 { + p.mu.Lock() + p.Message = "没有配置扫描路径" + p.mu.Unlock() + return + } + + for i := range scanPaths { + sp := &scanPaths[i] + p.mu.Lock() + p.Message = "正在扫描路径: " + sp.Name + p.mu.Unlock() + + if err := doScanPath(sp, p); err != nil { + log.Errorf("media scan error [%s] path[%s]: %v", cfg.MediaType, sp.Path, err) + } else { + // 更新最后扫描时间 + now := time.Now() + sp.LastScanAt = &now + _ = db.UpdateMediaScanPath(sp) + } + } + + p.mu.Lock() + if p.Skipped > 0 { + // 让管理端能直接看到「有 N 条因为路径过长没入库」 + p.Message = fmt.Sprintf("扫描完成(跳过 %d 条)", p.Skipped) + log.Warnf("media scan skipped %d items (path/name too long for unique index)", p.Skipped) + } else { + p.Message = "扫描完成" + } + p.mu.Unlock() + // 更新媒体库最后扫描时间 + now := time.Now() + cfg.LastScanAt = &now + _ = db.SaveMediaConfig(cfg) + + // 按需生成 STRM + TriggerStrmOnScan(cfg.MediaType, GetAPIRoot()) + }() +} + +// ScanMediaPath 扫描单个扫描路径(异步) +func ScanMediaPath(sp *model.MediaScanPath) { + p, ok := progressMap[sp.MediaType] + if !ok { + return + } + p.mu.Lock() + if p.Running { + p.mu.Unlock() + return + } + p.Running = true + p.Total = 0 + p.Done = 0 + p.Skipped = 0 + p.Error = "" + p.Message = "正在扫描路径: " + sp.Name + p.mu.Unlock() + + go func() { + defer func() { + p.mu.Lock() + p.Running = false + p.mu.Unlock() + }() + + if err := doScanPath(sp, p); err != nil { + p.mu.Lock() + p.Error = err.Error() + p.Message = "扫描失败" + p.mu.Unlock() + log.Errorf("media scan error [%s] path[%s]: %v", sp.MediaType, sp.Path, err) + } else { + p.mu.Lock() + if p.Skipped > 0 { + p.Message = fmt.Sprintf("扫描完成(跳过 %d 条)", p.Skipped) + log.Warnf("media scan skipped %d items (path/name too long for unique index)", p.Skipped) + } else { + p.Message = "扫描完成" + } + p.mu.Unlock() + now := time.Now() + sp.LastScanAt = &now + _ = db.UpdateMediaScanPath(sp) + } + }() +} + +func doScanPath(sp *model.MediaScanPath, p *ScanProgress) error { + scanRoot := sp.Path + if scanRoot == "" { + scanRoot = "/" + } + + // 收集所有待处理路径(VFS 虚拟路径) + var targets []string + + ctx := context.Background() + + if sp.PathMerge { + // 路径合并模式: + // - 子文件夹 → 作为一个合并条目(带 Episodes 选集信息) + // - 直接放在根目录下的单个媒体文件 → 正常作为独立条目扫描 + entries, err := fs.List(ctx, scanRoot, &fs.ListArgs{NoLog: true, Refresh: true}) + if err != nil { + return err + } + for _, e := range entries { + childPath := stdpath.Join(scanRoot, e.GetName()) + if e.IsDir() { + targets = append(targets, childPath) + } else if isMediaFile(e.GetName(), sp.MediaType) { + targets = append(targets, childPath) + } + } + } else { + // 普通模式:递归扫描所有匹配文件(每个目录都刷新缓存) + if err := walkVFS(ctx, scanRoot, sp.MediaType, &targets); err != nil { + return err + } + } + + p.mu.Lock() + p.Total += len(targets) + p.mu.Unlock() + + // 音乐类型:按专辑合并处理 + if sp.MediaType == model.MediaTypeMusic { + return doScanMusicByAlbum(ctx, targets, sp, p) + } + + for _, target := range targets { + item, err := buildMediaItemFromVFS(ctx, target, sp) + if err != nil { + log.Warnf("build media item error [%s]: %v", target, err) + continue + } + + // 路径合并模式下,扫描文件夹内的文件,填充选集信息 + if sp.PathMerge && item.IsFolder { + if episodes, err := buildEpisodesFromFolder(ctx, target, sp.MediaType); err == nil { + item.Episodes = episodes + } else { + log.Warnf("build episodes error [%s]: %v", target, err) + } + } + + if err := db.CreateOrUpdateMediaItem(item); err != nil { + log.Warnf("save media item error [%s]: %v", target, err) + p.mu.Lock() + p.Skipped++ + p.mu.Unlock() + } + p.mu.Lock() + p.Done++ + p.Message = "已扫描: " + stdpath.Base(target) + p.mu.Unlock() + } + return nil +} + +// doScanMusicByAlbum 音乐扫描 +// 存储规则: +// - 普通文件(is_folder=false): +// folder_path=文件所在真实目录,file_name=音乐文件名,episodes=空 +// - 合并文件夹(is_folder=true): +// folder_path=文件夹所在真实父目录,file_name=文件夹名, +// episodes=文件夹内所有音乐文件列表,封面/标签取第一首 +func doScanMusicByAlbum(ctx context.Context, targets []string, sp *model.MediaScanPath, p *ScanProgress) error { + for _, target := range targets { + obj, err := fs.Get(ctx, target, &fs.GetArgs{NoLog: true}) + if err != nil { + log.Warnf("get vfs object error [%s]: %v", target, err) + p.mu.Lock() + p.Done++ + p.mu.Unlock() + continue + } + + name := obj.GetName() + + if obj.IsDir() { + // ---- 合并文件夹模式:文件夹条目 ---- + // 列出文件夹内所有音乐文件 + entries, err := fs.List(ctx, target, &fs.ListArgs{NoLog: true}) + if err != nil { + log.Warnf("list music folder error [%s]: %v", target, err) + p.mu.Lock() + p.Done++ + p.mu.Unlock() + continue + } + + var musicFiles []string + var episodes []EpisodeInfo + for idx, e := range entries { + if e.IsDir() || !isMediaFile(e.GetName(), sp.MediaType) { + continue + } + musicFiles = append(musicFiles, stdpath.Join(target, e.GetName())) + ep := EpisodeInfo{ + FileName: e.GetName(), + Index: idx + 1, + Title: strings.TrimSuffix(e.GetName(), stdpath.Ext(e.GetName())), + } + episodes = append(episodes, ep) + } + + if len(musicFiles) == 0 { + p.mu.Lock() + p.Done++ + p.mu.Unlock() + continue + } + + // 取第一首音乐文件读取标签 + firstFile := musicFiles[0] + firstExt := strings.ToLower(stdpath.Ext(firstFile)) + var tag *MusicTag + readCtx, readCancel := context.WithTimeout(ctx, 15*time.Second) + if reader := FetchFileReader(readCtx, firstFile); reader != nil { + switch firstExt { + case ".flac": + tag, _ = ParseFLACVorbisComment(reader) + default: + tag, _ = ParseID3v2(reader) + } + _ = reader.Close() + } + readCancel() + + // 解析标签(用文件夹名作为默认专辑名) + albumName := name + albumArtist := "" + cover := "" + releaseDate := "" + genre := "" + authors := "" + if tag != nil { + if tag.Album != "" { + albumName = tag.Album + } + albumArtist = tag.AlbumArtist + if albumArtist == "" { + albumArtist = tag.Artist + } + if tag.Year != "" && len(tag.Year) >= 4 { + releaseDate = tag.Year[:4] + "-01-01" + } + if tag.Genre != "" { + genre = tag.Genre + } + if tag.Artist != "" { + if authorsJSON, err := json.Marshal([]string{tag.Artist}); err == nil { + authors = string(authorsJSON) + } + } + if len(tag.CoverData) > 0 { + mimeType := tag.CoverMIME + if mimeType == "" { + mimeType = "image/jpeg" + } + cover = "data:" + mimeType + ";base64," + base64.StdEncoding.EncodeToString(tag.CoverData) + } + } + + episodesJSON := "" + if b, err := json.Marshal(episodes); err == nil { + episodesJSON = string(b) + } + + // 加载歌词:同目录同名 .lrc 优先,其次内嵌歌词 + lyrics := loadLyricsForMusic(ctx, firstFile, tag) + + item := &model.MediaItem{ + MediaType: sp.MediaType, + ScanPathID: sp.ID, + FileName: name, // 文件夹名 + FolderPath: stdpath.Dir(target), // 文件夹所在的真实父目录 + IsFolder: true, + AlbumName: albumName, + AlbumArtist: albumArtist, + ScrapedName: albumName, + Cover: cover, + ReleaseDate: releaseDate, + Genre: genre, + Authors: authors, + Lyrics: lyrics, + Episodes: episodesJSON, + } + + if err := db.CreateOrUpdateMediaItem(item); err != nil { + log.Warnf("save music folder item error [%s]: %v", target, err) + } + } else { + // ---- 普通文件:每首歌独立一条记录 ---- + folderPath := stdpath.Dir(target) + ext := strings.ToLower(stdpath.Ext(name)) + + var tag *MusicTag + readCtx, readCancel := context.WithTimeout(ctx, 15*time.Second) + if reader := FetchFileReader(readCtx, target); reader != nil { + switch ext { + case ".flac": + tag, _ = ParseFLACVorbisComment(reader) + default: + tag, _ = ParseID3v2(reader) + } + _ = reader.Close() + } + readCancel() + + albumName := "" + albumArtist := "" + trackTitle := strings.TrimSuffix(name, stdpath.Ext(name)) + trackNumber := 0 + cover := "" + releaseDate := "" + genre := "" + authors := "" + + if tag != nil { + albumName = tag.Album + albumArtist = tag.AlbumArtist + if albumArtist == "" { + albumArtist = tag.Artist + } + if tag.Title != "" { + trackTitle = tag.Title + } + trackNumber = tag.TrackNumber + if tag.Year != "" && len(tag.Year) >= 4 { + releaseDate = tag.Year[:4] + "-01-01" + } + if tag.Genre != "" { + genre = tag.Genre + } + if tag.Artist != "" { + if authorsJSON, err := json.Marshal([]string{tag.Artist}); err == nil { + authors = string(authorsJSON) + } + } + if len(tag.CoverData) > 0 { + mimeType := tag.CoverMIME + if mimeType == "" { + mimeType = "image/jpeg" + } + cover = "data:" + mimeType + ";base64," + base64.StdEncoding.EncodeToString(tag.CoverData) + } + } + + // 加载歌词:同目录同名 .lrc 优先,其次内嵌歌词 + lyrics := loadLyricsForMusic(ctx, target, tag) + + item := &model.MediaItem{ + MediaType: sp.MediaType, + ScanPathID: sp.ID, + FileName: name, + FolderPath: folderPath, + IsFolder: false, + FileSize: obj.GetSize(), + MimeType: mime.TypeByExtension(ext), + AlbumName: albumName, + AlbumArtist: albumArtist, + ScrapedName: trackTitle, + Cover: cover, + ReleaseDate: releaseDate, + Genre: genre, + Authors: authors, + TrackNumber: trackNumber, + Lyrics: lyrics, + } + + if err := db.CreateOrUpdateMediaItem(item); err != nil { + log.Warnf("save music item error [%s]: %v", target, err) + p.mu.Lock() + p.Skipped++ + p.mu.Unlock() + } + } + + p.mu.Lock() + p.Done++ + p.Message = "已扫描: " + name + p.mu.Unlock() + } + return nil +} + +// loadLyricsForMusic 为音乐文件加载歌词: +// 1. 优先读取同目录同名 .lrc 文件(如 song.mp3 → song.lrc) +// 2. 如果不存在,使用 tag 中的内嵌歌词(USLT / Vorbis LYRICS) +// +// musicVfsPath 为音乐文件的 VFS 全路径;tag 可为 nil +func loadLyricsForMusic(ctx context.Context, musicVfsPath string, tag *MusicTag) string { + // 尝试同目录同名 .lrc + ext := stdpath.Ext(musicVfsPath) + if ext != "" { + lrcPath := strings.TrimSuffix(musicVfsPath, ext) + ".lrc" + lrcCtx, lrcCancel := context.WithTimeout(ctx, 8*time.Second) + reader := FetchFileReader(lrcCtx, lrcPath) + if reader != nil { + // 限制读取上限 1MB,避免异常文件压垮内存 + data, err := io.ReadAll(io.LimitReader(reader, 1024*1024)) + _ = reader.Close() + lrcCancel() + if err == nil && len(data) > 0 { + // 去掉可能的 UTF-8 BOM + if len(data) >= 3 && data[0] == 0xEF && data[1] == 0xBB && data[2] == 0xBF { + data = data[3:] + } + s := strings.TrimSpace(string(data)) + if s != "" { + return s + } + } + } else { + lrcCancel() + } + } + // 回退到内嵌歌词 + if tag != nil && tag.Lyrics != "" { + return tag.Lyrics + } + return "" +} + +// FetchFileReader 通过 VFS 路径获取文件内容流(用于刮削器读取文件内容) +// 优先使用 RangeReader 直接读取(本地存储无需 HTTP),失败时回退到 HTTP URL +// 返回 nil 表示无法获取(不影响主流程) +func FetchFileReader(ctx context.Context, vfsPath string) io.ReadCloser { + link, file, err := fs.Link(ctx, vfsPath, model.LinkArgs{}) + if err != nil || link == nil { + return nil + } + size := link.ContentLength + if size <= 0 { + size = file.GetSize() + } + rr, err := stream.GetRangeReaderFromLink(size, link) + if err != nil { + _ = link.Close() + return nil + } + rc, err := rr.RangeRead(ctx, http_range.Range{Length: size}) + if err != nil { + _ = link.Close() + return nil + } + link.SyncClosers.Add(rc) + return utils.ReadCloser{Reader: rc, Closer: link} +} + +// walkVFS 递归遍历 VFS 路径,收集匹配的媒体文件路径(每个目录都刷新缓存) +func walkVFS(ctx context.Context, dirPath string, mediaType model.MediaType, targets *[]string) error { + entries, err := fs.List(ctx, dirPath, &fs.ListArgs{NoLog: true, Refresh: true}) + if err != nil { + log.Warnf("media scan: list vfs path [%s] error: %v", dirPath, err) + return nil // 跳过无权限目录,不中断整体扫描 + } + for _, e := range entries { + childPath := stdpath.Join(dirPath, e.GetName()) + if e.IsDir() { + if err := walkVFS(ctx, childPath, mediaType, targets); err != nil { + return err + } + } else if isMediaFile(e.GetName(), mediaType) { + *targets = append(*targets, childPath) + } + } + return nil +} + +// buildMediaItemFromVFS 根据 VFS 路径构建 MediaItem(非音乐类型使用) +// 存储规则: +// - folder_path:文件/文件夹所在的真实父目录(VFS 父路径),用于前端按 folder_path + file_name 反查真实文件 +// - file_name:文件夹就是文件夹名,文件就是文件名 +// - episodes:文件夹时存里面每个文件的信息,文件时为空 +func buildMediaItemFromVFS(ctx context.Context, vfsPath string, sp *model.MediaScanPath) (*model.MediaItem, error) { + obj, err := fs.Get(ctx, vfsPath, &fs.GetArgs{NoLog: true}) + if err != nil { + return nil, err + } + + name := obj.GetName() + ext := strings.ToLower(stdpath.Ext(name)) + + item := &model.MediaItem{ + MediaType: sp.MediaType, + ScanPathID: sp.ID, + FileName: name, // 文件夹名 或 文件名 + FolderPath: stdpath.Dir(vfsPath), // 真实父目录 + IsFolder: obj.IsDir(), + ScrapedName: strings.TrimSuffix(name, stdpath.Ext(name)), // 去掉扩展名作为默认名称 + } + + if !obj.IsDir() { + // 普通文件:记录大小和 MIME 类型,episodes 为空 + item.FileSize = obj.GetSize() + item.MimeType = mime.TypeByExtension(ext) + } + + return item, nil +} + +// episodeNumRe 匹配文件名开头的数字序号,支持 "1、" "2." "3-" "4 " 等分隔符 +var episodeNumRe = regexp.MustCompile(`^(\d+)[、.\-\s_]+(.*)`) + +// EpisodeInfo 选集信息(定义在 model 包中,供协议层复用) +type EpisodeInfo = model.EpisodeInfo + +// buildEpisodesFromFolder 扫描文件夹内的媒体文件,构建选集信息 JSON 字符串 +func buildEpisodesFromFolder(ctx context.Context, folderPath string, mediaType model.MediaType) (string, error) { + entries, err := fs.List(ctx, folderPath, &fs.ListArgs{NoLog: true}) + if err != nil { + return "", err + } + + var episodes []EpisodeInfo + for _, e := range entries { + if e.IsDir() { + continue + } + name := e.GetName() + if !isMediaFile(name, mediaType) { + continue + } + + // 去掉扩展名得到裸文件名 + ext := stdpath.Ext(name) + baseName := strings.TrimSuffix(name, ext) + + ep := EpisodeInfo{ + FileName: name, + Index: 0, + Title: baseName, + } + + // 尝试从文件名开头提取数字序号 + if m := episodeNumRe.FindStringSubmatch(baseName); len(m) == 3 { + if idx := parseLeadingInt(m[1]); idx > 0 { + ep.Index = idx + ep.Title = strings.TrimSpace(m[2]) + } + } else { + // 文件名直接以纯数字开头(无分隔符),也尝试提取 + if idx, rest := splitLeadingNumber(baseName); idx > 0 { + ep.Index = idx + ep.Title = strings.TrimSpace(rest) + } + } + + episodes = append(episodes, ep) + } + + if len(episodes) == 0 { + return "", nil + } + + b, err := json.Marshal(episodes) + if err != nil { + return "", err + } + return string(b), nil +} + +// parseLeadingInt 将纯数字字符串解析为 int,失败返回 0 +func parseLeadingInt(s string) int { + v := 0 + for _, c := range s { + if c < '0' || c > '9' { + return 0 + } + v = v*10 + int(c-'0') + } + return v +} + +// splitLeadingNumber 从字符串开头提取连续数字,返回 (数字值, 剩余字符串) +// 仅当开头确实有数字时才返回非零值 +func splitLeadingNumber(s string) (int, string) { + i := 0 + for i < len(s) && s[i] >= '0' && s[i] <= '9' { + i++ + } + if i == 0 || i == len(s) { + // 没有数字,或者整个字符串都是数字(没有标题部分) + return 0, s + } + // 剩余部分必须以非字母数字字符开头,避免把 "1080p" 之类的误识别 + if unicode.IsLetter(rune(s[i])) || unicode.IsDigit(rune(s[i])) { + return 0, s + } + v := parseLeadingInt(s[:i]) + return v, s[i:] +} + +// isMediaFile 判断文件名是否为指定媒体类型(按扩展名判断) +func isMediaFile(name string, mediaType model.MediaType) bool { + ext := strings.ToLower(stdpath.Ext(name)) + switch mediaType { + case model.MediaTypeVideo: + return videoExts[ext] + case model.MediaTypeMusic: + return musicExts[ext] + case model.MediaTypeImage: + return imageExts[ext] + case model.MediaTypeBook: + return bookExts[ext] + } + return false +} + +// GetSupportedExts 获取指定媒体类型支持的扩展名列表 +func GetSupportedExts(mediaType model.MediaType) []string { + var extMap map[string]bool + switch mediaType { + case model.MediaTypeVideo: + extMap = videoExts + case model.MediaTypeMusic: + extMap = musicExts + case model.MediaTypeImage: + extMap = imageExts + case model.MediaTypeBook: + extMap = bookExts + default: + return nil + } + exts := make([]string, 0, len(extMap)) + for ext := range extMap { + exts = append(exts, ext) + } + return exts +} diff --git a/internal/media/scraper/book_local.go b/internal/media/scraper/book_local.go new file mode 100644 index 00000000..ca57580c --- /dev/null +++ b/internal/media/scraper/book_local.go @@ -0,0 +1,389 @@ +package scraper + +import ( + "archive/zip" + "bytes" + "context" + "encoding/base64" + "encoding/xml" + "errors" + "image/jpeg" + "io" + "strings" + "time" + + "github.com/disintegration/imaging" + "github.com/OpenListTeam/OpenList/v4/internal/media/imgmime" + "github.com/OpenListTeam/OpenList/v4/internal/model" +) + +// BookLocalScraper 书籍本地刮削器 +// 尝试从书籍文件中提取封面图片并以 base64 存入 Cover 字段,或保存到本地文件。 +// - EPUB:解压 zip,提取 OPF 声明的封面或 cover.jpg/cover.png +// - PDF:尝试提取内嵌 JPEG 图片流作为封面 +// - 其他/失败:降级为文件路径 +type BookLocalScraper struct { + // ThumbnailMode 缩略图存储方式:"base64"(默认,存入数据库)或 "local"(存到本地文件) + ThumbnailMode string + // ThumbnailPath 本地存储路径,ThumbnailMode 为 "local" 时有效,默认 "/.thumbnail" + ThumbnailPath string +} + +// NewBookLocalScraper 创建书籍本地刮削器 +func NewBookLocalScraper() *BookLocalScraper { + return &BookLocalScraper{ + ThumbnailMode: "base64", + ThumbnailPath: "/.thumbnail", + } +} + +// NewBookLocalScraperWithConfig 创建带完整配置的书籍本地刮削器 +func NewBookLocalScraperWithConfig(thumbnailMode, thumbnailPath string) *BookLocalScraper { + if thumbnailMode == "" { + thumbnailMode = "base64" + } + if thumbnailPath == "" { + thumbnailPath = "/.thumbnail" + } + return &BookLocalScraper{ + ThumbnailMode: thumbnailMode, + ThumbnailPath: thumbnailPath, + } +} + +// ScrapeBookLocal 从书籍文件流中提取封面图片,填充基本信息。 +// reader 为书籍文件内容流,不能为 nil。 +// 返回提取到的封面 base64 字符串(data URI),提取失败返回空字符串。 +// 注意:本函数不会将文件路径作为 cover 降级,cover 为空表示提取失败。 +func (s *BookLocalScraper) ScrapeBookLocal(item *model.MediaItem, reader io.Reader) error { + ext := strings.ToLower(getFileExt(item.FileName)) + + // 根据文件类型补充 MimeType + if item.MimeType == "" { + item.MimeType = mimeTypeByBookExt(ext) + } + // 去掉扩展名作为展示名 + if item.ScrapedName == "" { + item.ScrapedName = trimExt(item.FileName) + } + + // 从文件流提取封面,提取失败则 cover 保持为空 + if reader != nil { + data, err := readAllLimited(reader, MaxBookFileBytes) + if err == nil && len(data) > 0 { + var coverData []byte + switch ext { + case ".epub": + coverData = extractEpubCoverData(data) + case ".pdf": + coverData = extractPDFCoverData(data) + } + if len(coverData) > 0 { + cover := s.storeCover(strings.TrimSuffix(item.FolderPath, "/")+"/"+item.FileName, coverData) + if cover != "" { + item.Cover = cover + } + } + } + } + + now := time.Now() + item.ScrapedAt = &now + return nil +} + +// ExtractLocalCover 从书籍文件流中提取封面并根据配置存储(供外部调用)。 +// 提取失败返回空字符串,不降级为文件路径。 +func (s *BookLocalScraper) ExtractLocalCover(fileName string, filePath string, reader io.Reader) string { + if reader == nil { + return "" + } + ext := strings.ToLower(getFileExt(fileName)) + data, err := readAllLimited(reader, MaxBookFileBytes) + if err != nil || len(data) == 0 { + return "" + } + var coverData []byte + switch ext { + case ".epub": + coverData = extractEpubCoverData(data) + case ".pdf": + coverData = extractPDFCoverData(data) + } + if len(coverData) == 0 { + return "" + } + return s.storeCover(filePath, coverData) +} + +// storeCover 根据 ThumbnailMode 存储封面数据,返回 Cover 字段值 +func (s *BookLocalScraper) storeCover(filePath string, coverData []byte) string { + if NormalizeThumbMode(s.ThumbnailMode) == ThumbModeBase64 { + return imageDataToBase64Thumb(coverData) + } + thumb := resizeCoverBytes(coverData) + if len(thumb) == 0 { + thumb = coverData + } + return SaveThumb(context.Background(), s.ThumbnailMode, s.ThumbnailPath, filePath, "image/jpeg", thumb) +} + +// resizeCoverBytes 将封面缩放为 300px 宽,返回 JPEG 字节 +func resizeCoverBytes(data []byte) []byte { + img, err := decodeBounded(data) + if err != nil { + return nil + } + thumb := imaging.Resize(img, 300, 0, imaging.Lanczos) + var buf bytes.Buffer + if err := jpeg.Encode(&buf, thumb, &jpeg.Options{Quality: 80}); err != nil { + return nil + } + return buf.Bytes() +} + +// ── EPUB 封面提取 ───────────────────────────────────────────────────────────── + +// epubContainer 解析 META-INF/container.xml +type epubContainer struct { + Rootfiles []struct { + FullPath string `xml:"full-path,attr"` + } `xml:"rootfiles>rootfile"` +} + +// epubOPF 解析 OPF 文件,找封面 item +type epubOPF struct { + Metadata struct { + Metas []struct { + Name string `xml:"name,attr"` + Content string `xml:"content,attr"` + } `xml:"meta"` + } `xml:"metadata"` + Manifest struct { + Items []struct { + ID string `xml:"id,attr"` + Href string `xml:"href,attr"` + MediaType string `xml:"media-type,attr"` + } `xml:"item"` + } `xml:"manifest"` +} + +// extractEpubCoverData 从 EPUB(zip)数据中提取封面图片原始字节 +func extractEpubCoverData(data []byte) []byte { + r, err := zip.NewReader(bytes.NewReader(data), int64(len(data))) + if err != nil { + return nil + } + + // 建立文件名→ZipFile 映射(忽略大小写) + fileMap := make(map[string]*zip.File, len(r.File)) + for _, f := range r.File { + fileMap[strings.ToLower(f.Name)] = f + } + + // 1. 尝试读取 META-INF/container.xml 找 OPF 路径 + opfPath := "" + if cf, ok := fileMap["meta-inf/container.xml"]; ok { + if rc, err := cf.Open(); err == nil { + var container epubContainer + if xml.NewDecoder(rc).Decode(&container) == nil && len(container.Rootfiles) > 0 { + opfPath = container.Rootfiles[0].FullPath + } + rc.Close() + } + } + + // OPF 所在目录(用于拼接相对路径) + opfDir := "" + if idx := strings.LastIndex(opfPath, "/"); idx >= 0 { + opfDir = opfPath[:idx+1] + } + + // 2. 从 OPF 中找封面 item ID + coverHref := "" + if opfPath != "" { + if of, ok := fileMap[strings.ToLower(opfPath)]; ok { + if rc, err := of.Open(); err == nil { + var opf epubOPF + if xml.NewDecoder(rc).Decode(&opf) == nil { + // 先找 指向的 item id + coverItemID := "" + for _, m := range opf.Metadata.Metas { + if strings.EqualFold(m.Name, "cover") && m.Content != "" { + coverItemID = m.Content + break + } + } + // 在 manifest 中找对应 item + for _, item := range opf.Manifest.Items { + if coverItemID != "" && item.ID == coverItemID { + coverHref = opfDir + item.Href + break + } + // 如果没有指定 cover id,找第一个图片类型的 item + if coverItemID == "" && strings.Contains(strings.ToLower(item.MediaType), "image") { + // 优先选择 id 或 href 中包含 "cover" 的 + if strings.Contains(strings.ToLower(item.ID), "cover") || + strings.Contains(strings.ToLower(item.Href), "cover") { + coverHref = opfDir + item.Href + break + } + } + } + // 如果还没找到,取第一个图片 + if coverHref == "" && coverItemID == "" { + for _, item := range opf.Manifest.Items { + if strings.Contains(strings.ToLower(item.MediaType), "image") { + coverHref = opfDir + item.Href + break + } + } + } + } + rc.Close() + } + } + } + + // 3. 常见封面文件名候选(按优先级) + candidates := []string{} + if coverHref != "" { + candidates = append(candidates, strings.ToLower(coverHref)) + } + candidates = append(candidates, + "cover.jpg", "cover.jpeg", "cover.png", + "images/cover.jpg", "images/cover.jpeg", "images/cover.png", + "oebps/cover.jpg", "oebps/images/cover.jpg", + "oebps/cover.jpeg", "oebps/images/cover.jpeg", + ) + + for _, name := range candidates { + if f, ok := fileMap[name]; ok { + if rc, err := f.Open(); err == nil { + imgData, readErr := readAllLimited(rc, MaxEmbeddedEntryBytes) + rc.Close() + if readErr == nil && len(imgData) > 0 { + return imgData + } + } + } + } + return nil +} + +// extractEpubCoverBase64 从 EPUB(zip)数据中提取封面图片,返回 data URI base64(向后兼容) +func extractEpubCoverBase64(data []byte) string { + coverData := extractEpubCoverData(data) + if len(coverData) == 0 { + return "" + } + return imageDataToBase64Thumb(coverData) +} + +// ── PDF 封面提取 ────────────────────────────────────────────────────────────── + +// extractPDFCoverData 从 PDF 数据中提取第一个内嵌 JPEG 图片流,返回原始字节 +// PDF 内嵌图片通常以 JPEG 流存储(SOI marker: 0xFF 0xD8),直接扫描字节流提取 +func extractPDFCoverData(data []byte) []byte { + // 扫描 JPEG SOI (0xFF 0xD8 0xFF) 标记 + for i := 0; i < len(data)-2; i++ { + if data[i] == 0xFF && data[i+1] == 0xD8 && data[i+2] == 0xFF { + // 找对应的 EOI (0xFF 0xD9) + for j := i + 2; j < len(data)-1; j++ { + if data[j] == 0xFF && data[j+1] == 0xD9 { + jpegData := data[i : j+2] + // 验证是否为有效图片(至少 1KB,避免误匹配小缩略图) + if len(jpegData) > 1024 { + return jpegData + } + break + } + } + } + } + return nil +} + +// extractPDFCoverBase64 从 PDF 数据中提取第一个内嵌 JPEG 图片流,返回 data URI base64(向后兼容) +func extractPDFCoverBase64(data []byte) string { + coverData := extractPDFCoverData(data) + if len(coverData) == 0 { + return "" + } + return imageDataToBase64Thumb(coverData) +} + +// ── 公共辅助 ────────────────────────────────────────────────────────────────── + +// imageDataToBase64Thumb 将图片字节解码并缩放为 300px 宽缩略图,返回 data URI base64 +func imageDataToBase64Thumb(data []byte) string { + img, err := decodeBounded(data) + if err != nil { + // 超限图片不做任何保留;解码失败但魔数是合法栅格格式时,原样 base64(可能是解码器不支持的变体) + if errors.Is(err, ErrTooLarge) || len(data) > MaxRawCoverFallbackBytes { + return "" + } + mimeType := imgmime.Sniff(data) + if mimeType == "" { + return "" + } + return "data:" + mimeType + ";base64," + base64.StdEncoding.EncodeToString(data) + } + thumb := imaging.Resize(img, 300, 0, imaging.Lanczos) + var buf bytes.Buffer + if err := jpeg.Encode(&buf, thumb, &jpeg.Options{Quality: 80}); err != nil { + return "" + } + return "data:image/jpeg;base64," + base64.StdEncoding.EncodeToString(buf.Bytes()) +} + +// detectImageMime 根据文件头检测图片 MIME 类型 +func detectImageMime(data []byte) string { + if len(data) < 4 { + return "image/jpeg" + } + switch { + case data[0] == 0xFF && data[1] == 0xD8: + return "image/jpeg" + case data[0] == 0x89 && data[1] == 0x50 && data[2] == 0x4E && data[3] == 0x47: + return "image/png" + case data[0] == 0x47 && data[1] == 0x49 && data[2] == 0x46: + return "image/gif" + case data[0] == 0x52 && data[1] == 0x49 && data[2] == 0x46 && data[3] == 0x46: + return "image/webp" + default: + return "image/jpeg" + } +} + +// getFileExt 获取文件扩展名(含点号,小写) +func getFileExt(fileName string) string { + if idx := strings.LastIndex(fileName, "."); idx >= 0 { + return fileName[idx:] + } + return "" +} + +// mimeTypeByBookExt 根据书籍文件扩展名返回 MIME 类型 +func mimeTypeByBookExt(ext string) string { + switch ext { + case ".pdf": + return "application/pdf" + case ".epub": + return "application/epub+zip" + case ".mobi": + return "application/x-mobipocket-ebook" + case ".azw3": + return "application/vnd.amazon.ebook" + case ".txt": + return "text/plain" + case ".djvu": + return "image/vnd.djvu" + case ".cbz": + return "application/vnd.comicbook+zip" + case ".cbr": + return "application/vnd.comicbook-rar" + default: + return "application/octet-stream" + } +} diff --git a/internal/media/scraper/book_online.go b/internal/media/scraper/book_online.go new file mode 100644 index 00000000..30caff32 --- /dev/null +++ b/internal/media/scraper/book_online.go @@ -0,0 +1,278 @@ +package scraper + +import ( + "encoding/json" + "fmt" + "net/http" + "net/url" + "strings" + "time" + + "github.com/OpenListTeam/OpenList/v4/internal/model" +) + +// ==================== Google Books ==================== + +// GoogleBooksScraper 通过 Google Books API 刮削书籍 +type GoogleBooksScraper struct { + APIKey string + ThumbnailMode string + ThumbnailPath string + client *http.Client +} + +func NewGoogleBooksScraper(apiKey, thumbnailMode, thumbnailPath string) *GoogleBooksScraper { + if thumbnailMode == "" { + thumbnailMode = "base64" + } + if thumbnailPath == "" { + thumbnailPath = "imgs" + } + return &GoogleBooksScraper{ + APIKey: apiKey, + ThumbnailMode: thumbnailMode, + ThumbnailPath: thumbnailPath, + client: &http.Client{Timeout: 20 * time.Second}, + } +} + +type googleBooksResponse struct { + Items []struct { + ID string `json:"id"` + VolumeInfo struct { + Title string `json:"title"` + Authors []string `json:"authors"` + Description string `json:"description"` + PublishedDate string `json:"publishedDate"` + Publisher string `json:"publisher"` + Categories []string `json:"categories"` + AverageRating float64 `json:"averageRating"` + IndustryIDs []struct { + Type string `json:"type"` + Identifier string `json:"identifier"` + } `json:"industryIdentifiers"` + ImageLinks struct { + Thumbnail string `json:"thumbnail"` + SmallThumbnail string `json:"smallThumbnail"` + } `json:"imageLinks"` + } `json:"volumeInfo"` + } `json:"items"` +} + +// ScrapeBook 用 Google Books 填充书籍信息,封面保存为远程链接(由调用方决定是否落盘) +func (s *GoogleBooksScraper) ScrapeBook(item *model.MediaItem) error { + query := strings.TrimSpace(item.ISBN) + if query == "" { + query = strings.TrimSpace(item.ScrapedName) + } + if query == "" && item.FileName != "" { + query = trimExt(item.FileName) + } + if query == "" { + return fmt.Errorf("empty query") + } + api := "https://www.googleapis.com/books/v1/volumes?maxResults=1&printType=books&langRestrict=" + + url.QueryEscape(ScrapeLanguagePrimary()) + "&q=" + url.QueryEscape(query) + if s.APIKey != "" { + api += "&key=" + url.QueryEscape(s.APIKey) + } + var resp googleBooksResponse + if err := s.getJSON(api, &resp); err != nil { + return err + } + if len(resp.Items) == 0 { + return fmt.Errorf("google books: no result for %q", query) + } + it := resp.Items[0] + v := it.VolumeInfo + + if item.ScrapedName == "" { + item.ScrapedName = v.Title + } + if item.Plot == "" { + item.Plot = v.Description + } + if item.Description == "" { + item.Description = v.Description + } + if item.Publisher == "" { + item.Publisher = v.Publisher + } + if item.ReleaseDate == "" { + item.ReleaseDate = v.PublishedDate + } + if item.Authors == "" && len(v.Authors) > 0 { + item.Authors = toAuthorsJSON(v.Authors) + } + if item.Genre == "" && len(v.Categories) > 0 { + item.Genre = strings.Join(v.Categories, ",") + } + if item.ISBN == "" { + for _, id := range v.IndustryIDs { + if id.Type == "ISBN_13" || id.Type == "ISBN_10" { + item.ISBN = id.Identifier + break + } + } + } + if item.Rating == 0 && v.AverageRating > 0 { + item.Rating = float32(v.AverageRating * 2) + } + if item.ExternalID == "" { + item.ExternalID = "google:" + it.ID + } + if item.Cover == "" { + img := v.ImageLinks.Thumbnail + if img == "" { + img = v.ImageLinks.SmallThumbnail + } + item.Cover = normalizeGoogleImage(img) + } + return nil +} + +func (s *GoogleBooksScraper) getJSON(api string, out any) error { + req, err := http.NewRequest(http.MethodGet, api, nil) + if err != nil { + return err + } + req.Header.Set("User-Agent", "Mozilla/5.0 (compatible; OpenList-Media/1.0)") + resp, err := s.client.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("google books: unexpected status %d", resp.StatusCode) + } + return json.NewDecoder(resp.Body).Decode(out) +} + +func normalizeGoogleImage(img string) string { + if img == "" { + return "" + } + img = strings.Replace(img, "http://", "https://", 1) + if idx := strings.Index(img, "&edge="); idx > 0 { + img = img[:idx] + } + return img +} + +// ==================== OpenLibrary ==================== + +// OpenLibraryScraper 通过 OpenLibrary Search API 刮削书籍(无需凭据) +type OpenLibraryScraper struct { + ThumbnailMode string + ThumbnailPath string + client *http.Client +} + +func NewOpenLibraryScraper(thumbnailMode, thumbnailPath string) *OpenLibraryScraper { + if thumbnailMode == "" { + thumbnailMode = "base64" + } + if thumbnailPath == "" { + thumbnailPath = "imgs" + } + return &OpenLibraryScraper{ + ThumbnailMode: thumbnailMode, + ThumbnailPath: thumbnailPath, + client: &http.Client{Timeout: 20 * time.Second}, + } +} + +type openLibraryResponse struct { + Docs []struct { + Key string `json:"key"` + Title string `json:"title"` + AuthorName []string `json:"author_name"` + FirstPublish int `json:"first_publish_year"` + Publisher []string `json:"publisher"` + ISBN []string `json:"isbn"` + CoverID int `json:"cover_i"` + Subject []string `json:"subject"` + RatingsAverage float64 `json:"ratings_average"` + } `json:"docs"` +} + +// ScrapeBook 用 OpenLibrary 填充书籍信息,封面保存为远程链接(由调用方决定是否落盘) +func (s *OpenLibraryScraper) ScrapeBook(item *model.MediaItem) error { + query := strings.TrimSpace(item.ISBN) + if query == "" { + query = strings.TrimSpace(item.ScrapedName) + } + if query == "" && item.FileName != "" { + query = trimExt(item.FileName) + } + if query == "" { + return fmt.Errorf("empty query") + } + api := "https://openlibrary.org/search.json?limit=1&fields=key,title,author_name,first_publish_year,publisher,isbn,cover_i,subject,ratings_average&q=" + url.QueryEscape(query) + var resp openLibraryResponse + if err := s.getJSON(api, &resp); err != nil { + return err + } + if len(resp.Docs) == 0 { + return fmt.Errorf("openlibrary: no result for %q", query) + } + doc := resp.Docs[0] + + if item.ScrapedName == "" { + item.ScrapedName = doc.Title + } + if item.ReleaseDate == "" && doc.FirstPublish > 0 { + item.ReleaseDate = fmt.Sprintf("%d", doc.FirstPublish) + } + if item.Publisher == "" && len(doc.Publisher) > 0 { + item.Publisher = doc.Publisher[0] + } + if item.Authors == "" && len(doc.AuthorName) > 0 { + item.Authors = toAuthorsJSON(doc.AuthorName) + } + if item.Genre == "" && len(doc.Subject) > 0 { + limit := len(doc.Subject) + if limit > 8 { + limit = 8 + } + item.Genre = strings.Join(doc.Subject[:limit], ",") + } + if item.ISBN == "" && len(doc.ISBN) > 0 { + item.ISBN = doc.ISBN[0] + } + if item.Rating == 0 && doc.RatingsAverage > 0 { + item.Rating = float32(doc.RatingsAverage * 2) + } + if item.ExternalID == "" { + item.ExternalID = "openlibrary:" + doc.Key + } + if item.Cover == "" && doc.CoverID > 0 { + item.Cover = fmt.Sprintf("https://covers.openlibrary.org/b/id/%d-L.jpg", doc.CoverID) + } + return nil +} + +func (s *OpenLibraryScraper) getJSON(api string, out any) error { + req, err := http.NewRequest(http.MethodGet, api, nil) + if err != nil { + return err + } + req.Header.Set("User-Agent", "Mozilla/5.0 (compatible; OpenList-Media/1.0)") + resp, err := s.client.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("openlibrary: unexpected status %d", resp.StatusCode) + } + return json.NewDecoder(resp.Body).Decode(out) +} + +func toAuthorsJSON(authors []string) string { + b, err := json.Marshal(authors) + if err != nil { + return "" + } + return string(b) +} diff --git a/internal/media/scraper/discogs.go b/internal/media/scraper/discogs.go new file mode 100644 index 00000000..7e72c043 --- /dev/null +++ b/internal/media/scraper/discogs.go @@ -0,0 +1,317 @@ +package scraper + +import ( + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "strings" + "time" + + "github.com/OpenListTeam/OpenList/v4/internal/model" +) + +const discogsDefaultBaseURL = "https://api.discogs.com" + +// DiscogsScraper Discogs音乐刮削器 +type DiscogsScraper struct { + Token string + BaseURL string // 可自定义的 Discogs API 地址 + client *http.Client +} + +// NewDiscogsScraper 创建Discogs刮削器 +// +// 用户只需填写「域名 + API 前缀」,例如: +// 1. 留空:使用默认 https://api.discogs.com +// 2. 官方:api.discogs.com -> https://api.discogs.com +// 3. 反代:example.edgeone.run/api/discogs -> https://example.edgeone.run/api/discogs +// 4. 反代:example.edgeone.run/api/discogs/ -> https://example.edgeone.run/api/discogs(自动去尾斜杠) +// +// 后续会基于 BaseURL 直接拼接 /database/search、/releases/{id} 等具体路径。 +func NewDiscogsScraper(token string, baseURL string) *DiscogsScraper { + base := strings.TrimRight(strings.TrimSpace(baseURL), "/") + if base == "" { + base = discogsDefaultBaseURL + } + // 协议头容错:用户没填 http/https 时自动补 https:// + if !strings.HasPrefix(base, "http://") && !strings.HasPrefix(base, "https://") { + base = "https://" + base + } + // 再次去掉末尾多余的 /,避免后续拼接出 //database/search 这类双斜杠 + base = strings.TrimRight(base, "/") + return &DiscogsScraper{ + Token: token, + BaseURL: base, + client: &http.Client{Timeout: 15 * time.Second}, + } +} + +type discogsSearchResult struct { + Results []struct { + ID int `json:"id"` + Title string `json:"title"` + Type string `json:"type"` + Year string `json:"year"` + Thumb string `json:"thumb"` + CoverImage string `json:"cover_image"` + Genre []string `json:"genre"` + Style []string `json:"style"` + ResourceURL string `json:"resource_url"` + } `json:"results"` +} + +type discogsReleaseDetail struct { + ID int `json:"id"` + Title string `json:"title"` + Year int `json:"year"` + CoverImage string `json:"cover_image"` + Notes string `json:"notes"` + Artists []struct { + Name string `json:"name"` + } `json:"artists"` + Genres []string `json:"genres"` + Styles []string `json:"styles"` + Tracklist []struct { + Position string `json:"position"` + Title string `json:"title"` + Duration string `json:"duration"` + } `json:"tracklist"` + Community struct { + Rating struct { + Average float32 `json:"average"` + } `json:"rating"` + } `json:"community"` +} + +// parseMusicFileName 从音乐文件名中提取搜索关键词列表 +// 规则:去掉扩展名后,按" - "或"-"分割,返回各部分作为候选搜索词 +// 例如: +// +// "周杰伦 - 七里香.mp3" -> ["周杰伦 - 七里香", "周杰伦", "七里香"] +// "Coldplay-Yellow.flac" -> ["Coldplay-Yellow", "Coldplay", "Yellow"] +// "七里香.mp3" -> ["七里香"] +func parseMusicFileName(fileName string) []string { + // 去掉扩展名 + if idx := strings.LastIndex(fileName, "."); idx > 0 { + ext := strings.ToLower(fileName[idx:]) + if len(ext) <= 5 { + fileName = fileName[:idx] + } + } + fileName = strings.TrimSpace(fileName) + if fileName == "" { + return nil + } + + var queries []string + // 完整文件名作为第一候选 + queries = append(queries, fileName) + + // 按" - "(带空格)分割 + parts := strings.SplitN(fileName, " - ", 2) + if len(parts) == 2 { + before := strings.TrimSpace(parts[0]) + after := strings.TrimSpace(parts[1]) + if before != "" { + queries = append(queries, before) + } + if after != "" { + queries = append(queries, after) + } + } else { + // 按"-"(不带空格)分割 + parts = strings.SplitN(fileName, "-", 2) + if len(parts) == 2 { + before := strings.TrimSpace(parts[0]) + after := strings.TrimSpace(parts[1]) + if before != "" { + queries = append(queries, before) + } + if after != "" { + queries = append(queries, after) + } + } + } + + return queries +} + +// doDiscogsSearch 执行单次Discogs搜索 +func (s *DiscogsScraper) doDiscogsSearch(query string) (*discogsSearchResult, error) { + searchURL := fmt.Sprintf("%s/database/search?q=%s&type=release&token=%s", + s.BaseURL, url.QueryEscape(query), s.Token) + + req, _ := http.NewRequest("GET", searchURL, nil) + req.Header.Set("User-Agent", "OpenList/4.0 +https://github.com/OpenListTeam/OpenList") + + resp, err := s.client.Do(req) + if err != nil { + return nil, fmt.Errorf("Discogs搜索请求失败: %w", err) + } + defer resp.Body.Close() + + body, _ := io.ReadAll(resp.Body) + var result discogsSearchResult + if err := json.Unmarshal(body, &result); err != nil { + return nil, fmt.Errorf("Discogs搜索结果解析失败: %w", err) + } + return &result, nil +} + +// buildMusicQuery 根据 MediaItem 已有字段拼接 Discogs 搜索词 +// 优先级:专辑名 > ScrapedName > 文件名解析 +// 若有歌手或年份信息,则附加到搜索词中以提高精准度 +func buildMusicQuery(base string, item *model.MediaItem) string { + q := base + // 附加歌手 + if item.AlbumArtist != "" { + q = item.AlbumArtist + " " + q + } + // 附加年份(取 ReleaseDate 前4位) + if len(item.ReleaseDate) >= 4 { + q = q + " " + item.ReleaseDate[:4] + } + return strings.TrimSpace(q) +} + +// ScrapeMusic 刮削音乐/专辑信息 +func (s *DiscogsScraper) ScrapeMusic(item *model.MediaItem) error { + if s.Token == "" { + return fmt.Errorf("Discogs Token 未配置") + } + + // 构建候选搜索词列表 + var queries []string + if item.AlbumName != "" { + queries = append(queries, buildMusicQuery(item.AlbumName, item)) + // 同时保留不带附加信息的纯专辑名作为降级候选 + if item.AlbumArtist != "" || len(item.ReleaseDate) >= 4 { + queries = append(queries, item.AlbumName) + } + } else if item.ScrapedName != "" { + queries = append(queries, buildMusicQuery(item.ScrapedName, item)) + if item.AlbumArtist != "" || len(item.ReleaseDate) >= 4 { + queries = append(queries, item.ScrapedName) + } + } else { + // 从文件名解析,按"-"分割分别搜索 + baseQueries := parseMusicFileName(item.FileName) + for _, bq := range baseQueries { + enhanced := buildMusicQuery(bq, item) + if enhanced != bq { + // 先放带附加信息的精准词,再放原始词作为降级 + queries = append(queries, enhanced) + } + queries = append(queries, bq) + } + } + + if len(queries) == 0 { + return fmt.Errorf("无法从文件名中提取有效搜索词") + } + + // 依次尝试各候选词,找到结果即停止(模糊匹配降级) + var searchResult *discogsSearchResult + var lastQuery string + for _, q := range queries { + result, err := s.doDiscogsSearch(q) + if err != nil { + return err + } + lastQuery = q + if len(result.Results) > 0 { + searchResult = result + break + } + } + + if searchResult == nil || len(searchResult.Results) == 0 { + return fmt.Errorf("Discogs未找到匹配结果: %s", lastQuery) + } + + first := searchResult.Results[0] + + // 获取详情 + detailURL := fmt.Sprintf("%s/releases/%d?token=%s", s.BaseURL, first.ID, s.Token) + detailReq, _ := http.NewRequest("GET", detailURL, nil) + detailReq.Header.Set("User-Agent", "OpenList/4.0 +https://github.com/OpenListTeam/OpenList") + + detailResp, err := s.client.Do(detailReq) + if err != nil { + return fmt.Errorf("Discogs详情请求失败: %w", err) + } + defer detailResp.Body.Close() + + detailBody, _ := io.ReadAll(detailResp.Body) + var detail discogsReleaseDetail + if err := json.Unmarshal(detailBody, &detail); err != nil { + return fmt.Errorf("Discogs详情解析失败: %w", err) + } + + // 填充字段:已有值的字段不覆盖,仅补充空缺 + // Discogs 的 title 就是专辑名(不是 "Artist - Album" 格式) + if detail.Title != "" { + if item.AlbumName == "" { + item.AlbumName = detail.Title + } + if item.ScrapedName == "" { + item.ScrapedName = detail.Title + } + } + + if item.ReleaseDate == "" { + if detail.Year > 0 { + item.ReleaseDate = fmt.Sprintf("%d-01-01", detail.Year) + } else if first.Year != "" { + item.ReleaseDate = first.Year + "-01-01" + } + } + + if item.Rating == 0 { + item.Rating = detail.Community.Rating.Average + } + if item.Plot == "" { + item.Plot = detail.Notes + } + if item.ExternalID == "" { + item.ExternalID = fmt.Sprintf("discogs:%d", detail.ID) + } + + // 封面(存储 URL,前端直接展示) + if item.Cover == "" { + if detail.CoverImage != "" { + item.Cover = detail.CoverImage + } else if first.CoverImage != "" { + item.Cover = first.CoverImage + } else if first.Thumb != "" { + item.Cover = first.Thumb + } + } + + // 类型 + if item.Genre == "" { + genres := append(detail.Genres, detail.Styles...) + item.Genre = strings.Join(genres, ",") + } + + // 艺术家:仅在 Authors 和 AlbumArtist 均为空时才填充(ID3已有值则保留) + artists := make([]string, 0, len(detail.Artists)) + for _, a := range detail.Artists { + artists = append(artists, a.Name) + } + if len(artists) > 0 { + if item.Authors == "" { + authorsJSON, _ := json.Marshal(artists) + item.Authors = string(authorsJSON) + } + if item.AlbumArtist == "" { + item.AlbumArtist = artists[0] + } + } + + now := time.Now() + item.ScrapedAt = &now + return nil +} diff --git a/internal/media/scraper/douban.go b/internal/media/scraper/douban.go new file mode 100644 index 00000000..dd446a8a --- /dev/null +++ b/internal/media/scraper/douban.go @@ -0,0 +1,497 @@ +package scraper + +import ( + "bytes" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "image/jpeg" + "io" + "net/http" + "net/url" + "regexp" + "strings" + "time" + + "github.com/disintegration/imaging" + "github.com/OpenListTeam/OpenList/v4/internal/media/imgmime" + "github.com/OpenListTeam/OpenList/v4/internal/model" + "golang.org/x/net/html" +) + +const ( + doubanSearchURL = "https://www.douban.com/search" + doubanBookCat = "1001" + doubanBase = "https://book.douban.com/" + + // maxCoverDownloadBytes 单张远端封面最大读取字节数 + maxCoverDownloadBytes int64 = 16 << 20 +) + +var ( + doubanBookURLPattern = regexp.MustCompile(`.*/subject/(\d+)/?`) + doubanDatePattern = regexp.MustCompile(`(\d{4})-(\d+)`) + doubanDefaultHeaders = map[string]string{ + "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36", + "Referer": doubanBase, + } +) + +// DoubanScraper 豆瓣书籍刮削器(移植自 NewDouban.py) +type DoubanScraper struct { + client *http.Client + // ThumbnailMode 缩略图存储方式:"base64"(默认,存入数据库)或 "local"(存到本地文件) + ThumbnailMode string + // ThumbnailPath 本地存储路径,ThumbnailMode 为 "local" 时有效,默认 "/.thumbnail" + ThumbnailPath string +} + +// NewDoubanScraper 创建豆瓣刮削器 +func NewDoubanScraper() *DoubanScraper { + return &DoubanScraper{ + client: &http.Client{Timeout: 20 * time.Second}, + ThumbnailMode: "base64", + ThumbnailPath: "/.thumbnail", + } +} + +// NewDoubanScraperWithConfig 创建带完整配置的豆瓣刮削器 +func NewDoubanScraperWithConfig(thumbnailMode, thumbnailPath string) *DoubanScraper { + if thumbnailMode == "" { + thumbnailMode = "base64" + } + if thumbnailPath == "" { + thumbnailPath = "/.thumbnail" + } + return &DoubanScraper{ + client: &http.Client{Timeout: 20 * time.Second}, + ThumbnailMode: thumbnailMode, + ThumbnailPath: thumbnailPath, + } +} + +// ScrapeBook 刮削书籍信息 +func (s *DoubanScraper) ScrapeBook(item *model.MediaItem) error { + query := item.ScrapedName + if query == "" { + query = strings.TrimSuffix(item.FileName, strings.ToLower(item.FileName[strings.LastIndex(item.FileName, "."):])) + } + + // 搜索书籍URL列表 + bookURLs, err := s.searchBookURLs(query) + if err != nil { + return fmt.Errorf("豆瓣搜索失败: %w", err) + } + if len(bookURLs) == 0 { + return fmt.Errorf("豆瓣未找到匹配书籍: %s", query) + } + + // 取第一个结果 + bookDetail, err := s.loadBookDetail(bookURLs[0]) + if err != nil { + return fmt.Errorf("豆瓣获取书籍详情失败: %w", err) + } + + // 填充字段(仅填空字段,已有值不覆盖) + if item.ScrapedName == "" && bookDetail.Title != "" { + item.ScrapedName = bookDetail.Title + } + if item.Plot == "" { + item.Plot = bookDetail.Description + } + // 仅在豆瓣返回了有效封面图片URL且当前 item 还没有封面时才下载/覆盖 + if item.Cover == "" && bookDetail.Cover != "" { + if cover := s.downloadAndStoreCover(item.FolderPath, bookDetail.Cover); cover != "" { + item.Cover = cover + } else { + // 下载失败时保留原 URL(降级) + item.Cover = bookDetail.Cover + } + } + if item.Rating == 0 { + item.Rating = bookDetail.Rating + } + if item.ReleaseDate == "" { + item.ReleaseDate = bookDetail.PublishedDate + } + if item.Publisher == "" { + item.Publisher = bookDetail.Publisher + } + if item.ISBN == "" { + item.ISBN = bookDetail.ISBN + } + if item.ExternalID == "" { + item.ExternalID = "douban:" + bookDetail.ID + } + + if item.Authors == "" && len(bookDetail.Authors) > 0 { + authorsJSON, _ := json.Marshal(bookDetail.Authors) + item.Authors = string(authorsJSON) + } + if item.Genre == "" && len(bookDetail.Tags) > 0 { + item.Genre = strings.Join(bookDetail.Tags, ",") + } + + now := time.Now() + item.ScrapedAt = &now + return nil +} + +type doubanBookDetail struct { + ID string + Title string + Authors []string + Publisher string + PublishedDate string + Cover string + Rating float32 + Description string + Tags []string + ISBN string +} + +// searchBookURLs 搜索豆瓣书籍URL列表 +func (s *DoubanScraper) searchBookURLs(query string) ([]string, error) { + searchURL := fmt.Sprintf("%s?cat=%s&q=%s", doubanSearchURL, doubanBookCat, url.QueryEscape(query)) + body, err := s.doGet(searchURL) + if err != nil { + return nil, err + } + + doc, err := html.Parse(strings.NewReader(string(body))) + if err != nil { + return nil, err + } + + var bookURLs []string + var traverse func(*html.Node) + traverse = func(n *html.Node) { + if n.Type == html.ElementNode && n.Data == "a" { + for _, attr := range n.Attr { + if attr.Key == "class" && attr.Val == "nbg" { + for _, a := range n.Attr { + if a.Key == "href" { + parsed := s.calcURL(a.Val) + if parsed != "" && len(bookURLs) < 5 { + bookURLs = append(bookURLs, parsed) + } + } + } + } + } + } + for c := n.FirstChild; c != nil; c = c.NextSibling { + traverse(c) + } + } + traverse(doc) + return bookURLs, nil +} + +// calcURL 解析豆瓣搜索结果中的真实URL +func (s *DoubanScraper) calcURL(href string) string { + parsed, err := url.Parse(href) + if err != nil { + return "" + } + query := parsed.Query() + rawURL := query.Get("url") + if rawURL == "" { + return "" + } + decoded, err := url.QueryUnescape(rawURL) + if err != nil { + return "" + } + if doubanBookURLPattern.MatchString(decoded) { + return decoded + } + return "" +} + +// loadBookDetail 加载书籍详情页 +func (s *DoubanScraper) loadBookDetail(bookURL string) (*doubanBookDetail, error) { + body, err := s.doGet(bookURL) + if err != nil { + return nil, err + } + return s.parseBookHTML(bookURL, string(body)) +} + +// parseBookHTML 解析书籍HTML(移植自 DoubanBookHtmlParser.parse_book) +func (s *DoubanScraper) parseBookHTML(bookURL, content string) (*doubanBookDetail, error) { + doc, err := html.Parse(strings.NewReader(content)) + if err != nil { + return nil, err + } + + detail := &doubanBookDetail{} + + // 提取豆瓣ID + if m := doubanBookURLPattern.FindStringSubmatch(bookURL); len(m) > 1 { + detail.ID = m[1] + } + + var traverse func(*html.Node) + traverse = func(n *html.Node) { + if n.Type == html.ElementNode { + switch n.Data { + case "span": + // 标题 + if getAttr(n, "property") == "v:itemreviewed" { + detail.Title = getTextContent(n) + } + // 元数据字段 + if getAttr(n, "class") == "pl" { + text := getTextContent(n) + tail := getNextText(n) + switch { + case strings.HasPrefix(text, "作者") || strings.HasPrefix(text, "译者"): + // 作者通过链接提取 + collectAuthors(n, &detail.Authors) + case strings.HasPrefix(text, "出版社"): + detail.Publisher = strings.TrimSpace(tail) + case strings.HasPrefix(text, "出版年"): + detail.PublishedDate = parseDoubanDate(strings.TrimSpace(tail)) + case strings.HasPrefix(text, "ISBN"): + detail.ISBN = strings.TrimSpace(tail) + case strings.HasPrefix(text, "副标题"): + if detail.Title != "" { + detail.Title += ":" + strings.TrimSpace(tail) + } + } + } + // 评分 + if getAttr(n, "property") == "v:average" { + ratingStr := getTextContent(n) + var rating float32 + fmt.Sscanf(ratingStr, "%f", &rating) + detail.Rating = rating / 2 + } + case "img": + // 封面图片: + if getAttr(n, "class") == "nbg" { + src := getAttr(n, "src") + if src != "" && !strings.HasSuffix(src, "update_image") { + detail.Cover = src + } + } + // 备用:#mainpic 下的 img + if getAttr(n, "id") == "mainpic" || (n.Parent != nil && getAttr(n.Parent, "id") == "mainpic") { + src := getAttr(n, "src") + if src != "" && detail.Cover == "" { + detail.Cover = src + } + } + case "a": + // 标签(原封面逻辑移除,改为从img获取) + cls := getAttr(n, "class") + if strings.Contains(cls, "tag") { + tag := getTextContent(n) + if tag != "" { + detail.Tags = append(detail.Tags, tag) + } + } + case "div": + // 简介 + if getAttr(n, "id") == "link-report" { + detail.Description = extractIntroText(n) + } + } + } + for c := n.FirstChild; c != nil; c = c.NextSibling { + traverse(c) + } + } + traverse(doc) + return detail, nil +} + +// HTML辅助函数 + +func getAttr(n *html.Node, key string) string { + for _, attr := range n.Attr { + if attr.Key == key { + return attr.Val + } + } + return "" +} + +func getTextContent(n *html.Node) string { + var sb strings.Builder + var walk func(*html.Node) + walk = func(node *html.Node) { + if node.Type == html.TextNode { + sb.WriteString(node.Data) + } + for c := node.FirstChild; c != nil; c = c.NextSibling { + walk(c) + } + } + walk(n) + return strings.TrimSpace(sb.String()) +} + +func getNextText(n *html.Node) string { + if n.NextSibling != nil { + if n.NextSibling.Type == html.TextNode { + return strings.TrimSpace(n.NextSibling.Data) + } + return getTextContent(n.NextSibling) + } + return "" +} + +func collectAuthors(n *html.Node, authors *[]string) { + var walk func(*html.Node) + walk = func(node *html.Node) { + if node.Type == html.ElementNode && node.Data == "a" { + href := getAttr(node, "href") + if strings.Contains(href, "/author") || strings.Contains(href, "/search") { + name := getTextContent(node) + if name != "" { + *authors = append(*authors, name) + } + } + } + for c := node.FirstChild; c != nil; c = c.NextSibling { + walk(c) + } + } + // 从父节点开始找 + if n.Parent != nil { + walk(n.Parent) + } +} + +func extractIntroText(n *html.Node) string { + var sb strings.Builder + var walk func(*html.Node) + walk = func(node *html.Node) { + if node.Type == html.ElementNode && node.Data == "div" { + if getAttr(node, "class") == "intro" { + sb.WriteString(getTextContent(node)) + return + } + } + for c := node.FirstChild; c != nil; c = c.NextSibling { + walk(c) + } + } + walk(n) + return strings.TrimSpace(sb.String()) +} + +func parseDoubanDate(dateStr string) string { + if dateStr == "" { + return "" + } + if m := doubanDatePattern.FindStringSubmatch(dateStr); len(m) > 2 { + return fmt.Sprintf("%s-%s-01", m[1], m[2]) + } + return dateStr +} + +// downloadAndStoreCover 下载封面图片并根据 ThumbnailMode 存储 +// 失败时返回空字符串 +func (s *DoubanScraper) downloadAndStoreCover(filePath, imgURL string) string { + data, mimeType := s.downloadImage(imgURL) + if len(data) == 0 { + return "" + } + if NormalizeThumbMode(s.ThumbnailMode) != ThumbModeBase64 { + // 运行目录 / 挂载目录模式 + localScraper := NewBookLocalScraperWithConfig(s.ThumbnailMode, s.ThumbnailPath) + return localScraper.storeCover(filePath, data) + } + // Base64 模式(默认) + img, err := decodeBounded(data) + if err != nil { + // 解码失败时退回原始字节;超过阈值直接放弃,避免十几 MB 的图被 base64 写进 cover 列 + if errors.Is(err, ErrTooLarge) || len(data) > MaxRawCoverFallbackBytes { + return "" + } + return "data:" + mimeType + ";base64," + base64.StdEncoding.EncodeToString(data) + } + thumb := imaging.Resize(img, 300, 0, imaging.Lanczos) + var buf bytes.Buffer + if err := jpeg.Encode(&buf, thumb, &jpeg.Options{Quality: 80}); err != nil { + return "" + } + return "data:image/jpeg;base64," + base64.StdEncoding.EncodeToString(buf.Bytes()) +} + +// downloadCoverAsBase64 下载封面图片并转为 data URI Base64 字符串(向后兼容) +// 失败时返回空字符串 +func (s *DoubanScraper) downloadCoverAsBase64(imgURL string) string { + data, mimeType := s.downloadImage(imgURL) + if len(data) == 0 { + return "" + } + img, err := decodeBounded(data) + if err != nil { + // 解码失败时退回原始字节;超过阈值直接放弃,避免十几 MB 的图被 base64 写进 cover 列 + if errors.Is(err, ErrTooLarge) || len(data) > MaxRawCoverFallbackBytes { + return "" + } + return "data:" + mimeType + ";base64," + base64.StdEncoding.EncodeToString(data) + } + thumb := imaging.Resize(img, 300, 0, imaging.Lanczos) + var buf bytes.Buffer + if err := jpeg.Encode(&buf, thumb, &jpeg.Options{Quality: 80}); err != nil { + return "" + } + return "data:image/jpeg;base64," + base64.StdEncoding.EncodeToString(buf.Bytes()) +} + +// downloadImage 下载图片,返回原始字节和 MIME 类型 +func (s *DoubanScraper) downloadImage(imgURL string) ([]byte, string) { + req, err := http.NewRequest("GET", imgURL, nil) + if err != nil { + return nil, "" + } + for k, v := range doubanDefaultHeaders { + req.Header.Set(k, v) + } + resp, err := s.client.Do(req) + if err != nil { + return nil, "" + } + defer resp.Body.Close() + if resp.StatusCode != 200 { + return nil, "" + } + // 远端图片大小不可信,限制读取上限 + data, err := io.ReadAll(io.LimitReader(resp.Body, maxCoverDownloadBytes)) + if err != nil || len(data) == 0 { + return nil, "" + } + // 不信任远端 Content-Type:只接受魔数可识别的栅格图片,防止 SVG/HTML 被当作封面 + mimeType := imgmime.Sniff(data) + if mimeType == "" { + return nil, "" + } + return data, mimeType +} + +// doGet 发送GET请求 +func (s *DoubanScraper) doGet(reqURL string) ([]byte, error) { + req, err := http.NewRequest("GET", reqURL, nil) + if err != nil { + return nil, err + } + for k, v := range doubanDefaultHeaders { + req.Header.Set(k, v) + } + resp, err := s.client.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + if resp.StatusCode != 200 && resp.StatusCode != 201 { + return nil, fmt.Errorf("HTTP %d: %s", resp.StatusCode, reqURL) + } + return io.ReadAll(resp.Body) +} diff --git a/internal/media/scraper/image.go b/internal/media/scraper/image.go new file mode 100644 index 00000000..dba75915 --- /dev/null +++ b/internal/media/scraper/image.go @@ -0,0 +1,196 @@ +package scraper + +import ( + "bytes" + "context" + "fmt" + "image/jpeg" + "io" + "strings" + "time" + + "github.com/disintegration/imaging" + "github.com/rwcarlsen/goexif/exif" + + "github.com/OpenListTeam/OpenList/v4/internal/model" +) + +// ImageScraper 图片刮削器 +// 从图片 EXIF 中提取元数据,并生成缩略图写入 Cover 字段。 +type ImageScraper struct { + // StoreThumbnail 为 true 时生成缩略图写入 Cover; + // 为 false 时直接用文件路径作为 Cover,节省数据库空间。 + StoreThumbnail bool + // ThumbnailMode 缩略图存储方式:"base64"(默认,存入数据库)或 "local"(存到本地文件) + ThumbnailMode string + // ThumbnailPath 本地存储路径,ThumbnailMode 为 "local" 时有效,默认 "/.thumbnail" + ThumbnailPath string +} + +// NewImageScraper 创建图片刮削器 +// storeThumbnail 为 true 时生成缩略图写入 Cover。 +func NewImageScraper(storeThumbnail bool) *ImageScraper { + return &ImageScraper{ + StoreThumbnail: storeThumbnail, + ThumbnailMode: "base64", + ThumbnailPath: "/.thumbnail", + } +} + +// NewImageScraperWithConfig 创建带完整配置的图片刮削器 +func NewImageScraperWithConfig(storeThumbnail bool, thumbnailMode, thumbnailPath string) *ImageScraper { + if thumbnailMode == "" { + thumbnailMode = "base64" + } + if thumbnailPath == "" { + thumbnailPath = "/.thumbnail" + } + return &ImageScraper{ + StoreThumbnail: storeThumbnail, + ThumbnailMode: thumbnailMode, + ThumbnailPath: thumbnailPath, + } +} + +// ScrapeImage 刮削图片信息 +// reader 为图片文件内容流,必须传入有效流才能生成缩略图。 +// 当 reader 不为 nil 时: +// - 从 EXIF 读取拍摄时间、GPS 地点、相机型号+参数、评分 +// - 生成 300px 宽缩略图,根据 ThumbnailMode 存储为 Base64 或本地文件 +// +// 注意:绝不将文件路径作为 cover,reader 为 nil 或缩略图生成失败时 cover 保持为空。 +func (s *ImageScraper) ScrapeImage(item *model.MediaItem, reader io.Reader) error { + // 去掉扩展名作为展示名(如果尚未设置) + if item.ScrapedName == "" { + item.ScrapedName = trimExt(item.FileName) + } + + if reader == nil { + // 无文件流时无法生成缩略图,cover 保持为空,等待下次重试 + now := time.Now() + item.ScrapedAt = &now + return nil + } + + // 将流读入内存(EXIF 解析和图像解码都需要完整数据) + data, err := readAllLimited(reader, MaxImageFileBytes) + if err != nil || len(data) == 0 { + // 读取失败时 cover 保持为空 + now := time.Now() + item.ScrapedAt = &now + return nil + } + + // ── 1. 解析 EXIF ────────────────────────────────────────────── + s.parseEXIF(item, bytes.NewReader(data)) + + // ── 2. 生成缩略图并按配置存储 ──────────────────────────────── + if thumb := s.generateThumbBytes(data); len(thumb) > 0 { + fullPath := strings.TrimSuffix(item.FolderPath, "/") + "/" + item.FileName + item.Cover = SaveThumb(context.Background(), s.ThumbnailMode, s.ThumbnailPath, fullPath, "image/jpeg", thumb) + } + + now := time.Now() + item.ScrapedAt = &now + return nil +} + +// parseEXIF 从图片数据中解析 EXIF 元数据并写入 item +func (s *ImageScraper) parseEXIF(item *model.MediaItem, r io.Reader) { + x, err := exif.Decode(r) + if err != nil { + return + } + + // ── 拍摄时间 → ReleaseDate ──────────────────────────────────── + if item.ReleaseDate == "" { + if t, err := x.DateTime(); err == nil { + item.ReleaseDate = t.Format("2006-01-02") + } + } + + // ── GPS 地点 → Authors(借用作者字段存储地点信息)──────────── + if item.Authors == "" { + if lat, lon, err := x.LatLong(); err == nil { + item.Authors = fmt.Sprintf(`["GPS: %.6f, %.6f"]`, lat, lon) + } + } + + // ── 相机型号 + 参数 → Genre ─────────────────────────────────── + if item.Genre == "" { + var parts []string + if make, err := x.Get(exif.Make); err == nil { + if v, err := make.StringVal(); err == nil && v != "" { + parts = append(parts, strings.TrimSpace(v)) + } + } + if model_, err := x.Get(exif.Model); err == nil { + if v, err := model_.StringVal(); err == nil && v != "" { + parts = append(parts, strings.TrimSpace(v)) + } + } + if fnum, err := x.Get(exif.FNumber); err == nil { + if num, den, err := fnum.Rat2(0); err == nil && den != 0 { + parts = append(parts, fmt.Sprintf("f/%.1f", float64(num)/float64(den))) + } + } + if exp, err := x.Get(exif.ExposureTime); err == nil { + if num, den, err := exp.Rat2(0); err == nil && den != 0 { + if num == 1 { + parts = append(parts, fmt.Sprintf("1/%ds", den)) + } else { + parts = append(parts, fmt.Sprintf("%d/%ds", num, den)) + } + } + } + if iso, err := x.Get(exif.ISOSpeedRatings); err == nil { + if v, err := iso.Int(0); err == nil { + parts = append(parts, fmt.Sprintf("ISO%d", v)) + } + } + if focal, err := x.Get(exif.FocalLength); err == nil { + if num, den, err := focal.Rat2(0); err == nil && den != 0 { + parts = append(parts, fmt.Sprintf("%.0fmm", float64(num)/float64(den))) + } + } + if len(parts) > 0 { + item.Genre = strings.Join(parts, ",") + } + } + + // ── EXIF Rating(0-5 星)→ Rating(0-10 分)────────────────── + // Windows 资源管理器、Lightroom、Darktable 等软件会写入此标签 + if item.Rating == 0 { + if ratingTag, err := x.Get(exif.FieldName("Rating")); err == nil { + if v, err := ratingTag.Int(0); err == nil && v > 0 { + // 1-5 星映射到 2-10 分 + item.Rating = float32(v) * 2 + } + } + } +} + +// generateThumbBytes 将图片数据缩放为 300px 宽缩略图,返回 JPEG 字节 +func (s *ImageScraper) generateThumbBytes(data []byte) []byte { + img, err := decodeBounded(data) + if err != nil { + return nil + } + + // 缩放到宽度 300px,高度等比 + thumb := imaging.Resize(img, 300, 0, imaging.Lanczos) + + var buf bytes.Buffer + if err := jpeg.Encode(&buf, thumb, &jpeg.Options{Quality: 75}); err != nil { + return nil + } + return buf.Bytes() +} + +// trimExt 去掉文件名的扩展名 +func trimExt(fileName string) string { + if idx := strings.LastIndex(fileName, "."); idx > 0 { + return fileName[:idx] + } + return fileName +} diff --git a/internal/media/scraper/language.go b/internal/media/scraper/language.go new file mode 100644 index 00000000..df8c0745 --- /dev/null +++ b/internal/media/scraper/language.go @@ -0,0 +1,27 @@ +package scraper + +import ( + "strings" + + "github.com/OpenListTeam/OpenList/v4/internal/conf" + "github.com/OpenListTeam/OpenList/v4/internal/setting" +) + +// ScrapeLanguage 返回首选刮削语言(BCP-47,如 zh-CN / en-US),未配置时默认中文。 +func ScrapeLanguage() string { + lang := strings.TrimSpace(setting.GetStr(conf.MediaScrapeLanguage, "zh-CN")) + if lang == "" { + return "zh-CN" + } + return lang +} + +// ScrapeLanguagePrimary 返回 ISO 639-1 主语言代码(如 zh / en), +// 供只接受两位语言码的接口使用(Google Books 的 langRestrict)。 +func ScrapeLanguagePrimary() string { + lang := ScrapeLanguage() + if idx := strings.IndexAny(lang, "-_"); idx > 0 { + lang = lang[:idx] + } + return strings.ToLower(lang) +} diff --git a/internal/media/scraper/limits.go b/internal/media/scraper/limits.go new file mode 100644 index 00000000..a1883cc5 --- /dev/null +++ b/internal/media/scraper/limits.go @@ -0,0 +1,65 @@ +package scraper + +import ( + "bytes" + "errors" + "image" + "io" +) + +// 刮削阶段会把文件整体读入内存再解码,必须限制单文件大小与像素总量, +// 否则几路并发刮削一张超大图片/压缩炸弹就能把进程内存打满。 +const ( + // MaxImageFileBytes 单张待刮削图片的最大字节数 + MaxImageFileBytes int64 = 32 << 20 + // MaxBookFileBytes 单本待提取封面的书籍(epub/pdf)最大字节数 + MaxBookFileBytes int64 = 128 << 20 + // MaxEmbeddedEntryBytes epub(zip) 内单个封面条目的最大解压字节数,防止 zip 炸弹 + MaxEmbeddedEntryBytes int64 = 16 << 20 + // MaxImagePixels 解码前允许的最大像素总量(约 5657x5657)。 + // 解码后按 RGBA 计算约 128MB,默认 5 路并发刮削最坏约 640MB; + // 原来的 64MP 单张就要 256MB,并发时会接近 1.3GB。 + MaxImagePixels int64 = 32_000_000 + // MaxRawCoverFallbackBytes 解码失败时退回「原始字节 base64 写库」的上限。 + // 超过就直接放弃封面——否则单条记录可能写入 16MB 以上的 base64。 + MaxRawCoverFallbackBytes = 512 << 10 +) + +// ErrTooLarge 内容超过读取上限 +var ErrTooLarge = errors.New("scraper: content exceeds size limit") + +// decodeSem 限制同时解码的图片数:解码阶段的内存峰值 ≈ 像素数 × 4, +// 只靠单张上限无法约束多路并发叠加后的总量。 +// 容量必须小于默认刮削并发(5),否则信号量等于不起作用: +// 每路最坏 32MP × 4B ≈ 128MB,12 路就是 1.5GB。 +// 解码是 CPU 密集型操作,并发超过核数也不会更快,取 3 兼顾内存与吞吐。 +var decodeSem = make(chan struct{}, 3) + +// readAllLimited 最多读取 max 字节;超过上限返回 ErrTooLarge(不会继续读取剩余内容)。 +func readAllLimited(r io.Reader, max int64) ([]byte, error) { + data, err := io.ReadAll(io.LimitReader(r, max+1)) + if err != nil { + return nil, err + } + if int64(len(data)) > max { + return nil, ErrTooLarge + } + return data, nil +} + +// decodeBounded 先只读取图片头部获取尺寸,超过像素上限直接拒绝,再做完整解码。 +func decodeBounded(data []byte) (image.Image, error) { + cfg, _, err := image.DecodeConfig(bytes.NewReader(data)) + if err != nil { + return nil, err + } + // 用 int64 相乘:32 位平台上 int 溢出会让超限图片绕过检查 + if cfg.Width <= 0 || cfg.Height <= 0 || + int64(cfg.Width)*int64(cfg.Height) > MaxImagePixels { + return nil, ErrTooLarge + } + decodeSem <- struct{}{} + defer func() { <-decodeSem }() + img, _, err := image.Decode(bytes.NewReader(data)) + return img, err +} diff --git a/internal/media/scraper/thumb_store.go b/internal/media/scraper/thumb_store.go new file mode 100644 index 00000000..54db8610 --- /dev/null +++ b/internal/media/scraper/thumb_store.go @@ -0,0 +1,160 @@ +package scraper + +import ( + "bytes" + "context" + "crypto/sha1" + "encoding/base64" + "encoding/hex" + "fmt" + "io" + "net/http" + "net/url" + "os" + stdpath "path" + "path/filepath" + "strings" + "time" + + "github.com/OpenListTeam/OpenList/v4/internal/conf" + "github.com/OpenListTeam/OpenList/v4/internal/fs" + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/internal/sign" + "github.com/OpenListTeam/OpenList/v4/internal/stream" +) + +// 缩略图存储模式 +const ( + // ThumbModeBase64 缩略图以 base64 存入数据库 + ThumbModeBase64 = "base64" + // ThumbModeRuntime 缩略图存入程序数据目录(/) + ThumbModeRuntime = "runtime" + // ThumbModeVFS 缩略图写入挂载目录(VFS 路径) + ThumbModeVFS = "local" +) + +// NormalizeThumbMode 归一化缩略图模式,未知/空值按运行目录处理 +func NormalizeThumbMode(mode string) string { + switch strings.ToLower(strings.TrimSpace(mode)) { + case ThumbModeBase64: + return ThumbModeBase64 + case ThumbModeVFS, "vfs": + return ThumbModeVFS + default: + return ThumbModeRuntime + } +} + +// RuntimeThumbRoot 返回运行目录模式下缩略图的本地根目录 +func RuntimeThumbRoot(thumbPath string) string { + dir := strings.Trim(strings.TrimSpace(thumbPath), "/") + if dir == "" { + dir = "imgs" + } + return filepath.Join(conf.DataDir, filepath.FromSlash(dir)) +} + +// SaveThumb 按模式保存缩略图数据,返回 cover 字段值。 +// rawPath 为媒体文件的完整 VFS 路径,用于生成稳定且唯一的文件名。 +func SaveThumb(ctx context.Context, mode, thumbPath, rawPath, mimeType string, data []byte) string { + if len(data) == 0 { + return "" + } + mode = NormalizeThumbMode(mode) + if mimeType == "" { + mimeType = "image/jpeg" + } + if mode == ThumbModeBase64 { + return "data:" + mimeType + ";base64," + base64.StdEncoding.EncodeToString(data) + } + + dir := strings.Trim(strings.TrimSpace(thumbPath), "/") + if dir == "" { + dir = "imgs" + } + name := thumbName(rawPath) + + if mode == ThumbModeVFS { + vfsDir := "/" + dir + vfsFile := stdpath.Join(vfsDir, name) + file := &stream.FileStream{ + Ctx: ctx, + Obj: &model.Object{ + Name: name, + Size: int64(len(data)), + Modified: time.Now(), + }, + Reader: bytes.NewReader(data), + Mimetype: mimeType, + } + if err := fs.PutDirectly(ctx, vfsDir, file, true); err != nil { + return "" + } + // 带上签名,兼容开启「签名所有」的站点 + return "/d" + vfsFile + "?sign=" + sign.Sign(vfsFile) + } + + // 运行目录模式:// + root := RuntimeThumbRoot(dir) + if err := os.MkdirAll(root, 0755); err != nil { + return "" + } + if err := os.WriteFile(filepath.Join(root, name), data, 0644); err != nil { + return "" + } + return "thumb:/" + stdpath.Join(dir, name) +} + +// thumbName 根据媒体文件路径生成缩略图文件名 +func thumbName(rawPath string) string { + sum := sha1.Sum([]byte(rawPath)) + return hex.EncodeToString(sum[:]) + ".jpg" +} + +// DownloadCoverToStorage 下载远程封面并按缩略图配置保存,失败返回空字符串 +func DownloadCoverToStorage(ctx context.Context, imgURL, rawPath, thumbMode, thumbPath string) string { + if imgURL == "" || + !(strings.HasPrefix(imgURL, "http://") || strings.HasPrefix(imgURL, "https://")) { + return "" + } + data, contentType, err := fetchImageBytes(ctx, imgURL) + if err != nil { + // 部分网络环境无法直连图片域名,回退到公共图片网关 + if u, parseErr := url.Parse(imgURL); parseErr == nil { + gateway := "https://wsrv.nl/?url=" + url.QueryEscape( + strings.TrimPrefix(strings.TrimPrefix(u.String(), "https://"), "http://"), + ) + data, contentType, err = fetchImageBytes(ctx, gateway) + } + } + if err != nil || len(data) == 0 { + return "" + } + return SaveThumb(ctx, thumbMode, thumbPath, rawPath, contentType, data) +} + +func fetchImageBytes(ctx context.Context, raw string) ([]byte, string, error) { + client := &http.Client{Timeout: 20 * time.Second} + req, err := http.NewRequestWithContext(ctx, http.MethodGet, raw, nil) + if err != nil { + return nil, "", err + } + req.Header.Set("User-Agent", "Mozilla/5.0 (compatible; OpenList-Media/1.0)") + resp, err := client.Do(req) + if err != nil { + return nil, "", err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return nil, "", fmt.Errorf("unexpected status %d", resp.StatusCode) + } + contentType := resp.Header.Get("Content-Type") + if contentType == "" { + contentType = "image/jpeg" + } + if !strings.HasPrefix(contentType, "image/") { + return nil, "", fmt.Errorf("unexpected content type %s", contentType) + } + data, err := io.ReadAll(io.LimitReader(resp.Body, 10<<20)) + return data, contentType, err +} diff --git a/internal/media/scraper/tmdb.go b/internal/media/scraper/tmdb.go new file mode 100644 index 00000000..7b69eb44 --- /dev/null +++ b/internal/media/scraper/tmdb.go @@ -0,0 +1,1448 @@ +package scraper + +import ( + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + stdpath "path" + "regexp" + "sort" + "strings" + "sync/atomic" + "time" + "unicode/utf8" + + "github.com/OpenListTeam/OpenList/v4/internal/model" + log "github.com/sirupsen/logrus" +) + +// 年份正则(匹配 1900-2099) +var yearRegexp = regexp.MustCompile(`\b((?:19|20)\d{2})\b`) + +// chineseRegexp 匹配包含中文字符的片段 +var chineseRegexp = regexp.MustCompile(`[\p{Han}]`) + +// 噪声词正则:发布组、编码、分辨率、音轨等无意义片段 +// 清洗中文标题尾部常见的污染词 +// 注意:这里只用于「子串匹配」清洗,不要求整段命中 +var noiseTokenRegexp = regexp.MustCompile(`(?i)(双语字幕|双语|中字|中英字幕|中英双字|中英双语|国英双语|国英双轨|国粤双语|粤语中字|国语中字|英语中字|日语中字|韩语中字|国英|国粤|粤语|国语|英语|日语|韩语|内封字幕|外挂字幕|HDTV|HR-HDTV|BluRay|BDRip|WEB-?DL|HDRip|DVDRip|REMUX|x264|x265|h264|h265|HEVC|AVC|AAC|AC3|DTS|FLAC|10bit|8bit|HDR|SDR|4K|2160P|1080P|720P|480P|完整版|未删减版|加长版|导演剪辑版|蓝光版|高清版|正版上译公映|正版上译|上译公映|公映译制|译制公映|公映版|上译版|京译版|长译版|国配版|台配版|港配版|国语配音|粤语配音|国粤配音|原声版|原盘|配音版|译制版)`) + +// chineseReleaseGroupRe 中文字幕组/压制组/影视站名的整段匹配正则 +// 这些词出现在文件名中通常是发布信息,绝不应该被当成片名候选 +// 例:「-人人影视制作」「.YYeTs.」「-飞鸟影视」 +var chineseReleaseGroupRe = regexp.MustCompile(`^(?:` + + `人人影视制作|人人影视|人人字幕|人人字幕组|YYeTs|YYETs|FRDS|` + + `飞鸟影视|飞鸟影院|风行网|风行影视|远鉴字幕组|远鉴|深影字幕组|深影|` + + `破烂熊|圣城家园|TLF|HDS|MNHD|WiKi|CMCT|RARBG|YIFY|YTS|` + + `蓝色狂想|蓝色狂想字幕组|肥羊字幕组|衣柜字幕组|喵萌奶茶屋|` + + `众乐字幕组|猪猪乐园|猪猪字幕组|流鸣字幕|动漫国字幕组|诸神字幕组|` + + `字幕组|压制组|发布组|官方版本|高清影视|高清电影|蓝光原盘` + + `)$`) + +// chineseGarbageContainsRe 包含即视为发布组信息的中文片段(用于片段中嵌有这些字眼时整体丢弃) +// 例:「人人影视」、「天天美剧」即使附带其他字符,整段都不太可能是片名 +var chineseGarbageContainsRe = regexp.MustCompile(`人人影视|人人字幕|YYeTs|字幕组|压制组|发布组|正版上译|上译公映|公映译制|译制公映|公映版|上译版|国配版|台配版|港配版|配音版|译制版`) + +// 中文数字到阿拉伯数字的简单映射,用于「钢铁侠三」=>「钢铁侠3」类的归一化 +var cnNumMap = map[string]string{ + "〇": "0", "零": "0", "一": "1", "二": "2", "三": "3", "四": "4", + "五": "5", "六": "6", "七": "7", "八": "8", "九": "9", "十": "10", +} + +// normalizeTitle 对标题做模糊匹配前的归一化处理 +// - 去除括号及其中内容 +// - 去除版本/编码等噪声词 +// - 合并多余空白 +func normalizeTitle(s string) string { + if s == "" { + return s + } + // 去掉中英文括号包裹的内容 + bracketRe := regexp.MustCompile(`[\((\[【][^\))\]】]*[\))\]】]`) + s = bracketRe.ReplaceAllString(s, " ") + // 去掉常见噪声词 + s = noiseTokenRegexp.ReplaceAllString(s, " ") + // 替换分隔符为空格 + s = strings.NewReplacer(".", " ", "_", " ", "-", " ", "+", " ").Replace(s) + // 合并多余空白 + s = regexp.MustCompile(`\s+`).ReplaceAllString(s, " ") + return strings.TrimSpace(s) +} + +// cnNumToArabic 将标题中的中文数字归一化为阿拉伯数字(仅做轻量处理) +func cnNumToArabic(s string) string { + if s == "" { + return s + } + var b strings.Builder + for _, r := range s { + if v, ok := cnNumMap[string(r)]; ok { + b.WriteString(v) + } else { + b.WriteRune(r) + } + } + return b.String() +} + +// parsedVideoTitle 解析后的视频标题信息 +// +// 解析时除了主中文标题/英文标题之外,可能还会从文件名中识别到其他中文片段 +// (比如同时存在「[钢铁侠]」和「Iron Man」,或多段中文),这些都作为兜底候选保留。 +// 主中文/英文标题用于「常规」搜索路径;额外候选会在主候选搜不到时再尝试。 +type parsedVideoTitle struct { + EnglishTitle string // 英文标题(第一个中文片段之前、年份之前的部分) + ChineseTitle string // 中文标题(最优中文片段) + ExtraChineseTitles []string // 其他中文片段,作为兜底候选(如方括号外又出现的中文标题、副标题等) + Year string // 年份 +} + +// leadingSerialSpaceRe 匹配开头是 "数字 + 空格"(如 "01 [钢铁侠]"、"19 [复仇者联盟3:无限战争]") +// 数字 + 空格 几乎可以确定是序号(合法片名极少以"数字 + 空格"开头) +// 只匹配 1-3 位数字,避免把年份(2008) 或 4 位以上数字当成序号 +var leadingSerialSpaceRe = regexp.MustCompile(`^\s*\d{1,3}\s+`) + +// leadingSerialDotChineseRe 匹配开头是 "数字 + 点/-/_/、 + 中文" 的形式 +// 这种形式既可能是"序号 + 中文片名",也可能是"数字片名 + 中文别名",比如: +// +// "169.谍影重重3" -> 序号 + 中文片名 +// "1.漫威短片.47号物品" -> 序号 + 中文片名 +// "300.斯巴达勇士..." -> 数字片名 + 中文别名(300 是片名核心) +// "36总局" 这种中文里就含数字的也算 +// +// 启发式判定: +// - 1-2 位数字(1-99):序号概率高,删掉数字 +// - 3 位数字(100-999):片名概率高,保留数字 +// - 4 位以上:在外层正则就被 \d{1,3} 排除 +var leadingSerialDotChineseRe = regexp.MustCompile(`^\s*(\d{1,3})\s*[.\-_、]\s*(\p{Han})`) + +// stripLeadingSerial 去除文件名开头的序号前缀,返回剥离后的文件名 +// +// 处理两种序号情况: +// 1. "数字 + 空格":如 "01 [钢铁侠]Iron.Man..." -> "[钢铁侠]Iron.Man..." +// 2. "1-2 位数字 + 点/-/_/、 + 中文":如 "1.漫威短片..." -> "漫威短片..."、"169.谍影重重3" -> "谍影重重3" +// +// 不视为序号(保留原样): +// - "30.Days.Of.Night..."(数字 + 点 + 英文,数字是片名一部分) +// - "300.Rise.Of.An.Empire..."(同上) +// - "300.斯巴达勇士..."(3 位数字 + 点 + 中文,可能是片名 "300") +// - "3096.Days.2013..."(4 位数字直接被排除) +// +// 被剥离的序号本身不是片名关键词,不需要回填。 +func stripLeadingSerial(s string) string { + // 1) "数字 + 空格":稳定删除(序号信号最强) + if loc := leadingSerialSpaceRe.FindStringIndex(s); loc != nil && loc[0] == 0 { + return strings.TrimSpace(s[loc[1]:]) + } + // 2) "1-2 位数字 + 分隔符 + 中文":视为序号删除;3 位数字保守保留 + if m := leadingSerialDotChineseRe.FindStringSubmatchIndex(s); m != nil && m[0] == 0 { + // m[2]:m[3] 是数字捕获组的索引;m[4]:m[5] 是中文字符 + numStr := s[m[2]:m[3]] + // 3 位数字保守保留(避免删除 "300.斯巴达勇士" 中的 "300") + if len(numStr) >= 3 { + return s + } + // 1-2 位数字视为序号删除,从中文字符位置开始保留 + return strings.TrimSpace(s[m[4]:]) + } + return s +} + +// noiseTokenWholeRe 判断一个完整片段是否为“纯噪声词”(完全匹配,不包含其他字符) +// 覆盖常见发布组/字幕组/编码/音轨/分辨率/语言标记等。被识别为噪声的片段不会进入 EnglishTitle。 +// 例:"HR-HDTV"、"AC3"、"x264"、"X264"、"1024x576"、"2160p"、"BOBO"(发布组) 等 +var noiseTokenWholeRe = regexp.MustCompile(`(?i)^(?:` + + // 字幕/语言标记 + `双语字幕|中字|国英|国粤|粤语|国语|英语|日语|韩语|国英双轨|国粤英双轨|中英双字|` + + `English|Chinese|Cantonese|Mandarin|Japanese|Korean|CHS|CHT|ENG|JPN|KOR|CHS-ENG|CHS-JPN|CHT-ENG|` + + // 片源/容器 + `HDTV|HR-HDTV|HR\.HDTV|BluRay|Blu-Ray|BDRip|BDMV|WEB-?DL|WEBRip|HDRip|DVDRip|DVD|REMUX|UHD|RAW|TS|TC|CAM|HC|TVRip|` + + // 视频编码 + `x264|x265|h264|h265|HEVC|AVC|XviD|DivX|VP9|AV1|10bit|8bit|HDR10\+?|HDR|SDR|DV|DolbyVision|` + + // 音频编码 + `AAC|AAC2\.0|AC3|EAC3|DD5\.1|DD7\.1|DD\+|DDP|DTS|DTS-HD|DTS-X|DTSX|DTSHD|FLAC|MP3|TrueHD|Atmos|MA|` + + // 分辨率 / 画质 + `4K|2K|8K|2160p|2160P|1080p|1080P|720p|720P|480p|480P|\d{3,4}[xX×]\d{3,4}|` + + // 中文描述词 + `完整版|未删减版|院线版|导演剪辑版|加长版|年度佳作|` + + // 常见发布组名(可能不完全,但覆盖示例中出现过的) + `BOBO|SWTYBLZ|CMCT|HDS|FRDS|MNHD|WiKi|TLF|RARBG|YIFY|YTS|EVO|SPARKS` + + `)$`) + +// isNoiseToken 判断一个片段是否是完全的噪声词。 +// 同时覆盖一些“数字.数字”组合被 "." 拆出后的碎片,如“5”、“1”、“7”(来自 5.1/7.1 音轨拆分) +func isNoiseToken(p string) bool { + if p == "" { + return true + } + // 微小碎片(单个或两个数字)但不是年份,一般是被"."拆分开的 5.1/7.1/2.0 碎片。 + // 为避免误伤「4 Months 3 Weeks And 2 Days」这种中间字随意出现的数字,不在这里过滤, + // 仅依靠 noiseTokenWholeRe 覆盖。 + return noiseTokenWholeRe.MatchString(p) +} + +// extractBracketChinese 从原始文件名中提取出方括号/中文括号里的中文片段 +// +// "01 [钢铁侠]Iron.Man.2008..." -> ["钢铁侠"] +// "19 [复仇者联盟3:无限战争]Avengers..." -> ["复仇者联盟3:无限战争"] +// "[Pixar][玩具总动员]Toy.Story.1995..." -> ["玩具总动员"] +// +// 仅返回包含中文的括号内容,避免把发布组、字幕组等英文括号信息当成标题。 +func extractBracketChinese(fileName string) []string { + bracketRe := regexp.MustCompile(`[\((\[【]([^\))\]】]*)[\))\]】]`) + matches := bracketRe.FindAllStringSubmatch(fileName, -1) + var out []string + for _, m := range matches { + if len(m) < 2 { + continue + } + inner := strings.TrimSpace(m[1]) + if inner == "" { + continue + } + // 必须包含中文字符才视为中文标题候选 + if chineseRegexp.MatchString(inner) { + out = append(out, inner) + } + } + return out +} + +// parseVideoFileName 从视频文件名中提取标题和年份 +// 规则: +// - 自动去除文件名开头的序号(如 "01 "、"1."、"169."),避免污染英文标题 +// - 优先把方括号 [中文] 内的中文片段作为中文标题候选 +// - 文件名中允许使用 . / 空格 / _ / 的分隔多个字段 +// - 第一个含中文的片段即为中文标题(若上一步括号内未找到中文) +// - 中文标题之前的非中文、非年份片段拼接为英文标题 +// - 第一个 1900-2099 之间的 4 位数字识别为年份 +// +// 例如: +// +// "Inception.2010.盗梦空间.双语字幕.HR-HDTV.AC3.1024X576.X264-" -> {English:"Inception", Chinese:"盗梦空间", Year:"2010"} +// "Iron.Man.3.2013.钢铁侠3.国英音轨.双语字幕.HR-HDTV.AC3.x264-" -> {English:"Iron Man 3", Chinese:"钢铁侠3", Year:"2013"} +// "The.Dark.Knight.2008.1080p.BluRay" -> {English:"The Dark Knight", Chinese:"", Year:"2008"} +// "盗梦空间.2010.1080p.BluRay" -> {English:"", Chinese:"盗梦空间", Year:"2010"} +// "钢铁侠3 (2013).mkv" -> {English:"", Chinese:"钢铁侠3", Year:"2013"} +// "群星-演唱会现场.mp4" -> {English:"", Chinese:"群星", Year:""} +// "01 [钢铁侠]Iron.Man.2008.2160p.HDR.BluRay..." -> {English:"Iron Man", Chinese:"钢铁侠", Year:"2008"} +// "19 [复仇者联盟3:无限战争]Avengers.Infinity.War.2018.2160p..." -> {English:"Avengers Infinity War", Chinese:"复仇者联盟3:无限战争", Year:"2018"} +// "169.谍影重重3" -> {English:"", Chinese:"谍影重重3", Year:""} +// "1.漫威短片.47号物品" -> {English:"", Chinese:"漫威短片", Year:""}(额外候选含 "47号物品") +func parseVideoFileName(fileName string) parsedVideoTitle { + // 去掉扩展名(.mkv .mp4 .avi 等,扩展名长度 <= 5) + if idx := strings.LastIndex(fileName, "."); idx > 0 { + ext := strings.ToLower(fileName[idx:]) + if len(ext) <= 5 { + fileName = fileName[:idx] + } + } + + // 1) 先把开头的序号 "01 "、"1."、"169." 去掉,避免它们污染英文标题 + // 仅在能明确判定为序号的场景才剥离,避免误伤 "30.Days..."、"300.斯巴达勇士" 这类数字作为片名一部分的情况 + fileName = stripLeadingSerial(fileName) + + // 2) 在剥离括号之前,先尝试从方括号 [中文] 中抽取中文标题候选 + // 很多发布组习惯是 "01 [中文标题]English.Title.Year...",这里要保留 "中文标题" + bracketChinese := extractBracketChinese(fileName) + + // 3) 提取括号中的年份(如 "钢铁侠3 (2013).mkv"),把年份替换到括号外 + bracketYearRe := regexp.MustCompile(`[\((\[【]\s*((?:19|20)\d{2})\s*[\))\]】]`) + fileName = bracketYearRe.ReplaceAllString(fileName, " $1 ") + // 4) 再把剩余括号块替换成空格(其中的中文标题已在第 2 步保存到 bracketChinese) + bracketRe := regexp.MustCompile(`[\((\[【][^\))\]】]*[\))\]】]`) + fileName = bracketRe.ReplaceAllString(fileName, " ") + + // 把多种分隔符统一成 ".",方便后续按 "." 解析 + // 注意:中文之间通常没分隔符,应保留;这里只把 "." 之外的分隔符换成 "." + replacer := strings.NewReplacer( + "_", ".", + "+", ".", + ) + fileName = replacer.Replace(fileName) + + // 把空格、" - " 也作为分隔符(但只在 ASCII 段之间使用,避免破坏中文), + // 简化处理:直接把它们也变成 "." + fileName = regexp.MustCompile(`[\s]+`).ReplaceAllString(fileName, ".") + fileName = strings.ReplaceAll(fileName, "-", ".") + + // 按"."分割各字段 + parts := strings.Split(fileName, ".") + + var result parsedVideoTitle + var englishParts []string + var chineseParts []string // 收集所有中文片段,便于兜底 + foundYear := false + foundChinese := false // 一旦出现中文片段,后面的非中文片段都不再加入英文标题 + + // pureNoise 判断字符串经过噪声词清洗后是否为空(即整体都是噪声词) + // 用于过滤掉"双语字幕"、"国英音轨" 这种纯噪声片段,避免被当成中文标题 + pureNoise := func(s string) bool { + cleaned := noiseTokenRegexp.ReplaceAllString(s, "") + cleaned = strings.TrimSpace(cleaned) + return cleaned == "" + } + + // isChineseReleaseGroup 判断中文片段是否为字幕组/压制组名 + // 这类片段绝不应该被当成片名候选(如 "人人影视"、"YYeTs"、"飞鸟影视") + isChineseReleaseGroup := func(s string) bool { + s = strings.TrimSpace(s) + if s == "" { + return false + } + if chineseReleaseGroupRe.MatchString(s) { + return true + } + if chineseGarbageContainsRe.MatchString(s) { + return true + } + return false + } + + for _, p := range parts { + p = strings.TrimSpace(p) + if p == "" { + continue + } + + // 检测是否含中文 + if chineseRegexp.MatchString(p) { + // 整体被识别为噪声词的中文片段(如 "双语字幕"、"国英音轨")直接跳过 + if pureNoise(p) { + continue + } + // 字幕组/压制组中文名(如 "人人影视"、"飞鸟影院")也直接跳过 + if isChineseReleaseGroup(p) { + continue + } + chineseParts = append(chineseParts, p) + foundChinese = true + continue + } + + // 检测是否为年份(1900-2099) + if yearRegexp.MatchString(p) && !foundYear { + result.Year = yearRegexp.FindString(p) + foundYear = true + // 年份本身不加入英文标题 + continue + } + + // 后续非中文片段加入英文标题的判定: + // - 必须在年份出现之前(年份后的 ASCII 全是发布信息噪声) + // - 必须在中文出现之前(中文后的 ASCII 也是发布信息噪声,避免 "斯巴达勇士 HR HDTV AC3 X264" 这种污染) + // - 不能是完全的噪声词(如 HR-HDTV、AC3、x264、分辨率) + if foundYear || foundChinese { + continue + } + if isNoiseToken(p) { + continue + } + englishParts = append(englishParts, p) + } + + result.EnglishTitle = strings.Join(englishParts, " ") + + // 选定中文标题: + // - 优先使用方括号内的中文(最可靠的发布组标记) + // - 否则使用解析出的中文片段中第一个 + // - 同时把所有中文候选去重保留到 ExtraChineseTitles 里,留作兜底 + allChinese := append([]string{}, bracketChinese...) + allChinese = append(allChinese, chineseParts...) + allChinese = dedupStrings(allChinese) + + if len(allChinese) > 0 { + result.ChineseTitle = allChinese[0] + if len(allChinese) > 1 { + result.ExtraChineseTitles = allChinese[1:] + } + } + + // 当解析出多个连续中文片段时,它们很可能是「主标题.副标题」结构(如 "300勇士.帝国崛起" + // 实际是「300勇士:帝国崛起」),把合并后的候选也加入 ExtraChineseTitles 用于兜底搜索。 + // 注意:合并最多保留前两段(主标题 + 副标题),避免「主标题:副标题:发布版本」这类 + // 把发布信息(正版上译公映等)当成第二副标题的脏候选;电影/剧名极少有真正的三段式官方片名。 + if len(chineseParts) > 1 { + mergeParts := chineseParts + if len(mergeParts) > 2 { + mergeParts = mergeParts[:2] + } + merged := strings.Join(mergeParts, ":") + // 避免与已有候选重复 + exists := merged == result.ChineseTitle + for _, t := range result.ExtraChineseTitles { + if t == merged { + exists = true + break + } + } + if !exists { + result.ExtraChineseTitles = append(result.ExtraChineseTitles, merged) + } + } + + // 对中文标题做尾部噪声清洗(去除「钢铁侠3 国英音轨」末尾的污染词,但保留主标题) + if result.ChineseTitle != "" { + result.ChineseTitle = stripTrailingNoise(result.ChineseTitle) + } + for i, t := range result.ExtraChineseTitles { + result.ExtraChineseTitles[i] = stripTrailingNoise(t) + } + return result +} + +// dedupStrings 去除字符串切片中的空白与重复项,保留首次出现顺序 +func dedupStrings(in []string) []string { + seen := make(map[string]bool) + out := make([]string, 0, len(in)) + for _, s := range in { + s = strings.TrimSpace(s) + if s == "" || seen[s] { + continue + } + seen[s] = true + out = append(out, s) + } + return out +} + +// stripTrailingNoise 去除中文标题中嵌入的噪声词,保留主标题部分 +// 如「钢铁侠3 国英音轨」-> 「钢铁侠3」 +// 同时剖除末尾的 (1) (2) V2 这类重复/版本后缀、字幕组名等 +func stripTrailingNoise(s string) string { + cleaned := noiseTokenRegexp.ReplaceAllString(s, " ") + // 剖除「-人人影视」「.YYeTs」这种跟在中文后面的字幕组名 + cleaned = regexp.MustCompile(`[\s\-・]*`+ + `(?:人人影视制作|人人影视|人人字幕|YYeTs|YYETs|FRDS|飞鸟影视|飞鸟影院|风行网|深影|远鉴|字幕组|压制组)[s]?`+ + `(?:、|。|\.|,)?\s*$`).ReplaceAllString(cleaned, " ") + // 剖除末尾的 (1)(重复文件标记)、V2、v3 这类版本后缀 + cleaned = regexp.MustCompile(`(?i)\s*[\((]\s*\d+\s*[\))]\s*$`).ReplaceAllString(cleaned, " ") + cleaned = regexp.MustCompile(`(?i)\s*[Vv]\s*\d+\s*$`).ReplaceAllString(cleaned, " ") + cleaned = regexp.MustCompile(`\s+`).ReplaceAllString(cleaned, " ") + cleaned = strings.TrimSpace(cleaned) + if cleaned == "" { + return strings.TrimSpace(s) + } + return cleaned +} + +const tmdbDefaultBaseURL = "https://api.themoviedb.org" +const tmdbImageBase = "https://image.tmdb.org/t/p/w500" + +// TMDBStats TMDB 刮削统计(线程安全:原子计数) +// 仅记录一次进程级别的累积值;如需更精细可按 ScraperID 分桶 +type TMDBStats struct { + TotalScraped int64 // 总刮削请求次数(每次 ScrapeVideo 调用 +1) + HitCount int64 // 成功刮到结果的次数(搜索阶段命中即停) + NoMatchCount int64 // 完全 0 命中的次数 + LowConfidenceCount int64 // 命中但低于阈值,使用了「最佳兜底」结果的次数 + TotalAttempts int64 // 累计 TMDB 搜索请求次数(不含详情) + TotalSearchMillis int64 // 累计搜索耗时(毫秒) +} + +// 全局统计实例 +var globalTMDBStats = &TMDBStats{} + +// GetTMDBStats 返回当前进程的 TMDB 刮削统计快照 +func GetTMDBStats() TMDBStats { + return *globalTMDBStats +} + +// ResetTMDBStats 重置统计(仅用于测试或运维触发) +func ResetTMDBStats() { + *globalTMDBStats = TMDBStats{} +} + +// matchConfidenceThreshold 命中可信度阈值。 +// scoreMatch >= 该阈值视为「足够可信」,立即返回; +// 介于 fallbackThreshold 和该阈值之间则记为「最佳兜底」; +// 低于 fallbackThreshold 进入「降级重试」阶段;都失败才报错。 +const ( + matchConfidenceThreshold float32 = 0.60 + fallbackThreshold float32 = 0.30 + // desperateThreshold 「降级重试」阶段的阈值:当主流程完全无任何 score>=0.30 的命中时, + // 采用更宽的阈值(0.20)接受 score 在 [0.20, 0.30) 之间的结果作为最后的兜底。 + desperateThreshold float32 = 0.20 +) + +// TMDBScraper TMDB视频刮削器 +type TMDBScraper struct { + APIKey string + BaseURL string // 含 /3 版本路径的最终请求地址,例如 https://api.themoviedb.org/3 + client *http.Client +} + +// NewTMDBScraper 创建TMDB刮削器 +// baseURL 为可自定义的 TMDB API 服务地址,为空时使用默认 https://api.themoviedb.org/3 +// +// 支持的填写形式: +// 1. 留空:使用默认 https://api.themoviedb.org/3 +// 2. 官方主域名(不带路径):如 api.themoviedb.org / https://api.themoviedb.org,自动补全协议和 /3 版本路径 +// 3. 反代/自定义完整路径:如 https://eo-tmd.example.com/api/tmdb/3,会原样使用,不再追加 /3 +func NewTMDBScraper(apiKey string, baseURL string) *TMDBScraper { + base := strings.TrimRight(strings.TrimSpace(baseURL), "/") + if base == "" { + base = tmdbDefaultBaseURL + } + // 1. 协议头容错:用户没填 http/https 时自动补 https:// + if !strings.HasPrefix(base, "http://") && !strings.HasPrefix(base, "https://") { + base = "https://" + base + } + // 2. 版本号容错:约定前端要求填写「域名 + API 前缀」 + // - 官方:api.themoviedb.org -> https://api.themoviedb.org/3 + // - 反代:example.edgeone.run/api/tmdb/ -> https://example.edgeone.run/api/tmdb/3 + // - 反代:example.edgeone.run/api/tmdb -> https://example.edgeone.run/api/tmdb/3 + // - 用户已填到 /3:原样保留,不重复追加 + if !strings.HasSuffix(base, "/3") { + base = base + "/3" + } + return &TMDBScraper{ + APIKey: apiKey, + BaseURL: base, + client: &http.Client{Timeout: 15 * time.Second}, + } +} + +// tmdbSearchResult TMDB搜索结果 +type tmdbSearchResult struct { + Results []struct { + ID int `json:"id"` + Title string `json:"title"` + Name string `json:"name"` // 电视剧用name + Overview string `json:"overview"` + PosterPath string `json:"poster_path"` + ReleaseDate string `json:"release_date"` + FirstAirDate string `json:"first_air_date"` // 电视剧 + VoteAverage float32 `json:"vote_average"` + MediaType string `json:"media_type"` + GenreIDs []int `json:"genre_ids"` + } `json:"results"` +} + +// tmdbMovieDetail TMDB电影详情 +type tmdbMovieDetail struct { + ID int `json:"id"` + Title string `json:"title"` + Overview string `json:"overview"` + PosterPath string `json:"poster_path"` + ReleaseDate string `json:"release_date"` + VoteAverage float32 `json:"vote_average"` + Genres []struct { + ID int `json:"id"` + Name string `json:"name"` + } `json:"genres"` + Credits struct { + Cast []struct { + Name string `json:"name"` + } `json:"cast"` + } `json:"credits"` +} + +// doTMDBSearch 执行一次TMDB搜索请求 +// +// endpoint: +// - movie:/search/movie,参数参考 https://developer.themoviedb.org/reference/search-movie +// 支持 query / language / primary_release_year / year / region / include_adult / page +// - tv:/search/tv,参数参考 https://developer.themoviedb.org/reference/search-tv +// 支持 query / language / first_air_date_year / year / include_adult / page +// - multi:/search/multi,参数参考 https://developer.themoviedb.org/reference/search-multi +// 支持 query / language / include_adult / page / region(不支持按年份过滤) +// +// language 取值:zh-CN / en-US / 空字符串(不限定语言,让 TMDB 用所有别名匹配,模糊兜底) +// 注意:search_type=ngram 是早期废弃参数;TMDB 当前默认即支持子串/模糊匹配 +func (s *TMDBScraper) doTMDBSearch(endpoint, query, year, language string) (*tmdbSearchResult, error) { + if endpoint == "" { + endpoint = "movie" + } + if strings.TrimSpace(query) == "" { + return &tmdbSearchResult{}, nil + } + + params := url.Values{} + params.Set("api_key", s.APIKey) + params.Set("query", query) + params.Set("include_adult", "true") + params.Set("page", "1") + if language != "" { + params.Set("language", language) + } + if year != "" { + switch endpoint { + case "movie": + // movie 同时设置 year 和 primary_release_year,TMDB 文档建议优先使用 primary_release_year + params.Set("year", year) + params.Set("primary_release_year", year) + case "tv": + // tv 接口使用 first_air_date_year + params.Set("first_air_date_year", year) + params.Set("year", year) + case "multi": + // /search/multi 不支持按年份过滤;忽略 year 参数避免被反代/官方判为非法参数 + } + } + + searchURL := fmt.Sprintf("%s/search/%s?%s", s.BaseURL, endpoint, params.Encode()) + + // 输出脱敏后的请求 URL,便于排查刮削失败问题 + // (把 api_key 替换为 ***,避免日志泄漏密钥) + log.Debugf("[TMDB] 请求: %s", maskAPIKey(searchURL)) + + resp, err := s.client.Get(searchURL) + if err != nil { + log.Warnf("[TMDB] 请求失败 url=%s err=%v", maskAPIKey(searchURL), err) + return nil, fmt.Errorf("TMDB搜索请求失败: %w", err) + } + defer resp.Body.Close() + + body, _ := io.ReadAll(resp.Body) + var result tmdbSearchResult + if err := json.Unmarshal(body, &result); err != nil { + // 附带响应状态码和响应体片段,便于排查 API 地址错误、被代理拦截等问题 + snippet := string(body) + if len(snippet) > 200 { + snippet = snippet[:200] + } + log.Warnf("[TMDB] 解析失败 status=%d url=%s body=%s", resp.StatusCode, maskAPIKey(searchURL), snippet) + return nil, fmt.Errorf("TMDB搜索结果解析失败(status=%d, url=%s): %w, body=%s", + resp.StatusCode, searchURL, err, snippet) + } + // 输出响应概要 + log.Debugf("[TMDB] 响应 status=%d hits=%d url=%s", resp.StatusCode, len(result.Results), maskAPIKey(searchURL)) + + // /search/movie /search/tv 返回的结果不带 media_type,需要根据 endpoint 补齐 + if endpoint == "movie" || endpoint == "tv" { + for i := range result.Results { + if result.Results[i].MediaType == "" { + result.Results[i].MediaType = endpoint + } + } + } + return &result, nil +} + +// maskAPIKey 把 URL 中的 api_key 参数遮蔽为 ***,避免日志输出真实密钥 +var apiKeyMaskRe = regexp.MustCompile(`(api_key=)[^&]*`) + +func maskAPIKey(rawURL string) string { + return apiKeyMaskRe.ReplaceAllString(rawURL, "${1}***") +} + +// searchAttempt 单次搜索尝试参数 +type searchAttempt struct { + endpoint string // multi / movie / tv + query string + year string + language string +} + +// subtitleSepRe 副标题分隔符:「:」「:」「-」前后空白 +// 用于把「复仇者联盟3:无限战争」拆成「复仇者联盟3」和「无限战争」 +var subtitleSepRe = regexp.MustCompile(`\s*[::\-—]\s*`) + +// TitleCandidate 单个搜索候选 +// 用于把文件名解析出来的多组标题/语言/年份/置信度统一成一条「线索」, +// 然后按 Confidence 排序后,逐条尝试 TMDB 搜索。 +type TitleCandidate struct { + Name string // 待搜索的标题字符串 + Lang string // "zh-CN" / "en-US" / ""(不指定) + Year string // 候选年份(可空) + Confidence float32 // 0-1,越高越优先;用于排序与「命中可信度」校验 + Source string // 来源标签:bracket-cn / main-cn / merged-cn / sub-cn / extra-cn / main-en / degenerate-cn / degenerate-en +} + +// tvHintRe 检测文件名是否带电视剧特征(SxxExx、季、集、Episode) +// 命中则只走 /search/tv 端点,否则只走 /search/movie,节约一半请求量 +var tvHintRe = regexp.MustCompile(`(?i)(s\d{1,2}e\d{1,3}|season\s*\d+|episode\s*\d+|第\s*\d+\s*季|第\s*\d+\s*集)`) + +// pickEndpoint 根据原始文件名启发式判断走 movie 还是 tv +func pickEndpoint(rawFileName string) string { + if tvHintRe.MatchString(rawFileName) { + return "tv" + } + return "movie" +} + +// englishStopwords 英文冠词/介词等无信息量小词,提取关键词时可丢弃 +// 注意:仅丢弃整段完全等于这些词的部分,不做子串匹配,避免误伤 "and" 在 "Andromeda" 里 +var englishStopwords = map[string]bool{ + "a": true, "an": true, "the": true, + "of": true, "in": true, "on": true, "at": true, "by": true, "to": true, "for": true, + "and": true, "or": true, "but": true, + "is": true, "are": true, "was": true, "were": true, + "with": true, "from": true, "into": true, "onto": true, + "vs": true, "v": true, +} + +// extractEnglishKeywords 从英文标题中提取最有信息量的关键词 +// +// 规则: +// 1. 按空格分词,去掉 stopwords 与长度 < 3 的词 +// 2. 取剩余词的前 3 个,用空格拼接(更长的标题往往是 TMDB 上没有的精确组合) +// 3. 如果剔除后没有剩余词,返回空字符串(让调用方跳过) +// +// 例: +// +// "And Soon The Darkness" -> "Soon Darkness" +// "Alvin and the Chipmunks" -> "Alvin Chipmunks" +// "30 Days Of Night Dark Days" -> "30 Days Night" +// "Iron Man" -> "Iron Man"(原样保留,关键词都够长) +// "The Dark Knight" -> "Dark Knight" +func extractEnglishKeywords(s string) string { + if s == "" { + return "" + } + words := strings.Fields(s) + keepers := make([]string, 0, len(words)) + for _, w := range words { + lw := strings.ToLower(strings.Trim(w, ".,;:!?'\"")) + if englishStopwords[lw] { + continue + } + if utf8.RuneCountInString(lw) < 3 && !isAllDigits(lw) { + // 长度 < 3 且不是纯数字的词丢弃(数字往往是片名一部分如"300") + continue + } + keepers = append(keepers, w) + } + if len(keepers) == 0 { + return "" + } + if len(keepers) > 3 { + keepers = keepers[:3] + } + return strings.Join(keepers, " ") +} + +func isAllDigits(s string) bool { + if s == "" { + return false + } + for _, r := range s { + if r < '0' || r > '9' { + return false + } + } + return true +} + +// titleSimilarity 简易标题相似度(0-1) +// 规则: +// - 完全相等 -> 1.0 +// - 一个字符串包含另一个 -> 0.85 +// - 归一化(去空格、大小写)后相等 -> 0.95 +// - 归一化后一个包含另一个 -> 0.7 +// - 否则用「公共字符占比」粗算(避免引入完整编辑距离的开销) +func titleSimilarity(a, b string) float32 { + if a == "" || b == "" { + return 0 + } + if a == b { + return 1.0 + } + la := strings.ToLower(strings.TrimSpace(a)) + lb := strings.ToLower(strings.TrimSpace(b)) + if la == lb { + return 0.95 + } + if strings.Contains(la, lb) || strings.Contains(lb, la) { + return 0.85 + } + // 进一步归一化:去掉所有空白与标点,再比一次包含关系 + normA := normalizeForCompare(la) + normB := normalizeForCompare(lb) + if normA == normB && normA != "" { + return 0.92 + } + if normA != "" && normB != "" && (strings.Contains(normA, normB) || strings.Contains(normB, normA)) { + return 0.7 + } + // 兜底:公共字符比例(仅取较短字符串里的字符在较长字符串中出现的比例) + short, long := normA, normB + if len(long) < len(short) { + short, long = long, short + } + if short == "" { + return 0 + } + hit := 0 + for _, r := range short { + if strings.ContainsRune(long, r) { + hit++ + } + } + ratio := float32(hit) / float32(utf8.RuneCountInString(short)) + // 公共字符比例最高只贡献 0.5,避免冷门误判 + if ratio > 0.5 { + ratio = 0.5 + } + return ratio +} + +// normalizeForCompare 归一化字符串以做相似度比较: +// 去掉所有空白、标点(含中文标点),转小写。 +func normalizeForCompare(s string) string { + var b strings.Builder + for _, r := range s { + switch { + case r == ' ' || r == '\t' || r == '\n': + case r == '.' || r == ',' || r == '-' || r == '_' || r == ':' || r == ';' || r == '!' || r == '?': + case r == ':' || r == ',' || r == '。' || r == '、' || r == '!' || r == '?': + case r == '(' || r == ')' || r == '[' || r == ']' || r == '(' || r == ')' || r == '【' || r == '】': + default: + b.WriteRune(r) + } + } + return strings.ToLower(b.String()) +} + +// tmdbHit 从 tmdbSearchResult.Results 提取出来的单个命中项(仅用于内部传递) +type tmdbHit = struct { + ID int `json:"id"` + Title string `json:"title"` + Name string `json:"name"` + Overview string `json:"overview"` + PosterPath string `json:"poster_path"` + ReleaseDate string `json:"release_date"` + FirstAirDate string `json:"first_air_date"` + VoteAverage float32 `json:"vote_average"` + MediaType string `json:"media_type"` + GenreIDs []int `json:"genre_ids"` +} + +// scoreMatch 给一个 TMDB 搜索结果打分,作为「这个 hit 是否可信」的依据。 +// score = 0.5*titleSim + 0.3*yearScore + 0.2*voteScore (范围 0-1) +func scoreMatch(query string, year string, hit tmdbHit) float32 { + hitTitle := hit.Title + if hitTitle == "" { + hitTitle = hit.Name + } + hitYear := "" + if len(hit.ReleaseDate) >= 4 { + hitYear = hit.ReleaseDate[:4] + } else if len(hit.FirstAirDate) >= 4 { + hitYear = hit.FirstAirDate[:4] + } + + titleSim := titleSimilarity(query, hitTitle) + + var yearScore float32 + if year == "" { + // 无年份信号时给一个中性分(不奖励、不惩罚) + yearScore = 0.4 + } else if hitYear == "" { + // 文件名有年份但 TMDB 没填,给较低分 + yearScore = 0.2 + } else if hitYear == year { + yearScore = 1.0 + } else { + // 容忍 ±1 年(部分电影上映日期记录差一年) + diff := absInt(parseIntSafe(hitYear) - parseIntSafe(year)) + switch { + case diff == 1: + yearScore = 0.5 + case diff == 2: + yearScore = 0.2 + default: + yearScore = 0.0 + } + } + + // vote_average 0-10,用作冷门片名的最弱信号 + voteScore := hit.VoteAverage / 10.0 + if voteScore > 1 { + voteScore = 1 + } + + return 0.5*titleSim + 0.3*yearScore + 0.2*voteScore +} + +func absInt(x int) int { + if x < 0 { + return -x + } + return x +} + +func parseIntSafe(s string) int { + n := 0 + for _, r := range s { + if r < '0' || r > '9' { + return 0 + } + n = n*10 + int(r-'0') + } + return n +} + +// buildTitleCandidates 根据原始标题构造一组候选搜索词(按优先级返回) +// 候选生成策略,从精确到模糊: +// 1. 原始标题(保留所有信息) +// 2. 归一化标题(去括号/噪声词/分隔符 -> 空格合并) +// 3. 中文数字 -> 阿拉伯数字(钢铁侠三 -> 钢铁侠3) +// 4. 阿拉伯数字 -> 中文数字(钢铁侠3 -> 钢铁侠三,少数 TMDB 别名用中文数字) +// 5. 去除尾部数字/罗马数字(钢铁侠3 -> 钢铁侠,盗梦空间2 -> 盗梦空间) +// 6. 拆分多个词,每个非短词作为独立候选 +// 7. 副标题拆分(「复仇者联盟3:无限战争」 -> 「复仇者联盟3」、「无限战争」) +func buildTitleCandidates(title string) []string { + if title == "" { + return nil + } + seen := make(map[string]bool) + var out []string + add := func(s string) { + s = strings.TrimSpace(s) + if s == "" || seen[s] { + return + } + seen[s] = true + out = append(out, s) + } + + add(title) + norm := normalizeTitle(title) + add(norm) + + // 副标题拆分:以「:」「:」「-」分隔后得到多个候选 + // 例:「复仇者联盟3:无限战争」-> 主标题 "复仇者联盟3"、副标题 "无限战争" + // 「美国队长2:冬日战士」 -> 主标题 "美国队长2"、副标题 "冬日战士" + // 「雷神2:黑暗世界」 -> 主标题 "雷神2"、副标题 "黑暗世界" + subtitleParts := subtitleSepRe.Split(title, -1) + if len(subtitleParts) > 1 { + for _, p := range subtitleParts { + add(p) + } + } + + // 中文数字 <-> 阿拉伯数字 双向归一化 + arabic := cnNumToArabic(norm) + add(arabic) + add(arabicToCnNum(norm)) + + // 去除尾部数字(系列编号),帮助匹配主作品 + // 例:「钢铁侠3」-> 「钢铁侠」、「Iron Man 3」-> 「Iron Man」 + // 「异形1」-> 「异形」、「三十极夜2」-> 「三十极夜」 + // 这条对 title / norm / arabic 三者都做,提升 TMDB 中文别名不带后缀作品的命中率 + trailingNumRe := regexp.MustCompile(`[\s]*[0-90-9]+\s*$`) + add(trailingNumRe.ReplaceAllString(title, "")) + add(trailingNumRe.ReplaceAllString(arabic, "")) + add(trailingNumRe.ReplaceAllString(norm, "")) + + // 去除尾部罗马数字(II / III / IV 等) + trailingRomanRe := regexp.MustCompile(`(?i)\s+(?:I{1,3}|IV|V|VI{0,3}|IX|X)$`) + add(trailingRomanRe.ReplaceAllString(norm, "")) + + // 若中文标题里夹杂了空格分隔的多个词,把每个非短词单独作为候选 + for _, w := range strings.Fields(norm) { + if utf8.RuneCountInString(w) >= 2 { + add(w) + } + } + // 阿拉伯数字归一化版本同样拆词 + for _, w := range strings.Fields(arabic) { + if utf8.RuneCountInString(w) >= 2 { + add(w) + } + } + + // 数字 + 中文 / 中文 + 数字 的组合标题,把中文部分单独提取作为候选 + // 例:「300勇士」 -> 候选包含 "勇士" + // 「3096天」 -> 候选包含 "天"(虽然过短) + // 「36总局」 -> 候选包含 "总局" + // 「47号物品」 -> 候选包含 "号物品" 较弱,跳过 + // 同时把开头数字单独作为候选,匹配「300」「3096」这类纯数字片名 + digitChineseRe := regexp.MustCompile(`^(\d+)([\p{Han}].*)$`) + chineseDigitRe := regexp.MustCompile(`^([\p{Han}].*?)(\d+)$`) + for _, s := range []string{title, norm} { + if m := digitChineseRe.FindStringSubmatch(s); m != nil { + // 数字部分(如 "300") + add(m[1]) + // 中文部分(如 "勇士"),仅当中文长度 >= 2 才有用 + if utf8.RuneCountInString(m[2]) >= 2 { + add(m[2]) + } + } + if m := chineseDigitRe.FindStringSubmatch(s); m != nil { + // 中文部分(去掉尾部数字) + if utf8.RuneCountInString(m[1]) >= 2 { + add(m[1]) + } + // 数字部分单独作为候选 + add(m[2]) + } + } + + return out +} + +// arabicToCnNum 将标题中的单个阿拉伯数字归一化为中文数字(仅 0-9 的简单替换,10 以上不处理) +// 用于「钢铁侠3」-> 「钢铁侠三」,提升 TMDB 中文别名命中率 +func arabicToCnNum(s string) string { + if s == "" { + return s + } + rev := map[rune]string{ + '0': "零", '1': "一", '2': "二", '3': "三", '4': "四", + '5': "五", '6': "六", '7': "七", '8': "八", '9': "九", + } + // 仅当字符串包含中文且数字是单个字符(前后非数字)时才转换,避免 "2010" 年份被错误处理 + if !chineseRegexp.MatchString(s) { + return s + } + runes := []rune(s) + var b strings.Builder + for i, r := range runes { + if r >= '0' && r <= '9' { + // 检查是否是孤立的单个数字(前后不是数字) + prevDigit := i > 0 && runes[i-1] >= '0' && runes[i-1] <= '9' + nextDigit := i+1 < len(runes) && runes[i+1] >= '0' && runes[i+1] <= '9' + if !prevDigit && !nextDigit { + b.WriteString(rev[r]) + continue + } + } + b.WriteRune(r) + } + return b.String() +} + +// extractCandidates 把 parsedVideoTitle 转换成有序的 TitleCandidate 列表 +// +// 顺序原则:中文优先 -> 英文兜底;高置信度优先 -> 低置信度退化候选 +// 来源标签 (Source): +// - bracket-cn : 来自方括号内的中文,置信度最高(发布组明确标注) +// - main-cn : 文件名中第一个中文片段 +// - main-en : 主英文标题 +// - merged-cn : 多个中文片段合并(300勇士:帝国崛起) +// - sub-cn : 中文标题的副标题拆分(无限战争) +// - extra-cn : 其它中文片段 +// - degenerate-cn / degenerate-en : 去尾数/去括号等退化形式 +func extractCandidates(parsed parsedVideoTitle, bracketChinese []string) []TitleCandidate { + var list []TitleCandidate + seen := make(map[string]bool) + add := func(name, lang, source string, conf float32) { + name = strings.TrimSpace(name) + if name == "" || utf8.RuneCountInString(name) < 1 { + return + } + key := lang + "|" + name + if seen[key] { + return + } + seen[key] = true + list = append(list, TitleCandidate{ + Name: name, + Lang: lang, + Year: parsed.Year, + Confidence: conf, + Source: source, + }) + } + + // 1) 方括号中文:发布组明确标注的片名,最可信 + for _, b := range bracketChinese { + add(b, "zh-CN", "bracket-cn", 0.95) + } + + // 2) 主中文标题 + if parsed.ChineseTitle != "" { + add(parsed.ChineseTitle, "zh-CN", "main-cn", 0.90) + // 副标题拆分:「复仇者联盟3:无限战争」-> 「复仇者联盟3」+「无限战争」 + parts := subtitleSepRe.Split(parsed.ChineseTitle, -1) + if len(parts) > 1 { + for _, p := range parts { + add(p, "zh-CN", "sub-cn", 0.55) + } + } + } + + // 3) 主英文标题 + if parsed.EnglishTitle != "" { + add(parsed.EnglishTitle, "en-US", "main-en", 0.80) + } + + // 4) 额外中文候选(解析出多个中文片段时;包括 merged 合并版本) + for _, t := range parsed.ExtraChineseTitles { + source := "extra-cn" + conf := float32(0.50) + if strings.Contains(t, ":") { + source = "merged-cn" + conf = 0.65 + } + add(t, "zh-CN", source, conf) + } + + // 5) 退化候选:对中文/英文都做「去尾部数字」「归一化」 + trailingNumRe := regexp.MustCompile(`[\s]*[0-90-9]+\s*$`) + addDegenerate := func(src, lang, sourceTag string) { + if src == "" { + return + } + norm := normalizeTitle(src) + if norm != src { + add(norm, lang, sourceTag, 0.35) + } + if stripped := trailingNumRe.ReplaceAllString(src, ""); stripped != src { + add(stripped, lang, sourceTag, 0.30) + } + if stripped := trailingNumRe.ReplaceAllString(norm, ""); stripped != norm { + add(stripped, lang, sourceTag, 0.30) + } + // 中文数字 ⇄ 阿拉伯数字 + if arabic := cnNumToArabic(src); arabic != src { + add(arabic, lang, sourceTag, 0.35) + } + } + addDegenerate(parsed.ChineseTitle, "zh-CN", "degenerate-cn") + addDegenerate(parsed.EnglishTitle, "en-US", "degenerate-en") + + return list +} + +// searchWithFallback 带命中校验的多候选搜索 +// +// 流程: +// 1. 把候选按 Confidence 排序(中文高 > 中文低 > 英文 > 退化) +// 2. 对每个候选,最多发 2 次请求:带年份精搜 + 不带年份模糊搜 +// 3. 命中即用 scoreMatch 评分: +// - score >= matchConfidenceThreshold (0.60) → 立即返回 +// - score >= fallbackThreshold (0.30) → 暂存为「最佳兜底」继续尝试 +// - 其它 → 丢弃 +// 4. 全部尝试完仍无强命中 → 返回最佳兜底;都没有则报错 +// +// 端点选择:根据原始文件名启发式(含 SxxExx/季/集 -> tv,否则 movie),不再 movie/tv 互打 +func (s *TMDBScraper) searchWithFallback(parsed parsedVideoTitle, rawFileName string, bracketChinese []string) (*tmdbSearchResult, error) { + candidates := extractCandidates(parsed, bracketChinese) + // 按 Confidence 倒序,置信度高的先尝试 + sort.SliceStable(candidates, func(i, j int) bool { + return candidates[i].Confidence > candidates[j].Confidence + }) + + endpoint := pickEndpoint(rawFileName) + + // Debug 输出所有待尝试的候选 + log.Debugf("[TMDB] 候选生成 endpoint=%s year=%q candidates(%d):", endpoint, parsed.Year, len(candidates)) + for i, c := range candidates { + log.Debugf("[TMDB] #%d source=%-13s lang=%-5s conf=%.2f name=%q", i+1, c.Source, c.Lang, c.Confidence, c.Name) + } + + if len(candidates) == 0 { + return nil, fmt.Errorf("无法从文件名中提取有效标题") + } + + // 去重 + type attemptKey struct{ ep, q, y, l string } + done := make(map[attemptKey]bool) + attempts := int64(0) + + var bestFallback *tmdbSearchResult // 暂存的「分数过得去但不够强」的结果 + var bestScore float32 + var bestQuery string + var lastErr error + + tryOne := func(ep, q, year, lang, source string, candConf float32) (*tmdbSearchResult, float32, bool) { + k := attemptKey{ep, q, year, lang} + if done[k] { + return nil, 0, false + } + done[k] = true + attempts++ + + result, err := s.doTMDBSearch(ep, q, year, lang) + if err != nil { + lastErr = err + log.Debugf("[TMDB] try [%s] q=%q year=%q lang=%q -> 请求失败: %v", source, q, year, lang, err) + return nil, 0, false + } + if len(result.Results) == 0 { + log.Debugf("[TMDB] try [%s] q=%q year=%q lang=%q -> 0 hits", source, q, year, lang) + return nil, 0, false + } + first := result.Results[0] + score := scoreMatch(q, year, first) + hitTitle := first.Title + if hitTitle == "" { + hitTitle = first.Name + } + hitYear := "" + if len(first.ReleaseDate) >= 4 { + hitYear = first.ReleaseDate[:4] + } else if len(first.FirstAirDate) >= 4 { + hitYear = first.FirstAirDate[:4] + } + log.Debugf("[TMDB] try [%s] q=%q year=%q lang=%q -> hit{id=%d, title=%q, year=%s, vote=%.1f} score=%.2f", + source, q, year, lang, first.ID, hitTitle, hitYear, first.VoteAverage, score) + return result, score, true + } + + startTs := time.Now() + + for _, c := range candidates { + // 5.1 带年份精搜(如果有年份) + if c.Year != "" { + if res, score, ok := tryOne(endpoint, c.Name, c.Year, c.Lang, c.Source+"+y", c.Confidence); ok { + if score >= matchConfidenceThreshold { + recordHit(attempts, time.Since(startTs)) + log.Debugf("[TMDB] ✅ 强命中 (%.2f >= %.2f) source=%s q=%q", score, matchConfidenceThreshold, c.Source, c.Name) + return res, nil + } + if score >= fallbackThreshold && score > bestScore { + bestFallback = res + bestScore = score + bestQuery = c.Name + } + } + } + // 5.2 不带年份模糊搜(兜底) + if res, score, ok := tryOne(endpoint, c.Name, "", c.Lang, c.Source, c.Confidence); ok { + if score >= matchConfidenceThreshold { + recordHit(attempts, time.Since(startTs)) + log.Debugf("[TMDB] ✅ 强命中 (%.2f >= %.2f) source=%s q=%q", score, matchConfidenceThreshold, c.Source, c.Name) + return res, nil + } + if score >= fallbackThreshold && score > bestScore { + bestFallback = res + bestScore = score + bestQuery = c.Name + } + } + } + + // 没有强命中:使用 best fallback(可能仍然是错的,但至少有数据) + if bestFallback != nil { + recordLowConfidence(attempts, time.Since(startTs)) + log.Debugf("[TMDB] ⚠️ 使用低置信度兜底命中 score=%.2f q=%q (low confidence < %.2f)", + bestScore, bestQuery, matchConfidenceThreshold) + return bestFallback, nil + } + + // 主流程完全无命中 → 进入「降级重试」阶段 + // 这一阶段主动放宽限制,尝试更“街头魔法”的查询方式: + // L2: 主候选换 multi 端点 + 不指定语言 → 匹配 TMDB 全语言别名 + // L3: 英文标题去冠词/取关键词重新搜 + // L4: 阈值降到 desperateThreshold (0.20),换 multi 端点接纳「还有点像」的结果 + log.Debugf("[TMDB] 主流程无命中,进入降级重试阶段 attempts=%d", attempts) + + // L2: 主候选换 multi 端点 + 不指定语言 + for i, c := range candidates { + if i >= 3 { // 只对前 3 个高置信度候选走 multi,避免请求爆炸 + break + } + if res, score, ok := tryOne("multi", c.Name, "", "", c.Source+"@multi", c.Confidence); ok { + if score >= matchConfidenceThreshold { + recordHit(attempts, time.Since(startTs)) + log.Debugf("[TMDB] ✅ L2 multi 端点强命中 (%.2f >= %.2f) q=%q", score, matchConfidenceThreshold, c.Name) + return res, nil + } + if score >= fallbackThreshold && score > bestScore { + bestFallback = res + bestScore = score + bestQuery = c.Name + } + } + } + if bestFallback != nil { + recordLowConfidence(attempts, time.Since(startTs)) + log.Debugf("[TMDB] ⚠️ L2 低置信度兜底命中 score=%.2f q=%q", bestScore, bestQuery) + return bestFallback, nil + } + + // L3: 英文标题提取关键词(去冠词及过短词)重新搜索 + if kw := extractEnglishKeywords(parsed.EnglishTitle); kw != "" && kw != parsed.EnglishTitle { + log.Debugf("[TMDB] L3 英文关键词提取: %q -> %q", parsed.EnglishTitle, kw) + for _, ep := range []string{endpoint, "multi"} { + if res, score, ok := tryOne(ep, kw, "", "", "keywords-en", 0.4); ok { + if score >= matchConfidenceThreshold { + recordHit(attempts, time.Since(startTs)) + log.Debugf("[TMDB] ✅ L3 关键词强命中 (%.2f >= %.2f) q=%q", score, matchConfidenceThreshold, kw) + return res, nil + } + if score >= fallbackThreshold && score > bestScore { + bestFallback = res + bestScore = score + bestQuery = kw + } + } + } + if bestFallback != nil { + recordLowConfidence(attempts, time.Since(startTs)) + log.Debugf("[TMDB] ⚠️ L3 低置信度兜底命中 score=%.2f q=%q", bestScore, bestQuery) + return bestFallback, nil + } + } + + // L4: 「绝望式」阈值——陈障足以阁,重跳主候选但阈值降到 0.20 + // 说明:部分冷门片 vote_average=0、不同语言别名不及时补录,会被 0.30 阈值误杀, + // 这里使用更宽的 0.20 阈值接纳(阅读者可手动修正错误结果,总比空手好)。 + log.Debugf("[TMDB] L4 启用绝望式阈值 %.2f 重试主候选", desperateThreshold) + for i, c := range candidates { + if i >= 3 { + break + } + for _, ep := range []string{endpoint, "multi"} { + if res, score, ok := tryOne(ep, c.Name, "", "", c.Source+"@desperate", c.Confidence); ok { + if score >= desperateThreshold && score > bestScore { + bestFallback = res + bestScore = score + bestQuery = c.Name + } + } + } + } + if bestFallback != nil { + recordLowConfidence(attempts, time.Since(startTs)) + log.Debugf("[TMDB] ⚠️ L4 绝望式兜底命中 score=%.2f q=%q (阈值仅 %.2f,结果可能不准)", + bestScore, bestQuery, desperateThreshold) + return bestFallback, nil + } + + recordNoMatch(attempts, time.Since(startTs)) + titleInfo := parsed.ChineseTitle + if titleInfo == "" { + titleInfo = parsed.EnglishTitle + } + if lastErr != nil { + return nil, fmt.Errorf("TMDB未找到匹配结果: %s (last err: %v)", titleInfo, lastErr) + } + return nil, fmt.Errorf("TMDB未找到匹配结果: %s", titleInfo) +} + +// 统计辅助函数(使用 atomic 保证并发安全) +func recordHit(attempts int64, elapsed time.Duration) { + atomic.AddInt64(&globalTMDBStats.HitCount, 1) + atomic.AddInt64(&globalTMDBStats.TotalAttempts, attempts) + atomic.AddInt64(&globalTMDBStats.TotalSearchMillis, elapsed.Milliseconds()) +} + +func recordLowConfidence(attempts int64, elapsed time.Duration) { + atomic.AddInt64(&globalTMDBStats.LowConfidenceCount, 1) + atomic.AddInt64(&globalTMDBStats.TotalAttempts, attempts) + atomic.AddInt64(&globalTMDBStats.TotalSearchMillis, elapsed.Milliseconds()) +} + +func recordNoMatch(attempts int64, elapsed time.Duration) { + atomic.AddInt64(&globalTMDBStats.NoMatchCount, 1) + atomic.AddInt64(&globalTMDBStats.TotalAttempts, attempts) + atomic.AddInt64(&globalTMDBStats.TotalSearchMillis, elapsed.Milliseconds()) +} + +// ScrapeVideo 刮削视频信息 +func (s *TMDBScraper) ScrapeVideo(item *model.MediaItem) error { + if s.APIKey == "" { + return fmt.Errorf("TMDB API Key 未配置") + } + + atomic.AddInt64(&globalTMDBStats.TotalScraped, 1) + + // 始终从文件名中解析出标题和年份(ScrapedName 是刮削结果字段,不作为搜索输入) + parsed := parseVideoFileName(item.FileName) + // 在剥离括号前再次抽一次方括号中文,作为最高置信度候选源 + bracketChinese := extractBracketChinese(item.FileName) + log.Debugf("[TMDB] 开始刮削 file=%q parsed={chinese=%q english=%q year=%q extras=%v bracket=%v} baseURL=%s", + item.FileName, parsed.ChineseTitle, parsed.EnglishTitle, parsed.Year, parsed.ExtraChineseTitles, bracketChinese, s.BaseURL) + + // 搜索策略:中文标题优先,英文标题兜底,都搜不到才失败 + searchResult, err := s.searchWithFallback(parsed, item.FileName, bracketChinese) + if err != nil { + log.Debugf("[TMDB] ❌ 刮削失败 file=%q err=%v", item.FileName, err) + return err + } + + // 取第一个结果 + first := searchResult.Results[0] + mediaType := first.MediaType + if mediaType == "" { + mediaType = "movie" + } + + // 获取详情(语言跟随「首选刮削语言」设置) + detailURL := fmt.Sprintf("%s/%s/%d?api_key=%s&language=%s&append_to_response=credits", + s.BaseURL, mediaType, first.ID, s.APIKey, url.QueryEscape(ScrapeLanguage())) + log.Debugf("[TMDB] 详情请求: %s", maskAPIKey(detailURL)) + + detailResp, err := s.client.Get(detailURL) + if err != nil { + return fmt.Errorf("TMDB详情请求失败: %w", err) + } + defer detailResp.Body.Close() + + detailBody, _ := io.ReadAll(detailResp.Body) + var detail tmdbMovieDetail + if err := json.Unmarshal(detailBody, &detail); err != nil { + return fmt.Errorf("TMDB详情解析失败: %w", err) + } + + // 填充字段(仅填空字段,已有值不覆盖;ScrapedAt 由调用方统一设置) + title := detail.Title + if title == "" { + title = first.Name + } + // ScrapedName 处理: + // - 扫描阶段会把「去扩展名的文件名」作为默认占位写入 ScrapedName, + // 这种占位值应该被刮削得到的真实标题覆盖; + // - 用户手动编辑过的 ScrapedName 不会等于该占位值,保留不覆盖。 + if title != "" { + defaultName := strings.TrimSuffix(item.FileName, stdpath.Ext(item.FileName)) + if item.ScrapedName == "" || item.ScrapedName == defaultName { + item.ScrapedName = title + } + } + if item.Plot == "" { + item.Plot = detail.Overview + } + if item.Rating == 0 { + item.Rating = detail.VoteAverage + } + if item.ExternalID == "" { + item.ExternalID = fmt.Sprintf("tmdb:%d", detail.ID) + } + if item.VideoType == "" { + item.VideoType = mediaType + } + + releaseDate := detail.ReleaseDate + if releaseDate == "" { + releaseDate = first.FirstAirDate + } + if item.ReleaseDate == "" { + item.ReleaseDate = releaseDate + } + + if item.Cover == "" && detail.PosterPath != "" { + item.Cover = tmdbImageBase + detail.PosterPath + } + + // 类型 + if item.Genre == "" { + genres := make([]string, 0, len(detail.Genres)) + for _, g := range detail.Genres { + genres = append(genres, g.Name) + } + item.Genre = strings.Join(genres, ",") + } + + // 演员(取前10个) + if item.Authors == "" { + actors := make([]string, 0) + for i, cast := range detail.Credits.Cast { + if i >= 10 { + break + } + actors = append(actors, cast.Name) + } + authorsJSON, _ := json.Marshal(actors) + item.Authors = string(authorsJSON) + } + + now := time.Now() + item.ScrapedAt = &now + hitTitle := title + hitYear := "" + if len(detail.ReleaseDate) >= 4 { + hitYear = detail.ReleaseDate[:4] + } + log.Debugf("[TMDB] ✅ 刮削完成 file=%q -> tmdb_id=%d, title=%q, year=%s, mediaType=%s", + item.FileName, detail.ID, hitTitle, hitYear, mediaType) + return nil +} diff --git a/internal/media/strm.go b/internal/media/strm.go new file mode 100644 index 00000000..9b606f59 --- /dev/null +++ b/internal/media/strm.go @@ -0,0 +1,336 @@ +package media + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + stdpath "path" + "strings" + "sync/atomic" + "time" + + "github.com/OpenListTeam/OpenList/v4/internal/conf" + "github.com/OpenListTeam/OpenList/v4/internal/db" + "github.com/OpenListTeam/OpenList/v4/internal/fs" + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/internal/setting" + "github.com/OpenListTeam/OpenList/v4/internal/sign" + "github.com/OpenListTeam/OpenList/v4/internal/stream" + "github.com/OpenListTeam/OpenList/v4/pkg/utils" + log "github.com/sirupsen/logrus" +) + +var apiRootHolder atomic.Value + +// SetAPIRoot 记录本次运行使用的站点地址(由 HTTP 层注入,供异步任务复用) +func SetAPIRoot(u string) { + if u == "" { + return + } + apiRootHolder.Store(strings.TrimSuffix(u, "/")) +} + +// GetAPIRoot 返回当前站点地址 +func GetAPIRoot() string { + if v, ok := apiRootHolder.Load().(string); ok && v != "" { + return v + } + return strings.TrimSuffix(conf.Conf.SiteURL, "/") +} + +// StrmOptions STRM 生成参数 +type StrmOptions struct { + APIRoot string // 站点地址前缀,例如 https://example.com + Template string // 内容模板 + Overwrite bool // 是否覆盖已存在文件 + SameDir bool // 是否与媒体文件同目录(同名 .strm) + OutputPath string // 输出目录(VFS 路径,仅 SameDir=false 时使用) +} + +// StrmTarget 一个待生成的 .strm 目标 +type StrmTarget struct { + RelPath string // 相对输出目录的路径 + RawPath string // 对应媒体文件的 VFS 路径 + SameDirPath string // 与媒体文件同目录时的 .strm 路径 +} + +// strmSibling 计算与媒体文件同目录、同名的 .strm 路径 +func strmSibling(rawPath string, isDir bool) string { + if isDir { + return stdpath.Join(rawPath, stdpath.Base(rawPath)+".strm") + } + ext := stdpath.Ext(rawPath) + if ext == "" { + return rawPath + ".strm" + } + return strings.TrimSuffix(rawPath, ext) + ".strm" +} + +// DefaultStrmOptions 从设置中读取 STRM 参数 +func DefaultStrmOptions(apiRoot string) StrmOptions { + return StrmOptions{ + APIRoot: strings.TrimSuffix(apiRoot, "/"), + Template: setting.GetStr(conf.MediaStrmTemplate, "{site_url}/d{path}?sign={sign}"), + Overwrite: setting.GetBool(conf.MediaStrmOverwrite), + SameDir: setting.GetBool(conf.MediaStrmSameDir), + OutputPath: setting.GetStr(conf.MediaStrmPath, "/strm"), + } +} + +// RenderStrmContent 渲染 .strm 内容(单行 URL) +func RenderStrmContent(tpl, apiRoot, rawPath string) string { + if tpl == "" { + tpl = "{site_url}/d{path}?sign={sign}" + } + apiRoot = strings.TrimSuffix(apiRoot, "/") + p := rawPath + if !strings.HasPrefix(p, "/") { + p = "/" + p + } + replaced := strings.NewReplacer( + "{site_url}", apiRoot, + "{path}", utils.EncodePath(p), + "{sign}", sign.Sign(rawPath), + "{raw_path}", p, + "{name}", stdpath.Base(p), + ).Replace(tpl) + return strings.TrimSpace(replaced) +} + +// sanitizeName 清洗文件/目录名中的非法字符 +func sanitizeName(s string) string { + s = strings.TrimSpace(s) + if s == "" { + return "Unknown" + } + replacer := strings.NewReplacer( + "/", "_", "\\", "_", ":", "_", "*", "_", "?", "_", + "\"", "_", "<", "_", ">", "_", "|", "_", + "\n", " ", "\r", " ", + ) + s = replacer.Replace(s) + if runes := []rune(s); len(runes) > 120 { + s = string(runes[:120]) + } + return strings.TrimSpace(s) +} + +func itemTitle(item *model.MediaItem) string { + if item.ScrapedName != "" { + return item.ScrapedName + } + name := item.FileName + if !item.IsFolder { + name = strings.TrimSuffix(name, stdpath.Ext(name)) + } + return name +} + +func itemYear(item *model.MediaItem) string { + if len(item.ReleaseDate) >= 4 { + return item.ReleaseDate[:4] + } + return "" +} + +func parseEpisodes(item *model.MediaItem) []model.EpisodeInfo { + if item.Episodes == "" { + return nil + } + var eps []model.EpisodeInfo + if err := json.Unmarshal([]byte(item.Episodes), &eps); err != nil { + return nil + } + return eps +} + +// MediaItemFolder 条目所在目录 +func MediaItemFolder(item *model.MediaItem) string { + if item == nil { + return "/" + } + folder := strings.TrimSuffix(item.FolderPath, "/") + if folder == "" { + folder = "/" + } + return folder +} + +// StrmTargets 计算条目对应的 STRM 目标列表。 +// 图片/书籍返回空列表(不生成 STRM)。 +func StrmTargets(item *model.MediaItem) []StrmTarget { + if item == nil { + return nil + } + title := sanitizeName(itemTitle(item)) + year := itemYear(item) + folder := MediaItemFolder(item) + + switch item.MediaType { + case model.MediaTypeVideo: + if item.VideoType == "tv" { + series := title + season := item.Season + if season <= 0 { + season = 1 + } + var out []StrmTarget + if item.IsFolder { + for i, ep := range parseEpisodes(item) { + num := ep.Index + if num <= 0 { + num = i + 1 + } + epTitle := sanitizeName(ep.Title) + epRaw := stdpath.Join(folder, item.FileName, ep.FileName) + out = append(out, StrmTarget{ + RelPath: stdpath.Join(series, fmt.Sprintf("Season %02d", season), + fmt.Sprintf("%s S%02dE%02d - %s.strm", series, season, num, epTitle)), + RawPath: epRaw, + SameDirPath: strmSibling(epRaw, false), + }) + } + return out + } + num := item.Episode + if num <= 0 { + num = 1 + } + raw := stdpath.Join(folder, item.FileName) + return []StrmTarget{{ + RelPath: stdpath.Join(series, fmt.Sprintf("Season %02d", season), + fmt.Sprintf("%s S%02dE%02d.strm", series, season, num)), + RawPath: raw, + SameDirPath: strmSibling(raw, false), + }} + } + name := title + if year != "" { + name = fmt.Sprintf("%s (%s)", title, year) + } + raw := stdpath.Join(folder, item.FileName) + return []StrmTarget{{ + RelPath: stdpath.Join(name, name+".strm"), + RawPath: raw, + SameDirPath: strmSibling(raw, item.IsFolder), + }} + case model.MediaTypeMusic: + artist := sanitizeName(item.AlbumArtist) + if artist == "" { + artist = "Unknown Artist" + } + album := sanitizeName(item.AlbumName) + if album == "" { + album = "Unknown Album" + } + track := item.TrackNumber + if track <= 0 { + track = 1 + } + raw := stdpath.Join(folder, item.FileName) + return []StrmTarget{{ + RelPath: stdpath.Join(artist, album, fmt.Sprintf("%02d - %s.strm", track, title)), + RawPath: raw, + SameDirPath: strmSibling(raw, false), + }} + default: + return nil + } +} + +// GenerateStrmForItems 为一组条目生成 STRM 文件,返回写入数量 +func GenerateStrmForItems(ctx context.Context, items []model.MediaItem, opts StrmOptions) (int, error) { + if !opts.SameDir && opts.OutputPath == "" { + return 0, fmt.Errorf("strm output path is empty") + } + // 归一输出根目录,后续用它校验每个目标文件都在根目录之内 + outRoot := "" + if !opts.SameDir { + outRoot = stdpath.Clean("/" + strings.TrimSpace(opts.OutputPath)) + } + written := 0 + for i := range items { + item := &items[i] + if item.Hidden { + continue + } + for _, target := range StrmTargets(item) { + dst := target.SameDirPath + if !opts.SameDir { + dst = stdpath.Join(outRoot, target.RelPath) + // RelPath 含刮削得到的艺术家/专辑名等外部数据,必须保证不会借 ../ 逃出输出目录 + if dst != outRoot && !strings.HasPrefix(dst, strings.TrimSuffix(outRoot, "/")+"/") { + continue + } + } + if dst == "" { + continue + } + if !opts.Overwrite { + if _, err := fs.Get(ctx, dst, &fs.GetArgs{NoLog: true}); err == nil { + continue + } + } + data := []byte(RenderStrmContent(opts.Template, opts.APIRoot, target.RawPath)) + file := &stream.FileStream{ + Ctx: ctx, + Obj: &model.Object{ + Name: stdpath.Base(dst), + Size: int64(len(data)), + Modified: time.Now(), + }, + Reader: bytes.NewReader(data), + Mimetype: "text/plain", + } + if err := fs.PutDirectly(ctx, stdpath.Dir(dst), file, true); err != nil { + return written, fmt.Errorf("write %s: %w", dst, err) + } + written++ + } + } + return written, nil +} + +// GenerateStrmForType 按媒体类型(可选扫描路径)批量生成 STRM +func GenerateStrmForType(ctx context.Context, mediaType model.MediaType, scanPathID uint, opts StrmOptions) (int, error) { + var items []model.MediaItem + var err error + if scanPathID > 0 { + items, err = db.ListMediaItemsByScanPath(scanPathID) + } else { + items, err = db.ListAllMediaItems(mediaType) + } + if err != nil { + return 0, err + } + if mediaType != "" { + filtered := make([]model.MediaItem, 0, len(items)) + for _, it := range items { + if it.MediaType == mediaType { + filtered = append(filtered, it) + } + } + items = filtered + } + return GenerateStrmForItems(ctx, items, opts) +} + +// TriggerStrmOnScan 扫描结束后按需异步生成 STRM +func TriggerStrmOnScan(mediaType model.MediaType, apiRoot string) { + if !setting.GetBool(conf.MediaStrmOnScan) { + return + } + opts := DefaultStrmOptions(apiRoot) + if !opts.SameDir && opts.OutputPath == "" { + return + } + go func() { + n, err := GenerateStrmForType(context.Background(), mediaType, 0, opts) + if err != nil { + log.Errorf("[media] generate strm for %s failed: %+v", mediaType, err) + return + } + log.Infof("[media] generated %d strm files for %s", n, mediaType) + }() +} diff --git a/internal/model/args.go b/internal/model/args.go index 16a5c172..87a6342d 100644 --- a/internal/model/args.go +++ b/internal/model/args.go @@ -96,18 +96,21 @@ type ArchiveDecompressArgs struct { } type SharingListArgs struct { - Refresh bool - Pwd string + Refresh bool + Pwd string + ClientIP string } type SharingArchiveMetaArgs struct { ArchiveMetaArgs - Pwd string + Pwd string + ClientIP string } type SharingArchiveListArgs struct { ArchiveListArgs - Pwd string + Pwd string + ClientIP string } type SharingLinkArgs struct { diff --git a/internal/model/media.go b/internal/model/media.go new file mode 100644 index 00000000..56b0b357 --- /dev/null +++ b/internal/model/media.go @@ -0,0 +1,129 @@ +package model + +import ( + "time" + + "gorm.io/gorm" +) + +// MediaType 媒体类型,使用字符串便于后期扩展 +type MediaType string + +const ( + MediaTypeVideo MediaType = "video" + MediaTypeMusic MediaType = "music" + MediaTypeImage MediaType = "image" + MediaTypeBook MediaType = "book" +) + +// MediaScanPath 媒体库扫描路径(每个媒体库可配置多个扫描路径) +type MediaScanPath struct { + ID uint `gorm:"primarykey" json:"id"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` + DeletedAt gorm.DeletedAt `gorm:"index" json:"-"` + + MediaType MediaType `gorm:"size:32;index;not null" json:"media_type"` // 所属媒体类型 + Name string `json:"name"` // 路径名称(用于前端筛选显示) + Path string `gorm:"not null" json:"path"` // VFS 扫描路径 + PathMerge bool `gorm:"default:false" json:"path_merge"` // 路径合并模式(子文件夹作为一个条目) + TypeTag string `json:"type_tag"` // 类型标签:电影、电视剧等 + ContentTags string `json:"content_tags"` // 内容标签:喜剧、惊悚等(逗号分隔) + EnableScrape bool `gorm:"default:true" json:"enable_scrape"` // 是否启用刮削 + LastScanAt *time.Time `json:"last_scan_at"` // 最后扫描时间 +} + +// MediaIndexColumnMax 复合唯一索引列(folder_path / file_name / album_name)的最大字符数 +const MediaIndexColumnMax = 191 + +// MediaItem 媒体条目(统一表,通过 media_type 区分类型,便于后期扩展新类型) +type MediaItem struct { + // 显式声明基础字段,避免同时嵌入 gorm.Model 后重复生成 deleted_at 索引列; + // JSON 标签继续保持媒体 API 的字段命名约定。 + ID uint `gorm:"primarykey" json:"id"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` + DeletedAt gorm.DeletedAt `gorm:"index" json:"-"` + // 基础信息 + MediaType MediaType `gorm:"size:32;index;not null" json:"media_type"` + ScanPathID uint `gorm:"index" json:"scan_path_id"` // 关联的扫描路径ID + // 三个字段共同组成复合唯一索引。MySQL 对「有索引且未指定长度」的 string 会生成 longtext, + // 而 TEXT 列不能直接建索引(AutoMigrate 会失败);因此必须显式限定 size。 + // utf8mb4 下 InnoDB 索引总长上限 3072 字节:191*3*4 = 2292,留足余量。 + FileName string `gorm:"size:191;uniqueIndex:idx_media_folder_file_album" json:"file_name"` + FileSize int64 `json:"file_size"` + MimeType string `json:"mime_type"` + Hidden bool `gorm:"default:false" json:"hidden"` + + // 刮削/编辑信息 + ScrapedName string `json:"scraped_name"` + Description string `gorm:"type:text" json:"description"` + Cover string `json:"cover"` // 封面URL或本地路径 + ReleaseDate string `json:"release_date"` // 发布时间,格式 YYYY-MM-DD + Rating float32 `json:"rating"` // 评分 0-10 + Genre string `json:"genre"` // 类型,逗号分隔,如 "动作,科幻" + Authors string `gorm:"type:text" json:"authors"` // 作者/演员,JSON数组字符串 + Plot string `gorm:"type:text" json:"plot"` // 剧情/内容介绍 + Reviews string `gorm:"type:text" json:"reviews"` // 用户评价,JSON数组字符串 + + // 外部ID(用于刮削关联) + ExternalID string `json:"external_id"` // TMDB ID / Discogs ID / 豆瓣ID + + // 音乐专属字段 + AlbumName string `gorm:"size:191;uniqueIndex:idx_media_folder_file_album" json:"album_name"` // 所属专辑名 + AlbumArtist string `json:"album_artist"` // 专辑艺术家 + TrackNumber int `json:"track_number"` // 曲目编号 + Duration int `json:"duration"` // 时长(秒) + Lyrics string `gorm:"type:text" json:"lyrics"` // LRC格式歌词 + + // 视频专属字段 + VideoType string `json:"video_type"` // "movie" 或 "tv" + Season int `json:"season"` // 季(电视剧) + Episode int `json:"episode"` // 集(电视剧) + + // 书籍专属字段 + Publisher string `json:"publisher"` // 出版社 + ISBN string `json:"isbn"` // ISBN + + // 目录合并模式 + IsFolder bool `gorm:"default:false" json:"is_folder"` // 是否为文件夹模式条目 + FolderPath string `gorm:"size:191;index;uniqueIndex:idx_media_folder_file_album" json:"folder_path"` // 扫描根路径(恒定为扫描路径,与file_name+album_name组合唯一) + Episodes string `gorm:"type:text" json:"episodes"` // 选集信息,JSON数组,格式:[{file_name,index,title},...] + + ScrapedAt *time.Time `json:"scraped_at"` +} + +// MediaConfig 媒体库配置(每种类型一条记录) +type MediaConfig struct { + ID uint `gorm:"primarykey" json:"id"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` + DeletedAt gorm.DeletedAt `gorm:"index" json:"-"` + // 媒体类型只允许使用 MediaType 定义的短枚举值;显式长度使 MySQL 生成 + // varchar 而不是无法建立索引的 longtext,满足该字段的唯一约束。 + MediaType MediaType `gorm:"size:32;uniqueIndex;not null" json:"media_type"` + Enabled bool `gorm:"default:false" json:"enabled"` + LastScanAt *time.Time `json:"last_scan_at"` + LastScrapeAt *time.Time `json:"last_scrape_at"` + // ScraperConfig 该媒体类型独立的刮削配置(JSON): + // store_thumbnail / thumbnail_mode / thumbnail_path,未设置时回落到全局设置 + ScraperConfig string `gorm:"type:text" json:"scraper_config"` +} + +// EpisodeInfo 选集信息(文件夹型媒体的单集描述) +type EpisodeInfo struct { + FileName string `json:"file_name"` // 原始文件名(含扩展名) + Index int `json:"index"` // 序号,默认0,文件名开头有数字则取该数字 + Title string `json:"title"` // 选集标题(去掉序号后的文件名,不含扩展名) +} + +// MediaScanProgress 扫描进度(内存中维护,不持久化) +type MediaScanProgress struct { + MediaType MediaType `json:"media_type"` + ScanPathID uint `json:"scan_path_id,omitempty"` // 0 表示全部路径 + Running bool `json:"running"` + Total int `json:"total"` + Done int `json:"done"` + Message string `json:"message"` + Error string `json:"error,omitempty"` +} diff --git a/internal/model/media_extra.go b/internal/model/media_extra.go new file mode 100644 index 00000000..212b03ab --- /dev/null +++ b/internal/model/media_extra.go @@ -0,0 +1,94 @@ +package model + +import "time" + +// MediaAppKey 媒体应用密钥。 +// 用于无法走交互式登录的客户端(Subsonic 的 token 认证、Emby 的 api_key、 +// MoviePilot 对接等)。密钥本身以 AES-GCM 加密存储,数据库中不保留明文。 +type MediaAppKey struct { + ID uint `gorm:"primarykey" json:"id"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` + + UserID uint `gorm:"index;not null" json:"user_id"` + Name string `json:"name"` // 备注名,例如 "Symfonium"、"MoviePilot" + // 前缀由 AppKeyPrefix 固定为最多 8 个 ASCII 字符;显式长度使 MySQL + // 将索引列建为 varchar,而不是无法建立索引的 longtext。 + Prefix string `gorm:"size:32;index" json:"prefix"` // 明文前缀(便于识别/排查) + Secret string `gorm:"not null" json:"-"` // AES-GCM 密文(base64) + + LastUsedAt *time.Time `json:"last_used_at"` +} + +// MediaUserData 用户对媒体条目的播放状态,Emby 与 Subsonic 共用。 +type MediaUserData struct { + ID uint `gorm:"primarykey" json:"id"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` + + UserID uint `gorm:"uniqueIndex:idx_media_user_item_ep;not null" json:"user_id"` + ItemID uint `gorm:"uniqueIndex:idx_media_user_item_ep;not null" json:"item_id"` + EpisodeIndex int `gorm:"uniqueIndex:idx_media_user_item_ep;not null" json:"episode_index"` // 0 表示条目本身,>0 表示选集序号 + + Played bool `json:"played"` + PlayCount int `json:"play_count"` + IsFavorite bool `gorm:"index" json:"is_favorite"` + Rating int `json:"rating"` // 0-5,Subsonic 语义 + PositionTicks int64 `json:"position_ticks"` // 100ns tick,与 Emby 一致 + LastPlayedAt *time.Time `json:"last_played_at"` +} + +// MediaPlaylist 用户播放列表(Subsonic 需要,Emby 的 Playlist 复用同一份数据) +type MediaPlaylist struct { + ID uint `gorm:"primarykey" json:"id"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` + + UserID uint `gorm:"index;not null" json:"user_id"` + Name string `json:"name"` + Comment string `json:"comment"` + Public bool `json:"public"` + ItemCount int `json:"item_count"` +} + +// MediaPlaylistItem 播放列表条目(按 Order 排序) +type MediaPlaylistItem struct { + ID uint `gorm:"primarykey" json:"id"` + PlaylistID uint `gorm:"index;not null" json:"playlist_id"` + ItemID uint `gorm:"index;not null" json:"item_id"` + EpisodeIndex int `json:"episode_index"` + Order int `gorm:"column:sort_order" json:"order"` +} + +// MediaAnnotation 用户在 PDF 阅读器里添加的批注。 +// +// 只存查看器原生的批注对象(Anchor,JSON):页码、rect/quadPoints、颜色、 +// 文字、绘制点集都在里面,拆成独立列会出现两份可能不一致的数据。 +// 重新打开时原样回灌给 PDF 引擎,因此这里不重复建模。 +type MediaAnnotation struct { + ID uint `gorm:"primarykey" json:"id"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` + + UserID uint `gorm:"uniqueIndex:idx_media_annotation_key;not null" json:"-"` + ItemID uint `gorm:"uniqueIndex:idx_media_annotation_key;not null;index" json:"item_id"` + // RemoteID 是查看器分配的批注 id:多设备打开同一份 PDF 时靠它做去重与更新 + RemoteID string `gorm:"uniqueIndex:idx_media_annotation_key;size:128;not null" json:"remote_id"` + Anchor string `gorm:"type:text;not null" json:"anchor"` +} + +// EmbyDevice Emby 客户端会话(由 AuthenticateByName 签发) +type EmbyDevice struct { + ID uint `gorm:"primarykey" json:"id"` + CreatedAt time.Time `json:"created_at"` + + UserID uint `gorm:"index;not null" json:"user_id"` + DeviceID string `json:"device_id"` // 客户端上报的 DeviceId + DeviceName string `json:"device_name"` + Client string `json:"client"` + Version string `json:"version"` + // HashToken 始终返回 SHA-256 的 64 个十六进制字符,长度固定且适合唯一索引。 + TokenHash string `gorm:"size:64;uniqueIndex;not null" json:"-"` + AppKeyID uint `json:"app_key_id"` // 通过应用密钥认证时为对应 key,否则为 0 + LastSeenAt time.Time `json:"last_seen_at"` +} diff --git a/internal/model/setting.go b/internal/model/setting.go index 1e0fda0b..d367aa50 100644 --- a/internal/model/setting.go +++ b/internal/model/setting.go @@ -13,6 +13,8 @@ const ( S3 FTP TRAFFIC + MEDIA + TRANSCODE ) const ( diff --git a/internal/model/sharing.go b/internal/model/sharing.go index c5014944..0caa9304 100644 --- a/internal/model/sharing.go +++ b/internal/model/sharing.go @@ -24,12 +24,17 @@ type Sharing struct { } func (s *Sharing) Valid() bool { + return s.Usable() && !s.LimitReached() +} + +func (s *SharingDB) LimitReached() bool { + return s.MaxAccessed > 0 && s.Accessed >= s.MaxAccessed +} + +func (s *Sharing) Usable() bool { if s.Disabled { return false } - if s.MaxAccessed > 0 && s.Accessed >= s.MaxAccessed { - return false - } if len(s.Files) == 0 { return false } diff --git a/internal/model/storage.go b/internal/model/storage.go index a6b4745a..6bda023f 100644 --- a/internal/model/storage.go +++ b/internal/model/storage.go @@ -19,6 +19,7 @@ type Storage struct { Disabled bool `json:"disabled"` // if disabled DisableIndex bool `json:"disable_index"` EnableSign bool `json:"enable_sign"` + SeedPolicy string `json:"seed_policy" gorm:"default:inherit"` Sort Proxy } diff --git a/internal/model/webdav_property.go b/internal/model/webdav_property.go new file mode 100644 index 00000000..00a78b8b --- /dev/null +++ b/internal/model/webdav_property.go @@ -0,0 +1,35 @@ +package model + +import ( + "crypto/sha256" + "encoding/hex" + + "gorm.io/gorm" +) + +// WebDAVProperty stores a dead WebDAV property for a resource. +// +// Path, Namespace and Name are TEXT columns so that long WebDAV paths are +// never truncated or rejected. MySQL cannot index TEXT columns without a key +// prefix length, so the unique index uses a SHA-256 hash of the complete +// property key instead. +type WebDAVProperty struct { + ID uint `json:"id" gorm:"primaryKey"` + Path string `json:"path" gorm:"type:text"` + Namespace string `json:"namespace" gorm:"type:text"` + Name string `json:"name" gorm:"type:text"` + KeyHash string `json:"-" gorm:"size:64;uniqueIndex:idx_webdav_property"` + Lang string `json:"lang"` + InnerXML []byte `json:"inner_xml"` +} + +func (p *WebDAVProperty) BeforeSave(*gorm.DB) error { + p.KeyHash = HashWebDAVPropertyKey(p.Path, p.Namespace, p.Name) + return nil +} + +// HashWebDAVPropertyKey returns a stable hash for the complete property key. +func HashWebDAVPropertyKey(path, namespace, name string) string { + sum := sha256.Sum256([]byte(path + "\x00" + namespace + "\x00" + name)) + return hex.EncodeToString(sum[:]) +} diff --git a/internal/multipart/session_test.go b/internal/multipart/session_test.go index 282d6a48..daec7f80 100644 --- a/internal/multipart/session_test.go +++ b/internal/multipart/session_test.go @@ -364,6 +364,19 @@ func TestInitResume(t *testing.T) { if err != nil { t.Fatalf("Init: %v", err) } + m.mu.Lock() + firstSession := m.byID[snap.ID] + m.mu.Unlock() + sessions := []*Session{firstSession} + t.Cleanup(func() { + for _, session := range sessions { + if session == nil { + continue + } + m.terminate(session, context.Canceled) + <-session.done + } + }) sendChunk(t, m, user, snap.ID, data, 2, chunkSize) // same hashes prove the same file: resume with buffered chunks skippable @@ -383,6 +396,9 @@ func TestInitResume(t *testing.T) { if err != nil || resumed || other.ID == snap.ID { t.Fatalf("different-size Init = (id=%s, resumed=%v, err=%v)", other.ID, resumed, err) } + m.mu.Lock() + sessions = append(sessions, m.byID[other.ID]) + m.mu.Unlock() if _, err := m.Find(user, "/local/test.bin", totalSize); err != nil { t.Fatalf("Find: %v", err) @@ -399,6 +415,9 @@ func TestInitResume(t *testing.T) { if err != nil || resumed || fresh.ID == snap.ID { t.Fatalf("bare re-Init = (id=%s, resumed=%v, err=%v), want a fresh session", fresh.ID, resumed, err) } + m.mu.Lock() + sessions = append(sessions, m.byID[fresh.ID]) + m.mu.Unlock() if _, err := m.Status(user, snap.ID); !errors.Is(err, ErrSessionNotFound) { t.Fatalf("superseded session err = %v, want ErrSessionNotFound", err) } diff --git a/internal/net/oss_test.go b/internal/net/oss_test.go index 9001cd39..ee901ff8 100644 --- a/internal/net/oss_test.go +++ b/internal/net/oss_test.go @@ -18,7 +18,7 @@ func TestNewOSSClientUsesEnvironmentHTTPSProxy(t *testing.T) { t.Setenv("HTTP_PROXY", "") t.Setenv("http_proxy", "") t.Setenv("HTTPS_PROXY", "http://127.0.0.1:7890") - t.Setenv("https_proxy", "") + t.Setenv("https_proxy", "http://127.0.0.1:7890") t.Setenv("NO_PROXY", "") t.Setenv("no_proxy", "") @@ -31,9 +31,13 @@ func TestNewOSSClientUsesEnvironmentHTTPSProxy(t *testing.T) { t.Fatal("expected OSS client to use a custom HTTP client") } - transport, ok := client.HTTPClient.Transport.(*http.Transport) + safe, ok := client.HTTPClient.Transport.(*safeTransport) if !ok { - t.Fatalf("expected *http.Transport, got %T", client.HTTPClient.Transport) + t.Fatalf("expected *safeTransport, got %T", client.HTTPClient.Transport) + } + transport, ok := safe.base.(*http.Transport) + if !ok { + t.Fatalf("expected safe transport to wrap *http.Transport, got %T", safe.base) } if transport.Proxy == nil { diff --git a/internal/net/serve.go b/internal/net/serve.go index fe1a4a7e..53604c3e 100644 --- a/internal/net/serve.go +++ b/internal/net/serve.go @@ -268,7 +268,6 @@ var unsafeProxyHeaders = map[string]struct{}{ "x-real-ip": {}, } - func ProcessHeader(origin, override http.Header) http.Header { result := http.Header{} // client header @@ -345,9 +344,13 @@ func HttpClient() *http.Client { } func NewHttpClient() *http.Client { + insecureSkipVerify := false + if conf.Conf != nil { + insecureSkipVerify = conf.Conf.TlsInsecureSkipVerify + } transport := &http.Transport{ Proxy: http.ProxyFromEnvironment, - TLSClientConfig: &tls.Config{InsecureSkipVerify: conf.Conf.TlsInsecureSkipVerify}, + TLSClientConfig: &tls.Config{InsecureSkipVerify: insecureSkipVerify}, } SetProxyIfConfigured(transport) diff --git a/internal/net/util.go b/internal/net/util.go index f0f77033..6bb31612 100644 --- a/internal/net/util.go +++ b/internal/net/util.go @@ -69,7 +69,7 @@ func GetRangedHttpReader(readCloser io.ReadCloser, offset, length int64) (io.Rea // SetProxyIfConfigured sets proxy for HTTP Transport if configured func SetProxyIfConfigured(transport *http.Transport) { // If proxy address is configured, override environment variable settings - if conf.Conf.ProxyAddress != "" { + if conf.Conf != nil && conf.Conf.ProxyAddress != "" { if proxyURL, err := url.Parse(conf.Conf.ProxyAddress); err == nil { transport.Proxy = http.ProxyURL(proxyURL) } @@ -79,7 +79,7 @@ func SetProxyIfConfigured(transport *http.Transport) { // SetRestyProxyIfConfigured sets proxy for Resty client if configured func SetRestyProxyIfConfigured(client *resty.Client) { // If proxy address is configured, override environment variable settings - if conf.Conf.ProxyAddress != "" { + if conf.Conf != nil && conf.Conf.ProxyAddress != "" { if proxyURL, err := url.Parse(conf.Conf.ProxyAddress); err == nil { client.SetProxy(proxyURL.String()) } diff --git a/internal/offline_download/115/client.go b/internal/offline_download/115/client.go index d31971d7..c026ee7e 100644 --- a/internal/offline_download/115/client.go +++ b/internal/offline_download/115/client.go @@ -125,7 +125,7 @@ func (p *Cloud115) Status(task *tool.DownloadTask) (*tool.Status, error) { s.Completed = t.IsDone() s.TotalBytes = t.Size if t.IsFailed() { - s.Err = fmt.Errorf(t.GetStatus()) + s.Err = fmt.Errorf("%s", t.GetStatus()) } return s, nil } diff --git a/internal/offline_download/115_open/client.go b/internal/offline_download/115_open/client.go index 6f4c7fd0..dd179f31 100644 --- a/internal/offline_download/115_open/client.go +++ b/internal/offline_download/115_open/client.go @@ -122,7 +122,7 @@ func (o *Open115) Status(task *tool.DownloadTask) (*tool.Status, error) { s.Completed = t.IsDone() s.TotalBytes = t.Size if t.IsFailed() { - s.Err = fmt.Errorf(t.GetStatus()) + s.Err = fmt.Errorf("%s", t.GetStatus()) } return s, nil } diff --git a/internal/offline_download/pikpak/pikpak.go b/internal/offline_download/pikpak/pikpak.go index 8a669309..e223937e 100644 --- a/internal/offline_download/pikpak/pikpak.go +++ b/internal/offline_download/pikpak/pikpak.go @@ -127,7 +127,7 @@ func (p *PikPak) Status(task *tool.DownloadTask) (*tool.Status, error) { s.TotalBytes = 0 } if t.Phase == "PHASE_TYPE_ERROR" { - s.Err = fmt.Errorf(t.Message) + s.Err = fmt.Errorf("%s", t.Message) } return s, nil } diff --git a/internal/op/driver.go b/internal/op/driver.go index 5b79b0ae..38fe2f56 100644 --- a/internal/op/driver.go +++ b/internal/op/driver.go @@ -173,6 +173,14 @@ func getMainItems(config driver.Config) []driver.Item { Default: "false", Required: true, }) + items = append(items, driver.Item{ + Name: "seed_policy", + Type: conf.TypeSelect, + Options: "inherit,on,off", + Default: "inherit", + Required: true, + Help: "Override automatic transfer-seed generation for this storage", + }) return items } func getAdditionalItems(t reflect.Type, defaultRoot string) []driver.Item { diff --git a/internal/op/fs.go b/internal/op/fs.go index e6278ca0..519b2313 100644 --- a/internal/op/fs.go +++ b/internal/op/fs.go @@ -2,6 +2,7 @@ package op import ( "context" + "net/url" stdpath "path" "strconv" "strings" @@ -48,7 +49,11 @@ func list(ctx context.Context, storage driver.Driver, path string, args model.Li } } - objs, err, _ := listG.Do(key, func() ([]model.Obj, error) { + flightKey := key + if base, ok := ctx.Value(conf.ThumbnailBaseURLKey).(*url.URL); ok && storage.Config().NoCache { + flightKey += ":" + strconv.Quote(base.String()) + } + objs, err, _ := listG.Do(flightKey, func() ([]model.Obj, error) { dir, err := GetUnwrap(ctx, storage, path) if err != nil { return nil, errors.WithMessage(err, "failed get dir") @@ -519,7 +524,7 @@ func Copy(ctx context.Context, storage driver.Driver, srcPath, dstDirPath string srcPath = utils.FixAndCleanPath(srcPath) dstDirPath = utils.FixAndCleanPath(dstDirPath) if dstDirPath == stdpath.Dir(srcPath) { - return errors.New("copy in place") + return errors.WithStack(errs.NotImplement) } srcRawObj, err := Get(ctx, storage, srcPath, true) if err != nil { @@ -637,7 +642,7 @@ func Put(ctx context.Context, storage driver.Driver, dstDirPath string, file mod if err != nil { return errors.WithMessagef(err, "while uploading, failed remove existing file which size = 0") } - } else if storage.Config().NoOverwriteUpload { + } else if storage.Config().NoOverwriteUpload && ctx.Value(conf.SkipNoOverwriteKey) == nil { // try to rename old obj err = Rename(ctx, storage, dstPath, tempName) if err != nil { diff --git a/internal/op/sharing.go b/internal/op/sharing.go index 234ca415..95a9da0e 100644 --- a/internal/op/sharing.go +++ b/internal/op/sharing.go @@ -4,8 +4,10 @@ import ( "fmt" stdpath "path" "strings" + "time" "github.com/OpenListTeam/OpenList/v4/internal/db" + "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/pkg/singleflight" "github.com/OpenListTeam/OpenList/v4/pkg/utils" @@ -41,6 +43,67 @@ func makeJoined(sdb []model.SharingDB) []model.Sharing { var sharingCache = cache.NewMemCache(cache.WithShards[*model.Sharing](8)) var sharingG singleflight.Group[*model.Sharing] +var sharingAccessCache = cache.NewMemCache[struct{}]() + +const sharingAccessCountDelay = 30 * time.Minute + +func sharingAccessKey(id, ip string) string { + return id + ":" + ip +} + +var sharingAccessG singleflight.Group[struct{}] + +func validateSharing(pwd string, s *model.Sharing) error { + if !s.Usable() { + return errors.WithStack(errs.InvalidSharing) + } + if !s.Verify(pwd) { + return errors.WithStack(errs.WrongShareCode) + } + return nil +} + +func CheckSharing(ip, pwd string, s *model.Sharing) error { + if err := validateSharing(pwd, s); err != nil { + return err + } + key := sharingAccessKey(s.ID, ip) + if s.LimitReached() && !sharingAccessCache.Exists(key) { + _, err, _ := sharingAccessG.Do(key, func() (struct{}, error) { + return struct{}{}, nil + }) + if err != nil { + return err + } + if !sharingAccessCache.Exists(key) { + return errors.WithStack(errs.InvalidSharing) + } + } + return nil +} + +func AccessSharing(ip, pwd string, s *model.Sharing) error { + if err := validateSharing(pwd, s); err != nil { + return err + } + key := sharingAccessKey(s.ID, ip) + _, err, _ := sharingAccessG.Do(key, func() (struct{}, error) { + if sharingAccessCache.Exists(key) { + return struct{}{}, nil + } + ok, err := db.TakeSharingSlot(s.ID) + if err != nil { + return struct{}{}, err + } + if !ok { + return struct{}{}, errors.WithStack(errs.InvalidSharing) + } + sharingCache.Del(s.ID) + sharingAccessCache.Set(key, struct{}{}, cache.WithEx[struct{}](sharingAccessCountDelay)) + return struct{}{}, nil + }) + return err +} func GetSharingById(id string, refresh ...bool) (*model.Sharing, error) { if !utils.IsBool(refresh...) { diff --git a/internal/op/sharing_race_test.go b/internal/op/sharing_race_test.go new file mode 100644 index 00000000..04f0b398 --- /dev/null +++ b/internal/op/sharing_race_test.go @@ -0,0 +1,60 @@ +package op + +import ( + "testing" + "time" + + "github.com/OpenListTeam/OpenList/v4/internal/db" + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/pkg/utils/random" + "github.com/OpenListTeam/go-cache" +) + +func TestCheckSharingWaitsForAccessSlotPublication(t *testing.T) { + s := &model.Sharing{ + SharingDB: &model.SharingDB{MaxAccessed: 1}, + Files: []string{"/a"}, + Creator: &model.User{Permission: 1 << 14}, + } + if _, err := db.CreateSharing(s.SharingDB); err != nil { + t.Fatal(err) + } + + ip := random.String(16) + key := sharingAccessKey(s.ID, ip) + started := make(chan struct{}) + release := make(chan struct{}) + accessDone := make(chan struct{}) + go func() { + _, _, _ = sharingAccessG.Do(key, func() (struct{}, error) { + close(started) + <-release + sharingAccessCache.Set(key, struct{}{}, cache.WithEx[struct{}](sharingAccessCountDelay)) + return struct{}{}, nil + }) + close(accessDone) + }() + <-started + + result := make(chan error, 1) + s.Accessed = 1 + go func() { + result <- CheckSharing(ip, "", s) + }() + select { + case err := <-result: + t.Fatalf("check returned before access cache publication: %v", err) + case <-time.After(100 * time.Millisecond): + } + + close(release) + select { + case err := <-result: + if err != nil { + t.Fatalf("check after access cache publication: %v", err) + } + case <-time.After(time.Second): + t.Fatal("check did not complete after access cache publication") + } + <-accessDone +} diff --git a/internal/op/sharing_test.go b/internal/op/sharing_test.go new file mode 100644 index 00000000..a62618ab --- /dev/null +++ b/internal/op/sharing_test.go @@ -0,0 +1,129 @@ +package op_test + +import ( + "errors" + "fmt" + "sync" + "sync/atomic" + "testing" + + "github.com/OpenListTeam/OpenList/v4/internal/db" + "github.com/OpenListTeam/OpenList/v4/internal/errs" + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/internal/op" +) + +func newTestSharing(t *testing.T, maxAccessed int) *model.Sharing { + t.Helper() + s := &model.Sharing{ + SharingDB: &model.SharingDB{MaxAccessed: maxAccessed}, + Files: []string{"/a"}, + Creator: &model.User{Permission: 1 << 14}, + } + if _, err := db.CreateSharing(s.SharingDB); err != nil { + t.Fatal(err) + } + return s +} + +func accessed(t *testing.T, id string) int { + t.Helper() + s, err := db.GetSharingById(id) + if err != nil { + t.Fatal(err) + } + return s.Accessed +} + +func TestSharingAccessLimit(t *testing.T) { + s := newTestSharing(t, 1) + + // browsing never takes a slot + for range 3 { + if err := op.CheckSharing("1.1.1.1", "", s); err != nil { + t.Fatalf("check before download: %v", err) + } + } + if n := accessed(t, s.ID); n != 0 { + t.Fatalf("metadata took a slot: accessed = %d", n) + } + + if err := op.AccessSharing("1.1.1.1", "", s); err != nil { + t.Fatalf("first download: %v", err) + } + if n := accessed(t, s.ID); n != 1 { + t.Fatalf("accessed = %d, want 1", n) + } + + // #2143: the visitor holding the last slot keeps working + s.Accessed = 1 + if err := op.CheckSharing("1.1.1.1", "", s); err != nil { + t.Fatalf("slot holder check: %v", err) + } + if err := op.AccessSharing("1.1.1.1", "", s); err != nil { + t.Fatalf("slot holder download: %v", err) + } + + // new visitors are rejected + if err := op.CheckSharing("2.2.2.2", "", s); !errors.Is(err, errs.InvalidSharing) { + t.Fatalf("new visitor check: %v", err) + } + if err := op.AccessSharing("2.2.2.2", "", s); !errors.Is(err, errs.InvalidSharing) { + t.Fatalf("new visitor download: %v", err) + } + if n := accessed(t, s.ID); n != 1 { + t.Fatalf("accessed = %d, want 1", n) + } +} + +func TestSharingAccessUsesDatabaseState(t *testing.T) { + s := newTestSharing(t, 1) + + // The in-memory sharing metadata can be stale while the database is authoritative. + s.Accessed = 1 + if err := op.AccessSharing("3.3.3.3", "", s); err != nil { + t.Fatalf("stale metadata rejected access: %v", err) + } + if n := accessed(t, s.ID); n != 1 { + t.Fatalf("accessed = %d, want 1", n) + } +} + +func TestSharingAccessConcurrent(t *testing.T) { + t.Run("distinct ips never exceed limit", func(t *testing.T) { + s := newTestSharing(t, 3) + var ok atomic.Int32 + var wg sync.WaitGroup + for i := range 20 { + wg.Add(1) + go func() { + defer wg.Done() + if op.AccessSharing(fmt.Sprintf("10.0.0.%d", i), "", s) == nil { + ok.Add(1) + } + }() + } + wg.Wait() + if ok.Load() != 3 || accessed(t, s.ID) != 3 { + t.Fatalf("granted = %d, accessed = %d, want 3/3", ok.Load(), accessed(t, s.ID)) + } + }) + + t.Run("same ip counts once", func(t *testing.T) { + s := newTestSharing(t, 1) + var wg sync.WaitGroup + for range 20 { + wg.Add(1) + go func() { + defer wg.Done() + if err := op.AccessSharing("10.0.1.1", "", s); err != nil { + t.Errorf("same ip rejected: %v", err) + } + }() + } + wg.Wait() + if n := accessed(t, s.ID); n != 1 { + t.Fatalf("accessed = %d, want 1", n) + } + }) +} diff --git a/internal/sharing/archive.go b/internal/sharing/archive.go index ddfabd8c..ed0e649b 100644 --- a/internal/sharing/archive.go +++ b/internal/sharing/archive.go @@ -15,11 +15,8 @@ func archiveMeta(ctx context.Context, sid, path string, args model.SharingArchiv if err != nil { return nil, nil, errors.WithStack(errs.SharingNotFound) } - if !sharing.Valid() { - return sharing, nil, errors.WithStack(errs.InvalidSharing) - } - if !sharing.Verify(args.Pwd) { - return sharing, nil, errors.WithStack(errs.WrongShareCode) + if err = op.CheckSharing(args.ClientIP, args.Pwd, sharing); err != nil { + return sharing, nil, err } path = utils.FixAndCleanPath(path) if len(sharing.Files) == 1 || path != "/" { @@ -42,11 +39,8 @@ func archiveList(ctx context.Context, sid, path string, args model.SharingArchiv if err != nil { return nil, nil, errors.WithStack(errs.SharingNotFound) } - if !sharing.Valid() { - return sharing, nil, errors.WithStack(errs.InvalidSharing) - } - if !sharing.Verify(args.Pwd) { - return sharing, nil, errors.WithStack(errs.WrongShareCode) + if err = op.CheckSharing(args.ClientIP, args.Pwd, sharing); err != nil { + return sharing, nil, err } path = utils.FixAndCleanPath(path) if len(sharing.Files) == 1 || path != "/" { diff --git a/internal/sharing/get.go b/internal/sharing/get.go index 28815ad3..6091962c 100644 --- a/internal/sharing/get.go +++ b/internal/sharing/get.go @@ -17,11 +17,8 @@ func get(ctx context.Context, sid, path string, args model.SharingListArgs) (*mo if err != nil { return nil, nil, errors.WithStack(errs.SharingNotFound) } - if !sharing.Valid() { - return sharing, nil, errors.WithStack(errs.InvalidSharing) - } - if !sharing.Verify(args.Pwd) { - return sharing, nil, errors.WithStack(errs.WrongShareCode) + if err = op.CheckSharing(args.ClientIP, args.Pwd, sharing); err != nil { + return sharing, nil, err } path = utils.FixAndCleanPath(path) if len(sharing.Files) == 1 || path != "/" { diff --git a/internal/sharing/link.go b/internal/sharing/link.go index e021f3c2..03d2e198 100644 --- a/internal/sharing/link.go +++ b/internal/sharing/link.go @@ -17,30 +17,27 @@ func link(ctx context.Context, sid, path string, args *LinkArgs) (*model.Sharing if err != nil { return nil, nil, nil, errors.WithStack(errs.SharingNotFound) } - if !sharing.Valid() { - return sharing, nil, nil, errors.WithStack(errs.InvalidSharing) - } - if !sharing.Verify(args.Pwd) { - return sharing, nil, nil, errors.WithStack(errs.WrongShareCode) - } path = utils.FixAndCleanPath(path) - if len(sharing.Files) == 1 || path != "/" { - unwrapPath, err := op.GetSharingUnwrapPath(sharing, path) - if err != nil { - return nil, nil, nil, errors.WithMessage(err, "failed get sharing unwrap path") - } - storage, actualPath, err := op.GetStorageAndActualPath(unwrapPath) - if err != nil { - return nil, nil, nil, errors.WithMessage(err, "failed get sharing link") - } - l, obj, err := op.Link(ctx, storage, actualPath, args.LinkArgs) - if err != nil { - return nil, nil, nil, errors.WithMessage(err, "failed get sharing link") - } - if l.URL != "" && !strings.HasPrefix(l.URL, "http://") && !strings.HasPrefix(l.URL, "https://") { - l.URL = conf.GetApiUrl(ctx) + l.URL - } - return sharing, l, obj, nil + if len(sharing.Files) != 1 && path == "/" { + return nil, nil, nil, errors.New("cannot get sharing root link") } - return nil, nil, nil, errors.New("cannot get sharing root link") + if err = op.AccessSharing(args.ClientIP, args.Pwd, sharing); err != nil { + return sharing, nil, nil, err + } + unwrapPath, err := op.GetSharingUnwrapPath(sharing, path) + if err != nil { + return nil, nil, nil, errors.WithMessage(err, "failed get sharing unwrap path") + } + storage, actualPath, err := op.GetStorageAndActualPath(unwrapPath) + if err != nil { + return nil, nil, nil, errors.WithMessage(err, "failed get sharing link") + } + l, obj, err := op.Link(ctx, storage, actualPath, args.LinkArgs) + if err != nil { + return nil, nil, nil, errors.WithMessage(err, "failed get sharing link") + } + if l.URL != "" && !strings.HasPrefix(l.URL, "http://") && !strings.HasPrefix(l.URL, "https://") { + l.URL = conf.GetApiUrl(ctx) + l.URL + } + return sharing, l, obj, nil } diff --git a/internal/sharing/list.go b/internal/sharing/list.go index 0600e8a8..4c661f16 100644 --- a/internal/sharing/list.go +++ b/internal/sharing/list.go @@ -2,8 +2,10 @@ package sharing import ( "context" + "net/url" stdpath "path" + "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/op" @@ -16,11 +18,8 @@ func list(ctx context.Context, sid, path string, args model.SharingListArgs) (*m if err != nil { return nil, nil, errors.WithStack(errs.SharingNotFound) } - if !sharing.Valid() { - return sharing, nil, errors.WithStack(errs.InvalidSharing) - } - if !sharing.Verify(args.Pwd) { - return sharing, nil, errors.WithStack(errs.WrongShareCode) + if err = op.CheckSharing(args.ClientIP, args.Pwd, sharing); err != nil { + return sharing, nil, err } path = utils.FixAndCleanPath(path) if len(sharing.Files) == 1 || path != "/" { @@ -35,6 +34,14 @@ func list(ctx context.Context, sid, path string, args model.SharingListArgs) (*m } var objs []model.Obj if storage != nil { + query := url.Values{"type": {"thumb"}} + if sharing.Pwd != "" { + query.Set("pwd", sharing.Pwd) + } + ctx = context.WithValue(ctx, conf.ThumbnailBaseURLKey, &url.URL{ + Path: stdpath.Join("/sd", sid, path), + RawQuery: query.Encode(), + }) objs, err = op.List(ctx, storage, actualPath, model.ListArgs{ Refresh: args.Refresh, ReqPath: stdpath.Join(sid, path), diff --git a/internal/transcode/cache.go b/internal/transcode/cache.go new file mode 100644 index 00000000..918874ce --- /dev/null +++ b/internal/transcode/cache.go @@ -0,0 +1,489 @@ +package transcode + +import ( + "errors" + "fmt" + "io" + "os" + "path/filepath" + "regexp" + "sort" + "strconv" + "strings" + "sync" + "time" +) + +// SegmentInfo 单个切片元数据 +type SegmentInfo struct { + Seq int `json:"seq"` + Duration float64 `json:"duration"` + Size int64 `json:"size"` + Path string `json:"-"` // 本地实际文件路径 +} + +// JobCache 单个 Job 的切片缓存 +type JobCache struct { + JobID string + Profile string + BaseDir string + + mu sync.RWMutex + segments map[int]*SegmentInfo + final bool // 是否所有切片都到齐 + waiters map[int][]chan struct{} // seq -> 等待这一片的 channel 列表 +} + +// Cache 全局切片缓存 +type Cache struct { + rootDir string + maxBytes int64 + mu sync.RWMutex + jobs map[string]map[string]*JobCache // jobID -> profile -> cache +} + +func NewCache(rootDir string, maxGB int64) *Cache { + if maxGB <= 0 { + maxGB = 20 + } + return &Cache{ + rootDir: rootDir, + maxBytes: maxGB * 1024 * 1024 * 1024, + jobs: make(map[string]map[string]*JobCache), + } +} + +// Init 创建根目录 +func (c *Cache) Init() error { + return os.MkdirAll(c.rootDir, 0o755) +} + +const ( + // MaxSegmentBytes 单个切片允许写入的最大字节数,防止 callback_token 持有者打满磁盘 + MaxSegmentBytes int64 = 32 << 20 + // maxSegmentSeq 切片序号上限(按 2 秒/片约 6 天),拒绝负数与异常大值 + maxSegmentSeq = 200000 +) + +var ( + profileNameRe = regexp.MustCompile(`^[A-Za-z0-9_-]{1,32}$`) + jobIDRe = regexp.MustCompile(`^[A-Za-z0-9_-]{1,64}$`) + + // ErrInvalidProfile profile 名称不合法(可能是路径穿越尝试) + ErrInvalidProfile = errors.New("invalid profile name") + // ErrInvalidJobID job id 不合法 + ErrInvalidJobID = errors.New("invalid job id") + // ErrInvalidSegment 切片序号越界 + ErrInvalidSegment = errors.New("invalid segment seq") + // ErrSegmentTooLarge 切片超过大小上限 + ErrSegmentTooLarge = errors.New("segment too large") +) + +// ValidProfileName 判断 profile 是否为安全的目录名(与 BuildProfile 产出的 source/1080p/720p 对齐)。 +func ValidProfileName(profile string) bool { + return profileNameRe.MatchString(profile) +} + +// safeJoin 在 rootDir 下拼接子路径,并保证结果不会越出 rootDir。 +func (c *Cache) safeJoin(jobID, profile string) (string, error) { + if !jobIDRe.MatchString(jobID) { + return "", ErrInvalidJobID + } + if !ValidProfileName(profile) { + return "", ErrInvalidProfile + } + full := filepath.Join(c.rootDir, jobID, profile) + rel, err := filepath.Rel(c.rootDir, full) + if err != nil || rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) { + return "", ErrInvalidProfile + } + return full, nil +} + +// getOrCreate 获取或创建 (job, profile) 的 cache +func (c *Cache) getOrCreate(jobID, profile string) (*JobCache, error) { + baseDir, err := c.safeJoin(jobID, profile) + if err != nil { + return nil, err + } + c.mu.Lock() + defer c.mu.Unlock() + pmap, ok := c.jobs[jobID] + if !ok { + pmap = make(map[string]*JobCache) + c.jobs[jobID] = pmap + } + jc, ok := pmap[profile] + if !ok { + jc = &JobCache{ + JobID: jobID, + Profile: profile, + BaseDir: baseDir, + segments: make(map[int]*SegmentInfo), + waiters: make(map[int][]chan struct{}), + } + if err := os.MkdirAll(jc.BaseDir, 0o755); err != nil { + return nil, err + } + pmap[profile] = jc + } + return jc, nil +} + +// PutSegment 写入一个切片,并通知所有等待者 +func (c *Cache) PutSegment(jobID, profile string, seq int, duration float64, body io.Reader, isFinal bool) (*SegmentInfo, error) { + if seq < 0 || seq > maxSegmentSeq { + return nil, ErrInvalidSegment + } + jc, err := c.getOrCreate(jobID, profile) + if err != nil { + return nil, err + } + fname := fmt.Sprintf("seg-%d.ts", seq) + fpath := filepath.Join(jc.BaseDir, fname) + tmpPath := fpath + ".tmp" + f, err := os.Create(tmpPath) + if err != nil { + return nil, err + } + // 多读 1 字节用于判断是否超限 + n, err := io.Copy(f, io.LimitReader(body, MaxSegmentBytes+1)) + if err == nil && n > MaxSegmentBytes { + err = ErrSegmentTooLarge + } + cerr := f.Close() + if err != nil { + os.Remove(tmpPath) + return nil, err + } + if cerr != nil { + os.Remove(tmpPath) + return nil, cerr + } + if err := os.Rename(tmpPath, fpath); err != nil { + os.Remove(tmpPath) + return nil, err + } + info := &SegmentInfo{Seq: seq, Duration: duration, Size: n, Path: fpath} + + jc.mu.Lock() + jc.segments[seq] = info + if isFinal { + jc.final = true + } + // 通知所有等待 <= seq 的 waiter + var toClose []chan struct{} + for s, chs := range jc.waiters { + if s <= seq { + toClose = append(toClose, chs...) + delete(jc.waiters, s) + } + } + jc.mu.Unlock() + for _, ch := range toClose { + close(ch) + } + return info, nil +} + +// GetSegment 读取一个切片(不等待) +func (c *Cache) GetSegment(jobID, profile string, seq int) (*SegmentInfo, error) { + c.mu.RLock() + pmap, ok := c.jobs[jobID] + c.mu.RUnlock() + if !ok { + return nil, errors.New("job not found") + } + jc, ok := pmap[profile] + if !ok { + return nil, errors.New("profile not found") + } + jc.mu.RLock() + defer jc.mu.RUnlock() + info, ok := jc.segments[seq] + if !ok { + return nil, errors.New("segment not found") + } + return info, nil +} + +// WaitForSegment 等待某个 seq 切片就绪,timeout 后返回 nil +func (c *Cache) WaitForSegment(jobID, profile string, seq int, timeout time.Duration) (*SegmentInfo, error) { + jc, err := c.getOrCreate(jobID, profile) + if err != nil { + return nil, err + } + jc.mu.Lock() + if info, ok := jc.segments[seq]; ok { + jc.mu.Unlock() + return info, nil + } + if jc.final { + jc.mu.Unlock() + return nil, errors.New("segment never produced") + } + ch := make(chan struct{}) + jc.waiters[seq] = append(jc.waiters[seq], ch) + jc.mu.Unlock() + select { + case <-ch: + jc.mu.RLock() + info := jc.segments[seq] + jc.mu.RUnlock() + if info == nil { + return nil, errors.New("segment not produced") + } + return info, nil + case <-time.After(timeout): + // 【内存泄漏修复】超时后必须从 waiters map 中移除自己的 channel, + // 否则 channel 会永远留在 map 中无法被 GC + jc.mu.Lock() + if chs, ok := jc.waiters[seq]; ok { + for i, w := range chs { + if w == ch { + jc.waiters[seq] = append(chs[:i], chs[i+1:]...) + break + } + } + if len(jc.waiters[seq]) == 0 { + delete(jc.waiters, seq) + } + } + jc.mu.Unlock() + return nil, errors.New("timeout waiting segment") + } +} + +// SegmentCount 返回 (jobID, profile) 当前已生成的切片数量。 +// 用于 TCPlaylist 在首次返回前等待至少若干切片就绪,避免 playlist +// 中只有 seg-0 导致 HLS.js 反复请求填充缓冲。 +func (c *Cache) SegmentCount(jobID, profile string) int { + c.mu.RLock() + pmap, ok := c.jobs[jobID] + c.mu.RUnlock() + if !ok { + return 0 + } + jc, ok := pmap[profile] + if !ok { + return 0 + } + jc.mu.RLock() + defer jc.mu.RUnlock() + return len(jc.segments) +} + +// BuildPlaylist 生成 HLS playlist (m3u8) +// 【重要】当 totalDuration > 0 时,会根据总时长预先生成完整切片列表(VOD 模式), +// 即使切片还未生成,URL 也会出现在 playlist 中。HLS.js 请求未生成切片时 +// 会阻塞等待(见 TCSegment 的 WaitForSegment)。这样可以让 HLS.js 知道 +// 视频总时长,避免: +// 1. 时长显示错误(只显示已转码部分) +// 2. 拖动跳跃(playlist 增长导致切片序号重新对齐) +func (c *Cache) BuildPlaylist(jobID, profile string, segDur int, completed bool, baseURL string, totalDuration float64) string { + c.mu.RLock() + pmap, ok := c.jobs[jobID] + c.mu.RUnlock() + if !ok { + return "" + } + jc, ok := pmap[profile] + if !ok { + return "" + } + jc.mu.RLock() + defer jc.mu.RUnlock() + + var b strings.Builder + b.WriteString("#EXTM3U\n") + b.WriteString("#EXT-X-VERSION:3\n") + b.WriteString("#EXT-X-TARGETDURATION:" + strconv.Itoa(segDur) + "\n") + b.WriteString("#EXT-X-MEDIA-SEQUENCE:0\n") + + // 如果有总时长,按总时长生成完整切片列表 + if totalDuration > 0 { + // VOD 模式:完整列表,HLS.js 知道总时长 + b.WriteString("#EXT-X-PLAYLIST-TYPE:VOD\n") + fSegDur := float64(segDur) + // 总切片数 = ceil(总时长 / 每片时长) + totalSegs := int(totalDuration / fSegDur) + lastDur := totalDuration - float64(totalSegs)*fSegDur + if lastDur > 0.01 { + totalSegs++ + } + for i := 0; i < totalSegs; i++ { + dur := fSegDur + if i == totalSegs-1 && lastDur > 0.01 { + dur = lastDur + } + // 优先使用真实时长(如果已知) + if seg, ok := jc.segments[i]; ok && seg.Duration > 0 { + dur = seg.Duration + } + b.WriteString(fmt.Sprintf("#EXTINF:%.3f,\n", dur)) + b.WriteString(fmt.Sprintf("%sseg-%d.ts\n", baseURL, i)) + } + b.WriteString("#EXT-X-ENDLIST\n") + return b.String() + } + + // fallback: 没有总时长时退回到原 EVENT 模式(保留旧行为) + seqs := make([]int, 0, len(jc.segments)) + for s := range jc.segments { + seqs = append(seqs, s) + } + sort.Ints(seqs) + if jc.final || completed { + b.WriteString("#EXT-X-PLAYLIST-TYPE:VOD\n") + } else { + b.WriteString("#EXT-X-PLAYLIST-TYPE:EVENT\n") + } + for _, s := range seqs { + seg := jc.segments[s] + b.WriteString(fmt.Sprintf("#EXTINF:%.3f,\n", seg.Duration)) + b.WriteString(fmt.Sprintf("%sseg-%d.ts\n", baseURL, s)) + } + if jc.final || completed { + b.WriteString("#EXT-X-ENDLIST\n") + } + return b.String() +} + +// MasterPlaylist 多档位主 m3u8 +func (c *Cache) MasterPlaylist(profiles []Profile, baseURL string) string { + var b strings.Builder + b.WriteString("#EXTM3U\n#EXT-X-VERSION:3\n") + for _, p := range profiles { + bw, _ := parseBitrate(p.VideoBitrate) + ab, _ := parseBitrate(p.AudioBitrate) + // 【关键修复】根据实际编码动态生成 codec 字符串 + // 错误的 codec 声明会导致 HLS.js 用错误参数初始化 SourceBuffer + // 然后 appendBuffer 时数据格式不匹配 → bufferAppendError 死循环 + codecs := buildCodecString(p) + b.WriteString(fmt.Sprintf("#EXT-X-STREAM-INF:BANDWIDTH=%d,CODECS=\"%s\"\n", bw+ab, codecs)) + b.WriteString(fmt.Sprintf("%s%s/playlist.m3u8\n", baseURL, p.Name)) + } + return b.String() +} + +// buildCodecString 根据 Profile 生成 HLS CODECS 属性字符串 +// avc1.42E01E = H.264 Baseline Profile Level 3.0(兼容性最好) +// avc1.4D401E = H.264 Main Profile Level 3.0 +// avc1.640028 = H.264 High Profile Level 4.0 +// mp4a.40.2 = AAC-LC(HE-AAC v1 用 mp4a.40.5,HE-AAC v2 用 mp4a.40.29) +func buildCodecString(p Profile) string { + // 视频部分:默认 H.264 Main Profile Level 4.0,平衡兼容性与质量 + video := "avc1.4D4028" + switch strings.ToLower(p.VideoCodec) { + case "hevc", "h265": + // HEVC Main Profile Level 4.1,部分 Safari/Edge 支持 + video = "hvc1.1.6.L120.90" + case "av1": + video = "av01.0.05M.08" + } + // 音频部分:固定 AAC-LC(后端已强制转 AAC) + audio := "mp4a.40.2" + return video + "," + audio +} + +func parseBitrate(s string) (int64, error) { + s = strings.TrimSpace(strings.ToLower(s)) + if s == "" { + return 0, nil + } + mult := int64(1) + if strings.HasSuffix(s, "k") { + mult = 1000 + s = strings.TrimSuffix(s, "k") + } else if strings.HasSuffix(s, "m") { + mult = 1_000_000 + s = strings.TrimSuffix(s, "m") + } + v, err := strconv.ParseInt(s, 10, 64) + if err != nil { + return 0, err + } + return v * mult, nil +} + +// Cleanup 删除某个 job 的所有切片,并关闭所有阻塞的 waiter goroutine +func (c *Cache) Cleanup(jobID string) { + c.mu.Lock() + pmap, exists := c.jobs[jobID] + delete(c.jobs, jobID) + c.mu.Unlock() + // 【内存泄漏修复】关闭所有 profile 下阻塞等待的 waiter channel, + // 避免 goroutine 永远挂起(每个 goroutine 至少 8KB 栈空间) + if exists { + for _, jc := range pmap { + jc.mu.Lock() + jc.final = true // 标记为 final,防止新 waiter 进入 + for seq, chs := range jc.waiters { + for _, ch := range chs { + close(ch) + } + delete(jc.waiters, seq) + } + jc.mu.Unlock() + } + } + _ = os.RemoveAll(filepath.Join(c.rootDir, jobID)) +} + +// EnforceLRU 简易 LRU 容量控制:超过 maxBytes 时按目录修改时间从早到晚删除 +func (c *Cache) EnforceLRU() error { + entries, err := os.ReadDir(c.rootDir) + if err != nil { + return err + } + type item struct { + dir string + size int64 + mtime time.Time + } + var items []item + var total int64 + for _, e := range entries { + // 只管理本模块创建的 job 目录,避免误删缓存根目录下的其它内容 + if !e.IsDir() || !strings.HasPrefix(e.Name(), "job_") { + continue + } + full := filepath.Join(c.rootDir, e.Name()) + size := dirSize(full) + fi, _ := os.Stat(full) + mt := time.Now() + if fi != nil { + mt = fi.ModTime() + } + items = append(items, item{full, size, mt}) + total += size + } + if total <= c.maxBytes { + return nil + } + sort.Slice(items, func(i, j int) bool { return items[i].mtime.Before(items[j].mtime) }) + for _, it := range items { + if total <= c.maxBytes { + break + } + _ = os.RemoveAll(it.dir) + total -= it.size + // 同步从内存里删 + jobID := filepath.Base(it.dir) + c.mu.Lock() + delete(c.jobs, jobID) + c.mu.Unlock() + } + return nil +} + +func dirSize(dir string) int64 { + var size int64 + _ = filepath.Walk(dir, func(_ string, info os.FileInfo, err error) error { + if err == nil && info != nil && !info.IsDir() { + size += info.Size() + } + return nil + }) + return size +} diff --git a/internal/transcode/cache_security_test.go b/internal/transcode/cache_security_test.go new file mode 100644 index 00000000..79db6d96 --- /dev/null +++ b/internal/transcode/cache_security_test.go @@ -0,0 +1,70 @@ +package transcode + +import ( + "bytes" + "errors" + "os" + "path/filepath" + "strings" + "testing" +) + +func TestValidProfileName(t *testing.T) { + ok := []string{"source", "1080p", "720p", "a_b-c", "A1"} + for _, p := range ok { + if !ValidProfileName(p) { + t.Errorf("%q should be valid", p) + } + } + bad := []string{"", "..", "../x", "a/b", `a\b`, "a b", "a.b", strings.Repeat("a", 33), "%2e%2e"} + for _, p := range bad { + if ValidProfileName(p) { + t.Errorf("%q should be rejected", p) + } + } +} + +func TestPutSegmentRejectsPathTraversal(t *testing.T) { + root := filepath.Join(t.TempDir(), "cache") + c := NewCache(root, 1) + if err := c.Init(); err != nil { + t.Fatal(err) + } + _, err := c.PutSegment("job_x", "../../../outside", 0, 1, bytes.NewReader([]byte("x")), false) + if !errors.Is(err, ErrInvalidProfile) { + t.Fatalf("want ErrInvalidProfile, got %v", err) + } + // 目录之外不应出现任何文件 + parent := filepath.Dir(root) + entries, _ := os.ReadDir(parent) + for _, e := range entries { + if e.Name() != "cache" { + t.Fatalf("unexpected path created outside cache root: %s", e.Name()) + } + } + if _, err := c.PutSegment("../job", "source", 0, 1, bytes.NewReader([]byte("x")), false); !errors.Is(err, ErrInvalidJobID) { + t.Fatalf("want ErrInvalidJobID, got %v", err) + } +} + +func TestPutSegmentLimits(t *testing.T) { + c := NewCache(filepath.Join(t.TempDir(), "cache"), 1) + if err := c.Init(); err != nil { + t.Fatal(err) + } + if _, err := c.PutSegment("job_x", "source", -1, 1, bytes.NewReader([]byte("x")), false); !errors.Is(err, ErrInvalidSegment) { + t.Fatalf("negative seq: want ErrInvalidSegment, got %v", err) + } + big := bytes.NewReader(make([]byte, MaxSegmentBytes+10)) + if _, err := c.PutSegment("job_x", "source", 0, 1, big, false); !errors.Is(err, ErrSegmentTooLarge) { + t.Fatalf("oversize: want ErrSegmentTooLarge, got %v", err) + } + // 超限后不应残留 tmp 文件 + matches, _ := filepath.Glob(filepath.Join(c.rootDir, "job_x", "source", "*")) + if len(matches) != 0 { + t.Fatalf("leftover files after rejected upload: %v", matches) + } + if _, err := c.PutSegment("job_x", "source", 0, 1, bytes.NewReader([]byte("ok")), false); err != nil { + t.Fatalf("normal upload failed: %v", err) + } +} diff --git a/internal/transcode/chunk_manager.go b/internal/transcode/chunk_manager.go new file mode 100644 index 00000000..cc989ea1 --- /dev/null +++ b/internal/transcode/chunk_manager.go @@ -0,0 +1,520 @@ +// Package transcode chunk_manager.go:智能分段转码调度器 +// +// 设计目标: +// 1. 把视频按时间切成多个 chunk(默认每 60 秒一段),每个 chunk 独立用 ffmpeg 转码 +// 2. 用户拖动到任意位置时,按需启动对应 chunk 的 ffmpeg,不再线性等待 +// 3. 控制单 Job 最多并发 N 个 chunk ffmpeg,超出时 LRU 淘汰最久未访问的 chunk +// 4. 当前 chunk 接近末尾时自动预启动下一个 chunk,保证顺序播放无缝衔接 +// 5. chunk 空闲超过阈值时 kill ffmpeg 释放 CPU,但保留已转出的切片文件 +// +// 与 LocalRunner 的关系: +// - LocalRunner.runJob 不再直接启动一个完整的 ffmpeg,而是创建 ChunkSession 并阻塞等待 +// - ChunkManager 是 ChunkSession 的"大脑",负责调度 +// - TCSegment 收到切片请求时,调用 ChunkManager.EnsureChunkRunningForSeg(seq) 触发按需调度 +package transcode + +import ( + "context" + "fmt" + "os" + "os/exec" + "path/filepath" + "sync" + "time" + + "github.com/OpenListTeam/OpenList/v4/internal/conf" + "github.com/OpenListTeam/OpenList/v4/internal/setting" + "github.com/OpenListTeam/OpenList/v4/pkg/utils" +) + +// ChunkState chunk 当前状态 +type ChunkState int + +const ( + ChunkIdle ChunkState = iota // 未启动 + ChunkRunning // ffmpeg 正在转码 + ChunkDone // ffmpeg 自然退出(chunk 内所有切片已转完) + ChunkFailed // ffmpeg 异常失败 + ChunkKilled // 被 LRU 淘汰或空闲超时 kill +) + +// Chunk 表示视频中的一个时间段,对应一段连续的切片序号 +type Chunk struct { + Index int // chunk 序号(0 开始) + StartTime float64 // 起始秒 + Duration float64 // 持续时长(最后一个 chunk 可能不足 chunkDur) + StartSeq int // 起始切片序号(包含) + EndSeq int // 结束切片序号(不包含) + State ChunkState + StartedAt time.Time + LastAccess time.Time + Error string + + cancel context.CancelFunc // ffmpeg 进程的 cancel + doneCh chan struct{} // ffmpeg 进程结束时关闭 + doneOnce sync.Once +} + +// chunkDoneClose 关闭 doneCh(幂等) +func (c *Chunk) chunkDoneClose() { + c.doneOnce.Do(func() { + if c.doneCh != nil { + close(c.doneCh) + } + }) +} + +// ChunkSession 单个 (jobID, profile) 下的 chunk 调度上下文 +type ChunkSession struct { + JobID string + Profile string + Job *Job + ProfDef Profile + + // 静态参数 + TotalDuration float64 // 视频总时长 + SegDur int // 每片时长(秒) + ChunkDur int // 每个 chunk 时长(秒) + MaxConcurrent int // 最大并发 chunk 数 + IdleSec int // chunk 空闲多久就 kill + Prefetch bool // 是否启用预读 + + mu sync.Mutex + chunks map[int]*Chunk // chunkIndex -> Chunk + + // 给 LocalRunner 用:所有切片转完时关闭,让 runJob 能退出 + allDoneCh chan struct{} + allDoneOnce sync.Once + + // outDir / segPattern (供 ffmpeg 使用) + OutDir string +} + +// allDoneClose 关闭 allDoneCh(幂等) +func (s *ChunkSession) allDoneClose() { + s.allDoneOnce.Do(func() { + close(s.allDoneCh) + }) +} + +// ChunkManager 全局 chunk 管理器(一个 Manager 一个) +type ChunkManager struct { + mgr *Manager + + mu sync.Mutex + sessions map[string]*ChunkSession // key = jobID + ":" + profile + + stopCh chan struct{} +} + +// NewChunkManager 创建实例 +func NewChunkManager(mgr *Manager) *ChunkManager { + return &ChunkManager{ + mgr: mgr, + sessions: make(map[string]*ChunkSession), + stopCh: make(chan struct{}), + } +} + +func sessionKey(jobID, profile string) string { return jobID + ":" + profile } + +// StartSession 创建并启动一个 chunk session(由 LocalRunner.runJob 调用) +// 返回的 ChunkSession.allDoneCh 在所有切片都转完时关闭 +func (cm *ChunkManager) StartSession(job *Job, profile Profile, outDir string) (*ChunkSession, error) { + totalDur := job.Probe.Duration + if totalDur <= 0 { + return nil, fmt.Errorf("chunk session requires probe.duration > 0") + } + segDur := setting.GetInt(conf.TranscodeSegmentDuration, 6) + if segDur <= 0 { + segDur = 6 + } + chunkDur := setting.GetInt(conf.TranscodeChunkDurationSec, 60) + if chunkDur < segDur { + chunkDur = segDur * 10 + } + // 让 chunkDur 是 segDur 的整数倍,保证切片对齐 + if chunkDur%segDur != 0 { + chunkDur = (chunkDur / segDur) * segDur + if chunkDur < segDur { + chunkDur = segDur + } + } + maxConc := setting.GetInt(conf.TranscodeMaxChunkConcurrency, 2) + if maxConc <= 0 { + maxConc = 2 + } + idleSec := setting.GetInt(conf.TranscodeChunkIdleSec, 60) + if idleSec <= 0 { + idleSec = 60 + } + prefetch := setting.GetBool(conf.TranscodeChunkPrefetch) + + s := &ChunkSession{ + JobID: job.ID, + Profile: profile.Name, + Job: job, + ProfDef: profile, + TotalDuration: totalDur, + SegDur: segDur, + ChunkDur: chunkDur, + MaxConcurrent: maxConc, + IdleSec: idleSec, + Prefetch: prefetch, + chunks: make(map[int]*Chunk), + allDoneCh: make(chan struct{}), + OutDir: outDir, + } + + // 预先按总时长生成所有 chunk 元数据(不启动) + segsPerChunk := chunkDur / segDur + totalSegs := int(totalDur) / segDur + if int(totalDur)%segDur > 0 { + totalSegs++ + } + chunkCount := totalSegs / segsPerChunk + if totalSegs%segsPerChunk > 0 { + chunkCount++ + } + for i := 0; i < chunkCount; i++ { + startSeq := i * segsPerChunk + endSeq := startSeq + segsPerChunk + if endSeq > totalSegs { + endSeq = totalSegs + } + startT := float64(startSeq * segDur) + dur := float64((endSeq - startSeq) * segDur) + // 最后一个 chunk 可能不足 + if i == chunkCount-1 { + realDur := totalDur - startT + if realDur > 0 && realDur < dur { + dur = realDur + } + } + s.chunks[i] = &Chunk{ + Index: i, + StartTime: startT, + Duration: dur, + StartSeq: startSeq, + EndSeq: endSeq, + State: ChunkIdle, + } + } + + cm.mu.Lock() + cm.sessions[sessionKey(job.ID, profile.Name)] = s + cm.mu.Unlock() + + // 立刻启动第一个 chunk(保证 playlist 首切片可用) + cm.ensureChunk(s, 0) + + // 启动后台维护协程:空闲清理 + 完成检测 + go cm.sessionLoop(s) + + utils.Log.Infof("[transcode][chunk] session started job=%s profile=%s totalDur=%.1fs chunkDur=%ds chunks=%d maxConc=%d", + job.ID, profile.Name, totalDur, chunkDur, chunkCount, maxConc) + return s, nil +} + +// EnsureChunkRunningForSeg 确保覆盖 seq 的 chunk 正在转码(由 TCSegment 触发) +// 返回所属 chunk index;如果 seq 越界返回 -1 +func (cm *ChunkManager) EnsureChunkRunningForSeg(jobID, profile string, seq int) int { + cm.mu.Lock() + s, ok := cm.sessions[sessionKey(jobID, profile)] + cm.mu.Unlock() + if !ok { + utils.Log.Warnf("[transcode][chunk] EnsureChunk: session not found job=%s profile=%s seq=%d", jobID, profile, seq) + return -1 + } + idx := cm.findChunkBySeq(s, seq) + if idx < 0 { + utils.Log.Warnf("[transcode][chunk] EnsureChunk: seq=%d out of range job=%s", seq, jobID) + return -1 + } + cm.ensureChunk(s, idx) + return idx +} + +// findChunkBySeq 按切片序号找到所属 chunk +func (cm *ChunkManager) findChunkBySeq(s *ChunkSession, seq int) int { + s.mu.Lock() + defer s.mu.Unlock() + for idx, c := range s.chunks { + if seq >= c.StartSeq && seq < c.EndSeq { + return idx + } + } + return -1 +} + +// ensureChunk 确保某个 chunk 处于 Running 状态(已运行/已完成则跳过) +// 同时受 MaxConcurrent 限制,超出时 LRU 淘汰最久未访问的 chunk +func (cm *ChunkManager) ensureChunk(s *ChunkSession, idx int) { + s.mu.Lock() + c, ok := s.chunks[idx] + if !ok { + s.mu.Unlock() + utils.Log.Warnf("[transcode][chunk] ensureChunk: chunk %d not in session (job=%s)", idx, s.JobID) + return + } + now := time.Now() + c.LastAccess = now + prevState := c.State + switch c.State { + case ChunkRunning: + s.mu.Unlock() + utils.Log.Debugf("[transcode][chunk] ensureChunk: chunk %d already running (job=%s)", idx, s.JobID) + return + case ChunkDone: + s.mu.Unlock() + utils.Log.Debugf("[transcode][chunk] ensureChunk: chunk %d already done (job=%s)", idx, s.JobID) + return + } + // 【关键修复】ChunkFailed 也允许重启:因为可能是网络抖动、source URL 临时失效等 + // 而不是永久性故障;如果连续失败,runChunk 内部不会陷入死循环(每次都会重新尝试源) + // ChunkIdle / ChunkKilled / ChunkFailed → 需要启动 + running := 0 + var lruIdx = -1 + var lruTime = now + for i, ck := range s.chunks { + if ck.State == ChunkRunning { + running++ + if i != idx && ck.LastAccess.Before(lruTime) { + lruTime = ck.LastAccess + lruIdx = i + } + } + } + if running >= s.MaxConcurrent && lruIdx >= 0 { + victim := s.chunks[lruIdx] + utils.Log.Infof("[transcode][chunk] LRU evict chunk %d (job=%s) to make room for chunk %d", + lruIdx, s.JobID, idx) + if victim.cancel != nil { + victim.cancel() + } + victim.State = ChunkKilled + } + // 启动 chunk ffmpeg + c.State = ChunkRunning + c.StartedAt = now + c.doneCh = make(chan struct{}) + c.doneOnce = sync.Once{} + s.mu.Unlock() + + utils.Log.Infof("[transcode][chunk] ensureChunk: launching chunk %d (job=%s prevState=%v running=%d)", + idx, s.JobID, prevState, running) + go cm.runChunk(s, idx) +} + +// runChunk 实际启动 ffmpeg 转码该 chunk +func (cm *ChunkManager) runChunk(s *ChunkSession, idx int) { + s.mu.Lock() + c := s.chunks[idx] + s.mu.Unlock() + if c == nil { + return + } + + timeoutMin := setting.GetInt(conf.TranscodeJobTimeoutMin, 120) + ctx, cancel := context.WithTimeout(context.Background(), time.Duration(timeoutMin)*time.Minute) + s.mu.Lock() + c.cancel = cancel + s.mu.Unlock() + defer cancel() + + segPattern := filepath.Join(s.OutDir, "seg-%d.ts") + // 每个 chunk 写自己的 playlist 文件,防止互相覆盖;后端不读这个 playlist 用于响应, + // 真正的总 playlist 由 cache.BuildPlaylist 根据总时长生成。 + chunkPlaylist := filepath.Join(s.OutDir, fmt.Sprintf("_chunk_%d.m3u8", idx)) + + args := buildFFmpegChunkArgs(s.Job, s.ProfDef, s.SegDur, c.StartTime, c.Duration, c.StartSeq, segPattern, chunkPlaylist) + ffmpegPath := setting.GetStr(conf.TranscodeFFmpegPath, "ffmpeg") + cmd := exec.CommandContext(ctx, ffmpegPath, args...) + stderr := &limitedBuffer{maxSize: 32 * 1024} + cmd.Stderr = stderr + + utils.Log.Infof("[transcode][chunk] start ffmpeg job=%s chunk=%d ss=%.1f t=%.1f startSeq=%d", + s.JobID, idx, c.StartTime, c.Duration, c.StartSeq) + + startedAt := time.Now() + err := cmd.Run() + elapsed := time.Since(startedAt) + + s.mu.Lock() + switch { + case ctx.Err() != nil && c.State == ChunkKilled: + // 被 LRU 主动 kill,状态保持 Killed + utils.Log.Infof("[transcode][chunk] chunk %d killed (job=%s elapsed=%.1fs)", idx, s.JobID, elapsed.Seconds()) + case err != nil: + c.State = ChunkFailed + c.Error = truncate(stderr.String(), 512) + utils.Log.Errorf("[transcode][chunk] chunk %d failed (job=%s elapsed=%.1fs): %v stderr=%s", + idx, s.JobID, elapsed.Seconds(), err, c.Error) + default: + c.State = ChunkDone + utils.Log.Infof("[transcode][chunk] chunk %d done (job=%s elapsed=%.1fs)", idx, s.JobID, elapsed.Seconds()) + } + c.chunkDoneClose() + s.mu.Unlock() + + // 删除临时 playlist 文件 + _ = os.Remove(chunkPlaylist) + + // 【关键修复】chunk 自然完成时,必须用 finalScan-style 扫描,否则边界切片(最大 seq) + // 永远不会被登记到 cache,导致下次播放/拖动到该位置时 60 秒超时 + if cm.mgr != nil && cm.mgr.localRunner != nil { + cm.mgr.localRunner.publishChunkSegments(s.Job, s.ProfDef, s.OutDir, c.StartSeq, c.EndSeq) + } +} + +// sessionLoop 每个 session 的后台维护:空闲清理 + 全部完成检测 + 预读 +func (cm *ChunkManager) sessionLoop(s *ChunkSession) { + t := time.NewTicker(5 * time.Second) + defer t.Stop() + for { + select { + case <-cm.stopCh: + cm.killAllChunks(s) + return + case <-s.allDoneCh: + return + case <-t.C: + cm.maintainSession(s) + } + } +} + +// maintainSession 一次维护:检查空闲、预读下一 chunk、判断是否全部完成 +func (cm *ChunkManager) maintainSession(s *ChunkSession) { + now := time.Now() + idleThreshold := time.Duration(s.IdleSec) * time.Second + + s.mu.Lock() + // 1. 空闲清理:Running 状态但长时间未访问 → kill + for _, c := range s.chunks { + if c.State == ChunkRunning && now.Sub(c.LastAccess) > idleThreshold { + utils.Log.Infof("[transcode][chunk] chunk %d idle for %.0fs, killing (job=%s)", + c.Index, now.Sub(c.LastAccess).Seconds(), s.JobID) + if c.cancel != nil { + c.cancel() + } + c.State = ChunkKilled + } + } + // 2. 预读:找出最近被访问的 Running chunk,如果它的进度过半,启动下一个 chunk + var prefetchTarget = -1 + if s.Prefetch { + var hottestIdx = -1 + var hottestAccess time.Time + for i, c := range s.chunks { + if c.State == ChunkRunning && c.LastAccess.After(hottestAccess) { + hottestAccess = c.LastAccess + hottestIdx = i + } + } + if hottestIdx >= 0 { + hot := s.chunks[hottestIdx] + // 估算进度:用 ffmpeg 已经写出的切片数 / 该 chunk 总切片数 + // 由于 cache.segments 是全局的,我们检查范围 [StartSeq, EndSeq) 中已就绪数量 + done := cm.countReadySegments(s, hot.StartSeq, hot.EndSeq) + expected := hot.EndSeq - hot.StartSeq + if expected > 0 && done*2 >= expected { // 进度 >= 50% + next := hottestIdx + 1 + if nc, ok := s.chunks[next]; ok && nc.State == ChunkIdle { + prefetchTarget = next + } + } + } + } + // 3. 全部完成检测:所有 chunk 都 Done + allDone := true + for _, c := range s.chunks { + if c.State != ChunkDone { + allDone = false + break + } + } + s.mu.Unlock() + + if prefetchTarget >= 0 { + utils.Log.Infof("[transcode][chunk] prefetch chunk %d (job=%s)", prefetchTarget, s.JobID) + cm.ensureChunk(s, prefetchTarget) + } + + if allDone { + s.allDoneClose() + } +} + +// countReadySegments 统计 [start, end) 范围内已就绪的切片数 +func (cm *ChunkManager) countReadySegments(s *ChunkSession, start, end int) int { + if cm.mgr == nil || cm.mgr.Cache == nil { + return 0 + } + cnt := 0 + for seq := start; seq < end; seq++ { + if _, err := cm.mgr.Cache.GetSegment(s.JobID, s.Profile, seq); err == nil { + cnt++ + } + } + return cnt +} + +// killAllChunks 关闭某 session 下所有运行中的 ffmpeg +func (cm *ChunkManager) killAllChunks(s *ChunkSession) { + s.mu.Lock() + defer s.mu.Unlock() + for _, c := range s.chunks { + if c.State == ChunkRunning && c.cancel != nil { + c.cancel() + c.State = ChunkKilled + } + } +} + +// CancelSession 外部取消(job 被 cancel 或 idle 超时) +func (cm *ChunkManager) CancelSession(jobID, profile string) { + cm.mu.Lock() + s, ok := cm.sessions[sessionKey(jobID, profile)] + if ok { + delete(cm.sessions, sessionKey(jobID, profile)) + } + cm.mu.Unlock() + if !ok { + return + } + cm.killAllChunks(s) + s.allDoneClose() +} + +// CancelAllForJob 取消该 job 下所有 profile 的 session(CancelJob 时调用) +func (cm *ChunkManager) CancelAllForJob(jobID string) { + cm.mu.Lock() + var targets []*ChunkSession + for k, s := range cm.sessions { + if s.JobID == jobID { + targets = append(targets, s) + delete(cm.sessions, k) + } + } + cm.mu.Unlock() + for _, s := range targets { + cm.killAllChunks(s) + s.allDoneClose() + } +} + +// WaitDone 阻塞等待所有 chunk 完成(LocalRunner.runJob 用) +func (s *ChunkSession) WaitDone() <-chan struct{} { return s.allDoneCh } + +// Stop 停止 ChunkManager(Manager.Stop 调用) +func (cm *ChunkManager) Stop() { + close(cm.stopCh) + cm.mu.Lock() + defer cm.mu.Unlock() + for _, s := range cm.sessions { + cm.killAllChunks(s) + s.allDoneClose() + } + cm.sessions = nil +} diff --git a/internal/transcode/decide.go b/internal/transcode/decide.go new file mode 100644 index 00000000..456a32f4 --- /dev/null +++ b/internal/transcode/decide.go @@ -0,0 +1,131 @@ +package transcode + +import ( + "strings" + + "github.com/OpenListTeam/OpenList/v4/internal/conf" + "github.com/OpenListTeam/OpenList/v4/internal/setting" +) + +// DecideRequest 决策器输入 +type DecideRequest struct { + FileName string + FileSize int64 // 字节 + Probe SourceProbe // 可选;为空则只用大小+后缀判断 +} + +// DecideResult 决策结果 +type DecideResult struct { + NeedTranscode bool + Reason string +} + +// Decide 根据全局设置判断是否需要走转码 +func Decide(r DecideRequest) DecideResult { + if !setting.GetBool(conf.TranscodeEnabled) { + return DecideResult{false, "transcode disabled"} + } + // 1. 大小阈值 + minGB := setting.GetInt(conf.TranscodeMinSizeGB, 5) + if minGB > 0 { + minBytes := int64(minGB) * 1024 * 1024 * 1024 + if r.FileSize > 0 && r.FileSize < minBytes { + return DecideResult{false, "size below threshold"} + } + } + // 2. 后缀过滤 + exts := splitCSV(setting.GetStr(conf.TranscodeSourceExtensions)) + if len(exts) > 0 { + ext := strings.ToLower(strings.TrimPrefix(getExt(r.FileName), ".")) + if !containsFold(exts, ext) { + return DecideResult{false, "extension not in list"} + } + } + // 3. 编码过滤(仅当 probe 提供时) + codecs := splitCSV(setting.GetStr(conf.TranscodeSourceCodecs)) + if r.Probe.VideoCodec != "" && len(codecs) > 0 { + if !containsFold(codecs, r.Probe.VideoCodec) { + return DecideResult{false, "codec not in list"} + } + } + // 4. 码率过滤(仅当 probe 提供时) + minBR := setting.GetInt(conf.TranscodeMinBitrateMbps, 0) + if minBR > 0 && r.Probe.VideoBitrate > 0 { + if r.Probe.VideoBitrate < int64(minBR)*1_000_000 { + return DecideResult{false, "bitrate below threshold"} + } + } + return DecideResult{true, "ok"} +} + +func splitCSV(s string) []string { + var out []string + for _, p := range strings.Split(s, ",") { + p = strings.TrimSpace(p) + if p != "" { + out = append(out, p) + } + } + return out +} + +func containsFold(arr []string, v string) bool { + for _, a := range arr { + if strings.EqualFold(a, v) { + return true + } + } + return false +} + +func getExt(name string) string { + i := strings.LastIndex(name, ".") + if i < 0 { + return "" + } + return name[i:] +} + +// BuildProfile 根据当前全局设置构建默认 Profile +func BuildProfile() Profile { + res := setting.GetStr(conf.TranscodeOutputResolution) + scale := "" + name := "default" + switch res { + case "source": + scale = "" + name = "source" + case "3840x2160": + scale = "3840:-2" + name = "2160p" + case "2560x1440": + scale = "2560:-2" + name = "1440p" + case "1920x1080": + scale = "1920:-2" + name = "1080p" + case "1280x720": + scale = "1280:-2" + name = "720p" + case "854x480": + scale = "854:-2" + name = "480p" + } + return Profile{ + Name: name, + VideoCodec: setting.GetStr(conf.TranscodeOutputCodec), + VideoBitrate: setting.GetStr(conf.TranscodeOutputBitrate), + Scale: scale, + AudioCodec: setting.GetStr(conf.TranscodeOutputAudioCodec), + AudioBitrate: setting.GetStr(conf.TranscodeOutputAudioBitrate), + HWAccel: setting.GetStr(conf.TranscodeHWAccel), + } +} + +// BuildOutputSpec 根据当前全局设置构建输出规格 +func BuildOutputSpec() OutputSpec { + return OutputSpec{ + Format: OutputFormat(setting.GetStr(conf.TranscodeOutputFormat)), + SegmentDuration: setting.GetInt(conf.TranscodeSegmentDuration, 6), + } +} diff --git a/internal/transcode/local_runner.go b/internal/transcode/local_runner.go new file mode 100644 index 00000000..7768280a --- /dev/null +++ b/internal/transcode/local_runner.go @@ -0,0 +1,772 @@ +package transcode + +import ( + "context" + "encoding/json" + "fmt" + "os" + "os/exec" + "path/filepath" + "runtime" + "sort" + "strconv" + "strings" + "sync" + "time" + + "github.com/OpenListTeam/OpenList/v4/internal/conf" + "github.com/OpenListTeam/OpenList/v4/internal/setting" + "github.com/OpenListTeam/OpenList/v4/pkg/utils" +) + +// limitedBuffer 是一个有容量上限的 io.Writer,超出 maxSize 后静默丢弃。 +// 用于捕获 ffmpeg stderr,防止异常文件导致无限内存增长。 +type limitedBuffer struct { + data []byte + maxSize int +} + +func (b *limitedBuffer) Write(p []byte) (int, error) { + remain := b.maxSize - len(b.data) + if remain <= 0 { + return len(p), nil // 静默丢弃 + } + if len(p) > remain { + p = p[:remain] + } + b.data = append(b.data, p...) + return len(p), nil +} + +func (b *limitedBuffer) String() string { + return string(b.data) +} + +// LocalRunner 本地内置 worker,直接调用本机 ffmpeg +type LocalRunner struct { + mgr *Manager + workerID string + + stopCh chan struct{} + wg sync.WaitGroup + + // 运行中 job 的 cancel 管理:jobID -> cancelFunc + runMu sync.Mutex + runningJobs map[string]context.CancelFunc +} + +func NewLocalRunner(mgr *Manager) *LocalRunner { + return &LocalRunner{ + mgr: mgr, + stopCh: make(chan struct{}), + runningJobs: make(map[string]context.CancelFunc), + } +} + +// Start 注册本地 worker 并启动消费协程 +func (r *LocalRunner) Start() error { + concurrency := setting.GetInt(conf.TranscodeLocalConcurrency, 1) + if concurrency <= 0 { + concurrency = 1 + } + hwaccel := setting.GetStr(conf.TranscodeHWAccel, "none") + hostname, _ := os.Hostname() + w := r.mgr.Registry.Register(&RegisterRequest{ + Name: "local-" + hostname, + Version: "embedded", + Capacity: concurrency, + HWAccel: []string{hwaccel}, + CodecsDecode: []string{"h264", "hevc", "av1", "vp9", "mpeg4", "mpeg2video"}, + CodecsEncode: []string{"h264", "hevc"}, + MaxResolution: "3840x2160", + Tags: []string{"local", "embedded"}, + }, true) + r.workerID = w.ID + utils.Log.Infof("[transcode] local worker started: id=%s capacity=%d hwaccel=%s", w.ID, concurrency, hwaccel) + + for i := 0; i < concurrency; i++ { + r.wg.Add(1) + go r.loop() + } + return nil +} + +func (r *LocalRunner) Stop() { + close(r.stopCh) + // 取消所有运行中的 FFmpeg 进程 + r.runMu.Lock() + for id, cancel := range r.runningJobs { + utils.Log.Infof("[transcode] stopping ffmpeg for job %s (runner shutdown)", id) + cancel() + } + r.runMu.Unlock() + r.wg.Wait() + r.mgr.Registry.Unregister(r.workerID) +} + +// CancelJob 外部取消指定 job 的 FFmpeg 进程(由 Scheduler.Cancel / 空闲超时调用) +func (r *LocalRunner) CancelJob(jobID string) bool { + r.runMu.Lock() + cancel, ok := r.runningJobs[jobID] + r.runMu.Unlock() + if ok { + utils.Log.Infof("[transcode] killing ffmpeg for job %s (external cancel)", jobID) + cancel() + } + return ok +} + +// RunningJobIDs 返回当前正在运行的所有 job ID 列表 +func (r *LocalRunner) RunningJobIDs() []string { + r.runMu.Lock() + defer r.runMu.Unlock() + ids := make([]string, 0, len(r.runningJobs)) + for id := range r.runningJobs { + ids = append(ids, id) + } + return ids +} + +func (r *LocalRunner) loop() { + defer r.wg.Done() + for { + select { + case <-r.stopCh: + return + default: + } + jobs := r.mgr.Scheduler.Claim(r.workerID, 1, 5*time.Second) + if len(jobs) == 0 { + continue + } + for _, j := range jobs { + r.runJob(j) + } + } +} + +// runJob 执行一个转码任务(单 profile,HLS 输出,边转边写入 cache) +func (r *LocalRunner) runJob(job *Job) { + utils.Log.Infof("[transcode] local runner picked job %s (%s)", job.ID, job.Path) + if len(job.Profiles) == 0 { + _ = r.mgr.Scheduler.Finish(&FinishRequest{JobID: job.ID, Status: JobFailed, Error: "no profiles"}) + return + } + profile := job.Profiles[0] + timeoutMin := setting.GetInt(conf.TranscodeJobTimeoutMin, 120) + ctx, cancel := context.WithTimeout(context.Background(), time.Duration(timeoutMin)*time.Minute) + defer cancel() + + // 注册到 runningJobs,支持外部取消 + r.runMu.Lock() + r.runningJobs[job.ID] = cancel + r.runMu.Unlock() + defer func() { + r.runMu.Lock() + delete(r.runningJobs, job.ID) + r.runMu.Unlock() + }() + + // 输出目录 + jc, err := r.mgr.Cache.getOrCreate(job.ID, profile.Name) + if err != nil { + _ = r.mgr.Scheduler.Finish(&FinishRequest{JobID: job.ID, Status: JobFailed, Error: err.Error()}) + return + } + outDir := jc.BaseDir + + // 【智能调度】如果探测到了视频总时长,启用 chunk 模式(按需分段并发转码) + // 否则回退到旧的"一次性整段"模式(兼容 ffprobe 失败的场景) + if job.Probe.Duration > 0 && r.mgr.Chunks != nil { + r.runJobByChunks(ctx, job, profile, outDir) + return + } + + // === 旧路径:一次性整段转码 === + r.runJobLegacy(ctx, job, profile, outDir) +} + +// runJobByChunks 智能 chunk 调度模式:按需启动 ffmpeg 进程,支持随机访问 +func (r *LocalRunner) runJobByChunks(ctx context.Context, job *Job, profile Profile, outDir string) { + startedAt := time.Now() + + // 启动 watchSegments:周期扫描目录把切片登记到 cache + stopWatcher := make(chan struct{}) + var watcherWG sync.WaitGroup + watcherWG.Add(1) + go func() { + defer watcherWG.Done() + r.watchSegments(job, profile, outDir, stopWatcher) + }() + + // 创建 chunk session(会立刻启动 chunk-0 + 后台维护协程) + session, err := r.mgr.Chunks.StartSession(job, profile, outDir) + if err != nil { + close(stopWatcher) + watcherWG.Wait() + _ = r.mgr.Scheduler.Finish(&FinishRequest{JobID: job.ID, Status: JobFailed, Error: err.Error()}) + return + } + + // 阻塞等待全部 chunk 完成 / 任务被取消 + select { + case <-session.WaitDone(): + // 所有 chunk 都 Done,正常完成 + case <-ctx.Done(): + // 外部取消:通知 chunk session 杀掉所有 ffmpeg + r.mgr.Chunks.CancelSession(job.ID, profile.Name) + } + + close(stopWatcher) + watcherWG.Wait() + r.scanAndPublish(job, profile, outDir, true) + + if ctx.Err() != nil { + utils.Log.Infof("[transcode] job %s chunk session cancelled", job.ID) + _ = r.mgr.Scheduler.Finish(&FinishRequest{ + JobID: job.ID, + Status: JobCancelled, + Error: "cancelled: " + ctx.Err().Error(), + }) + return + } + utils.Log.Infof("[transcode] job %s completed in %.1fs (chunk mode)", job.ID, time.Since(startedAt).Seconds()) + _ = r.mgr.Scheduler.Finish(&FinishRequest{ + JobID: job.ID, + Status: JobFinished, + Stats: JobStats{Elapsed: time.Since(startedAt).Seconds()}, + }) +} + +// runJobLegacy 旧路径:一次性整段转码(不知道总时长时回退使用) +func (r *LocalRunner) runJobLegacy(ctx context.Context, job *Job, profile Profile, outDir string) { + playlistPath := filepath.Join(outDir, "playlist.m3u8") + segPattern := filepath.Join(outDir, "seg-%d.ts") + + args := buildFFmpegArgs(job, profile, segPattern, playlistPath) + ffmpegPath := setting.GetStr(conf.TranscodeFFmpegPath, "ffmpeg") + + cmd := exec.CommandContext(ctx, ffmpegPath, args...) + stderr := &limitedBuffer{maxSize: 64 * 1024} + cmd.Stderr = stderr + + startedAt := time.Now() + if err := cmd.Start(); err != nil { + _ = r.mgr.Scheduler.Finish(&FinishRequest{JobID: job.ID, Status: JobFailed, Error: err.Error()}) + return + } + + stopWatcher := make(chan struct{}) + var watcherWG sync.WaitGroup + watcherWG.Add(1) + go func() { + defer watcherWG.Done() + r.watchSegments(job, profile, outDir, stopWatcher) + }() + + err := cmd.Wait() + close(stopWatcher) + watcherWG.Wait() + r.scanAndPublish(job, profile, outDir, true) + + if err != nil { + if ctx.Err() != nil { + utils.Log.Infof("[transcode] job %s ffmpeg killed (context cancelled)", job.ID) + _ = r.mgr.Scheduler.Finish(&FinishRequest{ + JobID: job.ID, + Status: JobCancelled, + Error: "cancelled: " + ctx.Err().Error(), + }) + } else { + _ = r.mgr.Scheduler.Finish(&FinishRequest{ + JobID: job.ID, + Status: JobFailed, + Error: fmt.Sprintf("ffmpeg failed: %v, stderr=%s", err, truncate(stderr.String(), 1024)), + }) + } + return + } + utils.Log.Infof("[transcode] job %s completed in %.1fs (legacy mode)", job.ID, time.Since(startedAt).Seconds()) + _ = r.mgr.Scheduler.Finish(&FinishRequest{ + JobID: job.ID, + Status: JobFinished, + Stats: JobStats{Elapsed: time.Since(startedAt).Seconds()}, + }) +} + +// watchSegments 周期性扫描输出目录,把新切片登记到 Cache +// 【内存优化】把扫描频率从 500ms 降到 2s,减少每次 os.ReadDir 和 parseFFmpegPlaylistDurations +// 触发的字符串/字节切片分配。chunk 模式下每个 chunk 完成时还会主动调用 publishChunkSegments, +// 所以这里只需要兜底登记中间切片,2s 频率完全够用。 +func (r *LocalRunner) watchSegments(job *Job, profile Profile, dir string, stop chan struct{}) { + t := time.NewTicker(2 * time.Second) + defer t.Stop() + for { + select { + case <-stop: + return + case <-t.C: + r.scanAndPublish(job, profile, dir, false) + } + } +} + +func (r *LocalRunner) scanAndPublish(job *Job, profile Profile, dir string, finalScan bool) { + jc, err := r.mgr.Cache.getOrCreate(job.ID, profile.Name) + if err != nil { + return + } + entries, err := os.ReadDir(dir) + if err != nil { + return + } + type segFile struct { + seq int + path string + size int64 + } + var segs []segFile + for _, e := range entries { + if e.IsDir() { + continue + } + name := e.Name() + if !strings.HasPrefix(name, "seg-") || !strings.HasSuffix(name, ".ts") { + continue + } + numStr := strings.TrimSuffix(strings.TrimPrefix(name, "seg-"), ".ts") + seq, err := strconv.Atoi(numStr) + if err != nil { + continue + } + fi, err := e.Info() + if err != nil { + continue + } + segs = append(segs, segFile{seq: seq, path: filepath.Join(dir, name), size: fi.Size()}) + } + sort.Slice(segs, func(i, j int) bool { return segs[i].seq < segs[j].seq }) + + // 【优化】解析 ffmpeg 写出的 playlist.m3u8 获取每个切片的真实时长 + // 避免硬编码 segDur 导致 HLS.js 显示总时长不准确 + realDurations := parseFFmpegPlaylistDurations(filepath.Join(dir, "playlist.m3u8")) + + // ffmpeg 在写入 .ts 时是边写边写,稳妥起见仅登记除最后一片以外的切片(最后一片可能未写完) + limit := len(segs) + if !finalScan && limit > 0 { + limit-- + } + defaultDur := float64(setting.GetInt(conf.TranscodeSegmentDuration, 6)) + for i := 0; i < limit; i++ { + s := segs[i] + jc.mu.RLock() + _, exists := jc.segments[s.seq] + jc.mu.RUnlock() + if exists { + continue + } + // 优先用 ffmpeg playlist 中的真实时长 + dur := defaultDur + if d, ok := realDurations[s.seq]; ok && d > 0 { + dur = d + } + // 直接登记,不复制(已经在 cache 目录里了) + jc.mu.Lock() + jc.segments[s.seq] = &SegmentInfo{Seq: s.seq, Duration: dur, Size: s.size, Path: s.path} + var toClose []chan struct{} + for sk, chs := range jc.waiters { + if sk <= s.seq { + toClose = append(toClose, chs...) + delete(jc.waiters, sk) + } + } + jc.mu.Unlock() + for _, ch := range toClose { + close(ch) + } + // 第一片就绪 → 通知 job ready + if s.seq == 0 { + job.MarkReady() + } + } + if finalScan { + jc.mu.Lock() + jc.final = true + jc.mu.Unlock() + } +} + +// publishChunkSegments 把 [startSeq, endSeq) 范围内已经写完的切片登记到 cache。 +// 与 scanAndPublish 不同,这个方法专门用于"某个 chunk 的 ffmpeg 已经退出"的场景, +// 此时该范围的所有 .ts 文件都是完整的,不能再丢掉最大序号的那片(避免边界切片永远丢失)。 +func (r *LocalRunner) publishChunkSegments(job *Job, profile Profile, dir string, startSeq, endSeq int) { + jc, err := r.mgr.Cache.getOrCreate(job.ID, profile.Name) + if err != nil { + return + } + defaultDur := float64(setting.GetInt(conf.TranscodeSegmentDuration, 6)) + registered := 0 + for seq := startSeq; seq < endSeq; seq++ { + jc.mu.RLock() + _, exists := jc.segments[seq] + jc.mu.RUnlock() + if exists { + continue + } + fpath := filepath.Join(dir, fmt.Sprintf("seg-%d.ts", seq)) + fi, err := os.Stat(fpath) + if err != nil || fi.IsDir() || fi.Size() == 0 { + continue + } + jc.mu.Lock() + jc.segments[seq] = &SegmentInfo{Seq: seq, Duration: defaultDur, Size: fi.Size(), Path: fpath} + var toClose []chan struct{} + for sk, chs := range jc.waiters { + if sk <= seq { + toClose = append(toClose, chs...) + delete(jc.waiters, sk) + } + } + jc.mu.Unlock() + for _, ch := range toClose { + close(ch) + } + if seq == 0 { + job.MarkReady() + } + registered++ + } + if registered > 0 { + utils.Log.Infof("[transcode][chunk] published %d segments [%d, %d) for job=%s", + registered, startSeq, endSeq, job.ID) + } +} + +// playlistDurCache 缓存 m3u8 解析结果,按文件 mtime+size 失效。 +// 【内存优化】scanAndPublish 每 2 秒一次,每次都会解析同一个 m3u8(长视频几百 KB), +// strings.Split 会产生大量临时字符串切片造成 GC 压力。用缓存减少 99% 的重复解析。 +type playlistDurCacheEntry struct { + mtime time.Time + size int64 + durs map[int]float64 +} + +var ( + playlistDurCache = make(map[string]*playlistDurCacheEntry) + playlistDurCacheMu sync.Mutex +) + +// parseFFmpegPlaylistDurations 解析 ffmpeg 写出的 m3u8,返回 seq -> duration 的映射 +// 用于获取每个切片的真实时长,避免后端用硬编码 segDur 导致总时长显示偏差 +func parseFFmpegPlaylistDurations(path string) map[int]float64 { + fi, err := os.Stat(path) + if err != nil { + return map[int]float64{} + } + // 命中缓存:mtime + size 都没变就直接复用 + playlistDurCacheMu.Lock() + if e, ok := playlistDurCache[path]; ok && e.mtime.Equal(fi.ModTime()) && e.size == fi.Size() { + playlistDurCacheMu.Unlock() + return e.durs + } + playlistDurCacheMu.Unlock() + + out := make(map[int]float64) + data, err := os.ReadFile(path) + if err != nil { + return out + } + lines := strings.Split(string(data), "\n") + var pendingDur float64 + for _, line := range lines { + line = strings.TrimSpace(line) + if strings.HasPrefix(line, "#EXTINF:") { + // #EXTINF:6.000000, + body := strings.TrimPrefix(line, "#EXTINF:") + body = strings.TrimSuffix(body, ",") + d, err := strconv.ParseFloat(strings.TrimSpace(body), 64) + if err == nil { + pendingDur = d + } + continue + } + if strings.HasSuffix(line, ".ts") && strings.HasPrefix(line, "seg-") { + numStr := strings.TrimSuffix(strings.TrimPrefix(line, "seg-"), ".ts") + if seq, err := strconv.Atoi(numStr); err == nil && pendingDur > 0 { + out[seq] = pendingDur + } + pendingDur = 0 + } + } + // 写入缓存 + playlistDurCacheMu.Lock() + playlistDurCache[path] = &playlistDurCacheEntry{ + mtime: fi.ModTime(), + size: fi.Size(), + durs: out, + } + playlistDurCacheMu.Unlock() + return out +} + +// buildFFmpegArgs 构造 ffmpeg 命令行(一次性整段转码,兼容旧路径) +// 当 Probe.Duration 不可用时,LocalRunner.runJob 会回退到这条路径。 +func buildFFmpegArgs(job *Job, profile Profile, segPattern, playlistPath string) []string { + segDur := setting.GetInt(conf.TranscodeSegmentDuration, 6) + // startTime=0, duration=0(=不限) 表示整段转码;startSeq=0 + return buildFFmpegArgsCommon(job, profile, segDur, 0, 0, 0, segPattern, playlistPath) +} + +// buildFFmpegChunkArgs 构造 chunk 模式的 ffmpeg 命令行 +// startTime: 起始时间(秒);duration: 持续时长(秒,0=不限);startSeq: 切片起始编号 +// chunk 模式与一次性整段共用底层参数构造,差异仅在 -ss/-t/-start_number。 +func buildFFmpegChunkArgs(job *Job, profile Profile, segDur int, startTime, duration float64, startSeq int, segPattern, playlistPath string) []string { + return buildFFmpegArgsCommon(job, profile, segDur, startTime, duration, startSeq, segPattern, playlistPath) +} + +// buildFFmpegArgsCommon 通用 ffmpeg 命令行构造 +// 当 startTime > 0 或 duration > 0 时,启用"chunk 模式":使用 -ss / -t / -start_number 限定范围 +// 关键:-ss 必须放在 -i 之前(input seek,快),且需配合 force_key_frames 让 chunk 起始处生成 IDR 帧 +func buildFFmpegArgsCommon(job *Job, profile Profile, segDur int, startTime, duration float64, startSeq int, segPattern, playlistPath string) []string { + hw := strings.ToLower(profile.HWAccel) + args := []string{"-y", "-loglevel", "warning"} + + // 输入侧硬件加速 + switch hw { + case "nvenc": + args = append(args, "-hwaccel", "cuda") + case "qsv": + args = append(args, "-hwaccel", "qsv") + case "vaapi": + args = append(args, "-hwaccel", "vaapi", "-hwaccel_output_format", "vaapi", "-vaapi_device", "/dev/dri/renderD128") + case "amf": + args = append(args, "-hwaccel", "d3d11va") + case "videotoolbox": + args = append(args, "-hwaccel", "videotoolbox") + case "auto": + args = append(args, "-hwaccel", "auto") + } + + // 【chunk 模式】input seek:放在 -i 之前,速度快,会按关键帧对齐 + // 注意 -ss 在 -i 之前是 input seek(用容器索引快速跳转),但落点是关键帧; + // 我们通过 force_key_frames 保证每个 chunk 边界都有关键帧,所以这里精度足够。 + if startTime > 0.001 { + args = append(args, "-ss", fmt.Sprintf("%.3f", startTime)) + } + + // 输入:先限定协议白名单(输入选项,必须在 -i 之前),再指定输入 + args = append(args, "-protocol_whitelist", ProtocolWhitelistFor(job.SourceURL)) + args = append(args, "-i", job.SourceURL) + + // 【chunk 模式】限定时长(output side 更精确) + if duration > 0.001 { + args = append(args, "-t", fmt.Sprintf("%.3f", duration)) + } + + // 视频编码器 + vcodec := pickEncoder(profile.VideoCodec, hw) + args = append(args, "-c:v", vcodec) + // 【关键】强制浏览器兼容的 profile/level/pix_fmt,必须与 master.m3u8 声明的 + // CODECS="avc1.640028" (H.264 High Profile Level 4.0) 一致,否则 HLS.js 会触发 + // bufferAppendError 死循环。 + if strings.Contains(vcodec, "264") { + args = append(args, "-profile:v", "high", "-level:v", "4.0", "-pix_fmt", "yuv420p") + } + if profile.VideoBitrate != "" { + args = append(args, "-b:v", profile.VideoBitrate) + args = append(args, "-maxrate", profile.VideoBitrate, "-bufsize", profile.VideoBitrate) + } + if profile.Scale != "" { + // 软件 / 硬件缩放 + switch hw { + case "vaapi": + args = append(args, "-vf", "scale_vaapi="+profile.Scale) + case "qsv": + args = append(args, "-vf", "scale_qsv="+profile.Scale) + default: + args = append(args, "-vf", "scale="+profile.Scale) + } + } + args = append(args, "-preset", defaultPresetFor(vcodec)) + + // 音频 + // 【关键修复】不能使用 -c:a copy,因为浏览器 MSE 不支持 AC3/DTS/EAC3/TrueHD 等 + // 常见于 mkv 的音频编码。直接 copy 会导致 SourceBuffer.appendBuffer 失败 + // 触发 bufferAppendError 死循环。统一强制转 AAC 兼容所有浏览器。 + ac := profile.AudioCodec + if ac == "" || ac == "copy" { + ac = "aac" + } + args = append(args, "-c:a", ac) + ab := profile.AudioBitrate + if ab == "" { + ab = "160k" + } + args = append(args, "-b:a", ab) + // 强制立体声 + 48kHz,避免 5.1 声道 / 高采样率导致的兼容问题 + args = append(args, "-ac", "2", "-ar", "48000") + + // HLS 输出 + if segDur <= 0 { + segDur = setting.GetInt(conf.TranscodeSegmentDuration, 6) + } + // 【关键】强制关键帧对齐切片边界,确保每个切片以 IDR 帧开始可独立解码。 + // 没有这些参数时,第一个切片可能不含 IDR 帧,导致 HLS.js bufferAppendError。 + // fps 假设 25 (帧率未知时的保守值),gop = segDur * fps;force_key_frames 让 + // ffmpeg 在每个切片边界强制插入关键帧。 + gop := segDur * 25 + args = append(args, + "-g", strconv.Itoa(gop), + "-keyint_min", strconv.Itoa(gop), + "-sc_threshold", "0", + "-force_key_frames", fmt.Sprintf("expr:gte(t,n_forced*%d)", segDur), + ) + // 【chunk 模式 - 关键修复】input seek 后 ffmpeg 会把第一帧 PTS 重置为 0, + // 这会导致 chunk-30 的切片在播放器看来是 PTS=0~60s(而不是 1800~1860s), + // 进而触发 HLS.js bufferAppendError 或时间轴错位。 + // 解决方法:用 -output_ts_offset 让生成的切片时间戳偏移到原始位置。 + if startTime > 0.001 { + args = append(args, "-output_ts_offset", fmt.Sprintf("%.3f", startTime)) + } + args = append(args, + "-f", "hls", + "-hls_time", strconv.Itoa(segDur), + "-hls_playlist_type", "vod", + "-hls_segment_filename", segPattern, + "-hls_flags", "independent_segments+temp_file", + ) + // 【chunk 模式】指定起始切片编号,让全局切片编号连续 + if startSeq > 0 { + args = append(args, "-start_number", strconv.Itoa(startSeq)) + } + args = append(args, playlistPath) + return args +} + +// pickEncoder 根据视频编码 + HW 类型选择 ffmpeg 编码器 +func pickEncoder(codec, hw string) string { + codec = strings.ToLower(codec) + switch codec { + case "h264", "": + switch hw { + case "nvenc": + return "h264_nvenc" + case "qsv": + return "h264_qsv" + case "vaapi": + return "h264_vaapi" + case "amf": + return "h264_amf" + case "videotoolbox": + return "h264_videotoolbox" + default: + return "libx264" + } + case "hevc", "h265": + switch hw { + case "nvenc": + return "hevc_nvenc" + case "qsv": + return "hevc_qsv" + case "vaapi": + return "hevc_vaapi" + case "amf": + return "hevc_amf" + case "videotoolbox": + return "hevc_videotoolbox" + default: + return "libx265" + } + case "av1": + switch hw { + case "nvenc": + return "av1_nvenc" + case "qsv": + return "av1_qsv" + default: + return "libsvtav1" + } + } + return "libx264" +} + +func defaultPresetFor(encoder string) string { + if strings.Contains(encoder, "nvenc") { + return "p5" + } + if strings.Contains(encoder, "qsv") { + return "medium" + } + if strings.Contains(encoder, "amf") { + return "balanced" + } + if strings.Contains(encoder, "videotoolbox") { + return "medium" + } + return "veryfast" +} + +func truncate(s string, n int) string { + if len(s) <= n { + return s + } + return s[:n] + "...(truncated)" +} + +// ProbeSource 调用 ffprobe 探测源文件 +func ProbeSource(url string) (SourceProbe, error) { + if !ValidSourceInput(url) { + return SourceProbe{}, fmt.Errorf("unsupported probe source") + } + probePath := setting.GetStr(conf.TranscodeFFprobePath, "ffprobe") + args := []string{ + "-v", "error", + "-protocol_whitelist", ProtocolWhitelistFor(url), + "-show_entries", "format=duration,size,bit_rate:stream=codec_type,codec_name,bit_rate,width,height", + "-of", "json", + url, + } + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + cmd := exec.CommandContext(ctx, probePath, args...) + if runtime.GOOS == "windows" { + // no special handling + } + out, err := cmd.Output() + if err != nil { + return SourceProbe{}, err + } + var raw struct { + Format struct { + Duration string `json:"duration"` + Size string `json:"size"` + BitRate string `json:"bit_rate"` + } `json:"format"` + Streams []struct { + CodecType string `json:"codec_type"` + CodecName string `json:"codec_name"` + BitRate string `json:"bit_rate"` + Width int `json:"width"` + Height int `json:"height"` + } `json:"streams"` + } + if err := json.Unmarshal(out, &raw); err != nil { + return SourceProbe{}, err + } + p := SourceProbe{} + p.Duration, _ = strconv.ParseFloat(raw.Format.Duration, 64) + p.Size, _ = strconv.ParseInt(raw.Format.Size, 10, 64) + for _, s := range raw.Streams { + if s.CodecType == "video" && p.VideoCodec == "" { + p.VideoCodec = s.CodecName + p.Width = s.Width + p.Height = s.Height + p.VideoBitrate, _ = strconv.ParseInt(s.BitRate, 10, 64) + } + if s.CodecType == "audio" && p.AudioCodec == "" { + p.AudioCodec = s.CodecName + p.AudioBitrate, _ = strconv.ParseInt(s.BitRate, 10, 64) + } + } + return p, nil +} diff --git a/internal/transcode/manager.go b/internal/transcode/manager.go new file mode 100644 index 00000000..9436c825 --- /dev/null +++ b/internal/transcode/manager.go @@ -0,0 +1,177 @@ +package transcode + +import ( + "sync" + "time" + + "github.com/OpenListTeam/OpenList/v4/internal/conf" + "github.com/OpenListTeam/OpenList/v4/internal/setting" + "github.com/OpenListTeam/OpenList/v4/pkg/utils" +) + +// Manager 全局转码管理器单例 +type Manager struct { + Registry *WorkerRegistry + Scheduler *Scheduler + Cache *Cache + Chunks *ChunkManager // 智能 chunk 调度器(按需启动 ffmpeg,支持随机访问) + + localRunner *LocalRunner + + stopCh chan struct{} + once sync.Once +} + +var ( + defaultManager *Manager + managerOnce sync.Once +) + +// Default 返回全局 Manager(懒加载) +func Default() *Manager { + managerOnce.Do(func() { + defaultManager = newManager() + }) + return defaultManager +} + +func newManager() *Manager { + reg := NewWorkerRegistry() + sch := NewScheduler(reg) + cachePath := setting.GetStr(conf.TranscodeCachePath, "data/ctmp") + // 兼容旧默认值:统一迁移到 data/ctmp + if cachePath == "data/transcode_cache" || cachePath == "./data/transcode_cache" { + cachePath = "data/ctmp" + } + cacheMaxGB := int64(setting.GetInt(conf.TranscodeCacheMaxGB, 20)) + cache := NewCache(cachePath, cacheMaxGB) + _ = cache.Init() + m := &Manager{ + Registry: reg, + Scheduler: sch, + Cache: cache, + stopCh: make(chan struct{}), + } + m.Chunks = NewChunkManager(m) + return m +} + +// Start 启动后台维护协程:心跳清理 + 缓存 LRU + 完成任务清理 +func (m *Manager) Start() { + m.once.Do(func() { + go m.maintenance() + // 根据 run_mode 决定是否启动本地 worker + mode := setting.GetStr(conf.TranscodeRunMode, "local") + if mode == "local" || mode == "hybrid" { + m.startLocalWorker() + } + }) +} + +// Stop 停止 Manager +func (m *Manager) Stop() { + close(m.stopCh) + if m.Chunks != nil { + m.Chunks.Stop() + } + if m.localRunner != nil { + m.localRunner.Stop() + } +} + +func (m *Manager) maintenance() { + t1 := time.NewTicker(15 * time.Second) // 心跳清理 + t2 := time.NewTicker(5 * time.Minute) // 缓存 LRU + t3 := time.NewTicker(30 * time.Minute) // 历史 job 清理 + t4 := time.NewTicker(10 * time.Second) // 空闲超时检测 + defer t1.Stop() + defer t2.Stop() + defer t3.Stop() + defer t4.Stop() + for { + select { + case <-m.stopCh: + return + case <-t1.C: + removed := m.Registry.Sweep() + for _, id := range removed { + utils.Log.Infof("[transcode] worker %s removed (heartbeat timeout)", id) + } + case <-t2.C: + _ = m.Cache.EnforceLRU() + case <-t3.C: + purged := m.Scheduler.PurgeBefore(time.Now().Add(-2 * time.Hour)) + if len(purged) > 0 { + // 【内存泄漏修复】联动清理 Cache 中对应的 JobCache + segments map + for _, jobID := range purged { + m.Cache.Cleanup(jobID) + } + utils.Log.Infof("[transcode] purged %d finished jobs (cache cleaned)", len(purged)) + } + case <-t4.C: + m.sweepIdleJobs() + } + } +} + +// sweepIdleJobs 扫描所有活跃任务,超过空闲超时时间无播放端请求则自动取消并 kill FFmpeg +// 注意:这里只 cancel job + kill ffmpeg 释放 CPU,**不删除已转好的切片缓存**。 +// 当用户继续播放时,TCSegment / TCPlaylist 会通过 Scheduler.Reactivate 复用同一 jobID/token +// 重新启动转码;已存在的切片可以直接命中,不需要再次转码。 +func (m *Manager) sweepIdleJobs() { + idleTimeoutSec := setting.GetInt(conf.TranscodeIdleTimeoutSec, 90) + if idleTimeoutSec <= 0 { + return // 配置为 0 表示禁用空闲超时 + } + timeout := time.Duration(idleTimeoutSec) * time.Second + now := time.Now() + + activeJobs := m.Scheduler.ListActiveJobs() + for _, j := range activeJobs { + lastAccess := j.GetLastAccess() + idle := now.Sub(lastAccess) + if idle > timeout { + utils.Log.Infof("[transcode] job %s idle for %.0fs (threshold %ds), pausing ffmpeg (cache kept for resume)", + j.ID, idle.Seconds(), idleTimeoutSec) + // 先取消所有 chunk ffmpeg,再调度器层面取消 + if m.Chunks != nil { + m.Chunks.CancelAllForJob(j.ID) + } + m.Scheduler.Cancel(j.ID) + // 【重要】不再调用 m.Cache.Cleanup —— 已转好的切片必须保留, + // 这样浏览器恢复播放时旧的 seg-N.ts 仍然能命中,避免 404 和重新转码。 + // 真正的磁盘清理由更长周期的 PurgeBefore + Cache.EnforceLRU 负责。 + } + } +} + +func (m *Manager) startLocalWorker() { + r := NewLocalRunner(m) + if err := r.Start(); err != nil { + utils.Log.Errorf("[transcode] start local worker failed: %+v", err) + return + } + m.localRunner = r + // 注入取消回调:Scheduler.Cancel 时直接 kill 本地 FFmpeg 进程 + m.Scheduler.SetOnCancel(func(jobID string) { + // 先清理 chunk session(kill 所有 chunk ffmpeg) + if m.Chunks != nil { + m.Chunks.CancelAllForJob(jobID) + } + if m.localRunner != nil { + m.localRunner.CancelJob(jobID) + } + }) +} + +// IsEnabled 总开关 +func IsEnabled() bool { return setting.GetBool(conf.TranscodeEnabled) } + +// VerifyWorkerSecret 校验远程 worker 注册凭据 +func VerifyWorkerSecret(token string) bool { + expected := setting.GetStr(conf.TranscodeWorkerSecret) + if expected == "" { + return false // 未配置则禁用远程注册 + } + return token == expected +} diff --git a/internal/transcode/protocol.go b/internal/transcode/protocol.go new file mode 100644 index 00000000..7c771882 --- /dev/null +++ b/internal/transcode/protocol.go @@ -0,0 +1,75 @@ +package transcode + +// 协议版本 +const ProtocolVersion = "v1" + +// ---- 注册 ---- + +type RegisterRequest struct { + Name string `json:"name"` + Version string `json:"version"` + Capacity int `json:"capacity"` + HWAccel []string `json:"hwaccel"` + CodecsDecode []string `json:"codecs_decode"` + CodecsEncode []string `json:"codecs_encode"` + MaxResolution string `json:"max_resolution"` + Tags []string `json:"tags"` + Endpoint string `json:"endpoint,omitempty"` +} + +type RegisterResponse struct { + WorkerID string `json:"worker_id"` + HeartbeatInterval int `json:"heartbeat_interval"` + ClaimStrategy string `json:"claim_strategy"` // pull + ProtocolVersion string `json:"protocol_version"` +} + +// ---- 心跳 ---- + +type HeartbeatRequest struct { + WorkerID string `json:"worker_id"` + Load float64 `json:"load"` + Running []string `json:"running"` + FreeSlots int `json:"free_slots"` +} + +type HeartbeatResponse struct { + OK bool `json:"ok"` + Kick bool `json:"kick"` // 服务端要求 worker 退出 + Cancel []string `json:"cancel,omitempty"` // 需要取消的 job ids +} + +// ---- 领任务 ---- + +type ClaimRequest struct { + WorkerID string `json:"worker_id"` + Slots int `json:"slots"` + Wait int `json:"wait"` // 长轮询超时秒,0=不等 +} + +type ClaimResponse struct { + Jobs []*Job `json:"jobs"` +} + +// ---- 完成 ---- + +type FinishRequest struct { + JobID string `json:"job_id"` + Status JobStatus `json:"status"` + Error string `json:"error,omitempty"` + Stats JobStats `json:"stats"` +} + +type JobStats struct { + Elapsed float64 `json:"elapsed"` + FPS float64 `json:"fps"` + Speed float64 `json:"speed"` + BytesOut int64 `json:"bytes_out"` +} + +// ---- 通用响应 ---- + +type ErrorResponse struct { + Code int `json:"code"` + Message string `json:"message"` +} diff --git a/internal/transcode/protocol_guard.go b/internal/transcode/protocol_guard.go new file mode 100644 index 00000000..339c9d4d --- /dev/null +++ b/internal/transcode/protocol_guard.go @@ -0,0 +1,56 @@ +package transcode + +import "strings" + +// ffmpeg / ffprobe 支持 concat:、subfile:、data: 等大量协议。 +// SourceURL 最终来自请求中的路径或 Host 头,若不限制协议,攻击者就可能借 ffmpeg +// 读取服务器本地文件或访问内网地址。这里按输入类型收敛到最小协议集。 + +const ( + // protocolWhitelistLocal 本地文件输入只允许 file 协议 + protocolWhitelistLocal = "file" + // protocolWhitelistRemote 远程输入允许 HTTP(S) 及其底层传输;禁止 concat/subfile/data/pipe, + // 同时去掉 file 与 crypto:HTTP 源用不到它们,留着等于给被构造的播放列表留了读本地文件的通道。 + protocolWhitelistRemote = "http,https,tcp,tls" +) + +// ProtocolWhitelistFor 根据输入地址返回应传给 ffmpeg/ffprobe 的 -protocol_whitelist 值。 +func ProtocolWhitelistFor(source string) string { + s := strings.ToLower(strings.TrimSpace(source)) + if strings.HasPrefix(s, "http://") || strings.HasPrefix(s, "https://") { + return protocolWhitelistRemote + } + return protocolWhitelistLocal +} + +// ValidSourceInput 判断输入是否可以安全交给 ffmpeg: +// 只接受绝对本地路径或 http(s) URL,拒绝 concat:/subfile:/data:/pipe: 等协议前缀, +// 也拒绝以 - 开头(会被当作 ffmpeg 选项)的值。 +func ValidSourceInput(source string) bool { + s := strings.TrimSpace(source) + if s == "" || strings.HasPrefix(s, "-") { + return false + } + low := strings.ToLower(s) + if strings.HasPrefix(low, "http://") || strings.HasPrefix(low, "https://") { + return true + } + // 本地输入只接受绝对路径。 + // 不能按「冒号位置」判 scheme:文件名里带冒号非常常见 + // (/mnt/media/Star Wars: Episode IV.mkv),那样会被误判成协议前缀。 + // concat:/subfile:/data: 这些前缀都不是绝对路径,且还会被 -protocol_whitelist 挡住。 + // 注意不能用 filepath.IsAbs:它是平台相关的,Windows 上会拒绝 "/data/a.mkv" + // 这类 VFS 绝对路径(OpenList 的路径始终是正斜杠开头)。 + return strings.HasPrefix(s, "/") || isWindowsDrivePath(s) +} + +// isWindowsDrivePath 判断 C:\ 或 C:/ 形式的盘符绝对路径。 +// 跨平台使用:在 Linux 上构建时会话也可能处理 Windows 风格路径。 +func isWindowsDrivePath(s string) bool { + if len(s) < 3 || s[1] != ':' { + return false + } + c := s[0] + isLetter := (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') + return isLetter && (s[2] == '\\' || s[2] == '/') +} diff --git a/internal/transcode/registry.go b/internal/transcode/registry.go new file mode 100644 index 00000000..2888af2e --- /dev/null +++ b/internal/transcode/registry.go @@ -0,0 +1,122 @@ +package transcode + +import ( + "sync" + "time" + + "github.com/google/uuid" +) + +// WorkerRegistry 维护当前在线的 Worker 池 +type WorkerRegistry struct { + mu sync.RWMutex + workers map[string]*Worker + + heartbeatTimeout time.Duration +} + +func NewWorkerRegistry() *WorkerRegistry { + return &WorkerRegistry{ + workers: make(map[string]*Worker), + heartbeatTimeout: 30 * time.Second, + } +} + +// Register 注册一个新的 worker,返回分配的 ID +func (r *WorkerRegistry) Register(req *RegisterRequest, local bool) *Worker { + w := &Worker{ + ID: "wk_" + uuid.NewString(), + Name: req.Name, + Version: req.Version, + Capacity: req.Capacity, + HWAccel: req.HWAccel, + CodecsDecode: req.CodecsDecode, + CodecsEncode: req.CodecsEncode, + MaxResolution: req.MaxResolution, + Tags: req.Tags, + Endpoint: req.Endpoint, + FreeSlots: req.Capacity, + LastSeen: time.Now(), + RegisteredAt: time.Now(), + Local: local, + } + if w.Capacity <= 0 { + w.Capacity = 1 + w.FreeSlots = 1 + } + r.mu.Lock() + r.workers[w.ID] = w + r.mu.Unlock() + return w +} + +// Heartbeat 更新 worker 心跳信息,未注册返回 false +func (r *WorkerRegistry) Heartbeat(req *HeartbeatRequest) (*Worker, bool) { + r.mu.Lock() + defer r.mu.Unlock() + w, ok := r.workers[req.WorkerID] + if !ok { + return nil, false + } + w.LastSeen = time.Now() + w.Load = req.Load + w.Running = req.Running + w.FreeSlots = req.FreeSlots + return w, true +} + +// Get 获取 worker +func (r *WorkerRegistry) Get(id string) (*Worker, bool) { + r.mu.RLock() + defer r.mu.RUnlock() + w, ok := r.workers[id] + return w, ok +} + +// Unregister 主动移除 worker +func (r *WorkerRegistry) Unregister(id string) { + r.mu.Lock() + delete(r.workers, id) + r.mu.Unlock() +} + +// List 列出所有 worker(拷贝) +func (r *WorkerRegistry) List() []*Worker { + r.mu.RLock() + defer r.mu.RUnlock() + out := make([]*Worker, 0, len(r.workers)) + for _, w := range r.workers { + out = append(out, w) + } + return out +} + +// Sweep 清理超时未心跳的 worker;返回被移除的 worker_id 列表 +func (r *WorkerRegistry) Sweep() []string { + r.mu.Lock() + defer r.mu.Unlock() + now := time.Now() + var removed []string + for id, w := range r.workers { + if w.Local { + continue // 本地 Worker 不通过心跳判活 + } + if now.Sub(w.LastSeen) > r.heartbeatTimeout { + removed = append(removed, id) + delete(r.workers, id) + } + } + return removed +} + +// HasAvailable 是否存在有空闲槽位的 worker +func (r *WorkerRegistry) HasAvailable() bool { + r.mu.RLock() + defer r.mu.RUnlock() + for _, w := range r.workers { + if w.FreeSlots > 0 { + return true + } + } + return false +} diff --git a/internal/transcode/scheduler.go b/internal/transcode/scheduler.go new file mode 100644 index 00000000..6c14054f --- /dev/null +++ b/internal/transcode/scheduler.go @@ -0,0 +1,460 @@ +package transcode + +import ( + "errors" + "sort" + "strings" + "sync" + "time" + + "github.com/google/uuid" +) + +// Scheduler 任务调度器:维护 pending 队列并支持 worker pull 领取 +type Scheduler struct { + mu sync.Mutex + jobs map[string]*Job // 全部任务(含完成) + pending []*Job // 待领取队列 + + // 等待新任务的 worker 通知 + waiters []chan struct{} + + // cancellation 待派发给 worker 的取消列表(worker_id -> job_ids) + pendingCancels map[string][]string + + // onCancel 回调:Cancel 时同步通知本地 runner kill FFmpeg 进程 + // 由 Manager 在启动时注入 + onCancel func(jobID string) + + registry *WorkerRegistry +} + +func NewScheduler(reg *WorkerRegistry) *Scheduler { + return &Scheduler{ + jobs: make(map[string]*Job), + pendingCancels: make(map[string][]string), + registry: reg, + } +} + +// SetOnCancel 注入取消回调(由 Manager 在启动本地 worker 后调用) +func (s *Scheduler) SetOnCancel(fn func(jobID string)) { + s.mu.Lock() + s.onCancel = fn + s.mu.Unlock() +} + +// 提交配额。转码是 CPU/磁盘密集型操作,必须限制单用户并发与全局排队长度, +// 否则任何能访问播放接口的用户都能无限堆积任务(每个任务还会触发 30s 的 ffprobe)。 +const ( + // MaxPendingJobs 全局排队(未被 worker 领取)任务上限 + MaxPendingJobs = 200 + // MaxActiveJobsPerUser 单用户同时存在的未结束任务上限 + MaxActiveJobsPerUser = 3 +) + +var ( + // ErrQueueFull 全局队列已满 + ErrQueueFull = errors.New("transcode queue is full, try again later") + // ErrUserQuota 单用户并发任务已达上限 + ErrUserQuota = errors.New("too many active transcode jobs for this user") + // ErrJobNotFound 任务不存在或已被清理 + ErrJobNotFound = errors.New("transcode job not found") +) + +// isActiveStatus 任务是否仍占用资源(未结束) +func isActiveStatus(st JobStatus) bool { + return st == JobPending || st == JobRunning || st == JobReady +} + +// FindReusable 查找同一用户对同一路径、仍在进行中的任务,用于去重复用。 +func (s *Scheduler) FindReusable(ownerID string, path string) (*Job, bool) { + s.mu.Lock() + defer s.mu.Unlock() + return s.findReusableLocked(ownerID, path) +} + +func (s *Scheduler) findReusableLocked(ownerID string, path string) (*Job, bool) { + for _, j := range s.jobs { + if j.OwnerID == ownerID && j.Path == path && isActiveStatus(j.GetStatus()) { + return j, true + } + } + return nil, false +} + +// SubmitLimited 带去重与配额的提交: +// 1. 同一用户对同一路径已有进行中的任务时直接复用,不重复入队; +// 2. 全局排队过长或用户任务过多时拒绝。 +// +// 返回 (job, reused, err)。 +func (s *Scheduler) SubmitLimited(j *Job) (*Job, bool, error) { + s.mu.Lock() + defer s.mu.Unlock() + if exist, ok := s.findReusableLocked(j.OwnerID, j.Path); ok { + return exist, true, nil + } + if len(s.pending) >= MaxPendingJobs { + return nil, false, ErrQueueFull + } + active := 0 + for _, x := range s.jobs { + if x.OwnerID == j.OwnerID && isActiveStatus(x.GetStatus()) { + active++ + } + } + if active >= MaxActiveJobsPerUser { + return nil, false, ErrUserQuota + } + // 检查与入队在同一把锁内完成,避免并发请求同时越过配额 + return s.enqueueLocked(j), false, nil +} + +// Capacity 只读预检:在启动 ffprobe 这类昂贵操作之前先看配额是否已满。 +// +// 真正的配额判定仍由 SubmitLimited 在同一把锁内完成;这里的目的有两个: +// 1. 不让用户先等一次最长 30s 的 ffprobe 才收到 429; +// 2. 把「并发请求同时通过配额判定」这个窗口内的探测数量压下来。 +func (s *Scheduler) Capacity(ownerID string) error { + s.mu.Lock() + defer s.mu.Unlock() + if len(s.pending) >= MaxPendingJobs { + return ErrQueueFull + } + active := 0 + for _, x := range s.jobs { + if x.OwnerID == ownerID && isActiveStatus(x.GetStatus()) { + active++ + } + } + if active >= MaxActiveJobsPerUser { + return ErrUserQuota + } + return nil +} + +// Submit 提交一个任务并返回(带 ID + token)。不做配额限制,仅供内部流程(如重新激活)使用。 +func (s *Scheduler) Submit(j *Job) *Job { + s.mu.Lock() + defer s.mu.Unlock() + return s.enqueueLocked(j) +} + +// enqueueLocked 补全任务字段并入队,调用方必须已持有 s.mu。 +func (s *Scheduler) enqueueLocked(j *Job) *Job { + if j.ID == "" { + j.ID = "job_" + uuid.NewString() + } + if j.CallbackToken == "" { + j.CallbackToken = "tk_" + uuid.NewString() + } + // 播放凭据与 Worker 回写凭据分离:播放端只能读,不能上传切片或结束任务 + if j.PlayToken == "" { + j.PlayToken = "pt_" + uuid.NewString() + } + if j.readyCh == nil { + j.readyCh = make(chan struct{}) + } + if j.CreatedAt.IsZero() { + j.CreatedAt = time.Now() + } + j.Status = JobPending + + s.jobs[j.ID] = j + s.pending = append(s.pending, j) + // 唤醒等待的 worker + for _, ch := range s.waiters { + close(ch) + } + s.waiters = nil + return j +} + +// Get 通过 id 获取任务 +func (s *Scheduler) Get(id string) (*Job, bool) { + s.mu.Lock() + defer s.mu.Unlock() + j, ok := s.jobs[id] + return j, ok +} + +// Claim 由 worker 调用领取任务(最多 slots 个);如果没有任务则等待 wait 秒 +func (s *Scheduler) Claim(workerID string, slots int, wait time.Duration) []*Job { + worker, ok := s.registry.Get(workerID) + if !ok { + return nil + } + taken := s.claimNow(worker, slots) + if len(taken) > 0 || wait <= 0 { + return taken + } + // 长轮询等待 + s.mu.Lock() + notify := make(chan struct{}) + s.waiters = append(s.waiters, notify) + s.mu.Unlock() + select { + case <-notify: + case <-time.After(wait): + } + return s.claimNow(worker, slots) +} + +func (s *Scheduler) claimNow(worker *Worker, slots int) []*Job { + s.mu.Lock() + defer s.mu.Unlock() + if slots <= 0 || len(s.pending) == 0 { + return nil + } + // 按打分排序候选任务,选最匹配的前 slots 个 + candidates := append([]*Job(nil), s.pending...) + sort.SliceStable(candidates, func(i, j int) bool { + return scoreJobForWorker(candidates[i], worker) > scoreJobForWorker(candidates[j], worker) + }) + var taken []*Job + for _, j := range candidates { + if len(taken) >= slots { + break + } + if !workerCanRun(worker, j) { + continue + } + j.mu.Lock() + j.Status = JobRunning + j.WorkerID = worker.ID + j.StartedAt = time.Now() + j.mu.Unlock() + taken = append(taken, j) + // 从 pending 中移除 + for idx, pj := range s.pending { + if pj == j { + s.pending = append(s.pending[:idx], s.pending[idx+1:]...) + break + } + } + } + return taken +} + +// scoreJobForWorker 给 (worker, job) 打分(数值越大越匹配) +func scoreJobForWorker(j *Job, w *Worker) float64 { + score := 1.0 + // 硬件加速匹配 + 50 + if len(j.Profiles) > 0 { + req := strings.ToLower(j.Profiles[0].HWAccel) + if req != "" && req != "none" && req != "auto" { + for _, hw := range w.HWAccel { + if strings.EqualFold(hw, req) { + score += 50 + break + } + } + } + } + // 空闲槽位多 + + score += float64(w.FreeSlots) * 2 + // 负载低 + + score += (1 - w.Load) * 10 + return score +} + +// workerCanRun 检查 worker 是否能运行该 job +func workerCanRun(w *Worker, j *Job) bool { + if w.FreeSlots <= 0 { + return false + } + if j.Probe.VideoCodec != "" && len(w.CodecsDecode) > 0 { + ok := false + for _, c := range w.CodecsDecode { + if strings.EqualFold(c, j.Probe.VideoCodec) { + ok = true + break + } + } + if !ok { + return false + } + } + return true +} + +// Finish 由 worker 调用,标记任务完成/失败 +func (s *Scheduler) Finish(req *FinishRequest) error { + s.mu.Lock() + j, ok := s.jobs[req.JobID] + s.mu.Unlock() + if !ok { + return errors.New("job not found") + } + j.mu.Lock() + defer j.mu.Unlock() + j.Status = req.Status + j.Error = req.Error + j.FinishedAt = time.Now() + // 任务一旦完成或失败,确保 readyCh 一定会被关闭,避免播放端死等 + j.readyOnce.Do(func() { close(j.readyCh) }) + return nil +} + +// Cancel 主动取消任务(同时通知对应 worker 并 kill 本地 FFmpeg 进程) +func (s *Scheduler) Cancel(jobID string) { + s.mu.Lock() + j, ok := s.jobs[jobID] + if !ok { + s.mu.Unlock() + return + } + j.mu.Lock() + if j.Status == JobPending || j.Status == JobRunning || j.Status == JobReady { + j.Status = JobCancelled + } + j.readyOnce.Do(func() { close(j.readyCh) }) + wid := j.WorkerID + j.mu.Unlock() + if wid != "" { + s.pendingCancels[wid] = append(s.pendingCancels[wid], jobID) + } + // 从 pending 中剔除 + for idx, pj := range s.pending { + if pj.ID == jobID { + s.pending = append(s.pending[:idx], s.pending[idx+1:]...) + break + } + } + cancelFn := s.onCancel + s.mu.Unlock() + // 在锁外调用回调,避免死锁;回调会 kill 本地 FFmpeg 进程 + if cancelFn != nil { + cancelFn(jobID) + } +} + +// ListActiveJobs 列出所有活跃任务(pending/running/ready),用于空闲超时检测 +func (s *Scheduler) ListActiveJobs() []*Job { + s.mu.Lock() + defer s.mu.Unlock() + var out []*Job + for _, j := range s.jobs { + st := j.GetStatus() + if st == JobRunning || st == JobReady || st == JobPending { + out = append(out, j) + } + } + return out +} + +// PopCancellations 取出某个 worker 待处理的取消列表(心跳响应中下发) +func (s *Scheduler) PopCancellations(workerID string) []string { + s.mu.Lock() + defer s.mu.Unlock() + ids := s.pendingCancels[workerID] + delete(s.pendingCancels, workerID) + return ids +} + +// ListJobs 列出全部任务(仅用于调试/管理) +func (s *Scheduler) ListJobs() []*Job { + s.mu.Lock() + defer s.mu.Unlock() + out := make([]*Job, 0, len(s.jobs)) + for _, j := range s.jobs { + out = append(out, j) + } + return out +} + +// Reactivate 把一个已 Cancelled / Failed / Finished 的 job 重新激活回 Pending 状态, +// 让 worker 能再次领取并跑转码。用于"播放端重新请求时按需重启转码"的场景。 +// +// 设计点: +// 1. 复用原 jobID 和两套 token(播放用 PlayToken / Worker 回写用 CallbackToken), +// 前端持有的 master.m3u8 中的 URL 不需要变更 +// 2. 重置 readyCh / Status / WorkerID 等运行时字段,但保留 Path / SourceURL / Profiles / Probe +// 3. 重新加入 pending 队列并唤醒等待的 worker +// +// 返回被激活的 Job;job 不存在返回 ErrJobNotFound,单用户配额不足返回 ErrUserQuota, +// 全局排队已满返回 ErrQueueFull。 +func (s *Scheduler) Reactivate(jobID string) (*Job, error) { + s.mu.Lock() + defer s.mu.Unlock() + j, ok := s.jobs[jobID] + if !ok { + return nil, ErrJobNotFound + } + // 已经活跃:无需重新入队 + if isActiveStatus(j.GetStatus()) { + return j, nil + } + + // 重新激活同样受全局排队上限约束:否则可以靠「先让任务 idle 取消、再批量激活」 + // 把 pending 队列顶到 MaxPendingJobs 之外。 + if len(s.pending) >= MaxPendingJobs { + return nil, ErrQueueFull + } + + // 重新激活同样占用配额。否则持有 PlayToken 的用户可以先让任务因 idle 被取消, + // 再统一请求 playlist / seg-N.ts 把它们一起重新激活,绕过 SubmitLimited 的单用户上限。 + active := 0 + for _, x := range s.jobs { + if x == j { + continue + } + if x.OwnerID == j.OwnerID && isActiveStatus(x.GetStatus()) { + active++ + } + } + if active >= MaxActiveJobsPerUser { + return nil, ErrUserQuota + } + + j.mu.Lock() + // 期间可能已被其它请求激活 + if isActiveStatus(j.Status) { + j.mu.Unlock() + return j, nil + } + // 重置运行时状态 + j.Status = JobPending + j.WorkerID = "" + j.Error = "" + j.FinishedAt = time.Time{} + j.LastAccessAt = time.Now() + // 重置 readyCh:原 channel 已 closed,需要全新一份给后续 WaitReady + j.readyCh = make(chan struct{}) + j.readyOnce = sync.Once{} + j.mu.Unlock() + + // 重新入 pending 队列(先确认不在队列中以免重复) + inQueue := false + for _, pj := range s.pending { + if pj.ID == jobID { + inQueue = true + break + } + } + if !inQueue { + s.pending = append(s.pending, j) + } + // 唤醒等待的 worker + for _, ch := range s.waiters { + close(ch) + } + s.waiters = nil + return j, nil +} + +// PurgeBefore 删除 finishedAt 早于某时刻的任务,避免 jobs map 无限增长 +// 返回被清理的 job ID 列表,调用方需联动清理 Cache +func (s *Scheduler) PurgeBefore(t time.Time) []string { + s.mu.Lock() + defer s.mu.Unlock() + var purged []string + for id, j := range s.jobs { + if (j.Status == JobFinished || j.Status == JobFailed || j.Status == JobCancelled) && + !j.FinishedAt.IsZero() && j.FinishedAt.Before(t) { + delete(s.jobs, id) + purged = append(purged, id) + } + } + return purged +} diff --git a/internal/transcode/scheduler_quota_test.go b/internal/transcode/scheduler_quota_test.go new file mode 100644 index 00000000..cf8d6866 --- /dev/null +++ b/internal/transcode/scheduler_quota_test.go @@ -0,0 +1,266 @@ +package transcode + +import ( + "errors" + "fmt" + "sync" + "testing" +) + +func newTestScheduler() *Scheduler { + return NewScheduler(NewWorkerRegistry()) +} + +func TestSubmitLimitedReusesSameUserPath(t *testing.T) { + s := newTestScheduler() + a := NewJob() + a.OwnerID, a.Path = "u1", "/movie.mkv" + first, reused, err := s.SubmitLimited(a) + if err != nil || reused { + t.Fatalf("first submit: reused=%v err=%v", reused, err) + } + + b := NewJob() + b.OwnerID, b.Path = "u1", "/movie.mkv" + second, reused, err := s.SubmitLimited(b) + if err != nil || !reused { + t.Fatalf("second submit should reuse: reused=%v err=%v", reused, err) + } + if second.ID != first.ID { + t.Fatalf("reuse returned a different job: %s vs %s", second.ID, first.ID) + } + if got := len(s.ListJobs()); got != 1 { + t.Fatalf("duplicate job queued, jobs=%d", got) + } + + // 其他用户不应复用别人的任务(callback_token 属于发起者) + c := NewJob() + c.OwnerID, c.Path = "u2", "/movie.mkv" + third, reused, err := s.SubmitLimited(c) + if err != nil || reused || third.ID == first.ID { + t.Fatalf("other user must get own job: reused=%v err=%v", reused, err) + } +} + +func TestSubmitLimitedPerUserQuota(t *testing.T) { + s := newTestScheduler() + for i := 0; i < MaxActiveJobsPerUser; i++ { + j := NewJob() + j.OwnerID, j.Path = "u7", fmt.Sprintf("/f%d.mkv", i) + if _, _, err := s.SubmitLimited(j); err != nil { + t.Fatalf("submit %d: %v", i, err) + } + } + over := NewJob() + over.OwnerID, over.Path = "u7", "/over.mkv" + if _, _, err := s.SubmitLimited(over); !errors.Is(err, ErrUserQuota) { + t.Fatalf("want ErrUserQuota, got %v", err) + } + // 其它用户不受影响 + other := NewJob() + other.OwnerID, other.Path = "u8", "/other.mkv" + if _, _, err := s.SubmitLimited(other); err != nil { + t.Fatalf("other user blocked: %v", err) + } +} + +// Capacity 的判定必须与 SubmitLimited 一致:handler 靠它在启动 ffprobe 前预检配额。 +func TestCapacityMatchesSubmitLimitedQuota(t *testing.T) { + s := newTestScheduler() + if err := s.Capacity("cap-user"); err != nil { + t.Fatalf("empty scheduler should have capacity: %v", err) + } + for i := 0; i < MaxActiveJobsPerUser; i++ { + j := NewJob() + j.OwnerID, j.Path = "cap-user", fmt.Sprintf("/cap%d.mkv", i) + if _, _, err := s.SubmitLimited(j); err != nil { + t.Fatalf("submit %d: %v", i, err) + } + } + if err := s.Capacity("cap-user"); !errors.Is(err, ErrUserQuota) { + t.Fatalf("want ErrUserQuota, got %v", err) + } + // 其他用户不受影响 + if err := s.Capacity("cap-other"); err != nil { + t.Fatalf("other user should have capacity: %v", err) + } +} + +func TestSubmitLimitedQueueCap(t *testing.T) { + s := newTestScheduler() + for i := 0; i < MaxPendingJobs; i++ { + j := NewJob() + // 每个任务不同用户,避免触发单用户上限 + j.OwnerID, j.Path = fmt.Sprintf("u%d", 1000+i), "/x.mkv" + if _, _, err := s.SubmitLimited(j); err != nil { + t.Fatalf("submit %d: %v", i, err) + } + } + j := NewJob() + j.OwnerID, j.Path = "u99999", "/y.mkv" + if _, _, err := s.SubmitLimited(j); !errors.Is(err, ErrQueueFull) { + t.Fatalf("want ErrQueueFull, got %v", err) + } +} + +func TestSubmitLimitedConcurrentQuota(t *testing.T) { + s := newTestScheduler() + var wg sync.WaitGroup + var mu sync.Mutex + accepted := 0 + for i := 0; i < 50; i++ { + wg.Add(1) + go func(i int) { + defer wg.Done() + j := NewJob() + j.OwnerID, j.Path = "u5", fmt.Sprintf("/p%d.mkv", i) + if _, _, err := s.SubmitLimited(j); err == nil { + mu.Lock() + accepted++ + mu.Unlock() + } + }(i) + } + wg.Wait() + if accepted != MaxActiveJobsPerUser { + t.Fatalf("concurrent submits accepted=%d, want %d", accepted, MaxActiveJobsPerUser) + } +} + +// 重新激活(播放端请求 playlist / 切片时触发)也必须占配额, +// 否则可以先让任务闲置被取消,再统一激活来绕过单用户上限。 +func TestReactivateHonorsUserQuota(t *testing.T) { + s := newTestScheduler() + for i := 0; i < MaxActiveJobsPerUser; i++ { + j := NewJob() + j.OwnerID, j.Path = "u9", fmt.Sprintf("/r%d.mkv", i) + if _, _, err := s.SubmitLimited(j); err != nil { + t.Fatalf("submit %d: %v", i, err) + } + } + + // 构造一个"该用户此前创建、随后因闲置被取消"的任务 + idle := NewJob() + idle.OwnerID, idle.Path = "u10", "/idle.mkv" + sub, _, err := s.SubmitLimited(idle) + if err != nil { + t.Fatalf("submit idle: %v", err) + } + sub.OwnerID = "u9" + s.Cancel(sub.ID) + + if _, rerr := s.Reactivate(sub.ID); !errors.Is(rerr, ErrUserQuota) { + t.Fatalf("want ErrUserQuota, got %v", rerr) + } + + // 释放一个活跃任务后应能重新激活 + for _, j := range s.ListJobs() { + if j.Path != "/idle.mkv" && j.OwnerID == "u9" { + s.Cancel(j.ID) + break + } + } + if _, rerr := s.Reactivate(sub.ID); rerr != nil { + t.Fatalf("reactivate after freeing quota: %v", rerr) + } + if st := sub.GetStatus(); st != JobPending { + t.Fatalf("status=%v, want pending", st) + } +} + +// 重新激活同样不能突破全局排队上限:否则可以先让任务闲置取消,再批量激活把队列顶爆。 +func TestReactivateHonorsQueueLimit(t *testing.T) { + s := newTestScheduler() + + // 先造一个随后被取消的任务(Cancel 会把它从 pending 中移除) + idle := NewJob() + idle.OwnerID, idle.Path = "q1", "/idle.mkv" + sub, _, err := s.SubmitLimited(idle) + if err != nil { + t.Fatalf("submit idle: %v", err) + } + s.Cancel(sub.ID) + + // 用不同用户把全局队列填满(避开单用户配额) + for i := 0; i < MaxPendingJobs; i++ { + j := NewJob() + j.OwnerID, j.Path = fmt.Sprintf("q%d", i+2), fmt.Sprintf("/q%d.mkv", i) + if _, _, err := s.SubmitLimited(j); err != nil { + t.Fatalf("fill %d: %v", i, err) + } + } + + if _, rerr := s.Reactivate(sub.ID); !errors.Is(rerr, ErrQueueFull) { + t.Fatalf("want ErrQueueFull, got %v", rerr) + } + if st := sub.GetStatus(); st != JobCancelled { + t.Fatalf("status=%v, want cancelled (must not be requeued)", st) + } +} + +func TestValidSourceInput(t *testing.T) { + ok := []string{ + "/data/movies/a.mkv", + "http://127.0.0.1:5244/d/a.mkv?sign=x", + "https://example.com/d/a.mkv", + `C:\media\a.mkv`, + "C:/media/a.mkv", + // 文件名里的冒号很常见,不能被当成协议前缀 + "/mnt/media/Star Wars: Episode IV.mkv", + "/data/Spider-Man: Homecoming (2017).mkv", + `D:\Movies\Star Wars: IV.mkv`, + } + for _, s := range ok { + if !ValidSourceInput(s) { + t.Errorf("%q should be accepted", s) + } + } + bad := []string{ + "", "-i", "-f", "concat:/etc/passwd|/etc/hosts", "subfile,,start,0,end,0,,:/etc/passwd", + "data:text/plain;base64,AAAA", "pipe:0", "rtmp://x/y", "ftp://x/y", "file:///etc/passwd", + // 盘符后必须紧跟路径分隔符,否则不是合法的 Windows 绝对路径 + "C:foo", + } + for _, s := range bad { + if ValidSourceInput(s) { + t.Errorf("%q should be rejected", s) + } + } +} + +func TestProtocolWhitelistFor(t *testing.T) { + if got := ProtocolWhitelistFor("/data/a.mkv"); got != "file" { + t.Errorf("local = %q", got) + } + if got := ProtocolWhitelistFor("HTTPS://x/y"); got == "file" || got == "" { + t.Errorf("remote = %q", got) + } + // 远程输入不应包含 file / crypto:HTTP 源用不到,留着等于给构造过的播放列表 + // 留了一条读本地文件的通道 + for _, banned := range []string{"concat", "subfile", "data", "pipe", "rtmp", "ftp", "file", "crypto"} { + if contains := func(s string) bool { + for _, p := range splitComma(s) { + if p == banned { + return true + } + } + return false + }(ProtocolWhitelistFor("http://x/y")); contains { + t.Errorf("remote whitelist must not include %s", banned) + } + } +} + +func splitComma(s string) []string { + var out []string + cur := "" + for _, r := range s { + if r == ',' { + out = append(out, cur) + cur = "" + continue + } + cur += string(r) + } + return append(out, cur) +} diff --git a/internal/transcode/types.go b/internal/transcode/types.go new file mode 100644 index 00000000..4df99320 --- /dev/null +++ b/internal/transcode/types.go @@ -0,0 +1,178 @@ +// Package transcode 实现 OpenList 的 FFmpeg 云端/本地转码功能。 +// 设计目标: +// 1. 总开关默认关,开启后按文件大小、源码率、源编码、文件后缀过滤决定是否转码。 +// 2. 协议自研,纯 HTTP/JSON,可挂接本地内置 Worker 或远程 Worker 节点池。 +// 3. 切片流式回吐,支持边转边播。 +package transcode + +import ( + "sync" + "time" +) + +// JobStatus 转码任务状态 +type JobStatus string + +const ( + JobPending JobStatus = "pending" // 等待 Worker 领取 + JobRunning JobStatus = "running" // Worker 已领取并在转码 + JobReady JobStatus = "ready" // 至少首切片可用 + JobFinished JobStatus = "finished" // 全部切片完成 + JobFailed JobStatus = "failed" + JobCancelled JobStatus = "cancelled" +) + +// OutputFormat 输出封装格式 +type OutputFormat string + +const ( + FormatHLS OutputFormat = "hls" + FormatDASH OutputFormat = "dash" + FormatMP4 OutputFormat = "mp4" +) + +// HWAccel 硬件加速类型 +type HWAccel string + +const ( + HWNone HWAccel = "none" + HWAuto HWAccel = "auto" + HWNVENC HWAccel = "nvenc" // NVIDIA + HWQSV HWAccel = "qsv" // Intel QuickSync + HWVAAPI HWAccel = "vaapi" // Linux VA-API + HWAMF HWAccel = "amf" // AMD AMF + HWVideoToolbox HWAccel = "videotoolbox" // macOS +) + +// SourceProbe 源文件探测信息(由 ffprobe 得出) +type SourceProbe struct { + Duration float64 `json:"duration"` // 秒 + Size int64 `json:"size"` // 字节 + VideoCodec string `json:"video_codec"` // h264 / hevc / av1 / vp9 ... + VideoBitrate int64 `json:"video_bitrate"` // bps + Width int `json:"width"` + Height int `json:"height"` + AudioCodec string `json:"audio_codec"` + AudioBitrate int64 `json:"audio_bitrate"` +} + +// Profile 输出转码档位 +type Profile struct { + Name string `json:"name"` // 例如 1080p + VideoCodec string `json:"video_codec"` // h264 / hevc / av1 + VideoBitrate string `json:"video_bitrate"` // 例如 4000k + Scale string `json:"scale"` // 例如 1920:-2 + AudioCodec string `json:"audio_codec"` // aac / mp3 / opus / copy + AudioBitrate string `json:"audio_bitrate"` // 例如 160k + HWAccel string `json:"hwaccel"` // worker 端如何选择硬件加速 +} + +// OutputSpec 输出参数 +type OutputSpec struct { + Format OutputFormat `json:"format"` + SegmentDuration int `json:"segment_duration"` // 秒 +} + +// Job 一次转码任务 +type Job struct { + ID string `json:"id"` + Path string `json:"path"` // OpenList 文件路径 + SourceURL string `json:"source_url"` // 提供给 Worker 的签名 URL + Probe SourceProbe `json:"probe"` + Profiles []Profile `json:"profiles"` + Output OutputSpec `json:"output"` + CallbackToken string `json:"callback_token"` // Worker 回吐切片用(同时具备写权限,绝不外泄给播放端) + // PlayToken 播放凭据:只用于 master/variant playlist 与切片下发, + // 与 Worker 回写凭据(CallbackToken)分离,避免播放器拿到可写权限。 + // json:"-" 保证下发给 Worker 的任务描述里不会包含它。 + PlayToken string `json:"-"` + Deadline int64 `json:"deadline"` // unix 秒 + + // OwnerID 发起转码者的配额键,仅服务端用于去重与配额,不下发给 Worker。 + // 用字符串而不是用户 ID:访客共用一个账号,需要按 "guest:" 拆分配额, + // 否则单个匿名访客就能占满所有匿名转码配额。 + OwnerID string `json:"-"` + + // 运行时 + Status JobStatus `json:"status"` + WorkerID string `json:"worker_id"` + CreatedAt time.Time `json:"created_at"` + StartedAt time.Time `json:"started_at"` + FinishedAt time.Time `json:"finished_at"` + LastAccessAt time.Time `json:"last_access_at"` // 播放端最后一次请求 segment/playlist 的时间 + Error string `json:"error,omitempty"` + + // 内部:用于"边转边播"——首切片就绪后关闭该 channel + readyOnce sync.Once + readyCh chan struct{} + mu sync.RWMutex +} + +// NewJob 构造任务(自动初始化内部 channel) +func NewJob() *Job { + now := time.Now() + return &Job{ + Status: JobPending, + CreatedAt: now, + LastAccessAt: now, + readyCh: make(chan struct{}), + } +} + +// Touch 更新最后访问时间(播放端每次请求 segment/playlist 时调用) +func (j *Job) Touch() { + j.mu.Lock() + j.LastAccessAt = time.Now() + j.mu.Unlock() +} + +// GetLastAccess 获取最后访问时间(线程安全) +func (j *Job) GetLastAccess() time.Time { + j.mu.RLock() + defer j.mu.RUnlock() + return j.LastAccessAt +} + +// GetStatus 获取当前状态(线程安全) +func (j *Job) GetStatus() JobStatus { + j.mu.RLock() + defer j.mu.RUnlock() + return j.Status +} + +// MarkReady 标记任务已可播放(首切片就绪) +func (j *Job) MarkReady() { + j.readyOnce.Do(func() { + j.mu.Lock() + if j.Status == JobRunning || j.Status == JobPending { + j.Status = JobReady + } + j.mu.Unlock() + close(j.readyCh) + }) +} + +// WaitReady 阻塞等待首切片就绪 +func (j *Job) WaitReady() <-chan struct{} { return j.readyCh } + +// Worker 节点信息 +type Worker struct { + ID string `json:"id"` + Name string `json:"name"` + Version string `json:"version"` + Capacity int `json:"capacity"` // 最大并发任务数 + HWAccel []string `json:"hwaccel"` // 支持的硬件加速 + CodecsDecode []string `json:"codecs_decode"` // 可解码 + CodecsEncode []string `json:"codecs_encode"` // 可编码 + MaxResolution string `json:"max_resolution"` // 最大分辨率 + Tags []string `json:"tags"` + Endpoint string `json:"endpoint,omitempty"` // 反向连接地址(可选) + + // 运行时 + Load float64 `json:"load"` // 0~1 + Running []string `json:"running"` // 当前 job_ids + FreeSlots int `json:"free_slots"` + LastSeen time.Time `json:"last_seen"` + RegisteredAt time.Time `json:"registered_at"` + Local bool `json:"local"` // 是否本地内置 Worker +} diff --git a/pkg/torrent/bencode.go b/pkg/torrent/bencode.go index 2d4fd782..6e35690b 100644 --- a/pkg/torrent/bencode.go +++ b/pkg/torrent/bencode.go @@ -145,15 +145,24 @@ func bencodeEncodeOrderedDict(w io.Writer, d OrderedDict) error { // BencodeDecode 从字节数组解码 bencode 数据 func BencodeDecode(data []byte) (interface{}, error) { + if int64(len(data)) > DefaultMaxSeedSize { + return nil, fmt.Errorf("bencode: input exceeds %d bytes", DefaultMaxSeedSize) + } reader := bytes.NewReader(data) - val, err := bencodeDecodeValue(reader) + val, err := bencodeDecodeValue(reader, 0) if err != nil { return nil, err } + if reader.Len() != 0 { + return nil, fmt.Errorf("bencode: trailing data") + } return val, nil } -func bencodeDecodeValue(r *bytes.Reader) (interface{}, error) { +func bencodeDecodeValue(r *bytes.Reader, depth int) (interface{}, error) { + if depth > DefaultParseLimits().MaxDepth { + return nil, fmt.Errorf("bencode: nesting depth exceeds limit") + } b, err := r.ReadByte() if err != nil { return nil, err @@ -163,9 +172,9 @@ func bencodeDecodeValue(r *bytes.Reader) (interface{}, error) { case b == 'i': return bencodeDecodeInt(r) case b == 'l': - return bencodeDecodeList(r) + return bencodeDecodeList(r, depth+1) case b == 'd': - return bencodeDecodeDict(r) + return bencodeDecodeDict(r, depth+1) case b >= '0' && b <= '9': r.UnreadByte() return bencodeDecodeString(r) @@ -206,10 +215,14 @@ func bencodeDecodeString(r *bytes.Reader) ([]byte, error) { if err != nil { return nil, fmt.Errorf("bencode: invalid string length: %v", err) } - if length < 0 || length > 100*1024*1024 { - return nil, fmt.Errorf("bencode: string length out of bounds: %d", length) + // A single string can never exceed the whole input, which BencodeDecode + // already caps at DefaultMaxSeedSize. Deriving the bound from the same + // constant keeps the constraint self-consistent instead of maintaining a + // second, unreachable 100MB ceiling. + if length < 0 || length > DefaultMaxSeedSize { + return nil, fmt.Errorf("bencode: string length out of bounds: %d (limit %d)", length, DefaultMaxSeedSize) } - // Safe to convert to int: bounds check above ensures length <= 100MB which fits in int32 + // Bounded by DefaultMaxSeedSize, so the int conversion cannot truncate. data := make([]byte, int(length)) _, err = io.ReadFull(r, data) if err != nil { @@ -218,9 +231,12 @@ func bencodeDecodeString(r *bytes.Reader) ([]byte, error) { return data, nil } -func bencodeDecodeList(r *bytes.Reader) ([]interface{}, error) { +func bencodeDecodeList(r *bytes.Reader, depth int) ([]interface{}, error) { var list []interface{} for { + if len(list) >= DefaultMaxSeedFiles*4 { + return nil, fmt.Errorf("bencode: list item limit exceeded") + } b, err := r.ReadByte() if err != nil { return nil, err @@ -228,8 +244,8 @@ func bencodeDecodeList(r *bytes.Reader) ([]interface{}, error) { if b == 'e' { return list, nil } - r.UnreadByte() - val, err := bencodeDecodeValue(r) + _ = r.UnreadByte() + val, err := bencodeDecodeValue(r, depth) if err != nil { return nil, err } @@ -237,9 +253,12 @@ func bencodeDecodeList(r *bytes.Reader) ([]interface{}, error) { } } -func bencodeDecodeDict(r *bytes.Reader) (map[string]interface{}, error) { +func bencodeDecodeDict(r *bytes.Reader, depth int) (map[string]interface{}, error) { dict := make(map[string]interface{}) for { + if len(dict) >= DefaultMaxSeedFiles*4 { + return nil, fmt.Errorf("bencode: dictionary item limit exceeded") + } b, err := r.ReadByte() if err != nil { return nil, err @@ -247,12 +266,12 @@ func bencodeDecodeDict(r *bytes.Reader) (map[string]interface{}, error) { if b == 'e' { return dict, nil } - r.UnreadByte() + _ = r.UnreadByte() keyBytes, err := bencodeDecodeString(r) if err != nil { return nil, err } - val, err := bencodeDecodeValue(r) + val, err := bencodeDecodeValue(r, depth) if err != nil { return nil, err } diff --git a/pkg/torrent/generate.go b/pkg/torrent/generate.go index 566cad86..9e10aa46 100644 --- a/pkg/torrent/generate.go +++ b/pkg/torrent/generate.go @@ -1,9 +1,10 @@ package torrent import ( + "fmt" "io" "os" - "strings" + "path" ) // GenerateFromFile 从文件路径生成通用的 torrent 文件(不含 CAS 扩展) @@ -30,7 +31,7 @@ func GenerateFromReader(reader io.Reader, fileName string, fileSize int64, piece pieceSize = DefaultPieceSize } - hw := NewHashWriter(pieceSize, pieceSize) + hw := NewHashWriter(pieceSize, pieceSize, fileSize) buf := make([]byte, 32*1024) for { @@ -64,7 +65,7 @@ func GenerateFromReaderWithCAS(reader io.Reader, fileName string, fileSize int64 pieceSize = DefaultPieceSize } - hw := NewHashWriter(pieceSize, pieceSize) + hw := NewHashWriter(pieceSize, pieceSize, fileSize) buf := make([]byte, 32*1024) for { @@ -85,12 +86,8 @@ func GenerateFromReaderWithCAS(reader io.Reader, fileName string, fileSize int64 sliceMD5s := hw.GetSliceMD5s() pieceHashes := hw.GetPieceHashes() - // 计算 sliceMD5 - sliceMD5 := fileMD5 - if len(sliceMD5s) > 1 { - joined := strings.Join(sliceMD5s, "\n") - sliceMD5 = strings.ToUpper(GetMD5Str(joined)) - } + // 计算 sliceMD5(统一走规范实现) + sliceMD5 := SliceMD5FromPieces(sliceMD5s, fileMD5) t := NewTorrent(fileName, fileSize, fileMD5) t.Info.PieceLength = pieceSize @@ -100,7 +97,7 @@ func GenerateFromReaderWithCAS(reader io.Reader, fileName string, fileSize int64 SliceMD5: sliceMD5, SliceMD5s: sliceMD5s, SliceSize: pieceSize, - Cloud: "189", + Cloud: Cloud189, }) return t.Encode() @@ -121,3 +118,27 @@ func GenerateFromFileWithCAS(filePath string) ([]byte, error) { return GenerateFromReaderWithCAS(f, info.Name(), info.Size(), DefaultPieceSize) } + +// GenerateSeedFromReader computes the complete OSS hash matrix in one stream pass. +func GenerateSeedFromReader(reader io.Reader, filePath string, expectedSize, pieceSize int64, createdBy string) (*Seed, error) { + if pieceSize <= 0 { + pieceSize = DefaultPieceSize + } + if err := validateRelativeSeedPath(filePath); err != nil { + return nil, err + } + hw := NewHashWriter(pieceSize, pieceSize, expectedSize) + if _, err := CopyAndHash(nil, reader, hw); err != nil { + return nil, err + } + hw.Finish() + if expectedSize >= 0 && hw.GetTotalWritten() != expectedSize { + return nil, fmt.Errorf("stream size mismatch: read %d bytes, expected %d", hw.GetTotalWritten(), expectedSize) + } + seed := NewSeed(path.Base(filePath), createdBy, pieceSize) + seed.Files = []SeedFile{hw.BuildSeedFile(filePath, "")} + if err := ValidateSeed(seed, DefaultParseLimits()); err != nil { + return nil, err + } + return seed, nil +} diff --git a/pkg/torrent/hash_writer.go b/pkg/torrent/hash_writer.go index a62f42c6..44fda0f6 100644 --- a/pkg/torrent/hash_writer.go +++ b/pkg/torrent/hash_writer.go @@ -3,11 +3,14 @@ package torrent import ( "crypto/md5" "crypto/sha1" + "crypto/sha256" "encoding/hex" "fmt" "hash" "io" "strings" + + hash_extend "github.com/OpenListTeam/OpenList/v4/pkg/utils/hash" ) // HashWriter 同时计算文件的 MD5、分片 MD5 和 SHA-1 piece hash @@ -15,15 +18,24 @@ import ( type HashWriter struct { // 整文件 MD5 fileMD5 hash.Hash + // fileSHA1 and fileSHA256 complete the portable full-file matrix. + fileSHA1 hash.Hash + fileSHA256 hash.Hash + // fileGCID 用于迅雷、PikPak 等 + fileGCID hash.Hash // 当前分片 MD5 sliceMD5 hash.Hash - // 当前 piece 的 SHA-1 - pieceSHA1 hash.Hash + // Per-piece hashers are updated in the same pass as whole-file hashes. + pieceMD5 hash.Hash + pieceSHA1 hash.Hash + pieceSHA256 hash.Hash // 分片大小(默认 10MB) sliceSize int64 // piece 大小(与 sliceSize 相同,保持对齐) pieceSize int64 + // 文件总大小(用于 GCID 初始化) + fileSize int64 // 当前分片已写入字节数 sliceWritten int64 @@ -34,14 +46,19 @@ type HashWriter struct { // 每个分片的 MD5(大写十六进制) sliceMD5Hexs []string - // 所有 piece 的 SHA-1 哈希拼接 + // all standard BitTorrent SHA-1 piece hashes concatenated pieceHashes []byte + // portable per-file piece matrix + pieceMD5Hexs []string + pieceSHA1Hexs []string + pieceSHA256Hexs []string } // NewHashWriter 创建一个新的 HashWriter // sliceSize: CAS 分片大小(通常 10MB) // pieceSize: BT piece 大小(设为与 sliceSize 相同以保持对齐) -func NewHashWriter(sliceSize, pieceSize int64) *HashWriter { +// fileSize: 文件总大小(用于 GCID 初始化,0 表示未知) +func NewHashWriter(sliceSize, pieceSize, fileSize int64) *HashWriter { if sliceSize <= 0 { sliceSize = DefaultPieceSize } @@ -49,17 +66,23 @@ func NewHashWriter(sliceSize, pieceSize int64) *HashWriter { pieceSize = DefaultPieceSize } return &HashWriter{ - fileMD5: md5.New(), - sliceMD5: md5.New(), - pieceSHA1: sha1.New(), - sliceSize: sliceSize, - pieceSize: pieceSize, + fileMD5: md5.New(), + fileSHA1: sha1.New(), + fileSHA256: sha256.New(), + fileGCID: hash_extend.GCID.NewFunc(fileSize), + sliceMD5: md5.New(), + pieceMD5: md5.New(), + pieceSHA1: sha1.New(), + pieceSHA256: sha256.New(), + sliceSize: sliceSize, + pieceSize: pieceSize, + fileSize: fileSize, } } // NewDefaultHashWriter 创建默认的 HashWriter(10MB 分片) func NewDefaultHashWriter() *HashWriter { - return NewHashWriter(DefaultPieceSize, DefaultPieceSize) + return NewHashWriter(DefaultPieceSize, DefaultPieceSize, 0) } // Write 实现 io.Writer 接口 @@ -76,12 +99,15 @@ func (hw *HashWriter) Write(p []byte) (n int, err error) { chunk := p[offset : offset+int(canWrite)] - // 写入整文件 MD5 - hw.fileMD5.Write(chunk) - // 写入当前分片 MD5 - hw.sliceMD5.Write(chunk) - // 写入当前 piece SHA-1 - hw.pieceSHA1.Write(chunk) + // Write all whole-file and boundary-specific hashes in one pass. + _, _ = hw.fileMD5.Write(chunk) + _, _ = hw.fileSHA1.Write(chunk) + _, _ = hw.fileSHA256.Write(chunk) + _, _ = hw.fileGCID.Write(chunk) + _, _ = hw.sliceMD5.Write(chunk) + _, _ = hw.pieceMD5.Write(chunk) + _, _ = hw.pieceSHA1.Write(chunk) + _, _ = hw.pieceSHA256.Write(chunk) hw.sliceWritten += canWrite hw.pieceWritten += canWrite @@ -112,8 +138,16 @@ func (hw *HashWriter) finishSlice() { // finishPiece 完成当前 piece 的 SHA-1 计算 func (hw *HashWriter) finishPiece() { - hw.pieceHashes = append(hw.pieceHashes, hw.pieceSHA1.Sum(nil)...) + md5Sum := hw.pieceMD5.Sum(nil) + sha1Sum := hw.pieceSHA1.Sum(nil) + sha256Sum := hw.pieceSHA256.Sum(nil) + hw.pieceMD5Hexs = append(hw.pieceMD5Hexs, hex.EncodeToString(md5Sum)) + hw.pieceSHA1Hexs = append(hw.pieceSHA1Hexs, hex.EncodeToString(sha1Sum)) + hw.pieceSHA256Hexs = append(hw.pieceSHA256Hexs, hex.EncodeToString(sha256Sum)) + hw.pieceHashes = append(hw.pieceHashes, sha1Sum...) + hw.pieceMD5.Reset() hw.pieceSHA1.Reset() + hw.pieceSHA256.Reset() hw.pieceWritten = 0 } @@ -134,6 +168,56 @@ func (hw *HashWriter) GetFileMD5() string { return strings.ToUpper(hex.EncodeToString(hw.fileMD5.Sum(nil))) } +// GetFileSHA1 returns the lowercase whole-file SHA-1 digest. +func (hw *HashWriter) GetFileSHA1() string { + return hex.EncodeToString(hw.fileSHA1.Sum(nil)) +} + +// GetFileSHA256 returns the lowercase whole-file SHA-256 digest. +func (hw *HashWriter) GetFileSHA256() string { + return hex.EncodeToString(hw.fileSHA256.Sum(nil)) +} + +// GetFileGCID returns the uppercase GCID digest for Thunder/PikPak. +func (hw *HashWriter) GetFileGCID() string { + return strings.ToUpper(hex.EncodeToString(hw.fileGCID.Sum(nil))) +} + +// GetPieceMD5s returns independent per-file MD5 piece hashes. +func (hw *HashWriter) GetPieceMD5s() []string { + return append([]string(nil), hw.pieceMD5Hexs...) +} + +// GetPieceSHA1s returns independent per-file SHA-1 piece hashes. +func (hw *HashWriter) GetPieceSHA1s() []string { + return append([]string(nil), hw.pieceSHA1Hexs...) +} + +// GetPieceSHA256s returns independent per-file SHA-256 piece hashes. +func (hw *HashWriter) GetPieceSHA256s() []string { + return append([]string(nil), hw.pieceSHA256Hexs...) +} + +// BuildSeedFile exports all hashes accumulated during this single stream pass. +func (hw *HashWriter) BuildSeedFile(filePath string, modified string) SeedFile { + return SeedFile{ + Path: filePath, + Size: hw.totalWritten, + Modified: modified, + Hashes: SeedHashes{ + MD5: strings.ToLower(hw.GetFileMD5()), + SHA1: hw.GetFileSHA1(), + SHA256: hw.GetFileSHA256(), + GCID: strings.ToLower(hw.GetFileGCID()), + Pieces: &SeedPieceHashes{ + MD5: hw.GetPieceMD5s(), + SHA1: hw.GetPieceSHA1s(), + SHA256: hw.GetPieceSHA256s(), + }, + }, + } +} + // GetSliceMD5s 获取所有分片的 MD5 列表 func (hw *HashWriter) GetSliceMD5s() []string { return hw.sliceMD5Hexs @@ -141,11 +225,34 @@ func (hw *HashWriter) GetSliceMD5s() []string { // GetSliceMD5 获取最终的 sliceMD5(用于秒传) func (hw *HashWriter) GetSliceMD5(fileMD5 string) string { - if len(hw.sliceMD5Hexs) <= 1 { - return fileMD5 + return SliceMD5FromPieces(hw.sliceMD5Hexs, fileMD5) +} + +// SliceMD5FromPieces is the single canonical implementation of the sliceMd5 +// rule shared by every CAS producer and consumer: +// +// - no piece, or a single piece -> the whole-file MD5 +// - two or more pieces -> MD5 of the piece MD5s joined by "\n" +// +// Keeping one implementation matters because this value is what the remote +// provider compares against: a divergence between the hash-generation side and +// the torrent/CAS encoding side silently turns rapid uploads into mismatches. +// All comparisons and the returned value are upper-case. +func SliceMD5FromPieces(sliceMD5s []string, fileMD5 string) string { + switch len(sliceMD5s) { + case 0, 1: + // A single piece covers the whole file, so the two hashes coincide. + if len(sliceMD5s) == 1 && sliceMD5s[0] != "" { + return strings.ToUpper(sliceMD5s[0]) + } + return strings.ToUpper(fileMD5) + default: + upper := make([]string, len(sliceMD5s)) + for i, piece := range sliceMD5s { + upper[i] = strings.ToUpper(piece) + } + return strings.ToUpper(GetMD5Str(strings.Join(upper, "\n"))) } - joined := strings.Join(hw.sliceMD5Hexs, "\n") - return strings.ToUpper(GetMD5Str(joined)) } // GetPieceHashes 获取所有 piece 的 SHA-1 哈希拼接 @@ -170,7 +277,7 @@ func (hw *HashWriter) BuildTorrent(fileName string, fileSize int64) *Torrent { SliceMD5: sliceMD5, SliceMD5s: hw.GetSliceMD5s(), SliceSize: hw.sliceSize, - Cloud: "189", + Cloud: Cloud189, }) return t diff --git a/pkg/torrent/seed_security_test.go b/pkg/torrent/seed_security_test.go new file mode 100644 index 00000000..175f24d3 --- /dev/null +++ b/pkg/torrent/seed_security_test.go @@ -0,0 +1,156 @@ +package torrent + +import ( + "strings" + "testing" +) + +// --- path traversal / malformed path handling --------------------------------- + +func TestValidateSeedRejectsUnsafePaths(t *testing.T) { + cases := []struct { + name string + path string + }{ + {"parent traversal", "../secret"}, + {"nested traversal", "a/../../secret"}, + {"absolute unix", "/etc/passwd"}, + {"empty", ""}, + {"current dir", "."}, + {"nul byte", "a\x00b"}, + {"backslash traversal", `..\secret`}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + seed := testSeed() + seed.Files[0].Path = tc.path + if err := ValidateSeed(seed, DefaultParseLimits()); err == nil { + t.Fatalf("ValidateSeed() accepted unsafe path %q", tc.path) + } + }) + } +} + +func TestValidateSeedRejectsTooManyFiles(t *testing.T) { + seed := testSeed() + seed.Files = make([]SeedFile, DefaultMaxSeedFiles+1) + for i := range seed.Files { + seed.Files[i] = SeedFile{ + Path: "f" + strings.Repeat("0", i%3) + ".bin", + Size: 1, + Hashes: SeedHashes{MD5: strings.Repeat("1", 32)}, + } + } + if err := ValidateSeed(seed, DefaultParseLimits()); err == nil { + t.Fatal("ValidateSeed() accepted more files than DefaultMaxSeedFiles") + } +} + +// --- sliceMd5 canonical rule --------------------------------------------------- + +func TestSliceMD5FromPiecesMatchesSpec(t *testing.T) { + fileMD5 := strings.Repeat("a", 32) + + // Zero pieces: fall back to the whole-file MD5. + if got := SliceMD5FromPieces(nil, fileMD5); got != strings.ToUpper(fileMD5) { + t.Fatalf("SliceMD5FromPieces(nil) = %q, want %q", got, strings.ToUpper(fileMD5)) + } + // A single piece covers the whole file, so it equals the file MD5. + single := strings.ToUpper(fileMD5) + if got := SliceMD5FromPieces([]string{single}, fileMD5); got != single { + t.Fatalf("SliceMD5FromPieces(single) = %q, want %q", got, single) + } + // Two or more pieces: MD5 of the newline-joined, upper-cased piece list. + pieces := []string{strings.Repeat("b", 32), strings.Repeat("c", 32)} + want := strings.ToUpper(GetMD5Str(strings.Join([]string{strings.Repeat("B", 32), strings.Repeat("C", 32)}, "\n"))) + if got := SliceMD5FromPieces(pieces, fileMD5); got != want { + t.Fatalf("SliceMD5FromPieces(multi) = %q, want %q", got, want) + } +} + +// TestSliceMD5AgreesWithBuildCASInfo locks the rule shared by the hash-generation +// side and the CAS-encoding side. A divergence here silently turns rapid uploads +// into hash mismatches, so the two entry points must never drift apart. +func TestSliceMD5AgreesWithBuildCASInfo(t *testing.T) { + fileMD5 := strings.Repeat("a", 32) + sets := [][]string{ + nil, + {strings.Repeat("b", 32)}, + {strings.Repeat("b", 32), strings.Repeat("c", 32)}, + {strings.Repeat("b", 32), strings.Repeat("c", 32), strings.Repeat("d", 32)}, + } + for i, pieces := range sets { + hw := &HashWriter{sliceMD5Hexs: pieces} + fromWriter := hw.GetSliceMD5(fileMD5) + fromCAS := BuildCASInfoFromMD5s(fileMD5, pieces, DefaultPieceSize).SliceMD5 + if fromWriter != fromCAS { + t.Fatalf("case %d: GetSliceMD5() = %q, BuildCASInfoFromMD5s() = %q", i, fromWriter, fromCAS) + } + } +} + +// --- bencode robustness -------------------------------------------------------- + +func TestBencodeDecodeRejectsOversizedStringLength(t *testing.T) { + // Declares a 4GiB string while the buffer is empty; parsing must fail on the + // declared length instead of attempting a huge allocation. + payload := []byte("9999999999:") + if _, err := BencodeDecode(payload); err == nil { + t.Fatal("BencodeDecode() accepted an out-of-bounds string length") + } +} + +func TestBencodeDecodeRejectsDeepNesting(t *testing.T) { + depth := DefaultParseLimits().MaxDepth + 2 + payload := strings.Repeat("l", depth) + strings.Repeat("e", depth) + if _, err := BencodeDecode([]byte(payload)); err == nil { + t.Fatal("BencodeDecode() accepted nesting beyond MaxDepth") + } +} + +func TestBencodeDecodeRejectsTrailingData(t *testing.T) { + if _, err := BencodeDecode([]byte("i1eextra")); err == nil { + t.Fatal("BencodeDecode() accepted trailing data") + } +} + +// --- cross-format conversion consistency -------------------------------------- + +// TestConvertConsistencyAcrossFormats ensures a seed survives OSS -> torrent -> +// OSS and OSS -> CAS -> OSS without losing whole-file hashes. +func TestConvertConsistencyAcrossFormats(t *testing.T) { + original := testSeed() + + torrentData, err := EncodeSeed(original, "torrent") + if err != nil { + t.Fatalf("EncodeSeed(torrent) error = %v", err) + } + fromTorrent, err := DecodeSeed(torrentData, "torrent", DefaultParseLimits()) + if err != nil { + t.Fatalf("DecodeSeed(torrent) error = %v", err) + } + casData, err := EncodeCAS(fromTorrent) + if err != nil { + t.Fatalf("EncodeCAS() error = %v", err) + } + fromCAS, err := DecodeCAS(casData, DefaultParseLimits()) + if err != nil { + t.Fatalf("DecodeCAS() error = %v", err) + } + if got := fromCAS.Files[0].Hashes.MD5; got != strings.ToUpper(original.Files[0].Hashes.MD5) { + t.Fatalf("MD5 changed across formats: %q", got) + } + if len(fromCAS.Files) != len(original.Files) { + t.Fatalf("file count changed across formats: %d", len(fromCAS.Files)) + } +} + +func TestDecodeSeedRejectsUnknownFormat(t *testing.T) { + data, err := EncodeOSS(testSeed()) + if err != nil { + t.Fatalf("EncodeOSS() error = %v", err) + } + if _, err := DecodeSeed(data, "does-not-exist", DefaultParseLimits()); err == nil { + t.Fatal("DecodeSeed() accepted an unknown format") + } +} diff --git a/pkg/torrent/seed_test.go b/pkg/torrent/seed_test.go new file mode 100644 index 00000000..17ad34bf --- /dev/null +++ b/pkg/torrent/seed_test.go @@ -0,0 +1,160 @@ +package torrent + +import ( + "encoding/base64" + "encoding/json" + "strings" + "testing" + "time" +) + +func testSeed() *Seed { + return &Seed{ + Format: OSSFormat, + Version: OSSVersion, + Name: "example.bin", + CreatedAt: time.Unix(1, 0).UTC().Format(time.RFC3339), + CreatedBy: "OpenList", + PieceSize: DefaultPieceSize, + Files: []SeedFile{{ + Path: "example.bin", + Size: DefaultPieceSize + 1, + Hashes: SeedHashes{ + MD5: strings.Repeat("1", 32), + SHA1: strings.Repeat("2", 40), + SHA256: strings.Repeat("3", 64), + Pieces: &SeedPieceHashes{ + MD5: []string{strings.Repeat("4", 32), strings.Repeat("5", 32)}, + SHA1: []string{strings.Repeat("6", 40), strings.Repeat("7", 40)}, + SHA256: []string{strings.Repeat("8", 64), strings.Repeat("9", 64)}, + }, + }, + }}, + } +} + +func TestOSSRoundTrip(t *testing.T) { + encoded, err := EncodeOSS(testSeed()) + if err != nil { + t.Fatalf("EncodeOSS() error = %v", err) + } + decoded, err := DecodeOSS(encoded, DefaultParseLimits()) + if err != nil { + t.Fatalf("DecodeOSS() error = %v", err) + } + if decoded.Name != "example.bin" || len(decoded.Files) != 1 || decoded.Files[0].Hashes.SHA256 == "" { + t.Fatalf("DecodeOSS() returned incomplete seed: %#v", decoded) + } +} + +func TestCASWireFormatIsLegacyCompatible(t *testing.T) { + encoded, err := EncodeCAS(testSeed()) + if err != nil { + t.Fatalf("EncodeCAS() error = %v", err) + } + decodedJSON, err := base64.StdEncoding.DecodeString(string(encoded)) + if err != nil { + t.Fatalf("base64.DecodeString() error = %v", err) + } + var payload map[string]any + if err = json.Unmarshal(decodedJSON, &payload); err != nil { + t.Fatalf("json.Unmarshal() error = %v", err) + } + // The five legacy fields must always be present so the reference client can + // parse the payload. slice_md5s / slice_size are optional extensions. + for _, key := range []string{"name", "size", "md5", "sliceMd5", "create_time"} { + if _, ok := payload[key]; !ok { + t.Fatalf("CAS payload missing %q: %#v", key, payload) + } + } + decoded, err := DecodeCAS(encoded, DefaultParseLimits()) + if err != nil { + t.Fatalf("DecodeCAS() error = %v", err) + } + if decoded.Files[0].Hashes.MD5 != strings.Repeat("1", 32) { + t.Fatalf("DecodeCAS() MD5 = %q", decoded.Files[0].Hashes.MD5) + } + // Per-piece MD5 list must round-trip through the slice_md5s extension. + if decoded.Files[0].Hashes.Pieces == nil || len(decoded.Files[0].Hashes.Pieces.MD5) != 2 { + t.Fatalf("DecodeCAS() piece MD5 list = %#v", decoded.Files[0].Hashes.Pieces) + } +} + +func TestTorrentRoundTripPreservesOpenListExtension(t *testing.T) { + encoded, err := EncodeSeed(testSeed(), "torrent") + if err != nil { + t.Fatalf("EncodeSeed(torrent) error = %v", err) + } + decoded, err := DecodeSeed(encoded, "torrent", DefaultParseLimits()) + if err != nil { + t.Fatalf("DecodeSeed(torrent) error = %v", err) + } + if decoded.Files[0].Hashes.SHA256 != strings.Repeat("3", 64) { + t.Fatalf("torrent extension lost SHA-256: %#v", decoded.Files[0].Hashes) + } +} + +func TestCASCloudGeneralizationRoundTrip(t *testing.T) { + seed := testSeed() + seed.Files[0].CASSliceMD5 = strings.Repeat("a", 32) + seed.Files[0].CASCloud = CloudAliyundriveOpen + + // CAS (base64 JSON) round-trip must preserve the cloud identifier. + encoded, err := EncodeCAS(seed) + if err != nil { + t.Fatalf("EncodeCAS() error = %v", err) + } + decoded, err := DecodeCAS(encoded, DefaultParseLimits()) + if err != nil { + t.Fatalf("DecodeCAS() error = %v", err) + } + if got := decoded.Files[0].CASCloud; got != CloudAliyundriveOpen { + t.Fatalf("DecodeCAS() cloud = %q, want %q", got, CloudAliyundriveOpen) + } + + // Torrent bencode round-trip must preserve the cloud identifier too. + torrentData, err := EncodeSeed(seed, "torrent") + if err != nil { + t.Fatalf("EncodeSeed(torrent) error = %v", err) + } + decodedTorrent, err := DecodeSeed(torrentData, "torrent", DefaultParseLimits()) + if err != nil { + t.Fatalf("DecodeSeed(torrent) error = %v", err) + } + if got := decodedTorrent.Files[0].CASCloud; got != CloudAliyundriveOpen { + t.Fatalf("DecodeSeed(torrent) cloud = %q, want %q", got, CloudAliyundriveOpen) + } +} + +func TestBuildCASInfoFromMD5sDefaultsToCloud189(t *testing.T) { + info := BuildCASInfoFromMD5s(strings.Repeat("1", 32), []string{strings.Repeat("4", 32)}, DefaultPieceSize) + if info.Cloud != Cloud189 { + t.Fatalf("BuildCASInfoFromMD5s() cloud = %q, want %q", info.Cloud, Cloud189) + } + other := BuildCASInfoFromMD5sWithCloud(strings.Repeat("1", 32), []string{strings.Repeat("4", 32)}, DefaultPieceSize, Cloud115) + if other.Cloud != Cloud115 { + t.Fatalf("BuildCASInfoFromMD5sWithCloud() cloud = %q, want %q", other.Cloud, Cloud115) + } +} + +func TestValidateSeedRejectsTraversalAndInvalidHash(t *testing.T) { + seed := testSeed() + seed.Files[0].Path = "../secret" + if err := ValidateSeed(seed, DefaultParseLimits()); err == nil { + t.Fatal("ValidateSeed() accepted path traversal") + } + seed = testSeed() + seed.Files[0].Hashes.MD5 = "not-a-hash" + if err := ValidateSeed(seed, DefaultParseLimits()); err == nil { + t.Fatal("ValidateSeed() accepted an invalid hash") + } +} + +func TestDecodeSeedHonorsSizeLimit(t *testing.T) { + data := []byte(`{"format":"openlist-sharing-seed"}`) + limits := DefaultParseLimits() + limits.MaxBytes = int64(len(data) - 1) + if _, err := DecodeSeed(data, "oss", limits); err == nil { + t.Fatal("DecodeSeed() accepted input above MaxBytes") + } +} diff --git a/pkg/torrent/torrent.go b/pkg/torrent/torrent.go index 8744e636..dce495e4 100644 --- a/pkg/torrent/torrent.go +++ b/pkg/torrent/torrent.go @@ -1,12 +1,19 @@ package torrent import ( + "bytes" "crypto/md5" "crypto/sha1" + "encoding/base64" "encoding/hex" + "encoding/json" "fmt" + "io" + "net/url" + "path" "strings" "time" + "unicode/utf8" ) const ( @@ -26,8 +33,127 @@ const ( CASFileMD5Key = "file_md5" // CASCloudKey 云盘类型 key CASCloudKey = "cloud" + + // Cloud189 identifies the 天翼云 (189) PC driver CAS slice rule. + Cloud189 = "189" + // Cloud115 identifies the 115 driver rapid-upload rule (whole-file SHA1). + Cloud115 = "115" + // CloudAliyundriveOpen identifies the aliyundrive_open rapid-upload rule (whole-file SHA1). + CloudAliyundriveOpen = "aliyundrive_open" + + // OpenListExtensionKey is the optional root-level extension key. + OpenListExtensionKey = "x-openlist" + // OSSFormat identifies OpenList sharing seed JSON documents. + OSSFormat = "openlist-sharing-seed" + // OSSVersion is the currently supported sharing seed schema version. + OSSVersion = 1 + // DefaultMaxSeedSize bounds untrusted seed documents. + DefaultMaxSeedSize int64 = 10 * 1024 * 1024 + // DefaultMaxSeedFiles bounds file-list fan-out in untrusted seeds. + DefaultMaxSeedFiles = 100000 ) +// Seed describes the portable OpenList sharing seed contract. +type Seed struct { + Format string `json:"format"` + Version int `json:"version"` + Name string `json:"name"` + Comment string `json:"comment,omitempty"` + CreatedAt string `json:"created_at"` + CreatedBy string `json:"created_by"` + PieceSize int64 `json:"piece_size"` + Trackers []string `json:"trackers,omitempty"` + Channels []SeedChannel `json:"channels,omitempty"` + Files []SeedFile `json:"files"` +} + +// SeedChannel contains only public storage discovery metadata. +type SeedChannel struct { + Driver string `json:"driver"` + MountPath string `json:"mount_path,omitempty"` +} + +// SeedFile describes one relative file in a sharing seed. +type SeedFile struct { + Path string `json:"path"` + Size int64 `json:"size"` + Modified string `json:"modified,omitempty"` + Comment string `json:"comment,omitempty"` + Hashes SeedHashes `json:"hashes"` + Sources []SeedSource `json:"sources,omitempty"` + CASSliceMD5 string `json:"cas_slice_md5,omitempty"` + CASCreateTime string `json:"cas_create_time,omitempty"` + // CASCloud 标识该文件 CAS 元数据所属的云盘类型(如 "189")。留空视为 189。 + CASCloud string `json:"cas_cloud,omitempty"` + MissingChannels []string `json:"missing_channels,omitempty"` +} + +// SeedHashes contains whole-file and optional per-piece hashes. +type SeedHashes struct { + MD5 string `json:"md5,omitempty"` + SHA1 string `json:"sha1,omitempty"` + SHA256 string `json:"sha256,omitempty"` + GCID string `json:"gcid,omitempty"` + Pieces *SeedPieceHashes `json:"pieces,omitempty"` +} + +// SeedPieceHashes contains independent per-file piece hashes. +type SeedPieceHashes struct { + MD5 []string `json:"md5,omitempty"` + SHA1 []string `json:"sha1,omitempty"` + SHA256 []string `json:"sha256,omitempty"` +} + +// SeedSource is an optional retrievable public or signed source URL. +type SeedSource struct { + Type string `json:"type"` + URL string `json:"url"` + ExpiresAt string `json:"expires_at,omitempty"` + ShareID string `json:"share_id,omitempty"` +} + +// CASFileEntry describes one file inside a multi-file .cas payload. +type CASFileEntry struct { + Name string `json:"name"` + Size int64 `json:"size"` + MD5 string `json:"md5"` + SliceMD5 string `json:"sliceMd5"` + CreateTime string `json:"create_time"` + SliceMD5s []string `json:"slice_md5s,omitempty"` + SliceSize int64 `json:"slice_size,omitempty"` + // Cloud 标识该 CAS 条目所属的云盘类型(如 "189");留空视为 189。 + Cloud string `json:"cloud,omitempty"` +} + +// CASPayload matches the reference .cas JSON payload. The five legacy fields +// describe a single file (byte-for-byte compatible with the reference project); +// the optional "files" array extends it to multi-file seeds, and the optional +// slice_md5s/slice_size preserve the per-piece MD5 list. +type CASPayload struct { + Name string `json:"name"` + Size int64 `json:"size"` + MD5 string `json:"md5"` + SliceMD5 string `json:"sliceMd5"` + CreateTime string `json:"create_time"` + SliceMD5s []string `json:"slice_md5s,omitempty"` + SliceSize int64 `json:"slice_size,omitempty"` + // Cloud 标识该 CAS 载荷所属的云盘类型(如 "189");留空视为 189。 + Cloud string `json:"cloud,omitempty"` + Files []CASFileEntry `json:"files,omitempty"` +} + +// ParseLimits controls resource use while parsing untrusted seeds. +type ParseLimits struct { + MaxBytes int64 + MaxFiles int + MaxDepth int +} + +// DefaultParseLimits returns conservative public API limits. +func DefaultParseLimits() ParseLimits { + return ParseLimits{MaxBytes: DefaultMaxSeedSize, MaxFiles: DefaultMaxSeedFiles, MaxDepth: 64} +} + // CASInfo 天翼云 CAS 秒传所需信息 type CASInfo struct { // FileMD5 整文件 MD5(大写十六进制) @@ -38,7 +164,8 @@ type CASInfo struct { SliceMD5s []string // SliceSize 分片大小(字节) SliceSize int64 - // Cloud 云盘类型标识 + // Cloud 云盘类型标识。可为具体驱动名(如 "189"、"115"、"aliyundrive_open"), + // 或留空表示由使用方按目标驱动自行判定。不同类型的分片规则可能不同。 Cloud string } @@ -86,6 +213,8 @@ type Torrent struct { CreatedBy string // CAS 天翼云 CAS 扩展信息(存储在 info 字典外部,不影响 info_hash) CAS *CASInfo + // OpenList stores the portable extension outside info so info_hash stays standard. + OpenList *Seed } // NewTorrent 创建一个新的 torrent 结构 @@ -200,6 +329,12 @@ func (t *Torrent) Encode() ([]byte, error) { } rootDict[CASExtensionKey] = casDict } + if t.OpenList != nil { + if err := ValidateSeed(t.OpenList, DefaultParseLimits()); err != nil { + return nil, fmt.Errorf("validate x-openlist: %w", err) + } + rootDict[OpenListExtensionKey] = seedToBencode(t.OpenList) + } return BencodeEncode(rootDict) } @@ -376,6 +511,20 @@ func Decode(data []byte) (*Torrent, error) { } } + if extension, ok := rootDict[OpenListExtensionKey]; ok { + seed, err := seedFromBencode(extension) + if err != nil { + return nil, fmt.Errorf("decode x-openlist: %w", err) + } + if err = ValidateSeed(seed, DefaultParseLimits()); err != nil { + return nil, fmt.Errorf("validate x-openlist: %w", err) + } + t.OpenList = seed + } + if err := ValidateTorrent(t, DefaultParseLimits()); err != nil { + return nil, err + } + return t, nil } @@ -414,20 +563,23 @@ func (t *Torrent) HasCASInfo() bool { return t.CAS != nil && t.CAS.FileMD5 != "" && t.CAS.SliceMD5 != "" } -// BuildCASInfoFromMD5s 从分片 MD5 列表构建 CAS 信息 +// BuildCASInfoFromMD5s 从分片 MD5 列表构建 CAS 信息(默认按天翼云分片规则标记 cloud=189)。 +// 新代码应优先使用 BuildCASInfoFromMD5sWithCloud 显式指定云盘类型。 func BuildCASInfoFromMD5s(fileMD5 string, sliceMD5s []string, sliceSize int64) *CASInfo { - sliceMD5 := fileMD5 - if len(sliceMD5s) > 1 { - // 所有分片 MD5 用 \n 拼接后再取 MD5 - joined := strings.Join(sliceMD5s, "\n") - sliceMD5 = strings.ToUpper(GetMD5Str(joined)) - } + return BuildCASInfoFromMD5sWithCloud(fileMD5, sliceMD5s, sliceSize, Cloud189) +} + +// BuildCASInfoFromMD5sWithCloud 从分片 MD5 列表构建 CAS 信息,并指定云盘类型标识。 +func BuildCASInfoFromMD5sWithCloud(fileMD5 string, sliceMD5s []string, sliceSize int64, cloud string) *CASInfo { + fileMD5 = strings.ToUpper(fileMD5) + sliceMD5s = upperStrings(sliceMD5s) + sliceMD5 := SliceMD5FromPieces(sliceMD5s, fileMD5) return &CASInfo{ FileMD5: fileMD5, SliceMD5: sliceMD5, SliceMD5s: sliceMD5s, SliceSize: sliceSize, - Cloud: "189", + Cloud: cloud, } } @@ -437,3 +589,878 @@ func GetMD5Str(data string) string { h.Write([]byte(data)) return strings.ToUpper(hex.EncodeToString(h.Sum(nil))) } + +// ValidateTorrent checks standard BitTorrent invariants and portable paths. +func ValidateTorrent(t *Torrent, limits ParseLimits) error { + if t == nil { + return fmt.Errorf("missing torrent") + } + if limits.MaxFiles <= 0 { + limits.MaxFiles = DefaultMaxSeedFiles + } + if err := validateSeedName(t.Info.Name); err != nil { + return fmt.Errorf("torrent: %w", err) + } + if t.Info.PieceLength <= 0 || t.Info.PieceLength > 1<<30 { + return fmt.Errorf("torrent: invalid piece length") + } + if len(t.Info.Pieces)%sha1.Size != 0 { + return fmt.Errorf("torrent: pieces length must be a multiple of %d", sha1.Size) + } + if len(t.Info.Files) > limits.MaxFiles { + return fmt.Errorf("torrent: too many files") + } + var total int64 + if len(t.Info.Files) == 0 { + if t.Info.Length < 0 { + return fmt.Errorf("torrent: invalid file size") + } + total = t.Info.Length + } else { + seen := make(map[string]struct{}, len(t.Info.Files)) + for _, file := range t.Info.Files { + filePath := strings.Join(file.Path, "/") + if err := validateRelativeSeedPath(filePath); err != nil { + return fmt.Errorf("torrent: %w", err) + } + if _, ok := seen[filePath]; ok { + return fmt.Errorf("torrent: duplicate file path %q", filePath) + } + seen[filePath] = struct{}{} + if file.Length < 0 || total > int64(^uint64(0)>>1)-file.Length { + return fmt.Errorf("torrent: invalid file size") + } + total += file.Length + } + } + expectedPieces := int64(0) + if total > 0 { + expectedPieces = (total + t.Info.PieceLength - 1) / t.Info.PieceLength + } + if int64(len(t.Info.Pieces)/sha1.Size) != expectedPieces { + return fmt.Errorf("torrent: piece count mismatch") + } + return nil +} + +// NewSeed creates a versioned OSS document with stable defaults. +func NewSeed(name, createdBy string, pieceSize int64) *Seed { + if pieceSize <= 0 { + pieceSize = DefaultPieceSize + } + return &Seed{ + Format: OSSFormat, + Version: OSSVersion, + Name: name, + CreatedAt: time.Now().UTC().Format(time.RFC3339), + CreatedBy: createdBy, + PieceSize: pieceSize, + } +} + +// ValidateSeed rejects unsafe paths, malformed hashes and unreasonable resource use. +func ValidateSeed(seed *Seed, limits ParseLimits) error { + if seed == nil { + return fmt.Errorf("missing seed") + } + if limits.MaxBytes <= 0 { + limits.MaxBytes = DefaultMaxSeedSize + } + if limits.MaxFiles <= 0 { + limits.MaxFiles = DefaultMaxSeedFiles + } + if seed.Format != OSSFormat { + return fmt.Errorf("unsupported seed format %q", seed.Format) + } + if seed.Version != OSSVersion { + return fmt.Errorf("unsupported seed version %d", seed.Version) + } + if err := validateSeedName(seed.Name); err != nil { + return err + } + if seed.CreatedAt == "" || seed.CreatedBy == "" { + return fmt.Errorf("created_at and created_by are required") + } + if seed.PieceSize < 16*1024 || seed.PieceSize > 64*1024*1024 { + return fmt.Errorf("piece_size must be between 16384 and 67108864") + } + if len(seed.Files) == 0 || len(seed.Files) > limits.MaxFiles { + return fmt.Errorf("file count must be between 1 and %d", limits.MaxFiles) + } + seen := make(map[string]struct{}, len(seed.Files)) + var total int64 + for i := range seed.Files { + file := &seed.Files[i] + if err := validateRelativeSeedPath(file.Path); err != nil { + return fmt.Errorf("file %d: %w", i, err) + } + if _, ok := seen[file.Path]; ok { + return fmt.Errorf("duplicate file path %q", file.Path) + } + seen[file.Path] = struct{}{} + if file.Size < 0 || total > int64(^uint64(0)>>1)-file.Size { + return fmt.Errorf("file %q has invalid size", file.Path) + } + total += file.Size + if err := validateSeedHashes(file.Hashes, file.Size, seed.PieceSize); err != nil { + return fmt.Errorf("file %q: %w", file.Path, err) + } + if file.CASSliceMD5 != "" && !validHexHash(file.CASSliceMD5, 32) { + return fmt.Errorf("file %q has an invalid CAS slice MD5", file.Path) + } + for _, source := range file.Sources { + if source.Type == "" || source.URL == "" { + return fmt.Errorf("file %q has an incomplete source", file.Path) + } + u, err := url.Parse(source.URL) + if err != nil || u.Scheme == "" { + return fmt.Errorf("file %q has an invalid source URL", file.Path) + } + } + for _, channel := range file.MissingChannels { + if strings.TrimSpace(channel) == "" || strings.ContainsAny(channel, "/\\\x00") { + return fmt.Errorf("file %q has an invalid missing channel", file.Path) + } + } + } + for _, channel := range seed.Channels { + if strings.TrimSpace(channel.Driver) == "" { + return fmt.Errorf("channel driver is required") + } + if strings.ContainsAny(channel.MountPath, "?#\x00") { + return fmt.Errorf("channel mount_path contains invalid characters") + } + } + return nil +} + +func validateSeedName(name string) error { + name = strings.TrimSpace(name) + if name == "" || name == "." || name == ".." || strings.ContainsAny(name, "/\\\x00") { + return fmt.Errorf("invalid seed name %q", name) + } + return nil +} + +func validateRelativeSeedPath(name string) error { + if name == "" || !utf8.ValidString(name) || strings.ContainsAny(name, "\\\x00") || strings.HasPrefix(name, "/") { + return fmt.Errorf("invalid relative path %q", name) + } + cleaned := path.Clean(name) + if cleaned != name || cleaned == "." || cleaned == ".." || strings.HasPrefix(cleaned, "../") { + return fmt.Errorf("invalid relative path %q", name) + } + for _, part := range strings.Split(name, "/") { + if part == "" || part == "." || part == ".." { + return fmt.Errorf("invalid relative path %q", name) + } + } + return nil +} + +func validateSeedHashes(hashes SeedHashes, size, pieceSize int64) error { + for name, value := range map[string]string{"md5": hashes.MD5, "sha1": hashes.SHA1, "sha256": hashes.SHA256} { + if value != "" && !validHexHash(value, map[string]int{"md5": 32, "sha1": 40, "sha256": 64}[name]) { + return fmt.Errorf("invalid %s hash", name) + } + } + if hashes.Pieces == nil { + return nil + } + expected := 0 + if size > 0 { + expected = int((size + pieceSize - 1) / pieceSize) + } + pieceSets := []struct { + name string + width int + values []string + }{ + {"md5", 32, hashes.Pieces.MD5}, + {"sha1", 40, hashes.Pieces.SHA1}, + {"sha256", 64, hashes.Pieces.SHA256}, + } + for _, set := range pieceSets { + if len(set.values) != 0 && len(set.values) != expected { + return fmt.Errorf("%s piece count is %d, expected %d", set.name, len(set.values), expected) + } + for _, value := range set.values { + if !validHexHash(value, set.width) { + return fmt.Errorf("invalid %s piece hash", set.name) + } + } + } + return nil +} + +func validHexHash(value string, width int) bool { + if len(value) != width { + return false + } + _, err := hex.DecodeString(value) + return err == nil +} + +// EncodeOSS serializes a validated seed as UTF-8 JSON. +func EncodeOSS(seed *Seed) ([]byte, error) { + if err := ValidateSeed(seed, DefaultParseLimits()); err != nil { + return nil, err + } + return json.MarshalIndent(seed, "", " ") +} + +// DecodeOSS parses a bounded UTF-8 OSS document. +func DecodeOSS(data []byte, limits ParseLimits) (*Seed, error) { + if limits.MaxBytes <= 0 { + limits.MaxBytes = DefaultMaxSeedSize + } + if int64(len(data)) > limits.MaxBytes { + return nil, fmt.Errorf("seed exceeds %d bytes", limits.MaxBytes) + } + data = bytes.TrimPrefix(data, []byte{0xef, 0xbb, 0xbf}) + if !utf8.Valid(data) { + return nil, fmt.Errorf("seed is not valid UTF-8") + } + decoder := json.NewDecoder(bytes.NewReader(data)) + var seed Seed + if err := decoder.Decode(&seed); err != nil { + return nil, fmt.Errorf("decode OSS: %w", err) + } + var trailing any + if err := decoder.Decode(&trailing); err != io.EOF { + return nil, fmt.Errorf("decode OSS: trailing data") + } + if err := ValidateSeed(&seed, limits); err != nil { + return nil, err + } + return &seed, nil +} + +// buildCASFileEntry computes the reference-compatible five fields for one file, +// preserving the per-piece MD5 list and piece size when available. +func buildCASFileEntry(file SeedFile, pieceSize int64) (CASFileEntry, error) { + if file.Hashes.MD5 == "" { + return CASFileEntry{}, fmt.Errorf("CAS requires a whole-file MD5 for %s", file.Path) + } + var sliceMD5s []string + if file.Hashes.Pieces != nil && len(file.Hashes.Pieces.MD5) > 0 { + sliceMD5s = upperStrings(file.Hashes.Pieces.MD5) + } + // Prefer an explicitly supplied legacy slice MD5; otherwise derive it with + // the canonical rule so the value written here cannot drift from the one + // the hash-generation side computes. + sliceMD5 := strings.ToUpper(file.CASSliceMD5) + if sliceMD5 == "" && len(sliceMD5s) > 0 && pieceSize == DefaultPieceSize { + sliceMD5 = SliceMD5FromPieces(sliceMD5s, file.Hashes.MD5) + } + if sliceMD5 == "" { + if file.Size > DefaultPieceSize { + return CASFileEntry{}, fmt.Errorf("CAS requires a legacy slice MD5 or complete 10 MiB MD5 pieces for %s", file.Path) + } + sliceMD5 = strings.ToUpper(file.Hashes.MD5) + } + createTime := file.CASCreateTime + if createTime == "" { + createTime = fmt.Sprintf("%d", time.Now().Unix()) + } + entry := CASFileEntry{ + Name: path.Base(file.Path), Size: file.Size, MD5: strings.ToUpper(file.Hashes.MD5), + SliceMD5: sliceMD5, CreateTime: createTime, Cloud: file.CASCloud, + } + if len(sliceMD5s) > 0 { + entry.SliceMD5s = sliceMD5s + entry.SliceSize = pieceSize + if entry.SliceSize <= 0 { + entry.SliceSize = DefaultPieceSize + } + } + return entry, nil +} + +// EncodeCAS writes the reference-compatible base64 encoded JSON payload. A +// single-file seed uses the legacy five fields; multiple files are stored in +// the "files" array. +func EncodeCAS(seed *Seed) ([]byte, error) { + if err := ValidateSeed(seed, DefaultParseLimits()); err != nil { + return nil, err + } + if len(seed.Files) == 0 { + return nil, fmt.Errorf("CAS requires at least one file") + } + var payload CASPayload + if len(seed.Files) == 1 { + entry, err := buildCASFileEntry(seed.Files[0], seed.PieceSize) + if err != nil { + return nil, err + } + payload = CASPayload{ + Name: entry.Name, Size: entry.Size, MD5: entry.MD5, + SliceMD5: entry.SliceMD5, CreateTime: entry.CreateTime, + SliceMD5s: entry.SliceMD5s, SliceSize: entry.SliceSize, Cloud: entry.Cloud, + } + } else { + entries := make([]CASFileEntry, 0, len(seed.Files)) + var totalSize int64 + for _, file := range seed.Files { + entry, err := buildCASFileEntry(file, seed.PieceSize) + if err != nil { + return nil, err + } + entries = append(entries, entry) + totalSize += entry.Size + } + payload = CASPayload{Name: seed.Name, Size: totalSize, Files: entries} + } + content, err := json.Marshal(payload) + if err != nil { + return nil, err + } + encoded := make([]byte, base64.StdEncoding.EncodedLen(len(content))) + base64.StdEncoding.Encode(encoded, content) + return encoded, nil +} + +// DecodeCAS accepts both reference base64 JSON and raw JSON payloads. +func DecodeCAS(data []byte, limits ParseLimits) (*Seed, error) { + if limits.MaxBytes <= 0 { + limits.MaxBytes = DefaultMaxSeedSize + } + if int64(len(data)) > limits.MaxBytes { + return nil, fmt.Errorf("CAS seed exceeds %d bytes", limits.MaxBytes) + } + data = bytes.TrimSpace(data) + decoded, err := base64.StdEncoding.DecodeString(string(data)) + if err == nil { + data = decoded + } + var payload CASPayload + if err = json.Unmarshal(data, &payload); err != nil { + return nil, fmt.Errorf("decode CAS: %w", err) + } + seed := NewSeed(payload.Name, "OpenList CAS", DefaultPieceSize) + if len(payload.Files) > 0 { + // Multi-file extension. + if len(payload.Files) > limits.MaxFiles { + return nil, fmt.Errorf("CAS seed exceeds %d files", limits.MaxFiles) + } + for _, entry := range payload.Files { + file, err := casEntryToSeedFile(entry.Name, entry.Size, entry.MD5, entry.SliceMD5, entry.CreateTime, entry.SliceMD5s, entry.Cloud) + if err != nil { + return nil, err + } + if entry.SliceSize > 0 { + seed.PieceSize = entry.SliceSize + } + seed.Files = append(seed.Files, file) + } + return seed, ValidateSeed(seed, limits) + } + file, err := casEntryToSeedFile(payload.Name, payload.Size, payload.MD5, payload.SliceMD5, payload.CreateTime, payload.SliceMD5s, payload.Cloud) + if err != nil { + return nil, err + } + if payload.SliceSize > 0 { + seed.PieceSize = payload.SliceSize + } + seed.Files = []SeedFile{file} + return seed, ValidateSeed(seed, limits) +} + +// casEntryToSeedFile converts a CAS payload entry into a SeedFile, restoring the +// per-piece MD5 list when it is present. +func casEntryToSeedFile(name string, size int64, md5Hex, sliceMD5Hex, createTime string, sliceMD5s []string, cloud string) (SeedFile, error) { + if name == "" || size < 0 || !validHexHash(md5Hex, 32) { + return SeedFile{}, fmt.Errorf("invalid CAS payload") + } + sliceMD5 := sliceMD5Hex + if sliceMD5 == "" { + sliceMD5 = md5Hex + } + if !validHexHash(sliceMD5, 32) { + return SeedFile{}, fmt.Errorf("invalid CAS sliceMd5") + } + file := SeedFile{ + Path: name, Size: size, CASCreateTime: createTime, + CASSliceMD5: strings.ToLower(sliceMD5), + CASCloud: cloud, + Hashes: SeedHashes{MD5: strings.ToLower(md5Hex)}, + } + if len(sliceMD5s) > 0 { + file.Hashes.Pieces = &SeedPieceHashes{MD5: lowerStrings(sliceMD5s)} + } + return file, nil +} + +// DetectFormat determines the seed container from a file name and content. +func DetectFormat(fileName string, data []byte) string { + switch strings.ToLower(path.Ext(fileName)) { + case ".oss": + return "oss" + case ".torrent": + return "torrent" + case ".cas": + return "cas" + } + trimmed := bytes.TrimSpace(data) + if len(trimmed) > 0 && (trimmed[0] == '{' || bytes.HasPrefix(trimmed, []byte{0xef, 0xbb, 0xbf, '{'})) { + return "oss" + } + if len(trimmed) > 0 && trimmed[0] == 'd' { + return "torrent" + } + return "cas" +} + +// DecodeSeed normalizes OSS, torrent and CAS containers to the OSS contract. +func DecodeSeed(data []byte, format string, limits ParseLimits) (*Seed, error) { + if limits.MaxBytes <= 0 { + limits = DefaultParseLimits() + } + if int64(len(data)) > limits.MaxBytes { + return nil, fmt.Errorf("seed exceeds %d bytes", limits.MaxBytes) + } + switch strings.ToLower(strings.TrimPrefix(format, ".")) { + case "oss": + return DecodeOSS(data, limits) + case "torrent": + t, err := Decode(data) + if err != nil { + return nil, err + } + return SeedFromTorrent(t, limits) + case "cas": + return DecodeCAS(data, limits) + default: + return nil, fmt.Errorf("unsupported seed format %q", format) + } +} + +// EncodeSeed converts a normalized seed to the requested container. +func EncodeSeed(seed *Seed, format string) ([]byte, error) { + switch strings.ToLower(strings.TrimPrefix(format, ".")) { + case "oss": + return EncodeOSS(seed) + case "torrent": + t, diagnostics := TorrentFromSeed(seed) + if len(diagnostics) > 0 { + return nil, fmt.Errorf("cannot convert to torrent: %s", strings.Join(diagnostics, "; ")) + } + return t.Encode() + case "cas": + return EncodeCAS(seed) + default: + return nil, fmt.Errorf("unsupported seed format %q", format) + } +} + +// DiagnoseConversion reports information missing for a lossless target conversion. +func DiagnoseConversion(seed *Seed, format string) []string { + if err := ValidateSeed(seed, DefaultParseLimits()); err != nil { + return []string{err.Error()} + } + var diagnostics []string + switch strings.ToLower(strings.TrimPrefix(format, ".")) { + case "torrent": + for i, file := range seed.Files { + if file.Hashes.Pieces == nil || len(file.Hashes.Pieces.SHA1) == 0 { + diagnostics = append(diagnostics, fmt.Sprintf("%s: missing SHA-1 piece hashes", file.Path)) + } + if len(seed.Files) > 1 && i < len(seed.Files)-1 && file.Size%seed.PieceSize != 0 { + diagnostics = append(diagnostics, fmt.Sprintf("%s: piece boundary crosses the next file", file.Path)) + } + } + case "cas": + for _, file := range seed.Files { + if file.Hashes.MD5 == "" { + diagnostics = append(diagnostics, file.Path+": missing whole-file MD5") + } + if file.Size > DefaultPieceSize && file.CASSliceMD5 == "" && + (seed.PieceSize != DefaultPieceSize || file.Hashes.Pieces == nil || len(file.Hashes.Pieces.MD5) == 0) { + diagnostics = append(diagnostics, file.Path+": missing legacy slice MD5 or complete 10 MiB MD5 pieces") + } + } + case "oss": + default: + diagnostics = append(diagnostics, fmt.Sprintf("unsupported target format %q", format)) + } + return diagnostics +} + +// TorrentFromSeed constructs standard BitTorrent info/pieces plus x-openlist. +func TorrentFromSeed(seed *Seed) (*Torrent, []string) { + diagnostics := DiagnoseConversion(seed, "torrent") + if len(diagnostics) > 0 { + return nil, diagnostics + } + t := &Torrent{ + Info: TorrentInfo{PieceLength: seed.PieceSize, Name: seed.Name}, + Comment: seed.Comment, + CreatedBy: seed.CreatedBy, + CreationDate: time.Now().Unix(), + OpenList: seed, + } + if parsed, err := time.Parse(time.RFC3339, seed.CreatedAt); err == nil { + t.CreationDate = parsed.Unix() + } + if len(seed.Trackers) > 0 { + t.Announce = seed.Trackers[0] + for _, tracker := range seed.Trackers { + t.AnnounceList = append(t.AnnounceList, []string{tracker}) + } + } + for _, file := range seed.Files { + for _, piece := range file.Hashes.Pieces.SHA1 { + raw, _ := hex.DecodeString(piece) + t.Info.Pieces = append(t.Info.Pieces, raw...) + } + if len(seed.Files) == 1 { + t.Info.Length = file.Size + t.Info.MD5Sum = file.Hashes.MD5 + } else { + t.Info.Files = append(t.Info.Files, TorrentFile{ + Length: file.Size, + Path: strings.Split(file.Path, "/"), + MD5Sum: file.Hashes.MD5, + }) + } + } + if len(seed.Files) == 1 { + file := seed.Files[0] + if file.Hashes.MD5 != "" && file.CASSliceMD5 != "" { + cloud := file.CASCloud + if cloud == "" { + cloud = Cloud189 + } + t.CAS = &CASInfo{ + FileMD5: strings.ToUpper(file.Hashes.MD5), SliceMD5: strings.ToUpper(file.CASSliceMD5), + SliceSize: DefaultPieceSize, Cloud: cloud, + } + } else if file.Hashes.MD5 != "" && seed.PieceSize == DefaultPieceSize && file.Hashes.Pieces != nil && len(file.Hashes.Pieces.MD5) > 0 { + cloud := file.CASCloud + if cloud == "" { + cloud = Cloud189 + } + t.CAS = BuildCASInfoFromMD5sWithCloud(file.Hashes.MD5, upperStrings(file.Hashes.Pieces.MD5), DefaultPieceSize, cloud) + } + } + return t, nil +} + +func validateOpenListTorrentConsistency(t *Torrent, seed *Seed) error { + if seed.Name != t.Info.Name || seed.PieceSize != t.Info.PieceLength { + return fmt.Errorf("x-openlist metadata conflicts with torrent info") + } + if len(t.Info.Files) == 0 { + if len(seed.Files) != 1 || seed.Files[0].Size != t.Info.Length { + return fmt.Errorf("x-openlist file list conflicts with torrent info") + } + } else { + if len(seed.Files) != len(t.Info.Files) { + return fmt.Errorf("x-openlist file count conflicts with torrent info") + } + for i, file := range t.Info.Files { + if seed.Files[i].Path != strings.Join(file.Path, "/") || seed.Files[i].Size != file.Length { + return fmt.Errorf("x-openlist file %d conflicts with torrent info", i) + } + } + } + standardPieces := t.GetPieceHashes() + extensionPieces := make([]string, 0, len(standardPieces)) + pieceBoundariesAligned := true + for i, file := range seed.Files { + if i < len(seed.Files)-1 && file.Size%seed.PieceSize != 0 { + pieceBoundariesAligned = false + } + if file.Hashes.Pieces != nil { + extensionPieces = append(extensionPieces, file.Hashes.Pieces.SHA1...) + } + } + if pieceBoundariesAligned && len(extensionPieces) > 0 { + if len(extensionPieces) != len(standardPieces) { + return fmt.Errorf("x-openlist SHA-1 pieces conflict with torrent info") + } + for i, piece := range standardPieces { + if !strings.EqualFold(extensionPieces[i], hex.EncodeToString(piece)) { + return fmt.Errorf("x-openlist SHA-1 piece %d conflicts with torrent info", i) + } + } + } + return nil +} + +// SeedFromTorrent normalizes a standard torrent and preserves x-openlist when present. +func SeedFromTorrent(t *Torrent, limits ParseLimits) (*Seed, error) { + if t == nil { + return nil, fmt.Errorf("missing torrent") + } + if t.OpenList != nil { + if err := ValidateSeed(t.OpenList, limits); err != nil { + return nil, err + } + if err := validateOpenListTorrentConsistency(t, t.OpenList); err != nil { + return nil, err + } + return t.OpenList, nil + } + seed := NewSeed(t.Info.Name, t.CreatedBy, t.Info.PieceLength) + if seed.CreatedBy == "" { + seed.CreatedBy = "BitTorrent" + } + if t.CreationDate > 0 { + seed.CreatedAt = time.Unix(t.CreationDate, 0).UTC().Format(time.RFC3339) + } + seed.Comment = t.Comment + seed.Trackers = append(seed.Trackers, t.Announce) + for _, tier := range t.AnnounceList { + seed.Trackers = append(seed.Trackers, tier...) + } + seed.Trackers = uniqueNonEmpty(seed.Trackers) + pieceHex := make([]string, 0, len(t.Info.Pieces)/sha1.Size) + for _, piece := range t.GetPieceHashes() { + pieceHex = append(pieceHex, hex.EncodeToString(piece)) + } + if len(t.Info.Files) == 0 { + hashes := SeedHashes{MD5: t.Info.MD5Sum} + if len(pieceHex) > 0 { + hashes.Pieces = &SeedPieceHashes{SHA1: pieceHex} + } + if t.CAS != nil { + hashes.MD5 = t.CAS.FileMD5 + if hashes.Pieces == nil { + hashes.Pieces = &SeedPieceHashes{} + } + hashes.Pieces.MD5 = append([]string(nil), t.CAS.SliceMD5s...) + } + seed.Files = []SeedFile{{Path: t.Info.Name, Size: t.Info.Length, Hashes: hashes}} + } else { + pieceOffset := 0 + for i, file := range t.Info.Files { + hashes := SeedHashes{MD5: file.MD5Sum} + pieceCount := 0 + if file.Length > 0 && t.Info.PieceLength > 0 { + pieceCount = int((file.Length + t.Info.PieceLength - 1) / t.Info.PieceLength) + } + aligned := i == len(t.Info.Files)-1 || file.Length%t.Info.PieceLength == 0 + if aligned && pieceOffset+pieceCount <= len(pieceHex) { + hashes.Pieces = &SeedPieceHashes{SHA1: append([]string(nil), pieceHex[pieceOffset:pieceOffset+pieceCount]...)} + } + pieceOffset += pieceCount + seed.Files = append(seed.Files, SeedFile{Path: strings.Join(file.Path, "/"), Size: file.Length, Hashes: hashes}) + } + } + if err := ValidateSeed(seed, limits); err != nil { + return nil, err + } + return seed, nil +} + +func seedToBencode(seed *Seed) map[string]interface{} { + root := map[string]interface{}{ + "format": seed.Format, "version": int64(seed.Version), "name": seed.Name, + "created_at": seed.CreatedAt, "created_by": seed.CreatedBy, "piece_size": seed.PieceSize, + } + if seed.Comment != "" { + root["comment"] = seed.Comment + } + if len(seed.Trackers) > 0 { + root["trackers"] = stringsToInterfaces(seed.Trackers) + } + channels := make([]interface{}, 0, len(seed.Channels)) + for _, channel := range seed.Channels { + item := map[string]interface{}{"driver": channel.Driver} + if channel.MountPath != "" { + item["mount_path"] = channel.MountPath + } + channels = append(channels, item) + } + if len(channels) > 0 { + root["channels"] = channels + } + files := make([]interface{}, 0, len(seed.Files)) + for _, file := range seed.Files { + item := map[string]interface{}{"path": file.Path, "size": file.Size, "hashes": hashesToBencode(file.Hashes)} + if file.Modified != "" { + item["modified"] = file.Modified + } + if file.Comment != "" { + item["comment"] = file.Comment + } + if file.CASSliceMD5 != "" { + item["cas_slice_md5"] = file.CASSliceMD5 + } + if file.CASCreateTime != "" { + item["cas_create_time"] = file.CASCreateTime + } + if file.CASCloud != "" { + item["cas_cloud"] = file.CASCloud + } + if len(file.MissingChannels) > 0 { + item["missing_channels"] = stringsToInterfaces(file.MissingChannels) + } + sources := make([]interface{}, 0, len(file.Sources)) + for _, source := range file.Sources { + s := map[string]interface{}{"type": source.Type, "url": source.URL} + if source.ExpiresAt != "" { + s["expires_at"] = source.ExpiresAt + } + if source.ShareID != "" { + s["share_id"] = source.ShareID + } + sources = append(sources, s) + } + if len(sources) > 0 { + item["sources"] = sources + } + files = append(files, item) + } + root["files"] = files + return root +} + +func hashesToBencode(hashes SeedHashes) map[string]interface{} { + result := make(map[string]interface{}) + if hashes.MD5 != "" { + result["md5"] = hashes.MD5 + } + if hashes.SHA1 != "" { + result["sha1"] = hashes.SHA1 + } + if hashes.SHA256 != "" { + result["sha256"] = hashes.SHA256 + } + if hashes.Pieces != nil { + pieces := make(map[string]interface{}) + if len(hashes.Pieces.MD5) > 0 { + pieces["md5"] = stringsToInterfaces(hashes.Pieces.MD5) + } + if len(hashes.Pieces.SHA1) > 0 { + pieces["sha1"] = stringsToInterfaces(hashes.Pieces.SHA1) + } + if len(hashes.Pieces.SHA256) > 0 { + pieces["sha256"] = stringsToInterfaces(hashes.Pieces.SHA256) + } + if len(pieces) > 0 { + result["pieces"] = pieces + } + } + return result +} + +func seedFromBencode(value interface{}) (*Seed, error) { + root, ok := value.(map[string]interface{}) + if !ok { + return nil, fmt.Errorf("extension is not a dictionary") + } + seed := &Seed{ + Format: bString(root["format"]), Version: int(bInt(root["version"])), Name: bString(root["name"]), + Comment: bString(root["comment"]), CreatedAt: bString(root["created_at"]), + CreatedBy: bString(root["created_by"]), PieceSize: bInt(root["piece_size"]), + Trackers: bStrings(root["trackers"]), + } + for _, value := range bList(root["channels"]) { + item, ok := value.(map[string]interface{}) + if !ok { + return nil, fmt.Errorf("channel is not a dictionary") + } + seed.Channels = append(seed.Channels, SeedChannel{Driver: bString(item["driver"]), MountPath: bString(item["mount_path"])}) + } + for _, value := range bList(root["files"]) { + item, ok := value.(map[string]interface{}) + if !ok { + return nil, fmt.Errorf("file is not a dictionary") + } + file := SeedFile{ + Path: bString(item["path"]), Size: bInt(item["size"]), Modified: bString(item["modified"]), Comment: bString(item["comment"]), + CASSliceMD5: bString(item["cas_slice_md5"]), CASCreateTime: bString(item["cas_create_time"]), + CASCloud: bString(item["cas_cloud"]), + MissingChannels: bStrings(item["missing_channels"]), + } + if hashes, ok := item["hashes"].(map[string]interface{}); ok { + file.Hashes = SeedHashes{MD5: bString(hashes["md5"]), SHA1: bString(hashes["sha1"]), SHA256: bString(hashes["sha256"])} + if pieces, ok := hashes["pieces"].(map[string]interface{}); ok { + file.Hashes.Pieces = &SeedPieceHashes{MD5: bStrings(pieces["md5"]), SHA1: bStrings(pieces["sha1"]), SHA256: bStrings(pieces["sha256"])} + } + } + for _, sourceValue := range bList(item["sources"]) { + source, ok := sourceValue.(map[string]interface{}) + if !ok { + return nil, fmt.Errorf("source is not a dictionary") + } + file.Sources = append(file.Sources, SeedSource{Type: bString(source["type"]), URL: bString(source["url"]), ExpiresAt: bString(source["expires_at"]), ShareID: bString(source["share_id"])}) + } + seed.Files = append(seed.Files, file) + } + return seed, nil +} + +func stringsToInterfaces(values []string) []interface{} { + result := make([]interface{}, len(values)) + for i := range values { + result[i] = values[i] + } + return result +} + +func bString(value interface{}) string { + switch value := value.(type) { + case []byte: + return string(value) + case string: + return value + default: + return "" + } +} + +func bInt(value interface{}) int64 { + valueInt, _ := value.(int64) + return valueInt +} + +func bList(value interface{}) []interface{} { + list, _ := value.([]interface{}) + return list +} + +func bStrings(value interface{}) []string { + list := bList(value) + result := make([]string, 0, len(list)) + for _, item := range list { + if text := bString(item); text != "" { + result = append(result, text) + } + } + return result +} + +func upperStrings(values []string) []string { + result := make([]string, len(values)) + for i, value := range values { + result[i] = strings.ToUpper(value) + } + return result +} + +func lowerStrings(values []string) []string { + result := make([]string, len(values)) + for i, value := range values { + result[i] = strings.ToLower(value) + } + return result +} + +func uniqueNonEmpty(values []string) []string { + seen := make(map[string]struct{}, len(values)) + result := make([]string, 0, len(values)) + for _, value := range values { + if value == "" { + continue + } + if _, ok := seen[value]; ok { + continue + } + seen[value] = struct{}{} + result = append(result, value) + } + return result +} diff --git a/server/handles/auth.go b/server/handles/auth.go index 2013a5c2..75d3b4d7 100644 --- a/server/handles/auth.go +++ b/server/handles/auth.go @@ -11,6 +11,7 @@ import ( "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/OpenListTeam/OpenList/v4/server/common" + "github.com/OpenListTeam/go-cache" "github.com/gin-gonic/gin" "github.com/pquerna/otp/totp" "gorm.io/gorm" @@ -48,21 +49,22 @@ func loginHash(c *gin.Context, req *LoginReq) { ip := c.ClientIP() count, ok := model.LoginCache.Get(ip) if ok && count >= model.DefaultMaxAuthRetries { + // 被拒时不能再续期:自动重试的客户端(如凭据失效的 Subsonic/Web 轮询) + // 会每来一次就刷新锁定窗口,把自己永久锁死。锁定窗口只在计数时设定。 common.ErrorStrResp(c, model.TooManyAttempts, 429) - model.LoginCache.Expire(ip, model.DefaultLockDuration) return } // check username user, err := op.GetUserByName(req.Username) if err != nil { common.ErrorStrResp(c, model.InvalidUsernameOrPassword, 401) - model.LoginCache.Set(ip, count+1) + model.LoginCache.Set(ip, count+1, cache.WithEx[int](model.DefaultLockDuration)) return } // validate password hash if err := user.ValidatePwdStaticHash(req.Password); err != nil { common.ErrorStrResp(c, model.InvalidUsernameOrPassword, 401) - model.LoginCache.Set(ip, count+1) + model.LoginCache.Set(ip, count+1, cache.WithEx[int](model.DefaultLockDuration)) return } // check 2FA @@ -70,7 +72,7 @@ func loginHash(c *gin.Context, req *LoginReq) { if !totp.Validate(req.OtpCode, user.OtpSecret) { // 402 - need opt common.ErrorStrResp(c, model.Invalid2FACode, 402) - model.LoginCache.Set(ip, count+1) + model.LoginCache.Set(ip, count+1, cache.WithEx[int](model.DefaultLockDuration)) return } } diff --git a/server/handles/down.go b/server/handles/down.go index 50025a0b..82c44304 100644 --- a/server/handles/down.go +++ b/server/handles/down.go @@ -51,7 +51,9 @@ func Proxy(c *gin.Context) { common.ErrorPage(c, err, 500) return } - if canProxy(storage, filename) { + // 支持 ?force 参数强制代理(用于媒体库等需要 JS 加载文件内容的场景,避免 CORS 问题) + _, forceProxy := c.GetQuery("force") + if forceProxy || canProxy(storage, filename) { if _, ok := c.GetQuery("d"); !ok { if url := common.GenerateDownProxyURL(storage.GetStorage(), rawPath); url != "" { c.Redirect(302, url) diff --git a/server/handles/emby.go b/server/handles/emby.go new file mode 100644 index 00000000..85c0389d --- /dev/null +++ b/server/handles/emby.go @@ -0,0 +1,1246 @@ +package handles + +import ( + "crypto/md5" + "crypto/sha1" + "encoding/hex" + "encoding/json" + "hash/fnv" + "net" + "net/http" + stdpath "path" + "regexp" + "strconv" + "strings" + "sync" + "time" + + "github.com/gorilla/websocket" + + "github.com/OpenListTeam/OpenList/v4/internal/conf" + "github.com/OpenListTeam/OpenList/v4/internal/db" + "github.com/OpenListTeam/OpenList/v4/internal/media" + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/internal/op" + "github.com/OpenListTeam/OpenList/v4/internal/setting" + "github.com/OpenListTeam/OpenList/v4/pkg/utils" + "github.com/OpenListTeam/OpenList/v4/server/common" + "github.com/gin-gonic/gin" +) + +// ==================== 数据结构(Emby 使用 PascalCase 字段名) ==================== + +type embyNameID struct { + Name string + // Id 在官方 DTO 中是 Guid?,空字符串必须省略 + Id string `json:",omitempty"` +} + +type embyUserData struct { + PlaybackPositionTicks int64 + PlayCount int + IsFavorite bool + Played bool + UnplayedItemCount int + PlayedPercentage float64 + Rating float64 `json:",omitempty"` + Likes bool + // LastPlayedDate 是 DateTime?,空字符串必须省略 + LastPlayedDate string `json:",omitempty"` +} + +// embyMediaSource 对应官方 MediaSourceInfo。 +// 客户端按 SupportsDirectPlay -> SupportsDirectStream -> TranscodingUrl 的顺序判定播放方式, +// 同时读取 MediaStreams(视频/音频轨)和 RequiresOpening 等字段,缺失会被判定为"不可播放"。 +type embyMediaSource struct { + Protocol string + Id string + Path string `json:",omitempty"` + Type string + Container string `json:",omitempty"` + Name string `json:",omitempty"` + Size int64 `json:",omitempty"` + IsRemote bool + ETag string `json:",omitempty"` + RunTimeTicks int64 `json:",omitempty"` + ReadAtNativeFramerate bool + IgnoreDts bool + IgnoreIndex bool + GenPtsInput bool + SupportsTranscoding bool + SupportsDirectStream bool + SupportsDirectPlay bool + IsInfiniteStream bool + RequiresOpening bool + RequiresClosing bool + RequiresLooping bool + SupportsProbing bool + VideoType string `json:",omitempty"` + MediaStreams []embyMediaStream + MediaAttachments []any + Formats []string + Bitrate int `json:",omitempty"` + RequiredHttpHeaders map[string]string + DirectStreamUrl string `json:",omitempty"` + AddApiKeyToDirectStreamUrl bool + DefaultAudioStreamIndex int `json:",omitempty"` + // DefaultSubtitleStreamIndex 指向歌词字幕轨:客户端歌词面板只认 + // Type=="Subtitle" 且 Index 等于该值的流,缺失就不请求歌词 + DefaultSubtitleStreamIndex *int `json:",omitempty"` + ItemId string `json:",omitempty"` +} + +// embyMediaStream 对应官方 MediaStream,只填写客户端判定播放所需的最小字段。 +type embyMediaStream struct { + Codec string `json:",omitempty"` + Type string + Index int + IsDefault bool + IsForced bool + IsExternal bool + IsInterlaced bool + IsTextSubtitleStream bool + SupportsExternalStream bool + Channels int `json:",omitempty"` + SampleRate int `json:",omitempty"` + BitRate int `json:",omitempty"` + DisplayTitle string `json:",omitempty"` + Language string `json:",omitempty"` + DeliveryMethod string `json:",omitempty"` + DeliveryUrl string `json:",omitempty"` +} + +// embyItem 字段名与类型严格对应 Emby/Jellyfin 的 BaseItemDto: +// Id/ParentId/SeriesId/SeasonId/AlbumId 是 Guid,DateCreated/PremiereDate/EndDate 是 DateTime?。 +// 空值必须省略(omitempty),否则客户端会把 "" 当作 Guid/DateTime 解析并抛异常。 +type embyItem struct { + Name string `json:",omitempty"` + ServerId string `json:",omitempty"` + Id string `json:",omitempty"` + DateCreated string `json:",omitempty"` + Container string `json:",omitempty"` + SortName string `json:",omitempty"` + PremiereDate string `json:",omitempty"` + Overview string `json:",omitempty"` + ProductionYear int `json:",omitempty"` + CommunityRating float32 `json:",omitempty"` + OfficialRating string `json:",omitempty"` + RunTimeTicks int64 `json:",omitempty"` + IndexNumber int `json:",omitempty"` + ParentIndexNumber int `json:",omitempty"` + Type string `json:",omitempty"` + MediaType string `json:",omitempty"` + IsFolder bool + ParentId string `json:",omitempty"` + SeriesId string `json:",omitempty"` + SeriesName string `json:",omitempty"` + SeasonId string `json:",omitempty"` + SeasonName string `json:",omitempty"` + Album string `json:",omitempty"` + AlbumId string `json:",omitempty"` + AlbumArtist string `json:",omitempty"` + AlbumArtists []embyNameID + Artists []string + ArtistItems []embyNameID + Genres []string + GenreItems []embyNameID + Studios []embyNameID + People []any + MediaStreams []any + Taglines []string + ProductionLocations []string + Keywords []string + RemoteTrailers []any + ExternalUrls []any + LockedFields []string + Path string `json:",omitempty"` + ImageTags map[string]string + BackdropImageTags []string + ProviderIds map[string]string + UserData *embyUserData + MediaSources []embyMediaSource + ChildCount int `json:",omitempty"` + LocationType string `json:",omitempty"` + CollectionType string `json:",omitempty"` + ViewType string `json:",omitempty"` + // Subviews 是电影库页面的必需字段:客户端 videos.js 直接读取 + // item.Subviews 并调用 .includes(),缺失会抛 undefined.includes 导致整页报错 + Subviews []string `json:",omitempty"` + EndDate string `json:",omitempty"` + Status string `json:",omitempty"` + PlayAccess string `json:",omitempty"` + Video3DFormat string `json:",omitempty"` + PreferredMetadataLanguage string `json:",omitempty"` + PreferredMetadataCountryCode string `json:",omitempty"` + CleanName string `json:",omitempty"` + PresentationUniqueKey string `json:",omitempty"` + ShortOverview string `json:",omitempty"` + HomePageUrl string `json:",omitempty"` + SeriesPrimaryImageTag string `json:",omitempty"` + PrimaryImageTag string `json:",omitempty"` + ParentThumbItemId string `json:",omitempty"` + ParentThumbImageTag string `json:",omitempty"` + ParentPrimaryImageItemId string `json:",omitempty"` + ParentPrimaryImageTag string `json:",omitempty"` + AlbumPrimaryImageTag string `json:",omitempty"` + ParentBackdropItemId string `json:",omitempty"` + AirTime string `json:",omitempty"` + PrimaryImageAspectRatio float64 `json:",omitempty"` + CriticRating float64 `json:",omitempty"` + MediaSourceCount int `json:",omitempty"` + LocalTrailerCount int `json:",omitempty"` + RecursiveItemCount int `json:",omitempty"` + SpecialFeatureCount int `json:",omitempty"` + PartCount int `json:",omitempty"` + IsHD bool + CanDelete bool + CanDownload bool + SupportsSync bool + IsPlaceHolder bool + LockData bool + ImageBlurHashes map[string]any + TagItems []embyNameID + AirDays []string + ParentBackdropImageTags []string + OriginalTitle string `json:",omitempty"` + ForcedSortName string `json:",omitempty"` + Etag string `json:",omitempty"` + DateLastMediaAdded string `json:",omitempty"` + CustomRating string `json:",omitempty"` + ChannelId string `json:",omitempty"` + ChannelName string `json:",omitempty"` + FileName string `json:",omitempty"` + Number string `json:",omitempty"` + SeriesStudio string `json:",omitempty"` + AspectRatio string `json:",omitempty"` + DisplayPreferencesId string `json:",omitempty"` + ParentLogoItemId string `json:",omitempty"` + ParentLogoImageTag string `json:",omitempty"` + IsoType string `json:",omitempty"` + Software string `json:",omitempty"` + CameraMake string `json:",omitempty"` + CameraModel string `json:",omitempty"` + Size int64 `json:",omitempty"` + Bitrate int `json:",omitempty"` + IndexNumberEnd int `json:",omitempty"` + DisplayOrder string `json:",omitempty"` + Width int `json:",omitempty"` + Height int `json:",omitempty"` + Chapters []any + Tags []string +} + +// MarshalJSON 把 nil 切片/映射输出为空数组/空对象。 +// Emby 客户端会对部分字段直接调用 .includes(), +// JSON 里出现 null 或字段缺失都会让客户端抛异常。 +func (e embyItem) MarshalJSON() ([]byte, error) { + type alias embyItem + a := alias(e) + if a.AlbumArtists == nil { + a.AlbumArtists = []embyNameID{} + } + if a.Artists == nil { + a.Artists = []string{} + } + if a.ArtistItems == nil { + a.ArtistItems = []embyNameID{} + } + if a.Genres == nil { + a.Genres = []string{} + } + if a.GenreItems == nil { + a.GenreItems = []embyNameID{} + } + if a.Studios == nil { + a.Studios = []embyNameID{} + } + if a.People == nil { + a.People = []any{} + } + if a.MediaStreams == nil { + a.MediaStreams = []any{} + } + if a.Taglines == nil { + a.Taglines = []string{} + } + if a.ProductionLocations == nil { + a.ProductionLocations = []string{} + } + if a.Keywords == nil { + a.Keywords = []string{} + } + if a.RemoteTrailers == nil { + a.RemoteTrailers = []any{} + } + if a.ExternalUrls == nil { + a.ExternalUrls = []any{} + } + if a.LockedFields == nil { + a.LockedFields = []string{} + } + if a.ImageTags == nil { + a.ImageTags = map[string]string{} + } + // 没有封面时不输出 Primary:空 tag 会让 Theater 拼出 ?tag= 的无效图片请求, + // openemby_tv 等客户端也只在 tag 非空时才请求图片 + if v, ok := a.ImageTags["Primary"]; ok && v == "" { + delete(a.ImageTags, "Primary") + } + // 剧集/歌曲等子项没有自己的封面时,让客户端回退到父级封面,保证列表不出现空白图 + if a.ImageTags["Primary"] == "" && a.ParentPrimaryImageTag != "" { + if a.ParentPrimaryImageItemId == "" { + a.ParentPrimaryImageItemId = a.ParentId + } + } + if a.BackdropImageTags == nil { + a.BackdropImageTags = []string{} + } + if a.ProviderIds == nil { + a.ProviderIds = map[string]string{} + } + if a.MediaSources == nil { + a.MediaSources = []embyMediaSource{} + } + if a.ImageBlurHashes == nil { + a.ImageBlurHashes = map[string]any{} + } + if a.TagItems == nil { + a.TagItems = []embyNameID{} + } + if a.AirDays == nil { + a.AirDays = []string{} + } + if a.ParentBackdropImageTags == nil { + a.ParentBackdropImageTags = []string{} + } + if a.Chapters == nil { + a.Chapters = []any{} + } + if a.Tags == nil { + a.Tags = []string{} + } + if a.UserData == nil { + a.UserData = &embyUserData{} + } + // PlayAccess 为空会让客户端判定"播放受限" + if a.PlayAccess == "" { + a.PlayAccess = "Full" + } + if !a.IsFolder { + a.CanDownload = true + } + // 客户端会直接展示 CleanName,并把它当作排序/查找键 + if a.CleanName == "" { + a.CleanName = a.Name + } + if a.PresentationUniqueKey == "" { + a.PresentationUniqueKey = a.Id + } + return json.Marshal(a) +} + +type embyQueryResult struct { + Items []*embyItem + TotalRecordCount int + StartIndex int +} + +type embyUserDto struct { + Name string + ServerId string + Id string + HasPassword bool + HasConfiguredPassword bool + Configuration map[string]any + Policy map[string]any + PrimaryImageTag string `json:",omitempty"` + // DateTime? 字段空值必须省略 + LastLoginDate string `json:",omitempty"` + LastActivityDate string `json:",omitempty"` + IsAdministrator bool + IsDisabled bool +} + +type embyAuthResult struct { + User *model.User + AccessToken string + ServerId string +} + +// embyPublicUserDto 登录前展示的用户信息。 +// Emby 客户端依赖 /Users/Public 渲染用户选择列表,返回空数组会跳过登录界面。 +type embyPublicUserDto struct { + Name string + ServerId string + Id string + HasPassword bool + PrimaryImageTag string +} + +// ==================== 基础工具 ==================== + +func embyServerID() string { + seed := conf.Conf.SiteURL + conf.Conf.JwtSecret + strconv.Itoa(conf.Conf.Scheme.HttpPort) + sum := sha1.Sum([]byte("openlist-emby:" + seed)) + // Emby 的 ServerId 是 32 位 hex GUID,客户端本地库会用 Guid 解析 + return hex.EncodeToString(sum[:16]) +} + +// embyCompatVersion 是返回给 Emby 客户端的兼容版本号。 +// Emby 客户端会把该值与其要求的最低服务器版本比较,直接返回 OpenList 版本 +// (如 v4.1.0 / dev)会被判定为"服务器需要升级"而拒绝连接。 +// 客户端在 4.10.0.4 起改用服务端 HomeSections API; +// 低于该版本会走本地生成逻辑,其 section 缺少 ListOptions.fields 会直接报错。 +const embyCompatVersion = "4.10.0.5" + +func embyVersion() string { + return embyCompatVersion +} + +func embyTicks(seconds int) int64 { + if seconds <= 0 { + return 0 + } + return int64(seconds) * 10_000_000 +} + +func embyTimeStr(t time.Time) string { + return t.UTC().Format("2006-01-02T15:04:05.0000000Z") +} + +func embyExt(name string) string { + ext := strings.TrimPrefix(strings.ToLower(stdpath.Ext(name)), ".") + if ext == "" { + return "mkv" + } + return ext +} + +var embyAuthHeaderRe = regexp.MustCompile(`(?i)([A-Za-z]+)\s*=\s*"([^"]*)"`) + +func embyParseAuthHeader(v string) map[string]string { + out := map[string]string{} + for _, m := range embyAuthHeaderRe.FindAllStringSubmatch(v, -1) { + out[strings.ToLower(m[1])] = m[2] + } + return out +} + +func embyRequestToken(c *gin.Context) string { + // Emby Windows 等客户端会把 X-Emby-Token 直接放在 URL query 里, + // 必须与 api_key / ApiKey 一起解析,否则登录成功后每个请求仍是 401 + for key, values := range c.Request.URL.Query() { + if len(values) == 0 || values[0] == "" { + continue + } + switch { + case strings.EqualFold(key, "api_key"), + strings.EqualFold(key, "ApiKey"), + strings.EqualFold(key, "X-Emby-Token"), + strings.EqualFold(key, "X-MediaBrowser-Token"): + return strings.TrimSpace(values[0]) + } + } + for _, h := range []string{"X-Emby-Token", "X-MediaBrowser-Token"} { + if t := strings.TrimSpace(c.GetHeader(h)); t != "" { + return t + } + } + if v := c.GetHeader("X-Emby-Authorization"); v != "" { + if t, ok := embyParseAuthHeader(v)["token"]; ok && t != "" { + return t + } + } + if v := c.GetHeader("Authorization"); strings.HasPrefix(strings.ToLower(v), "mediabrowser ") { + if t, ok := embyParseAuthHeader(v)["token"]; ok && t != "" { + return t + } + } + return "" +} + +// embyAuth 解析请求身份,token 可以是设备会话令牌或媒体应用密钥 +func embyAuth(c *gin.Context) (*model.User, bool) { + token := embyRequestToken(c) + if token == "" { + return nil, false + } + if dev, err := db.GetEmbyDeviceByTokenHash(media.HashToken(token)); err == nil { + db.TouchEmbyDevice(dev.ID) + user, err := op.GetUserById(dev.UserID) + if err != nil || user.Disabled { + return nil, false + } + return user, true + } + user, _, ok := LookUpAppKeyUser(token) + if !ok { + return nil, false + } + return user, true +} + +// embyUserGUID 返回给客户端的用户 ID(Emby 规范为 32 位 hex GUID) +func embyUserGUID(id uint) string { + return embyGUID("user-" + strconv.FormatUint(uint64(id), 10)) +} + +// embyResolveUserID 兼容数字 ID 与 GUID 两种形式 +func embyResolveUserID(idStr string) (uint, bool) { + if n, err := strconv.ParseUint(idStr, 10, 64); err == nil { + return uint(n), true + } + users, _, err := op.GetUsers(1, -1) + if err != nil { + return 0, false + } + for i := range users { + if embyUserGUID(users[i].ID) == idStr { + return users[i].ID, true + } + } + return 0, false +} + +func embyUserDtoOf(u *model.User) *embyUserDto { + // Emby 客户端的 PlayAccessValidation 会直接读取 Policy.EnableMediaPlayback, + // 缺失即判定为禁止播放,并弹出 "Playback of this content is currently restricted"。 + // 这里按 Emby 的 UserPolicy 结构给全默认值,避免任何字段被读成 undefined。 + policy := map[string]any{ + "IsAdministrator": u.IsAdmin(), + "IsDisabled": u.Disabled, + "IsHidden": false, + "EnableMediaPlayback": true, + "EnableAudioPlaybackTranscoding": true, + "EnableVideoPlaybackTranscoding": true, + "EnablePlaybackRemuxing": true, + "EnableContentDownloading": true, + "EnableSyncTranscoding": true, + "EnableSubtitleDownloading": true, + "EnableSubtitleManagement": true, + "EnableRemoteControlOfOtherUsers": false, + "EnableSharedDeviceControl": true, + "EnableRemoteAccess": true, + "EnableLiveTvManagement": false, + "EnableLiveTvAccess": false, + "EnableAllChannels": true, + "EnableAllFolders": true, + "EnableAllDevices": true, + "ForceRemoteSourceTranscoding": false, + "EnabledFolders": []string{}, + "EnabledChannels": []string{}, + "EnabledDevices": []string{}, + "BlockedChannels": []string{}, + "BlockedMediaFolders": []string{}, + "BlockedTags": []string{}, + "AllowedTags": []string{}, + "MaxParentalRating": nil, + "AuthenticationProviderId": "Default", + "PasswordResetProviderId": "Default", + } + return &embyUserDto{ + Name: u.Username, + ServerId: embyServerID(), + Id: embyUserGUID(u.ID), + HasPassword: u.PwdHash != "", + HasConfiguredPassword: u.PwdHash != "", + Configuration: map[string]any{}, + Policy: policy, + IsAdministrator: u.IsAdmin(), + IsDisabled: u.Disabled, + } +} + +// embyWebNoticeHTML 浏览器直接打开 @emby 时显示的提示页。 +// OpenList 只实现 Emby API,没有 Emby Web 客户端静态资源。 +const embyWebNoticeHTML = ` + + +OpenList - Emby API + +

OpenList 仅提供 Emby 兼容 API

+

当前地址没有 Emby Web 客户端界面,请使用支持 Emby 协议的客户端添加服务器。

+ + + + +
服务器地址http://你的地址:端口/@emby
用户名OpenList 用户名
密码媒体应用密钥(管理 - 媒体库 - 应用密钥)
+

推荐客户端:Infuse、VidHub、SenPlayer、Emby 官方 App 等。

+
+

This endpoint only serves the Emby-compatible API. Please use an Emby client instead of a browser.

+` + +// ==================== 请求分发 ==================== + +// EmbyAPI Emby 兼容 API 入口,兼容 /@emby 与 /@mediabrowser 前缀,路径大小写不敏感 +func EmbyAPI(c *gin.Context) { + if !MediaEmbyEnabled() { + c.AbortWithStatus(404) + return + } + raw := strings.Trim(c.Param("path"), "/") + if raw == "" { + c.JSON(200, gin.H{"Service": "OpenList", "Version": embyVersion()}) + return + } + // 兼容把接入地址填成 .../@emby 后客户端又自动追加 /emby(或 /mediabrowser)的情况 + if first, rest, found := strings.Cut(raw, "/"); found { + switch strings.ToLower(first) { + case "emby", "mediabrowser": + raw = rest + } + } + segs := strings.Split(raw, "/") + lower := make([]string, len(segs)) + for i, s := range segs { + lower[i] = strings.ToLower(s) + } + method := c.Request.Method + + // 公共端点:无需认证 + switch { + case len(lower) == 3 && lower[0] == "system" && lower[1] == "info" && lower[2] == "public": + embyPublicSystemInfo(c) + return + case len(lower) == 2 && lower[0] == "system" && lower[1] == "ping": + c.String(200, "Emby Server") + return + // 客户端连接阶段还会拉取公共配置,返回空对象即可,否则同样会 401 + case len(lower) == 3 && lower[0] == "system" && lower[1] == "configuration" && lower[2] == "public": + c.JSON(200, gin.H{}) + return + case len(lower) == 2 && lower[0] == "branding" && lower[1] == "configuration": + c.JSON(200, gin.H{}) + return + // 离线同步接口:OpenList 不支持客户端下载同步,返回空结果而不是 401 + case len(lower) >= 1 && lower[0] == "sync": + if method == "GET" { + c.JSON(200, []any{}) + } else { + c.Status(204) + } + return + // 浏览器直接打开 @emby 时会拉取 Emby Web 静态资源,这里返回说明页而不是 401 + case lower[0] == "web": + if strings.HasSuffix(strings.ToLower(c.Request.URL.Path), ".json") { + c.JSON(200, gin.H{}) + return + } + c.Header("Content-Type", "text/html; charset=utf-8") + c.String(200, embyWebNoticeHTML) + return + // 没有 token 的连接探测:部分客户端直接请求 /System/Info(非 Public)。 + // 必须返回完整结构(含 WebPath),精简版会让客户端在处理 sections 时崩溃 + case len(lower) == 2 && lower[0] == "system" && lower[1] == "info": + embySystemInfo(c) + return + case len(lower) == 2 && lower[0] == "system" && lower[1] == "endpoint": + // 按实际来源判断,不要固定返回 true:硬编码「内网直连」会让客户端 + // 误以为可直连并跳过中转,也是对匿名访问者宣称内网可达。 + local := embyIsLocalRequest(c) + c.JSON(200, gin.H{"IsLocal": local, "IsInNetwork": local}) + return + case len(lower) == 2 && lower[0] == "localization" && lower[1] == "options": + c.JSON(200, []any{}) + return + case len(lower) == 2 && lower[0] == "sessions" && lower[1] == "capabilities": + c.Status(204) + return + case len(lower) == 3 && lower[0] == "sessions" && lower[1] == "capabilities" && lower[2] == "full": + c.Status(204) + return + // 播放进度上报:部分客户端的原生播放器不带 token, + // 这类请求不能 401,否则客户端会中断播放状态跟踪 + case len(lower) >= 2 && lower[0] == "sessions" && lower[1] == "playing": + if user, ok := embyAuth(c); ok { + embySessionPlaying(c, user, strings.Join(lower[2:], "/")) + } else { + c.Status(204) + } + return + // 电影库页面可能请求的推荐/字母索引接口,返回空结果 + case method == "GET" && len(lower) == 2 && lower[0] == "movies" && lower[1] == "recommendations": + c.JSON(200, []any{}) + return + case method == "GET" && len(lower) == 2 && lower[0] == "items" && lower[1] == "prefixes": + c.JSON(200, []any{}) + return + // 添加服务器 / 登录流程必须在鉴权前放行,否则客户端只会看到 Unauthorized + case len(lower) == 2 && lower[0] == "users" && lower[1] == "public": + embyPublicUsers(c) + return + case len(lower) == 2 && lower[0] == "users" && lower[1] == "authenticatebyname": + embyAuthenticateByName(c) + return + // 旧版客户端登录:POST /Users/{userId}/Authenticate,密码在 body 或 query + case len(lower) == 3 && lower[0] == "users" && lower[2] == "authenticate": + embyAuthenticateByID(c, segs[1]) + return + case len(lower) >= 3 && lower[0] == "items" && strings.EqualFold(lower[2], "images"): + // /Items/{id}/Images/Primary 等:图片 URL 不带 token,必须匿名可访问 + embyImage(c, segs[1], segs, true) + return + } + + user, ok := embyAuth(c) + if !ok { + embyUnauthorized(c) + return + } + + switch { + // Emby 客户端用它做会话与实时通知,缺失时会持续重连 + case len(lower) == 1 && lower[0] == "embywebsocket": + embyWebSocket(c) + // 客户端启动/仪表盘探测接口,返回空数据避免 404 弹通用错误 + case method == "GET" && len(lower) == 1 && lower[0] == "sessions": + c.JSON(200, []any{}) + case method == "GET" && len(lower) == 1 && lower[0] == "scheduledtasks": + c.JSON(200, []any{}) + case len(lower) == 3 && lower[0] == "system" && lower[1] == "activitylog" && lower[2] == "entries": + c.JSON(200, gin.H{"Items": []any{}, "TotalRecordCount": 0, "StartIndex": 0}) + // LiveTv:会话页/剧集页会查询录像、节目单等,全部返回空结果 + case len(lower) >= 2 && lower[0] == "livetv": + c.JSON(200, gin.H{"Items": []any{}, "TotalRecordCount": 0, "StartIndex": 0}) + // Emby 4.10 客户端首页读取用户设置:GET /usersettings/{userId} + case len(lower) == 2 && lower[0] == "usersettings": + if method == "GET" { + c.JSON(200, gin.H{ + "Screensaver": gin.H{}, + "HomeSections": []any{}, + "Playback": gin.H{}, + }) + } else { + c.Status(204) + } + // 客户端登录后会读取/保存界面偏好,字段缺失会让 sectionscontroller 抛异常 + case len(lower) >= 1 && lower[0] == "displaypreferences": + if method == "GET" { + c.JSON(200, gin.H{ + "Id": "usersettings", + "ViewType": "", + "SortBy": "SortName", + "IndexBy": "", + "RememberIndexing": false, + "PrimaryImageHeight": 250, + "PrimaryImageWidth": 250, + "CustomPrefs": gin.H{ + "homeSections": "", + }, + "ScrollDirection": "Horizontal", + "ShowBackdrop": true, + "RememberSorting": false, + "SortOrder": "Ascending", + "ShowSidebar": false, + "Client": "emby", + }) + } else { + c.Status(204) + } + case len(lower) == 2 && lower[0] == "system" && lower[1] == "info": + embySystemInfo(c) + case len(lower) == 2 && lower[0] == "system" && lower[1] == "endpoint": + c.JSON(200, gin.H{"IsLocal": true, "IsInNetwork": true}) + case len(lower) == 1 && lower[0] == "users" && method == "GET": + embyListUsers(c, user) + case len(lower) == 2 && lower[0] == "users" && lower[1] == "query": + embyQueryUsers(c) + case len(lower) == 2 && lower[0] == "users" && method == "GET": + embyGetUser(c, segs[1], user) + case len(lower) == 3 && lower[0] == "users" && lower[2] == "views": + embyViews(c, user) + case len(lower) == 2 && lower[0] == "items" && lower[1] == "counts": + embyItemCounts(c, user) + case len(lower) == 2 && lower[0] == "library" && lower[1] == "refresh" && method == "POST": + embyLibraryRefresh(c) + case len(lower) == 2 && lower[0] == "library" && lower[1] == "selectablemediafolders": + embySelectableFolders(c, user) + case len(lower) == 3 && lower[0] == "library" && lower[1] == "virtualfolders" && lower[2] == "query": + embyVirtualFolders(c, user) + case len(lower) == 2 && lower[0] == "search" && lower[1] == "hints": + embySearchHints(c, user) + // 首页「收藏的艺术家 / 人物」分区查询,无对应数据时返回空结果 + case method == "GET" && len(lower) == 1 && lower[0] == "artists": + c.JSON(200, gin.H{"Items": []any{}, "TotalRecordCount": 0, "StartIndex": 0}) + case method == "GET" && len(lower) == 1 && lower[0] == "persons": + c.JSON(200, gin.H{"Items": []any{}, "TotalRecordCount": 0, "StartIndex": 0}) + case len(lower) == 2 && lower[0] == "artists": + embyArtists(c, user, lower[1] == "albumartists") + case len(lower) == 3 && lower[0] == "sessions" && lower[1] == "playing": + embySessionPlaying(c, user, lower[2]) + case len(lower) == 2 && lower[0] == "sessions" && lower[1] == "playing" && method == "POST": + embySessionPlaying(c, user, "") + case len(lower) == 2 && lower[0] == "sessions" && lower[1] == "logout" && method == "POST": + embyLogout(c, user) + case len(lower) == 2 && lower[0] == "sessions" && lower[1] == "capabilities": + c.Status(204) + case len(lower) == 3 && lower[0] == "sessions" && lower[1] == "capabilities" && lower[2] == "full": + c.Status(204) + // 媒体库页(电影/剧集/音乐)筛选栏依赖的分面接口,缺失返回 404 会让整页报错 + case method == "GET" && len(lower) == 2 && lower[0] == "items" && (lower[1] == "filters" || lower[1] == "filters2"): + c.JSON(200, gin.H{"Genres": []any{}, "Tags": []any{}, "OfficialRatings": []any{}, "Years": []any{}}) + case method == "GET" && len(lower) == 1 && embyEmptyListRoot[lower[0]]: + c.JSON(200, gin.H{"Items": []any{}, "TotalRecordCount": 0, "StartIndex": 0}) + // 用户态条目 + case len(lower) >= 3 && lower[0] == "users": + embyUserScoped(c, user, segs, lower, method) + // 全局条目 + case len(lower) >= 2 && lower[0] == "items": + embyItemsScoped(c, user, segs, lower, method) + // 播放流 + case len(lower) >= 3 && (lower[0] == "videos" || lower[0] == "audio") && + (lower[2] == "stream" || strings.HasPrefix(lower[2], "stream.") || lower[2] == "universal"): + embyStream(c, user, segs[1], lower[0] == "audio") + case len(lower) == 3 && lower[0] == "audio" && lower[2] == "lyrics": + embyLyrics(c, user, segs[1]) + // 剧集:/Shows/NextUp 只有两段,/Shows/{id}/Seasons 为三段 + case len(lower) >= 2 && lower[0] == "shows": + embyShows(c, user, segs, lower, method) + default: + embyNotFound(c) + } +} + +func embyUnauthorized(c *gin.Context) { + // 记录客户端实际请求,便于排查"连接后被判未认证"的兼容问题。 + // 只能记 Path 不能记 RequestURI:query 里可能带 Pw/Password/api_key, + // 而 X-Emby-Authorization 里是可直接使用的设备令牌,都不能落到日志里。 + utils.Log.Warnf("emby unauthorized: %s %s client=%q ua=%q hasToken=%v", + c.Request.Method, c.Request.URL.Path, c.ClientIP(), + c.GetHeader("User-Agent"), embyRequestToken(c) != "") + c.JSON(401, gin.H{"Error": "Unauthorized", "ResponseStatus": gin.H{"ErrorCode": "Unauthorized"}}) +} + +func embyNotFound(c *gin.Context) { + c.JSON(404, gin.H{"Error": "Not Found"}) +} + +// embyEmptyListRoot 本服务无对应数据、但客户端页面一定会请求的顶层集合接口 +var embyEmptyListRoot = map[string]bool{ + "genres": true, "musicgenres": true, "studios": true, "tags": true, + "officialratings": true, "years": true, "collections": true, + "playlists": true, "trailers": true, "channels": true, "livetv": true, +} + +// embyAuxItem 处理详情页附带请求(相似、预告片、特辑、片头等)。 +// 这类接口缺失时客户端详情页和列表页都会弹"error processing the request"。 +// 返回 true 表示已响应。 +func embyAuxItem(c *gin.Context, name string) bool { + switch name { + case "similar", "suggestions", "instantmix", "intros": + c.JSON(200, gin.H{"Items": []any{}, "TotalRecordCount": 0, "StartIndex": 0}) + case "localtrailers", "specialfeatures", "additionalparts", "remoteimages", "thememedia": + c.JSON(200, []any{}) + case "criticreviews": + c.JSON(200, gin.H{"Items": []any{}, "TotalRecordCount": 0}) + default: + return false + } + return true +} + +var embyWSUpgrader = websocket.Upgrader{ + ReadBufferSize: 1024, + WriteBufferSize: 1024, + CheckOrigin: func(r *http.Request) bool { return true }, +} + +// embyWebSocket 提供最小可用的 Emby WebSocket 通道。 +// 客户端用它做会话/实时通知,缺失时会持续重连并产生大量 404。 +func embyWebSocket(c *gin.Context) { + conn, err := embyWSUpgrader.Upgrade(c.Writer, c.Request, nil) + if err != nil { + return + } + defer conn.Close() + conn.SetReadLimit(4096) + _ = conn.SetReadDeadline(time.Now().Add(90 * time.Second)) + conn.SetPongHandler(func(string) error { + return conn.SetReadDeadline(time.Now().Add(90 * time.Second)) + }) + stop := make(chan struct{}) + defer close(stop) + go func() { + ticker := time.NewTicker(30 * time.Second) + defer ticker.Stop() + for { + select { + case <-ticker.C: + if err := conn.WriteMessage(websocket.TextMessage, []byte(`{"MessageType":"KeepAlive"}`)); err != nil { + return + } + case <-stop: + return + } + } + }() + for { + if _, _, err := conn.ReadMessage(); err != nil { + return + } + } +} + +// ==================== 系统 / 用户 ==================== + +func embyPublicSystemInfo(c *gin.Context) { + api := common.GetApiUrl(c.Request.Context()) + c.JSON(200, gin.H{ + "ServerName": setting.GetStr(conf.SiteTitle, "OpenList"), + "Version": embyVersion(), + "Id": embyServerID(), + "OperatingSystem": "Linux", + "LocalAddress": api, + "WebPath": "", + "StartupWizardCompleted": true, + }) +} + +func embySystemInfo(c *gin.Context) { + api := common.GetApiUrl(c.Request.Context()) + c.JSON(200, gin.H{ + "ServerName": setting.GetStr(conf.SiteTitle, "OpenList"), + "Version": embyVersion(), + "Id": embyServerID(), + "OperatingSystem": "Linux", + "OperatingSystemDisplayName": "Linux", + "LocalAddress": api, + "WebPath": "", + "StartupWizardCompleted": true, + "HasPendingRestart": false, + "IsShuttingDown": false, + "SupportsLibraryMonitor": false, + "CanSelfRestart": false, + "CanLaunchWebBrowser": false, + "SupportsHttps": false, + "HasUpdateAvailable": false, + "EncoderLocation": "System", + "SystemArchitecture": "X64", + "SystemUpdateLevel": "Release", + // 不返回真实监听端口:这个端点匿名可访问,客户端本来就是从已知端口连进来的, + // 回了真实端口只是白给一份端口探测信息。 + "HttpServerPortNumber": 0, + "HttpsPortNumber": 0, + "ProgramDataPath": "", + "ItemsByNamePath": "", + "CachePath": "", + "LogPath": "", + "InternalMetadataPath": "", + "TranscodingTempPath": "", + }) +} + +// embyIsLocalRequest 判断请求是否来自本机 / 内网,用于 /System/Endpoint。 +func embyIsLocalRequest(c *gin.Context) bool { + ip := net.ParseIP(c.ClientIP()) + if ip == nil { + return false + } + return ip.IsLoopback() || ip.IsPrivate() +} + +// embyReadAuthBody 读取登录凭据,兼容 JSON / 表单 / query 三种提交方式 +func embyReadAuthBody(c *gin.Context) (username, pw string) { + if strings.Contains(strings.ToLower(c.ContentType()), "json") { + var req struct { + Username string `json:"Username"` + Pw string `json:"Pw"` + Password string `json:"Password"` + } + _ = c.ShouldBindJSON(&req) + username, pw = req.Username, req.Pw + if pw == "" { + pw = req.Password + } + } else { + _ = c.Request.ParseForm() + username = c.PostForm("Username") + pw = c.PostForm("Pw") + if pw == "" { + pw = c.PostForm("Password") + } + } + if username == "" { + username = c.Query("Username") + } + if pw == "" { + pw = c.Query("Pw") + } + if pw == "" { + pw = c.Query("Password") + } + return strings.TrimSpace(username), strings.TrimSpace(pw) +} + +// embyAuthenticateByName 标准登录接口 POST /Users/AuthenticateByName +func embyAuthenticateByName(c *gin.Context) { + username, pw := embyReadAuthBody(c) + // 除 IP 外还要检查用户名维度:IP 可被 X-Forwarded-For 伪造 + if mediaAuthLockedFor(c, username) { + c.JSON(429, gin.H{"Error": model.TooManyAttempts}) + return + } + if username == "" || pw == "" { + embyUnauthorized(c) + return + } + user, err := op.GetUserByName(username) + if err != nil || user.Disabled || user.IsGuest() { + mediaAuthFailedFor(c, username) + embyUnauthorized(c) + return + } + embyCompleteAuth(c, user, pw) +} + +// embyAuthenticateByID 旧版登录接口 POST /Users/{userId}/Authenticate +func embyAuthenticateByID(c *gin.Context, idStr string) { + if mediaAuthLocked(c) { + c.JSON(429, gin.H{"Error": model.TooManyAttempts}) + return + } + id, ok := embyResolveUserID(idStr) + if !ok { + embyUnauthorized(c) + return + } + user, err := op.GetUserById(id) + if err != nil || user.Disabled || user.IsGuest() { + mediaAuthFailed(c) + embyUnauthorized(c) + return + } + // 按 id 登录同样要受用户名维度的锁定约束,否则可以换 id 绕过按用户名的计数 + if mediaAuthLockedFor(c, user.Username) { + c.JSON(429, gin.H{"Error": model.TooManyAttempts}) + return + } + _, pw := embyReadAuthBody(c) + if pw == "" { + embyUnauthorized(c) + return + } + embyCompleteAuth(c, user, pw) +} + +// embyCompleteAuth 校验密码并签发设备令牌。 +// 密码位置既接受 OpenList 登录密码(无 2FA 时),也接受属于该用户的应用密钥。 +func embyCompleteAuth(c *gin.Context, user *model.User, pw string) { + keyUser, _, keyOK := LookUpAppKeyUser(pw) + switch { + case keyOK && keyUser.ID == user.ID: + // 应用密钥认证通过,2FA 账号也允许(密钥本身即第二因素) + case keyOK: + mediaAuthFailedFor(c, user.Username) + embyUnauthorized(c) + return + case user.OtpSecret != "": + // 2FA 账号不允许用登录密码通过 Emby 认证 + mediaAuthFailedFor(c, user.Username) + c.JSON(403, gin.H{"Error": "Accounts with two-factor authentication must use an app key"}) + return + case user.ValidateRawPassword(pw) != nil: + mediaAuthFailedFor(c, user.Username) + embyUnauthorized(c) + return + } + mediaAuthSucceededFor(c, user.Username) + authHeader := embyParseAuthHeader(c.GetHeader("X-Emby-Authorization")) + token, err := media.NewDeviceToken() + if err != nil { + c.JSON(500, gin.H{"Error": err.Error()}) + return + } + dev := &model.EmbyDevice{ + UserID: user.ID, + DeviceID: authHeader["deviceid"], + DeviceName: authHeader["device"], + Client: authHeader["client"], + Version: authHeader["version"], + TokenHash: media.HashToken(token), + LastSeenAt: time.Now(), + } + _ = db.CreateEmbyDevice(dev) + c.JSON(200, gin.H{ + "User": embyUserDtoOf(user), + "AccessToken": token, + "ServerId": embyServerID(), + }) +} + +// embyPublicUsers 匿名可访问,只返回登录界面需要的公开字段, +// 不包含权限、策略、是否管理员等敏感信息。 +func embyPublicUsers(c *gin.Context) { + out := make([]*embyPublicUserDto, 0) + users, _, err := op.GetUsers(1, -1) + if err == nil { + for i := range users { + if users[i].IsGuest() || users[i].Disabled { + continue + } + out = append(out, &embyPublicUserDto{ + Name: users[i].Username, + ServerId: embyServerID(), + Id: embyUserGUID(users[i].ID), + HasPassword: users[i].PwdHash != "", + }) + } + } + c.JSON(200, out) +} + +func embyListUsers(c *gin.Context, current *model.User) { + users, _, err := op.GetUsers(1, -1) + if err != nil { + c.JSON(500, gin.H{"Error": err.Error()}) + return + } + out := make([]*embyUserDto, 0, len(users)) + for i := range users { + if users[i].IsGuest() { + continue + } + if !current.IsAdmin() && users[i].ID != current.ID { + continue + } + out = append(out, embyUserDtoOf(&users[i])) + } + c.JSON(200, out) +} + +func embyQueryUsers(c *gin.Context) { + users, _, err := op.GetUsers(1, -1) + if err != nil { + c.JSON(500, gin.H{"Error": err.Error()}) + return + } + out := make([]*embyUserDto, 0, len(users)) + for i := range users { + if users[i].IsGuest() { + continue + } + out = append(out, embyUserDtoOf(&users[i])) + } + c.JSON(200, gin.H{"Items": out, "TotalRecordCount": len(out), "StartIndex": 0}) +} + +func embyGetUser(c *gin.Context, idStr string, current *model.User) { + // Jellyfin/部分 Emby 客户端使用 /Users/Me 获取当前用户 + if strings.EqualFold(idStr, "me") { + c.JSON(200, embyUserDtoOf(current)) + return + } + id, ok := embyResolveUserID(idStr) + if !ok { + embyNotFound(c) + return + } + user, err := op.GetUserById(id) + if err != nil { + embyNotFound(c) + return + } + c.JSON(200, embyUserDtoOf(user)) +} + +func embyLogout(c *gin.Context, user *model.User) { + token := embyRequestToken(c) + if token != "" { + if dev, err := db.GetEmbyDeviceByTokenHash(media.HashToken(token)); err == nil { + _ = db.DeleteEmbyDevice(user.ID, dev.DeviceID) + } + } + c.Status(204) +} + +// ==================== 媒体库刷新 ==================== + +var ( + embyRefreshMu sync.Mutex + embyRefreshTimer *time.Timer +) + +func scheduleLibraryRefresh() { + embyRefreshMu.Lock() + defer embyRefreshMu.Unlock() + if embyRefreshTimer != nil { + embyRefreshTimer.Stop() + } + embyRefreshTimer = time.AfterFunc(10*time.Second, func() { + for _, mt := range MediaTypeEnabledList() { + cfg, err := db.GetMediaConfig(mt) + if err != nil || cfg == nil { + continue + } + media.ScanMedia(cfg) + } + }) +} + +func embyLibraryRefresh(c *gin.Context) { + scheduleLibraryRefresh() + c.Status(204) +} + +// ==================== 内部工具 ==================== + +func embyPaging(c *gin.Context) (start, limit int) { + start, _ = strconv.Atoi(c.DefaultQuery("StartIndex", "0")) + limit, _ = strconv.Atoi(c.DefaultQuery("Limit", "100")) + if start < 0 { + start = 0 + } + if limit <= 0 || limit > 1000 { + limit = 200 + } + return +} + +func embyBoolQuery(c *gin.Context, key string) bool { + v := strings.ToLower(c.Query(key)) + return v == "true" || v == "1" +} + +func embyIntQuery(c *gin.Context, key string) int { + v, _ := strconv.Atoi(c.Query(key)) + return v +} + +func embyJson(v any) string { + b, err := json.Marshal(v) + if err != nil { + return "" + } + return string(b) +} + +func embyHash(s string) string { + sum := sha1.Sum([]byte(s)) + return hex.EncodeToString(sum[:6]) +} + +// embyGUID 把内部可读 ID 映射为 32 位 hex GUID。 +// embyGUID 用于用户 ID(Emby 规范为 32 位 hex GUID)。 +// 注意:条目的 Item Id 不能用 GUID,见 embyNumericID。 +func embyGUID(s string) string { + sum := md5.Sum([]byte("openlist-emby-id:" + s)) + return hex.EncodeToString(sum[:]) +} + +// embyNumericID 生成稳定的十进制数字 ID,供条目(影片/剧集/专辑等)使用。 +// Emby 真实服务器的 Item Id 就是数字字符串;Emby Theater 的原生 +// LocalItemRepository 会把它当 long 解析(System.Number.ThrowFormatException), +// 传 GUID 或 mv-1 这类可读 ID 都会让客户端播放/取图直接抛异常。 +func embyNumericID(s string) string { + h := fnv.New64a() + _, _ = h.Write([]byte("openlist-emby-item:" + s)) + // 取 47 位:既在 .NET long / JS 安全整数范围内,又足够大避免与真实 ID 混淆 + return strconv.FormatUint(h.Sum64()&0x7FFFFFFFFFFF, 10) +} + +func embyImageTag(item *model.MediaItem) string { + if item.Cover == "" { + return "" + } + return embyHash(item.Cover + item.UpdatedAt.String()) +} diff --git a/server/handles/emby_browse.go b/server/handles/emby_browse.go new file mode 100644 index 00000000..6473a329 --- /dev/null +++ b/server/handles/emby_browse.go @@ -0,0 +1,1654 @@ +package handles + +import ( + "encoding/json" + "fmt" + stdpath "path" + "sort" + "strconv" + "strings" + "time" + + "github.com/OpenListTeam/OpenList/v4/internal/db" + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/gin-gonic/gin" +) + +// ==================== 媒体库索引 ==================== + +type embyLibrary struct { + user *model.User + videos []model.MediaItem + music []model.MediaItem + + userData map[[2]uint]*model.MediaUserData + // ids 保存 GUID -> 内部可读 ID 的映射,客户端只会回传 GUID + ids map[string]string +} + +func loadEmbyLibrary(user *model.User) *embyLibrary { + l := &embyLibrary{ + user: user, + userData: map[[2]uint]*model.MediaUserData{}, + ids: map[string]string{}, + } + if MediaTypeEnabled(model.MediaTypeVideo) { + items, err := db.ListAllMediaItems(model.MediaTypeVideo) + if err == nil { + for i := range items { + if CanUserAccessItem(user, &items[i]) { + l.videos = append(l.videos, items[i]) + } + } + } + } + if MediaTypeEnabled(model.MediaTypeMusic) { + items, err := db.ListAllMediaItems(model.MediaTypeMusic) + if err == nil { + for i := range items { + if CanUserAccessItem(user, &items[i]) { + l.music = append(l.music, items[i]) + } + } + } + } + ids := make([]uint, 0, len(l.videos)+len(l.music)) + for _, it := range l.videos { + ids = append(ids, it.ID) + } + for _, it := range l.music { + ids = append(ids, it.ID) + } + if user != nil && len(ids) > 0 { + if list, err := db.ListMediaUserDataByItems(user.ID, ids); err == nil { + for i := range list { + d := list[i] + l.userData[[2]uint{d.ItemID, uint(d.EpisodeIndex)}] = &d + } + } + } + l.registerIDs() + return l +} + +// guid 返回内部 ID 对应的对外数字 ID,并登记反查映射。 +// 名称保留 guid 是为了少改调用点,实际输出的是纯数字(Emby Item Id 规范)。 +func (l *embyLibrary) guid(orig string) string { + g := embyNumericID(orig) + if l.ids != nil { + l.ids[g] = orig + // 兼容旧版本缓存下来的 GUID 形式 ID:客户端没清缓存时仍能解析, + // 否则会一直拿旧 ID 请求并得到 {"Error":"Not Found"} + l.ids[embyGUID(orig)] = orig + } + return g +} + +// origID 把客户端回传的数字 ID 还原为内部 ID;已是内部 ID 时原样返回 +func (l *embyLibrary) origID(id string) string { + id = strings.TrimSpace(id) + if l.ids != nil { + if orig, ok := l.ids[id]; ok { + return orig + } + } + return id +} + +// registerIDs 预先登记全部条目的对外 ID,保证任意请求都能反查出内部 ID +func (l *embyLibrary) registerIDs() { + register := func(orig string) { + l.ids[embyNumericID(orig)] = orig + l.ids[embyGUID(orig)] = orig + } + register("root") + register("lib-movie") + register("lib-tv") + register("lib-music") + for i := range l.videos { + register(fmt.Sprintf("mv-%d", l.videos[i].ID)) + } + for _, s := range l.seriesList() { + register("sr-" + s.Key) + for _, season := range l.seasonList(s) { + register(fmt.Sprintf("sn-%s-%d", s.Key, season)) + } + for _, ep := range l.episodesOf(s.Key, 0) { + register(ep.ID) + } + } + for i := range l.music { + it := &l.music[i] + register(fmt.Sprintf("so-%d", it.ID)) + register("al-" + albumKeyOf(it)) + register("ar-" + artistKeyOf(it)) + } +} + +func (l *embyLibrary) userDataOf(itemID uint, episodeIndex int) *embyUserData { + if l == nil || l.user == nil { + return &embyUserData{} + } + d, ok := l.userData[[2]uint{itemID, uint(episodeIndex)}] + if !ok { + return &embyUserData{} + } + return &embyUserData{ + PlaybackPositionTicks: d.PositionTicks, + PlayCount: d.PlayCount, + IsFavorite: d.IsFavorite, + Played: d.Played, + } +} + +type embySeries struct { + Key string + Name string + Items []model.MediaItem + Year int + Overview string + Cover string +} + +// isEmbySeriesItem 判断视频条目是否属于剧集。 +// 电视剧按目录扫描(IsFolder=true,带 Episodes 选集),或被刮削为 tv; +// 其余单文件视频一律按电影处理。仅看 video_type 会把目录型剧集误判为电影。 +func isEmbySeriesItem(it *model.MediaItem) bool { + return it.IsFolder || it.VideoType == "tv" +} + +func seriesKeyOf(item *model.MediaItem) string { + name := item.ScrapedName + if name == "" { + name = strings.TrimSuffix(stdpath.Base(mediaFolderOf(item)), "/") + } + return embyHash("series:" + strings.ToLower(name)) +} + +func seriesNameOf(item *model.MediaItem) string { + if item.ScrapedName != "" { + return item.ScrapedName + } + if item.IsFolder { + return strings.TrimSuffix(item.FileName, stdpath.Ext(item.FileName)) + } + return strings.TrimSuffix(stdpath.Base(mediaFolderOf(item)), "/") +} + +// seriesList 汇总剧集(按剧名去重) +func (l *embyLibrary) seriesList() []*embySeries { + index := map[string]*embySeries{} + var order []string + for i := range l.videos { + it := &l.videos[i] + if !isEmbySeriesItem(it) { + continue + } + key := seriesKeyOf(it) + s, ok := index[key] + if !ok { + s = &embySeries{Key: key, Name: seriesNameOf(it), Cover: it.Cover} + if len(it.ReleaseDate) >= 4 { + s.Year, _ = strconv.Atoi(it.ReleaseDate[:4]) + } + s.Overview = it.Plot + index[key] = s + order = append(order, key) + } + if s.Cover == "" { + s.Cover = it.Cover + } + if s.Overview == "" { + s.Overview = it.Plot + } + s.Items = append(s.Items, *it) + } + out := make([]*embySeries, 0, len(order)) + for _, k := range order { + out = append(out, index[k]) + } + return out +} + +func (l *embyLibrary) seriesByKey(key string) *embySeries { + for _, s := range l.seriesList() { + if s.Key == key { + return s + } + } + return nil +} + +type embyEpisode struct { + ID string + Item *model.MediaItem + Episode model.EpisodeInfo + Index int + Season int + SeriesKey string + SeriesName string +} + +func parseEpisodeList(item *model.MediaItem) []model.EpisodeInfo { + if item.Episodes == "" { + return nil + } + var eps []model.EpisodeInfo + if err := json.Unmarshal([]byte(item.Episodes), &eps); err != nil { + return nil + } + return eps +} + +// episodesOf 返回某部剧(可按季过滤)的集列表 +func (l *embyLibrary) episodesOf(seriesKey string, season int) []*embyEpisode { + var out []*embyEpisode + for i := range l.videos { + it := &l.videos[i] + if !isEmbySeriesItem(it) || seriesKeyOf(it) != seriesKey { + continue + } + s := it.Season + if s <= 0 { + s = 1 + } + if season > 0 && s != season { + continue + } + if it.IsFolder { + for idx, ep := range parseEpisodeList(it) { + num := ep.Index + if num <= 0 { + num = idx + 1 + } + out = append(out, &embyEpisode{ + ID: fmt.Sprintf("ep-%d-%d", it.ID, num), + Item: it, + Episode: ep, + Index: num, + Season: s, + SeriesKey: seriesKey, + SeriesName: seriesNameOf(it), + }) + } + continue + } + num := it.Episode + if num <= 0 { + num = 1 + } + out = append(out, &embyEpisode{ + ID: fmt.Sprintf("ep-%d-%d", it.ID, num), + Item: it, + Index: num, + Season: s, + SeriesKey: seriesKey, + SeriesName: seriesNameOf(it), + }) + } + sort.Slice(out, func(i, j int) bool { + if out[i].Season != out[j].Season { + return out[i].Season < out[j].Season + } + return out[i].Index < out[j].Index + }) + return out +} + +func (l *embyLibrary) findEpisode(id string) *embyEpisode { + // id: ep-- + parts := strings.Split(id, "-") + if len(parts) != 3 { + return nil + } + itemID, err := strconv.ParseUint(parts[1], 10, 64) + if err != nil { + return nil + } + num, _ := strconv.Atoi(parts[2]) + item, err := db.GetMediaItemByID(uint(itemID)) + if err != nil || !CanUserAccessItem(l.user, item) { + return nil + } + season := item.Season + if season <= 0 { + season = 1 + } + if item.IsFolder { + for i, ep := range parseEpisodeList(item) { + n := ep.Index + if n <= 0 { + n = i + 1 + } + if n == num { + return &embyEpisode{ID: id, Item: item, Episode: ep, Index: n, Season: season, + SeriesKey: seriesKeyOf(item), SeriesName: seriesNameOf(item)} + } + } + return nil + } + return &embyEpisode{ID: id, Item: item, Index: num, Season: season, + SeriesKey: seriesKeyOf(item), SeriesName: seriesNameOf(item)} +} + +// music 索引 + +type embyAlbum struct { + Key string + Name string + Artist string + Cover string + Year int + Items []model.MediaItem +} + +func albumArtistOf(item *model.MediaItem) string { + if item.AlbumArtist != "" { + return item.AlbumArtist + } + if item.Authors != "" { + return item.Authors + } + return "Unknown Artist" +} + +func albumKeyOf(item *model.MediaItem) string { + album := item.AlbumName + if album == "" { + album = "Unknown Album" + } + return embyHash("album:" + strings.ToLower(albumArtistOf(item)) + "\x00" + strings.ToLower(album)) +} + +func artistKeyOf(item *model.MediaItem) string { + return embyHash("artist:" + strings.ToLower(albumArtistOf(item))) +} + +func (l *embyLibrary) albums() []*embyAlbum { + index := map[string]*embyAlbum{} + var order []string + for i := range l.music { + it := &l.music[i] + key := albumKeyOf(it) + al, ok := index[key] + if !ok { + name := it.AlbumName + if name == "" { + name = "Unknown Album" + } + al = &embyAlbum{Key: key, Name: name, Artist: albumArtistOf(it), Cover: it.Cover} + if len(it.ReleaseDate) >= 4 { + al.Year, _ = strconv.Atoi(it.ReleaseDate[:4]) + } + index[key] = al + order = append(order, key) + } + if al.Cover == "" { + al.Cover = it.Cover + } + al.Items = append(al.Items, *it) + } + out := make([]*embyAlbum, 0, len(order)) + for _, k := range order { + out = append(out, index[k]) + } + sort.Slice(out, func(i, j int) bool { return strings.ToLower(out[i].Name) < strings.ToLower(out[j].Name) }) + return out +} + +func (l *embyLibrary) albumByKey(key string) *embyAlbum { + for _, a := range l.albums() { + if a.Key == key { + return a + } + } + return nil +} + +// ==================== DTO 构造 ==================== + +func (l *embyLibrary) movieItem(it *model.MediaItem) *embyItem { + year := 0 + if len(it.ReleaseDate) >= 4 { + year, _ = strconv.Atoi(it.ReleaseDate[:4]) + } + name := it.ScrapedName + if name == "" { + name = strings.TrimSuffix(it.FileName, stdpath.Ext(it.FileName)) + } + e := &embyItem{ + Name: name, + SortName: name, + ServerId: embyServerID(), + Id: l.guid(fmt.Sprintf("mv-%d", it.ID)), + ParentId: l.guid("lib-movie"), + Type: "Movie", + MediaType: "Video", + IsFolder: false, + Path: MediaItemFilePath(it), + Container: embyExt(it.FileName), + RunTimeTicks: embyTicks(it.Duration), + ProductionYear: year, + Overview: it.Plot, + CommunityRating: it.Rating, + PremiereDate: it.ReleaseDate, + DateCreated: embyTimeStr(it.CreatedAt), + UserData: l.userDataOf(it.ID, 0), + ImageTags: map[string]string{}, + ProviderIds: map[string]string{}, + LocationType: "FileSystem", + } + if tag := embyImageTag(it); tag != "" { + e.ImageTags["Primary"] = tag + } + if it.ExternalID != "" { + e.ProviderIds["Tmdb"] = it.ExternalID + } + return e +} + +func (l *embyLibrary) seriesItem(s *embySeries) *embyItem { + e := &embyItem{ + Name: s.Name, + SortName: s.Name, + ServerId: embyServerID(), + Id: l.guid("sr-" + s.Key), + ParentId: l.guid("lib-tv"), + Type: "Series", + IsFolder: true, + ProductionYear: s.Year, + Overview: s.Overview, + DateCreated: embyTimeStr(s.Items[0].CreatedAt), + ChildCount: l.seasonCount(s), + LocationType: "FileSystem", + ImageTags: map[string]string{}, + ProviderIds: map[string]string{}, + UserData: &embyUserData{}, + } + if tag := embyImageTag(&s.Items[0]); tag != "" { + e.ImageTags["Primary"] = tag + } + if s.Items[0].ExternalID != "" { + e.ProviderIds["Tmdb"] = s.Items[0].ExternalID + } + return e +} + +func (l *embyLibrary) seasonCount(s *embySeries) int { + seasons := map[int]bool{} + for i := range s.Items { + season := s.Items[i].Season + if season <= 0 { + season = 1 + } + seasons[season] = true + } + return len(seasons) +} + +func (l *embyLibrary) seasonList(s *embySeries) []int { + set := map[int]bool{} + for i := range s.Items { + season := s.Items[i].Season + if season <= 0 { + season = 1 + } + set[season] = true + } + out := make([]int, 0, len(set)) + for n := range set { + out = append(out, n) + } + sort.Ints(out) + return out +} + +func (l *embyLibrary) seasonItem(s *embySeries, season int) *embyItem { + eps := l.episodesOf(s.Key, season) + unplayed := 0 + for _, ep := range eps { + if !l.userDataOf(ep.Item.ID, ep.Index).Played { + unplayed++ + } + } + e := &embyItem{ + Name: fmt.Sprintf("Season %d", season), + SortName: fmt.Sprintf("Season %02d", season), + ServerId: embyServerID(), + Id: l.guid(fmt.Sprintf("sn-%s-%d", s.Key, season)), + Type: "Season", + IsFolder: true, + ParentId: l.guid("sr-" + s.Key), + SeriesId: l.guid("sr-" + s.Key), + SeriesName: s.Name, + IndexNumber: season, + ChildCount: len(eps), + LocationType: "FileSystem", + ImageTags: map[string]string{}, + UserData: &embyUserData{UnplayedItemCount: unplayed}, + } + if len(s.Items) > 0 { + if tag := embyImageTag(&s.Items[0]); tag != "" { + e.ImageTags["Primary"] = tag + } + } + return e +} + +func (l *embyLibrary) episodeItem(ep *embyEpisode) *embyItem { + title := ep.Episode.Title + if title == "" { + title = fmt.Sprintf("Episode %d", ep.Index) + } + seriesID := "sr-" + ep.SeriesKey + seasonID := fmt.Sprintf("sn-%s-%d", ep.SeriesKey, ep.Season) + rawPath := MediaItemFilePath(ep.Item) + if ep.Episode.FileName != "" { + rawPath = stdpath.Join(mediaFolderOf(ep.Item), ep.Item.FileName, ep.Episode.FileName) + } + e := &embyItem{ + Name: title, + SortName: fmt.Sprintf("%02d-%03d", ep.Season, ep.Index), + ServerId: embyServerID(), + Id: l.guid(ep.ID), + Type: "Episode", + MediaType: "Video", + IsFolder: false, + ParentId: l.guid(seasonID), + SeriesId: l.guid(seriesID), + SeriesName: ep.SeriesName, + SeasonId: l.guid(seasonID), + SeasonName: fmt.Sprintf("Season %d", ep.Season), + IndexNumber: ep.Index, + ParentIndexNumber: ep.Season, + Path: rawPath, + Container: embyExt(rawPath), + RunTimeTicks: embyTicks(ep.Item.Duration), + Overview: ep.Item.Plot, + DateCreated: embyTimeStr(ep.Item.CreatedAt), + UserData: l.userDataOf(ep.Item.ID, ep.Index), + ImageTags: map[string]string{}, + ProviderIds: map[string]string{}, + LocationType: "FileSystem", + } + if tag := embyImageTag(ep.Item); tag != "" { + e.ImageTags["Primary"] = tag + } + return e +} + +func (l *embyLibrary) albumItem(al *embyAlbum) *embyItem { + e := &embyItem{ + Name: al.Name, + SortName: al.Name, + ServerId: embyServerID(), + Id: l.guid("al-" + al.Key), + ParentId: l.guid("lib-music"), + Type: "MusicAlbum", + IsFolder: true, + AlbumArtist: al.Artist, + AlbumArtists: []embyNameID{{Name: al.Artist, Id: l.guid("ar-" + artistKeyOf(&al.Items[0]))}}, + ProductionYear: al.Year, + ChildCount: len(al.Items), + DateCreated: embyTimeStr(al.Items[0].CreatedAt), + LocationType: "FileSystem", + ImageTags: map[string]string{}, + UserData: &embyUserData{}, + } + if tag := embyImageTag(&al.Items[0]); tag != "" { + e.ImageTags["Primary"] = tag + } + return e +} + +func (l *embyLibrary) songItem(it *model.MediaItem) *embyItem { + name := it.ScrapedName + if name == "" { + name = strings.TrimSuffix(it.FileName, stdpath.Ext(it.FileName)) + } + albumName := it.AlbumName + if albumName == "" { + albumName = "Unknown Album" + } + artist := albumArtistOf(it) + albumKey := albumKeyOf(it) + artistKey := artistKeyOf(it) + e := &embyItem{ + Name: name, + SortName: name, + ServerId: embyServerID(), + Id: l.guid(fmt.Sprintf("so-%d", it.ID)), + Type: "Audio", + MediaType: "Audio", + IsFolder: false, + ParentId: l.guid("al-" + albumKey), + Album: albumName, + AlbumId: l.guid("al-" + albumKey), + AlbumArtist: artist, + AlbumArtists: []embyNameID{{Name: artist, Id: l.guid("ar-" + artistKey)}}, + Artists: []string{artist}, + ArtistItems: []embyNameID{{Name: artist, Id: l.guid("ar-" + artistKey)}}, + IndexNumber: it.TrackNumber, + RunTimeTicks: embyTicks(it.Duration), + Path: MediaItemFilePath(it), + Container: embyExt(it.FileName), + DateCreated: embyTimeStr(it.CreatedAt), + CommunityRating: it.Rating, + UserData: l.userDataOf(it.ID, 0), + ImageTags: map[string]string{}, + ProviderIds: map[string]string{}, + LocationType: "FileSystem", + } + if tag := embyImageTag(it); tag != "" { + e.ImageTags["Primary"] = tag + } + return e +} + +func (l *embyLibrary) songByID(id string) (*model.MediaItem, bool) { + id = l.origID(id) + n, err := strconv.ParseUint(strings.TrimPrefix(id, "so-"), 10, 64) + if err != nil { + return nil, false + } + for i := range l.music { + if l.music[i].ID == uint(n) { + return &l.music[i], true + } + } + item, err := db.GetMediaItemByID(uint(n)) + if err != nil || !CanUserAccessItem(l.user, item) { + return nil, false + } + return item, true +} + +// ==================== 视图 / 列表 ==================== + +func (l *embyLibrary) viewItems() []*embyItem { + movieCount, seriesCount := 0, 0 + for i := range l.videos { + if isEmbySeriesItem(&l.videos[i]) { + continue + } + movieCount++ + } + seriesCount = len(l.seriesList()) + albumCount := len(l.albums()) + + var out []*embyItem + if MediaTypeEnabled(model.MediaTypeVideo) { + out = append(out, &embyItem{ + Name: "Movies", SortName: "Movies", ServerId: embyServerID(), + Id: l.guid("lib-movie"), ParentId: l.guid("root"), Type: "CollectionFolder", IsFolder: true, + CollectionType: "movies", ViewType: "movies", LocationType: "FileSystem", + ChildCount: movieCount, RecursiveItemCount: movieCount, + // 只声明客户端能正常处理的子视图,避免生成 OpenList 没有实现的 tab + Subviews: []string{"movies"}, + ImageTags: map[string]string{}, UserData: &embyUserData{}, + }) + out = append(out, &embyItem{ + Name: "TV Shows", SortName: "TV Shows", ServerId: embyServerID(), + Id: l.guid("lib-tv"), ParentId: l.guid("root"), Type: "CollectionFolder", IsFolder: true, + CollectionType: "tvshows", ViewType: "tvshows", LocationType: "FileSystem", + ChildCount: seriesCount, RecursiveItemCount: seriesCount, + ImageTags: map[string]string{}, UserData: &embyUserData{}, + }) + } + if MediaTypeEnabled(model.MediaTypeMusic) { + out = append(out, &embyItem{ + Name: "Music", SortName: "Music", ServerId: embyServerID(), + Id: l.guid("lib-music"), ParentId: l.guid("root"), Type: "CollectionFolder", IsFolder: true, + CollectionType: "music", ViewType: "music", LocationType: "FileSystem", + ChildCount: albumCount, RecursiveItemCount: albumCount, + ImageTags: map[string]string{}, UserData: &embyUserData{}, + }) + } + return out +} + +func embyViews(c *gin.Context, user *model.User) { + lib := loadEmbyLibrary(user) + items := lib.viewItems() + c.JSON(200, gin.H{"Items": items, "TotalRecordCount": len(items), "StartIndex": 0}) +} + +// embyHomeSections 返回服务端首页分区配置(Emby 4.10+ 客户端直接使用)。 +// ListOptions.fields 必须存在:客户端会无条件调用 fields.includes()。 +func embyHomeSections() []gin.H { + return []gin.H{ + { + "Id": "userviews", + "SectionType": "userviews", + "Name": "Media", + "CollectionType": "", + "ViewType": "cards", + "ScrollDirection": "Horizontal", + "LimitItems": false, + "ListOptions": gin.H{ + "fields": []string{}, + "shape": "auto", + "context": "home", + "preferThumb": false, + "preferSeriesImage": false, + "showChildCountIndicator": false, + "image": nil, + "sideFooter": nil, + "smallSideFooter": nil, + "showCurrentProgramImage": false, + "showAirDateTime": false, + "showAirEndTime": false, + "defaultShape": nil, + "action": nil, + "multiSelect": false, + "albumFirst": false, + "lines": 1, + "focusTransformTitleAdjust": true, + "contextMenu": true, + "hoverMenu": true, + "hoverPlayButton": false, + }, + "QueryOptions": gin.H{}, + }, + } +} + +// embySectionItems 返回首页某个分区的条目 +func embySectionItems(user *model.User, sectionID string) gin.H { + if sectionID == "userviews" { + lib := loadEmbyLibrary(user) + items := lib.viewItems() + return gin.H{"Items": items, "TotalRecordCount": len(items), "StartIndex": 0} + } + return gin.H{"Items": []any{}, "TotalRecordCount": 0, "StartIndex": 0} +} + +func (l *embyLibrary) rootItem() *embyItem { + return &embyItem{ + Name: "Media", SortName: "Media", ServerId: embyServerID(), + Id: l.guid("root"), Type: "CollectionFolder", IsFolder: true, + LocationType: "FileSystem", ImageTags: map[string]string{}, UserData: &embyUserData{}, + } +} + +// childrenOf 返回指定父级下的子项 +func (l *embyLibrary) childrenOf(parentID string) []*embyItem { + parentID = l.origID(parentID) + switch { + case parentID == "" || parentID == "root": + return l.viewItems() + case parentID == "lib-movie": + var out []*embyItem + for i := range l.videos { + if !isEmbySeriesItem(&l.videos[i]) { + out = append(out, l.movieItem(&l.videos[i])) + } + } + return out + case parentID == "lib-tv": + var out []*embyItem + for _, s := range l.seriesList() { + out = append(out, l.seriesItem(s)) + } + return out + case parentID == "lib-music": + var out []*embyItem + for _, a := range l.albums() { + out = append(out, l.albumItem(a)) + } + return out + } + if strings.HasPrefix(parentID, "sr-") { + key := strings.TrimPrefix(parentID, "sr-") + s := l.seriesByKey(key) + if s == nil { + return nil + } + var out []*embyItem + for _, season := range l.seasonList(s) { + out = append(out, l.seasonItem(s, season)) + } + return out + } + if strings.HasPrefix(parentID, "sn-") { + // sn-- + rest := strings.TrimPrefix(parentID, "sn-") + idx := strings.LastIndex(rest, "-") + if idx < 0 { + return nil + } + key := rest[:idx] + season, _ := strconv.Atoi(rest[idx+1:]) + var out []*embyItem + for _, ep := range l.episodesOf(key, season) { + out = append(out, l.episodeItem(ep)) + } + return out + } + if strings.HasPrefix(parentID, "al-") { + al := l.albumByKey(strings.TrimPrefix(parentID, "al-")) + if al == nil { + return nil + } + var out []*embyItem + for i := range al.Items { + out = append(out, l.songItem(&al.Items[i])) + } + return out + } + if strings.HasPrefix(parentID, "ar-") { + key := strings.TrimPrefix(parentID, "ar-") + var out []*embyItem + for _, al := range l.albums() { + if artistKeyOf(&al.Items[0]) == key { + out = append(out, l.albumItem(al)) + } + } + return out + } + return nil +} + +// allItems 返回媒体库全部条目(用于搜索/递归) +func (l *embyLibrary) allItems() []*embyItem { + var out []*embyItem + for i := range l.videos { + if isEmbySeriesItem(&l.videos[i]) { + continue + } + out = append(out, l.movieItem(&l.videos[i])) + } + for _, s := range l.seriesList() { + out = append(out, l.seriesItem(s)) + } + for i := range l.music { + out = append(out, l.songItem(&l.music[i])) + } + return out +} + +// embySplitCSV 解析 Emby 的逗号分隔参数,同时兼容 | 分隔 +func embySplitCSV(v string) []string { + var out []string + for _, p := range strings.FieldsFunc(v, func(r rune) bool { return r == ',' || r == '|' }) { + if p = strings.TrimSpace(p); p != "" { + out = append(out, p) + } + } + return out +} + +func embyInList(list []string, v string) bool { + for _, s := range list { + if strings.EqualFold(s, v) { + return true + } + } + return false +} + +// filterEmbyItems 实现 /Items 查询的通用过滤,覆盖 openemby_tv / Theater / Tsukimi 实际会发送的参数: +// IncludeItemTypes、ExcludeItemTypes、MediaTypes、Ids、IsFolder、SearchTerm、NameStartsWith、Filters +// allItemsFor 按请求的 IncludeItemTypes 只构造需要的类型。 +// 客户端无 ParentId 的递归查询(如 Type=Episode / MusicAlbum)必须能返回对应类型, +// 且不应把其他媒体类型混进来。 +func (l *embyLibrary) allItemsFor(include []string) []*embyItem { + if len(include) == 0 { + return l.allItems() + } + want := func(t string) bool { return embyInList(include, t) } + var out []*embyItem + if want("Movie") { + for i := range l.videos { + if !isEmbySeriesItem(&l.videos[i]) { + out = append(out, l.movieItem(&l.videos[i])) + } + } + } + if want("Series") || want("Season") || want("Episode") { + for _, s := range l.seriesList() { + if want("Series") { + out = append(out, l.seriesItem(s)) + } + if want("Season") { + for _, season := range l.seasonList(s) { + out = append(out, l.seasonItem(s, season)) + } + } + if want("Episode") { + for _, ep := range l.episodesOf(s.Key, 0) { + out = append(out, l.episodeItem(ep)) + } + } + } + } + if want("Audio") { + for i := range l.music { + out = append(out, l.songItem(&l.music[i])) + } + } + if want("MusicAlbum") { + for _, a := range l.albums() { + out = append(out, l.albumItem(a)) + } + } + return out +} + +func filterEmbyItems(items []*embyItem, c *gin.Context) []*embyItem { + include := embySplitCSV(c.Query("IncludeItemTypes")) + exclude := embySplitCSV(c.Query("ExcludeItemTypes")) + mediaTypes := embySplitCSV(c.Query("MediaTypes")) + searchTerm := strings.ToLower(strings.TrimSpace(c.Query("SearchTerm"))) + namePrefix := strings.ToLower(strings.TrimSpace(c.Query("NameStartsWith"))) + filters := strings.ToLower(c.Query("Filters")) + isFolder := c.Query("IsFolder") + idSet := map[string]bool{} + for _, id := range embySplitCSV(c.Query("Ids")) { + idSet[strings.ToLower(id)] = true + } + + var out []*embyItem + for _, it := range items { + if len(include) > 0 && !embyInList(include, it.Type) { + continue + } + if len(exclude) > 0 && embyInList(exclude, it.Type) { + continue + } + if len(mediaTypes) > 0 && !embyInList(mediaTypes, it.MediaType) { + continue + } + if len(idSet) > 0 && !idSet[strings.ToLower(it.Id)] { + continue + } + lname := strings.ToLower(it.Name) + if searchTerm != "" && !strings.Contains(lname, searchTerm) { + continue + } + if namePrefix != "" && !strings.HasPrefix(lname, namePrefix) { + continue + } + if isFolder != "" { + want := strings.EqualFold(isFolder, "true") + if it.IsFolder != want { + continue + } + } + if strings.Contains(filters, "isfavorite") { + if it.UserData == nil || !it.UserData.IsFavorite { + continue + } + } + if strings.Contains(filters, "isunplayed") { + if it.UserData != nil && it.UserData.Played { + continue + } + } else if strings.Contains(filters, "isplayed") { + if it.UserData == nil || !it.UserData.Played { + continue + } + } + out = append(out, it) + } + return out +} + +func embyTypeMatches(itemType, wanted string) bool { + return embyInList(embySplitCSV(wanted), itemType) +} + +// sortEmbyItems 支持多个排序键(SortBy=SeriesSortName,ParentIndexNumber,IndexNumber,SortName) +func sortEmbyItems(items []*embyItem, sortBy, order string) { + if sortBy == "" { + return + } + keys := embySplitCSV(sortBy) + desc := strings.EqualFold(order, "Descending") + if len(keys) > 0 && strings.EqualFold(keys[0], "random") { + for i := len(items) - 1; i > 0; i-- { + j := int(timeNowUnixNano() % int64(i+1)) + items[i], items[j] = items[j], items[i] + } + return + } + cmp := func(a, b *embyItem, key string) int { + switch strings.ToLower(key) { + case "sortname", "name", "seriessortname": + la, lb := strings.ToLower(a.SortName), strings.ToLower(b.SortName) + if strings.EqualFold(key, "seriessortname") { + la, lb = strings.ToLower(a.SeriesName), strings.ToLower(b.SeriesName) + } + switch { + case la < lb: + return -1 + case la > lb: + return 1 + } + case "datecreated", "dateplayed": + switch { + case a.DateCreated < b.DateCreated: + return -1 + case a.DateCreated > b.DateCreated: + return 1 + } + case "productionyear": + return a.ProductionYear - b.ProductionYear + case "parentindexnumber": + return a.ParentIndexNumber - b.ParentIndexNumber + case "indexnumber": + return a.IndexNumber - b.IndexNumber + case "communityrating": + switch { + case a.CommunityRating < b.CommunityRating: + return -1 + case a.CommunityRating > b.CommunityRating: + return 1 + } + } + return 0 + } + sort.SliceStable(items, func(i, j int) bool { + for _, k := range keys { + if r := cmp(items[i], items[j], k); r != 0 { + if desc { + return r > 0 + } + return r < 0 + } + } + return false + }) +} + +func embyPaginate(items []*embyItem, start, limit int) []*embyItem { + if start >= len(items) { + return []*embyItem{} + } + end := start + limit + if end > len(items) { + end = len(items) + } + return items[start:end] +} + +func timeNowUnixNano() int64 { + return time.Now().UnixNano() +} + +// ==================== 条目分发 ==================== + +// embyUserScoped 处理 /Users/{uid}/... 下的条目相关请求 +func embyUserScoped(c *gin.Context, user *model.User, segs, lower []string, method string) { + // 服务端首页分区(Emby 4.10+ 客户端) + if len(lower) >= 3 && lower[2] == "homesections" { + if method == "GET" && len(lower) == 3 { + c.JSON(200, embyHomeSections()) + } else { + c.Status(204) + } + return + } + // 首页分区条目:/Users/{uid}/Sections/{sectionId}/Items + if len(lower) == 5 && lower[2] == "sections" && lower[4] == "items" { + c.JSON(200, embySectionItems(user, lower[3])) + return + } + // /users/{uid}/items... + if len(lower) >= 3 && lower[2] == "items" { + lib := loadEmbyLibrary(user) + switch { + case len(lower) == 3 && (method == "GET" || method == "POST"): + // /Users/{uid}/Items?ParentId=...&Recursive=...&IncludeItemTypes=... + // 必须按查询条件返回真实数据;Playlist/BoxSet 等无对应类型会被过滤为空 + embyItemList(c, lib, c.Query("ParentId")) + case len(lower) == 4 && lower[3] == "root": + c.JSON(200, lib.rootItem()) + case len(lower) == 4 && lower[3] == "latest": + embyLatest(c, lib) + case len(lower) == 4 && lower[3] == "resume": + embyResume(c, lib) + case len(lower) == 4 && method == "GET": + // 单条目 + embyGetItem(c, lib, segs[3]) + case len(lower) == 5 && lower[4] == "children": + embyChildren(c, lib, segs[3]) + case len(lower) == 5 && lower[4] == "playbackinfo": + embyPlaybackInfo(c, lib, segs[3]) + case len(lower) == 5 && lower[4] == "userdata": + embyUpdateUserData(c, user, segs[3]) + case len(lower) == 4: + embyItemList(c, lib, segs[3]) + case len(lower) == 5 && embyAuxItem(c, lower[4]): + default: + embyNotFound(c) + } + return + } + if len(lower) == 3 && embyAuxItem(c, lower[2]) { + return + } + // /users/{uid}/settings:客户端保存/读取首页与播放设置 + if len(lower) == 3 && lower[2] == "settings" { + if method == "GET" { + c.JSON(200, gin.H{"HomeSections": []any{}}) + } else { + c.Status(204) + } + return + } + // /users/{uid}/views 已在外部处理 + if len(lower) >= 3 && (lower[2] == "playeditems" || lower[2] == "favoriteitems" || lower[2] == "playingitems") { + if len(lower) < 4 { + embyNotFound(c) + return + } + kind := lower[2] + id := segs[3] + action := "" + if len(lower) == 5 { + action = lower[4] + } + switch kind { + case "playeditems": + played := method == "POST" && action != "delete" + embySetPlayed(c, user, id, played) + case "favoriteitems": + fav := method == "POST" && action != "delete" + embySetFavorite(c, user, id, fav) + case "playingitems": + if len(lower) == 5 && lower[4] == "progress" { + embySaveProgress(c, user, id, false) + return + } + embySaveProgress(c, user, id, true) + } + return + } + embyNotFound(c) +} + +// embyItemsScoped 处理 /Items/... 全局请求 +func embyItemsScoped(c *gin.Context, user *model.User, segs, lower []string, method string) { + lib := loadEmbyLibrary(user) + switch { + case len(lower) == 2 && lower[1] == "counts": + embyItemCounts(c, user) + case len(lower) == 2 && method == "GET": + embyGetItem(c, lib, segs[1]) + case len(lower) == 3 && lower[2] == "children": + embyChildren(c, lib, segs[1]) + case len(lower) == 3 && lower[2] == "playbackinfo": + embyPlaybackInfo(c, lib, segs[1]) + case len(lower) == 3 && lower[2] == "refresh" && method == "POST": + embyRefreshItem(c, lib.origID(segs[1])) + // 歌词:/Items/{id}/{mediaSourceId}/Subtitles/{index}/Stream.js + case len(lower) >= 4 && lower[3] == "subtitles": + embySubtitleStream(c, user, segs[1]) + case len(lower) >= 3 && lower[2] == "subtitles": + embySubtitleStream(c, user, segs[1]) + case len(lower) >= 3 && lower[2] == "images": + embyImage(c, segs[1], segs, true) + case len(lower) == 3 && embyAuxItem(c, lower[2]): + default: + embyNotFound(c) + } +} + +func embyGetItem(c *gin.Context, lib *embyLibrary, id string) { + item, ok := lib.resolveItem(id) + if !ok { + embyNotFound(c) + return + } + // 有歌词的歌曲在详情里也带上媒体源:客户端歌词面板读取 + // item.MediaSources[0].DefaultSubtitleStreamIndex 来定位歌词轨 + if item.Type == "Audio" { + if song, ok := lib.songByID(lib.origID(id)); ok && song.Lyrics != "" { + idx := 1 + item.MediaSources = []embyMediaSource{{ + Protocol: "Http", Id: item.Id, ItemId: item.Id, Type: "Default", + SupportsDirectPlay: true, SupportsDirectStream: true, + MediaStreams: []embyMediaStream{ + {Type: "Audio", Index: 0, IsDefault: true}, + embyLyricsMediaStream(idx), + }, + MediaAttachments: []any{}, Formats: []string{}, + RequiredHttpHeaders: map[string]string{}, + DefaultSubtitleStreamIndex: &idx, + }} + } + } + c.JSON(200, item) +} + +func (l *embyLibrary) resolveItem(id string) (*embyItem, bool) { + id = l.origID(id) + switch { + case strings.HasPrefix(id, "mv-"): + n, err := strconv.ParseUint(strings.TrimPrefix(id, "mv-"), 10, 64) + if err != nil { + return nil, false + } + for i := range l.videos { + if l.videos[i].ID == uint(n) { + return l.movieItem(&l.videos[i]), true + } + } + case strings.HasPrefix(id, "so-"): + if it, ok := l.songByID(id); ok { + return l.songItem(it), true + } + case strings.HasPrefix(id, "sr-"): + if s := l.seriesByKey(strings.TrimPrefix(id, "sr-")); s != nil { + return l.seriesItem(s), true + } + case strings.HasPrefix(id, "sn-"): + rest := strings.TrimPrefix(id, "sn-") + idx := strings.LastIndex(rest, "-") + if idx > 0 { + key := rest[:idx] + season, _ := strconv.Atoi(rest[idx+1:]) + if s := l.seriesByKey(key); s != nil { + return l.seasonItem(s, season), true + } + } + case strings.HasPrefix(id, "al-"): + if al := l.albumByKey(strings.TrimPrefix(id, "al-")); al != nil { + return l.albumItem(al), true + } + case strings.HasPrefix(id, "ep-"): + if ep := l.findEpisode(id); ep != nil { + return l.episodeItem(ep), true + } + case strings.HasPrefix(id, "ar-"): + key := strings.TrimPrefix(id, "ar-") + for i := range l.music { + if artistKeyOf(&l.music[i]) != key { + continue + } + name := albumArtistOf(&l.music[i]) + e := &embyItem{ + Name: name, SortName: name, ServerId: embyServerID(), + Id: l.guid("ar-" + key), ParentId: l.guid("lib-music"), + Type: "MusicArtist", IsFolder: true, + ImageTags: map[string]string{}, UserData: &embyUserData{}, + LocationType: "FileSystem", + } + if tag := embyImageTag(&l.music[i]); tag != "" { + e.ImageTags["Primary"] = tag + } + return e, true + } + case strings.HasPrefix(id, "lib-") || id == "root": + for _, v := range l.viewItems() { + if strings.EqualFold(v.Id, l.guid(id)) { + return v, true + } + } + if id == "root" { + return l.rootItem(), true + } + } + return nil, false +} + +// descendantsOf 递归展开 parentID 下的全部后代条目。 +// 递归必须以 ParentId 为范围:此前直接返回全库条目,导致电影/剧集/音乐库互相串内容。 +func (l *embyLibrary) descendantsOf(parentID string) []*embyItem { + var out []*embyItem + var walk func(id string, depth int) + walk = func(id string, depth int) { + if depth > 6 { + return + } + for _, ch := range l.childrenOf(id) { + out = append(out, ch) + if ch.IsFolder { + walk(ch.Id, depth+1) + } + } + } + walk(parentID, 0) + return out +} + +func embyItemList(c *gin.Context, lib *embyLibrary, parentID string) { + if q := strings.TrimSpace(c.Query("ParentId")); q != "" { + parentID = q + } + var items []*embyItem + if embyBoolQuery(c, "Recursive") { + if lib.origID(parentID) == "" || lib.origID(parentID) == "root" { + items = lib.allItemsFor(embySplitCSV(c.Query("IncludeItemTypes"))) + } else { + items = lib.descendantsOf(parentID) + } + } else { + items = lib.childrenOf(parentID) + } + items = filterEmbyItems(items, c) + sortEmbyItems(items, c.Query("SortBy"), c.Query("SortOrder")) + total := len(items) + start, limit := embyPaging(c) + items = embyPaginate(items, start, limit) + c.JSON(200, embyQueryResult{Items: items, TotalRecordCount: total, StartIndex: start}) +} + +func embyChildren(c *gin.Context, lib *embyLibrary, parentID string) { + items := lib.childrenOf(parentID) + items = filterEmbyItems(items, c) + start, limit := embyPaging(c) + total := len(items) + c.JSON(200, embyQueryResult{Items: embyPaginate(items, start, limit), TotalRecordCount: total, StartIndex: start}) +} + +func embyLatest(c *gin.Context, lib *embyLibrary) { + // 客户端按媒体库分别请求"最近添加":带 ParentId 时只返回该库内容,避免电影/电视剧/音乐互相串 + if pid := lib.origID(c.Query("ParentId")); pid == "lib-movie" || pid == "lib-tv" || pid == "lib-music" { + items := filterEmbyItems(lib.childrenOf(pid), c) + sort.SliceStable(items, func(i, j int) bool { return items[i].DateCreated > items[j].DateCreated }) + _, limit := embyPaging(c) + c.JSON(200, embyPaginate(items, 0, limit)) + return + } + var items []*embyItem + if MediaTypeEnabled(model.MediaTypeVideo) { + var movies []model.MediaItem + for i := range lib.videos { + if !isEmbySeriesItem(&lib.videos[i]) { + movies = append(movies, lib.videos[i]) + } + } + sort.Slice(movies, func(i, j int) bool { return movies[i].CreatedAt.After(movies[j].CreatedAt) }) + for i := range movies { + items = append(items, lib.movieItem(&movies[i])) + if len(items) >= 50 { + break + } + } + } + if MediaTypeEnabled(model.MediaTypeMusic) { + sort.Slice(lib.music, func(i, j int) bool { return lib.music[i].CreatedAt.After(lib.music[j].CreatedAt) }) + for i := range lib.music { + items = append(items, lib.songItem(&lib.music[i])) + if len(items) >= 100 { + break + } + } + } + _, limit := embyPaging(c) + c.JSON(200, embyPaginate(items, 0, limit)) +} + +func embyResume(c *gin.Context, lib *embyLibrary) { + if lib.user == nil { + c.JSON(200, []*embyItem{}) + return + } + rows, err := db.ListMediaResume(lib.user.ID, 100) + if err != nil { + c.JSON(200, []*embyItem{}) + return + } + var items []*embyItem + for _, row := range rows { + id := fmt.Sprintf("mv-%d", row.ItemID) + if row.EpisodeIndex > 0 { + id = fmt.Sprintf("ep-%d-%d", row.ItemID, row.EpisodeIndex) + } + if it, ok := lib.resolveItem(id); ok { + if it.UserData == nil { + it.UserData = &embyUserData{} + } + it.UserData.PlaybackPositionTicks = row.PositionTicks + items = append(items, it) + } + } + c.JSON(200, items) +} + +func embyItemCounts(c *gin.Context, user *model.User) { + lib := loadEmbyLibrary(user) + movies, series, episodes, songs, albums := 0, 0, 0, 0, 0 + for i := range lib.videos { + if isEmbySeriesItem(&lib.videos[i]) { + continue + } + movies++ + } + for range lib.seriesList() { + series++ + } + for _, s := range lib.seriesList() { + episodes += len(lib.episodesOf(s.Key, 0)) + } + songs = len(lib.music) + albums = len(lib.albums()) + c.JSON(200, gin.H{ + "MovieCount": movies, "SeriesCount": series, "EpisodeCount": episodes, + "SongCount": songs, "AlbumCount": albums, + "ArtistCount": len(embyArtistKeys(lib)), "BookCount": 0, "TrailerCount": 0, + }) +} + +// ==================== 剧集接口 ==================== + +func embyShows(c *gin.Context, user *model.User, segs, lower []string, method string) { + lib := loadEmbyLibrary(user) + // /Shows/NextUp 只有两段路径 + if len(lower) == 2 && lower[1] == "nextup" { + embyNextUp(c, lib) + return + } + // /Shows/{id} 返回剧集详情 + if len(lower) == 2 { + if item, ok := lib.resolveItem(segs[1]); ok { + c.JSON(200, item) + return + } + embyNotFound(c) + return + } + seriesID := lib.origID(segs[1]) + key := strings.TrimPrefix(seriesID, "sr-") + s := lib.seriesByKey(key) + if s == nil { + embyNotFound(c) + return + } + switch lower[2] { + case "seasons": + var items []*embyItem + for _, season := range lib.seasonList(s) { + items = append(items, lib.seasonItem(s, season)) + } + c.JSON(200, gin.H{"Items": items, "TotalRecordCount": len(items), "StartIndex": 0}) + case "episodes": + season := embyIntQuery(c, "Season") + // 客户端也会传 SeasonId(GUID)按季过滤 + if sid := lib.origID(c.Query("SeasonId")); strings.HasPrefix(sid, "sn-") { + if idx := strings.LastIndex(sid, "-"); idx > 0 { + season, _ = strconv.Atoi(sid[idx+1:]) + } + } + var items []*embyItem + for _, ep := range lib.episodesOf(key, season) { + items = append(items, lib.episodeItem(ep)) + } + c.JSON(200, gin.H{"Items": items, "TotalRecordCount": len(items), "StartIndex": 0}) + default: + embyNotFound(c) + } +} + +func embyNextUp(c *gin.Context, lib *embyLibrary) { + var items []*embyItem + seriesFilter := strings.TrimPrefix(lib.origID(c.Query("SeriesId")), "sr-") + for _, s := range lib.seriesList() { + if seriesFilter != "" && s.Key != seriesFilter { + continue + } + if lib.user != nil { + if rows, err := db.ListMediaResume(lib.user.ID, 200); err == nil { + nextIdx := -1 + for _, row := range rows { + if seriesKeyOf(&model.MediaItem{ScrapedName: s.Name}) == s.Key && + row.ItemID == s.Items[0].ID && row.EpisodeIndex > 0 { + nextIdx = row.EpisodeIndex + 1 + break + } + } + eps := lib.episodesOf(s.Key, 0) + if nextIdx > 0 { + for _, ep := range eps { + if ep.Index == nextIdx { + items = append(items, lib.episodeItem(ep)) + break + } + } + } else if len(eps) > 0 { + items = append(items, lib.episodeItem(eps[0])) + } + continue + } + } + eps := lib.episodesOf(s.Key, 0) + if len(eps) > 0 { + items = append(items, lib.episodeItem(eps[0])) + } + } + c.JSON(200, gin.H{"Items": items, "TotalRecordCount": len(items)}) +} + +// ==================== 音乐 / 搜索 ==================== + +func embyArtistKeys(lib *embyLibrary) []string { + set := map[string]string{} + for i := range lib.music { + key := artistKeyOf(&lib.music[i]) + set[key] = albumArtistOf(&lib.music[i]) + } + out := make([]string, 0, len(set)) + for k := range set { + out = append(out, k) + } + sort.Strings(out) + return out +} + +func embyArtists(c *gin.Context, user *model.User, albumOnly bool) { + lib := loadEmbyLibrary(user) + seen := map[string]bool{} + var items []*embyItem + for i := range lib.music { + it := &lib.music[i] + key := artistKeyOf(it) + if seen[key] { + continue + } + seen[key] = true + name := albumArtistOf(it) + e := &embyItem{ + Name: name, SortName: name, ServerId: embyServerID(), + Id: lib.guid("ar-" + key), ParentId: lib.guid("lib-music"), Type: "MusicArtist", IsFolder: true, + ImageTags: map[string]string{}, UserData: &embyUserData{}, + LocationType: "FileSystem", + } + if tag := embyImageTag(it); tag != "" { + e.ImageTags["Primary"] = tag + } + items = append(items, e) + } + start, limit := embyPaging(c) + c.JSON(200, embyQueryResult{Items: embyPaginate(items, start, limit), TotalRecordCount: len(items), StartIndex: start}) +} + +func embySearchHints(c *gin.Context, user *model.User) { + lib := loadEmbyLibrary(user) + term := strings.ToLower(strings.TrimSpace(c.Query("SearchTerm"))) + items := lib.allItems() + items = filterEmbyItems(items, c) + var hints []gin.H + for _, it := range items { + if term != "" && !strings.Contains(strings.ToLower(it.Name), term) { + continue + } + hints = append(hints, gin.H{ + "ItemId": it.Id, + "Id": it.Id, + "Name": it.Name, + "Type": it.Type, + "MediaType": it.MediaType, + "RunTimeTicks": it.RunTimeTicks, + "ProductionYear": it.ProductionYear, + "ImageTags": it.ImageTags, + }) + if len(hints) >= 50 { + break + } + } + c.JSON(200, gin.H{"SearchHints": hints, "TotalRecordCount": len(hints)}) +} + +// ==================== 媒体库目录信息(MoviePilot 需要) ==================== + +func embySelectableFolders(c *gin.Context, user *model.User) { + paths, err := db.ListMediaScanPaths("") + if err != nil { + c.JSON(500, gin.H{"Error": err.Error()}) + return + } + var out []gin.H + for _, p := range paths { + if !CanUserAccessPath(user, p.Path) { + continue + } + out = append(out, gin.H{ + "Name": p.Name, + "SubFolders": []gin.H{{ + "Name": p.Name, + "Path": p.Path, + "Id": strconv.FormatUint(uint64(p.ID), 10), + }}, + }) + } + c.JSON(200, out) +} + +func embyVirtualFolders(c *gin.Context, user *model.User) { + paths, err := db.ListMediaScanPaths("") + if err != nil { + c.JSON(500, gin.H{"Error": err.Error()}) + return + } + var out []gin.H + for _, p := range paths { + if !CanUserAccessPath(user, p.Path) { + continue + } + collectionType := "movies" + switch p.MediaType { + case model.MediaTypeVideo: + if p.TypeTag == "电视剧" || p.TypeTag == "tv" { + collectionType = "tvshows" + } + case model.MediaTypeMusic: + collectionType = "music" + case model.MediaTypeImage: + collectionType = "homevideos" + case model.MediaTypeBook: + collectionType = "books" + } + out = append(out, gin.H{ + "Name": p.Name, + "ItemId": "libpath-" + strconv.FormatUint(uint64(p.ID), 10), + "CollectionType": collectionType, + "Locations": []string{p.Path}, + "LibraryOptions": gin.H{ + "PathInfos": []gin.H{{"Path": p.Path}}, + }, + }) + } + c.JSON(200, out) +} diff --git a/server/handles/emby_play.go b/server/handles/emby_play.go new file mode 100644 index 00000000..eef8d916 --- /dev/null +++ b/server/handles/emby_play.go @@ -0,0 +1,681 @@ +package handles + +import ( + "encoding/base64" + "encoding/json" + "fmt" + "net/http" + "net/url" + stdpath "path" + "regexp" + "strconv" + "strings" + "time" + + "github.com/OpenListTeam/OpenList/v4/internal/db" + "github.com/OpenListTeam/OpenList/v4/internal/media" + "github.com/OpenListTeam/OpenList/v4/internal/media/imgmime" + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/internal/op" + "github.com/OpenListTeam/OpenList/v4/server/common" + "github.com/gin-gonic/gin" +) + +// embyPlayTarget 播放目标(条目或某一集) +type embyPlayTarget struct { + Item *model.MediaItem + Episode *model.EpisodeInfo + RawPath string + Container string + Size int64 + Name string + Duration int + ItemID uint + EpisodeNo int +} + +func (l *embyLibrary) playTarget(id string) (*embyPlayTarget, bool) { + id = l.origID(id) + switch { + case strings.HasPrefix(id, "mv-"): + n, err := strconv.ParseUint(strings.TrimPrefix(id, "mv-"), 10, 64) + if err != nil { + return nil, false + } + for i := range l.videos { + if l.videos[i].ID == uint(n) { + it := &l.videos[i] + return &embyPlayTarget{ + Item: it, RawPath: MediaItemFilePath(it), Container: embyExt(it.FileName), + Size: it.FileSize, Name: it.FileName, Duration: it.Duration, ItemID: it.ID, + }, true + } + } + case strings.HasPrefix(id, "so-"): + it, ok := l.songByID(id) + if !ok { + return nil, false + } + return &embyPlayTarget{ + Item: it, RawPath: MediaItemFilePath(it), Container: embyExt(it.FileName), + Size: it.FileSize, Name: it.FileName, Duration: it.Duration, ItemID: it.ID, + }, true + case strings.HasPrefix(id, "ep-"): + ep := l.findEpisode(id) + if ep == nil { + return nil, false + } + raw := MediaItemFilePath(ep.Item) + if ep.Episode.FileName != "" { + raw = stdpath.Join(mediaFolderOf(ep.Item), ep.Item.FileName, ep.Episode.FileName) + } + return &embyPlayTarget{ + Item: ep.Item, Episode: &ep.Episode, RawPath: raw, Container: embyExt(raw), + Size: ep.Item.FileSize, Name: ep.Episode.FileName, Duration: ep.Item.Duration, + ItemID: ep.Item.ID, EpisodeNo: ep.Index, + }, true + } + return nil, false +} + +// embyPlaybackInfo 返回直连播放信息 +func embyPlaybackInfo(c *gin.Context, lib *embyLibrary, id string) { + target, ok := lib.playTarget(id) + if !ok { + embyNotFound(c) + return + } + api := common_ApiBase(c) + isAudio := target.Item.MediaType == model.MediaTypeMusic + kind, streamType := "Videos", "Video" + if isAudio { + kind, streamType = "Audio", "Audio" + } + // 流地址必须自带 api_key:播放器取流时不会再附加请求头, + // 缺失会得到 401,客户端表现为"Playback of this content is currently restricted" + q := url.Values{} + q.Set("Static", "true") + q.Set("MediaSourceId", id) + if tok := embyRequestToken(c); tok != "" { + q.Set("api_key", tok) + } + streamURL := fmt.Sprintf("%s/@emby/%s/%s/stream.%s?%s", api, kind, id, target.Container, q.Encode()) + if target.Container == "" { + streamURL = fmt.Sprintf("%s/@emby/%s/%s/stream?%s", api, kind, id, q.Encode()) + } + // 必须是 Http:客户端看到 File 会当成"服务器本地文件", + // 转而调用原生 LocalItemRepository 解析本地路径,必然失败(Playback Error) + streams := []embyMediaStream{{Type: streamType, Index: 0, IsDefault: true}} + var subtitleIndex *int + if isAudio && target.Item.Lyrics != "" { + // 歌词以"外挂字幕轨"的形式提供给客户端(Emby Theater 的歌词面板 + // 只遍历 Subtitle 流,并请求 Subtitles/{index}/Stream.js) + lyricsIndex := len(streams) + streams = append(streams, embyLyricsMediaStream(lyricsIndex)) + subtitleIndex = &lyricsIndex + } + source := embyMediaSource{ + Protocol: "Http", + Id: id, + ItemId: id, + // Emby Theater 在本地播放时会把 Path 直接交给 mpv; + // 填文件系统路径会让 mpv 去开本机不存在的文件(一直转圈/Playback Error), + // 这里填可播放的 HTTP 流地址。 + Path: streamURL, + Type: "Default", + Container: target.Container, + Name: target.Name, + Size: target.Size, + ETag: embyHash(fmt.Sprintf("%s|%d", target.RawPath, target.Size)), + RunTimeTicks: embyTicks(target.Duration), + SupportsDirectPlay: true, + SupportsDirectStream: true, + SupportsTranscoding: false, + SupportsProbing: false, + MediaStreams: streams, + MediaAttachments: []any{}, + Formats: []string{}, + RequiredHttpHeaders: map[string]string{}, + DirectStreamUrl: streamURL, + AddApiKeyToDirectStreamUrl: false, + DefaultSubtitleStreamIndex: subtitleIndex, + } + c.JSON(200, gin.H{ + "MediaSources": []embyMediaSource{source}, + "PlaySessionId": fmt.Sprintf("openlist-%d-%d", target.ItemID, target.EpisodeNo), + }) +} + +// embyStream 302 到签名直链 +func embyStream(c *gin.Context, user *model.User, id string, isAudio bool) { + lib := loadEmbyLibrary(user) + target, ok := lib.playTarget(id) + if !ok { + embyNotFound(c) + return + } + url := SignedStreamURL(c, target.RawPath) + c.Redirect(http.StatusFound, url) +} + +// ==================== 图片 ==================== + +var dataURIPrefix = "data:image/" + +// embyImage 返回条目封面 +func embyImage(c *gin.Context, id string, segs []string, allowAnon bool) { + var lib *embyLibrary + if user, ok := embyAuth(c); ok { + lib = loadEmbyLibrary(user) + } else if allowAnon { + // Emby 客户端的封面 URL 不带 token,只能允许匿名访问; + // 但必须按访客身份做权限过滤,否则等于把整个媒体库(含 hidden、 + // base_path 之外、meta 受保护的条目)的封面暴露给未登录用户。 + // 访客被禁用时直接拒绝,不能退化成「全库可见」。 + guest, err := op.GetGuest() + if err != nil || guest.Disabled { + embyUnauthorized(c) + return + } + lib = loadEmbyLibrary(guest) + } else { + embyUnauthorized(c) + return + } + // 客户端回传的是 GUID,这里先还原成内部 ID 再做前缀判断。 + // 注意不能先走 resolveItem:匿名库的 user 为 nil,剧集条目会被权限校验判为不可见。 + id = lib.origID(id) + var mediaItem *model.MediaItem + switch { + case strings.HasPrefix(id, "mv-"): + n, _ := strconv.ParseUint(strings.TrimPrefix(id, "mv-"), 10, 64) + for i := range lib.videos { + if lib.videos[i].ID == uint(n) { + mediaItem = &lib.videos[i] + } + } + case strings.HasPrefix(id, "so-"): + mediaItem, _ = lib.songByID(id) + case strings.HasPrefix(id, "ep-"): + // 图片 URL 不带 token,匿名库的 user 为 nil; + // findEpisode 会做用户权限校验并直接失败,这里只按库内条目定位封面来源 + if it, ok := lib.episodeSourceItem(id); ok { + mediaItem = it + } + case strings.HasPrefix(id, "sr-"): + if s := lib.seriesByKey(strings.TrimPrefix(id, "sr-")); s != nil && len(s.Items) > 0 { + mediaItem = &s.Items[0] + } + case strings.HasPrefix(id, "sn-"): + rest := strings.TrimPrefix(id, "sn-") + if idx := strings.LastIndex(rest, "-"); idx > 0 { + if s := lib.seriesByKey(rest[:idx]); s != nil && len(s.Items) > 0 { + mediaItem = &s.Items[0] + } + } + case strings.HasPrefix(id, "al-"): + if al := lib.albumByKey(strings.TrimPrefix(id, "al-")); al != nil && len(al.Items) > 0 { + mediaItem = &al.Items[0] + } + case strings.HasPrefix(id, "ar-"): + key := strings.TrimPrefix(id, "ar-") + for i := range lib.music { + if artistKeyOf(&lib.music[i]) == key { + mediaItem = &lib.music[i] + break + } + } + } + if mediaItem == nil { + c.Status(404) + return + } + serveEmbyCover(c, mediaItem) +} + +// episodeSourceItem 按 ep-- 在库内定位剧集所属条目,不做权限校验。 +// 仅用于封面这类匿名可访问的资源,播放与条目详情仍走 findEpisode 的权限校验。 +func (l *embyLibrary) episodeSourceItem(id string) (*model.MediaItem, bool) { + parts := strings.Split(strings.TrimPrefix(id, "ep-"), "-") + if len(parts) != 2 { + return nil, false + } + n, err := strconv.ParseUint(parts[0], 10, 64) + if err != nil { + return nil, false + } + for i := range l.videos { + if l.videos[i].ID == uint(n) { + return &l.videos[i], true + } + } + return nil, false +} + +func serveEmbyCover(c *gin.Context, item *model.MediaItem) { + cover := strings.TrimSpace(item.Cover) + if cover == "" { + c.Status(404) + return + } + if strings.HasPrefix(cover, dataURIPrefix) { + idx := strings.Index(cover, ";base64,") + if idx < 0 { + c.Status(404) + return + } + data, err := base64.StdEncoding.DecodeString(cover[idx+8:]) + if err != nil { + c.Status(404) + return + } + // 数据库里的 data URI 声明的 MIME 不可信(历史数据可能来自内嵌标签/远端响应), + // 响应头只使用按魔数嗅探出的四种栅格类型,SVG/HTML 一律拒绝。 + mime := imgmime.Sniff(data) + if mime == "" { + c.Status(404) + return + } + c.Header("X-Content-Type-Options", "nosniff") + c.Header("Content-Disposition", `inline; filename="cover"`) + c.Header("Content-Security-Policy", "default-src 'none'; sandbox") + c.Header("Cache-Control", "private, max-age=86400") + c.Data(200, mime, data) + return + } + if strings.HasPrefix(cover, "http://") || strings.HasPrefix(cover, "https://") { + c.Redirect(302, cover) + return + } + // 本地缩略图路径 + c.Redirect(302, SignedStreamURL(c, cover)) +} + +// ==================== 歌词 ==================== + +var lrcLineRe = regexp.MustCompile(`\[(\d{1,3}):(\d{1,2})(?:[.:](\d{1,3}))?\](.*)`) + +func parseLrc(content string) []gin.H { + var out []gin.H + for _, line := range strings.Split(content, "\n") { + m := lrcLineRe.FindStringSubmatch(strings.TrimSpace(line)) + if len(m) == 0 { + continue + } + min, _ := strconv.Atoi(m[1]) + sec, _ := strconv.Atoi(m[2]) + ms := 0 + if m[3] != "" { + frac := m[3] + for len(frac) < 3 { + frac += "0" + } + ms, _ = strconv.Atoi(frac[:3]) + } + text := strings.TrimSpace(m[4]) + out = append(out, gin.H{ + "Text": text, + "Start": int64(min*60000 + sec*1000 + ms), + }) + } + return out +} + +func embyLyrics(c *gin.Context, user *model.User, id string) { + lib := loadEmbyLibrary(user) + item, ok := lib.songByID(id) + if !ok || item.Lyrics == "" { + c.Status(404) + return + } + lines := parseLrc(item.Lyrics) + c.JSON(200, gin.H{"Lyrics": lines, "IsSynced": len(lines) > 0}) +} + +// embyLyricsMediaStream 构造歌词字幕轨(Emby 用外挂字幕承载歌词)。 +func embyLyricsMediaStream(index int) embyMediaStream { + return embyMediaStream{ + Type: "Subtitle", + Index: index, + Codec: "lrc", + Language: "und", + DisplayTitle: "Lyrics", + IsExternal: true, + IsTextSubtitleStream: true, + SupportsExternalStream: true, + DeliveryMethod: "External", + DeliveryUrl: "", + } +} + +// embyLyricsTrackEvents 把 LRC 转成客户端歌词面板使用的 TrackEvent 列表。 +// 客户端按 StartPositionTicks 与当前播放位置比较(1ms = 10000 ticks)。 +// 没有时间轴的纯文本歌词会按行均分到歌曲时长,至少能顺序展示。 +func embyLyricsTrackEvents(content string, durationSec int) []gin.H { + lines := parseLrc(content) + if len(lines) == 0 { + texts := make([]string, 0, 32) + for _, l := range strings.Split(strings.ReplaceAll(content, "\r\n", "\n"), "\n") { + if t := strings.TrimSpace(l); t != "" { + texts = append(texts, t) + } + } + if len(texts) > 0 { + total := int64(durationSec) * 1000 + if total <= 0 { + total = int64(len(texts)) * 5000 + } + step := total / int64(len(texts)) + for i, t := range texts { + lines = append(lines, gin.H{"Text": t, "Start": int64(i) * step}) + } + } + } + events := make([]gin.H, 0, len(lines)) + for _, l := range lines { + ms, _ := l["Start"].(int64) + events = append(events, gin.H{ + "StartPositionTicks": ms * 10000, + "Text": l["Text"], + }) + } + return events +} + +// embySubtitleStream 处理 /Items/{id}/{mediaSourceId}/Subtitles/{index}/Stream.js +// Emby Theater 用它拉取歌词行(JSON 形式的 TrackEvents) +func embySubtitleStream(c *gin.Context, user *model.User, id string) { + lib := loadEmbyLibrary(user) + item, ok := lib.songByID(lib.origID(id)) + if !ok || item.Lyrics == "" { + c.JSON(200, gin.H{"TrackEvents": []any{}}) + return + } + events := embyLyricsTrackEvents(item.Lyrics, item.Duration) + c.JSON(200, gin.H{"TrackEvents": events, "TotalRecordCount": len(events)}) +} + +// ==================== 播放会话 / 用户状态 ==================== + +type embyPlaybackBody struct { + ItemId string `json:"ItemId"` + ItemID string `json:"ItemID"` + PositionTicks int64 `json:"PositionTicks"` + PlayMethod string `json:"PlayMethod"` + EventName string `json:"EventName"` + MediaSourceId string `json:"MediaSourceId"` +} + +func (b embyPlaybackBody) itemID() string { + if b.ItemId != "" { + return b.ItemId + } + return b.ItemID +} + +func embySessionPlaying(c *gin.Context, user *model.User, action string) { + var body embyPlaybackBody + _ = c.ShouldBindJSON(&body) + if action == "ping" { + c.Status(204) + return + } + id := body.itemID() + if id == "" { + id = c.Query("ItemId") + } + if id == "" { + c.Status(204) + return + } + stopped := action == "stopped" || action == "stop" + lib := loadEmbyLibrary(user) + target, ok := lib.playTarget(id) + if !ok { + c.Status(204) + return + } + position := body.PositionTicks + if position == 0 { + if v, err := strconv.ParseInt(c.DefaultQuery("PositionTicks", "0"), 10, 64); err == nil { + position = v + } + } + played := false + if stopped && target.Duration > 0 { + ratio := float64(position) / float64(embyTicks(target.Duration)) + played = ratio >= 0.9 + } + applyProgress(user, target, position, played, stopped) + c.Status(204) +} + +func embySaveProgress(c *gin.Context, user *model.User, id string, start bool) { + var body embyPlaybackBody + _ = c.ShouldBindJSON(&body) + lib := loadEmbyLibrary(user) + target, ok := lib.playTarget(id) + if !ok { + c.Status(204) + return + } + position := body.PositionTicks + if position == 0 { + position, _ = strconv.ParseInt(c.DefaultQuery("PositionTicks", "0"), 10, 64) + } + played := false + stopped := c.Request.Method == "DELETE" + if stopped && target.Duration > 0 { + played = float64(position)/float64(embyTicks(target.Duration)) >= 0.9 + } + applyProgress(user, target, position, played, stopped) + c.Status(204) +} + +func applyProgress(user *model.User, target *embyPlayTarget, position int64, played bool, stopped bool) { + if user == nil || target == nil { + return + } + d, err := db.GetMediaUserData(user.ID, target.ItemID, target.EpisodeNo) + if err != nil || d == nil { + d = &model.MediaUserData{UserID: user.ID, ItemID: target.ItemID, EpisodeIndex: target.EpisodeNo} + } + d.PositionTicks = position + if played { + d.Played = true + d.PositionTicks = 0 + d.PlayCount++ + } + now := time.Now() + d.LastPlayedAt = &now + _ = db.SaveMediaUserData(d) +} + +func embySetPlayed(c *gin.Context, user *model.User, id string, played bool) { + lib := loadEmbyLibrary(user) + id = lib.origID(id) + target, ok := lib.playTarget(id) + if !ok { + // 剧集/季整体标记 + if strings.HasPrefix(id, "sr-") || strings.HasPrefix(id, "sn-") { + embyMarkContainerPlayed(c, user, lib, id, played) + return + } + embyNotFound(c) + return + } + d, err := db.GetMediaUserData(user.ID, target.ItemID, target.EpisodeNo) + if err != nil || d == nil { + d = &model.MediaUserData{UserID: user.ID, ItemID: target.ItemID, EpisodeIndex: target.EpisodeNo} + } + d.Played = played + if played { + d.PositionTicks = 0 + d.PlayCount++ + } else { + d.PlayCount = 0 + } + now := time.Now() + d.LastPlayedAt = &now + _ = db.SaveMediaUserData(d) + c.JSON(200, gin.H{ + "PlaybackPositionTicks": d.PositionTicks, + "PlayCount": d.PlayCount, + "Played": d.Played, + "IsFavorite": d.IsFavorite, + }) +} + +func embyMarkContainerPlayed(c *gin.Context, user *model.User, lib *embyLibrary, id string, played bool) { + var targets []*embyPlayTarget + if strings.HasPrefix(id, "sr-") { + key := strings.TrimPrefix(id, "sr-") + for _, ep := range lib.episodesOf(key, 0) { + if t, ok := lib.playTarget(ep.ID); ok { + targets = append(targets, t) + } + } + } else { + rest := strings.TrimPrefix(id, "sn-") + idx := strings.LastIndex(rest, "-") + if idx > 0 { + key := rest[:idx] + season, _ := strconv.Atoi(rest[idx+1:]) + for _, ep := range lib.episodesOf(key, season) { + if t, ok := lib.playTarget(ep.ID); ok { + targets = append(targets, t) + } + } + } + } + for _, t := range targets { + d, err := db.GetMediaUserData(user.ID, t.ItemID, t.EpisodeNo) + if err != nil || d == nil { + d = &model.MediaUserData{UserID: user.ID, ItemID: t.ItemID, EpisodeIndex: t.EpisodeNo} + } + d.Played = played + d.PositionTicks = 0 + if !played { + d.PlayCount = 0 + } + _ = db.SaveMediaUserData(d) + } + c.Status(204) +} + +func embySetFavorite(c *gin.Context, user *model.User, id string, fav bool) { + lib := loadEmbyLibrary(user) + target, ok := lib.playTarget(id) + if !ok { + embyNotFound(c) + return + } + d, err := db.GetMediaUserData(user.ID, target.ItemID, target.EpisodeNo) + if err != nil || d == nil { + d = &model.MediaUserData{UserID: user.ID, ItemID: target.ItemID, EpisodeIndex: target.EpisodeNo} + } + d.IsFavorite = fav + _ = db.SaveMediaUserData(d) + c.JSON(200, gin.H{ + "PlaybackPositionTicks": d.PositionTicks, + "PlayCount": d.PlayCount, + "Played": d.Played, + "IsFavorite": d.IsFavorite, + }) +} + +// embyUpdateUserData 部分更新用户状态 +func embyUpdateUserData(c *gin.Context, user *model.User, id string) { + lib := loadEmbyLibrary(user) + target, ok := lib.playTarget(id) + if !ok { + embyNotFound(c) + return + } + var body map[string]any + _ = c.ShouldBindJSON(&body) + d, err := db.GetMediaUserData(user.ID, target.ItemID, target.EpisodeNo) + if err != nil || d == nil { + d = &model.MediaUserData{UserID: user.ID, ItemID: target.ItemID, EpisodeIndex: target.EpisodeNo} + } + if v, ok := body["Played"].(bool); ok { + d.Played = v + if v { + d.PositionTicks = 0 + } + } + if v, ok := body["IsFavorite"].(bool); ok { + d.IsFavorite = v + } + if v, ok := body["PlaybackPositionTicks"].(float64); ok { + d.PositionTicks = int64(v) + } + if v, ok := body["PlayCount"].(float64); ok { + d.PlayCount = int(v) + } + _ = db.SaveMediaUserData(d) + c.JSON(200, gin.H{ + "PlaybackPositionTicks": d.PositionTicks, + "PlayCount": d.PlayCount, + "Played": d.Played, + "IsFavorite": d.IsFavorite, + }) +} + +// embyRefreshItem 触发条目所属扫描路径的增量扫描 +func embyRefreshItem(c *gin.Context, id string) { + lib := loadEmbyLibrary(nil) + var itemID uint + switch { + case strings.HasPrefix(id, "mv-"), strings.HasPrefix(id, "so-"): + n, _ := strconv.ParseUint(id[3:], 10, 64) + itemID = uint(n) + case strings.HasPrefix(id, "ep-"): + parts := strings.Split(id, "-") + if len(parts) == 3 { + n, _ := strconv.ParseUint(parts[1], 10, 64) + itemID = uint(n) + } + case strings.HasPrefix(id, "sr-"): + if s := lib.seriesByKey(strings.TrimPrefix(id, "sr-")); s != nil && len(s.Items) > 0 { + itemID = s.Items[0].ID + } + case strings.HasPrefix(id, "al-"): + if al := lib.albumByKey(strings.TrimPrefix(id, "al-")); al != nil && len(al.Items) > 0 { + itemID = al.Items[0].ID + } + } + if itemID == 0 { + c.Status(204) + return + } + item, err := db.GetMediaItemByID(itemID) + if err != nil { + c.Status(204) + return + } + if sp, err := db.GetMediaScanPath(item.ScanPathID); err == nil && sp != nil { + media.ScanMediaPath(sp) + } + c.Status(204) +} + +// ==================== 内部工具 ==================== + +func common_ApiBase(c *gin.Context) string { + return strings.TrimSuffix(common.GetApiUrl(c.Request.Context()), "/") +} + +// embyBodyJSON 便于调试时输出请求体 +func embyBodyJSON(c *gin.Context) string { + raw, _ := c.GetRawData() + if len(raw) == 0 { + return "" + } + var v any + if err := json.Unmarshal(raw, &v); err != nil { + return string(raw) + } + b, _ := json.Marshal(v) + return string(b) +} diff --git a/server/handles/fsmanage.go b/server/handles/fsmanage.go index 11e904be..1a027027 100644 --- a/server/handles/fsmanage.go +++ b/server/handles/fsmanage.go @@ -63,6 +63,7 @@ type MoveCopyReq struct { Overwrite bool `json:"overwrite"` SkipExisting bool `json:"skip_existing"` Merge bool `json:"merge"` + FollowSeed bool `json:"follow_seed"` } // FsMove performs batch move (individual item permission checks skipped for performance). @@ -156,6 +157,14 @@ func FsMove(c *gin.Context) { common.ErrorResp(c, err, 500) return } + if req.FollowSeed { + seedTasks, followErr := followSeedTransfer(c, "move", p, dstDir) + if followErr != nil { + common.ErrorResp(c, followErr, 500) + return + } + addedTasks = append(addedTasks, seedTasks...) + } } // Return immediately with task information @@ -269,6 +278,14 @@ func FsCopy(c *gin.Context) { common.ErrorResp(c, err, 500) return } + if req.FollowSeed { + seedTasks, followErr := followSeedTransfer(c, "copy", p, dstDir) + if followErr != nil { + common.ErrorResp(c, followErr, 500) + return + } + addedTasks = append(addedTasks, seedTasks...) + } } // Return immediately with task information @@ -285,9 +302,10 @@ func FsCopy(c *gin.Context) { } type RenameReq struct { - Path string `json:"path"` - Name string `json:"name"` - Overwrite bool `json:"overwrite"` + Path string `json:"path"` + Name string `json:"name"` + Overwrite bool `json:"overwrite"` + FollowSeed bool `json:"follow_seed"` } func FsRename(c *gin.Context) { @@ -332,6 +350,12 @@ func FsRename(c *gin.Context) { common.ErrorResp(c, err, 500) return } + if req.FollowSeed { + if err := followSeedRename(c, reqPath, req.Name); err != nil { + common.ErrorResp(c, err, 500) + return + } + } common.SuccessResp(c) } @@ -343,8 +367,9 @@ func checkRelativePath(path string) error { } type RemoveReq struct { - Dir string `json:"dir"` - Names []string `json:"names"` + Dir string `json:"dir"` + Names []string `json:"names"` + FollowSeed bool `json:"follow_seed"` } // FsRemove performs batch remove (individual item permission checks skipped for performance). @@ -396,16 +421,88 @@ func FsRemove(c *gin.Context) { if path == "" { continue } + source, _ := fs.Get(c.Request.Context(), path, &fs.GetArgs{NoLog: true}) err := fs.Remove(c.Request.Context(), path) if err != nil { common.ErrorResp(c, err, 500) return } + if req.FollowSeed && source != nil && !source.IsDir() { + if err = followSeedRemove(c, path); err != nil { + common.ErrorResp(c, err, 500) + return + } + } } //fs.ClearCache(req.Dir) common.SuccessResp(c) } +func seedSidecarPaths(filePath string) []string { + return []string{filePath + ".oss", filePath + ".torrent", filePath + ".cas", filePath + ".cas.torrent"} +} + +func followSeedTransfer(c *gin.Context, operation, srcPath, dstDir string) ([]task.TaskExtensionInfo, error) { + source, err := fs.Get(c.Request.Context(), srcPath, &fs.GetArgs{NoLog: true}) + if err != nil || source == nil || source.IsDir() { + return nil, nil + } + var tasks []task.TaskExtensionInfo + for _, sidecarPath := range seedSidecarPaths(srcPath) { + obj, err := fs.Get(c.Request.Context(), sidecarPath, &fs.GetArgs{NoLog: true}) + if err != nil || obj == nil || obj.IsDir() { + continue + } + var current task.TaskExtensionInfo + switch operation { + case "copy": + current, err = fs.Copy(c.Request.Context(), sidecarPath, dstDir, true) + case "move": + current, err = fs.Move(c.Request.Context(), sidecarPath, dstDir, true) + default: + return nil, fmt.Errorf("unsupported seed sidecar operation %q", operation) + } + if err != nil { + return tasks, fmt.Errorf("%s seed sidecar %s: %w", operation, sidecarPath, err) + } + if current != nil { + tasks = append(tasks, current) + } + } + return tasks, nil +} + +func followSeedRename(c *gin.Context, srcPath, newName string) error { + source, err := fs.Get(c.Request.Context(), srcPath, &fs.GetArgs{NoLog: true}) + if err != nil || source == nil || source.IsDir() { + return nil + } + for _, sidecarPath := range seedSidecarPaths(srcPath) { + obj, err := fs.Get(c.Request.Context(), sidecarPath, &fs.GetArgs{NoLog: true}) + if err != nil || obj == nil || obj.IsDir() { + continue + } + suffix := strings.TrimPrefix(sidecarPath, srcPath) + if err = fs.Rename(c.Request.Context(), sidecarPath, newName+suffix, true); err != nil { + return fmt.Errorf("rename seed sidecar %s: %w", sidecarPath, err) + } + } + return nil +} + +func followSeedRemove(c *gin.Context, srcPath string) error { + for _, sidecarPath := range seedSidecarPaths(srcPath) { + obj, err := fs.Get(c.Request.Context(), sidecarPath, &fs.GetArgs{NoLog: true}) + if err != nil || obj == nil || obj.IsDir() { + continue + } + if err = fs.Remove(c.Request.Context(), sidecarPath); err != nil { + return fmt.Errorf("remove seed sidecar %s: %w", sidecarPath, err) + } + } + return nil +} + type RemoveEmptyDirectoryReq struct { SrcDir string `json:"src_dir"` } diff --git a/server/handles/fsread.go b/server/handles/fsread.go index 841cb5af..bf4089ff 100644 --- a/server/handles/fsread.go +++ b/server/handles/fsread.go @@ -109,8 +109,9 @@ func FsList(c *gin.Context, req *ListReq, user *model.User) { total, objs := pagination(objs, &req.PageReq) provider := "unknown" var directUploadTools []string - if canWriteContentAtPath { - if storage, err := fs.GetStorage(reqPath, &fs.GetStoragesArgs{}); err == nil { + if storage, err := fs.GetStorage(reqPath, &fs.GetStoragesArgs{}); err == nil { + provider = storage.GetStorage().Driver + if canWriteContentAtPath { directUploadTools = op.GetDirectUploadTools(storage) } } diff --git a/server/handles/fsread_test.go b/server/handles/fsread_test.go index 3947ae27..89b7c46b 100644 --- a/server/handles/fsread_test.go +++ b/server/handles/fsread_test.go @@ -1,9 +1,11 @@ package handles import ( + "encoding/json" "testing" "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/gin-gonic/gin" ) func TestGetReadme(t *testing.T) { @@ -253,3 +255,30 @@ func TestIsEncrypt(t *testing.T) { }) } } + +func TestFsListReportsProvider(t *testing.T) { + gin.SetMode(gin.TestMode) + root, _ := prepareBackslashTraversalFs(t) + for _, tt := range []struct { + name string + permission int32 + }{ + {name: "read only", permission: 0}, + {name: "write content", permission: 1 << 3}, + } { + t.Run(tt.name, func(t *testing.T) { + user := setupBackslashTraversalTest(t, root, tt.permission) + recorder := invokeHandler(t, user, map[string]any{"path": "/"}, FsListSplit) + var resp struct { + Code int `json:"code"` + Data FsListResp `json:"data"` + } + if err := json.Unmarshal(recorder.Body.Bytes(), &resp); err != nil { + t.Fatal(err) + } + if resp.Code != 200 || resp.Data.Provider != "Local" { + t.Fatalf("got code=%d provider=%q, want 200 and Local; body=%s", resp.Code, resp.Data.Provider, recorder.Body.String()) + } + }) + } +} diff --git a/server/handles/fsup.go b/server/handles/fsup.go index 0f46398c..6e429c8d 100644 --- a/server/handles/fsup.go +++ b/server/handles/fsup.go @@ -1,19 +1,25 @@ package handles import ( + "bytes" + "encoding/json" + "fmt" "io" "net/url" stdpath "path" "strconv" + "strings" "time" "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/fs" "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/OpenListTeam/OpenList/v4/internal/setting" "github.com/OpenListTeam/OpenList/v4/internal/stream" "github.com/OpenListTeam/OpenList/v4/internal/task" + "github.com/OpenListTeam/OpenList/v4/pkg/torrent" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/OpenListTeam/OpenList/v4/server/common" "github.com/gin-gonic/gin" @@ -97,6 +103,23 @@ func FsStream(c *gin.Context) { if len(mimetype) == 0 { mimetype = utils.GetMimeType(name) } + generateSeed := shouldGenerateUploadSeed(c, dir) + if generateSeed { + if err = validateUploadSeedRequest(c); err != nil { + common.ErrorStrResp(c, err.Error(), 400) + return + } + } + if generateSeed && asTask { + common.ErrorStrResp(c, "seed sidecar generation requires synchronous upload", 400) + return + } + var seedHasher *torrent.HashWriter + var uploadReader io.Reader = c.Request.Body + if generateSeed { + seedHasher = torrent.NewHashWriter(seedPieceSize(c), seedPieceSize(c), 0) + uploadReader = io.TeeReader(c.Request.Body, seedHasher) + } s := &stream.FileStream{ Obj: &model.Object{ Name: name, @@ -104,7 +127,7 @@ func FsStream(c *gin.Context) { Modified: getLastModified(c), HashInfo: utils.NewHashInfoByMap(h), }, - Reader: c.Request.Body, + Reader: uploadReader, Mimetype: mimetype, WebPutAsTask: asTask, } @@ -118,6 +141,12 @@ func FsStream(c *gin.Context) { common.ErrorResp(c, err, 500) return } + if generateSeed { + if err = writeUploadSeedSidecar(c, dir, name, size, seedHasher); err != nil { + common.ErrorResp(c, err, 500) + return + } + } if t == nil { common.SuccessResp(c) return @@ -194,6 +223,23 @@ func FsForm(c *gin.Context) { if len(mimetype) == 0 { mimetype = utils.GetMimeType(name) } + generateSeed := shouldGenerateUploadSeed(c, dir) + if generateSeed { + if err = validateUploadSeedRequest(c); err != nil { + common.ErrorStrResp(c, err.Error(), 400) + return + } + } + if generateSeed && asTask { + common.ErrorStrResp(c, "seed sidecar generation requires synchronous upload", 400) + return + } + var seedHasher *torrent.HashWriter + var uploadReader io.Reader = f + if generateSeed { + seedHasher = torrent.NewHashWriter(seedPieceSize(c), seedPieceSize(c), 0) + uploadReader = io.TeeReader(f, seedHasher) + } s := &stream.FileStream{ Obj: &model.Object{ Name: name, @@ -201,7 +247,7 @@ func FsForm(c *gin.Context) { Modified: getLastModified(c), HashInfo: utils.NewHashInfoByMap(h), }, - Reader: f, + Reader: uploadReader, Mimetype: mimetype, WebPutAsTask: asTask, } @@ -218,6 +264,12 @@ func FsForm(c *gin.Context) { common.ErrorResp(c, err, 500) return } + if generateSeed { + if err = writeUploadSeedSidecar(c, dir, name, file.Size, seedHasher); err != nil { + common.ErrorResp(c, err, 500) + return + } + } if t == nil { common.SuccessResp(c) return @@ -226,3 +278,130 @@ func FsForm(c *gin.Context) { "task": getTaskInfo(t), }) } + +func shouldGenerateUploadSeed(c *gin.Context, path string) bool { + if strings.TrimSpace(c.GetHeader("X-Seed-Sidecars")) != "" { + return true + } + policy := strings.ToLower(strings.TrimSpace(c.GetHeader("X-Generate-Seed"))) + if policy != "" && policy != "inherit" { + return policy == "on" || policy == "true" || policy == "1" + } + if storage := op.GetBalancedStorage(path); storage != nil { + policy = strings.ToLower(strings.TrimSpace(storage.GetStorage().SeedPolicy)) + } + if policy == "" || policy == "inherit" { + policy = strings.ToLower(setting.GetStr(conf.SeedAutoGeneratePolicy, "off")) + } + return (policy == "on" || policy == "true" || policy == "1") && configuredSeedFormats() != "" +} + +func seedPieceSize(c *gin.Context) int64 { + for _, format := range strings.Split(strings.ToLower(c.GetHeader("X-Seed-Sidecars")), ",") { + if strings.TrimSpace(format) == "cas" { + return torrent.DefaultPieceSize + } + } + value := c.GetHeader("X-Seed-Piece-Size") + if value == "" { + return torrent.DefaultPieceSize + } + size, err := strconv.ParseInt(value, 10, 64) + if err != nil || size <= 0 || size > 1<<30 { + return torrent.DefaultPieceSize + } + return size +} + +func configuredSeedFormats() string { + policies := make(map[string]string) + if err := json.Unmarshal([]byte(setting.GetStr(conf.SeedFormatPolicies)), &policies); err != nil { + return "" + } + formats := make([]string, 0, 3) + for _, format := range []string{"oss", "torrent", "cas"} { + if strings.EqualFold(strings.TrimSpace(policies[format]), "on") { + formats = append(formats, format) + } + } + return strings.Join(formats, ",") +} + +// validateUploadSeedRequest rejects invalid seed sidecar headers before the +// main file is stored, so a bad X-Seed-* header cannot leave a partially +// uploaded file behind. +func validateUploadSeedRequest(c *gin.Context) error { + formatsHeader := strings.TrimSpace(c.GetHeader("X-Seed-Sidecars")) + if formatsHeader == "" { + formatsHeader = strings.TrimSpace(c.GetHeader("X-Seed-Format")) + } + if formatsHeader == "" { + formatsHeader = configuredSeedFormats() + } + if _, err := fs.NormalizeSeedFormats(strings.Split(formatsHeader, ",")); err != nil { + return err + } + if raw := strings.TrimSpace(c.GetHeader("X-Seed-Hash-Matrix")); raw != "" { + var matrix SeedHashMatrix + if err := json.Unmarshal([]byte(raw), &matrix); err != nil { + return fmt.Errorf("invalid seed hash matrix: %w", err) + } + } + return nil +} + +func writeUploadSeedSidecar(c *gin.Context, dir, name string, expectedSize int64, hasher *torrent.HashWriter) error { + if hasher == nil { + return nil + } + hasher.Finish() + if expectedSize >= 0 && hasher.GetTotalWritten() != expectedSize { + return fmt.Errorf("seed sidecar requires a complete stream: read %d of %d bytes", hasher.GetTotalWritten(), expectedSize) + } + seed := torrent.NewSeed(name, "OpenList", seedPieceSize(c)) + formatsHeader := strings.TrimSpace(c.GetHeader("X-Seed-Sidecars")) + if formatsHeader == "" { + formatsHeader = strings.TrimSpace(c.GetHeader("X-Seed-Format")) + } + if formatsHeader == "" { + formatsHeader = configuredSeedFormats() + } + formats, err := fs.NormalizeSeedFormats(strings.Split(formatsHeader, ",")) + if err != nil { + return err + } + matrix := SeedHashMatrix{} + if rawMatrix := strings.TrimSpace(c.GetHeader("X-Seed-Hash-Matrix")); rawMatrix != "" { + if err = json.Unmarshal([]byte(rawMatrix), &matrix); err != nil { + return fmt.Errorf("invalid seed hash matrix: %w", err) + } + } + matrix = normalizedSeedMatrix(matrix, formats) + seedFile := hasher.BuildSeedFile(name, getLastModified(c).UTC().Format(time.RFC3339)) + applySeedMatrix(&seedFile, matrix) + seed.Files = []torrent.SeedFile{seedFile} + seen := make(map[string]struct{}, 3) + for _, rawFormat := range formats { + format := strings.ToLower(strings.TrimPrefix(strings.TrimSpace(rawFormat), ".")) + if format == "bt" { + format = "torrent" + } + if _, exists := seen[format]; exists { + continue + } + seen[format] = struct{}{} + data, err := fs.EncodeGeneratedSeed(seed, format, hasher.GetPieceHashes()) + if err != nil { + return fmt.Errorf("generate %s seed sidecar: %w", format, err) + } + sidecar := &stream.FileStream{ + Ctx: c.Request.Context(), + Obj: &model.Object{Name: name + "." + format, Size: int64(len(data)), Modified: time.Now()}, + Reader: bytes.NewReader(data), Mimetype: "application/octet-stream", + } + if err = fs.PutDirectly(c.Request.Context(), dir, sidecar, true); err != nil { + return fmt.Errorf("upload %s seed sidecar: %w", format, err) + } + } + return nil +} diff --git a/server/handles/media.go b/server/handles/media.go new file mode 100644 index 00000000..b6f981b5 --- /dev/null +++ b/server/handles/media.go @@ -0,0 +1,1101 @@ +package handles + +import ( + "context" + "encoding/json" + "strconv" + "strings" + "sync" + "time" + + stdpath "path" + + "github.com/OpenListTeam/OpenList/v4/internal/conf" + "github.com/OpenListTeam/OpenList/v4/internal/db" + "github.com/OpenListTeam/OpenList/v4/internal/errs" + "github.com/OpenListTeam/OpenList/v4/internal/fs" + "github.com/OpenListTeam/OpenList/v4/internal/media" + "github.com/OpenListTeam/OpenList/v4/internal/media/scraper" + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/internal/setting" + "github.com/OpenListTeam/OpenList/v4/server/common" + "github.com/gin-gonic/gin" + log "github.com/sirupsen/logrus" +) + +// scrapeConcurrency 刮削并发度(默认 5) +// 走 setting 读取,未配置时使用 5;远端 API 调用密集型任务,并发 5 在限流与吞吐间较平衡。 +const defaultScrapeConcurrency = 5 + +// ==================== 配置管理 ==================== + +// ListMediaConfigs 获取所有媒体库配置 +func ListMediaConfigs(c *gin.Context) { + types := []model.MediaType{ + model.MediaTypeVideo, + model.MediaTypeMusic, + model.MediaTypeImage, + model.MediaTypeBook, + } + cfgs := make([]*model.MediaConfig, 0, len(types)) + for _, t := range types { + cfg, err := db.GetMediaConfig(t) + if err != nil { + common.ErrorResp(c, err, 500) + return + } + cfgs = append(cfgs, cfg) + } + common.SuccessResp(c, cfgs) +} + +// SaveMediaConfigReq 保存配置请求 +type SaveMediaConfigReq struct { + MediaType model.MediaType `json:"media_type" binding:"required"` + Enabled bool `json:"enabled"` + ScraperConfig *string `json:"scraper_config"` +} + +// SaveMediaConfig 保存媒体库配置 +func SaveMediaConfig(c *gin.Context) { + var req SaveMediaConfigReq + if err := c.ShouldBindJSON(&req); err != nil { + common.ErrorResp(c, err, 400) + return + } + // 读取已有配置,避免用零值覆盖 last_scan_at / last_scrape_at + cfg, err := db.GetMediaConfig(req.MediaType) + if err != nil { + common.ErrorResp(c, err, 500) + return + } + cfg.Enabled = req.Enabled + if req.ScraperConfig != nil { + cfg.ScraperConfig = *req.ScraperConfig + } + if err := db.SaveMediaConfig(cfg); err != nil { + common.ErrorResp(c, err, 500) + return + } + common.SuccessResp(c) +} + +// ==================== 扫描路径管理 ==================== + +// ListMediaScanPaths 获取指定媒体类型的扫描路径列表 +func ListMediaScanPaths(c *gin.Context) { + mediaType := model.MediaType(c.Query("media_type")) + paths, err := db.ListMediaScanPaths(mediaType) + if err != nil { + common.ErrorResp(c, err, 500) + return + } + common.SuccessResp(c, paths) +} + +// CreateMediaScanPathReq 创建扫描路径请求 +type CreateMediaScanPathReq struct { + MediaType model.MediaType `json:"media_type" binding:"required"` + Name string `json:"name"` + Path string `json:"path" binding:"required"` + PathMerge bool `json:"path_merge"` + TypeTag string `json:"type_tag"` + ContentTags string `json:"content_tags"` + EnableScrape bool `json:"enable_scrape"` +} + +// CreateMediaScanPath 创建扫描路径 +func CreateMediaScanPath(c *gin.Context) { + var req CreateMediaScanPathReq + if err := c.ShouldBindJSON(&req); err != nil { + common.ErrorResp(c, err, 400) + return + } + if req.Path == "" { + req.Path = "/" + } + name := req.Name + if name == "" { + name = req.Path + } + sp := &model.MediaScanPath{ + MediaType: req.MediaType, + Name: name, + Path: req.Path, + PathMerge: req.PathMerge, + TypeTag: req.TypeTag, + ContentTags: req.ContentTags, + EnableScrape: req.EnableScrape, + } + if err := db.CreateMediaScanPath(sp); err != nil { + common.ErrorResp(c, err, 500) + return + } + common.SuccessResp(c, sp) +} + +// UpdateMediaScanPathReq 更新扫描路径请求 +type UpdateMediaScanPathReq struct { + ID uint `json:"id" binding:"required"` + Name string `json:"name"` + Path string `json:"path"` + PathMerge bool `json:"path_merge"` + TypeTag string `json:"type_tag"` + ContentTags string `json:"content_tags"` + EnableScrape bool `json:"enable_scrape"` +} + +// UpdateMediaScanPath 更新扫描路径 +func UpdateMediaScanPath(c *gin.Context) { + var req UpdateMediaScanPathReq + if err := c.ShouldBindJSON(&req); err != nil { + common.ErrorResp(c, err, 400) + return + } + sp, err := db.GetMediaScanPath(req.ID) + if err != nil { + common.ErrorResp(c, err, 404) + return + } + sp.Name = req.Name + sp.Path = req.Path + sp.PathMerge = req.PathMerge + sp.TypeTag = req.TypeTag + sp.ContentTags = req.ContentTags + sp.EnableScrape = req.EnableScrape + if err := db.UpdateMediaScanPath(sp); err != nil { + common.ErrorResp(c, err, 500) + return + } + common.SuccessResp(c) +} + +// DeleteMediaScanPath 删除扫描路径 +func DeleteMediaScanPath(c *gin.Context) { + idStr := c.Query("id") + id, err := strconv.ParseUint(idStr, 10, 64) + if err != nil { + common.ErrorStrResp(c, "无效的ID", 400) + return + } + if err := db.DeleteMediaScanPath(uint(id)); err != nil { + common.ErrorResp(c, err, 500) + return + } + common.SuccessResp(c) +} + +// ClearMediaScanPathDB 清空指定扫描路径的媒体数据 +func ClearMediaScanPathDB(c *gin.Context) { + idStr := c.Query("id") + id, err := strconv.ParseUint(idStr, 10, 64) + if err != nil { + common.ErrorStrResp(c, "无效的ID", 400) + return + } + if err := db.ClearMediaItemsByScanPath(uint(id)); err != nil { + common.ErrorResp(c, err, 500) + return + } + common.SuccessResp(c) +} + +// ==================== 媒体条目管理(后台) ==================== + +// clampMediaPage 限制分页参数,避免 page_size 过大或为负导致整表扫描 / 响应体爆炸 +func clampMediaPage(page, pageSize, defaultSize, maxSize int) (int, int) { + if page < 1 { + page = 1 + } + if pageSize <= 0 { + pageSize = defaultSize + } + if pageSize > maxSize { + pageSize = maxSize + } + return page, pageSize +} + +// ListMediaItemsAdmin 后台分页查询媒体条目 +func ListMediaItemsAdmin(c *gin.Context) { + mediaType := model.MediaType(c.Query("media_type")) + page, _ := strconv.Atoi(c.DefaultQuery("page", "1")) + pageSize, _ := strconv.Atoi(c.DefaultQuery("page_size", "20")) + page, pageSize = clampMediaPage(page, pageSize, 20, 500) + keyword := c.Query("keyword") + orderBy := c.DefaultQuery("order_by", "name") + orderDir := c.DefaultQuery("order_dir", "asc") + scanPathIDStr := c.Query("scan_path_id") + var scanPathID uint + if scanPathIDStr != "" { + if id, err := strconv.ParseUint(scanPathIDStr, 10, 64); err == nil { + scanPathID = uint(id) + } + } + + q := db.MediaItemQuery{ + MediaType: mediaType, + ScanPathID: scanPathID, + Keyword: keyword, + OrderBy: orderBy, + OrderDir: orderDir, + Page: page, + PageSize: pageSize, + } + items, total, err := db.ListMediaItems(q) + if err != nil { + common.ErrorResp(c, err, 500) + return + } + common.SuccessResp(c, common.PageResp{Content: items, Total: total}) +} + +// UpdateMediaItemReq 更新媒体条目请求 +type UpdateMediaItemReq struct { + ID uint `json:"id" binding:"required"` + ScrapedName string `json:"scraped_name"` + Description string `json:"description"` + Cover string `json:"cover"` + ReleaseDate string `json:"release_date"` + Rating float32 `json:"rating"` + Genre string `json:"genre"` + Authors string `json:"authors"` + Plot string `json:"plot"` + Reviews string `json:"reviews"` + AlbumName string `json:"album_name"` + AlbumArtist string `json:"album_artist"` + Publisher string `json:"publisher"` + ISBN string `json:"isbn"` + Hidden bool `json:"hidden"` +} + +// UpdateMediaItemAdmin 后台更新媒体条目 +func UpdateMediaItemAdmin(c *gin.Context) { + var req UpdateMediaItemReq + if err := c.ShouldBindJSON(&req); err != nil { + common.ErrorResp(c, err, 400) + return + } + item, err := db.GetMediaItemByID(req.ID) + if err != nil { + common.ErrorResp(c, err, 404) + return + } + item.ScrapedName = req.ScrapedName + item.Description = req.Description + item.Cover = req.Cover + item.ReleaseDate = req.ReleaseDate + item.Rating = req.Rating + item.Genre = req.Genre + item.Authors = req.Authors + item.Plot = req.Plot + item.Reviews = req.Reviews + item.AlbumName = req.AlbumName + item.AlbumArtist = req.AlbumArtist + item.Publisher = req.Publisher + item.ISBN = req.ISBN + item.Hidden = req.Hidden + + if err := db.UpdateMediaItem(item); err != nil { + common.ErrorResp(c, err, 500) + return + } + common.SuccessResp(c) +} + +// DeleteMediaItemAdmin 后台删除媒体条目 +func DeleteMediaItemAdmin(c *gin.Context) { + idStr := c.Query("id") + id, err := strconv.ParseUint(idStr, 10, 64) + if err != nil { + common.ErrorStrResp(c, "无效的ID", 400) + return + } + if err := db.DeleteMediaItem(uint(id)); err != nil { + common.ErrorResp(c, err, 500) + return + } + common.SuccessResp(c) +} + +// ClearMediaDB 清空指定类型媒体数据库 +func ClearMediaDB(c *gin.Context) { + mediaType := model.MediaType(c.Query("media_type")) + if mediaType == "" { + common.ErrorStrResp(c, "media_type 不能为空", 400) + return + } + if err := db.ClearMediaItems(mediaType); err != nil { + common.ErrorResp(c, err, 500) + return + } + common.SuccessResp(c) +} + +// ClearMediaScrape 清空刮削数据(保留扫描记录,但清空所有刮削结果字段) +// 参数:media_type 可选,留空表示所有类型;item_id 暂不支持(统一为类型/全部) +func ClearMediaScrape(c *gin.Context) { + mediaType := model.MediaType(c.Query("media_type")) + affected, err := db.ClearMediaScrapedData(mediaType) + if err != nil { + common.ErrorResp(c, err, 500) + return + } + // 同步清掉对应媒体配置中的最近刮削时间 + if mediaType != "" { + if cfg, e := db.GetMediaConfig(mediaType); e == nil && cfg != nil { + cfg.LastScrapeAt = nil + _ = db.SaveMediaConfig(cfg) + } + } else { + if cfgs, e := db.GetAllMediaConfigs(); e == nil { + for i := range cfgs { + cfgs[i].LastScrapeAt = nil + _ = db.SaveMediaConfig(&cfgs[i]) + } + } + } + common.SuccessResp(c, gin.H{"affected": affected}) +} + +// DeleteInvalidMedia 删除已失效的媒体条目(即对应文件 / 文件夹在存储中已不存在) +// 参数:media_type 可选,留空表示扫描所有类型 +// 返回:检测总数、被删除条目数 +func DeleteInvalidMedia(c *gin.Context) { + mediaType := model.MediaType(c.Query("media_type")) + + items, err := db.ListAllValidMediaItemsForCheck(mediaType) + if err != nil { + common.ErrorResp(c, err, 500) + return + } + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) + defer cancel() + + var invalidIDs []uint + var skipped int + for i := range items { + it := &items[i] + // 拼接文件 / 文件夹的完整 VFS 路径 + // - 文件夹模式 (is_folder=true): folder_path 即文件夹自身(扫描根 + 文件夹名 视情况而定);这里直接使用 folder_path/file_name + // - 普通文件: folder_path 是所在目录,file_name 是文件名 + fullPath := stdpath.Join(it.FolderPath, it.FileName) + if fullPath == "" || fullPath == "/" { + continue + } + if _, gerr := fs.Get(ctx, fullPath, &fs.GetArgs{NoLog: true}); gerr != nil { + // 只有「确认不存在」才算失效条目;驱动故障、超时、鉴权失败等可能是临时问题, + // 一律跳过(保留记录),否则一次网络抖动就会把有效的媒体记录硬删除。 + if errs.IsObjectNotFound(gerr) { + invalidIDs = append(invalidIDs, it.ID) + continue + } + skipped++ + log.Warnf("跳过无法确认状态的媒体条目 %s(%s): %v", fullPath, it.FileName, gerr) + } + } + + if len(invalidIDs) > 0 { + if err := db.DeleteMediaItemsByIDs(invalidIDs); err != nil { + common.ErrorResp(c, err, 500) + return + } + } + log.Infof("删除已失效媒体条目:检测 %d 条,删除 %d 条,跳过 %d 条 (media_type=%q)", len(items), len(invalidIDs), skipped, mediaType) + common.SuccessResp(c, gin.H{ + "checked": len(items), + "deleted": len(invalidIDs), + "skipped": skipped, + }) +} + +// ==================== 扫描与刮削 ==================== + +// ScanMediaReq 扫描请求 +type ScanMediaReq struct { + MediaType model.MediaType `json:"media_type" binding:"required"` + ScanPathID uint `json:"scan_path_id"` // 0 表示扫描全部路径 +} + +// StartMediaScan 开始扫描 +func StartMediaScan(c *gin.Context) { + var req ScanMediaReq + if err := c.ShouldBindJSON(&req); err != nil { + common.ErrorResp(c, err, 400) + return + } + cfg, err := db.GetMediaConfig(req.MediaType) + if err != nil { + common.ErrorResp(c, err, 500) + return + } + if !cfg.Enabled { + common.ErrorStrResp(c, "该媒体库未启用", 400) + return + } + + if req.ScanPathID > 0 { + // 扫描单个路径 + sp, err := db.GetMediaScanPath(req.ScanPathID) + if err != nil { + common.ErrorResp(c, err, 404) + return + } + media.ScanMediaPath(sp) + } else { + // 扫描全部路径 + media.ScanMedia(cfg) + } + common.SuccessResp(c) +} + +// GetMediaScanProgress 获取扫描进度 +func GetMediaScanProgress(c *gin.Context) { + mediaType := model.MediaType(c.Query("media_type")) + if mediaType == "" { + common.ErrorStrResp(c, "media_type 不能为空", 400) + return + } + progress := media.GetProgress(mediaType) + common.SuccessResp(c, progress) +} + +// MediaScrapeStatsResp 刮削统计响应 +// 包含 TMDB 视频刮削的累计指标,用于运维观测与命中率评估 +type MediaScrapeStatsResp struct { + TMDB scraper.TMDBStats `json:"tmdb"` +} + +// GetMediaScrapeStats 获取刮削统计(进程级累计值,重启清零) +// 返回字段: +// - total_scraped : 累计触发刮削次数 +// - hit_count : 命中(含强命中)次数 +// - low_confidence_count : 仅低置信度兜底命中次数 +// - no_match_count : 完全未命中次数 +// - total_attempts : 累计 TMDB 搜索请求次数 +// - total_search_millis : 累计搜索耗时(毫秒) +func GetMediaScrapeStats(c *gin.Context) { + common.SuccessResp(c, MediaScrapeStatsResp{ + TMDB: scraper.GetTMDBStats(), + }) +} + +// ScrapeMediaReq 刮削请求 +type ScrapeMediaReq struct { + MediaType model.MediaType `json:"media_type" binding:"required"` + ItemID uint `json:"item_id"` // 0 表示刮削全部未刮削的 +} + +// StartMediaScrape 开始刮削 +func StartMediaScrape(c *gin.Context) { + var req ScrapeMediaReq + if err := c.ShouldBindJSON(&req); err != nil { + common.ErrorResp(c, err, 400) + return + } + + cfg, err := db.GetMediaConfig(req.MediaType) + if err != nil { + common.ErrorResp(c, err, 500) + return + } + + // 从系统设置中读取刮削配置 + tmdbKey := setting.GetStr(conf.MediaTMDBKey) + tmdbAPIURL := setting.GetStr(conf.MediaTMDBAPIURL, "https://api.themoviedb.org") + discogsToken := setting.GetStr(conf.MediaDiscogsToken) + discogsAPIURL := setting.GetStr(conf.MediaDiscogsAPIURL, "https://api.discogs.com") + thumbnailMode := setting.GetStr(conf.MediaThumbnailMode, "runtime") + thumbnailPath := setting.GetStr(conf.MediaThumbnailPath, "/imgs") + storeThumbnail := setting.GetBool(conf.MediaStoreThumbnail) + bookSource := strings.ToLower(setting.GetStr(conf.MediaBookSource, "douban")) + googleBooksKey := setting.GetStr(conf.MediaGoogleBooksKey) + downloadCover := setting.GetBool(conf.MediaDownloadCover) + + // 该媒体类型如果单独配置过缩略图,覆盖全局设置 + if cfg.ScraperConfig != "" { + var sc struct { + StoreThumbnail string `json:"store_thumbnail"` + ThumbnailMode string `json:"thumbnail_mode"` + ThumbnailPath string `json:"thumbnail_path"` + } + if json.Unmarshal([]byte(cfg.ScraperConfig), &sc) == nil { + if sc.ThumbnailMode != "" { + thumbnailMode = sc.ThumbnailMode + } + if sc.ThumbnailPath != "" { + thumbnailPath = sc.ThumbnailPath + } + if sc.StoreThumbnail != "" { + storeThumbnail = sc.StoreThumbnail == "true" + } + } + } + + log.Infof("[media] scrape %s: thumbnail mode=%s, path=%s, runtime dir=%s", + req.MediaType, thumbnailMode, thumbnailPath, scraper.RuntimeThumbRoot(thumbnailPath)) + + var preloadedItems []model.MediaItem + if req.ItemID > 0 { + item, e := db.GetMediaItemByID(req.ItemID) + if e != nil { + common.ErrorResp(c, e, 404) + return + } + preloadedItems = []model.MediaItem{*item} + } + + go func() { + var items []model.MediaItem + var err error + + if req.ItemID > 0 { + items = preloadedItems + } else { + // limit 传 -1,GORM 不会拼接 LIMIT 子句,从而一次性取出全部未刮削条目 + items, err = db.GetUnscrappedItems(req.MediaType, -1) + if err != nil { + log.Errorf("获取未刮削条目失败: %v", err) + return + } + } + + // 读取并发度(默认5),并启动 worker pool + concurrency := setting.GetInt(conf.MediaScrapeConcurrency, defaultScrapeConcurrency) + if concurrency <= 0 { + concurrency = defaultScrapeConcurrency + } + if concurrency > len(items) && len(items) > 0 { + concurrency = len(items) + } + + var wg sync.WaitGroup + jobs := make(chan int, len(items)) + // 落库串行化:所有 worker 把刮削成功的 item 推到 saveCh,由单协程负责落库, + // 这样既能并发抓 API、又避免多 goroutine 同时写 sqlite/mysql 引发锁竞争。 + saveCh := make(chan *model.MediaItem, concurrency*2) + var saveWg sync.WaitGroup + + // 单写入协程:刮一个写一个 + saveWg.Add(1) + go func() { + defer saveWg.Done() + for it := range saveCh { + if err := db.UpdateMediaItem(it); err != nil { + log.Warnf("保存刮削结果失败 [id=%d, %s/%s]: %v", it.ID, it.FolderPath, it.FileName, err) + } + } + }() + + worker := func() { + defer wg.Done() + for idx := range jobs { + item := &items[idx] + var scrapeErr error + + switch req.MediaType { + case model.MediaTypeVideo: + s := scraper.NewTMDBScraper(tmdbKey, tmdbAPIURL) + scrapeErr = s.ScrapeVideo(item) + case model.MediaTypeMusic: + // 音乐:刮削前先用文件本身的 ID3/Vorbis tag + .lrc 做本地兜底, + // 仅填空字段;这样用户清空刮削后重新刮削也能恢复封面/歌词等。 + localCtx, localCancel := context.WithTimeout(context.Background(), 30*time.Second) + media.FillMusicFromLocal(localCtx, item) + localCancel() + + s := scraper.NewDiscogsScraper(discogsToken, discogsAPIURL) + scrapeErr = s.ScrapeMusic(item) + case model.MediaTypeBook: + var bookErr error + switch bookSource { + case "google": + bookErr = scraper.NewGoogleBooksScraper( + googleBooksKey, + thumbnailMode, + thumbnailPath, + ).ScrapeBook(item) + case "openlibrary": + bookErr = scraper.NewOpenLibraryScraper( + thumbnailMode, + thumbnailPath, + ).ScrapeBook(item) + default: + bookErr = scraper.NewDoubanScraperWithConfig( + thumbnailMode, + thumbnailPath, + ).ScrapeBook(item) + } + if bookErr != nil { + log.Debugf("书籍刮削失败 [%s/%s]: %v,将尝试本地提取封面", item.FolderPath, item.FileName, bookErr) + } + + if item.Cover == "" { + bookCtx, bookCancel := context.WithTimeout(context.Background(), 60*time.Second) + bookFilePath := item.FolderPath + "/" + item.FileName + bookReader := media.FetchFileReader(bookCtx, bookFilePath) + if bookReader != nil { + localScraper := scraper.NewBookLocalScraperWithConfig( + thumbnailMode, + thumbnailPath, + ) + if localCover := localScraper.ExtractLocalCover(item.FileName, bookFilePath, bookReader); localCover != "" { + item.Cover = localCover + } + _ = bookReader.Close() + } + bookCancel() + } + + if bookErr != nil && item.Cover == "" { + scrapeErr = bookErr + } + case model.MediaTypeImage: + imgCtx, imgCancel := context.WithTimeout(context.Background(), 30*time.Second) + imgFilePath := item.FolderPath + "/" + item.FileName + imgReader := media.FetchFileReader(imgCtx, imgFilePath) + s := scraper.NewImageScraperWithConfig( + storeThumbnail, + thumbnailMode, + thumbnailPath, + ) + scrapeErr = s.ScrapeImage(item, imgReader) + if imgReader != nil { + _ = imgReader.Close() + } + imgCancel() + } + + // 远程封面按需下载到本地存储(或 base64),避免直接保存图片链接 + if downloadCover && strings.HasPrefix(item.Cover, "http") { + rawPath := strings.TrimSuffix(item.FolderPath, "/") + "/" + item.FileName + if cover := scraper.DownloadCoverToStorage( + context.Background(), + item.Cover, + rawPath, + thumbnailMode, + thumbnailPath, + ); cover != "" { + item.Cover = cover + } + } + + if scrapeErr != nil { + log.Warnf("刮削失败 [%s] %s: %v", req.MediaType, item.FolderPath, scrapeErr) + // 即使远端刮削失败,只要本地兜底已经写入了任意有效字段, + // 也保存一次(不标记 ScrapedAt,便于下次还能重试), + // 避免 cover/lyrics 等被丢弃。 + if hasAnyScrapedField(item) { + saveCh <- item + } + continue + } + now := time.Now() + item.ScrapedAt = &now + saveCh <- item + } + } + + for i := 0; i < concurrency; i++ { + wg.Add(1) + go worker() + } + + for i := range items { + jobs <- i + } + close(jobs) + wg.Wait() + close(saveCh) + saveWg.Wait() + + log.Infof("刮削完成 [%s],共处理 %d 条(并发=%d)", req.MediaType, len(items), concurrency) + }() + + _ = cfg + common.SuccessResp(c) +} + +// hasAnyScrapedField 判断 item 是否已经有任意一个有意义的刮削字段被填充 +// 用于:远端刮削失败但本地兜底已经写入了内容时,仍然落库保留这些信息 +func hasAnyScrapedField(item *model.MediaItem) bool { + if item == nil { + return false + } + return item.Cover != "" || + item.Lyrics != "" || + item.Description != "" || + item.Plot != "" || + item.Genre != "" || + item.Authors != "" || + item.AlbumArtist != "" || + item.ReleaseDate != "" || + item.Rating > 0 || + item.Publisher != "" || + item.ISBN != "" || + item.ExternalID != "" +} + +// ==================== 公开API(前端媒体库浏览) ==================== + +// PublicListMedia 公开媒体列表(前端浏览用) +func PublicListMedia(c *gin.Context) { + mediaType := model.MediaType(c.Query("media_type")) + page, _ := strconv.Atoi(c.DefaultQuery("page", "1")) + pageSize, _ := strconv.Atoi(c.DefaultQuery("page_size", "40")) + page, pageSize = clampMediaPage(page, pageSize, 40, 200) + orderBy := c.DefaultQuery("order_by", "name") + orderDir := c.DefaultQuery("order_dir", "asc") + folderPath := c.Query("folder_path") + keyword := c.Query("keyword") + typeTag := c.Query("type_tag") + contentTag := c.Query("content_tag") + scanPathIDStr := c.Query("scan_path_id") + var scanPathID uint + if scanPathIDStr != "" { + if id, err := strconv.ParseUint(scanPathIDStr, 10, 64); err == nil { + scanPathID = uint(id) + } + } + + hidden := false + user := mediaUser(c) + q := db.MediaItemQuery{ + MediaType: mediaType, + ScanPathID: scanPathID, + FolderPath: folderPath, + TypeTag: typeTag, + ContentTag: contentTag, + PathPrefix: userPathPrefix(user), + Hidden: &hidden, + Keyword: keyword, + OrderBy: orderBy, + OrderDir: orderDir, + Page: page, + PageSize: pageSize, + } + items, total, err := db.ListMediaItems(q) + if err != nil { + common.ErrorResp(c, err, 500) + return + } + // 元信息(meta) 级别的读权限无法下推到 SQL,这里对当前页做二次过滤 + filtered := make([]model.MediaItem, 0, len(items)) + for i := range items { + if CanUserAccessItem(user, &items[i]) { + filtered = append(filtered, items[i]) + } + } + common.SuccessResp(c, common.PageResp{Content: filtered, Total: total}) +} + +// PublicGetMedia 公开获取媒体详情 +func PublicGetMedia(c *gin.Context) { + idStr := c.Param("id") + id, err := strconv.ParseUint(idStr, 10, 64) + if err != nil { + common.ErrorStrResp(c, "无效的ID", 400) + return + } + item, err := db.GetMediaItemByID(uint(id)) + if err != nil { + common.ErrorResp(c, err, 404) + return + } + if item.Hidden || !CanUserAccessItem(mediaUser(c), item) { + common.ErrorStrResp(c, "资源不存在", 404) + return + } + common.SuccessResp(c, item) +} + +// PublicListAlbums 公开专辑列表(音乐) +func PublicListAlbums(c *gin.Context) { + page, _ := strconv.Atoi(c.DefaultQuery("page", "1")) + pageSize, _ := strconv.Atoi(c.DefaultQuery("page_size", "40")) + page, pageSize = clampMediaPage(page, pageSize, 40, 200) + keyword := c.Query("keyword") + scanPathIDStr := c.Query("scan_path_id") + var scanPathID uint + if scanPathIDStr != "" { + if id, err := strconv.ParseUint(scanPathIDStr, 10, 64); err == nil { + scanPathID = uint(id) + } + } + + hidden := false + q := db.MediaItemQuery{ + MediaType: model.MediaTypeMusic, + ScanPathID: scanPathID, + PathPrefix: userPathPrefix(mediaUser(c)), + Hidden: &hidden, + Keyword: keyword, + Page: page, + PageSize: pageSize, + } + albums, total, err := db.ListAlbums(q) + if err != nil { + common.ErrorResp(c, err, 500) + return + } + common.SuccessResp(c, common.PageResp{Content: albums, Total: total}) +} + +// PublicGetAlbum 公开获取专辑详情及曲目 +func PublicGetAlbum(c *gin.Context) { + albumName := c.Query("album_name") + albumArtist := c.Query("album_artist") + if albumName == "" && albumArtist == "" { + common.ErrorStrResp(c, "album_name 和 album_artist 不能同时为空", 400) + return + } + tracks, err := db.GetAlbumTracks(albumName, albumArtist) + if err != nil { + common.ErrorResp(c, err, 500) + return + } + user := mediaUser(c) + filtered := make([]model.MediaItem, 0, len(tracks)) + for i := range tracks { + if CanUserAccessItem(user, &tracks[i]) { + filtered = append(filtered, tracks[i]) + } + } + common.SuccessResp(c, filtered) +} + +// PublicListFolders 公开文件夹列表(目录浏览模式) +func PublicListFolders(c *gin.Context) { + mediaType := model.MediaType(c.Query("media_type")) + if mediaType == "" { + common.ErrorStrResp(c, "media_type 不能为空", 400) + return + } + paths, err := db.ListFolderPaths(mediaType) + if err != nil { + common.ErrorResp(c, err, 500) + return + } + user := mediaUser(c) + filtered := make([]string, 0, len(paths)) + for _, p := range paths { + if CanUserAccessPath(user, p) { + filtered = append(filtered, p) + } + } + common.SuccessResp(c, filtered) +} + +// PublicListScanPaths 公开获取扫描路径列表(前端筛选用) +func PublicListScanPaths(c *gin.Context) { + mediaType := model.MediaType(c.Query("media_type")) + paths, err := db.ListMediaScanPaths(mediaType) + if err != nil { + common.ErrorResp(c, err, 500) + return + } + user := mediaUser(c) + filtered := make([]model.MediaScanPath, 0, len(paths)) + for _, p := range paths { + if CanUserAccessPath(user, p.Path) { + filtered = append(filtered, p) + } + } + common.SuccessResp(c, filtered) +} + +// ==================== 导入 / 导出 ==================== + +// MediaExportData 导出数据结构 +// 包含版本号便于后续兼容性升级;导出粒度由 scope 字段说明: +// - "all" :所有类型的全部数据 +// - "media_type" :单个媒体类型(video/music/image/book)的全部数据 +// - "scan_path" :单个扫描路径下的数据(仅 1 个路径 + 该路径下的条目) +type MediaExportData struct { + Version int `json:"version"` + ExportAt time.Time `json:"export_at"` + Scope string `json:"scope"` + MediaType model.MediaType `json:"media_type,omitempty"` + ScanPaths []model.MediaScanPath `json:"scan_paths"` + Items []model.MediaItem `json:"items"` +} + +const mediaExportVersion = 1 + +// ExportMediaDB 导出媒体数据 +// query: media_type(可选,为空表示全部类型), scan_path_id(可选,单路径导出) +// 返回 JSON 文件下载流 +func ExportMediaDB(c *gin.Context) { + mediaType := model.MediaType(c.Query("media_type")) + scanPathIDStr := c.Query("scan_path_id") + + export := MediaExportData{ + Version: mediaExportVersion, + ExportAt: time.Now(), + } + + if scanPathIDStr != "" { + // 按扫描路径导出 + id, err := strconv.ParseUint(scanPathIDStr, 10, 64) + if err != nil { + common.ErrorStrResp(c, "无效的 scan_path_id", 400) + return + } + sp, err := db.GetMediaScanPath(uint(id)) + if err != nil { + common.ErrorResp(c, err, 404) + return + } + items, err := db.ListMediaItemsByScanPath(uint(id)) + if err != nil { + common.ErrorResp(c, err, 500) + return + } + export.Scope = "scan_path" + export.MediaType = sp.MediaType + export.ScanPaths = []model.MediaScanPath{*sp} + export.Items = items + } else if mediaType != "" { + // 按媒体类型导出 + paths, err := db.ListMediaScanPaths(mediaType) + if err != nil { + common.ErrorResp(c, err, 500) + return + } + items, err := db.ListAllMediaItems(mediaType) + if err != nil { + common.ErrorResp(c, err, 500) + return + } + export.Scope = "media_type" + export.MediaType = mediaType + export.ScanPaths = paths + export.Items = items + } else { + // 全部导出 + paths, err := db.ListAllMediaScanPaths() + if err != nil { + common.ErrorResp(c, err, 500) + return + } + items, err := db.ListAllMediaItems("") + if err != nil { + common.ErrorResp(c, err, 500) + return + } + export.Scope = "all" + export.ScanPaths = paths + export.Items = items + } + + // 直接以 JSON 流下载 + filename := buildExportFilename(export) + c.Header("Content-Disposition", `attachment; filename="`+filename+`"`) + c.Header("Content-Type", "application/json; charset=utf-8") + if err := json.NewEncoder(c.Writer).Encode(export); err != nil { + log.Errorf("导出媒体数据失败: %v", err) + } +} + +// buildExportFilename 根据导出范围生成下载文件名 +func buildExportFilename(d MediaExportData) string { + ts := d.ExportAt.Format("20060102_150405") + switch d.Scope { + case "scan_path": + spName := "" + if len(d.ScanPaths) > 0 { + spName = d.ScanPaths[0].Name + if spName == "" { + spName = d.ScanPaths[0].Path + } + } + // 去掉路径中的斜杠等字符,避免文件名非法 + safe := strings.NewReplacer("/", "_", "\\", "_", ":", "_", "?", "_", "*", "_", "\"", "_", "<", "_", ">", "_", "|", "_").Replace(spName) + return "media_export_" + string(d.MediaType) + "_" + safe + "_" + ts + ".json" + case "media_type": + return "media_export_" + string(d.MediaType) + "_" + ts + ".json" + default: + return "media_export_all_" + ts + ".json" + } +} + +// ImportMediaDB 导入媒体数据 +// 支持两种调用方式: +// 1. multipart/form-data:file 字段携带导出生成的 JSON 文件 +// 2. application/json:直接 POST 整个 MediaExportData 结构 +// +// 可选 query 参数 scan_path_id:当导入"scan_path"范围数据时, +// 把所有条目重定向到该扫描路径ID(覆盖文件中原有的 scan_path_id)。 +func ImportMediaDB(c *gin.Context) { + var data MediaExportData + + contentType := c.GetHeader("Content-Type") + if strings.HasPrefix(contentType, "multipart/form-data") { + fileHeader, err := c.FormFile("file") + if err != nil { + common.ErrorStrResp(c, "缺少 file 文件参数", 400) + return + } + f, err := fileHeader.Open() + if err != nil { + common.ErrorResp(c, err, 400) + return + } + defer f.Close() + if err := json.NewDecoder(f).Decode(&data); err != nil { + common.ErrorStrResp(c, "导入文件解析失败: "+err.Error(), 400) + return + } + } else { + if err := c.ShouldBindJSON(&data); err != nil { + common.ErrorResp(c, err, 400) + return + } + } + + if data.Version <= 0 || data.Version > mediaExportVersion { + common.ErrorStrResp(c, "不支持的导入文件版本", 400) + return + } + + // 1) 先导入扫描路径,建立 oldID -> newID 的映射 + idMap, spCreated, spUpdated, err := db.ImportMediaScanPaths(data.ScanPaths) + if err != nil { + common.ErrorResp(c, err, 500) + return + } + + // 2) 再导入条目,根据 idMap 把 scan_path_id 重定向到新ID + // 修正每个条目的 scan_path_id: + // - 如果 query 指定了 scan_path_id,全部覆盖为该值(按扫描路径重定向导入) + // - 否则按 idMap 自动重定向;找不到映射的保留原值(通常发生在该路径未一并导入时) + scanPathIDStr := c.Query("scan_path_id") + var override *uint + if scanPathIDStr != "" { + if id, err := strconv.ParseUint(scanPathIDStr, 10, 64); err == nil { + v := uint(id) + override = &v + } + } + if override == nil { + for i := range data.Items { + if newID, ok := idMap[data.Items[i].ScanPathID]; ok { + data.Items[i].ScanPathID = newID + } + } + } + + itemCreated, itemUpdated, err := db.ImportMediaItems(data.Items, override) + if err != nil { + common.ErrorResp(c, err, 500) + return + } + + common.SuccessResp(c, gin.H{ + "scan_paths_created": spCreated, + "scan_paths_updated": spUpdated, + "items_created": itemCreated, + "items_updated": itemUpdated, + }) +} diff --git a/server/handles/media_extra.go b/server/handles/media_extra.go new file mode 100644 index 00000000..54e0974d --- /dev/null +++ b/server/handles/media_extra.go @@ -0,0 +1,277 @@ +package handles + +import ( + "encoding/json" + "fmt" + "net/url" + stdpath "path" + "strconv" + "strings" + + "github.com/OpenListTeam/OpenList/v4/internal/conf" + "github.com/OpenListTeam/OpenList/v4/internal/db" + "github.com/OpenListTeam/OpenList/v4/internal/media" + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/internal/setting" + "github.com/OpenListTeam/OpenList/v4/server/common" + "github.com/gin-gonic/gin" +) + +// MediaStatus 媒体库状态(前端据此决定是否渲染媒体库入口/侧栏) +type MediaStatus struct { + Enabled bool `json:"enabled"` + Types []string `json:"types"` + EmbyEnabled bool `json:"emby_enabled"` + SubsonicEnabled bool `json:"subsonic_enabled"` + EmbyBaseURL string `json:"emby_base_url"` + SubsonicBaseURL string `json:"subsonic_base_url"` + StrmPath string `json:"strm_path"` + StrmOnScan bool `json:"strm_on_scan"` + StrmSameDir bool `json:"strm_same_dir"` + ThumbPath string `json:"thumb_path"` + Version string `json:"version"` + ProxyCover bool `json:"proxy_cover"` +} + +// PublicMediaStatus 公开的媒体库状态 +func PublicMediaStatus(c *gin.Context) { + // Types 必须是非 nil 切片:前端会直接调用 .includes(),nil 序列化成 null 会抛异常 + status := MediaStatus{Enabled: MediaEnabled(), Types: []string{}} + apiBase := common.GetApiUrl(c.Request.Context()) + status.StrmPath = media.DefaultStrmOptions(apiBase).OutputPath + status.StrmOnScan = setting.GetBool(conf.MediaStrmOnScan) + status.StrmSameDir = setting.GetBool(conf.MediaStrmSameDir) + status.ThumbPath = setting.GetStr(conf.MediaThumbnailPath, "/imgs") + status.Version = conf.Version + status.ProxyCover = setting.GetBool(conf.MediaProxyCover) + if !status.Enabled { + common.SuccessResp(c, status) + return + } + for _, mt := range MediaTypeEnabledList() { + status.Types = append(status.Types, string(mt)) + } + status.EmbyEnabled = MediaEmbyEnabled() + status.SubsonicEnabled = MediaSubsonicEnabled() + status.EmbyBaseURL = strings.TrimSuffix(apiBase, "/") + "/@emby" + status.SubsonicBaseURL = strings.TrimSuffix(apiBase, "/") + "/@rest" + common.SuccessResp(c, status) +} + +// ==================== 应用密钥 ==================== + +// ListMediaAppKeys 列出当前管理员的媒体应用密钥 +func ListMediaAppKeys(c *gin.Context) { + user := mediaUser(c) + if user == nil { + common.ErrorStrResp(c, "未登录", 401) + return + } + keys, err := db.ListMediaAppKeys(user.ID) + if err != nil { + common.ErrorResp(c, err, 500) + return + } + common.SuccessResp(c, keys) +} + +// CreateMediaAppKey 创建应用密钥(明文只在创建时返回一次) +func CreateMediaAppKey(c *gin.Context) { + user := mediaUser(c) + if user == nil { + common.ErrorStrResp(c, "未登录", 401) + return + } + var req struct { + Name string `json:"name"` + } + _ = c.ShouldBindJSON(&req) + name := strings.TrimSpace(req.Name) + if name == "" { + name = "Media Key" + } + plain, err := media.NewAppKeyPlain() + if err != nil { + common.ErrorResp(c, err, 500) + return + } + secret, err := media.EncryptAppKey(plain) + if err != nil { + common.ErrorResp(c, err, 500) + return + } + key := &model.MediaAppKey{ + UserID: user.ID, + Name: name, + Prefix: media.AppKeyPrefix(plain), + Secret: secret, + } + if err := db.CreateMediaAppKey(key); err != nil { + common.ErrorResp(c, err, 500) + return + } + common.SuccessResp(c, gin.H{ + "id": key.ID, + "name": key.Name, + "prefix": key.Prefix, + "key": plain, + }) +} + +// DeleteMediaAppKey 删除应用密钥 +func DeleteMediaAppKey(c *gin.Context) { + user := mediaUser(c) + if user == nil { + common.ErrorStrResp(c, "未登录", 401) + return + } + id, err := strconv.ParseUint(c.Query("id"), 10, 64) + if err != nil { + common.ErrorStrResp(c, "无效的ID", 400) + return + } + if err := db.DeleteMediaAppKey(user.ID, uint(id)); err != nil { + common.ErrorResp(c, err, 500) + return + } + common.SuccessResp(c, nil) +} + +// ==================== STRM ==================== + +// GenerateMediaStrm 手动批量生成 STRM +func GenerateMediaStrm(c *gin.Context) { + var req struct { + MediaType model.MediaType `json:"media_type"` + ScanPathID uint `json:"scan_path_id"` + } + _ = c.ShouldBindJSON(&req) + apiRoot := common.GetApiUrl(c.Request.Context()) + media.SetAPIRoot(apiRoot) + opts := media.DefaultStrmOptions(apiRoot) + if !opts.SameDir && opts.OutputPath == "" { + common.ErrorStrResp(c, "未配置 STRM 输出目录", 400) + return + } + n, err := media.GenerateStrmForType(c.Request.Context(), req.MediaType, req.ScanPathID, opts) + if err != nil { + common.ErrorResp(c, err, 500) + return + } + outPath := opts.OutputPath + if opts.SameDir { + outPath = "与媒体文件同目录" + } + common.SuccessResp(c, gin.H{"written": n, "path": outPath}) +} + +// GetMediaStreamURL 返回条目(或其中一集)的签名直链,供本地播放器/外链使用 +func GetMediaStreamURL(c *gin.Context) { + // 直接按路径取签名链接(用于选集等场景) + if rawPath := c.Query("path"); rawPath != "" { + if !CanUserAccessPath(mediaUser(c), rawPath) { + common.ErrorStrResp(c, "资源不存在", 404) + return + } + common.SuccessResp(c, gin.H{ + "url": SignedStreamURL(c, rawPath), + "path": rawPath, + "name": stdpath.Base(rawPath), + "proxied": SignedProxyURL(c, rawPath), + }) + return + } + id, err := strconv.ParseUint(c.Query("id"), 10, 64) + if err != nil { + common.ErrorStrResp(c, "无效的ID", 400) + return + } + item, err := db.GetMediaItemByID(uint(id)) + if err != nil || !CanUserAccessItem(mediaUser(c), item) { + common.ErrorStrResp(c, "资源不存在", 404) + return + } + episodeIndex, _ := strconv.Atoi(c.DefaultQuery("episode", "0")) + rawPath := MediaItemFilePath(item) + if episodeIndex > 0 { + var eps []model.EpisodeInfo + if err := json.Unmarshal([]byte(item.Episodes), &eps); err == nil { + for i := range eps { + if eps[i].Index == episodeIndex || i+1 == episodeIndex { + rawPath = MediaEpisodeFilePath(item, &eps[i]) + break + } + } + } + } + // 选集是目录里的另一个文件,必须对最终文件再校验一次: + // 目录可见不代表其中每个文件都可见(例如 meta 隐藏规则单独禁止某个文件), + // 而签名之后的下载链路只验签名,不会重新执行用户的隐藏规则。 + if !CanUserAccessPath(mediaUser(c), rawPath) { + common.ErrorStrResp(c, "资源不存在", 404) + return + } + common.SuccessResp(c, gin.H{ + "url": SignedStreamURL(c, rawPath), + "path": rawPath, + "name": stdpath.Base(rawPath), + "proxied": SignedProxyURL(c, rawPath), + }) +} + +// GetMediaStrm 动态生成单个条目的 .strm 内容(详情页下载) +func GetMediaStrm(c *gin.Context) { + id, err := strconv.ParseUint(c.Query("id"), 10, 64) + if err != nil { + common.ErrorStrResp(c, "无效的ID", 400) + return + } + item, err := db.GetMediaItemByID(uint(id)) + if err != nil { + common.ErrorStrResp(c, "资源不存在", 404) + return + } + if !CanUserAccessItem(mediaUser(c), item) { + common.ErrorStrResp(c, "资源不存在", 404) + return + } + episodeIndex, _ := strconv.Atoi(c.DefaultQuery("episode", "0")) + + var rawPath string + var filename string + if episodeIndex > 0 { + var eps []model.EpisodeInfo + if err := json.Unmarshal([]byte(item.Episodes), &eps); err == nil { + for i := range eps { + if eps[i].Index == episodeIndex || i+1 == episodeIndex { + rawPath = MediaEpisodeFilePath(item, &eps[i]) + filename = fmt.Sprintf("%s S%02dE%02d.strm", item.ScrapedName, item.Season, episodeIndex) + break + } + } + } + } + if rawPath == "" { + rawPath = MediaItemFilePath(item) + if item.IsFolder { + filename = item.ScrapedName + ".strm" + } else { + filename = strings.TrimSuffix(item.FileName, stdpath.Ext(item.FileName)) + ".strm" + } + } + // 与 GetMediaStreamURL 同理:选集解析出来的是另一个文件,需要按最终路径再校验一次 + if !CanUserAccessPath(mediaUser(c), rawPath) { + common.ErrorStrResp(c, "资源不存在", 404) + return + } + if filename == "" { + filename = "media.strm" + } + + apiRoot := common.GetApiUrl(c.Request.Context()) + opts := media.DefaultStrmOptions(apiRoot) + content := media.RenderStrmContent(opts.Template, apiRoot, rawPath) + c.Header("Content-Type", "text/plain; charset=utf-8") + c.Header("Content-Disposition", fmt.Sprintf("attachment; filename*=UTF-8''%s", url.PathEscape(filename))) + c.String(200, content) +} diff --git a/server/handles/media_gate.go b/server/handles/media_gate.go new file mode 100644 index 00000000..c6fccbef --- /dev/null +++ b/server/handles/media_gate.go @@ -0,0 +1,222 @@ +package handles + +import ( + "crypto/subtle" + "fmt" + "net/url" + stdpath "path" + "strings" + + "github.com/OpenListTeam/OpenList/v4/internal/conf" + "github.com/OpenListTeam/OpenList/v4/internal/db" + "github.com/OpenListTeam/OpenList/v4/internal/errs" + "github.com/OpenListTeam/OpenList/v4/internal/media" + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/internal/op" + "github.com/OpenListTeam/OpenList/v4/internal/setting" + "github.com/OpenListTeam/OpenList/v4/internal/sign" + "github.com/OpenListTeam/OpenList/v4/pkg/utils" + "github.com/OpenListTeam/OpenList/v4/server/common" + "github.com/gin-gonic/gin" + "github.com/pkg/errors" +) + +// MediaEnabled 媒体库总开关是否开启 +func MediaEnabled() bool { + return setting.GetBool(conf.MediaEnabled) +} + +// MediaEmbyEnabled Emby 兼容 API 是否可用(依赖总开关) +func MediaEmbyEnabled() bool { + return MediaEnabled() && setting.GetBool(conf.MediaEmbyEnabled) +} + +// MediaSubsonicEnabled OpenSubsonic API 是否可用(依赖总开关) +func MediaSubsonicEnabled() bool { + return MediaEnabled() && setting.GetBool(conf.MediaSubsonicEnabled) +} + +// RequireMediaEnabled 媒体库关闭时统一返回 404,使其表现为"功能不存在" +func RequireMediaEnabled(c *gin.Context) { + if !MediaEnabled() { + c.AbortWithStatus(404) + } +} + +// RequireMediaProtocol 校验指定协议开关 +func RequireMediaProtocol(enabled func() bool) gin.HandlerFunc { + return func(c *gin.Context) { + if !enabled() { + c.AbortWithStatus(404) + } + } +} + +// mediaUser 返回当前请求的登录用户,未登录时为 nil +func mediaUser(c *gin.Context) *model.User { + user, _ := c.Request.Context().Value(conf.UserKey).(*model.User) + return user +} + +// CanUserAccessPath 判断用户是否有权访问指定 VFS 路径 +// 规则与文件浏览保持一致:base_path 前缀 + 元信息(meta) 读权限 + 隐藏文件规则。 +// +// user 为 nil 一律拒绝(fail-closed):所有调用点都应经过 Auth / embyAuth 拿到用户, +// 缺失用户说明鉴权链路出了问题,不能默认放行;被禁用的访客同样拒绝。 +func CanUserAccessPath(user *model.User, reqPath string) bool { + if user == nil || (user.IsGuest() && user.Disabled) { + return false + } + reqPath = utils.FixAndCleanPath(reqPath) + base := utils.FixAndCleanPath(user.BasePath) + if base == "" { + base = "/" + } + if base != "/" && !utils.IsSubPath(base, reqPath) && !utils.PathEqual(base, reqPath) { + return false + } + meta, err := op.GetNearestMeta(reqPath) + if err != nil && !errors.Is(errors.Cause(err), errs.MetaNotFound) { + return false + } + // media 播放链路无法携带目录密码,因此受密码保护的目录视为不可访问 + return common.CanAccess(user, meta, reqPath, "") +} + +// userPathPrefix 返回用于 SQL 前缀过滤的用户目录,全盘用户返回空串 +func userPathPrefix(user *model.User) string { + if user == nil { + return "" + } + base := utils.FixAndCleanPath(user.BasePath) + if base == "" || base == "/" { + return "" + } + return base +} + +// mediaFolderOf 返回条目所在目录 +func mediaFolderOf(item *model.MediaItem) string { + if item == nil { + return "/" + } + folder := strings.TrimSuffix(item.FolderPath, "/") + if folder == "" { + folder = "/" + } + return folder +} + +// MediaItemFilePath 拼接媒体条目对应的实际文件路径 +func MediaItemFilePath(item *model.MediaItem) string { + if item == nil { + return "" + } + folder := strings.TrimSuffix(item.FolderPath, "/") + if folder == "" { + folder = "/" + } + return stdpath.Join(folder, item.FileName) +} + +// MediaEpisodeFilePath 拼接文件夹型条目的单集文件路径 +func MediaEpisodeFilePath(item *model.MediaItem, ep *model.EpisodeInfo) string { + if item == nil || ep == nil { + return "" + } + return stdpath.Join(MediaItemFilePath(item), ep.FileName) +} + +// CanUserAccessItem 判断用户是否可访问某个媒体条目(含所属媒体类型开关) +func CanUserAccessItem(user *model.User, item *model.MediaItem) bool { + if item == nil || item.Hidden { + return false + } + if !MediaTypeEnabled(item.MediaType) { + return false + } + return CanUserAccessPath(user, MediaItemFilePath(item)) +} + +// MediaTypeEnabled 判断某媒体类型是否启用扫描 +func MediaTypeEnabled(mediaType model.MediaType) bool { + cfg, err := db.GetMediaConfig(mediaType) + if err != nil || cfg == nil { + return false + } + return cfg.Enabled +} + +// MediaTypeEnabledList 返回已启用的媒体类型 +func MediaTypeEnabledList() []model.MediaType { + all := []model.MediaType{model.MediaTypeVideo, model.MediaTypeMusic, model.MediaTypeImage, model.MediaTypeBook} + enabled := make([]model.MediaType, 0, len(all)) + for _, mt := range all { + if MediaTypeEnabled(mt) { + enabled = append(enabled, mt) + } + } + return enabled +} + +// LookUpAppKeyUser 按明文应用密钥查找对应用户与密钥记录。 +// Emby 与 Subsonic 的所有密钥认证都走这里,避免两套实现产生行为差异。 +func LookUpAppKeyUser(key string) (*model.User, *model.MediaAppKey, bool) { + key = strings.TrimSpace(key) + if !strings.HasPrefix(key, "olmp_") { + return nil, nil, false + } + keys, err := db.ListMediaAppKeysByPrefix(media.AppKeyPrefix(key)) + if err != nil { + return nil, nil, false + } + for i := range keys { + plain, err := media.DecryptAppKey(keys[i].Secret) + if err != nil { + continue + } + if subtle.ConstantTimeCompare([]byte(plain), []byte(key)) != 1 { + continue + } + user, err := op.GetUserById(keys[i].UserID) + if err != nil || user.Disabled || user.IsGuest() { + return nil, nil, false + } + db.TouchMediaAppKey(keys[i].ID) + return user, &keys[i], true + } + return nil, nil, false +} + +// SignedStreamURL 构造带签名的 /d 直链(绝对地址,供 Emby/Subsonic/STRM 使用) +func SignedStreamURL(c *gin.Context, rawPath string) string { + return SignedStreamURLWithBase(common.GetApiUrl(c.Request.Context()), rawPath) +} + +// SignedStreamURLWithBase 使用指定站点前缀构造签名直链 +func SignedStreamURLWithBase(apiBase, rawPath string) string { + apiBase = strings.TrimSuffix(apiBase, "/") + p := rawPath + if !strings.HasPrefix(p, "/") { + p = "/" + p + } + return fmt.Sprintf("%s/d%s?sign=%s", apiBase, utils.EncodePath(p), url.QueryEscape(sign.Sign(rawPath))) +} + +// SignedProxyURL 构造经服务端代理的签名链接(/p) +func SignedProxyURL(c *gin.Context, rawPath string) string { + apiBase := strings.TrimSuffix(common.GetApiUrl(c.Request.Context()), "/") + p := rawPath + if !strings.HasPrefix(p, "/") { + p = "/" + p + } + return fmt.Sprintf("%s/p%s?sign=%s", apiBase, utils.EncodePath(p), url.QueryEscape(sign.Sign(rawPath))) +} + +// MediaStreamURL 返回媒体条目(或其中一集)的签名直链 +func MediaStreamURL(c *gin.Context, item *model.MediaItem, ep *model.EpisodeInfo) string { + if ep != nil { + return SignedStreamURL(c, MediaEpisodeFilePath(item, ep)) + } + return SignedStreamURL(c, MediaItemFilePath(item)) +} diff --git a/server/handles/media_image.go b/server/handles/media_image.go new file mode 100644 index 00000000..87d50273 --- /dev/null +++ b/server/handles/media_image.go @@ -0,0 +1,201 @@ +package handles + +import ( + "context" + "fmt" + "io" + "net" + "net/http" + "net/url" + stdpath "path" + "path/filepath" + "strings" + "time" + + "github.com/OpenListTeam/OpenList/v4/internal/conf" + "github.com/OpenListTeam/OpenList/v4/internal/media/imgmime" + "github.com/OpenListTeam/OpenList/v4/internal/media/scraper" + "github.com/OpenListTeam/OpenList/v4/internal/setting" + "github.com/OpenListTeam/OpenList/v4/pkg/utils" + "github.com/gin-gonic/gin" +) + +// 允许代理的封面来源域名(TMDB / 豆瓣 / Discogs)。 +// 仅代理图片,避免该接口被当作通用反向代理滥用。 +var coverAllowedHosts = []string{ + "image.tmdb.org", + "tmdb.org", + "doubanio.com", + "douban.com", + "discogs.com", +} + +func hostAllowed(host string) bool { + host = strings.ToLower(strings.TrimSuffix(host, ".")) + if host == "" || host == "localhost" || strings.HasSuffix(host, ".local") { + return false + } + // 显式拒绝 IP 字面量,避免白名单域名被解析/重定向到内网 + if ip := net.ParseIP(host); ip != nil { + return false + } + for _, h := range coverAllowedHosts { + if host == h || strings.HasSuffix(host, "."+h) { + return true + } + } + return false +} + +// safeCoverURL 校验封面地址的协议与域名(含重定向后的每一跳) +func safeCoverURL(u *url.URL) error { + if u == nil { + return fmt.Errorf("empty url") + } + if u.Scheme != "http" && u.Scheme != "https" { + return fmt.Errorf("unsupported scheme") + } + if !hostAllowed(u.Hostname()) { + return fmt.Errorf("host not allowed") + } + return nil +} + +// PublicMediaProxy 代理外部封面图(TMDB/豆瓣/Discogs),解决客户端直连被墙的问题。 +// 用法:/api/public/media/proxy?url=https%3A%2F%2Fimage.tmdb.org%2F... +func PublicMediaProxy(c *gin.Context) { + if !MediaEnabled() { + c.Status(http.StatusNotFound) + return + } + raw := c.Query("url") + if raw == "" { + c.Status(http.StatusBadRequest) + return + } + u, err := url.Parse(raw) + if err != nil || safeCoverURL(u) != nil { + c.Status(http.StatusForbidden) + return + } + resp, err := fetchCover(c.Request.Context(), u.String()) + if err != nil && setting.GetBool(conf.MediaCoverGateway) { + // 直连失败时回退到公共图片网关(部分网络环境下 TMDB 图片域名不可达)。 + // 这会把封面 URL 交给第三方 wsrv.nl,因此默认关闭,需管理员显式开启。 + gateway := "https://wsrv.nl/?url=" + url.QueryEscape(strings.TrimPrefix(strings.TrimPrefix(u.String(), "https://"), "http://")) + resp, err = fetchCover(c.Request.Context(), gateway) + } + if err != nil { + utils.Log.Warnf("media cover proxy failed: %s: %v", u.String(), err) + c.Status(http.StatusBadGateway) + return + } + defer resp.Body.Close() + // 不信任上游 Content-Type:读取有限字节后按魔数判定,只输出四种栅格图片, + // 避免 image/svg+xml 等可执行脚本的类型以本站源被浏览器渲染。 + data, err := io.ReadAll(io.LimitReader(resp.Body, 8<<20)) + if err != nil || len(data) == 0 { + c.Status(http.StatusBadGateway) + return + } + ct := imgmime.Sniff(data) + if ct == "" { + c.Status(http.StatusUnsupportedMediaType) + return + } + setSafeImageHeaders(c) + c.Header("Cache-Control", "public, max-age=604800") + c.Data(http.StatusOK, ct, data) +} + +// setSafeImageHeaders 为用户可影响内容的图片响应加上防 XSS 头。 +func setSafeImageHeaders(c *gin.Context) { + c.Header("X-Content-Type-Options", "nosniff") + c.Header("Content-Disposition", `inline; filename="cover"`) + c.Header("Content-Security-Policy", "default-src 'none'; sandbox") +} + +// fetchCover 拉取图片,仅接受 200 且 Content-Type 为 image/* +// 重定向的每一跳都重新做域名校验,避免白名单域名 302 到内网造成 SSRF。 +func fetchCover(ctx context.Context, raw string) (*http.Response, error) { + client := &http.Client{ + Timeout: 20 * time.Second, + CheckRedirect: func(req *http.Request, via []*http.Request) error { + if len(via) >= 3 { + return fmt.Errorf("too many redirects") + } + return safeCoverURL(req.URL) + }, + } + req, err := http.NewRequestWithContext(ctx, http.MethodGet, raw, nil) + if err != nil { + return nil, err + } + req.Header.Set("User-Agent", "Mozilla/5.0 (compatible; OpenList-Media/1.0)") + resp, err := client.Do(req) + if err != nil { + return nil, err + } + if resp.StatusCode != http.StatusOK { + _ = resp.Body.Close() + return nil, fmt.Errorf("unexpected status %d", resp.StatusCode) + } + return resp, nil +} + +// thumbnailRoot 返回运行目录模式下缩略图的本地根目录(/) +func thumbnailRoot() string { + dir := setting.GetStr(conf.MediaThumbnailPath, "imgs") + return scraper.RuntimeThumbRoot(dir) +} + +// PublicMediaThumb 读取本地缩略图目录中的文件。 +// 缩略图由刮削器写入本地磁盘(非 VFS),因此不能用 /d 读取。 +// 用法:/api/public/media/thumb?path=/imgs/xxx.jpg +func PublicMediaThumb(c *gin.Context) { + if !MediaEnabled() { + c.Status(http.StatusNotFound) + return + } + reqPath := strings.TrimPrefix(stdpath.Clean("/"+c.Query("path")), "/") + // stdpath.Clean 已消解 ../ 段;这里不能再用 Contains("..") 粗暴过滤,会误杀 a..b.jpg 这类合法文件名。 + // 越界由下方 filepath.Rel 校验兜底。 + if reqPath == "" || reqPath == "." { + c.Status(http.StatusBadRequest) + return + } + base := strings.Trim(setting.GetStr(conf.MediaThumbnailPath, "/imgs"), "/") + if base != "" { + if !strings.HasPrefix(reqPath, base+"/") { + c.Status(http.StatusNotFound) + return + } + reqPath = strings.TrimPrefix(reqPath, base+"/") + } + root := thumbnailRoot() + full := filepath.Join(root, filepath.FromSlash(reqPath)) + rel, err := filepath.Rel(root, full) + if err != nil || rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) { + c.Status(http.StatusForbidden) + return + } + // 只按扩展名白名单输出栅格图片,拒绝 svg / html 等可执行类型 + var ct string + switch strings.ToLower(filepath.Ext(full)) { + case ".jpg", ".jpeg": + ct = "image/jpeg" + case ".png": + ct = "image/png" + case ".gif": + ct = "image/gif" + case ".webp": + ct = "image/webp" + default: + c.Status(http.StatusUnsupportedMediaType) + return + } + setSafeImageHeaders(c) + c.Header("Content-Type", ct) + c.Header("Cache-Control", "public, max-age=604800") + c.File(full) +} diff --git a/server/handles/media_ratelimit.go b/server/handles/media_ratelimit.go new file mode 100644 index 00000000..b8a37d70 --- /dev/null +++ b/server/handles/media_ratelimit.go @@ -0,0 +1,92 @@ +package handles + +import ( + "strings" + + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/go-cache" + "github.com/gin-gonic/gin" +) + +// 媒体协议(Emby / Subsonic)的认证入口不经过 /api/auth/login, +// 因此必须自行复用 LoginCache 的失败计数与锁定策略,否则等于附送一个无限次数的爆破接口。 +// 使用独立的 key 前缀,避免 Subsonic 客户端的失败重试把 Web 登录一并锁死。 + +func mediaAuthKey(c *gin.Context) string { + return "media:" + c.ClientIP() +} + +// mediaAuthLocked 当前 IP 是否已达到失败上限(需要拒绝认证尝试) +// 注意:这里绝不能刷新过期时间。客户端(尤其是 Subsonic)会持续轮询重试, +// 每次被拒都续期会把 IP 永久锁死,连之后正确的密钥也无法登录。 +func mediaAuthLocked(c *gin.Context) bool { + count, ok := model.LoginCache.Get(mediaAuthKey(c)) + return ok && count >= model.DefaultMaxAuthRetries +} + +// bumpMediaAuthCount 增加一次失败计数。 +// 计数必须带 TTL:key 里包含 IP / 用户名(后者来自请求参数,可被随意伪造), +// 不带过期时间会让缓存随请求无限增长;固定窗口也让锁定到期后计数自动归零。 +// 达到上限后请求会被 mediaAuthLocked / mediaAuthLockedFor 拦截,不会再走到这里, +// 因此锁定窗口不会被持续刷新。 +func bumpMediaAuthCount(key string) { + count, _ := model.LoginCache.Get(key) + count++ + model.LoginCache.Set(key, count, cache.WithEx[int](model.DefaultLockDuration)) +} + +// mediaAuthFailed 记录一次认证失败(IP 维度) +func mediaAuthFailed(c *gin.Context) { + bumpMediaAuthCount(mediaAuthKey(c)) +} + +// mediaAuthSucceeded 认证成功后清除失败计数 +func mediaAuthSucceeded(c *gin.Context) { + model.LoginCache.Del(mediaAuthKey(c)) +} + +// ==================== 用户名维度 ==================== +// 只按 IP 计数是可以被绕过的:gin 默认信任所有代理(仓库里没有设置 +// SetTrustedProxies),客户端自带 X-Forwarded-For 就能改写 c.ClientIP(), +// 轮换这个头即可无限次尝试。因此再按用户名记一份失败次数:它与 IP 无关, +// 即使 IP 被伪造,对同一账号的爆破依然会被锁住。 +// (代理信任范围属于项目层面的问题,主干 /api/auth/login 现状相同。) + +func mediaAuthUserKey(username string) string { + return "media:user:" + strings.ToLower(strings.TrimSpace(username)) +} + +// mediaAuthLockedFor 本次请求是否应被限流:IP 或用户名任一维度达到上限即拒绝。 +func mediaAuthLockedFor(c *gin.Context, username string) bool { + if mediaAuthLocked(c) { + return true + } + if strings.TrimSpace(username) == "" { + return false + } + count, ok := model.LoginCache.Get(mediaAuthUserKey(username)) + return ok && count >= model.DefaultMaxAuthRetries +} + +// bumpMediaAuthUser 给用户名维度记一次失败 +func bumpMediaAuthUser(username string) { + name := strings.TrimSpace(username) + if name == "" { + return + } + bumpMediaAuthCount(mediaAuthUserKey(name)) +} + +// mediaAuthFailedFor 记录一次认证失败:同时计入 IP 与用户名两个维度。 +func mediaAuthFailedFor(c *gin.Context, username string) { + mediaAuthFailed(c) + bumpMediaAuthUser(username) +} + +// mediaAuthSucceededFor 认证成功后清除 IP 与用户名两个维度的计数。 +func mediaAuthSucceededFor(c *gin.Context, username string) { + mediaAuthSucceeded(c) + if name := strings.TrimSpace(username); name != "" { + model.LoginCache.Del(mediaAuthUserKey(name)) + } +} diff --git a/server/handles/media_reader.go b/server/handles/media_reader.go new file mode 100644 index 00000000..b5b6f1b2 --- /dev/null +++ b/server/handles/media_reader.go @@ -0,0 +1,140 @@ +package handles + +import ( + "strings" + + "github.com/OpenListTeam/OpenList/v4/internal/db" + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/server/common" + "github.com/gin-gonic/gin" +) + +const ( + // 单个条目最多保存的批注数,防止恶意请求把表撑大 + maxMediaAnnotations = 500 + // 单条批注对象(含绘制点集)的 JSON 上限 + maxMediaAnnotationAnchor = 512 << 10 + // 单次请求的批注对象总量上限,避免一次塞进几百个大体积批注 + maxMediaAnnotationTotal = 8 << 20 +) + +// mediaAnnotationTarget 校验条目存在且当前用户可以写入批注。 +// 批注是用户私有数据,不允许访客:访客是公共账号,多设备之间的批注会互相覆盖。 +func mediaAnnotationTarget(c *gin.Context, itemID uint) (*model.MediaItem, *model.User, bool) { + user := mediaUser(c) + if user == nil { + common.ErrorStrResp(c, "unauthorized", 401) + return nil, nil, false + } + if user.IsGuest() { + common.ErrorStrResp(c, "guest cannot save annotations", 403) + return nil, nil, false + } + if itemID == 0 { + common.ErrorStrResp(c, "invalid request", 400) + return nil, nil, false + } + item, err := db.GetMediaItemByID(itemID) + if err != nil || item.Hidden || item.DeletedAt.Valid || !CanUserAccessItem(user, item) { + common.ErrorStrResp(c, "not found", 404) + return nil, nil, false + } + return item, user, true +} + +// ListMediaAnnotationsAPI 列出当前用户对某个条目保存的批注 +// GET /fs/media/annotations?item_id=1 +func ListMediaAnnotationsAPI(c *gin.Context) { + user := mediaUser(c) + itemID := uint(atoiDefault(c.Query("item_id"), 0)) + // 未登录 / 访客 / 无权访问一律返回空列表:这是私有数据,不对访客暴露 + if user == nil || user.IsGuest() || itemID == 0 { + common.SuccessResp(c, []model.MediaAnnotation{}) + return + } + item, err := db.GetMediaItemByID(itemID) + if err != nil || item.Hidden || item.DeletedAt.Valid || !CanUserAccessItem(user, item) { + common.SuccessResp(c, []model.MediaAnnotation{}) + return + } + list, err := db.ListMediaAnnotations(user.ID, item.ID) + if err != nil { + common.ErrorResp(c, err, 500) + return + } + common.SuccessResp(c, list) +} + +type mediaAnnotationPayload struct { + RemoteID string `json:"remote_id"` + Anchor string `json:"anchor"` +} + +type mediaAnnotationSyncReq struct { + ItemID uint `json:"item_id"` + Upserts []mediaAnnotationPayload `json:"upserts"` + Deletes []string `json:"deletes"` +} + +// SyncMediaAnnotationsAPI 增量保存批注:upserts 覆盖同一 remote_id,deletes 按 remote_id 删除。 +// 查看器每次改动只会带上变化的那几条,不需要整份文档重传。 +// POST /fs/media/annotations +func SyncMediaAnnotationsAPI(c *gin.Context) { + var req mediaAnnotationSyncReq + if err := c.ShouldBindJSON(&req); err != nil { + common.ErrorStrResp(c, "invalid request", 400) + return + } + item, user, ok := mediaAnnotationTarget(c, req.ItemID) + if !ok { + return + } + if len(req.Upserts) > maxMediaAnnotations || len(req.Deletes) > maxMediaAnnotations { + common.ErrorStrResp(c, "too many annotations", 400) + return + } + + upserts := make([]model.MediaAnnotation, 0, len(req.Upserts)) + seen := make(map[string]struct{}, len(req.Upserts)) + anchorBytes := 0 + for _, p := range req.Upserts { + remoteID := strings.TrimSpace(p.RemoteID) + // remote_id 是查看器分配的批注主键,多设备同步时靠它去重,必须存在且唯一 + if remoteID == "" || p.Anchor == "" || len(remoteID) > 128 { + common.ErrorStrResp(c, "invalid annotation", 400) + return + } + if _, dup := seen[remoteID]; dup { + continue + } + seen[remoteID] = struct{}{} + if len(p.Anchor) > maxMediaAnnotationAnchor { + common.ErrorStrResp(c, "annotation too large", 400) + return + } + anchorBytes += len(p.Anchor) + if anchorBytes > maxMediaAnnotationTotal { + common.ErrorStrResp(c, "annotations too large", 400) + return + } + upserts = append(upserts, model.MediaAnnotation{ + RemoteID: remoteID, + Anchor: p.Anchor, + }) + } + + deletes := make([]string, 0, len(req.Deletes)) + for _, id := range req.Deletes { + id = strings.TrimSpace(id) + if id == "" || len(id) > 128 { + continue + } + deletes = append(deletes, id) + } + + if err := db.SyncMediaAnnotations(user.ID, item.ID, upserts, deletes); err != nil { + common.ErrorResp(c, err, 500) + return + } + common.SuccessResp(c, gin.H{"saved": len(upserts), "deleted": len(deletes)}) +} diff --git a/server/handles/media_user.go b/server/handles/media_user.go new file mode 100644 index 00000000..87ba65ad --- /dev/null +++ b/server/handles/media_user.go @@ -0,0 +1,268 @@ +package handles + +import ( + "strconv" + "strings" + "time" + + "github.com/OpenListTeam/OpenList/v4/internal/db" + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/server/common" + "github.com/gin-gonic/gin" +) + +const mediaTicksPerSecond = 10_000_000 + +// GetMediaUserDataBatch 批量查询当前用户对条目的播放状态(收藏/进度/已看) +// GET /fs/media/userdata?ids=1,2,3 +func GetMediaUserDataBatch(c *gin.Context) { + user := mediaUser(c) + if user == nil { + common.SuccessResp(c, []model.MediaUserData{}) + return + } + raw := c.Query("ids") + ids := make([]uint, 0) + for _, part := range strings.Split(raw, ",") { + part = strings.TrimSpace(part) + if part == "" { + continue + } + if id, err := strconv.ParseUint(part, 10, 64); err == nil { + ids = append(ids, uint(id)) + } + } + if len(ids) == 0 { + common.SuccessResp(c, []model.MediaUserData{}) + return + } + // 限制批量查询数量,避免超长 IN 子句造成数据库压力 + if len(ids) > 500 { + ids = ids[:500] + } + list, err := db.ListMediaUserDataByItems(user.ID, ids) + if err != nil { + common.ErrorResp(c, err, 500) + return + } + common.SuccessResp(c, list) +} + +// GetMediaProgress 查询单个条目(或选集)的播放进度 +func GetMediaProgress(c *gin.Context) { + user := mediaUser(c) + itemID := uint(atoiDefault(c.Query("item_id"), 0)) + episode := atoiDefault(c.Query("episode_index"), 0) + if user == nil || itemID == 0 { + common.SuccessResp(c, nil) + return + } + item, err := db.GetMediaItemByID(itemID) + if err != nil || item.Hidden || !CanUserAccessItem(user, item) { + common.SuccessResp(c, nil) + return + } + data, err := db.GetMediaUserData(user.ID, itemID, episode) + if err != nil { + common.SuccessResp(c, nil) + return + } + common.SuccessResp(c, data) +} + +// SaveMediaProgress 上报播放进度 +type saveProgressReq struct { + ItemID uint `json:"item_id"` + EpisodeIndex int `json:"episode_index"` + PositionSec float64 `json:"position"` + DurationSec float64 `json:"duration"` + Played bool `json:"played"` +} + +func SaveMediaProgress(c *gin.Context) { + user := mediaUser(c) + if user == nil { + common.ErrorStrResp(c, "unauthorized", 401) + return + } + var req saveProgressReq + if err := c.ShouldBindJSON(&req); err != nil || req.ItemID == 0 { + common.ErrorStrResp(c, "invalid request", 400) + return + } + item, err := db.GetMediaItemByID(req.ItemID) + if err != nil || item.Hidden || !CanUserAccessItem(user, item) { + common.ErrorStrResp(c, "not found", 404) + return + } + data, err := db.GetMediaUserData(user.ID, req.ItemID, req.EpisodeIndex) + if err != nil { + data = &model.MediaUserData{ + UserID: user.ID, + ItemID: req.ItemID, + EpisodeIndex: req.EpisodeIndex, + } + } + now := time.Now() + data.PositionTicks = int64(req.PositionSec * mediaTicksPerSecond) + if req.DurationSec > 0 { + data.PositionTicks = int64(minFloat(req.PositionSec, req.DurationSec) * mediaTicksPerSecond) + } + data.LastPlayedAt = &now + finished := req.Played + if req.DurationSec > 0 && req.PositionSec/req.DurationSec >= 0.9 { + finished = true + } + if finished && !data.Played { + data.Played = true + data.PlayCount++ + } + // 重新播放时清空已看标记 + if !finished && req.PositionSec < 5 && data.Played { + data.Played = false + } + if err := db.SaveMediaUserData(data); err != nil { + common.ErrorResp(c, err, 500) + return + } + common.SuccessResp(c, data) +} + +// SetMediaFavorite 收藏 / 取消收藏 +type favoriteReq struct { + ItemID uint `json:"item_id"` + EpisodeIndex int `json:"episode_index"` + Favorite bool `json:"favorite"` +} + +func SetMediaFavorite(c *gin.Context) { + user := mediaUser(c) + if user == nil { + common.ErrorStrResp(c, "unauthorized", 401) + return + } + var req favoriteReq + if err := c.ShouldBindJSON(&req); err != nil || req.ItemID == 0 { + common.ErrorStrResp(c, "invalid request", 400) + return + } + item, err := db.GetMediaItemByID(req.ItemID) + if err != nil || item.Hidden || !CanUserAccessItem(user, item) { + common.ErrorStrResp(c, "not found", 404) + return + } + data, err := db.GetMediaUserData(user.ID, req.ItemID, req.EpisodeIndex) + if err != nil { + data = &model.MediaUserData{ + UserID: user.ID, + ItemID: req.ItemID, + EpisodeIndex: req.EpisodeIndex, + } + } + data.IsFavorite = req.Favorite + if err := db.SaveMediaUserData(data); err != nil { + common.ErrorResp(c, err, 500) + return + } + common.SuccessResp(c, gin.H{"favorite": data.IsFavorite}) +} + +// ListMediaFavoritesAPI 列出当前用户收藏的媒体条目 +// GET /fs/media/favorites?media_type=video&page=1&page_size=20 +func ListMediaFavoritesAPI(c *gin.Context) { + user := mediaUser(c) + mediaType := model.MediaType(c.Query("media_type")) + page := atoiDefault(c.DefaultQuery("page", "1"), 1) + pageSize := atoiDefault(c.DefaultQuery("page_size", "20"), 20) + if page < 1 { + page = 1 + } + if pageSize <= 0 || pageSize > 100 { + pageSize = 20 + } + items := []model.MediaItem{} + if user != nil { + data, err := db.ListMediaFavorites(user.ID) + if err != nil { + common.ErrorResp(c, err, 500) + return + } + for i := range data { + row := data[i] + if row.EpisodeIndex != 0 { + continue + } + item, err := db.GetMediaItemByID(row.ItemID) + if err != nil || item.Hidden || item.DeletedAt.Valid { + continue + } + if mediaType != "" && item.MediaType != mediaType { + continue + } + if !CanUserAccessItem(user, item) { + continue + } + items = append(items, *item) + } + } + total := len(items) + start := (page - 1) * pageSize + if start > total { + start = total + } + end := start + pageSize + if end > total { + end = total + } + common.SuccessResp(c, common.PageResp{Content: items[start:end], Total: int64(total)}) +} + +// ListMediaResumeAPI 继续观看列表 +func ListMediaResumeAPI(c *gin.Context) { + user := mediaUser(c) + mediaType := model.MediaType(c.Query("media_type")) + limit := atoiDefault(c.DefaultQuery("limit", "20"), 20) + items := []model.MediaItem{} + if user != nil { + data, err := db.ListMediaResume(user.ID, 200) + if err != nil { + common.ErrorResp(c, err, 500) + return + } + for i := range data { + row := data[i] + if row.PositionTicks <= 0 { + continue + } + item, err := db.GetMediaItemByID(row.ItemID) + if err != nil || item.Hidden || item.DeletedAt.Valid { + continue + } + if mediaType != "" && item.MediaType != mediaType { + continue + } + if !CanUserAccessItem(user, item) { + continue + } + items = append(items, *item) + if len(items) >= limit { + break + } + } + } + common.SuccessResp(c, items) +} + +func atoiDefault(s string, def int) int { + if v, err := strconv.Atoi(s); err == nil { + return v + } + return def +} + +func minFloat(a, b float64) float64 { + if a < b { + return a + } + return b +} diff --git a/server/handles/search_test.go b/server/handles/search_test.go index 3e22b332..1faa34c9 100644 --- a/server/handles/search_test.go +++ b/server/handles/search_test.go @@ -25,6 +25,7 @@ func fakeResolveMeta(metas map[string]*model.Meta) func(string) (*model.Meta, er func TestIsSearchNodeAccessible(t *testing.T) { tests := []struct { name string + basePath string metas map[string]*model.Meta node model.SearchNode want bool @@ -53,14 +54,19 @@ func TestIsSearchNodeAccessible(t *testing.T) { }, { name: "outside base path", + basePath: "/team", node: model.SearchNode{Parent: "/other", Name: "private", IsDir: true}, want: false, wantMetaPath: "", }, } - user := &model.User{ID: 2, BasePath: "/"} for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { + basePath := tt.basePath + if basePath == "" { + basePath = "/" + } + user := &model.User{ID: 2, BasePath: basePath} resolve := fakeResolveMeta(tt.metas) var gotMetaPath string spy := func(p string) (*model.Meta, error) { diff --git a/server/handles/sharing.go b/server/handles/sharing.go index 221bc127..af96b25a 100644 --- a/server/handles/sharing.go +++ b/server/handles/sharing.go @@ -16,9 +16,9 @@ import ( "github.com/OpenListTeam/OpenList/v4/internal/sharing" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/OpenListTeam/OpenList/v4/server/common" - "github.com/OpenListTeam/go-cache" "github.com/gin-gonic/gin" "github.com/pkg/errors" + log "github.com/sirupsen/logrus" ) func SharingGet(c *gin.Context, req *FsGetReq) { @@ -28,13 +28,13 @@ func SharingGet(c *gin.Context, req *FsGetReq) { return } s, obj, err := sharing.Get(c.Request.Context(), sid, path, model.SharingListArgs{ - Refresh: false, - Pwd: req.Password, + Refresh: false, + Pwd: req.Password, + ClientIP: c.ClientIP(), }) if dealError(c, err) { return } - _ = countAccess(c.ClientIP(), s) url := "" if !obj.IsDir() { fakePath := fmt.Sprintf("/%s/%s", sid, path) @@ -72,13 +72,13 @@ func SharingList(c *gin.Context, req *ListReq) { return } s, objs, err := sharing.List(c.Request.Context(), sid, path, model.SharingListArgs{ - Refresh: req.Refresh, - Pwd: req.Password, + Refresh: req.Refresh, + Pwd: req.Password, + ClientIP: c.ClientIP(), }) if dealError(c, err) { return } - _ = countAccess(c.ClientIP(), s) total, objs := pagination(objs, &req.PageReq) common.SuccessResp(c, FsListResp{ Content: utils.MustSliceConvert(objs, func(obj model.Obj) ObjResp { @@ -126,12 +126,12 @@ func SharingArchiveMeta(c *gin.Context, req *ArchiveMetaReq) { ArchiveArgs: archiveArgs, Refresh: req.Refresh, }, - Pwd: req.Password, + Pwd: req.Password, + ClientIP: c.ClientIP(), }) if dealError(c, err) { return } - _ = countAccess(c.ClientIP(), s) fakePath := fmt.Sprintf("/%s/%s", sid, path) url := fmt.Sprintf("%s/sad%s", common.GetApiUrl(c), utils.EncodePath(fakePath, true)) if s.Pwd != "" { @@ -167,17 +167,17 @@ func SharingArchiveList(c *gin.Context, req *ArchiveListReq) { }, InnerPath: utils.FixAndCleanPath(req.InnerPath), } - s, objs, err := sharing.ArchiveList(c.Request.Context(), sid, path, model.SharingArchiveListArgs{ + _, objs, err := sharing.ArchiveList(c.Request.Context(), sid, path, model.SharingArchiveListArgs{ ArchiveListArgs: model.ArchiveListArgs{ ArchiveInnerArgs: innerArgs, Refresh: req.Refresh, }, - Pwd: req.Password, + Pwd: req.Password, + ClientIP: c.ClientIP(), }) if dealError(c, err) { return } - _ = countAccess(c.ClientIP(), s) total, objs := pagination(objs, &req.PageReq) ret, _ := utils.SliceConvert(objs, func(src model.Obj) (ObjResp, error) { return toObjsRespWithoutSignAndThumb(src), nil @@ -195,12 +195,10 @@ func SharingDown(c *gin.Context) { pwd := c.Query("pwd") s, err := op.GetSharingById(sid) if err == nil { - if !s.Valid() { - err = errs.InvalidSharing - } else if !s.Verify(pwd) { - err = errs.WrongShareCode - } else if len(s.Files) != 1 && path == "/" { + if len(s.Files) != 1 && path == "/" { err = errors.New("cannot get sharing root link") + } else { + err = op.AccessSharing(c.ClientIP(), pwd, s) } } if dealErrorPage(c, err) { @@ -212,14 +210,15 @@ func SharingDown(c *gin.Context) { return } storage, actualPath, err := op.GetStorageAndActualPath(unwrapPath) - if dealErrorPage(c, err) { + if err != nil { + log.Errorf("failed get sharing storage: %+v", err) + common.ErrorPage(c, errors.New("failed get sharing link"), 500) return } if setting.GetBool(conf.ShareForceProxy) || common.ShouldProxy(storage, stdpath.Base(actualPath)) { if _, ok := c.GetQuery("d"); !ok { if url := common.GenerateDownProxyURL(storage.GetStorage(), unwrapPath); url != "" { c.Redirect(302, url) - _ = countAccess(c.ClientIP(), s) return } } @@ -228,10 +227,10 @@ func SharingDown(c *gin.Context) { Type: c.Query("type"), }) if err != nil { - common.ErrorPage(c, errors.WithMessage(err, "failed get sharing link"), 500) + log.Errorf("failed get sharing link: %+v", err) + common.ErrorPage(c, errors.New("failed get sharing link"), 500) return } - _ = countAccess(c.ClientIP(), s) proxy(c, link, obj, storage.GetStorage().ProxyRange) } else { link, _, err := op.Link(c.Request.Context(), storage, actualPath, model.LinkArgs{ @@ -241,10 +240,10 @@ func SharingDown(c *gin.Context) { Redirect: true, }) if err != nil { - common.ErrorPage(c, errors.WithMessage(err, "failed get sharing link"), 500) + log.Errorf("failed get sharing link: %+v", err) + common.ErrorPage(c, errors.New("failed get sharing link"), 500) return } - _ = countAccess(c.ClientIP(), s) redirect(c, link) } } @@ -262,12 +261,10 @@ func SharingArchiveExtract(c *gin.Context) { archivePass := c.Query("pass") s, err := op.GetSharingById(sid) if err == nil { - if !s.Valid() { - err = errs.InvalidSharing - } else if !s.Verify(pwd) { - err = errs.WrongShareCode - } else if len(s.Files) != 1 && path == "/" { + if len(s.Files) != 1 && path == "/" { err = errors.New("cannot extract sharing root") + } else { + err = op.AccessSharing(c.ClientIP(), pwd, s) } } if dealErrorPage(c, err) { @@ -611,19 +608,3 @@ func SetEnableSharing(disable bool) func(ctx *gin.Context) { } } } - -var ( - AccessCache = cache.NewMemCache[interface{}]() - AccessCountDelay = 30 * time.Minute -) - -func countAccess(ip string, s *model.Sharing) error { - key := fmt.Sprintf("%s:%s", s.ID, ip) - _, ok := AccessCache.Get(key) - if !ok { - AccessCache.Set(key, struct{}{}, cache.WithEx[interface{}](AccessCountDelay)) - s.Accessed += 1 - return op.UpdateSharing(s, true) - } - return nil -} diff --git a/server/handles/subsonic.go b/server/handles/subsonic.go new file mode 100644 index 00000000..01c0f131 --- /dev/null +++ b/server/handles/subsonic.go @@ -0,0 +1,772 @@ +package handles + +import ( + "encoding/hex" + "encoding/json" + "encoding/xml" + "fmt" + stdpath "path" + "sort" + "strconv" + "strings" + "time" + + "github.com/OpenListTeam/OpenList/v4/internal/conf" + "github.com/OpenListTeam/OpenList/v4/internal/db" + "github.com/OpenListTeam/OpenList/v4/internal/media" + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/internal/op" + "github.com/OpenListTeam/OpenList/v4/internal/setting" + "github.com/gin-gonic/gin" +) + +const subsonicAPIVersion = "1.16.1" + +// ==================== 响应结构 ==================== + +type subsonicError struct { + Code int `json:"code" xml:"code,attr"` + Message string `json:"message" xml:"message,attr"` +} + +type subsonicResponse struct { + XMLName xml.Name `json:"-" xml:"subsonic-response"` + + Status string `json:"status" xml:"status,attr"` + Version string `json:"version" xml:"version,attr"` + Type string `json:"type" xml:"type,attr"` + ServerVersion string `json:"serverVersion" xml:"serverVersion,attr"` + OpenSubsonic bool `json:"openSubsonic" xml:"openSubsonic,attr"` + + Error *subsonicError `json:"error,omitempty" xml:"error,omitempty"` + + License *subsonicLicense `json:"license,omitempty" xml:"license,omitempty"` + MusicFolders *subsonicMusicFolders `json:"musicFolders,omitempty" xml:"musicFolders,omitempty"` + Indexes *subsonicIndexes `json:"indexes,omitempty" xml:"indexes,omitempty"` + Artists *subsonicArtists `json:"artists,omitempty" xml:"artists,omitempty"` + Artist *subsonicArtist `json:"artist,omitempty" xml:"artist,omitempty"` + Album *subsonicAlbum `json:"album,omitempty" xml:"album,omitempty"` + Song *subsonicChild `json:"song,omitempty" xml:"song,omitempty"` + AlbumList *subsonicAlbumList `json:"albumList,omitempty" xml:"albumList,omitempty"` + AlbumList2 *subsonicAlbumList `json:"albumList2,omitempty" xml:"albumList2,omitempty"` + SearchResult2 *subsonicSearchResult `json:"searchResult2,omitempty" xml:"searchResult2,omitempty"` + SearchResult3 *subsonicSearchResult `json:"searchResult3,omitempty" xml:"searchResult3,omitempty"` + Playlists *subsonicPlaylists `json:"playlists,omitempty" xml:"playlists,omitempty"` + Playlist *subsonicPlaylist `json:"playlist,omitempty" xml:"playlist,omitempty"` + Lyrics *subsonicLyrics `json:"lyrics,omitempty" xml:"lyrics,omitempty"` + LyricsList *subsonicLyricsList `json:"lyricsList,omitempty" xml:"lyricsList,omitempty"` + Starred *subsonicStarred `json:"starred,omitempty" xml:"starred,omitempty"` + Starred2 *subsonicStarred `json:"starred2,omitempty" xml:"starred2,omitempty"` + Genres *subsonicGenres `json:"genres,omitempty" xml:"genres,omitempty"` + RandomSongs *subsonicSongs `json:"randomSongs,omitempty" xml:"randomSongs,omitempty"` + SongsByGenre *subsonicSongs `json:"songsByGenre,omitempty" xml:"songsByGenre,omitempty"` + ScanStatus *subsonicScanStatus `json:"scanStatus,omitempty" xml:"scanStatus,omitempty"` + User *subsonicUser `json:"user,omitempty" xml:"user,omitempty"` + Users *subsonicUsers `json:"users,omitempty" xml:"users,omitempty"` + OpenSubsonicExtensions *subsonicExtensions `json:"openSubsonicExtensions,omitempty" xml:"openSubsonicExtensions,omitempty"` + NowPlaying *subsonicNowPlaying `json:"nowPlaying,omitempty" xml:"nowPlaying,omitempty"` + PlayQueue *subsonicPlayQueue `json:"playQueue,omitempty" xml:"playQueue,omitempty"` + ArtistInfo2 *subsonicArtistInfo `json:"artistInfo2,omitempty" xml:"artistInfo2,omitempty"` + AlbumInfo *subsonicAlbumInfo `json:"albumInfo,omitempty" xml:"albumInfo,omitempty"` + TopSongs *subsonicSongs `json:"topSongs,omitempty" xml:"topSongs,omitempty"` + SimilarSongs2 *subsonicSongs `json:"similarSongs2,omitempty" xml:"similarSongs2,omitempty"` + MusicDirectory *subsonicDirectory `json:"directory,omitempty" xml:"directory,omitempty"` + Bookmarks *subsonicBookmarks `json:"bookmarks,omitempty" xml:"bookmarks,omitempty"` +} + +type subsonicLicense struct { + Valid bool `json:"valid" xml:"valid,attr"` +} + +type subsonicMusicFolder struct { + Id string `json:"id" xml:"id,attr"` + Name string `json:"name" xml:"name,attr"` +} + +type subsonicMusicFolders struct { + MusicFolder []subsonicMusicFolder `json:"musicFolder" xml:"musicFolder"` +} + +type subsonicIndex struct { + Name string `json:"name" xml:"name,attr"` + Artist []subsonicChild `json:"artist" xml:"artist"` +} + +type subsonicIndexes struct { + LastModified int64 `json:"lastModified" xml:"lastModified,attr"` + IgnoredArticles string `json:"ignoredArticles" xml:"ignoredArticles,attr"` + Index []subsonicIndex `json:"index" xml:"index"` +} + +type subsonicArtist struct { + Id string `json:"id" xml:"id,attr"` + Name string `json:"name" xml:"name,attr"` + AlbumCount int `json:"albumCount,omitempty" xml:"albumCount,attr,omitempty"` + CoverArt string `json:"coverArt,omitempty" xml:"coverArt,attr,omitempty"` + Album []subsonicAlbum `json:"album" xml:"album"` +} + +type subsonicArtists struct { + IgnoredArticles string `json:"ignoredArticles" xml:"ignoredArticles,attr"` + Index []subsonicIndex2 `json:"index" xml:"index"` +} + +type subsonicIndex2 struct { + Name string `json:"name" xml:"name,attr"` + Artist []subsonicArtist `json:"artist" xml:"artist"` +} + +type subsonicAlbum struct { + Id string `json:"id" xml:"id,attr"` + Name string `json:"name" xml:"name,attr"` + Artist string `json:"artist,omitempty" xml:"artist,attr,omitempty"` + ArtistId string `json:"artistId,omitempty" xml:"artistId,attr,omitempty"` + CoverArt string `json:"coverArt,omitempty" xml:"coverArt,attr,omitempty"` + SongCount int `json:"songCount" xml:"songCount,attr"` + Duration int `json:"duration" xml:"duration,attr"` + Created string `json:"created" xml:"created,attr"` + Year int `json:"year,omitempty" xml:"year,attr,omitempty"` + Song []subsonicChild `json:"song" xml:"song"` +} + +type subsonicChild struct { + Id string `json:"id" xml:"id,attr"` + Parent string `json:"parent,omitempty" xml:"parent,attr,omitempty"` + IsDir bool `json:"isDir" xml:"isDir,attr"` + Title string `json:"title" xml:"title,attr"` + Album string `json:"album,omitempty" xml:"album,attr,omitempty"` + Artist string `json:"artist,omitempty" xml:"artist,attr,omitempty"` + Track int `json:"track,omitempty" xml:"track,attr,omitempty"` + Year int `json:"year,omitempty" xml:"year,attr,omitempty"` + Genre string `json:"genre,omitempty" xml:"genre,attr,omitempty"` + CoverArt string `json:"coverArt,omitempty" xml:"coverArt,attr,omitempty"` + Size int64 `json:"size,omitempty" xml:"size,attr,omitempty"` + ContentType string `json:"contentType,omitempty" xml:"contentType,attr,omitempty"` + Suffix string `json:"suffix,omitempty" xml:"suffix,attr,omitempty"` + Duration int `json:"duration,omitempty" xml:"duration,attr,omitempty"` + BitRate int `json:"bitRate,omitempty" xml:"bitRate,attr,omitempty"` + Path string `json:"path,omitempty" xml:"path,attr,omitempty"` + AlbumId string `json:"albumId,omitempty" xml:"albumId,attr,omitempty"` + ArtistId string `json:"artistId,omitempty" xml:"artistId,attr,omitempty"` + Type string `json:"type,omitempty" xml:"type,attr,omitempty"` + IsVideo bool `json:"isVideo,omitempty" xml:"isVideo,attr,omitempty"` + PlayCount int `json:"playCount,omitempty" xml:"playCount,attr,omitempty"` + DiscNumber int `json:"discNumber,omitempty" xml:"discNumber,attr,omitempty"` + Created string `json:"created,omitempty" xml:"created,attr,omitempty"` + Starred string `json:"starred,omitempty" xml:"starred,attr,omitempty"` + UserRating int `json:"userRating,omitempty" xml:"userRating,attr,omitempty"` +} + +type subsonicAlbumList struct { + Album []subsonicAlbum `json:"album" xml:"album"` +} + +type subsonicSearchResult struct { + Offset int `json:"offset" xml:"offset,attr"` + Total int `json:"total" xml:"total,attr"` + Artist []subsonicArtist `json:"artist" xml:"artist"` + Album []subsonicAlbum `json:"album" xml:"album"` + Song []subsonicChild `json:"song" xml:"song"` +} + +type subsonicSongs struct { + Song []subsonicChild `json:"song" xml:"song"` +} + +type subsonicDirectory struct { + Id string `json:"id" xml:"id,attr"` + Parent string `json:"parent,omitempty" xml:"parent,attr,omitempty"` + Name string `json:"name" xml:"name,attr"` + Starred string `json:"starred,omitempty" xml:"starred,attr,omitempty"` + Child []subsonicChild `json:"child" xml:"child"` +} + +type subsonicPlaylists struct { + Playlist []subsonicPlaylist `json:"playlist" xml:"playlist"` +} + +type subsonicPlaylist struct { + Id string `json:"id" xml:"id,attr"` + Name string `json:"name" xml:"name,attr"` + Comment string `json:"comment,omitempty" xml:"comment,attr,omitempty"` + Owner string `json:"owner,omitempty" xml:"owner,attr,omitempty"` + Public bool `json:"public" xml:"public,attr"` + SongCount int `json:"songCount" xml:"songCount,attr"` + Duration int `json:"duration" xml:"duration,attr"` + Created string `json:"created" xml:"created,attr"` + Entry []subsonicChild `json:"entry" xml:"entry"` +} + +type subsonicLyrics struct { + Artist string `json:"artist,omitempty" xml:"artist,attr,omitempty"` + Title string `json:"title,omitempty" xml:"title,attr,omitempty"` + Value string `json:"value" xml:",chardata"` +} + +type subsonicLyricsList struct { + StructuredLyrics []subsonicStructuredLyrics `json:"structuredLyrics" xml:"structuredLyrics"` +} + +type subsonicStructuredLyrics struct { + DisplayArtist string `json:"displayArtist,omitempty" xml:"displayArtist,attr,omitempty"` + DisplayTitle string `json:"displayTitle,omitempty" xml:"displayTitle,attr,omitempty"` + Lang string `json:"lang" xml:"lang,attr"` + Synced bool `json:"synced" xml:"synced,attr"` + Line []subsonicLyricLine `json:"line" xml:"line"` +} + +type subsonicLyricLine struct { + Start int64 `json:"start" xml:"start,attr"` + Value string `json:"value" xml:"value,attr"` +} + +type subsonicStarred struct { + Artist []subsonicArtist `json:"artist" xml:"artist"` + Album []subsonicAlbum `json:"album" xml:"album"` + Song []subsonicChild `json:"song" xml:"song"` +} + +type subsonicGenre struct { + Value string `json:"value" xml:",chardata"` + SongCount int `json:"songCount" xml:"songCount,attr"` + AlbumCount int `json:"albumCount" xml:"albumCount,attr"` +} + +type subsonicGenres struct { + Genre []subsonicGenre `json:"genre" xml:"genre"` +} + +type subsonicScanStatus struct { + Scanning bool `json:"scanning" xml:"scanning,attr"` + Count int64 `json:"count" xml:"count,attr"` +} + +type subsonicUser struct { + Username string `json:"username" xml:"username,attr"` + Email string `json:"email,omitempty" xml:"email,attr,omitempty"` + ScrobblingEnabled bool `json:"scrobblingEnabled" xml:"scrobblingEnabled,attr"` + AdminRole bool `json:"adminRole" xml:"adminRole,attr"` + DownloadRole bool `json:"downloadRole" xml:"downloadRole,attr"` + PlaylistRole bool `json:"playlistRole" xml:"playlistRole,attr"` + StreamRole bool `json:"streamRole" xml:"streamRole,attr"` + CoverArtRole bool `json:"coverArtRole" xml:"coverArtRole,attr"` +} + +type subsonicUsers struct { + User []subsonicUser `json:"user" xml:"user"` +} + +type subsonicExtension struct { + Name string `json:"name" xml:"name,attr"` + Versions []int `json:"versions" xml:"versions"` +} + +type subsonicExtensions struct { + Extension []subsonicExtension `json:"openSubsonicExtension" xml:"openSubsonicExtension"` +} + +type subsonicNowPlaying struct { + Entry []subsonicChild `json:"entry" xml:"entry"` +} + +type subsonicPlayQueue struct { + Current string `json:"current,omitempty" xml:"current,attr,omitempty"` + Position int64 `json:"position,omitempty" xml:"position,attr,omitempty"` + Username string `json:"username,omitempty" xml:"username,attr,omitempty"` + Changed string `json:"changed,omitempty" xml:"changed,attr,omitempty"` + Entry []subsonicChild `json:"entry" xml:"entry"` +} + +type subsonicArtistInfo struct { + Biography string `json:"biography,omitempty" xml:"biography,omitempty"` + MusicBrainzId string `json:"musicBrainzId,omitempty" xml:"musicBrainzId,omitempty"` +} + +type subsonicAlbumInfo struct { + Notes string `json:"notes,omitempty" xml:"notes,omitempty"` +} + +type subsonicBookmarks struct { + Bookmark []any `json:"bookmark" xml:"bookmark"` +} + +// ==================== 分发 ==================== + +// SubsonicAPI OpenSubsonic API 入口(/@rest/*) +func SubsonicAPI(c *gin.Context) { + if !MediaSubsonicEnabled() { + c.AbortWithStatus(404) + return + } + name := strings.Trim(c.Param("path"), "/") + // 兼容接入地址已带 /@rest、客户端又自动追加 /rest 的情况 + if first, rest, found := strings.Cut(name, "/"); found && strings.EqualFold(first, "rest") { + name = rest + } + name = strings.TrimSuffix(name, ".view") + name = strings.ToLower(name) + if name == "" { + subError(c, 0, "missing method") + return + } + + // 免认证端点 + if name == "getopensubsonicextensions" { + subRespond(c, &subsonicResponse{OpenSubsonicExtensions: &subsonicExtensions{ + Extension: []subsonicExtension{ + {Name: "apiKeyAuthentication", Versions: []int{1}}, + {Name: "formPost", Versions: []int{1}}, + {Name: "songLyrics", Versions: []int{1}}, + }, + }}) + return + } + + user, _, ok := subAuth(c) + if !ok { + // 锁定判定同样要看用户名维度,否则被锁的账号只会收到「密码错误」 + if mediaAuthLockedFor(c, subParam(c, "u")) { + subError(c, 40, "Too many failed attempts, please try again later") + return + } + subError(c, 40, "Wrong username or password; use an app key as the password or enable password auth") + return + } + + switch name { + case "ping": + subRespond(c, &subsonicResponse{}) + case "getlicense": + subRespond(c, &subsonicResponse{License: &subsonicLicense{Valid: true}}) + case "tokeninfo": + subRespond(c, &subsonicResponse{User: subUserOf(user)}) + case "getuser": + subRespond(c, &subsonicResponse{User: subUserOf(user)}) + case "getusers": + if !user.IsAdmin() { + subError(c, 50, "not authorized") + return + } + users, _, err := op.GetUsers(1, -1) + if err != nil { + subError(c, 0, err.Error()) + return + } + out := &subsonicUsers{} + for i := range users { + if users[i].IsGuest() { + continue + } + out.User = append(out.User, *subUserOf(&users[i])) + } + subRespond(c, &subsonicResponse{Users: out}) + case "getmusicfolders": + subRespond(c, &subsonicResponse{MusicFolders: &subsonicMusicFolders{ + MusicFolder: []subsonicMusicFolder{{Id: "1", Name: "Music"}}, + }}) + case "getindexes": + subGetIndexes(c, user) + case "getartists": + subGetArtists(c, user) + case "getartist": + subGetArtist(c, user) + case "getalbum": + subGetAlbum(c, user) + case "getsong": + subGetSong(c, user) + case "getmusicdirectory": + subGetMusicDirectory(c, user) + case "getalbumlist", "getalbumlist2": + subGetAlbumList(c, user, name == "getalbumlist") + case "getrandomsongs": + subGetRandomSongs(c, user) + case "getsongsbygenre": + subGetSongsByGenre(c, user) + case "search2", "search3": + subSearch(c, user, name == "search2") + case "getstarred", "getstarred2": + subGetStarred(c, user, name == "getstarred") + case "stream", "download": + subStream(c, user) + case "getcoverart": + subGetCoverArt(c, user) + case "getlyrics": + subGetLyrics(c, user) + case "getlyricsbysongid": + subGetLyricsBySongID(c, user) + case "star", "unstar": + subStar(c, user, name == "star") + case "setrating": + subSetRating(c, user) + case "scrobble": + subScrobble(c, user) + case "getnowplaying": + subRespond(c, &subsonicResponse{NowPlaying: &subsonicNowPlaying{Entry: subNowPlayingEntries()}}) + case "getplaylists": + subGetPlaylists(c, user) + case "getplaylist": + subGetPlaylist(c, user) + case "createplaylist": + subCreatePlaylist(c, user) + case "updateplaylist": + subUpdatePlaylist(c, user) + case "deleteplaylist": + subDeletePlaylist(c, user) + case "getscanstatus": + subRespond(c, &subsonicResponse{ScanStatus: &subsonicScanStatus{Scanning: false, Count: subSongCount(user)}}) + case "startscan": + if !user.IsAdmin() { + subError(c, 50, "not authorized") + return + } + subStartScan(c) + case "saveplayqueue": + subSavePlayQueue(c, user) + case "getplayqueue": + subRespond(c, &subsonicResponse{PlayQueue: subLoadPlayQueue(user)}) + case "getartistinfo2": + subRespond(c, &subsonicResponse{ArtistInfo2: &subsonicArtistInfo{}}) + case "getalbuminfo", "getalbuminfo2": + subRespond(c, &subsonicResponse{AlbumInfo: &subsonicAlbumInfo{}}) + case "gettopsongs", "getsimilarsongs2": + subRespond(c, &subsonicResponse{TopSongs: &subsonicSongs{}}) + case "getbookmarks": + subRespond(c, &subsonicResponse{Bookmarks: &subsonicBookmarks{}}) + case "getgenres": + subGetGenres(c, user) + case "getavatars": + c.Status(404) + // 明确不支持 + case "getpodcasts", "getnewestpodcasts", "refreshpodcasts", "createpodcastchannel", + "deletepodcastchannel", "deletepodcastepisode", "downloadpodcastepisode", + "createuser", "updateuser", "deleteuser", "changepassword", + "getshares", "createshare", "updateshare", "deleteshare", "jukeboxcontrol": + c.String(501, "This endpoint is not implemented, but may be in future releases") + case "search", "getchatmessages", "addchatmessage", "getvideos", "getvideoinfo", "getcaptions", "hls": + c.String(410, "This endpoint will not be implemented") + default: + subError(c, 0, "unknown method: "+name) + } +} + +// ==================== 参数 / 认证 / 响应 ==================== + +func subParam(c *gin.Context, key string) string { + if v := c.Query(key); v != "" { + return v + } + if c.Request.Method == "POST" { + _ = c.Request.ParseForm() + if v := c.Request.PostForm.Get(key); v != "" { + return v + } + } + return "" +} + +func subIntParam(c *gin.Context, key string, def int) int { + v := subParam(c, key) + if v == "" { + return def + } + n, err := strconv.Atoi(v) + if err != nil { + return def + } + return n +} + +// subAuth 支持 apiKey / u+t+s / u+p(默认关闭)认证,并对失败次数做 IP + 用户名双维度限流 +func subAuth(c *gin.Context) (*model.User, *model.MediaAppKey, bool) { + username := strings.TrimSpace(subParam(c, "u")) + if mediaAuthLockedFor(c, username) { + return nil, nil, false + } + u, k, ok := subAuthInner(c) + if ok { + // 用真实用户名清除计数:apiKey 认证时 u 参数是空的 + mediaAuthSucceededFor(c, u.Username) + } else if subHasCredential(c) { + // 只有「确实带了凭据但校验失败」才计数; + // 完全裸请求(部分客户端启动时会先发一次不带参数的探测)不计入失败次数。 + mediaAuthFailedFor(c, username) + } + return u, k, ok +} + +// subHasCredential 判断请求是否真的携带了认证凭据 +func subHasCredential(c *gin.Context) bool { + return subParam(c, "apiKey") != "" || subParam(c, "u") != "" || + subParam(c, "t") != "" || subParam(c, "p") != "" +} + +func subAuthInner(c *gin.Context) (*model.User, *model.MediaAppKey, bool) { + // OpenSubsonic apiKey 扩展:直接把应用密钥放在 apiKey 参数里 + if key := strings.TrimSpace(subParam(c, "apiKey")); key != "" { + if u, k, ok := subAuthByAppKey(key); ok { + return u, k, true + } + return nil, nil, false + } + username := strings.TrimSpace(subParam(c, "u")) + password := decodeSubsonicPassword(subParam(c, "p")) + // 用户名留空、只填了应用密钥(部分客户端只允许填一个"密码"字段) + if username == "" { + if password != "" { + if u, k, ok := subAuthByAppKey(password); ok { + return u, k, true + } + } + return nil, nil, false + } + user, err := op.GetUserByName(username) + if err != nil || user.Disabled || user.IsGuest() { + return nil, nil, false + } + // t+s:用应用密钥的明文做 md5(密码+salt) 校验(Subsonic token 认证) + if salt := subParam(c, "s"); salt != "" { + if token := subParam(c, "t"); token != "" { + keys, _ := db.ListMediaAppKeys(user.ID) + for i := range keys { + plain, err := media.DecryptAppKey(keys[i].Secret) + if err != nil { + continue + } + if media.MatchSubsonicToken(plain, salt, token) { + db.TouchMediaAppKey(keys[i].ID) + return user, &keys[i], true + } + } + // 不直接返回:个别客户端会同时带上 p,继续按明文校验 + } + } + // p:OpenList 密码或应用密钥明文 + if password != "" { + // 使用主账号密码认证会把登录密码暴露给第三方客户端,必须由管理员显式开启 + if setting.GetBool(conf.MediaSubsonicAllowPwd) && user.OtpSecret == "" && + user.ValidateRawPassword(password) == nil { + return user, nil, true + } + if u, k, ok := subAuthByAppKey(password); ok && u.ID == user.ID { + return u, k, true + } + } + return nil, nil, false +} + +// decodeSubsonicPassword 解析 p 参数,兼容 Subsonic 的 enc: 编码 +func decodeSubsonicPassword(p string) string { + p = strings.TrimSpace(p) + if strings.HasPrefix(p, "enc:") { + if raw, err := hex.DecodeString(strings.TrimPrefix(p, "enc:")); err == nil { + return string(raw) + } + } + return p +} + +func subAuthByAppKey(key string) (*model.User, *model.MediaAppKey, bool) { + return LookUpAppKeyUser(key) +} + +// subsonicServerVersion 返回给 Subsonic 客户端的服务器版本(仅用于展示) +func subsonicServerVersion() string { + if conf.Version == "" { + return "OpenList" + } + return conf.Version +} + +func subRespond(c *gin.Context, resp *subsonicResponse) { + resp.Status = "ok" + resp.Version = subsonicAPIVersion + resp.Type = "OpenList" + resp.ServerVersion = subsonicServerVersion() + resp.OpenSubsonic = true + format := strings.ToLower(subParam(c, "f")) + switch format { + case "json": + c.JSON(200, gin.H{"subsonic-response": resp}) + case "jsonp": + callback := subParam(c, "callback") + if !validJsonpCallback(callback) { + c.JSON(200, gin.H{"subsonic-response": &subsonicResponse{ + Status: "failed", Version: subsonicAPIVersion, Type: "OpenList", + ServerVersion: subsonicServerVersion(), OpenSubsonic: true, + Error: &subsonicError{Code: 0, Message: "invalid callback parameter"}, + }}) + return + } + body, _ := json.Marshal(gin.H{"subsonic-response": resp}) + c.Header("Content-Type", "application/javascript; charset=utf-8") + c.String(200, "%s(%s)", callback, string(body)) + default: + c.XML(200, resp) + } +} + +func subError(c *gin.Context, code int, message string) { + resp := &subsonicResponse{ + Status: "failed", + Version: subsonicAPIVersion, + Type: "OpenList", + ServerVersion: subsonicServerVersion(), + OpenSubsonic: true, + Error: &subsonicError{Code: code, Message: message}, + } + format := strings.ToLower(subParam(c, "f")) + switch format { + case "json": + c.JSON(200, gin.H{"subsonic-response": resp}) + case "jsonp": + body, _ := json.Marshal(gin.H{"subsonic-response": resp}) + c.String(200, "%s(%s)", subParam(c, "callback"), string(body)) + default: + c.XML(200, resp) + } +} + +func validJsonpCallback(s string) bool { + if s == "" { + return false + } + for i, r := range s { + if r >= 'a' && r <= 'z' || r >= 'A' && r <= 'Z' || r == '_' || r == '$' { + continue + } + if i > 0 && (r >= '0' && r <= '9' || r == '.') { + continue + } + return false + } + return true +} + +// ==================== 数据映射 ==================== + +func subUserOf(u *model.User) *subsonicUser { + return &subsonicUser{ + Username: u.Username, + ScrobblingEnabled: true, + AdminRole: u.IsAdmin(), + DownloadRole: true, + PlaylistRole: true, + StreamRole: true, + CoverArtRole: true, + } +} + +func (l *embyLibrary) subChild(it *model.MediaItem) subsonicChild { + title := it.ScrapedName + if title == "" { + title = strings.TrimSuffix(it.FileName, stdpath.Ext(it.FileName)) + } + album := it.AlbumName + if album == "" { + album = "Unknown Album" + } + artist := albumArtistOf(it) + image := fmt.Sprintf("al-%s", albumKeyOf(it)) + year := 0 + if len(it.ReleaseDate) >= 4 { + year, _ = strconv.Atoi(it.ReleaseDate[:4]) + } + userData := l.userDataOf(it.ID, 0) + c := subsonicChild{ + Id: fmt.Sprintf("so-%d", it.ID), + Parent: image, + IsDir: false, + Title: title, + Album: album, + Artist: artist, + Track: it.TrackNumber, + Year: year, + Genre: it.Genre, + CoverArt: image, + Size: it.FileSize, + ContentType: it.MimeType, + Suffix: strings.TrimPrefix(strings.ToLower(stdpath.Ext(it.FileName)), "."), + Duration: it.Duration, + Path: MediaItemFilePath(it), + AlbumId: image, + ArtistId: "ar-" + artistKeyOf(it), + Type: "music", + PlayCount: userData.PlayCount, + Created: it.CreatedAt.UTC().Format("2006-01-02T15:04:05.000Z"), + UserRating: userDataRating(it.ID, l), + } + if userData.IsFavorite { + c.Starred = it.UpdatedAt.UTC().Format("2006-01-02T15:04:05.000Z") + } + return c +} + +func userDataRating(itemID uint, l *embyLibrary) int { + if l == nil { + return 0 + } + if d, ok := l.userData[[2]uint{itemID, 0}]; ok { + return d.Rating + } + return 0 +} + +func (l *embyLibrary) subAlbum(al *embyAlbum) subsonicAlbum { + duration := 0 + for i := range al.Items { + duration += al.Items[i].Duration + } + created := time.Now().UTC().Format("2006-01-02T15:04:05.000Z") + if len(al.Items) > 0 { + created = al.Items[0].CreatedAt.UTC().Format("2006-01-02T15:04:05.000Z") + } + return subsonicAlbum{ + Id: "al-" + al.Key, + Name: al.Name, + Artist: al.Artist, + ArtistId: "ar-" + artistKeyOf(&al.Items[0]), + CoverArt: "al-" + al.Key, + SongCount: len(al.Items), + Duration: duration, + Created: created, + Year: al.Year, + } +} + +func (l *embyLibrary) subAlbumWithSongs(al *embyAlbum) subsonicAlbum { + out := l.subAlbum(al) + for i := range al.Items { + out.Song = append(out.Song, l.subChild(&al.Items[i])) + } + return out +} + +func (l *embyLibrary) subArtist(key, name string) subsonicArtist { + out := subsonicArtist{Id: "ar-" + key, Name: name} + for _, al := range l.albums() { + if artistKeyOf(&al.Items[0]) != key { + continue + } + out.Album = append(out.Album, l.subAlbum(al)) + } + out.AlbumCount = len(out.Album) + if len(out.Album) > 0 { + out.CoverArt = out.Album[0].CoverArt + } + return out +} + +func (l *embyLibrary) subArtists() []subsonicArtist { + seen := map[string]bool{} + var out []subsonicArtist + for i := range l.music { + key := artistKeyOf(&l.music[i]) + if seen[key] { + continue + } + seen[key] = true + out = append(out, l.subArtist(key, albumArtistOf(&l.music[i]))) + } + sort.Slice(out, func(i, j int) bool { return strings.ToLower(out[i].Name) < strings.ToLower(out[j].Name) }) + return out +} diff --git a/server/handles/subsonic_handlers.go b/server/handles/subsonic_handlers.go new file mode 100644 index 00000000..81ba06e8 --- /dev/null +++ b/server/handles/subsonic_handlers.go @@ -0,0 +1,667 @@ +package handles + +import ( + "fmt" + "math/rand" + "net/http" + stdpath "path" + "sort" + "strconv" + "strings" + "sync" + "time" + + "github.com/OpenListTeam/OpenList/v4/internal/conf" + "github.com/OpenListTeam/OpenList/v4/internal/db" + "github.com/OpenListTeam/OpenList/v4/internal/media" + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/internal/setting" + "github.com/gin-gonic/gin" +) + +// ==================== 浏览 ==================== + +func subInitial(name string) string { + if name == "" { + return "#" + } + r := strings.ToUpper(string([]rune(name)[0])) + if r < "A" || r > "Z" { + return "#" + } + return r +} + +func subGetIndexes(c *gin.Context, user *model.User) { + lib := loadEmbyLibrary(user) + artists := lib.subArtists() + groups := map[string][]subsonicChild{} + var keys []string + for _, a := range artists { + idx := subInitial(a.Name) + if _, ok := groups[idx]; !ok { + keys = append(keys, idx) + } + groups[idx] = append(groups[idx], subsonicChild{ + Id: a.Id, + Title: a.Name, + IsDir: true, + Album: a.Name, + }) + } + sort.Strings(keys) + out := &subsonicIndexes{IgnoredArticles: "The El La Los Las Le Les", LastModified: time.Now().UnixMilli()} + for _, k := range keys { + out.Index = append(out.Index, subsonicIndex{Name: k, Artist: groups[k]}) + } + subRespond(c, &subsonicResponse{Indexes: out}) +} + +func subGetArtists(c *gin.Context, user *model.User) { + lib := loadEmbyLibrary(user) + artists := lib.subArtists() + groups := map[string][]subsonicArtist{} + var keys []string + for _, a := range artists { + idx := subInitial(a.Name) + if _, ok := groups[idx]; !ok { + keys = append(keys, idx) + } + groups[idx] = append(groups[idx], a) + } + sort.Strings(keys) + out := &subsonicArtists{IgnoredArticles: "The El La Los Las Le Les"} + for _, k := range keys { + out.Index = append(out.Index, subsonicIndex2{Name: k, Artist: groups[k]}) + } + subRespond(c, &subsonicResponse{Artists: out}) +} + +func subGetArtist(c *gin.Context, user *model.User) { + id := subParam(c, "id") + if !strings.HasPrefix(id, "ar-") { + subError(c, 70, "artist not found") + return + } + lib := loadEmbyLibrary(user) + key := strings.TrimPrefix(id, "ar-") + for _, a := range lib.subArtists() { + if a.Id == id { + _ = key + subRespond(c, &subsonicResponse{Artist: &a}) + return + } + } + subError(c, 70, "artist not found") +} + +func subGetAlbum(c *gin.Context, user *model.User) { + id := subParam(c, "id") + if !strings.HasPrefix(id, "al-") { + subError(c, 70, "album not found") + return + } + lib := loadEmbyLibrary(user) + al := lib.albumByKey(strings.TrimPrefix(id, "al-")) + if al == nil { + subError(c, 70, "album not found") + return + } + out := lib.subAlbumWithSongs(al) + subRespond(c, &subsonicResponse{Album: &out}) +} + +func subGetSong(c *gin.Context, user *model.User) { + id := subParam(c, "id") + lib := loadEmbyLibrary(user) + it, ok := lib.songByID(id) + if !ok { + subError(c, 70, "song not found") + return + } + child := lib.subChild(it) + subRespond(c, &subsonicResponse{Song: &child}) +} + +func subGetMusicDirectory(c *gin.Context, user *model.User) { + id := subParam(c, "id") + lib := loadEmbyLibrary(user) + switch { + case strings.HasPrefix(id, "ar-"): + key := strings.TrimPrefix(id, "ar-") + dir := &subsonicDirectory{Id: id, Name: id} + for _, al := range lib.albums() { + if artistKeyOf(&al.Items[0]) != key { + continue + } + a := lib.subAlbum(al) + dir.Child = append(dir.Child, subsonicChild{ + Id: a.Id, IsDir: true, Title: a.Name, Album: a.Name, + Artist: a.Artist, CoverArt: a.CoverArt, Year: a.Year, + }) + if dir.Name == id { + dir.Name = a.Artist + } + } + subRespond(c, &subsonicResponse{MusicDirectory: dir}) + case strings.HasPrefix(id, "al-"): + al := lib.albumByKey(strings.TrimPrefix(id, "al-")) + if al == nil { + subError(c, 70, "directory not found") + return + } + dir := &subsonicDirectory{Id: id, Name: al.Name, Parent: "ar-" + artistKeyOf(&al.Items[0])} + for i := range al.Items { + dir.Child = append(dir.Child, lib.subChild(&al.Items[i])) + } + subRespond(c, &subsonicResponse{MusicDirectory: dir}) + default: + subError(c, 70, "directory not found") + } +} + +func subGetAlbumList(c *gin.Context, user *model.User, legacy bool) { + lib := loadEmbyLibrary(user) + listType := strings.ToLower(subParam(c, "type")) + albums := lib.albums() + switch listType { + case "newest": + sort.SliceStable(albums, func(i, j int) bool { return albums[i].Items[0].CreatedAt.After(albums[j].Items[0].CreatedAt) }) + case "recent": + sort.SliceStable(albums, func(i, j int) bool { return albums[i].Items[0].UpdatedAt.After(albums[j].Items[0].UpdatedAt) }) + case "alphabeticalbyartist": + sort.SliceStable(albums, func(i, j int) bool { + if albums[i].Artist == albums[j].Artist { + return strings.ToLower(albums[i].Name) < strings.ToLower(albums[j].Name) + } + return strings.ToLower(albums[i].Artist) < strings.ToLower(albums[j].Artist) + }) + case "random": + rand.Shuffle(len(albums), func(i, j int) { albums[i], albums[j] = albums[j], albums[i] }) + case "byyear": + from := subIntParam(c, "fromYear", 0) + to := subIntParam(c, "toYear", 0) + var filtered []*embyAlbum + for _, al := range albums { + if from > 0 && al.Year < from { + continue + } + if to > 0 && al.Year > to { + continue + } + filtered = append(filtered, al) + } + albums = filtered + default: + sort.SliceStable(albums, func(i, j int) bool { return strings.ToLower(albums[i].Name) < strings.ToLower(albums[j].Name) }) + } + size := subIntParam(c, "size", 50) + offset := subIntParam(c, "offset", 0) + if offset > len(albums) { + offset = len(albums) + } + end := offset + size + if end > len(albums) { + end = len(albums) + } + out := &subsonicAlbumList{} + for _, al := range albums[offset:end] { + out.Album = append(out.Album, lib.subAlbum(al)) + } + if legacy { + subRespond(c, &subsonicResponse{AlbumList: out}) + return + } + subRespond(c, &subsonicResponse{AlbumList2: out}) +} + +func (l *embyLibrary) allSongs() []subsonicChild { + out := make([]subsonicChild, 0, len(l.music)) + for i := range l.music { + out = append(out, l.subChild(&l.music[i])) + } + return out +} + +func subGetRandomSongs(c *gin.Context, user *model.User) { + lib := loadEmbyLibrary(user) + songs := lib.allSongs() + rand.Shuffle(len(songs), func(i, j int) { songs[i], songs[j] = songs[j], songs[i] }) + size := subIntParam(c, "size", 10) + if size > len(songs) { + size = len(songs) + } + subRespond(c, &subsonicResponse{RandomSongs: &subsonicSongs{Song: songs[:size]}}) +} + +func subGetSongsByGenre(c *gin.Context, user *model.User) { + lib := loadEmbyLibrary(user) + genre := strings.ToLower(subParam(c, "genre")) + var songs []subsonicChild + for i := range lib.music { + if genre != "" && !strings.Contains(strings.ToLower(lib.music[i].Genre), genre) { + continue + } + songs = append(songs, lib.subChild(&lib.music[i])) + } + subRespond(c, &subsonicResponse{SongsByGenre: &subsonicSongs{Song: songs}}) +} + +func subSearch(c *gin.Context, user *model.User, legacy bool) { + lib := loadEmbyLibrary(user) + term := strings.ToLower(strings.TrimSpace(subParam(c, "query"))) + if term == "" { + term = strings.ToLower(strings.TrimSpace(subParam(c, "any"))) + } + artistCount := subIntParam(c, "artistCount", 20) + albumCount := subIntParam(c, "albumCount", 20) + songCount := subIntParam(c, "songCount", 20) + + out := &subsonicSearchResult{} + for _, a := range lib.subArtists() { + if len(out.Artist) >= artistCount { + break + } + if term == "" || strings.Contains(strings.ToLower(a.Name), term) { + out.Artist = append(out.Artist, a) + } + } + for _, al := range lib.albums() { + if len(out.Album) >= albumCount { + break + } + if term == "" || strings.Contains(strings.ToLower(al.Name), term) || + strings.Contains(strings.ToLower(al.Artist), term) { + out.Album = append(out.Album, lib.subAlbum(al)) + } + } + for i := range lib.music { + if len(out.Song) >= songCount { + break + } + name := lib.music[i].ScrapedName + if name == "" { + name = lib.music[i].FileName + } + if term == "" || strings.Contains(strings.ToLower(name), term) { + out.Song = append(out.Song, lib.subChild(&lib.music[i])) + } + } + out.Total = len(out.Artist) + len(out.Album) + len(out.Song) + if legacy { + subRespond(c, &subsonicResponse{SearchResult2: out}) + return + } + subRespond(c, &subsonicResponse{SearchResult3: out}) +} + +func subGetStarred(c *gin.Context, user *model.User, legacy bool) { + lib := loadEmbyLibrary(user) + out := &subsonicStarred{} + for i := range lib.music { + if d, ok := lib.userData[[2]uint{lib.music[i].ID, 0}]; ok && d.IsFavorite { + out.Song = append(out.Song, lib.subChild(&lib.music[i])) + } + } + if legacy { + subRespond(c, &subsonicResponse{Starred: out}) + return + } + subRespond(c, &subsonicResponse{Starred2: out}) +} + +func subGetGenres(c *gin.Context, user *model.User) { + lib := loadEmbyLibrary(user) + counts := map[string]int{} + for i := range lib.music { + for _, g := range strings.Split(lib.music[i].Genre, ",") { + g = strings.TrimSpace(g) + if g == "" { + continue + } + counts[g]++ + } + } + out := &subsonicGenres{} + for g, n := range counts { + out.Genre = append(out.Genre, subsonicGenre{Value: g, SongCount: n}) + } + sort.Slice(out.Genre, func(i, j int) bool { return out.Genre[i].Value < out.Genre[j].Value }) + subRespond(c, &subsonicResponse{Genres: out}) +} + +// ==================== 播放 ==================== + +func subStream(c *gin.Context, user *model.User) { + id := subParam(c, "id") + lib := loadEmbyLibrary(user) + target, ok := lib.playTarget(id) + if !ok { + subError(c, 70, "song not found") + return + } + if setting.GetBool(conf.MediaSubsonicProxyStream) { + c.Redirect(http.StatusFound, SignedProxyURL(c, target.RawPath)) + return + } + c.Redirect(http.StatusFound, SignedStreamURL(c, target.RawPath)) +} + +func subGetCoverArt(c *gin.Context, user *model.User) { + id := subParam(c, "id") + lib := loadEmbyLibrary(user) + var item *model.MediaItem + switch { + case strings.HasPrefix(id, "so-"): + item, _ = lib.songByID(id) + case strings.HasPrefix(id, "al-"): + if al := lib.albumByKey(strings.TrimPrefix(id, "al-")); al != nil && len(al.Items) > 0 { + item = &al.Items[0] + } + case strings.HasPrefix(id, "ar-"): + key := strings.TrimPrefix(id, "ar-") + for i := range lib.music { + if artistKeyOf(&lib.music[i]) == key { + item = &lib.music[i] + break + } + } + case strings.HasPrefix(id, "mv-"), strings.HasPrefix(id, "ep-"): + if it, ok := lib.resolveItem(id); ok { + _ = it + if p, ok := lib.playTarget(id); ok { + item = p.Item + } + } + } + if item == nil { + c.Status(404) + return + } + serveEmbyCover(c, item) +} + +func subGetLyrics(c *gin.Context, user *model.User) { + artist := subParam(c, "artist") + title := subParam(c, "title") + lib := loadEmbyLibrary(user) + for i := range lib.music { + it := &lib.music[i] + name := it.ScrapedName + if name == "" { + name = strings.TrimSuffix(it.FileName, stdpath.Ext(it.FileName)) + } + if title != "" && !strings.EqualFold(title, name) { + continue + } + if artist != "" && !strings.EqualFold(artist, albumArtistOf(it)) { + continue + } + subRespond(c, &subsonicResponse{Lyrics: &subsonicLyrics{ + Artist: albumArtistOf(it), Title: name, Value: it.Lyrics, + }}) + return + } + subError(c, 70, "lyrics not found") +} + +func subGetLyricsBySongID(c *gin.Context, user *model.User) { + id := subParam(c, "id") + lib := loadEmbyLibrary(user) + it, ok := lib.songByID(id) + if !ok || it.Lyrics == "" { + subError(c, 70, "lyrics not found") + return + } + name := it.ScrapedName + if name == "" { + name = strings.TrimSuffix(it.FileName, stdpath.Ext(it.FileName)) + } + lines := parseLrc(it.Lyrics) + structured := subsonicStructuredLyrics{ + DisplayArtist: albumArtistOf(it), + DisplayTitle: name, + Lang: "und", + Synced: len(lines) > 0, + } + for _, l := range lines { + structured.Line = append(structured.Line, subsonicLyricLine{ + Start: l["Start"].(int64), + Value: l["Text"].(string), + }) + } + subRespond(c, &subsonicResponse{LyricsList: &subsonicLyricsList{ + StructuredLyrics: []subsonicStructuredLyrics{structured}, + }}) +} + +// ==================== 状态 ==================== + +func subStar(c *gin.Context, user *model.User, star bool) { + id := subParam(c, "id") + lib := loadEmbyLibrary(user) + target, ok := lib.playTarget(id) + if !ok { + if album := strings.HasPrefix(id, "al-"); album { + subError(c, 70, "not supported") + return + } + subError(c, 70, "song not found") + return + } + d, err := db.GetMediaUserData(user.ID, target.ItemID, target.EpisodeNo) + if err != nil || d == nil { + d = &model.MediaUserData{UserID: user.ID, ItemID: target.ItemID, EpisodeIndex: target.EpisodeNo} + } + d.IsFavorite = star + if err := db.SaveMediaUserData(d); err != nil { + subError(c, 0, err.Error()) + return + } + subRespond(c, &subsonicResponse{}) +} + +func subSetRating(c *gin.Context, user *model.User) { + id := subParam(c, "id") + rating := subIntParam(c, "rating", 0) + lib := loadEmbyLibrary(user) + target, ok := lib.playTarget(id) + if !ok { + subError(c, 70, "song not found") + return + } + d, err := db.GetMediaUserData(user.ID, target.ItemID, target.EpisodeNo) + if err != nil || d == nil { + d = &model.MediaUserData{UserID: user.ID, ItemID: target.ItemID, EpisodeIndex: target.EpisodeNo} + } + d.Rating = rating + _ = db.SaveMediaUserData(d) + subRespond(c, &subsonicResponse{}) +} + +func subScrobble(c *gin.Context, user *model.User) { + id := subParam(c, "id") + submission := strings.EqualFold(subParam(c, "submission"), "true") + lib := loadEmbyLibrary(user) + target, ok := lib.playTarget(id) + if !ok { + subRespond(c, &subsonicResponse{}) + return + } + d, err := db.GetMediaUserData(user.ID, target.ItemID, target.EpisodeNo) + if err != nil || d == nil { + d = &model.MediaUserData{UserID: user.ID, ItemID: target.ItemID, EpisodeIndex: target.EpisodeNo} + } + now := time.Now() + d.LastPlayedAt = &now + if submission { + d.Played = true + d.PlayCount++ + } + _ = db.SaveMediaUserData(d) + subRespond(c, &subsonicResponse{}) +} + +// ==================== 播放列表 ==================== + +func subGetPlaylists(c *gin.Context, user *model.User) { + list, err := db.ListMediaPlaylists(user.ID) + if err != nil { + subError(c, 0, err.Error()) + return + } + out := &subsonicPlaylists{} + for i := range list { + out.Playlist = append(out.Playlist, subPlaylistOf(&list[i])) + } + subRespond(c, &subsonicResponse{Playlists: out}) +} + +func subPlaylistOf(p *model.MediaPlaylist) subsonicPlaylist { + return subsonicPlaylist{ + Id: strconv.FormatUint(uint64(p.ID), 10), + Name: p.Name, + Comment: p.Comment, + Public: p.Public, + SongCount: p.ItemCount, + Created: p.CreatedAt.UTC().Format("2006-01-02T15:04:05.000Z"), + } +} + +func subGetPlaylist(c *gin.Context, user *model.User) { + id, err := strconv.ParseUint(subParam(c, "id"), 10, 64) + if err != nil { + subError(c, 70, "playlist not found") + return + } + p, err := db.GetMediaPlaylist(uint(id)) + if err != nil || (p.UserID != user.ID && !user.IsAdmin()) { + subError(c, 70, "playlist not found") + return + } + out := subPlaylistOf(p) + items, _ := db.ListMediaPlaylistItems(p.ID) + lib := loadEmbyLibrary(user) + for _, it := range items { + if song, ok := lib.songByID(fmt.Sprintf("so-%d", it.ItemID)); ok { + out.Entry = append(out.Entry, lib.subChild(song)) + } + } + out.SongCount = len(out.Entry) + subRespond(c, &subsonicResponse{Playlist: &out}) +} + +func subCreatePlaylist(c *gin.Context, user *model.User) { + name := subParam(c, "name") + if name == "" { + subError(c, 10, "missing name") + return + } + p := &model.MediaPlaylist{UserID: user.ID, Name: name} + if err := db.CreateMediaPlaylist(p); err != nil { + subError(c, 0, err.Error()) + return + } + subError(c, 0, "created:"+strconv.FormatUint(uint64(p.ID), 10)) +} + +func subUpdatePlaylist(c *gin.Context, user *model.User) { + id, err := strconv.ParseUint(subParam(c, "playlistId"), 10, 64) + if err != nil { + subError(c, 10, "missing playlistId") + return + } + p, err := db.GetMediaPlaylist(uint(id)) + if err != nil || (p.UserID != user.ID && !user.IsAdmin()) { + subError(c, 70, "playlist not found") + return + } + if name := subParam(c, "name"); name != "" { + p.Name = name + } + if comment := subParam(c, "comment"); comment != "" { + p.Comment = comment + } + if pub := subParam(c, "public"); pub != "" { + p.Public = strings.EqualFold(pub, "true") + } + _ = db.UpdateMediaPlaylist(p) + subRespond(c, &subsonicResponse{}) +} + +func subDeletePlaylist(c *gin.Context, user *model.User) { + id, err := strconv.ParseUint(subParam(c, "id"), 10, 64) + if err != nil { + subError(c, 10, "missing id") + return + } + p, err := db.GetMediaPlaylist(uint(id)) + if err != nil || (p.UserID != user.ID && !user.IsAdmin()) { + subError(c, 70, "playlist not found") + return + } + _ = db.DeleteMediaPlaylist(p.ID) + subRespond(c, &subsonicResponse{}) +} + +// ==================== 扫描 ==================== + +func subSongCount(user *model.User) int64 { + lib := loadEmbyLibrary(user) + return int64(len(lib.music)) +} + +func subStartScan(c *gin.Context) { + for _, mt := range MediaTypeEnabledList() { + cfg, err := db.GetMediaConfig(mt) + if err != nil || cfg == nil { + continue + } + media.ScanMedia(cfg) + } + subRespond(c, &subsonicResponse{ScanStatus: &subsonicScanStatus{Scanning: true}}) +} + +// ==================== 播放队列(内存) ==================== + +var ( + subQueueMu sync.Mutex + subQueues = map[uint]*subsonicPlayQueue{} +) + +func subSavePlayQueue(c *gin.Context, user *model.User) { + ids := c.Query("id") + if ids == "" { + _ = c.Request.ParseForm() + ids = c.Request.PostForm.Get("id") + } + q := &subsonicPlayQueue{ + Current: subParam(c, "current"), + Position: int64(subIntParam(c, "position", 0)), + Username: user.Username, + Changed: time.Now().UTC().Format("2006-01-02T15:04:05.000Z"), + } + if ids != "" { + lib := loadEmbyLibrary(user) + for _, id := range strings.Split(ids, ",") { + if song, ok := lib.songByID(strings.TrimSpace(id)); ok { + q.Entry = append(q.Entry, lib.subChild(song)) + } + } + } + subQueueMu.Lock() + subQueues[user.ID] = q + subQueueMu.Unlock() + subRespond(c, &subsonicResponse{}) +} + +func subLoadPlayQueue(user *model.User) *subsonicPlayQueue { + subQueueMu.Lock() + defer subQueueMu.Unlock() + if q, ok := subQueues[user.ID]; ok { + return q + } + return &subsonicPlayQueue{Username: user.Username} +} + +func subNowPlayingEntries() []subsonicChild { return []subsonicChild{} } diff --git a/server/handles/torrent.go b/server/handles/torrent.go index 8b6ee1b6..99f539c5 100644 --- a/server/handles/torrent.go +++ b/server/handles/torrent.go @@ -1,29 +1,256 @@ package handles import ( + "bytes" + "context" "encoding/base64" + "encoding/json" "fmt" "io" + "net/http" + "net/url" + stdpath "path" "strings" + "time" - _189pc "github.com/OpenListTeam/OpenList/v4/drivers/189pc" "github.com/OpenListTeam/OpenList/v4/internal/conf" + "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/errs" + "github.com/OpenListTeam/OpenList/v4/internal/fs" "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/internal/offline_download/tool" "github.com/OpenListTeam/OpenList/v4/internal/op" + "github.com/OpenListTeam/OpenList/v4/internal/setting" + "github.com/OpenListTeam/OpenList/v4/internal/stream" "github.com/OpenListTeam/OpenList/v4/pkg/http_range" "github.com/OpenListTeam/OpenList/v4/pkg/torrent" + "github.com/OpenListTeam/OpenList/v4/pkg/utils" + hash_extend "github.com/OpenListTeam/OpenList/v4/pkg/utils/hash" "github.com/OpenListTeam/OpenList/v4/server/common" "github.com/gin-gonic/gin" "github.com/pkg/errors" ) +// seedRapidUploaderFor probes a storage driver for the generic +// driver.SeedRapidUploader capability. It returns nil when the driver does not +// support hash-driven rapid upload (秒传/CAS). +func seedRapidUploaderFor(storage driver.Driver) driver.SeedRapidUploader { + if storage == nil { + return nil + } + if u, ok := storage.(driver.SeedRapidUploader); ok { + return u + } + return nil +} + +// buildSeedRapidUploadRequest converts a parsed torrent's CAS metadata into a +// driver.SeedRapidUploadRequest. open lazily provides the file content and may +// be nil when no content source is available. +func buildSeedRapidUploadRequest(t *torrent.Torrent, open func() (model.FileStreamer, error)) *driver.SeedRapidUploadRequest { + if t == nil { + return nil + } + + wholeHashes := make(map[*utils.HashType]string) + req := &driver.SeedRapidUploadRequest{ + Name: t.Info.Name, + Size: t.GetTotalSize(), + Open: open, + } + + if t.CAS != nil { + req.SliceSize = t.CAS.SliceSize + req.SliceMD5s = t.CAS.SliceMD5s + if t.CAS.FileMD5 != "" { + wholeHashes[utils.MD5] = strings.ToUpper(t.CAS.FileMD5) + } + } + + // 从种子文件的哈希矩阵补充整文件与分片哈希。 + // + // 秒传请求一次只描述一个文件:req.Size / 哈希都必须属于同一个对象。 + // 多文件 torrent 的哈希属于不同文件,若取 Files[0] 而 Size 取总大小, + // 会向云端提交一个大小与哈希互相矛盾的对象。这里显式拒绝,由调用方 + // 拆成单文件种子后逐个处理(saveSeedFilesToPath 正是这样做的)。 + if t.OpenList != nil { + if len(t.OpenList.Files) > 1 { + return nil + } + if req.SliceSize == 0 { + req.SliceSize = t.OpenList.PieceSize + } + if len(t.OpenList.Files) == 1 { + sf := t.OpenList.Files[0] + if sf.Size > 0 && sf.Size != t.GetTotalSize() { + return nil + } + if sf.Hashes.MD5 != "" { + wholeHashes[utils.MD5] = strings.ToUpper(sf.Hashes.MD5) + } + if sf.Hashes.SHA1 != "" { + wholeHashes[utils.SHA1] = strings.ToUpper(sf.Hashes.SHA1) + } + if sf.Hashes.SHA256 != "" { + wholeHashes[utils.SHA256] = strings.ToUpper(sf.Hashes.SHA256) + } + if sf.Hashes.GCID != "" { + wholeHashes[hash_extend.GCID] = strings.ToUpper(sf.Hashes.GCID) + } + if sf.Hashes.Pieces != nil { + if len(req.SliceMD5s) == 0 && len(sf.Hashes.Pieces.MD5) > 0 { + req.SliceMD5s = sf.Hashes.Pieces.MD5 + } + if len(sf.Hashes.Pieces.SHA1) > 0 { + req.SliceSHA1s = sf.Hashes.Pieces.SHA1 + } + } + } + } + + req.Whole = &utils.HashInfo{} + *req.Whole = utils.NewHashInfoByMap(wholeHashes) + return req +} + +// seedFileSupportsRapidUpload reports whether a seed file carries enough hash +// metadata for the given rapid-upload driver to attempt an instant upload. +func seedFileSupportsRapidUpload(uploader driver.SeedRapidUploader, file torrent.SeedFile) bool { + if uploader == nil { + return false + } + hasMD5 := file.Hashes.MD5 != "" + hasSHA1 := file.Hashes.SHA1 != "" + for _, ht := range uploader.RapidHashAlgos() { + switch ht.Name { + case utils.MD5.Name: + if hasMD5 { + return true + } + case utils.SHA1.Name: + if hasSHA1 { + return true + } + } + } + return false +} + // maxTorrentBase64Len is the max allowed Base64-encoded torrent size (~10MB decoded) const maxTorrentBase64Len = 14 * 1024 * 1024 // maxTorrentGenFileSize is the max file size allowed for synchronous torrent generation (1GB) const maxTorrentGenFileSize = 1 * 1024 * 1024 * 1024 +// maxSeedSourceRedirects bounds how many redirects a seed source fetch may follow. +const maxSeedSourceRedirects = 3 + +// seedSourceHTTPClient fetches seed sources without any credentials. +// +// Seed sources are deliberately anonymous: a direct link (/d/) or share link +// (/sd/) embedded in a seed must stay usable from another instance that has no +// knowledge of this instance's session, so no Authorization header, cookie or +// signing query is ever attached here. +// +// The only guarantee we must hold is that the request cannot be used to reach +// an arbitrary internal endpoint. firstUsableSeedSource/validateSeedSource only +// pin the *initial* host, so every redirect hop is re-validated against the +// configured site with the exact same rule. Without this, a benign-looking +// source could 302 to a metadata/IPC endpoint and turn seed fetching into SSRF. +var seedSourceHTTPClient = &http.Client{ + Timeout: 5 * time.Minute, + CheckRedirect: func(req *http.Request, via []*http.Request) error { + if len(via) >= maxSeedSourceRedirects { + return fmt.Errorf("种子来源重定向次数过多(最多 %d 次)", maxSeedSourceRedirects) + } + if err := validateSeedHost(req.URL); err != nil { + return fmt.Errorf("种子来源重定向被拒绝: %w", err) + } + // Redirects must not silently drop to a weaker scheme. + if origin := via[0].URL.Scheme; !strings.EqualFold(req.URL.Scheme, origin) { + return fmt.Errorf("种子来源重定向不允许切换协议: %s -> %s", origin, req.URL.Scheme) + } + return nil + }, +} + +// seedSiteURLProvider supplies the configured seed_site_url. It is a variable so +// tests can exercise the validation rules without a settings database; the +// production value reads the setting on every call so config changes apply +// without a restart. +var seedSiteURLProvider = func() string { + return setting.GetStr(conf.SeedSiteURL) +} + +// seedSiteConfig returns the parsed seed_site_url, or an error when it is unset +// or malformed. All seed source validation is relative to this origin. +func seedSiteConfig() (*url.URL, error) { + configured := strings.TrimSpace(seedSiteURLProvider()) + if configured == "" { + return nil, fmt.Errorf("seed_site_url 未配置") + } + site, err := url.Parse(configured) + if err != nil { + return nil, fmt.Errorf("seed_site_url 解析失败: %w", err) + } + if site.Scheme == "" || site.Hostname() == "" { + return nil, fmt.Errorf("seed_site_url 缺少协议或主机名") + } + return site, nil +} + +// sameSeedHost reports whether two URLs point at the same site. +// +// Comparison is on hostname plus the *effective* port (80/443 for http/https), +// not on url.URL.Host: a configured "https://pan.example.com" and an embedded +// "https://pan.example.com:443/..." are the same origin, and treating them as +// different silently drops otherwise valid sources. +func sameSeedHost(a, b *url.URL) bool { + if !strings.EqualFold(a.Scheme, b.Scheme) { + return false + } + if !strings.EqualFold(a.Hostname(), b.Hostname()) { + return false + } + return effectivePort(a) == effectivePort(b) +} + +func effectivePort(u *url.URL) string { + if port := u.Port(); port != "" { + return port + } + switch strings.ToLower(u.Scheme) { + case "http": + return "80" + case "https": + return "443" + } + return "" +} + +// validateSeedHost checks that a seed source URL belongs to the configured site +// and carries no embedded credentials. It is shared by the pre-flight +// validation and by the redirect guard, so both enforce identical rules. +func validateSeedHost(u *url.URL) error { + if u == nil { + return fmt.Errorf("空 URL") + } + if u.Scheme != "http" && u.Scheme != "https" { + return fmt.Errorf("不支持的协议 %q", u.Scheme) + } + if u.User != nil { + return fmt.Errorf("种子来源不允许携带用户凭据") + } + site, err := seedSiteConfig() + if err != nil { + return err + } + if !sameSeedHost(u, site) { + return fmt.Errorf("种子来源主机 %q 与配置站点 %q 不一致", u.Host, site.Host) + } + return nil +} + // validateParsedTorrent checks that basic torrent invariants hold. func validateParsedTorrent(t *torrent.Torrent) error { if len(t.Info.Pieces)%20 != 0 { @@ -172,10 +399,14 @@ func TorrentRapidUpload(c *gin.Context) { common.ErrorResp(c, err, 500, true) return } - if !common.CanWrite(user, meta, reqPath) { + if (!user.CanWriteContent() && !common.CanWriteContentBypassUserPerms(meta, reqPath)) || !common.CanWrite(user, meta, reqPath) { common.ErrorResp(c, errs.PermissionDenied, 403) return } + if len(req.TorrentData) > maxTorrentBase64Len { + common.ErrorStrResp(c, "torrent data exceeds the maximum size", 413) + return + } // Base64 解码 torrentData, err := base64.StdEncoding.DecodeString(req.TorrentData) @@ -214,15 +445,22 @@ func TorrentRapidUpload(c *gin.Context) { return } - // 检查是否是天翼云 PC 驱动 - cloud189PC, ok := storage.(*_189pc.Cloud189PC) + // 探测目标驱动是否支持通用 CAS 秒传 + rapid, ok := storage.(driver.SeedRapidUploader) if !ok { - common.ErrorResp(c, fmt.Errorf("目标存储不是天翼云PC驱动,不支持 CAS 秒传"), 400) + common.ErrorResp(c, fmt.Errorf("目标存储驱动 %s 不支持哈希秒传(CAS)", storage.GetStorage().Driver), 400) return } // 尝试秒传 - obj, err := cloud189PC.RapidUploadFromTorrent(c.Request.Context(), dstDir, torrentData, true) + // 注意:overwrite 刻意固定为 true。秒传本身就是「把已存在的云端数据 + // 挂载到目标目录」,语义上等价于覆盖,放开成用户可选没有实际意义。 + rapidReq := buildSeedRapidUploadRequest(t, nil) + if rapidReq == nil { + common.ErrorResp(c, fmt.Errorf("该种子无法用于秒传:仅支持单文件且大小与哈希一致的种子"), 400) + return + } + obj, err := rapid.RapidUploadByHashes(c.Request.Context(), dstDir, rapidReq, true) if err != nil { common.ErrorResp(c, fmt.Errorf("秒传失败: %w", err), 400) return @@ -358,10 +596,10 @@ func GenerateTorrentForPath(c *gin.Context) { return } - // with_cas 仅支持天翼云PC驱动 + // with_cas 需要目标驱动支持通用哈希秒传(CAS) if req.WithCAS { - if _, is189pc := storage.(*_189pc.Cloud189PC); !is189pc { - common.ErrorResp(c, fmt.Errorf("CAS 秒传扩展仅支持天翼云PC驱动"), 400) + if _, ok := storage.(driver.SeedRapidUploader); !ok { + common.ErrorResp(c, fmt.Errorf("存储驱动 %s 不支持 CAS 秒传扩展", storage.GetStorage().Driver), 400) return } } @@ -392,13 +630,15 @@ func GenerateTorrentForPath(c *gin.Context) { defer link.Close() // 通过 RangeReader 获取文件内容并计算哈希生成 torrent - if link.RangeReader == nil { - common.ErrorResp(c, fmt.Errorf("该存储不支持流式读取,无法生成 torrent(请先下载文件到本地)"), 400) + // 对于仅返回 URL(无 RangeReader)的驱动,GetRangeReaderFromLink 会退化为 HTTP Range 流式读取 + rangeReader, err := stream.GetRangeReaderFromLink(obj.GetSize(), link) + if err != nil { + common.ErrorResp(c, fmt.Errorf("该存储不支持流式读取,无法生成 torrent: %w", err), 400) return } // 读取整个文件 - rc, err := link.RangeReader.RangeRead(c.Request.Context(), http_range.Range{Length: obj.GetSize()}) + rc, err := rangeReader.RangeRead(c.Request.Context(), http_range.Range{Length: obj.GetSize()}) if err != nil { common.ErrorResp(c, fmt.Errorf("读取文件失败: %w", err), 500) return @@ -431,3 +671,1266 @@ func GenerateTorrentForPath(c *gin.Context) { "with_cas": req.WithCAS, }) } + +// SeedDataReq carries a bounded base64-encoded seed document. +// SeedData is intentionally NOT bound with `required` here: some requests that +// embed SeedDataReq (e.g. SeedCapabilityReq preflight) accept `paths` instead of +// `seed_data`. Required-ness is enforced inside decodeSeedData for the handlers +// that actually consume a seed document. +type SeedDataReq struct { + SeedData string `json:"seed_data"` + Format string `json:"format"` + FileName string `json:"file_name"` +} + +// SeedConvertReq requests a container conversion without changing metadata. +type SeedConvertReq struct { + SeedDataReq + TargetFormat string `json:"target_format"` + Path string `json:"path"` +} + +// SeedHashSelection controls whole-file and piece hash inclusion. +type SeedHashSelection struct { + Whole bool `json:"whole"` + Pieces bool `json:"pieces"` +} + +// SeedHashMatrix controls the optional hash metadata stored in a seed. +type SeedHashMatrix struct { + MD5 SeedHashSelection `json:"md5"` + SHA1 SeedHashSelection `json:"sha1"` + SHA256 SeedHashSelection `json:"sha256"` +} + +// SeedGenerateReq generates one or more seed containers for existing files. +type SeedGenerateReq struct { + Paths []string `json:"paths" binding:"required"` + Format string `json:"format"` + Formats []string `json:"formats"` + Name string `json:"name"` + Comment string `json:"comment"` + FileComments map[string]string `json:"file_comments"` + HashMatrix SeedHashMatrix `json:"hash_matrix"` + PieceSize int64 `json:"piece_size"` + Trackers []string `json:"trackers"` + Channels []torrent.SeedChannel `json:"channels"` + OutputPath string `json:"output_path"` + SavePath string `json:"save_path"` + IncludeShare bool `json:"include_share"` + IncludeDirectSource bool `json:"include_direct_source"` + ShareFiles []string `json:"share_files"` + DirectFiles []string `json:"direct_files"` +} + +// SeedCapabilityReq supports source preflight and destination import planning. +type SeedCapabilityReq struct { + SeedDataReq + Paths []string `json:"paths"` + Path string `json:"path"` + Override string `json:"policy"` +} + +// SeedUpdateChannelsReq replaces only public channel metadata. +type SeedUpdateChannelsReq struct { + SeedDataReq + Channels []torrent.SeedChannel `json:"channels"` +} + +// SeedRecalcFile maps one seed file to a server-side path for re-hashing. +type SeedRecalcFile struct { + Path string `json:"path"` // seed file relative path + SourcePath string `json:"source_path"` // server-side readable file path +} + +// SeedUpdateReq edits metadata and optionally recalculates hashes. +type SeedUpdateReq struct { + SeedDataReq + Comment *string `json:"comment"` + Trackers []string `json:"trackers"` + Channels []torrent.SeedChannel `json:"channels"` + FileComments map[string]string `json:"file_comments"` + FileSources map[string][]torrent.SeedSource `json:"file_sources"` + RemoveFiles []string `json:"remove_files"` + Recalculate bool `json:"recalculate"` + RecalcFiles []SeedRecalcFile `json:"recalc_files"` + HashMatrix SeedHashMatrix `json:"hash_matrix"` + PieceSize int64 `json:"piece_size"` + OutputPath string `json:"output_path"` + SavePath string `json:"save_path"` + Options map[string]any `json:"options"` +} + +// SeedQuickSaveReq imports selected seed files using rapid upload or an existing source URL. +type SeedQuickSaveReq struct { + SeedDataReq + Path string `json:"path" binding:"required"` + Files []string `json:"files"` + SelectedFiles []int `json:"selected_files"` + Tool string `json:"tool"` + DeletePolicy string `json:"delete_policy"` + Overwrite bool `json:"overwrite"` + TransitPath string `json:"transit_path"` + UpdateChannel bool `json:"update_channel"` + Options map[string]any `json:"options"` +} + +func decodeSeedData(req SeedDataReq) ([]byte, *torrent.Seed, string, error) { + if strings.TrimSpace(req.SeedData) == "" { + return nil, nil, "", fmt.Errorf("seed_data is required") + } + if len(req.SeedData) > maxTorrentBase64Len { + return nil, nil, "", fmt.Errorf("seed data is too large") + } + data, err := base64.StdEncoding.DecodeString(req.SeedData) + if err != nil { + return nil, nil, "", fmt.Errorf("invalid base64 seed data: %w", err) + } + format := strings.ToLower(strings.TrimPrefix(req.Format, ".")) + if format == "" { + format = torrent.DetectFormat(req.FileName, data) + } + seed, err := torrent.DecodeSeed(data, format, torrent.DefaultParseLimits()) + if err != nil { + return nil, nil, "", err + } + return data, seed, format, nil +} + +// ParseSeed parses OSS, torrent or CAS data into one preview contract. +func ParseSeed(c *gin.Context) { + var req SeedDataReq + if err := c.ShouldBindJSON(&req); err != nil { + common.ErrorResp(c, err, 400) + return + } + _, seed, format, err := decodeSeedData(req) + if err != nil { + common.ErrorResp(c, err, 400) + return + } + hasSource := false + hasRapidHashes := false + for _, file := range seed.Files { + hasSource = hasSource || firstUsableSeedSource(file) != "" + hasRapidHashes = hasRapidHashes || file.Hashes.MD5 != "" || file.Hashes.SHA1 != "" || file.Hashes.SHA256 != "" + } + common.SuccessResp(c, gin.H{ + "format": format, + "seed": seed, + "files": seed.Files, + "total_size": seedTotalSize(seed), + "diagnostics": seedDiagnostics(seed), + "conversions": seedConversionStates(seed), + "capabilities": gin.H{ + "rapid_upload": hasRapidHashes, "offline_download": hasSource || format == "torrent", + "transfer": hasSource, "convert": true, "edit": true, "recalculate": true, + }, + "direct_preview": len(seed.Files) == 1 && setting.GetBool(conf.SeedSingleDirectPreview), + }) +} + +// UploadSeedAndParse parses a multipart seed upload with the same limits as JSON parsing. +func UploadSeedAndParse(c *gin.Context) { + file, err := c.FormFile("seed") + if err != nil { + common.ErrorResp(c, err, 400) + return + } + if file.Size < 0 || file.Size > torrent.DefaultMaxSeedSize { + common.ErrorStrResp(c, "seed file is too large", 400) + return + } + r, err := file.Open() + if err != nil { + common.ErrorResp(c, err, 400) + return + } + defer r.Close() + data, err := io.ReadAll(io.LimitReader(r, torrent.DefaultMaxSeedSize+1)) + if err != nil || int64(len(data)) > torrent.DefaultMaxSeedSize { + common.ErrorStrResp(c, "failed to read bounded seed file", 400) + return + } + format := torrent.DetectFormat(file.Filename, data) + seed, err := torrent.DecodeSeed(data, format, torrent.DefaultParseLimits()) + if err != nil { + common.ErrorResp(c, err, 400) + return + } + common.SuccessResp(c, gin.H{ + "format": format, + "seed": seed, + "seed_data": base64.StdEncoding.EncodeToString(data), + "diagnostics": seedDiagnostics(seed), + }) +} + +// ConvertSeed converts between OSS, standard torrent+x-openlist and CAS containers. +func ConvertSeed(c *gin.Context) { + var req SeedConvertReq + if err := c.ShouldBindJSON(&req); err != nil { + common.ErrorResp(c, err, 400) + return + } + _, seed, _, err := decodeSeedData(req.SeedDataReq) + if err != nil { + common.ErrorResp(c, err, 400) + return + } + targetFormat := req.TargetFormat + if targetFormat == "" { + targetFormat = req.Format + } + if targetFormat == "" { + common.ErrorStrResp(c, "target format is required", 400) + return + } + diagnostics := torrent.DiagnoseConversion(seed, targetFormat) + if len(diagnostics) > 0 { + common.SuccessResp(c, gin.H{"convertible": false, "diagnostics": diagnostics}) + return + } + data, err := torrent.EncodeSeed(seed, targetFormat) + if err != nil { + common.ErrorResp(c, err, 400) + return + } + format := strings.ToLower(strings.TrimPrefix(targetFormat, ".")) + fileName := stdpath.Base(seed.Name) + "." + format + result := gin.H{ + "convertible": true, "format": format, "name": fileName, + "data": base64.StdEncoding.EncodeToString(data), "seed_data": base64.StdEncoding.EncodeToString(data), "size": len(data), + } + if req.Path != "" { + user := c.Request.Context().Value(conf.UserKey).(*model.User) + dstDir, joinErr := user.JoinPath(req.Path) + if joinErr != nil { + common.ErrorResp(c, joinErr, 403) + return + } + meta, metaErr := op.GetNearestMeta(dstDir) + if metaErr != nil && !errors.Is(errors.Cause(metaErr), errs.MetaNotFound) { + common.ErrorResp(c, metaErr, 500, true) + return + } + if (!user.CanWriteContent() && !common.CanWriteContentBypassUserPerms(meta, dstDir)) || !common.CanWrite(user, meta, dstDir) { + common.ErrorResp(c, errs.PermissionDenied, 403) + return + } + fileStream := &stream.FileStream{Ctx: c.Request.Context(), Obj: &model.Object{Name: fileName, Size: int64(len(data)), Modified: time.Now()}, Reader: bytes.NewReader(data), Mimetype: "application/octet-stream"} + if err = fs.PutDirectly(c.Request.Context(), dstDir, fileStream); err != nil { + common.ErrorResp(c, err, 500) + return + } + result["path"] = stdpath.Join(req.Path, fileName) + } + common.SuccessResp(c, result) +} + +// DiagnoseSeed reports conversion requirements for every supported container. +func DiagnoseSeed(c *gin.Context) { + var req SeedDataReq + if err := c.ShouldBindJSON(&req); err != nil { + common.ErrorResp(c, err, 400) + return + } + _, seed, _, err := decodeSeedData(req) + if err != nil { + common.ErrorResp(c, err, 400) + return + } + common.SuccessResp(c, seedDiagnostics(seed)) +} + +func seedMatrixEmpty(matrix SeedHashMatrix) bool { + return !matrix.MD5.Whole && !matrix.MD5.Pieces && !matrix.SHA1.Whole && !matrix.SHA1.Pieces && !matrix.SHA256.Whole && !matrix.SHA256.Pieces +} + +// loadSeedDefaultMatrix parses the configured default right-click hash matrix. +func loadSeedDefaultMatrix() SeedHashMatrix { + var matrix SeedHashMatrix + if raw := strings.TrimSpace(setting.GetStr(conf.SeedDefaultMatrix)); raw != "" { + _ = json.Unmarshal([]byte(raw), &matrix) + } + return matrix +} + +func normalizedSeedMatrix(matrix SeedHashMatrix, formats []string) SeedHashMatrix { + if seedMatrixEmpty(matrix) { + matrix = loadSeedDefaultMatrix() + } + if seedMatrixEmpty(matrix) { + matrix = SeedHashMatrix{ + MD5: SeedHashSelection{Whole: true, Pieces: true}, SHA1: SeedHashSelection{Whole: true, Pieces: true}, + SHA256: SeedHashSelection{Whole: true, Pieces: true}, + } + } + for _, format := range formats { + switch format { + case "torrent": + matrix.SHA1 = SeedHashSelection{Whole: true, Pieces: true} + case "cas": + matrix.MD5 = SeedHashSelection{Whole: true, Pieces: true} + } + } + return matrix +} + +func applySeedMatrix(file *torrent.SeedFile, matrix SeedHashMatrix) { + if !matrix.MD5.Whole { + file.Hashes.MD5 = "" + } + if !matrix.SHA1.Whole { + file.Hashes.SHA1 = "" + } + if !matrix.SHA256.Whole { + file.Hashes.SHA256 = "" + } + if file.Hashes.Pieces == nil { + return + } + if !matrix.MD5.Pieces { + file.Hashes.Pieces.MD5 = nil + } + if !matrix.SHA1.Pieces { + file.Hashes.Pieces.SHA1 = nil + } + if !matrix.SHA256.Pieces { + file.Hashes.Pieces.SHA256 = nil + } + if len(file.Hashes.Pieces.MD5) == 0 && len(file.Hashes.Pieces.SHA1) == 0 && len(file.Hashes.Pieces.SHA256) == 0 { + file.Hashes.Pieces = nil + } +} + +// toSeedGenerateParams converts the HTTP request into the transport-agnostic +// params consumed by fs.GenerateSeedArtifacts. +func toSeedGenerateParams(req SeedGenerateReq) fs.SeedGenerateParams { + return fs.SeedGenerateParams{ + Paths: req.Paths, + Formats: req.Formats, + Name: req.Name, + Comment: req.Comment, + FileComments: req.FileComments, + HashMatrix: fs.SeedHashMatrix{ + MD5: fs.SeedHashSelection(req.HashMatrix.MD5), + SHA1: fs.SeedHashSelection(req.HashMatrix.SHA1), + SHA256: fs.SeedHashSelection(req.HashMatrix.SHA256), + }, + PieceSize: req.PieceSize, + Trackers: req.Trackers, + Channels: req.Channels, + OutputPath: req.OutputPath, + IncludeShare: req.IncludeShare, + IncludeDirectSource: req.IncludeDirectSource, + ShareFiles: req.ShareFiles, + DirectFiles: req.DirectFiles, + } +} + +// GenerateSeedForPaths reads each file once while calculating complete hashes. +// Requests exceeding the synchronous size limit are queued as a background task. +func GenerateSeedForPaths(c *gin.Context) { + user := c.Request.Context().Value(conf.UserKey).(*model.User) + var req SeedGenerateReq + if err := c.ShouldBindJSON(&req); err != nil { + common.ErrorResp(c, err, 400) + return + } + if len(req.Paths) == 0 || len(req.Paths) > torrent.DefaultMaxSeedFiles { + common.ErrorStrResp(c, "invalid seed file count", 400) + return + } + if len(req.Formats) == 0 { + req.Formats = []string{req.Format} + } + params := toSeedGenerateParams(req) + async, err := fs.SeedGenerateNeedsAsync(c.Request.Context(), user, req.Paths) + if err != nil { + common.ErrorResp(c, err, 400) + return + } + if async { + outputPath := strings.TrimSpace(req.OutputPath) + if outputPath == "" { + outputPath = strings.TrimSpace(req.SavePath) + } + if outputPath == "" { + common.ErrorStrResp(c, "asynchronous seed generation requires an output_path", 400) + return + } + params.OutputPath = outputPath + taskInfo, addErr := fs.AddSeedGenerateTask(c.Request.Context(), user, params) + if addErr != nil { + common.ErrorResp(c, addErr, 500) + return + } + common.SuccessResp(c, gin.H{"task": getTaskInfo(taskInfo), "async": true}) + return + } + artifacts, seed, genErr := fs.GenerateSeedArtifacts(c.Request.Context(), user, params) + if genErr != nil { + status := 400 + if errors.Is(genErr, errs.PermissionDenied) { + status = 403 + } + common.ErrorResp(c, genErr, status) + return + } + common.SuccessResp(c, gin.H{"artifacts": artifacts, "seed": seed}) +} + +// loadSeedDefaultTrackers returns the configured default tracker list, one per line. +func loadSeedDefaultTrackers() []string { + raw := strings.TrimSpace(setting.GetStr(conf.SeedDefaultTrackers)) + if raw == "" { + return nil + } + var trackers []string + for _, line := range strings.Split(raw, "\n") { + if trimmed := strings.TrimSpace(line); trimmed != "" { + trackers = append(trackers, trimmed) + } + } + return trackers +} + +// directSourceAvailable reports whether an unauthenticated /d/ direct source can be embedded. +// Direct sources are only safe when the file is not encrypted and guest access is allowed. +func directSourceAvailable(meta *model.Meta, path string) bool { + if setting.GetBool(conf.SignAll) { + return false + } + if isEncrypt(meta, path) { + return false + } + guest, err := op.GetGuest() + return err == nil && !guest.Disabled +} + +// SeedCapabilities reports whether files can use hashes, source URLs or require downloading. +func SeedCapabilities(c *gin.Context) { + user := c.Request.Context().Value(conf.UserKey).(*model.User) + var req SeedCapabilityReq + if err := c.ShouldBindJSON(&req); err != nil { + common.ErrorResp(c, err, 400) + return + } + if len(req.Paths) > 0 { + if len(req.Paths) > torrent.DefaultMaxSeedFiles { + common.ErrorStrResp(c, "invalid seed file count", 400) + return + } + files := make([]gin.H, 0, len(req.Paths)) + var estimatedTraffic int64 + existing := make(map[string]bool) + for _, requestedPath := range req.Paths { + fullPath, joinErr := user.JoinPath(requestedPath) + if joinErr != nil { + common.ErrorResp(c, joinErr, 403) + return + } + meta, metaErr := op.GetNearestMeta(fullPath) + if metaErr != nil && !errors.Is(errors.Cause(metaErr), errs.MetaNotFound) { + common.ErrorResp(c, metaErr, 500, true) + return + } + if !common.CanRead(user, meta, fullPath) { + common.ErrorResp(c, errs.PermissionDenied, 403) + return + } + storage, actualPath, getErr := op.GetStorageAndActualPath(fullPath) + if getErr != nil { + common.ErrorResp(c, getErr, 400) + return + } + obj, getErr := op.Get(c.Request.Context(), storage, actualPath) + if getErr != nil || obj.IsDir() { + common.ErrorResp(c, fmt.Errorf("seed path must be a readable file: %s", requestedPath), 400) + return + } + available := make([]string, 0, 3) + hashInfo := obj.GetHash() + for _, hashType := range []*utils.HashType{utils.MD5, utils.SHA1, utils.SHA256} { + if hashInfo.GetHash(hashType) != "" { + available = append(available, hashType.Name) + existing[hashType.Name] = true + } + } + requiresDownload := len(available) < 3 + var fileTraffic int64 + if requiresDownload { + fileTraffic = obj.GetSize() + estimatedTraffic += fileTraffic + } + // Probe whether the storage can be streamed for server-side hashing. + streamable := false + if link, _, linkErr := op.Link(c.Request.Context(), storage, actualPath, model.LinkArgs{}); linkErr == nil && link != nil { + if _, rrErr := stream.GetRangeReaderFromLink(obj.GetSize(), link); rrErr == nil { + streamable = true + } + _ = link.Close() + } + files = append(files, gin.H{ + "path": requestedPath, "name": obj.GetName(), "size": obj.GetSize(), + "available_hashes": available, "requires_download": requiresDownload, + "requires_fetch": requiresDownload, "estimated_traffic": fileTraffic, + "streamable": streamable, "share_available": user.CanShare(), + "direct_source_available": directSourceAvailable(meta, fullPath), + }) + } + existingHashes := make([]string, 0, len(existing)) + for _, name := range []string{"md5", "sha1", "sha256"} { + if existing[name] { + existingHashes = append(existingHashes, name) + } + } + common.SuccessResp(c, gin.H{ + "formats": gin.H{"oss": true, "torrent": true, "cas": true}, + "files": files, "existing_hashes": existingHashes, "estimated_traffic": estimatedTraffic, + "default_matrix": loadSeedDefaultMatrix(), "trackers": loadSeedDefaultTrackers(), + }) + return + } + _, seed, _, err := decodeSeedData(req.SeedDataReq) + if err != nil { + common.ErrorResp(c, err, 400) + return + } + dstPath, err := user.JoinPath(req.Path) + if err != nil { + common.ErrorResp(c, err, 403) + return + } + meta, err := op.GetNearestMeta(dstPath) + if err != nil && !errors.Is(errors.Cause(err), errs.MetaNotFound) { + common.ErrorResp(c, err, 500, true) + return + } + if (!user.CanWriteContent() && !common.CanWriteContentBypassUserPerms(meta, dstPath)) || !common.CanWrite(user, meta, dstPath) { + common.ErrorResp(c, errs.PermissionDenied, 403) + return + } + storage, _, err := op.GetStorageAndActualPath(dstPath) + if err != nil { + common.ErrorResp(c, err, 400) + return + } + rapidUploader := seedRapidUploaderFor(storage) + _, putURL := storage.(driver.PutURL) + _, putURLResult := storage.(driver.PutURLResult) + files := make([]gin.H, 0, len(seed.Files)) + for _, file := range seed.Files { + hasCAS := seedFileSupportsRapidUpload(rapidUploader, file) + hasSource := firstUsableSeedSource(file) != "" + method := "download_required" + if hasCAS { + method = "rapid_upload" + } else if hasSource && (putURL || putURLResult) { + method = "put_url" + } else if hasSource { + method = "offline_download" + } + files = append(files, gin.H{"path": file.Path, "method": method, "requires_download": method == "download_required"}) + } + globalPolicy := setting.GetStr(conf.SeedAutoGeneratePolicy, "off") + policy := strings.ToLower(strings.TrimSpace(req.Override)) + if policy == "" || policy == "inherit" { + policy = globalPolicy + } + if policy != "on" && policy != "off" { + common.ErrorStrResp(c, "policy must be off, on, or inherit", 400) + return + } + // Describe the destination driver's rapid-transfer capability surface so the + // frontend can show which hashes are reusable for instant upload. + rapidAlgos := []string{} + usesPieces := false + if rapidUploader != nil { + for _, ht := range rapidUploader.RapidHashAlgos() { + rapidAlgos = append(rapidAlgos, ht.Name) + } + usesPieces = rapidUploader.RapidHashNeedsPieces() + } + driverSupports := gin.H{ + "cas_rapid": rapidUploader != nil, + "put_url": putURL || putURLResult, + "offline_download": true, + "rapid_hash_algos": rapidAlgos, + "rapid_uses_pieces": usesPieces, + } + common.SuccessResp(c, gin.H{ + "driver": storage.Config().Name, "global_policy": globalPolicy, + "resolved_policy": policy, "files": files, "driver_supports": driverSupports, + }) +} + +// UpdateSeedChannels updates public discovery metadata and keeps the requested container. +func UpdateSeedChannels(c *gin.Context) { + UpdateSeed(c) +} + +// applySeedUpdateOptions folds the frontend options passthrough into typed fields. +func applySeedUpdateOptions(req *SeedUpdateReq) { + if req.Options == nil { + return + } + if req.Comment == nil { + if v, ok := req.Options["comment"].(string); ok { + req.Comment = &v + } + } + if !req.Recalculate { + if v, ok := req.Options["recalculate"].(bool); ok { + req.Recalculate = v + } + } +} + +// validateSeedSource enforces the metadata contract for editable share/direct sources. +func validateSeedSource(src torrent.SeedSource) error { + if src.Type != "openlist-direct" && src.Type != "openlist-share" { + return fmt.Errorf("unsupported seed source type %q", src.Type) + } + u, err := url.Parse(src.URL) + if err != nil { + return fmt.Errorf("invalid seed source URL for %q", src.Type) + } + if err := validateSeedHost(u); err != nil { + return err + } + if src.Type == "openlist-direct" && !strings.HasPrefix(u.EscapedPath(), "/d/") { + return fmt.Errorf("openlist-direct source URL must start with /d/") + } + if src.Type == "openlist-share" && !strings.HasPrefix(u.EscapedPath(), "/sd/") { + return fmt.Errorf("openlist-share source URL must start with /sd/") + } + return nil +} + +// rehashSeedFile reads one server-side file and recomputes its whole and piece hashes. +func rehashSeedFile(c *gin.Context, user *model.User, sourcePath string, pieceSize int64) (torrent.SeedFile, error) { + var out torrent.SeedFile + fullPath, err := user.JoinPath(sourcePath) + if err != nil { + return out, err + } + meta, err := op.GetNearestMeta(fullPath) + if err != nil && !errors.Is(errors.Cause(err), errs.MetaNotFound) { + return out, err + } + if !common.CanRead(user, meta, fullPath) { + return out, errs.PermissionDenied + } + storage, actualPath, err := op.GetStorageAndActualPath(fullPath) + if err != nil { + return out, err + } + obj, err := op.Get(c.Request.Context(), storage, actualPath) + if err != nil || obj.IsDir() { + return out, fmt.Errorf("recalculate path must be a readable file: %s", sourcePath) + } + if obj.GetSize() < 0 || obj.GetSize() > maxTorrentGenFileSize { + return out, fmt.Errorf("recalculate file exceeds 1GB limit: %s", sourcePath) + } + link, _, err := op.Link(c.Request.Context(), storage, actualPath, model.LinkArgs{}) + if err != nil { + return out, fmt.Errorf("storage cannot stream %s: %v", sourcePath, err) + } + rangeReader, err := stream.GetRangeReaderFromLink(obj.GetSize(), link) + if err != nil { + return out, fmt.Errorf("storage cannot stream %s", sourcePath) + } + rc, err := rangeReader.RangeRead(c.Request.Context(), http_range.Range{Length: obj.GetSize()}) + if err != nil { + return out, err + } + defer rc.Close() + hasher := torrent.NewHashWriter(pieceSize, pieceSize, obj.GetSize()) + n, copyErr := io.Copy(hasher, io.LimitReader(rc, obj.GetSize()+1)) + if copyErr != nil { + return out, fmt.Errorf("read %s: %w", sourcePath, copyErr) + } + if n != obj.GetSize() { + return out, fmt.Errorf("read %s: got %d of %d bytes", sourcePath, n, obj.GetSize()) + } + hasher.Finish() + modified := "" + if !obj.ModTime().IsZero() { + modified = obj.ModTime().UTC().Format(time.RFC3339) + } + return hasher.BuildSeedFile(stdpath.Base(sourcePath), modified), nil +} + +// UpdateSeed edits seed metadata and optionally recalculates hashes from server files. +func UpdateSeed(c *gin.Context) { + user := c.Request.Context().Value(conf.UserKey).(*model.User) + var req SeedUpdateReq + if err := c.ShouldBindJSON(&req); err != nil { + common.ErrorResp(c, err, 400) + return + } + applySeedUpdateOptions(&req) + _, seed, format, err := decodeSeedData(req.SeedDataReq) + if err != nil { + common.ErrorResp(c, err, 400) + return + } + if req.Comment != nil { + seed.Comment = strings.TrimSpace(*req.Comment) + } + if req.Trackers != nil { + seed.Trackers = req.Trackers + } + if req.Channels != nil { + seed.Channels = req.Channels + } + for i := range seed.Files { + if comment, ok := req.FileComments[seed.Files[i].Path]; ok { + seed.Files[i].Comment = strings.TrimSpace(comment) + } + if sources, ok := req.FileSources[seed.Files[i].Path]; ok { + for _, src := range sources { + if srcErr := validateSeedSource(src); srcErr != nil { + common.ErrorResp(c, srcErr, 400) + return + } + } + seed.Files[i].Sources = sources + } + } + if len(req.RemoveFiles) > 0 { + removeSet := make(map[string]struct{}, len(req.RemoveFiles)) + for _, path := range req.RemoveFiles { + if trimmed := strings.TrimSpace(path); trimmed != "" { + removeSet[trimmed] = struct{}{} + } + } + if len(removeSet) == 0 { + common.ErrorStrResp(c, "remove_files requires at least one non-empty path", 400) + return + } + kept := make([]torrent.SeedFile, 0, len(seed.Files)) + for _, file := range seed.Files { + if _, removed := removeSet[file.Path]; !removed { + kept = append(kept, file) + } + } + if len(kept) == 0 { + common.ErrorStrResp(c, "cannot remove all files from a seed", 400) + return + } + seed.Files = kept + } + if req.Recalculate { + pieceSize := req.PieceSize + if pieceSize <= 0 { + pieceSize = seed.PieceSize + } + if pieceSize <= 0 { + pieceSize = torrent.DefaultPieceSize + } + if pieceSize < 16*1024 || pieceSize > 64*1024*1024 { + common.ErrorStrResp(c, "piece_size must be between 16384 and 67108864", 400) + return + } + seed.PieceSize = pieceSize + matrix := normalizedSeedMatrix(req.HashMatrix, []string{format}) + recalcMap := make(map[string]string, len(req.RecalcFiles)) + for _, rf := range req.RecalcFiles { + if strings.TrimSpace(rf.SourcePath) != "" { + recalcMap[rf.Path] = rf.SourcePath + } + } + if len(recalcMap) == 0 { + common.ErrorStrResp(c, "recalculate requires at least one source_path", 400) + return + } + for i := range seed.Files { + srcPath, ok := recalcMap[seed.Files[i].Path] + if !ok { + continue + } + rehashed, rehashErr := rehashSeedFile(c, user, srcPath, pieceSize) + if rehashErr != nil { + common.ErrorResp(c, rehashErr, 400) + return + } + rehashed.Path = seed.Files[i].Path + rehashed.Comment = seed.Files[i].Comment + rehashed.Sources = seed.Files[i].Sources + rehashed.CASSliceMD5 = "" + rehashed.CASCreateTime = "" + applySeedMatrix(&rehashed, matrix) + seed.Files[i] = rehashed + } + } + data, err := torrent.EncodeSeed(seed, format) + if err != nil { + common.ErrorResp(c, err, 400) + return + } + result := gin.H{"format": format, "seed_data": base64.StdEncoding.EncodeToString(data), "seed": seed} + outputPath := strings.TrimSpace(req.OutputPath) + if outputPath == "" { + outputPath = strings.TrimSpace(req.SavePath) + } + if outputPath != "" { + fileName := stdpath.Base(seed.Name) + "." + format + dstDir, joinErr := user.JoinPath(outputPath) + if joinErr != nil { + common.ErrorResp(c, joinErr, 403) + return + } + meta, metaErr := op.GetNearestMeta(dstDir) + if metaErr != nil && !errors.Is(errors.Cause(metaErr), errs.MetaNotFound) { + common.ErrorResp(c, metaErr, 500, true) + return + } + if (!user.CanWriteContent() && !common.CanWriteContentBypassUserPerms(meta, dstDir)) || !common.CanWrite(user, meta, dstDir) { + common.ErrorResp(c, errs.PermissionDenied, 403) + return + } + fileStream := &stream.FileStream{Ctx: c.Request.Context(), Obj: &model.Object{Name: fileName, Size: int64(len(data)), Modified: time.Now()}, Reader: bytes.NewReader(data), Mimetype: "application/octet-stream"} + if putErr := fs.PutDirectly(c.Request.Context(), dstDir, fileStream); putErr != nil { + common.ErrorResp(c, putErr, 500) + return + } + result["path"] = stdpath.Join(outputPath, fileName) + } + shareStatus := checkSeedShareValidity(seed) + if len(shareStatus) > 0 { + result["share_status"] = shareStatus + } + common.SuccessResp(c, result) +} + +// saveSeedFilesToPath saves selected seed files into one target directory. +func saveSeedFilesToPath(c *gin.Context, user *model.User, seed *torrent.Seed, req SeedQuickSaveReq, dstPath string, rapidOnly bool, allowOffline bool) ([]gin.H, string, string, error) { + results := make([]gin.H, 0, len(seed.Files)) + fullPath, err := user.JoinPath(dstPath) + if err != nil { + return nil, "", "", err + } + meta, err := op.GetNearestMeta(fullPath) + if err != nil && !errors.Is(errors.Cause(err), errs.MetaNotFound) { + return nil, "", "", err + } + if (!user.CanWriteContent() && !common.CanWriteContentBypassUserPerms(meta, fullPath)) || !common.CanWrite(user, meta, fullPath) { + return nil, "", "", errs.PermissionDenied + } + storage, actualPath, err := op.GetStorageAndActualPath(fullPath) + if err != nil { + return nil, "", "", err + } + dstDir, err := op.Get(c.Request.Context(), storage, actualPath) + if err != nil || !dstDir.IsDir() { + return nil, "", "", errs.NotFolder + } + driverName, mountPath := "", "" + if mounted := storage.GetStorage(); mounted != nil { + driverName = strings.TrimSpace(mounted.Driver) + mountPath = strings.TrimSpace(mounted.MountPath) + } + selected := make(map[string]struct{}, len(req.Files)) + for _, name := range req.Files { + selected[name] = struct{}{} + } + selectedIndexes := make(map[int]struct{}, len(req.SelectedFiles)) + for _, index := range req.SelectedFiles { + if index < 0 || index >= len(seed.Files) { + return nil, "", "", fmt.Errorf("selected_files contains an invalid index") + } + selectedIndexes[index] = struct{}{} + } + rapidUploader := seedRapidUploaderFor(storage) + _, putURL := storage.(driver.PutURL) + _, putURLResult := storage.(driver.PutURLResult) + for index, file := range seed.Files { + if len(selected) > 0 { + if _, ok := selected[file.Path]; !ok { + continue + } + } else if len(selectedIndexes) > 0 { + if _, ok := selectedIndexes[index]; !ok { + continue + } + } + name := stdpath.Base(file.Path) + if rapidUploader != nil && seedFileSupportsRapidUpload(rapidUploader, file) { + one := *seed + one.Name = name + one.Files = []torrent.SeedFile{file} + if data, encodeErr := torrent.EncodeSeed(&one, "torrent"); encodeErr == nil { + if t, decErr := torrent.Decode(data); decErr == nil { + if rapidReq := buildSeedRapidUploadRequest(t, seedContentOpener(seed, file, c.Request.Context())); rapidReq != nil { + if obj, rapidErr := rapidUploader.RapidUploadByHashes(c.Request.Context(), dstDir, rapidReq, req.Overwrite); rapidErr == nil { + results = append(results, gin.H{"path": file.Path, "name": obj.GetName(), "method": "rapid_upload"}) + continue + } + } + } + } + } + if rapidOnly { + results = append(results, gin.H{"path": file.Path, "name": name, "method": "unavailable", "error": "target driver cannot reuse the available hashes"}) + continue + } + source := firstUsableSeedSource(file) + if source == "" { + results = append(results, gin.H{"path": file.Path, "name": name, "method": "unavailable", "error": "no usable source URL or compatible rapid-upload hashes"}) + continue + } + if putURL || putURLResult { + if putErr := fs.PutURL(c.Request.Context(), fullPath, name, source); putErr == nil { + results = append(results, gin.H{"path": file.Path, "name": name, "method": "put_url"}) + continue + } + } + if !allowOffline { + results = append(results, gin.H{"path": file.Path, "name": name, "method": "offline_download", "error": "transit requires a synchronous intermediate save (rapid upload or PutURL)"}) + continue + } + if !user.CanAddOfflineDownloadTasks() { + results = append(results, gin.H{"path": file.Path, "name": name, "method": "offline_download", "error": "offline download permission is required"}) + continue + } + if req.Tool == "" { + results = append(results, gin.H{"path": file.Path, "name": name, "method": "offline_download", "error": "offline download tool is required"}) + continue + } + t, addErr := tool.AddURL(c, &tool.AddURLArgs{URL: source, DstDirPath: fullPath, Tool: req.Tool, DeletePolicy: tool.DeletePolicy(req.DeletePolicy)}) + if addErr != nil { + results = append(results, gin.H{"path": file.Path, "name": name, "method": "offline_download", "error": addErr.Error()}) + continue + } + result := gin.H{"path": file.Path, "name": name, "method": "offline_download"} + if t != nil { + result["task"] = getTaskInfo(t) + } + results = append(results, result) + } + return results, driverName, mountPath, nil +} + +// resolveSeedTransitPath extracts the optional transit destination for a relayed save. +func resolveSeedTransitPath(req SeedQuickSaveReq) string { + transitPath := strings.TrimSpace(req.TransitPath) + if transitPath == "" && req.Options != nil { + if mode, ok := req.Options["mode"].(string); ok && mode == "transfer" { + if tp, ok := req.Options["transit_path"].(string); ok { + transitPath = strings.TrimSpace(tp) + } + } + } + return transitPath +} + +// transferSeedViaTransit first saves to an intermediate directory then relays to the final target. +func transferSeedViaTransit(c *gin.Context, user *model.User, seed *torrent.Seed, req SeedQuickSaveReq, transitPath string) { + if !user.CanCopy() { + common.ErrorResp(c, errs.PermissionDenied, 403) + return + } + transitResults, _, _, err := saveSeedFilesToPath(c, user, seed, req, transitPath, false, false) + if err != nil { + common.ErrorResp(c, err, seedSaveErrorCode(err)) + return + } + finalDst, err := user.JoinPath(req.Path) + if err != nil { + common.ErrorResp(c, err, 403) + return + } + finalMeta, err := op.GetNearestMeta(finalDst) + if err != nil && !errors.Is(errors.Cause(err), errs.MetaNotFound) { + common.ErrorResp(c, err, 500, true) + return + } + if (!user.CanWriteContent() && !common.CanWriteContentBypassUserPerms(finalMeta, finalDst)) || !common.CanWrite(user, finalMeta, finalDst) { + common.ErrorResp(c, errs.PermissionDenied, 403) + return + } + transitFullPath, err := user.JoinPath(transitPath) + if err != nil { + common.ErrorResp(c, err, 403) + return + } + finalResults := make([]gin.H, 0, len(transitResults)) + for _, res := range transitResults { + method, _ := res["method"].(string) + if method == "unavailable" { + finalResults = append(finalResults, res) + continue + } + name, _ := res["name"].(string) + if name == "" { + continue + } + // Asynchronous offline downloads cannot be relayed synchronously. + if method == "offline_download" { + finalResults = append(finalResults, gin.H{"path": res["path"], "name": name, "method": "transfer", "error": "transit requires a synchronous intermediate save (rapid upload or PutURL)"}) + continue + } + srcObjPath := stdpath.Join(transitFullPath, name) + _, copyErr := fs.Copy(context.WithValue(c.Request.Context(), conf.NoTaskKey, struct{}{}), srcObjPath, finalDst) + if copyErr != nil { + finalResults = append(finalResults, gin.H{"path": res["path"], "name": name, "method": "transfer", "error": copyErr.Error()}) + continue + } + finalResults = append(finalResults, gin.H{"path": res["path"], "name": name, "method": "transfer"}) + } + common.SuccessResp(c, gin.H{"results": finalResults}) +} + +// seedSaveErrorCode maps a save error to an HTTP status code. +func seedSaveErrorCode(err error) int { + if errors.Is(errors.Cause(err), errs.PermissionDenied) { + return 403 + } + if errors.Is(errors.Cause(err), errs.NotFolder) || errors.Is(errors.Cause(err), errs.ObjectNotFound) { + return 400 + } + return 500 +} + +// checkSeedShareValidity reports whether each openlist-share source still resolves to a valid sharing. +func checkSeedShareValidity(seed *torrent.Seed) gin.H { + status := make(gin.H) + for _, file := range seed.Files { + for _, source := range file.Sources { + if source.Type != "openlist-share" || strings.TrimSpace(source.ShareID) == "" { + continue + } + shareID := strings.TrimSpace(source.ShareID) + if _, checked := status[shareID]; checked { + continue + } + valid := false + if sharing, err := op.GetSharingById(shareID, true); err == nil && sharing != nil { + valid = sharing.Valid() + } + status[shareID] = valid + } + } + return status +} + +// applyChannelUpdate reflects successful saves as channels and failed saves as missing channels. +func applyChannelUpdate(seed *torrent.Seed, driverName, mountPath string, results []gin.H) { + if driverName == "" { + return + } + successByPath := make(map[string]bool) + for _, res := range results { + filePath, _ := res["path"].(string) + if filePath == "" { + continue + } + method, _ := res["method"].(string) + successByPath[filePath] = method != "unavailable" + } + anySuccess := false + for _, ok := range successByPath { + if ok { + anySuccess = true + break + } + } + if anySuccess { + exists := false + for _, channel := range seed.Channels { + if channel.Driver == driverName { + exists = true + break + } + } + if !exists { + seed.Channels = append(seed.Channels, torrent.SeedChannel{Driver: driverName, MountPath: mountPath}) + } + } + for i := range seed.Files { + file := &seed.Files[i] + ok, tracked := successByPath[file.Path] + if !tracked { + continue + } + if ok { + file.MissingChannels = removeString(file.MissingChannels, driverName) + } else if !containsString(file.MissingChannels, driverName) { + file.MissingChannels = append(file.MissingChannels, driverName) + } + } +} + +func containsString(values []string, target string) bool { + for _, value := range values { + if value == target { + return true + } + } + return false +} + +func removeString(values []string, target string) []string { + result := values[:0] + for _, value := range values { + if value != target { + result = append(result, value) + } + } + return result +} + +// QuickSaveSeed imports selected files without downloading when the target supports it. +func QuickSaveSeed(c *gin.Context) { + user := c.Request.Context().Value(conf.UserKey).(*model.User) + var req SeedQuickSaveReq + if err := c.ShouldBindJSON(&req); err != nil { + common.ErrorResp(c, err, 400) + return + } + _, seed, format, err := decodeSeedData(req.SeedDataReq) + if err != nil { + common.ErrorResp(c, err, 400) + return + } + rapidOnly := strings.HasSuffix(c.FullPath(), "/rapid_upload") + if transitPath := resolveSeedTransitPath(req); transitPath != "" { + transferSeedViaTransit(c, user, seed, req, transitPath) + return + } + results, driverName, mountPath, err := saveSeedFilesToPath(c, user, seed, req, req.Path, rapidOnly, true) + if err != nil { + common.ErrorResp(c, err, seedSaveErrorCode(err)) + return + } + resp := gin.H{"results": results} + if req.UpdateChannel { + applyChannelUpdate(seed, driverName, mountPath, results) + if data, encodeErr := torrent.EncodeSeed(seed, format); encodeErr == nil { + resp["seed_data"] = base64.StdEncoding.EncodeToString(data) + resp["seed"] = seed + } + } + common.SuccessResp(c, resp) +} + +// seedContentOpener returns a lazy content provider for a seed file. +// +// The source bytes are fetched from the file's first usable source URL, but +// only on the first call, letting hash-only drivers (e.g. 189pc) skip content +// entirely. Requests are deliberately anonymous and routed through +// seedSourceHTTPClient so redirects are re-validated (see its doc comment). +// +// The returned streamer is backed by a range reader rather than a fully +// buffered copy: drivers such as aliyundrive_open and quark_open only read a +// small proof window, and 115 reads a 128KiB prefix, so buffering up to 1GB in +// memory would be pure waste. +func seedContentOpener(seed *torrent.Seed, file torrent.SeedFile, ctx context.Context) func() (model.FileStreamer, error) { + source := firstUsableSeedSource(file) + if source == "" { + return nil + } + if ctx == nil { + ctx = context.Background() + } + return func() (model.FileStreamer, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, source, nil) + if err != nil { + return nil, err + } + resp, err := seedSourceHTTPClient.Do(req) + if err != nil { + return nil, fmt.Errorf("下载种子内容失败: %w", err) + } + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + resp.Body.Close() + return nil, fmt.Errorf("下载种子内容失败: HTTP %d", resp.StatusCode) + } + // Content-Length is authoritative when present; fall back to the seed + // metadata. Reject up-front rather than streaming an oversized body. + size := resp.ContentLength + if size < 0 { + size = file.Size + } + if size > maxTorrentGenFileSize { + resp.Body.Close() + return nil, fmt.Errorf("种子内容超过大小上限: %d", size) + } + if file.Size > 0 && resp.ContentLength >= 0 && resp.ContentLength != file.Size { + resp.Body.Close() + return nil, fmt.Errorf("种子内容大小不匹配: 期望 %d,实际 %d", file.Size, resp.ContentLength) + } + + obj := &model.Object{ + Name: stdpath.Base(file.Path), + Size: size, + Modified: time.Now(), + IsFolder: false, + } + fs := &stream.FileStream{ + Ctx: ctx, + Obj: obj, + Reader: io.LimitReader(resp.Body, maxTorrentGenFileSize), + } + // Keep the body alive for the lifetime of the streamer and expose range + // reads over an in-memory prefix so proof/hash windows can be re-read. + fs.Add(resp.Body) + return fs, nil + } +} + +func firstUsableSeedSource(file torrent.SeedFile) string { + if _, err := seedSiteConfig(); err != nil { + return "" + } + for _, source := range file.Sources { + candidate, parseErr := url.Parse(source.URL) + if parseErr != nil { + continue + } + if err := validateSeedHost(candidate); err != nil { + continue + } + validPath := (source.Type == "openlist-direct" && strings.HasPrefix(candidate.EscapedPath(), "/d/")) || + (source.Type == "openlist-share" && strings.HasPrefix(candidate.EscapedPath(), "/sd/")) + if !validPath { + continue + } + if source.ExpiresAt != "" { + expires, parseErr := time.Parse(time.RFC3339, source.ExpiresAt) + if parseErr != nil || time.Now().After(expires) { + continue + } + } + return candidate.String() + } + return "" +} + +func seedTotalSize(seed *torrent.Seed) int64 { + var total int64 + for _, file := range seed.Files { + total += file.Size + } + return total +} + +func seedDiagnostics(seed *torrent.Seed) gin.H { + return gin.H{ + "oss": torrent.DiagnoseConversion(seed, "oss"), + "torrent": torrent.DiagnoseConversion(seed, "torrent"), + "cas": torrent.DiagnoseConversion(seed, "cas"), + } +} + +func seedConversionStates(seed *torrent.Seed) gin.H { + states := make(gin.H, 3) + for _, format := range []string{"oss", "torrent", "cas"} { + missing := torrent.DiagnoseConversion(seed, format) + states[format] = gin.H{"feasible": len(missing) == 0, "missing": missing} + } + return states +} diff --git a/server/handles/torrent_seed_test.go b/server/handles/torrent_seed_test.go new file mode 100644 index 00000000..ea4c9583 --- /dev/null +++ b/server/handles/torrent_seed_test.go @@ -0,0 +1,259 @@ +package handles + +import ( + "net/http" + "net/url" + "strings" + "testing" + + "github.com/OpenListTeam/OpenList/v4/pkg/torrent" + "github.com/OpenListTeam/OpenList/v4/pkg/utils" +) + +// withSeedSite pins the configured seed site for the duration of a test. +func withSeedSite(t *testing.T, site string) { + t.Helper() + previous := seedSiteURLProvider + seedSiteURLProvider = func() string { return site } + t.Cleanup(func() { seedSiteURLProvider = previous }) +} + +func mustParse(t *testing.T, raw string) *url.URL { + t.Helper() + u, err := url.Parse(raw) + if err != nil { + t.Fatalf("url.Parse(%q) error = %v", raw, err) + } + return u +} + +// TestSameSeedHostNormalizesDefaultPort guards against the regression where a +// configured "https://pan.example.com" and an embedded +// "https://pan.example.com:443/..." were treated as different hosts, silently +// discarding otherwise valid sources. +func TestSameSeedHostNormalizesDefaultPort(t *testing.T) { + cases := []struct { + a, b string + want bool + }{ + {"https://pan.example.com", "https://pan.example.com:443/x", true}, + {"http://pan.example.com", "http://pan.example.com:80/x", true}, + {"https://pan.example.com:8443", "https://pan.example.com:8443/x", true}, + {"https://pan.example.com:8443", "https://pan.example.com", false}, + {"https://pan.example.com", "http://pan.example.com", false}, + {"https://pan.example.com", "https://evil.example.com", false}, + {"https://pan.example.com", "https://pan.example.com.evil.com", false}, + } + for _, tc := range cases { + if got := sameSeedHost(mustParse(t, tc.a), mustParse(t, tc.b)); got != tc.want { + t.Errorf("sameSeedHost(%q, %q) = %v, want %v", tc.a, tc.b, got, tc.want) + } + } +} + +// TestValidateSeedHostRejectsForeignHosts is the core SSRF guarantee: a seed +// source may only ever point at the operator-configured site. +func TestValidateSeedHostRejectsForeignHosts(t *testing.T) { + withSeedSite(t, "https://pan.example.com") + + rejected := []string{ + "http://169.254.169.254/latest/meta-data/", // cloud metadata + "http://127.0.0.1:5244/api/fs/list", // local admin API + "http://localhost:5244/d/secret", + "https://evil.example.com/d/secret", + "https://pan.example.com.evil.com/d/x", + "file:///etc/passwd", + "ftp://pan.example.com/x", + "https://user:pass@pan.example.com/d/x", // embedded credentials + } + for _, raw := range rejected { + if err := validateSeedHost(mustParse(t, raw)); err == nil { + t.Errorf("validateSeedHost(%q) accepted a disallowed URL", raw) + } + } + + allowed := []string{ + "https://pan.example.com/d/some/file", + "https://pan.example.com:443/sd/abc123", + } + for _, raw := range allowed { + if err := validateSeedHost(mustParse(t, raw)); err != nil { + t.Errorf("validateSeedHost(%q) rejected a valid URL: %v", raw, err) + } + } +} + +func TestValidateSeedHostWithoutConfiguredSite(t *testing.T) { + withSeedSite(t, "") + if err := validateSeedHost(mustParse(t, "https://pan.example.com/d/x")); err == nil { + t.Fatal("validateSeedHost() accepted a source while seed_site_url is unset") + } +} + +// TestRedirectGuardBlocksSSRF is the regression test for the bypass: the first +// hop passes the host allow-list, but a redirect must not be allowed to escape +// to an internal address. +func TestRedirectGuardBlocksSSRF(t *testing.T) { + withSeedSite(t, "https://pan.example.com") + + origin := mustParse(t, "https://pan.example.com/d/file") + check := seedSourceHTTPClient.CheckRedirect + + // A redirect staying on the configured site is fine. + sameHost := &http.Request{URL: mustParse(t, "https://pan.example.com/d/file-2")} + if err := check(sameHost, []*http.Request{{URL: origin}}); err != nil { + t.Fatalf("CheckRedirect() rejected a same-host redirect: %v", err) + } + + // A redirect to the cloud metadata endpoint must be refused. + metadata := &http.Request{URL: mustParse(t, "http://169.254.169.254/latest/meta-data/")} + if err := check(metadata, []*http.Request{{URL: origin}}); err == nil { + t.Fatal("CheckRedirect() allowed a redirect to the cloud metadata endpoint") + } + + // A redirect to localhost must be refused. + local := &http.Request{URL: mustParse(t, "http://127.0.0.1:5244/api/fs/list")} + if err := check(local, []*http.Request{{URL: origin}}); err == nil { + t.Fatal("CheckRedirect() allowed a redirect to localhost") + } + + // A same-host redirect that downgrades https -> http must be refused. + downgrade := &http.Request{URL: mustParse(t, "http://pan.example.com/d/file")} + if err := check(downgrade, []*http.Request{{URL: origin}}); err == nil { + t.Fatal("CheckRedirect() allowed a scheme downgrade") + } +} + +func TestRedirectGuardLimitsHopCount(t *testing.T) { + withSeedSite(t, "https://pan.example.com") + + origin := mustParse(t, "https://pan.example.com/d/file") + via := make([]*http.Request, maxSeedSourceRedirects) + for i := range via { + via[i] = &http.Request{URL: origin} + } + next := &http.Request{URL: mustParse(t, "https://pan.example.com/d/file-2")} + if err := seedSourceHTTPClient.CheckRedirect(next, via); err == nil { + t.Fatal("CheckRedirect() accepted more redirects than maxSeedSourceRedirects") + } +} + +// TestValidateSeedSourceEnforcesPathPrefix documents the per-type path contract. +func TestValidateSeedSourceEnforcesPathPrefix(t *testing.T) { + withSeedSite(t, "https://pan.example.com") + + if err := validateSeedSource(torrent.SeedSource{ + Type: "openlist-direct", + URL: "https://pan.example.com/sd/abc", + }); err == nil { + t.Fatal("validateSeedSource() accepted a share path for a direct source") + } + if err := validateSeedSource(torrent.SeedSource{ + Type: "openlist-share", + URL: "https://pan.example.com/d/file", + }); err == nil { + t.Fatal("validateSeedSource() accepted a direct path for a share source") + } + if err := validateSeedSource(torrent.SeedSource{ + Type: "openlist-direct", + URL: "https://evil.example.com/d/file", + }); err == nil { + t.Fatal("validateSeedSource() accepted a foreign host") + } + if err := validateSeedSource(torrent.SeedSource{ + Type: "openlist-direct", + URL: "https://pan.example.com/d/file", + }); err != nil { + t.Fatalf("validateSeedSource() rejected a valid direct source: %v", err) + } +} + +// TestFirstUsableSeedSourceSkipsExpiredAndForeign verifies the selection logic +// only returns sources that are both on-site and not expired. +func TestFirstUsableSeedSourceSkipsExpiredAndForeign(t *testing.T) { + withSeedSite(t, "https://pan.example.com") + + file := torrent.SeedFile{ + Sources: []torrent.SeedSource{ + {Type: "openlist-direct", URL: "https://evil.example.com/d/a"}, + {Type: "openlist-direct", URL: "https://pan.example.com/d/b", ExpiresAt: "2000-01-01T00:00:00Z"}, + {Type: "openlist-share", URL: "https://pan.example.com/sd/good"}, + }, + } + if got := firstUsableSeedSource(file); got != "https://pan.example.com/sd/good" { + t.Fatalf("firstUsableSeedSource() = %q, want the valid share source", got) + } + + // No usable source at all. + none := torrent.SeedFile{ + Sources: []torrent.SeedSource{ + {Type: "openlist-direct", URL: "https://evil.example.com/d/a"}, + }, + } + if got := firstUsableSeedSource(none); got != "" { + t.Fatalf("firstUsableSeedSource() = %q, want empty", got) + } +} + +// TestBuildSeedRapidUploadRequestRejectsMultiFile documents that a multi-file +// torrent cannot be described by a single rapid-upload request. Returning nil +// (instead of silently using Files[0] with the aggregate size) prevents sending +// the destination a size/hash combination that contradicts itself. +func TestBuildSeedRapidUploadRequestRejectsMultiFile(t *testing.T) { + const md5Hex = "0123456789abcdef0123456789abcdef" + + multi := &torrent.Torrent{ + OpenList: &torrent.Seed{ + PieceSize: torrent.DefaultPieceSize, + Files: []torrent.SeedFile{ + {Path: "a.bin", Size: 10, Hashes: torrent.SeedHashes{MD5: md5Hex}}, + {Path: "b.bin", Size: 20, Hashes: torrent.SeedHashes{MD5: md5Hex}}, + }, + }, + } + if req := buildSeedRapidUploadRequest(multi, nil); req != nil { + t.Fatalf("buildSeedRapidUploadRequest() accepted a multi-file torrent: %#v", req) + } + + // A single file must still work. + single := &torrent.Torrent{ + Info: torrent.TorrentInfo{Name: "a.bin", Length: 10}, + OpenList: &torrent.Seed{ + PieceSize: torrent.DefaultPieceSize, + Files: []torrent.SeedFile{ + {Path: "a.bin", Size: 10, Hashes: torrent.SeedHashes{MD5: md5Hex}}, + }, + }, + } + req := buildSeedRapidUploadRequest(single, nil) + if req == nil { + t.Fatal("buildSeedRapidUploadRequest() rejected a valid single-file torrent") + } + if req.Size != 10 { + t.Fatalf("buildSeedRapidUploadRequest() size = %d, want 10", req.Size) + } + if got := req.Whole.GetHash(utils.MD5); !strings.EqualFold(got, md5Hex) { + t.Fatalf("buildSeedRapidUploadRequest() MD5 = %q, want %q", got, md5Hex) + } + + // A single file whose metadata size disagrees with the torrent length is + // internally inconsistent and must also be refused. + mismatched := &torrent.Torrent{ + Info: torrent.TorrentInfo{Name: "a.bin", Length: 99}, + OpenList: &torrent.Seed{ + PieceSize: torrent.DefaultPieceSize, + Files: []torrent.SeedFile{ + {Path: "a.bin", Size: 10, Hashes: torrent.SeedHashes{MD5: md5Hex}}, + }, + }, + } + if req := buildSeedRapidUploadRequest(mismatched, nil); req != nil { + t.Fatalf("buildSeedRapidUploadRequest() accepted a size/hash mismatch: %#v", req) + } +} + +func TestBuildSeedRapidUploadRequestHandlesNil(t *testing.T) { + if req := buildSeedRapidUploadRequest(nil, nil); req != nil { + t.Fatalf("buildSeedRapidUploadRequest(nil) = %#v, want nil", req) + } +} diff --git a/server/handles/transcode.go b/server/handles/transcode.go new file mode 100644 index 00000000..dfe8f258 --- /dev/null +++ b/server/handles/transcode.go @@ -0,0 +1,656 @@ +package handles + +import ( + "crypto/subtle" + "errors" + "fmt" + "net/http" + "net/url" + "os" + "path/filepath" + "strconv" + "strings" + "sync" + "time" + + "github.com/OpenListTeam/OpenList/v4/internal/conf" + "github.com/OpenListTeam/OpenList/v4/internal/driver" + "github.com/OpenListTeam/OpenList/v4/internal/fs" + "github.com/OpenListTeam/OpenList/v4/internal/op" + "github.com/OpenListTeam/OpenList/v4/internal/setting" + "github.com/OpenListTeam/OpenList/v4/internal/sign" + "github.com/OpenListTeam/OpenList/v4/internal/transcode" + "github.com/OpenListTeam/OpenList/v4/server/common" + "github.com/gin-gonic/gin" +) + +// 探测阶段的独立并发上限。 +// +// 转码配额(每用户 3 个活跃任务)只约束「入队」,而配额判定发生在探测之后: +// 并发请求可能同时通过配额预检、各起一个最长 30s 的 ffprobe。这里给探测单独 +// 设一道闸,保证这段窗口内的进程数有界。每用户上限取任务配额值,正常并发播放 +// 不会被误伤(用户有 3 个任务时预检已经先拒绝了)。 +const ( + maxConcurrentProbes = 8 + maxProbesPerUser = transcode.MaxActiveJobsPerUser +) + +type probeLimiter struct { + mu sync.Mutex + active int + users map[string]int +} + +var probeSlots = &probeLimiter{users: make(map[string]int)} + +func (p *probeLimiter) acquire(owner string) bool { + p.mu.Lock() + defer p.mu.Unlock() + if p.active >= maxConcurrentProbes || p.users[owner] >= maxProbesPerUser { + return false + } + p.active++ + p.users[owner]++ + return true +} + +func (p *probeLimiter) release(owner string) { + p.mu.Lock() + defer p.mu.Unlock() + if p.active > 0 { + p.active-- + } + if n := p.users[owner]; n <= 1 { + delete(p.users, owner) + } else { + p.users[owner] = n - 1 + } +} + +// ============================================================ +// 播放端:/api/transcode/play 发起或复用一个转码任务 +// ============================================================ + +// TranscodePlay 入口:POST /api/fs/transcode body: {path} +// +// 行为: +// 1. 检查 transcode_enabled +// 2. fs.Get 获取文件并跑 Decide +// 3. 不需要转码 → 返回 {transcode:false, direct: true} +// 4. 需要转码 → 创建 Job 入队,返回 {transcode:true, job_id, master_url} +func TranscodePlay(c *gin.Context) { + if !transcode.IsEnabled() { + common.SuccessResp(c, gin.H{"transcode": false, "reason": "disabled"}) + return + } + var req struct { + Path string `json:"path" binding:"required"` + } + if err := c.ShouldBindJSON(&req); err != nil { + common.ErrorResp(c, err, 400) + return + } + // 该路由挂在 Auth(true)(允许被禁用的访客通过)之下,必须在这里自行拒绝, + // 否则关闭访客后匿名请求仍能触发 ffprobe/ffmpeg 并占用转码缓存。 + user := mediaUser(c) + if user == nil || (user.IsGuest() && user.Disabled) { + common.ErrorStrResp(c, "Guest user is disabled, login please", 401) + return + } + if !CanUserAccessPath(user, req.Path) { + common.ErrorStrResp(c, "path is not accessible", 403) + return + } + obj, err := fs.Get(c.Request.Context(), req.Path, &fs.GetArgs{NoLog: true}) + if err != nil { + common.ErrorResp(c, err, 500) + return + } + if obj.IsDir() { + common.ErrorStrResp(c, "path is a directory", 400) + return + } + dec := transcode.Decide(transcode.DecideRequest{ + FileName: obj.GetName(), + FileSize: obj.GetSize(), + }) + if !dec.NeedTranscode { + common.SuccessResp(c, gin.H{ + "transcode": false, + "reason": dec.Reason, + }) + return + } + + mgr := transcode.Default() + mgr.Start() + + // apiURL 仅用于返回给浏览器的 master_url(需要跟随请求 Host 才能被客户端访问)。 + apiURL := common.GetApiUrlFromRequest(c.Request) + + // 配额键:普通用户按 ID,访客按 IP——访客共用一个账号, + // 否则一个匿名访客就能占满全部匿名转码配额。 + ownerID := "user:" + strconv.FormatUint(uint64(user.ID), 10) + if user.IsGuest() { + ownerID = "guest:" + c.ClientIP() + } + + // 同一用户重复请求同一文件:直接复用进行中的任务,避免重复 ffprobe 与重复转码。 + if exist, ok := mgr.Scheduler.FindReusable(ownerID, req.Path); ok { + respondTranscodeJob(c, apiURL, exist) + return + } + + // 【内存优化】优先尝试解析为本地文件路径,让 ffmpeg 直接读取本地文件, + // 完全绕过 HTTP /d 代理路径——避免 net.Downloader 给每个 range 请求分配 + // 高达 MaxBufferLimit (默认系统内存 5%) 的大缓冲。这是降低 Go 进程内存 + // 占用最有效的方法。如果不是本地驱动则退回到 HTTP 签名 URL。 + var sourceURL string + if localFile, ok := resolveLocalFilePath(req.Path); ok { + // ffmpeg 接受本地路径作为输入,无需 file:// 前缀 + sourceURL = localFile + fmt.Printf("[transcode] using local file path for ffmpeg: %s\n", localFile) + } else { + // 远程驱动:构造签名 HTTP URL。 + // 这个 URL 会被服务端 ffmpeg 直接访问,绝不能使用 Host / X-Forwarded-Host: + // 未配置 site_url 时攻击者可借此让服务器去拉取任意地址(SSRF)。 + base := trustedInternalBaseURL() + if base == "" { + // 仅 HTTPS 监听且未配置 site_url 时无可用基址; + // 若继续拼出 "/d/..." 会被当成绝对本地路径,ffmpeg 只会报文件不存在。 + common.ErrorStrResp( + c, + "transcode needs site_url (or an HTTP port) to fetch the source", + 503, + ) + return + } + signedPath := sign.Sign(req.Path) + sourceURL = fmt.Sprintf("%s/d%s?sign=%s", base, encodePath(req.Path), signedPath) + } + if !transcode.ValidSourceInput(sourceURL) { + common.ErrorStrResp(c, "unsupported transcode source", 400) + return + } + + // 创建 Job + job := transcode.NewJob() + job.Path = req.Path + job.OwnerID = ownerID + job.SourceURL = sourceURL + // 配额预检:ProbeSource 会起一个最长 30s 的 ffprobe 子进程, + // 不能等探测完才告诉用户「已有 3 个活跃任务」。最终判定仍由 SubmitLimited 原子完成。 + if err := mgr.Scheduler.Capacity(ownerID); err != nil { + common.ErrorResp(c, err, 429) + return + } + if !probeSlots.acquire(ownerID) { + common.ErrorStrResp(c, "too many concurrent transcode requests", 429) + return + } + // 【关键】探测视频总时长(异步会让首响应变慢,但只有一次,必须等待) + // 总时长用于生成完整的 playlist,避免 HLS.js 时长显示错误和拖动跳跃问题 + probe, perr := func() (transcode.SourceProbe, error) { + // 用 defer 释放:探测内部 panic 时也不能把槽位永久占住 + defer probeSlots.release(ownerID) + return transcode.ProbeSource(sourceURL) + }() + if perr != nil { + // 探测失败不阻塞流程,但记录日志;后续 playlist 会缺少总时长信息 + fmt.Printf("[transcode] probe source failed: %v\n", perr) + probe = transcode.SourceProbe{Size: obj.GetSize()} + } else if probe.Size == 0 { + probe.Size = obj.GetSize() + } + job.Probe = probe + profile := transcode.BuildProfile() + job.Profiles = []transcode.Profile{profile} + job.Output = transcode.BuildOutputSpec() + timeoutMin := setting.GetInt(conf.TranscodeJobTimeoutMin, 120) + job.Deadline = time.Now().Add(time.Duration(timeoutMin) * time.Minute).Unix() + submitted, _, err := mgr.Scheduler.SubmitLimited(job) + if err != nil { + // 队列已满 / 用户配额超限:返回 429,前端可降级为直链播放 + common.ErrorResp(c, err, 429) + return + } + respondTranscodeJob(c, apiURL, submitted) +} + +// respondTranscodeJob 返回转码任务的播放信息 +func respondTranscodeJob(c *gin.Context, apiURL string, job *transcode.Job) { + profileName := "" + if len(job.Profiles) > 0 { + profileName = job.Profiles[0].Name + } + masterURL := fmt.Sprintf("%s/tc/%s/%s/master.m3u8", apiURL, job.ID, job.PlayToken) + common.SuccessResp(c, gin.H{ + "transcode": true, + "job_id": job.ID, + "master_url": masterURL, + "profile": profileName, + }) +} + +func encodePath(p string) string { + // 与 down.go 保持一致:保留 '/' 不转义 + parts := strings.Split(p, "/") + for i, s := range parts { + parts[i] = url.PathEscape(s) + } + return strings.Join(parts, "/") +} + +// trustedInternalBaseURL 返回供服务端 ffmpeg 访问本站 /d 接口的可信基址,不读取任何请求头。 +// 优先使用管理员配置的 site_url;未配置时回环到本机实际监听的 HTTP 端口。 +// 两者都不可用(例如仅监听 Unix socket)时返回空串,调用方应拒绝转码。 +func trustedInternalBaseURL() string { + if site := strings.TrimSuffix(conf.Conf.SiteURL, "/"); strings.HasPrefix(site, "http") { + return site + } + if p := conf.Conf.Scheme.HttpPort; p > 0 { + return fmt.Sprintf("http://127.0.0.1:%d", p) + } + return "" +} + +// resolveLocalFilePath 尝试把 OpenList 路径解析为宿主机文件系统的实际路径。 +// 仅对本地驱动 (Local) 有效,其他驱动(云盘等)返回 false。 +// +// 这是内存优化的关键路径:本地文件场景下让 ffmpeg 直接读文件,可以完全绕过 +// HTTP /d 代理路径上的 net.Downloader 大块缓冲(默认每个 range 请求最高 +// 占用 MaxBufferLimit = 系统内存 5%,多 chunk 并发 + 多 range 累计可达数 GB)。 +func resolveLocalFilePath(rawPath string) (string, bool) { + storage, actualPath, err := op.GetStorageAndActualPath(rawPath) + if err != nil || storage == nil { + return "", false + } + if storage.Config().Name != "Local" { + return "", false + } + rooter, ok := storage.(driver.IRootPath) + if !ok { + return "", false + } + root := rooter.GetRootPath() + if root == "" { + return "", false + } + full := filepath.Join(root, actualPath) + // 只接受真实存在的文件,避免传给 ffmpeg 一个无效路径 + fi, err := os.Stat(full) + if err != nil || fi.IsDir() { + return "", false + } + return full, true +} + +// ============================================================ +// 播放端:/tc/:job/:token/... Master/Variant playlist & 切片 +// ============================================================ + +// TCMaster GET /tc/:job/:token/master.m3u8 +func TCMaster(c *gin.Context) { + jobID := c.Param("job") + token := c.Param("token") + job, ok := transcode.Default().Scheduler.Get(jobID) + if !ok { + c.String(http.StatusNotFound, "job not found") + return + } + if !tokenMatches(job.PlayToken, token) { + c.String(http.StatusForbidden, "bad token") + return + } + if len(job.Profiles) == 0 { + c.String(http.StatusInternalServerError, "no profiles") + return + } + apiURL := common.GetApiUrlFromRequest(c.Request) + baseURL := fmt.Sprintf("%s/tc/%s/%s/", apiURL, jobID, token) + body := transcode.Default().Cache.MasterPlaylist(job.Profiles, baseURL) + job.Touch() + c.Header("Content-Type", "application/vnd.apple.mpegurl") + // Master playlist 内容在 job 生命周期内不变,设置较长缓存 + c.Header("Cache-Control", "public, max-age=3600") + c.String(http.StatusOK, body) +} + +// TCPlaylist GET /tc/:job/:token/:profile/playlist.m3u8 +// 等待首切片就绪后返回;阻塞最多 30s +func TCPlaylist(c *gin.Context) { + jobID := c.Param("job") + token := c.Param("token") + profile := c.Param("profile") + mgr := transcode.Default() + job, ok := mgr.Scheduler.Get(jobID) + if !ok { + c.String(http.StatusNotFound, "job not found") + return + } + if !tokenMatches(job.PlayToken, token) { + c.String(http.StatusForbidden, "bad token") + return + } + if !transcode.ValidProfileName(profile) { + c.String(http.StatusBadRequest, "bad profile") + return + } + // 【关键】如果 job 之前因 idle 被 cancel,这里重新激活:复用 jobID/token 重新入队 + // 让 worker 再次启动 ffmpeg 转码,避免播放器看到 404 后无法恢复。 + // 配额/队列已满时不再起新的 ffmpeg,但 playlist 与已缓存的切片仍应可读: + // 这里只记下 quotaBlocked,交给切片请求在「缓存也没有」时才返回 429。 + quotaBlocked := false + if st := job.GetStatus(); st == transcode.JobCancelled || st == transcode.JobFinished || st == transcode.JobFailed { + if _, rerr := mgr.Scheduler.Reactivate(jobID); rerr == nil { + fmt.Printf("[transcode] reactivate job %s on playlist request (was %s)\n", jobID, st) + } else if errors.Is(rerr, transcode.ErrUserQuota) || errors.Is(rerr, transcode.ErrQueueFull) { + quotaBlocked = true + fmt.Printf("[transcode] reactivate job %s on playlist request rejected (%v), serving from cache only\n", jobID, rerr) + } + } + job.Touch() + // 等首切片;配额已满时无法启动新的转码,没必要干等 30s + if !quotaBlocked { + select { + case <-job.WaitReady(): + case <-time.After(30 * time.Second): + c.String(http.StatusGatewayTimeout, "timeout waiting first segment") + return + } + } + if job.Status == transcode.JobFailed { + c.String(http.StatusInternalServerError, "transcode failed: "+job.Error) + return + } + apiURL := common.GetApiUrlFromRequest(c.Request) + baseURL := fmt.Sprintf("%s/tc/%s/%s/%s/", apiURL, jobID, token, profile) + isFinished := job.Status == transcode.JobFinished + // 【关键】传入总时长,让 BuildPlaylist 能够生成完整切片列表(VOD 模式), + // 这样 HLS.js 就知道视频真实总时长,避免时长显示错误和拖动跳跃问题。 + body := transcode.Default().Cache.BuildPlaylist(jobID, profile, job.Output.SegmentDuration, + isFinished, baseURL, job.Probe.Duration) + c.Header("Content-Type", "application/vnd.apple.mpegurl") + if isFinished || job.Probe.Duration > 0 { + // VOD 模式:playlist 完整不变,长缓存 + c.Header("Cache-Control", "public, max-age=3600") + } else { + // 没有总时长时退回 EVENT 模式:短缓存 + c.Header("Cache-Control", "public, max-age=2") + } + c.String(http.StatusOK, body) +} + +// TCSegment GET /tc/:job/:token/:profile/seg-:seq.ts +// 切片就绪则直接返回;否则等待最多 60s +func TCSegment(c *gin.Context) { + jobID := c.Param("job") + token := c.Param("token") + profile := c.Param("profile") + segName := c.Param("seg") + mgr := transcode.Default() + job, ok := mgr.Scheduler.Get(jobID) + if !ok { + c.String(http.StatusNotFound, "job not found") + return + } + if !tokenMatches(job.PlayToken, token) { + c.String(http.StatusForbidden, "bad token") + return + } + if !transcode.ValidProfileName(profile) { + c.String(http.StatusBadRequest, "bad profile") + return + } + // 【关键】job 之前因 idle 被 cancel 时,复用同一 jobID/token 重新入队启动转码。 + // 这样浏览器持有的旧 master.m3u8 + 切片 URL 仍然有效,暂停再继续也不会 404。 + // 配额/队列已满时不重新起 ffmpeg,但已缓存的切片依然要能命中(回看已转完的视频)。 + quotaBlocked := false + if st := job.GetStatus(); st == transcode.JobCancelled || st == transcode.JobFinished || st == transcode.JobFailed { + if _, rerr := mgr.Scheduler.Reactivate(jobID); rerr == nil { + fmt.Printf("[transcode] reactivate job %s on segment request (was %s)\n", jobID, st) + } else if errors.Is(rerr, transcode.ErrUserQuota) || errors.Is(rerr, transcode.ErrQueueFull) { + quotaBlocked = true + fmt.Printf("[transcode] reactivate job %s on segment request rejected (%v), serving from cache only\n", jobID, rerr) + } + } + // seg-N.ts + name := strings.TrimSuffix(strings.TrimPrefix(segName, "seg-"), ".ts") + seq, err := strconv.Atoi(name) + if err != nil { + c.String(http.StatusBadRequest, "bad segment name") + return + } + cache := mgr.Cache + // 【智能调度】触发对应 chunk 的 ffmpeg 启动(如已运行则只更新 LastAccess) + // 这是用户拖动进度条到任意位置时能快速响应的关键。 + // 配额已满时不能触发新的 ffmpeg,否则「限制并发」就没有意义了。 + if mgr.Chunks != nil && !quotaBlocked { + chunkIdx := mgr.Chunks.EnsureChunkRunningForSeg(jobID, profile, seq) + fmt.Printf("[tc-segment] req seq=%d chunk=%d job=%s\n", seq, chunkIdx, jobID) + } + info, err := cache.GetSegment(jobID, profile, seq) + if err != nil { + if quotaBlocked { + // 配额已满且该切片不在缓存里:只能拒绝,让前端降级为直链播放 + c.String(http.StatusTooManyRequests, "too many active transcode jobs") + return + } + // 等待 + waitStart := time.Now() + info, err = cache.WaitForSegment(jobID, profile, seq, 60*time.Second) + if err != nil { + fmt.Printf("[tc-segment] req seq=%d FAILED after %.1fs: %v\n", seq, time.Since(waitStart).Seconds(), err) + c.String(http.StatusNotFound, "segment not available: "+err.Error()) + return + } + fmt.Printf("[tc-segment] req seq=%d ready after %.1fs\n", seq, time.Since(waitStart).Seconds()) + } + job.Touch() + // 切片内容不可变,设置强缓存避免浏览器/HLS.js 反复请求同一切片 + c.Header("Cache-Control", "public, max-age=86400, immutable") + // 【内存优化】流式发送切片文件,避免 gin c.File() 将整个切片读入内存 + // 注意:不能使用 http.ServeFile,它会自动嗅探 Content-Type 并处理 Range 请求, + // 与 gin 已写入的响应头冲突,导致 HLS.js 收到异常数据触发 bufferAppendError。 + f, err := os.Open(info.Path) + if err != nil { + c.String(http.StatusInternalServerError, "open segment: "+err.Error()) + return + } + defer f.Close() + fi, err := f.Stat() + if err != nil { + c.String(http.StatusInternalServerError, "stat segment: "+err.Error()) + return + } + c.DataFromReader(http.StatusOK, fi.Size(), "video/mp2t", f, nil) +} + +// ============================================================ +// Worker 协议端 +// ============================================================ + +// WorkerRegister POST /api/transcode/worker/register +func WorkerRegister(c *gin.Context) { + if !transcode.IsEnabled() { + common.ErrorStrResp(c, "transcode disabled", 403) + return + } + if !checkWorkerSecret(c) { + common.ErrorStrResp(c, "unauthorized", 401) + return + } + var req transcode.RegisterRequest + if err := c.ShouldBindJSON(&req); err != nil { + common.ErrorResp(c, err, 400) + return + } + w := transcode.Default().Registry.Register(&req, false) + common.SuccessResp(c, transcode.RegisterResponse{ + WorkerID: w.ID, + HeartbeatInterval: 10, + ClaimStrategy: "pull", + ProtocolVersion: transcode.ProtocolVersion, + }) +} + +// WorkerHeartbeat POST /api/transcode/worker/heartbeat +func WorkerHeartbeat(c *gin.Context) { + if !checkWorkerSecret(c) { + common.ErrorStrResp(c, "unauthorized", 401) + return + } + var req transcode.HeartbeatRequest + if err := c.ShouldBindJSON(&req); err != nil { + common.ErrorResp(c, err, 400) + return + } + mgr := transcode.Default() + if _, ok := mgr.Registry.Heartbeat(&req); !ok { + common.SuccessResp(c, transcode.HeartbeatResponse{OK: false, Kick: true}) + return + } + cancellations := mgr.Scheduler.PopCancellations(req.WorkerID) + common.SuccessResp(c, transcode.HeartbeatResponse{OK: true, Cancel: cancellations}) +} + +// WorkerClaim POST /api/transcode/worker/claim +func WorkerClaim(c *gin.Context) { + if !checkWorkerSecret(c) { + common.ErrorStrResp(c, "unauthorized", 401) + return + } + var req transcode.ClaimRequest + if err := c.ShouldBindJSON(&req); err != nil { + common.ErrorResp(c, err, 400) + return + } + wait := time.Duration(req.Wait) * time.Second + if wait > 30*time.Second { + wait = 30 * time.Second + } + jobs := transcode.Default().Scheduler.Claim(req.WorkerID, req.Slots, wait) + common.SuccessResp(c, transcode.ClaimResponse{Jobs: jobs}) +} + +// WorkerSegmentUpload PUT /api/transcode/worker/segment?job=&profile=&seq=&duration=&final= +func WorkerSegmentUpload(c *gin.Context) { + jobID := c.Query("job") + profile := c.Query("profile") + seq, _ := strconv.Atoi(c.Query("seq")) + dur, _ := strconv.ParseFloat(c.Query("duration"), 64) + isFinal, _ := strconv.ParseBool(c.Query("final")) + job, ok := transcode.Default().Scheduler.Get(jobID) + if !ok { + common.ErrorStrResp(c, "job not found", 404) + return + } + // 校验 callback_token (Bearer),使用恒定时间比较避免时序侧信道 + auth := c.GetHeader("Authorization") + expect := "Bearer " + job.CallbackToken + if subtle.ConstantTimeCompare([]byte(auth), []byte(expect)) != 1 { + common.ErrorStrResp(c, "unauthorized", 401) + return + } + // profile 会被拼进缓存目录路径,必须是白名单字符,防止 ../ 路径穿越 + if !transcode.ValidProfileName(profile) { + common.ErrorStrResp(c, "invalid profile", 400) + return + } + // 只接受任务声明过的档位,避免持有 callback_token 的一方往任意档位目录写数据 + if !jobHasProfile(job, profile) { + common.ErrorStrResp(c, "profile not declared for this job", 400) + return + } + defer c.Request.Body.Close() + _, err := transcode.Default().Cache.PutSegment(jobID, profile, seq, dur, c.Request.Body, isFinal) + if err != nil { + switch err { + case transcode.ErrInvalidProfile, transcode.ErrInvalidJobID, transcode.ErrInvalidSegment: + common.ErrorResp(c, err, 400) + case transcode.ErrSegmentTooLarge: + common.ErrorResp(c, err, 413) + default: + common.ErrorResp(c, err, 500) + } + return + } + if seq == 0 { + job.MarkReady() + } + c.Status(http.StatusNoContent) +} + +// WorkerJobFinish POST /api/transcode/worker/job/finish +func WorkerJobFinish(c *gin.Context) { + var req transcode.FinishRequest + if err := c.ShouldBindJSON(&req); err != nil { + common.ErrorResp(c, err, 400) + return + } + job, ok := transcode.Default().Scheduler.Get(req.JobID) + if !ok { + common.ErrorStrResp(c, "job not found", 404) + return + } + auth := c.GetHeader("Authorization") + if !tokenMatches("Bearer "+job.CallbackToken, auth) { + common.ErrorStrResp(c, "unauthorized", 401) + return + } + if err := transcode.Default().Scheduler.Finish(&req); err != nil { + common.ErrorResp(c, err, 500) + return + } + common.SuccessResp(c) +} + +// ListTranscodeWorkers GET /api/admin/transcode/workers (admin) +func ListTranscodeWorkers(c *gin.Context) { + common.SuccessResp(c, transcode.Default().Registry.List()) +} + +// ListTranscodeJobs GET /api/admin/transcode/jobs (admin) +func ListTranscodeJobs(c *gin.Context) { + common.SuccessResp(c, transcode.Default().Scheduler.ListJobs()) +} + +// CancelTranscodeJob POST /api/admin/transcode/cancel +func CancelTranscodeJob(c *gin.Context) { + var req struct { + JobID string `json:"job_id" binding:"required"` + } + if err := c.ShouldBindJSON(&req); err != nil { + common.ErrorResp(c, err, 400) + return + } + transcode.Default().Scheduler.Cancel(req.JobID) + common.SuccessResp(c) +} + +// jobHasProfile 判断 profile 是否属于任务声明的输出档位 +func jobHasProfile(job *transcode.Job, profile string) bool { + if job == nil { + return false + } + for _, p := range job.Profiles { + if p.Name == profile { + return true + } + } + return false +} + +// tokenMatches 恒定时间比较凭据,避免按字节比较造成的时序侧信道 +func tokenMatches(expected, got string) bool { + if expected == "" || got == "" { + return false + } + return subtle.ConstantTimeCompare([]byte(expected), []byte(got)) == 1 +} + +func checkWorkerSecret(c *gin.Context) bool { + auth := c.GetHeader("Authorization") + auth = strings.TrimPrefix(auth, "Bearer ") + return transcode.VerifyWorkerSecret(auth) +} diff --git a/server/mcp/handler_test.go b/server/mcp/handler_test.go index d3369576..a7515773 100644 --- a/server/mcp/handler_test.go +++ b/server/mcp/handler_test.go @@ -200,8 +200,10 @@ func TestInitializeNegotiatesUnsupportedProtocolVersion(t *testing.T) { func TestCreateSessionPrunesUserSessionLimit(t *testing.T) { srv := newTestServer(nil) var firstSessionID string + baseTime := time.Now() for i := range maxUserSessions { currentSession := srv.createSession(1) + currentSession.lastUsedAt = baseTime.Add(time.Duration(i)) if i == 0 { firstSessionID = currentSession.id } @@ -223,8 +225,10 @@ func TestCreateSessionPrunesUserSessionLimit(t *testing.T) { func TestCreateSessionPrunesGlobalSessionLimit(t *testing.T) { srv := newTestServer(nil) var firstSessionID string + baseTime := time.Now() for i := range maxSessions { currentSession := srv.createSession(uint(i + 1)) + currentSession.lastUsedAt = baseTime.Add(time.Duration(i)) if i == 0 { firstSessionID = currentSession.id } diff --git a/server/router.go b/server/router.go index 4be0dd84..fe210e6e 100644 --- a/server/router.go +++ b/server/router.go @@ -67,6 +67,11 @@ func Init(e *gin.Engine) { g.HEAD("/sad/:sid", middlewares.EmptyPathParse, middlewares.SharingIdParse, handles.SharingArchiveExtract) g.HEAD("/sad/:sid/*path", middlewares.PathParse, middlewares.SharingIdParse, handles.SharingArchiveExtract) + // 转码播放:master/variant playlist 与切片下发;token 已包含在路径中作为鉴权 + g.GET("/tc/:job/:token/master.m3u8", handles.TCMaster) + g.GET("/tc/:job/:token/:profile/playlist.m3u8", handles.TCPlaylist) + g.GET("/tc/:job/:token/:profile/:seg", handles.TCSegment) + api := g.Group("/api") auth := api.Group("", middlewares.Auth(false)) webauthn := api.Group("/authn", middlewares.Authn) @@ -76,6 +81,10 @@ func Init(e *gin.Engine) { api.POST("/auth/login/ldap", handles.LoginLdap) auth.GET("/me", handles.CurrentUser) auth.POST("/me/update", handles.UpdateCurrent) + // 媒体应用密钥(每个登录用户都可以为自己的客户端签发密钥) + auth.GET("/me/appkey/list", handles.ListMediaAppKeys) + auth.POST("/me/appkey/create", handles.CreateMediaAppKey) + auth.POST("/me/appkey/delete", handles.DeleteMediaAppKey) auth.GET("/me/sshkey/list", handles.ListMyPublicKey) auth.POST("/me/sshkey/add", handles.AddMyPublicKey) auth.POST("/me/sshkey/delete", handles.DeleteMyPublicKey) @@ -100,12 +109,51 @@ func Init(e *gin.Engine) { // no need auth public := api.Group("/public") public.Any("/settings", handles.PublicSettings) + public.GET("/media_status", handles.PublicMediaStatus) + // 封面代理与本地缩略图(媒体库页面使用,免鉴权) + public.GET("/media/proxy", handles.PublicMediaProxy) + public.GET("/media/thumb", handles.PublicMediaThumb) public.Any("/offline_download_tools", handles.OfflineDownloadTools) public.Any("/archive_extensions", handles.ArchiveExtensions) public.Any("/init_status", handles.InitStatus) public.POST("/init/setup", handles.InitSetup) + // 转码 Worker 协议端点(远程 Worker 节点对接,由共享密钥/token 鉴权,无需登录) + transcodeWorker := api.Group("/transcode/worker") + transcodeWorker.POST("/register", handles.WorkerRegister) + transcodeWorker.POST("/heartbeat", handles.WorkerHeartbeat) + transcodeWorker.POST("/claim", handles.WorkerClaim) + transcodeWorker.PUT("/segment", handles.WorkerSegmentUpload) + transcodeWorker.POST("/job/finish", handles.WorkerJobFinish) + + // Emby 兼容 API(自带鉴权与开关判断,路径大小写由 handler 归一化) + // 使用 @ 前缀,避免与用户自己创建的 /emby、/rest 目录冲突 + g.Any("/@emby/*path", handles.EmbyAPI) + g.Any("/@mediabrowser/*path", handles.EmbyAPI) + // OpenSubsonic API(自带鉴权与开关判断) + g.Any("/@rest/*path", handles.SubsonicAPI) + _fs(auth.Group("/fs")) + // 媒体库公开 API:与文件浏览一致,允许访客访问(访客仍受自身 base_path/权限限制) + mediaPublic := auth.Group("/fs/media", handles.RequireMediaEnabled) + mediaPublic.GET("/list", handles.PublicListMedia) + mediaPublic.GET("/item/:id", handles.PublicGetMedia) + mediaPublic.GET("/albums", handles.PublicListAlbums) + mediaPublic.GET("/album", handles.PublicGetAlbum) + mediaPublic.GET("/folders", handles.PublicListFolders) + mediaPublic.GET("/scan_paths", handles.PublicListScanPaths) + mediaPublic.GET("/strm", handles.GetMediaStrm) + mediaPublic.GET("/stream_url", handles.GetMediaStreamURL) + // 播放进度与收藏(与 Emby / Subsonic 共用同一份用户数据) + mediaPublic.GET("/userdata", handles.GetMediaUserDataBatch) + mediaPublic.GET("/progress", handles.GetMediaProgress) + mediaPublic.POST("/progress", handles.SaveMediaProgress) + mediaPublic.POST("/favorite", handles.SetMediaFavorite) + mediaPublic.GET("/favorites", handles.ListMediaFavoritesAPI) + mediaPublic.GET("/resume", handles.ListMediaResumeAPI) + // 阅读器批注(登录用户私有数据;访客只读返回空列表,写入直接 403) + mediaPublic.GET("/annotations", handles.ListMediaAnnotationsAPI) + mediaPublic.POST("/annotations", handles.SyncMediaAnnotationsAPI) fsAndShare(api.Group("/fs", middlewares.Auth(true))) _task(auth.Group("/task", middlewares.AuthNotGuest)) _sharing(auth.Group("/share", middlewares.AuthNotGuest)) @@ -190,6 +238,42 @@ func admin(g *gin.RouterGroup) { scan.POST("/start", handles.StartManualScan) scan.POST("/stop", handles.StopManualScan) scan.GET("/progress", handles.GetManualScanProgress) + + // 媒体库管理 + mediaAdmin := g.Group("/media") + mediaAdmin.GET("/config/list", handles.ListMediaConfigs) + mediaAdmin.POST("/config/save", handles.SaveMediaConfig) + mediaAdmin.GET("/items", handles.ListMediaItemsAdmin) + mediaAdmin.POST("/items/update", handles.UpdateMediaItemAdmin) + mediaAdmin.POST("/items/delete", handles.DeleteMediaItemAdmin) + mediaAdmin.POST("/scan/start", handles.StartMediaScan) + mediaAdmin.GET("/scan/progress", handles.GetMediaScanProgress) + mediaAdmin.POST("/scrape/start", handles.StartMediaScrape) + mediaAdmin.GET("/scrape/stats", handles.GetMediaScrapeStats) + mediaAdmin.POST("/clear", handles.ClearMediaDB) + mediaAdmin.POST("/clear_scrape", handles.ClearMediaScrape) + mediaAdmin.POST("/delete_invalid", handles.DeleteInvalidMedia) + // 扫描路径管理 + mediaAdmin.GET("/scan_paths", handles.ListMediaScanPaths) + mediaAdmin.POST("/scan_paths/create", handles.CreateMediaScanPath) + mediaAdmin.POST("/scan_paths/update", handles.UpdateMediaScanPath) + mediaAdmin.POST("/scan_paths/delete", handles.DeleteMediaScanPath) + mediaAdmin.POST("/scan_paths/clear", handles.ClearMediaScanPathDB) + // 导入 / 导出 + mediaAdmin.GET("/export", handles.ExportMediaDB) + mediaAdmin.POST("/import", handles.ImportMediaDB) + // 媒体应用密钥(Subsonic/Emby/MoviePilot 等无交互客户端) + mediaAdmin.GET("/appkey/list", handles.ListMediaAppKeys) + mediaAdmin.POST("/appkey/create", handles.CreateMediaAppKey) + mediaAdmin.POST("/appkey/delete", handles.DeleteMediaAppKey) + // STRM 批量生成 + mediaAdmin.POST("/strm/generate", handles.GenerateMediaStrm) + + // 转码管理 + transcodeAdmin := g.Group("/transcode") + transcodeAdmin.GET("/workers", handles.ListTranscodeWorkers) + transcodeAdmin.GET("/jobs", handles.ListTranscodeJobs) + transcodeAdmin.POST("/cancel", handles.CancelTranscodeJob) } func fsAndShare(g *gin.RouterGroup) { @@ -198,6 +282,9 @@ func fsAndShare(g *gin.RouterGroup) { a := g.Group("/archive") a.Any("/meta", handles.FsArchiveMetaSplit) a.Any("/list", handles.FsArchiveListSplit) + + // 转码 - 用户态:申请播放 + g.POST("/transcode/play", handles.RequireMediaEnabled, handles.TranscodePlay) } func _fs(g *gin.RouterGroup) { @@ -234,6 +321,19 @@ func _fs(g *gin.RouterGroup) { g.POST("/torrent/upload_parse", handles.UploadTorrentAndParse) g.POST("/torrent/rapid_upload", handles.TorrentRapidUpload) g.POST("/torrent/generate", handles.GenerateTorrentForPath) + // Unified transfer seed APIs. Legacy torrent routes above remain supported. + seed := g.Group("/seed") + seed.POST("/parse", handles.ParseSeed) + seed.POST("/upload_parse", handles.UploadSeedAndParse) + seed.POST("/generate", handles.GenerateSeedForPaths) + seed.POST("/convert", handles.ConvertSeed) + seed.POST("/diagnose", handles.DiagnoseSeed) + seed.POST("/capabilities", handles.SeedCapabilities) + seed.POST("/rapid_upload", handles.QuickSaveSeed) + seed.POST("/offline_download", handles.QuickSaveSeed) + seed.POST("/update", handles.UpdateSeed) + seed.POST("/quick_save", handles.QuickSaveSeed) + seed.POST("/update_channels", handles.UpdateSeedChannels) // Direct upload (client-side upload to storage) g.POST("/get_direct_upload_info", handles.FsGetDirectUploadInfo) } diff --git a/server/s3/redirect.go b/server/s3/redirect.go index 6d8d430b..b61a9df1 100644 --- a/server/s3/redirect.go +++ b/server/s3/redirect.go @@ -159,9 +159,18 @@ func s3RequestAuthorized(r *http.Request, authPairs map[string]string) bool { if len(authPairs) == 0 { return true } - result := signature.V4SignVerify(r) + // Verify against the keys this server was configured with. V4SignVerify and + // V2SignVerify read the signature package's process-wide key store, which + // gofakes3 never writes (keys are kept per instance), so they always + // returned InvalidAccessKeyId and the 302/307 direct-transfer redirects + // never ran. Same V4-then-V2 order the auth middleware uses. + lookup := func(accessKey string) (string, bool) { + secret, ok := authPairs[accessKey] + return secret, ok + } + result := signature.V4SignVerifyWithLookup(r, lookup) if result == signature.ErrUnsupportAlgorithm { - result = signature.V2SignVerify(r) + result = signature.V2SignVerifyWithLookup(r, lookup) } return result == signature.ErrNone } diff --git a/server/static/static.go b/server/static/static.go index 38eee2df..200dc397 100644 --- a/server/static/static.go +++ b/server/static/static.go @@ -7,6 +7,7 @@ import ( "io" "io/fs" "net/http" + "net/url" "os" "strings" @@ -20,8 +21,8 @@ import ( type ManifestIcon struct { Src string `json:"src"` - Sizes string `json:"sizes"` - Type string `json:"type"` + Sizes string `json:"sizes,omitempty"` + Type string `json:"type,omitempty"` } type Manifest struct { @@ -131,6 +132,19 @@ func UpdateIndex() { utils.Log.Debug("Index.html update completed") } +func manifestIcon(src string) ManifestIcon { + icon := ManifestIcon{Src: src, Sizes: "any"} + u, err := url.Parse(src) + if err != nil { + return icon + } + mimeType := utils.GetMimeType(u.Path) + if strings.HasPrefix(mimeType, "image/") { + icon.Type = mimeType + } + return icon +} + func ManifestJSON(c *gin.Context) { // Get site configuration to ensure consistent base path handling siteConfig := getSiteConfig() @@ -156,13 +170,7 @@ func ManifestJSON(c *gin.Context) { Scope: scope, StartURL: startURL, Name: siteTitle, - Icons: []ManifestIcon{ - { - Src: logoUrl, - Sizes: "512x512", - Type: "image/png", - }, - }, + Icons: []ManifestIcon{manifestIcon(logoUrl)}, } c.Header("Content-Type", "application/json") diff --git a/server/webdav/file.go b/server/webdav/file.go index ea609973..ed4c3a70 100644 --- a/server/webdav/file.go +++ b/server/webdav/file.go @@ -6,9 +6,12 @@ package webdav import ( "context" + stderrors "errors" + "fmt" "net/http" "path" "path/filepath" + "strings" "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/errs" @@ -16,7 +19,10 @@ import ( "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/OpenListTeam/OpenList/v4/server/common" + "github.com/google/uuid" "github.com/pkg/errors" + log "github.com/sirupsen/logrus" + "gorm.io/gorm" ) // slashClean is equivalent to but slightly more efficient than @@ -28,6 +34,271 @@ func slashClean(name string) string { return path.Clean(name) } +func moveResource(ctx context.Context, src, dstDir, srcDir, srcName, dstName string) error { + if srcDir == dstDir { + return fs.Rename(ctx, src, dstName) + } + if _, err := fs.Move(context.WithValue(ctx, conf.NoTaskKey, struct{}{}), src, dstDir); err != nil { + return err + } + if srcName != dstName { + return fs.Rename(ctx, path.Join(dstDir, srcName), dstName) + } + return nil +} + +type fileMutation struct { + rollback func() error + finalize func() + backupPaths []string +} + +type fileBackup struct { + originalPath string + backupPath string + renamed bool +} + +func stagingContext(ctx context.Context) context.Context { + ctx = context.WithValue(ctx, conf.NoTaskKey, struct{}{}) + return context.WithValue(ctx, conf.SkipHookKey, struct{}{}) +} + +func removeIfExists(ctx context.Context, name string) error { + if _, err := fs.Get(ctx, name, &fs.GetArgs{}); err != nil { + if errs.IsObjectNotFound(err) { + return nil + } + return err + } + return fs.Remove(ctx, name) +} + +func cleanupAfterFailure(ctx context.Context, name string, cause error) error { + if err := removeIfExists(stagingContext(context.WithoutCancel(ctx)), name); err != nil { + log.WithError(err).Errorf("failed to clean up WebDAV staging path %s", name) + return fmt.Errorf("%w; failed to clean up %s: %w", cause, name, err) + } + return cause +} + +func newFileBackup(ctx context.Context, originalPath string, keepOriginal bool) (*fileBackup, error) { + backupPath := path.Join(path.Dir(originalPath), ".openlist-backup-"+uuid.NewString()) + backup := &fileBackup{originalPath: originalPath, backupPath: backupPath} + if !keepOriginal { + err := fs.Rename(stagingContext(ctx), originalPath, path.Base(backupPath)) + if err == nil { + backup.renamed = true + return backup, nil + } + if !errs.IsNotImplementError(err) && !errs.IsNotSupportError(err) { + return nil, errors.WithMessage(err, "failed to rename object into rollback backup") + } + } + if _, err := fs.CopyTo(stagingContext(ctx), originalPath, path.Dir(originalPath), path.Base(backupPath)); err != nil { + return nil, cleanupAfterFailure(ctx, backupPath, errors.WithMessage(err, "failed to copy object into rollback backup")) + } + return backup, nil +} + +func (b *fileBackup) restore(ctx context.Context) error { + if b == nil { + return nil + } + ctx = context.WithoutCancel(ctx) + if err := removeIfExists(ctx, b.originalPath); err != nil { + return err + } + if b.renamed { + if err := fs.Rename(ctx, b.backupPath, path.Base(b.originalPath)); err != nil { + return errors.WithMessagef(err, "failed to restore %s from %s", b.originalPath, b.backupPath) + } + return nil + } + if _, err := fs.CopyTo(context.WithValue(ctx, conf.NoTaskKey, struct{}{}), b.backupPath, path.Dir(b.originalPath), path.Base(b.originalPath)); err != nil { + return errors.WithMessagef(err, "failed to restore %s from %s", b.originalPath, b.backupPath) + } + if err := fs.Remove(stagingContext(ctx), b.backupPath); err != nil { + if !errs.IsObjectNotFound(err) { + log.Warnf("failed to remove restored WebDAV backup %s: %v", b.backupPath, err) + } + } + return nil +} + +func (b *fileBackup) discard(ctx context.Context) { + if b == nil { + return + } + if err := removeIfExists(stagingContext(context.WithoutCancel(ctx)), b.backupPath); err != nil { + log.Warnf("failed to remove WebDAV backup %s: %v", b.backupPath, err) + } +} + +func installStagedObject(ctx context.Context, stagePath, dstDir, dstName string) error { + if err := fs.Rename(ctx, stagePath, dstName); err == nil { + return nil + } else if !errs.IsNotImplementError(err) && !errs.IsNotSupportError(err) { + return err + } + copyCtx := context.WithValue(ctx, conf.NoTaskKey, struct{}{}) + if _, err := fs.CopyTo(copyCtx, stagePath, dstDir, dstName); err != nil { + return err + } + if err := fs.Remove(stagingContext(context.WithoutCancel(ctx)), stagePath); err != nil { + log.Warnf("failed to remove installed WebDAV staging object %s: %v", stagePath, err) + } + return nil +} + +func copyWithRollback(ctx context.Context, src, dstDir, dstName string, dstExisted, shallow bool) (*fileMutation, error) { + stagePath := path.Join(dstDir, ".openlist-copy-"+uuid.NewString()) + var err error + if shallow { + err = fs.MakeDir(stagingContext(ctx), stagePath) + } else { + _, err = fs.CopyTo(stagingContext(ctx), src, dstDir, path.Base(stagePath)) + } + if err != nil { + return nil, cleanupAfterFailure(ctx, stagePath, err) + } + dstPath := path.Join(dstDir, dstName) + var dstBackup *fileBackup + if dstExisted { + var err error + dstBackup, err = newFileBackup(ctx, dstPath, false) + if err != nil { + return nil, cleanupAfterFailure(ctx, stagePath, err) + } + if !dstBackup.renamed { + if err := removeIfExists(ctx, dstPath); err != nil { + mutation := copyMutation(ctx, dstPath, stagePath, dstBackup) + return mutation, errors.WithMessage(err, "failed to remove destination after creating rollback copy") + } + } + } + mutation := copyMutation(ctx, dstPath, stagePath, dstBackup) + if err := installStagedObject(ctx, stagePath, dstDir, dstName); err != nil { + return mutation, errors.WithMessage(err, "failed to install staged COPY") + } + return mutation, nil +} + +func copyMutation(ctx context.Context, dstPath, stagePath string, dstBackup *fileBackup) *fileMutation { + paths := []string{stagePath} + if dstBackup != nil { + paths = append(paths, dstBackup.backupPath) + } + return &fileMutation{ + backupPaths: paths, + rollback: func() error { + var rollbackErrs []error + if err := removeIfExists(context.WithoutCancel(ctx), dstPath); err != nil { + rollbackErrs = append(rollbackErrs, err) + } + if err := dstBackup.restore(ctx); err != nil { + rollbackErrs = append(rollbackErrs, err) + } + if err := removeIfExists(stagingContext(context.WithoutCancel(ctx)), stagePath); err != nil { + rollbackErrs = append(rollbackErrs, err) + } + return stderrors.Join(rollbackErrs...) + }, + finalize: func() { + if err := removeIfExists(stagingContext(context.WithoutCancel(ctx)), stagePath); err != nil { + log.Warnf("failed to remove WebDAV COPY staging object %s: %v", stagePath, err) + } + dstBackup.discard(ctx) + }, + } +} + +func moveWithRollback(ctx context.Context, src, dst, dstDir, srcDir, srcName, dstName string, dstExisted bool) (*fileMutation, error) { + // Keep a source copy so a partially completed cross-storage MOVE can be undone. + srcBackup, err := newFileBackup(ctx, src, true) + if err != nil { + return nil, err + } + var dstBackup *fileBackup + if dstExisted { + dstBackup, err = newFileBackup(ctx, dst, false) + if err != nil { + srcBackup.discard(ctx) + return nil, err + } + if !dstBackup.renamed { + if err := removeIfExists(ctx, dst); err != nil { + mutation := moveMutation(ctx, dst, dstDir, srcName, dstName, srcBackup, dstBackup) + return mutation, errors.WithMessage(err, "failed to remove destination after creating rollback copy") + } + } + } + mutation := moveMutation(ctx, dst, dstDir, srcName, dstName, srcBackup, dstBackup) + if err := moveResource(ctx, src, dstDir, srcDir, srcName, dstName); err != nil { + return mutation, err + } + return mutation, nil +} + +func moveMutation(ctx context.Context, dst, dstDir, srcName, dstName string, srcBackup, dstBackup *fileBackup) *fileMutation { + paths := []string{srcBackup.backupPath} + if dstBackup != nil { + paths = append(paths, dstBackup.backupPath) + } + return &fileMutation{ + backupPaths: paths, + rollback: func() error { + rollbackCtx := context.WithoutCancel(ctx) + var rollbackErrs []error + candidates := []string{dst} + if srcName != dstName { + candidates = append(candidates, path.Join(dstDir, srcName)) + } + for _, candidate := range candidates { + if err := removeIfExists(rollbackCtx, candidate); err != nil { + rollbackErrs = append(rollbackErrs, err) + } + } + if err := srcBackup.restore(rollbackCtx); err != nil { + rollbackErrs = append(rollbackErrs, err) + } + if err := dstBackup.restore(rollbackCtx); err != nil { + rollbackErrs = append(rollbackErrs, err) + } + return stderrors.Join(rollbackErrs...) + }, + finalize: func() { + srcBackup.discard(ctx) + dstBackup.discard(ctx) + }, + } +} + +func runDeadPropsFileMutation(ctx context.Context, update func(*gorm.DB) error, apply func() (*fileMutation, error)) error { + var mutation *fileMutation + err := withDeadPropsTransaction(ctx, func(tx *gorm.DB) error { + if err := update(tx); err != nil { + return err + } + var err error + mutation, err = apply() + return err + }) + if err != nil { + if mutation != nil && mutation.rollback != nil { + if rollbackErr := mutation.rollback(); rollbackErr != nil { + log.WithError(rollbackErr).Errorf("failed to compensate WebDAV file operation; backups may remain at %s", strings.Join(mutation.backupPaths, ", ")) + return fmt.Errorf("%w; filesystem compensation failed: %w", err, rollbackErr) + } + } + return err + } + if mutation != nil && mutation.finalize != nil { + mutation.finalize() + } + return nil +} + // moveFiles moves files and/or directories from src to dst. // Individual item permission checks are skipped for performance reasons. // @@ -55,21 +326,41 @@ func moveFiles(ctx context.Context, src, dst string, overwrite bool) (status int if !common.CanWrite(user, srcMeta, srcDir) || !common.CanWrite(user, dstMeta, dstDir) { return http.StatusForbidden, nil } - if srcDir == dstDir { - err = fs.Rename(ctx, src, dstName) - } else { - _, err = fs.Move(context.WithValue(ctx, conf.NoTaskKey, struct{}{}), src, dstDir) - if err != nil { - return http.StatusInternalServerError, err - } - if srcName != dstName { - err = fs.Rename(ctx, path.Join(dstDir, srcName), dstName) + if _, err = fs.Get(ctx, src, &fs.GetArgs{}); err != nil { + if errs.IsObjectNotFound(err) { + return http.StatusNotFound, err } + return http.StatusInternalServerError, err } + if dstDirInfo, err := fs.Get(ctx, dstDir, &fs.GetArgs{}); err != nil { + if errs.IsObjectNotFound(err) { + return http.StatusConflict, err + } + return http.StatusMethodNotAllowed, err + } else if !dstDirInfo.IsDir() { + return http.StatusConflict, nil + } + dstExisted := false + if _, err = fs.Get(ctx, dst, &fs.GetArgs{}); err == nil { + dstExisted = true + if !overwrite { + return http.StatusPreconditionFailed, nil + } + } else if !errs.IsObjectNotFound(err) { + return http.StatusInternalServerError, err + } + err = runDeadPropsFileMutation(ctx, + func(tx *gorm.DB) error { return moveDeadProps(tx, src, dst) }, + func() (*fileMutation, error) { + return moveWithRollback(ctx, src, dst, dstDir, srcDir, srcName, dstName, dstExisted && overwrite) + }, + ) if err != nil { return http.StatusInternalServerError, err } - // TODO if there are no files copy, should return 204 + if dstExisted { + return http.StatusNoContent, nil + } return http.StatusCreated, nil } @@ -77,9 +368,10 @@ func moveFiles(ctx context.Context, src, dst string, overwrite bool) (status int // Individual item permission checks are skipped for performance reasons. // // See section 9.8.5 for when various HTTP status codes apply. -func copyFiles(ctx context.Context, src, dst string, overwrite bool) (status int, err error) { +func copyFiles(ctx context.Context, src, dst string, overwrite bool, depth int) (status int, err error) { srcDir := path.Dir(src) dstDir := path.Dir(dst) + dstName := path.Base(dst) user := ctx.Value(conf.UserKey).(*model.User) if !user.CanCopy() { return http.StatusForbidden, nil @@ -98,11 +390,44 @@ func copyFiles(ctx context.Context, src, dst string, overwrite bool) (status int if !common.CanWrite(user, dstMeta, dstDir) { return http.StatusForbidden, nil } - _, err = fs.Copy(context.WithValue(ctx, conf.NoTaskKey, struct{}{}), src, dstDir) + srcInfo, err := fs.Get(ctx, src, &fs.GetArgs{}) + if err != nil { + if errs.IsObjectNotFound(err) { + return http.StatusNotFound, err + } + return http.StatusInternalServerError, err + } + shallow := depth == 0 && srcInfo.IsDir() + if dstDirInfo, err := fs.Get(ctx, dstDir, &fs.GetArgs{}); err != nil { + if errs.IsObjectNotFound(err) { + return http.StatusConflict, err + } + return http.StatusMethodNotAllowed, err + } else if !dstDirInfo.IsDir() { + return http.StatusConflict, nil + } + dstExisted := false + if _, err = fs.Get(ctx, dst, &fs.GetArgs{}); err == nil { + dstExisted = true + if !overwrite { + return http.StatusPreconditionFailed, nil + } + } else if !errs.IsObjectNotFound(err) { + return http.StatusInternalServerError, err + } + + err = runDeadPropsFileMutation(ctx, + func(tx *gorm.DB) error { return copyDeadProps(tx, src, dst) }, + func() (*fileMutation, error) { + return copyWithRollback(ctx, src, dstDir, dstName, dstExisted && overwrite, shallow) + }, + ) if err != nil { return http.StatusInternalServerError, err } - // TODO if there are no files copy, should return 204 + if dstExisted { + return http.StatusNoContent, nil + } return http.StatusCreated, nil } diff --git a/server/webdav/lock.go b/server/webdav/lock.go index 344ac5ce..0d279e06 100644 --- a/server/webdav/lock.go +++ b/server/webdav/lock.go @@ -109,13 +109,15 @@ type LockDetails struct { // ZeroDepth is whether the lock has zero depth. If it does not have zero // depth, it has infinite depth. ZeroDepth bool + // Shared is whether the lock may coexist with other shared locks on Root. + Shared bool } // NewMemLS returns a new in-memory LockSystem. func NewMemLS() LockSystem { return &memLS{ byName: make(map[string]*memLSNode), - byToken: make(map[string]*memLSNode), + byToken: make(map[string]*memLSLock), gen: uint64(time.Now().Unix()), } } @@ -123,7 +125,7 @@ func NewMemLS() LockSystem { type memLS struct { mu sync.Mutex byName map[string]*memLSNode - byToken map[string]*memLSNode + byToken map[string]*memLSLock gen uint64 // byExpiry only contains those nodes whose LockDetails have a finite // Duration and are yet to expire. @@ -149,7 +151,7 @@ func (m *memLS) Confirm(now time.Time, name0, name1 string, conditions ...Condit defer m.mu.Unlock() m.collectExpiredNodes(now) - var n0, n1 *memLSNode + var n0, n1 *memLSLock if name0 != "" { if n0 = m.lookup(slashClean(name0), conditions...); n0 == nil { return nil, ErrConfirmationFailed @@ -184,48 +186,43 @@ func (m *memLS) Confirm(now time.Time, name0, name1 string, conditions ...Condit }, nil } -// lookup returns the node n that locks the named resource, provided that n -// matches at least one of the given conditions and that lock isn't held by -// another party. Otherwise, it returns nil. -// -// n may be a parent of the named resource, if n is an infinite depth lock. -func (m *memLS) lookup(name string, conditions ...Condition) (n *memLSNode) { +func (m *memLS) lookup(name string, conditions ...Condition) *memLSLock { // TODO: support Condition.Not and Condition.ETag. for _, c := range conditions { - n = m.byToken[c.Token] - if n == nil || n.held { + lock := m.byToken[c.Token] + if lock == nil || lock.held { continue } - if name == n.details.Root { - return n + if name == lock.details.Root { + return lock } - if n.details.ZeroDepth { + if lock.details.ZeroDepth { continue } - if n.details.Root == "/" || strings.HasPrefix(name, n.details.Root+"/") { - return n + if lock.details.Root == "/" || strings.HasPrefix(name, lock.details.Root+"/") { + return lock } } return nil } -func (m *memLS) hold(n *memLSNode) { - if n.held { +func (m *memLS) hold(lock *memLSLock) { + if lock.held { panic("webdav: memLS inconsistent held state") } - n.held = true - if n.details.Duration >= 0 && n.byExpiryIndex >= 0 { - heap.Remove(&m.byExpiry, n.byExpiryIndex) + lock.held = true + if lock.details.Duration >= 0 && lock.byExpiryIndex >= 0 { + heap.Remove(&m.byExpiry, lock.byExpiryIndex) } } -func (m *memLS) unhold(n *memLSNode) { - if !n.held { +func (m *memLS) unhold(lock *memLSLock) { + if !lock.held { panic("webdav: memLS inconsistent held state") } - n.held = false - if n.details.Duration >= 0 { - heap.Push(&m.byExpiry, n) + lock.held = false + if lock.details.Duration >= 0 { + heap.Push(&m.byExpiry, lock) } } @@ -235,18 +232,23 @@ func (m *memLS) Create(now time.Time, details LockDetails) (string, error) { m.collectExpiredNodes(now) details.Root = slashClean(details.Root) - if !m.canCreate(details.Root, details.ZeroDepth) { + if !m.canCreate(details.Root, details.ZeroDepth, details.Shared) { return "", ErrLocked } - n := m.create(details.Root) - n.token = m.nextToken() - m.byToken[n.token] = n - n.details = details - if n.details.Duration >= 0 { - n.expiry = now.Add(n.details.Duration) - heap.Push(&m.byExpiry, n) + node := m.create(details.Root) + lock := &memLSLock{ + token: m.nextToken(), + details: details, + node: node, + byExpiryIndex: -1, } - return n.token, nil + node.locks[lock.token] = lock + m.byToken[lock.token] = lock + if lock.details.Duration >= 0 { + lock.expiry = now.Add(lock.details.Duration) + heap.Push(&m.byExpiry, lock) + } + return lock.token, nil } func (m *memLS) Refresh(now time.Time, token string, duration time.Duration) (LockDetails, error) { @@ -254,22 +256,22 @@ func (m *memLS) Refresh(now time.Time, token string, duration time.Duration) (Lo defer m.mu.Unlock() m.collectExpiredNodes(now) - n := m.byToken[token] - if n == nil { + lock := m.byToken[token] + if lock == nil { return LockDetails{}, ErrNoSuchLock } - if n.held { + if lock.held { return LockDetails{}, ErrLocked } - if n.byExpiryIndex >= 0 { - heap.Remove(&m.byExpiry, n.byExpiryIndex) + if lock.byExpiryIndex >= 0 { + heap.Remove(&m.byExpiry, lock.byExpiryIndex) } - n.details.Duration = duration - if n.details.Duration >= 0 { - n.expiry = now.Add(n.details.Duration) - heap.Push(&m.byExpiry, n) + lock.details.Duration = duration + if lock.details.Duration >= 0 { + lock.expiry = now.Add(lock.details.Duration) + heap.Push(&m.byExpiry, lock) } - return n.details, nil + return lock.details, nil } func (m *memLS) Unlock(now time.Time, token string) error { @@ -277,34 +279,42 @@ func (m *memLS) Unlock(now time.Time, token string) error { defer m.mu.Unlock() m.collectExpiredNodes(now) - n := m.byToken[token] - if n == nil { + lock := m.byToken[token] + if lock == nil { return ErrNoSuchLock } - if n.held { + if lock.held { return ErrLocked } - m.remove(n) + m.remove(lock) return nil } -func (m *memLS) canCreate(name string, zeroDepth bool) bool { +func (m *memLS) canCreate(name string, zeroDepth bool, shared ...bool) bool { + wantShared := len(shared) > 0 && shared[0] return walkToRoot(name, func(name0 string, first bool) bool { n := m.byName[name0] if n == nil { return true } if first { - if n.token != "" { - // The target node is already locked. - return false + if len(n.locks) != 0 { + if !wantShared || (!zeroDepth && n.refCount != len(n.locks)) { + return false + } + for _, lock := range n.locks { + if !lock.details.Shared || lock.details.ZeroDepth != zeroDepth { + return false + } + } + return true } - if !zeroDepth { + if !zeroDepth && n.refCount > 0 { // The requested lock depth is infinite, and the fact that n exists // (n != nil) means that a descendent of the target node is locked. return false } - } else if n.token != "" && !n.details.ZeroDepth { + } else if n.hasInfiniteLock() { // An ancestor of the target node is locked with infinite depth. return false } @@ -317,10 +327,8 @@ func (m *memLS) create(name string) (ret *memLSNode) { n := m.byName[name0] if n == nil { n = &memLSNode{ - details: LockDetails{ - Root: name0, - }, - byExpiryIndex: -1, + details: LockDetails{Root: name0}, + locks: make(map[string]*memLSLock), } m.byName[name0] = n } @@ -333,10 +341,10 @@ func (m *memLS) create(name string) (ret *memLSNode) { return ret } -func (m *memLS) remove(n *memLSNode) { - delete(m.byToken, n.token) - n.token = "" - walkToRoot(n.details.Root, func(name0 string, first bool) bool { +func (m *memLS) remove(lock *memLSLock) { + delete(m.byToken, lock.token) + delete(lock.node.locks, lock.token) + walkToRoot(lock.details.Root, func(name0 string, first bool) bool { x := m.byName[name0] x.refCount-- if x.refCount == 0 { @@ -344,8 +352,8 @@ func (m *memLS) remove(n *memLSNode) { } return true }) - if n.byExpiryIndex >= 0 { - heap.Remove(&m.byExpiry, n.byExpiryIndex) + if lock.byExpiryIndex >= 0 { + heap.Remove(&m.byExpiry, lock.byExpiryIndex) } } @@ -366,24 +374,33 @@ func walkToRoot(name string, f func(name0 string, first bool) bool) bool { } type memLSNode struct { - // details are the lock metadata. Even if this node's name is not explicitly locked, - // details.Root will still equal the node's name. + // details identifies the resource path represented by this node. details LockDetails - // token is the unique identifier for this node's lock. An empty token means that - // this node is not explicitly locked. - token string - // refCount is the number of self-or-descendent nodes that are explicitly locked. + // locks contains the independent lock instances rooted at this resource. + locks map[string]*memLSLock + // refCount is the number of self-or-descendent lock instances. refCount int - // expiry is when this node's lock expires. - expiry time.Time - // byExpiryIndex is the index of this node in memLS.byExpiry. It is -1 - // if this node does not expire, or has expired. - byExpiryIndex int - // held is whether this node's lock is actively held by a Confirm call. - held bool } -type byExpiry []*memLSNode +type memLSLock struct { + token string + details LockDetails + node *memLSNode + expiry time.Time + byExpiryIndex int + held bool +} + +func (n *memLSNode) hasInfiniteLock() bool { + for _, lock := range n.locks { + if !lock.details.ZeroDepth { + return true + } + } + return false +} + +type byExpiry []*memLSLock func (b *byExpiry) Len() int { return len(*b) @@ -400,18 +417,18 @@ func (b *byExpiry) Swap(i, j int) { } func (b *byExpiry) Push(x interface{}) { - n := x.(*memLSNode) - n.byExpiryIndex = len(*b) - *b = append(*b, n) + lock := x.(*memLSLock) + lock.byExpiryIndex = len(*b) + *b = append(*b, lock) } func (b *byExpiry) Pop() interface{} { i := len(*b) - 1 - n := (*b)[i] + lock := (*b)[i] (*b)[i] = nil - n.byExpiryIndex = -1 + lock.byExpiryIndex = -1 *b = (*b)[:i] - return n + return lock } const infiniteTimeout = -1 diff --git a/server/webdav/lock_test.go b/server/webdav/lock_test.go index e7fe9706..8353c158 100644 --- a/server/webdav/lock_test.go +++ b/server/webdav/lock_test.go @@ -182,7 +182,10 @@ func TestMemLSLookup(t *testing.T) { goodToken := "" base := m.byName[baseName] if base != nil && (suffix == "" || !lockTestZeroDepth(baseName)) { - goodToken = base.token + for token := range base.locks { + goodToken = token + break + } } for _, token := range []string{badToken, goodToken} { @@ -191,9 +194,9 @@ func TestMemLSLookup(t *testing.T) { } got := m.lookup(name, Condition{Token: token}) - want := base - if token == badToken { - want = nil + var want *memLSLock + if token != badToken { + want = m.byToken[token] } if got != want { t.Errorf("name=%-20qtoken=%q (bad=%t): got %p, want %p", @@ -449,6 +452,46 @@ func TestMemLSExpiry(t *testing.T) { } } +func TestMemLSSharedLockExpiry(t *testing.T) { + m := NewMemLS().(*memLS) + now := time.Unix(0, 0) + first, err := m.Create(now, LockDetails{Root: "/shared", Duration: infiniteTimeout, ZeroDepth: true, Shared: true}) + if err != nil { + t.Fatal(err) + } + second, err := m.Create(now, LockDetails{Root: "/shared", Duration: 2 * time.Second, ZeroDepth: true, Shared: true}) + if err != nil { + t.Fatal(err) + } + third, err := m.Create(now, LockDetails{Root: "/shared", Duration: 4 * time.Second, ZeroDepth: true, Shared: true}) + if err != nil { + t.Fatal(err) + } + + if got := m.byToken[first].details.Duration; got != infiniteTimeout { + t.Fatalf("first lock duration = %v, want infinite", got) + } + if got := m.byToken[second].details.Duration; got != 2*time.Second { + t.Fatalf("second lock duration = %v, want 2s", got) + } + if got := m.byToken[third].details.Duration; got != 4*time.Second { + t.Fatalf("third lock duration = %v, want 4s", got) + } + + m.mu.Lock() + m.collectExpiredNodes(now.Add(2 * time.Second)) + m.mu.Unlock() + if m.byToken[second] != nil { + t.Fatal("second shared lock did not expire independently") + } + if m.byToken[first] == nil || m.byToken[third] == nil { + t.Fatal("independent shared lock expired with second lock") + } + if err := m.consistent(); err != nil { + t.Fatal(err) + } +} + func TestMemLS(t *testing.T) { now := time.Unix(0, 0) m := NewMemLS().(*memLS) @@ -578,8 +621,10 @@ func (m *memLS) consistent() error { // so strings.HasPrefix is equivalent to self-or-descendent name match. // We don't have to worry about "/foo/bar" being a false positive match // for "/foo/b". - if strings.HasPrefix(name0, name) && n0.token != "" { - list = append(list, name0) + if strings.HasPrefix(name0, name) { + for range n0.locks { + list = append(list, name0) + } } } if n.refCount != len(list) { @@ -588,50 +633,34 @@ func (m *memLS) consistent() error { name, n.refCount, list, len(list)) } - // A node n is in m.byToken if it has a non-empty token. - if n.token != "" { - if _, ok := m.byToken[n.token]; !ok { - return fmt.Errorf("node at name %q has token %q but not in m.byToken", name, n.token) - } - } - - // A node n is in m.byExpiry if it has a non-negative byExpiryIndex. - if n.byExpiryIndex >= 0 { - if n.byExpiryIndex >= len(m.byExpiry) { - return fmt.Errorf("node at name %q has byExpiryIndex %d but m.byExpiry has length %d", name, n.byExpiryIndex, len(m.byExpiry)) - } - if n != m.byExpiry[n.byExpiryIndex] { - return fmt.Errorf("node at name %q has byExpiryIndex %d but that indexes a different node", name, n.byExpiryIndex) + for token, lock := range n.locks { + if lock.token != token || lock.node != n || m.byToken[token] != lock { + return fmt.Errorf("lock %q at node %q is inconsistent", token, name) } } } - for token, n := range m.byToken { - // The map keys should be consistent with the node's copy of the key. - if n.token != token { - return fmt.Errorf("node token %q != byToken map key %q", n.token, token) + for token, lock := range m.byToken { + if lock.token != token { + return fmt.Errorf("lock token %q != byToken map key %q", lock.token, token) } - - // Every node in m.byToken is in m.byName. - if _, ok := m.byName[n.details.Root]; !ok { - return fmt.Errorf("node at name %q in m.byToken but not in m.byName", n.details.Root) + if lock.node == nil || lock.node.locks[token] != lock { + return fmt.Errorf("lock %q is missing from its node", token) + } + if m.byName[lock.details.Root] != lock.node { + return fmt.Errorf("lock %q has inconsistent root %q", token, lock.details.Root) } } - for i, n := range m.byExpiry { - // The slice indices should be consistent with the node's copy of the index. - if n.byExpiryIndex != i { - return fmt.Errorf("node byExpiryIndex %d != byExpiry slice index %d", n.byExpiryIndex, i) + for i, lock := range m.byExpiry { + if lock.byExpiryIndex != i { + return fmt.Errorf("lock byExpiryIndex %d != byExpiry slice index %d", lock.byExpiryIndex, i) } - - // Every node in m.byExpiry is in m.byName. - if _, ok := m.byName[n.details.Root]; !ok { - return fmt.Errorf("node at name %q in m.byExpiry but not in m.byName", n.details.Root) + if m.byToken[lock.token] != lock { + return fmt.Errorf("lock %q is missing from byToken", lock.token) } - - // No node in m.byExpiry should be held. - if n.held { - return fmt.Errorf("node at name %q in m.byExpiry is held", n.details.Root) + if lock.held { + return fmt.Errorf("lock at name %q is held", lock.details.Root) } } return nil @@ -733,3 +762,22 @@ func TestParseTimeout(t *testing.T) { } } } + +func TestHasNegatedTokenCondition(t *testing.T) { + tests := []struct { + name string + conditions []Condition + want bool + }{ + {name: "positive token", conditions: []Condition{{Token: "DAV:no-lock"}}}, + {name: "negated token", conditions: []Condition{{Not: true, Token: "DAV:no-lock"}}, want: true}, + {name: "negated etag", conditions: []Condition{{Not: true, ETag: `"etag"`}}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := hasNegatedTokenCondition(tt.conditions); got != tt.want { + t.Fatalf("hasNegatedTokenCondition(%v) = %t, want %t", tt.conditions, got, tt.want) + } + }) + } +} diff --git a/server/webdav/prop.go b/server/webdav/prop.go index 5c888934..8fc0bae4 100644 --- a/server/webdav/prop.go +++ b/server/webdav/prop.go @@ -16,9 +16,11 @@ import ( "time" "github.com/OpenListTeam/OpenList/v4/internal/conf" + "github.com/OpenListTeam/OpenList/v4/internal/db" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/OpenListTeam/OpenList/v4/server/common" + "gorm.io/gorm" ) // Proppatch describes a property update instruction as defined in RFC 4918. @@ -170,79 +172,39 @@ var liveProps = map[xml.Name]struct { // TODO(nigeltao) merge props and allprop? // Props returns the status of the properties named pnames for resource name. -// -// Each Propstat has a unique status and each property name will only be part -// of one Propstat element. -func props(ctx context.Context, ls LockSystem, fi model.Obj, pnames []xml.Name) ([]Propstat, error) { - //f, err := fs.OpenFile(ctx, name, os.O_RDONLY, 0) - //if err != nil { - // return nil, err - //} - //defer f.Close() - //fi, err := f.Stat() - //if err != nil { - // return nil, err - //} +func props(tx *gorm.DB, ctx context.Context, ls LockSystem, name string, fi model.Obj, pnames []xml.Name) ([]Propstat, error) { isDir := fi.IsDir() - - var deadProps map[xml.Name]Property - // ??? what is this for? - //if dph, ok := f.(DeadPropsHolder); ok { - // deadProps, err = dph.DeadProps() - // if err != nil { - // return nil, err - // } - //} - + deadProps, err := getDeadProps(tx, name) + if err != nil { + return nil, err + } pstatOK := Propstat{Status: http.StatusOK} pstatNotFound := Propstat{Status: http.StatusNotFound} for _, pn := range pnames { - // If this file has dead properties, check if they contain pn. if dp, ok := deadProps[pn]; ok { pstatOK.Props = append(pstatOK.Props, dp) continue } - // Otherwise, it must either be a live property or we don't know it. if prop := liveProps[pn]; prop.findFn != nil && (prop.dir || !isDir) { innerXML, err := prop.findFn(ctx, ls, fi.GetName(), fi) if err != nil { return nil, err } - pstatOK.Props = append(pstatOK.Props, Property{ - XMLName: pn, - InnerXML: []byte(innerXML), - }) + pstatOK.Props = append(pstatOK.Props, Property{XMLName: pn, InnerXML: []byte(innerXML)}) } else { - pstatNotFound.Props = append(pstatNotFound.Props, Property{ - XMLName: pn, - }) + pstatNotFound.Props = append(pstatNotFound.Props, Property{XMLName: pn}) } } return makePropstats(pstatOK, pstatNotFound), nil } // Propnames returns the property names defined for resource name. -func propnames(ctx context.Context, ls LockSystem, fi model.Obj) ([]xml.Name, error) { - //f, err := fs.OpenFile(ctx, name, os.O_RDONLY, 0) - //if err != nil { - // return nil, err - //} - //defer f.Close() - //fi, err := f.Stat() - //if err != nil { - // return nil, err - //} +func propnames(tx *gorm.DB, name string, fi model.Obj) ([]xml.Name, error) { isDir := fi.IsDir() - - var deadProps map[xml.Name]Property - // ??? what is this for? - //if dph, ok := f.(DeadPropsHolder); ok { - // deadProps, err = dph.DeadProps() - // if err != nil { - // return nil, err - // } - //} - + deadProps, err := getDeadProps(tx, name) + if err != nil { + return nil, err + } pnames := make([]xml.Name, 0, len(liveProps)+len(deadProps)) for pn, prop := range liveProps { if prop.findFn != nil && (prop.dir || !isDir) { @@ -255,20 +217,12 @@ func propnames(ctx context.Context, ls LockSystem, fi model.Obj) ([]xml.Name, er return pnames, nil } -// Allprop returns the properties defined for resource name and the properties -// named in include. -// -// Note that RFC 4918 defines 'allprop' to return the DAV: properties defined -// within the RFC plus dead properties. Other live properties should only be -// returned if they are named in 'include'. -// -// See http://www.webdav.org/specs/rfc4918.html#METHOD_PROPFIND -func allprop(ctx context.Context, ls LockSystem, fi model.Obj, include []xml.Name) ([]Propstat, error) { - pnames, err := propnames(ctx, ls, fi) +// Allprop returns the properties defined for resource name and the properties named in include. +func allprop(tx *gorm.DB, ctx context.Context, ls LockSystem, name string, fi model.Obj, include []xml.Name) ([]Propstat, error) { + pnames, err := propnames(tx, name, fi) if err != nil { return nil, err } - // Add names from include if they are not already covered in pnames. nameset := make(map[xml.Name]bool) for _, pn := range pnames { nameset[pn] = true @@ -278,11 +232,10 @@ func allprop(ctx context.Context, ls LockSystem, fi model.Obj, include []xml.Nam pnames = append(pnames, pn) } } - return props(ctx, ls, fi, pnames) + return props(tx, ctx, ls, name, fi, pnames) } -// Patch patches the properties of resource name. The return values are -// constrained in the same manner as DeadPropsHolder.Patch. +// Patch patches the properties of resource name. func patch(ctx context.Context, ls LockSystem, name string, patches []Proppatch) ([]Propstat, error) { conflict := false loop: @@ -314,53 +267,93 @@ loop: return makePropstats(pstatForbidden, pstatFailedDep), nil } - // ------------------------------------------------------------ - //f, err := fs.OpenFile(ctx, name, os.O_RDWR, 0) - //if err != nil { - // return nil, err - //} - //defer f.Close() - //if dph, ok := f.(DeadPropsHolder); ok { - // ret, err := dph.Patch(patches) - // if err != nil { - // return nil, err - // } - // // http://www.webdav.org/specs/rfc4918.html#ELEMENT_propstat says that - // // "The contents of the prop XML element must only list the names of - // // properties to which the result in the status element applies." - // for _, pstat := range ret { - // for i, p := range pstat.Props { - // pstat.Props[i] = Property{XMLName: p.XMLName} - // } - // } - // return ret, nil - //} - // ------------------------------------------------------------ - - // The file doesn't implement the optional DeadPropsHolder interface, so - // all patches are forbidden. - pstat := Propstat{Status: http.StatusForbidden} - for _, patch := range patches { - for _, p := range patch.Props { - pstat.Props = append(pstat.Props, Property{XMLName: p.XMLName}) + pstat := Propstat{Status: http.StatusOK} + var err error + for attempt := 0; attempt < 10; attempt++ { + pstat.Props = nil + err = withDeadPropsTransaction(ctx, func(tx *gorm.DB) error { + for _, patch := range patches { + for _, prop := range patch.Props { + if patch.Remove { + if err := tx.Where("path = ? AND namespace = ? AND name = ?", name, prop.XMLName.Space, prop.XMLName.Local).Delete(&model.WebDAVProperty{}).Error; err != nil { + return err + } + } else { + row := model.WebDAVProperty{ + Path: name, + Namespace: prop.XMLName.Space, + Name: prop.XMLName.Local, + } + if err := tx.Where("path = ? AND namespace = ? AND name = ?", row.Path, row.Namespace, row.Name). + Assign(model.WebDAVProperty{Lang: prop.Lang, InnerXML: prop.InnerXML}). + FirstOrCreate(&row).Error; err != nil { + return err + } + } + pstat.Props = append(pstat.Props, Property{XMLName: prop.XMLName}) + } + } + return nil + }) + if err == nil || !strings.Contains(err.Error(), "database is locked") { + break } + time.Sleep(time.Duration(attempt+1) * 20 * time.Millisecond) + } + if err != nil { + return nil, err } return []Propstat{pstat}, nil } +func getDeadProps(tx *gorm.DB, path string) (map[xml.Name]Property, error) { + var rows []model.WebDAVProperty + if tx == nil { + return nil, errors.New("webdav property transaction is not initialized") + } + if err := tx.Where("path = ?", path).Find(&rows).Error; err != nil { + return nil, err + } + props := make(map[xml.Name]Property, len(rows)) + for _, row := range rows { + props[xml.Name{Space: row.Namespace, Local: row.Name}] = Property{ + XMLName: xml.Name{Space: row.Namespace, Local: row.Name}, + Lang: row.Lang, + InnerXML: row.InnerXML, + } + } + return props, nil +} + +func withDeadPropsTransaction(ctx context.Context, fn func(*gorm.DB) error) error { + database := db.GetDb() + if database == nil { + return errors.New("webdav property database is not initialized") + } + var transaction *gorm.DB + err := database.WithContext(ctx).Transaction(func(tx *gorm.DB) error { + transaction = tx + var probe int + if err := tx.Raw("SELECT 1").Scan(&probe).Error; err != nil { + return err + } + return fn(tx) + }) + if err != nil && transaction != nil { + _ = transaction.Rollback().Error + } + return err +} + func escapeXML(s string) string { for i := 0; i < len(s); i++ { - // As an optimization, if s contains only ASCII letters, digits or a - // few special characters, the escaped value is s itself and we don't - // need to allocate a buffer and convert between string and []byte. switch c := s[i]; { case c == ' ' || c == '_' || - ('+' <= c && c <= '9') || // Digits as well as + , - . and / + ('+' <= c && c <= '9') || ('A' <= c && c <= 'Z') || ('a' <= c && c <= 'z'): continue } - // Otherwise, go through the full escaping process. var buf bytes.Buffer xml.EscapeText(&buf, []byte(s)) return buf.String() @@ -368,6 +361,72 @@ func escapeXML(s string) string { return s } +func deadPropsLikePath(path string) string { + return strings.ReplaceAll(strings.ReplaceAll(strings.ReplaceAll(path, "\\", "\\\\"), "%", "\\%"), "_", "\\_") +} + +func deadPropsSubtree(tx *gorm.DB, path string) *gorm.DB { + return tx.Where("path = ? OR path LIKE ? ESCAPE '\\'", path, deadPropsLikePath(path)+"/%") +} + +func deleteDeadProps(tx *gorm.DB, path string) error { + return deadPropsSubtree(tx, path).Delete(&model.WebDAVProperty{}).Error +} + +func removeResourceWithDeadProps(ctx context.Context, path string, remove func() error) (removeErr, transactionErr error) { + transactionErr = withDeadPropsTransaction(ctx, func(tx *gorm.DB) error { + if err := deleteDeadProps(tx, path); err != nil { + return err + } + removeErr = remove() + return removeErr + }) + return removeErr, transactionErr +} + +func copyDeadProps(tx *gorm.DB, src, dst string) error { + var rows []model.WebDAVProperty + if err := deadPropsSubtree(tx, src).Find(&rows).Error; err != nil { + return err + } + if err := deadPropsSubtree(tx, dst).Delete(&model.WebDAVProperty{}).Error; err != nil { + return err + } + for _, row := range rows { + copy := row + copy.ID = 0 + copy.Path = dst + strings.TrimPrefix(row.Path, src) + if err := tx.Create(©).Error; err != nil { + return err + } + } + return nil +} + +func moveDeadProps(tx *gorm.DB, src, dst string) error { + var rows []model.WebDAVProperty + if err := deadPropsSubtree(tx, src).Find(&rows).Error; err != nil { + return err + } + // Clear destination only after the source rows are staged in memory. + if err := deadPropsSubtree(tx, dst).Delete(&model.WebDAVProperty{}).Error; err != nil { + return err + } + for _, row := range rows { + newPath := dst + strings.TrimPrefix(row.Path, src) + copy := row + copy.ID = 0 + copy.Path = newPath + if err := tx.Create(©).Error; err != nil { + return err + } + if err := tx.Delete(&row).Error; err != nil { + return err + } + } + return nil +} + func findResourceType(ctx context.Context, ls LockSystem, name string, fi model.Obj) (string, error) { if fi.IsDir() { return ``, nil diff --git a/server/webdav/webdav.go b/server/webdav/webdav.go index 5729543e..323476b4 100644 --- a/server/webdav/webdav.go +++ b/server/webdav/webdav.go @@ -29,6 +29,7 @@ import ( "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/OpenListTeam/OpenList/v4/server/common" + "gorm.io/gorm" ) type Handler struct { @@ -82,10 +83,6 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { status, err = h.handleUnlock(brw, r) case "PROPFIND": status, err = h.handlePropfind(brw, r) - // if there is a error for PROPFIND, we should be as an empty folder to the client - if err != nil { - status = http.StatusNotFound - } case "PROPPATCH": status, err = h.handleProppatch(brw, r) } @@ -119,14 +116,64 @@ func (h *Handler) lock(now time.Time, root string) (token string, status int, er return token, 0, nil } +func ifETagConditionsMatch(ctx context.Context, ls LockSystem, name string, conditions []Condition) (bool, error) { + var obj model.Obj + objLoaded, objMissing := false, false + for _, condition := range conditions { + if condition.ETag == "" { + continue + } + if !objLoaded { + var err error + obj, err = fs.Get(ctx, name, &fs.GetArgs{}) + objLoaded = true + if err != nil { + if !errs.IsObjectNotFound(err) { + return false, err + } + objMissing = true + } + } + matches := false + if !objMissing { + etag, err := findETag(ctx, ls, name, obj) + if err != nil { + return false, err + } + matches = etag == condition.ETag + } + if condition.Not { + matches = !matches + } + if !matches { + return false, nil + } + } + return true, nil +} + +func positiveTokenConditions(conditions []Condition) []Condition { + ret := make([]Condition, 0, len(conditions)) + for _, condition := range conditions { + if condition.Token != "" && !condition.Not { + ret = append(ret, condition) + } + } + return ret +} + +func hasNegatedTokenCondition(conditions []Condition) bool { + for _, condition := range conditions { + if condition.Token != "" && condition.Not { + return true + } + } + return false +} + func (h *Handler) confirmLocks(r *http.Request, src, dst string) (release func(), status int, err error) { hdr := r.Header.Get("If") if hdr == "" { - // An empty If header means that the client hasn't previously created locks. - // Even if this client doesn't care about locks, we still need to check that - // the resources aren't locked by another client, so we create temporary - // locks that would conflict with another client's locks. These temporary - // locks are unlocked at the end of the HTTP request. now, srcToken, dstToken := time.Now(), "", "" if src != "" { srcToken, status, err = h.lock(now, src) @@ -143,7 +190,6 @@ func (h *Handler) confirmLocks(r *http.Request, src, dst string) (release func() return nil, status, err } } - return func() { if dstToken != "" { h.LockSystem.Unlock(now, dstToken) @@ -158,7 +204,9 @@ func (h *Handler) confirmLocks(r *http.Request, src, dst string) (release func() if !ok { return nil, http.StatusBadRequest, errInvalidIfHeader } - // ih is a disjunction (OR) of ifLists, so any ifList will do. + ctx := r.Context() + user, _ := ctx.Value(conf.UserKey).(*model.User) + etagMismatch, hasNegatedToken := false, false for _, l := range ih.lists { lsrc := l.resourceTag if lsrc == "" { @@ -175,8 +223,23 @@ func (h *Handler) confirmLocks(r *http.Request, src, dst string) (release func() if err != nil { return nil, status, err } + if user != nil { + lsrc, err = user.JoinPath(lsrc) + if err != nil { + return nil, http.StatusForbidden, err + } + } } - release, err = h.LockSystem.Confirm(time.Now(), lsrc, dst, l.conditions...) + matches, err := ifETagConditionsMatch(ctx, h.LockSystem, lsrc, l.conditions) + if err != nil { + return nil, http.StatusInternalServerError, err + } + if !matches { + etagMismatch = true + continue + } + hasNegatedToken = hasNegatedToken || hasNegatedTokenCondition(l.conditions) + release, err = h.LockSystem.Confirm(time.Now(), lsrc, dst, positiveTokenConditions(l.conditions)...) if err == ErrConfirmationFailed { continue } @@ -185,10 +248,28 @@ func (h *Handler) confirmLocks(r *http.Request, src, dst string) (release func() } return release, 0, nil } - // Section 10.4.1 says that "If this header is evaluated and all state lists - // fail, then the request must fail with a 412 (Precondition Failed) status." - // We follow the spec even though the cond_put_corrupt_token test case from - // the litmus test warns on seeing a 412 instead of a 423 (Locked). + if etagMismatch || !hasNegatedToken { + return nil, http.StatusPreconditionFailed, ErrLocked + } + + // Litmus expects a corrupt token paired with Not on an + // actively locked resource to report the lock conflict. + now := time.Now() + for _, name := range []string{src, dst} { + if name == "" { + continue + } + token, lockStatus, lockErr := h.lock(now, name) + if lockErr == ErrLocked { + return nil, lockStatus, lockErr + } + if lockErr != nil { + return nil, lockStatus, lockErr + } + if unlockErr := h.LockSystem.Unlock(now, token); unlockErr != nil { + return nil, http.StatusInternalServerError, unlockErr + } + } return nil, http.StatusPreconditionFailed, ErrLocked } @@ -212,9 +293,7 @@ func (h *Handler) handleOptions(w http.ResponseWriter, r *http.Request) (status } } w.Header().Set("Allow", allow) - // http://www.webdav.org/specs/rfc4918.html#dav.compliance.classes w.Header().Set("DAV", "1, 2") - // http://msdn.microsoft.com/en-au/library/cc250217.aspx w.Header().Set("MS-Author-Via", "DAV") return 0, nil } @@ -295,12 +374,6 @@ func (h *Handler) handleDelete(w http.ResponseWriter, r *http.Request) (status i if err != nil { return status, err } - release, status, err := h.confirmLocks(r, reqPath, "") - if err != nil { - return status, err - } - defer release() - ctx := r.Context() user := ctx.Value(conf.UserKey).(*model.User) if !user.CanRemove() { @@ -310,18 +383,29 @@ func (h *Handler) handleDelete(w http.ResponseWriter, r *http.Request) (status i if err != nil { return http.StatusForbidden, err } + release, status, err := h.confirmLocks(r, reqPath, "") + if err != nil { + return status, err + } + defer release() // TODO: return MultiStatus where appropriate. // "godoc os RemoveAll" says that "If the path does not exist, RemoveAll // returns nil (no error)." WebDAV semantics are that it should return a // "404 Not Found". We therefore have to Stat before we RemoveAll. + parentPath := path.Dir(reqPath) + if _, err := fs.List(ctx, parentPath, &fs.ListArgs{Refresh: true, NoLog: true}); err != nil { + if errs.IsNotFoundError(err) { + return http.StatusNotFound, err + } + return http.StatusMethodNotAllowed, err + } if _, err := fs.Get(ctx, reqPath, &fs.GetArgs{}); err != nil { - if errs.IsObjectNotFound(err) { + if errs.IsNotFoundError(err) { return http.StatusNotFound, err } return http.StatusMethodNotAllowed, err } - parentPath := path.Dir(reqPath) parentMeta, err := op.GetNearestMeta(parentPath) if err != nil && !errors.Is(errors.Cause(err), errs.MetaNotFound) { return http.StatusInternalServerError, err @@ -329,8 +413,14 @@ func (h *Handler) handleDelete(w http.ResponseWriter, r *http.Request) (status i if !common.CanWrite(user, parentMeta, parentPath) { return http.StatusForbidden, errs.PermissionDenied } - if err := fs.Remove(ctx, reqPath); err != nil { - return http.StatusMethodNotAllowed, err + removeErr, transactionErr := removeResourceWithDeadProps(ctx, reqPath, func() error { + return fs.Remove(ctx, reqPath) + }) + if removeErr != nil { + return http.StatusMethodNotAllowed, removeErr + } + if transactionErr != nil { + return http.StatusInternalServerError, transactionErr } //fs.ClearCache(path.Dir(reqPath)) return http.StatusNoContent, nil @@ -350,17 +440,19 @@ func (h *Handler) handlePut(w http.ResponseWriter, r *http.Request) (status int, if reqPath == "" { return http.StatusMethodNotAllowed, nil } - release, status, err := h.confirmLocks(r, reqPath, "") - if err != nil { - return status, err - } - defer release() + // TODO(rost): Support the If-Match, If-None-Match headers? See bradfitz' + // comments in http.checkEtag. ctx := r.Context() user := ctx.Value(conf.UserKey).(*model.User) reqPath, err = user.JoinPath(reqPath) if err != nil { return http.StatusForbidden, err } + release, status, err := h.confirmLocks(r, reqPath, "") + if err != nil { + return status, err + } + defer release() size := r.ContentLength if size < 0 { sizeStr := r.Header.Get("X-File-Size") @@ -382,6 +474,16 @@ func (h *Handler) handlePut(w http.ResponseWriter, r *http.Request) (status int, return http.StatusForbidden, errs.IgnoredSystemFile } parentPath := path.Dir(reqPath) + parent, err := fs.Get(ctx, parentPath, &fs.GetArgs{}) + if err != nil { + if errs.IsNotFoundError(err) { + return http.StatusConflict, err + } + return http.StatusInternalServerError, err + } + if !parent.IsDir() { + return http.StatusConflict, errs.NotFolder + } parentMeta, err := op.GetNearestMeta(parentPath) if err != nil && !errors.Is(errors.Cause(err), errs.MetaNotFound) { return http.StatusInternalServerError, err @@ -455,18 +557,17 @@ func (h *Handler) handleMkcol(w http.ResponseWriter, r *http.Request) (status in if err != nil { return status, err } - release, status, err := h.confirmLocks(r, reqPath, "") - if err != nil { - return status, err - } - defer release() - ctx := r.Context() user := ctx.Value(conf.UserKey).(*model.User) reqPath, err = user.JoinPath(reqPath) if err != nil { return http.StatusForbidden, err } + release, status, err := h.confirmLocks(r, reqPath, "") + if err != nil { + return status, err + } + defer release() if r.ContentLength > 0 { return http.StatusUnsupportedMediaType, nil @@ -476,15 +577,21 @@ func (h *Handler) handleMkcol(w http.ResponseWriter, r *http.Request) (status in //405 (Method Not Allowed) - MKCOL can only be executed on an unmapped URL if _, err := fs.Get(ctx, reqPath, &fs.GetArgs{}); err == nil { return http.StatusMethodNotAllowed, err + } else if !errs.IsNotFoundError(err) { + return http.StatusInternalServerError, err } // RFC 4918 9.3.1 // 409 (Conflict) The server MUST NOT create those intermediate collections automatically. parentPath := path.Dir(reqPath) - if _, err := fs.Get(ctx, parentPath, &fs.GetArgs{}); err != nil { - if errs.IsObjectNotFound(err) { + parent, err := fs.Get(ctx, parentPath, &fs.GetArgs{}) + if err != nil { + if errs.IsNotFoundError(err) { return http.StatusConflict, err } - return http.StatusMethodNotAllowed, err + return http.StatusInternalServerError, err + } + if !parent.IsDir() { + return http.StatusConflict, errs.NotFolder } parentMeta, err := op.GetNearestMeta(parentPath) if err != nil && !errors.Is(errors.Cause(err), errs.MetaNotFound) { @@ -569,7 +676,7 @@ func (h *Handler) handleCopyMove(w http.ResponseWriter, r *http.Request) (status return http.StatusBadRequest, errInvalidDepth } } - return copyFiles(ctx, src, dst, r.Header.Get("Overwrite") != "F") + return copyFiles(ctx, src, dst, r.Header.Get("Overwrite") != "F", depth) } release, status, err := h.confirmLocks(r, src, dst) @@ -649,11 +756,18 @@ func (h *Handler) handleLock(w http.ResponseWriter, r *http.Request) (retStatus if !common.CanWrite(user, meta, reqPath) { return http.StatusForbidden, errs.PermissionDenied } + if _, err := fs.Get(ctx, reqPath, &fs.GetArgs{}); err != nil { + if !errs.IsObjectNotFound(err) { + return http.StatusInternalServerError, err + } + created = true + } ld = LockDetails{ Root: reqPath, Duration: duration, OwnerXML: li.Owner.InnerXML, ZeroDepth: depth == 0, + Shared: li.Shared != nil, } token, err = h.LockSystem.Create(now, ld) if err != nil { @@ -685,6 +799,10 @@ func (h *Handler) handleLock(w http.ResponseWriter, r *http.Request) (retStatus w.Header().Set("Lock-Token", "<"+token+">") } + // ld.Root is the storage path; the response's lockroot is the client's URL. + obj, err := fs.Get(ctx, ld.Root, &fs.GetArgs{}) + ld.Root = lockRootHref(h.Prefix, user.BasePath, ld.Root, err == nil && obj.IsDir()) + w.Header().Set("Content-Type", "application/xml; charset=utf-8") if created { // This is "w.WriteHeader(http.StatusCreated)" and not "return @@ -696,6 +814,16 @@ func (h *Handler) handleLock(w http.ResponseWriter, r *http.Request) (retStatus return 0, nil } +// lockRootHref maps the storage path a lock is kept under back to the URL path +// the client sees, like the hrefs PROPFIND writes. +func lockRootHref(prefix, basePath, root string, isDir bool) string { + href := path.Join(prefix, strings.TrimPrefix(root, basePath)) + if href != "/" && isDir { + href += "/" + } + return (&url.URL{Path: href}).EscapedPath() +} + func (h *Handler) handleUnlock(w http.ResponseWriter, r *http.Request) (status int, err error) { // http://www.webdav.org/specs/rfc4918.html#HEADER_Lock-Token says that the // Lock-Token value is a Coded-URL. We strip its angle brackets. @@ -779,43 +907,44 @@ func (h *Handler) handlePropfind(w http.ResponseWriter, r *http.Request) (status mw := multistatusWriter{w: w} - walkFn := func(reqPath string, info model.Obj, err error) error { - if err != nil { - return err - } - var pstats []Propstat - if pf.Propname != nil { - pnames, err := propnames(ctx, h.LockSystem, info) - if err != nil { - return err + err = withDeadPropsTransaction(ctx, func(tx *gorm.DB) error { + walkFn := func(currentPath string, info model.Obj, walkErr error) error { + if walkErr != nil { + return walkErr } - pstat := Propstat{Status: http.StatusOK} - for _, xmlname := range pnames { - pstat.Props = append(pstat.Props, Property{XMLName: xmlname}) + var pstats []Propstat + if pf.Propname != nil { + pnames, err := propnames(tx, currentPath, info) + if err != nil { + return err + } + pstat := Propstat{Status: http.StatusOK} + for _, xmlname := range pnames { + pstat.Props = append(pstat.Props, Property{XMLName: xmlname}) + } + pstats = append(pstats, pstat) + } else if pf.Allprop != nil { + pstats, walkErr = allprop(tx, ctx, h.LockSystem, currentPath, info, pf.Prop) + } else { + pstats, walkErr = props(tx, ctx, h.LockSystem, currentPath, info, pf.Prop) } - pstats = append(pstats, pstat) - } else if pf.Allprop != nil { - pstats, err = allprop(ctx, h.LockSystem, info, pf.Prop) - } else { - pstats, err = props(ctx, h.LockSystem, info, pf.Prop) + if walkErr != nil { + return walkErr + } + href := path.Join(h.Prefix, strings.TrimPrefix(currentPath, user.BasePath)) + if href != "/" && info.IsDir() { + href += "/" + } + return mw.write(makePropstatResponse(href, pstats)) } - if err != nil { - return err - } - href := path.Join(h.Prefix, strings.TrimPrefix(reqPath, user.BasePath)) - if href != "/" && info.IsDir() { - href += "/" - } - return mw.write(makePropstatResponse(href, pstats)) - } - walkErr := walkFS(ctx, depth, reqPath, fi, walkFn) - closeErr := mw.close() - if walkErr != nil { - return http.StatusInternalServerError, walkErr - } - if closeErr != nil { - return http.StatusInternalServerError, closeErr + if walkErr := walkFS(ctx, depth, reqPath, fi, walkFn); walkErr != nil { + return walkErr + } + return mw.close() + }) + if err != nil { + return http.StatusInternalServerError, err } return 0, nil } @@ -825,18 +954,17 @@ func (h *Handler) handleProppatch(w http.ResponseWriter, r *http.Request) (statu if err != nil { return status, err } - release, status, err := h.confirmLocks(r, reqPath, "") - if err != nil { - return status, err - } - defer release() - ctx := r.Context() user := ctx.Value(conf.UserKey).(*model.User) reqPath, err = user.JoinPath(reqPath) if err != nil { return http.StatusForbidden, err } + release, status, err := h.confirmLocks(r, reqPath, "") + if err != nil { + return status, err + } + defer release() meta, err := op.GetNearestMeta(reqPath) if err != nil && !errors.Is(errors.Cause(err), errs.MetaNotFound) { return http.StatusInternalServerError, err diff --git a/server/webdav/webdav_test.go b/server/webdav/webdav_test.go new file mode 100644 index 00000000..337f84eb --- /dev/null +++ b/server/webdav/webdav_test.go @@ -0,0 +1,24 @@ +package webdav + +import "testing" + +func TestLockRootHref(t *testing.T) { + testCases := []struct { + prefix, basePath, root string + isDir bool + want string + }{ + {"/dav", "/storage", "/storage/lock-probe", true, "/dav/lock-probe/"}, + {"/dav", "/storage", "/storage/a.txt", false, "/dav/a.txt"}, + {"/dav", "/", "/storage/a.txt", false, "/dav/storage/a.txt"}, + {"/dav", "/storage", "/storage", true, "/dav/"}, + {"", "/", "/", true, "/"}, + {"/dav", "/", "/new folder/文件.txt", false, "/dav/new%20folder/%E6%96%87%E4%BB%B6.txt"}, + } + for _, tc := range testCases { + got := lockRootHref(tc.prefix, tc.basePath, tc.root, tc.isDir) + if got != tc.want { + t.Errorf("lockRootHref(%q, %q, %q, %v) = %q, want %q", tc.prefix, tc.basePath, tc.root, tc.isDir, got, tc.want) + } + } +} diff --git a/server/webdav/xml.go b/server/webdav/xml.go index c9ec61df..d87eb863 100644 --- a/server/webdav/xml.go +++ b/server/webdav/xml.go @@ -62,9 +62,7 @@ func readLockInfo(r io.Reader) (li lockInfo, status int, err error) { } return lockInfo{}, http.StatusBadRequest, err } - // We only support exclusive (non-shared) write locks. In practice, these are - // the only types of locks that seem to matter. - if li.Exclusive == nil || li.Shared != nil || li.Write == nil { + if (li.Exclusive == nil) == (li.Shared == nil) || li.Write == nil { return lockInfo{}, http.StatusNotImplemented, errUnsupportedLockInfo } return li, 0, nil @@ -86,18 +84,22 @@ func writeLockInfo(w io.Writer, token string, ld LockDetails) (int, error) { if ld.ZeroDepth { depth = "0" } + scope := "exclusive" + if ld.Shared { + scope = "shared" + } timeout := ld.Duration / time.Second return fmt.Fprintf(w, "\n"+ "\n"+ - " \n"+ - " \n"+ - " %s\n"+ - " %s\n"+ - " Second-%d\n"+ - " %s\n"+ - " %s\n"+ + "\t\n"+ + "\t\n"+ + "\t%s\n"+ + "\t%s\n"+ + "\tSecond-%d\n"+ + "\t%s\n"+ + "\t%s\n"+ "", - depth, ld.OwnerXML, timeout, escape(token), escape(ld.Root), + scope, depth, ld.OwnerXML, timeout, escape(token), escape(ld.Root), ) } @@ -176,19 +178,31 @@ type propfind struct { } func readPropfind(r io.Reader) (pf propfind, status int, err error) { - c := countingReader{r: r} - if err = ixml.NewDecoder(&c).Decode(&pf); err != nil { + // 64KB is more than enough for a well-formed PROPFIND body. + const maxBody = 64 << 10 + body, err := io.ReadAll(io.LimitReader(r, maxBody)) + if err != nil { + return propfind{}, http.StatusBadRequest, err + } + // If the limit was reached, the body was too large. + if len(body) >= maxBody { + // Drain any remaining bytes so the connection stays usable. + _, _ = io.Copy(io.Discard, r) + return propfind{}, http.StatusRequestEntityTooLarge, nil + } + if len(body) == 0 { + // An empty body means to propfind allprop. + return propfind{Allprop: new(struct{})}, 0, nil + } + if hasEmptyNamespacePrefix(body) { + return propfind{}, http.StatusBadRequest, errInvalidPropfind + } + if err = ixml.NewDecoder(bytes.NewReader(body)).Decode(&pf); err != nil { if err == io.EOF { - if c.n == 0 { - // An empty body means to propfind allprop. - // http://www.webdav.org/specs/rfc4918.html#METHOD_PROPFIND - return propfind{Allprop: new(struct{})}, 0, nil - } err = errInvalidPropfind } return propfind{}, http.StatusBadRequest, err } - if pf.Allprop == nil && pf.Include != nil { return propfind{}, http.StatusBadRequest, errInvalidPropfind } @@ -204,6 +218,33 @@ func readPropfind(r io.Reader) (pf propfind, status int, err error) { return pf, 0, nil } +func hasEmptyNamespacePrefix(body []byte) bool { + for offset := 0; ; { + i := bytes.Index(body[offset:], []byte("xmlns:")) + if i < 0 { + return false + } + i += offset + len("xmlns:") + j := i + for j < len(body) && body[j] != '=' && body[j] != '>' && body[j] != '/' && body[j] != ' ' && body[j] != '\t' && body[j] != '\n' && body[j] != '\r' { + j++ + } + for j < len(body) && (body[j] == ' ' || body[j] == '\t' || body[j] == '\n' || body[j] == '\r') { + j++ + } + if j < len(body) && body[j] == '=' { + j++ + for j < len(body) && (body[j] == ' ' || body[j] == '\t' || body[j] == '\n' || body[j] == '\r') { + j++ + } + if j+1 < len(body) && (body[j] == '\'' || body[j] == '"') && body[j+1] == body[j] { + return true + } + } + offset = i + } +} + // Property represents a single DAV resource property as defined in RFC 4918. // See http://www.webdav.org/specs/rfc4918.html#data.model.for.resource.properties type Property struct {