diff --git a/.github/ISSUE_TEMPLATE/00-bug_report_zh.yml b/.github/ISSUE_TEMPLATE/00-bug_report_zh.yml index ec18a650..42c61d51 100644 --- a/.github/ISSUE_TEMPLATE/00-bug_report_zh.yml +++ b/.github/ISSUE_TEMPLATE/00-bug_report_zh.yml @@ -82,9 +82,17 @@ body: label: 复现链接（可选） description: | 请提供能复现此问题的链接。 - - type: textarea + - type: checkboxes id: aigenerated attributes: - label: AI生成内容（可选） - description: | - 如果此问题是由AI辅助您发现的，请提供全部聊天记录，包括使用的模型信息。 + label: AI生成内容 + description: 必须且只能勾选一项，请勿删除或修改声明文字。 + options: + - label: 我使用了AI工具生成此内容 + - label: 我没有使用AI工具生成此内容 + - type: input + id: ai-model + attributes: + label: AI模型是 + description: 如果使用了AI工具，请填写模型名称；未使用则留空。 + placeholder: xxx diff --git a/.github/ISSUE_TEMPLATE/01-bug_report_en.yml b/.github/ISSUE_TEMPLATE/01-bug_report_en.yml index 0085d4de..3e900191 100644 --- a/.github/ISSUE_TEMPLATE/01-bug_report_en.yml +++ b/.github/ISSUE_TEMPLATE/01-bug_report_en.yml @@ -82,9 +82,17 @@ body: label: Reproduction Link (optional) description: | Please provide a link to a repo or page that can reproduce this issue. - - type: textarea + - type: checkboxes id: aigenerated attributes: - label: AI Generated Content (optional) - description: | - If this issue was identified with the assistance of AI, please provide the complete chat log, including information about the model used. + label: AI Generated Content + description: Select exactly one option. Do not delete or modify the disclosure text. + options: + - label: I used AI tools to generate this content + - label: I did not use AI tools to generate this content + - type: input + id: ai-model + attributes: + label: AI model used + description: If you used AI tools, enter the model name; otherwise leave this blank. + placeholder: xxx diff --git a/.github/ISSUE_TEMPLATE/02-feature_request_zh.yml b/.github/ISSUE_TEMPLATE/02-feature_request_zh.yml index 90724026..def688e2 100644 --- a/.github/ISSUE_TEMPLATE/02-feature_request_zh.yml +++ b/.github/ISSUE_TEMPLATE/02-feature_request_zh.yml @@ -48,9 +48,17 @@ body: label: 附加信息 description: | 相关的任何其他上下文或截图，或者你觉得有帮助的信息 - - type: textarea + - type: checkboxes id: aigenerated attributes: - label: AI生成内容（可选） - description: | - 如果此请求是由AI辅助您提交的，请提供全部聊天记录，包括使用的模型信息。 + label: AI生成内容 + description: 必须且只能勾选一项，请勿删除或修改声明文字。 + options: + - label: 我使用了AI工具生成此内容 + - label: 我没有使用AI工具生成此内容 + - type: input + id: ai-model + attributes: + label: AI模型是 + description: 如果使用了AI工具，请填写模型名称；未使用则留空。 + placeholder: xxx diff --git a/.github/ISSUE_TEMPLATE/03-feature_request_en.yml b/.github/ISSUE_TEMPLATE/03-feature_request_en.yml index 39311859..796f0e6c 100644 --- a/.github/ISSUE_TEMPLATE/03-feature_request_en.yml +++ b/.github/ISSUE_TEMPLATE/03-feature_request_en.yml @@ -48,9 +48,17 @@ body: label: Additional Information description: | Any other context or screenshots related to this feature request, or information you find helpful. - - type: textarea + - type: checkboxes id: aigenerated attributes: - label: AI Generated Content (optional) - description: | - If this request was submitted with the assistance of an AI, please provide the complete chat log, including information about the model used. + label: AI Generated Content + description: Select exactly one option. Do not delete or modify the disclosure text. + options: + - label: I used AI tools to generate this content + - label: I did not use AI tools to generate this content + - type: input + id: ai-model + attributes: + label: AI model used + description: If you used AI tools, enter the model name; otherwise leave this blank. + placeholder: xxx diff --git a/.github/workflows/beta_release.yml b/.github/workflows/beta_release.yml index 7827f1da..dc9c5e1b 100644 --- a/.github/workflows/beta_release.yml +++ b/.github/workflows/beta_release.yml @@ -18,13 +18,13 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v6 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 with: fetch-depth: 0 - name: Create or update ref id: create-or-update-ref - uses: ovsds/create-or-update-ref-action@v1 + uses: ovsds/create-or-update-ref-action@1157b2991820188d5a8e6f22656775d1d171f762 # v1 with: ref: tags/beta sha: ${{ github.sha }} @@ -51,7 +51,7 @@ jobs: gh release upload beta "CHANGELOG.md" --clobber - name: Upload assets to github artifact - uses: actions/upload-artifact@v7 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: name: beta changelog path: ${{ github.workspace }}/CHANGELOG.md @@ -108,14 +108,14 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v6 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 with: fetch-depth: 0 - name: Setup Go - uses: actions/setup-go@v6 + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7 with: - go-version: "1.25.0" + go-version: "1.27.1" - name: Setup web run: bash build.sh dev web @@ -124,7 +124,7 @@ jobs: FRONTEND_REPO: ${{ vars.FRONTEND_REPO }} - name: Build - uses: OpenListTeam/cgo-actions@v1.2.5 + uses: OpenListTeam/cgo-actions@d760a8ec1a6be1f8ec181229e11cb2671797f9e0 # v1.3.0 with: targets: ${{ matrix.target }} flags: ${{ matrix.flags || '-ldflags=' }} @@ -136,7 +136,7 @@ jobs: musl-base-url: "https://github.com/OpenListTeam/musl-compilers/releases/latest/download/" x-flags: | github.com/OpenListTeam/OpenList/v4/internal/conf.BuiltAt=$built_at - github.com/OpenListTeam/OpenList/v4/internal/conf.GitAuthor=The OpenList Projects Contributors + github.com/OpenListTeam/OpenList/v4/internal/conf.GitAuthor=The OpenList Projects Contributors github.com/OpenListTeam/OpenList/v4/internal/conf.GitCommit=$git_commit github.com/OpenListTeam/OpenList/v4/internal/conf.Version=$tag github.com/OpenListTeam/OpenList/v4/internal/conf.WebVersion=rolling @@ -185,7 +185,7 @@ jobs: echo "cleaned_target=$CLEANED_TARGET" >> $GITHUB_ENV - name: Upload assets to github artifact - uses: actions/upload-artifact@v7 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: name: beta builds for ${{ env.cleaned_target }} path: ${{ github.workspace }}/build/compress/* diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 58cc3ef6..caa833ce 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -25,15 +25,15 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v6 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - - uses: benjlevesque/short-sha@v3.0 + - uses: benjlevesque/short-sha@dbe07338b37c456ce06d23409b35a56a7815eef4 # v4.0 id: short-sha - name: Setup Go - uses: actions/setup-go@v6 + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7 with: - go-version: "1.25.0" + go-version: "1.27.1" - name: Setup web run: bash build.sh dev web @@ -42,7 +42,7 @@ jobs: FRONTEND_REPO: ${{ vars.FRONTEND_REPO }} - name: Build - uses: OpenListTeam/cgo-actions@v1.2.5 + uses: OpenListTeam/cgo-actions@d760a8ec1a6be1f8ec181229e11cb2671797f9e0 # v1.3.0 with: targets: ${{ matrix.target }} flags: ${{ contains(matrix.target, '-musl') && '-ldflags=-linkmode external -extldflags ''-static -fpic''' || '-ldflags=' }} @@ -52,7 +52,7 @@ jobs: out-dir: build x-flags: | github.com/OpenListTeam/OpenList/v4/internal/conf.BuiltAt=$built_at - github.com/OpenListTeam/OpenList/v4/internal/conf.GitAuthor=The OpenList Projects Contributors + github.com/OpenListTeam/OpenList/v4/internal/conf.GitAuthor=The OpenList Projects Contributors github.com/OpenListTeam/OpenList/v4/internal/conf.GitCommit=$git_commit github.com/OpenListTeam/OpenList/v4/internal/conf.Version=$tag github.com/OpenListTeam/OpenList/v4/internal/conf.WebVersion=rolling @@ -69,7 +69,7 @@ jobs: fi - name: Upload artifact - uses: actions/upload-artifact@v7 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: name: openlist_${{ steps.short-sha.outputs.sha }}_${{ matrix.target }} path: build/* diff --git a/.github/workflows/changelog.yml b/.github/workflows/changelog.yml index e14eaaa7..423bd106 100644 --- a/.github/workflows/changelog.yml +++ b/.github/workflows/changelog.yml @@ -14,7 +14,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout code - uses: actions/checkout@v6 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 with: fetch-depth: 0 diff --git a/.github/workflows/issue_comment.yml b/.github/workflows/issue_comment.yml new file mode 100644 index 00000000..f8b20903 --- /dev/null +++ b/.github/workflows/issue_comment.yml @@ -0,0 +1,75 @@ +name: Issue Auto Reply + +on: + issues: + types: [opened] + +permissions: + issues: write + +jobs: + auto-reply: + runs-on: ubuntu-latest + steps: + - name: Check issue for unchecked tasks and reply + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9 + with: + script: | + if (context.payload.issue.title.startsWith('[Announcements]')) return; + const titleNotEdited = /(请修改标题|Please modify the title)/i.test(context.payload.issue.title); + const issueBody = context.payload.issue.body || ""; + const aiSection = issueBody.match(/^### (AI生成内容|AI Generated Content)\r?\n([\s\S]*?)(?=^### |$(?![\s\S]))/m); + const aiOptionsPattern = aiSection?.[1] === 'AI生成内容' + ? /^\s*- \[([ xX])\] 我使用了AI工具生成此内容\r?\n- \[([ xX])\] 我没有使用AI工具生成此内容\s*$/ + : /^\s*- \[([ xX])\] I used AI tools to generate this content\r?\n- \[([ xX])\] I did not use AI tools to generate this content\s*$/; + const aiOptions = (aiSection?.[2] || '').match(aiOptionsPattern); + const validAiDisclosure = aiOptions !== null && (aiOptions[1] !== ' ') !== (aiOptions[2] !== ' '); + const aiModelSection = issueBody.match(/^### (AI模型是|AI model used)\r?\n([\s\S]*?)(?=^### |$(?![\s\S]))/m); + const aiModel = (aiModelSection?.[2] || '').trim(); + const missingAiModel = validAiDisclosure && aiOptions[1] !== ' ' && (aiModel === '' || aiModel === '_No response_'); + const confirmNotRead = /- \[[xX]\] (?:我没有阅读这个清单|I have not read these checkboxes)/.test(issueBody); + const closeIssue = titleNotEdited || confirmNotRead || !validAiDisclosure || missingAiModel; + let comment; + if (titleNotEdited) { + comment = `⚠️ 请修改标题以更好地描述您的问题或需求，并删除示例提示。当前 Issue 将被自动关闭并锁定。如需继续提交，请创建新的 Issue。 + ⚠️ Please modify the title to better describe your issue or request, and remove the example prompt. This issue will be automatically closed and locked. If you wish to proceed, please create a new issue. + `; + } else if (confirmNotRead || !validAiDisclosure || missingAiModel) { + comment = `⚠️ 你的 Issue 不符合提交规则。请先阅读相关规范后再重新提交。当前 Issue 将被自动关闭并锁定。如需继续提交，请确认已了解规则后创建新的 Issue。 + ⚠️ Your issue does not comply with the submission rules. Please read the guidelines before submitting again. This issue will be automatically closed and locked. If you wish to proceed, please confirm that you have reviewed the rules before creating a new issue. + `; + } else if (/- \[ \] (?!我没有阅读这个清单|I have not read these checkboxes)/.test(issueBody.replace(aiSection[0], ''))) { + comment = `感谢您联系OpenList。我们会尽快回复您。 + Thanks for contacting OpenList. We will reply to you as soon as possible. + + 由于您提出的 Issue 中包含部分未确认的项目，为了更好地管理项目，在人工审核后可能会直接关闭此问题。 + 如果您能确认并补充相关未确认项目的信息，欢迎随时重新提交。我们会及时关注并处理。感谢您的理解与支持！ + Since your issue contains some unchecked tasks, it may be closed after manual review. + If you can confirm and provide information for the unchecked tasks, feel free to resubmit. + We will pay attention and handle it in a timely manner. + + 感谢您的理解与支持！ + Thank you for your understanding and support! + `; + } else { + return; + } + + await github.rest.issues.createComment({ + ...context.repo, + issue_number: context.issue.number, + body: comment + }); + if (closeIssue) { + await github.rest.issues.update({ + ...context.repo, + issue_number: context.issue.number, + state: 'closed', + state_reason: 'not_planned', + labels: ['invalid'] + }); + await github.rest.issues.lock({ + ...context.repo, + issue_number: context.issue.number + }); + } diff --git a/.github/workflows/issue_pr_comment.yml b/.github/workflows/issue_pr_comment.yml deleted file mode 100644 index 1b51e23b..00000000 --- a/.github/workflows/issue_pr_comment.yml +++ /dev/null @@ -1,101 +0,0 @@ -name: Issue or PR Auto Reply - -on: - issues: - types: [opened] - pull_request: - types: [opened] - -permissions: - issues: write - pull-requests: write - -jobs: - auto-reply: - runs-on: ubuntu-latest - if: github.event_name == 'issues' - steps: - - name: Check issue for unchecked tasks and reply - uses: actions/github-script@v7 - with: - script: | - let comment = ""; - const issueTitle = context.payload.issue.title || ""; - const titleNotEdited = /(请修改标题|Please modify the title)/i.test(issueTitle); - if (titleNotEdited) { - comment = "⚠️ 请修改标题以更好地描述您的问题或需求，并删除示例提示。当前 Issue 将被自动关闭。如需继续提交，请创建新的 Issue。\n"; - comment += "⚠️ Please modify the title to better describe your issue or request, and remove the example prompt. This issue will be automatically closed. If you wish to proceed, please create a new issue.\n"; - await github.rest.issues.createComment({ - ...context.repo, - issue_number: context.issue.number, - body: comment - }); - await github.rest.issues.update({ - ...context.repo, - issue_number: context.issue.number, - state: 'closed', - state_reason: 'not_planned', - labels: ['invalid'] - }); - return; - } - const issueBody = context.payload.issue.body || ""; - const confirmHasRead = /- \[ \] (?!我没有阅读这个清单|I have not read these checkboxes)/.test(issueBody); - const confirmNotRead = /- \[[xX]\] (?:我没有阅读这个清单|I have not read these checkboxes)/.test(issueBody); - if (confirmNotRead) { - comment = "⚠️ 你的 Issue 不符合提交规则。请先阅读相关规范后再重新提交。当前 Issue 将被自动关闭。如需继续提交，请确认已了解规则后重新打开或创建新的 Issue。\n"; - comment += "⚠️ Your issue does not comply with the submission rules. Please read the guidelines before submitting again. This issue will be automatically closed. If you wish to proceed, please confirm that you have reviewed the rules before reopening or creating a new issue.\n"; - await github.rest.issues.createComment({ - ...context.repo, - issue_number: context.issue.number, - body: comment - }); - await github.rest.issues.update({ - ...context.repo, - issue_number: context.issue.number, - state: 'closed', - state_reason: 'not_planned', - labels: ['invalid'] - }); - return; - } - if (confirmHasRead) { - comment = "感谢您联系OpenList。我们会尽快回复您。\n"; - comment += "Thanks for contacting OpenList. We will reply to you as soon as possible.\n\n"; - comment += "由于您提出的 Issue 中包含部分未确认的项目，为了更好地管理项目，在人工审核后可能会直接关闭此问题。\n"; - comment += "如果您能确认并补充相关未确认项目的信息，欢迎随时重新提交。我们会及时关注并处理。感谢您的理解与支持！\n"; - comment += "Since your issue contains some unchecked tasks, it may be closed after manual review.\n"; - comment += "If you can confirm and provide information for the unchecked tasks, feel free to resubmit.\n"; - comment += "We will pay attention and handle it in a timely manner.\n\n"; - comment += "感谢您的理解与支持！\n"; - comment += "Thank you for your understanding and support!\n"; - await github.rest.issues.createComment({ - ...context.repo, - issue_number: context.issue.number, - body: comment - }); - } - - pr-title-check: - runs-on: ubuntu-latest - if: github.event_name == 'pull_request' - steps: - - name: Check PR title for required prefix and comment - uses: actions/github-script@v7 - with: - script: | - const title = context.payload.pull_request.title || ""; - const ok = /^(feat|docs|fix|style|refactor|chore)\(.+?\)!?: /i.test(title); - if (!ok) { - let comment = "⚠️ PR 标题需以 `feat(): `, `docs(): `, `fix(): `, `style(): `, `refactor(): `, `chore(): ` 其中之一开头，例如：`feat(component): 新增功能`。\n"; - comment += "⚠️ The PR title must start with `feat(): `, `docs(): `, `fix(): `, `style(): `, or `refactor(): `, `chore(): `. For example: `feat(component): add new feature`.\n\n"; - comment += "如果跨多个组件，请使用主要组件作为前缀，并在标题中枚举、描述中说明。\n"; - comment += "If it spans multiple components, use the main component as the prefix and enumerate in the title, describe in the body.\n\n"; - comment += "如果是破坏性变更，请在类型后添加 `!`，例如 `feat(component)!: 破坏性变更`。\n"; - comment += "For breaking changes, add `!` after the type, e.g., `feat(component)!: breaking change`.\n\n"; - await github.rest.issues.createComment({ - ...context.repo, - issue_number: context.issue.number, - body: comment - }); - } diff --git a/.github/workflows/pr_title_check.yml b/.github/workflows/pr_title_check.yml new file mode 100644 index 00000000..45707af7 --- /dev/null +++ b/.github/workflows/pr_title_check.yml @@ -0,0 +1,34 @@ +name: PR Title Check + +on: + pull_request_target: + types: [opened] + +permissions: + pull-requests: write + +jobs: + pr-title-check: + runs-on: ubuntu-latest + steps: + - name: Check PR title for required prefix and comment + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9 + with: + script: | + const title = context.payload.pull_request.title; + if (/^(feat|docs|fix|style|refactor|chore)\(.+?\)!?: /i.test(title)) return; + const comment = `⚠️ PR 标题需以 \`feat(): \`, \`docs(): \`, \`fix(): \`, \`style(): \`, \`refactor(): \`, \`chore(): \` 其中之一开头，例如：\`feat(component): 新增功能\`。 + ⚠️ The PR title must start with \`feat(): \`, \`docs(): \`, \`fix(): \`, \`style(): \`, or \`refactor(): \`, \`chore(): \`. For example: \`feat(component): add new feature\`. + + 如果跨多个组件，请使用主要组件作为前缀，并在标题中枚举、描述中说明。 + If it spans multiple components, use the main component as the prefix and enumerate in the title, describe in the body. + + 如果是破坏性变更，请在类型后添加 \`!\`，例如 \`feat(component)!: 破坏性变更\`。 + For breaking changes, add \`!\` after the type, e.g., \`feat(component)!: breaking change\`. + + `; + await github.rest.issues.createComment({ + ...context.repo, + issue_number: context.issue.number, + body: comment + }); diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 3eed1faf..99b721bc 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -14,7 +14,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Prerelease - uses: irongut/EditRelease@v1.2.0 + uses: irongut/EditRelease@ccf529ad26dddf9996e7dd0f24ca5da4ea507cc2 # v1.2.0 with: token: ${{ secrets.GITHUB_TOKEN }} id: ${{ github.event.release.id }} @@ -33,7 +33,7 @@ jobs: - name: Free Disk Space (Ubuntu) if: matrix.target-platform == '' - uses: jlumbroso/free-disk-space@main + uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be # main with: tool-cache: false android: true @@ -44,14 +44,15 @@ jobs: swap-storage: true - name: Setup Go - uses: actions/setup-go@v6 + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7 with: - go-version: '1.25.0' + go-version: '1.27.1' - name: Checkout - uses: actions/checkout@v6 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 with: - fetch-depth: 0 + fetch-depth: 1 + fetch-tags: true - name: Install dependencies if: matrix.target-platform == '' diff --git a/.github/workflows/release_docker.yml b/.github/workflows/release_docker.yml index 1badaf45..e8af8ebf 100644 --- a/.github/workflows/release_docker.yml +++ b/.github/workflows/release_docker.yml @@ -43,15 +43,15 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v6 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - - uses: actions/setup-go@v6 + - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7 with: - go-version: '1.25.0' + go-version: '1.27' - name: Cache Musl id: cache-musl - uses: actions/cache@v4 + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6 with: path: build/musl-libs key: docker-musl-libs-v2 @@ -69,7 +69,7 @@ jobs: FRONTEND_REPO: ${{ vars.FRONTEND_REPO }} - name: Upload artifacts - uses: actions/upload-artifact@v7 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: name: ${{ env.ARTIFACT_NAME }} overwrite: true @@ -83,15 +83,15 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v6 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - - uses: actions/setup-go@v6 + - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7 with: - go-version: '1.25.0' + go-version: '1.27.1' - name: Cache Musl id: cache-musl - uses: actions/cache@v4 + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6 with: path: build/musl-libs key: docker-musl-libs-v2 @@ -109,7 +109,7 @@ jobs: FRONTEND_REPO: ${{ vars.FRONTEND_REPO }} - name: Upload artifacts - uses: actions/upload-artifact@v7 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: name: ${{ env.ARTIFACT_NAME_LITE }} overwrite: true @@ -146,21 +146,21 @@ jobs: tag_favor: "suffix=-aio,onlatest=true" steps: - name: Checkout - uses: actions/checkout@v6 - - uses: actions/download-artifact@v4 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 with: name: ${{ env.ARTIFACT_NAME }} path: 'build/' - name: Set up QEMU - uses: docker/setup-qemu-action@v3 + uses: docker/setup-qemu-action@1f40c72289eff860ee54a304f1438e3cff362e0a # v4 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 + uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4 - name: Login to GitHub Container Registry if: env.IMAGE_PUSH == 'true' - uses: docker/login-action@v3 + uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4 with: registry: ${{ env.REGISTRY }} username: ${{ github.actor }} @@ -168,14 +168,14 @@ jobs: - name: Login to DockerHub Container Registry if: env.IMAGE_PUSH == 'true' - uses: docker/login-action@v3 + uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4 with: username: ${{ vars.DOCKERHUB_ORG_NAME_BACKUP || env.DOCKERHUB_ORG_NAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Docker meta id: meta - uses: docker/metadata-action@v5 + uses: docker/metadata-action@dc802804100637a589fabce1cb79ff13a1411302 # v6 with: images: | ${{ env.REGISTRY }}/${{ env.GHCR_ORG_NAME }}/${{ env.IMAGE_NAME }} @@ -190,7 +190,7 @@ jobs: - name: Build and push id: docker_build - uses: docker/build-push-action@v6 + uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7 with: context: . file: Dockerfile.ci @@ -230,21 +230,21 @@ jobs: tag_favor: "suffix=-lite-aio,onlatest=true" steps: - name: Checkout - uses: actions/checkout@v6 - - uses: actions/download-artifact@v4 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 with: name: ${{ env.ARTIFACT_NAME_LITE }} path: 'build/' - name: Set up QEMU - uses: docker/setup-qemu-action@v3 + uses: docker/setup-qemu-action@1f40c72289eff860ee54a304f1438e3cff362e0a # v4 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 + uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4 - name: Login to GitHub Container Registry if: env.IMAGE_PUSH == 'true' - uses: docker/login-action@v3 + uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4 with: registry: ${{ env.REGISTRY }} username: ${{ github.actor }} @@ -252,14 +252,14 @@ jobs: - name: Login to DockerHub Container Registry if: env.IMAGE_PUSH == 'true' - uses: docker/login-action@v3 + uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4 with: username: ${{ vars.DOCKERHUB_ORG_NAME_BACKUP || env.DOCKERHUB_ORG_NAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Docker meta id: meta - uses: docker/metadata-action@v5 + uses: docker/metadata-action@dc802804100637a589fabce1cb79ff13a1411302 # v6 with: images: | ${{ env.REGISTRY }}/${{ env.GHCR_ORG_NAME }}/${{ env.IMAGE_NAME }} @@ -274,7 +274,7 @@ jobs: - name: Build and push id: docker_build - uses: docker/build-push-action@v6 + uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7 with: context: . file: Dockerfile.ci diff --git a/.github/workflows/sync_repo.yml b/.github/workflows/sync_repo.yml index fe57905d..4f5481b0 100644 --- a/.github/workflows/sync_repo.yml +++ b/.github/workflows/sync_repo.yml @@ -12,7 +12,7 @@ jobs: name: Sync GitHub to Gitee steps: - name: Checkout - uses: actions/checkout@v6 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 with: fetch-depth: 0 diff --git a/.github/workflows/test_docker.yml b/.github/workflows/test_docker.yml index 5448adc5..84556617 100644 --- a/.github/workflows/test_docker.yml +++ b/.github/workflows/test_docker.yml @@ -32,15 +32,15 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v6 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - - uses: actions/setup-go@v6 + - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7 with: - go-version: '1.25.0' + go-version: '1.27.1' - name: Cache Musl id: cache-musl - uses: actions/cache@v4 + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6 with: path: build/musl-libs key: docker-musl-libs-v2 @@ -58,7 +58,7 @@ jobs: FRONTEND_REPO: ${{ vars.FRONTEND_REPO }} - name: Upload artifacts - uses: actions/upload-artifact@v7 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: name: ${{ env.ARTIFACT_NAME }} overwrite: true @@ -97,21 +97,21 @@ jobs: tag_favor: "suffix=-aio,onlatest=true" steps: - name: Checkout - uses: actions/checkout@v6 - - uses: actions/download-artifact@v4 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 with: name: ${{ env.ARTIFACT_NAME }} path: 'build/' - name: Set up QEMU - uses: docker/setup-qemu-action@v3 + uses: docker/setup-qemu-action@1f40c72289eff860ee54a304f1438e3cff362e0a # v4 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 + uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4 - name: Login to GitHub Container Registry if: env.IMAGE_PUSH == 'true' - uses: docker/login-action@v3 + uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4 with: registry: ${{ env.REGISTRY }} username: ${{ github.actor }} @@ -119,14 +119,14 @@ jobs: - name: Login to DockerHub Container Registry if: env.IMAGE_PUSH == 'true' - uses: docker/login-action@v3 + uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4 with: username: ${{ vars.DOCKERHUB_ORG_NAME_BACKUP || env.DOCKERHUB_ORG_NAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Docker meta id: meta - uses: docker/metadata-action@v5 + uses: docker/metadata-action@dc802804100637a589fabce1cb79ff13a1411302 # v6 with: images: | ${{ env.REGISTRY }}/${{ env.GHCR_ORG_NAME }}/${{ env.IMAGE_NAME }} @@ -137,7 +137,7 @@ jobs: - name: Build and push id: docker_build - uses: docker/build-push-action@v6 + uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7 with: context: . file: Dockerfile.ci diff --git a/.github/workflows/trigger-makefile-update.yml b/.github/workflows/trigger-makefile-update.yml index 58b53b48..e35343dd 100644 --- a/.github/workflows/trigger-makefile-update.yml +++ b/.github/workflows/trigger-makefile-update.yml @@ -16,20 +16,33 @@ jobs: runs-on: ubuntu-latest steps: - name: Trigger Makefile hash update - uses: peter-evans/repository-dispatch@v3 + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9 with: - token: ${{ secrets.EXTERNAL_REPO_TOKEN_LUCI_APP_OPENLIST }} - repository: ${{ vars.HOOK_REPO || 'OpenListTeam/OpenList-OpenWRT' }} - event-type: update-hashes - client-payload: | - { - "source_repository": "${{ github.repository }}", - "release_tag": "${{ inputs.tag || github.ref_name }}", - "release_name": "${{ inputs.tag || github.ref_name }}", - "release_url": "${{ github.server_url }}/${{ github.repository }}/releases/tag/${{ inputs.tag || github.ref_name }}", - "triggered_by": "${{ github.actor }}", - "trigger_reason": "${{ github.event_name }}" - } + github-token: ${{ secrets.EXTERNAL_REPO_TOKEN_LUCI_APP_OPENLIST }} + script: | + const repository = process.env.REPOSITORY; + const [owner, repo] = repository.split('/'); + + await github.rest.repos.createDispatchEvent({ + owner: owner, + repo: repo, + event_type: 'update-hashes', + client_payload: { + source_repository: process.env.SOURCE_REPOSITORY, + release_tag: process.env.RELEASE_TAG, + release_name: process.env.RELEASE_TAG, + release_url: process.env.RELEASE_URL, + triggered_by: process.env.ACTOR, + trigger_reason: process.env.EVENT_NAME + } + }); + env: + REPOSITORY: ${{ vars.HOOK_REPO || 'OpenListTeam/OpenList-OpenWRT' }} + SOURCE_REPOSITORY: ${{ github.repository }} + RELEASE_TAG: ${{ inputs.tag || github.ref_name }} + RELEASE_URL: ${{ github.server_url }}/${{ github.repository }}/releases/tag/${{ inputs.tag || github.ref_name }} + ACTOR: ${{ github.actor }} + EVENT_NAME: ${{ github.event_name }} - name: Log trigger information run: | diff --git a/.gitignore b/.gitignore index 1d71f0d6..06129e0a 100644 --- a/.gitignore +++ b/.gitignore @@ -22,6 +22,7 @@ output/ # vendor/ /bin/* *.json +!renovate.json /build /data/ /tmp/ @@ -29,6 +30,7 @@ output/ /lang/ /daemon/ /public/dist/* -/!public/dist/README.md +!/public/dist/README.md -.VSCodeCounter \ No newline at end of file +.VSCodeCounter +*.syso \ No newline at end of file diff --git a/README.md b/README.md index cbe127d8..3b0ea099 100644 --- a/README.md +++ b/README.md @@ -67,7 +67,7 @@ Thank you for your support and understanding of the OpenList project. - [x] [MediaFire](https://www.mediafire.com) - [x] [Mediatrack](https://www.mediatrack.cn) - [x] [ProtonDrive](https://proton.me/drive) - - [x] [139yun](https://yun.139.com) (Personal, Family, Group) + - [x] [139yun](https://yun.139.com) (Personal, Family, Group, Share) - [x] [YandexDisk](https://disk.yandex.com) - [x] [BaiduNetdisk](http://pan.baidu.com) - [x] [Terabox](https://www.terabox.com/main) diff --git a/README/README_ar.md b/README/README_ar.md index 32d9e959..ddd4782e 100644 --- a/README/README_ar.md +++ b/README/README_ar.md @@ -50,7 +50,7 @@ OpenList has no official association with third-party derivative projects that u - [x] [MediaFire](https://www.mediafire.com) - [x] [Mediatrack](https://www.mediatrack.cn) - [x] [ProtonDrive](https://proton.me/drive) - - [x] [139yun](https://yun.139.com) (Personal, Family, Group) + - [x] [139yun](https://yun.139.com) (Personal, Family, Group, Share) - [x] [YandexDisk](https://disk.yandex.com) - [x] [BaiduNetdisk](http://pan.baidu.com) - [x] [Terabox](https://www.terabox.com/main) diff --git a/README/README_cn.md b/README/README_cn.md index 5bf8af49..d4ed120e 100644 --- a/README/README_cn.md +++ b/README/README_cn.md @@ -67,7 +67,7 @@ OpenList 是一个由 OpenList 团队独立维护的开源项目，遵循 AGPL-3 - [x] [MediaFire](https://www.mediafire.com) - [x] [分秒帧](https://www.mediatrack.cn) - [x] [ProtonDrive](https://proton.me/drive) - - [x] [和彩云](https://yun.139.com)（个人、家庭、群组） + - [x] [和彩云](https://yun.139.com)（个人、家庭、群组、分享） - [x] [YandexDisk](https://disk.yandex.com) - [x] [百度网盘](http://pan.baidu.com) - [x] [Terabox](https://www.terabox.com/main) diff --git a/README/README_de.md b/README/README_de.md index a0e5a941..93cd891f 100644 --- a/README/README_de.md +++ b/README/README_de.md @@ -50,7 +50,7 @@ OpenList has no official association with third-party derivative projects that u - [x] [MediaFire](https://www.mediafire.com) - [x] [Mediatrack](https://www.mediatrack.cn) - [x] [ProtonDrive](https://proton.me/drive) - - [x] [139yun](https://yun.139.com) (Personal, Family, Group) + - [x] [139yun](https://yun.139.com) (Personal, Family, Group, Share) - [x] [YandexDisk](https://disk.yandex.com) - [x] [BaiduNetdisk](http://pan.baidu.com) - [x] [Terabox](https://www.terabox.com/main) diff --git a/README/README_es.md b/README/README_es.md index fe62a456..84880a60 100644 --- a/README/README_es.md +++ b/README/README_es.md @@ -50,7 +50,7 @@ OpenList has no official association with third-party derivative projects that u - [x] [MediaFire](https://www.mediafire.com) - [x] [Mediatrack](https://www.mediatrack.cn) - [x] [ProtonDrive](https://proton.me/drive) - - [x] [139yun](https://yun.139.com) (Personal, Family, Group) + - [x] [139yun](https://yun.139.com) (Personal, Family, Group, Share) - [x] [YandexDisk](https://disk.yandex.com) - [x] [BaiduNetdisk](http://pan.baidu.com) - [x] [Terabox](https://www.terabox.com/main) diff --git a/README/README_fr.md b/README/README_fr.md index 686de981..304c8086 100644 --- a/README/README_fr.md +++ b/README/README_fr.md @@ -50,7 +50,7 @@ OpenList has no official association with third-party derivative projects that u - [x] [MediaFire](https://www.mediafire.com) - [x] [Mediatrack](https://www.mediatrack.cn) - [x] [ProtonDrive](https://proton.me/drive) - - [x] [139yun](https://yun.139.com) (Personal, Family, Group) + - [x] [139yun](https://yun.139.com) (Personal, Family, Group, Share) - [x] [YandexDisk](https://disk.yandex.com) - [x] [BaiduNetdisk](http://pan.baidu.com) - [x] [Terabox](https://www.terabox.com/main) diff --git a/README/README_ja.md b/README/README_ja.md index d1e5b838..87e261c0 100644 --- a/README/README_ja.md +++ b/README/README_ja.md @@ -66,7 +66,7 @@ OpenListプロジェクトへのご支援とご理解をありがとうござい - [x] Teambition([中国](https://www.teambition.com), [国際](https://us.teambition.com)) - [x] [Mediatrack](https://www.mediatrack.cn) - [x] [ProtonDrive](https://proton.me/drive) - - [x] [139yun](https://yun.139.com)（個人、家族、グループ） + - [x] [139yun](https://yun.139.com)（個人、家族、グループ、共有） - [x] [YandexDisk](https://disk.yandex.com) - [x] [BaiduNetdisk](http://pan.baidu.com) - [x] [Terabox](https://www.terabox.com/main) diff --git a/README/README_ko.md b/README/README_ko.md index 647eb22c..91f6bd88 100644 --- a/README/README_ko.md +++ b/README/README_ko.md @@ -50,7 +50,7 @@ OpenList has no official association with third-party derivative projects that u - [x] [MediaFire](https://www.mediafire.com) - [x] [Mediatrack](https://www.mediatrack.cn) - [x] [ProtonDrive](https://proton.me/drive) - - [x] [139yun](https://yun.139.com) (Personal, Family, Group) + - [x] [139yun](https://yun.139.com) (Personal, Family, Group, Share) - [x] [YandexDisk](https://disk.yandex.com) - [x] [BaiduNetdisk](http://pan.baidu.com) - [x] [Terabox](https://www.terabox.com/main) diff --git a/README/README_nl.md b/README/README_nl.md index af7a9afc..28ab0669 100644 --- a/README/README_nl.md +++ b/README/README_nl.md @@ -67,7 +67,7 @@ Dank u voor uw ondersteuning en begrip - [x] [MediaFire](https://www.mediafire.com) - [x] [Mediatrack](https://www.mediatrack.cn) - [x] [ProtonDrive](https://proton.me/drive) - - [x] [139yun](https://yun.139.com) (Persoonlijk, Familie, Groep) + - [x] [139yun](https://yun.139.com) (Persoonlijk, Familie, Groep, Delen) - [x] [YandexDisk](https://disk.yandex.com) - [x] [BaiduNetdisk](http://pan.baidu.com) - [x] [Terabox](https://www.terabox.com/main) diff --git a/README/README_ru.md b/README/README_ru.md index 782282e3..6ffeb776 100644 --- a/README/README_ru.md +++ b/README/README_ru.md @@ -50,7 +50,7 @@ OpenList has no official association with third-party derivative projects that u - [x] [MediaFire](https://www.mediafire.com) - [x] [Mediatrack](https://www.mediatrack.cn) - [x] [ProtonDrive](https://proton.me/drive) - - [x] [139yun](https://yun.139.com) (Personal, Family, Group) + - [x] [139yun](https://yun.139.com) (Personal, Family, Group, Share) - [x] [YandexDisk](https://disk.yandex.com) - [x] [BaiduNetdisk](http://pan.baidu.com) - [x] [Terabox](https://www.terabox.com/main) diff --git a/SECURITY.md b/SECURITY.md index ff558d64..65446a56 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -38,6 +38,22 @@ To keep your OpenList instance secure: - Regularly review mounted storage permissions and revoke unused API tokens. - When using Docker, avoid running the container as root if possible. +## Security Response Scope (Applicable to 4.x) + +This policy applies only to version 4.x. + +In 4.x, privilege isolation between authenticated users (including administrators and regular users) is not currently considered a security boundary that this project commits to maintaining. + +In principle, only security vulnerabilities that can be exploited without valid authentication (including authentication bypass vulnerabilities) fall within the scope of this project's dedicated security response process. + +Privilege isolation issues that require valid authentication to exploit should preferably be submitted and fixed through regular Pull Requests rather than handled through Security Advisories. + +This policy only defines the scope of the project's security response and does not imply that such issues are not security vulnerabilities. For exceptional cases that may have severe security implications, the maintenance team reserves the right to initiate the security response process at its discretion. + +This policy does not retroactively affect previously accepted security vulnerability reports. + +This project is currently preparing version 5.x. In the future, we may redefine the security boundaries between authenticated users by refactoring the user and permission systems and adjust the security response scope accordingly. + ## Acknowledgments We sincerely thank all security researchers and community members who responsibly disclose vulnerabilities and help make OpenList safer for everyone. @@ -84,6 +100,22 @@ We sincerely thank all security researchers and community members who responsibl - 定期检查已挂载存储的权限，并撤销未使用的 API 令牌。 - 使用 Docker 部署时，尽可能避免以 root 用户运行容器。 +## 安全响应范围 (适用于 4.x) + +本政策仅适用于 4.x 版本。 + +在 4.x 中，已认证用户 (包括管理员和普通用户) 之间的权限隔离暂不属于本项目承诺维护的安全边界。 + +只有无需有效身份认证即可利用的安全漏洞 (包括身份认证绕过漏洞)，原则上才纳入本项目的专门安全响应流程。 + +需要有效身份认证才能利用的权限隔离问题，应优先通过普通 Pull Request 提交和修复，而非通过安全公告 (Security Advisory) 处理。 + +本政策仅限定项目维护团队的安全响应范围，不代表此类问题不构成安全漏洞。对于可能造成严重安全影响的特殊情况，维护团队保留酌情启动安全响应流程的权利。 + +本政策不追溯影响此前已正式接受的安全漏洞报告。 + +本项目正在准备 5.x 版本，未来可能通过重构用户及权限系统，重新定义已认证用户之间的安全边界，并相应调整安全响应范围。 + ## 致谢 我们衷心感谢所有负责任地披露漏洞、帮助 OpenList 变得更加安全的安全研究人员和社区成员。 diff --git a/build.sh b/build.sh index b6baca1f..2ba5f1cc 100644 --- a/build.sh +++ b/build.sh @@ -1,7 +1,7 @@ set -e appName="openlist" builtAt="$(date +'%F %T %z')" -gitAuthor="The OpenList Projects Contributors " +gitAuthor="The OpenList Projects Contributors " gitCommit=$(git log --pretty=format:"%h" -1) # Set frontend repository, default to OpenListTeam/OpenList-Frontend @@ -97,7 +97,7 @@ AssertStaticBinary() { } FetchWebRolling() { - pre_release_json=$(eval "curl -fsSL --max-time 2 $githubAuthArgs -H \"Accept: application/vnd.github.v3+json\" \"https://api.github.com/repos/$frontendRepo/releases/tags/rolling\"") + pre_release_json=$(eval "curl -fsSL --max-time 2 $githubAuthArgs -H \"Accept: application/vnd.github.v3+json\" \"https://api.github.com/repos/$frontendRepo/releases/tags/edge\"") pre_release_assets=$(echo "$pre_release_json" | jq -r '.assets[].browser_download_url') # There is no lite for rolling @@ -178,7 +178,6 @@ BuildWin7() { } BuildDev() { - rm -rf .git/ mkdir -p "dist" muslflags="$(GetMuslStaticLdflags)" BASE="https://github.com/OpenListTeam/musl-compilers/releases/latest/download/" @@ -269,7 +268,6 @@ BuildDockerMultiplatform() { } BuildRelease() { - rm -rf .git/ mkdir -p "build" BuildWinArm64 ./build/"$appName"-windows-arm64.exe BuildWin7 ./build/"$appName"-windows7 @@ -289,7 +287,7 @@ BuildRelease() { BuildLoongGLIBC() { local target_abi="$2" local output_file="$1" - local oldWorldGoVersion="1.25.0" + local oldWorldGoVersion="1.26.4" local loong_tags loong_tags=$(GetBuildTagsForTarget "linux-loong64") @@ -309,13 +307,13 @@ BuildLoongGLIBC() { # Download and setup patched Go compiler for old-world if ! curl -fsSL --retry 3 -H "Authorization: Bearer $GITHUB_TOKEN" \ - "https://github.com/loong64/loong64-abi1.0-toolchains/releases/download/20250821/go${oldWorldGoVersion}.linux-amd64.tar.gz" \ + "https://github.com/loong64-abi1-0/golang/releases/download/go${oldWorldGoVersion}/go${oldWorldGoVersion}.linux-amd64.tar.gz" \ -o go-loong64-abi1.0.tar.gz; then echo "Error: Failed to download patched Go compiler for old-world ABI1.0" if [ -n "$GITHUB_TOKEN" ]; then echo "Error output from curl:" curl -fsSL --retry 3 -H "Authorization: Bearer $GITHUB_TOKEN" \ - "https://github.com/loong64/loong64-abi1.0-toolchains/releases/download/20250821/go${oldWorldGoVersion}.linux-amd64.tar.gz" \ + "https://github.com/loong64-abi1-0/golang/releases/download/go${oldWorldGoVersion}/go${oldWorldGoVersion}.linux-amd64.tar.gz" \ -o go-loong64-abi1.0.tar.gz || true fi return 1 @@ -442,7 +440,6 @@ BuildLoongGLIBC() { } BuildReleaseLinuxMusl() { - rm -rf .git/ mkdir -p "build" muslflags="$(GetMuslStaticLdflags)" BASE="https://github.com/OpenListTeam/musl-compilers/releases/latest/download/" @@ -471,7 +468,6 @@ BuildReleaseLinuxMusl() { } BuildReleaseLinuxMuslArm() { - rm -rf .git/ mkdir -p "build" muslflags="$(GetMuslStaticLdflags)" BASE="https://github.com/OpenListTeam/musl-compilers/releases/latest/download/" @@ -502,7 +498,6 @@ BuildReleaseLinuxMuslArm() { BuildReleaseAndroid() { - rm -rf .git/ mkdir -p "build" wget https://dl.google.com/android/repository/android-ndk-r26b-linux.zip unzip android-ndk-r26b-linux.zip @@ -523,7 +518,6 @@ BuildReleaseAndroid() { } BuildReleaseFreeBSD() { - rm -rf .git/ mkdir -p "build/freebsd" # Get latest FreeBSD 14.x release version from GitHub @@ -537,8 +531,8 @@ BuildReleaseFreeBSD() { sed 's/\.0$//') if [ -z "$freebsd_version" ]; then - echo "Failed to get FreeBSD version, falling back to 14.3" - freebsd_version="14.3" + echo "Failed to get FreeBSD version, falling back to 14.4" + freebsd_version="14.4" fi echo "Using FreeBSD version: $freebsd_version" diff --git a/cmd/crypt.go b/cmd/crypt.go index 12f51f5c..7cdb2925 100644 --- a/cmd/crypt.go +++ b/cmd/crypt.go @@ -23,7 +23,7 @@ type options struct { pwd string //de/encrypt password salt string - filenameEncryption string //reference drivers\crypt\meta.go Addtion + filenameEncryption string //reference drivers\crypt\meta.go Addition dirnameEncryption string filenameEncode string suffix string diff --git a/cmd/start.go b/cmd/start.go index 57edff38..a1838ead 100644 --- a/cmd/start.go +++ b/cmd/start.go @@ -8,6 +8,7 @@ import ( "os/exec" "path/filepath" "strconv" + "strings" log "github.com/sirupsen/logrus" "github.com/spf13/cobra" @@ -31,14 +32,23 @@ func start() { return } } - args := os.Args - args[1] = "server" - args = append(args, "--force-bin-dir") - cmd := &exec.Cmd{ - Path: args[0], - Args: args, - Env: os.Environ(), + exe, err := os.Executable() + if err != nil { + log.Fatal("failed to resolve executable path: ", err) } + childArgs := append([]string{"server"}, os.Args[2:]...) + hasForceBinDir := false + for _, arg := range childArgs { + if arg == "--force-bin-dir" || strings.HasPrefix(arg, "--force-bin-dir=") { + hasForceBinDir = true + break + } + } + if !hasForceBinDir { + childArgs = append(childArgs, "--force-bin-dir") + } + cmd := exec.Command(exe, childArgs...) + cmd.Env = os.Environ() stdout, err := os.OpenFile(filepath.Join(filepath.Dir(pidFile), "start.log"), os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0666) if err != nil { log.Fatal(os.Getpid(), ": failed to open start log file:", err) diff --git a/drivers/115/driver.go b/drivers/115/driver.go index 7c7c581a..d4f5741d 100644 --- a/drivers/115/driver.go +++ b/drivers/115/driver.go @@ -5,7 +5,6 @@ import ( "strings" "sync" - "github.com/OpenListTeam/OpenList/v4/drivers/base" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/model" streamPkg "github.com/OpenListTeam/OpenList/v4/internal/stream" @@ -69,9 +68,6 @@ func (d *Pan115) Link(ctx context.Context, file model.Obj, args model.LinkArgs) return nil, err } userAgent := args.Header.Get("User-Agent") - if userAgent == "" { - userAgent = base.UserAgent - } downloadInfo, err := d.client.DownloadWithUA(file.(*FileObj).PickCode, userAgent) if err != nil { return nil, err diff --git a/drivers/115/util.go b/drivers/115/util.go index 57c98734..104ebe8e 100644 --- a/drivers/115/util.go +++ b/drivers/115/util.go @@ -337,9 +337,11 @@ func (d *Pan115) UploadByMultipart(ctx context.Context, params *driver115.Upload for chunk := range chunksCh { var part oss.UploadPart // 出现错误就继续尝试，共尝试3次 for retry := 0; retry < 3; retry++ { - select { - case <-ctx.Done(): + if utils.IsCanceled(ctx) { + err = ctx.Err() break + } + select { case <-ticker.C: if ossToken, err = d.client.GetOSSToken(); err != nil { // 到时重新获取ossToken errCh <- errors.Wrap(err, "刷新token时出现错误") diff --git a/drivers/115_open/driver.go b/drivers/115_open/driver.go index e27b950f..65fc5eef 100644 --- a/drivers/115_open/driver.go +++ b/drivers/115_open/driver.go @@ -172,17 +172,59 @@ func (d *Open115) Get(ctx context.Context, path string) (model.Obj, error) { resp, err := d.client.GetFolderInfoByPath(ctx, path) if err != nil { if errors.Is(err, sdk.ErrObjectNotFound) { - return nil, errs.ObjectNotFound + return d.getFromParent(ctx, path, "") } return nil, err } - return &Obj{ + obj := &Obj{ Fid: resp.FileID, Fn: resp.FileName, Fc: resp.FileCategory, Sha1: resp.Sha1, Pc: resp.PickCode, - }, nil + FS: resp.SizeByte, + Upt: parseTime(resp.UTime), + UpPt: parseTime(resp.PTime), + } + if !obj.IsDir() && obj.ModTime().Unix() <= 0 { + return d.getFromParent(ctx, path, obj.GetID()) + } + return obj, nil +} + +func (d *Open115) getFromParent(ctx context.Context, path, id string) (model.Obj, error) { + path = stdpath.Clean(path) + parent, name := stdpath.Split(path) + parent = stdpath.Clean(parent) + parentID := d.GetRootId() + if stdpath.Clean(parent) != stdpath.Clean(d.parentPath) { + if err := d.WaitLimit(ctx); err != nil { + return nil, err + } + parentInfo, err := d.client.GetFolderInfoByPath(ctx, parent) + if err != nil { + if !errors.Is(err, sdk.ErrObjectNotFound) { + return nil, err + } + parentObj, err := d.getFromParent(ctx, parent, "") + if err != nil { + return nil, err + } + parentID = parentObj.GetID() + } else { + parentID = parentInfo.FileID + } + } + files, err := d.List(ctx, &Obj{Fid: parentID, Fc: "0"}, model.ListArgs{}) + if err != nil { + return nil, err + } + for _, file := range files { + if (id != "" && file.GetID() == id) || (id == "" && file.GetName() == name) { + return file, nil + } + } + return nil, errs.ObjectNotFound } func (d *Open115) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) (model.Obj, error) { diff --git a/drivers/115_open/upload.go b/drivers/115_open/upload.go index d02640e2..ac0aad38 100644 --- a/drivers/115_open/upload.go +++ b/drivers/115_open/upload.go @@ -3,6 +3,7 @@ package _115_open import ( "context" "encoding/base64" + "errors" "io" "time" @@ -70,6 +71,19 @@ func (d *Open115) singleUpload(ctx context.Context, tempF model.File, tokenResp // } `json:"data"` // } +// retryExpiredToken retries only the rejected OSS operation, preserving the upload ID. +func retryExpiredToken(refresh func() error, operation func() error) error { + err := operation() + var serviceErr oss.ServiceError + if !errors.As(err, &serviceErr) || serviceErr.Code != "SecurityTokenExpired" { + return err + } + if err := refresh(); err != nil { + return err + } + return operation() +} + func (d *Open115) multpartUpload(ctx context.Context, stream model.FileStreamer, up driver.UpdateProgress, tokenResp *sdk.UploadGetTokenResp, initResp *sdk.UploadInitResp) error { ossClient, err := netutil.NewOSSClient(tokenResp.Endpoint, tokenResp.AccessKeyId, tokenResp.AccessKeySecret, oss.SecurityToken(tokenResp.SecurityToken)) if err != nil { @@ -80,7 +94,32 @@ func (d *Open115) multpartUpload(ctx context.Context, stream model.FileStreamer, return err } - imur, err := bucket.InitiateMultipartUpload(initResp.Object, oss.Sequential()) + refresh := func() error { + if err := d.WaitLimit(ctx); err != nil { + return err + } + token, err := d.client.UploadGetToken(ctx) + if err != nil { + return err + } + client, err := netutil.NewOSSClient(token.Endpoint, token.AccessKeyId, token.AccessKeySecret, oss.SecurityToken(token.SecurityToken)) + if err != nil { + return err + } + newBucket, err := client.Bucket(initResp.Bucket) + if err != nil { + return err + } + bucket = newBucket + return nil + } + + var imur oss.InitiateMultipartUploadResult + err = retryExpiredToken(refresh, func() error { + var err error + imur, err = bucket.InitiateMultipartUpload(initResp.Object, oss.Sequential(), oss.WithContext(ctx)) + return err + }) if err != nil { return err } @@ -109,13 +148,17 @@ func (d *Open115) multpartUpload(ctx context.Context, stream model.FileStreamer, return err } err = retry.Do(func() error { - rd.Seek(0, io.SeekStart) - part, err := bucket.UploadPart(imur, driver.NewLimitedUploadStream(ctx, rd), partSize, int(i)) - if err != nil { - return err - } - parts[i-1] = part - return nil + return retryExpiredToken(refresh, func() error { + if _, err := rd.Seek(0, io.SeekStart); err != nil { + return err + } + part, err := bucket.UploadPart(imur, driver.NewLimitedUploadStream(ctx, rd), partSize, int(i), oss.WithContext(ctx)) + if err != nil { + return err + } + parts[i-1] = part + return nil + }) }, retry.Context(ctx), retry.Attempts(3), @@ -134,14 +177,16 @@ func (d *Open115) multpartUpload(ctx context.Context, stream model.FileStreamer, up(float64(offset) * 100 / float64(fileSize)) } - // callbackRespBytes := make([]byte, 1024) - _, err = bucket.CompleteMultipartUpload( - imur, - parts, - oss.Callback(base64.StdEncoding.EncodeToString([]byte(initResp.Callback.Value.Callback))), - oss.CallbackVar(base64.StdEncoding.EncodeToString([]byte(initResp.Callback.Value.CallbackVar))), - // oss.CallbackResult(&callbackRespBytes), - ) + err = retryExpiredToken(refresh, func() error { + _, err := bucket.CompleteMultipartUpload( + imur, + parts, + oss.Callback(base64.StdEncoding.EncodeToString([]byte(initResp.Callback.Value.Callback))), + oss.CallbackVar(base64.StdEncoding.EncodeToString([]byte(initResp.Callback.Value.CallbackVar))), + oss.WithContext(ctx), + ) + return err + }) if err != nil { return err } diff --git a/drivers/115_open/util.go b/drivers/115_open/util.go index a7a333c9..a12ed8b1 100644 --- a/drivers/115_open/util.go +++ b/drivers/115_open/util.go @@ -1,6 +1,9 @@ package _115_open -import "encoding/json" +import ( + "encoding/json" + "strconv" +) func ParseInt64(v json.Number) (int64, error) { i, err := v.Int64() @@ -13,3 +16,8 @@ func ParseInt64(v json.Number) (int64, error) { } return int64(0), err } + +func parseTime(value string) int64 { + timestamp, _ := strconv.ParseInt(value, 10, 64) + return timestamp +} diff --git a/drivers/115_share/driver.go b/drivers/115_share/driver.go index 8f8cac01..a97c13ca 100644 --- a/drivers/115_share/driver.go +++ b/drivers/115_share/driver.go @@ -90,9 +90,6 @@ func (d *Pan115Share) Link(ctx context.Context, file model.Obj, args model.LinkA if args.Header != nil { ua = args.Header.Get("User-Agent") } - if ua == "" { - ua = base.UserAgent - } downloadInfo, err := d.client.DownloadByShareCodeWithUA(ua, d.ShareCode, d.ReceiveCode, file.GetID()) if err != nil { return nil, err diff --git a/drivers/123/util.go b/drivers/123/util.go index 4b10b719..fef7483b 100644 --- a/drivers/123/util.go +++ b/drivers/123/util.go @@ -181,7 +181,7 @@ func (d *Pan123) login() error { return err } if utils.Json.Get(res.Body(), "code").ToInt() != 200 { - err = fmt.Errorf(utils.Json.Get(res.Body(), "message").ToString()) + err = fmt.Errorf("%s", utils.Json.Get(res.Body(), "message").ToString()) } else { d.AccessToken = utils.Json.Get(res.Body(), "data", "token").ToString() } diff --git a/drivers/139/driver.go b/drivers/139/driver.go index 6386a2f2..a647fed9 100644 --- a/drivers/139/driver.go +++ b/drivers/139/driver.go @@ -19,6 +19,7 @@ import ( "github.com/OpenListTeam/OpenList/v4/pkg/cron" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/OpenListTeam/OpenList/v4/pkg/utils/random" + "github.com/avast/retry-go" log "github.com/sirupsen/logrus" ) @@ -29,7 +30,9 @@ type Yun139 struct { Account string ref *Yun139 PersonalCloudHost string - RootPath string + FamilyCloudHost string + GroupCloudHost string + ProviderRoot string } func (d *Yun139) Config() driver.Config { @@ -42,53 +45,62 @@ func (d *Yun139) GetAddition() driver.Additional { func (d *Yun139) Init(ctx context.Context) error { if d.ref == nil { - if len(d.Authorization) == 0 { - if d.Username != "" && d.Password != "" { - log.Infof("139yun: authorization is empty, trying to login with password.") - newAuth, err := d.loginWithPassword() - log.Debugf("newAuth: Ok: %s", newAuth) - if err != nil { - return fmt.Errorf("login with password failed: %w", err) - } - } else { - return fmt.Errorf("authorization is empty and username/password is not provided") + // Share links may be browsed without account credentials. All other + // storage types need an authorization or a supported login fallback. + if !d.isShare() { + if err := d.validateAndInitCredentials(); err != nil { + return err } } - err := d.refreshToken() - if err != nil { - return err - } - - // Query Route Policy - var resp QueryRoutePolicyResp - _, err = d.requestRoute(base.Json{ - "userInfo": base.Json{ - "userType": 1, - "accountType": 1, - "accountName": d.Account, - }, - "modAddrType": 1, - }, &resp) - if err != nil { - return err - } - for _, policyItem := range resp.Data.RoutePolicyList { - if policyItem.ModName == "personal" { - d.PersonalCloudHost = policyItem.HttpsUrl - break - } - } - if len(d.PersonalCloudHost) == 0 { - return fmt.Errorf("PersonalCloudHost is empty") - } - - d.cron = cron.NewCron(time.Hour * 12) - d.cron.Do(func() { + if d.Authorization != "" { err := d.refreshToken() if err != nil { - log.Errorf("%+v", err) + return err } - }) + + // Query Route Policy + var resp QueryRoutePolicyResp + _, err = d.requestRoute(base.Json{ + "userInfo": base.Json{ + "userType": 1, + "accountType": 1, + "accountName": d.Account, + }, + "modAddrType": 1, + }, &resp) + if err != nil { + return err + } + for _, policyItem := range resp.Data.RoutePolicyList { + switch policyItem.ModName { + case "personal": + d.PersonalCloudHost = policyItem.HttpsUrl + case "group": + d.GroupCloudHost = policyItem.HttpsUrl + case "family": + d.FamilyCloudHost = policyItem.HttpsUrl + } + } + if len(d.PersonalCloudHost) == 0 { + return fmt.Errorf("PersonalCloudHost is empty") + } + if d.isGroup() || d.isFamily() { + if len(d.GroupCloudHost) == 0 { + return fmt.Errorf("GroupCloudHost is empty") + } + if len(d.FamilyCloudHost) == 0 { + return fmt.Errorf("FamilyCloudHost is empty") + } + } + + d.cron = cron.NewCron(time.Hour * 12) + d.cron.Do(func() { + err := d.refreshToken() + if err != nil { + log.Errorf("%+v", err) + } + }) + } } switch d.Addition.Type { case MetaPersonalNew: @@ -107,15 +119,25 @@ func (d *Yun139) Init(ctx context.Context) error { if err != nil { return err } - case MetaFamily: + case MetaShare: if len(d.Addition.RootFolderID) == 0 { - // Attempt to obtain data.path as the root via a query and persist it. - if root, err := d.getFamilyRootPath(d.CloudID); err == nil && root != "" { - d.RootFolderID = root - op.MustSaveDriverStorage(d) - } + d.RootFolderID = "root" } - _, err := d.familyGetFiles(d.RootFolderID) + if len(d.shareEntries()) == 0 { + return fmt.Errorf("link_id is empty") + } + case MetaFamily: + // Attempt to obtain data.path as the root via a query and persist it. + root, err := d.getFamilyRootPath(d.CloudID) + if err != nil || root == "" { + return fmt.Errorf("failed to get family root path: %w", err) + } + d.ProviderRoot = root + if len(d.Addition.RootFolderID) == 0 { + d.RootFolderID = root + op.MustSaveDriverStorage(d) + } + _, err = d.familyGetFiles(d.RootFolderID) if err != nil { return err } @@ -134,6 +156,19 @@ func (d *Yun139) InitReference(storage driver.Driver) error { return errs.NotSupport } +func (d *Yun139) Get(ctx context.Context, path string) (model.Obj, error) { + if !d.isShare() { + return nil, errs.NotImplement + } + if path == "/" { + return &model.Object{ID: "root", Name: "root", IsFolder: true, Path: "/"}, nil + } + if obj, err := d.shareGetObj(path); err == nil { + return obj, nil + } + return nil, errs.ObjectNotFound +} + func (d *Yun139) Drop(ctx context.Context) error { if d.cron != nil { d.cron.Stop() @@ -152,12 +187,23 @@ func (d *Yun139) List(ctx context.Context, dir model.Obj, args model.ListArgs) ( return d.familyGetFiles(dir.GetID()) case MetaGroup: return d.groupGetFiles(dir.GetID()) + case MetaShare: + if dir.GetID() == "root" { + return d.shareGetMergedFiles(d.shareRootEntries()) + } + if refs, ok := decodeShareRefs(dir.GetID()); ok { + return d.shareGetMergedFiles(refs) + } + return d.shareGetFilesWithRef(shareRef{LinkID: d.LinkID, NodeID: dir.GetID()}, dir.GetID()) default: return nil, errs.NotImplement } } func (d *Yun139) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { + if file.IsDir() { + return nil, errs.NotFile + } var url string var err error switch d.Addition.Type { @@ -169,6 +215,16 @@ func (d *Yun139) Link(ctx context.Context, file model.Obj, args model.LinkArgs) url, err = d.familyGetLink(file.GetID(), file.GetPath()) case MetaGroup: url, err = d.groupGetLink(file.GetID(), file.GetPath()) + case MetaShare: + if refs, ok := decodeShareRefs(file.GetID()); ok && len(refs) > 0 { + return d.shareGetLinkWithRef(refs[0], refs[0].NodeID, args.Type) + } + fallbackRef := shareRef{LinkID: d.LinkID, NodeID: "root"} + if entries := d.shareEntries(); len(entries) > 0 { + fallbackRef.LinkID = entries[0].LinkID + fallbackRef.Password = entries[0].Password + } + return d.shareGetLinkWithRef(fallbackRef, file.GetID(), args.Type) default: return nil, errs.NotImplement } @@ -179,6 +235,9 @@ func (d *Yun139) Link(ctx context.Context, file model.Obj, args model.LinkArgs) } func (d *Yun139) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error { + if d.isShare() { + return errs.NotImplement + } var err error switch d.Addition.Type { case MetaPersonalNew: @@ -212,7 +271,7 @@ func (d *Yun139) MakeDir(ctx context.Context, parentDir model.Obj, dirName strin "accountType": 1, }, "docLibName": dirName, - "path": path.Join(parentDir.GetPath(), parentDir.GetID()), + "path": d.dirPath(parentDir), } pathname := "/orchestration/familyCloud-rebuild/cloudCatalog/v1.0/createCloudDoc" _, err = d.post(pathname, data, nil) @@ -225,7 +284,7 @@ func (d *Yun139) MakeDir(ctx context.Context, parentDir model.Obj, dirName strin }, "groupID": d.CloudID, "parentFileId": parentDir.GetID(), - "path": path.Join(parentDir.GetPath(), parentDir.GetID()), + "path": d.dirPath(parentDir), } pathname := "/orchestration/group-rebuild/catalog/v1.0/createGroupCatalog" _, err = d.post(pathname, data, nil) @@ -236,6 +295,9 @@ func (d *Yun139) MakeDir(ctx context.Context, parentDir model.Obj, dirName strin } func (d *Yun139) Move(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { + if d.isShare() { + return nil, errs.NotImplement + } switch d.Addition.Type { case MetaPersonalNew: data := base.Json{ @@ -310,9 +372,9 @@ func (d *Yun139) Move(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, var contentList []string var catalogList []string if srcObj.IsDir() { - catalogList = append(catalogList, path.Join(srcObj.GetPath(), srcObj.GetID())) + catalogList = append(catalogList, d.dirPath(srcObj)) } else { - contentList = append(contentList, path.Join(srcObj.GetPath(), srcObj.GetID())) + contentList = append(contentList, d.dirPath(srcObj)) } body := base.Json{ @@ -324,7 +386,7 @@ func (d *Yun139) Move(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, "contentList": contentList, "destCatalogID": dstDir.GetID(), "destGroupID": d.CloudID, - "destPath": path.Join(dstDir.GetPath(), dstDir.GetID()), + "destPath": d.dirPath(dstDir), "destType": 0, "srcGroupID": d.CloudID, "srcType": 0, @@ -347,6 +409,9 @@ func (d *Yun139) Move(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, } func (d *Yun139) Rename(ctx context.Context, srcObj model.Obj, newName string) error { + if d.isShare() { + return errs.NotImplement + } var err error switch d.Addition.Type { case MetaPersonalNew: @@ -425,7 +490,7 @@ func (d *Yun139) Rename(ctx context.Context, srcObj model.Obj, newName string) e }, "docLibName": newName, "docLibraryID": srcObj.GetID(), - "path": path.Join(srcObj.GetPath(), srcObj.GetID()), + "path": d.dirPath(srcObj), } var resp ModifyCloudDocV2Resp _, err = d.andAlbumRequest(pathname, data, &resp) @@ -456,6 +521,9 @@ func (d *Yun139) Rename(ctx context.Context, srcObj model.Obj, newName string) e } func (d *Yun139) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { + if d.isShare() { + return errs.NotImplement + } var err error switch d.Addition.Type { case MetaPersonalNew: @@ -525,6 +593,9 @@ func (d *Yun139) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { } func (d *Yun139) Remove(ctx context.Context, obj model.Obj) error { + if d.isShare() { + return errs.NotImplement + } switch d.Addition.Type { case MetaPersonalNew: data := base.Json{ @@ -615,8 +686,25 @@ func (d *Yun139) getPartSize(size int64) int64 { } func (d *Yun139) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) error { - switch d.Addition.Type { - case MetaPersonalNew: + if d.isShare() { + return errs.NotImplement + } + // PersonalNew 以及 Group/Family 在非旧流模式时走新上传路径 + if d.Addition.Type == MetaPersonalNew || + ((d.isGroup() || d.isFamily()) && !d.UseOldStreamUpload) { + var createPath, getUploadUrlPath, completePath string + if d.isGroup() || d.isFamily() { + // 家庭云和共享群共用同一套新上传 API + createPath = "/dynamic/file/create" + getUploadUrlPath = "/dynamic/file/getUploadUrl" + completePath = "/dynamic/file/complete" + } else { + // MetaPersonalNew + createPath = "/file/create" + getUploadUrlPath = "/file/getUploadUrl" + completePath = "/file/complete" + } + var err error fullHash := stream.GetHash().GetHash(utils.SHA256) if len(fullHash) != utils.SHA256.Width { @@ -671,9 +759,22 @@ func (d *Yun139) Put(ctx context.Context, dstDir model.Obj, stream model.FileStr "type": "file", "fileRenameMode": "auto_rename", } - pathname := "/file/create" + // 家庭云和共享群需要额外的参数 + if d.isGroup() || d.isFamily() { + if d.CloudID == "" { + return fmt.Errorf("cloud_id is required for group/family upload") + } + data["groupId"] = d.CloudID + if d.isGroup() { + data["groupType"] = 2 + } else if d.isFamily() { + data["groupType"] = 1 + } + data["catalogType"] = 3 + data["seqNo"] = random.String(32) + } var resp PersonalUploadResp - _, err = d.personalPost(pathname, data, &resp) + _, err = d.newPost(createPath, data, &resp) if err != nil { return err } @@ -690,10 +791,19 @@ func (d *Yun139) Put(ctx context.Context, dstDir model.Obj, stream model.FileStr if resp.Data.PartInfos != nil { // Progress p := driver.NewProgress(size, up) + rateLimited := driver.NewLimitedUploadStream(ctx, stream) + ss, err := streamPkg.NewStreamSectionReader(&streamPkg.FileStream{ + Ctx: ctx, + Reader: rateLimited, + Obj: &model.Object{Size: size}, + }, int(partSize), &up) + if err != nil { + return err + } // 先上传前100个分片 - err = d.uploadPersonalParts(ctx, partInfos, resp.Data.PartInfos, rateLimited, p) + err = d.uploadPersonalParts(ctx, partInfos, resp.Data.PartInfos, ss, p) if err != nil { return err } @@ -711,13 +821,12 @@ func (d *Yun139) Put(ctx context.Context, dstDir model.Obj, stream model.FileStr "accountType": 1, }, } - pathname := "/file/getUploadUrl" var moreresp PersonalUploadUrlResp - _, err = d.personalPost(pathname, moredata, &moreresp) + _, err = d.newPost(getUploadUrlPath, moredata, &moreresp) if err != nil { return err } - err = d.uploadPersonalParts(ctx, partInfos, moreresp.Data.PartInfos, rateLimited, p) + err = d.uploadPersonalParts(ctx, partInfos, moreresp.Data.PartInfos, ss, p) if err != nil { return err } @@ -730,7 +839,11 @@ func (d *Yun139) Put(ctx context.Context, dstDir model.Obj, stream model.FileStr "fileId": resp.Data.FileId, "uploadId": resp.Data.UploadId, } - _, err = d.personalPost("/file/complete", data, nil) + // 家庭云和共享群需要额外的参数 + if d.isGroup() || d.isFamily() { + data["groupId"] = d.CloudID + } + _, err = d.newPost(completePath, data, nil) if err != nil { return err } @@ -775,11 +888,11 @@ func (d *Yun139) Put(ctx context.Context, dstDir model.Obj, stream model.FileStr } } return nil - case MetaPersonal: - fallthrough - case MetaGroup: - fallthrough - case MetaFamily: + } + + // 旧上传路径 + switch d.Addition.Type { + case MetaPersonal, MetaGroup, MetaFamily: // 处理冲突 // 获取文件列表 files, err := d.List(ctx, dstDir, model.ListArgs{}) @@ -826,11 +939,11 @@ func (d *Yun139) Put(ctx context.Context, dstDir model.Obj, stream model.FileStr }, } pathname := "/orchestration/personalCloud/uploadAndDownload/v1.0/pcUploadFileRequest" - if d.isFamily() || d.Addition.Type == MetaGroup { - uploadPath := path.Join(dstDir.GetPath(), dstDir.GetID()) - // if dstDir is root folder - if dstDir.GetID() == d.RootFolderID { - uploadPath = d.RootPath + if d.isFamily() || d.isGroup() { + uploadPath := d.dirPath(dstDir) + // 共享群的根目录上传路径为 0 + if d.isGroup() && dstDir.GetID() == d.RootFolderID { + uploadPath = "0" } data = d.newJson(base.Json{ "fileCount": 1, @@ -858,56 +971,86 @@ func (d *Yun139) Put(ctx context.Context, dstDir model.Obj, stream model.FileStr } size := stream.GetSize() + partSize := d.getPartSize(size) + // Progress p := driver.NewProgress(size, up) - partSize := d.getPartSize(size) + rateLimited := driver.NewLimitedUploadStream(ctx, stream) + + // StreamSectionReader for per-chunk buffering and retry + ss, err := streamPkg.NewStreamSectionReader(&streamPkg.FileStream{ + Ctx: ctx, + Reader: rateLimited, + Obj: &model.Object{Size: size}, + }, int(partSize), &up) + if err != nil { + return err + } + part := int64(1) if size > partSize { part = (size + partSize - 1) / partSize } - rateLimited := driver.NewLimitedUploadStream(ctx, stream) for i := int64(0); i < part; i++ { if utils.IsCanceled(ctx) { return ctx.Err() } - start := i * partSize byteSize := min(size-start, partSize) - limitReader := io.LimitReader(rateLimited, byteSize) - // Update Progress - r := io.TeeReader(limitReader, p) - req, err := http.NewRequestWithContext(ctx, http.MethodPost, resp.Data.UploadResult.RedirectionURL, r) - if err != nil { - return err + rd, getErr := ss.GetSectionReader(start, byteSize) + if getErr != nil { + return getErr } - req.Header.Set("Content-Type", "text/plain;name="+unicode(stream.GetName())) - req.Header.Set("contentSize", strconv.FormatInt(size, 10)) - req.Header.Set("range", fmt.Sprintf("bytes=%d-%d", start, start+byteSize-1)) - req.Header.Set("uploadtaskID", resp.Data.UploadResult.UploadTaskID) - req.Header.Set("rangeType", "0") - req.ContentLength = byteSize - res, err := base.HttpClient.Do(req) + err = retry.Do( + func() error { + if _, err := rd.Seek(0, io.SeekStart); err != nil { + return err + } + req, reqErr := http.NewRequestWithContext(ctx, http.MethodPost, resp.Data.UploadResult.RedirectionURL, + io.TeeReader(rd, p)) + if reqErr != nil { + return reqErr + } + req.Header.Set("Content-Type", "text/plain;name="+unicode(stream.GetName())) + req.Header.Set("contentSize", strconv.FormatInt(size, 10)) + req.Header.Set("range", fmt.Sprintf("bytes=%d-%d", start, start+byteSize-1)) + req.Header.Set("uploadtaskID", resp.Data.UploadResult.UploadTaskID) + req.Header.Set("rangeType", "0") + req.ContentLength = byteSize + + res, doErr := base.HttpClient.Do(req) + if doErr != nil { + return doErr + } + defer res.Body.Close() + bodyBytes, readErr := io.ReadAll(res.Body) + if readErr != nil { + return fmt.Errorf("error reading response body: %v", readErr) + } + if res.StatusCode != http.StatusOK { + return fmt.Errorf("unexpected status code: %d, body: %s", res.StatusCode, string(bodyBytes)) + } + var result InterLayerUploadResult + xmlErr := xml.Unmarshal(bodyBytes, &result) + if xmlErr != nil { + return fmt.Errorf("error parsing XML: %v", xmlErr) + } + if result.ResultCode != 0 { + return fmt.Errorf("upload failed with result code: %d, message: %s", result.ResultCode, result.Msg) + } + return nil + }, + retry.Context(ctx), + retry.Attempts(3), + retry.DelayType(retry.BackOffDelay), + retry.Delay(time.Second), + ) + ss.FreeSectionReader(rd) if err != nil { return err } - if res.StatusCode != http.StatusOK { - res.Body.Close() - return fmt.Errorf("unexpected status code: %d", res.StatusCode) - } - bodyBytes, err := io.ReadAll(res.Body) - if err != nil { - return fmt.Errorf("error reading response body: %v", err) - } - var result InterLayerUploadResult - err = xml.Unmarshal(bodyBytes, &result) - if err != nil { - return fmt.Errorf("error parsing XML: %v", err) - } - if result.ResultCode != 0 { - return fmt.Errorf("upload failed with result code: %d, message: %s", result.ResultCode, result.Msg) - } } return nil default: diff --git a/drivers/139/meta.go b/drivers/139/meta.go index 91d54fd3..184c3e12 100644 --- a/drivers/139/meta.go +++ b/drivers/139/meta.go @@ -7,17 +7,21 @@ import ( type Addition struct { //Account string `json:"account" required:"true"` - Authorization string `json:"authorization" type:"text" required:"true"` - Username string `json:"username" required:"true"` - Password string `json:"password" required:"true" secret:"true"` - MailCookies string `json:"mail_cookies" required:"true" type:"text" help:"Cookies from mail.139.com used for login authentication."` + Authorization string `json:"authorization" type:"text" help:"Authorization can be used alone. If empty, use username + password; mail_cookies is optional and will be established/updated automatically. Existing mail_cookies can also be used alone for fast login."` + Username string `json:"username" help:"Use together with password when Authorization is empty. mail_cookies may be left empty on the first login."` + Password string `json:"password" secret:"true" help:"Use together with username when Authorization is empty. mail_cookies may be left empty on the first login."` + SmsCode string `json:"sms_code" secret:"true" help:"Fill this only after OpenList reports that a 139 Mail SMS verification code was sent, then save the storage again."` + MailCookies string `json:"mail_cookies" type:"text" help:"Optional cookies from mail.10086.cn. Leave empty for a first username/password login; cookies created or updated by password/SMS login are persisted and reused as device context. Existing cookies may also be used alone for fast login."` driver.RootID - Type string `json:"type" type:"select" options:"personal_new,family,group,personal" default:"personal_new"` + Type string `json:"type" type:"select" options:"personal_new,family,group,personal,share" default:"personal_new"` + LinkID string `json:"link_id" type:"text" help:"Multiple shares are separated by commas or new lines. Use link_id#password for password-protected shares."` CloudID string `json:"cloud_id"` UserDomainID string `json:"user_domain_id" help:"ud_id in Cookie, fill in to show disk usage"` CustomUploadPartSize int64 `json:"custom_upload_part_size" type:"number" default:"0" help:"0 for auto"` ReportRealSize bool `json:"report_real_size" type:"bool" default:"true" help:"Enable to report the real file size during upload"` UseLargeThumbnail bool `json:"use_large_thumbnail" type:"bool" default:"false" help:"Enable to use large thumbnail for images"` + UseOldStreamUpload bool `json:"use_old_stream_upload" type:"bool" default:"false" help:"Enable to use old stream upload method (not support rapid upload)"` + ClientType string `json:"client_type" type:"select" options:"web,pc" default:"web" help:"Client type reported to the personal cloud API. web (9) allows uploading files up to 50 GB; pc (13) raises the single file upload limit to 500 GB."` } var config = driver.Config{ diff --git a/drivers/139/share_test.go b/drivers/139/share_test.go new file mode 100644 index 00000000..484f6f65 --- /dev/null +++ b/drivers/139/share_test.go @@ -0,0 +1,35 @@ +package _139 + +import ( + "testing" +) + +func TestShareEntriesAndRefEncoding(t *testing.T) { + d := &Yun139{Addition: Addition{ + Type: MetaShare, + LinkID: "share-a,share-b,share-c#pass", + }} + + entries := d.shareEntries() + if len(entries) != 3 { + t.Fatalf("expected 3 share entries, got %d", len(entries)) + } + if entries[2].LinkID != "share-c" || entries[2].Password != "pass" { + t.Fatalf("unexpected password share entry: %+v", entries[2]) + } + + refs := []shareRef{ + {LinkID: entries[0].LinkID, Password: entries[0].Password, NodeID: "root-a"}, + {LinkID: entries[2].LinkID, Password: entries[2].Password, NodeID: "root-b"}, + } + encoded := encodeShareRefs(refs) + decoded, ok := decodeShareRefs(encoded) + if !ok || len(decoded) != len(refs) { + t.Fatalf("failed to decode merged share refs: %q", encoded) + } + for i := range refs { + if decoded[i] != refs[i] { + t.Fatalf("unexpected decoded ref at %d: %+v", i, decoded[i]) + } + } +} diff --git a/drivers/139/types.go b/drivers/139/types.go index 8c23cdcb..a7401a3a 100644 --- a/drivers/139/types.go +++ b/drivers/139/types.go @@ -9,6 +9,7 @@ const ( MetaFamily string = "family" MetaGroup string = "group" MetaPersonalNew string = "personal_new" + MetaShare string = "share" ) type BaseResp struct { @@ -285,6 +286,56 @@ type PersonalUploadUrlResp struct { } } +type ShareCatalog struct { + CaID string `json:"caId"` + CaName string `json:"caName"` + UdTime string `json:"udTime"` +} + +type ShareContent struct { + CoID string `json:"coId"` + CoName string `json:"coName"` + CoSize int64 `json:"coSize"` + CoType int `json:"coType"` + UdTime string `json:"udTime"` + CoPath string `json:"coPath"` + PresentURL string `json:"presentURL"` + DownloadURL string `json:"downloadURL"` +} + +type ShareListResp struct { + BaseResp + Data struct { + LKName string `json:"lkName"` + Passwd string `json:"password"` + CaLst []ShareCatalog `json:"caLst"` + CoLst []ShareContent `json:"coLst"` + } `json:"data"` +} + +type ShareContentInfo struct { + PresentURL string `json:"presentURL"` + DownloadURL string `json:"cdnDownLoadUrl"` +} + +type ShareDownloadResp struct { + BaseResp + Data struct { + DownloadURL string `json:"downloadURL"` + RedrURL string `json:"redrUrl"` + ExtInfo struct { + CDNDownloadURL string `json:"cdnDownloadUrl"` + } `json:"extInfo"` + } `json:"data"` +} + +type ShareContentInfoResp struct { + BaseResp + Data struct { + ContentInfo ShareContentInfo `json:"contentInfo"` + } `json:"data"` +} + type QueryRoutePolicyResp struct { Success bool `json:"success"` Code string `json:"code"` diff --git a/drivers/139/util.go b/drivers/139/util.go index 3ff123ba..a53f99e9 100644 --- a/drivers/139/util.go +++ b/drivers/139/util.go @@ -6,9 +6,12 @@ import ( "crypto/aes" "crypto/cipher" crypto_rand "crypto/rand" + "crypto/rsa" "crypto/sha1" + "crypto/x509" "encoding/base64" "encoding/hex" + "encoding/xml" "errors" "fmt" "io" @@ -23,23 +26,49 @@ import ( "github.com/OpenListTeam/OpenList/v4/drivers/base" "github.com/OpenListTeam/OpenList/v4/internal/driver" + "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/op" + streamPkg "github.com/OpenListTeam/OpenList/v4/internal/stream" + cookiepkg "github.com/OpenListTeam/OpenList/v4/pkg/cookie" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/OpenListTeam/OpenList/v4/pkg/utils/random" + "github.com/avast/retry-go" "github.com/go-resty/resty/v2" jsoniter "github.com/json-iterator/go" log "github.com/sirupsen/logrus" ) const ( - KEY_HEX_1 = "73634235495062495331515373756c734e7253306c673d3d" // 第一层 AES 解密密钥 - KEY_HEX_2 = "7150714477323633586746674c337538" // 第二层 AES 解密密钥 + KEY_HEX_1 = "73634235495062495331515373756c734e7253306c673d3d" // 第一层 AES 解密密钥 + KEY_HEX_2 = "7150714477323633586746674c337538" // 第二层 AES 解密密钥 + mailPublicKey = "MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAnsOHTFtwW5rq/8gGhPlM5Z3RPdeN/d+FYIHb5JmcfBOCozXuT8c+0anvxtkjzghixwNlnmBuhN8OYfS789YuH/qReQbHC7OdlisLildNWPHRNUYcPa0W3lXSG3+81CXK7FDXPvXo5ubw2GqVbIsccMarI1dyfXdi4ITiCXvmM9wYBdUs9yXtoorhlpyYUI2GV8HNsQjWK9P5QZHT3ox5Qy+mjRmvv6RUFJLPOMkOS/pGZ0DwC1ypFZBxstW0/ftVupdOmGWvW7J2/e3dq3A/UvIkC4YUY/diL1wighJx1G9MiRROISjNMvNyUDSTqPJy516+l3sgHEbc067QIJx2NQIDAQAB" +) + +var ( + mailPasswordURL = "https://mail.10086.cn/Login/Login.ashx" + mailSMSURL = "https://mail.10086.cn/s" +) + +type credentialState int + +const ( + credentialStateAuthorization credentialState = iota + credentialStateFullLogin + credentialStateCookiesOnly ) // do others that not defined in Driver interface func (d *Yun139) isFamily() bool { - return d.Type == "family" + return d.Type == MetaFamily +} + +func (d *Yun139) isGroup() bool { + return d.Type == MetaGroup +} + +func (d *Yun139) isShare() bool { + return d.Type == MetaShare } func encodeURIComponent(str string) string { @@ -75,29 +104,28 @@ func (d *Yun139) refreshToken() error { } decode, err := base64.StdEncoding.DecodeString(d.Authorization) if err != nil { - return fmt.Errorf("authorization decode failed: %s", err) + return d.loginAfterAuthorizationFailure(fmt.Errorf("authorization decode failed: %w", err)) } decodeStr := string(decode) splits := strings.Split(decodeStr, ":") if len(splits) < 3 { - return fmt.Errorf("authorization is invalid, splits < 3") + return d.loginAfterAuthorizationFailure(errors.New("authorization is invalid, splits < 3")) } d.Account = splits[1] strs := strings.Split(splits[2], "|") if len(strs) < 4 { - return fmt.Errorf("authorization is invalid, strs < 4") + return d.loginAfterAuthorizationFailure(errors.New("authorization is invalid, strs < 4")) } expiration, err := strconv.ParseInt(strs[3], 10, 64) if err != nil { - return fmt.Errorf("authorization is invalid") + return d.loginAfterAuthorizationFailure(errors.New("authorization expiration is invalid")) } expiration -= time.Now().UnixMilli() if expiration > 1000*60*60*24*15 { - // Authorization有效期大于15天无需刷新 return nil } if expiration < 0 { - return fmt.Errorf("authorization has expired") + return d.loginAfterAuthorizationFailure(errors.New("authorization has expired")) } url := "https://aas.caiyun.feixin.10086.cn:443/tellin/authTokenRefresh.do" @@ -109,13 +137,7 @@ func (d *Yun139) refreshToken() error { SetResult(&resp). Post(url) if err != nil || resp.Return != "0" { - log.Warnf("139yun: failed to refresh token with old token: %v, desc: %s. trying to login with password.", err, resp.Desc) - newAuth, loginErr := d.loginWithPassword() - log.Debugf("newAuth: Ok: %s", newAuth) - if loginErr != nil { - return fmt.Errorf("failed to login with password after refresh failed: %w", loginErr) - } - return nil + return d.loginAfterAuthorizationFailure(fmt.Errorf("token refresh failed: %v, desc: %s", err, resp.Desc)) } d.Authorization = base64.StdEncoding.EncodeToString([]byte(splits[0] + ":" + splits[1] + ":" + resp.Token)) @@ -123,6 +145,18 @@ func (d *Yun139) refreshToken() error { return nil } +// loginAfterAuthorizationFailure deliberately skips cookie fast login. Mail +// cookies are only reused as device context for the password login request. +func (d *Yun139) loginAfterAuthorizationFailure(cause error) error { + log.Warnf("139yun: %v; trying password login.", cause) + newAuth, err := d.loginWithPassword() + log.Debugf("139yun: password fallback generated authorization: %t", newAuth != "") + if err != nil { + return fmt.Errorf("%v; password login failed: %w", cause, err) + } + return nil +} + func (d *Yun139) request(url string, method string, callback base.ReqCallback, resp interface{}) ([]byte, error) { req := base.RestyClient.R() randStr := random.String(16) @@ -339,15 +373,18 @@ func (d *Yun139) familyGetFiles(catalogID string) ([]model.Obj, error) { }, "sortDirection": 1, }) + // 传入 catalogID 是文件夹的ID，而不是完整路径 + // 当传入catalogID为家庭云根目录时，直接留空 + if catalogID == d.ProviderRoot { + data["catalogID"] = "" + } var resp QueryContentListResp _, err := d.post("/orchestration/familyCloud-rebuild/content/v1.2/queryContentList", data, &resp) if err != nil { return nil, err } + // 返回的是完整的Path: root:///.../ path := resp.Data.Path - if catalogID == d.RootFolderID { - d.RootPath = path - } for _, catalog := range resp.Data.CloudCatalogList { f := model.Object{ ID: catalog.CatalogID, @@ -403,9 +440,6 @@ func (d *Yun139) groupGetFiles(catalogID string) ([]model.Obj, error) { return nil, err } path := resp.Data.GetGroupContentResult.ParentCatalogID - if catalogID == d.RootFolderID { - d.RootPath = path - } for _, catalog := range resp.Data.GetGroupContentResult.CatalogList { f := model.Object{ ID: catalog.CatalogID, @@ -485,14 +519,349 @@ func (d *Yun139) groupGetLink(contentId string, path string) (string, error) { return jsoniter.Get(res, "data", "downloadURL").ToString(), nil } +var shareAesKeyHex = hex.EncodeToString([]byte("PVGDwmcvfs1uV3d1")) + +func (d *Yun139) shareHeaders() map[string]string { + auth := d.getAuthorization() + if auth != "" && !strings.HasPrefix(strings.ToLower(auth), "basic ") { + auth = "Basic " + auth + } + headers := map[string]string{ + "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:140.0) Gecko/20100101 Firefox/140.0", + "Accept": "application/json, text/plain, */*", + "Content-Type": "application/json;charset=UTF-8", + "X-Deviceinfo": "||9|12.27.0|firefox|140.0|||linux unknown|1920X526|zh-CN|||", + "hcy-cool-flag": "1", + "CMS-DEVICE": "default", + "x-m4c-caller": "PC", + "X-Yun-Api-Version": "v1", + "Origin": "https://yun.139.com", + "Referer": "https://yun.139.com/", + } + if auth != "" { + headers["Authorization"] = auth + } + return headers +} + +func (d *Yun139) sharePost(pathname string, data interface{}, resp interface{}) ([]byte, error) { + url := "https://share-kd-njs.yun.139.com/yun-share" + pathname + return d.yun139EncryptedRequest(url, data, d.shareHeaders(), shareAesKeyHex, resp) +} + +type shareRef struct { + LinkID string + Password string + NodeID string +} + +const multiShareRefPrefix = "shares:" + +func encodeShareRef(linkID, password, nodeID string) string { + return url.PathEscape(linkID) + "|" + url.PathEscape(password) + "|" + url.PathEscape(nodeID) +} + +func decodeShareRef(id string) (shareRef, bool) { + parts := strings.SplitN(id, "|", 3) + if len(parts) != 3 { + return shareRef{}, false + } + linkID, err1 := url.PathUnescape(parts[0]) + password, err2 := url.PathUnescape(parts[1]) + nodeID, err3 := url.PathUnescape(parts[2]) + if err1 != nil || err2 != nil || err3 != nil || linkID == "" { + return shareRef{}, false + } + return shareRef{LinkID: linkID, Password: password, NodeID: nodeID}, true +} + +func encodeShareRefs(refs []shareRef) string { + if len(refs) == 1 { + ref := refs[0] + return encodeShareRef(ref.LinkID, ref.Password, ref.NodeID) + } + data, err := utils.Json.Marshal(refs) + if err != nil { + return "" + } + return multiShareRefPrefix + base64.RawURLEncoding.EncodeToString(data) +} + +func decodeShareRefs(id string) ([]shareRef, bool) { + if !strings.HasPrefix(id, multiShareRefPrefix) { + ref, ok := decodeShareRef(id) + if !ok { + return nil, false + } + return []shareRef{ref}, true + } + data, err := base64.RawURLEncoding.DecodeString(strings.TrimPrefix(id, multiShareRefPrefix)) + if err != nil { + return nil, false + } + var refs []shareRef + if err = utils.Json.Unmarshal(data, &refs); err != nil || len(refs) == 0 { + return nil, false + } + return refs, true +} + +func (d *Yun139) shareEntries() []struct{ LinkID, Password string } { + raw := strings.TrimSpace(d.LinkID) + if raw == "" { + return nil + } + parts := strings.FieldsFunc(raw, func(r rune) bool { + return r == '\n' || r == '\r' || r == ',' || r == ';' + }) + entries := make([]struct{ LinkID, Password string }, 0, len(parts)) + for _, part := range parts { + part = strings.TrimSpace(part) + if part == "" { + continue + } + entry := struct{ LinkID, Password string }{LinkID: part} + if linkID, password, ok := strings.Cut(part, "#"); ok { + entry.LinkID = strings.TrimSpace(linkID) + entry.Password = strings.TrimSpace(password) + } + if entry.LinkID != "" { + entries = append(entries, entry) + } + } + return entries +} + +func (d *Yun139) shareRootEntries() []shareRef { + entries := d.shareEntries() + refs := make([]shareRef, 0, len(entries)) + for _, entry := range entries { + refs = append(refs, shareRef{LinkID: entry.LinkID, Password: entry.Password, NodeID: "root"}) + } + return refs +} + +func (d *Yun139) shareGetFilesWithRef(ref shareRef, pCaID string) ([]model.Obj, error) { + if ref.NodeID == "" { + ref.NodeID = "root" + } + if pCaID == "" { + pCaID = ref.NodeID + } + data := base.Json{ + "getOutLinkInfoReq": base.Json{ + "account": d.getAccount(), + "linkID": ref.LinkID, + "passwd": ref.Password, + "pCaID": pCaID, + }, + } + var resp ShareListResp + _, err := d.sharePost("/richlifeApp/devapp/IOutLink/getOutLinkInfoV6", data, &resp) + if err != nil { + return nil, err + } + files := make([]model.Obj, 0) + for _, catalog := range resp.Data.CaLst { + modTime, _ := time.ParseInLocation("20060102150405", catalog.UdTime, utils.CNLoc) + f := model.Object{ + ID: encodeShareRef(ref.LinkID, ref.Password, catalog.CaID), + Name: catalog.CaName, + Modified: modTime, + IsFolder: true, + } + files = append(files, &f) + } + for _, content := range resp.Data.CoLst { + name := content.CoName + size := content.CoSize + modTime, _ := time.ParseInLocation("20060102150405", content.UdTime, utils.CNLoc) + f := model.Object{ + ID: encodeShareRef(ref.LinkID, ref.Password, content.CoID), + Name: name, + Size: size, + Modified: modTime, + } + files = append(files, &f) + } + + return files, nil +} + +func (d *Yun139) shareGetMergedFiles(refs []shareRef) ([]model.Obj, error) { + files := make([]model.Obj, 0) + indices := make(map[string]int) + var firstErr error + for _, ref := range refs { + items, err := d.shareGetFilesWithRef(ref, ref.NodeID) + if err != nil { + if firstErr == nil { + firstErr = err + } + continue + } + for _, item := range items { + idx, exists := indices[item.GetName()] + if !exists { + indices[item.GetName()] = len(files) + files = append(files, item) + continue + } + if !files[idx].IsDir() || !item.IsDir() { + continue + } + existingRefs, ok1 := decodeShareRefs(files[idx].GetID()) + itemRefs, ok2 := decodeShareRefs(item.GetID()) + if !ok1 || !ok2 { + continue + } + mergedRefs := append(existingRefs, itemRefs...) + files[idx] = &model.Object{ + ID: encodeShareRefs(mergedRefs), + Name: item.GetName(), + Modified: item.ModTime(), + IsFolder: true, + } + } + } + if len(files) == 0 && firstErr != nil { + return nil, firstErr + } + return files, nil +} + +func (d *Yun139) shareGetObj(reqPath string) (model.Obj, error) { + reqPath = utils.FixAndCleanPath(reqPath) + if reqPath == "/" { + return &model.Object{ID: "root", Name: "root", IsFolder: true, Path: "/"}, nil + } + parts := strings.Split(strings.Trim(reqPath, "/"), "/") + currentRefs := d.shareRootEntries() + currentPath := "" + var currentObj model.Obj + for idx, part := range parts { + items, err := d.shareGetMergedFiles(currentRefs) + if err != nil { + return nil, err + } + matched := false + for _, item := range items { + if item.GetName() != part { + continue + } + matched = true + currentObj = item + currentPath = path.Join(currentPath, item.GetName()) + if item.IsDir() { + itemRefs, ok := decodeShareRefs(item.GetID()) + if !ok { + return nil, errs.ObjectNotFound + } + currentRefs = itemRefs + break + } + if idx != len(parts)-1 { + return nil, errs.ObjectNotFound + } + } + if !matched { + return nil, errs.ObjectNotFound + } + } + if currentObj == nil { + return nil, errs.ObjectNotFound + } + if setter, ok := currentObj.(model.SetPath); ok { + setter.SetPath(currentPath) + } + return currentObj, nil +} + +func (d *Yun139) shareGetLinkWithRef(ref shareRef, coID string, linkType string) (*model.Link, error) { + data := base.Json{ + "getContentInfoFromOutLinkReq": base.Json{ + "contentId": coID, + "linkID": ref.LinkID, + "passwd": ref.Password, + "account": d.getAccount(), + }, + } + var resp ShareContentInfoResp + body, err := d.sharePost("/richlifeApp/devapp/IOutLink/getContentInfoFromOutLink", data, &resp) + if err != nil { + return nil, err + } + + res := resp.Data.ContentInfo + if linkType == "video_preview" || linkType == "preview" || linkType == "thumb" { + if res.PresentURL == "" { + return nil, fmt.Errorf("failed to get preview link") + } + return &model.Link{URL: res.PresentURL}, nil + } + + if d.getAccount() == "" { + if res.PresentURL != "" { + return &model.Link{URL: res.PresentURL}, nil + } + return nil, fmt.Errorf("139 share download requires account authentication") + } + + downloadReq := base.Json{ + "dlFromOutLinkReqV3": base.Json{ + "account": d.getAccount(), + "linkID": ref.LinkID, + "passwd": ref.Password, + "coIDLst": base.Json{ + "item": []string{coID}, + }, + }, + } + var downloadResp ShareDownloadResp + downloadBody, err := d.sharePost("/richlifeApp/devapp/IOutLink/dlFromOutLinkV3", downloadReq, &downloadResp) + if err != nil { + return nil, err + } + if downloadResp.Data.ExtInfo.CDNDownloadURL != "" { + return &model.Link{URL: downloadResp.Data.ExtInfo.CDNDownloadURL}, nil + } + if downloadResp.Data.RedrURL != "" { + return &model.Link{URL: downloadResp.Data.RedrURL}, nil + } + if downloadResp.Data.DownloadURL != "" { + return &model.Link{URL: downloadResp.Data.DownloadURL}, nil + } + + log.Debugf("[139Share] content info without embedded download url: %s", string(body)) + log.Debugf("[139Share] download response without direct url: %s", string(downloadBody)) + return nil, fmt.Errorf("failed to get link") +} + func unicode(str string) string { textQuoted := strconv.QuoteToASCII(str) textUnquoted := textQuoted[1 : len(textQuoted)-1] return textUnquoted } -func (d *Yun139) personalRequest(pathname string, method string, callback base.ReqCallback, resp interface{}) ([]byte, error) { - url := d.getPersonalCloudHost() + pathname +// Client types reported in the X-Yun-Client-Info header. The web client is the +// default with a 50 GB single file upload limit, while the PC client raises the +// limit to 500 GB. +const ( + yunClientTypeWeb = "9" + yunClientTypePC = "13" +) + +// yunClientInfo returns the X-Yun-Client-Info header value for the configured +// client_type addition. Anything other than pc falls back to the web client. +func (d *Yun139) yunClientInfo() string { + clientType := yunClientTypeWeb + if d.ClientType == "pc" { + clientType = yunClientTypePC + } + return "||" + clientType + "|7.14.0|chrome|120.0.0.0|||windows 10||zh-CN|||dW5kZWZpbmVk||" +} + +func (d *Yun139) newRequest(url string, method string, callback base.ReqCallback, resp interface{}) ([]byte, error) { req := base.RestyClient.R() randStr := random.String(16) ts := time.Now().Format("2006-01-02 15:04:05") @@ -527,7 +896,7 @@ func (d *Yun139) personalRequest(pathname string, method string, callback base.R "X-Yun-Api-Version": "v1", "X-Yun-App-Channel": "10000034", "X-Yun-Channel-Source": "10000034", - "X-Yun-Client-Info": "||9|7.14.0|chrome|120.0.0.0|||windows 10||zh-CN|||dW5kZWZpbmVk||", + "X-Yun-Client-Info": d.yunClientInfo(), "X-Yun-Module-Type": "100", "X-Yun-Svc-Type": "1", }) @@ -554,7 +923,21 @@ func (d *Yun139) personalRequest(pathname string, method string, callback base.R } func (d *Yun139) personalPost(pathname string, data interface{}, resp interface{}) ([]byte, error) { - return d.personalRequest(pathname, http.MethodPost, func(req *resty.Request) { + return d.newRequest(d.getPersonalCloudHost()+pathname, http.MethodPost, func(req *resty.Request) { + req.SetBody(data) + }, resp) +} + +func (d *Yun139) newPost(pathname string, data interface{}, resp interface{}) ([]byte, error) { + var url string + switch d.Type { + case MetaFamily, MetaGroup: + // this is on purpose + url = d.getGroupCloudHost() + pathname + default: + url = d.getPersonalCloudHost() + pathname + } + return d.newRequest(url, http.MethodPost, func(req *resty.Request) { req.SetBody(data) }, resp) } @@ -681,7 +1064,21 @@ func (d *Yun139) getPersonalCloudHost() string { return d.PersonalCloudHost } -func (d *Yun139) uploadPersonalParts(ctx context.Context, partInfos []PartInfo, uploadPartInfos []PersonalPartInfo, rateLimited *driver.RateLimitReader, p *driver.Progress) error { +func (d *Yun139) getFamilyCloudHost() string { + if d.ref != nil { + return d.ref.getFamilyCloudHost() + } + return d.FamilyCloudHost +} + +func (d *Yun139) getGroupCloudHost() string { + if d.ref != nil { + return d.ref.getGroupCloudHost() + } + return d.GroupCloudHost +} + +func (d *Yun139) uploadPersonalParts(ctx context.Context, partInfos []PartInfo, uploadPartInfos []PersonalPartInfo, ss streamPkg.StreamSectionReader, p *driver.Progress) error { // 确保数组以 PartNumber 从小到大排序 sort.Slice(uploadPartInfos, func(i, j int) bool { return uploadPartInfos[i].PartNumber < uploadPartInfos[j].PartNumber @@ -693,31 +1090,51 @@ func (d *Yun139) uploadPersonalParts(ctx context.Context, partInfos []PartInfo, return fmt.Errorf("invalid PartNumber %d: index out of bounds (partInfos length: %d)", uploadPartInfo.PartNumber, len(partInfos)) } partSize := partInfos[index].PartSize + offset := partInfos[index].ParallelHashCtx.PartOffset log.Debugf("[139] uploading part %+v/%+v", index, len(partInfos)) - limitReader := io.LimitReader(rateLimited, partSize) - r := io.TeeReader(limitReader, p) - req, err := http.NewRequestWithContext(ctx, http.MethodPut, uploadPartInfo.UploadUrl, r) - if err != nil { - return err + + rd, getErr := ss.GetSectionReader(offset, partSize) + if getErr != nil { + return getErr } - req.Header.Set("Content-Type", "application/octet-stream") - req.Header.Set("Content-Length", fmt.Sprint(partSize)) - req.Header.Set("Origin", "https://yun.139.com") - req.Header.Set("Referer", "https://yun.139.com/") - req.ContentLength = partSize - err = func() error { - res, err := base.HttpClient.Do(req) - if err != nil { - return err - } - defer res.Body.Close() - log.Debugf("[139] uploaded: %+v", res) - if res.StatusCode != http.StatusOK { - body, _ := io.ReadAll(res.Body) - return fmt.Errorf("unexpected status code: %d, body: %s", res.StatusCode, string(body)) - } - return nil - }() + + // Save progress before this part so retries don't double-count bytes + partDoneStart := p.Done + err := retry.Do( + func() error { + // Reset progress to the start of this part on each attempt + p.Done = partDoneStart + if _, seekErr := rd.Seek(0, io.SeekStart); seekErr != nil { + return seekErr + } + req, reqErr := http.NewRequestWithContext(ctx, http.MethodPut, uploadPartInfo.UploadUrl, io.TeeReader(rd, p)) + if reqErr != nil { + return reqErr + } + req.Header.Set("Content-Type", "application/octet-stream") + req.Header.Set("Content-Length", fmt.Sprint(partSize)) + req.Header.Set("Origin", "https://yun.139.com") + req.Header.Set("Referer", "https://yun.139.com/") + req.ContentLength = partSize + + res, doErr := base.HttpClient.Do(req) + if doErr != nil { + return doErr + } + defer res.Body.Close() + log.Debugf("[139] uploaded: %+v", res) + if res.StatusCode != http.StatusOK { + body, _ := io.ReadAll(res.Body) + return fmt.Errorf("unexpected status code: %d, body: %s", res.StatusCode, string(body)) + } + return nil + }, + retry.Context(ctx), + retry.Attempts(3), + retry.DelayType(retry.BackOffDelay), + retry.Delay(time.Second), + ) + ss.FreeSectionReader(rd) if err != nil { return err } @@ -740,15 +1157,212 @@ func (d *Yun139) getDiskQuotaDetail(ctx context.Context) (*DiskQuotaDetail, erro return &resp, nil } +type mailLoginResponse struct { + Code string `json:"code"` + Summary string `json:"summary"` + Var struct { + LoginSuccessURL string `json:"loginSuccessUrl"` + } `json:"var"` +} + +func parseMailLoginResponse(body []byte) mailLoginResponse { + var response mailLoginResponse + if utils.Json.Unmarshal(body, &response) == nil && (response.Code != "" || response.Summary != "" || response.Var.LoginSuccessURL != "") { + return response + } + if match := regexp.MustCompile(`['"]?code['"]?\s*:\s*['"]([^'"]+)`).FindSubmatch(body); len(match) == 2 { + response.Code = string(match[1]) + } + if match := regexp.MustCompile(`['"]?summary['"]?\s*:\s*['"]([^'"]+)`).FindSubmatch(body); len(match) == 2 { + response.Summary = string(match[1]) + } + if match := regexp.MustCompile(`['"]?loginSuccessUrl['"]?\s*:\s*['"]([^'"]+)`).FindSubmatch(body); len(match) == 2 { + response.Var.LoginSuccessURL = string(match[1]) + } + return response +} + +func mergeMailCookieHeader(existing string, responseCookies []*http.Cookie) string { + cookies := cookiepkg.Parse(existing) + for _, responseCookie := range responseCookies { + if responseCookie != nil && responseCookie.Name != "" { + cookies = cookiepkg.SetCookie(cookies, responseCookie.Name, responseCookie.Value) + } + } + return cookiepkg.ToString(cookies) +} + +func mailRiskCode(location string) string { + parsed, err := url.Parse(location) + if err != nil { + return "" + } + return parsed.Query().Get("ec") +} + +func smsSceneForRisk(riskCode string) (int, bool) { + switch riskCode { + case "PML401010062": + return 2, true + case "MW0016": + return 4, true + case "S025", "S035": + return 1, true + default: + return 0, false + } +} + +func encryptMailLoginName(account string) (string, error) { + der, err := base64.StdEncoding.DecodeString(mailPublicKey) + if err != nil { + return "", fmt.Errorf("decode mail public key: %w", err) + } + parsed, err := x509.ParsePKIXPublicKey(der) + if err != nil { + return "", fmt.Errorf("parse mail public key: %w", err) + } + publicKey, ok := parsed.(*rsa.PublicKey) + if !ok { + return "", errors.New("mail public key is not RSA") + } + encrypted, err := rsa.EncryptPKCS1v15(crypto_rand.Reader, publicKey, []byte(account)) + if err != nil { + return "", fmt.Errorf("encrypt mail login name: %w", err) + } + return base64.StdEncoding.EncodeToString(encrypted), nil +} + +func mailXMLHeaders(cookie string) map[string]string { + return map[string]string{ + "Cookie": cookie, + "Content-Type": "application/xml; charset=utf-8", + "Accept-Encoding": "gzip", + "User-Agent": "okhttp/4.12.0", + } +} + +func new139RestyClient() *resty.Client { + if base.RestyClient != nil { + return base.RestyClient.Clone() + } + return base.NewRestyClient() +} + +func (d *Yun139) sendSMSVerificationCode(riskCode string) error { + scene, ok := smsSceneForRisk(riskCode) + if !ok { + return fmt.Errorf("139 Mail risk control does not support SMS verification: %s", riskCode) + } + loginName, err := encryptMailLoginName(d.Username) + if err != nil { + return err + } + body := strings.Join([]string{ + "", + mailXMLField("loginName", loginName), + mailXMLField("fv", "4"), + mailXMLField("clientId", "1003"), + mailXMLField("eMode", "1"), + mailXMLField("loginFailureUrl", ""), + mailXMLField("loginSuccessUrl", ""), + mailXMLField("verifyCode", ""), + mailXMLField("version", "1.0"), + mailXMLField("scene", strconv.Itoa(scene)), + "", + }, "") + res, err := new139RestyClient().SetRetryCount(0).R(). + SetHeaders(mailXMLHeaders(d.MailCookies)). + SetBody(body). + Post(mailSMSURL + "?func=" + url.QueryEscape("login:sendSmsCodeByScene") + "&cguid=" + strconv.FormatInt(time.Now().UnixMilli(), 10)) + if err != nil { + return fmt.Errorf("send 139 Mail SMS verification code: %w", err) + } + d.MailCookies = mergeMailCookieHeader(d.MailCookies, res.Cookies()) + response := parseMailLoginResponse(res.Body()) + if response.Code == "S_OK" { + return nil + } + switch response.Code { + case "PML401010021", "PML401010022": + return fmt.Errorf("139 Mail requires picture verification before SMS can be sent: %s", response.Code) + case "PML404010001": + return errors.New("139 Mail SMS verification code was requested too frequently") + case "PML401010002": + return errors.New("139 Mail rejected the SMS verification parameters") + default: + return fmt.Errorf("send 139 Mail SMS verification code failed: code=%s summary=%s", response.Code, response.Summary) + } +} + +func (d *Yun139) verifySMSCode(riskCode string) (string, error) { + if _, ok := smsSceneForRisk(riskCode); !ok { + return "", fmt.Errorf("139 Mail risk control does not support SMS verification: %s", riskCode) + } + loginName, err := encryptMailLoginName(d.Username) + if err != nil { + return "", err + } + pwdType := "" + if riskCode == "MW0016" { + pwdType = mailXMLField("pwdType", "1") + } + body := strings.Join([]string{ + "", + mailXMLField("clientId", "1003"), + mailXMLField("version", "4"), + mailXMLField("loginType", "0"), + mailXMLField("authType", "2"), + mailXMLField("loginName", loginName), + mailXMLField("eMode", "1"), + mailXMLField("loginPassword", sha1Hash("fetion.com.cn:"+d.SmsCode)), + mailXMLField("createAutoLoginSecretKey", "1"), + mailXMLField("verifyCode", ""), + mailXMLField("verifyAgentId", ""), + mailXMLField("reqFrom", "3"), + mailXMLField("needWCookie", "1"), + pwdType, + "", + }, "") + res, err := new139RestyClient().SetRetryCount(0).R(). + SetHeaders(mailXMLHeaders(d.MailCookies)). + SetBody(body). + Post(mailSMSURL + "?func=" + url.QueryEscape("/login/inlogin.action") + "&cguid=" + strconv.FormatInt(time.Now().UnixMilli(), 10)) + if err != nil { + return "", fmt.Errorf("verify 139 Mail SMS code: %w", err) + } + d.MailCookies = mergeMailCookieHeader(d.MailCookies, res.Cookies()) + response := parseMailLoginResponse(res.Body()) + if response.Code != "S_OK" { + return "", fmt.Errorf("verify 139 Mail SMS code failed: code=%s summary=%s", response.Code, response.Summary) + } + sid := "" + if response.Var.LoginSuccessURL != "" { + if successURL, parseErr := url.Parse(response.Var.LoginSuccessURL); parseErr == nil { + sid = successURL.Query().Get("sid") + } + } + if sid == "" { + for _, cookie := range cookiepkg.Parse(d.MailCookies) { + if cookie.Name == "Os_SSo_Sid" || cookie.Name == "sid" { + sid = cookie.Value + break + } + } + } + if sid == "" { + return "", errors.New("139 Mail SMS verification succeeded but did not return sid") + } + d.SmsCode = "" + return sid, nil +} + func (d *Yun139) step1_password_login() (string, error) { log.Debugf("--- 执行步骤 1: 登录 API ---") - loginURL := "https://mail.10086.cn/Login/Login.ashx" + loginURL := mailPasswordURL // 密码 SHA1 哈希 hashedPassword := sha1Hash(fmt.Sprintf("fetion.com.cn:%s", d.Password)) - log.Debugf("DEBUG: 原始密码: %s", d.Password) - log.Debugf("DEBUG: SHA1 输入: fetion.com.cn:%s", d.Password) - log.Debugf("DEBUG: 生成的 Password 哈希: %s", hashedPassword) cguid := strconv.FormatInt(time.Now().UnixMilli(), 10) // 随机生成 cguid @@ -784,79 +1398,56 @@ func (d *Yun139) step1_password_login() (string, error) { loginData.Set("authType", "2") log.Debugf("DEBUG: 登录请求 URL: %s", loginURL) - log.Debugf("DEBUG: 登录请求 Headers: %+v", loginHeaders) - log.Debugf("DEBUG: 登录请求 Body: %s", loginData.Encode()) + log.Debugf("DEBUG: 登录请求已准备") - // 设置客户端不跟随重定向 - client := base.RestyClient.SetRedirectPolicy(resty.NoRedirectPolicy()) - res, err := client.R(). + res, err := new139RestyClient(). + SetRetryCount(0). + SetRedirectPolicy(resty.RedirectPolicyFunc(func(_ *http.Request, _ []*http.Request) error { + return http.ErrUseLastResponse + })).R(). SetHeaders(loginHeaders). SetFormDataFromValues(loginData). Post(loginURL) - if err != nil { - // 如果是重定向错误，则不作为失败处理，因为我们禁止了自动重定向 - if res != nil && res.StatusCode() >= 300 && res.StatusCode() < 400 { - log.Debugf("DEBUG: 登录响应 Status Code: %d (Redirect)", res.StatusCode()) - } else { - return "", fmt.Errorf("step1 login request failed: %w", err) - } - } else { - log.Debugf("DEBUG: 登录响应 Status Code: %d", res.StatusCode()) + return "", fmt.Errorf("step1 login request failed: %w", err) } - // 恢复客户端的默认重定向策略，以免影响后续请求 - base.RestyClient.SetRedirectPolicy(resty.FlexibleRedirectPolicy(10)) - log.Debugf("DEBUG: 登录响应 Headers: %+v", res.Header()) + log.Debugf("DEBUG: 登录响应 Status Code: %d", res.StatusCode()) + log.Debugf("DEBUG: 登录响应 Location present: %t", res.Header().Get("Location") != "") - var sid, extractedCguid string + d.MailCookies = mergeMailCookieHeader(d.MailCookies, res.Cookies()) - // 从 Location 头部提取 sid 和 cguid + sid := "" locationHeader := res.Header().Get("Location") if locationHeader != "" { - sidMatch := regexp.MustCompile(`sid=([^&]+)`).FindStringSubmatch(locationHeader) - cguidMatch := regexp.MustCompile(`cguid=([^&]+)`).FindStringSubmatch(locationHeader) - if len(sidMatch) > 1 { - sid = sidMatch[1] - log.Debugf("DEBUG: 从 Location 提取到 sid: %s", sid) + if riskCode := mailRiskCode(locationHeader); riskCode != "" { + if _, ok := smsSceneForRisk(riskCode); !ok { + return "", fmt.Errorf("139 Mail risk control triggered: %s", riskCode) + } + if strings.TrimSpace(d.SmsCode) == "" { + if sendErr := d.sendSMSVerificationCode(riskCode); sendErr != nil { + return "", sendErr + } + op.MustSaveDriverStorage(d) + return "", errors.New("139 Mail SMS verification code sent; fill sms_code and save the storage again") + } + return d.verifySMSCode(riskCode) } - if len(cguidMatch) > 1 { - extractedCguid = cguidMatch[1] - log.Debugf("DEBUG: 从 Location 提取到 cguid: %s", extractedCguid) + if redirectURL, parseErr := url.Parse(locationHeader); parseErr == nil { + sid = redirectURL.Query().Get("sid") } } - // 如果 Location 中没有，尝试从 Set-Cookie 中提取 - if sid == "" || extractedCguid == "" { - setCookieHeaders := res.Header().Values("Set-Cookie") - for _, cookieStr := range setCookieHeaders { - ssoSidMatch := regexp.MustCompile(`Os_SSo_Sid=([^;]+)`).FindStringSubmatch(cookieStr) - cookieCguidMatch := regexp.MustCompile(`cguid=([^;]+)`).FindStringSubmatch(cookieStr) - if len(ssoSidMatch) > 1 && sid == "" { - sid = ssoSidMatch[1] - log.Debugf("DEBUG: 从 Set-Cookie 提取到 sid: %s", sid) - } - if len(cookieCguidMatch) > 1 && extractedCguid == "" { - extractedCguid = cookieCguidMatch[1] - log.Debugf("DEBUG: 从 Set-Cookie 提取到 cguid: %s", extractedCguid) + if sid == "" { + for _, cookie := range res.Cookies() { + if cookie.Name == "Os_SSo_Sid" || cookie.Name == "sid" { + sid = cookie.Value + break } } } - - if sid == "" || extractedCguid == "" { - return "", errors.New("failed to extract sid or cguid from login response") + if sid == "" { + return "", errors.New("failed to extract sid from login response") } - - // 提取并记录 cookies - loginUrlObj, _ := url.Parse(loginURL) - cookies := base.RestyClient.GetClient().Jar.Cookies(loginUrlObj) - var cookieStrings []string - for _, cookie := range cookies { - cookieStrings = append(cookieStrings, cookie.Name+"="+cookie.Value) - } - cookieStr := strings.Join(cookieStrings, "; ") - log.Debugf("DEBUG: 提取到的 Cookies: %s", cookieStr) - d.MailCookies = cookieStr - return sid, nil } @@ -881,7 +1472,6 @@ func (d *Yun139) step2_get_single_token(sid string) (string, error) { } exchangePassidHeaders := map[string]string{ - "Host": "smsrebuild1.mail.10086.cn", "Cookie": rmkey, "Content-Type": "text/xml; charset=utf-8", "Accept-Encoding": "gzip", @@ -912,6 +1502,16 @@ func (d *Yun139) step2_get_single_token(sid string) (string, error) { return dycpwd, nil } +func mailXMLField(name, value string) string { + return `` + escapeXML(value) + `` +} + +func escapeXML(value string) string { + var escaped bytes.Buffer + _ = xml.EscapeText(&escaped, []byte(value)) + return escaped.String() +} + // --- 辅助函数：加密/解密 --- // sha1Hash 计算 SHA1 哈希值，返回十六进制字符串。 @@ -1156,7 +1756,6 @@ func (d *Yun139) step3_third_party_login(dycpwd string) (string, error) { "x-UserAgent": "android|23116PN5BC|android15|1.2.6|||1440x3200|10246600", "x-DeviceInfo": "4|127.0.0.1|5|1.2.6|Xiaomi|23116PN5BC||02-00-00-00-00-00|android 15|1440x3200|android|||", "Content-Type": "text/plain;charset=UTF-8", - "Host": "user-njs.yun.139.com", "Accept-Encoding": "gzip", "User-Agent": "okhttp/3.12.2", } @@ -1208,9 +1807,126 @@ func (d *Yun139) step3_third_party_login(dycpwd string) (string, error) { return newAuthorization, nil } +func extractFastLoginCookies(mailCookies string) (sid string, rmkey string) { + for _, c := range cookiepkg.Parse(mailCookies) { + switch c.Name { + case "Os_SSo_Sid": + sid = c.Value + case "RMKEY": + rmkey = c.Value + } + if sid != "" && rmkey != "" { + return sid, rmkey + } + } + return sid, rmkey +} + +func hasCookiePair(raw string) bool { + for _, part := range strings.Split(raw, ";") { + name, value, ok := strings.Cut(strings.TrimSpace(part), "=") + if ok && strings.TrimSpace(name) != "" && value != "" { + return true + } + } + return false +} + +func (d *Yun139) tryFastLoginWithCookies() bool { + sid, rmkey := extractFastLoginCookies(d.MailCookies) + if sid == "" || rmkey == "" { + log.Warnf("139yun: fast login skipped, required cookies missing: Os_SSo_Sid=%t RMKEY=%t", sid != "", rmkey != "") + return false + } + + log.Infof("139yun: attempting fast login using existing SID/Cookies (Step 2).") + token, err := d.step2_get_single_token(sid) + if err != nil || token == "" { + log.Warnf("139yun: fast login Step 2 failed: %v", err) + return false + } + + log.Infof("139yun: Step 2 success. Proceeding to Step 3.") + auth, err := d.step3_third_party_login(token) + if err != nil { + log.Warnf("139yun: fast login Step 3 failed: %v", err) + return false + } + + d.Authorization = auth + op.MustSaveDriverStorage(d) + log.Infof("139yun: fast login success (Step 2 -> Step 3).") + return true +} + +func (d *Yun139) validateAndInitCredentials() error { + state, err := d.credentialState() + if err != nil { + return err + } + + switch state { + case credentialStateAuthorization: + // Authorization is refreshed by Init immediately after this helper returns. + log.Debugf("139yun: Authorization exists, skipping initialization login.") + return nil + case credentialStateFullLogin, credentialStateCookiesOnly: + log.Infof("139yun: Authorization missing, attempting login...") + if d.MailCookies != "" && d.tryFastLoginWithCookies() { + return nil + } + + if state == credentialStateCookiesOnly { + return fmt.Errorf("fast login with cookies failed, and cannot fallback to password login (missing username/password)") + } + + log.Infof("139yun: fast login failed or not possible, performing full password login (Step 1).") + _, err := d.loginWithPassword() + if err != nil { + return fmt.Errorf("login with password failed: %w", err) + } + return nil + default: + return fmt.Errorf("unsupported credential state: %d", state) + } +} + +func (d *Yun139) credentialState() (credentialState, error) { + d.Authorization = strings.TrimSpace(d.Authorization) + d.Username = strings.TrimSpace(d.Username) + d.MailCookies = strings.TrimSpace(d.MailCookies) + + if d.Authorization != "" { + if strings.HasPrefix(strings.ToLower(d.Authorization), "basic ") { + return 0, fmt.Errorf("authorization should not include Basic prefix") + } + return credentialStateAuthorization, nil + } + + if d.MailCookies != "" && !hasCookiePair(d.MailCookies) { + return 0, fmt.Errorf("MailCookies format is invalid, please check your configuration") + } + + hasUsername := d.Username != "" + hasPassword := strings.TrimSpace(d.Password) != "" + + if hasUsername != hasPassword { + return 0, fmt.Errorf("username and password must be provided together") + } + if hasUsername { + return credentialStateFullLogin, nil + } + + if d.MailCookies != "" { + return credentialStateCookiesOnly, nil + } + + return 0, fmt.Errorf("authorization is empty and credentials are not provided") +} + func (d *Yun139) loginWithPassword() (string, error) { - if d.Username == "" || d.Password == "" || d.MailCookies == "" { - return "", errors.New("username, password or mail_cookies is empty") + if d.Username == "" || d.Password == "" { + return "", errors.New("username or password is empty") } passId, err := d.step1_password_login() @@ -1237,10 +1953,9 @@ func (d *Yun139) loginWithPassword() (string, error) { } func (d *Yun139) andAlbumRequest(pathname string, body interface{}, resp interface{}) ([]byte, error) { - url := "https://group.yun.139.com/hcy/family/adapter/andAlbum/openApi" + pathname + url := d.getFamilyCloudHost() + "/andAlbum/openApi" + pathname headers := map[string]string{ - "Host": "group.yun.139.com", "authorization": "Basic " + d.getAuthorization(), "x-svctype": "2", "hcy-cool-flag": "1", @@ -1325,6 +2040,21 @@ func (d *Yun139) getGroupRootByCloudID(cloudID string) (string, error) { return "", fmt.Errorf("no root found in group response") } +// dirPath returns the full path for a directory object. +// For family root (Path="" from framework), needs "root:/"+id prefix. +// Non-root objects already have their API path in GetPath() from List responses. +func (d *Yun139) dirPath(dir model.Obj) string { + p := dir.GetPath() + id := dir.GetID() + if p == "" { + if d.isFamily() { + return "root:/" + id + } + return id + } + return path.Join(p, id) +} + // getFamilyRootPath 查询 family 的上层 path（data.path） // 返回值已去除前缀 "root:/"（或 "root:"），直接返回纯 ID 或 path 部分，便于持久化为 RootFolderID。 func (d *Yun139) getFamilyRootPath(cloudID string) (string, error) { diff --git a/drivers/139/util_test.go b/drivers/139/util_test.go new file mode 100644 index 00000000..6514a042 --- /dev/null +++ b/drivers/139/util_test.go @@ -0,0 +1,501 @@ +package _139 + +import ( + "fmt" + "io" + "net/http" + "net/http/httptest" + "os" + "strings" + "testing" + "time" + + "github.com/OpenListTeam/OpenList/v4/drivers/base" + "github.com/go-resty/resty/v2" +) + +func useRetryingTestClient(t *testing.T) { + t.Helper() + oldClient := base.RestyClient + base.RestyClient = resty.New().SetRetryCount(3) + t.Cleanup(func() { + base.RestyClient = oldClient + }) +} + +func TestMergeMailCookiesPreservesExistingOrderAndAppendsNewNames(t *testing.T) { + got := mergeMailCookieHeader("z=zv; behaviorid=b; Os_SSo_Sid=old", []*http.Cookie{ + {Name: "RMKEY", Value: "rm"}, + {Name: "Os_SSo_Sid", Value: "sid"}, + {Name: "a", Value: "av"}, + }) + want := "z=zv;behaviorid=b;Os_SSo_Sid=sid;RMKEY=rm;a=av" + if got != want { + t.Fatalf("mergeMailCookieHeader() = %q, want %q", got, want) + } +} + +func TestExtractFastLoginCookies(t *testing.T) { + sid, rmkey := extractFastLoginCookies("RMKEY=rm; Os_SSo_Sid=sid") + if sid != "sid" || rmkey != "rm" { + t.Fatalf("extractFastLoginCookies() = %q, %q; want sid, rm", sid, rmkey) + } +} + +func TestYunClientInfoClientType(t *testing.T) { + tests := []struct { + name string + clientType string + want string + }{ + { + name: "default web client", + want: "||9|7.14.0|chrome|120.0.0.0|||windows 10||zh-CN|||dW5kZWZpbmVk||", + }, + { + name: "pc client", + clientType: "pc", + want: "||13|7.14.0|chrome|120.0.0.0|||windows 10||zh-CN|||dW5kZWZpbmVk||", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + d := Yun139{Addition: Addition{ClientType: tt.clientType}} + if got := d.yunClientInfo(); got != tt.want { + t.Fatalf("yunClientInfo() = %q, want %q", got, tt.want) + } + }) + } +} + +func TestCredentialState(t *testing.T) { + tests := []struct { + name string + d Yun139 + want credentialState + err bool + }{ + { + name: "authorization", + d: Yun139{Addition: Addition{Authorization: " auth "}}, + want: credentialStateAuthorization, + }, + { + name: "full login", + d: Yun139{Addition: Addition{ + MailCookies: "RMKEY=rm; Os_SSo_Sid=sid", + Username: "user", + Password: "password", + }}, + want: credentialStateFullLogin, + }, + { + name: "full login without initial cookies", + d: Yun139{Addition: Addition{ + Username: "user", + Password: "password", + }}, + want: credentialStateFullLogin, + }, + { + name: "cookies only", + d: Yun139{Addition: Addition{MailCookies: "RMKEY=rm; Os_SSo_Sid=sid"}}, + want: credentialStateCookiesOnly, + }, + { + name: "partial password login", + d: Yun139{Addition: Addition{Username: "user"}}, + err: true, + }, + { + name: "missing credentials", + d: Yun139{}, + err: true, + }, + { + name: "invalid cookie", + d: Yun139{Addition: Addition{MailCookies: "invalid-cookie"}}, + err: true, + }, + { + name: "authorization with basic prefix", + d: Yun139{Addition: Addition{Authorization: "Basic abc"}}, + err: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := tt.d.credentialState() + if tt.err { + if err == nil { + t.Fatal("credentialState() expected error") + } + return + } + if err != nil { + t.Fatalf("credentialState() unexpected error: %v", err) + } + if got != tt.want { + t.Fatalf("credentialState() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestIntegrationLoginObtainsAuthorization(t *testing.T) { + if os.Getenv("OPENLIST_139_INTEGRATION") != "1" { + t.Skip("set OPENLIST_139_INTEGRATION=1 to run live 139Yun login checks") + } + base.RestyClient = resty.New(). + SetHeader("user-agent", base.UserAgent). + SetRetryCount(3). + SetRetryResetReaders(true). + SetTimeout(30 * time.Second) + + username := os.Getenv("OPENLIST_139_USERNAME") + password := os.Getenv("OPENLIST_139_PASSWORD") + mailCookies := os.Getenv("OPENLIST_139_MAIL_COOKIES") + authorization := strings.TrimSpace(os.Getenv("OPENLIST_139_AUTHORIZATION")) + + if authorization != "" { + t.Run("authorization", func(t *testing.T) { + d := Yun139{Addition: Addition{Authorization: authorization}} + state, err := d.credentialState() + if err != nil { + t.Fatalf("credentialState() unexpected error: %v", err) + } + if state != credentialStateAuthorization { + t.Fatalf("credentialState() = %v, want authorization", state) + } + if d.Authorization == "" || strings.HasPrefix(strings.ToLower(d.Authorization), "basic ") { + t.Fatal("authorization should be present without Basic prefix") + } + }) + } + + if mailCookies == "" { + t.Fatal("OPENLIST_139_MAIL_COOKIES is required") + } + + runFastLogin := func(mailCookies string) error { + d := Yun139{Addition: Addition{MailCookies: mailCookies}} + state, err := d.credentialState() + if err != nil { + return fmt.Errorf("credentialState() error: %w", err) + } + if state != credentialStateCookiesOnly { + return fmt.Errorf("credentialState() = %v, want cookies only", state) + } + sid, rmkey := extractFastLoginCookies(d.MailCookies) + if sid == "" || rmkey == "" { + return fmt.Errorf("mail cookies are missing Os_SSo_Sid or RMKEY") + } + token, err := d.step2_get_single_token(sid) + if err != nil { + return fmt.Errorf("step2_get_single_token() error: %w", err) + } + auth, err := d.step3_third_party_login(token) + if err != nil { + return fmt.Errorf("step3_third_party_login() error: %w", err) + } + if auth == "" { + return fmt.Errorf("authorization is empty after fast login") + } + return nil + } + + if username == "" || password == "" { + t.Fatal("OPENLIST_139_USERNAME and OPENLIST_139_PASSWORD are required for password fallback") + } + + var refreshedMailCookies string + var generatedAuthorization string + t.Run("password login fallback", func(t *testing.T) { + d := Yun139{Addition: Addition{ + MailCookies: mailCookies, + Username: username, + Password: password, + }} + state, err := d.credentialState() + if err != nil { + t.Fatalf("credentialState() unexpected error: %v", err) + } + if state != credentialStateFullLogin { + t.Fatalf("credentialState() = %v, want full login", state) + } + passId, err := d.step1_password_login() + if err != nil { + t.Fatalf("step1_password_login() error: %v", err) + } + token, err := d.step2_get_single_token(passId) + if err != nil { + t.Fatalf("step2_get_single_token() error: %v", err) + } + auth, err := d.step3_third_party_login(token) + if err != nil { + t.Fatalf("step3_third_party_login() error: %v", err) + } + d.Authorization = auth + if auth == "" || d.Authorization == "" { + t.Fatal("authorization is empty after password login") + } + generatedAuthorization = auth + refreshedMailCookies = d.MailCookies + }) + + t.Run("authorization generated by password login", func(t *testing.T) { + if generatedAuthorization == "" { + t.Fatal("password login did not generate authorization") + } + d := Yun139{Addition: Addition{Authorization: generatedAuthorization}} + state, err := d.credentialState() + if err != nil { + t.Fatalf("credentialState() unexpected error: %v", err) + } + if state != credentialStateAuthorization { + t.Fatalf("credentialState() = %v, want authorization", state) + } + if strings.HasPrefix(strings.ToLower(d.Authorization), "basic ") { + t.Fatal("authorization should not include Basic prefix") + } + }) + + t.Run("mail cookies fast login from input", func(t *testing.T) { + sid, rmkey := extractFastLoginCookies(mailCookies) + if sid == "" || rmkey == "" { + t.Skip("input mail cookies are missing Os_SSo_Sid or RMKEY") + } + if err := runFastLogin(mailCookies); err != nil { + t.Skipf("input mail cookies are stale or unusable: %v", err) + } + }) + + t.Run("mail cookies fast login after password login", func(t *testing.T) { + if refreshedMailCookies == "" { + t.Fatal("password login did not refresh mail cookies") + } + if err := runFastLogin(refreshedMailCookies); err != nil { + t.Fatalf("refreshed mail cookies fast login failed: %v", err) + } + }) +} + +func TestInvalidAuthorizationDoesNotUseCookieFastLogin(t *testing.T) { + d := Yun139{Addition: Addition{ + Authorization: "not-base64", + MailCookies: "Os_SSo_Sid=sid; RMKEY=rmkey", + }} + + err := d.refreshToken() + if err == nil || !strings.Contains(err.Error(), "password login failed") { + t.Fatalf("refreshToken() error = %v, want password login fallback error", err) + } + if d.Authorization != "not-base64" { + t.Fatalf("Authorization = %q, want original invalid value retained", d.Authorization) + } +} + +func TestSMSSceneForRisk(t *testing.T) { + tests := map[string]int{ + "S025": 1, + "S035": 1, + "PML401010062": 2, + "MW0016": 4, + } + for riskCode, want := range tests { + got, ok := smsSceneForRisk(riskCode) + if !ok || got != want { + t.Fatalf("smsSceneForRisk(%q) = %d, %t; want %d, true", riskCode, got, ok, want) + } + } + if _, ok := smsSceneForRisk("PICTURE_ONLY"); ok { + t.Fatal("smsSceneForRisk() accepted unsupported risk code") + } +} + +func TestSendSMSVerificationCode(t *testing.T) { + useRetryingTestClient(t) + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body, err := io.ReadAll(r.Body) + if err != nil { + t.Fatalf("read request body: %v", err) + } + if r.URL.Query().Get("func") != "login:sendSmsCodeByScene" { + t.Errorf("func = %q", r.URL.Query().Get("func")) + } + if !strings.Contains(string(body), `1`) { + t.Errorf("request body = %s", body) + } + if !strings.Contains(r.Header.Get("Cookie"), "device=fingerprint") { + t.Errorf("Cookie = %q", r.Header.Get("Cookie")) + } + http.SetCookie(w, &http.Cookie{Name: "challenge", Value: "sms"}) + _, _ = io.WriteString(w, `{"code":"S_OK"}`) + })) + defer server.Close() + + oldURL := mailSMSURL + mailSMSURL = server.URL + defer func() { mailSMSURL = oldURL }() + + d := Yun139{Addition: Addition{Username: "18800000000", MailCookies: "device=fingerprint"}} + if err := d.sendSMSVerificationCode("S025"); err != nil { + t.Fatalf("sendSMSVerificationCode() error = %v", err) + } + if !strings.Contains(d.MailCookies, "challenge=sms") { + t.Fatalf("MailCookies = %q", d.MailCookies) + } +} + +func TestSendSMSVerificationCodeStopsAtPictureChallenge(t *testing.T) { + useRetryingTestClient(t) + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = io.WriteString(w, `{"code":"PML401010021"}`) + })) + defer server.Close() + + oldURL := mailSMSURL + mailSMSURL = server.URL + defer func() { mailSMSURL = oldURL }() + + d := Yun139{Addition: Addition{Username: "18800000000"}} + err := d.sendSMSVerificationCode("S025") + if err == nil || !strings.Contains(err.Error(), "requires picture verification") { + t.Fatalf("sendSMSVerificationCode() error = %v", err) + } +} + +func TestVerifySMSCode(t *testing.T) { + useRetryingTestClient(t) + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body, err := io.ReadAll(r.Body) + if err != nil { + t.Fatalf("read request body: %v", err) + } + if r.URL.Query().Get("func") != "/login/inlogin.action" { + t.Errorf("func = %q", r.URL.Query().Get("func")) + } + wantHash := sha1Hash("fetion.com.cn:123456") + if !strings.Contains(string(body), ``+wantHash+``) { + t.Errorf("request body does not contain SMS code hash: %s", body) + } + http.SetCookie(w, &http.Cookie{Name: "RMKEY", Value: "new-rmkey"}) + _, _ = io.WriteString(w, `{"code":"S_OK","var":{"loginSuccessUrl":"https://mail.10086.cn/?sid=sms-sid"}}`) + })) + defer server.Close() + + oldURL := mailSMSURL + mailSMSURL = server.URL + defer func() { mailSMSURL = oldURL }() + + d := Yun139{Addition: Addition{ + Username: "18800000000", + SmsCode: "123456", + MailCookies: "challenge=sms", + }} + sid, err := d.verifySMSCode("S025") + if err != nil { + t.Fatalf("verifySMSCode() error = %v", err) + } + if sid != "sms-sid" { + t.Fatalf("sid = %q", sid) + } + if d.SmsCode != "" { + t.Fatalf("SmsCode = %q, want cleared", d.SmsCode) + } + if !strings.Contains(d.MailCookies, "RMKEY=new-rmkey") { + t.Fatalf("MailCookies = %q", d.MailCookies) + } +} + +func TestPasswordLoginWithoutInitialCookiesStopsAtRedirectAndPersistsCookies(t *testing.T) { + useRetryingTestClient(t) + + var loginRequests, redirectRequests int + var server *httptest.Server + server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/Login/Login.ashx": + loginRequests++ + http.SetCookie(w, &http.Cookie{Name: "RMKEY", Value: "new-rm"}) + http.SetCookie(w, &http.Cookie{Name: "Os_SSo_Sid", Value: "new-sid"}) + w.Header().Set("Location", server.URL+"/appmail?sid=single-sid") + w.WriteHeader(http.StatusFound) + case "/appmail": + redirectRequests++ + w.WriteHeader(http.StatusOK) + default: + http.NotFound(w, r) + } + })) + defer server.Close() + + oldURL := mailPasswordURL + mailPasswordURL = server.URL + "/Login/Login.ashx" + defer func() { mailPasswordURL = oldURL }() + + d := Yun139{Addition: Addition{Username: "18800000000", Password: "password"}} + sid, err := d.step1_password_login() + if err != nil || sid != "single-sid" { + t.Fatalf("sid=%q err=%v", sid, err) + } + if loginRequests != 1 || redirectRequests != 0 { + t.Fatalf("login/redirect requests=%d/%d, want 1/0", loginRequests, redirectRequests) + } + if !strings.Contains(d.MailCookies, "RMKEY=new-rm") || !strings.Contains(d.MailCookies, "Os_SSo_Sid=new-sid") { + t.Fatalf("response cookies were not persisted: %q", d.MailCookies) + } +} + +func TestPasswordLoginReusesRawRefreshedCookies(t *testing.T) { + useRetryingTestClient(t) + + var n int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + n++ + cookie := r.Header.Get("Cookie") + if !strings.Contains(cookie, "JSESSIONID=stale") || !strings.Contains(cookie, "behaviorid=device") { + t.Errorf("login %d lost raw cookie context: %q", n, cookie) + } + if n == 1 { + http.SetCookie(w, &http.Cookie{Name: "RMKEY", Value: "rm-1"}) + http.SetCookie(w, &http.Cookie{Name: "Os_SSo_Sid", Value: "sid-1"}) + w.Header().Set("Location", "https://mail.10086.cn/?sid=first") + } else { + if !strings.Contains(cookie, "RMKEY=rm-1") || !strings.Contains(cookie, "Os_SSo_Sid=sid-1") { + t.Errorf("second login did not reuse first refreshed cookies: %q", cookie) + } + http.SetCookie(w, &http.Cookie{Name: "RMKEY", Value: "rm-2"}) + http.SetCookie(w, &http.Cookie{Name: "Os_SSo_Sid", Value: "sid-2"}) + w.Header().Set("Location", "https://mail.10086.cn/?sid=second") + } + w.WriteHeader(http.StatusFound) + })) + defer server.Close() + + oldURL := mailPasswordURL + mailPasswordURL = server.URL + defer func() { mailPasswordURL = oldURL }() + + d := Yun139{Addition: Addition{Username: "18800000000", Password: "password", MailCookies: "Os_SSo_Sid=old; RMKEY=old; JSESSIONID=stale; behaviorid=device"}} + if sid, err := d.step1_password_login(); err != nil || sid != "first" { + t.Fatalf("first login sid=%q err=%v", sid, err) + } + refreshed := d.MailCookies + if !strings.Contains(refreshed, "RMKEY=rm-1") || !strings.Contains(refreshed, "Os_SSo_Sid=sid-1") { + t.Fatalf("first login did not persist refreshed cookies: %q", refreshed) + } + d.Authorization = "" + d.MailCookies = refreshed + if sid, err := d.step1_password_login(); err != nil || sid != "second" { + t.Fatalf("second login sid=%q err=%v", sid, err) + } + if n != 2 { + t.Fatalf("login count=%d, want 2", n) + } +} diff --git a/drivers/189/util.go b/drivers/189/util.go index a98b0d01..ae168137 100644 --- a/drivers/189/util.go +++ b/drivers/189/util.go @@ -238,7 +238,7 @@ func (d *Cloud189) oldUpload(dstDir model.Obj, file model.FileStreamer) error { if utils.Json.Get(res.Body(), "MD5").ToString() != "" { return nil } - log.Debugf(res.String()) + log.Debug(res.String()) return errors.New(res.String()) } diff --git a/drivers/189_tv/help.go b/drivers/189_tv/help.go index cfd9e18a..1f130c56 100644 --- a/drivers/189_tv/help.go +++ b/drivers/189_tv/help.go @@ -5,12 +5,15 @@ import ( "crypto/hmac" "crypto/sha1" "encoding/hex" + "encoding/json" "encoding/xml" "fmt" "net/http" "regexp" "strings" "time" + + "github.com/OpenListTeam/OpenList/v4/pkg/utils" ) func clientSuffix() map[string]string { @@ -57,7 +60,13 @@ func timestamp() int64 { type Time time.Time -func (t *Time) UnmarshalJSON(b []byte) error { return t.Unmarshal(b) } +func (t *Time) UnmarshalJSON(b []byte) error { + var s string + if err := json.Unmarshal(b, &s); err != nil { + return err + } + return t.Unmarshal([]byte(s)) +} func (t *Time) UnmarshalXML(e *xml.Decoder, ee xml.StartElement) error { b, err := e.Token() if err != nil { @@ -72,10 +81,19 @@ func (t *Time) UnmarshalXML(e *xml.Decoder, ee xml.StartElement) error { } func (t *Time) Unmarshal(b []byte) error { bs := strings.Trim(string(b), "\"") + // 189 时间串里 AM/PM 前可能使用 U+202F 窄不换行空格或 U+00A0 不换行空格，统一替换为普通空格 + bs = strings.ReplaceAll(bs, "\u202f", " ") + bs = strings.ReplaceAll(bs, "\u00a0", " ") var v time.Time var err error - for _, f := range []string{"2006-01-02 15:04:05 -07", "Jan 2, 2006 15:04:05 PM -07"} { - v, err = time.ParseInLocation(f, bs+" +08", time.Local) + // 189 返回的时间可能自带时区（如 "Aug 11, 2026, 10:37:18 PM +08"），也可能不带，分别尝试 + for _, s := range []string{bs, bs + " +08"} { + for _, f := range []string{"2006-01-02 15:04:05 -07", "Jan 2, 2006 3:04:05 PM -07", "Jan 2, 2006, 3:04:05 PM -07"} { + v, err = time.ParseInLocation(f, s, utils.CNLoc) + if err == nil { + break + } + } if err == nil { break } diff --git a/drivers/189_tv/help_test.go b/drivers/189_tv/help_test.go new file mode 100644 index 00000000..6643a5aa --- /dev/null +++ b/drivers/189_tv/help_test.go @@ -0,0 +1,65 @@ +package _189_tv + +import ( + "encoding/json" + "encoding/xml" + "testing" + "time" +) + +func TestTimeUnmarshal(t *testing.T) { + tests := []struct { + name string + input string + want time.Time + }{ + {"numeric date", `"2026-08-11 10:37:18"`, time.Date(2026, 8, 11, 10, 37, 18, 0, time.FixedZone("", 8*3600))}, + {"legacy month date", `"Aug 11, 2026 10:37:18 PM"`, time.Date(2026, 8, 11, 22, 37, 18, 0, time.FixedZone("", 8*3600))}, + {"new format with tz", `"Aug 11, 2026, 10:37:18 PM +08"`, time.Date(2026, 8, 11, 22, 37, 18, 0, time.FixedZone("", 8*3600))}, + {"new format no tz", `"Aug 11, 2026, 10:37:18 PM"`, time.Date(2026, 8, 11, 22, 37, 18, 0, time.FixedZone("", 8*3600))}, + {"narrow no-break space (U+202F)", "\"Aug 12, 2026, 12:35:41\u202fAM +08\"", time.Date(2026, 8, 12, 0, 35, 41, 0, time.FixedZone("", 8*3600))}, + {"no-break space (U+00A0)", "\"Aug 12, 2026, 12:35:41\u00a0AM +08\"", time.Date(2026, 8, 12, 0, 35, 41, 0, time.FixedZone("", 8*3600))}, + {"JSON escaped narrow space", `"Sep 4, 2026, 10:59:33\u202fPM +08"`, time.Date(2026, 9, 4, 22, 59, 33, 0, time.FixedZone("", 8*3600))}, + {"JSON escaped space without timezone", `"Sep 4, 2026, 12:59:33\u00a0AM"`, time.Date(2026, 9, 4, 0, 59, 33, 0, time.FixedZone("", 8*3600))}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var tm Time + if err := json.Unmarshal([]byte(tt.input), &tm); err != nil { + t.Fatalf("Unmarshal(%s) error: %v", tt.input, err) + } + if !tt.want.Equal(time.Time(tm)) { + t.Fatalf("Unmarshal(%s) = %v, want %v", tt.input, time.Time(tm), tt.want) + } + }) + } +} + +func TestTimeUnmarshalRejectsInvalid(t *testing.T) { + var tm Time + if err := tm.Unmarshal([]byte("Aug 12, 2026, 25:32:44 AM")); err == nil { + t.Fatal("Unmarshal accepted an invalid time") + } +} + +func TestTimeUnmarshalJSONRejectsInvalid(t *testing.T) { + for _, input := range []string{`"invalid"`, `123`, `null`, `"Sep 4, 2026, 10:59:33\uZZZZPM +08"`} { + t.Run(input, func(t *testing.T) { + var tm Time + if err := tm.UnmarshalJSON([]byte(input)); err == nil { + t.Fatalf("UnmarshalJSON(%s) accepted invalid input", input) + } + }) + } +} + +func TestTimeUnmarshalXML(t *testing.T) { + var tm Time + if err := xml.Unmarshal([]byte(``), &tm); err != nil { + t.Fatal(err) + } + want := time.Date(2026, 9, 4, 22, 59, 33, 0, time.FixedZone("", 8*3600)) + if !want.Equal(time.Time(tm)) { + t.Fatalf("UnmarshalXML = %v, want %v", time.Time(tm), want) + } +} diff --git a/drivers/189_tv/utils.go b/drivers/189_tv/utils.go index cea9411e..313f1dc5 100644 --- a/drivers/189_tv/utils.go +++ b/drivers/189_tv/utils.go @@ -184,6 +184,7 @@ func (y *Cloud189TV) getFiles(ctx context.Context, fileId string, isFamily bool) } fullUrl += "/listFiles.action" + pageSize := 130 // 每一页返回的文件数量 res := make([]model.Obj, 0, 130) for pageNum := 1; ; pageNum++ { var resp Cloud189FilesResp @@ -195,7 +196,7 @@ func (y *Cloud189TV) getFiles(ctx context.Context, fileId string, isFamily bool) "mediaAttr": "0", "iconOption": "5", "pageNum": fmt.Sprint(pageNum), - "pageSize": "130", + "pageSize": fmt.Sprint(pageSize), }) if isFamily { r.SetQueryParams(map[string]string{ @@ -218,13 +219,22 @@ func (y *Cloud189TV) getFiles(ctx context.Context, fileId string, isFamily bool) if resp.FileListAO.Count == 0 { break } + + FolderCount := len(resp.FileListAO.FolderList) // 当前文件夹总数 + FileCount := len(resp.FileListAO.FileList) // 当前文件总数 + PageCount := FolderCount + FileCount // 当前页数总数 - for i := 0; i < len(resp.FileListAO.FolderList); i++ { + for i := 0; i < FolderCount; i++ { res = append(res, &resp.FileListAO.FolderList[i]) } - for i := 0; i < len(resp.FileListAO.FileList); i++ { + for i := 0; i < FileCount; i++ { res = append(res, &resp.FileListAO.FileList[i]) } + + // 文件数量小于设定数量时跳出 + if PageCount < pageSize { + break + } } return res, nil } diff --git a/drivers/189pc/driver.go b/drivers/189pc/driver.go index f557cefd..549f72ed 100644 --- a/drivers/189pc/driver.go +++ b/drivers/189pc/driver.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "net/http" + "path" "strconv" "strings" "time" @@ -262,6 +263,16 @@ func (y *Cloud189PC) Move(ctx context.Context, srcObj, dstDir model.Obj) (model. if err = y.WaitBatchTask("MOVE", resp.TaskID, time.Millisecond*400); err != nil { return nil, err } + + // 跟随移动 torrent 文件 + if !srcObj.IsDir() { + var srcFolderId string + if f, ok := srcObj.(*Cloud189File); ok { + srcFolderId = f.ParentID + } + y.torrentFollowMove(srcFolderId, srcObj.GetName(), dstDir) + } + return srcObj, nil } @@ -298,6 +309,12 @@ func (y *Cloud189PC) Rename(ctx context.Context, srcObj model.Obj, newName strin } return nil, err } + + // 跟随重命名 torrent 文件 + if f, ok := srcObj.(*Cloud189File); ok { + y.torrentFollowRename(f.ParentID, srcObj.GetName(), newName) + } + switch f := srcObj.(type) { case *Cloud189File: return resp.toFile(f), nil @@ -319,7 +336,20 @@ func (y *Cloud189PC) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { if err != nil { return err } - return y.WaitBatchTask("COPY", resp.TaskID, time.Second) + if err = y.WaitBatchTask("COPY", resp.TaskID, time.Second); err != nil { + return err + } + + // 跟随复制 torrent 文件 + if !srcObj.IsDir() { + var srcFolderId string + if f, ok := srcObj.(*Cloud189File); ok { + srcFolderId = f.ParentID + } + y.torrentFollowCopy(srcFolderId, srcObj.GetName(), dstDir) + } + + return nil } func (y *Cloud189PC) Remove(ctx context.Context, obj model.Obj) error { @@ -337,6 +367,14 @@ func (y *Cloud189PC) Remove(ctx context.Context, obj model.Obj) error { return y.WaitBatchTask("DELETE", resp.TaskID, time.Millisecond*200) } +func familyTransferTempName(srcName string) string { + ext := path.Ext(srcName) + if ext == "" { + ext = ".transfer" + } + return fmt.Sprintf("0%s%s", uuid.NewString(), ext) +} + func (y *Cloud189PC) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) (newObj model.Obj, err error) { overwrite := true isFamily := y.isFamily() @@ -370,7 +408,7 @@ func (y *Cloud189PC) Put(ctx context.Context, dstDir model.Obj, stream model.Fil srcName := stream.GetName() stream = &WrapFileStreamer{ FileStreamer: stream, - Name: fmt.Sprintf("0%s.transfer", uuid.NewString()), + Name: familyTransferTempName(srcName), } // 使用家庭云上传 diff --git a/drivers/189pc/driver_test.go b/drivers/189pc/driver_test.go new file mode 100644 index 00000000..03611f6e --- /dev/null +++ b/drivers/189pc/driver_test.go @@ -0,0 +1,45 @@ +package _189pc + +import ( + "path" + "slices" + "strings" + "testing" +) + +func TestFamilyTransferTempNamePreservesExtension(t *testing.T) { + tests := []struct { + name string + want string + }{ + {name: "photo.jpg", want: ".jpg"}, + {name: "PHOTO.PNG", want: ".PNG"}, + {name: "archive.tar.gz", want: ".gz"}, + {name: "no-extension", want: ".transfer"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := familyTransferTempName(tt.name) + if !strings.HasPrefix(got, "0") { + t.Fatalf("temporary name %q must start with 0", got) + } + if ext := path.Ext(got); ext != tt.want { + t.Fatalf("temporary name extension = %q, want %q", ext, tt.want) + } + }) + } +} + +func TestUploadProgressKeysIncludeExtension(t *testing.T) { + jpgKeys := uploadProgressKeys("session", "same-md5", "photo.jpg") + txtKeys := uploadProgressKeys("session", "same-md5", "photo.txt") + retryKeys := uploadProgressKeys("session", "same-md5", "different-name.jpg") + + if slices.Equal(jpgKeys, txtKeys) { + t.Fatal("files with different extensions must not share upload progress") + } + if !slices.Equal(jpgKeys, retryKeys) { + t.Fatal("files with the same session, MD5, and extension should share upload progress") + } +} diff --git a/drivers/189pc/help.go b/drivers/189pc/help.go index 6f6c59f3..5d695940 100644 --- a/drivers/189pc/help.go +++ b/drivers/189pc/help.go @@ -9,6 +9,7 @@ import ( "crypto/sha1" "crypto/x509" "encoding/hex" + "encoding/json" "encoding/pem" "encoding/xml" "fmt" @@ -19,6 +20,7 @@ import ( "time" "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/OpenListTeam/OpenList/v4/pkg/utils/random" ) @@ -101,7 +103,13 @@ func MustParseTime(str string) *time.Time { type Time time.Time -func (t *Time) UnmarshalJSON(b []byte) error { return t.Unmarshal(b) } +func (t *Time) UnmarshalJSON(b []byte) error { + var s string + if err := json.Unmarshal(b, &s); err != nil { + return err + } + return t.Unmarshal([]byte(s)) +} func (t *Time) UnmarshalXML(e *xml.Decoder, ee xml.StartElement) error { b, err := e.Token() if err != nil { @@ -116,10 +124,19 @@ func (t *Time) UnmarshalXML(e *xml.Decoder, ee xml.StartElement) error { } func (t *Time) Unmarshal(b []byte) error { bs := strings.Trim(string(b), "\"") + // 189 时间串里 AM/PM 前可能使用 U+202F 窄不换行空格或 U+00A0 不换行空格，统一替换为普通空格 + bs = strings.ReplaceAll(bs, "\u202f", " ") + bs = strings.ReplaceAll(bs, "\u00a0", " ") var v time.Time var err error - for _, f := range []string{"2006-01-02 15:04:05 -07", "Jan 2, 2006 15:04:05 PM -07"} { - v, err = time.ParseInLocation(f, bs+" +08", time.Local) + // 189 返回的时间可能自带时区（如 "Aug 11, 2026, 10:37:18 PM +08"），也可能不带，分别尝试 + for _, s := range []string{bs, bs + " +08"} { + for _, f := range []string{"2006-01-02 15:04:05 -07", "Jan 2, 2006 3:04:05 PM -07", "Jan 2, 2006, 3:04:05 PM -07"} { + v, err = time.ParseInLocation(f, s, utils.CNLoc) + if err == nil { + break + } + } if err == nil { break } diff --git a/drivers/189pc/help_test.go b/drivers/189pc/help_test.go new file mode 100644 index 00000000..6c91df12 --- /dev/null +++ b/drivers/189pc/help_test.go @@ -0,0 +1,65 @@ +package _189pc + +import ( + "encoding/json" + "encoding/xml" + "testing" + "time" +) + +func TestTimeUnmarshal(t *testing.T) { + tests := []struct { + name string + input string + want time.Time + }{ + {"numeric date", `"2026-08-11 10:37:18"`, time.Date(2026, 8, 11, 10, 37, 18, 0, time.FixedZone("", 8*3600))}, + {"legacy month date", `"Aug 11, 2026 10:37:18 PM"`, time.Date(2026, 8, 11, 22, 37, 18, 0, time.FixedZone("", 8*3600))}, + {"new format with tz", `"Aug 11, 2026, 10:37:18 PM +08"`, time.Date(2026, 8, 11, 22, 37, 18, 0, time.FixedZone("", 8*3600))}, + {"new format no tz", `"Aug 11, 2026, 10:37:18 PM"`, time.Date(2026, 8, 11, 22, 37, 18, 0, time.FixedZone("", 8*3600))}, + {"narrow no-break space (U+202F)", "\"Aug 12, 2026, 12:35:41\u202fAM +08\"", time.Date(2026, 8, 12, 0, 35, 41, 0, time.FixedZone("", 8*3600))}, + {"no-break space (U+00A0)", "\"Aug 12, 2026, 12:35:41\u00a0AM +08\"", time.Date(2026, 8, 12, 0, 35, 41, 0, time.FixedZone("", 8*3600))}, + {"JSON escaped narrow space", `"Sep 4, 2026, 10:59:33\u202fPM +08"`, time.Date(2026, 9, 4, 22, 59, 33, 0, time.FixedZone("", 8*3600))}, + {"JSON escaped space without timezone", `"Sep 4, 2026, 12:59:33\u00a0AM"`, time.Date(2026, 9, 4, 0, 59, 33, 0, time.FixedZone("", 8*3600))}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var tm Time + if err := json.Unmarshal([]byte(tt.input), &tm); err != nil { + t.Fatalf("Unmarshal(%s) error: %v", tt.input, err) + } + if !tt.want.Equal(time.Time(tm)) { + t.Fatalf("Unmarshal(%s) = %v, want %v", tt.input, time.Time(tm), tt.want) + } + }) + } +} + +func TestTimeUnmarshalRejectsInvalid(t *testing.T) { + var tm Time + if err := tm.Unmarshal([]byte("Aug 12, 2026, 25:32:44 AM")); err == nil { + t.Fatal("Unmarshal accepted an invalid time") + } +} + +func TestTimeUnmarshalJSONRejectsInvalid(t *testing.T) { + for _, input := range []string{`"invalid"`, `123`, `null`, `"Sep 4, 2026, 10:59:33\uZZZZPM +08"`} { + t.Run(input, func(t *testing.T) { + var tm Time + if err := tm.UnmarshalJSON([]byte(input)); err == nil { + t.Fatalf("UnmarshalJSON(%s) accepted invalid input", input) + } + }) + } +} + +func TestTimeUnmarshalXML(t *testing.T) { + var tm Time + if err := xml.Unmarshal([]byte(``), &tm); err != nil { + t.Fatal(err) + } + want := time.Date(2026, 9, 4, 22, 59, 33, 0, time.FixedZone("", 8*3600)) + if !want.Equal(time.Time(tm)) { + t.Fatalf("UnmarshalXML = %v, want %v", time.Time(tm), want) + } +} diff --git a/drivers/189pc/meta.go b/drivers/189pc/meta.go index 3a5e0297..5d9dc350 100644 --- a/drivers/189pc/meta.go +++ b/drivers/189pc/meta.go @@ -7,11 +7,16 @@ import ( type Addition struct { LoginType string `json:"login_type" type:"select" options:"password,qrcode" default:"password" required:"true"` - Username string `json:"username" required:"true"` - Password string `json:"password" required:"true"` + Username string `json:"username" help:"Not needed when an access token or refresh token is provided"` + Password string `json:"password" help:"Not needed when an access token or refresh token is provided"` VCode string `json:"validate_code"` + SmsCode string `json:"sms_code" help:"SMS code for the second device verification, fill it in and save again when login asks for it"` AccessToken string `json:"access_token" required:"false"` RefreshToken string `json:"refresh_token" help:"To switch accounts, please clear this field"` + DeviceID string `json:"device_id" help:"DEVICEID cookie issued after the second device verification, keep it to avoid verifying again"` + ClientSn string `json:"client_sn" help:"Device serial number captured from the official client, leave it empty if you do not have one"` + JgOpenId string `json:"jg_open_id" help:"Optional push id reported by the official client"` + UserFinger string `json:"user_finger" help:"Device fingerprint sent with login requests, generated and kept automatically when empty"` driver.RootID OrderBy string `json:"order_by" type:"select" options:"filename,filesize,lastOpTime" default:"filename"` OrderDirection string `json:"order_direction" type:"select" options:"asc,desc" default:"asc"` diff --git a/drivers/189pc/torrent.go b/drivers/189pc/torrent.go index 8d591cf4..30c195e8 100644 --- a/drivers/189pc/torrent.go +++ b/drivers/189pc/torrent.go @@ -1,6 +1,7 @@ package _189pc import ( + "bytes" "context" "crypto/sha1" "encoding/hex" @@ -8,10 +9,13 @@ import ( "io" "net/url" "strings" + "time" "github.com/go-resty/resty/v2" "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/internal/op" + "github.com/OpenListTeam/OpenList/v4/internal/stream" "github.com/OpenListTeam/OpenList/v4/pkg/torrent" "github.com/OpenListTeam/OpenList/v4/pkg/utils" ) @@ -162,6 +166,36 @@ func (y *Cloud189PC) RapidUploadFromTorrent(ctx context.Context, dstDir model.Ob return nil, fmt.Errorf("提交上传失败: %w", err) } + // 秒传成功后，将 torrent 文件上传到目标目录（异步，不影响秒传结果） + if y.Addition.GenerateTorrent { + capturedDstDir := dstDir + capturedIsFamily := isFamily + go func() { + torrentName := fileName + ".cas.torrent" + infoHash, _ := GetInfoHashHex(torrentData) + utils.Log.Infof("秒传成功，上传 torrent: %s (info_hash: %s, size: %d bytes)", + torrentName, infoHash, len(torrentData)) + + torrentFileStream := &stream.FileStream{ + Ctx: context.Background(), + Obj: &model.Object{ + Name: torrentName, + Size: int64(len(torrentData)), + IsFolder: false, + }, + Reader: bytes.NewReader(torrentData), + Mimetype: "application/x-bittorrent", + } + _, uploadErr := y.fastUpload(context.Background(), capturedDstDir, torrentFileStream, func(p float64) {}, capturedIsFamily, false, false) + if uploadErr != nil { + utils.Log.Warnf("上传 torrent 文件失败: %v", uploadErr) + } else { + utils.Log.Infof("torrent 文件已上传: %s", torrentName) + op.Cache.DeleteDirectory(y, capturedDstDir.GetPath()) + } + }() + } + return resp.toFile(), nil } @@ -258,6 +292,10 @@ func GetInfoHashHex(torrentData []byte) (string, error) { return hex.EncodeToString(t.InfoHash), nil } +func isCASTorrentFile(fileName string) bool { + return strings.HasSuffix(fileName, ".cas.torrent") +} + // ComputeSliceMD5sFromReader 从 reader 中计算每个 10MB 分片的 MD5 // 返回：整文件 MD5、分片 MD5 列表 func ComputeSliceMD5sFromReader(reader io.Reader, sliceSize int64) (string, []string, error) { @@ -289,3 +327,123 @@ func ComputeSliceMD5sFromReader(reader io.Reader, sliceSize int64) (string, []st fileMD5Hex := strings.ToUpper(hex.EncodeToString(fileMD5Hash.Sum(nil))) return fileMD5Hex, sliceMD5s, nil } + +// torrentFollowCopy 跟随复制 torrent 文件（异步，不影响主操作） +// srcFolderId: 源文件所在目录 ID +// srcFileName: 源文件名 +// dstDir: 目标目录 +func (y *Cloud189PC) torrentFollowCopy(srcFolderId string, srcFileName string, dstDir model.Obj) { + if !y.Addition.GenerateTorrent { + return + } + if srcFolderId == "" { + return + } + torrentName := srcFileName + ".cas.torrent" + isFamily := y.isFamily() + + go func() { + torrentFile, err := y.findFileByName(context.Background(), torrentName, srcFolderId, isFamily) + if err != nil { + utils.Log.Debugf("查找 torrent 文件失败（跟随复制）: %v", err) + return + } + // 复制 torrent 文件到目标目录 + resp, copyErr := y.CreateBatchTask("COPY", IF(isFamily, y.FamilyID, ""), dstDir.GetID(), + map[string]string{"targetFileName": dstDir.GetName()}, + BatchTaskInfo{ + FileId: torrentFile.GetID(), + FileName: torrentFile.GetName(), + IsFolder: 0, + }) + if copyErr != nil { + utils.Log.Warnf("跟随复制 torrent 文件失败: %v", copyErr) + return + } + if err = y.WaitBatchTask("COPY", resp.TaskID, time.Second); err != nil { + utils.Log.Warnf("等待跟随复制 torrent 文件失败: %v", err) + } + }() +} + +// torrentFollowMove 跟随移动 torrent 文件（异步，不影响主操作） +// srcFolderId: 源文件所在目录 ID +// srcFileName: 源文件名 +// dstDir: 目标目录 +func (y *Cloud189PC) torrentFollowMove(srcFolderId string, srcFileName string, dstDir model.Obj) { + if !y.Addition.GenerateTorrent { + return + } + if srcFolderId == "" { + return + } + torrentName := srcFileName + ".cas.torrent" + isFamily := y.isFamily() + + go func() { + torrentFile, err := y.findFileByName(context.Background(), torrentName, srcFolderId, isFamily) + if err != nil { + utils.Log.Debugf("查找 torrent 文件失败（跟随移动）: %v", err) + return + } + // 移动 torrent 文件到目标目录 + resp, moveErr := y.CreateBatchTask("MOVE", IF(isFamily, y.FamilyID, ""), dstDir.GetID(), + map[string]string{"targetFileName": dstDir.GetName()}, + BatchTaskInfo{ + FileId: torrentFile.GetID(), + FileName: torrentFile.GetName(), + IsFolder: 0, + }) + if moveErr != nil { + utils.Log.Warnf("跟随移动 torrent 文件失败: %v", moveErr) + return + } + if err = y.WaitBatchTask("MOVE", resp.TaskID, time.Millisecond*400); err != nil { + utils.Log.Warnf("等待跟随移动 torrent 文件失败: %v", err) + } + }() +} + +// torrentFollowRename 跟随重命名 torrent 文件（异步，不影响主操作） +// folderId: 文件所在目录 ID +// oldFileName: 原文件名 +// newFileName: 新文件名 +func (y *Cloud189PC) torrentFollowRename(folderId string, oldFileName string, newFileName string) { + if !y.Addition.GenerateTorrent { + return + } + if folderId == "" { + return + } + oldTorrentName := oldFileName + ".cas.torrent" + newTorrentName := newFileName + ".cas.torrent" + isFamily := y.isFamily() + + go func() { + torrentFile, err := y.findFileByName(context.Background(), oldTorrentName, folderId, isFamily) + if err != nil { + utils.Log.Debugf("查找 torrent 文件失败（跟随重命名）: %v", err) + return + } + + // 重命名 torrent 文件 + queryParam := make(map[string]string) + fullUrl := API_URL + method := "POST" + if isFamily { + fullUrl += "/family/file" + method = "GET" + queryParam["familyId"] = y.FamilyID + } + fullUrl += "/renameFile.action" + queryParam["fileId"] = torrentFile.GetID() + queryParam["destFileName"] = newTorrentName + + _, renameErr := y.request(fullUrl, method, func(req *resty.Request) { + req.SetContext(context.Background()).SetQueryParams(queryParam) + }, nil, &RenameResp{}, isFamily) + if renameErr != nil { + utils.Log.Warnf("跟随重命名 torrent 文件失败: %v", renameErr) + } + }() +} diff --git a/drivers/189pc/types.go b/drivers/189pc/types.go index 7952cb78..6607c408 100644 --- a/drivers/189pc/types.go +++ b/drivers/189pc/types.go @@ -72,6 +72,8 @@ type BaseLoginParam struct { // 请求头参数 Lt string ReqId string + // logbox页面地址，作为后续请求的Referer，缺失会被判定为陌生设备 + Referer string // 表单参数 ParamId string @@ -97,10 +99,20 @@ type LoginParam struct { // rsa密钥 jRsaKey string + // 加密字段的前缀，服务端下发（如 {NRP}） + rsaPrefix string + + // 设备二次校验时服务端返回的加密手机号 + SecondAuthMobile string BaseLoginParam } +// encryptSecret 用登陆时拿到的公钥加密敏感值，格式与userName/epd一致 +func (p *LoginParam) encryptSecret(value string) string { + return p.rsaPrefix + RsaEncrypt(p.jRsaKey, value) +} + // 登陆加密相关 type EncryptConfResp struct { Result int `json:"result"` @@ -116,6 +128,35 @@ type LoginResp struct { Msg string `json:"msg"` Result int `json:"result"` ToUrl string `json:"toUrl"` + // 设备二次校验时返回的加密手机号 + Mobile string `json:"mobile"` +} + +// 登陆页配置，新版登陆页的paramId由该接口下发 +// 该接口的result可能是数字也可能是字符串 +type AppConfResp struct { + Result any `json:"result"` + Msg string `json:"msg"` + Data struct { + ParamId string `json:"paramId"` + AccountType string `json:"accountType"` + ReturnUrl string `json:"returnUrl"` + MailSuffix string `json:"mailSuffix"` + } `json:"data"` +} + +func (r *AppConfResp) Succeeded() bool { + switch v := r.Result.(type) { + case nil: + return true + case string: + return v == "0" || v == "" + case float64: + return v == 0 + case int: + return v == 0 + } + return false } // 刷新session返回 @@ -149,6 +190,27 @@ type AppSessionResp struct { RefreshToken string `json:"refreshToken"` } +// 刷新token返回，失败时以HTTP 200返回result/msg，需要单独判断 +type RefreshTokenResp struct { + AccessToken string `json:"accessToken"` + RefreshToken string `json:"refreshToken"` + ExpiresIn int `json:"expiresIn"` + + Result int `json:"result"` + Msg string `json:"msg"` +} + +func (r *RefreshTokenResp) HasError() bool { + return r.Result != 0 || r.AccessToken == "" +} + +func (r *RefreshTokenResp) Error() string { + if r.Msg != "" { + return fmt.Sprintf("refresh token failed, result: %d, msg: %s", r.Result, r.Msg) + } + return fmt.Sprintf("refresh token failed, result: %d", r.Result) +} + // 家庭云账户 type FamilyInfoListResp struct { FamilyInfoResp []FamilyInfoResp `json:"familyInfoResp"` @@ -166,10 +228,11 @@ type FamilyInfoResp struct { /*文件部分*/ // 文件 type Cloud189File struct { - ID String `json:"id"` - Name string `json:"name"` - Size int64 `json:"size"` - Md5 string `json:"md5"` + ID String `json:"id"` + Name string `json:"name"` + Size int64 `json:"size"` + Md5 string `json:"md5"` + ParentID string `json:"-"` // 由 getFiles 设置，不从 JSON 解析 LastOpTime Time `json:"lastOpTime"` CreateDate Time `json:"createDate"` @@ -190,6 +253,10 @@ type Cloud189File struct { // StarLabel int64 `json:"starLabel"` } +func normalizeCloud189Name(name string) string { + return strings.ReplaceAll(name, "\\'", "'") +} + func (c *Cloud189File) CreateTime() time.Time { return time.Time(c.CreateDate) } @@ -206,6 +273,8 @@ func (c *Cloud189File) GetID() string { return string(c.ID) } func (c *Cloud189File) GetPath() string { return "" } func (c *Cloud189File) Thumb() string { return c.Icon.SmallUrl } +func (c *Cloud189File) GetDisplayName() string { return normalizeCloud189Name(c.Name) } + // 文件夹 type Cloud189Folder struct { ID String `json:"id"` @@ -237,6 +306,8 @@ func (c *Cloud189Folder) IsDir() bool { return true } func (c *Cloud189Folder) GetID() string { return string(c.ID) } func (c *Cloud189Folder) GetPath() string { return "" } +func (c *Cloud189Folder) GetDisplayName() string { return normalizeCloud189Name(c.Name) } + type Cloud189FilesResp struct { //ResCode int `json:"res_code"` //ResMessage string `json:"res_message"` diff --git a/drivers/189pc/utils.go b/drivers/189pc/utils.go index e863060a..713fa78b 100644 --- a/drivers/189pc/utils.go +++ b/drivers/189pc/utils.go @@ -14,6 +14,7 @@ import ( "net/http/cookiejar" "net/url" "os" + "path" "regexp" "sort" "strconv" @@ -30,6 +31,7 @@ import ( "github.com/OpenListTeam/OpenList/v4/internal/stream" "github.com/OpenListTeam/OpenList/v4/pkg/errgroup" "github.com/OpenListTeam/OpenList/v4/pkg/utils" + "github.com/OpenListTeam/OpenList/v4/pkg/utils/random" "github.com/skip2/go-qrcode" "github.com/avast/retry-go" @@ -41,9 +43,13 @@ import ( const ( ACCOUNT_TYPE = "02" - APP_ID = "8025431004" - CLIENT_TYPE = "10020" - VERSION = "6.2" + // 官方 PC 端（cloud.189.cn 网页/客户端）使用的 appId， + // 登录、生成二维码、换取 session 必须全程使用同一个 appId + APP_ID = "9317140619" + CLIENT_TYPE = "10020" + // 扫码状态轮询使用的 clientType，与密码登录的 10020 不同 + QR_CLIENT_TYPE = "1" + VERSION = "7.2.4.0" WEB_URL = "https://cloud.189.cn" AUTH_URL = "https://open.e.189.cn" @@ -57,8 +63,18 @@ const ( CHANNEL_ID = "web_cloud.189.cn" + // 服务端通过短信二次校验后下发的设备标识，复用它可以避免再次触发校验 + DEVICE_ID_COOKIE = "DEVICEID" + + // 扫码登录本地轮询参数，超时后把二维码交回前端，避免请求被反向代理掐断 + QRCODE_POLL_INTERVAL = 2 * time.Second + QRCODE_POLL_TIMEOUT = 20 * time.Second + // Error codes UserInvalidOpenTokenError = "UserInvalidOpenToken" + + // 密码登录返回该结果表示需要设备二次校验 + SecondDeviceAuthResult = -133 ) func (y *Cloud189PC) SignatureHeader(url, method, params string, isFamily bool) map[string]string { @@ -195,9 +211,10 @@ func (y *Cloud189PC) put(ctx context.Context, url string, headers map[string]str } func (y *Cloud189PC) getFiles(ctx context.Context, fileId string, isFamily bool) ([]model.Obj, error) { + pageSize := 1000 // 每一页返回的文件数量 res := make([]model.Obj, 0, 100) for pageNum := 1; ; pageNum++ { - resp, err := y.getFilesWithPage(ctx, fileId, isFamily, pageNum, 1000, y.OrderBy, y.OrderDirection) + resp, err := y.getFilesWithPage(ctx, fileId, isFamily, pageNum, pageSize, y.OrderBy, y.OrderDirection) if err != nil { return nil, err } @@ -205,13 +222,23 @@ func (y *Cloud189PC) getFiles(ctx context.Context, fileId string, isFamily bool) if resp.FileListAO.Count == 0 { break } + + FolderCount := len(resp.FileListAO.FolderList) // 当前文件夹总数 + FileCount := len(resp.FileListAO.FileList) // 当前文件总数 + PageCount := FolderCount + FileCount // 当前页数总数 - for i := 0; i < len(resp.FileListAO.FolderList); i++ { + for i := 0; i < FolderCount; i++ { res = append(res, &resp.FileListAO.FolderList[i]) } - for i := 0; i < len(resp.FileListAO.FileList); i++ { + for i := 0; i < FileCount; i++ { + resp.FileListAO.FileList[i].ParentID = fileId res = append(res, &resp.FileListAO.FileList[i]) } + + // 当前文件数量小于设定数量则跳出 + if PageCount < pageSize { + break + } } return res, nil } @@ -277,9 +304,72 @@ func (y *Cloud189PC) login() error { if y.LoginType == "qrcode" { return y.loginByQRCode() } + if y.Username == "" || y.Password == "" { + return errors.New("please fill in the username and password, or provide an access token / refresh token") + } return y.loginByPassword() } +// 设备指纹，为空时生成并保存，服务端以此识别是否为同一台设备 +func (y *Cloud189PC) getUserFinger() string { + if y.Addition.UserFinger == "" { + y.Addition.UserFinger = fmt.Sprint(random.Rand.Int63n(9e9) + 1e9) + op.MustSaveDriverStorage(y) + } + return y.Addition.UserFinger +} + +// 换取会话时携带的设备参数，与官方PC客户端保持一致 +// clientSn/jgOpenId 只在用户从官方客户端抓到并填写后才发送，避免上报一个服务端不认识的设备号 +func (y *Cloud189PC) deviceParams() map[string]string { + params := map[string]string{"returnType": "JSON"} + if y.Addition.ClientSn != "" { + params["clientSn"] = y.Addition.ClientSn + } + if y.Addition.JgOpenId != "" { + params["jgOpenId"] = y.Addition.JgOpenId + } + return params +} + +// logbox接口的公共请求头，缺少user-finger和Referer会被判定为陌生设备 +func (y *Cloud189PC) loginHeaders(param BaseLoginParam) map[string]string { + return map[string]string{ + "REQID": param.ReqId, + "lt": param.Lt, + "user-finger": y.getUserFinger(), + "Referer": IF(param.Referer != "", param.Referer, AUTH_URL), + } +} + +// 把已保存的设备标识写入cookie，避免重复触发设备二次校验 +func (y *Cloud189PC) applyDeviceID(jar http.CookieJar) { + if y.Addition.DeviceID == "" { + return + } + authUrl, err := url.Parse(AUTH_URL) + if err != nil { + return + } + jar.SetCookies(authUrl, []*http.Cookie{{ + Name: DEVICE_ID_COOKIE, + Value: y.Addition.DeviceID, + Domain: "e.189.cn", + Path: "/", + }}) +} + +// 保存服务端下发的设备标识，下次登陆复用即可跳过设备二次校验 +func (y *Cloud189PC) saveDeviceID(res *resty.Response) { + for _, cookie := range res.Cookies() { + if cookie.Name == DEVICE_ID_COOKIE && cookie.Value != "" && cookie.Value != y.Addition.DeviceID { + y.Addition.DeviceID = cookie.Value + op.MustSaveDriverStorage(y) + return + } + } +} + func (y *Cloud189PC) loginByPassword() (err error) { // 初始化登陆所需参数 if y.loginParam == nil { @@ -288,9 +378,16 @@ func (y *Cloud189PC) loginByPassword() (err error) { return err } } + // 设备二次校验必须复用同一套登陆参数，此时不能销毁也不能重新初始化 + keepLoginParam := false defer func() { // 销毁验证码 y.VCode = "" + if keepLoginParam { + y.Status = err.Error() + op.MustSaveDriverStorage(y) + return + } // 销毁登陆参数 y.loginParam = nil // 遇到错误，重新加载登陆参数(刷新验证码) @@ -308,17 +405,18 @@ func (y *Cloud189PC) loginByPassword() (err error) { param := y.loginParam var loginresp LoginResp - _, err = y.client.R(). + res, err := y.client.R(). ForceContentType("application/json;charset=UTF-8").SetResult(&loginresp). - SetHeaders(map[string]string{ - "REQID": param.ReqId, - "lt": param.Lt, - }). + SetHeaders(y.loginHeaders(param.BaseLoginParam)). SetFormData(map[string]string{ + "version": "v2.0", + "apToken": "", "appKey": APP_ID, + "pageKey": "normal", "accountType": ACCOUNT_TYPE, "userName": param.RsaUsername, "password": param.RsaPassword, + "epd": param.RsaPassword, "validateCode": y.VCode, "captchaToken": param.CaptchaToken, "returnUrl": RETURN_URL, @@ -334,17 +432,106 @@ func (y *Cloud189PC) loginByPassword() (err error) { if err != nil { return err } + y.saveDeviceID(res) + + // 设备二次校验：服务端要求短信验证，保留登陆参数并引导填写短信验证码 + if loginresp.Result == SecondDeviceAuthResult { + err = y.secondDeviceAuth(loginresp.Mobile) + // 校验未完成时保留登陆参数，等待用户回填短信验证码 + keepLoginParam = err != nil && y.loginParam != nil + return err + } + if loginresp.ToUrl == "" { return fmt.Errorf("login failed,No toUrl obtained, msg: %s", loginresp.Msg) } - // 获取Session + return y.getSessionByRedirectURL(loginresp.ToUrl) +} + +// 设备二次校验：先发短信，用户回填验证码后再提交 +func (y *Cloud189PC) secondDeviceAuth(mobile string) error { + param := y.loginParam + if mobile != "" { + param.SecondAuthMobile = mobile + } + if param.SecondAuthMobile == "" { + return errors.New("second device verification is required, but no mobile was returned") + } + + // 已填写短信验证码，直接提交校验 + if y.SmsCode != "" { + smsCode := y.SmsCode + y.SmsCode = "" + op.MustSaveDriverStorage(y) + return y.submitSecondDeviceAuth(smsCode) + } + + var smsResp LoginResp + _, err := y.client.R(). + ForceContentType("application/json;charset=UTF-8").SetResult(&smsResp). + SetHeaders(y.loginHeaders(param.BaseLoginParam)). + SetFormData(map[string]string{ + "mobile": param.SecondAuthMobile, + "appKey": APP_ID, + }). + Post(AUTH_URL + "/api/logbox/oauth2/sendSmsCodeForSecondAuth.do") + if err != nil { + return err + } + if smsResp.Result != 0 { + return fmt.Errorf("failed to send the verification SMS: %s", smsResp.Msg) + } + // 保留登陆参数，等待用户回填短信验证码后重新保存 + return errors.New("second device verification is required, an SMS code has been sent, please fill it into `sms_code` and save again") +} + +// 提交短信验证码完成设备二次校验 +// 注意：该接口没有独立的短信码字段，短信码要加密后放在epd里（登陆时epd装的是密码） +func (y *Cloud189PC) submitSecondDeviceAuth(smsCode string) error { + param := y.loginParam + var authResp LoginResp + res, err := y.client.R(). + ForceContentType("application/json;charset=UTF-8").SetResult(&authResp). + SetHeaders(y.loginHeaders(param.BaseLoginParam)). + SetFormData(map[string]string{ + "mobile": param.SecondAuthMobile, + "appKey": APP_ID, + "userName": param.RsaUsername, + "epd": param.encryptSecret(smsCode), + "accountType": ACCOUNT_TYPE, + "returnUrl": RETURN_URL, + "isOauth2": "false", + "cb_SaveName": "1", + "state": "", + "paramId": param.ParamId, + }). + Post(AUTH_URL + "/api/logbox/oauth2/submitForSecondAuth.do") + if err != nil { + return err + } + // 校验通过后服务端会下发DEVICEID，保存下来以后就不会再触发二次校验 + y.saveDeviceID(res) + + if authResp.Result != 0 { + return fmt.Errorf("second device verification failed: %s", authResp.Msg) + } + if authResp.ToUrl == "" { + return fmt.Errorf("second device verification failed, no toUrl obtained, msg: %s", authResp.Msg) + } + return y.getSessionByRedirectURL(authResp.ToUrl) +} + +// 用登陆结果的跳转地址换取会话 +func (y *Cloud189PC) getSessionByRedirectURL(redirectURL string) error { var erron RespErr var tokenInfo AppSessionResp - _, err = y.client.R(). + _, err := y.client.R(). SetResult(&tokenInfo).SetError(&erron). SetQueryParams(clientSuffix()). - SetQueryParam("redirectURL", loginresp.ToUrl). + SetQueryParams(y.deviceParams()). + SetQueryParam("redirectURL", redirectURL). + SetHeader("X-Request-ID", uuid.NewString()). Post(API_URL + "/getSessionForPC.action") if err != nil { return err @@ -354,14 +541,13 @@ func (y *Cloud189PC) loginByPassword() (err error) { return &erron } if tokenInfo.ResCode != 0 { - err = fmt.Errorf(tokenInfo.ResMessage) - return err + return errors.New(tokenInfo.ResMessage) } y.Addition.AccessToken = tokenInfo.AccessToken y.Addition.RefreshToken = tokenInfo.RefreshToken y.tokenInfo = &tokenInfo op.MustSaveDriverStorage(y) - return err + return nil } func (y *Cloud189PC) loginByQRCode() error { @@ -372,66 +558,74 @@ func (y *Cloud189PC) loginByQRCode() error { } } - var state struct { - Status int `json:"status"` - RedirectUrl string `json:"redirectUrl"` - Msg string `json:"msg"` + // 本地轮询，扫码确认后自动继续，不需要用户反复保存 + deadline := time.Now().Add(QRCODE_POLL_TIMEOUT) + lastStatus := -106 + for { + state, err := y.checkQRCodeState() + if err != nil { + return fmt.Errorf("failed to check QR code state: %w", err) + } + lastStatus = state.Status + + switch state.Status { + case 0: // 登录成功 + y.qrcodeParam = nil + return y.getSessionByRedirectURL(state.RedirectUrl) + case -106, -11002: // -106 等待扫描，-11002 已扫描等待确认 + case -11001: // 二维码过期 + y.qrcodeParam = nil + return errors.New("QR code expired, please try again") + default: // 其他错误 + y.qrcodeParam = nil + return fmt.Errorf("QR code login failed with status %d: %s", state.Status, state.Msg) + } + + if time.Now().Add(QRCODE_POLL_INTERVAL).After(deadline) { + break + } + time.Sleep(QRCODE_POLL_INTERVAL) } + // 轮询超时，把二维码交回前端等待下一次保存 + if lastStatus == -11002 { + return y.genQRCode("QR code has been scanned, please confirm the login on your phone and save again") + } + return y.genQRCode("QR code has not been scanned yet, please scan and save again") +} + +type qrCodeState struct { + Status int `json:"status"` + RedirectUrl string `json:"redirectUrl"` + Msg string `json:"msg"` +} + +// 查询扫码状态，参数需与官方PC端一致，否则服务端不会返回授权结果 +func (y *Cloud189PC) checkQRCodeState() (*qrCodeState, error) { now := time.Now() + var state qrCodeState _, err := y.client.R(). - SetHeaders(map[string]string{ - "Referer": AUTH_URL, - "Reqid": y.qrcodeParam.ReqId, - "lt": y.qrcodeParam.Lt, - }). + SetHeaders(y.loginHeaders(y.qrcodeParam.BaseLoginParam)). SetFormData(map[string]string{ - "appId": APP_ID, - "clientType": CLIENT_TYPE, - "returnUrl": RETURN_URL, - "paramId": y.qrcodeParam.ParamId, - "uuid": y.qrcodeParam.UUID, - "encryuuid": y.qrcodeParam.EncryUUID, - "date": formatDate(now), - "timeStamp": fmt.Sprint(now.UTC().UnixNano() / 1e6), + "appId": APP_ID, + "clientType": QR_CLIENT_TYPE, + "returnUrl": RETURN_URL, + "paramId": y.qrcodeParam.ParamId, + "uuid": y.qrcodeParam.UUID, + "encryuuid": y.qrcodeParam.EncryUUID, + "cb_SaveName": "3", + "isOauth2": "false", + "state": "", + "date": formatDate(now), + "timeStamp": fmt.Sprint(now.UTC().UnixNano() / 1e6), }). ForceContentType("application/json;charset=UTF-8"). SetResult(&state). Post(AUTH_URL + "/api/logbox/oauth2/qrcodeLoginState.do") if err != nil { - return fmt.Errorf("failed to check QR code state: %w", err) - } - - switch state.Status { - case 0: // 登录成功 - var tokenInfo AppSessionResp - _, err = y.client.R(). - SetResult(&tokenInfo). - SetQueryParams(clientSuffix()). - SetQueryParam("redirectURL", state.RedirectUrl). - Post(API_URL + "/getSessionForPC.action") - if err != nil { - return err - } - if tokenInfo.ResCode != 0 { - return fmt.Errorf(tokenInfo.ResMessage) - } - y.Addition.AccessToken = tokenInfo.AccessToken - y.Addition.RefreshToken = tokenInfo.RefreshToken - y.tokenInfo = &tokenInfo - op.MustSaveDriverStorage(y) - return nil - case -11001: // 二维码过期 - y.qrcodeParam = nil - return errors.New("QR code expired, please try again") - case -106: // 等待扫描 - return y.genQRCode("QR code has not been scanned yet, please scan and save again") - case -11002: // 等待确认 - return y.genQRCode("QR code has been scanned, please confirm the login on your phone and save again") - default: // 其他错误 - y.qrcodeParam = nil - return fmt.Errorf("QR code login failed with status %d: %s", state.Status, state.Msg) + return nil, err } + return &state, nil } func (y *Cloud189PC) genQRCode(text string) error { @@ -457,8 +651,9 @@ func (y *Cloud189PC) genQRCode(text string) error { } func (y *Cloud189PC) initBaseParams() (*BaseLoginParam, error) { - // 清除cookie + // 清除cookie，并带上已保存的设备标识 jar, _ := cookiejar.New(nil) + y.applyDeviceID(jar) y.client.SetCookieJar(jar) res, err := y.client.R(). @@ -473,14 +668,98 @@ func (y *Cloud189PC) initBaseParams() (*BaseLoginParam, error) { return nil, err } + // 当前登陆页把lt/reqId放在跳转地址上，老页面则写在页内变量里，两种都要支持 + param, err := parseBaseParamFromRedirect(res.RawResponse.Request.URL) + if err != nil { + param, err = parseBaseParamFromPage(res.String()) + if err != nil { + return nil, err + } + return param, nil + } + + // 跳转地址上没有paramId，需要再问一次appConf.do + var appConf AppConfResp + _, err = y.client.R(). + SetHeaders(y.loginHeaders(*param)). + ForceContentType("application/json;charset=UTF-8"). + SetResult(&appConf). + SetFormData(map[string]string{ + "version": "2.0", + "appKey": APP_ID, + }). + Post(AUTH_URL + "/api/logbox/oauth2/appConf.do") + if err != nil { + return nil, err + } + if !appConf.Succeeded() || appConf.Data.ParamId == "" { + return nil, fmt.Errorf("failed to get the login paramId: %s", appConf.Msg) + } + param.ParamId = appConf.Data.ParamId + return param, nil +} + +// parseBaseParamFromRedirect 从logbox跳转地址提取登陆参数，并以该地址作为后续请求的Referer +func parseBaseParamFromRedirect(finalUrl *url.URL) (*BaseLoginParam, error) { + if finalUrl == nil { + return nil, errors.New("no login page redirect") + } + query := finalUrl.Query() + lt, reqId := query.Get("lt"), query.Get("reqId") + if lt == "" || reqId == "" { + return nil, errors.New("no lt/reqId in the login page redirect") + } return &BaseLoginParam{ - CaptchaToken: regexp.MustCompile(`'captchaToken' value='(.+?)'`).FindStringSubmatch(res.String())[1], - Lt: regexp.MustCompile(`lt = "(.+?)"`).FindStringSubmatch(res.String())[1], - ParamId: regexp.MustCompile(`paramId = "(.+?)"`).FindStringSubmatch(res.String())[1], - ReqId: regexp.MustCompile(`reqId = "(.+?)"`).FindStringSubmatch(res.String())[1], + Lt: lt, + ReqId: reqId, + Referer: finalUrl.String(), }, nil } +// parseBaseParamFromPage 兼容把参数写在页内变量里的老登陆页 +func parseBaseParamFromPage(body string) (*BaseLoginParam, error) { + lt, err := matchLoginParam(body, `lt = "(.+?)"`, "lt") + if err != nil { + return nil, err + } + reqId, err := matchLoginParam(body, `reqId = "(.+?)"`, "reqId") + if err != nil { + return nil, err + } + paramId, err := matchLoginParam(body, `paramId = "(.+?)"`, "paramId") + if err != nil { + return nil, err + } + // 老页面才有内嵌的图形验证码token + captchaToken, _ := matchLoginParam(body, `'captchaToken' value='(.+?)'`, "captchaToken") + + param := &BaseLoginParam{ + CaptchaToken: captchaToken, + Lt: lt, + ParamId: paramId, + ReqId: reqId, + } + encryptUrl, _ := matchLoginParam(body, `encryptUrl = "(.+?)"`, "encryptUrl") + param.Referer = AUTH_URL + "/api/logbox/separate/web/index.html?" + strings.Join([]string{ + "appId=" + url.QueryEscape(APP_ID), + "lt=" + url.QueryEscape(param.Lt), + "reqId=" + url.QueryEscape(param.ReqId), + }, "&") + if encryptUrl != "" { + param.Referer += "&encryptUrl=" + url.QueryEscape(encryptUrl) + } + return param, nil +} + +// matchLoginParam 从登陆页面提取参数，缺失时返回可读的错误而不是panic +func matchLoginParam(body, pattern, name string) (string, error) { + matches := regexp.MustCompile(pattern).FindStringSubmatch(body) + if len(matches) < 2 { + return "", fmt.Errorf("failed to get %s from the login page", name) + } + return matches[1], nil +} + /* 初始化登陆需要的参数 * 如果遇到验证码返回错误 */ @@ -505,12 +784,13 @@ func (y *Cloud189PC) initLoginParam() error { } y.loginParam.jRsaKey = fmt.Sprintf("-----BEGIN PUBLIC KEY-----\n%s\n-----END PUBLIC KEY-----", encryptConf.Data.PubKey) - y.loginParam.RsaUsername = encryptConf.Data.Pre + RsaEncrypt(y.loginParam.jRsaKey, y.Username) - y.loginParam.RsaPassword = encryptConf.Data.Pre + RsaEncrypt(y.loginParam.jRsaKey, y.Password) + y.loginParam.rsaPrefix = encryptConf.Data.Pre + y.loginParam.RsaUsername = y.loginParam.encryptSecret(y.Username) + y.loginParam.RsaPassword = y.loginParam.encryptSecret(y.Password) // 判断是否需要验证码 resp, err := y.client.R(). - SetHeader("REQID", y.loginParam.ReqId). + SetHeaders(y.loginHeaders(y.loginParam.BaseLoginParam)). SetFormData(map[string]string{ "appKey": APP_ID, "accountType": ACCOUNT_TYPE, @@ -565,6 +845,7 @@ func (y *Cloud189PC) initQRCodeParam() (err error) { var qrcodeParam QRLoginParam _, err = y.client.R(). + SetHeaders(y.loginHeaders(*baseParam)). SetFormData(map[string]string{"appId": APP_ID}). ForceContentType("application/json;charset=UTF-8"). SetResult(&qrcodeParam). @@ -572,6 +853,9 @@ func (y *Cloud189PC) initQRCodeParam() (err error) { if err != nil { return err } + if qrcodeParam.UUID == "" { + return errors.New("failed to get the QR code uuid") + } qrcodeParam.BaseLoginParam = *baseParam y.qrcodeParam = &qrcodeParam @@ -592,6 +876,7 @@ func (y *Cloud189PC) refreshSessionWithRetry(retryCount int) (err error) { _, err = y.client.R(). SetResult(&userSessionResp).SetError(&erron). SetQueryParams(clientSuffix()). + SetQueryParams(y.deviceParams()). SetQueryParams(map[string]string{ "appId": APP_ID, "accessToken": y.tokenInfo.AccessToken, @@ -632,12 +917,11 @@ func (y *Cloud189PC) refreshTokenWithRetry(retryCount int) (err error) { return errors.New("refresh token failed after maximum retries") } - var erron RespErr - var tokenInfo AppSessionResp + // 该接口刷新失败时以HTTP 200返回 result/msg，SetError不会触发，必须解析响应体判断 + var tokenInfo RefreshTokenResp _, err = y.client.R(). SetResult(&tokenInfo). ForceContentType("application/json;charset=UTF-8"). - SetError(&erron). SetFormData(map[string]string{ "clientId": APP_ID, "refreshToken": y.tokenInfo.RefreshToken, @@ -650,7 +934,8 @@ func (y *Cloud189PC) refreshTokenWithRetry(retryCount int) (err error) { } // 如果刷新失败，返回错误给上层处理 - if erron.HasError() { + if tokenInfo.HasError() { + refreshErr := tokenInfo.Error() if y.Addition.RefreshToken != "" { y.Addition.RefreshToken = "" op.MustSaveDriverStorage(y) @@ -658,7 +943,11 @@ func (y *Cloud189PC) refreshTokenWithRetry(retryCount int) (err error) { // 根据登录类型决定下一步行为 if y.LoginType == "qrcode" { - return errors.New("QR code session has expired, please re-scan the code to log in") + return fmt.Errorf("QR code session has expired, please re-scan the code to log in: %s", refreshErr) + } + // 没有账号密码时无法回退到完整登录，直接把刷新失败的原因返回 + if y.Username == "" || y.Password == "" { + return errors.New(refreshErr) } // 密码登录模式下，尝试回退到完整登录 return y.login() @@ -666,7 +955,8 @@ func (y *Cloud189PC) refreshTokenWithRetry(retryCount int) (err error) { y.Addition.AccessToken = tokenInfo.AccessToken y.Addition.RefreshToken = tokenInfo.RefreshToken - y.tokenInfo = &tokenInfo + y.tokenInfo.AccessToken = tokenInfo.AccessToken + y.tokenInfo.RefreshToken = tokenInfo.RefreshToken op.MustSaveDriverStorage(y) return y.refreshSessionWithRetry(retryCount + 1) } @@ -910,6 +1200,15 @@ func (y *Cloud189PC) RapidUpload(ctx context.Context, dstDir model.Obj, stream m // 快传 func (y *Cloud189PC) FastUpload(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress, isFamily bool, overwrite bool) (model.Obj, error) { + generateTorrent := y.Addition.GenerateTorrent && !isCASTorrentFile(file.GetName()) + return y.fastUpload(ctx, dstDir, file, up, isFamily, overwrite, generateTorrent) +} + +func uploadProgressKeys(sessionKey, fileMD5, fileName string) []string { + return []string{sessionKey, fileMD5, path.Ext(fileName)} +} + +func (y *Cloud189PC) fastUpload(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress, isFamily bool, overwrite bool, generateTorrent bool) (model.Obj, error) { var ( cache = file.GetFile() tmpF *os.File @@ -947,6 +1246,9 @@ func (y *Cloud189PC) FastUpload(ctx context.Context, dstDir model.Obj, file mode if tmpF != nil { writers = append(writers, tmpF) } + + pieceSHA1Hashes := make([]byte, 0, count*20) + written := int64(0) for i := 1; i <= count; i++ { if utils.IsCanceled(ctx) { @@ -957,7 +1259,17 @@ func (y *Cloud189PC) FastUpload(ctx context.Context, dstDir model.Obj, file mode byteSize = lastSliceSize } - n, err := utils.CopyWithBufferN(io.MultiWriter(writers...), file, byteSize) + // 如果需要生成 torrent，同时计算 SHA-1 + var sha1Writer hash.Hash + var multiWriter io.Writer + if generateTorrent { + sha1Writer = sha1Pkg.New() + multiWriter = io.MultiWriter(append(writers, sha1Writer)...) + } else { + multiWriter = io.MultiWriter(writers...) + } + + n, err := utils.CopyWithBufferN(multiWriter, file, byteSize) written += n if err != nil && err != io.EOF { return nil, err @@ -966,6 +1278,11 @@ func (y *Cloud189PC) FastUpload(ctx context.Context, dstDir model.Obj, file mode sliceMd5Hexs = append(sliceMd5Hexs, strings.ToUpper(hex.EncodeToString(md5Byte))) partInfos = append(partInfos, fmt.Sprint(i, "-", base64.StdEncoding.EncodeToString(md5Byte))) sliceMd5.Reset() + + // 收集 SHA-1 piece hash（仅在本次分片实际写入了数据时追加） + if generateTorrent && n > 0 { + pieceSHA1Hashes = append(pieceSHA1Hashes, sha1Writer.Sum(nil)...) + } } if tmpF != nil { @@ -993,7 +1310,8 @@ func (y *Cloud189PC) FastUpload(ctx context.Context, dstDir model.Obj, file mode } // 尝试恢复进度 - uploadProgress, ok := base.GetUploadProgress[*UploadProgress](y, y.getTokenInfo().SessionKey, fileMd5Hex) + progressKeys := uploadProgressKeys(y.getTokenInfo().SessionKey, fileMd5Hex, file.GetName()) + uploadProgress, ok := base.GetUploadProgress[*UploadProgress](y, progressKeys...) if !ok { // step.2 预上传 params := Params{ @@ -1061,7 +1379,7 @@ func (y *Cloud189PC) FastUpload(ctx context.Context, dstDir model.Obj, file mode if err = threadG.Wait(); err != nil { if errors.Is(err, context.Canceled) { uploadProgress.UploadParts = utils.SliceFilter(uploadProgress.UploadParts, func(s string) bool { return s != "" }) - base.SaveUploadProgress(y, uploadProgress, y.getTokenInfo().SessionKey, fileMd5Hex) + base.SaveUploadProgress(y, uploadProgress, progressKeys...) } return nil, err } @@ -1081,6 +1399,44 @@ func (y *Cloud189PC) FastUpload(ctx context.Context, dstDir model.Obj, file mode if err != nil { return nil, err } + + // 生成 torrent 文件（异步，不影响上传结果） + if generateTorrent && size > 0 && len(pieceSHA1Hashes) > 0 { + capturedDstDir := dstDir + capturedIsFamily := isFamily + capturedFileName := file.GetName() + go func() { + torrentData, err := GenerateTorrent(capturedFileName, size, fileMd5Hex, sliceMd5Hexs, sliceSize, pieceSHA1Hashes) + if err != nil { + utils.Log.Warnf("生成 torrent 失败: %v", err) + return + } + infoHash, _ := GetInfoHashHex(torrentData) + torrentName := capturedFileName + ".cas.torrent" + utils.Log.Infof("已生成 torrent: %s (info_hash: %s, size: %d bytes)", + torrentName, infoHash, len(torrentData)) + + // 将 torrent 文件上传到同一目录 + torrentFileStream := &stream.FileStream{ + Ctx: context.Background(), + Obj: &model.Object{ + Name: torrentName, + Size: int64(len(torrentData)), + IsFolder: false, + }, + Reader: bytes.NewReader(torrentData), + Mimetype: "application/x-bittorrent", + } + _, uploadErr := y.fastUpload(context.Background(), capturedDstDir, torrentFileStream, func(p float64) {}, capturedIsFamily, false, false) + if uploadErr != nil { + utils.Log.Warnf("上传 torrent 文件失败: %v", uploadErr) + } else { + utils.Log.Infof("torrent 文件已上传: %s", torrentName) + op.Cache.DeleteDirectory(y, capturedDstDir.GetPath()) + } + }() + } + return resp.toFile(), nil } diff --git a/drivers/alias/driver.go b/drivers/alias/driver.go index 7008495c..d69d6cf5 100644 --- a/drivers/alias/driver.go +++ b/drivers/alias/driver.go @@ -328,7 +328,6 @@ func (d *Alias) Link(ctx context.Context, file model.Obj, args model.LinkArgs) ( return nil, err } resultLink := link.Clone() // 复制一份，避免修改到原始link - resultLink.Expiration = nil if args.Redirect { return resultLink, nil } diff --git a/drivers/alist_v3/driver.go b/drivers/alist_v3/driver.go index a9fc44fc..ddfb0a96 100644 --- a/drivers/alist_v3/driver.go +++ b/drivers/alist_v3/driver.go @@ -95,6 +95,7 @@ func (d *AListV3) List(ctx context.Context, dir model.Obj, args model.ListArgs) file := model.ObjThumb{ Object: model.Object{ Name: f.Name, + Path: path.Join(dir.GetPath(), f.Name), Modified: f.Modified, Ctime: f.Created, Size: f.Size, diff --git a/drivers/alist_v3/driver_test.go b/drivers/alist_v3/driver_test.go new file mode 100644 index 00000000..bb8e009e --- /dev/null +++ b/drivers/alist_v3/driver_test.go @@ -0,0 +1,65 @@ +package alist_v3 + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/OpenListTeam/OpenList/v4/drivers/base" + "github.com/OpenListTeam/OpenList/v4/internal/driver" + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/go-resty/resty/v2" +) + +// TestListSetsChildPaths descends two levels the way op.Get does, feeding an +// object from one listing back into List as dir. Without a Path on that object +// the driver asks upstream for "", which a real server answers with its own +// root -- hence the fake upstream's fallback, and the endless self-similar tree. +func TestListSetsChildPaths(t *testing.T) { + tree := map[string][]ObjResp{ + "/": {{Name: "root-marker", IsDir: true}}, + "/drive": {{Name: "concerts", IsDir: true}}, + "/drive/concerts": {{Name: "show.mkv"}}, + } + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var req ListReq + _ = json.NewDecoder(r.Body).Decode(&req) + content, ok := tree[req.Path] + if !ok { + content = tree["/"] + } + w.Header().Set("Content-Type", "application/json") // resty only unmarshals JSON + _ = json.NewEncoder(w).Encode(map[string]any{ + "code": 200, "message": "success", + "data": map[string]any{"content": content, "total": len(content)}, + }) + })) + t.Cleanup(srv.Close) + + // conf.Conf is nil outside a booted server, so base.InitClient() is unusable. + prev := base.RestyClient + base.RestyClient = resty.New().SetTimeout(5 * time.Second) + t.Cleanup(func() { base.RestyClient = prev }) + + d := &AListV3{Addition: Addition{ + RootPath: driver.RootPath{RootFolderPath: "/drive"}, + Address: srv.URL, + }} + dir := model.Obj(&model.Object{Path: "/drive", IsFolder: true}) + for _, want := range []string{"/drive/concerts", "/drive/concerts/show.mkv"} { + objs, err := d.List(context.Background(), dir, model.ListArgs{}) + if err != nil { + t.Fatalf("List(%q): %v", dir.GetPath(), err) + } + if len(objs) != 1 { + t.Fatalf("List(%q) returned %d objects, want 1", dir.GetPath(), len(objs)) + } + if got := objs[0].GetPath(); got != want { + t.Fatalf("child of %q has path %q, want %q", dir.GetPath(), got, want) + } + dir = objs[0] + } +} diff --git a/drivers/aliyundrive_open/callback.go b/drivers/aliyundrive_open/callback.go new file mode 100644 index 00000000..54b0e88b --- /dev/null +++ b/drivers/aliyundrive_open/callback.go @@ -0,0 +1,295 @@ +package aliyundrive_open + +import ( + "context" + "fmt" + "io" + "math/rand/v2" + "net/http" + "strings" + "sync" + "time" + + "github.com/OpenListTeam/OpenList/v4/internal/conf" + "github.com/OpenListTeam/OpenList/v4/internal/errs" + anet "github.com/OpenListTeam/OpenList/v4/internal/net" + "github.com/OpenListTeam/OpenList/v4/internal/stream" + "github.com/OpenListTeam/OpenList/v4/pkg/http_range" +) + +const ( + defaultCallbackConcurrency = 1 + callbackAcquireTimeout = time.Second + callbackRequestAttempts = 3 + callbackRetryBaseDelay = 200 * time.Millisecond + callbackErrorBodyLimit = 64 << 10 +) + +var callbackLimiters = struct { + sync.Mutex + byUser map[string]*callbackLimiter +}{byUser: make(map[string]*callbackLimiter)} + +type callbackLimiter struct { + userID string + mu sync.Mutex + active int + nextID uint64 + registrations map[uint64]int + changed chan struct{} +} + +type callbackRegistration struct { + limiter *callbackLimiter + id uint64 + once sync.Once +} + +type callbackPermit struct { + limiter *callbackLimiter + once sync.Once +} + +func normalizeCallbackConcurrency(limit int) int { + if limit <= 0 { + return defaultCallbackConcurrency + } + return limit +} + +func registerCallbackLimiter(userID string, limit int) *callbackRegistration { + callbackLimiters.Lock() + defer callbackLimiters.Unlock() + + limiter := callbackLimiters.byUser[userID] + if limiter == nil { + limiter = &callbackLimiter{ + userID: userID, + registrations: make(map[uint64]int), + changed: make(chan struct{}), + } + callbackLimiters.byUser[userID] = limiter + } + limiter.mu.Lock() + limiter.nextID++ + id := limiter.nextID + limiter.registrations[id] = normalizeCallbackConcurrency(limit) + limiter.signalLocked() + limiter.mu.Unlock() + return &callbackRegistration{limiter: limiter, id: id} +} + +func (r *callbackRegistration) unregister() { + if r == nil || r.limiter == nil { + return + } + r.once.Do(func() { + callbackLimiters.Lock() + defer callbackLimiters.Unlock() + r.limiter.mu.Lock() + delete(r.limiter.registrations, r.id) + r.limiter.signalLocked() + if len(r.limiter.registrations) == 0 && r.limiter.active == 0 { + delete(callbackLimiters.byUser, r.limiter.userID) + } + r.limiter.mu.Unlock() + }) +} + +func (r *callbackRegistration) acquire(ctx context.Context) (*callbackPermit, error) { + if r == nil || r.limiter == nil { + return nil, errs.NewErr(errs.TemporaryCapacity, "callback limiter is unavailable") + } + if err := ctx.Err(); err != nil { + return nil, err + } + waitCtx, cancel := context.WithTimeout(ctx, callbackAcquireTimeout) + defer cancel() + for { + r.limiter.mu.Lock() + if r.limiter.active < r.limiter.limitLocked() { + r.limiter.active++ + r.limiter.mu.Unlock() + return &callbackPermit{limiter: r.limiter}, nil + } + changed := r.limiter.changed + r.limiter.mu.Unlock() + select { + case <-ctx.Done(): + return nil, ctx.Err() + case <-waitCtx.Done(): + if err := ctx.Err(); err != nil { + return nil, err + } + return nil, errs.NewErr(errs.TemporaryCapacity, "timed out waiting for callback admission") + case <-changed: + } + } +} + +func (l *callbackLimiter) limitLocked() int { + limit := 0 + for _, registered := range l.registrations { + if limit == 0 || registered < limit { + limit = registered + } + } + return limit +} + +func (l *callbackLimiter) signalLocked() { + close(l.changed) + l.changed = make(chan struct{}) +} + +func (p *callbackPermit) release() { + if p == nil || p.limiter == nil { + return + } + p.once.Do(func() { + callbackLimiters.Lock() + defer callbackLimiters.Unlock() + p.limiter.mu.Lock() + p.limiter.active-- + p.limiter.signalLocked() + if len(p.limiter.registrations) == 0 && p.limiter.active == 0 { + delete(callbackLimiters.byUser, p.limiter.userID) + } + p.limiter.mu.Unlock() + }) +} + +func (d *AliyundriveOpen) callbackRegistration() *callbackRegistration { + if d.callback != nil { + return d.callback + } + if d.ref != nil { + return d.ref.callbackRegistration() + } + return nil +} + +func (d *AliyundriveOpen) callbackRangeReader(url string, size int64) stream.RangeReaderFunc { + return func(ctx context.Context, requested http_range.Range) (io.ReadCloser, error) { + if requested.Length < 0 || requested.Start+requested.Length > size { + requested.Length = size - requested.Start + } + for attempt := 0; attempt < callbackRequestAttempts; attempt++ { + permit, err := d.callbackRegistration().acquire(ctx) + if err != nil { + return nil, err + } + body, retry, err := openCallbackRange(ctx, url, size, requested) + if !retry && err == nil { + return newCallbackBody(ctx, body, permit.release), nil + } + permit.release() + if !retry { + return nil, err + } + if attempt+1 == callbackRequestAttempts { + return nil, errs.NewErr(errs.TemporaryCapacity, "Aliyun callback concurrency limit rejected %d attempts", callbackRequestAttempts) + } + delay := callbackRetryBaseDelay << attempt + delay += time.Duration(rand.Int64N(int64(delay / 2))) + timer := time.NewTimer(delay) + select { + case <-ctx.Done(): + timer.Stop() + return nil, ctx.Err() + case <-timer.C: + } + } + return nil, errs.NewErr(errs.TemporaryCapacity, "callback attempts exhausted") + } +} + +func openCallbackRange(ctx context.Context, url string, size int64, requested http_range.Range) (io.ReadCloser, bool, error) { + requestHeader, _ := ctx.Value(conf.RequestHeaderKey).(http.Header) + header := anet.ProcessHeader(requestHeader, nil) + header = http_range.ApplyRangeToHttpHeader(requested, header) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return nil, false, fmt.Errorf("create Aliyun callback request: %w", err) + } + req.Header = header + response, err := anet.HttpClient().Do(req) + if err != nil { + return nil, false, fmt.Errorf("Aliyun callback request failed: %w", err) + } + if response.StatusCode >= http.StatusBadRequest { + defer response.Body.Close() + body, readErr := io.ReadAll(io.LimitReader(response.Body, callbackErrorBodyLimit)) + if readErr != nil { + return nil, false, fmt.Errorf("read Aliyun callback error response: %w", readErr) + } + if isCallbackCapacityRejection(response.StatusCode, body) { + return nil, true, nil + } + message := strings.ReplaceAll(strings.TrimSpace(string(body)), url, "") + return nil, false, fmt.Errorf("Aliyun callback request failed: %w; response: %s", anet.HttpStatusCodeError(response.StatusCode), message) + } + if requested.Start == 0 && requested.Length == size || response.StatusCode == http.StatusPartialContent || callbackContentRangeStartsAt(response.Header, requested.Start) { + return response.Body, false, nil + } + if response.StatusCode == http.StatusOK { + body, rangeErr := anet.GetRangedHttpReader(response.Body, requested.Start, requested.Length) + if rangeErr != nil { + response.Body.Close() + return nil, false, rangeErr + } + return body, false, nil + } + return response.Body, false, nil +} + +func isCallbackCapacityRejection(status int, body []byte) bool { + return status == http.StatusForbidden && + strings.Contains(string(body), "RequestDeniedByCallback") && + strings.Contains(string(body), "ExceedMaxConcurrency") +} + +func callbackContentRangeStartsAt(header http.Header, offset int64) bool { + start, _, err := http_range.ParseContentRange(header.Get("Content-Range")) + return err == nil && start == offset +} + +type callbackBody struct { + body io.ReadCloser + release func() + once sync.Once + mu sync.Mutex + stop func() bool +} + +func newCallbackBody(ctx context.Context, body io.ReadCloser, release func()) *callbackBody { + b := &callbackBody{body: body, release: release} + stop := context.AfterFunc(ctx, func() { _ = b.Close() }) + b.mu.Lock() + b.stop = stop + b.mu.Unlock() + return b +} + +func (b *callbackBody) Read(p []byte) (int, error) { + n, err := b.body.Read(p) + if err != nil { + _ = b.Close() + } + return n, err +} + +func (b *callbackBody) Close() error { + var err error + b.once.Do(func() { + b.mu.Lock() + stop := b.stop + b.mu.Unlock() + if stop != nil { + stop() + } + err = b.body.Close() + b.release() + }) + return err +} diff --git a/drivers/aliyundrive_open/callback_test.go b/drivers/aliyundrive_open/callback_test.go new file mode 100644 index 00000000..d16d22cc --- /dev/null +++ b/drivers/aliyundrive_open/callback_test.go @@ -0,0 +1,365 @@ +package aliyundrive_open + +import ( + "context" + "errors" + "fmt" + "io" + "net/http" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/OpenListTeam/OpenList/v4/drivers/base" + "github.com/OpenListTeam/OpenList/v4/internal/conf" + "github.com/OpenListTeam/OpenList/v4/internal/errs" + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/internal/stream" + "github.com/OpenListTeam/OpenList/v4/pkg/http_range" +) + +func TestLinkSeparatesRedirectAndProxyRepresentations(t *testing.T) { + oldConf := conf.Conf + conf.Conf = &conf.Config{} + t.Cleanup(func() { conf.Conf = oldConf }) + base.InitClient() + + var server *httptest.Server + server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/adrive/v1.0/user/getDriveInfo": + _, _ = fmt.Fprint(w, `{"user_id":"user-1","resource_drive_id":"drive-1"}`) + case "/adrive/v1.0/openFile/getDownloadUrl": + _, _ = fmt.Fprintf(w, `{"url":%q}`, server.URL+"/callback") + default: + http.NotFound(w, r) + } + })) + defer server.Close() + + oldAPIURL := API_URL + API_URL = server.URL + defer func() { API_URL = oldAPIURL }() + + d := &AliyundriveOpen{Addition: Addition{AccessToken: "token"}} + if err := d.Init(t.Context()); err != nil { + t.Fatal(err) + } + defer d.Drop(context.Background()) + if d.CallbackConcurrency != defaultCallbackConcurrency { + t.Fatalf("normalized callback concurrency = %d, want %d", d.CallbackConcurrency, defaultCallbackConcurrency) + } + + link, err := d.Link(t.Context(), &model.Object{ID: "file-1", Name: "file", Size: 1}, model.LinkArgs{}) + if err != nil { + t.Fatal(err) + } + if link.RangeReader == nil { + t.Fatal("proxy link must own callback acquisition through a range reader") + } + if _, ok := link.RangeReader.(stream.RateLimitRangeReaderFunc); !ok { + t.Fatalf("proxy range reader type = %T, want server-rate-limited reader", link.RangeReader) + } + direct, err := d.Link(t.Context(), &model.Object{ID: "file-1", Name: "file", Size: 1}, model.LinkArgs{Redirect: true}) + if err != nil { + t.Fatal(err) + } + if direct.URL == "" || direct.RangeReader != nil { + t.Fatal("redirect link must remain URL-only") + } +} + +func TestCallbackRangeHoldsPermitUntilBodyClose(t *testing.T) { + oldConf := conf.Conf + conf.Conf = &conf.Config{} + t.Cleanup(func() { conf.Conf = oldConf }) + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Length", "1") + w.Header().Set("Content-Range", "bytes 0-0/1") + w.WriteHeader(http.StatusPartialContent) + _, _ = io.WriteString(w, "x") + })) + defer server.Close() + + registration := registerCallbackLimiter(t.Name(), 1) + t.Cleanup(registration.unregister) + d := &AliyundriveOpen{callback: registration} + body, err := d.callbackRangeReader(server.URL, 1).RangeRead(t.Context(), http_range.Range{Length: 1}) + if err != nil { + t.Fatal(err) + } + registration.limiter.mu.Lock() + active := registration.limiter.active + registration.limiter.mu.Unlock() + if active != 1 { + t.Fatalf("active callback bodies = %d, want 1", active) + } + if err := body.Close(); err != nil { + t.Fatal(err) + } + registration.limiter.mu.Lock() + active = registration.limiter.active + registration.limiter.mu.Unlock() + if active != 0 { + t.Fatalf("active callback bodies after Close = %d, want 0", active) + } +} + +func TestCallbackLimiterUsesMinimumRegisteredLimit(t *testing.T) { + firstRegistration := registerCallbackLimiter(t.Name(), 2) + t.Cleanup(firstRegistration.unregister) + first, err := firstRegistration.acquire(t.Context()) + if err != nil { + t.Fatal(err) + } + second, err := firstRegistration.acquire(t.Context()) + if err != nil { + t.Fatal(err) + } + defer first.release() + defer second.release() + + lowerRegistration := registerCallbackLimiter(t.Name(), 1) + t.Cleanup(lowerRegistration.unregister) + acquired := make(chan *callbackPermit, 1) + go func() { + permit, acquireErr := lowerRegistration.acquire(t.Context()) + if acquireErr == nil { + acquired <- permit + } + }() + + first.release() + select { + case permit := <-acquired: + permit.release() + t.Fatal("lowering the shared limit must wait for all excess bodies to drain") + case <-time.After(100 * time.Millisecond): + } + second.release() + select { + case permit := <-acquired: + permit.release() + case <-time.After(time.Second): + t.Fatal("admission did not resume after active bodies drained below the new limit") + } +} + +func TestCallbackLimiterSeparatesUsers(t *testing.T) { + firstUser := registerCallbackLimiter(t.Name()+"-first", 1) + secondUser := registerCallbackLimiter(t.Name()+"-second", 1) + t.Cleanup(firstUser.unregister) + t.Cleanup(secondUser.unregister) + first, err := firstUser.acquire(t.Context()) + if err != nil { + t.Fatal(err) + } + defer first.release() + second, err := secondUser.acquire(t.Context()) + if err != nil { + t.Fatalf("independent user was blocked: %v", err) + } + second.release() +} + +func TestCallbackLimiterReconfigureWaitsForOldBodies(t *testing.T) { + userID := t.Name() + oldRegistration := registerCallbackLimiter(userID, 2) + first, err := oldRegistration.acquire(t.Context()) + if err != nil { + t.Fatal(err) + } + second, err := oldRegistration.acquire(t.Context()) + if err != nil { + t.Fatal(err) + } + oldRegistration.unregister() + + newRegistration := registerCallbackLimiter(userID, 1) + t.Cleanup(newRegistration.unregister) + acquired := make(chan *callbackPermit, 1) + go func() { + permit, acquireErr := newRegistration.acquire(t.Context()) + if acquireErr == nil { + acquired <- permit + } + }() + first.release() + select { + case permit := <-acquired: + permit.release() + t.Fatal("reconfigured limiter admitted while an old body still occupied the new limit") + case <-time.After(100 * time.Millisecond): + } + second.release() + select { + case permit := <-acquired: + permit.release() + case <-time.After(time.Second): + t.Fatal("reconfigured limiter did not admit after old bodies drained") + } +} + +func TestCallbackLimiterDistinguishesTimeoutAndCancellation(t *testing.T) { + registration := registerCallbackLimiter(t.Name(), 1) + t.Cleanup(registration.unregister) + permit, err := registration.acquire(t.Context()) + if err != nil { + t.Fatal(err) + } + defer permit.release() + + started := time.Now() + _, err = registration.acquire(t.Context()) + if !errors.Is(err, errs.TemporaryCapacity) { + t.Fatalf("admission timeout error = %v, want TemporaryCapacity", err) + } + if time.Since(started) < callbackAcquireTimeout { + t.Fatal("admission timed out before the configured wait elapsed") + } + + ctx, cancel := context.WithCancel(t.Context()) + cancel() + _, err = registration.acquire(ctx) + if !errors.Is(err, context.Canceled) || errors.Is(err, errs.TemporaryCapacity) { + t.Fatalf("canceled admission error = %v, want only context.Canceled", err) + } +} + +func TestCallbackCapacityRejectionRequiresBothExactMarkers(t *testing.T) { + tests := []struct { + name string + body string + want bool + }{ + {name: "both", body: `{"code":"RequestDeniedByCallback","message":"ExceedMaxConcurrency"}`, want: true}, + {name: "code only", body: `{"code":"RequestDeniedByCallback"}`}, + {name: "message only", body: `{"message":"ExceedMaxConcurrency"}`}, + {name: "case differs", body: `{"code":"requestdeniedbycallback","message":"ExceedMaxConcurrency"}`}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + if got := isCallbackCapacityRejection(http.StatusForbidden, []byte(test.body)); got != test.want { + t.Fatalf("classification = %v, want %v", got, test.want) + } + }) + } + if isCallbackCapacityRejection(http.StatusTooManyRequests, []byte(`RequestDeniedByCallback ExceedMaxConcurrency`)) { + t.Fatal("non-403 response must not be classified as callback capacity") + } +} + +func TestCallbackRangeRetriesOnlyVerifiedCapacityRejections(t *testing.T) { + var requests atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requests.Add(1) + w.WriteHeader(http.StatusForbidden) + _, _ = io.WriteString(w, `{"code":"RequestDeniedByCallback","message":"ExceedMaxConcurrency"}`) + })) + defer server.Close() + + registration := registerCallbackLimiter(t.Name(), 1) + t.Cleanup(registration.unregister) + d := &AliyundriveOpen{callback: registration} + _, err := d.callbackRangeReader(server.URL+"?token=secret", 1).RangeRead(t.Context(), http_range.Range{Length: 1}) + if !errors.Is(err, errs.TemporaryCapacity) { + t.Fatalf("verified rejection error = %v, want TemporaryCapacity", err) + } + if requests.Load() != callbackRequestAttempts { + t.Fatalf("requests = %d, want %d", requests.Load(), callbackRequestAttempts) + } + if strings.Contains(err.Error(), "secret") { + t.Fatal("capacity error leaked the signed callback URL") + } + permit, acquireErr := registration.acquire(t.Context()) + if acquireErr != nil { + t.Fatalf("capacity retries leaked admission: %v", acquireErr) + } + permit.release() + + requests.Store(0) + permanent := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requests.Add(1) + w.WriteHeader(http.StatusForbidden) + _, _ = io.WriteString(w, `{"code":"RequestDeniedByCallback","message":"denied"}`) + })) + defer permanent.Close() + _, err = d.callbackRangeReader(permanent.URL+"?token=secret", 1).RangeRead(t.Context(), http_range.Range{Length: 1}) + if errors.Is(err, errs.TemporaryCapacity) { + t.Fatalf("permanent 403 error = %v, must not be TemporaryCapacity", err) + } + if requests.Load() != 1 { + t.Fatalf("permanent 403 requests = %d, want 1", requests.Load()) + } + if strings.Contains(err.Error(), "secret") { + t.Fatal("permanent error leaked the signed callback URL") + } +} + +type countingReadCloser struct { + reader io.Reader + closed atomic.Int32 +} + +func (r *countingReadCloser) Read(p []byte) (int, error) { return r.reader.Read(p) } +func (r *countingReadCloser) Close() error { + r.closed.Add(1) + return nil +} + +func TestCallbackBodyReleasesExactlyOnce(t *testing.T) { + underlying := &countingReadCloser{reader: strings.NewReader("x")} + var released atomic.Int32 + body := newCallbackBody(t.Context(), underlying, func() { released.Add(1) }) + _, _ = io.ReadAll(body) + if err := body.Close(); err != nil { + t.Fatal(err) + } + if err := body.Close(); err != nil { + t.Fatal(err) + } + if underlying.closed.Load() != 1 || released.Load() != 1 { + t.Fatalf("close count = %d, release count = %d; want 1, 1", underlying.closed.Load(), released.Load()) + } +} + +type failingReadCloser struct { + closed atomic.Int32 +} + +func (*failingReadCloser) Read([]byte) (int, error) { return 0, errors.New("read failed") } +func (r *failingReadCloser) Close() error { + r.closed.Add(1) + return nil +} + +func TestCallbackBodyReadFailureReleasesPermit(t *testing.T) { + underlying := &failingReadCloser{} + var released atomic.Int32 + body := newCallbackBody(t.Context(), underlying, func() { released.Add(1) }) + if _, err := body.Read(make([]byte, 1)); err == nil { + t.Fatal("read unexpectedly succeeded") + } + if underlying.closed.Load() != 1 || released.Load() != 1 { + t.Fatalf("close count = %d, release count = %d; want 1, 1", underlying.closed.Load(), released.Load()) + } +} + +func TestCallbackBodyCancellationReleasesPermit(t *testing.T) { + ctx, cancel := context.WithCancel(t.Context()) + underlying := &countingReadCloser{reader: strings.NewReader("x")} + released := make(chan struct{}, 1) + _ = newCallbackBody(ctx, underlying, func() { released <- struct{}{} }) + cancel() + select { + case <-released: + case <-time.After(time.Second): + t.Fatal("context cancellation did not release callback admission") + } + if underlying.closed.Load() != 1 { + t.Fatalf("underlying close count = %d, want 1", underlying.closed.Load()) + } +} diff --git a/drivers/aliyundrive_open/driver.go b/drivers/aliyundrive_open/driver.go index ee93b330..707eb5b2 100644 --- a/drivers/aliyundrive_open/driver.go +++ b/drivers/aliyundrive_open/driver.go @@ -11,6 +11,7 @@ import ( "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/internal/stream" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/go-resty/resty/v2" log "github.com/sirupsen/logrus" @@ -22,8 +23,9 @@ type AliyundriveOpen struct { DriveId string - limiter *limiter - ref *AliyundriveOpen + limiter *limiter + ref *AliyundriveOpen + callback *callbackRegistration } func (d *AliyundriveOpen) Config() driver.Config { @@ -35,6 +37,7 @@ func (d *AliyundriveOpen) GetAddition() driver.Additional { } func (d *AliyundriveOpen) Init(ctx context.Context) error { + d.CallbackConcurrency = normalizeCallbackConcurrency(d.CallbackConcurrency) d.limiter = getLimiterForUser(globalLimiterUserID) // First create a globally shared limiter to limit the initial requests. if d.LIVPDownloadFormat == "" { d.LIVPDownloadFormat = "jpeg" @@ -52,6 +55,7 @@ func (d *AliyundriveOpen) Init(ctx context.Context) error { userid := utils.Json.Get(res, "user_id").ToString() d.limiter.free() d.limiter = getLimiterForUser(userid) // Allocate a corresponding limiter for each user. + d.callback = registerCallbackLimiter(userid, d.CallbackConcurrency) return nil } @@ -65,6 +69,10 @@ func (d *AliyundriveOpen) InitReference(storage driver.Driver) error { } func (d *AliyundriveOpen) Drop(ctx context.Context) error { + if d.callback != nil { + d.callback.unregister() + d.callback = nil + } d.limiter.free() d.limiter = nil d.ref = nil @@ -119,10 +127,16 @@ func (d *AliyundriveOpen) Link(ctx context.Context, file model.Obj, args model.L url = utils.Json.Get(res, "streamsUrl", d.LIVPDownloadFormat).ToString() } exp := time.Minute - return &model.Link{ + link := &model.Link{ URL: url, Expiration: &exp, - }, nil + } + if args.Redirect { + return link, nil + } + link.URL = "" + link.RangeReader = stream.RateLimitRangeReaderFunc(d.callbackRangeReader(url, file.GetSize())) + return link, nil } func (d *AliyundriveOpen) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) (model.Obj, error) { diff --git a/drivers/aliyundrive_open/meta.go b/drivers/aliyundrive_open/meta.go index d76fc2ea..becfef09 100644 --- a/drivers/aliyundrive_open/meta.go +++ b/drivers/aliyundrive_open/meta.go @@ -8,19 +8,20 @@ import ( type Addition struct { DriveType string `json:"drive_type" type:"select" options:"default,resource,backup" default:"resource"` driver.RootID - RefreshToken string `json:"refresh_token" required:"true"` - OrderBy string `json:"order_by" type:"select" options:"name,size,updated_at,created_at"` - OrderDirection string `json:"order_direction" type:"select" options:"ASC,DESC"` - UseOnlineAPI bool `json:"use_online_api" default:"true"` - AlipanType string `json:"alipan_type" required:"true" type:"select" default:"default" options:"default,alipanTV"` - APIAddress string `json:"api_url_address" default:"https://api.oplist.org/alicloud/renewapi"` - ClientID string `json:"client_id" help:"Keep it empty if you don't have one"` - ClientSecret string `json:"client_secret" help:"Keep it empty if you don't have one"` - RemoveWay string `json:"remove_way" required:"true" type:"select" options:"trash,delete"` - RapidUpload bool `json:"rapid_upload" help:"If you enable this option, the file will be uploaded to the server first, so the progress will be incorrect"` - InternalUpload bool `json:"internal_upload" help:"If you are using Aliyun ECS is located in Beijing, you can turn it on to boost the upload speed"` - LIVPDownloadFormat string `json:"livp_download_format" type:"select" options:"jpeg,mov" default:"jpeg"` - AccessToken string + RefreshToken string `json:"refresh_token" required:"true"` + OrderBy string `json:"order_by" type:"select" options:"name,size,updated_at,created_at"` + OrderDirection string `json:"order_direction" type:"select" options:"ASC,DESC"` + UseOnlineAPI bool `json:"use_online_api" default:"true"` + AlipanType string `json:"alipan_type" required:"true" type:"select" default:"default" options:"default,alipanTV"` + APIAddress string `json:"api_url_address" default:"https://api.oplist.org/alicloud/renewapi"` + ClientID string `json:"client_id" help:"Keep it empty if you don't have one"` + ClientSecret string `json:"client_secret" help:"Keep it empty if you don't have one"` + RemoveWay string `json:"remove_way" required:"true" type:"select" options:"trash,delete"` + RapidUpload bool `json:"rapid_upload" help:"If you enable this option, the file will be uploaded to the server first, so the progress will be incorrect"` + InternalUpload bool `json:"internal_upload" help:"If you are using Aliyun ECS is located in Beijing, you can turn it on to boost the upload speed"` + LIVPDownloadFormat string `json:"livp_download_format" type:"select" options:"jpeg,mov" default:"jpeg"` + CallbackConcurrency int `json:"callback_concurrency" type:"number" default:"1" help:"Maximum active proxied downloads shared by Aliyun user"` + AccessToken string } var config = driver.Config{ diff --git a/drivers/all.go b/drivers/all.go index 4eb7987d..601c7bfc 100644 --- a/drivers/all.go +++ b/drivers/all.go @@ -22,6 +22,7 @@ import ( _ "github.com/OpenListTeam/OpenList/v4/drivers/azure_blob" _ "github.com/OpenListTeam/OpenList/v4/drivers/baidu_netdisk" _ "github.com/OpenListTeam/OpenList/v4/drivers/baidu_photo" + _ "github.com/OpenListTeam/OpenList/v4/drivers/bunny_storage" _ "github.com/OpenListTeam/OpenList/v4/drivers/chaoxing" _ "github.com/OpenListTeam/OpenList/v4/drivers/chunk" _ "github.com/OpenListTeam/OpenList/v4/drivers/cloudflare_imgbed" @@ -41,6 +42,7 @@ import ( _ "github.com/OpenListTeam/OpenList/v4/drivers/github_releases" _ "github.com/OpenListTeam/OpenList/v4/drivers/google_drive" _ "github.com/OpenListTeam/OpenList/v4/drivers/google_photo" + _ "github.com/OpenListTeam/OpenList/v4/drivers/guangyapan" _ "github.com/OpenListTeam/OpenList/v4/drivers/halalcloud" _ "github.com/OpenListTeam/OpenList/v4/drivers/halalcloud_open" _ "github.com/OpenListTeam/OpenList/v4/drivers/ilanzou" diff --git a/drivers/azure_blob/driver.go b/drivers/azure_blob/driver.go index 1c0bb9a9..7e5ce943 100644 --- a/drivers/azure_blob/driver.go +++ b/drivers/azure_blob/driver.go @@ -2,6 +2,7 @@ package azure_blob import ( "context" + "encoding/hex" "fmt" "io" "path" @@ -12,10 +13,13 @@ import ( "github.com/Azure/azure-sdk-for-go/sdk/azcore" "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob" "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/container" "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/sas" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/pkg/utils" + log "github.com/sirupsen/logrus" ) // Azure Blob Storage based on the blob APIs @@ -283,6 +287,26 @@ func (d *AzureBlob) Put(ctx context.Context, dstDir model.Obj, stream model.File // Determine optimal upload options based on file size options := optimizedUploadOptions(stream.GetSize()) + // Preserve the file content type for direct Azure Blob downloads. + contentType := stream.GetMimetype() + if contentType == "" { + contentType = "application/octet-stream" + } + options.HTTPHeaders = &blob.HTTPHeaders{ + BlobContentType: &contentType, + } + + // Preserve the MD5 checksum when it is available. + md5Hex := stream.GetHash().GetHash(utils.MD5) + if md5Hex != "" { + md5, err := hex.DecodeString(md5Hex) + if err == nil && len(md5) == 16 { + options.HTTPHeaders.BlobContentMD5 = md5 + } else { + log.Warnf("Invalid MD5 hash: %s, error: %v", md5Hex, err) + } + } + // Track upload progress progressTracker := &progressTracker{ total: stream.GetSize(), diff --git a/drivers/bunny_storage/driver.go b/drivers/bunny_storage/driver.go new file mode 100644 index 00000000..c097da80 --- /dev/null +++ b/drivers/bunny_storage/driver.go @@ -0,0 +1,226 @@ +package bunny_storage + +import ( + "bytes" + "context" + "fmt" + "net/http" + "net/url" + stdpath "path" + "strings" + "time" + + "github.com/OpenListTeam/OpenList/v4/drivers/base" + "github.com/OpenListTeam/OpenList/v4/internal/driver" + "github.com/OpenListTeam/OpenList/v4/internal/errs" + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/go-resty/resty/v2" +) + +type BunnyStorage struct { + model.Storage + Addition + client *resty.Client + endpoint *url.URL + cdnBase *url.URL +} + +func (d *BunnyStorage) Config() driver.Config { + cfg := config + if d.StorageZoneName != "" && d.CDNBaseURL == "" { + cfg.OnlyProxy = true + cfg.PreferProxy = true + } + if d.CDNTokenKey != "" && d.CDNTokenIncludeIP { + cfg.LinkCacheMode = driver.LinkCacheIP + } + return cfg +} + +func (d *BunnyStorage) GetAddition() driver.Additional { + return &d.Addition +} + +func (d *BunnyStorage) Init(ctx context.Context) error { + if d.RootFolderPath == "" { + d.RootFolderPath = "/" + } + if d.Endpoint == "" { + d.Endpoint = defaultEndpoint + } + if d.SignURLExpire <= 0 { + d.SignURLExpire = 4 + } + if d.CDNTokenMethod == "" { + d.CDNTokenMethod = cdnTokenMethodSHA256 + } + endpoint, err := normalizeBaseURL(d.Endpoint, defaultEndpoint) + if err != nil { + return fmt.Errorf("invalid endpoint: %w", err) + } + d.endpoint = endpoint + if d.CDNBaseURL != "" { + cdnBase, err := normalizeBaseURL(d.CDNBaseURL, "") + if err != nil { + return fmt.Errorf("invalid cdn_base_url: %w", err) + } + d.cdnBase = cdnBase + } + d.client = base.RestyClient + if d.client == nil { + d.client = base.NewRestyClient() + } + return nil +} + +func (d *BunnyStorage) Drop(ctx context.Context) error { + return nil +} + +func (d *BunnyStorage) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { + var items []bunnyObject + resp, err := d.authRequest(). + SetContext(ctx). + SetResult(&items). + Get(d.storageURL(dir.GetPath(), true)) + if err != nil { + return nil, err + } + if err := d.handleResponseError(resp); err != nil { + return nil, err + } + result := make([]model.Obj, 0, len(items)) + placeholder := d.placeholderName() + for _, item := range items { + if item.ObjectName == "" { + continue + } + if !args.S3ShowPlaceholder && !item.IsDirectory && item.ObjectName == placeholder { + continue + } + result = append(result, d.toObj(dir.GetPath(), item)) + } + return result, nil +} + +func (d *BunnyStorage) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { + if file.IsDir() { + return nil, errs.NotFile + } + cacheTTL := time.Duration(0) + if d.cdnBase != nil { + linkURL := d.cdnURL(d.cdnObjectPath(file.GetPath())) + link := &model.Link{ + URL: linkURL, + ContentLength: file.GetSize(), + Expiration: &cacheTTL, + } + if d.CDNTokenKey != "" { + signedURL, _, err := d.signCDNURL(linkURL, args.IP) + if err != nil { + return nil, err + } + link.URL = signedURL + } + return link, nil + } + return &model.Link{ + URL: d.storageURL(file.GetPath(), false), + Header: http.Header{"AccessKey": []string{d.AccessKey}}, + ContentLength: file.GetSize(), + Expiration: &cacheTTL, + }, nil +} + +func (d *BunnyStorage) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) (model.Obj, error) { + dirPath := stdpath.Join(parentDir.GetPath(), dirName) + placeholderPath := stdpath.Join(dirPath, d.placeholderName()) + if err := d.putReader(ctx, placeholderPath, bytes.NewReader(nil), 0, "application/octet-stream", nil); err != nil { + return nil, err + } + now := time.Now() + return &model.Object{ + Path: dirPath, + Name: dirName, + Modified: now, + Ctime: now, + IsFolder: true, + }, nil +} + +func (d *BunnyStorage) Remove(ctx context.Context, obj model.Obj) error { + resp, err := d.authRequest(). + SetContext(ctx). + Delete(d.storageURL(obj.GetPath(), obj.IsDir())) + if err != nil { + return err + } + return d.handleResponseError(resp) +} + +func (d *BunnyStorage) Put(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) { + if up == nil { + up = func(float64) {} + } + dstPath := stdpath.Join(dstDir.GetPath(), file.GetName()) + err := d.putReader(ctx, dstPath, driver.NewLimitedUploadStream(ctx, &driver.ReaderUpdatingProgress{ + Reader: file, + UpdateProgress: up, + }), file.GetSize(), file.GetMimetype(), nil) + if err != nil { + return nil, err + } + now := time.Now() + return &model.Object{ + Path: dstPath, + Name: file.GetName(), + Size: file.GetSize(), + Modified: now, + Ctime: now, + }, nil +} + +func (d *BunnyStorage) putReader(ctx context.Context, path string, body any, size int64, contentType string, extraHeaders http.Header) error { + if contentType == "" { + contentType = "application/octet-stream" + } + req := d.authRequest(). + SetContext(ctx). + SetBody(body). + SetHeader("Content-Type", contentType) + if size >= 0 { + req.SetHeader("Content-Length", fmt.Sprint(size)) + } + for key, values := range extraHeaders { + for _, value := range values { + req.SetHeader(key, value) + } + } + resp, err := req.Put(d.storageURL(path, false)) + if err != nil { + return err + } + return d.handleResponseError(resp) +} + +func (d *BunnyStorage) Get(ctx context.Context, path string) (model.Obj, error) { + fullPath := stdpath.Join(d.GetRootPath(), path) + parentPath, name := stdpath.Split(fullPath) + parentPath = strings.TrimSuffix(parentPath, "/") + if parentPath == "" { + parentPath = "/" + } + objs, err := d.List(ctx, &model.Object{Path: parentPath, IsFolder: true}, model.ListArgs{S3ShowPlaceholder: true}) + if err != nil { + return nil, err + } + for _, obj := range objs { + if obj.GetName() == name { + return obj, nil + } + } + return nil, errs.ObjectNotFound +} + +var _ driver.Driver = (*BunnyStorage)(nil) +var _ driver.Getter = (*BunnyStorage)(nil) diff --git a/drivers/bunny_storage/meta.go b/drivers/bunny_storage/meta.go new file mode 100644 index 00000000..46e13590 --- /dev/null +++ b/drivers/bunny_storage/meta.go @@ -0,0 +1,32 @@ +package bunny_storage + +import ( + "github.com/OpenListTeam/OpenList/v4/internal/driver" + "github.com/OpenListTeam/OpenList/v4/internal/op" +) + +type Addition struct { + driver.RootPath + StorageZoneName string `json:"storage_zone_name" required:"true"` + AccessKey string `json:"access_key" required:"true"` + Endpoint string `json:"endpoint" required:"true" default:"storage.bunnycdn.com"` + CDNBaseURL string `json:"cdn_base_url"` + CDNTokenKey string `json:"cdn_token_key"` + CDNTokenMethod string `json:"cdn_token_method" type:"select" options:"sha256,hmac_sha256" default:"sha256"` + CDNTokenIncludeIP bool `json:"cdn_token_include_ip" default:"false"` + SignURLExpire int `json:"sign_url_expire" type:"number" default:"4"` + Placeholder string `json:"placeholder" default:".openlist"` +} + +var config = driver.Config{ + Name: "Bunny Storage", + LocalSort: true, + DefaultRoot: "/", + CheckStatus: true, +} + +func init() { + op.RegisterDriver(func() driver.Driver { + return &BunnyStorage{} + }) +} diff --git a/drivers/bunny_storage/types.go b/drivers/bunny_storage/types.go new file mode 100644 index 00000000..0a4d1016 --- /dev/null +++ b/drivers/bunny_storage/types.go @@ -0,0 +1,27 @@ +package bunny_storage + +import "time" + +type bunnyObject struct { + Guid string `json:"Guid"` + StorageZoneName string `json:"StorageZoneName"` + Path string `json:"Path"` + ObjectName string `json:"ObjectName"` + Length int64 `json:"Length"` + LastChanged string `json:"LastChanged"` + IsDirectory bool `json:"IsDirectory"` + ServerID int `json:"ServerId"` + UserID string `json:"UserId"` + DateCreated string `json:"DateCreated"` + StorageZoneID int64 `json:"StorageZoneId"` +} + +type apiError struct { + HttpCode int `json:"HttpCode"` + Message string `json:"Message"` +} + +type parsedTimes struct { + modified time.Time + created time.Time +} diff --git a/drivers/bunny_storage/util.go b/drivers/bunny_storage/util.go new file mode 100644 index 00000000..dd8b5da6 --- /dev/null +++ b/drivers/bunny_storage/util.go @@ -0,0 +1,275 @@ +package bunny_storage + +import ( + "crypto/hmac" + "crypto/sha256" + "encoding/base64" + "encoding/json" + "fmt" + "net/http" + "net/url" + stdpath "path" + "sort" + "strconv" + "strings" + "time" + + "github.com/OpenListTeam/OpenList/v4/internal/errs" + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/go-resty/resty/v2" +) + +const ( + defaultEndpoint = "storage.bunnycdn.com" + defaultPlaceholder = ".openlist" + + cdnTokenMethodSHA256 = "sha256" + cdnTokenMethodHMACSHA256 = "hmac_sha256" +) + +func normalizeBaseURL(raw string, fallback string) (*url.URL, error) { + raw = strings.TrimSpace(raw) + if raw == "" { + raw = fallback + } + if raw == "" { + return nil, fmt.Errorf("empty url") + } + if !strings.Contains(raw, "://") { + raw = "https://" + raw + } + u, err := url.Parse(raw) + if err != nil { + return nil, err + } + if u.Host == "" { + return nil, fmt.Errorf("invalid url: %s", raw) + } + u.Path = strings.TrimRight(u.Path, "/") + return u, nil +} + +func cleanObjectPath(path string) string { + if path == "" { + return "/" + } + return stdpath.Clean("/" + strings.TrimPrefix(path, "/")) +} + +func stripObjectPathPrefix(path string, prefix string) (string, bool) { + path = cleanObjectPath(path) + prefix = cleanObjectPath(prefix) + if prefix == "/" { + return path, false + } + if path == prefix { + return "/", true + } + if strings.HasPrefix(path, prefix+"/") { + return cleanObjectPath(strings.TrimPrefix(path, prefix)), true + } + return path, false +} + +func isObjectPathOrChild(path string, parent string) bool { + path = cleanObjectPath(path) + parent = cleanObjectPath(parent) + return path == parent || strings.HasPrefix(path, parent+"/") +} + +func trimCDNBasePath(path string, mountPath string) string { + path = cleanObjectPath(path) + if path == "/" { + return "" + } + if stripped, ok := stripObjectPathPrefix(path, mountPath); ok { + path = stripped + } + if path == "/" { + return "" + } + return strings.TrimRight(path, "/") +} + +func (d *BunnyStorage) cdnObjectPath(path string) string { + objectPath := cleanObjectPath(path) + if stripped, ok := stripObjectPathPrefix(objectPath, d.GetStorage().MountPath); ok { + objectPath = stripped + } + rootPath := cleanObjectPath(d.GetRootPath()) + if rootPath != "/" && !isObjectPathOrChild(objectPath, rootPath) { + objectPath = cleanObjectPath(stdpath.Join(rootPath, objectPath)) + } + return objectPath +} + +func (d *BunnyStorage) placeholderName() string { + if d.Placeholder == "" { + return defaultPlaceholder + } + return d.Placeholder +} + +func (d *BunnyStorage) storageURL(path string, dir bool) string { + u := *d.endpoint + cleanPath := cleanObjectPath(path) + zone := strings.Trim(d.StorageZoneName, "/") + if cleanPath == "/" { + u.Path = "/" + zone + "/" + return u.String() + } + u.Path = "/" + zone + "/" + strings.TrimPrefix(cleanPath, "/") + if dir && !strings.HasSuffix(u.Path, "/") { + u.Path += "/" + } + return u.String() +} + +func (d *BunnyStorage) cdnURL(path string) string { + u := *d.cdnBase + cleanPath := cleanObjectPath(path) + basePath := trimCDNBasePath(u.Path, d.GetStorage().MountPath) + if cleanPath == "/" { + if basePath == "" { + u.Path = "/" + } else { + u.Path = basePath + "/" + } + return u.String() + } + u.Path = basePath + "/" + strings.TrimPrefix(cleanPath, "/") + return u.String() +} + +func (d *BunnyStorage) authRequest() *resty.Request { + return d.client.R().SetHeader("AccessKey", d.AccessKey) +} + +func (d *BunnyStorage) handleResponseError(resp *resty.Response) error { + if resp == nil { + return fmt.Errorf("empty response") + } + if resp.StatusCode() >= http.StatusOK && resp.StatusCode() < http.StatusMultipleChoices { + return nil + } + message := strings.TrimSpace(resp.String()) + var apiErrors []apiError + if err := json.Unmarshal(resp.Body(), &apiErrors); err == nil && len(apiErrors) > 0 && apiErrors[0].Message != "" { + message = apiErrors[0].Message + } + switch resp.StatusCode() { + case http.StatusUnauthorized, http.StatusForbidden: + return errs.NewErr(errs.PermissionDenied, "bunny storage request failed: %s", message) + case http.StatusNotFound: + return errs.NewErr(errs.ObjectNotFound, "bunny storage request failed: %s", message) + default: + return fmt.Errorf("bunny storage request failed: %s: %s", resp.Status(), message) + } +} + +func (d *BunnyStorage) parseTimes(item bunnyObject) parsedTimes { + return parsedTimes{ + modified: parseBunnyTime(item.LastChanged, d.Modified), + created: parseBunnyTime(item.DateCreated, time.Time{}), + } +} + +func parseBunnyTime(value string, fallback time.Time) time.Time { + if value == "" { + return fallback + } + if t, err := time.Parse(time.RFC3339Nano, value); err == nil { + return t + } + if t, err := time.Parse("2006-01-02T15:04:05.999999999", value); err == nil { + return t + } + if t, err := time.Parse("2006-01-02T15:04:05", value); err == nil { + return t + } + return fallback +} + +func (d *BunnyStorage) toObj(parentPath string, item bunnyObject) model.Obj { + times := d.parseTimes(item) + return &model.Object{ + ID: item.Guid, + Path: stdpath.Join(parentPath, item.ObjectName), + Name: item.ObjectName, + Size: item.Length, + Modified: times.modified, + Ctime: times.created, + IsFolder: item.IsDirectory, + } +} + +func canonicalQuery(values url.Values) (string, error) { + keys := make([]string, 0, len(values)) + for key := range values { + if key == "token" || key == "expires" { + continue + } + keys = append(keys, key) + } + sort.Strings(keys) + parts := make([]string, 0, len(keys)) + for _, key := range keys { + vals := values[key] + if len(vals) > 1 { + return "", fmt.Errorf("duplicate query parameter %q is not supported", key) + } + value := "" + if len(vals) == 1 { + value = vals[0] + } + parts = append(parts, key+"="+value) + } + return strings.Join(parts, "&"), nil +} + +func (d *BunnyStorage) signCDNURL(rawURL string, clientIP string) (string, time.Duration, error) { + return d.signCDNURLAt(rawURL, clientIP, time.Now()) +} + +func (d *BunnyStorage) signCDNURLAt(rawURL string, clientIP string, now time.Time) (string, time.Duration, error) { + expire := time.Hour * time.Duration(d.SignURLExpire) + if expire <= 0 { + expire = 4 * time.Hour + } + expires := now.Add(expire).Unix() + u, err := url.Parse(rawURL) + if err != nil { + return "", 0, err + } + query := u.Query() + parameterData, err := canonicalQuery(query) + if err != nil { + return "", 0, err + } + signaturePath, err := url.PathUnescape(u.EscapedPath()) + if err != nil { + signaturePath = u.Path + } + if !d.CDNTokenIncludeIP { + clientIP = "" + } + token := d.signCDNToken(signaturePath, strconv.FormatInt(expires, 10), parameterData, clientIP) + query.Set("token", token) + query.Set("expires", strconv.FormatInt(expires, 10)) + u.RawQuery = query.Encode() + return u.String(), expire, nil +} + +func (d *BunnyStorage) signCDNToken(signaturePath string, expires string, parameterData string, clientIP string) string { + switch strings.ToLower(strings.TrimSpace(d.CDNTokenMethod)) { + case cdnTokenMethodHMACSHA256: + message := signaturePath + expires + parameterData + clientIP + mac := hmac.New(sha256.New, []byte(d.CDNTokenKey)) + _, _ = mac.Write([]byte(message)) + return "HS256-" + base64.RawURLEncoding.EncodeToString(mac.Sum(nil)) + default: + hashableBase := d.CDNTokenKey + signaturePath + expires + parameterData + clientIP + sum := sha256.Sum256([]byte(hashableBase)) + return base64.RawURLEncoding.EncodeToString(sum[:]) + } +} diff --git a/drivers/bunny_storage/util_test.go b/drivers/bunny_storage/util_test.go new file mode 100644 index 00000000..ba9fe0b7 --- /dev/null +++ b/drivers/bunny_storage/util_test.go @@ -0,0 +1,255 @@ +package bunny_storage + +import ( + "context" + "net/url" + "testing" + "time" + + "github.com/OpenListTeam/OpenList/v4/internal/model" +) + +func TestStorageURL(t *testing.T) { + endpoint, err := normalizeBaseURL("ny.storage.bunnycdn.com", defaultEndpoint) + if err != nil { + t.Fatal(err) + } + driver := &BunnyStorage{ + Addition: Addition{ + StorageZoneName: "my-zone", + }, + endpoint: endpoint, + } + if got, want := driver.storageURL("/", true), "https://ny.storage.bunnycdn.com/my-zone/"; got != want { + t.Fatalf("root list url = %q, want %q", got, want) + } + if got, want := driver.storageURL("/dir/a file.txt", false), "https://ny.storage.bunnycdn.com/my-zone/dir/a%20file.txt"; got != want { + t.Fatalf("file url = %q, want %q", got, want) + } + if got, want := driver.storageURL("/dir", true), "https://ny.storage.bunnycdn.com/my-zone/dir/"; got != want { + t.Fatalf("dir url = %q, want %q", got, want) + } +} + +func TestCDNURLWithBasePath(t *testing.T) { + cdnBase, err := normalizeBaseURL("https://cdn.example.com/prefix/", "") + if err != nil { + t.Fatal(err) + } + driver := &BunnyStorage{cdnBase: cdnBase} + if got, want := driver.cdnURL("/dir/a file.txt"), "https://cdn.example.com/prefix/dir/a%20file.txt"; got != want { + t.Fatalf("cdn url = %q, want %q", got, want) + } +} + +func TestCDNURLUsesObjectPathWithoutMountPath(t *testing.T) { + cdnBase, err := normalizeBaseURL("https://cdn.firmant.me", "") + if err != nil { + t.Fatal(err) + } + driver := &BunnyStorage{ + Storage: model.Storage{MountPath: "/BS"}, + cdnBase: cdnBase, + } + if got, want := driver.cdnURL(driver.cdnObjectPath("/BS/Video")), "https://cdn.firmant.me/Video"; got != want { + t.Fatalf("cdn url = %q, want %q", got, want) + } +} + +func TestCDNURLDropsMountPathFromBaseURL(t *testing.T) { + cdnBase, err := normalizeBaseURL("https://cdn.firmant.me/BS/", "") + if err != nil { + t.Fatal(err) + } + driver := &BunnyStorage{ + Storage: model.Storage{MountPath: "/BS"}, + cdnBase: cdnBase, + } + if got, want := driver.cdnURL(driver.cdnObjectPath("/Video")), "https://cdn.firmant.me/Video"; got != want { + t.Fatalf("cdn url = %q, want %q", got, want) + } +} + +func TestCDNObjectPathKeepsRootFolderPath(t *testing.T) { + driver := &BunnyStorage{} + driver.MountPath = "/BS" + driver.RootFolderPath = "/library" + if got, want := driver.cdnObjectPath("/BS/Video"), "/library/Video"; got != want { + t.Fatalf("cdn object path = %q, want %q", got, want) + } + if got, want := driver.cdnObjectPath("/library/Video"), "/library/Video"; got != want { + t.Fatalf("cdn object path = %q, want %q", got, want) + } +} + +func TestLinkDisablesLongLivedCache(t *testing.T) { + cdnBase, err := normalizeBaseURL("https://cdn.example.com", "") + if err != nil { + t.Fatal(err) + } + driver := &BunnyStorage{cdnBase: cdnBase} + link, err := driver.Link(context.Background(), &model.Object{ + Path: "/video.mp4", + Name: "video.mp4", + Size: 123, + }, model.LinkArgs{}) + if err != nil { + t.Fatal(err) + } + if link.Expiration == nil || *link.Expiration != 0 { + t.Fatalf("link expiration = %v, want immediate cache expiration", link.Expiration) + } +} + +func TestSignCDNURL(t *testing.T) { + driver := &BunnyStorage{ + Addition: Addition{ + CDNTokenKey: "secret", + CDNTokenIncludeIP: true, + SignURLExpire: 1, + }, + } + signed, expire, err := driver.signCDNURLAt("https://zone.b-cdn.net/video.mp4?quality=high", "192.0.2.1", time.Unix(1700000000, 0)) + if err != nil { + t.Fatal(err) + } + if expire <= 0 { + t.Fatal("expected positive expiration") + } + parsed, err := url.Parse(signed) + if err != nil { + t.Fatal(err) + } + token := parsed.Query().Get("token") + if want := "FxSpFem88zFo6uHFziwTuoMQTgDaD2PEn5n1zTMBUBI"; token != want { + t.Fatalf("token = %q, want %q", token, want) + } + if parsed.Query().Get("expires") != "1700003600" { + t.Fatalf("expires = %q, want 1700003600", parsed.Query().Get("expires")) + } + if parsed.Query().Get("quality") != "high" { + t.Fatal("expected existing query parameters to be preserved") + } +} + +func TestSignCDNURLSupportsHMACSHA256(t *testing.T) { + driver := &BunnyStorage{ + Addition: Addition{ + CDNTokenKey: "secret", + CDNTokenMethod: cdnTokenMethodHMACSHA256, + CDNTokenIncludeIP: true, + SignURLExpire: 1, + }, + } + signed, _, err := driver.signCDNURLAt("https://zone.b-cdn.net/video.mp4?quality=high", "192.0.2.1", time.Unix(1700000000, 0)) + if err != nil { + t.Fatal(err) + } + parsed, err := url.Parse(signed) + if err != nil { + t.Fatal(err) + } + if got, want := parsed.Query().Get("token"), "HS256-sdrSSJE2JVwhSk2AoDUrmTV1muH6R5UHpZVcVfHeNxg"; got != want { + t.Fatalf("token = %q, want %q", got, want) + } +} + +func TestSignCDNURLUsesDecodedPathForSHA256(t *testing.T) { + driver := &BunnyStorage{ + Addition: Addition{ + CDNTokenKey: "secret", + SignURLExpire: 1, + }, + } + signed, _, err := driver.signCDNURLAt("https://zone.b-cdn.net/%E8%A7%86%E9%A2%91/%5Ba%20b%5D.mp4", "", time.Unix(1700000000, 0)) + if err != nil { + t.Fatal(err) + } + parsed, err := url.Parse(signed) + if err != nil { + t.Fatal(err) + } + if got, want := parsed.Query().Get("token"), "yq1evD7klw0e3DjCbv8dJptbW4S4JwVW3GKLnxfeKGM"; got != want { + t.Fatalf("token = %q, want %q", got, want) + } +} + +func TestSignCDNURLTreatsPlusAsLiteralPathCharacter(t *testing.T) { + driver := &BunnyStorage{ + Addition: Addition{ + CDNTokenKey: "secret", + SignURLExpire: 1, + }, + } + now := time.Unix(1700000000, 0) + literal, _, err := driver.signCDNURLAt("https://zone.b-cdn.net/a+b.mp4", "", now) + if err != nil { + t.Fatal(err) + } + encoded, _, err := driver.signCDNURLAt("https://zone.b-cdn.net/a%2Bb.mp4", "", now) + if err != nil { + t.Fatal(err) + } + space, _, err := driver.signCDNURLAt("https://zone.b-cdn.net/a%20b.mp4", "", now) + if err != nil { + t.Fatal(err) + } + literalURL, err := url.Parse(literal) + if err != nil { + t.Fatal(err) + } + encodedURL, err := url.Parse(encoded) + if err != nil { + t.Fatal(err) + } + spaceURL, err := url.Parse(space) + if err != nil { + t.Fatal(err) + } + if got, want := literalURL.Query().Get("token"), encodedURL.Query().Get("token"); got != want { + t.Fatalf("literal plus token = %q, encoded plus token = %q", got, want) + } + if got, notWant := literalURL.Query().Get("token"), spaceURL.Query().Get("token"); got == notWant { + t.Fatalf("literal plus token = %q, space token should differ", got) + } +} + +func TestSignCDNURLRejectsDuplicateQueryParameters(t *testing.T) { + driver := &BunnyStorage{ + Addition: Addition{ + CDNTokenKey: "secret", + SignURLExpire: 1, + }, + } + _, _, err := driver.signCDNURLAt("https://zone.b-cdn.net/video.mp4?quality=high&quality=low", "", time.Unix(1700000000, 0)) + if err == nil { + t.Fatal("expected duplicate query parameters to be rejected") + } + if got, want := err.Error(), `duplicate query parameter "quality" is not supported`; got != want { + t.Fatalf("error = %q, want %q", got, want) + } +} + +func TestParseBunnyTimeSupportsFractionalSecondsWithoutTimezone(t *testing.T) { + fallback := time.Unix(1, 0) + got := parseBunnyTime("2023-03-21T13:38:31.693", fallback) + want := time.Date(2023, 3, 21, 13, 38, 31, 693000000, time.UTC) + if !got.Equal(want) { + t.Fatalf("parsed time = %s, want %s", got.Format(time.RFC3339Nano), want.Format(time.RFC3339Nano)) + } +} + +func TestConfigProxyMode(t *testing.T) { + registeredConfig := (&BunnyStorage{}).Config() + if registeredConfig.OnlyProxy { + t.Fatal("driver registration config should allow users to choose proxy policy") + } + withoutCDN := (&BunnyStorage{Addition: Addition{StorageZoneName: "my-zone"}}).Config() + if !withoutCDN.OnlyProxy { + t.Fatal("storage API links require AccessKey headers and should be proxied without CDN") + } + withCDN := (&BunnyStorage{Addition: Addition{StorageZoneName: "my-zone", CDNBaseURL: "https://zone.b-cdn.net"}}).Config() + if withCDN.OnlyProxy { + t.Fatal("CDN links should be allowed to redirect directly") + } +} diff --git a/drivers/chaoxing/driver.go b/drivers/chaoxing/driver.go index dfd25d19..36fd0bde 100644 --- a/drivers/chaoxing/driver.go +++ b/drivers/chaoxing/driver.go @@ -55,13 +55,13 @@ func (d *ChaoXing) refreshCookie() error { func (d *ChaoXing) Init(ctx context.Context) error { err := d.refreshCookie() if err != nil { - log.Errorf(ctx, err.Error()) + log.Errorf(ctx, "%v", err) } d.cron = cron.NewCron(time.Hour * 12) d.cron.Do(func() { err = d.refreshCookie() if err != nil { - log.Errorf(ctx, err.Error()) + log.Errorf(ctx, "%v", err) } }) return nil diff --git a/drivers/crypt/driver.go b/drivers/crypt/driver.go index 9e8b5c5c..7dca8428 100644 --- a/drivers/crypt/driver.go +++ b/drivers/crypt/driver.go @@ -251,7 +251,7 @@ func (d *Crypt) Link(ctx context.Context, file model.Obj, _ model.LinkArgs) (*mo rrf, err := stream.GetRangeReaderFromLink(remoteSize, remoteLink) if err != nil { _ = remoteLink.Close() - return nil, fmt.Errorf("the remote storage driver need to be enhanced to support encrytion") + return nil, fmt.Errorf("the remote storage driver need to be enhanced to support encryption") } mu := &sync.Mutex{} diff --git a/drivers/emby/driver.go b/drivers/emby/driver.go index b2406aa6..39122d43 100644 --- a/drivers/emby/driver.go +++ b/drivers/emby/driver.go @@ -56,8 +56,8 @@ func (d *Emby) Init(ctx context.Context) error { return nil } - if strings.TrimSpace(d.Username) == "" || strings.TrimSpace(d.Password) == "" { - return fmt.Errorf("please provide api_key+user_id or username+password") + if strings.TrimSpace(d.Username) == "" { + return fmt.Errorf("please provide api_key+user_id or username(+password)") } if err := d.login(ctx); err != nil { diff --git a/drivers/github/driver.go b/drivers/github/driver.go index 7c35aee1..adf4a002 100644 --- a/drivers/github/driver.go +++ b/drivers/github/driver.go @@ -154,15 +154,17 @@ func (d *Github) List(ctx context.Context, dir model.Obj, args model.ListArgs) ( } } return ret, nil - } else { - ret := make([]model.Obj, 0, len(obj.Entries)) - for _, entry := range obj.Entries { - if entry.Name != ".gitkeep" { - ret = append(ret, entry.toModelObj()) - } - } - return ret, nil } + + ret := make([]model.Obj, 0, len(obj.Entries)) + for _, entry := range obj.Entries { + if entry.Name == ".gitkeep" { + continue + } + ret = append(ret, entry.toModelObj()) + } + d.fetchAccurateModifiedTimes(ctx, dir.GetPath(), ret) + return ret, nil } func (d *Github) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { diff --git a/drivers/github/meta.go b/drivers/github/meta.go index ae313dcc..366d3850 100644 --- a/drivers/github/meta.go +++ b/drivers/github/meta.go @@ -7,23 +7,24 @@ import ( type Addition struct { driver.RootPath - Token string `json:"token" type:"string" required:"true"` - Owner string `json:"owner" type:"string" required:"true"` - Repo string `json:"repo" type:"string" required:"true"` - Ref string `json:"ref" type:"string" help:"A branch, a tag or a commit SHA, main branch by default."` - GitHubProxy string `json:"gh_proxy" type:"string" help:"GitHub proxy, e.g. https://ghproxy.net/raw.githubusercontent.com or https://gh-proxy.com/raw.githubusercontent.com"` - GPGPrivateKey string `json:"gpg_private_key" type:"text"` - GPGKeyPassphrase string `json:"gpg_key_passphrase" type:"string"` - CommitterName string `json:"committer_name" type:"string"` - CommitterEmail string `json:"committer_email" type:"string"` - AuthorName string `json:"author_name" type:"string"` - AuthorEmail string `json:"author_email" type:"string"` - MkdirCommitMsg string `json:"mkdir_commit_message" type:"text" default:"{{.UserName}} mkdir {{.ObjPath}}"` - DeleteCommitMsg string `json:"delete_commit_message" type:"text" default:"{{.UserName}} remove {{.ObjPath}}"` - PutCommitMsg string `json:"put_commit_message" type:"text" default:"{{.UserName}} upload {{.ObjPath}}"` - RenameCommitMsg string `json:"rename_commit_message" type:"text" default:"{{.UserName}} rename {{.ObjPath}} to {{.TargetName}}"` - CopyCommitMsg string `json:"copy_commit_message" type:"text" default:"{{.UserName}} copy {{.ObjPath}} to {{.TargetPath}}"` - MoveCommitMsg string `json:"move_commit_message" type:"text" default:"{{.UserName}} move {{.ObjPath}} to {{.TargetPath}}"` + Token string `json:"token" type:"string" required:"true"` + Owner string `json:"owner" type:"string" required:"true"` + Repo string `json:"repo" type:"string" required:"true"` + Ref string `json:"ref" type:"string" help:"A branch, a tag or a commit SHA, main branch by default."` + AccurateModifiedTime bool `json:"accurate_modified_time" type:"bool" default:"false" help:"Best-effort accurate modified time for small directory listings. Default disabled. Adds one GitHub GraphQL request and falls back to legacy zero-time values on failure."` + GitHubProxy string `json:"gh_proxy" type:"string" help:"GitHub proxy, e.g. https://ghproxy.net/raw.githubusercontent.com or https://gh-proxy.com/raw.githubusercontent.com"` + GPGPrivateKey string `json:"gpg_private_key" type:"text"` + GPGKeyPassphrase string `json:"gpg_key_passphrase" type:"string"` + CommitterName string `json:"committer_name" type:"string"` + CommitterEmail string `json:"committer_email" type:"string"` + AuthorName string `json:"author_name" type:"string"` + AuthorEmail string `json:"author_email" type:"string"` + MkdirCommitMsg string `json:"mkdir_commit_message" type:"text" default:"{{.UserName}} mkdir {{.ObjPath}}"` + DeleteCommitMsg string `json:"delete_commit_message" type:"text" default:"{{.UserName}} remove {{.ObjPath}}"` + PutCommitMsg string `json:"put_commit_message" type:"text" default:"{{.UserName}} upload {{.ObjPath}}"` + RenameCommitMsg string `json:"rename_commit_message" type:"text" default:"{{.UserName}} rename {{.ObjPath}} to {{.TargetName}}"` + CopyCommitMsg string `json:"copy_commit_message" type:"text" default:"{{.UserName}} copy {{.ObjPath}} to {{.TargetPath}}"` + MoveCommitMsg string `json:"move_commit_message" type:"text" default:"{{.UserName}} move {{.ObjPath}} to {{.TargetPath}}"` } var config = driver.Config{ diff --git a/drivers/github/mtime.go b/drivers/github/mtime.go new file mode 100644 index 00000000..f2ec3975 --- /dev/null +++ b/drivers/github/mtime.go @@ -0,0 +1,106 @@ +package github + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "strings" + "time" + + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/pkg/utils" + log "github.com/sirupsen/logrus" +) + +const ( + mtimeMaxEntries = 200 + githubGraphQLEndpoint = "https://api.github.com/graphql" +) + +var githubZeroTime = time.Unix(0, 0) + +type graphQLHistory struct { + Nodes []struct { + CommittedDate time.Time `json:"committedDate"` + } `json:"nodes"` +} + +type graphQLMtimeResponse struct { + Data struct { + Repository struct { + Commit map[string]graphQLHistory `json:"commit"` + } `json:"repository"` + } `json:"data"` + Errors []struct{} `json:"errors"` +} + +func quoteGraphQLString(value string) string { + quoted, _ := json.Marshal(value) + return string(quoted) +} + +func buildMtimeQuery(owner, repo, ref string, objs []model.Obj) string { + histories := make([]string, 0, len(objs)) + for i, obj := range objs { + path := strings.TrimPrefix(obj.GetPath(), "/") + histories = append(histories, fmt.Sprintf(`p%d: history(first: 1, path: %s) { nodes { committedDate } }`, i, quoteGraphQLString(path))) + } + + return fmt.Sprintf(`query { + repository(owner: %s, name: %s) { + commit: object(expression: %s) { + ... on Commit { + %s + } + } + } +}`, + quoteGraphQLString(owner), + quoteGraphQLString(repo), + quoteGraphQLString(ref+"^{commit}"), + strings.Join(histories, "\n\t\t\t\t"), + ) +} + +func (d *Github) fetchAccurateModifiedTimes(ctx context.Context, dirPath string, objs []model.Obj) { + token := strings.TrimSpace(d.Token) + if !d.AccurateModifiedTime || token == "" || len(objs) == 0 || len(objs) > mtimeMaxEntries { + return + } + + res, err := d.client.R(). + SetContext(ctx). + SetHeader("Accept", "application/vnd.github+json"). + SetHeader("Authorization", "Bearer "+token). + SetBody(map[string]string{"query": buildMtimeQuery(d.Owner, d.Repo, d.Ref, objs)}). + Post(githubGraphQLEndpoint) + if err != nil { + log.WithError(err).Warnf("github accurate mtime failed for %s: transport", dirPath) + return + } + if res.StatusCode() != http.StatusOK { + log.Warnf("github accurate mtime failed for %s: http_%d", dirPath, res.StatusCode()) + return + } + + var response graphQLMtimeResponse + if err := utils.Json.Unmarshal(res.Body(), &response); err != nil { + log.WithError(err).Warnf("github accurate mtime failed for %s: graphql", dirPath) + return + } + if len(response.Errors) > 0 || response.Data.Repository.Commit == nil { + log.Warnf("github accurate mtime failed for %s: graphql", dirPath) + return + } + + for i, obj := range objs { + history := response.Data.Repository.Commit[fmt.Sprintf("p%d", i)] + if len(history.Nodes) == 0 { + continue + } + if raw, ok := obj.(*model.Object); ok { + raw.Modified = history.Nodes[0].CommittedDate + } + } +} diff --git a/drivers/github/mtime_test.go b/drivers/github/mtime_test.go new file mode 100644 index 00000000..a6b2bdc6 --- /dev/null +++ b/drivers/github/mtime_test.go @@ -0,0 +1,378 @@ +package github + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "strings" + "testing" + "time" + + odriver "github.com/OpenListTeam/OpenList/v4/internal/driver" + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/internal/op" + "github.com/go-resty/resty/v2" +) + +func TestDriverInfoIncludesAccurateModifiedTimeDefault(t *testing.T) { + info := op.GetDriverInfoMap()["GitHub API"] + for _, item := range info.Additional { + if item.Name != "accurate_modified_time" { + continue + } + if item.Default != "false" { + t.Fatalf("unexpected default: %q", item.Default) + } + return + } + t.Fatal("accurate_modified_time item not registered") +} + +type roundTripFunc func(*http.Request) (*http.Response, error) + +func (f roundTripFunc) RoundTrip(r *http.Request) (*http.Response, error) { + return f(r) +} + +func newGithubTestDriver(rt roundTripFunc, token string, enabled bool) *Github { + return &Github{ + Storage: model.Storage{MountPath: "/github-test", CacheExpiration: 10}, + Addition: Addition{ + RootPath: odriver.RootPath{RootFolderPath: "/"}, + Token: token, + Owner: "owner", + Repo: "repo", + Ref: "main", + AccurateModifiedTime: enabled, + }, + client: resty.New().SetTransport(rt), + } +} + +func newJSONResponse(status int, body string) *http.Response { + return &http.Response{ + StatusCode: status, + Header: make(http.Header), + Body: io.NopCloser(strings.NewReader(body)), + } +} + +func mustJSON(t *testing.T, value any) string { + t.Helper() + data, err := json.Marshal(value) + if err != nil { + t.Fatalf("marshal json: %v", err) + } + return string(data) +} + +func graphQLQueryFromRequest(t *testing.T, r *http.Request) string { + t.Helper() + body, err := io.ReadAll(r.Body) + if err != nil { + t.Fatalf("read graphql request body: %v", err) + } + var payload map[string]string + if err := json.Unmarshal(body, &payload); err != nil { + t.Fatalf("decode graphql request body: %v", err) + } + query := payload["query"] + if query == "" { + t.Fatalf("graphql request missing query: %s", string(body)) + } + return query +} + +func newContentsPayload(t *testing.T, entries []Object) string { + t.Helper() + return mustJSON(t, map[string]any{ + "type": "dir", + "sha": "tree-sha", + "entries": entries, + }) +} + +func newTreePayload(t *testing.T, sha string, trees []TreeObjResp) string { + t.Helper() + return mustJSON(t, map[string]any{ + "sha": sha, + "truncated": false, + "tree": trees, + }) +} + +func newCommitGraphQLPayload(t *testing.T, histories map[string][]string) string { + t.Helper() + commit := make(map[string]any, len(histories)) + for alias, dates := range histories { + nodes := make([]map[string]string, 0, len(dates)) + for _, date := range dates { + nodes = append(nodes, map[string]string{"committedDate": date}) + } + commit[alias] = map[string]any{"nodes": nodes} + } + return mustJSON(t, map[string]any{ + "data": map[string]any{ + "repository": map[string]any{"commit": commit}, + }, + }) +} + +func newSequentialEntries(count int) []Object { + entries := make([]Object, 0, count) + for i := range count { + name := fmt.Sprintf("%03d.md", i) + entries = append(entries, Object{Name: name, Path: "docs/" + name, Type: "file", Size: 1}) + } + return entries +} + +func mustObject(t *testing.T, obj model.Obj) *model.Object { + t.Helper() + raw, ok := model.UnwrapObjName(obj).(*model.Object) + if !ok { + t.Fatalf("unexpected obj type %T", obj) + } + return raw +} + +func TestListAppliesAccurateModifiedTimeInOneRequest(t *testing.T) { + stamp := time.Date(2025, 12, 22, 4, 52, 41, 0, time.UTC) + entries := []Object{ + {Name: "a.md", Path: "docs/a.md", Type: "file", Size: 1}, + {Name: ".gitkeep", Path: "docs/.gitkeep", Type: "file"}, + {Name: `quote " 文.md`, Path: `docs/quote " 文.md`, Type: "file", Size: 1}, + {Name: "control.md", Path: "docs/control\x01.md", Type: "file", Size: 1}, + } + graphqlCalls := 0 + var query string + drv := newGithubTestDriver(roundTripFunc(func(r *http.Request) (*http.Response, error) { + switch { + case r.Method == http.MethodGet && strings.Contains(r.URL.Path, "/contents/"): + return newJSONResponse(http.StatusOK, newContentsPayload(t, entries)), nil + case r.Method == http.MethodPost && r.URL.String() == githubGraphQLEndpoint: + graphqlCalls++ + query = graphQLQueryFromRequest(t, r) + if got := r.Header.Get("Authorization"); got != "Bearer token" { + t.Fatalf("unexpected authorization header: %q", got) + } + return newJSONResponse(http.StatusOK, newCommitGraphQLPayload(t, map[string][]string{ + "p0": {stamp.Format(time.RFC3339)}, + "p1": {}, + "p2": {}, + })), nil + default: + return nil, fmt.Errorf("unexpected request %s %s", r.Method, r.URL.String()) + } + }), "token", true) + + objs, err := drv.List(context.Background(), &model.Object{Path: "/docs", Name: "docs", IsFolder: true}, model.ListArgs{}) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if graphqlCalls != 1 { + t.Fatalf("expected one GraphQL request, got %d", graphqlCalls) + } + if strings.Count(query, "history(first: 1") != 3 || + !strings.Contains(query, `object(expression: "main^{commit}")`) || + !strings.Contains(query, `p0: history(first: 1, path: "docs/a.md")`) || + !strings.Contains(query, `p1: history(first: 1, path: "docs/quote \" 文.md")`) || + !strings.Contains(query, `p2: history(first: 1, path: "docs/control\u0001.md")`) { + t.Fatalf("query should peel the ref and contain all listed paths once:\n%s", query) + } + if len(objs) != 3 { + t.Fatalf("expected three objects after .gitkeep filtering, got %d", len(objs)) + } + if first := mustObject(t, objs[0]); !first.ModTime().Equal(stamp) || !first.CreateTime().Equal(githubZeroTime) { + t.Fatalf("unexpected first timestamps: mod=%v create=%v", first.ModTime(), first.CreateTime()) + } + if second := mustObject(t, objs[1]); !second.ModTime().Equal(githubZeroTime) || !second.CreateTime().Equal(githubZeroTime) { + t.Fatalf("unmatched entry should retain legacy timestamps: mod=%v create=%v", second.ModTime(), second.CreateTime()) + } +} + +func TestListSkipsAccurateModifiedTime(t *testing.T) { + tests := []struct { + name string + enabled bool + token string + entries int + }{ + {name: "disabled", enabled: false, token: "token", entries: 1}, + {name: "missing token", enabled: true, token: "", entries: 1}, + {name: "over entry limit", enabled: true, token: "token", entries: 201}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + graphqlCalls := 0 + payload := newContentsPayload(t, newSequentialEntries(tc.entries)) + drv := newGithubTestDriver(roundTripFunc(func(r *http.Request) (*http.Response, error) { + switch { + case r.Method == http.MethodGet && strings.Contains(r.URL.Path, "/contents/"): + return newJSONResponse(http.StatusOK, payload), nil + case r.Method == http.MethodPost && r.URL.String() == githubGraphQLEndpoint: + graphqlCalls++ + return newJSONResponse(http.StatusOK, newCommitGraphQLPayload(t, nil)), nil + default: + return nil, fmt.Errorf("unexpected request %s %s", r.Method, r.URL.String()) + } + }), tc.token, tc.enabled) + + objs, err := drv.List(context.Background(), &model.Object{Path: "/docs", Name: "docs", IsFolder: true}, model.ListArgs{}) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if graphqlCalls != 0 { + t.Fatalf("expected zero GraphQL requests, got %d", graphqlCalls) + } + for _, obj := range objs { + raw := mustObject(t, obj) + if !raw.ModTime().Equal(githubZeroTime) || !raw.CreateTime().Equal(githubZeroTime) { + t.Fatalf("legacy timestamps should be preserved: mod=%v create=%v", raw.ModTime(), raw.CreateTime()) + } + } + }) + } +} + +func TestListFallsBackWhenGraphQLFails(t *testing.T) { + graphqlCalls := 0 + drv := newGithubTestDriver(roundTripFunc(func(r *http.Request) (*http.Response, error) { + switch { + case r.Method == http.MethodGet && strings.Contains(r.URL.Path, "/contents/"): + return newJSONResponse(http.StatusOK, newContentsPayload(t, newSequentialEntries(1))), nil + case r.Method == http.MethodPost && r.URL.String() == githubGraphQLEndpoint: + graphqlCalls++ + return newJSONResponse(http.StatusOK, `{"errors":[{"message":"rate limited"}]}`), nil + default: + return nil, fmt.Errorf("unexpected request %s %s", r.Method, r.URL.String()) + } + }), "token", true) + + objs, err := drv.List(context.Background(), &model.Object{Path: "/docs", Name: "docs", IsFolder: true}, model.ListArgs{}) + if err != nil { + t.Fatalf("GraphQL failure should be best-effort: %v", err) + } + if graphqlCalls != 1 { + t.Fatalf("expected one GraphQL request, got %d", graphqlCalls) + } + obj := mustObject(t, objs[0]) + if !obj.ModTime().Equal(githubZeroTime) || !obj.CreateTime().Equal(githubZeroTime) { + t.Fatalf("failed GraphQL request should retain legacy timestamps: mod=%v create=%v", obj.ModTime(), obj.CreateTime()) + } +} + +func TestListUsesOneGraphQLRequestAtEntryLimit(t *testing.T) { + stamp := time.Date(2025, 12, 22, 4, 52, 41, 0, time.UTC) + graphqlCalls := 0 + var query string + drv := newGithubTestDriver(roundTripFunc(func(r *http.Request) (*http.Response, error) { + switch { + case r.Method == http.MethodGet && strings.Contains(r.URL.Path, "/contents/"): + return newJSONResponse(http.StatusOK, newContentsPayload(t, newSequentialEntries(200))), nil + case r.Method == http.MethodPost && r.URL.String() == githubGraphQLEndpoint: + graphqlCalls++ + query = graphQLQueryFromRequest(t, r) + return newJSONResponse(http.StatusOK, newCommitGraphQLPayload(t, map[string][]string{ + "p199": {stamp.Format(time.RFC3339)}, + })), nil + default: + return nil, fmt.Errorf("unexpected request %s %s", r.Method, r.URL.String()) + } + }), "token", true) + + objs, err := drv.List(context.Background(), &model.Object{Path: "/docs", Name: "docs", IsFolder: true}, model.ListArgs{}) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if graphqlCalls != 1 || strings.Count(query, "history(first: 1") != 200 { + t.Fatalf("200 entries should share one request: calls=%d histories=%d", graphqlCalls, strings.Count(query, "history(first: 1")) + } + if !strings.Contains(query, `p199: history(first: 1, path: "docs/199.md")`) { + t.Fatalf("query missing final entry:\n%s", query) + } + if len(objs) != 200 || !mustObject(t, objs[199]).ModTime().Equal(stamp) { + t.Fatalf("unexpected final object timestamp") + } +} + +func TestListKeepsTreeFallbackOnLegacyPath(t *testing.T) { + entries := make([]Object, 0, 1000) + for i := range 1000 { + name := fmt.Sprintf("dir-%d", i) + entries = append(entries, Object{Name: name, Path: "docs/" + name, Type: "dir"}) + } + graphqlCalls := 0 + drv := newGithubTestDriver(roundTripFunc(func(r *http.Request) (*http.Response, error) { + switch { + case r.Method == http.MethodGet && strings.Contains(r.URL.Path, "/contents/"): + return newJSONResponse(http.StatusOK, newContentsPayload(t, entries)), nil + case r.Method == http.MethodGet && strings.Contains(r.URL.Path, "/git/trees/"): + return newJSONResponse(http.StatusOK, newTreePayload(t, "tree-sha", []TreeObjResp{{TreeObjReq: TreeObjReq{Path: "child.md", Mode: "100644", Type: "blob", Sha: "blob-sha"}, Size: 1, URL: "https://example.invalid/blob"}})), nil + case r.Method == http.MethodPost && r.URL.String() == githubGraphQLEndpoint: + graphqlCalls++ + return newJSONResponse(http.StatusOK, newCommitGraphQLPayload(t, nil)), nil + default: + return nil, fmt.Errorf("unexpected request %s %s", r.Method, r.URL.String()) + } + }), "token", true) + + objs, err := drv.List(context.Background(), &model.Object{Path: "/docs", Name: "docs", IsFolder: true}, model.ListArgs{}) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(objs) != 1 { + t.Fatalf("unexpected tree fallback result length: %d", len(objs)) + } + first := mustObject(t, objs[0]) + if first.GetPath() != "/child.md" { + t.Fatalf("unexpected tree fallback path: %s", first.GetPath()) + } + if !first.ModTime().Equal(githubZeroTime) || !first.CreateTime().Equal(githubZeroTime) { + t.Fatalf("tree fallback should preserve legacy timestamps: mod=%v create=%v", first.ModTime(), first.CreateTime()) + } + if graphqlCalls != 0 { + t.Fatalf("tree fallback should skip GraphQL, got %d calls", graphqlCalls) + } +} + +func TestOpListCacheHitDoesNotRepeatGraphQL(t *testing.T) { + op.Cache.ClearAll() + defer op.Cache.ClearAll() + stamp := time.Date(2025, 12, 22, 4, 52, 41, 0, time.UTC) + graphqlCalls := 0 + drv := newGithubTestDriver(roundTripFunc(func(r *http.Request) (*http.Response, error) { + switch { + case r.Method == http.MethodGet && strings.Contains(r.URL.Path, "/contents/"): + return newJSONResponse(http.StatusOK, newContentsPayload(t, []Object{{Name: "a.md", Path: "a.md", Type: "file", Size: 1}})), nil + case r.Method == http.MethodPost && r.URL.String() == githubGraphQLEndpoint: + graphqlCalls++ + return newJSONResponse(http.StatusOK, newCommitGraphQLPayload(t, map[string][]string{"p0": {stamp.Format(time.RFC3339)}})), nil + default: + return nil, fmt.Errorf("unexpected request %s %s", r.Method, r.URL.String()) + } + }), "token", true) + + first, err := op.List(context.Background(), drv, "/", model.ListArgs{}) + if err != nil { + t.Fatalf("unexpected first list error: %v", err) + } + second, err := op.List(context.Background(), drv, "/", model.ListArgs{}) + if err != nil { + t.Fatalf("unexpected second list error: %v", err) + } + if len(first) != 1 || len(second) != 1 { + t.Fatalf("unexpected cached results: first=%d second=%d", len(first), len(second)) + } + if graphqlCalls != 1 { + t.Fatalf("expected one GraphQL call across cached lists, got %d", graphqlCalls) + } + if !mustObject(t, first[0]).ModTime().Equal(stamp) { + t.Fatalf("expected first list to include backfilled modified time, got %v", mustObject(t, first[0]).ModTime()) + } + if !mustObject(t, second[0]).ModTime().Equal(stamp) { + t.Fatalf("expected cached list to retain modified time, got %v", mustObject(t, second[0]).ModTime()) + } +} diff --git a/drivers/github/types.go b/drivers/github/types.go index d0d24839..d6d30642 100644 --- a/drivers/github/types.go +++ b/drivers/github/types.go @@ -1,8 +1,6 @@ package github import ( - "time" - "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/pkg/utils" ) @@ -35,7 +33,8 @@ func (o *Object) toModelObj() *model.Object { return &model.Object{ Name: o.Name, Size: o.Size, - Modified: time.Unix(0, 0), + Modified: githubZeroTime, + Ctime: githubZeroTime, IsFolder: o.Type == "dir", Path: utils.FixAndCleanPath(o.Path), } @@ -69,7 +68,8 @@ func (o *TreeObjResp) toModelObj() *model.Object { return &model.Object{ Name: o.Path, Size: o.Size, - Modified: time.Unix(0, 0), + Modified: githubZeroTime, + Ctime: githubZeroTime, IsFolder: o.Type == "tree", Path: utils.FixAndCleanPath(o.Path), } diff --git a/drivers/github_releases/driver.go b/drivers/github_releases/driver.go index 92d94b4d..e19f7998 100644 --- a/drivers/github_releases/driver.go +++ b/drivers/github_releases/driver.go @@ -45,134 +45,126 @@ func (d *GithubReleases) List(ctx context.Context, dir model.Obj, args model.Lis for i := range d.points { point := &d.points[i] - if !d.Addition.ShowAllVersion { // latest - err := point.RequestRelease(d.GetRequest, args.Refresh) + if !d.Addition.ShowAllVersion { + // latest version mode + release, err := d.getLatestRelease(point.Repo) if err != nil { log.Warnf("failed to request release for %s: %v", point.Repo, err) + continue + } + if release == nil { + continue } - if point.Point == path { // 与仓库路径相同 - if point.Release == nil { - if err != nil { - return nil, fmt.Errorf("failed to get release for %s: %w", point.Repo, err) - } - return nil, fmt.Errorf("failed to get release for %s: unknown error", point.Repo) - } - files = append(files, point.GetLatestRelease()...) + if point.Point == path { + // 当前目录就是仓库挂载点 + files = append(files, releaseToFiles(point.Point, release)...) if d.Addition.ShowReadme { - otherFiles, err := point.GetOtherFile(d.GetRequest, args.Refresh) - if err != nil { - return nil, fmt.Errorf("failed to get other files for %s: %w", point.Repo, err) + other, err := d.fetchRepoFiles(point.Repo) + if err == nil { + files = append(files, otherFiles(point.Point, other)...) + } else { + log.Warnf("failed to get other files for %s: %v", point.Repo, err) } - files = append(files, otherFiles...) } if d.Addition.ShowSourceCode { - files = append(files, point.GetSourceCode()...) + files = append(files, sourceCodeFiles(point.Point, release)...) } - } else if strings.HasPrefix(point.Point, path) { // 仓库目录的父目录 + } else if strings.HasPrefix(point.Point, path) { + // 仓库目录的父目录，需要聚合显示 nextDir := GetNextDir(point.Point, path) if nextDir == "" { continue } - if err != nil { - return nil, fmt.Errorf("failed to get release for %s: %w", point.Repo, err) - } hasSameDir := false for index := range files { if files[index].GetName() == nextDir { hasSameDir = true - files[index].Size += point.GetLatestSize() + files[index].Size += releaseSize(release) break } } if !hasSameDir { - var updateAt, createAt string - if point.Release != nil { - updateAt = point.Release.PublishedAt - createAt = point.Release.CreatedAt - } files = append(files, File{ Path: stdpath.Join(path, nextDir), FileName: nextDir, - Size: point.GetLatestSize(), - UpdateAt: updateAt, - CreateAt: createAt, + Size: releaseSize(release), + UpdateAt: release.PublishedAt, + CreateAt: release.CreatedAt, Type: "dir", Url: "", }) } } - } else { // all version - err := point.RequestReleases(d.GetRequest, args.Refresh) + } else { + // all versions mode + releases, err := d.getAllReleases(point.Repo) if err != nil { log.Warnf("failed to request releases for %s: %v", point.Repo, err) + continue + } + if len(releases) == 0 { + // no releases but may still have repo files (e.g. README) + if point.Point == path && d.Addition.ShowReadme { + other, err := d.fetchRepoFiles(point.Repo) + if err == nil { + files = append(files, otherFiles(point.Point, other)...) + } else { + log.Warnf("failed to get other files for %s: %v", point.Repo, err) + } + } + continue } - if point.Point == path { // 与仓库路径相同 - if point.Releases == nil { - if err != nil { - return nil, fmt.Errorf("failed to get releases for %s: %w", point.Repo, err) - } - return nil, fmt.Errorf("failed to get releases for %s: unknown error", point.Repo) - } - files = append(files, point.GetAllVersion()...) + if point.Point == path { + // 当前目录就是仓库挂载点 + files = append(files, releasesToVersionDirs(point.Point, releases)...) if d.Addition.ShowReadme { - otherFiles, err := point.GetOtherFile(d.GetRequest, args.Refresh) - if err != nil { - return nil, fmt.Errorf("failed to get other files for %s: %w", point.Repo, err) + other, err := d.fetchRepoFiles(point.Repo) + if err == nil { + files = append(files, otherFiles(point.Point, other)...) + } else { + log.Warnf("failed to get other files for %s: %v", point.Repo, err) } - files = append(files, otherFiles...) } - } else if strings.HasPrefix(point.Point, path) { // 仓库目录的父目录 + } else if strings.HasPrefix(point.Point, path) { + // 仓库目录的父目录 nextDir := GetNextDir(point.Point, path) if nextDir == "" { continue } - if err != nil { - return nil, fmt.Errorf("failed to get releases for %s: %w", point.Repo, err) - } hasSameDir := false for index := range files { if files[index].GetName() == nextDir { hasSameDir = true - files[index].Size += point.GetAllVersionSize() + files[index].Size += releasesTotalSize(releases) break } } if !hasSameDir { - var updateAt, createAt string - if point.Releases != nil && len(*point.Releases) > 0 { - updateAt = (*point.Releases)[0].PublishedAt - createAt = (*point.Releases)[0].CreatedAt - } files = append(files, File{ FileName: nextDir, Path: stdpath.Join(path, nextDir), - Size: point.GetAllVersionSize(), - UpdateAt: updateAt, - CreateAt: createAt, + Size: releasesTotalSize(releases), + UpdateAt: releases[0].PublishedAt, + CreateAt: releases[0].CreatedAt, Type: "dir", Url: "", }) } - } else if strings.HasPrefix(path, point.Point) { // 仓库目录的子目录 + } else if strings.HasPrefix(path, point.Point) { + // 仓库目录的子目录（某个版本） tagName := GetNextDir(path, point.Point) if tagName == "" { continue } - if point.Releases == nil { - if err != nil { - return nil, fmt.Errorf("failed to get releases for %s: %w", point.Repo, err) - } - return nil, fmt.Errorf("failed to get releases for %s: unknown error", point.Repo) - } - files = append(files, point.GetReleaseByTagName(tagName)...) + files = append(files, releaseAssetsByTag(point.Point, tagName, releases)...) if d.Addition.ShowSourceCode { - files = append(files, point.GetSourceCodeByTagName(tagName)...) + files = append(files, sourceCodeFilesByTag(point.Point, releases, tagName)...) } } } diff --git a/drivers/github_releases/meta.go b/drivers/github_releases/meta.go index fc5fad69..36f8b3cf 100644 --- a/drivers/github_releases/meta.go +++ b/drivers/github_releases/meta.go @@ -12,7 +12,9 @@ type Addition struct { Token string `json:"token" type:"string" required:"false" help:"GitHub token, if you want to access private repositories or increase the rate limit"` ShowSourceCode bool `json:"show_source_code" type:"bool" default:"false" help:"show Source code (zip/tar.gz)"` ShowAllVersion bool `json:"show_all_version" type:"bool" default:"false" help:"show all versions"` - GitHubProxy string `json:"gh_proxy" type:"string" default:"" help:"GitHub proxy, e.g. https://ghproxy.net/github.com or https://gh-proxy.com/github.com "` + PerPage int `json:"per_page" type:"number" default:"30" help:"releases per page (max 100), only works when show all versions"` + MaxPage int `json:"max_page" type:"number" default:"0" help:"max pages to fetch (0 = unlimited), only works when show all versions"` + GitHubProxy string `json:"gh_proxy" type:"string" default:"" help:"GitHub proxy, e.g. https://ghproxy.net/https://github.com or https://gh-proxy.com/https://github.com"` } var config = driver.Config{ diff --git a/drivers/github_releases/types.go b/drivers/github_releases/types.go index abc024cc..c03108d6 100644 --- a/drivers/github_releases/types.go +++ b/drivers/github_releases/types.go @@ -1,76 +1,28 @@ package github_releases import ( - "encoding/json" "path" "strings" "time" "github.com/OpenListTeam/OpenList/v4/pkg/utils" - "github.com/go-resty/resty/v2" ) +// MountPoint 表示一个仓库挂载点 type MountPoint struct { - Point string // 挂载点 - Repo string // 仓库名 owner/repo - Release *Release // Release 指针 latest - Releases *[]Release // []Release 指针 - OtherFile *[]FileInfo // 仓库根目录下的其他文件 + Point string // 挂载点路径 + Repo string // 仓库名 owner/repo } -// 请求最新版本 -func (m *MountPoint) RequestRelease(get func(url string) (*resty.Response, error), refresh bool) error { - if m.Repo == "" { +// Release 转为 File 列表 +func releaseToFiles(point string, release *Release) []File { + if release == nil { return nil } - - if m.Release == nil || refresh { - resp, err := get("https://api.github.com/repos/" + m.Repo + "/releases/latest") - if err != nil { - m.Release = nil - return err - } - release := new(Release) - if err := json.Unmarshal(resp.Body(), release); err != nil { - m.Release = nil - return err - } - m.Release = release - } - return nil -} - -// 请求所有版本 -func (m *MountPoint) RequestReleases(get func(url string) (*resty.Response, error), refresh bool) error { - if m.Repo == "" { - return nil - } - - if m.Releases == nil || refresh { - resp, err := get("https://api.github.com/repos/" + m.Repo + "/releases") - if err != nil { - m.Releases = nil - return err - } - releases := new([]Release) - if err := json.Unmarshal(resp.Body(), releases); err != nil { - m.Releases = nil - return err - } - m.Releases = releases - } - return nil -} - -// 获取最新版本 -func (m *MountPoint) GetLatestRelease() []File { - if m.Release == nil { - return nil - } - files := make([]File, 0, len(m.Release.Assets)) - for _, asset := range m.Release.Assets { + files := make([]File, 0, len(release.Assets)) + for _, asset := range release.Assets { files = append(files, File{ - Path: path.Join(m.Point, asset.Name), + Path: path.Join(point, asset.Name), FileName: asset.Name, Size: asset.Size, Type: "file", @@ -82,53 +34,43 @@ func (m *MountPoint) GetLatestRelease() []File { return files } -// 获取最新版本大小 -func (m *MountPoint) GetLatestSize() int64 { - if m.Release == nil { +// 计算 release 的 asset 总大小 +func releaseSize(release *Release) int64 { + if release == nil { return 0 } size := int64(0) - for _, asset := range m.Release.Assets { + for _, asset := range release.Assets { size += asset.Size } return size } -// 获取所有版本 -func (m *MountPoint) GetAllVersion() []File { - if m.Releases == nil { - return nil - } - files := make([]File, 0) - for _, release := range *m.Releases { - file := File{ - Path: path.Join(m.Point, release.TagName), +// Releases 列表转为版本目录 File 列表 +func releasesToVersionDirs(point string, releases []Release) []File { + files := make([]File, 0, len(releases)) + for _, release := range releases { + files = append(files, File{ + Path: path.Join(point, release.TagName), FileName: release.TagName, - Size: m.GetSizeByTagName(release.TagName), + Size: releaseSize(&release), Type: "dir", UpdateAt: release.PublishedAt, CreateAt: release.CreatedAt, Url: release.HtmlUrl, - } - for _, asset := range release.Assets { - file.Size += asset.Size - } - files = append(files, file) + }) } return files } -// 根据版本号获取版本 -func (m *MountPoint) GetReleaseByTagName(tagName string) []File { - if m.Releases == nil { - return nil - } - for _, item := range *m.Releases { +// 根据 tagName 查找 release 的 asset 文件列表 +func releaseAssetsByTag(point, tagName string, releases []Release) []File { + for _, item := range releases { if item.TagName == tagName { - files := make([]File, 0) + files := make([]File, 0, len(item.Assets)) for _, asset := range item.Assets { files = append(files, File{ - Path: path.Join(m.Point, tagName, asset.Name), + Path: path.Join(point, tagName, asset.Name), FileName: asset.Name, Size: asset.Size, Type: "file", @@ -143,118 +85,74 @@ func (m *MountPoint) GetReleaseByTagName(tagName string) []File { return nil } -// 根据版本号获取版本大小 -func (m *MountPoint) GetSizeByTagName(tagName string) int64 { - if m.Releases == nil { - return 0 - } - for _, item := range *m.Releases { +// 根据 tagName 计算 asset 总大小 +func releasesSizeByTag(releases []Release, tagName string) int64 { + for _, item := range releases { if item.TagName == tagName { - size := int64(0) - for _, asset := range item.Assets { - size += asset.Size - } - return size + return releaseSize(&item) } } return 0 } -// 获取所有版本大小 -func (m *MountPoint) GetAllVersionSize() int64 { - if m.Releases == nil { - return 0 - } +// 计算所有 releases 的 asset 总大小 +func releasesTotalSize(releases []Release) int64 { size := int64(0) - for _, release := range *m.Releases { - for _, asset := range release.Assets { - size += asset.Size - } + for _, release := range releases { + size += releaseSize(&release) } return size } -func (m *MountPoint) GetSourceCode() []File { - if m.Release == nil { +// Source code 文件 +func sourceCodeFiles(point string, release *Release) []File { + if release == nil { return nil } - files := make([]File, 0) - - // 无法获取文件大小，此处设为 1 - files = append(files, File{ - Path: path.Join(m.Point, "Source code (zip)"), - FileName: "Source code (zip)", - Size: 1, - Type: "file", - UpdateAt: m.Release.CreatedAt, - CreateAt: m.Release.CreatedAt, - Url: m.Release.ZipballUrl, - }) - files = append(files, File{ - Path: path.Join(m.Point, "Source code (tar.gz)"), - FileName: "Source code (tar.gz)", - Size: 1, - Type: "file", - UpdateAt: m.Release.CreatedAt, - CreateAt: m.Release.CreatedAt, - Url: m.Release.TarballUrl, - }) - - return files + return []File{ + { + Path: path.Join(point, "Source code (zip)"), + FileName: "Source code (zip)", + Size: 1, + Type: "file", + UpdateAt: release.CreatedAt, + CreateAt: release.CreatedAt, + Url: release.ZipballUrl, + }, + { + Path: path.Join(point, "Source code (tar.gz)"), + FileName: "Source code (tar.gz)", + Size: 1, + Type: "file", + UpdateAt: release.CreatedAt, + CreateAt: release.CreatedAt, + Url: release.TarballUrl, + }, + } } -func (m *MountPoint) GetSourceCodeByTagName(tagName string) []File { - if m.Releases == nil { - return nil - } - for _, item := range *m.Releases { +// 根据 tagName 获取 Source Code 文件 +func sourceCodeFilesByTag(point string, releases []Release, tagName string) []File { + for _, item := range releases { if item.TagName == tagName { - files := make([]File, 0) - files = append(files, File{ - Path: path.Join(m.Point, "Source code (zip)"), - FileName: "Source code (zip)", - Size: 1, - Type: "file", - UpdateAt: item.CreatedAt, - CreateAt: item.CreatedAt, - Url: item.ZipballUrl, - }) - files = append(files, File{ - Path: path.Join(m.Point, "Source code (tar.gz)"), - FileName: "Source code (tar.gz)", - Size: 1, - Type: "file", - UpdateAt: item.CreatedAt, - CreateAt: item.CreatedAt, - Url: item.TarballUrl, - }) - return files + return sourceCodeFiles(point, &item) } } return nil } -func (m *MountPoint) GetOtherFile(get func(url string) (*resty.Response, error), refresh bool) ([]File, error) { - if m.OtherFile == nil || refresh { - resp, err := get("https://api.github.com/repos/" + m.Repo + "/contents") - if err != nil { - m.OtherFile = nil - return nil, err - } - otherFile := new([]FileInfo) - if err := json.Unmarshal(resp.Body(), otherFile); err != nil { - m.OtherFile = nil - return nil, err - } - m.OtherFile = otherFile - } - +// 仓库根目录下的 README/LICENSE 文件 +func otherFiles(point string, fileInfos []FileInfo) []File { files := make([]File, 0) defaultTime := "1970-01-01T00:00:00Z" - for _, file := range *m.OtherFile { - if strings.HasSuffix(file.Name, ".md") || strings.HasPrefix(file.Name, "LICENSE") { + for _, file := range fileInfos { + if file.Type == "dir" { + continue + } + name := file.Name + if strings.EqualFold(name, "README.md") || strings.HasPrefix(name, "LICENSE") { files = append(files, File{ - Path: path.Join(m.Point, file.Name), + Path: path.Join(point, file.Name), FileName: file.Name, Size: file.Size, Type: "file", @@ -264,7 +162,7 @@ func (m *MountPoint) GetOtherFile(get func(url string) (*resty.Response, error), }) } } - return files, nil + return files } type File struct { diff --git a/drivers/github_releases/util.go b/drivers/github_releases/util.go index a6401e6c..cbaafb08 100644 --- a/drivers/github_releases/util.go +++ b/drivers/github_releases/util.go @@ -1,8 +1,8 @@ package github_releases import ( + "encoding/json" "fmt" - "path/filepath" "strings" "github.com/OpenListTeam/OpenList/v4/drivers/base" @@ -49,10 +49,8 @@ func (d *GithubReleases) ParseRepos(text string) ([]MountPoint, error) { } points = append(points, MountPoint{ - Point: path, - Repo: repo, - Release: nil, - Releases: nil, + Point: path, + Repo: repo, }) } d.points = points @@ -76,9 +74,79 @@ func GetNextDir(wholePath string, basePath string) string { return "" } -// 判断当前目录是否是目标目录的祖先目录 -func IsAncestorDir(parentDir string, targetDir string) bool { - absTargetDir, _ := filepath.Abs(targetDir) - absParentDir, _ := filepath.Abs(parentDir) - return strings.HasPrefix(absTargetDir, absParentDir) +// getLatestRelease 获取最新 release +func (d *GithubReleases) getLatestRelease(repo string) (*Release, error) { + resp, err := d.GetRequest("https://api.github.com/repos/" + repo + "/releases/latest") + if err != nil { + return nil, err + } + release := new(Release) + if err := json.Unmarshal(resp.Body(), release); err != nil { + return nil, err + } + return release, nil +} + +// getAllReleases 获取所有 releases（支持自动翻页） +func (d *GithubReleases) getAllReleases(repo string) ([]Release, error) { + perPage := d.Addition.PerPage + if perPage < 1 { + perPage = 30 + } else if perPage > 100 { + perPage = 100 + } + + maxPage := d.Addition.MaxPage + if maxPage < 0 { + maxPage = 0 + } + + allReleases := make([]Release, 0) + page := 1 + + for { + url := fmt.Sprintf("https://api.github.com/repos/%s/releases?per_page=%d&page=%d", repo, perPage, page) + resp, err := d.GetRequest(url) + if err != nil { + return nil, err + } + + releases := make([]Release, 0) + if err := json.Unmarshal(resp.Body(), &releases); err != nil { + return nil, err + } + + if len(releases) == 0 { + break + } + + allReleases = append(allReleases, releases...) + + // 达到最大页数限制 + if maxPage > 0 && page >= maxPage { + break + } + + // 如果返回数量小于 perPage，说明是最后一页 + if len(releases) < perPage { + break + } + + page++ + } + + return allReleases, nil +} + +// fetchRepoFiles 获取仓库根目录文件列表 +func (d *GithubReleases) fetchRepoFiles(repo string) ([]FileInfo, error) { + resp, err := d.GetRequest("https://api.github.com/repos/" + repo + "/contents") + if err != nil { + return nil, err + } + files := make([]FileInfo, 0) + if err := json.Unmarshal(resp.Body(), &files); err != nil { + return nil, err + } + return files, nil } diff --git a/drivers/google_drive/util.go b/drivers/google_drive/util.go index 042abafa..e4f5f1a0 100644 --- a/drivers/google_drive/util.go +++ b/drivers/google_drive/util.go @@ -170,7 +170,7 @@ func (d *GoogleDrive) refreshToken() error { } log.Debug(res.String()) if e.Error != "" { - return fmt.Errorf(e.Error) + return fmt.Errorf("%s", e.Error) } d.AccessToken = resp.AccessToken return nil @@ -192,7 +192,7 @@ func (d *GoogleDrive) refreshToken() error { } log.Debug(res.String()) if e.Error != "" { - return fmt.Errorf(e.Error) + return fmt.Errorf("%s", e.Error) } d.AccessToken = resp.AccessToken return nil diff --git a/drivers/google_photo/util.go b/drivers/google_photo/util.go index 3a9b66ab..81d05b49 100644 --- a/drivers/google_photo/util.go +++ b/drivers/google_photo/util.go @@ -32,7 +32,7 @@ func (d *GooglePhoto) refreshToken() error { return err } if e.Error != "" { - return fmt.Errorf(e.Error) + return fmt.Errorf("%s", e.Error) } d.AccessToken = resp.AccessToken return nil diff --git a/drivers/guangyapan/driver.go b/drivers/guangyapan/driver.go new file mode 100644 index 00000000..dff2809b --- /dev/null +++ b/drivers/guangyapan/driver.go @@ -0,0 +1,792 @@ +package guangyapan + +import ( + "context" + "errors" + "fmt" + "strings" + "sync" + "time" + + "github.com/OpenListTeam/OpenList/v4/drivers/base" + "github.com/OpenListTeam/OpenList/v4/internal/driver" + "github.com/OpenListTeam/OpenList/v4/internal/errs" + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/internal/op" + "github.com/aliyun/aliyun-oss-go-sdk/oss" + "github.com/go-resty/resty/v2" + log "github.com/sirupsen/logrus" +) + +const ( + accountBaseURL = "https://account.guangyapan.com" + apiBaseURL = "https://api.guangyapan.com" +) + +type GuangYaPan struct { + model.Storage + Addition + + accountClient *resty.Client + apiClient *resty.Client + + resolvedRootFolderID string + rootFolderResolved bool + + // refreshMu protects concurrent access to AccessToken/RefreshToken during refresh. + refreshMu sync.Mutex + // statusTimer tracks delayed status updates for cancellation on Drop. + statusTimer *time.Timer + + // apiRateLimit throttles requests per API endpoint so that batch operations + // (e.g. copying many files cross-storage) don't flood the upstream API. + apiRateLimit sync.Map +} + +// apiRateInterval is the minimum gap between two requests to the same endpoint. +const apiRateInterval = 500 * time.Millisecond + +func (d *GuangYaPan) Config() driver.Config { + return config +} + +func (d *GuangYaPan) GetAddition() driver.Additional { + return &d.Addition +} + +func (d *GuangYaPan) Init(ctx context.Context) error { + d.ClientID = strings.TrimSpace(d.ClientID) + if d.ClientID == "" { + return errors.New("client_id is required, please provide a valid client_id") + } + d.DeviceID = normalizeDeviceID(d.DeviceID) + if d.DeviceID == "" { + d.DeviceID = randomDeviceID() + } + deviceSign := strings.TrimSpace(d.DeviceSign) + if deviceSign == "" { + deviceSign = "wdi10." + d.DeviceID + } + if d.PageSize <= 0 { + d.PageSize = 100 + } + if d.OrderBy < 0 { + d.OrderBy = 3 + } + if d.SortType != 0 && d.SortType != 1 { + d.SortType = 1 + } + + d.RootPath = strings.TrimSpace(d.RootPath) + d.AccessToken = strings.TrimSpace(d.AccessToken) + d.RefreshToken = strings.TrimSpace(d.RefreshToken) + d.PhoneNumber = strings.TrimSpace(d.PhoneNumber) + d.VerifyCode = strings.TrimSpace(d.VerifyCode) + d.CaptchaToken = strings.TrimSpace(d.CaptchaToken) + d.VerificationID = strings.TrimSpace(d.VerificationID) + d.resolvedRootFolderID = "" + d.rootFolderResolved = false + + d.accountClient = base.NewRestyClient(). + SetBaseURL(accountBaseURL). + SetHeader("Accept", "application/json, text/plain, */*"). + SetHeader("Content-Type", "application/json"). + SetHeader("X-Device-Model", "chrome%2F147.0.0.0"). + SetHeader("X-Device-Name", "PC-Chrome"). + SetHeader("X-Device-Sign", deviceSign). + SetHeader("X-Net-Work-Type", "NONE"). + SetHeader("X-OS-Version", "MacIntel"). + SetHeader("X-Platform-Version", "1"). + SetHeader("X-Protocol-Version", "301"). + SetHeader("X-Provider-Name", "NONE"). + SetHeader("X-SDK-Version", "9.0.2"). + SetHeader("X-Client-Id", d.ClientID). + SetHeader("X-Client-Version", "0.0.1"). + SetHeader("X-Device-Id", d.DeviceID) + + d.apiClient = base.NewRestyClient(). + SetBaseURL(apiBaseURL). + SetHeader("Accept", "application/json, text/plain, */*"). + SetHeader("Content-Type", "application/json"). + SetHeader("Did", d.DeviceID). + SetHeader("Dt", "4") + + // Priority: access_token -> refresh_token -> sms login. + if d.AccessToken != "" { + if err := d.validateToken(ctx); err == nil { + return d.prepareRootFolder(ctx) + } + d.AccessToken = "" + } + if d.RefreshToken != "" { + if err := d.refreshToken(ctx); err == nil { + if err2 := d.validateToken(ctx); err2 == nil { + return d.prepareRootFolder(ctx) + } + } + } + // Two-stage SMS flow: + // 1) phone only + send_code=true: send code and cache verification_id (do not fail init). + // 2) phone + verify_code: complete login and save tokens. + if d.PhoneNumber != "" { + if d.canSMSLogin() { + if err := d.loginBySMSCode(ctx); err != nil { + return err + } + if err := d.validateToken(ctx); err != nil { + return err + } + return d.prepareRootFolder(ctx) + } + if d.SendCode { + d.setTempStatus("SMS sending in progress...") + if err := d.prepareSMSCode(ctx); err != nil { + d.setTempStatus(fmt.Sprintf("SMS send failed: %v. Please check captcha/meta and set send_code=true to retry.", err)) + log.Warnf("guangyapan: prepare sms code failed: %v", err) + } else { + d.setTempStatus("SMS sent successfully. Please fill verify_code and save to complete login.") + } + } + return nil + } + return errors.New("login failed: provide a valid access_token, or refresh_token, or phone_number + verify_code + captcha_token") +} + +func (d *GuangYaPan) Drop(ctx context.Context) error { + if d.statusTimer != nil { + d.statusTimer.Stop() + } + return nil +} + +func (d *GuangYaPan) GetRoot(ctx context.Context) (model.Obj, error) { + rootID, err := d.getRootFolderID(ctx) + if err != nil { + return nil, err + } + return &model.Object{ + ID: rootID, + Path: "/", + Name: "root", + Size: 0, + Modified: d.Modified, + IsFolder: true, + }, nil +} + +func (d *GuangYaPan) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { + if err := d.ensureAccessToken(ctx); err != nil { + return nil, err + } + + parentID := dir.GetID() + + const maxPage = 10000 + res := make([]model.Obj, 0, d.PageSize) + for page := 0; page < maxPage; page++ { + var resp listResp + body := map[string]any{ + "parentId": parentID, + "page": page, + "pageSize": d.PageSize, + "orderBy": d.OrderBy, + "sortType": d.SortType, + } + if err := d.postAPI(ctx, "/userres/v1/file/get_file_list", body, &resp); err != nil { + return nil, err + } + for _, item := range resp.Data.List { + res = append(res, &model.Object{ + ID: item.FileID, + Path: parentID, + Name: item.FileName, + Size: item.FileSize, + Modified: unixOrZero(item.UTime), + Ctime: unixOrZero(item.CTime), + IsFolder: item.ResType == 2, + }) + } + if len(resp.Data.List) < d.PageSize { + break + } + if resp.Data.Total > 0 && len(res) >= resp.Data.Total { + break + } + } + return res, nil +} + +func (d *GuangYaPan) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { + if file.IsDir() { + return nil, errs.NotFile + } + if err := d.ensureAccessToken(ctx); err != nil { + return nil, err + } + + var resp downloadResp + if err := d.postAPI(ctx, "/nd.bizuserres.s/v1/get_res_download_url", map[string]any{ + "fileId": file.GetID(), + }, &resp); err != nil { + return nil, err + } + + url := strings.TrimSpace(resp.Data.SignedURL) + if url == "" { + url = strings.TrimSpace(resp.Data.DownloadURL) + } + if url == "" { + return nil, errors.New("empty download url") + } + return &model.Link{URL: url}, nil +} + +func (d *GuangYaPan) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error { + if err := d.ensureAccessToken(ctx); err != nil { + return err + } + + name := strings.TrimSpace(dirName) + if name == "" { + return errors.New("dir name is empty") + } + + parentID := parentDir.GetID() + + var out createDirResp + if err := d.postAPI(ctx, "/nd.bizuserres.s/v1/file/create_dir", map[string]any{ + "parentId": parentID, + "dirName": name, + }, &out); err != nil { + return err + } + if !isSuccessMsg(out.Msg) { + return fmt.Errorf("make dir failed: %s", strings.TrimSpace(out.Msg)) + } + return nil +} + +func (d *GuangYaPan) Rename(ctx context.Context, srcObj model.Obj, newName string) error { + if err := d.ensureAccessToken(ctx); err != nil { + return err + } + + fileID := strings.TrimSpace(srcObj.GetID()) + if fileID == "" { + return errors.New("file id is empty") + } + name := strings.TrimSpace(newName) + if name == "" { + return errors.New("new name is empty") + } + + var out commonResp + if err := d.postAPI(ctx, "/nd.bizuserres.s/v1/file/rename", map[string]any{ + "fileId": fileID, + "newName": name, + }, &out); err != nil { + return err + } + if !isSuccessMsg(out.Msg) { + return fmt.Errorf("rename failed: %s", strings.TrimSpace(out.Msg)) + } + return nil +} + +func (d *GuangYaPan) Remove(ctx context.Context, obj model.Obj) error { + if err := d.ensureAccessToken(ctx); err != nil { + return err + } + + fileID := strings.TrimSpace(obj.GetID()) + if fileID == "" { + return errors.New("file id is empty") + } + + var del taskResp + if err := d.postAPI(ctx, "/nd.bizuserres.s/v1/file/delete_file", map[string]any{ + "fileIds": []string{fileID}, + }, &del); err != nil { + return err + } + if !isSuccessMsg(del.Msg) { + return fmt.Errorf("delete failed: %s", strings.TrimSpace(del.Msg)) + } + + taskID := strings.TrimSpace(del.Data.TaskID) + if taskID == "" { + // Some backends may apply deletion synchronously. + return nil + } + return d.waitTaskDone(ctx, taskID) +} + +func (d *GuangYaPan) Move(ctx context.Context, srcObj, dstDir model.Obj) error { + if err := d.ensureAccessToken(ctx); err != nil { + return err + } + + fileID := strings.TrimSpace(srcObj.GetID()) + if fileID == "" { + return errors.New("file id is empty") + } + parentID := dstDir.GetID() + + var out taskResp + if err := d.postAPI(ctx, "/nd.bizuserres.s/v1/file/move_file", map[string]any{ + "fileIds": []string{fileID}, + "parentId": parentID, + }, &out); err != nil { + return err + } + if !isSuccessMsg(out.Msg) { + return fmt.Errorf("move failed: %s", strings.TrimSpace(out.Msg)) + } + taskID := strings.TrimSpace(out.Data.TaskID) + if taskID == "" { + return nil + } + return d.waitTaskDone(ctx, taskID) +} + +func (d *GuangYaPan) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { + if err := d.ensureAccessToken(ctx); err != nil { + return err + } + + fileID := strings.TrimSpace(srcObj.GetID()) + if fileID == "" { + return errors.New("file id is empty") + } + parentID := dstDir.GetID() + + var out taskResp + if err := d.postAPI(ctx, "/nd.bizuserres.s/v1/file/copy_file", map[string]any{ + "fileIds": []string{fileID}, + "parentId": parentID, + }, &out); err != nil { + return err + } + if !isSuccessMsg(out.Msg) { + return fmt.Errorf("copy failed: %s", strings.TrimSpace(out.Msg)) + } + taskID := strings.TrimSpace(out.Data.TaskID) + if taskID == "" { + return nil + } + return d.waitTaskDone(ctx, taskID) +} + +func (d *GuangYaPan) Put(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) { + if err := d.ensureAccessToken(ctx); err != nil { + return nil, err + } + if file == nil { + return nil, errors.New("file is nil") + } + if file.GetSize() < 0 { + return nil, errors.New("invalid file size") + } + name := strings.TrimSpace(file.GetName()) + if name == "" { + return nil, errors.New("file name is empty") + } + + parentID := dstDir.GetID() + + // 优先秒传：先计算文件 MD5，后端命中相同文件时直接秒传完成，无需真实上传。 + md5sum, err := d.fileMD5(file, up) + if err != nil { + return nil, err + } + + token, code, err := d.getUploadToken(ctx, parentID, name, file.GetSize(), md5sum) + if err != nil { + return nil, err + } + taskID := strings.TrimSpace(token.TaskID) + // code == 156 (instant upload) or AlreadyDone mean the backend has already + // finished/imported the file; there is no OSS upload to perform. + if code == 156 || token.AlreadyDone { + if taskID == "" { + return nil, errors.New("instant upload returns empty task id") + } + if err := d.waitUploadTaskInfo(ctx, taskID); err != nil { + return nil, err + } + return nil, nil + } + + if token.ObjectPath == "" || token.BucketName == "" || token.EndPoint == "" || token.AccessKeyID == "" || token.SecretAccessKey == "" { + return nil, errors.New("upload token is incomplete") + } + + ossEndpoint := normalizeOSSEndpoint(token.EndPoint, token.BucketName) + client, err := oss.New(ossEndpoint, token.AccessKeyID, token.SecretAccessKey, oss.SecurityToken(token.SessionToken)) + if err != nil { + return nil, fmt.Errorf("create oss client failed: %w", err) + } + bucket, err := client.Bucket(token.BucketName) + if err != nil { + return nil, fmt.Errorf("create oss bucket failed: %w", err) + } + + if file.GetSize() == 0 { + if err := bucket.PutObject(token.ObjectPath, strings.NewReader("")); err != nil { + return nil, err + } + } else { + if err := d.multipartUploadToOSS(ctx, bucket, token.ObjectPath, file, up); err != nil { + return nil, err + } + } + + if taskID == "" { + return nil, nil + } + if err := d.waitUploadTaskInfo(ctx, taskID); err != nil { + return nil, err + } + return nil, nil +} + +func (d *GuangYaPan) GetDetails(ctx context.Context) (*model.StorageDetails, error) { + if err := d.ensureAccessToken(ctx); err != nil { + return nil, err + } + + var resp assetsInfoResp + if err := d.postAPI(ctx, "/nd.bizassets.s/v1/get_assets", nil, &resp); err != nil { + return nil, err + } + if resp.IsSuccess() && resp.Data.TotalSpaceSize > 0 { + return &model.StorageDetails{ + DiskUsage: model.DiskUsage{ + TotalSpace: resp.Data.TotalSpaceSize, + UsedSpace: resp.Data.UsedSpaceSize, + }, + }, nil + } + return nil, errors.New("failed to get storage details") +} + +func (d *GuangYaPan) getRootFolderID(ctx context.Context) (string, error) { + if d.rootFolderResolved { + return d.resolvedRootFolderID, nil + } + if err := d.ensureAccessToken(ctx); err != nil { + return "", err + } + if err := d.prepareRootFolder(ctx); err != nil { + return "", err + } + return d.resolvedRootFolderID, nil +} + +func (d *GuangYaPan) prepareRootFolder(ctx context.Context) error { + rootID, err := d.resolveConfiguredRootFolderID(ctx) + if err != nil { + return err + } + d.resolvedRootFolderID = rootID + d.rootFolderResolved = true + return nil +} + +func (d *GuangYaPan) resolveConfiguredRootFolderID(ctx context.Context) (string, error) { + root := strings.TrimSpace(d.RootPath) + if root == "" { + return "", nil + } + return d.resolveFolderPath(ctx, root) +} + +func (d *GuangYaPan) resolveFolderPath(ctx context.Context, rootPath string) (string, error) { + cleanPath := strings.Trim(strings.ReplaceAll(strings.TrimSpace(rootPath), "\\", "/"), "/") + if cleanPath == "" { + return "", nil + } + + parentID := "" + for _, name := range strings.Split(cleanPath, "/") { + if name == "" { + continue + } + childID, err := d.findChildFolderID(ctx, parentID, name) + if err != nil { + return "", err + } + parentID = childID + } + return parentID, nil +} + +func (d *GuangYaPan) findChildFolderID(ctx context.Context, parentID, name string) (string, error) { + pageSize := d.PageSize + if pageSize <= 0 { + pageSize = 100 + } + + const maxPage = 10000 + seen := 0 + for page := 0; page < maxPage; page++ { + var resp listResp + body := map[string]any{ + "parentId": parentID, + "page": page, + "pageSize": pageSize, + "orderBy": d.OrderBy, + "sortType": d.SortType, + } + if err := d.postAPI(ctx, "/nd.bizuserres.s/v1/file/get_file_list", body, &resp); err != nil { + return "", err + } + for _, item := range resp.Data.List { + seen++ + if item.ResType == 2 && item.FileName == name { + return item.FileID, nil + } + } + if len(resp.Data.List) < pageSize { + break + } + if resp.Data.Total > 0 && seen >= resp.Data.Total { + break + } + } + + if parentID == "" { + return "", fmt.Errorf("resolve root folder path failed: folder %q not found under /", name) + } + return "", fmt.Errorf("resolve root folder path failed: folder %q not found under parent %s", name, parentID) +} + +func (d *GuangYaPan) ensureAccessToken(ctx context.Context) error { + if strings.TrimSpace(d.AccessToken) != "" { + return nil + } + if strings.TrimSpace(d.RefreshToken) == "" { + return errors.New("not logged in, please re-init storage") + } + return d.refreshToken(ctx) +} + +func (d *GuangYaPan) validateToken(ctx context.Context) error { + var me userMeResp + resp, err := d.accountClient.R(). + SetContext(ctx). + SetHeader("Authorization", "Bearer "+d.AccessToken). + SetResult(&me). + Get("/v1/user/me") + if err != nil { + return err + } + if resp.IsError() { + return fmt.Errorf("validate token failed: status=%d body=%s", resp.StatusCode(), resp.String()) + } + if strings.TrimSpace(me.Sub) == "" { + return errors.New("validate token failed: empty user sub") + } + return nil +} + +func (d *GuangYaPan) refreshToken(ctx context.Context) error { + if strings.TrimSpace(d.RefreshToken) == "" { + return errors.New("refresh_token is empty") + } + + d.refreshMu.Lock() + defer d.refreshMu.Unlock() + + // Double-check after acquiring lock (may have been refreshed by another goroutine) + if strings.TrimSpace(d.AccessToken) != "" { + if err := d.validateToken(ctx); err == nil { + return nil + } + } + + var out tokenResp + resp, err := d.accountClient.R(). + SetContext(ctx). + SetBody(map[string]any{ + "client_id": d.ClientID, + "grant_type": "refresh_token", + "refresh_token": d.RefreshToken, + }). + SetResult(&out). + Post("/v1/auth/token") + if err != nil { + return err + } + if resp.IsError() || out.Error != "" || strings.TrimSpace(out.AccessToken) == "" { + errMsg := strings.TrimSpace(out.ErrorDesc) + if errMsg == "" { + errMsg = strings.TrimSpace(out.Error) + } + if errMsg == "" { + errMsg = strings.TrimSpace(resp.String()) + } + if errMsg == "" { + errMsg = fmt.Sprintf("status=%d", resp.StatusCode()) + } + return fmt.Errorf("refresh token failed: %s", errMsg) + } + + d.AccessToken = strings.TrimSpace(out.AccessToken) + if strings.TrimSpace(out.RefreshToken) != "" { + d.RefreshToken = strings.TrimSpace(out.RefreshToken) + } + op.MustSaveDriverStorage(d) + return nil +} + +func (d *GuangYaPan) canSMSLogin() bool { + return d.PhoneNumber != "" && d.VerifyCode != "" +} + +func (d *GuangYaPan) loginBySMSCode(ctx context.Context) error { + verificationID := strings.TrimSpace(d.VerificationID) + if verificationID == "" { + var err error + verificationID, err = d.requestVerificationID(ctx) + if err != nil { + return err + } + } + + var step2 verifyResp + resp, err := d.accountClient.R(). + SetContext(ctx). + SetBody(map[string]any{ + "verification_id": verificationID, + "verification_code": d.VerifyCode, + "client_id": d.ClientID, + }). + SetResult(&step2). + Post("/v1/auth/verification/verify") + if err != nil { + return err + } + if resp.IsError() || step2.Error != "" || strings.TrimSpace(step2.VerificationToken) == "" { + return fmt.Errorf("verify code failed: %s", d.accountErr(step2.ErrorDesc, step2.Error, resp)) + } + + var out tokenResp + resp, err = d.accountClient.R(). + SetContext(ctx). + SetBody(map[string]any{ + "verification_code": d.VerifyCode, + "verification_token": step2.VerificationToken, + "username": normalizePhoneE164(d.PhoneNumber), + "client_id": d.ClientID, + }). + SetResult(&out). + Post("/v1/auth/signin") + if err != nil { + return err + } + if resp.IsError() || out.Error != "" || strings.TrimSpace(out.AccessToken) == "" { + return fmt.Errorf("signin failed: %s", d.accountErr(out.ErrorDesc, out.Error, resp)) + } + + d.AccessToken = strings.TrimSpace(out.AccessToken) + d.RefreshToken = strings.TrimSpace(out.RefreshToken) + d.VerificationID = "" + // One-time SMS code should not be reused after successful login. + d.VerifyCode = "" + op.MustSaveDriverStorage(d) + return nil +} + +func (d *GuangYaPan) prepareSMSCode(ctx context.Context) error { + // Explicit send action should always refresh verification_id. + d.VerificationID = "" + if err := d.ensureCaptchaToken(ctx, false); err != nil { + return err + } + verificationID, err := d.requestVerificationID(ctx) + if err != nil { + return err + } + d.VerificationID = verificationID + d.SendCode = false + op.MustSaveDriverStorage(d) + return nil +} + +func (d *GuangYaPan) requestVerificationID(ctx context.Context) (string, error) { + req := d.accountClient.R().SetContext(ctx) + if d.CaptchaToken != "" { + req.SetHeader("X-Captcha-Token", d.CaptchaToken) + } + + var step1 verificationResp + resp, err := req. + SetBody(map[string]any{ + "phone_number": normalizePhoneE164(d.PhoneNumber), + "target": "ANY", + "client_id": d.ClientID, + }). + SetResult(&step1). + Post("/v1/auth/verification") + if err != nil { + return "", err + } + if resp.IsError() || step1.Error != "" || strings.TrimSpace(step1.VerificationID) == "" { + // If captcha token is expired/invalid, refresh it once and retry. + if strings.Contains(step1.Error, "captcha_invalid") || strings.Contains(step1.ErrorDesc, "captcha_token expired") { + if err := d.ensureCaptchaToken(ctx, true); err == nil { + return d.requestVerificationID(ctx) + } + } + return "", fmt.Errorf("request verification failed: %s", d.accountErr(step1.ErrorDesc, step1.Error, resp)) + } + return strings.TrimSpace(step1.VerificationID), nil +} + +func (d *GuangYaPan) ensureCaptchaToken(ctx context.Context, force bool) error { + if !force && d.CaptchaToken != "" { + return nil + } + + var out captchaInitResp + req := d.accountClient.R().SetContext(ctx) + if d.CaptchaToken != "" { + req.SetHeader("X-Captcha-Token", d.CaptchaToken) + } + resp, err := req. + SetBody(map[string]any{ + "client_id": d.ClientID, + "action": "POST:/v1/auth/verification", + "device_id": d.DeviceID, + "meta": map[string]any{ + "username": normalizePhoneE164(d.PhoneNumber), + "phone_number": normalizePhoneE164(d.PhoneNumber), + "VERIFICATION_PHONE": normalizePhoneE164(d.PhoneNumber), + }, + }). + SetResult(&out). + Post("/v1/shield/captcha/init") + if err != nil { + return err + } + if resp.IsError() || out.Error != "" || strings.TrimSpace(out.CaptchaToken) == "" { + return fmt.Errorf("init captcha token failed: %s", d.accountErr(out.ErrorDesc, out.Error, resp)) + } + d.CaptchaToken = strings.TrimSpace(out.CaptchaToken) + op.MustSaveDriverStorage(d) + return nil +} + +// Interface compliance checks +var ( + _ driver.Driver = (*GuangYaPan)(nil) + _ driver.GetRooter = (*GuangYaPan)(nil) + _ driver.Mkdir = (*GuangYaPan)(nil) + _ driver.Move = (*GuangYaPan)(nil) + _ driver.Copy = (*GuangYaPan)(nil) + _ driver.Rename = (*GuangYaPan)(nil) + _ driver.Remove = (*GuangYaPan)(nil) + _ driver.PutResult = (*GuangYaPan)(nil) + _ driver.WithDetails = (*GuangYaPan)(nil) +) diff --git a/drivers/guangyapan/meta.go b/drivers/guangyapan/meta.go new file mode 100644 index 00000000..2cd7f8be --- /dev/null +++ b/drivers/guangyapan/meta.go @@ -0,0 +1,37 @@ +package guangyapan + +import ( + "github.com/OpenListTeam/OpenList/v4/internal/driver" + "github.com/OpenListTeam/OpenList/v4/internal/op" +) + +type Addition struct { + RootPath string `json:"root_path" help:"Full path in GuangYaPan cloud drive"` + PhoneNumber string `json:"phone_number" type:"text" help:"Phone number for SMS login, e.g. +86 13800000000"` + CaptchaToken string `json:"captcha_token" help:"Captcha token required by /v1/auth/verification"` + SendCode bool `json:"send_code" type:"bool" help:"Set true and save to send SMS code, it auto-resets to false after sending"` + VerifyCode string `json:"verify_code" type:"text" help:"SMS verification code used with phone_number; fill then save to finish login"` + VerificationID string `json:"verification_id" type:"text" help:"Auto-generated after sending SMS code; do not edit manually"` + AccessToken string `json:"access_token" help:"Bearer access token (optional if refresh_token is provided)"` + RefreshToken string `json:"refresh_token" help:"Refresh token for auto-login/auto-refresh"` + ClientID string `json:"client_id" required:"true" help:"Client ID for GuangYaPan API, must be provided"` + DeviceID string `json:"device_id" help:"Optional custom device id (32 hex chars), auto-generated when empty"` + DeviceSign string `json:"device_sign" help:"Optional custom X-Device-Sign header (generated from device_id when empty)"` + PageSize int `json:"page_size" type:"number" default:"100"` + OrderBy int `json:"order_by" type:"number" options:"0,1,2,3,4" default:"3" help:"Sort field used by the file list"` + SortType int `json:"sort_type" type:"number" options:"0,1" default:"1" help:"Sort direction used by the file list"` +} + +var config = driver.Config{ + Name: "GuangYaPan", + DefaultRoot: "", + CheckStatus: true, + Alert: "info|Two-stage SMS login: (1) fill phone_number (+ captcha_token if needed), set send_code=true and save; (2) fill verify_code and save to finish login and auto-save access_token/refresh_token.", + NoOverwriteUpload: true, +} + +func init() { + op.RegisterDriver(func() driver.Driver { + return &GuangYaPan{} + }) +} diff --git a/drivers/guangyapan/offline.go b/drivers/guangyapan/offline.go new file mode 100644 index 00000000..4dd4500b --- /dev/null +++ b/drivers/guangyapan/offline.go @@ -0,0 +1,169 @@ +package guangyapan + +import ( + "context" + "errors" + "fmt" + "net/url" + stdpath "path" + "strings" + + "github.com/OpenListTeam/OpenList/v4/internal/model" +) + +func (d *GuangYaPan) ResolveOfflineResource(ctx context.Context, fileURL string) (*OfflineResolveData, error) { + if err := d.ensureAccessToken(ctx); err != nil { + return nil, err + } + fileURL = strings.TrimSpace(fileURL) + if fileURL == "" { + return nil, errors.New("offline url is empty") + } + + var resp offlineResolveResp + if err := d.postAPI(ctx, "/cloudcollection/v1/resolve_res", map[string]any{ + "url": fileURL, + }, &resp); err != nil { + return nil, err + } + if !isSuccessMsg(resp.Msg) { + return nil, fmt.Errorf("resolve offline resource failed: %s", strings.TrimSpace(resp.Msg)) + } + return &resp.Data, nil +} + +func (d *GuangYaPan) OfflineDownload(ctx context.Context, fileURL string, parentDir model.Obj, fileName string) (*OfflineTask, error) { + resolved, err := d.ResolveOfflineResource(ctx, fileURL) + if err != nil { + return nil, err + } + + parentID := parentDir.GetID() + rootID, err := d.getRootFolderID(ctx) + if err != nil { + return nil, err + } + if parentID == rootID { + parentID = "" + } + + taskURL := strings.TrimSpace(resolved.URL) + if taskURL == "" { + taskURL = strings.TrimSpace(fileURL) + } + name := strings.TrimSpace(fileName) + if name == "" { + name = resolved.defaultName(taskURL) + } + + body := map[string]any{ + "url": taskURL, + "parentId": parentID, + "newName": name, + } + if indexes := resolved.fileIndexes(); len(indexes) > 0 { + body["fileIndexes"] = indexes + } + + var resp offlineCreateResp + if err := d.postAPI(ctx, "/cloudcollection/v1/create_task", body, &resp); err != nil { + return nil, err + } + if !isSuccessMsg(resp.Msg) { + return nil, fmt.Errorf("create offline task failed: %s", strings.TrimSpace(resp.Msg)) + } + taskID := strings.TrimSpace(resp.Data.TaskID) + if taskID == "" { + return nil, errors.New("create offline task failed: empty task id") + } + return &OfflineTask{ + TaskID: taskID, + FileName: name, + Res: taskURL, + }, nil +} + +func (d *GuangYaPan) OfflineList(ctx context.Context, taskIDs []string, statuses []int, cursor string, pageSize int) ([]OfflineTask, error) { + if err := d.ensureAccessToken(ctx); err != nil { + return nil, err + } + body := map[string]any{} + if len(taskIDs) > 0 { + body["taskIds"] = taskIDs + } + if len(statuses) > 0 { + body["status"] = statuses + } + if cursor = strings.TrimSpace(cursor); cursor != "" { + body["cursor"] = cursor + } + if pageSize > 0 { + body["pageSize"] = pageSize + } + + var resp offlineListResp + if err := d.postAPI(ctx, "/cloudcollection/v1/list_task", body, &resp); err != nil { + return nil, err + } + if !isSuccessMsg(resp.Msg) { + return nil, fmt.Errorf("list offline tasks failed: %s", strings.TrimSpace(resp.Msg)) + } + return resp.Data.List, nil +} + +func (d *GuangYaPan) DeleteOfflineTasks(ctx context.Context, taskIDs []string) error { + if err := d.ensureAccessToken(ctx); err != nil { + return err + } + if len(taskIDs) == 0 { + return nil + } + + var resp offlineDeleteResp + if err := d.postAPI(ctx, "/cloudcollection/v2/delete_task", map[string]any{ + "taskIds": taskIDs, + }, &resp); err != nil { + return err + } + if !isSuccessMsg(resp.Msg) { + return fmt.Errorf("delete offline tasks failed: %s", strings.TrimSpace(resp.Msg)) + } + return nil +} + +func (d OfflineResolveData) defaultName(fileURL string) string { + if d.BTResInfo != nil && strings.TrimSpace(d.BTResInfo.FileName) != "" { + return strings.TrimSpace(d.BTResInfo.FileName) + } + u, err := url.Parse(fileURL) + if err == nil { + name := strings.TrimSpace(stdpath.Base(u.Path)) + if name != "" && name != "." && name != "/" { + if decoded, err := url.PathUnescape(name); err == nil { + name = decoded + } + return name + } + } + return "offline_download" +} + +func (d OfflineResolveData) fileIndexes() []int { + if d.BTResInfo == nil || len(d.BTResInfo.Subfiles) == 0 { + return nil + } + indexes := make([]int, 0, len(d.BTResInfo.Subfiles)) + for i, file := range d.BTResInfo.Subfiles { + if file.FileIndex != nil { + indexes = append(indexes, *file.FileIndex) + continue + } + indexes = append(indexes, i) + } + return indexes +} + +func isSuccessMsg(msg string) bool { + msg = strings.TrimSpace(msg) + return msg == "" || strings.EqualFold(msg, "success") +} diff --git a/drivers/guangyapan/types.go b/drivers/guangyapan/types.go new file mode 100644 index 00000000..d175ce9d --- /dev/null +++ b/drivers/guangyapan/types.go @@ -0,0 +1,232 @@ +package guangyapan + +import "time" + +type tokenResp struct { + AccessToken string `json:"access_token"` + RefreshToken string `json:"refresh_token"` + TokenType string `json:"token_type"` + ExpiresIn int64 `json:"expires_in"` + Sub string `json:"sub"` + Error string `json:"error"` + ErrorCode int `json:"error_code"` + ErrorDesc string `json:"error_description"` +} + +type verificationResp struct { + VerificationID string `json:"verification_id"` + Error string `json:"error"` + ErrorCode int `json:"error_code"` + ErrorDesc string `json:"error_description"` +} + +type captchaInitResp struct { + CaptchaToken string `json:"captcha_token"` + ExpiresIn int64 `json:"expires_in"` + Error string `json:"error"` + ErrorCode int `json:"error_code"` + ErrorDesc string `json:"error_description"` +} + +type verifyResp struct { + VerificationToken string `json:"verification_token"` + Error string `json:"error"` + ErrorCode int `json:"error_code"` + ErrorDesc string `json:"error_description"` +} + +type userMeResp struct { + Sub string `json:"sub"` +} + +type listResp struct { + Code int `json:"code"` + Msg string `json:"msg"` + Data struct { + Total int `json:"total"` + List []fileItem `json:"list"` + } `json:"data"` +} + +type fileItem struct { + FileID string `json:"fileId"` + ParentID string `json:"parentId"` + FileName string `json:"fileName"` + FileSize int64 `json:"fileSize"` + ResType int `json:"resType"` + CTime int64 `json:"ctime"` + UTime int64 `json:"utime"` +} + +type downloadResp struct { + Code int `json:"code"` + Msg string `json:"msg"` + Data struct { + SignedURL string `json:"signedURL"` + DownloadURL string `json:"downloadUrl"` + } `json:"data"` +} + +type createDirResp struct { + Code int `json:"code"` + Msg string `json:"msg"` + Data struct { + FileID string `json:"fileId"` + FileName string `json:"fileName"` + ResType int `json:"resType"` + CTime int64 `json:"ctime"` + UTime int64 `json:"utime"` + } `json:"data"` +} + +type commonResp struct { + Code int `json:"code"` + Msg string `json:"msg"` +} + +// taskResp is used for async operations (delete, move, copy) that return a task ID. +type taskResp struct { + Code int `json:"code"` + Msg string `json:"msg"` + Data struct { + TaskID string `json:"taskId"` + } `json:"data"` +} + +type taskStatusResp struct { + Code int `json:"code"` + Msg string `json:"msg"` + Data struct { + Status int `json:"status"` + } `json:"data"` +} + +type uploadTokenResp struct { + Code int `json:"code"` + Msg string `json:"msg"` + Data uploadTokenData `json:"data"` +} + +type uploadTokenData struct { + TaskID string `json:"taskId"` + AlreadyDone bool `json:"-"` + ObjectPath string `json:"objectPath"` + Provider any `json:"provider"` + Region string `json:"region"` + BucketName string `json:"bucketName"` + EndPoint string `json:"endPoint"` + FullEndPoint string `json:"fullEndPoint"` + CallbackVar string `json:"callbackVar"` + AccessKeyID string `json:"accessKeyID"` + SecretAccessKey string `json:"secretAccessKey"` + SessionToken string `json:"sessionToken"` + Creds struct { + AccessKeyID string `json:"accessKeyID"` + SecretAccessKey string `json:"secretAccessKey"` + SessionToken string `json:"sessionToken"` + } `json:"creds"` +} + +type taskInfoResp struct { + Code int `json:"code"` + Msg string `json:"msg"` + Data struct { + FileID string `json:"fileId"` + } `json:"data"` +} + +// assetsInfoResp is the response for storage details query. +type assetsInfoResp struct { + Code int `json:"code"` + Msg string `json:"msg"` + Data struct { + TotalSpaceSize int64 `json:"totalSpaceSize"` + UsedSpaceSize int64 `json:"usedSpaceSize"` + } `json:"data"` +} + +func (r assetsInfoResp) IsSuccess() bool { + return isSuccessMsg(r.Msg) +} + +// Offline download types + +type offlineResolveResp struct { + Code int `json:"code"` + Msg string `json:"msg"` + Data OfflineResolveData `json:"data"` +} + +type OfflineResolveData struct { + ResType int `json:"resType"` + BTResInfo *OfflineBTResInfo `json:"btResInfo"` + URL string `json:"url"` +} + +type OfflineBTResInfo struct { + InfoHash string `json:"infoHash"` + FileName string `json:"fileName"` + FileSize int64 `json:"fileSize"` + SubfilesNum int `json:"subfilesNum"` + Subfiles []OfflineSubfile `json:"subfiles"` + CreateTime int64 `json:"createTime"` + ExcludeIndices []int `json:"excludeIndices"` +} + +type OfflineSubfile struct { + FileName string `json:"fileName"` + FileIndex *int `json:"fileIndex"` + FileSize int64 `json:"fileSize"` +} + +type offlineCreateResp struct { + Code int `json:"code"` + Msg string `json:"msg"` + Data struct { + TaskID string `json:"taskId"` + URL string `json:"url"` + } `json:"data"` +} + +type offlineDeleteResp struct { + Code int `json:"code"` + Msg string `json:"msg"` + Data struct { + TaskIDs []string `json:"taskIds"` + } `json:"data"` +} + +type offlineListResp struct { + Code int `json:"code"` + Msg string `json:"msg"` + Data struct { + StatusCounts []struct { + Status int `json:"status"` + Count int `json:"count"` + } `json:"statusCounts"` + Cursor string `json:"cursor"` + List []OfflineTask `json:"list"` + Total int `json:"total"` + } `json:"data"` +} + +type OfflineTask struct { + TaskID string `json:"taskId"` + FileName string `json:"fileName"` + TotalSize int64 `json:"totalSize"` + Status int `json:"status"` + CreateTime int64 `json:"createTime"` + Res string `json:"res"` + ResType int `json:"resType"` + Progress int `json:"progress"` + FileID string `json:"fileId"` + IsDir bool `json:"isDir"` + Exist bool `json:"exist"` +} + +func unixOrZero(v int64) time.Time { + if v <= 0 { + return time.Time{} + } + return time.Unix(v, 0) +} diff --git a/drivers/guangyapan/util.go b/drivers/guangyapan/util.go new file mode 100644 index 00000000..e98c2bdd --- /dev/null +++ b/drivers/guangyapan/util.go @@ -0,0 +1,436 @@ +package guangyapan + +import ( + "context" + "crypto/rand" + "encoding/hex" + "errors" + "fmt" + "io" + "net/url" + "strings" + "time" + + "github.com/OpenListTeam/OpenList/v4/internal/driver" + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/internal/op" + streamPkg "github.com/OpenListTeam/OpenList/v4/internal/stream" + "github.com/OpenListTeam/OpenList/v4/pkg/utils" + "github.com/aliyun/aliyun-oss-go-sdk/oss" + "github.com/avast/retry-go" + "github.com/go-resty/resty/v2" + "golang.org/x/time/rate" +) + +// --- HTTP request helpers --- + +func (d *GuangYaPan) accountErr(desc, short string, resp *resty.Response) string { + msg := strings.TrimSpace(desc) + if msg == "" { + msg = strings.TrimSpace(short) + } + if msg == "" && resp != nil { + msg = strings.TrimSpace(resp.String()) + } + if msg == "" && resp != nil { + msg = fmt.Sprintf("status=%d", resp.StatusCode()) + } + if msg == "" { + msg = "unknown error" + } + return msg +} + +func (d *GuangYaPan) apiRateLimitWait(ctx context.Context, path string) error { + value, _ := d.apiRateLimit.LoadOrStore(path, rate.NewLimiter(rate.Every(apiRateInterval), 1)) + return value.(*rate.Limiter).Wait(ctx) +} + +func (d *GuangYaPan) postAPI(ctx context.Context, path string, body any, out any) error { + if err := d.ensureAccessToken(ctx); err != nil { + return err + } + if err := d.apiRateLimitWait(ctx, path); err != nil { + return err + } + resp, err := d.apiClient.R(). + SetContext(ctx). + SetHeader("Authorization", "Bearer "+d.AccessToken). + SetBody(body). + SetResult(out). + Post(path) + if err != nil { + return err + } + if resp.StatusCode() == 401 || resp.StatusCode() == 403 { + if strings.TrimSpace(d.RefreshToken) == "" { + return fmt.Errorf("request failed: status=%d body=%s", resp.StatusCode(), resp.String()) + } + if err := d.refreshToken(ctx); err != nil { + return err + } + resp, err = d.apiClient.R(). + SetContext(ctx). + SetHeader("Authorization", "Bearer "+d.AccessToken). + SetBody(body). + SetResult(out). + Post(path) + if err != nil { + return err + } + } + if resp.IsError() { + return fmt.Errorf("request failed: status=%d body=%s", resp.StatusCode(), resp.String()) + } + return nil +} + +func (d *GuangYaPan) setTempStatus(status string) { + if d.statusTimer != nil { + d.statusTimer.Stop() + } + // initStorage sets status to WORK after Init returns, so we update it shortly after. + d.statusTimer = time.AfterFunc(200*time.Millisecond, func() { + d.GetStorage().SetStatus(status) + op.MustSaveDriverStorage(d) + }) +} + +// --- Task polling helpers --- + +func (d *GuangYaPan) waitTaskDone(ctx context.Context, taskID string) error { + const ( + maxTry = 30 + interval = 300 * time.Millisecond + ) + for i := 0; i < maxTry; i++ { + var out taskStatusResp + if err := d.postAPI(ctx, "/nd.bizuserres.s/v1/get_task_status", map[string]any{ + "taskId": taskID, + }, &out); err != nil { + return err + } + if !isSuccessMsg(out.Msg) { + return fmt.Errorf("get task status failed: %s", strings.TrimSpace(out.Msg)) + } + switch out.Data.Status { + case 2: + return nil + case -1, 3: + return fmt.Errorf("task %s failed with status=%d", taskID, out.Data.Status) + } + if i == maxTry-1 { + break + } + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(interval): + } + } + return fmt.Errorf("task %s timeout", taskID) +} + +// --- Upload helpers --- + +// fileMD5 returns the MD5 of the file content, computing it from the stream when +// the caller did not provide it (e.g. local uploads). The computed hash is used +// to attempt instant upload (秒传) before falling back to the real OSS upload. +// For force-stream uploads the whole file would have to be buffered just to hash +// it, which defeats the purpose of streaming, so we skip 秒传 in that case. +func (d *GuangYaPan) fileMD5(file model.FileStreamer, up driver.UpdateProgress) (string, error) { + if md5sum := file.GetHash().GetHash(utils.MD5); len(md5sum) == utils.MD5.Width { + return md5sum, nil + } + if file.IsForceStreamUpload() { + return "", nil + } + _, md5sum, err := streamPkg.CacheFullAndHash(file, &up, utils.MD5) + if err != nil { + return "", err + } + return md5sum, nil +} + +func (d *GuangYaPan) getUploadToken(ctx context.Context, parentID, name string, size int64, md5sum string) (*uploadTokenData, int, error) { + var out uploadTokenResp + res := map[string]any{ + "fileSize": size, + } + if md5sum != "" { + // 秒传：后端根据 md5 判断文件是否已存在，命中则返回 code 156 秒传完成。 + res["md5"] = md5sum + } + err := d.postAPI(ctx, "/nd.bizuserres.s/v1/get_res_center_token", map[string]any{ + "capacity": 2, + "name": name, + "parentId": parentID, + "res": res, + }, &out) + if err != nil { + return nil, 0, err + } + msg := strings.TrimSpace(out.Msg) + if !isSuccessMsg(msg) && !isUploadAlreadyDone(msg) { + return nil, out.Code, fmt.Errorf("get upload token failed: %s", msg) + } + if out.Data.TaskID == "" { + return nil, out.Code, errors.New("get upload token failed: empty task id") + } + // When the backend reports the file is already uploaded/instant-uploaded, + // it returns a valid TaskID without OSS credentials. + // Mark it so the caller can skip the real upload and just wait for the task. + if out.Code == 156 || isUploadAlreadyDone(msg) { + out.Data.AlreadyDone = true + } + if out.Data.AccessKeyID == "" { + out.Data.AccessKeyID = out.Data.Creds.AccessKeyID + } + if out.Data.SecretAccessKey == "" { + out.Data.SecretAccessKey = out.Data.Creds.SecretAccessKey + } + if out.Data.SessionToken == "" { + out.Data.SessionToken = out.Data.Creds.SessionToken + } + if strings.TrimSpace(out.Data.EndPoint) == "" { + out.Data.EndPoint = strings.TrimSpace(out.Data.FullEndPoint) + } + if strings.TrimSpace(out.Data.EndPoint) != "" && !strings.HasPrefix(out.Data.EndPoint, "http://") && !strings.HasPrefix(out.Data.EndPoint, "https://") { + if strings.TrimSpace(out.Data.FullEndPoint) != "" { + out.Data.EndPoint = strings.TrimSpace(out.Data.FullEndPoint) + } else if strings.TrimSpace(out.Data.BucketName) != "" { + host := strings.TrimSpace(out.Data.EndPoint) + prefix := strings.TrimSpace(out.Data.BucketName) + "." + if strings.HasPrefix(host, prefix) { + out.Data.EndPoint = "https://" + host + } else { + out.Data.EndPoint = "https://" + strings.TrimSpace(out.Data.BucketName) + "." + host + } + } else { + out.Data.EndPoint = "https://" + strings.TrimSpace(out.Data.EndPoint) + } + } + return &out.Data, out.Code, nil +} + +func (d *GuangYaPan) waitUploadTaskInfo(ctx context.Context, taskID string) error { + const ( + maxTry = 300 + interval = 1 * time.Second + ) + for i := 0; i < maxTry; i++ { + var out taskInfoResp + if err := d.postAPI(ctx, "/nd.bizuserres.s/v1/file/get_info_by_task_id", map[string]any{ + "taskId": taskID, + }, &out); err != nil { + return err + } + if out.Data.FileID != "" { + return nil + } + switch out.Code { + case 145, 146, 147, 155, 163, 0: + // uploading/verifying/processing + default: + if strings.TrimSpace(out.Msg) != "" { + return fmt.Errorf("upload task failed: code=%d msg=%s", out.Code, strings.TrimSpace(out.Msg)) + } + } + if i == maxTry-1 { + break + } + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(interval): + } + } + return fmt.Errorf("upload task %s timeout", taskID) +} + +func (d *GuangYaPan) multipartUploadToOSS(ctx context.Context, bucket *oss.Bucket, objectPath string, file model.FileStreamer, up driver.UpdateProgress) error { + partSize := calcUploadPartSize(file.GetSize()) + imur, err := bucket.InitiateMultipartUpload(objectPath, oss.Sequential()) + if err != nil { + return err + } + + total := file.GetSize() + partCount := int((total + partSize - 1) / partSize) + + // Use StreamSectionReader for seekable, retryable chunk reads (hybrid cache). + ss, err := streamPkg.NewStreamSectionReader(file, int(partSize), &up) + if err != nil { + return err + } + + parts := make([]oss.UploadPart, 0, partCount) + var uploaded int64 + for i := 0; i < partCount; i++ { + if utils.IsCanceled(ctx) { + return ctx.Err() + } + + offset := int64(i) * partSize + length := partSize + if remain := total - offset; length > remain { + length = remain + } + + rd, err := ss.GetSectionReader(offset, length) + if err != nil { + return err + } + + var part oss.UploadPart + err = retry.Do(func() error { + rd.Seek(0, io.SeekStart) + var uploadErr error + part, uploadErr = bucket.UploadPart(imur, driver.NewLimitedUploadStream(ctx, rd), length, i+1) + return uploadErr + }, + retry.Context(ctx), + retry.Attempts(3), + retry.DelayType(retry.BackOffDelay), + retry.Delay(time.Second)) + ss.FreeSectionReader(rd) + if err != nil { + return fmt.Errorf("failed to upload part %d: %w", i+1, err) + } + parts = append(parts, part) + uploaded += length + if total > 0 { + up(100 * float64(uploaded) / float64(total)) + } + } + + _, err = bucket.CompleteMultipartUpload(imur, parts) + return err +} + +// --- Normalization helpers --- + +func normalizeOSSEndpoint(endpoint, bucket string) string { + ep := strings.TrimSpace(endpoint) + if ep == "" { + return ep + } + if !strings.HasPrefix(ep, "http://") && !strings.HasPrefix(ep, "https://") { + ep = "https://" + ep + } + u, err := url.Parse(ep) + if err != nil || u.Host == "" { + return ep + } + host := u.Host + prefix := strings.TrimSpace(bucket) + if prefix != "" && strings.HasPrefix(host, prefix+".") { + host = strings.TrimPrefix(host, prefix+".") + } + u.Host = host + return u.String() +} + +func normalizeDeviceID(v string) string { + v = strings.ToLower(strings.TrimSpace(v)) + v = strings.ReplaceAll(v, "-", "") + if len(v) != 32 { + return "" + } + for _, ch := range v { + if (ch < '0' || ch > '9') && (ch < 'a' || ch > 'f') { + return "" + } + } + return v +} + +func randomDeviceID() string { + b := make([]byte, 16) + if _, err := rand.Read(b); err != nil { + return "0123456789abcdef0123456789abcdef" + } + return hex.EncodeToString(b) +} + +func normalizeCaptchaUsername(phone string) string { + p := strings.TrimSpace(phone) + p = strings.ReplaceAll(p, " ", "") + p = strings.TrimPrefix(p, "+") + // Keep only digits. + b := make([]rune, 0, len(p)) + for _, ch := range p { + if ch >= '0' && ch <= '9' { + b = append(b, ch) + } + } + digits := string(b) + // Mainland number normalization: +86xxxxxxxxxxx -> xxxxxxxxxxx + if strings.HasPrefix(digits, "86") && len(digits) > 11 { + digits = digits[2:] + } + return digits +} + +func normalizePhoneE164(phone string) string { + p := strings.TrimSpace(phone) + if p == "" { + return "" + } + p = strings.ReplaceAll(p, " ", "") + if strings.HasPrefix(p, "+") { + // Format as "+86 1xxxxxxxxxx" to match browser payload expectations. + if strings.HasPrefix(p, "+86") && len(p) > 3 { + rest := strings.TrimPrefix(p, "+86") + return "+86 " + rest + } + return p + } + // If raw mainland number is provided, normalize with +86 prefix. + digits := normalizeCaptchaUsername(p) + if len(digits) == 11 { + return "+86 " + digits + } + return p +} + +func calcUploadPartSize(size int64) int64 { + const ( + mb = int64(1024 * 1024) + gb = int64(1024 * 1024 * 1024) + ) + switch { + case size <= 100*mb: + return 1 * mb + case size <= 16*gb: + return 2 * mb + case size <= 160*gb: + return 4 * mb + default: + return 8 * mb + } +} + +// isUploadAlreadyDone reports whether the upload-token response indicates the +// file was already uploaded (instant upload). In that case the backend returns +// a valid TaskID but no OSS credentials, and we should just wait for the task +// instead of starting a real upload. +func isUploadAlreadyDone(msg string) bool { + msg = strings.TrimSpace(msg) + if msg == "" { + return false + } + if strings.EqualFold(msg, "上传已完成") { + return true + } + if strings.EqualFold(msg, "upload completed") { + return true + } + if strings.EqualFold(msg, "already uploaded") { + return true + } + if strings.EqualFold(msg, "秒传成功") { + return true + } + return false +} diff --git a/drivers/ilanzou/driver.go b/drivers/ilanzou/driver.go index 73b31014..a640d866 100644 --- a/drivers/ilanzou/driver.go +++ b/drivers/ilanzou/driver.go @@ -1,4 +1,4 @@ -package template +package ilanzou import ( "context" @@ -7,6 +7,7 @@ import ( "fmt" "io" "net/http" + "net/http/cookiejar" "net/url" "strconv" "strings" @@ -27,11 +28,13 @@ type ILanZou struct { model.Storage Addition - userID string - account string - upClient *resty.Client - conf Conf - config driver.Config + userID string + account string + apiClient *resty.Client + linkClient *resty.Client + upClient *resty.Client + conf Conf + config driver.Config } func (d *ILanZou) Config() driver.Config { @@ -43,6 +46,18 @@ func (d *ILanZou) GetAddition() driver.Additional { } func (d *ILanZou) Init(ctx context.Context) error { + jar, err := cookiejar.New(nil) + if err != nil { + return err + } + // Keep state isolated per storage. The console and CDN issue cookies that + // must survive a retry but must never leak to another configured account. + d.apiClient = base.NewRestyClient().SetCookieJar(jar) + d.linkClient = base.NewRestyClient().SetCookieJar(jar).SetRedirectPolicy( + resty.RedirectPolicyFunc(func(_ *http.Request, _ []*http.Request) error { + return http.ErrUseLastResponse + }), + ) d.upClient = base.NewRestyClient().SetTimeout(time.Minute * 10) if d.UUID == "" { res, err := d.unproved("/getUuid", http.MethodGet, nil) @@ -77,8 +92,7 @@ func (d *ILanZou) List(ctx context.Context, dir model.Obj, args model.ListArgs) "folderId=" + dir.GetID(), "type=0", } - queryString := strings.Join(params, "&") - req.SetQueryString(queryString).SetResult(&resp) + req.SetQueryString(strings.Join(params, "&")).SetResult(&resp) }) if err != nil { return nil, err @@ -96,14 +110,12 @@ func (d *ILanZou) List(ctx context.Context, dir model.Obj, args model.ListArgs) return nil, err } obj := model.Object{ - ID: strconv.FormatInt(f.FileId, 10), - // Path: "", + ID: strconv.FormatInt(f.FileId, 10), Name: f.FileName, Size: f.FileSize * 1024, Modified: updTime, Ctime: updTime, IsFolder: false, - // HashInfo: utils.HashInfo{}, } if f.FileType == 2 { obj.IsFolder = true @@ -133,26 +145,23 @@ func (d *ILanZou) Link(ctx context.Context, file model.Obj, args model.LinkArgs) "devVersion=" + url.QueryEscape(d.conf.devVersion), "appVersion=", "timestamp=" + tsStr, - "appToken=" + url.QueryEscape(d.Token), + "appToken=" + appTokenQueryValue(d.Token), "enable=1", } - - downloadId, err := mopan.AesEncrypt([]byte(fmt.Sprintf("%s|%s", file.GetID(), d.userID)), d.conf.secret) + downloadID, err := mopan.AesEncrypt([]byte(fmt.Sprintf("%s|%s", file.GetID(), d.userID)), d.conf.secret) if err != nil { return nil, err } - params = append(params, "downloadId="+url.QueryEscape(hex.EncodeToString(downloadId))) - + params = append(params, "downloadId="+url.QueryEscape(hex.EncodeToString(downloadID))) auth, err := mopan.AesEncrypt([]byte(fmt.Sprintf("%s|%d", file.GetID(), ts)), d.conf.secret) if err != nil { return nil, err } params = append(params, "auth="+url.QueryEscape(hex.EncodeToString(auth))) - u.RawQuery = strings.Join(params, "&") realURL := u.String() - // get the url after redirect - req := base.NoRedirectClient.R().SetContext(ctx) + + req := d.linkClient.R().SetContext(ctx) req.SetHeaders(map[string]string{ "Origin": d.conf.site, "Referer": d.conf.site + "/", @@ -162,7 +171,6 @@ func (d *ILanZou) Link(ctx context.Context, file model.Obj, args model.LinkArgs) if d.Addition.Ip != "" { req.SetHeader("X-Forwarded-For", d.Addition.Ip) } - res, err := req.Get(realURL) if err != nil { return nil, err @@ -172,49 +180,50 @@ func (d *ILanZou) Link(ctx context.Context, file model.Obj, args model.LinkArgs) realURL = location } else if res.StatusCode() == http.StatusOK && location != "" { realURL = location + } else if res.StatusCode() == http.StatusOK { + // Some file types return a 200 JSON resolver response instead of a 3xx. + realURL = utils.Json.Get(res.Body(), "url").ToString() + if realURL == "" { + realURL = utils.Json.Get(res.Body(), "data", "url").ToString() + } + if realURL == "" { + return nil, fmt.Errorf("download resolver returned no URL: %s", utils.Json.Get(res.Body(), "msg").ToString()) + } } else { return nil, fmt.Errorf("redirect failed, status: %d, location: %s, msg: %s", res.StatusCode(), location, utils.Json.Get(res.Body(), "msg").ToString()) } - link := model.Link{URL: realURL} - return &link, nil + link := &model.Link{URL: realURL} + // Probe the CDN for the actual object size; API metadata can differ from + // the bytes served by the final URL. The timeout bounds Link latency. + headCtx, cancel := context.WithTimeout(ctx, linkHeadTimeout) + defer cancel() + if response, err := d.apiClient.R().SetContext(headCtx).Head(realURL); err == nil && response.StatusCode() >= http.StatusOK && response.StatusCode() < http.StatusMultipleChoices { + if size, parseErr := strconv.ParseInt(response.Header().Get("Content-Length"), 10, 64); parseErr == nil && size > 0 { + link.ContentLength = size + } + } + return link, nil } func (d *ILanZou) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) (model.Obj, error) { res, err := d.proved("/file/folder/save", http.MethodPost, func(req *resty.Request) { - req.SetBody(base.Json{ - "folderDesc": "", - "folderId": parentDir.GetID(), - "folderName": dirName, - }) + req.SetBody(base.Json{"folderDesc": "", "folderId": parentDir.GetID(), "folderName": dirName}) }) if err != nil { return nil, err } - return &model.Object{ - ID: utils.Json.Get(res, "list", 0, "id").ToString(), - // Path: "", - Name: dirName, - Size: 0, - Modified: time.Now(), - Ctime: time.Now(), - IsFolder: true, - // HashInfo: utils.HashInfo{}, - }, nil + return &model.Object{ID: utils.Json.Get(res, "list", 0, "id").ToString(), Name: dirName, Modified: time.Now(), Ctime: time.Now(), IsFolder: true}, nil } func (d *ILanZou) Move(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { - var fileIds, folderIds []string + var fileIDs, folderIDs []string if srcObj.IsDir() { - folderIds = []string{srcObj.GetID()} + folderIDs = []string{srcObj.GetID()} } else { - fileIds = []string{srcObj.GetID()} + fileIDs = []string{srcObj.GetID()} } _, err := d.proved("/file/folder/move", http.MethodPost, func(req *resty.Request) { - req.SetBody(base.Json{ - "folderIds": strings.Join(folderIds, ","), - "fileIds": strings.Join(fileIds, ","), - "targetId": dstDir.GetID(), - }) + req.SetBody(base.Json{"folderIds": strings.Join(folderIDs, ","), "fileIds": strings.Join(fileIDs, ","), "targetId": dstDir.GetID()}) }) if err != nil { return nil, err @@ -226,58 +235,47 @@ func (d *ILanZou) Rename(ctx context.Context, srcObj model.Obj, newName string) var err error if srcObj.IsDir() { _, err = d.proved("/file/folder/edit", http.MethodPost, func(req *resty.Request) { - req.SetBody(base.Json{ - "folderDesc": "", - "folderId": srcObj.GetID(), - "folderName": newName, - }) + req.SetBody(base.Json{"folderDesc": "", "folderId": srcObj.GetID(), "folderName": newName}) }) } else { _, err = d.proved("/file/edit", http.MethodPost, func(req *resty.Request) { - req.SetBody(base.Json{ - "fileDesc": "", - "fileId": srcObj.GetID(), - "fileName": newName, - }) + req.SetBody(base.Json{"fileDesc": "", "fileId": srcObj.GetID(), "fileName": newName}) }) } if err != nil { return nil, err } - return &model.Object{ - ID: srcObj.GetID(), - // Path: "", - Name: newName, - Size: srcObj.GetSize(), - Modified: time.Now(), - Ctime: srcObj.CreateTime(), - IsFolder: srcObj.IsDir(), - }, nil + return &model.Object{ID: srcObj.GetID(), Name: newName, Size: srcObj.GetSize(), Modified: time.Now(), Ctime: srcObj.CreateTime(), IsFolder: srcObj.IsDir()}, nil } +// iLanzou has no server-side copy primitive. Returning NotImplement delegates +// to OpenList's persistent CopyTaskManager instead of blocking the HTTP request. func (d *ILanZou) Copy(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { - // TODO copy obj, optional return nil, errs.NotImplement } func (d *ILanZou) Remove(ctx context.Context, obj model.Obj) error { - var fileIds, folderIds []string + var fileIDs, folderIDs []string if obj.IsDir() { - folderIds = []string{obj.GetID()} + folderIDs = []string{obj.GetID()} } else { - fileIds = []string{obj.GetID()} + fileIDs = []string{obj.GetID()} } _, err := d.proved("/file/delete", http.MethodPost, func(req *resty.Request) { - req.SetBody(base.Json{ - "folderIds": strings.Join(folderIds, ","), - "fileIds": strings.Join(fileIds, ","), - "status": 0, - }) + req.SetBody(base.Json{"folderIds": strings.Join(folderIDs, ","), "fileIds": strings.Join(fileIDs, ","), "status": 0}) }) return err } -const DefaultPartSize = 1024 * 1024 * 8 +const ( + DefaultPartSize = 1024 * 1024 * 8 + + // The results endpoint is eventually consistent after the upload commit. + maxUploadCommitRetries = 10 + uploadCommitRetryDelay = time.Second + + linkHeadTimeout = 5 * time.Second +) func (d *ILanZou) Put(ctx context.Context, dstDir model.Obj, s model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) { etag := s.GetHash().GetHash(utils.MD5) @@ -288,55 +286,31 @@ func (d *ILanZou) Put(ctx context.Context, dstDir model.Obj, s model.FileStreame return nil, err } } - // get upToken + fileSizeKiB := (s.GetSize() + 1023) / 1024 + if fileSizeKiB < 1 { + fileSizeKiB = 1 + } res, err := d.proved("/7n/getUpToken", http.MethodPost, func(req *resty.Request) { - req.SetBody(base.Json{ - "fileId": "", - "fileName": s.GetName(), - "fileSize": s.GetSize()/1024 + 1, - "folderId": dstDir.GetID(), - "md5": etag, - "type": 1, - }) + req.SetBody(base.Json{"fileId": "", "fileName": s.GetName(), "fileSize": fileSizeKiB, "folderId": dstDir.GetID(), "md5": etag, "type": 1}) }) if err != nil { return nil, err } upToken := utils.Json.Get(res, "upToken").ToString() if upToken == "-1" { - // 支持秒传 var resp UploadTokenRapidResp - err := utils.Json.Unmarshal(res, &resp) - if err != nil { + if err := utils.Json.Unmarshal(res, &resp); err != nil { return nil, err } - return &model.Object{ - ID: strconv.FormatInt(resp.Map.FileID, 10), - Name: resp.Map.FileName, - Size: s.GetSize(), - Modified: s.ModTime(), - Ctime: s.CreateTime(), - IsFolder: false, - HashInfo: utils.NewHashInfo(utils.MD5, etag), - }, nil + return &model.Object{ID: strconv.FormatInt(resp.Map.FileID, 10), Name: resp.Map.FileName, Size: s.GetSize(), Modified: s.ModTime(), Ctime: s.CreateTime(), IsFolder: false, HashInfo: utils.NewHashInfo(utils.MD5, etag)}, nil } now := time.Now() - key := fmt.Sprintf("disk/%d/%d/%d/%s/%016d", now.Year(), now.Month(), now.Day(), d.account, now.UnixMilli()) - reader := driver.NewLimitedUploadStream(ctx, &driver.ReaderUpdatingProgress{ - Reader: &driver.SimpleReaderWithSize{ - Reader: s, - Size: s.GetSize(), - }, - UpdateProgress: up, - }) + // Match the current console's generated Qiniu object key. + key := fmt.Sprintf("disk/%04d/%02d/%02d/%s/%d.rar", now.Year(), now.Month(), now.Day(), d.account, now.UnixMilli()) + reader := driver.NewLimitedUploadStream(ctx, &driver.ReaderUpdatingProgress{Reader: &driver.SimpleReaderWithSize{Reader: s, Size: s.GetSize()}, UpdateProgress: up}) var token string if s.GetSize() <= DefaultPartSize { - res, err := d.upClient.R().SetContext(ctx).SetMultipartFormData(map[string]string{ - "token": upToken, - "key": key, - "fname": s.GetName(), - }).SetMultipartField("file", s.GetName(), s.GetMimetype(), reader). - Post("https://upload.qiniup.com/") + res, err := d.upClient.R().SetContext(ctx).SetMultipartFormData(map[string]string{"token": upToken, "key": key, "fname": s.GetName()}).SetMultipartField("file", s.GetName(), s.GetMimetype(), reader).Post("https://upload.qiniup.com/") if err != nil { return nil, err } @@ -347,40 +321,27 @@ func (d *ILanZou) Put(ctx context.Context, dstDir model.Obj, s model.FileStreame if err != nil { return nil, err } - uploadId := utils.Json.Get(res.Body(), "uploadId").ToString() + uploadID := utils.Json.Get(res.Body(), "uploadId").ToString() parts := make([]Part, 0) partNum := (s.GetSize() + DefaultPartSize - 1) / DefaultPartSize for i := 1; i <= int(partNum); i++ { - u := fmt.Sprintf("https://upload.qiniup.com/buckets/%s/objects/%s/uploads/%s/%d", d.conf.bucket, keyBase64, uploadId, i) + u := fmt.Sprintf("https://upload.qiniup.com/buckets/%s/objects/%s/uploads/%s/%d", d.conf.bucket, keyBase64, uploadID, i) res, err = d.upClient.R().SetContext(ctx).SetHeader("Authorization", "UpToken "+upToken).SetBody(io.LimitReader(reader, DefaultPartSize)).Put(u) if err != nil { return nil, err } - etag := utils.Json.Get(res.Body(), "etag").ToString() - parts = append(parts, Part{ - PartNumber: i, - ETag: etag, - }) + parts = append(parts, Part{PartNumber: i, ETag: utils.Json.Get(res.Body(), "etag").ToString()}) } - res, err = d.upClient.R().SetHeader("Authorization", "UpToken "+upToken).SetBody(base.Json{ - "fnmae": s.GetName(), - "parts": parts, - }).Post(fmt.Sprintf("https://upload.qiniup.com/buckets/%s/objects/%s/uploads/%s", d.conf.bucket, keyBase64, uploadId)) + res, err = d.upClient.R().SetHeader("Authorization", "UpToken "+upToken).SetBody(base.Json{"fnmae": s.GetName(), "parts": parts}).Post(fmt.Sprintf("https://upload.qiniup.com/buckets/%s/objects/%s/uploads/%s", d.conf.bucket, keyBase64, uploadID)) if err != nil { return nil, err } token = utils.Json.Get(res.Body(), "token").ToString() } - // commit upload var resp UploadResultResp - for i := 0; i < 10; i++ { + for i := 0; i < maxUploadCommitRetries; i++ { _, err = d.unproved("/7n/results", http.MethodPost, func(req *resty.Request) { - params := []string{ - "tokenList=" + token, - "tokenTime=" + time.Now().Format("Mon Jan 02 2006 15:04:05 GMT-0700 (MST)"), - } - queryString := strings.Join(params, "&") - req.SetQueryString(queryString).SetResult(&resp) + req.SetQueryString("tokenList=" + token + "&tokenTime=" + time.Now().Format("Mon Jan 02 2006 15:04:05 GMT-0700 (MST)")).SetResult(&resp) }) if err != nil { return nil, err @@ -391,22 +352,17 @@ func (d *ILanZou) Put(ctx context.Context, dstDir model.Obj, s model.FileStreame if resp.List[0].Status == 1 { break } - time.Sleep(time.Second * 1) + select { + case <-ctx.Done(): + return nil, ctx.Err() + case <-time.After(uploadCommitRetryDelay): + } } file := resp.List[0] if file.Status != 1 { - return nil, fmt.Errorf("upload failed, status: %d", resp.List[0].Status) + return nil, fmt.Errorf("upload failed, status: %d", file.Status) } - return &model.Object{ - ID: strconv.FormatInt(file.FileId, 10), - // Path: , - Name: file.FileName, - Size: s.GetSize(), - Modified: s.ModTime(), - Ctime: s.CreateTime(), - IsFolder: false, - HashInfo: utils.NewHashInfo(utils.MD5, etag), - }, nil + return &model.Object{ID: strconv.FormatInt(file.FileId, 10), Name: file.FileName, Size: s.GetSize(), Modified: s.ModTime(), Ctime: s.CreateTime(), IsFolder: false, HashInfo: utils.NewHashInfo(utils.MD5, etag)}, nil } func (d *ILanZou) GetDetails(ctx context.Context) (*model.StorageDetails, error) { @@ -419,18 +375,8 @@ func (d *ILanZou) GetDetails(ctx context.Context) (*model.StorageDetails, error) vipSize := utils.Json.Get(res, "map", "vipSize").ToInt64() * 1024 totalSize := utils.Json.Get(res, "map", "totalSize").ToInt64() * 1024 rewardSize := utils.Json.Get(res, "map", "rewardSize").ToInt64() * 1024 - total := totalSize + rewardSize + vipSize used := utils.Json.Get(res, "map", "usedSize").ToInt64() * 1024 - return &model.StorageDetails{ - DiskUsage: model.DiskUsage{ - TotalSpace: total, - UsedSpace: used, - }, - }, nil + return &model.StorageDetails{DiskUsage: model.DiskUsage{TotalSpace: totalSize + rewardSize + vipSize, UsedSpace: used}}, nil } -//func (d *ILanZou) Other(ctx context.Context, args model.OtherArgs) (interface{}, error) { -// return nil, errs.NotSupport -//} - var _ driver.Driver = (*ILanZou)(nil) diff --git a/drivers/ilanzou/meta.go b/drivers/ilanzou/meta.go index 35637aa5..b20146b1 100644 --- a/drivers/ilanzou/meta.go +++ b/drivers/ilanzou/meta.go @@ -1,4 +1,4 @@ -package template +package ilanzou import ( "github.com/OpenListTeam/OpenList/v4/internal/driver" diff --git a/drivers/ilanzou/types.go b/drivers/ilanzou/types.go index 04579c36..70dc8a03 100644 --- a/drivers/ilanzou/types.go +++ b/drivers/ilanzou/types.go @@ -1,4 +1,4 @@ -package template +package ilanzou type ListResp struct { Msg string `json:"msg"` diff --git a/drivers/ilanzou/util.go b/drivers/ilanzou/util.go index ea03cfd0..11087ac4 100644 --- a/drivers/ilanzou/util.go +++ b/drivers/ilanzou/util.go @@ -1,4 +1,4 @@ -package template +package ilanzou import ( "encoding/hex" @@ -18,10 +18,7 @@ import ( func (d *ILanZou) login() error { res, err := d.unproved("/login", http.MethodPost, func(req *resty.Request) { - req.SetBody(base.Json{ - "loginName": d.Username, - "loginPwd": d.Password, - }) + req.SetBody(base.Json{"loginName": d.Username, "loginPwd": d.Password}) }) if err != nil { return err @@ -35,20 +32,33 @@ func (d *ILanZou) login() error { func getTimestamp(secret []byte) (int64, string, error) { ts := time.Now().UnixMilli() - tsStr := strconv.FormatInt(ts, 10) - res, err := mopan.AesEncrypt([]byte(tsStr), secret) + res, err := mopan.AesEncrypt([]byte(strconv.FormatInt(ts, 10)), secret) if err != nil { return 0, "", err } return ts, hex.EncodeToString(res), nil } +// isCDNChallenge detects a transient CDN challenge (409 Conflict + HTML 403). +// A retry with the session cookie jar can resolve transient challenges. +func isCDNChallenge(res *resty.Response) bool { + if res == nil || res.StatusCode() != http.StatusConflict { + return false + } + return strings.Contains(res.Header().Get("Content-Type"), "text/html") && strings.Contains(string(res.Body()), "403") +} + +func appTokenQueryValue(token string) string { + // iLanzou requires a literal colon, while other reserved token characters + // must remain query-escaped. + return strings.ReplaceAll(url.QueryEscape(token), "%3A", ":") +} + func (d *ILanZou) request(pathname, method string, callback base.ReqCallback, proved bool, retry ...bool) ([]byte, error) { - _, ts_str, err := getTimestamp(d.conf.secret) + _, timestamp, err := getTimestamp(d.conf.secret) if err != nil { return nil, err } - params := []string{ "uuid=" + url.QueryEscape(d.UUID), "devType=6", @@ -56,48 +66,51 @@ func (d *ILanZou) request(pathname, method string, callback base.ReqCallback, pr "devModel=chrome", "devVersion=" + url.QueryEscape(d.conf.devVersion), "appVersion=", - "timestamp=" + ts_str, + "timestamp=" + timestamp, } - if proved { - params = append(params, "appToken="+url.QueryEscape(d.Token)) + params = append(params, "appToken="+appTokenQueryValue(d.Token)) } - params = append(params, "extra=2") - queryString := strings.Join(params, "&") - - req := base.RestyClient.R() + if d.apiClient == nil { + return nil, fmt.Errorf("iLanzou driver is not initialized") + } + req := d.apiClient.R() req.SetHeaders(map[string]string{ "Origin": d.conf.site, "Referer": d.conf.site + "/", "Accept-Encoding": "gzip", "Accept-Language": "zh-CN,zh;q=0.9,en-US,en;q=0.8", }) - if d.Addition.Ip != "" { req.SetHeader("X-Forwarded-For", d.Addition.Ip) } - if callback != nil { callback(req) } - - res, err := req.Execute(method, d.conf.base+pathname+"?"+queryString) + res, err := req.Execute(method, d.conf.base+pathname+"?"+strings.Join(params, "&")) if err != nil { if res != nil { - log.Errorf("[iLanZou] request error: %s", res.String()) + log.Errorf("[ilanzou] request error: %s", res.String()) } return nil, err } isRetry := len(retry) > 0 && retry[0] + if isCDNChallenge(res) { + // Resty's cookie jar records the challenge cookie from the first response. + // A second request proves whether it is a transient challenge or hard block. + if !isRetry { + return d.request(pathname, method, callback, proved, true) + } + return nil, fmt.Errorf("iLanzou CDN rejected %s with HTML 403; retry later or delete from the console", pathname) + } body := res.Body() code := utils.Json.Get(body, "code").ToInt() msg := utils.Json.Get(body, "msg").ToString() if code != 200 { if !isRetry && proved && (utils.SliceContains([]int{-1, -2}, code) || d.Token == "") { - err = d.login() - if err != nil { + if err := d.login(); err != nil { return nil, err } return d.request(pathname, method, callback, proved, true) diff --git a/drivers/ilanzou/util_test.go b/drivers/ilanzou/util_test.go new file mode 100644 index 00000000..10d6eb23 --- /dev/null +++ b/drivers/ilanzou/util_test.go @@ -0,0 +1,11 @@ +package ilanzou + +import "testing" + +func TestAppTokenQueryValue(t *testing.T) { + got := appTokenQueryValue("alpha:beta+gamma&delta#percent%") + want := "alpha:beta%2Bgamma%26delta%23percent%25" + if got != want { + t.Fatalf("appTokenQueryValue() = %q, want %q", got, want) + } +} diff --git a/drivers/lanzou/help.go b/drivers/lanzou/help.go index 930c7248..e702ebca 100644 --- a/drivers/lanzou/help.go +++ b/drivers/lanzou/help.go @@ -170,23 +170,17 @@ var findKVReg = regexp.MustCompile(`'(.+?)':('?([^' },]*)'?)`) // 拆分kv // 根据key查询js变量 func findJSVarFunc(key, data string) string { - var values []string - if key != "sasign" { - values = regexp.MustCompile(`var ` + key + `\s*=\s*['"]?(.+?)['"]?;`).FindStringSubmatch(data) - } else { - matches := regexp.MustCompile(`var `+key+`\s*=\s*['"]?(.+?)['"]?;`).FindAllStringSubmatch(data, -1) - if len(matches) == 3 { - values = matches[1] - } else { - if len(matches) > 0 { - values = matches[0] - } - } - } - if len(values) == 0 { + re := regexp.MustCompile(`var\s+` + regexp.QuoteMeta(key) + `\s*=\s*['"]?([^'"]*)['"]?\s*;`) + matches := re.FindAllStringSubmatch(data, -1) + if len(matches) == 0 { return "" } - return values[1] + for i := len(matches) - 1; i >= 0; i-- { + if matches[i][1] != "" { + return matches[i][1] + } + } + return matches[len(matches)-1][1] } var findFunction = regexp.MustCompile(`(?ims)^function[^{]+`) @@ -304,7 +298,7 @@ var findFromReg = regexp.MustCompile(`data : '(.+?)'`) // 查找from字符串 func htmlFormToMap(html string) (map[string]string, error) { forms := findFromReg.FindStringSubmatch(html) if len(forms) != 2 { - return nil, fmt.Errorf("not find file sgin") + return nil, fmt.Errorf("not find file sign") } return formToMap(forms[1]), nil } diff --git a/drivers/lanzou/types.go b/drivers/lanzou/types.go index dbce7887..4e6f481f 100644 --- a/drivers/lanzou/types.go +++ b/drivers/lanzou/types.go @@ -132,7 +132,7 @@ type FileOrFolderByShareUrl struct { //T int `json:"t"` // 文件夹特有 - IsFloder bool `json:"-"` + IsFolder bool `json:"-"` // Url string `json:"-"` @@ -162,7 +162,7 @@ func (f *FileOrFolderByShareUrl) GetSize() int64 { } return *f.size } -func (f *FileOrFolderByShareUrl) IsDir() bool { return f.IsFloder } +func (f *FileOrFolderByShareUrl) IsDir() bool { return f.IsFolder } func (f *FileOrFolderByShareUrl) ModTime() time.Time { if f.time == nil { time := MustParseTime(f.Time) diff --git a/drivers/lanzou/util.go b/drivers/lanzou/util.go index e7015440..c02df953 100644 --- a/drivers/lanzou/util.go +++ b/drivers/lanzou/util.go @@ -90,7 +90,7 @@ func (d *LanZou) _post(url string, callback base.ReqCallback, resp interface{}, if info == "" { info = utils.Json.Get(data, "info").ToString() } - return data, fmt.Errorf(info) + return data, fmt.Errorf("%s", info) } } @@ -157,25 +157,43 @@ func (d *LanZou) request(url string, method string, callback base.ReqCallback, u } func (d *LanZou) Login() ([]*http.Cookie, error) { - resp, err := base.NewRestyClient().SetRedirectPolicy(resty.NoRedirectPolicy()). - R().SetFormData(map[string]string{ - "task": "3", - "uid": d.Account, - "pwd": d.Password, - "setSessionId": "", - "setSig": "", - "setScene": "", - "setTocen": "", - "formhash": "", - }).Post("https://up.woozooo.com/mlogin.php") - if err != nil { - return nil, err + var vs string + for retry := 0; retry < 3; retry++ { + req := base.NewRestyClient().SetRedirectPolicy(resty.NoRedirectPolicy()).R() + + // 如果已计算出 acw_sc__v2，通过 cookie 携带 + if vs != "" { + req.SetHeader("cookie", "acw_sc__v2="+vs) + } + + resp, err := req.SetFormData(map[string]string{ + "task": "3", + "uid": d.Account, + "pwd": d.Password, + "setSessionId": "", + "setSig": "", + "setScene": "", + "setTocen": "", + "formhash": "", + }).Post("https://up.woozooo.com/mlogin.php") + if err != nil { + return nil, err + } + bodyStr := resp.String() + if strings.Contains(bodyStr, "acw_sc__v2") { + vs, err = CalcAcwScV2(bodyStr) + if err != nil { + return nil, err + } + continue + } + if utils.Json.Get(resp.Body(), "zt").ToInt() != 1 { + return nil, fmt.Errorf("login err: %s", resp.Body()) + } + d.Cookie = CookieToString(resp.Cookies()) + return resp.Cookies(), nil } - if utils.Json.Get(resp.Body(), "zt").ToInt() != 1 { - return nil, fmt.Errorf("login err: %s", resp.Body()) - } - d.Cookie = CookieToString(resp.Cookies()) - return resp.Cookies(), nil + return nil, errors.New("acw_sc__v2 validation error") } /* @@ -295,7 +313,37 @@ var findSubFolderReg = regexp.MustCompile(`(?i)(?:folderlink|mbxfolder).+href="/ var findDownPageParamReg = regexp.MustCompile(` 1 { - fileID = fileIDs[1] - } else { + matches := findFileIDReg.FindStringSubmatch(sharePageData) + if len(matches) < 2 { return nil, fmt.Errorf("not find file id") } + ajaxUrl := d.ShareUrl + matches[0][1:len(matches[0])-1] var resp FileShareInfoAndUrlResp[string] - _, err = d.post(d.ShareUrl+"/ajaxm.php?file="+fileID, func(req *resty.Request) { req.SetFormData(param) }, &resp) + _, err = d.post(ajaxUrl, func(req *resty.Request) { req.SetFormData(param) }, &resp) if err != nil { return nil, err } @@ -421,20 +467,23 @@ func (d *LanZou) getFilesByShareUrl(shareID, pwd string, sharePageData string) ( return nil, err } nextPageData := RemoveNotes(string(data)) - param, err = htmlJsonToMap(nextPageData) - if err != nil { - return nil, err - } - fileIDs := findFileIDReg.FindStringSubmatch(nextPageData) - var fileID string - if len(fileIDs) > 1 { - fileID = fileIDs[1] + var resp FileShareInfoAndUrlResp[int] + matches := findFileIDReg.FindStringSubmatch(nextPageData) + if len(matches) >= 2 { + // 旧版结构：相对路径 /ajaxm.php?file=N + param, err = htmlJsonToMap(nextPageData) + if err != nil { + return nil, err + } + ajaxUrl := d.ShareUrl + matches[0][1:len(matches[0])-1] + _, err = d.post(ajaxUrl, func(req *resty.Request) { req.SetFormData(param) }, &resp) + } else if fnUrl, fnForm, ferr := parseFnPage(nextPageData); ferr == nil { + // 2026-10 改版结构：/fn? 内页携带 apifile 绝对地址与签名参数 + _, err = d.post(fnUrl, func(req *resty.Request) { req.SetFormData(fnForm) }, &resp) } else { return nil, fmt.Errorf("not find file id") } - var resp FileShareInfoAndUrlResp[int] - _, err = d.post(d.ShareUrl+"/ajaxm.php?file="+fileID, func(req *resty.Request) { req.SetFormData(param) }, &resp) if err != nil { return nil, err } @@ -572,14 +621,14 @@ func (d *LanZou) getFolderByShareUrl(pwd string, sharePageData string) ([]FileOr files := make([]FileOrFolderByShareUrl, 0) // vip获取文件夹 - floders := findSubFolderReg.FindAllStringSubmatch(sharePageData, -1) - for _, floder := range floders { - if len(floder) == 3 { + folders := findSubFolderReg.FindAllStringSubmatch(sharePageData, -1) + for _, folder := range folders { + if len(folder) == 3 { files = append(files, FileOrFolderByShareUrl{ // Pwd: pwd, // 子文件夹不加密 - ID: floder[1], - NameAll: floder[2], - IsFloder: true, + ID: folder[1], + NameAll: folder[2], + IsFolder: true, }) } } diff --git a/drivers/local/driver.go b/drivers/local/driver.go index ba7c0850..696e2480 100644 --- a/drivers/local/driver.go +++ b/drivers/local/driver.go @@ -14,7 +14,6 @@ import ( "strings" "time" - "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" @@ -153,8 +152,7 @@ func (d *Local) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([ func (d *Local) FileInfoToObj(ctx context.Context, f fs.FileInfo, reqPath string, fullPath string) model.Obj { thumb := "" if d.Thumbnail { - typeName := utils.GetFileType(f.Name()) - if typeName == conf.IMAGE || typeName == conf.VIDEO { + if d.supportsThumbnail(f.Name()) { thumb = common.GetApiUrl(ctx) + stdpath.Join("/d", reqPath, f.Name()) thumb = utils.EncodePath(thumb, true) thumb += "?type=thumb&sign=" + sign.Sign(stdpath.Join(reqPath, f.Name())) @@ -240,7 +238,7 @@ func (d *Local) Link(ctx context.Context, file model.Obj, args model.LinkArgs) ( var thumbPath *string err := d.thumbTokenBucket.Do(ctx, func() error { var err error - buf, thumbPath, err = d.getThumb(file) + buf, thumbPath, err = d.getThumb(ctx, file) return err }) if err != nil { diff --git a/drivers/local/meta.go b/drivers/local/meta.go index a27e9eec..f4536bd2 100644 --- a/drivers/local/meta.go +++ b/drivers/local/meta.go @@ -9,6 +9,7 @@ type Addition struct { driver.RootPath DirectorySize bool `json:"directory_size" default:"false" help:"This might impact host performance"` Thumbnail bool `json:"thumbnail" required:"true" help:"enable thumbnail"` + PDFThumbnail bool `json:"pdf_thumbnail" default:"false" required:"false" help:"Generate PDF first-page thumbnails with Quick Look on macOS"` ThumbCacheFolder string `json:"thumb_cache_folder"` ThumbConcurrency string `json:"thumb_concurrency" default:"16" required:"false" help:"Number of concurrent thumbnail generation goroutines. This controls how many thumbnails can be generated in parallel."` VideoThumbPos string `json:"video_thumb_pos" default:"20%" required:"false" help:"The position of the video thumbnail. If the value is a number (integer ot floating point), it represents the time in seconds. If the value ends with '%', it represents the percentage of the video duration."` diff --git a/drivers/local/pdf_thumb_darwin.go b/drivers/local/pdf_thumb_darwin.go new file mode 100644 index 00000000..6542cb39 --- /dev/null +++ b/drivers/local/pdf_thumb_darwin.go @@ -0,0 +1,45 @@ +//go:build darwin + +package local + +import ( + "bytes" + "context" + "fmt" + "os" + "os/exec" + "path/filepath" + "time" +) + +func pdfThumbnailSupported() bool { + return true +} + +func renderPDFThumbnail(ctx context.Context, fullPath string) (*bytes.Buffer, error) { + tempDir, err := os.MkdirTemp("", "openlist-pdf-thumb-*") + if err != nil { + return nil, err + } + defer os.RemoveAll(tempDir) + + renderCtx, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() + cmd := exec.CommandContext(renderCtx, "/usr/bin/qlmanage", "-t", "-s", "512", "-o", tempDir, fullPath) + if output, err := cmd.CombinedOutput(); err != nil { + if renderCtx.Err() == context.DeadlineExceeded { + return nil, fmt.Errorf("render PDF thumbnail timed out: %w", renderCtx.Err()) + } + if renderCtx.Err() != nil { + return nil, fmt.Errorf("render PDF thumbnail canceled: %w", renderCtx.Err()) + } + return nil, fmt.Errorf("render PDF thumbnail: %w: %s", err, bytes.TrimSpace(output)) + } + + thumbPath := filepath.Join(tempDir, filepath.Base(fullPath)+".png") + data, err := os.ReadFile(thumbPath) + if err != nil { + return nil, fmt.Errorf("read rendered PDF thumbnail: %w", err) + } + return bytes.NewBuffer(data), nil +} diff --git a/drivers/local/pdf_thumb_darwin_test.go b/drivers/local/pdf_thumb_darwin_test.go new file mode 100644 index 00000000..ec00ed63 --- /dev/null +++ b/drivers/local/pdf_thumb_darwin_test.go @@ -0,0 +1,53 @@ +//go:build darwin + +package local + +import ( + "bytes" + "context" + "errors" + "image/png" + "os" + "os/exec" + "path/filepath" + "testing" +) + +func TestRenderPDFThumbnailDarwin(t *testing.T) { + tempDir := t.TempDir() + textPath := filepath.Join(tempDir, "source.txt") + pdfPath := filepath.Join(tempDir, "source 文件.pdf") + if err := os.WriteFile(textPath, []byte("OpenList PDF thumbnail integration test\n"), 0o600); err != nil { + t.Fatal(err) + } + + cmd := exec.Command("/usr/sbin/cupsfilter", textPath) + pdfData, err := cmd.Output() + if err != nil { + t.Fatalf("create fixture PDF: %v", err) + } + if err := os.WriteFile(pdfPath, pdfData, 0o600); err != nil { + t.Fatal(err) + } + + thumb, err := renderPDFThumbnail(context.Background(), pdfPath) + if err != nil { + t.Fatal(err) + } + if !bytes.HasPrefix(thumb.Bytes(), []byte("\x89PNG\r\n\x1a\n")) { + t.Fatal("rendered thumbnail is not PNG") + } + cfg, err := png.DecodeConfig(bytes.NewReader(thumb.Bytes())) + if err != nil { + t.Fatalf("decode thumbnail: %v", err) + } + if cfg.Width <= 0 || cfg.Height <= 0 { + t.Fatalf("invalid thumbnail dimensions: %dx%d", cfg.Width, cfg.Height) + } + + canceledCtx, cancel := context.WithCancel(context.Background()) + cancel() + if _, err := renderPDFThumbnail(canceledCtx, pdfPath); !errors.Is(err, context.Canceled) { + t.Fatalf("renderPDFThumbnail with canceled context returned %v, want context.Canceled", err) + } +} diff --git a/drivers/local/pdf_thumb_test.go b/drivers/local/pdf_thumb_test.go new file mode 100644 index 00000000..614232fb --- /dev/null +++ b/drivers/local/pdf_thumb_test.go @@ -0,0 +1,40 @@ +package local + +import ( + "testing" + + "github.com/OpenListTeam/OpenList/v4/internal/conf" +) + +func TestSupportsThumbnail(t *testing.T) { + oldImages := conf.SlicesMap[conf.ImageTypes] + oldVideos := conf.SlicesMap[conf.VideoTypes] + conf.SlicesMap[conf.ImageTypes] = []string{"jpg"} + conf.SlicesMap[conf.VideoTypes] = []string{"mp4"} + t.Cleanup(func() { + conf.SlicesMap[conf.ImageTypes] = oldImages + conf.SlicesMap[conf.VideoTypes] = oldVideos + }) + + tests := []struct { + name string + fileName string + pdfThumbnail bool + want bool + }{ + {name: "image", fileName: "cover.jpg", want: true}, + {name: "video", fileName: "movie.mp4", want: true}, + {name: "PDF disabled by default", fileName: "document.pdf", want: false}, + {name: "unrelated document", fileName: "document.txt", pdfThumbnail: true, want: false}, + {name: "PDF enabled when renderer is available", fileName: "document.PDF", pdfThumbnail: true, want: pdfThumbnailSupported()}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + d := &Local{Addition: Addition{PDFThumbnail: tt.pdfThumbnail}} + if got := d.supportsThumbnail(tt.fileName); got != tt.want { + t.Fatalf("supportsThumbnail(%q) = %v, want %v", tt.fileName, got, tt.want) + } + }) + } +} diff --git a/drivers/local/pdf_thumb_unsupported.go b/drivers/local/pdf_thumb_unsupported.go new file mode 100644 index 00000000..deb8e99f --- /dev/null +++ b/drivers/local/pdf_thumb_unsupported.go @@ -0,0 +1,17 @@ +//go:build !darwin + +package local + +import ( + "bytes" + "context" + "errors" +) + +func pdfThumbnailSupported() bool { + return false +} + +func renderPDFThumbnail(context.Context, string) (*bytes.Buffer, error) { + return nil, errors.New("PDF thumbnails are not supported on this platform") +} diff --git a/drivers/local/util.go b/drivers/local/util.go index f9f4f4c4..b3ccc1fd 100644 --- a/drivers/local/util.go +++ b/drivers/local/util.go @@ -2,6 +2,7 @@ package local import ( "bytes" + "context" "encoding/json" "errors" "fmt" @@ -127,7 +128,19 @@ func (d *Local) removeThumbCache(fullPath string) { _ = os.Remove(thumbPath) } -func (d *Local) getThumb(file model.Obj) (*bytes.Buffer, *string, error) { +func (d *Local) supportsThumbnail(name string) bool { + typeName := utils.GetFileType(name) + if typeName == conf.IMAGE || typeName == conf.VIDEO { + return true + } + return d.supportsPDFThumbnail(name) +} + +func (d *Local) supportsPDFThumbnail(name string) bool { + return d.PDFThumbnail && pdfThumbnailSupported() && strings.EqualFold(filepath.Ext(name), ".pdf") +} + +func (d *Local) getThumb(ctx context.Context, file model.Obj) (*bytes.Buffer, *string, error) { fullPath := file.GetPath() if d.ThumbCacheFolder != "" { // skip if the file is a thumbnail @@ -140,12 +153,19 @@ func (d *Local) getThumb(file model.Obj) (*bytes.Buffer, *string, error) { } } var srcBuf *bytes.Buffer - if utils.GetFileType(file.GetName()) == conf.VIDEO { + typeName := utils.GetFileType(file.GetName()) + if typeName == conf.VIDEO { videoBuf, err := d.GetSnapshot(fullPath) if err != nil { return nil, nil, err } srcBuf = videoBuf + } else if d.supportsPDFThumbnail(file.GetName()) { + pdfBuf, err := renderPDFThumbnail(ctx, fullPath) + if err != nil { + return nil, nil, err + } + srcBuf = pdfBuf } else { imgData, err := os.ReadFile(fullPath) if err != nil { diff --git a/drivers/onedrive/driver.go b/drivers/onedrive/driver.go index e01c1f7c..1d82b8c2 100644 --- a/drivers/onedrive/driver.go +++ b/drivers/onedrive/driver.go @@ -52,16 +52,20 @@ func (d *Onedrive) InitReference(refStorage driver.Driver) error { } func (d *Onedrive) Drop(ctx context.Context) error { + d.mutex.Lock() + defer d.mutex.Unlock() + // RootFolderPath may change before this driver instance is reinitialized. + d.root = nil d.ref = nil return nil } func (d *Onedrive) GetRoot(ctx context.Context) (model.Obj, error) { + d.mutex.Lock() + defer d.mutex.Unlock() if d.root != nil { return d.root, nil } - d.mutex.Lock() - defer d.mutex.Unlock() root := &Object{ ObjThumb: model.ObjThumb{ Object: model.Object{ diff --git a/drivers/onedrive_sharelink/util.go b/drivers/onedrive_sharelink/util.go index 13785939..a0f7450d 100644 --- a/drivers/onedrive_sharelink/util.go +++ b/drivers/onedrive_sharelink/util.go @@ -2,7 +2,6 @@ package onedrive_sharelink import ( "context" - "crypto/tls" "encoding/json" "fmt" "io" @@ -13,30 +12,28 @@ import ( "time" "github.com/OpenListTeam/OpenList/v4/drivers/base" - "github.com/OpenListTeam/OpenList/v4/internal/conf" + internalNet "github.com/OpenListTeam/OpenList/v4/internal/net" log "github.com/sirupsen/logrus" "golang.org/x/net/html" ) // NewNoRedirectClient creates an HTTP client that doesn't follow redirects func NewNoRedirectCLient() *http.Client { - return &http.Client{ - Timeout: time.Hour * 48, - Transport: &http.Transport{ - Proxy: http.ProxyFromEnvironment, - TLSClientConfig: &tls.Config{InsecureSkipVerify: conf.Conf.TlsInsecureSkipVerify}, - }, - // Prevent following redirects - CheckRedirect: func(req *http.Request, via []*http.Request) error { - return http.ErrUseLastResponse - }, + client := internalNet.NewHttpClient() + client.CheckRedirect = func(req *http.Request, via []*http.Request) error { + return http.ErrUseLastResponse } + return client } // getCookiesWithPassword fetches cookies required for authenticated access using the provided password -func getCookiesWithPassword(link, password string) (string, error) { +func getCookiesWithPassword(ctx context.Context, link, password string) (string, error) { // Send GET request - resp, err := http.Get(link) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, link, nil) + if err != nil { + return "", err + } + resp, err := base.HttpClient.Do(req) if err != nil { return "", err } @@ -95,16 +92,18 @@ func getCookiesWithPassword(link, password string) (string, error) { "__VIEWSTATEENCRYPTED": []string{""}, } - client := &http.Client{ - CheckRedirect: func(req *http.Request, via []*http.Request) error { - return http.ErrUseLastResponse - }, - } + client := NewNoRedirectCLient() // Send the POST request, preventing redirects - resp, err = client.PostForm(newURL, data) + req, err = http.NewRequestWithContext(ctx, http.MethodPost, newURL, strings.NewReader(data.Encode())) if err != nil { return "", err } + req.Header.Set("Content-Type", "application/x-www-form-urlencoded") + resp, err = client.Do(req) + if err != nil { + return "", err + } + defer resp.Body.Close() // Extract the desired cookie value cookie := resp.Cookies() @@ -153,6 +152,7 @@ func (d *OnedriveSharelink) getHeaders(ctx context.Context) (http.Header, error) if err != nil { return nil, err } + defer answerNoRedirect.Body.Close() redirectUrl := answerNoRedirect.Header.Get("Location") log.Debugln("redirectUrl:", redirectUrl) if redirectUrl == "" { @@ -169,7 +169,7 @@ func (d *OnedriveSharelink) getHeaders(ctx context.Context) (http.Header, error) header.Set("authority", u.Host) return header, nil } else { - cookie, err := getCookiesWithPassword(d.ShareLinkURL, d.ShareLinkPassword) + cookie, err := getCookiesWithPassword(ctx, d.ShareLinkURL, d.ShareLinkPassword) if err != nil { return nil, err } @@ -197,6 +197,7 @@ func (d *OnedriveSharelink) getFiles(ctx context.Context, path string) ([]Item, if err != nil { return nil, err } + _ = answerNoRedirect.Body.Close() redirectUrl = answerNoRedirect.Header.Get("Location") } else { header = d.Headers @@ -205,6 +206,7 @@ func (d *OnedriveSharelink) getFiles(ctx context.Context, path string) ([]Item, if err != nil { return nil, err } + _ = answerNoRedirect.Body.Close() redirectUrl = answerNoRedirect.Header.Get("Location") } redirectSplitURL := strings.Split(redirectUrl, "/") @@ -289,9 +291,9 @@ func (d *OnedriveSharelink) getFiles(ctx context.Context, path string) ([]Item, } tempHeader["Content-Type"] = []string{"application/json;odata=verbose"} - client := &http.Client{} + client := base.HttpClient postUrl := strings.Join(redirectSplitURL[:len(redirectSplitURL)-3], "/") + "/_api/v2.1/graphql" - req, err = http.NewRequest(http.MethodPost, postUrl, strings.NewReader(graphqlVar)) + req, err = http.NewRequestWithContext(ctx, http.MethodPost, postUrl, strings.NewReader(graphqlVar)) if err != nil { return nil, err } @@ -326,7 +328,10 @@ func (d *OnedriveSharelink) getFiles(ctx context.Context, path string) ([]Item, log.Debugln("nextHref:", nextHref) graphqlReqNEW := GraphQLNEWRequest{} postUrl = strings.Join(redirectSplitURL[:len(redirectSplitURL)-3], "/") + "/_api/web/GetListUsingPath(DecodedUrl=@a1)/RenderListDataAsStream" + nextHref - req, _ = http.NewRequest(http.MethodPost, postUrl, strings.NewReader(renderListDataAsStreamVar)) + req, err = http.NewRequestWithContext(ctx, http.MethodPost, postUrl, strings.NewReader(renderListDataAsStreamVar)) + if err != nil { + return nil, err + } req.Header = tempHeader resp, err := client.Do(req) diff --git a/drivers/onedrive_sharelink/util_test.go b/drivers/onedrive_sharelink/util_test.go new file mode 100644 index 00000000..e7756844 --- /dev/null +++ b/drivers/onedrive_sharelink/util_test.go @@ -0,0 +1,46 @@ +package onedrive_sharelink + +import ( + "net/http" + "net/http/httptest" + "reflect" + "testing" + + internalNet "github.com/OpenListTeam/OpenList/v4/internal/net" +) + +func TestNoRedirectClientUsesSharedSettings(t *testing.T) { + client := NewNoRedirectCLient() + sharedClient := internalNet.NewHttpClient() + if client.Timeout != sharedClient.Timeout { + t.Fatalf("expected shared timeout %s, got %s", sharedClient.Timeout, client.Timeout) + } + if reflect.TypeOf(client.Transport) != reflect.TypeOf(sharedClient.Transport) { + t.Fatalf("expected shared transport type %T, got %T", sharedClient.Transport, client.Transport) + } +} + +func TestNoRedirectClientStopsRedirects(t *testing.T) { + targetRequested := false + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/target" { + targetRequested = true + w.WriteHeader(http.StatusNoContent) + return + } + http.Redirect(w, r, "/target", http.StatusFound) + })) + defer server.Close() + + resp, err := NewNoRedirectCLient().Get(server.URL) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusFound { + t.Fatalf("expected redirect response, got %d", resp.StatusCode) + } + if targetRequested { + t.Fatal("redirect target was requested") + } +} diff --git a/drivers/pikpak/meta.go b/drivers/pikpak/meta.go index c602bd2a..da3da551 100644 --- a/drivers/pikpak/meta.go +++ b/drivers/pikpak/meta.go @@ -10,10 +10,11 @@ type Addition struct { Username string `json:"username" required:"true"` Password string `json:"password" required:"true"` Platform string `json:"platform" required:"true" default:"web" type:"select" options:"android,web,pc"` - RefreshToken string `json:"refresh_token" required:"true" default:""` + RefreshToken string `json:"refresh_token" required:"false" default:""` CaptchaToken string `json:"captcha_token" default:""` DeviceID string `json:"device_id" required:"false" default:""` DisableMediaLink bool `json:"disable_media_link" default:"true"` + SkipVerification bool `json:"skip_verification" default:"false" help:"ignore the human verification URL returned by the captcha API instead of failing; enabling this may trigger PikPak risk control"` } var config = driver.Config{ diff --git a/drivers/pikpak/util.go b/drivers/pikpak/util.go index 1d091217..2d5b0729 100644 --- a/drivers/pikpak/util.go +++ b/drivers/pikpak/util.go @@ -100,12 +100,13 @@ func (d *PikPak) login() error { return errors.New("username or password is empty") } + // Clear expired access token so captcha requests don't carry a stale bearer + d.AccessToken = "" + url := "https://user.mypikpak.net/v1/auth/signin" - // 使用 用户填写的 CaptchaToken —————— (验证后的captcha_token) - if d.GetCaptchaToken() == "" { - if err := d.RefreshCaptchaTokenInLogin(GetAction(http.MethodPost, url), d.Username); err != nil { - return err - } + // Always refresh captcha token before signin (it may be expired) + if err := d.RefreshCaptchaTokenInLogin(GetAction(http.MethodPost, url), d.Username); err != nil { + return err } var e ErrResp @@ -125,7 +126,12 @@ func (d *PikPak) login() error { data := res.Body() d.RefreshToken = jsoniter.Get(data, "refresh_token").ToString() d.AccessToken = jsoniter.Get(data, "access_token").ToString() + if d.AccessToken == "" || d.RefreshToken == "" { + return errors.New("login failed: server returned empty tokens") + } d.Common.SetUserID(jsoniter.Get(data, "sub").ToString()) + d.Addition.RefreshToken = d.RefreshToken + op.MustSaveDriverStorage(d) return nil } @@ -159,9 +165,14 @@ func (d *PikPak) refreshToken(refreshToken string) error { return errors.New(e.Error()) } data := res.Body() + newAccessToken := jsoniter.Get(data, "access_token").ToString() + newRefreshToken := jsoniter.Get(data, "refresh_token").ToString() + if newAccessToken == "" || newRefreshToken == "" { + return errors.New("refresh failed: server returned empty tokens") + } d.Status = "work" - d.RefreshToken = jsoniter.Get(data, "refresh_token").ToString() - d.AccessToken = jsoniter.Get(data, "access_token").ToString() + d.RefreshToken = newRefreshToken + d.AccessToken = newAccessToken d.Common.SetUserID(jsoniter.Get(data, "sub").ToString()) d.Addition.RefreshToken = d.RefreshToken op.MustSaveDriverStorage(d) @@ -197,12 +208,18 @@ func (d *PikPak) request(url string, method string, callback base.ReqCallback, r case 0: return res.Body(), nil case 4122, 4121, 16: - // access_token 过期 + if strings.Contains(url, "/v1/auth/") || strings.Contains(url, "/v1/shield/captcha/") { + return nil, errors.New(e.Error()) + } + // access_token expired, refresh and retry if err1 := d.refreshToken(d.RefreshToken); err1 != nil { return nil, err1 } return d.request(url, method, callback, resp) - case 9: // 验证码token过期 + case 9: // captcha token expired + if strings.Contains(url, "/v1/shield/captcha/") { + return nil, errors.New(e.Error()) + } if err = d.RefreshCaptchaTokenAtLogin(GetAction(method, url), d.GetUserID()); err != nil { return nil, err } @@ -369,6 +386,9 @@ func (d *PikPak) RefreshCaptchaTokenInLogin(action, username string) error { } else { metas["username"] = username } + metas["client_version"] = d.ClientVersion + metas["package_name"] = d.PackageName + metas["timestamp"], metas["captcha_sign"] = d.Common.GetCaptchaSign() return d.refreshCaptchaToken(action, metas) } @@ -407,7 +427,7 @@ func (d *PikPak) refreshCaptchaToken(action string, metas map[string]string) err return errors.New(e.Error()) } - if resp.Url != "" { + if resp.Url != "" && !d.Addition.SkipVerification { return fmt.Errorf(`need verify: Click Here`, resp.Url) } @@ -503,9 +523,11 @@ func (d *PikPak) UploadByMultipart(ctx context.Context, params *S3Params, fileSi for chunk := range chunksCh { var part oss.UploadPart // 出现错误就继续尝试，共尝试3次 for retry := 0; retry < 3; retry++ { - select { - case <-ctx.Done(): + if utils.IsCanceled(ctx) { + err = ctx.Err() break + } + select { case <-ticker.C: errCh <- errors.Wrap(err, "ossToken 过期") default: diff --git a/drivers/pikpak/util_test.go b/drivers/pikpak/util_test.go new file mode 100644 index 00000000..7c69ea53 --- /dev/null +++ b/drivers/pikpak/util_test.go @@ -0,0 +1,720 @@ +package pikpak + +import ( + "encoding/json" + "fmt" + "io" + "net/http" + "net/http/httptest" + "strings" + "sync" + "testing" + + "github.com/OpenListTeam/OpenList/v4/drivers/base" + "github.com/OpenListTeam/OpenList/v4/internal/conf" + "github.com/OpenListTeam/OpenList/v4/internal/db" + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/glebarez/sqlite" + "github.com/go-resty/resty/v2" + "gorm.io/gorm" +) + +// --- Helper function tests --- + +func TestGetAction(t *testing.T) { + tests := []struct { + method string + url string + want string + }{ + {"GET", "https://api-drive.mypikpak.net/drive/v1/files", "GET:/drive/v1/files"}, + {"POST", "https://user.mypikpak.net/v1/auth/signin", "POST:/v1/auth/signin"}, + {"POST", "https://user.mypikpak.net/v1/shield/captcha/init", "POST:/v1/shield/captcha/init"}, + {"GET", "https://api-drive.mypikpak.net/drive/v1/files?page_token=abc", "GET:/drive/v1/files"}, + {"POST", "https://user.mypikpak.net/v1/auth/token", "POST:/v1/auth/token"}, + } + for _, tt := range tests { + t.Run(tt.method+":"+tt.url, func(t *testing.T) { + got := GetAction(tt.method, tt.url) + if got != tt.want { + t.Errorf("GetAction(%q, %q) = %q, want %q", tt.method, tt.url, got, tt.want) + } + }) + } +} + +func TestGetCaptchaSign(t *testing.T) { + c := &Common{ + ClientID: "YNxT9w7GMdWvEOKa", + ClientVersion: "1.53.2", + PackageName: "com.pikcloud.pikpak", + DeviceID: "test-device-id", + Algorithms: AndroidAlgorithms, + } + + timestamp, sign := c.GetCaptchaSign() + if timestamp == "" { + t.Fatal("timestamp should not be empty") + } + if len(sign) != 34 { + t.Fatalf("sign length should be 34 (\"1.\" + 32 hex), got %d: %q", len(sign), sign) + } + if sign[:2] != "1." { + t.Errorf("sign should start with '1.', got %q", sign[:2]) + } +} + +func TestGenerateDeviceSign(t *testing.T) { + sign := generateDeviceSign("test-device", "com.pikcloud.pikpak") + if len(sign) < 7 { + t.Fatal("device sign too short") + } + if sign[:7] != "div101." { + t.Errorf("device sign should start with 'div101.', got %q", sign[:7]) + } + // Deterministic + if sign != generateDeviceSign("test-device", "com.pikcloud.pikpak") { + t.Error("generateDeviceSign should be deterministic") + } +} + +func TestBuildCustomUserAgent(t *testing.T) { + ua := BuildCustomUserAgent("dev123", AndroidClientID, AndroidPackageName, + AndroidSdkVersion, AndroidClientVersion, AndroidPackageName, "user456") + for _, want := range []string{"ANDROID-", "clientid/", "deviceid/dev123", "usrno/user456"} { + if !strings.Contains(ua, want) { + t.Errorf("user agent should contain %q", want) + } + } +} + +// --- Auth recovery behavior tests --- + +func TestErrRespErrorClassification(t *testing.T) { + tests := []struct { + name string + resp ErrResp + wantError bool + wantCode int64 + }{ + {"success", ErrResp{ErrorCode: 0}, false, 0}, + {"access_token_expired_4122", ErrResp{ErrorCode: 4122, ErrorMsg: "access_token expired"}, true, 4122}, + {"access_token_expired_4121", ErrResp{ErrorCode: 4121, ErrorMsg: "access_token expired"}, true, 4121}, + {"unauthenticated_16", ErrResp{ErrorCode: 16, ErrorMsg: "unauthenticated"}, true, 16}, + {"refresh_token_invalid_4126", ErrResp{ErrorCode: 4126, ErrorMsg: "invalid_grant"}, true, 4126}, + {"captcha_expired_9", ErrResp{ErrorCode: 9, ErrorMsg: "captcha_invalid"}, true, 9}, + {"rate_limit_10", ErrResp{ErrorCode: 10, ErrorDescription: "too frequent"}, true, 10}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + gotError := tt.resp.IsError() + if gotError != tt.wantError { + t.Errorf("IsError() = %v, want %v", gotError, tt.wantError) + } + if tt.resp.ErrorCode != tt.wantCode { + t.Errorf("ErrorCode = %d, want %d", tt.resp.ErrorCode, tt.wantCode) + } + }) + } +} + +// TestGuardClauseOnAuthURLDoesNotRefresh verifies that when the auth endpoint +// itself reports 4122, request() fails fast instead of calling refreshToken() +// (which would recurse). Real behavior, real code path: with the guard +// removed from request(), the token endpoint would be hit a second time. +func TestGuardClauseOnAuthURLDoesNotRefresh(t *testing.T) { + m := newPikpakMock(t) + defer m.close() + restore := installMockClient(m) + defer restore() + + d, _ := newTestDriver(t) + m.tokenStatus = http.StatusBadRequest + m.tokenBody = map[string]any{"error_code": 4122, "error": "access_token_expired"} + + _, err := d.request("https://user.mypikpak.net/v1/auth/token", http.MethodPost, nil, nil) + if err == nil { + t.Fatal("request() to an auth URL must fail on 4122 instead of refreshing") + } + if got := m.count(pathToken); got != 1 { + t.Errorf("guard clause violated: token endpoint hit %d times, want exactly 1 (no refreshToken recursion)", got) + } + if got := m.count(pathSignin); got != 0 { + t.Errorf("no re-login expected, got %d signin calls", got) + } +} + +// --- Integration scaffolding: in-memory DB + mock PikPak endpoints --- + +var ( + setupDBOnce sync.Once + setupDBErr error + rowSeq int64 +) + +// setupTestDB mirrors internal/op/storage_test.go: an in-memory SQLite +// database behind internal/db, so op.MustSaveDriverStorage really persists +// and tests can assert on the saved row instead of on comments. +func setupTestDB(t *testing.T) { + t.Helper() + setupDBOnce.Do(func() { + var gormDB *gorm.DB + gormDB, setupDBErr = gorm.Open(sqlite.Open("file::memory:?cache=shared"), &gorm.Config{}) + if setupDBErr != nil { + return + } + conf.Conf = conf.DefaultConfig("testdata") + db.Init(gormDB) + }) + if setupDBErr != nil { + t.Fatalf("failed to set up test database: %v", setupDBErr) + } +} + +// createStorageRow inserts a fresh storage row and returns it, so that +// MustSaveDriverStorage during a test performs an UPDATE that can be read +// back afterwards. +func createStorageRow(t *testing.T) *model.Storage { + t.Helper() + rowSeq++ + st := &model.Storage{ + Driver: "PikPak", + MountPath: fmt.Sprintf("/pikpak-test-%d", rowSeq), + Addition: `{"username":"tester@example.com","password":"pw"}`, + } + if err := db.CreateStorage(st); err != nil { + t.Fatalf("failed to create storage row: %v", err) + } + return st +} + +func persistedRefreshToken(t *testing.T, id uint) string { + t.Helper() + st, err := db.GetStorageById(id) + if err != nil { + t.Fatalf("failed to read storage back: %v", err) + } + var a Addition + if err := json.Unmarshal([]byte(st.Addition), &a); err != nil { + t.Fatalf("failed to decode persisted addition %q: %v", st.Addition, err) + } + return a.RefreshToken +} + +// mockCall records one request received by the mock server. +type mockCall struct { + headers http.Header + body map[string]any +} + +func (c mockCall) captchaToken() string { + s, _ := c.body["captcha_token"].(string) + return s +} + +// pikpakMock emulates the captcha/auth endpoints used by login() and +// refreshToken(), plus one drive endpoint that serves as the entry point of +// the recovery chain. The drive endpoint fails exactly once (with the code +// configured in driveFirstStatus) and succeeds afterwards, so request() can +// only complete if recovery actually ran. +type pikpakMock struct { + t *testing.T + srv *httptest.Server + mu sync.Mutex + calls map[string][]mockCall + + captchaTokenOut string + captchaURL string + + tokenStatus int + tokenBody map[string]any + + signinStatus int + signinBody map[string]any + + driveFirstStatus int + driveFirstBody map[string]any // body served on the first drive call only + driveBody map[string]any // body served afterwards + driveHits int +} + +func newPikpakMock(t *testing.T) *pikpakMock { + t.Helper() + m := &pikpakMock{ + t: t, + calls: map[string][]mockCall{}, + captchaTokenOut: "cap-fresh", + tokenStatus: http.StatusOK, + tokenBody: map[string]any{"access_token": "at-2", "refresh_token": "rt-2", "sub": "user-1"}, + signinStatus: http.StatusOK, + signinBody: map[string]any{"access_token": "at-new", "refresh_token": "rt-new", "sub": "user-1"}, + driveFirstStatus: http.StatusOK, + driveFirstBody: map[string]any{"files": []any{}, "next_page_token": ""}, + driveBody: map[string]any{"files": []any{}, "next_page_token": ""}, + } + m.srv = httptest.NewServer(http.HandlerFunc(m.serve)) + return m +} + +func (m *pikpakMock) close() { m.srv.Close() } + +func (m *pikpakMock) serve(w http.ResponseWriter, r *http.Request) { + body := map[string]any{} + if raw, err := io.ReadAll(r.Body); err == nil && len(raw) > 0 { + _ = json.Unmarshal(raw, &body) + } + m.mu.Lock() + m.calls[r.URL.Path] = append(m.calls[r.URL.Path], mockCall{headers: r.Header.Clone(), body: body}) + status := http.StatusOK + payload := any(map[string]any{}) + switch { + case strings.HasSuffix(r.URL.Path, "/v1/shield/captcha/init"): + payload = map[string]any{"captcha_token": m.captchaTokenOut, "expires_in": 3600, "url": m.captchaURL} + case strings.HasSuffix(r.URL.Path, "/v1/auth/signin"): + status = m.signinStatus + payload = m.signinBody + case strings.HasSuffix(r.URL.Path, "/v1/auth/token"): + status = m.tokenStatus + payload = m.tokenBody + case strings.HasSuffix(r.URL.Path, "/drive/v1/files"): + m.driveHits++ + if m.driveHits == 1 { + status = m.driveFirstStatus + payload = m.driveFirstBody + } else { + payload = m.driveBody + } + default: + m.mu.Unlock() + m.t.Errorf("unexpected request to %s", r.URL.Path) + w.WriteHeader(http.StatusNotFound) + return + } + m.mu.Unlock() + + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(status) + _ = json.NewEncoder(w).Encode(payload) +} + +func (m *pikpakMock) count(path string) int { + m.mu.Lock() + defer m.mu.Unlock() + return len(m.calls[path]) +} + +func (m *pikpakMock) reset() { + m.mu.Lock() + defer m.mu.Unlock() + m.calls = map[string][]mockCall{} + m.driveHits = 0 +} + +func (m *pikpakMock) last(path string) mockCall { + m.mu.Lock() + defer m.mu.Unlock() + calls := m.calls[path] + if len(calls) == 0 { + m.t.Fatalf("no recorded call for %s", path) + } + return calls[len(calls)-1] +} + +// installMockClient replaces base.RestyClient with a client whose requests to +// the hard-coded PikPak hosts are rewritten onto the mock server, and returns +// a restore function. The rewrite happens in OnBeforeRequest, which resty +// runs before its internal parseRequestURL/createHTTPRequest middlewares. +func installMockClient(m *pikpakMock) func() { + old := base.RestyClient + client := resty.New() + client.OnBeforeRequest(func(_ *resty.Client, req *resty.Request) error { + for _, host := range []string{"https://user.mypikpak.net", "https://api-drive.mypikpak.net"} { + if strings.HasPrefix(req.URL, host) { + req.URL = strings.Replace(req.URL, host, m.srv.URL, 1) + } + } + return nil + }) + base.RestyClient = client + return func() { base.RestyClient = old } +} + +// newTestDriver builds a PikPak with a fully initialized Common (web platform +// constants) and a fresh storage row in the DB, ready for auth-flow tests. +func newTestDriver(t *testing.T) (*PikPak, uint) { + t.Helper() + setupTestDB(t) + st := createStorageRow(t) + d := &PikPak{} + d.SetStorage(*st) + d.Platform = "web" + d.Username = "tester@example.com" + d.Password = "pw" + d.Common = &Common{ + ClientID: WebClientID, + ClientSecret: WebClientSecret, + ClientVersion: WebClientVersion, + PackageName: WebPackageName, + DeviceID: "test-device", + UserAgent: "test-agent", + Algorithms: WebAlgorithms, + } + d.Common.RefreshCTokenCk = func(token string) { + d.Common.CaptchaToken = token + } + return d, st.ID +} + +const ( + pathCaptchaInit = "/v1/shield/captcha/init" + pathSignin = "/v1/auth/signin" + pathToken = "/v1/auth/token" + pathFiles = "/drive/v1/files" +) + +// --- Main auth recovery path --- + +// TestMainRecoveryPath exercises the full chain the PR is about: a drive +// request fails with 4122, refreshToken fails with 4126, login() runs (fresh +// captcha + password signin), the new refresh token is persisted to the DB, +// and request() retries the original call successfully with the new tokens. +func TestMainRecoveryPath(t *testing.T) { + m := newPikpakMock(t) + defer m.close() + restore := installMockClient(m) + defer restore() + + d, id := newTestDriver(t) + d.RefreshToken = "rt-old" + d.AccessToken = "at-stale" + d.SetCaptchaToken("cap-stale") + d.Addition.RefreshToken = "rt-old" + + // refresh attempt fails with "refresh token invalid" + m.tokenStatus = http.StatusBadRequest + m.tokenBody = map[string]any{"error_code": 4126, "error": "invalid_grant"} + // the first drive call reports an expired access token; the retry succeeds + m.driveFirstStatus = http.StatusBadRequest + m.driveFirstBody = map[string]any{"error_code": 4122, "error": "access_token_expired"} + m.driveBody = map[string]any{"files": []any{}, "next_page_token": ""} + + var resp Files + if _, err := d.request("https://api-drive.mypikpak.net/drive/v1/files", http.MethodGet, nil, &resp); err != nil { + t.Fatalf("request() returned error even though recovery should succeed: %v", err) + } + + if got := m.count(pathToken); got != 1 { + t.Errorf("expected exactly 1 refresh request, got %d", got) + } + if got := m.count(pathSignin); got != 1 { + t.Errorf("expected exactly 1 signin (re-login), got %d", got) + } + if got := m.count(pathFiles); got != 2 { + t.Errorf("expected 2 files requests (failed + retried), got %d", got) + } + if got := m.count(pathCaptchaInit); got != 1 { + t.Errorf("expected exactly 1 captcha/init call during re-login, got %d", got) + } + + // The retry must carry the tokens obtained via re-login, not the stale ones. + lastFiles := m.last(pathFiles) + if got := lastFiles.headers.Get("Authorization"); got != "Bearer at-new" { + t.Errorf("retried request Authorization = %q, want %q", got, "Bearer at-new") + } + if got := lastFiles.headers.Get("X-Captcha-Token"); got != "cap-fresh" { + t.Errorf("retried request X-Captcha-Token = %q, want %q", got, "cap-fresh") + } + + // Tokens were rotated in memory... + if d.AccessToken != "at-new" { + t.Errorf("AccessToken = %q, want %q", d.AccessToken, "at-new") + } + if d.RefreshToken != "rt-new" { + t.Errorf("RefreshToken = %q, want %q", d.RefreshToken, "rt-new") + } + // ...and the rotated refresh token was really persisted. + if got := persistedRefreshToken(t, id); got != "rt-new" { + t.Errorf("persisted addition refresh_token = %q, want %q", got, "rt-new") + } +} + +// TestRefreshToken4126WithoutCredentialsDoesNotLogin checks that a 4126 with +// empty username/password yields the "re-provide refresh_token" error instead +// of attempting a password login. +func TestRefreshToken4126WithoutCredentialsDoesNotLogin(t *testing.T) { + m := newPikpakMock(t) + defer m.close() + restore := installMockClient(m) + defer restore() + + d, _ := newTestDriver(t) + d.Username = "" + d.Password = "" + + m.tokenStatus = http.StatusBadRequest + m.tokenBody = map[string]any{"error_code": 4126, "error": "invalid_grant"} + + err := d.refreshToken("rt-old") + if err == nil { + t.Fatal("refreshToken() with invalid refresh token and no credentials must fail") + } + if !strings.Contains(err.Error(), "re-provide") { + t.Errorf("unexpected error text: %v", err) + } + if got := m.count(pathSignin); got != 0 { + t.Errorf("signin must not be attempted without credentials, got %d calls", got) + } +} + +// TestRefreshTokenOtherErrorDoesNotLogin checks that a non-4126 refresh +// failure propagates without triggering a re-login (4126 is the single +// documented trigger). +func TestRefreshTokenOtherErrorDoesNotLogin(t *testing.T) { + m := newPikpakMock(t) + defer m.close() + restore := installMockClient(m) + defer restore() + + d, _ := newTestDriver(t) + m.tokenStatus = http.StatusBadRequest + m.tokenBody = map[string]any{"error_code": 10, "error_description": "too frequent"} + + if err := d.refreshToken("rt-old"); err == nil { + t.Fatal("refreshToken() must propagate a non-4126 error") + } + if got := m.count(pathSignin); got != 0 { + t.Errorf("signin must not be attempted for non-4126 errors, got %d calls", got) + } +} + +// --- Token validation (replaces TestTokenValidationRejectsEmpty) --- + +// TestTokenValidationRejectsEmpty drives login() and refreshToken() against +// 200 responses that carry empty tokens and requires both paths to refuse +// them without persisting anything. +func TestTokenValidationRejectsEmpty(t *testing.T) { + m := newPikpakMock(t) + defer m.close() + restore := installMockClient(m) + defer restore() + + // login(): signin answers 200 but with an empty access_token. + d, id := newTestDriver(t) + m.signinBody = map[string]any{"access_token": "", "refresh_token": "rt-x", "sub": "user-1"} + if err := d.login(); err == nil { + t.Fatal("login() must reject empty access_token") + } + if got := persistedRefreshToken(t, id); got != "" { + t.Errorf("login() must not persist tokens when validation fails, persisted %q", got) + } + + // login(): symmetric case — empty refresh_token but non-empty access_token. + d3, id3 := newTestDriver(t) + m.reset() + m.signinBody = map[string]any{"access_token": "at-x", "refresh_token": "", "sub": "user-1"} + if err := d3.login(); err == nil { + t.Fatal("login() must reject empty refresh_token") + } + if got := persistedRefreshToken(t, id3); got != "" { + t.Errorf("login() must not persist tokens when validation fails, persisted %q", got) + } + + // refreshToken(): 200 but empty refresh_token. + d2, id2 := newTestDriver(t) + m.tokenStatus = http.StatusOK + m.tokenBody = map[string]any{"access_token": "at-x", "refresh_token": "", "sub": "user-1"} + if err := d2.refreshToken("rt-old"); err == nil { + t.Fatal("refreshToken() must reject empty refresh_token") + } + if got := persistedRefreshToken(t, id2); got != "" { + t.Errorf("refreshToken() must not persist tokens when validation fails, persisted %q", got) + } +} + +// --- Captcha refresh (replaces TestCaptchaAlwaysRefreshedBeforeLogin) --- + +// TestCaptchaAlwaysRefreshedBeforeLogin proves login() fetches a fresh captcha +// even when a (possibly expired) token is already present, and that signin is +// performed with the fresh token rather than the stale one. +func TestCaptchaAlwaysRefreshedBeforeLogin(t *testing.T) { + m := newPikpakMock(t) + defer m.close() + restore := installMockClient(m) + defer restore() + + d, _ := newTestDriver(t) + d.SetCaptchaToken("cap-stale") // non-empty and (conceptually) expired + + if err := d.login(); err != nil { + t.Fatalf("login() failed: %v", err) + } + + if got := m.count(pathCaptchaInit); got != 1 { + t.Fatalf("expected exactly 1 captcha/init call despite a non-empty stale token, got %d", got) + } + if got := m.last(pathSignin).captchaToken(); got != "cap-fresh" { + t.Errorf("signin used captcha_token %q, want the fresh %q", got, "cap-fresh") + } + if got := d.GetCaptchaToken(); got != "cap-fresh" { + t.Errorf("driver CaptchaToken = %q after login, want %q", got, "cap-fresh") + } +} + +// --- Stale bearer cleared before login --- + +// TestLoginClearsStaleAccessToken checks that the captcha/init and signin +// requests issued by login() do not carry the expired bearer token. +func TestLoginClearsStaleAccessToken(t *testing.T) { + m := newPikpakMock(t) + defer m.close() + restore := installMockClient(m) + defer restore() + + d, _ := newTestDriver(t) + d.AccessToken = "at-stale" + + if err := d.login(); err != nil { + t.Fatalf("login() failed: %v", err) + } + + for _, path := range []string{pathCaptchaInit, pathSignin} { + if got := m.last(path).headers.Get("Authorization"); got != "" { + t.Errorf("%s request carried Authorization %q, want it cleared before login", path, got) + } + } +} + +// --- Captcha meta completeness --- + +// TestCaptchaMetaCompleteness asserts captcha/init on the login path carries +// the same meta fields RefreshCaptchaTokenAtLogin sends on main. +func TestCaptchaMetaCompleteness(t *testing.T) { + m := newPikpakMock(t) + defer m.close() + restore := installMockClient(m) + defer restore() + + d, _ := newTestDriver(t) + if err := d.login(); err != nil { + t.Fatalf("login() failed: %v", err) + } + + meta, _ := m.last(pathCaptchaInit).body["meta"].(map[string]any) + for _, key := range []string{"email", "client_version", "package_name", "timestamp", "captcha_sign"} { + if v, ok := meta[key]; !ok || v == "" { + t.Errorf("captcha meta missing or empty %q (got %#v)", key, meta) + } + } +} + +// --- refreshToken success path (highest-frequency production path) --- + +// TestRefreshTokenSuccessRotatesAndPersists covers 4122 -> refreshToken() +// succeeding: rotated tokens land in memory, the retry carries the new bearer, +// and the new refresh token is persisted to the DB. +func TestRefreshTokenSuccessRotatesAndPersists(t *testing.T) { + m := newPikpakMock(t) + defer m.close() + restore := installMockClient(m) + defer restore() + + d, id := newTestDriver(t) + d.RefreshToken = "rt-old" + d.AccessToken = "at-stale" + d.Addition.RefreshToken = "rt-old" + + m.tokenStatus = http.StatusOK + m.tokenBody = map[string]any{"access_token": "at-2", "refresh_token": "rt-2", "sub": "user-1"} + m.driveFirstStatus = http.StatusBadRequest + m.driveFirstBody = map[string]any{"error_code": 4122, "error": "access_token_expired"} + m.driveBody = map[string]any{"files": []any{}, "next_page_token": ""} + + var resp Files + if _, err := d.request("https://api-drive.mypikpak.net/drive/v1/files", http.MethodGet, nil, &resp); err != nil { + t.Fatalf("request() failed even though refresh should succeed: %v", err) + } + + if got := m.count(pathSignin); got != 0 { + t.Errorf("a successful refresh must not fall through to password login, got %d signin calls", got) + } + if d.AccessToken != "at-2" { + t.Errorf("AccessToken = %q, want %q", d.AccessToken, "at-2") + } + if d.RefreshToken != "rt-2" { + t.Errorf("RefreshToken = %q, want %q", d.RefreshToken, "rt-2") + } + if got := m.last(pathFiles).headers.Get("Authorization"); got != "Bearer at-2" { + t.Errorf("retried request Authorization = %q, want %q", got, "Bearer at-2") + } + if got := persistedRefreshToken(t, id); got != "rt-2" { + t.Errorf("persisted addition refresh_token = %q, want %q", got, "rt-2") + } +} + +// --- captcha expired (case 9) --- + +// TestCaptchaExpiredRefreshesAndRetries covers request() case 9: a captcha +// error on a drive call triggers a captcha refresh and one retry. +func TestCaptchaExpiredRefreshesAndRetries(t *testing.T) { + m := newPikpakMock(t) + defer m.close() + restore := installMockClient(m) + defer restore() + + d, _ := newTestDriver(t) + d.AccessToken = "at-ok" + d.RefreshToken = "rt-ok" + d.SetCaptchaToken("cap-stale") + + m.driveFirstStatus = http.StatusBadRequest + m.driveFirstBody = map[string]any{"error_code": 9, "error": "captcha_invalid"} + m.driveBody = map[string]any{"files": []any{}, "next_page_token": ""} + + var resp Files + if _, err := d.request("https://api-drive.mypikpak.net/drive/v1/files", http.MethodGet, nil, &resp); err != nil { + t.Fatalf("request() failed even though captcha refresh should recover: %v", err) + } + + if got := m.count(pathCaptchaInit); got == 0 { + t.Fatal("expected a captcha refresh after error code 9") + } + if got := m.count(pathFiles); got != 2 { + t.Errorf("expected 2 files requests (failed + retried), got %d", got) + } + if got := m.count(pathSignin); got != 0 { + t.Errorf("captcha recovery must not re-login, got %d signin calls", got) + } + if got := m.last(pathFiles).headers.Get("X-Captcha-Token"); got != "cap-fresh" { + t.Errorf("retried request X-Captcha-Token = %q, want %q", got, "cap-fresh") + } +} + +// --- SkipVerification (added by this PR) --- + +// TestSkipVerificationControlsVerificationURL covers the new config option: +// a captcha/init response carrying a human-verification url is fatal by +// default and ignored only when skip_verification is enabled. +func TestSkipVerificationControlsVerificationURL(t *testing.T) { + m := newPikpakMock(t) + defer m.close() + restore := installMockClient(m) + defer restore() + + m.captchaURL = "https://user.mypikpak.net/forbidden/test" + + d, _ := newTestDriver(t) + if err := d.login(); err == nil { + t.Fatal("login() must fail on a verification url by default") + } else if !strings.Contains(err.Error(), "need verify") { + t.Errorf("unexpected error: %v", err) + } + + d2, _ := newTestDriver(t) + d2.SkipVerification = true + if err := d2.login(); err != nil { + t.Fatalf("login() with skip_verification must ignore the url, got: %v", err) + } + if d2.AccessToken != "at-new" { + t.Errorf("AccessToken = %q after skipped verification, want %q", d2.AccessToken, "at-new") + } +} diff --git a/drivers/s3/driver.go b/drivers/s3/driver.go index a037912f..253d3c0a 100644 --- a/drivers/s3/driver.go +++ b/drivers/s3/driver.go @@ -9,6 +9,7 @@ import ( "strings" "time" + "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" @@ -178,7 +179,7 @@ func (d *S3) Move(ctx context.Context, srcObj, dstDir model.Obj) error { } func (d *S3) Rename(ctx context.Context, srcObj model.Obj, newName string) error { - err := d.copy(ctx, srcObj.GetPath(), stdpath.Join(stdpath.Dir(srcObj.GetPath()), newName), srcObj.IsDir()) + err := d.copy(ctx, srcObj.GetPath(), stdpath.Join(stdpath.Dir(srcObj.GetPath()), newName), srcObj.GetSize(), srcObj.IsDir()) if err != nil { return err } @@ -186,14 +187,14 @@ func (d *S3) Rename(ctx context.Context, srcObj model.Obj, newName string) error } func (d *S3) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { - return d.copy(ctx, srcObj.GetPath(), stdpath.Join(dstDir.GetPath(), srcObj.GetName()), srcObj.IsDir()) + return d.copy(ctx, srcObj.GetPath(), stdpath.Join(dstDir.GetPath(), srcObj.GetName()), srcObj.GetSize(), srcObj.IsDir()) } func (d *S3) Remove(ctx context.Context, obj model.Obj) error { if obj.IsDir() { return d.removeDir(ctx, obj.GetPath()) } - return d.removeFile(obj.GetPath()) + return d.removeFile(ctx, obj.GetPath()) } func (d *S3) Put(ctx context.Context, dstDir model.Obj, s model.FileStreamer, up driver.UpdateProgress) error { @@ -229,9 +230,14 @@ func (d *S3) GetDirectUploadInfo(ctx context.Context, _ string, dstDir model.Obj return nil, errs.NotImplement } path := getKey(stdpath.Join(dstDir.GetPath(), fileName), false) + contentType, _ := ctx.Value(conf.DirectUploadContentTypeKey).(string) + if contentType == "" { + contentType = "application/octet-stream" + } req, _ := d.directUploadClient.PutObjectRequest(&s3.PutObjectInput{ - Bucket: &d.Bucket, - Key: &path, + Bucket: &d.Bucket, + Key: &path, + ContentType: &contentType, }) if req == nil { return nil, fmt.Errorf("failed to create PutObject request") @@ -243,13 +249,22 @@ func (d *S3) GetDirectUploadInfo(ctx context.Context, _ string, dstDir model.Obj return &model.HttpDirectUploadInfo{ UploadURL: link, Method: "PUT", + Headers: map[string]string{"Content-Type": contentType}, }, nil } // implements driver.Getter interface func (d *S3) Get(ctx context.Context, path string) (model.Obj, error) { // try to get object as a file using HeadObject - path = stdpath.Join(d.GetRootPath(), path) + rootPath := d.GetRootPath() + // Avoid double-prepending root path when path already contains it. + // This happens when obj.GetPath() from a previous Get call is passed back + // to op.List → op.Get → d.Get. + if rootPath != "/" && utils.IsSubPath(rootPath, path) { + // path already includes the root prefix + } else { + path = stdpath.Join(rootPath, path) + } key := getKey(path, false) headInput := &s3.HeadObjectInput{ Bucket: &d.Bucket, diff --git a/drivers/s3/meta.go b/drivers/s3/meta.go index b2439c68..4243f12d 100644 --- a/drivers/s3/meta.go +++ b/drivers/s3/meta.go @@ -23,6 +23,7 @@ type Addition struct { AddFilenameToDisposition bool `json:"add_filename_to_disposition" help:"Add filename to Content-Disposition header."` EnableDirectUpload bool `json:"enable_direct_upload" default:"false"` DirectUploadHost string `json:"direct_upload_host" required:"false"` + UserAgent string `json:"user_agent" required:"false" default:"" help:"Custom User-Agent for S3 requests."` } func init() { diff --git a/drivers/s3/util.go b/drivers/s3/util.go index c8438078..cba8698f 100644 --- a/drivers/s3/util.go +++ b/drivers/s3/util.go @@ -3,6 +3,7 @@ package s3 import ( "context" "errors" + "fmt" "net/http" "net/url" "path" @@ -19,6 +20,13 @@ import ( log "github.com/sirupsen/logrus" ) +const ( + maxCopyObjectSize int64 = 5 * 1000 * 1000 * 1000 + defaultCopyPartSize int64 = 100 * 1024 * 1024 + maxCopyPartSize int64 = 5 * 1024 * 1024 * 1024 + maxCopyParts int64 = 10000 +) + // do others that not defined in Driver interface func (d *S3) initSession() error { @@ -49,6 +57,11 @@ const ( func (d *S3) getClient(clientType int) *s3.S3 { client := s3.New(d.Session) + if d.UserAgent != "" { + client.Handlers.Build.PushBack(func(r *request.Request) { + r.HTTPRequest.Header.Set("User-Agent", d.UserAgent) + }) + } if clientType == ClientTypeLink && d.CustomHost != "" { client.Handlers.Build.PushBack(func(r *request.Request) { if r.HTTPRequest.Method != http.MethodGet { @@ -207,26 +220,129 @@ func (d *S3) listV2(dirPath string, args model.ListArgs) ([]model.Obj, error) { return files, nil } -func (d *S3) copy(ctx context.Context, src string, dst string, isDir bool) error { +func (d *S3) copy(ctx context.Context, src string, dst string, size int64, isDir bool) error { if isDir { return d.copyDir(ctx, src, dst) } - return d.copyFile(ctx, src, dst) + return d.copyFile(ctx, src, dst, size) } -func (d *S3) copyFile(ctx context.Context, src string, dst string) error { +func (d *S3) copyFile(ctx context.Context, src string, dst string, size int64) error { srcKey := getKey(src, false) dstKey := getKey(dst, false) encodedKey := strings.ReplaceAll(url.PathEscape(d.Bucket+"/"+srcKey), "+", "%2B") + if size > maxCopyObjectSize { + return d.copyFileMultipart(ctx, srcKey, dstKey, encodedKey, size) + } input := &s3.CopyObjectInput{ Bucket: &d.Bucket, CopySource: aws.String(encodedKey), Key: &dstKey, } - _, err := d.client.CopyObject(input) + _, err := d.client.CopyObjectWithContext(ctx, input) return err } +func (d *S3) copyFileMultipart(ctx context.Context, srcKey, dstKey, encodedKey string, size int64) (err error) { + head, err := d.client.HeadObjectWithContext(ctx, &s3.HeadObjectInput{ + Bucket: &d.Bucket, + Key: &srcKey, + }) + if err != nil { + return err + } + if head.ContentLength != nil { + size = *head.ContentLength + } + partSize, err := getCopyPartSize(size) + if err != nil { + return err + } + createInput := &s3.CreateMultipartUploadInput{ + Bucket: &d.Bucket, + Key: &dstKey, + CacheControl: head.CacheControl, + ContentDisposition: head.ContentDisposition, + ContentEncoding: head.ContentEncoding, + ContentLanguage: head.ContentLanguage, + ContentType: head.ContentType, + Metadata: head.Metadata, + WebsiteRedirectLocation: head.WebsiteRedirectLocation, + } + if head.Expires != nil { + if expires, parseErr := http.ParseTime(*head.Expires); parseErr == nil { + createInput.Expires = &expires + } + } + created, err := d.client.CreateMultipartUploadWithContext(ctx, createInput) + if err != nil { + return err + } + uploadID := aws.StringValue(created.UploadId) + if uploadID == "" { + return errors.New("create multipart upload returned an empty upload ID") + } + completed := false + defer func() { + if completed { + return + } + _, abortErr := d.client.AbortMultipartUploadWithContext(context.WithoutCancel(ctx), &s3.AbortMultipartUploadInput{ + Bucket: &d.Bucket, + Key: &dstKey, + UploadId: &uploadID, + }) + if abortErr != nil { + err = errors.Join(err, fmt.Errorf("failed to abort multipart copy: %w", abortErr)) + } + }() + + parts := make([]*s3.CompletedPart, 0, (size+partSize-1)/partSize) + for start, partNumber := int64(0), int64(1); start < size; start, partNumber = start+partSize, partNumber+1 { + end := min(start+partSize, size) - 1 + copied, copyErr := d.client.UploadPartCopyWithContext(ctx, &s3.UploadPartCopyInput{ + Bucket: &d.Bucket, + CopySource: &encodedKey, + CopySourceRange: aws.String(fmt.Sprintf("bytes=%d-%d", start, end)), + Key: &dstKey, + PartNumber: &partNumber, + UploadId: &uploadID, + }) + if copyErr != nil { + return copyErr + } + if copied.CopyPartResult == nil || aws.StringValue(copied.CopyPartResult.ETag) == "" { + return fmt.Errorf("multipart copy part %d returned an empty ETag", partNumber) + } + parts = append(parts, &s3.CompletedPart{ + ETag: copied.CopyPartResult.ETag, + PartNumber: &partNumber, + }) + } + + _, err = d.client.CompleteMultipartUploadWithContext(ctx, &s3.CompleteMultipartUploadInput{ + Bucket: &d.Bucket, + Key: &dstKey, + UploadId: &uploadID, + MultipartUpload: &s3.CompletedMultipartUpload{ + Parts: parts, + }, + }) + if err != nil { + return err + } + completed = true + return nil +} + +func getCopyPartSize(size int64) (int64, error) { + partSize := max(defaultCopyPartSize, (size-1)/maxCopyParts+1) + if partSize > maxCopyPartSize { + return 0, fmt.Errorf("object size %d exceeds multipart copy limit", size) + } + return partSize, nil +} + func (d *S3) copyDir(ctx context.Context, src string, dst string) error { objs, err := op.List(ctx, d, src, model.ListArgs{S3ShowPlaceholder: true}) if err != nil { @@ -238,7 +354,7 @@ func (d *S3) copyDir(ctx context.Context, src string, dst string) error { if obj.IsDir() { err = d.copyDir(ctx, cSrc, cDst) } else { - err = d.copyFile(ctx, cSrc, cDst) + err = d.copyFile(ctx, cSrc, cDst, obj.GetSize()) } if err != nil { return err @@ -257,23 +373,23 @@ func (d *S3) removeDir(ctx context.Context, src string) error { if obj.IsDir() { err = d.removeDir(ctx, cSrc) } else { - err = d.removeFile(cSrc) + err = d.removeFile(ctx, cSrc) } if err != nil { return err } } - _ = d.removeFile(path.Join(src, getPlaceholderName(d.Placeholder))) - _ = d.removeFile(path.Join(src, d.Placeholder)) + _ = d.removeFile(ctx, path.Join(src, getPlaceholderName(d.Placeholder))) + _ = d.removeFile(ctx, path.Join(src, d.Placeholder)) return nil } -func (d *S3) removeFile(src string) error { +func (d *S3) removeFile(ctx context.Context, src string) error { key := getKey(src, false) input := &s3.DeleteObjectInput{ Bucket: &d.Bucket, Key: &key, } - _, err := d.client.DeleteObject(input) + _, err := d.client.DeleteObjectWithContext(ctx, input) return err } diff --git a/drivers/s3/util_test.go b/drivers/s3/util_test.go new file mode 100644 index 00000000..6c718a2f --- /dev/null +++ b/drivers/s3/util_test.go @@ -0,0 +1,215 @@ +package s3 + +import ( + "context" + "fmt" + "io" + "net/http" + "net/http/httptest" + "strconv" + "strings" + "testing" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/credentials" + "github.com/aws/aws-sdk-go/aws/session" + awss3 "github.com/aws/aws-sdk-go/service/s3" +) + +func TestCopyFileUsesCopyObjectAtLimit(t *testing.T) { + copyRequests := 0 + d := newTestS3Driver(t, func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPut || r.URL.Query().Get("uploadId") != "" { + t.Errorf("unexpected request: %s %s", r.Method, r.URL.String()) + w.WriteHeader(http.StatusBadRequest) + return + } + copyRequests++ + writeTestXML(t, w, `"copy"`) + }) + + if err := d.copyFile(context.Background(), "source+file", "destination", maxCopyObjectSize); err != nil { + t.Fatalf("copyFile: %v", err) + } + if copyRequests != 1 { + t.Fatalf("copy requests = %d, want 1", copyRequests) + } +} + +func TestCopyFileUsesMultipartCopyAboveLimit(t *testing.T) { + size := maxCopyObjectSize + 1 + wantParts := int((size + defaultCopyPartSize - 1) / defaultCopyPartSize) + ranges := make(map[int]string, wantParts) + completed := false + aborted := false + + d := newTestS3Driver(t, func(w http.ResponseWriter, r *http.Request) { + switch { + case r.Method == http.MethodHead: + w.Header().Set("Content-Length", strconv.FormatInt(size, 10)) + w.Header().Set("Content-Type", "application/octet-stream") + w.Header().Set("Cache-Control", "max-age=60") + w.Header().Set("Content-Disposition", "attachment") + w.Header().Set("Expires", "Wed, 21 Oct 2015 07:28:00 GMT") + w.Header().Set("X-Amz-Meta-Source", "preserved") + w.Header().Set("X-Amz-Website-Redirect-Location", "/redirect") + w.WriteHeader(http.StatusOK) + case r.Method == http.MethodPost && r.URL.Query().Has("uploads"): + if got := r.Header.Get("Cache-Control"); got != "max-age=60" { + t.Errorf("Cache-Control = %q, want %q", got, "max-age=60") + } + if got := r.Header.Get("Content-Disposition"); got != "attachment" { + t.Errorf("Content-Disposition = %q, want %q", got, "attachment") + } + if got := r.Header.Get("Content-Type"); got != "application/octet-stream" { + t.Errorf("Content-Type = %q, want %q", got, "application/octet-stream") + } + if got := r.Header.Get("Expires"); got != "Wed, 21 Oct 2015 07:28:00 GMT" { + t.Errorf("Expires = %q, want an unchanged HTTP date", got) + } + if got := r.Header.Get("X-Amz-Meta-Source"); got != "preserved" { + t.Errorf("metadata = %q, want %q", got, "preserved") + } + if got := r.Header.Get("X-Amz-Website-Redirect-Location"); got != "/redirect" { + t.Errorf("website redirect = %q, want %q", got, "/redirect") + } + writeTestXML(t, w, `upload-id`) + case r.Method == http.MethodPut && r.URL.Query().Get("uploadId") == "upload-id": + partNumber, err := strconv.Atoi(r.URL.Query().Get("partNumber")) + if err != nil { + t.Errorf("invalid part number: %v", err) + w.WriteHeader(http.StatusBadRequest) + return + } + if got := r.Header.Get("X-Amz-Copy-Source"); !strings.Contains(got, "source%2Bfile") { + t.Errorf("copy source = %q, want encoded source key", got) + } + ranges[partNumber] = r.Header.Get("X-Amz-Copy-Source-Range") + writeTestXML(t, w, fmt.Sprintf(`"part-%d"`, partNumber)) + case r.Method == http.MethodPost && r.URL.Query().Get("uploadId") == "upload-id": + body, err := io.ReadAll(r.Body) + if err != nil { + t.Errorf("read complete body: %v", err) + } + if got := strings.Count(string(body), ""); got != wantParts { + t.Errorf("completed parts = %d, want %d", got, wantParts) + } + completed = true + writeTestXML(t, w, `"complete"`) + case r.Method == http.MethodDelete && r.URL.Query().Get("uploadId") == "upload-id": + aborted = true + w.WriteHeader(http.StatusNoContent) + default: + t.Errorf("unexpected request: %s %s", r.Method, r.URL.String()) + w.WriteHeader(http.StatusBadRequest) + } + }) + + if err := d.copyFile(context.Background(), "source+file", "destination", size); err != nil { + t.Fatalf("copyFile: %v", err) + } + if !completed { + t.Fatal("multipart upload was not completed") + } + if aborted { + t.Fatal("successful multipart upload was aborted") + } + if len(ranges) != wantParts { + t.Fatalf("copied parts = %d, want %d", len(ranges), wantParts) + } + if got := ranges[1]; got != fmt.Sprintf("bytes=0-%d", defaultCopyPartSize-1) { + t.Errorf("first range = %q", got) + } + lastStart := int64(wantParts-1) * defaultCopyPartSize + if got := ranges[wantParts]; got != fmt.Sprintf("bytes=%d-%d", lastStart, size-1) { + t.Errorf("last range = %q", got) + } +} + +func TestCopyFileMultipartAbortsOnPartFailure(t *testing.T) { + size := maxCopyObjectSize + 1 + aborted := false + completed := false + + d := newTestS3Driver(t, func(w http.ResponseWriter, r *http.Request) { + switch { + case r.Method == http.MethodHead: + w.Header().Set("Content-Length", strconv.FormatInt(size, 10)) + w.WriteHeader(http.StatusOK) + case r.Method == http.MethodPost && r.URL.Query().Has("uploads"): + writeTestXML(t, w, `upload-id`) + case r.Method == http.MethodPut && r.URL.Query().Get("uploadId") == "upload-id": + w.WriteHeader(http.StatusInternalServerError) + writeTestXML(t, w, `InternalErrorcopy failed`) + case r.Method == http.MethodDelete && r.URL.Query().Get("uploadId") == "upload-id": + aborted = true + w.WriteHeader(http.StatusNoContent) + case r.Method == http.MethodPost && r.URL.Query().Get("uploadId") == "upload-id": + completed = true + w.WriteHeader(http.StatusOK) + default: + t.Errorf("unexpected request: %s %s", r.Method, r.URL.String()) + w.WriteHeader(http.StatusBadRequest) + } + }) + + if err := d.copyFile(context.Background(), "source", "destination", size); err == nil { + t.Fatal("copyFile returned nil error") + } + if !aborted { + t.Fatal("failed multipart upload was not aborted") + } + if completed { + t.Fatal("failed multipart upload was completed") + } +} + +func TestGetCopyPartSize(t *testing.T) { + partSize, err := getCopyPartSize(defaultCopyPartSize * maxCopyParts) + if err != nil { + t.Fatalf("getCopyPartSize: %v", err) + } + if partSize != defaultCopyPartSize { + t.Fatalf("part size = %d, want %d", partSize, defaultCopyPartSize) + } + + partSize, err = getCopyPartSize(defaultCopyPartSize*maxCopyParts + 1) + if err != nil { + t.Fatalf("getCopyPartSize: %v", err) + } + if partSize != defaultCopyPartSize+1 { + t.Fatalf("grown part size = %d, want %d", partSize, defaultCopyPartSize+1) + } + + if _, err := getCopyPartSize(maxCopyPartSize*maxCopyParts + 1); err == nil { + t.Fatal("getCopyPartSize returned nil error for an oversized object") + } +} + +func newTestS3Driver(t *testing.T, handler http.HandlerFunc) *S3 { + t.Helper() + server := httptest.NewServer(handler) + t.Cleanup(server.Close) + sess, err := session.NewSession(&aws.Config{ + Credentials: credentials.NewStaticCredentials("access-key", "secret-key", ""), + Endpoint: aws.String(server.URL), + Region: aws.String("us-east-1"), + S3ForcePathStyle: aws.Bool(true), + MaxRetries: aws.Int(0), + }) + if err != nil { + t.Fatalf("create AWS session: %v", err) + } + return &S3{ + Addition: Addition{Bucket: "bucket"}, + client: awss3.New(sess), + } +} + +func writeTestXML(t *testing.T, w http.ResponseWriter, body string) { + t.Helper() + w.Header().Set("Content-Type", "application/xml") + if _, err := io.WriteString(w, body); err != nil { + t.Errorf("write response: %v", err) + } +} diff --git a/drivers/strm/driver.go b/drivers/strm/driver.go index 1189fd9a..3d243c15 100644 --- a/drivers/strm/driver.go +++ b/drivers/strm/driver.go @@ -27,6 +27,7 @@ type Strm struct { supportSuffix map[string]struct{} downloadSuffix map[string]struct{} + minSizeBytes int64 } func (d *Strm) Config() driver.Config { @@ -120,6 +121,7 @@ func (d *Strm) Init(ctx context.Context) error { if len(d.SaveLocalMode) == 0 { d.SaveLocalMode = SaveLocalInsertMode } + d.minSizeBytes = d.MinFileSize * 1024 * 1024 return nil } diff --git a/drivers/strm/hook.go b/drivers/strm/hook.go index 78ec1e7e..24b31ee3 100644 --- a/drivers/strm/hook.go +++ b/drivers/strm/hook.go @@ -8,6 +8,7 @@ import ( "io" "os" stdpath "path" + "path/filepath" "strings" "github.com/OpenListTeam/OpenList/v4/internal/model" @@ -26,6 +27,10 @@ func UpdateLocalStrm(ctx context.Context, path string, objs []model.Obj) { updateLocal := func(driver *Strm, basePath string, objs []model.Obj) { relParent := strings.TrimPrefix(basePath, utils.GetActualMountPath(driver.MountPath)) localParentPath := stdpath.Join(driver.SaveStrmLocalPath, relParent) + if err := createLocalDirectory(localParentPath); err != nil { + log.Warnf("failed to create local strm directory %s: %v", localParentPath, err) + return + } for _, obj := range objs { localPath := stdpath.Join(localParentPath, obj.GetName()) generateStrm(ctx, driver, obj, localPath) @@ -92,55 +97,68 @@ func RemoveStrm(dstPath string, d *Strm) { } func generateStrm(ctx context.Context, driver *Strm, obj model.Obj, localPath string) { - if !obj.IsDir() { - if utils.Exists(localPath) && driver.SaveLocalMode == SaveLocalInsertMode { - return - } - link, err := driver.Link(ctx, obj, model.LinkArgs{}) - if err != nil { - log.Warnf("failed to generate strm of obj %s: failed to link: %v", localPath, err) - return - } - defer link.Close() - size := link.ContentLength - if size <= 0 { - size = obj.GetSize() - } - rrf, err := stream.GetRangeReaderFromLink(size, link) - if err != nil { - log.Warnf("failed to generate strm of obj %s: failed to get range reader: %v", localPath, err) - return - } - rc, err := rrf.RangeRead(ctx, http_range.Range{Length: -1}) - if err != nil { - log.Warnf("failed to generate strm of obj %s: failed to read range: %v", localPath, err) - return - } - defer rc.Close() - same, err := isSameContent(localPath, size, rc) - if err != nil { - log.Warnf("failed to compare content of obj %s: %v", localPath, err) - return - } - if same { - return - } - rc, err = rrf.RangeRead(ctx, http_range.Range{Length: -1}) - if err != nil { - log.Warnf("failed to generate strm of obj %s: failed to reread range: %v", localPath, err) - return - } - defer rc.Close() - file, err := utils.CreateNestedFile(localPath) - if err != nil { - log.Warnf("failed to generate strm of obj %s: failed to create local file: %v", localPath, err) - return - } - defer file.Close() - if _, err := utils.CopyWithBuffer(file, rc); err != nil { - log.Warnf("failed to generate strm of obj %s: copy failed: %v", localPath, err) + if obj.IsDir() { + if err := createLocalDirectory(localPath); err != nil { + log.Warnf("failed to create local strm directory %s: %v", localPath, err) } + return } + + if utils.Exists(localPath) && driver.SaveLocalMode == SaveLocalInsertMode { + return + } + link, err := driver.Link(ctx, obj, model.LinkArgs{}) + if err != nil { + log.Warnf("failed to generate strm of obj %s: failed to link: %v", localPath, err) + return + } + defer link.Close() + size := link.ContentLength + if size <= 0 { + size = obj.GetSize() + } + rrf, err := stream.GetRangeReaderFromLink(size, link) + if err != nil { + log.Warnf("failed to generate strm of obj %s: failed to get range reader: %v", localPath, err) + return + } + rc, err := rrf.RangeRead(ctx, http_range.Range{Length: -1}) + if err != nil { + log.Warnf("failed to generate strm of obj %s: failed to read range: %v", localPath, err) + return + } + defer rc.Close() + same, err := isSameContent(localPath, size, rc) + if err != nil { + log.Warnf("failed to compare content of obj %s: %v", localPath, err) + return + } + if same { + return + } + rc, err = rrf.RangeRead(ctx, http_range.Range{Length: -1}) + if err != nil { + log.Warnf("failed to generate strm of obj %s: failed to reread range: %v", localPath, err) + return + } + defer rc.Close() + if err := createLocalDirectory(filepath.Dir(localPath)); err != nil { + log.Warnf("failed to generate strm of obj %s: failed to create parent directory: %v", localPath, err) + return + } + file, err := os.Create(localPath) + if err != nil { + log.Warnf("failed to generate strm of obj %s: failed to create local file: %v", localPath, err) + return + } + defer file.Close() + if _, err := utils.CopyWithBuffer(file, rc); err != nil { + log.Warnf("failed to generate strm of obj %s: copy failed: %v", localPath, err) + } +} + +func createLocalDirectory(path string) error { + return os.MkdirAll(path, 0o777) } func isSameContent(localPath string, size int64, rc io.Reader) (bool, error) { diff --git a/drivers/strm/hook_test.go b/drivers/strm/hook_test.go new file mode 100644 index 00000000..eccdf209 --- /dev/null +++ b/drivers/strm/hook_test.go @@ -0,0 +1,43 @@ +package strm + +import ( + "os" + "path/filepath" + "runtime" + "testing" +) + +func TestCreateLocalDirectory(t *testing.T) { + root := t.TempDir() + nested := filepath.Join(root, "library", "movie") + if err := createLocalDirectory(nested); err != nil { + t.Fatalf("createLocalDirectory() error = %v", err) + } + + if _, err := os.Stat(nested); err != nil { + t.Fatalf("stat created directory: %v", err) + } +} + +func TestCreateLocalDirectoryPreservesExistingPermissions(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("directory permission bits are not supported on Windows") + } + + root := t.TempDir() + nested := filepath.Join(root, "library", "movie") + if err := os.MkdirAll(nested, 0o700); err != nil { + t.Fatalf("create existing directory: %v", err) + } + if err := createLocalDirectory(nested); err != nil { + t.Fatalf("createLocalDirectory() error = %v", err) + } + + info, err := os.Stat(nested) + if err != nil { + t.Fatalf("stat existing directory: %v", err) + } + if got := info.Mode().Perm(); got != 0o700 { + t.Errorf("existing directory permissions = %#o, want %#o", got, 0o700) + } +} diff --git a/drivers/strm/hook_umask_unix_test.go b/drivers/strm/hook_umask_unix_test.go new file mode 100644 index 00000000..b693f9f1 --- /dev/null +++ b/drivers/strm/hook_umask_unix_test.go @@ -0,0 +1,30 @@ +//go:build aix || darwin || dragonfly || freebsd || linux || netbsd || openbsd || solaris + +package strm + +import ( + "os" + "path/filepath" + "syscall" + "testing" +) + +func TestCreateLocalDirectoryRespectsUmask(t *testing.T) { + root := t.TempDir() + nested := filepath.Join(root, "library", "movie") + + originalUmask := syscall.Umask(0o027) + defer syscall.Umask(originalUmask) + + if err := createLocalDirectory(nested); err != nil { + t.Fatalf("createLocalDirectory() error = %v", err) + } + + info, err := os.Stat(nested) + if err != nil { + t.Fatalf("stat created directory: %v", err) + } + if got := info.Mode().Perm(); got != 0o750 { + t.Errorf("created directory permissions = %#o, want %#o", got, 0o750) + } +} diff --git a/drivers/strm/meta.go b/drivers/strm/meta.go index 843ff7c6..2e2e8a08 100644 --- a/drivers/strm/meta.go +++ b/drivers/strm/meta.go @@ -17,6 +17,7 @@ type Addition struct { PathPrefix string `json:"PathPrefix" type:"text" required:"false" default:"/d" help:"Path prefix"` DownloadFileTypes string `json:"downloadFileTypes" type:"text" default:"ass,srt,vtt,sub,strm" required:"false" help:"Files need to download with strm (usally subtitles)"` FilterFileTypes string `json:"filterFileTypes" type:"text" default:"mp4,mkv,flv,avi,wmv,ts,rmvb,webm,mp3,flac,aac,wav,ogg,m4a,wma,alac" required:"false" help:"Supports suffix name of strm file"` + MinFileSize int64 `json:"minFileSize" type:"number" required:"false" default:"0" help:"Filter out files smaller than this size (unit: MB, 0 to disable)"` EncodePath bool `json:"encodePath" default:"true" required:"true" help:"encode the path in the strm file"` WithoutUrl bool `json:"withoutUrl" default:"false" help:"strm file content without URL prefix"` WithSign bool `json:"withSign" default:"false"` diff --git a/drivers/strm/util.go b/drivers/strm/util.go index b9a40da5..7dfc7dbe 100644 --- a/drivers/strm/util.go +++ b/drivers/strm/util.go @@ -73,6 +73,9 @@ func (d *Strm) convert2strmObjs(ctx context.Context, reqPath string, objs []mode if _, ok := d.downloadSuffix[ext]; ok { size = obj.GetSize() } else if _, ok := d.supportSuffix[ext]; ok { + if d.minSizeBytes > 0 && obj.GetSize() < d.minSizeBytes { + continue + } id = "strm" name = strings.TrimSuffix(name, sourceExt) + "strm" size = int64(len(d.getLink(ctx, path))) diff --git a/drivers/teldrive/driver.go b/drivers/teldrive/driver.go index ea0093b7..849d9170 100644 --- a/drivers/teldrive/driver.go +++ b/drivers/teldrive/driver.go @@ -55,10 +55,18 @@ func (d *Teldrive) Drop(ctx context.Context) error { } func (d *Teldrive) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { + dirPath := dir.GetPath() + if dirPath == "" { + dirPath = d.GetRootPath() + } + if dirPath == "" { + dirPath = "/" + } + var firstResp ListResp err := d.request(http.MethodGet, "/api/files", func(req *resty.Request) { req.SetQueryParams(map[string]string{ - "path": dir.GetPath(), + "path": dirPath, "limit": "500", "page": "1", }) @@ -87,7 +95,7 @@ func (d *Teldrive) List(ctx context.Context, dir model.Obj, args model.ListArgs) var resp ListResp err := d.request(http.MethodGet, "/api/files", func(req *resty.Request) { req.SetQueryParams(map[string]string{ - "path": dir.GetPath(), + "path": dirPath, "limit": "500", "page": strconv.Itoa(page), }) @@ -114,7 +122,7 @@ func (d *Teldrive) List(ctx context.Context, dir model.Obj, args model.ListArgs) return utils.SliceConvert(allItems, func(src Object) (model.Obj, error) { return &model.Object{ - Path: path.Join(dir.GetPath(), src.Name), + Path: path.Join(dirPath, src.Name), ID: src.ID, Name: src.Name, Size: func() int64 { diff --git a/drivers/teldrive/driver_test.go b/drivers/teldrive/driver_test.go index a5e4de9d..3d073c69 100644 --- a/drivers/teldrive/driver_test.go +++ b/drivers/teldrive/driver_test.go @@ -4,9 +4,11 @@ import ( "context" "net/http" "net/http/httptest" + "sync" "testing" "github.com/OpenListTeam/OpenList/v4/drivers/base" + "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/go-resty/resty/v2" ) @@ -36,3 +38,44 @@ func TestListEmptyDir(t *testing.T) { t.Fatalf("expected no entries for an empty dir, got %d", len(objs)) } } + +func TestListRootUsesConfiguredRootPath(t *testing.T) { + var ( + mu sync.Mutex + paths []string + ) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + mu.Lock() + paths = append(paths, r.URL.Query().Get("path")) + mu.Unlock() + + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"items":[{"id":"child","name":"child","type":"folder"}],"meta":{"count":1,"totalPages":1,"currentPage":1}}`)) + })) + defer srv.Close() + + oldClient := base.RestyClient + base.RestyClient = resty.New() + defer func() { base.RestyClient = oldClient }() + + d := &Teldrive{ + Addition: Addition{ + RootPath: driver.RootPath{RootFolderPath: "/configured-root"}, + }, + } + d.Address = srv.URL + + objs, err := d.List(context.Background(), &model.Object{}, model.ListArgs{}) + if err != nil { + t.Fatalf("List returned error: %v", err) + } + + mu.Lock() + defer mu.Unlock() + if len(paths) != 1 || paths[0] != "/configured-root" { + t.Fatalf("expected request path %q, got %q", "/configured-root", paths) + } + if len(objs) != 1 || objs[0].GetPath() != "/configured-root/child" { + t.Fatalf("expected child path %q, got %#v", "/configured-root/child", objs) + } +} diff --git a/drivers/wps/driver.go b/drivers/wps/driver.go index 810048a7..8d76e106 100644 --- a/drivers/wps/driver.go +++ b/drivers/wps/driver.go @@ -46,9 +46,6 @@ func (d *Wps) Init(ctx context.Context) error { if !resp.IsSuccess() { return fmt.Errorf("failed to check login status, status code: %d, body: %s", resp.StatusCode(), resp.String()) } - if d.login.CompanyID == 0 { - return fmt.Errorf("wps company id is empty, please check business account login") - } return nil } diff --git a/go.mod b/go.mod index 5487a17f..b9ea73ca 100644 --- a/go.mod +++ b/go.mod @@ -1,49 +1,50 @@ module github.com/OpenListTeam/OpenList/v4 -go 1.24.0 +go 1.26.0 -toolchain go1.24.13 +toolchain go1.27.1 require ( - github.com/Azure/azure-sdk-for-go/sdk/azcore v1.18.1 - github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.2 + github.com/Azure/azure-sdk-for-go/sdk/azcore v1.23.1 + github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.8.0 github.com/KarpelesLab/reflink v1.0.2 github.com/KirCute/zip v1.0.1 github.com/OpenListTeam/go-cache v0.1.0 + github.com/OpenListTeam/gofakes3 v0.8.2-0.20260911142347-cd3c030a83b4 github.com/OpenListTeam/sftpd-openlist v1.0.1 github.com/OpenListTeam/tache v0.2.2 github.com/OpenListTeam/times v0.1.0 github.com/OpenListTeam/wopan-sdk-go v0.1.5 - github.com/ProtonMail/go-crypto v1.3.0 - github.com/ProtonMail/gopenpgp/v2 v2.9.0 - github.com/SheltonZhu/115driver v1.3.3 + github.com/ProtonMail/go-crypto v1.4.1 + github.com/ProtonMail/gopenpgp/v2 v2.10.0 + github.com/SheltonZhu/115driver v1.3.5 github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible github.com/antchfx/htmlquery v1.3.6 - github.com/antchfx/xpath v1.3.6 + github.com/antchfx/xpath v1.3.8 github.com/avast/retry-go v3.0.0+incompatible github.com/aws/aws-sdk-go v1.55.8 - github.com/blevesearch/bleve/v2 v2.5.2 - github.com/bmatcuk/doublestar/v4 v4.9.1 + github.com/blevesearch/bleve/v2 v2.6.1 + github.com/bmatcuk/doublestar/v4 v4.10.0 github.com/caarlos0/env/v9 v9.0.0 - github.com/charmbracelet/bubbles v0.21.0 - github.com/charmbracelet/bubbletea v1.3.6 + github.com/charmbracelet/bubbles v0.21.1 + github.com/charmbracelet/bubbletea v1.3.10 github.com/charmbracelet/lipgloss v1.1.0 github.com/city404/v6-public-rpc-proto/go v0.0.0-20240817070657-90f8e24b653e - github.com/cloudsoda/go-smb2 v0.0.0-20250228001242-d4c70e6251cc - github.com/coreos/go-oidc v2.3.0+incompatible - github.com/deckarep/golang-set/v2 v2.8.0 + github.com/cloudsoda/go-smb2 v0.0.0-20260803221621-0b399b9d036c + github.com/coreos/go-oidc v2.5.0+incompatible + github.com/deckarep/golang-set/v2 v2.9.0 github.com/dhowden/tag v0.0.0-20240417053706-3d75831295e8 github.com/disintegration/imaging v1.6.2 - github.com/dlclark/regexp2 v1.11.5 + github.com/dlclark/regexp2 v1.12.0 github.com/dustinxie/ecc v0.0.0-20210511000915-959544187564 github.com/fclairamb/ftpserverlib v0.26.1-0.20250709223522-4a925d79caf6 github.com/foxxorcat/mopan-sdk-go v0.1.6 github.com/foxxorcat/weiyun-sdk-go v0.1.4 - github.com/gin-contrib/cors v1.7.6 - github.com/gin-gonic/gin v1.10.1 + github.com/gin-contrib/cors v1.7.7 + github.com/gin-gonic/gin v1.12.0 github.com/glebarez/sqlite v1.11.0 - github.com/go-resty/resty/v2 v2.16.5 - github.com/go-webauthn/webauthn v0.13.4 + github.com/go-resty/resty/v2 v2.17.2 + github.com/go-webauthn/webauthn v0.18.0 github.com/golang-jwt/jwt/v4 v4.5.2 github.com/google/uuid v1.6.0 github.com/gorilla/websocket v1.5.3 @@ -51,67 +52,74 @@ require ( github.com/hekmon/transmissionrpc/v3 v3.0.0 github.com/henrybear327/go-proton-api v1.0.0 github.com/ipfs/go-ipfs-api v0.7.0 - github.com/itsHenry35/gofakes3 v0.0.8 - github.com/jlaffaye/ftp v0.2.1-0.20240918233326-1b970516f5d3 + github.com/jlaffaye/ftp v0.2.4 github.com/json-iterator/go v1.1.12 github.com/kdomanski/iso9660 v0.4.0 - github.com/maruel/natural v1.1.1 + github.com/maruel/natural v1.3.0 github.com/meilisearch/meilisearch-go v0.32.0 - github.com/mholt/archives v0.1.3 + github.com/mholt/archives v0.1.5 github.com/natefinch/lumberjack v2.0.0+incompatible - github.com/ncw/swift/v2 v2.0.4 + github.com/ncw/swift/v2 v2.0.5 github.com/pkg/errors v0.9.1 - github.com/pkg/sftp v1.13.9 + github.com/pkg/sftp v1.13.11 github.com/pquerna/otp v1.5.0 - github.com/quic-go/quic-go v0.54.1 - github.com/rclone/rclone v1.70.3 + github.com/quic-go/quic-go v0.61.0 + github.com/rclone/rclone v1.75.1 github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd - github.com/shirou/gopsutil/v4 v4.25.5 - github.com/sirupsen/logrus v1.9.3 + github.com/shirou/gopsutil/v4 v4.26.8 + github.com/sirupsen/logrus v1.10.2 github.com/spf13/afero v1.15.0 github.com/spf13/cobra v1.10.2 - github.com/stretchr/testify v1.11.1 - github.com/t3rm1n4l/go-mega v0.0.0-20241213151442-a19cff0ec7b5 + github.com/stretchr/testify v1.12.1 + github.com/t3rm1n4l/go-mega v0.0.0-20260717075258-c6acd6a5bd04 github.com/tchap/go-patricia/v2 v2.3.3 github.com/u2takey/ffmpeg-go v0.5.0 github.com/upyun/go-sdk/v3 v3.0.4 - github.com/winfsp/cgofuse v1.6.0 github.com/zzzhr1990/go-common-entity v0.0.0-20250202070650-1a200048f0d3 - golang.org/x/crypto v0.46.0 - golang.org/x/image v0.29.0 - golang.org/x/net v0.48.0 - golang.org/x/oauth2 v0.34.0 - golang.org/x/time v0.14.0 + golang.org/x/crypto v0.56.0 + golang.org/x/image v0.45.0 + golang.org/x/net v0.58.0 + golang.org/x/oauth2 v0.36.0 + golang.org/x/time v0.15.0 google.golang.org/appengine v1.6.8 gopkg.in/ldap.v3 v3.1.0 - gorm.io/driver/mysql v1.5.7 - gorm.io/driver/postgres v1.5.9 + gorm.io/driver/mysql v1.6.0 + gorm.io/driver/postgres v1.6.2 gorm.io/driver/sqlite v1.6.0 - gorm.io/gorm v1.30.0 + gorm.io/gorm v1.31.2 ) require ( - github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.1 // indirect + filippo.io/edwards25519 v1.1.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/internal v1.12.0 // indirect github.com/BurntSushi/toml v1.6.0 // indirect github.com/ProtonMail/bcrypt v0.0.0-20211005172633-e235017c1baf // indirect github.com/ProtonMail/gluon v0.17.1-0.20230724134000-308be39be96e // indirect github.com/ProtonMail/go-mime v0.0.0-20230322103455-7d82a3887f2f // indirect github.com/ProtonMail/go-srp v0.0.7 // indirect - github.com/PuerkitoBio/goquery v1.10.3 // indirect - github.com/RoaringBitmap/roaring/v2 v2.4.5 // indirect - github.com/andybalholm/cascadia v1.3.3 // indirect + github.com/PuerkitoBio/goquery v1.12.0 // indirect + github.com/RoaringBitmap/roaring/v2 v2.14.5 // indirect + github.com/adrg/xdg v0.5.3 // indirect + github.com/andybalholm/cascadia v1.3.4 // indirect + github.com/blevesearch/zapx/v17 v17.2.3 // indirect github.com/bradenaw/juniper v0.15.3 // indirect - github.com/charmbracelet/colorprofile v0.2.3-0.20250311203215-f60798e515dc // indirect - github.com/charmbracelet/x/cellbuf v0.0.13-0.20250311204145-2c3ea96c31dd // indirect + github.com/bytedance/gopkg v0.1.3 // indirect + github.com/charmbracelet/colorprofile v0.4.1 // indirect + github.com/charmbracelet/x/cellbuf v0.0.15 // indirect + github.com/clipperhouse/displaywidth v0.9.0 // indirect + github.com/clipperhouse/stringish v0.1.1 // indirect + github.com/clipperhouse/uax29/v2 v2.7.0 // indirect github.com/cloudsoda/sddl v0.0.0-20250224235906-926454e91efc // indirect - github.com/coreos/go-systemd/v22 v22.5.0 // indirect - github.com/cronokirby/saferith v0.33.0 // indirect + github.com/coreos/go-systemd/v22 v22.6.0 // indirect + github.com/cronokirby/saferith v0.33.1-0.20250226174546-1f11f94ce488 // indirect github.com/dustin/go-humanize v1.0.1 // indirect - github.com/ebitengine/purego v0.8.4 // indirect + github.com/ebitengine/purego v0.10.2 // indirect github.com/emersion/go-message v0.18.2 // indirect - github.com/emersion/go-vcard v0.0.0-20241024213814-c9703dde27ff // indirect + github.com/emersion/go-vcard v0.0.0-20260618161152-d854b7e0e2d3 // indirect github.com/geoffgarside/ber v1.2.0 // indirect github.com/glebarez/go-sqlite v1.21.2 // indirect + github.com/go-viper/mapstructure/v2 v2.5.0 // indirect + github.com/goccy/go-yaml v1.19.2 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/hashicorp/go-uuid v1.0.3 // indirect github.com/jcmturner/aescts/v2 v2.0.0 // indirect @@ -120,19 +128,24 @@ require ( github.com/jcmturner/goidentity/v6 v6.0.1 // indirect github.com/jcmturner/gokrb5/v8 v8.4.4 // indirect github.com/jcmturner/rpc/v2 v2.0.3 // indirect - github.com/lanrat/extsort v1.0.2 // indirect - github.com/mattn/go-sqlite3 v1.14.22 // indirect + github.com/lanrat/extsort v1.4.2 // indirect github.com/mikelolasagasti/xz v1.0.1 // indirect - github.com/minio/minlz v1.0.0 // indirect + github.com/minio/minlz v1.2.0 // indirect github.com/minio/xxml v0.0.3 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect - github.com/quic-go/qpack v0.5.1 // indirect - github.com/relvacode/iso8601 v1.6.0 // indirect + github.com/philhofer/fwd v1.2.0 // indirect + github.com/quic-go/qpack v0.6.0 // indirect + github.com/relvacode/iso8601 v1.7.0 // indirect github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect + github.com/stangelandcl/ppmd v0.1.1 // indirect + github.com/tinylib/msgp v1.6.4 // indirect github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect - go.uber.org/mock v0.5.0 // indirect - golang.org/x/exp v0.0.0-20250606033433-dcc06ee1d476 // indirect - golang.org/x/mod v0.30.0 // indirect + github.com/zeebo/blake3 v0.2.4 // indirect + github.com/zeebo/xxh3 v1.1.0 // indirect + go.mongodb.org/mongo-driver v1.17.9 // indirect + go.mongodb.org/mongo-driver/v2 v2.5.0 // indirect + go.yaml.in/yaml/v3 v3.0.5 // indirect + golang.org/x/exp v0.0.0-20260709172345-9ea1abe57597 // indirect gopkg.in/go-jose/go-jose.v2 v2.6.3 // indirect modernc.org/libc v1.22.5 // indirect modernc.org/mathutil v1.5.0 // indirect @@ -141,19 +154,19 @@ require ( ) require ( - github.com/OpenListTeam/115-sdk-go v0.2.4 - github.com/STARRY-S/zip v0.2.1 // indirect + github.com/OpenListTeam/115-sdk-go v0.2.6 + github.com/STARRY-S/zip v0.2.3 // indirect github.com/aymerick/douceur v0.2.0 // indirect - github.com/blevesearch/go-faiss v1.0.25 // indirect - github.com/blevesearch/zapx/v16 v16.2.4 // indirect + github.com/blevesearch/go-faiss v1.1.5 // indirect + github.com/blevesearch/zapx/v16 v16.3.4 // indirect github.com/bodgit/plumbing v1.3.0 // indirect - github.com/bodgit/sevenzip v1.6.1 + github.com/bodgit/sevenzip v1.6.5 github.com/bodgit/windows v1.0.1 // indirect - github.com/bytedance/sonic/loader v0.2.4 // indirect - github.com/charmbracelet/x/ansi v0.9.3 // indirect - github.com/charmbracelet/x/term v0.2.1 // indirect - github.com/cloudflare/circl v1.6.1 // indirect - github.com/cloudwego/base64x v0.1.5 // indirect + github.com/bytedance/sonic/loader v0.5.0 // indirect + github.com/charmbracelet/x/ansi v0.11.5 // indirect + github.com/charmbracelet/x/term v0.2.2 // indirect + github.com/cloudflare/circl v1.6.4 // indirect + github.com/cloudwego/base64x v0.1.6 // indirect github.com/dsnet/compress v0.0.2-0.20230904184137-39efe44ab707 // indirect github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f // indirect github.com/fclairamb/go-log v0.6.0 // indirect @@ -162,14 +175,14 @@ require ( github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect github.com/hekmon/cunits/v2 v2.1.0 // indirect github.com/ipfs/boxo v0.12.0 // indirect - github.com/jackc/puddle/v2 v2.2.1 // indirect + github.com/jackc/puddle/v2 v2.2.2 // indirect github.com/klauspost/pgzip v1.2.6 // indirect github.com/matoous/go-nanoid/v2 v2.1.0 // indirect github.com/microcosm-cc/bluemonday v1.0.27 - github.com/nwaples/rardecode/v2 v2.1.1 - github.com/sorairolake/lzip-go v0.3.5 // indirect + github.com/nwaples/rardecode/v2 v2.4.1 + github.com/sorairolake/lzip-go v0.3.8 // indirect github.com/taruti/bytepool v0.0.0-20160310082835-5e3a9ea56543 // indirect - github.com/ulikunitz/xz v0.5.12 // indirect + github.com/ulikunitz/xz v0.5.15 // indirect go4.org v0.0.0-20260112195520-a5071408f32f resty.dev/v3 v3.0.0-beta.2 // indirect ) @@ -180,84 +193,81 @@ require ( github.com/abbot/go-http-auth v0.4.0 // indirect github.com/aead/ecdh v0.2.0 // indirect github.com/andreburgaud/crypt2go v1.8.0 // indirect - github.com/andybalholm/brotli v1.1.2-0.20250424173009-453214e765f3 // indirect + github.com/andybalholm/brotli v1.2.2 // indirect github.com/axgle/mahonia v0.0.0-20180208002826-3358181d7394 github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect github.com/benbjohnson/clock v1.3.0 // indirect github.com/beorn7/perks v1.0.1 // indirect - github.com/bits-and-blooms/bitset v1.22.0 // indirect + github.com/bits-and-blooms/bitset v1.24.4 // indirect github.com/blang/semver/v4 v4.0.0 // indirect - github.com/blevesearch/bleve_index_api v1.2.8 // indirect - github.com/blevesearch/geo v0.2.3 // indirect + github.com/blevesearch/bleve_index_api v1.4.1 // indirect + github.com/blevesearch/geo v0.2.6 // indirect github.com/blevesearch/go-porterstemmer v1.0.3 // indirect github.com/blevesearch/gtreap v0.1.1 // indirect - github.com/blevesearch/mmap-go v1.0.4 // indirect - github.com/blevesearch/scorch_segment_api/v2 v2.3.10 // indirect + github.com/blevesearch/mmap-go v1.2.0 // indirect + github.com/blevesearch/scorch_segment_api/v2 v2.4.10 // indirect github.com/blevesearch/segment v0.9.1 // indirect github.com/blevesearch/snowballstem v0.9.0 // indirect github.com/blevesearch/upsidedown_store_api v1.0.2 // indirect - github.com/blevesearch/vellum v1.1.0 // indirect - github.com/blevesearch/zapx/v11 v11.4.2 // indirect - github.com/blevesearch/zapx/v12 v12.4.2 // indirect - github.com/blevesearch/zapx/v13 v13.4.2 // indirect - github.com/blevesearch/zapx/v14 v14.4.2 // indirect - github.com/blevesearch/zapx/v15 v15.4.2 // indirect - github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc // indirect - github.com/bytedance/sonic v1.13.3 // indirect + github.com/blevesearch/vellum v1.2.0 // indirect + github.com/blevesearch/zapx/v11 v11.4.3 // indirect + github.com/blevesearch/zapx/v12 v12.4.3 // indirect + github.com/blevesearch/zapx/v13 v13.4.3 // indirect + github.com/blevesearch/zapx/v14 v14.4.3 // indirect + github.com/blevesearch/zapx/v15 v15.4.3 // indirect + github.com/boombuler/barcode v1.1.0 // indirect + github.com/bytedance/sonic v1.15.0 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/coreos/go-semver v0.3.1 // indirect github.com/crackcomm/go-gitignore v0.0.0-20170627025303-887ab5e44cc3 // indirect - github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0 // indirect - github.com/fxamacker/cbor/v2 v2.9.0 // indirect - github.com/gabriel-vasile/mimetype v1.4.9 // indirect + github.com/fxamacker/cbor/v2 v2.9.3 // indirect + github.com/gabriel-vasile/mimetype v1.4.13 // indirect github.com/gin-contrib/sse v1.1.0 // indirect - github.com/go-chi/chi/v5 v5.2.2 // indirect + github.com/go-chi/chi/v5 v5.3.1 // indirect github.com/go-ole/go-ole v1.3.0 // indirect github.com/go-playground/locales v0.14.1 // indirect github.com/go-playground/universal-translator v0.18.1 // indirect - github.com/go-playground/validator/v10 v10.26.0 // indirect - github.com/go-sql-driver/mysql v1.7.0 // indirect - github.com/go-webauthn/x v0.1.23 // indirect - github.com/goccy/go-json v0.10.5 // indirect - github.com/golang-jwt/jwt/v5 v5.2.3 // indirect + github.com/go-playground/validator/v10 v10.30.3 // indirect + github.com/go-sql-driver/mysql v1.8.1 // indirect + github.com/go-webauthn/x v0.3.0 // indirect + github.com/goccy/go-json v0.10.6 // indirect + github.com/golang-jwt/jwt/v5 v5.3.1 // indirect github.com/golang/protobuf v1.5.4 // indirect - github.com/golang/snappy v0.0.4 // indirect - github.com/google/go-tpm v0.9.5 // indirect - github.com/hashicorp/errwrap v1.1.0 // indirect - github.com/hashicorp/go-multierror v1.1.1 // indirect + github.com/golang/snappy v1.0.0 // indirect + github.com/google/go-tpm v0.9.8 // indirect github.com/hashicorp/go-version v1.6.0 // indirect github.com/henrybear327/Proton-API-Bridge v1.0.0 github.com/inconshreveable/mousetrap v1.1.0 // indirect - github.com/ipfs/go-cid v0.5.0 + github.com/ipfs/go-cid v0.6.2 github.com/jackc/pgpassfile v1.0.0 // indirect - github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect - github.com/jackc/pgx/v5 v5.5.5 // indirect + github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect + github.com/jackc/pgx/v5 v5.10.0 // indirect github.com/jinzhu/inflection v1.0.0 // indirect github.com/jinzhu/now v1.1.5 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/jzelinskie/whirlpool v0.0.0-20201016144138-0675e54bb004 // indirect - github.com/klauspost/compress v1.18.0 // indirect - github.com/klauspost/cpuid/v2 v2.2.10 // indirect + github.com/klauspost/compress v1.19.0 // indirect + github.com/klauspost/cpuid/v2 v2.4.0 // indirect github.com/kr/fs v0.1.0 // indirect github.com/leodido/go-urn v1.4.0 // indirect github.com/libp2p/go-buffer-pool v0.1.0 // indirect github.com/libp2p/go-flow-metrics v0.1.0 // indirect github.com/libp2p/go-libp2p v0.27.8 // indirect - github.com/lucasb-eyer/go-colorful v1.2.0 // indirect - github.com/lufia/plan9stats v0.0.0-20250317134145-8bc96cf8fc35 // indirect - github.com/mailru/easyjson v0.9.0 // indirect - github.com/mattn/go-colorable v0.1.14 // indirect - github.com/mattn/go-isatty v0.0.20 // indirect + github.com/lucasb-eyer/go-colorful v1.4.0 // indirect + github.com/lufia/plan9stats v0.0.0-20260627054121-477a66015f15 // indirect + github.com/mailru/easyjson v0.9.2 // indirect + github.com/mattn/go-colorable v0.1.15 // indirect + github.com/mattn/go-isatty v0.0.23 // indirect github.com/mattn/go-localereader v0.0.1 // indirect - github.com/mattn/go-runewidth v0.0.16 // indirect + github.com/mattn/go-runewidth v0.0.24 // indirect + github.com/mattn/go-sqlite3 v1.14.22 // indirect github.com/minio/sha256-simd v1.0.1 // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect - github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect - github.com/mr-tron/base58 v1.2.0 // indirect + github.com/mr-tron/base58 v1.3.0 // indirect github.com/mschoch/smat v0.2.0 // indirect github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6 // indirect github.com/muesli/cancelreader v0.2.2 // indirect @@ -265,46 +275,44 @@ require ( github.com/multiformats/go-base32 v0.1.0 // indirect github.com/multiformats/go-base36 v0.2.0 // indirect github.com/multiformats/go-multiaddr v0.9.0 // indirect - github.com/multiformats/go-multibase v0.2.0 // indirect + github.com/multiformats/go-multibase v0.3.0 // indirect github.com/multiformats/go-multicodec v0.9.0 // indirect github.com/multiformats/go-multihash v0.2.3 // indirect github.com/multiformats/go-multistream v0.4.1 // indirect - github.com/multiformats/go-varint v0.0.7 // indirect + github.com/multiformats/go-varint v0.1.0 // indirect github.com/pelletier/go-toml/v2 v2.2.4 // indirect - github.com/pierrec/lz4/v4 v4.1.22 // indirect - github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect + github.com/pierrec/lz4/v4 v4.1.27 // indirect github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect github.com/pquerna/cachecontrol v0.1.0 // indirect - github.com/prometheus/client_golang v1.22.0 // indirect + github.com/prometheus/client_golang v1.23.2 // indirect github.com/prometheus/client_model v0.6.2 // indirect - github.com/prometheus/common v0.64.0 // indirect - github.com/prometheus/procfs v0.16.1 // indirect - github.com/rfjakob/eme v1.1.2 // indirect + github.com/prometheus/common v0.70.0 // indirect + github.com/prometheus/procfs v0.21.1 // indirect + github.com/rfjakob/eme v1.2.0 // indirect github.com/rivo/uniseg v0.4.7 // indirect github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46 // indirect github.com/shabbyrobe/gocovmerge v0.0.0-20230507112040-c3350d9342df // indirect github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e github.com/spaolacci/murmur3 v1.1.0 // indirect github.com/spf13/pflag v1.0.10 // indirect - github.com/tklauser/go-sysconf v0.3.15 // indirect - github.com/tklauser/numcpus v0.10.0 // indirect + github.com/tklauser/go-sysconf v0.4.0 // indirect + github.com/tklauser/numcpus v0.12.0 // indirect github.com/twitchyliquid64/golang-asm v0.15.1 // indirect github.com/u2takey/go-utils v0.3.1 // indirect - github.com/ugorji/go/codec v1.3.0 // indirect + github.com/ugorji/go/codec v1.3.1 // indirect github.com/x448/float16 v0.8.4 // indirect github.com/yusufpapurcu/wmi v1.2.4 // indirect - go.etcd.io/bbolt v1.4.0 // indirect - golang.org/x/arch v0.18.0 // indirect - golang.org/x/sync v0.19.0 - golang.org/x/sys v0.40.0 - golang.org/x/term v0.38.0 // indirect - golang.org/x/text v0.32.0 - golang.org/x/tools v0.39.0 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20251222181119-0a764e51fe1b // indirect - google.golang.org/grpc v1.78.0 + go.etcd.io/bbolt v1.5.0 // indirect + golang.org/x/arch v0.23.0 // indirect + golang.org/x/sync v0.22.0 + golang.org/x/sys v0.47.0 + golang.org/x/term v0.45.0 // indirect + golang.org/x/text v0.41.0 + golang.org/x/tools v0.48.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260715232425-e75dac1f907d // indirect + google.golang.org/grpc v1.85.0-dev google.golang.org/protobuf v1.36.11 // indirect gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d // indirect - gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect lukechampine.com/blake3 v1.1.7 // indirect ) @@ -315,4 +323,4 @@ replace github.com/cronokirby/saferith => github.com/Da3zKi7/saferith v0.33.0-fi // replace github.com/OpenListTeam/115-sdk-go => ../../OpenListTeam/115-sdk-go -replace github.com/KarpelesLab/reflink => github.com/OpenListTeam/reflink v0.0.0-20260520031008-ed3c0dbe8009 +replace github.com/KarpelesLab/reflink => github.com/OpenListTeam/reflink v0.0.0-20260701021214-78760eaeafef diff --git a/go.sum b/go.sum index 5030c3ce..937fbb57 100644 --- a/go.sum +++ b/go.sum @@ -1,22 +1,24 @@ cloud.google.com/go v0.121.6 h1:waZiuajrI28iAf40cWgycWNgaXPO06dupuS+sgibK6c= -cloud.google.com/go/auth v0.18.0 h1:wnqy5hrv7p3k7cShwAU/Br3nzod7fxoqG+k0VZ+/Pk0= -cloud.google.com/go/auth v0.18.0/go.mod h1:wwkPM1AgE1f2u6dG443MiWoD8C3BtOywNsUMcUTVDRo= +cloud.google.com/go/auth v0.20.0 h1:kXTssoVb4azsVDoUiF8KvxAqrsQcQtB53DcSgta74CA= +cloud.google.com/go/auth v0.20.0/go.mod h1:942/yi/itH1SsmpyrbnTMDgGfdy2BUqIKyd0cyYLc5Q= cloud.google.com/go/auth/oauth2adapt v0.2.8 h1:keo8NaayQZ6wimpNSmW5OPc283g65QNIiLpZnkHRbnc= cloud.google.com/go/auth/oauth2adapt v0.2.8/go.mod h1:XQ9y31RkqZCcwJWNSx2Xvric3RrU88hAYYbjDWYDL+c= cloud.google.com/go/compute/metadata v0.9.0 h1:pDUj4QMoPejqq20dK0Pg2N4yG9zIkYGdBtwLoEkH9Zs= cloud.google.com/go/compute/metadata v0.9.0/go.mod h1:E0bWwX5wTnLPedCKqk3pJmVgCBSM6qQI1yTBdEb3C10= -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.18.1 h1:Wc1ml6QlJs2BHQ/9Bqu1jiyggbsSjramq2oUmp5WeIo= -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.18.1/go.mod h1:Ot/6aikWnKWi4l9QB7qVSwa8iMphQNqkWALMoNT3rzM= -github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.10.1 h1:B+blDbyVIG3WaikNxPnhPiJ1MThR03b3vKGtER95TP4= -github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.10.1/go.mod h1:JdM5psgjfBf5fo2uWOZhflPWyDBZ/O/CNAH9CtsuZE4= -github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.1 h1:FPKJS1T+clwv+OLGt13a8UjqeRuh0O4SJ3lUriThc+4= -github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.1/go.mod h1:j2chePtV91HrC22tGoRX3sGY42uF13WzmmV80/OdVAA= +filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA= +filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.23.1 h1:zvXfGJCWvywnCA814d8ZiVyt+fm9nnTE8xSb99zRyfo= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.23.1/go.mod h1:iptorS+VYKFL2N6PnebpS91dubG35eAOEERnT4PJbQU= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.14.0 h1:CU4+EJeJi3TKYWEcYuSdWsjzw0nVsK/H0MSQOiPcymU= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.14.0/go.mod h1:q0+UTSRvShwUCrR/s5HtyInYphN7Wvxb7snFM3u+SLA= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.12.0 h1:fhqpLE3UEXi9lPaBRpQ6XuRW0nU7hgg4zlmZZa+a9q4= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.12.0/go.mod h1:7dCRMLwisfRH3dBupKeNCioWYUZ4SS09Z14H+7i8ZoY= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.1 h1:/Zt+cDPnpC3OVDm/JKLOs7M2DKmLRIIp3XIx9pHHiig= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.1/go.mod h1:Ng3urmn6dYe8gnbCMoHHVl5APYz2txho3koEkV2o2HA= -github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.2 h1:FwladfywkNirM+FZYLBR2kBz5C8Tg0fw5w5Y7meRXWI= -github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.2/go.mod h1:vv5Ad0RrIoT1lJFdWBZwt4mB1+j+V8DUroixmKDTCdk= -github.com/AzureAD/microsoft-authentication-library-for-go v1.4.2 h1:oygO0locgZJe7PpYPXT5A29ZkwJaPqcva7BVeemZOZs= -github.com/AzureAD/microsoft-authentication-library-for-go v1.4.2/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.8.0 h1:irsmOWwkp0KCTTNS5e2hdFeIvSQClQo2No3IaNmL3Vw= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.8.0/go.mod h1:GWcBkQj3MqN7ozHKLaCCAuNLiXoIGv2RtanfAwSjY/Y= +github.com/AzureAD/microsoft-authentication-library-for-go v1.7.2 h1:RHK7bS+HQMslb1sZpAokUt+zTVmue0hKSs2C791hhzU= +github.com/AzureAD/microsoft-authentication-library-for-go v1.7.2/go.mod h1:HKpQxkWaGLJ+D/5H8QRpyQXA1eKjxkFlOMwck5+33Jk= github.com/BurntSushi/toml v1.6.0 h1:dRaEfpa2VI55EwlIW72hMRHdWouJeRF7TPYhI+AUQjk= github.com/BurntSushi/toml v1.6.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho= github.com/Da3zKi7/saferith v0.33.0-fixed h1:fnIWTk7EP9mZAICf7aQjeoAwpfrlCrkOvqmi6CbWdTk= @@ -27,14 +29,16 @@ github.com/Masterminds/semver/v3 v3.2.0 h1:3MEsd0SM6jqZojhjLWWeBY+Kcjy9i6MQAeY7Y github.com/Masterminds/semver/v3 v3.2.0/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ= github.com/Max-Sum/base32768 v0.0.0-20230304063302-18e6ce5945fd h1:nzE1YQBdx1bq9IlZinHa+HVffy+NmVRoKr+wHN8fpLE= github.com/Max-Sum/base32768 v0.0.0-20230304063302-18e6ce5945fd/go.mod h1:C8yoIfvESpM3GD07OCHU7fqI7lhwyZ2Td1rbNbTAhnc= -github.com/OpenListTeam/115-sdk-go v0.2.4 h1:dVoEz3Pm996n/ZCuRckQvz36w938uNHjPrS7E/SVpBM= -github.com/OpenListTeam/115-sdk-go v0.2.4/go.mod h1:cfvitk2lwe6036iNi2h+iNxwxWDifKZsSvNtrur5BqU= +github.com/OpenListTeam/115-sdk-go v0.2.6 h1:ehXyStvncvn4qRBuknor3kyGZtUmHc0+stjF2ByVdFo= +github.com/OpenListTeam/115-sdk-go v0.2.6/go.mod h1:cfvitk2lwe6036iNi2h+iNxwxWDifKZsSvNtrur5BqU= github.com/OpenListTeam/go-cache v0.1.0 h1:eV2+FCP+rt+E4OCJqLUW7wGccWZNJMV0NNkh+uChbAI= github.com/OpenListTeam/go-cache v0.1.0/go.mod h1:AHWjKhNK3LE4rorVdKyEALDHoeMnP8SjiNyfVlB+Pz4= +github.com/OpenListTeam/gofakes3 v0.8.2-0.20260911142347-cd3c030a83b4 h1:Zy7/qg6aCS0OF/FPIoJh9/d0IgcIxpWRvn79ACm2R/Y= +github.com/OpenListTeam/gofakes3 v0.8.2-0.20260911142347-cd3c030a83b4/go.mod h1:mS9Ywbo6aId6BrRzeYjOIOpK0QDVnMoKOIb0hpaQZ3U= github.com/OpenListTeam/gsync v0.1.0 h1:ywzGybOvA3lW8K1BUjKZ2IUlT2FSlzPO4DOazfYXjcs= github.com/OpenListTeam/gsync v0.1.0/go.mod h1:h/Rvv9aX/6CdW/7B8di3xK3xNV8dUg45Fehrd/ksZ9s= -github.com/OpenListTeam/reflink v0.0.0-20260520031008-ed3c0dbe8009 h1:qLqJPr/FAsZTJiqy65JKKuFJP0V9pRVtSaIE0kqaQ8w= -github.com/OpenListTeam/reflink v0.0.0-20260520031008-ed3c0dbe8009/go.mod h1:WGkTOKNjd1FsJKBw3mu4JvrPEDJyJJ+JPtxBkbPoCok= +github.com/OpenListTeam/reflink v0.0.0-20260701021214-78760eaeafef h1:67uGHancMF/abMrnkc8abVUWQiG73Wk5d8CKt3RzkFo= +github.com/OpenListTeam/reflink v0.0.0-20260701021214-78760eaeafef/go.mod h1:WGkTOKNjd1FsJKBw3mu4JvrPEDJyJJ+JPtxBkbPoCok= github.com/OpenListTeam/sftpd-openlist v1.0.1 h1:j4S3iPFOpnXCUKRPS7uCT4mF2VCl34GyqvH6lqwnkUU= github.com/OpenListTeam/sftpd-openlist v1.0.1/go.mod h1:uO/wKnbvbdq3rBLmClMTZXuCnw7XW4wlAq4dZe91a40= github.com/OpenListTeam/tache v0.2.2 h1:CWFn6sr1AIYaEjC8ONdKs+LrxHyuErheenAjEqRhh4k= @@ -49,24 +53,26 @@ github.com/ProtonMail/bcrypt v0.0.0-20211005172633-e235017c1baf/go.mod h1:o0ESU9 github.com/ProtonMail/gluon v0.17.1-0.20230724134000-308be39be96e h1:lCsqUUACrcMC83lg5rTo9Y0PnPItE61JSfvMyIcANwk= github.com/ProtonMail/gluon v0.17.1-0.20230724134000-308be39be96e/go.mod h1:Og5/Dz1MiGpCJn51XujZwxiLG7WzvvjE5PRpZBQmAHo= github.com/ProtonMail/go-crypto v0.0.0-20230321155629-9a39f2531310/go.mod h1:8TI4H3IbrackdNgv+92dI+rhpCaLqM0IfpgCgenFvRE= -github.com/ProtonMail/go-crypto v1.3.0 h1:ILq8+Sf5If5DCpHQp4PbZdS1J7HDFRXz/+xKBiRGFrw= -github.com/ProtonMail/go-crypto v1.3.0/go.mod h1:9whxjD8Rbs29b4XWbB8irEcE8KHMqaR2e7GWU1R+/PE= +github.com/ProtonMail/go-crypto v1.4.1 h1:9RfcZHqEQUvP8RzecWEUafnZVtEvrBVL9BiF67IQOfM= +github.com/ProtonMail/go-crypto v1.4.1/go.mod h1:e1OaTyu5SYVrO9gKOEhTc+5UcXtTUa+P3uLudwcgPqo= github.com/ProtonMail/go-mime v0.0.0-20230322103455-7d82a3887f2f h1:tCbYj7/299ekTTXpdwKYF8eBlsYsDVoggDAuAjoK66k= github.com/ProtonMail/go-mime v0.0.0-20230322103455-7d82a3887f2f/go.mod h1:gcr0kNtGBqin9zDW9GOHcVntrwnjrK+qdJ06mWYBybw= github.com/ProtonMail/go-srp v0.0.7 h1:Sos3Qk+th4tQR64vsxGIxYpN3rdnG9Wf9K4ZloC1JrI= github.com/ProtonMail/go-srp v0.0.7/go.mod h1:giCp+7qRnMIcCvI6V6U3S1lDDXDQYx2ewJ6F/9wdlJk= -github.com/ProtonMail/gopenpgp/v2 v2.9.0 h1:ruLzBmwe4dR1hdnrsEJ/S7psSBmV15gFttFUPP/+/kE= -github.com/ProtonMail/gopenpgp/v2 v2.9.0/go.mod h1:IldDyh9Hv1ZCCYatTuuEt1XZJ0OPjxLpTarDfglih7s= -github.com/PuerkitoBio/goquery v1.10.3 h1:pFYcNSqHxBD06Fpj/KsbStFRsgRATgnf3LeXiUkhzPo= -github.com/PuerkitoBio/goquery v1.10.3/go.mod h1:tMUX0zDMHXYlAQk6p35XxQMqMweEKB7iK7iLNd4RH4Y= -github.com/RoaringBitmap/roaring/v2 v2.4.5 h1:uGrrMreGjvAtTBobc0g5IrW1D5ldxDQYe2JW2gggRdg= -github.com/RoaringBitmap/roaring/v2 v2.4.5/go.mod h1:FiJcsfkGje/nZBZgCu0ZxCPOKD/hVXDS2dXi7/eUFE0= -github.com/STARRY-S/zip v0.2.1 h1:pWBd4tuSGm3wtpoqRZZ2EAwOmcHK6XFf7bU9qcJXyFg= -github.com/STARRY-S/zip v0.2.1/go.mod h1:xNvshLODWtC4EJ702g7cTYn13G53o1+X9BWnPFpcWV4= -github.com/SheltonZhu/115driver v1.3.3 h1:Bqs86D2MziYPgIOuOJF+HzG4d7GBr71ZhSCrs/U17UU= -github.com/SheltonZhu/115driver v1.3.3/go.mod h1:OujS7azslg1/bn85sPSHnNsp4/WBI9/TiijtZL9kuSQ= +github.com/ProtonMail/gopenpgp/v2 v2.10.0 h1:llCzLvntC9+iH+if/na4AgKTef/Zm4vpaRrR3+JdKvo= +github.com/ProtonMail/gopenpgp/v2 v2.10.0/go.mod h1:dc0h9Pg3ftfN0U4pfRzujilfh61A2R52wgMkZWcWm2I= +github.com/PuerkitoBio/goquery v1.12.0 h1:pAcL4g3WRXekcB9AU/y1mbKez2dbY2AajVhtkO8RIBo= +github.com/PuerkitoBio/goquery v1.12.0/go.mod h1:802ej+gV2y7bbIhOIoPY5sT183ZW0YFofScC4q/hIpQ= +github.com/RoaringBitmap/roaring/v2 v2.14.5 h1:ckd0o545JqDPeVJDgeFoaM21eBixUnlWfYgjE5VnyWw= +github.com/RoaringBitmap/roaring/v2 v2.14.5/go.mod h1:eq4wdNXxtJIS/oikeCzdX1rBzek7ANzbth041hrU8Q4= +github.com/STARRY-S/zip v0.2.3 h1:luE4dMvRPDOWQdeDdUxUoZkzUIpTccdKdhHHsQJ1fm4= +github.com/STARRY-S/zip v0.2.3/go.mod h1:lqJ9JdeRipyOQJrYSOtpNAiaesFO6zVDsE8GIGFaoSk= +github.com/SheltonZhu/115driver v1.3.5 h1:NlJA61RskdwHhIZyg4519696lCk1n4aid0lKARsMwFg= +github.com/SheltonZhu/115driver v1.3.5/go.mod h1:ag0B8Vs6TPFQjTNFLVVxA7dINPt4q+WR47+JIPJhmTk= github.com/abbot/go-http-auth v0.4.0 h1:QjmvZ5gSC7jm3Zg54DqWE/T5m1t2AfDu6QlXJT0EVT0= github.com/abbot/go-http-auth v0.4.0/go.mod h1:Cz6ARTIzApMJDzh5bRMSUou6UMSp0IEXg9km/ci7TJM= +github.com/adrg/xdg v0.5.3 h1:xRnxJXne7+oWDatRhR1JLnvuccuIeCoBu2rtuLqQB78= +github.com/adrg/xdg v0.5.3/go.mod h1:nlTsY+NNiCBGCK2tpm09vRqfVzrc2fLmXGpBLF0zlTQ= github.com/aead/ecdh v0.2.0 h1:pYop54xVaq/CEREFEcukHRZfTdjiWvYIsZDXXrBapQQ= github.com/aead/ecdh v0.2.0/go.mod h1:a9HHtXuSo8J1Js1MwLQx2mBhkXMT6YwUmVVEY4tTB8U= github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible h1:8psS8a+wKfiLt1iVDX79F7Y6wUM49Lcha2FMXt4UM8g= @@ -74,156 +80,165 @@ github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible/go.mod h1:T/Aws4fEfogEE9 github.com/andreburgaud/crypt2go v1.8.0 h1:J73vGTb1P6XL69SSuumbKs0DWn3ulbl9L92ZXBjw6pc= github.com/andreburgaud/crypt2go v1.8.0/go.mod h1:L5nfShQ91W78hOWhUH2tlGRPO+POAPJAF5fKOLB9SXg= github.com/andybalholm/brotli v1.1.1/go.mod h1:05ib4cKhjx3OQYUY22hTVd34Bc8upXjOLL2rKwwZBoA= -github.com/andybalholm/brotli v1.1.2-0.20250424173009-453214e765f3 h1:8PmGpDEZl9yDpcdEr6Odf23feCxK3LNUNMxjXg41pZQ= -github.com/andybalholm/brotli v1.1.2-0.20250424173009-453214e765f3/go.mod h1:05ib4cKhjx3OQYUY22hTVd34Bc8upXjOLL2rKwwZBoA= -github.com/andybalholm/cascadia v1.3.3 h1:AG2YHrzJIm4BZ19iwJ/DAua6Btl3IwJX+VI4kktS1LM= -github.com/andybalholm/cascadia v1.3.3/go.mod h1:xNd9bqTn98Ln4DwST8/nG+H0yuB8Hmgu1YHNnWw0GeA= +github.com/andybalholm/brotli v1.2.2 h1:HzTuoo2ErYQqf5qvcJInB8uvqSVxRttzkFexPWtnceM= +github.com/andybalholm/brotli v1.2.2/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY= +github.com/andybalholm/cascadia v1.3.4 h1:vM2lgh0Vru9Vwyfm4cQqWP2HHMW0u0+2PAW7Q38Qufg= +github.com/andybalholm/cascadia v1.3.4/go.mod h1:BLRmbRjpEtNKieZOCCvYj4RqN+KRA41GBe/5O+G93kM= github.com/antchfx/htmlquery v1.3.6 h1:RNHHL7YehO5XdO8IM8CynwLKONwRHWkrghbYhQIk9ag= github.com/antchfx/htmlquery v1.3.6/go.mod h1:kcVUqancxPygm26X2rceEcagZFFVkLEE7xgLkGSDl/4= -github.com/antchfx/xpath v1.3.6 h1:s0y+ElRRtTQdfHP609qFu0+c6bglDv20pqOViQjjdPI= github.com/antchfx/xpath v1.3.6/go.mod h1:i54GszH55fYfBmoZXapTHN8T8tkcHfRgLyVwwqzXNcs= +github.com/antchfx/xpath v1.3.8 h1:RQlkLaJDKk1Ew1H6CUPUTKM+IQxm+6HTyOgcrfqOU9c= +github.com/antchfx/xpath v1.3.8/go.mod h1:i54GszH55fYfBmoZXapTHN8T8tkcHfRgLyVwwqzXNcs= github.com/avast/retry-go v3.0.0+incompatible h1:4SOWQ7Qs+oroOTQOYnAHqelpCO0biHSxpiH9JdtuBj0= github.com/avast/retry-go v3.0.0+incompatible/go.mod h1:XtSnn+n/sHqQIpZ10K1qAevBhOOCWBLXXy3hyiqqBrY= github.com/aws/aws-sdk-go v1.38.20/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= github.com/aws/aws-sdk-go v1.55.8 h1:JRmEUbU52aJQZ2AjX4q4Wu7t4uZjOu71uyNmaWlUkJQ= github.com/aws/aws-sdk-go v1.55.8/go.mod h1:ZkViS9AqA6otK+JBBNH2++sx1sgxrPKcSzPPvQkUtXk= -github.com/aws/aws-sdk-go-v2 v1.36.3 h1:mJoei2CxPutQVxaATCzDUjcZEjVRdpsiiXi2o38yqWM= -github.com/aws/aws-sdk-go-v2 v1.36.3/go.mod h1:LLXuLpgzEbD766Z5ECcRmi8AzSwfZItDtmABVkRLGzg= -github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.10 h1:zAybnyUQXIZ5mok5Jqwlf58/TFE7uvd3IAsa1aF9cXs= -github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.10/go.mod h1:qqvMj6gHLR/EXWZw4ZbqlPbQUyenf4h82UQUlKc+l14= -github.com/aws/aws-sdk-go-v2/credentials v1.17.67 h1:9KxtdcIA/5xPNQyZRgUSpYOE6j9Bc4+D7nZua0KGYOM= -github.com/aws/aws-sdk-go-v2/credentials v1.17.67/go.mod h1:p3C44m+cfnbv763s52gCqrjaqyPikj9Sg47kUVaNZQQ= -github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.49 h1:7gss+6H2mrrFtBrkokJRR2TzQD9qkpGA4N6BvIP/pCM= -github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.49/go.mod h1:30PBx0ENoUCJm2AxzgCue8j7KEjb9ci4enxy6CCOjbE= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.34 h1:ZK5jHhnrioRkUNOc+hOgQKlUL5JeC3S6JgLxtQ+Rm0Q= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.34/go.mod h1:p4VfIceZokChbA9FzMbRGz5OV+lekcVtHlPKEO0gSZY= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.34 h1:SZwFm17ZUNNg5Np0ioo/gq8Mn6u9w19Mri8DnJ15Jf0= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.34/go.mod h1:dFZsC0BLo346mvKQLWmoJxT+Sjp+qcVR1tRVHQGOH9Q= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.34 h1:ZNTqv4nIdE/DiBfUUfXcLZ/Spcuz+RjeziUtNJackkM= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.34/go.mod h1:zf7Vcd1ViW7cPqYWEHLHJkS50X0JS2IKz9Cgaj6ugrs= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.3 h1:eAh2A4b5IzM/lum78bZ590jy36+d/aFLgKF/4Vd1xPE= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.3/go.mod h1:0yKJC/kb8sAnmlYa6Zs3QVYqaC8ug2AbnNChv5Ox3uA= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.7.2 h1:BCG7DCXEXpNCcpwCxg1oi9pkJWH2+eZzTn9MY56MbVw= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.7.2/go.mod h1:iu6FSzgt+M2/x3Dk8zhycdIcHjEFb36IS8HVUVFoMg0= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.15 h1:dM9/92u2F1JbDaGooxTq18wmmFzbJRfXfVfy96/1CXM= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.15/go.mod h1:SwFBy2vjtA0vZbjjaFtfN045boopadnoVPhu4Fv66vY= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.15 h1:moLQUoVq91LiqT1nbvzDukyqAlCv89ZmwaHw/ZFlFZg= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.15/go.mod h1:ZH34PJUc8ApjBIfgQCFvkWcUDBtl/WTD+uiYHjd8igA= -github.com/aws/aws-sdk-go-v2/service/s3 v1.72.3 h1:WZOmJfCDV+4tYacLxpiojoAdT5sxTfB3nTqQNtZu+J4= -github.com/aws/aws-sdk-go-v2/service/s3 v1.72.3/go.mod h1:xMekrnhmJ5aqmyxtmALs7mlvXw5xRh+eYjOjvrIIFJ4= -github.com/aws/smithy-go v1.22.3 h1:Z//5NuZCSW6R4PhQ93hShNbyBbn8BWCmCVCt+Q8Io5k= -github.com/aws/smithy-go v1.22.3/go.mod h1:t1ufH5HMublsJYulve2RKmHDC15xu1f26kHCp/HgceI= +github.com/aws/aws-sdk-go-v2 v1.43.7 h1:msCzvkeYJA9ehbV8mRRmkZLo/zJg/+yDVLNtflg83hQ= +github.com/aws/aws-sdk-go-v2 v1.43.7/go.mod h1:tXpPM+v0D1lndmga+HqqLDIzUFJlEeR21aspVklHF00= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.18 h1:LAfOuhAH331fmOjTQpAaOlH+Ftn7RzSDJ2VFwjdMMy4= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.18/go.mod h1:4e5xhuXHx1e4U9EthvbPP1r/DIMp5c2823OL8karzcM= +github.com/aws/aws-sdk-go-v2/credentials v1.19.29 h1:WHZGssHH887cO0ox07SIQZsFx3MKD4ps6w0xUEmnKYQ= +github.com/aws/aws-sdk-go-v2/credentials v1.19.29/go.mod h1:Mhl0xR6zjguiuj00XRx2wMx22sAltk7oya39sT7fdg8= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.22.34 h1:Pn7OsMwBLbkZ6OnCxWHAjf0L/22H8cnhxZC0uPwtMtg= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.22.34/go.mod h1:eToXR/Gk1uqpn04eSmdgVXwfS0WvH8aG4eBFr8ygbpU= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.38 h1:MBMg0zJ6i4TkAJ0dVFLKKn2cOkY6FkicmUDM67BRr6g= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.38/go.mod h1:9MWuJbyiUyj6eA7W1/zm1zuePDPSB3g+xcgRQeMWsXc= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.38 h1:lHm4jPf3k1Lz5ZWc+Vcn3MKVwym+26kWCba9FkJ4f0Y= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.38/go.mod h1:Rn+P2XR+FbyZzjmWKjg/KUZNxmGfr5oZwh5jQiE+CzI= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.39 h1:vo4xvMRs/F6h1E52qsgLqCQgWIQXgIJUauG6rlZEh4U= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.39/go.mod h1:jB03R1ij/A+OE2e1dz6vgj076gd7vlYcfstAzj3HcnU= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.17 h1:OvYZOB3qA6zvfdRFiRFRzVSiElMYrz3GdntkXZxlp1o= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.17/go.mod h1:JgR/2Ew50ACfIWau1oeMRX59tMtC0kM+PYQGEaT04cY= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.31 h1:uZOinZb+h7lZw8IYzP1z1IuEnueB76/EFkcf/fEW4Ag= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.31/go.mod h1:NRtwAM/p5VRt03TlEUs0pH3TeWamWdf4YyJpSrzPYLc= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.38 h1:H/5TI1jqaHsNoDQ60UwvPvJBg4GURkinXI3Qga29t2w= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.38/go.mod h1:PTVFf+XH++7NJOky+RLBYQx0QA5NcaeEYFQ2fsi0nwo= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.39 h1:HLPAVrlLDaN2boN0xJx7MgaQDNEO3Q+c9L6kl/8m47Q= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.39/go.mod h1:Pg/dVfsNkm1hsIDK/gMvCKtmyNfNTV12mrgHqVE/6Oo= +github.com/aws/aws-sdk-go-v2/service/s3 v1.107.3 h1:IKoCZqfWfZzSBi16QFQ+QcbQ3LRQ7QgB1S5tDAyPBQQ= +github.com/aws/aws-sdk-go-v2/service/s3 v1.107.3/go.mod h1:RBpRcXiM4s2pOInVs32GsBonnje+fiAj4mcrStRmlCA= +github.com/aws/smithy-go v1.27.8 h1:FR0dxZfIlV7Z8eh2iHfIofdunw382XsDV3Mxt9nUvRY= +github.com/aws/smithy-go v1.27.8/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= github.com/axgle/mahonia v0.0.0-20180208002826-3358181d7394 h1:OYA+5W64v3OgClL+IrOD63t4i/RW7RqrAVl9LTZ9UqQ= github.com/axgle/mahonia v0.0.0-20180208002826-3358181d7394/go.mod h1:Q8n74mJTIgjX4RBBcHnJ05h//6/k6foqmgE45jTQtxg= github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiErDT4WkJ2k= github.com/aymanbagabas/go-osc52/v2 v2.0.1/go.mod h1:uYgXzlJ7ZpABp8OJ+exZzJJhRNQ2ASbcXHWsFqH8hp8= -github.com/aymanbagabas/go-udiff v0.2.0 h1:TK0fH4MteXUDspT88n8CKzvK0X9O2xu9yQjWpi6yML8= -github.com/aymanbagabas/go-udiff v0.2.0/go.mod h1:RE4Ex0qsGkTAJoQdQQCA0uG+nAzJO/pI/QwceO5fgrA= +github.com/aymanbagabas/go-udiff v0.3.1 h1:LV+qyBQ2pqe0u42ZsUEtPiCaUoqgA9gYRDs3vj1nolY= +github.com/aymanbagabas/go-udiff v0.3.1/go.mod h1:G0fsKmG+P6ylD0r6N/KgQD/nWzgfnl8ZBcNLgcbrw8E= github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk= github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4= github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A= github.com/benbjohnson/clock v1.3.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= -github.com/bits-and-blooms/bitset v1.12.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8= -github.com/bits-and-blooms/bitset v1.22.0 h1:Tquv9S8+SGaS3EhyA+up3FXzmkhxPGjQQCkcs2uw7w4= -github.com/bits-and-blooms/bitset v1.22.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8= +github.com/bits-and-blooms/bitset v1.24.4 h1:95H15Og1clikBrKr/DuzMXkQzECs1M6hhoGXLwLQOZE= +github.com/bits-and-blooms/bitset v1.24.4/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8= github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= -github.com/blevesearch/bleve/v2 v2.5.2 h1:Ab0r0MODV2C5A6BEL87GqLBySqp/s9xFgceCju6BQk8= -github.com/blevesearch/bleve/v2 v2.5.2/go.mod h1:5Dj6dUQxZM6aqYT3eutTD/GpWKGFSsV8f7LDidFbwXo= -github.com/blevesearch/bleve_index_api v1.2.8 h1:Y98Pu5/MdlkRyLM0qDHostYo7i+Vv1cDNhqTeR4Sy6Y= -github.com/blevesearch/bleve_index_api v1.2.8/go.mod h1:rKQDl4u51uwafZxFrPD1R7xFOwKnzZW7s/LSeK4lgo0= -github.com/blevesearch/geo v0.2.3 h1:K9/vbGI9ehlXdxjxDRJtoAMt7zGAsMIzc6n8zWcwnhg= -github.com/blevesearch/geo v0.2.3/go.mod h1:K56Q33AzXt2YExVHGObtmRSFYZKYGv0JEN5mdacJJR8= -github.com/blevesearch/go-faiss v1.0.25 h1:lel1rkOUGbT1CJ0YgzKwC7k+XH0XVBHnCVWahdCXk4U= -github.com/blevesearch/go-faiss v1.0.25/go.mod h1:OMGQwOaRRYxrmeNdMrXJPvVx8gBnvE5RYrr0BahNnkk= +github.com/blevesearch/bleve/v2 v2.6.1 h1:47vLskRTqxvQEtxVPYHjf5KpOgzD2msslXFjvUQCgWQ= +github.com/blevesearch/bleve/v2 v2.6.1/go.mod h1:Dvvx6ZoEBTOj6RSzfk0lEz0wce/qhe2yOUubXeuzd2c= +github.com/blevesearch/bleve_index_api v1.4.1 h1:CYIyecFlI+/RYjzUm+NmDjYbSvk870Bb7f+Vl4b12q8= +github.com/blevesearch/bleve_index_api v1.4.1/go.mod h1:xvd48t5XMeeioWQ5/jZvgLrV98flT2rdvEJ3l/ki4Ko= +github.com/blevesearch/geo v0.2.6 h1:7K1oyQKYlauC+mJuo2AfNPyjN/4mihEoJMfyClVH1Mo= +github.com/blevesearch/geo v0.2.6/go.mod h1:6qzVUiB4BK47QkSZcRqiXEP2W3EeXuzM5XFTF8AdZ8A= +github.com/blevesearch/go-faiss v1.1.5 h1:/IU5lkOahH9Ghfk9n3F6N0XD7PYVXZJWmNDc9TtXuco= +github.com/blevesearch/go-faiss v1.1.5/go.mod h1:w3W9AiWsFRGVaMG+/cmJi7iHEAuGyC6blsgO1EzCK/M= github.com/blevesearch/go-porterstemmer v1.0.3 h1:GtmsqID0aZdCSNiY8SkuPJ12pD4jI+DdXTAn4YRcHCo= github.com/blevesearch/go-porterstemmer v1.0.3/go.mod h1:angGc5Ht+k2xhJdZi511LtmxuEf0OVpvUUNrwmM1P7M= github.com/blevesearch/gtreap v0.1.1 h1:2JWigFrzDMR+42WGIN/V2p0cUvn4UP3C4Q5nmaZGW8Y= github.com/blevesearch/gtreap v0.1.1/go.mod h1:QaQyDRAT51sotthUWAH4Sj08awFSSWzgYICSZ3w0tYk= -github.com/blevesearch/mmap-go v1.0.4 h1:OVhDhT5B/M1HNPpYPBKIEJaD0F3Si+CrEKULGCDPWmc= -github.com/blevesearch/mmap-go v1.0.4/go.mod h1:EWmEAOmdAS9z/pi/+Toxu99DnsbhG1TIxUoRmJw/pSs= -github.com/blevesearch/scorch_segment_api/v2 v2.3.10 h1:Yqk0XD1mE0fDZAJXTjawJ8If/85JxnLd8v5vG/jWE/s= -github.com/blevesearch/scorch_segment_api/v2 v2.3.10/go.mod h1:Z3e6ChN3qyN35yaQpl00MfI5s8AxUJbpTR/DL8QOQ+8= +github.com/blevesearch/mmap-go v1.2.0 h1:l33nNKPFcBjJUMwem6sAYJPUzhUCABoK9FxZDGiFNBI= +github.com/blevesearch/mmap-go v1.2.0/go.mod h1:Vd6+20GBhEdwJnU1Xohgt88XCD/CTWcqbCNxkZpyBo0= +github.com/blevesearch/scorch_segment_api/v2 v2.4.10 h1:C3873+iWZ0YJM2ijaSHhJJzSvD4x1k+5UaQdGygZVhM= +github.com/blevesearch/scorch_segment_api/v2 v2.4.10/go.mod h1:WUUkAocbkDlNK/kgAE13NvS9oxe+u618mYZ8sOvcCc4= github.com/blevesearch/segment v0.9.1 h1:+dThDy+Lvgj5JMxhmOVlgFfkUtZV2kw49xax4+jTfSU= github.com/blevesearch/segment v0.9.1/go.mod h1:zN21iLm7+GnBHWTao9I+Au/7MBiL8pPFtJBJTsk6kQw= github.com/blevesearch/snowballstem v0.9.0 h1:lMQ189YspGP6sXvZQ4WZ+MLawfV8wOmPoD/iWeNXm8s= github.com/blevesearch/snowballstem v0.9.0/go.mod h1:PivSj3JMc8WuaFkTSRDW2SlrulNWPl4ABg1tC/hlgLs= github.com/blevesearch/upsidedown_store_api v1.0.2 h1:U53Q6YoWEARVLd1OYNc9kvhBMGZzVrdmaozG2MfoB+A= github.com/blevesearch/upsidedown_store_api v1.0.2/go.mod h1:M01mh3Gpfy56Ps/UXHjEO/knbqyQ1Oamg8If49gRwrQ= -github.com/blevesearch/vellum v1.1.0 h1:CinkGyIsgVlYf8Y2LUQHvdelgXr6PYuvoDIajq6yR9w= -github.com/blevesearch/vellum v1.1.0/go.mod h1:QgwWryE8ThtNPxtgWJof5ndPfx0/YMBh+W2weHKPw8Y= -github.com/blevesearch/zapx/v11 v11.4.2 h1:l46SV+b0gFN+Rw3wUI1YdMWdSAVhskYuvxlcgpQFljs= -github.com/blevesearch/zapx/v11 v11.4.2/go.mod h1:4gdeyy9oGa/lLa6D34R9daXNUvfMPZqUYjPwiLmekwc= -github.com/blevesearch/zapx/v12 v12.4.2 h1:fzRbhllQmEMUuAQ7zBuMvKRlcPA5ESTgWlDEoB9uQNE= -github.com/blevesearch/zapx/v12 v12.4.2/go.mod h1:TdFmr7afSz1hFh/SIBCCZvcLfzYvievIH6aEISCte58= -github.com/blevesearch/zapx/v13 v13.4.2 h1:46PIZCO/ZuKZYgxI8Y7lOJqX3Irkc3N8W82QTK3MVks= -github.com/blevesearch/zapx/v13 v13.4.2/go.mod h1:knK8z2NdQHlb5ot/uj8wuvOq5PhDGjNYQQy0QDnopZk= -github.com/blevesearch/zapx/v14 v14.4.2 h1:2SGHakVKd+TrtEqpfeq8X+So5PShQ5nW6GNxT7fWYz0= -github.com/blevesearch/zapx/v14 v14.4.2/go.mod h1:rz0XNb/OZSMjNorufDGSpFpjoFKhXmppH9Hi7a877D8= -github.com/blevesearch/zapx/v15 v15.4.2 h1:sWxpDE0QQOTjyxYbAVjt3+0ieu8NCE0fDRaFxEsp31k= -github.com/blevesearch/zapx/v15 v15.4.2/go.mod h1:1pssev/59FsuWcgSnTa0OeEpOzmhtmr/0/11H0Z8+Nw= -github.com/blevesearch/zapx/v16 v16.2.4 h1:tGgfvleXTAkwsD5mEzgM3zCS/7pgocTCnO1oyAUjlww= -github.com/blevesearch/zapx/v16 v16.2.4/go.mod h1:Rti/REtuuMmzwsI8/C/qIzRaEoSK/wiFYw5e5ctUKKs= -github.com/bmatcuk/doublestar/v4 v4.9.1 h1:X8jg9rRZmJd4yRy7ZeNDRnM+T3ZfHv15JiBJ/avrEXE= -github.com/bmatcuk/doublestar/v4 v4.9.1/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc= +github.com/blevesearch/vellum v1.2.0 h1:xkDiOEsHc2t3Cp0NsNZZ36pvc130sCzcGKOPMzXe+e0= +github.com/blevesearch/vellum v1.2.0/go.mod h1:uEcfBJz7mAOf0Kvq6qoEKQQkLODBF46SINYNkZNae4k= +github.com/blevesearch/zapx/v11 v11.4.3 h1:PTZOO5loKpHC/x/GzmPZNa9cw7GZIQxd5qRjwij9tHY= +github.com/blevesearch/zapx/v11 v11.4.3/go.mod h1:4gdeyy9oGa/lLa6D34R9daXNUvfMPZqUYjPwiLmekwc= +github.com/blevesearch/zapx/v12 v12.4.3 h1:eElXvAaAX4m04t//CGBQAtHNPA+Q6A1hHZVrN3LSFYo= +github.com/blevesearch/zapx/v12 v12.4.3/go.mod h1:TdFmr7afSz1hFh/SIBCCZvcLfzYvievIH6aEISCte58= +github.com/blevesearch/zapx/v13 v13.4.3 h1:qsdhRhaSpVnqDFlRiH9vG5+KJ+dE7KAW9WyZz/KXAiE= +github.com/blevesearch/zapx/v13 v13.4.3/go.mod h1:knK8z2NdQHlb5ot/uj8wuvOq5PhDGjNYQQy0QDnopZk= +github.com/blevesearch/zapx/v14 v14.4.3 h1:GY4Hecx0C6UTmiNC2pKdeA2rOKiLR5/rwpU9WR51dgM= +github.com/blevesearch/zapx/v14 v14.4.3/go.mod h1:rz0XNb/OZSMjNorufDGSpFpjoFKhXmppH9Hi7a877D8= +github.com/blevesearch/zapx/v15 v15.4.3 h1:iJiMJOHrz216jyO6lS0m9RTCEkprUnzvqAI2lc/0/CU= +github.com/blevesearch/zapx/v15 v15.4.3/go.mod h1:1pssev/59FsuWcgSnTa0OeEpOzmhtmr/0/11H0Z8+Nw= +github.com/blevesearch/zapx/v16 v16.3.4 h1:hDAqA8qusZTNbPEL7//w5P65UZ2de6yhSeUaTbp0Po0= +github.com/blevesearch/zapx/v16 v16.3.4/go.mod h1:zqkPPqs9GS9FzVWzCO3Wf1X044yWAV17+4zb+FTiEHg= +github.com/blevesearch/zapx/v17 v17.2.3 h1:UYYJPAt5b2tVxldx5h0jmv23RMsg8/UZKFVya7v92po= +github.com/blevesearch/zapx/v17 v17.2.3/go.mod h1:r7mb4QWbDQSkbAnOjCb9iCfkcrzajB4yBdJpuBIo/fE= +github.com/bmatcuk/doublestar/v4 v4.10.0 h1:zU9WiOla1YA122oLM6i4EXvGW62DvKZVxIe6TYWexEs= +github.com/bmatcuk/doublestar/v4 v4.10.0/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc= github.com/bodgit/plumbing v1.3.0 h1:pf9Itz1JOQgn7vEOE7v7nlEfBykYqvUYioC61TwWCFU= github.com/bodgit/plumbing v1.3.0/go.mod h1:JOTb4XiRu5xfnmdnDJo6GmSbSbtSyufrsyZFByMtKEs= -github.com/bodgit/sevenzip v1.6.1 h1:kikg2pUMYC9ljU7W9SaqHXhym5HyKm8/M/jd31fYan4= -github.com/bodgit/sevenzip v1.6.1/go.mod h1:GVoYQbEVbOGT8n2pfqCIMRUaRjQ8F9oSqoBEqZh5fQ8= +github.com/bodgit/sevenzip v1.6.5 h1:7H7BxgmeX0j6UX42lH+KXQ92WgMQJ49DoocFdfHbCng= +github.com/bodgit/sevenzip v1.6.5/go.mod h1:GhuB6Lq1xCpP1sps+horjZ8lgiKPJcy2zUX3prla9wc= github.com/bodgit/windows v1.0.1 h1:tF7K6KOluPYygXa3Z2594zxlkbKPAOvqr97etrGNIz4= github.com/bodgit/windows v1.0.1/go.mod h1:a6JLwrB4KrTR5hBpp8FI9/9W9jJfeQ2h4XDXU74ZCdM= -github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc h1:biVzkmvwrH8WK8raXaxBx6fRVTlJILwEwQGL1I/ByEI= github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8= +github.com/boombuler/barcode v1.1.0 h1:ChaYjBR63fr4LFyGn8E8nt7dBSt3MiU3zMOZqFvVkHo= +github.com/boombuler/barcode v1.1.0/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8= github.com/bradenaw/juniper v0.15.3 h1:RHIAMEDTpvmzV1wg1jMAHGOoI2oJUSPx3lxRldXnFGo= github.com/bradenaw/juniper v0.15.3/go.mod h1:UX4FX57kVSaDp4TPqvSjkAAewmRFAfXf27BOs5z9dq8= github.com/bwesterb/go-ristretto v1.2.0/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0= -github.com/bytedance/sonic v1.13.3 h1:MS8gmaH16Gtirygw7jV91pDCN33NyMrPbN7qiYhEsF0= -github.com/bytedance/sonic v1.13.3/go.mod h1:o68xyaF9u2gvVBuGHPlUVCy+ZfmNNO5ETf1+KgkJhz4= -github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU= -github.com/bytedance/sonic/loader v0.2.4 h1:ZWCw4stuXUsn1/+zQDqeE7JKP+QO47tz7QCNan80NzY= -github.com/bytedance/sonic/loader v0.2.4/go.mod h1:N8A3vUdtUebEY2/VQC0MyhYeKUFosQU6FxH2JmUe6VI= +github.com/bytedance/gopkg v0.1.3 h1:TPBSwH8RsouGCBcMBktLt1AymVo2TVsBVCY4b6TnZ/M= +github.com/bytedance/gopkg v0.1.3/go.mod h1:576VvJ+eJgyCzdjS+c4+77QF3p7ubbtiKARP3TxducM= +github.com/bytedance/sonic v1.15.0 h1:/PXeWFaR5ElNcVE84U0dOHjiMHQOwNIx3K4ymzh/uSE= +github.com/bytedance/sonic v1.15.0/go.mod h1:tFkWrPz0/CUCLEF4ri4UkHekCIcdnkqXw9VduqpJh0k= +github.com/bytedance/sonic/loader v0.5.0 h1:gXH3KVnatgY7loH5/TkeVyXPfESoqSBSBEiDd5VjlgE= +github.com/bytedance/sonic/loader v0.5.0/go.mod h1:AR4NYCk5DdzZizZ5djGqQ92eEhCCcdf5x77udYiSJRo= github.com/caarlos0/env/v9 v9.0.0 h1:SI6JNsOA+y5gj9njpgybykATIylrRMklbs5ch6wO6pc= github.com/caarlos0/env/v9 v9.0.0/go.mod h1:ye5mlCVMYh6tZ+vCgrs/B95sj88cg5Tlnc0XIzgZ020= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/charmbracelet/bubbles v0.21.0 h1:9TdC97SdRVg/1aaXNVWfFH3nnLAwOXr8Fn6u6mfQdFs= -github.com/charmbracelet/bubbles v0.21.0/go.mod h1:HF+v6QUR4HkEpz62dx7ym2xc71/KBHg+zKwJtMw+qtg= -github.com/charmbracelet/bubbletea v1.3.6 h1:VkHIxPJQeDt0aFJIsVxw8BQdh/F/L2KKZGsK6et5taU= -github.com/charmbracelet/bubbletea v1.3.6/go.mod h1:oQD9VCRQFF8KplacJLo28/jofOI2ToOfGYeFgBBxHOc= -github.com/charmbracelet/colorprofile v0.2.3-0.20250311203215-f60798e515dc h1:4pZI35227imm7yK2bGPcfpFEmuY1gc2YSTShr4iJBfs= -github.com/charmbracelet/colorprofile v0.2.3-0.20250311203215-f60798e515dc/go.mod h1:X4/0JoqgTIPSFcRA/P6INZzIuyqdFY5rm8tb41s9okk= +github.com/charmbracelet/bubbles v0.21.1 h1:nj0decPiixaZeL9diI4uzzQTkkz1kYY8+jgzCZXSmW0= +github.com/charmbracelet/bubbles v0.21.1/go.mod h1:HHvIYRCpbkCJw2yo0vNX1O5loCwSr9/mWS8GYSg50Sk= +github.com/charmbracelet/bubbletea v1.3.10 h1:otUDHWMMzQSB0Pkc87rm691KZ3SWa4KUlvF9nRvCICw= +github.com/charmbracelet/bubbletea v1.3.10/go.mod h1:ORQfo0fk8U+po9VaNvnV95UPWA1BitP1E0N6xJPlHr4= +github.com/charmbracelet/colorprofile v0.4.1 h1:a1lO03qTrSIRaK8c3JRxJDZOvhvIeSco3ej+ngLk1kk= +github.com/charmbracelet/colorprofile v0.4.1/go.mod h1:U1d9Dljmdf9DLegaJ0nGZNJvoXAhayhmidOdcBwAvKk= github.com/charmbracelet/lipgloss v1.1.0 h1:vYXsiLHVkK7fp74RkV7b2kq9+zDLoEU4MZoFqR/noCY= github.com/charmbracelet/lipgloss v1.1.0/go.mod h1:/6Q8FR2o+kj8rz4Dq0zQc3vYf7X+B0binUUBwA0aL30= -github.com/charmbracelet/x/ansi v0.9.3 h1:BXt5DHS/MKF+LjuK4huWrC6NCvHtexww7dMayh6GXd0= -github.com/charmbracelet/x/ansi v0.9.3/go.mod h1:3RQDQ6lDnROptfpWuUVIUG64bD2g2BgntdxH0Ya5TeE= -github.com/charmbracelet/x/cellbuf v0.0.13-0.20250311204145-2c3ea96c31dd h1:vy0GVL4jeHEwG5YOXDmi86oYw2yuYUGqz6a8sLwg0X8= -github.com/charmbracelet/x/cellbuf v0.0.13-0.20250311204145-2c3ea96c31dd/go.mod h1:xe0nKWGd3eJgtqZRaN9RjMtK7xUYchjzPr7q6kcvCCs= +github.com/charmbracelet/x/ansi v0.11.5 h1:NBWeBpj/lJPE3Q5l+Lusa4+mH6v7487OP8K0r1IhRg4= +github.com/charmbracelet/x/ansi v0.11.5/go.mod h1:2JNYLgQUsyqaiLovhU2Rv/pb8r6ydXKS3NIttu3VGZQ= +github.com/charmbracelet/x/cellbuf v0.0.15 h1:ur3pZy0o6z/R7EylET877CBxaiE1Sp1GMxoFPAIztPI= +github.com/charmbracelet/x/cellbuf v0.0.15/go.mod h1:J1YVbR7MUuEGIFPCaaZ96KDl5NoS0DAWkskup+mOY+Q= github.com/charmbracelet/x/exp/golden v0.0.0-20241011142426-46044092ad91 h1:payRxjMjKgx2PaCWLZ4p3ro9y97+TVLZNaRZgJwSVDQ= github.com/charmbracelet/x/exp/golden v0.0.0-20241011142426-46044092ad91/go.mod h1:wDlXFlCrmJ8J+swcL/MnGUuYnqgQdW9rhSD61oNMb6U= -github.com/charmbracelet/x/term v0.2.1 h1:AQeHeLZ1OqSXhrAWpYUtZyX1T3zVxfpZuEQMIQaGIAQ= -github.com/charmbracelet/x/term v0.2.1/go.mod h1:oQ4enTYFV7QN4m0i9mzHrViD7TQKvNEEkHUMCmsxdUg= +github.com/charmbracelet/x/term v0.2.2 h1:xVRT/S2ZcKdhhOuSP4t5cLi5o+JxklsoEObBSgfgZRk= +github.com/charmbracelet/x/term v0.2.2/go.mod h1:kF8CY5RddLWrsgVwpw4kAa6TESp6EB5y3uxGLeCqzAI= github.com/cheekybits/is v0.0.0-20150225183255-68e9c0620927 h1:SKI1/fuSdodxmNNyVBR8d7X/HuLnRpvvFO0AgyQk764= github.com/cheekybits/is v0.0.0-20150225183255-68e9c0620927/go.mod h1:h/aW8ynjgkuj+NQRlZcDbAbM1ORAbXjXX77sX7T289U= github.com/city404/v6-public-rpc-proto/go v0.0.0-20240817070657-90f8e24b653e h1:GLC8iDDcbt1H8+RkNao2nRGjyNTIo81e1rAJT9/uWYA= github.com/city404/v6-public-rpc-proto/go v0.0.0-20240817070657-90f8e24b653e/go.mod h1:ln9Whp+wVY/FTbn2SK0ag+SKD2fC0yQCF/Lqowc1LmU= +github.com/clipperhouse/displaywidth v0.9.0 h1:Qb4KOhYwRiN3viMv1v/3cTBlz3AcAZX3+y9OLhMtAtA= +github.com/clipperhouse/displaywidth v0.9.0/go.mod h1:aCAAqTlh4GIVkhQnJpbL0T/WfcrJXHcj8C0yjYcjOZA= +github.com/clipperhouse/stringish v0.1.1 h1:+NSqMOr3GR6k1FdRhhnXrLfztGzuG+VuFDfatpWHKCs= +github.com/clipperhouse/stringish v0.1.1/go.mod h1:v/WhFtE1q0ovMta2+m+UbpZ+2/HEXNWYXQgCt4hdOzA= +github.com/clipperhouse/uax29/v2 v2.7.0 h1:+gs4oBZ2gPfVrKPthwbMzWZDaAFPGYK72F0NJv2v7Vk= +github.com/clipperhouse/uax29/v2 v2.7.0/go.mod h1:EFJ2TJMRUaplDxHKj1qAEhCtQPW2tJSwu5BF98AuoVM= github.com/cloudflare/circl v1.1.0/go.mod h1:prBCrKB9DV4poKZY1l9zBXg2QJY7mvgRvtMxxK7fi4I= -github.com/cloudflare/circl v1.6.1 h1:zqIqSPIndyBh1bjLVVDHMPpVKqp8Su/V+6MeDzzQBQ0= -github.com/cloudflare/circl v1.6.1/go.mod h1:uddAzsPgqdMAYatqJ0lsjX1oECcQLIlRpzZh3pJrofs= -github.com/cloudsoda/go-smb2 v0.0.0-20250228001242-d4c70e6251cc h1:t8YjNUCt1DimB4HCIXBztwWMhgxr5yG5/YaRl9Afdfg= -github.com/cloudsoda/go-smb2 v0.0.0-20250228001242-d4c70e6251cc/go.mod h1:CgWpFCFWzzEA5hVkhAc6DZZzGd3czx+BblvOzjmg6KA= +github.com/cloudflare/circl v1.6.4 h1:pOXuDTCEYyzydgUpQ0CQz3LsinKjiSk6nNP5Lt5K64U= +github.com/cloudflare/circl v1.6.4/go.mod h1:YxarevkLlbaHuWsxG6vmYNWBEsSp4pnp7j+4VljMavY= +github.com/cloudsoda/go-smb2 v0.0.0-20260803221621-0b399b9d036c h1:7VByb9X2X3LXbk89eMavoQO9uD5dcAjY6uPZhAR9Yso= +github.com/cloudsoda/go-smb2 v0.0.0-20260803221621-0b399b9d036c/go.mod h1:1pQXB0vAlzRlqcY7LYKOOZMw0wKfJPFxTLsJRF2Gswo= github.com/cloudsoda/sddl v0.0.0-20250224235906-926454e91efc h1:0xCWmFKBmarCqqqLeM7jFBSw/Or81UEElFqO8MY+GDs= github.com/cloudsoda/sddl v0.0.0-20250224235906-926454e91efc/go.mod h1:uvR42Hb/t52HQd7x5/ZLzZEK8oihrFpgnodIJ1vte2E= -github.com/cloudwego/base64x v0.1.5 h1:XPciSp1xaq2VCSt6lF0phncD4koWyULpl5bUxbfCyP4= -github.com/cloudwego/base64x v0.1.5/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJgA0rcu/8w= -github.com/cloudwego/iasm v0.2.0/go.mod h1:8rXZaNYT2n95jn+zTI1sDr+IgcD2GVs0nlbbQPiEFhY= -github.com/coreos/go-oidc v2.3.0+incompatible h1:+5vEsrgprdLjjQ9FzIKAzQz1wwPD+83hQRfUIPh7rO0= -github.com/coreos/go-oidc v2.3.0+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc= +github.com/cloudwego/base64x v0.1.6 h1:t11wG9AECkCDk5fMSoxmufanudBtJ+/HemLstXDLI2M= +github.com/cloudwego/base64x v0.1.6/go.mod h1:OFcloc187FXDaYHvrNIjxSe8ncn0OOM8gEHfghB2IPU= +github.com/coreos/go-oidc v2.5.0+incompatible h1:6W0vGJR3Tu0r0PwfmjOrRZSlfxeEln8dsejt3ZWIvwo= +github.com/coreos/go-oidc v2.5.0+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc= github.com/coreos/go-semver v0.3.1 h1:yi21YpKnrx1gt5R+la8n5WgS0kCrsPp33dmEyHReZr4= github.com/coreos/go-semver v0.3.1/go.mod h1:irMmmIw/7yzSRPWryHsK7EYSg09caPQL03VsM8rvUec= -github.com/coreos/go-systemd/v22 v22.5.0 h1:RrqgGjYQKalulkV8NGVIfkXQf6YYmOyiJKk8iXXhfZs= -github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= +github.com/coreos/go-systemd/v22 v22.6.0 h1:aGVa/v8B7hpb0TKl0MWoAavPDmHvobFe5R5zn0bCJWo= +github.com/coreos/go-systemd/v22 v22.6.0/go.mod h1:iG+pp635Fo7ZmV/j14KUcmEyWF+0X7Lua8rrTWzYgWU= github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= github.com/crackcomm/go-gitignore v0.0.0-20170627025303-887ab5e44cc3 h1:HVTnpeuvF6Owjd5mniCL8DEXo7uYXdQEmOP4FJbV5tg= github.com/crackcomm/go-gitignore v0.0.0-20170627025303-887ab5e44cc3/go.mod h1:p1d6YEZWvFzEh4KLyvBcVSnrfNDDvK2zfK/4x2v/4pE= @@ -231,8 +246,8 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/deckarep/golang-set/v2 v2.8.0 h1:swm0rlPCmdWn9mESxKOjWk8hXSqoxOp+ZlfuyaAdFlQ= -github.com/deckarep/golang-set/v2 v2.8.0/go.mod h1:VAky9rY/yGXJOLEDv3OMci+7wtDpOF4IN+y82NBOac4= +github.com/deckarep/golang-set/v2 v2.9.0 h1:prva4eP9UysWagLyKrtn074ughi0NnkIf0A4M5yOCKI= +github.com/deckarep/golang-set/v2 v2.9.0/go.mod h1:EWknQXbs0mcFpat2QOoXV0Ee57cD+w6ZEN76BR2JVrM= github.com/decred/dcrd/crypto/blake256 v1.0.0 h1:/8DMNYp9SGi5f0w7uCm6d6M4OU2rGFK09Y2A4Xv7EE0= github.com/decred/dcrd/crypto/blake256 v1.0.0/go.mod h1:sQl2p6Y26YV+ZOcSTP6thNdn47hh8kt6rqSlvmrXFAc= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0 h1:HbphB4TFFXpv7MNrT52FGrrgVXF1owhMVTHFZIlnvd4= @@ -241,8 +256,8 @@ github.com/dhowden/tag v0.0.0-20240417053706-3d75831295e8 h1:OtSeLS5y0Uy01jaKK4m github.com/dhowden/tag v0.0.0-20240417053706-3d75831295e8/go.mod h1:apkPC/CR3s48O2D7Y++n1XWEpgPNNCjXYga3PPbJe2E= github.com/disintegration/imaging v1.6.2 h1:w1LecBlG2Lnp8B3jk5zSuNqd7b4DXhcjwek1ei82L+c= github.com/disintegration/imaging v1.6.2/go.mod h1:44/5580QXChDfwIclfc/PCwrr44amcmDAg8hxG0Ewe4= -github.com/dlclark/regexp2 v1.11.5 h1:Q/sSnsKerHeCkc/jSTNq1oCm7KiVgUMZRDUoRu0JQZQ= -github.com/dlclark/regexp2 v1.11.5/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= +github.com/dlclark/regexp2 v1.12.0 h1:0j4c5qQmnC6XOWNjP3PIXURXN2gWx76rd3KvgdPkCz8= +github.com/dlclark/regexp2 v1.12.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= github.com/dsnet/compress v0.0.2-0.20230904184137-39efe44ab707 h1:2tV76y6Q9BB+NEBasnqvs7e49aEBFI8ejC89PSnWH+4= github.com/dsnet/compress v0.0.2-0.20230904184137-39efe44ab707/go.mod h1:qssHWj60/X5sZFNxpG4HBPDHVqxNm4DfnCKgrbZOT+s= github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdfkVLjJ8T6VcRQv3SXugXy999NBtR9aFY= @@ -250,43 +265,43 @@ github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkp github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/dustinxie/ecc v0.0.0-20210511000915-959544187564 h1:I6KUy4CI6hHjqnyJLNCEi7YHVMkwwtfSr2k9splgdSM= github.com/dustinxie/ecc v0.0.0-20210511000915-959544187564/go.mod h1:yekO+3ZShy19S+bsmnERmznGy9Rfg6dWWWpiGJjNAz8= -github.com/ebitengine/purego v0.8.4 h1:CF7LEKg5FFOsASUj0+QwaXf8Ht6TlFxg09+S9wz0omw= -github.com/ebitengine/purego v0.8.4/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ= +github.com/ebitengine/purego v0.10.2 h1:W809HbnvzAxgdm+aOvlSekrM16wGCdT/e76+9tS7gzE= +github.com/ebitengine/purego v0.10.2/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ= github.com/emersion/go-message v0.18.2 h1:rl55SQdjd9oJcIoQNhubD2Acs1E6IzlZISRTK7x/Lpg= github.com/emersion/go-message v0.18.2/go.mod h1:XpJyL70LwRvq2a8rVbHXikPgKj8+aI0kGdHlg16ibYA= -github.com/emersion/go-vcard v0.0.0-20241024213814-c9703dde27ff h1:4N8wnS3f1hNHSmFD5zgFkWCyA4L1kCDkImPAtK7D6tg= -github.com/emersion/go-vcard v0.0.0-20241024213814-c9703dde27ff/go.mod h1:HMJKR5wlh/ziNp+sHEDV2ltblO4JD2+IdDOWtGcQBTM= +github.com/emersion/go-vcard v0.0.0-20260618161152-d854b7e0e2d3 h1:B9YK+Tck5mTccyDhtxBzWyqGYcFxLyB6+noMNW4/VgI= +github.com/emersion/go-vcard v0.0.0-20260618161152-d854b7e0e2d3/go.mod h1:HMJKR5wlh/ziNp+sHEDV2ltblO4JD2+IdDOWtGcQBTM= github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f h1:Y/CXytFA4m6baUTXGLOoWe4PQhGxaX0KpnayAqC48p4= github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f/go.mod h1:vw97MGsxSvLiUE2X8qFplwetxpGLQrlU1Q9AUEIzCaM= github.com/fclairamb/ftpserverlib v0.26.1-0.20250709223522-4a925d79caf6 h1:q1b+gv6AG2TDPN+f0QAkbRrAvJ3ZosnwRLTKNxSXlaA= github.com/fclairamb/ftpserverlib v0.26.1-0.20250709223522-4a925d79caf6/go.mod h1:MAsn6OKL24MLbGdCjt1t44XMGgX3sFqukYTKmTUOci8= github.com/fclairamb/go-log v0.6.0 h1:1V7BJ75P2PvanLHRyGBBFjncB6d4AgEmu+BPWKbMkaU= github.com/fclairamb/go-log v0.6.0/go.mod h1:cyXxOw4aJwO6lrZb8GRELSw+sxO6wwkLJdsjY5xYCWA= -github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= -github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/felixge/httpsnoop v1.1.0 h1:3YtUj32ZZkqZtt3sZZsClsymw/QDuVfpNhoA31zeORc= +github.com/felixge/httpsnoop v1.1.0/go.mod h1:Zqxgdd+1Rkcz8euOqdr7lqgCRJztwr5hp9vDSi5UZCE= github.com/foxxorcat/mopan-sdk-go v0.1.6 h1:6J37oI4wMZLj8EPgSCcSTTIbnI5D6RCNW/srX8vQd1Y= github.com/foxxorcat/mopan-sdk-go v0.1.6/go.mod h1:UaY6D88yBXWGrcu/PcyLWyL4lzrk5pSxSABPHftOvxs= github.com/foxxorcat/weiyun-sdk-go v0.1.4 h1:X2tFvdqikkJ7awCBbMH7XXk7+uQoJlQksJz9CUU6ZgA= github.com/foxxorcat/weiyun-sdk-go v0.1.4/go.mod h1:TPxzN0d2PahweUEHlOBWlwZSA+rELSUlGYMWgXRn9ps= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= -github.com/fxamacker/cbor/v2 v2.9.0 h1:NpKPmjDBgUfBms6tr6JZkTHtfFGcMKsw3eGcmD/sapM= -github.com/fxamacker/cbor/v2 v2.9.0/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ= -github.com/gabriel-vasile/mimetype v1.4.9 h1:5k+WDwEsD9eTLL8Tz3L0VnmVh9QxGjRmjBvAG7U/oYY= -github.com/gabriel-vasile/mimetype v1.4.9/go.mod h1:WnSQhFKJuBlRyLiKohA/2DtIlPFAbguNaG7QCHcyGok= +github.com/fxamacker/cbor/v2 v2.9.3 h1:oQBnFATpNdY8gJHTndDDv5Xl4QqNaz51G5LLEPhng3Q= +github.com/fxamacker/cbor/v2 v2.9.3/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ= +github.com/gabriel-vasile/mimetype v1.4.13 h1:46nXokslUBsAJE/wMsp5gtO500a4F3Nkz9Ufpk2AcUM= +github.com/gabriel-vasile/mimetype v1.4.13/go.mod h1:d+9Oxyo1wTzWdyVUPMmXFvp4F9tea18J8ufA774AB3s= github.com/geoffgarside/ber v1.2.0 h1:/loowoRcs/MWLYmGX9QtIAbA+V/FrnVLsMMPhwiRm64= github.com/geoffgarside/ber v1.2.0/go.mod h1:jVPKeCbj6MvQZhwLYsGwaGI52oUorHoHKNecGT85ZCc= -github.com/gin-contrib/cors v1.7.6 h1:3gQ8GMzs1Ylpf70y8bMw4fVpycXIeX1ZemuSQIsnQQY= -github.com/gin-contrib/cors v1.7.6/go.mod h1:Ulcl+xN4jel9t1Ry8vqph23a60FwH9xVLd+3ykmTjOk= +github.com/gin-contrib/cors v1.7.7 h1:Oh9joP463x7Mw72vhvJ61YQm8ODh9b04YR7vsOErD0Q= +github.com/gin-contrib/cors v1.7.7/go.mod h1:K5tW0RkzJtWSiOdikXloy8VEZlgdVNpHNw8FpjUPNrE= github.com/gin-contrib/sse v1.1.0 h1:n0w2GMuUpWDVp7qSpvze6fAu9iRxJY4Hmj6AmBOU05w= github.com/gin-contrib/sse v1.1.0/go.mod h1:hxRZ5gVpWMT7Z0B0gSNYqqsSCNIJMjzvm6fqCz9vjwM= -github.com/gin-gonic/gin v1.10.1 h1:T0ujvqyCSqRopADpgPgiTT63DUQVSfojyME59Ei63pQ= -github.com/gin-gonic/gin v1.10.1/go.mod h1:4PMNQiOhvDRa013RKVbsiNwoyezlm2rm0uX/T7kzp5Y= +github.com/gin-gonic/gin v1.12.0 h1:b3YAbrZtnf8N//yjKeU2+MQsh2mY5htkZidOM7O0wG8= +github.com/gin-gonic/gin v1.12.0/go.mod h1:VxccKfsSllpKshkBWgVgRniFFAzFb9csfngsqANjnLc= github.com/glebarez/go-sqlite v1.21.2 h1:3a6LFC4sKahUunAmynQKLZceZCOzUthkRkEAl9gAXWo= github.com/glebarez/go-sqlite v1.21.2/go.mod h1:sfxdZyhQjTM2Wry3gVYWaW072Ri1WMdWJi0k6+3382k= github.com/glebarez/sqlite v1.11.0 h1:wSG0irqzP6VurnMEpFGer5Li19RpIRi2qvQz++w0GMw= github.com/glebarez/sqlite v1.11.0/go.mod h1:h8/o8j5wiAsqSPoWELDUdJXhjAhsVliSn7bWZjOhrgQ= -github.com/go-chi/chi/v5 v5.2.2 h1:CMwsvRVTbXVytCk1Wd72Zy1LAsAh9GxMmSNWLHCG618= -github.com/go-chi/chi/v5 v5.2.2/go.mod h1:L2yAIGWB3H+phAw1NxKwWM+7eUH/lU8pOMm5hHcoops= +github.com/go-chi/chi/v5 v5.3.1 h1:3j4HZLGZQ3JpMCrPJF/Jl3mYJfWLKBfNJ6quurUGCf8= +github.com/go-chi/chi/v5 v5.3.1/go.mod h1:R+tYY2hNuVUUjxoPtqUdgBqevM9s9njzkTLutVsOCto= github.com/go-darwin/apfs v0.0.0-20211011131704-f84b94dbf348 h1:JnrjqG5iR07/8k7NqrLNilRsl3s1EPRQEGvbPyOce68= github.com/go-darwin/apfs v0.0.0-20211011131704-f84b94dbf348/go.mod h1:Czxo/d1g948LtrALAZdL04TL/HnkopquAjxYUuI02bo= github.com/go-kit/log v0.2.1 h1:MRVx0/zhvdseW+Gza6N9rVzU/IVzaeE1SFI4raAhmBU= @@ -307,39 +322,44 @@ github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/o github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= -github.com/go-playground/validator/v10 v10.26.0 h1:SP05Nqhjcvz81uJaRfEV0YBSSSGMc/iMaVtFbr3Sw2k= -github.com/go-playground/validator/v10 v10.26.0/go.mod h1:I5QpIEbmr8On7W0TktmJAumgzX4CA1XNl4ZmDuVHKKo= -github.com/go-resty/resty/v2 v2.16.5 h1:hBKqmWrr7uRc3euHVqmh1HTHcKn99Smr7o5spptdhTM= -github.com/go-resty/resty/v2 v2.16.5/go.mod h1:hkJtXbA2iKHzJheXYvQ8snQES5ZLGKMwQ07xAwp/fiA= -github.com/go-sql-driver/mysql v1.7.0 h1:ueSltNNllEqE3qcWBTD0iQd3IpL/6U+mJxLkazJ7YPc= -github.com/go-sql-driver/mysql v1.7.0/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI= -github.com/go-webauthn/webauthn v0.13.4 h1:q68qusWPcqHbg9STSxBLBHnsKaLxNO0RnVKaAqMuAuQ= -github.com/go-webauthn/webauthn v0.13.4/go.mod h1:MglN6OH9ECxvhDqoq1wMoF6P6JRYDiQpC9nc5OomQmI= -github.com/go-webauthn/x v0.1.23 h1:9lEO0s+g8iTyz5Vszlg/rXTGrx3CjcD0RZQ1GPZCaxI= -github.com/go-webauthn/x v0.1.23/go.mod h1:AJd3hI7NfEp/4fI6T4CHD753u91l510lglU7/NMN6+E= -github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4= -github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= -github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/go-playground/validator/v10 v10.30.3 h1:4MU6YkEwx7GbcPJOZxrtbu+QfF3pJLJuaYTeAH0DYy8= +github.com/go-playground/validator/v10 v10.30.3/go.mod h1:4Axh7oCNGcoGkqLoE4YWt6n20mcEIsPRlB7vPk3lpyc= +github.com/go-resty/resty/v2 v2.17.2 h1:FQW5oHYcIlkCNrMD2lloGScxcHJ0gkjshV3qcQAyHQk= +github.com/go-resty/resty/v2 v2.17.2/go.mod h1:kCKZ3wWmwJaNc7S29BRtUhJwy7iqmn+2mLtQrOyQlVA= +github.com/go-sql-driver/mysql v1.8.1 h1:LedoTUt/eveggdHS9qUFC1EFSa8bU2+1pZjSRpvNJ1Y= +github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg= +github.com/go-viper/mapstructure/v2 v2.5.0 h1:vM5IJoUAy3d7zRSVtIwQgBj7BiWtMPfmPEgAXnvj1Ro= +github.com/go-viper/mapstructure/v2 v2.5.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= +github.com/go-webauthn/webauthn v0.18.0 h1:PC8R3PNLEmjZf++WwcQlo1Z39S9rf8ma69rlwkypZhA= +github.com/go-webauthn/webauthn v0.18.0/go.mod h1:ymzZQhx3D/PrDjznemBdQJ23gHTaSDxUchM7sH1lUCg= +github.com/go-webauthn/x v0.3.0 h1:Q2X9vbrlP0Ed+QGEzixh1hthGZlDnzVT0XH/9IIQ0kE= +github.com/go-webauthn/x v0.3.0/go.mod h1:5OkdSQdOy7taRXWqvNHggtaPffmW94ybu3rZEER4I+I= +github.com/goccy/go-json v0.10.6 h1:p8HrPJzOakx/mn/bQtjgNjdTcN+/S6FcG2CTtQOrHVU= +github.com/goccy/go-json v0.10.6/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= +github.com/goccy/go-yaml v1.19.2 h1:PmFC1S6h8ljIz6gMRBopkjP1TVT7xuwrButHID66PoM= +github.com/goccy/go-yaml v1.19.2/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7LkFRi1kA= github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/golang-jwt/jwt/v4 v4.5.1/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang-jwt/jwt/v4 v4.5.2 h1:YtQM7lnr8iZ+j5q71MGKkNw9Mn7AjHM68uc9g5fXeUI= github.com/golang-jwt/jwt/v4 v4.5.2/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= -github.com/golang-jwt/jwt/v5 v5.2.3 h1:kkGXqQOBSDDWRhWNXTFpqGSCMyh/PLnqUvMGJPDJDs0= -github.com/golang-jwt/jwt/v5 v5.2.3/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= +github.com/golang-jwt/jwt/v5 v5.3.1 h1:kYf81DTWFe7t+1VvL7eS+jKFVWaUnK9cB1qbwn63YCY= +github.com/golang-jwt/jwt/v5 v5.3.1/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= -github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= -github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v1.0.0 h1:Oy607GVXHs7RtbggtPBnr2RmDArIsAefDwvrdWvRhGs= +github.com/golang/snappy v1.0.0/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= -github.com/google/go-tpm v0.9.5 h1:ocUmnDebX54dnW+MQWGQRbdaAcJELsa6PqZhJ48KwVU= -github.com/google/go-tpm v0.9.5/go.mod h1:h9jEsEECg7gtLis0upRBQU+GhYVH6jMjrFxI8u6bVUY= +github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= +github.com/google/go-tpm v0.9.8/go.mod h1:h9jEsEECg7gtLis0upRBQU+GhYVH6jMjrFxI8u6bVUY= +github.com/google/go-tpm-tools v0.3.13-0.20230620182252-4639ecce2aba h1:qJEJcuLzH5KDR0gKc0zcktin6KSAwL7+jWKBYceddTc= +github.com/google/go-tpm-tools v0.3.13-0.20230620182252-4639ecce2aba/go.mod h1:EFYHy8/1y2KfgTAsx7Luu7NGhoxtuVHnNo8jE7FikKc= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/pprof v0.0.0-20230405160723-4a4c7d95572b h1:Qcx5LM0fSiks9uCyFZwDBUasd3lxd1RM0GYpL+Li5o4= github.com/google/pprof v0.0.0-20230405160723-4a4c7d95572b/go.mod h1:79YE0hCXdHag9sBkw2o+N/YnZtTkXi0UT9Nnixa5eYk= @@ -348,10 +368,10 @@ github.com/google/s2a-go v0.1.9/go.mod h1:YA0Ei2ZQL3acow2O62kdp9UlnvMmU7kA6Eutn0 github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/googleapis/enterprise-certificate-proxy v0.3.7 h1:zrn2Ee/nWmHulBx5sAVrGgAa0f2/R35S4DJwfFaUPFQ= -github.com/googleapis/enterprise-certificate-proxy v0.3.7/go.mod h1:MkHOF77EYAE7qfSuSS9PU6g4Nt4e11cnsDUowfwewLA= -github.com/googleapis/gax-go/v2 v2.16.0 h1:iHbQmKLLZrexmb0OSsNGTeSTS0HO4YvFOG8g5E4Zd0Y= -github.com/googleapis/gax-go/v2 v2.16.0/go.mod h1:o1vfQjjNZn4+dPnRdl/4ZD7S9414Y4xA+a/6Icj6l14= +github.com/googleapis/enterprise-certificate-proxy v0.3.18 h1:hvVi34VucdrV1IIsiWuqYM8kutw/92MxNEFxCJZEh0k= +github.com/googleapis/enterprise-certificate-proxy v0.3.18/go.mod h1:rSEsBUemEBZEexP2y6jPp16LUmUbjmSbcPMQizR0o4k= +github.com/googleapis/gax-go/v2 v2.22.0 h1:PjIWBpgGIVKGoCXuiCoP64altEJCj3/Ei+kSU5vlZD4= +github.com/googleapis/gax-go/v2 v2.22.0/go.mod h1:irWBbALSr0Sk3qlqb9SyJ1h68WjgeFuiOzI4Rqw5+aY= github.com/gorilla/css v1.0.1 h1:ntNaBIghp6JmvWnxbZKANoLyuXTPZ4cAMlo6RyhlbO8= github.com/gorilla/css v1.0.1/go.mod h1:BvnYkspnSzMmwRK+b8/xgNPLiIuNZr6vbZBTPQ2A3b0= github.com/gorilla/securecookie v1.1.1 h1:miw7JPhV+b/lAHSXz4qd/nN9jRiAFV5FwjeKyCS8BvQ= @@ -362,13 +382,8 @@ github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aN github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/halalcloud/golang-sdk-lite v0.0.0-20251105081800-78cbb6786c38 h1:lsK2GVgI2Ox0NkRpQnN09GBOH7jtsjFK5tcIgxXlLr0= github.com/halalcloud/golang-sdk-lite v0.0.0-20251105081800-78cbb6786c38/go.mod h1:8x1h4rm3s8xMcTyJrq848sQ6BJnKzl57mDY4CNshdPM= -github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= -github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= -github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= -github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= -github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= github.com/hashicorp/go-uuid v1.0.2/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-uuid v1.0.3 h1:2gKiV6YVmrJ1i2CKKa9obLvRieoRGviZFL26PcT/Co8= github.com/hashicorp/go-uuid v1.0.3/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= @@ -389,20 +404,18 @@ github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/ipfs/boxo v0.12.0 h1:AXHg/1ONZdRQHQLgG5JHsSC3XoE4DjCAMgK+asZvUcQ= github.com/ipfs/boxo v0.12.0/go.mod h1:xAnfiU6PtxWCnRqu7dcXQ10bB5/kvI1kXRotuGqGBhg= -github.com/ipfs/go-cid v0.5.0 h1:goEKKhaGm0ul11IHA7I6p1GmKz8kEYniqFopaB5Otwg= -github.com/ipfs/go-cid v0.5.0/go.mod h1:0L7vmeNXpQpUS9vt+yEARkJ8rOg43DF3iPgn4GIN0mk= +github.com/ipfs/go-cid v0.6.2 h1:VuGwJd+KJTaMJ4S4d5EEf9SXc17YUblS5axCbocn9YE= +github.com/ipfs/go-cid v0.6.2/go.mod h1:Xhwg8NzHeK9xPCEZkCw4idzPiuNMpX3fARuI5Iwj1Lo= github.com/ipfs/go-ipfs-api v0.7.0 h1:CMBNCUl0b45coC+lQCXEVpMhwoqjiaCwUIrM+coYW2Q= github.com/ipfs/go-ipfs-api v0.7.0/go.mod h1:AIxsTNB0+ZhkqIfTZpdZ0VR/cpX5zrXjATa3prSay3g= -github.com/itsHenry35/gofakes3 v0.0.8 h1:1AgOl04IgoUV5r/WSK7ycnvwfpgharYLfVTmnzk5miw= -github.com/itsHenry35/gofakes3 v0.0.8/go.mod h1:gQwOJ7LoH5QSpCVmjzC6oKp+MS71utLS7GHtonsvD0c= github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= -github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a h1:bbPeKD0xmW/Y25WS6cokEszi5g+S0QxI/d45PkRi7Nk= -github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= -github.com/jackc/pgx/v5 v5.5.5 h1:amBjrZVmksIdNjxGW/IiIMzxMKZFelXbUoPNb+8sjQw= -github.com/jackc/pgx/v5 v5.5.5/go.mod h1:ez9gk+OAat140fv9ErkZDYFWmXLfV+++K0uAOiwgm1A= -github.com/jackc/puddle/v2 v2.2.1 h1:RhxXJtFG022u4ibrCSMSiu5aOq1i77R3OHKNJj77OAk= -github.com/jackc/puddle/v2 v2.2.1/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= +github.com/jackc/pgx/v5 v5.10.0 h1:VhSvgU2jSli8o3AqIEOTJr7rZwAEUVo4E4XhR94Zfr0= +github.com/jackc/pgx/v5 v5.10.0/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4= +github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= +github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= github.com/jcmturner/aescts/v2 v2.0.0 h1:9YKLH6ey7H4eDBXW8khjYslgyqG2xZikXP0EQFKrle8= github.com/jcmturner/aescts/v2 v2.0.0/go.mod h1:AiaICIRyfYg35RUkr8yESTqvSy7csK90qZ5xfvvsoNs= github.com/jcmturner/dnsutils/v2 v2.0.0 h1:lltnkeZGL0wILNvrNiVCR6Ro5PGU/SeBvVO/8c/iPbo= @@ -419,8 +432,8 @@ github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ= github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= -github.com/jlaffaye/ftp v0.2.1-0.20240918233326-1b970516f5d3 h1:ZxO6Qr2GOXPdcW80Mcn3nemvilMPvpWqxrNfK2ZnNNs= -github.com/jlaffaye/ftp v0.2.1-0.20240918233326-1b970516f5d3/go.mod h1:dvLUr/8Fs9a2OBrEnCC5duphbkz/k/mSy5OkXg3PAgI= +github.com/jlaffaye/ftp v0.2.4 h1:JqI85DdkfZj8ntaHk8W9U2SC3jNfiPUU70+wtIWmlfE= +github.com/jlaffaye/ftp v0.2.4/go.mod h1:Y1ZnkzxownGIuX7xQ1mQzzkZ21+DbjVIyeKL/V+IIz4= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= @@ -437,15 +450,14 @@ github.com/kdomanski/iso9660 v0.4.0/go.mod h1:OxUSupHsO9ceI8lBLPJKWBTphLemjrCQY8 github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= -github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= -github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= +github.com/klauspost/compress v1.19.0 h1:sXLILfc9jV2QYWkzFOPWStmcUVH2RHEB1JCdY2oVvCQ= +github.com/klauspost/compress v1.19.0/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= -github.com/klauspost/cpuid/v2 v2.2.10 h1:tBs3QSyvjDyFTq3uoc/9xFpCuOsJQFNPiAhYdw2skhE= -github.com/klauspost/cpuid/v2 v2.2.10/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= +github.com/klauspost/cpuid/v2 v2.4.0 h1:S6Hrbc7+ywsr0r+RLapfGBHfyefhCTwEh3A0tV913Dw= +github.com/klauspost/cpuid/v2 v2.4.0/go.mod h1:19jmZ9mjzoF//ddRSUsv0zfBTJWh3QJh9FNxZTMrGxU= github.com/klauspost/pgzip v1.2.6 h1:8RXeL5crjEUFnR2/Sn6GJNWtSQ3Dk8pq4CL3jvdDyjU= github.com/klauspost/pgzip v1.2.6/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= -github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M= github.com/kr/fs v0.1.0 h1:Jskdu9ieNAYnjxsi0LbQp1ulIKZV1LAFgK1tWhpZgl8= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= @@ -454,8 +466,8 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= -github.com/lanrat/extsort v1.0.2 h1:p3MLVpQEPwEGPzeLBb+1eSErzRl6Bgjgr+qnIs2RxrU= -github.com/lanrat/extsort v1.0.2/go.mod h1:ivzsdLm8Tv+88qbdpMElV6Z15StlzPUtZSKsGb51hnQ= +github.com/lanrat/extsort v1.4.2 h1:akbLIdo4PhNZtvjpaWnbXtGMmLtnGzXplkzfgl+XTTY= +github.com/lanrat/extsort v1.4.2/go.mod h1:hceP6kxKPKebjN1RVrDBXMXXECbaI41Y94tt6MDazc4= github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= github.com/libp2p/go-buffer-pool v0.1.0 h1:oK4mSFcQz7cTQIfqbe4MIj9gLW+mnanjyFtc6cdF0Y8= @@ -464,44 +476,43 @@ github.com/libp2p/go-flow-metrics v0.1.0 h1:0iPhMI8PskQwzh57jB9WxIuIOQ0r+15PChFG github.com/libp2p/go-flow-metrics v0.1.0/go.mod h1:4Xi8MX8wj5aWNDAZttg6UPmc0ZrnFNsMtpsYUClFtro= github.com/libp2p/go-libp2p v0.27.8 h1:IX5x/4yKwyPQeVS2AXHZ3J4YATM9oHBGH1gBc23jBAI= github.com/libp2p/go-libp2p v0.27.8/go.mod h1:eCFFtd0s5i/EVKR7+5Ki8bM7qwkNW3TPTTSSW9sz8NE= -github.com/lucasb-eyer/go-colorful v1.2.0 h1:1nnpGOrhyZZuNyfu1QjKiUICQ74+3FNCN69Aj6K7nkY= -github.com/lucasb-eyer/go-colorful v1.2.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0= -github.com/lufia/plan9stats v0.0.0-20250317134145-8bc96cf8fc35 h1:PpXWgLPs+Fqr325bN2FD2ISlRRztXibcX6e8f5FR5Dc= -github.com/lufia/plan9stats v0.0.0-20250317134145-8bc96cf8fc35/go.mod h1:autxFIvghDt3jPTLoqZ9OZ7s9qTGNAWmYCjVFWPX/zg= -github.com/mailru/easyjson v0.9.0 h1:PrnmzHw7262yW8sTBwxi1PdJA3Iw/EKBa8psRf7d9a4= +github.com/lucasb-eyer/go-colorful v1.4.0 h1:UtrWVfLdarDgc44HcS7pYloGHJUjHV/4FwW4TvVgFr4= +github.com/lucasb-eyer/go-colorful v1.4.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0= +github.com/lufia/plan9stats v0.0.0-20260627054121-477a66015f15 h1:YkjVPl/YH5XlJ+/NiwzJtPYXXKRcyjmEUhsDci6YK3c= +github.com/lufia/plan9stats v0.0.0-20260627054121-477a66015f15/go.mod h1:autxFIvghDt3jPTLoqZ9OZ7s9qTGNAWmYCjVFWPX/zg= github.com/mailru/easyjson v0.9.0/go.mod h1:1+xMtQp2MRNVL/V1bOzuP3aP8VNwRW55fQUto+XFtTU= -github.com/maruel/natural v1.1.1 h1:Hja7XhhmvEFhcByqDoHz9QZbkWey+COd9xWfCfn1ioo= -github.com/maruel/natural v1.1.1/go.mod h1:v+Rfd79xlw1AgVBjbO0BEQmptqb5HvL/k9GRHB7ZKEg= +github.com/mailru/easyjson v0.9.2 h1:dX8U45hQsZpxd80nLvDGihsQ/OxlvTkVUXH2r/8cb2M= +github.com/mailru/easyjson v0.9.2/go.mod h1:1+xMtQp2MRNVL/V1bOzuP3aP8VNwRW55fQUto+XFtTU= +github.com/maruel/natural v1.3.0 h1:VsmCsBmEyrR46RomtgHs5hbKADGRVtliHTyCOLFBpsg= +github.com/maruel/natural v1.3.0/go.mod h1:v+Rfd79xlw1AgVBjbO0BEQmptqb5HvL/k9GRHB7ZKEg= github.com/matoous/go-nanoid/v2 v2.1.0 h1:P64+dmq21hhWdtvZfEAofnvJULaRR1Yib0+PnU669bE= github.com/matoous/go-nanoid/v2 v2.1.0/go.mod h1:KlbGNQ+FhrUNIHUxZdL63t7tl4LaPkZNpUULS8H4uVM= -github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE= -github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8= -github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= -github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-colorable v0.1.15 h1:+u9SLTRGnXv73cEsnsmoZBom+dMU88B2M0aDcWy0/jY= +github.com/mattn/go-colorable v0.1.15/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8= +github.com/mattn/go-isatty v0.0.23 h1:cYwCQTQf3HB6xUC+BtyCLZNr7IzbOmoZbmssVNzSyiQ= +github.com/mattn/go-isatty v0.0.23/go.mod h1:nMCL3Zebbrt45jsMDgnfIwz6ydEQApk5oEI3HqDio6A= github.com/mattn/go-localereader v0.0.1 h1:ygSAOl7ZXTx4RdPYinUpg6W99U8jWvWi9Ye2JC/oIi4= github.com/mattn/go-localereader v0.0.1/go.mod h1:8fBrzywKY7BI3czFoHkuzRoWE9C+EiG4R1k4Cjx5p88= -github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc= -github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= +github.com/mattn/go-runewidth v0.0.24 h1:cpokDiIn0MGnhdHwuWnJBITySJ20QyNGnY2kR/ay2DU= +github.com/mattn/go-runewidth v0.0.24/go.mod h1:XBkDxAl56ILZc9knddidhrOlY5R/pDhgLpndooCuJAs= github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU= github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= github.com/meilisearch/meilisearch-go v0.32.0 h1:cWcycpONSH3VLTZ5npUl1O5aXPkNM0vUx6bywnYqGbE= github.com/meilisearch/meilisearch-go v0.32.0/go.mod h1:aNtyuwurDg/ggxQIcKqWH6G9g2ptc8GyY7PLY4zMn/g= -github.com/mholt/archives v0.1.3 h1:aEAaOtNra78G+TvV5ohmXrJOAzf++dIlYeDW3N9q458= -github.com/mholt/archives v0.1.3/go.mod h1:LUCGp++/IbV/I0Xq4SzcIR6uwgeh2yjnQWamjRQfLTU= +github.com/mholt/archives v0.1.5 h1:Fh2hl1j7VEhc6DZs2DLMgiBNChUux154a1G+2esNvzQ= +github.com/mholt/archives v0.1.5/go.mod h1:3TPMmBLPsgszL+1As5zECTuKwKvIfj6YcwWPpeTAXF4= github.com/microcosm-cc/bluemonday v1.0.27 h1:MpEUotklkwCSLeH+Qdx1VJgNqLlpY2KXwXFM08ygZfk= github.com/microcosm-cc/bluemonday v1.0.27/go.mod h1:jFi9vgW+H7c3V0lb6nR74Ib/DIB5OBs92Dimizgw2cA= github.com/mikelolasagasti/xz v1.0.1 h1:Q2F2jX0RYJUG3+WsM+FJknv+6eVjsjXNDV0KJXZzkD0= github.com/mikelolasagasti/xz v1.0.1/go.mod h1:muAirjiOUxPRXwm9HdDtB3uoRPrGnL85XHtokL9Hcgc= -github.com/minio/minlz v1.0.0 h1:Kj7aJZ1//LlTP1DM8Jm7lNKvvJS2m74gyyXXn3+uJWQ= -github.com/minio/minlz v1.0.0/go.mod h1:qT0aEB35q79LLornSzeDH75LBf3aH1MV+jB5w9Wasec= +github.com/minio/minlz v1.2.0 h1:6IOBuiHg04QxvbFfgFLT/9sMaO/UhL7S+ApW1mK8q5A= +github.com/minio/minlz v1.2.0/go.mod h1:Ls9H7nlkASeCcdl5thjVD5Eraj6z+zGa7xtq57jIKD4= github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM= github.com/minio/sha256-simd v1.0.1/go.mod h1:Pz6AKMiUdngCLpeTL/RJY1M9rUuPMYujV5xJjtbRSN8= github.com/minio/xxml v0.0.3 h1:ZIpPQpfyG5uZQnqqC0LZuWtPk/WT8G/qkxvO6jb7zMU= github.com/minio/xxml v0.0.3/go.mod h1:wcXErosl6IezQIMEWSK/LYC2VS7LJ1dAkgvuyIN3aH4= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= -github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -509,8 +520,8 @@ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lN github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= -github.com/mr-tron/base58 v1.2.0 h1:T/HDJBh4ZCPbU39/+c3rRvE0uKBQlU27+QI8LJ4t64o= -github.com/mr-tron/base58 v1.2.0/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= +github.com/mr-tron/base58 v1.3.0 h1:K6Y13R2h+dku0wOqKtecgRnBUBPrZzLZy5aIj8lCcJI= +github.com/mr-tron/base58 v1.3.0/go.mod h1:2BuubE67DCSWwVfx37JWNG8emOC0sHEU4/HpcYgCLX8= github.com/mschoch/smat v0.2.0 h1:8imxQsjDm8yFEAVBe7azKmKSgzSkZXDuKkSq9374khM= github.com/mschoch/smat v0.2.0/go.mod h1:kc9mz7DoBKqDyiRL7VZN8KvXQMWeTaVnttLRXOlotKw= github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6 h1:ZK8zHtRHOkbHy6Mmr5D264iyp3TiX5OmNcI5cIARiQI= @@ -525,38 +536,42 @@ github.com/multiformats/go-base36 v0.2.0 h1:lFsAbNOGeKtuKozrtBsAkSVhv1p9D0/qedU9 github.com/multiformats/go-base36 v0.2.0/go.mod h1:qvnKE++v+2MWCfePClUEjE78Z7P2a1UV0xHgWc0hkp4= github.com/multiformats/go-multiaddr v0.9.0 h1:3h4V1LHIk5w4hJHekMKWALPXErDfz/sggzwC/NcqbDQ= github.com/multiformats/go-multiaddr v0.9.0/go.mod h1:mI67Lb1EeTOYb8GQfL/7wpIZwc46ElrvzhYnoJOmTT0= -github.com/multiformats/go-multibase v0.2.0 h1:isdYCVLvksgWlMW9OZRYJEa9pZETFivncJHmHnnd87g= -github.com/multiformats/go-multibase v0.2.0/go.mod h1:bFBZX4lKCA/2lyOFSAoKH5SS6oPyjtnzK/XTFDPkNuk= +github.com/multiformats/go-multibase v0.3.0 h1:8helZD2+4Db7NNWFiktk2NePbF0boolBe6bDQvM4r68= +github.com/multiformats/go-multibase v0.3.0/go.mod h1:MoBLQPCkRTOL3eveIPO81860j2AQY8JwcnNlRkGRUfI= github.com/multiformats/go-multicodec v0.9.0 h1:pb/dlPnzee/Sxv/j4PmkDRxCOi3hXTz3IbPKOXWJkmg= github.com/multiformats/go-multicodec v0.9.0/go.mod h1:L3QTQvMIaVBkXOXXtVmYE+LI16i14xuaojr/H7Ai54k= github.com/multiformats/go-multihash v0.2.3 h1:7Lyc8XfX/IY2jWb/gI7JP+o7JEq9hOa7BFvVU9RSh+U= github.com/multiformats/go-multihash v0.2.3/go.mod h1:dXgKXCXjBzdscBLk9JkjINiEsCKRVch90MdaGiKsvSM= github.com/multiformats/go-multistream v0.4.1 h1:rFy0Iiyn3YT0asivDUIR05leAdwZq3de4741sbiSdfo= github.com/multiformats/go-multistream v0.4.1/go.mod h1:Mz5eykRVAjJWckE2U78c6xqdtyNUEhKSM0Lwar2p77Q= -github.com/multiformats/go-varint v0.0.7 h1:sWSGR+f/eu5ABZA2ZpYKBILXTTs9JWpdEM/nEGOHFS8= -github.com/multiformats/go-varint v0.0.7/go.mod h1:r8PUYw/fD/SjBCiKOoDlGF6QawOELpZAu9eioSos/OU= +github.com/multiformats/go-varint v0.1.0 h1:i2wqFp4sdl3IcIxfAonHQV9qU5OsZ4Ts9IOoETFs5dI= +github.com/multiformats/go-varint v0.1.0/go.mod h1:5KVAVXegtfmNQQm/lCY+ATvDzvJJhSkUlGQV9wgObdI= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/natefinch/lumberjack v2.0.0+incompatible h1:4QJd3OLAMgj7ph+yZTuX13Ld4UpgHp07nNdFX7mqFfM= github.com/natefinch/lumberjack v2.0.0+incompatible/go.mod h1:Wi9p2TTF5DG5oU+6YfsmYQpsTIOm0B1VNzQg9Mw6nPk= -github.com/ncw/swift/v2 v2.0.4 h1:hHWVFxn5/YaTWAASmn4qyq2p6OyP/Hm3vMLzkjEqR7w= -github.com/ncw/swift/v2 v2.0.4/go.mod h1:cbAO76/ZwcFrFlHdXPjaqWZ9R7Hdar7HpjRXBfbjigk= -github.com/nwaples/rardecode/v2 v2.1.1 h1:OJaYalXdliBUXPmC8CZGQ7oZDxzX1/5mQmgn0/GASew= -github.com/nwaples/rardecode/v2 v2.1.1/go.mod h1:7uz379lSxPe6j9nvzxUZ+n7mnJNgjsRNb6IbvGVHRmw= +github.com/ncw/swift/v2 v2.0.5 h1:9o5Gsd7bInAFEqsGPcaUdsboMbqf8lnNtxqWKFT9iz8= +github.com/ncw/swift/v2 v2.0.5/go.mod h1:cbAO76/ZwcFrFlHdXPjaqWZ9R7Hdar7HpjRXBfbjigk= +github.com/nwaples/rardecode/v2 v2.4.1 h1:F7zNW2LdAuuBThHWXQaiFUGVD/sef299NfWSB1nHAl4= +github.com/nwaples/rardecode/v2 v2.4.1/go.mod h1:7uz379lSxPe6j9nvzxUZ+n7mnJNgjsRNb6IbvGVHRmw= github.com/panjf2000/ants/v2 v2.4.2/go.mod h1:f6F0NZVFsGCp5A7QW/Zj/m92atWwOkY0OIhFxRNFr4A= github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4= github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY= -github.com/pierrec/lz4/v4 v4.1.22 h1:cKFw6uJDK+/gfw5BcDL0JL5aBsAFdsIT18eRtLj7VIU= -github.com/pierrec/lz4/v4 v4.1.22/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= +github.com/peterh/liner v1.2.2 h1:aJ4AOodmL+JxOZZEL2u9iJf8omNRpqHc/EbrK+3mAXw= +github.com/peterh/liner v1.2.2/go.mod h1:xFwJyiKIXJZUKItq5dGHZSTBRAuG/CpeNpWLyiNRNwI= +github.com/philhofer/fwd v1.2.0 h1:e6DnBTl7vGY+Gz322/ASL4Gyp1FspeMvx1RNDoToZuM= +github.com/philhofer/fwd v1.2.0/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM= +github.com/pierrec/lz4/v4 v4.1.27 h1:+PhzhWDrjRj89TH2sw43nE3+4+W8lSxIuQadEHZyjUk= +github.com/pierrec/lz4/v4 v4.1.27/go.mod h1:EoQMVJgeeEOMsCqCzqFm2O0cJvljX2nGZjcRIPL34O4= github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ= github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/sftp v1.13.9 h1:4NGkvGudBL7GteO3m6qnaQ4pC0Kvf0onSVc9gR3EWBw= -github.com/pkg/sftp v1.13.9/go.mod h1:OBN7bVXdstkFFN/gdnHPUb5TE8eb8G1Rp9wCItqjkkA= -github.com/pkg/xattr v0.4.10 h1:Qe0mtiNFHQZ296vRgUjRCoPHPqH7VdTOrZx3g0T+pGA= -github.com/pkg/xattr v0.4.10/go.mod h1:di8WF84zAKk8jzR1UBTEWh9AUlIZZ7M/JNt8e9B6ktU= +github.com/pkg/sftp v1.13.11 h1:0N92SLTB8JqASJB14ZLHHzFnBV8mG9zw4K7jghEFWuE= +github.com/pkg/sftp v1.13.11/go.mod h1:uNkH9roSXglNJqM+glJJi+TQXQUm0fXFWqCFmT8hsN0= +github.com/pkg/xattr v0.4.12 h1:rRTkSyFNTRElv6pkA3zpjHpQ90p/OdHQC1GmGh1aTjM= +github.com/pkg/xattr v0.4.12/go.mod h1:di8WF84zAKk8jzR1UBTEWh9AUlIZZ7M/JNt8e9B6ktU= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= @@ -566,32 +581,33 @@ github.com/pquerna/cachecontrol v0.1.0 h1:yJMy84ti9h/+OEWa752kBTKv4XC30OtVVHYv/8 github.com/pquerna/cachecontrol v0.1.0/go.mod h1:NrUG3Z7Rdu85UNR3vm7SOsl1nFIeSiQnrHV5K9mBcUI= github.com/pquerna/otp v1.5.0 h1:NMMR+WrmaqXU4EzdGJEE1aUUI0AMRzsp96fFFWNPwxs= github.com/pquerna/otp v1.5.0/go.mod h1:dkJfzwRKNiegxyNb54X/3fLwhCynbMspSyWKnvi1AEg= -github.com/prometheus/client_golang v1.22.0 h1:rb93p9lokFEsctTys46VnV1kLCDpVZ0a/Y92Vm0Zc6Q= -github.com/prometheus/client_golang v1.22.0/go.mod h1:R7ljNsLXhuQXYZYtw6GAE9AZg8Y7vEW5scdCXrWRXC0= +github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o= +github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg= github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= -github.com/prometheus/common v0.64.0 h1:pdZeA+g617P7oGv1CzdTzyeShxAGrTBsolKNOLQPGO4= -github.com/prometheus/common v0.64.0/go.mod h1:0gZns+BLRQ3V6NdaerOhMbwwRbNh9hkGINtQAsP5GS8= -github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzMyRg= -github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is= -github.com/quic-go/qpack v0.5.1 h1:giqksBPnT/HDtZ6VhtFKgoLOWmlyo9Ei6u9PqzIMbhI= -github.com/quic-go/qpack v0.5.1/go.mod h1:+PC4XFrEskIVkcLzpEkbLqq1uCoxPhQuvK5rH1ZgaEg= -github.com/quic-go/quic-go v0.54.1 h1:4ZAWm0AhCb6+hE+l5Q1NAL0iRn/ZrMwqHRGQiFwj2eg= -github.com/quic-go/quic-go v0.54.1/go.mod h1:e68ZEaCdyviluZmy44P6Iey98v/Wfz6HCjQEm+l8zTY= -github.com/rclone/rclone v1.70.3 h1:rg/WNh4DmSVZyKP2tHZ4lAaWEyMi7h/F0r7smOMA3IE= -github.com/rclone/rclone v1.70.3/go.mod h1:nLyN+hpxAsQn9Rgt5kM774lcRDad82x/KqQeBZ83cMo= -github.com/relvacode/iso8601 v1.6.0 h1:eFXUhMJN3Gz8Rcq82f9DTMW0svjtAVuIEULglM7QHTU= -github.com/relvacode/iso8601 v1.6.0/go.mod h1:FlNp+jz+TXpyRqgmM7tnzHHzBnz776kmAH2h3sZCn0I= +github.com/prometheus/common v0.70.0 h1:bcpru3tWPVnxGnETLgOV5jbp/JRXgYEyv65CuBLAMMI= +github.com/prometheus/common v0.70.0/go.mod h1:S/SFasQmgGiYH6C81LKCtYa8QACgthGg5zxL2udV7SY= +github.com/prometheus/procfs v0.21.1 h1:GljZCt+zSTS+NZq88cyQ1LjZ+RCHp3uVuabBWA5+OJI= +github.com/prometheus/procfs v0.21.1/go.mod h1:aB55Cww9pdSJVHk0hUf0inxWyyjPogFIjmHKYgMKmtY= +github.com/quic-go/go-ossfuzz-seeds v0.1.0 h1:APacT+iIaNF6fd8AGEiN3bT/Jtkd2jz4v4TzM7MFjy0= +github.com/quic-go/go-ossfuzz-seeds v0.1.0/go.mod h1:3IOHRbJIc+L6YKMwfDtJAM9Vj9k0YY4muhuyUYk5tbk= +github.com/quic-go/qpack v0.6.0 h1:g7W+BMYynC1LbYLSqRt8PBg5Tgwxn214ZZR34VIOjz8= +github.com/quic-go/qpack v0.6.0/go.mod h1:lUpLKChi8njB4ty2bFLX2x4gzDqXwUpaO1DP9qMDZII= +github.com/quic-go/quic-go v0.61.0 h1:ui88A53s8MSVYLC56en0KQ17HARk+9986Dn0SBfKNvA= +github.com/quic-go/quic-go v0.61.0/go.mod h1:9So2anK4Tp22URSQq00k+Vo2PNkle96ycDPDHL4s9vs= +github.com/rclone/rclone v1.75.1 h1:kIxQcoDLj2Gke/gMSHK7OnxhX1Gu1cJBLP1kJZoaFp0= +github.com/rclone/rclone v1.75.1/go.mod h1:4zmMjGatCkSJPRZDpo+7y3xOl8S29EMUyKvZop5mHr4= +github.com/relvacode/iso8601 v1.7.0 h1:BXy+V60stMP6cpswc+a93Mq3e65PfXCgDFfhvNNGrdo= +github.com/relvacode/iso8601 v1.7.0/go.mod h1:FlNp+jz+TXpyRqgmM7tnzHHzBnz776kmAH2h3sZCn0I= github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= -github.com/rfjakob/eme v1.1.2 h1:SxziR8msSOElPayZNFfQw4Tjx/Sbaeeh3eRvrHVMUs4= -github.com/rfjakob/eme v1.1.2/go.mod h1:cVvpasglm/G3ngEfcfT/Wt0GwhkuO32pf/poW6Nyk1k= -github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= +github.com/rfjakob/eme v1.2.0 h1:8dAHL+WVAw06+7DkRKnRiFp1JL3QjcJEZFqDnndUaSI= +github.com/rfjakob/eme v1.2.0/go.mod h1:cVvpasglm/G3ngEfcfT/Wt0GwhkuO32pf/poW6Nyk1k= github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= -github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= -github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= +github.com/rogpeppe/go-internal v1.15.0 h1:D0RCU5rMAp+SpgkiNdrjfJ+LX4J1M32V2NeCY7EJ6hc= +github.com/rogpeppe/go-internal v1.15.0/go.mod h1:DrUVZyrJU+txYW5/1kwtXQSMFio52ZOxX7yM1VHvnxs= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd h1:CmH9+J6ZSsIjUK3dcGsnCnO41eRBOnY12zwkn5qVwgc= github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd/go.mod h1:hPqNNc0+uJM6H+SuU8sEs5K5IQeKccPqeSjfgcKGgPk= @@ -601,16 +617,16 @@ github.com/secsy/goftp v0.0.0-20200609142545-aa2de14babf4 h1:PT+ElG/UUFMfqy5HrxJ github.com/secsy/goftp v0.0.0-20200609142545-aa2de14babf4/go.mod h1:MnkX001NG75g3p8bhFycnyIjeQoOjGL6CEIsdE/nKSY= github.com/shabbyrobe/gocovmerge v0.0.0-20230507112040-c3350d9342df h1:S77Pf5fIGMa7oSwp8SQPp7Hb4ZiI38K3RNBKD2LLeEM= github.com/shabbyrobe/gocovmerge v0.0.0-20230507112040-c3350d9342df/go.mod h1:dcuzJZ83w/SqN9k4eQqwKYMgmKWzg/KzJAURBhRL1tc= -github.com/shirou/gopsutil/v4 v4.25.5 h1:rtd9piuSMGeU8g1RMXjZs9y9luK5BwtnG7dZaQUJAsc= -github.com/shirou/gopsutil/v4 v4.25.5/go.mod h1:PfybzyydfZcN+JMMjkF6Zb8Mq1A/VcogFFg7hj50W9c= -github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= -github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/shirou/gopsutil/v4 v4.26.8 h1:YQMTF/1J50B5+Y0vlo1eDRf5DoR7Gk69hY+8wjYkQeo= +github.com/shirou/gopsutil/v4 v4.26.8/go.mod h1:5O9FjBiXoTDFatIWjZZosqj4pV0DRtLx598xGbBehzM= +github.com/sirupsen/logrus v1.10.2 h1:G2SED73/qrAu6YwbdxOD6peLkCBI3z7L+ykJFTXJBBo= +github.com/sirupsen/logrus v1.10.2/go.mod h1:SLEg8TqYulVKKfIGHldVp2K2aYz2DKSVBq4g/H5bR7Q= github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e h1:MRM5ITcdelLK2j1vwZ3Je0FKVCfqOLp5zO6trqMLYs0= github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e/go.mod h1:XV66xRDqSt+GTGFMVlhk3ULuV0y9ZmzeVGR4mloJI3M= github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966 h1:JIAuq3EEf9cgbU6AtGPK4CTG3Zf6CKMNqf0MHTggAUA= github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966/go.mod h1:sUM3LWHvSMaG192sy56D9F7CNvL7jUJVXoqM1QKLnog= -github.com/sorairolake/lzip-go v0.3.5 h1:ms5Xri9o1JBIWvOFAorYtUNik6HI3HgBTkISiqu0Cwg= -github.com/sorairolake/lzip-go v0.3.5/go.mod h1:N0KYq5iWrMXI0ZEXKXaS9hCyOjZUQdBDEIbXfoUwbdk= +github.com/sorairolake/lzip-go v0.3.8 h1:j5Q2313INdTA80ureWYRhX+1K78mUXfMoPZCw/ivWik= +github.com/sorairolake/lzip-go v0.3.8/go.mod h1:JcBqGMV0frlxwrsE9sMWXDjqn3EeVf0/54YPsw66qkU= github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= @@ -621,11 +637,14 @@ github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiT github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk= github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stangelandcl/ppmd v0.1.1 h1:c25QazhlWUn5nmR1QOzafKhQxBicAr7GGCKER2aJ8H8= +github.com/stangelandcl/ppmd v0.1.1/go.mod h1:Rrv7M+/2P5jYr/GMLhBl7Ug3uJ1bUiVzr5LbbaV6xgY= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= -github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= +github.com/stretchr/objx v0.5.3 h1:jmXUvGomnU1o3W/V5h2VEradbpJDwGrzugQQvL0POH4= +github.com/stretchr/objx v0.5.3/go.mod h1:rDQraq+vQZU7Fde9LOZLr8Tax6zZvy4kuNKF+QYS+U0= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= @@ -635,72 +654,88 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= -github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= -github.com/t3rm1n4l/go-mega v0.0.0-20241213151442-a19cff0ec7b5 h1:Sa+sR8aaAMFwxhXWENEnE6ZpqhZ9d7u1RT2722Rw6hc= -github.com/t3rm1n4l/go-mega v0.0.0-20241213151442-a19cff0ec7b5/go.mod h1:UdZiFUFu6e2WjjtjxivwXWcwc1N/8zgbkBR9QNucUOY= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE= +github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg= +github.com/t3rm1n4l/go-mega v0.0.0-20260717075258-c6acd6a5bd04 h1:s30A8dMuZ55lUOi5xUTh1hlfLqHpVBsRMfpk2iKCayk= +github.com/t3rm1n4l/go-mega v0.0.0-20260717075258-c6acd6a5bd04/go.mod h1:BF/l2jNyK+2h/BJZ7VLMAz6m/IWjA2F67gTjV1C/+Bo= github.com/taruti/bytepool v0.0.0-20160310082835-5e3a9ea56543 h1:6Y51mutOvRGRx6KqyMNo//xk8B8o6zW9/RVmy1VamOs= github.com/taruti/bytepool v0.0.0-20160310082835-5e3a9ea56543/go.mod h1:jpwqYA8KUVEvSUJHkCXsnBRJCSKP1BMa81QZ6kvRpow= github.com/tchap/go-patricia/v2 v2.3.3 h1:xfNEsODumaEcCcY3gI0hYPZ/PcpVv5ju6RMAhgwZDDc= github.com/tchap/go-patricia/v2 v2.3.3/go.mod h1:VZRHKAb53DLaG+nA9EaYYiaEx6YztwDlLElMsnSHD4k= -github.com/tklauser/go-sysconf v0.3.15 h1:VE89k0criAymJ/Os65CSn1IXaol+1wrsFHEB8Ol49K4= -github.com/tklauser/go-sysconf v0.3.15/go.mod h1:Dmjwr6tYFIseJw7a3dRLJfsHAMXZ3nEnL/aZY+0IuI4= -github.com/tklauser/numcpus v0.10.0 h1:18njr6LDBk1zuna922MgdjQuJFjrdppsZG60sHGfjso= -github.com/tklauser/numcpus v0.10.0/go.mod h1:BiTKazU708GQTYF4mB+cmlpT2Is1gLk7XVuEeem8LsQ= +github.com/tinylib/msgp v1.6.4 h1:mOwYbyYDLPj35mkA2BjjYejgJk9BuHxDdvRnb6v2ZcQ= +github.com/tinylib/msgp v1.6.4/go.mod h1:RSp0LW9oSxFut3KzESt5Voq4GVWyS+PSulT77roAqEA= +github.com/tklauser/go-sysconf v0.4.0 h1:7H0uAN+7RkwWRaxhYXDLqa5V3LPrJeV8wmD9dRUgPQU= +github.com/tklauser/go-sysconf v0.4.0/go.mod h1:8mTNWyog7H+MpKijp4VmKJAd2bbYQ2zuUwkYRbUArPI= +github.com/tklauser/numcpus v0.12.0 h1:NR85qdvHA9pFse3x3weVZ0r0ST8R6l5RHbZrlRaqob4= +github.com/tklauser/numcpus v0.12.0/go.mod h1:ABHeXzJnr/qqwguhClkZKT1/8VABcYrsyUiUGobwWJg= github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= github.com/u2takey/ffmpeg-go v0.5.0 h1:r7d86XuL7uLWJ5mzSeQ03uvjfIhiJYvsRAJFCW4uklU= github.com/u2takey/ffmpeg-go v0.5.0/go.mod h1:ruZWkvC1FEiUNjmROowOAps3ZcWxEiOpFoHCvk97kGc= github.com/u2takey/go-utils v0.3.1 h1:TaQTgmEZZeDHQFYfd+AdUT1cT4QJgJn/XVPELhHw4ys= github.com/u2takey/go-utils v0.3.1/go.mod h1:6e+v5vEZ/6gu12w/DC2ixZdZtCrNokVxD0JUklcqdCs= -github.com/ugorji/go/codec v1.3.0 h1:Qd2W2sQawAfG8XSvzwhBeoGq71zXOC/Q1E9y/wUcsUA= -github.com/ugorji/go/codec v1.3.0/go.mod h1:pRBVtBSKl77K30Bv8R2P+cLSGaTtex6fsA2Wjqmfxj4= +github.com/ugorji/go/codec v1.3.1 h1:waO7eEiFDwidsBN6agj1vJQ4AG7lh2yqXyOXqhgQuyY= +github.com/ugorji/go/codec v1.3.1/go.mod h1:pRBVtBSKl77K30Bv8R2P+cLSGaTtex6fsA2Wjqmfxj4= github.com/ulikunitz/xz v0.5.8/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= -github.com/ulikunitz/xz v0.5.12 h1:37Nm15o69RwBkXM0J6A5OlE67RZTfzUxTj8fB3dfcsc= -github.com/ulikunitz/xz v0.5.12/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= +github.com/ulikunitz/xz v0.5.15 h1:9DNdB5s+SgV3bQ2ApL10xRc35ck0DuIX/isZvIk+ubY= +github.com/ulikunitz/xz v0.5.15/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= github.com/unknwon/goconfig v1.0.0 h1:rS7O+CmUdli1T+oDm7fYj1MwqNWtEJfNj+FqcUHML8U= github.com/unknwon/goconfig v1.0.0/go.mod h1:qu2ZQ/wcC/if2u32263HTVC39PeOQRSmidQk3DuDFQ8= github.com/upyun/go-sdk/v3 v3.0.4 h1:2DCJa/Yi7/3ZybT9UCPATSzvU3wpPPxhXinNlb1Hi8Q= github.com/upyun/go-sdk/v3 v3.0.4/go.mod h1:P/SnuuwhrIgAVRd/ZpzDWqCsBAf/oHg7UggbAxyZa0E= -github.com/winfsp/cgofuse v1.6.0 h1:re3W+HTd0hj4fISPBqfsrwyvPFpzqhDu8doJ9nOPDB0= -github.com/winfsp/cgofuse v1.6.0/go.mod h1:uxjoF2jEYT3+x+vC2KJddEGdk/LU8pRowXmyVMHSV5I= github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e h1:JVG44RsyaB9T2KIHavMF/ppJZNG9ZpyihvCd0w101no= github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e/go.mod h1:RbqR21r5mrJuqunuUZ/Dhy/avygyECGrLceyNeo4LiM= github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU= github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E= +github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 h1:ilQV1hzziu+LLM3zUTJ0trRztfwgjqKnBWNtSRkbmwM= +github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78/go.mod h1:aL8wCCfTfSfmXjznFBSZNN13rSJjlIOI1fUNAtF7rmI= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0= github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= +github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ= +github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= +github.com/zeebo/blake3 v0.2.4 h1:KYQPkhpRtcqh0ssGYcKLG1JYvddkEA8QwCM/yBqhaZI= +github.com/zeebo/blake3 v0.2.4/go.mod h1:7eeQ6d2iXWRGF6npfaxl2CU+xy2Fjo2gxeyZGCRUjcE= +github.com/zeebo/pcg v1.0.1 h1:lyqfGeWiv4ahac6ttHs+I5hwtH/+1mrhlCtVNQM2kHo= +github.com/zeebo/pcg v1.0.1/go.mod h1:09F0S9iiKrwn9rlI5yjLkmrug154/YRW6KnnXVDM/l4= +github.com/zeebo/xxh3 v1.1.0 h1:s7DLGDK45Dyfg7++yxI0khrfwq9661w9EN78eP/UZVs= +github.com/zeebo/xxh3 v1.1.0/go.mod h1:IisAie1LELR4xhVinxWS5+zf1lA4p0MW4T+w+W07F5s= github.com/zzzhr1990/go-common-entity v0.0.0-20250202070650-1a200048f0d3 h1:PSRwrE5QBufPnOjdgIkRs5KBV1Avq3SY8oksj2Z+k3o= github.com/zzzhr1990/go-common-entity v0.0.0-20250202070650-1a200048f0d3/go.mod h1:CKriYB8bkNgSbYUQF1khSpejKb5IsV6cR7MdaAR7Fc0= -go.etcd.io/bbolt v1.4.0 h1:TU77id3TnN/zKr7CO/uk+fBCwF2jGcMuw2B/FMAzYIk= -go.etcd.io/bbolt v1.4.0/go.mod h1:AsD+OCi/qPN1giOX1aiLAha3o1U8rAz65bvN4j0sRuk= +go.etcd.io/bbolt v1.5.0 h1:S7GAl7Fxv12yohbwFfIbQCGDWbQbtDGPET4P/bD4lxU= +go.etcd.io/bbolt v1.5.0/go.mod h1:mkltfYE5aUHQxUct9N9V+Kp7aSjFqjgrhcXIS70Lrdk= +go.mongodb.org/mongo-driver v1.17.9 h1:IexDdCuuNJ3BHrELgBlyaH9p60JXAvdzWR128q+U5tU= +go.mongodb.org/mongo-driver v1.17.9/go.mod h1:LlOhpH5NUEfhxcAwG0UEkMqwYcc4JU18gtCdGudk/tQ= +go.mongodb.org/mongo-driver/v2 v2.5.0 h1:yXUhImUjjAInNcpTcAlPHiT7bIXhshCTL3jVBkF3xaE= +go.mongodb.org/mongo-driver/v2 v2.5.0/go.mod h1:yOI9kBsufol30iFsl1slpdq1I0eHPzybRWdyYUs8K/0= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0 h1:F7Jx+6hwnZ41NSFTO5q4LYDtJRXBf2PD0rNBkeB/lus= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0/go.mod h1:UHB22Z8QsdRDrnAtX4PntOl36ajSxcdUMt1sF7Y6E7Q= -go.opentelemetry.io/otel v1.38.0 h1:RkfdswUDRimDg0m2Az18RKOsnI8UDzppJAtj01/Ymk8= -go.opentelemetry.io/otel v1.38.0/go.mod h1:zcmtmQ1+YmQM9wrNsTGV/q/uyusom3P8RxwExxkZhjM= -go.opentelemetry.io/otel/metric v1.38.0 h1:Kl6lzIYGAh5M159u9NgiRkmoMKjvbsKtYRwgfrA6WpA= -go.opentelemetry.io/otel/metric v1.38.0/go.mod h1:kB5n/QoRM8YwmUahxvI3bO34eVtQf2i4utNVLr9gEmI= -go.opentelemetry.io/otel/sdk v1.38.0 h1:l48sr5YbNf2hpCUj/FoGhW9yDkl+Ma+LrVl8qaM5b+E= -go.opentelemetry.io/otel/sdk v1.38.0/go.mod h1:ghmNdGlVemJI3+ZB5iDEuk4bWA3GkTpW+DOoZMYBVVg= -go.opentelemetry.io/otel/sdk/metric v1.38.0 h1:aSH66iL0aZqo//xXzQLYozmWrXxyFkBJ6qT5wthqPoM= -go.opentelemetry.io/otel/sdk/metric v1.38.0/go.mod h1:dg9PBnW9XdQ1Hd6ZnRz689CbtrUp0wMMs9iPcgT9EZA= -go.opentelemetry.io/otel/trace v1.38.0 h1:Fxk5bKrDZJUH+AMyyIXGcFAPah0oRcT+LuNtJrmcNLE= -go.opentelemetry.io/otel/trace v1.38.0/go.mod h1:j1P9ivuFsTceSWe1oY+EeW3sc+Pp42sO++GHkg4wwhs= -go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A= -go.uber.org/goleak v1.2.1/go.mod h1:qlT2yGI9QafXHhZZLxlSuNsMw3FFLxBr+tBRlmO1xH4= -go.uber.org/mock v0.5.0 h1:KAMbZvZPyBPWgD14IrIQ38QCyjwpvVVV6K/bHl1IwQU= -go.uber.org/mock v0.5.0/go.mod h1:ge71pBPLYDk7QIi1LupWxdAykm7KIEFchiOqd6z7qMM= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.69.0 h1:8tvICD4vSTOOsNrsI4Ljf6C+6UKvpTEH5XY3JMoyPoo= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.69.0/go.mod h1:z9+yiacE0IHRqM4qFfkbt/JYlmYXgss8GY/jXoNuPJI= +go.opentelemetry.io/otel v1.44.0 h1:JjwHmHpA4iZ3wBxluu2fbbE7j4kqlE8jXyAyPXH7HqU= +go.opentelemetry.io/otel v1.44.0/go.mod h1:BMgjTHL9WPRlRjL2oZCBTL4whCGtXch2H4BhOPIAyYc= +go.opentelemetry.io/otel/metric v1.44.0 h1:1w0gILTcHdr3YI+ixLyjemwrVnsMURbTZFrSYCdDdmc= +go.opentelemetry.io/otel/metric v1.44.0/go.mod h1:8O7hanEPBNgEMmybD3s2VBKcgWOCsA6tzHBPODAiquo= +go.opentelemetry.io/otel/trace v1.44.0 h1:jxF5CsGYCe74MCRx2X4g7WsY/VBKRqqpNvXlX/6gtIk= +go.opentelemetry.io/otel/trace v1.44.0/go.mod h1:oLl1jrMQAVo6v3GAggN+1VH9VIz9iUSvW53sW1Q8PIE= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.uber.org/mock v0.6.0 h1:hyF9dfmbgIX5EfOdasqLsWD6xqpNZlXblLB/Dbnwv3Y= +go.uber.org/mock v0.6.0/go.mod h1:KiVJ4BqZJaMj4svdfmHM0AUx4NJYO8ZNpPnZn1Z+BBU= +go.yaml.in/yaml/v2 v2.4.4 h1:tuyd0P+2Ont/d6e2rl3be67goVK4R6deVxCUX5vyPaQ= +go.yaml.in/yaml/v2 v2.4.4/go.mod h1:gMZqIpDtDqOfM0uNfy0SkpRhvUryYH0Z6wdMYcacYXQ= go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= +go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw= +go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg= go4.org v0.0.0-20260112195520-a5071408f32f h1:ziUVAjmTPwQMBmYR1tbdRFJPtTcQUI12fH9QQjfb0Sw= go4.org v0.0.0-20260112195520-a5071408f32f/go.mod h1:ZRJnO5ZI4zAwMFp+dS1+V6J6MSyAowhRqAE+DPa1Xp0= gocv.io/x/gocv v0.25.0/go.mod h1:Rar2PS6DV+T4FL+PM535EImD/h13hGVaHhnCu1xarBs= -golang.org/x/arch v0.18.0 h1:WN9poc33zL4AzGxqf8VtpKUnGvMi8O9lhNyBMF/85qc= -golang.org/x/arch v0.18.0/go.mod h1:bdwinDaKcfZUGpH09BB7ZmOfhalA8lQdzl62l8gGWsk= +golang.org/x/arch v0.23.0 h1:lKF64A2jF6Zd8L0knGltUnegD62JMFBiCPBmQpToHhg= +golang.org/x/arch v0.23.0/go.mod h1:dNHoOeKiyja7GTvF9NJS1l3Z2yntpQNzgrjh1cU103A= golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= @@ -711,20 +746,18 @@ golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliY golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= -golang.org/x/crypto v0.46.0 h1:cKRW/pmt1pKAfetfu+RCEvjvZkA9RimPbh7bhFjGVBU= -golang.org/x/crypto v0.46.0/go.mod h1:Evb/oLKmMraqjZ2iQTwDwvCtJkczlDuTmdJXoZVzqU0= -golang.org/x/exp v0.0.0-20250606033433-dcc06ee1d476 h1:bsqhLWFR6G6xiQcb+JoGqdKdRU6WzPWmK8E0jxTjzo4= -golang.org/x/exp v0.0.0-20250606033433-dcc06ee1d476/go.mod h1:3//PLf8L/X+8b4vuAfHzxeRUl04Adcb341+IGKfnqS8= +golang.org/x/crypto v0.56.0 h1:GUh5Ii4J5jtcseSMiRqr1jXCNHoxjeV9Fmekc2oLy6Y= +golang.org/x/crypto v0.56.0/go.mod h1:OMW5y6CY9l38uPLmxU6l6pwcXp1obtLo3e6gT7gQR2I= +golang.org/x/exp v0.0.0-20260709172345-9ea1abe57597 h1:qLvzZeaANDgyVOA8pyHCOStGlXn0rseXma+GQjeuv2g= +golang.org/x/exp v0.0.0-20260709172345-9ea1abe57597/go.mod h1:EdfpwwqSu+0Li0mzskwHU6FWDV3t9Q+RZDo3QMUtL3Q= golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= -golang.org/x/image v0.29.0 h1:HcdsyR4Gsuys/Axh0rDEmlBmB68rW1U9BUdB3UVHsas= -golang.org/x/image v0.29.0/go.mod h1:RVJROnf3SLK8d26OW91j4FrIHGbsJ8QnbEocVTOWQDA= +golang.org/x/image v0.45.0 h1:FMb1nTbH5H9vF55SriQHgFw5GnNL9Jg6L25BwXKzhB0= +golang.org/x/image v0.45.0/go.mod h1:n62x/7RqlwXDvGsSU4u6IUTUf6KghUZ9Bt7cG/T9Fx4= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= -golang.org/x/mod v0.30.0 h1:fDEXFVZ/fmCKProc/yAXXUijritrDzahmwwefnjoPFk= -golang.org/x/mod v0.30.0/go.mod h1:lAsf5O2EvJeSFMiBxXDki7sCgAxEUcZHXoXMKT4GJKc= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -739,10 +772,10 @@ golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= -golang.org/x/net v0.48.0 h1:zyQRTTrjc33Lhh0fBgT/H3oZq9WuvRR5gPC70xpDiQU= -golang.org/x/net v0.48.0/go.mod h1:+ndRgGjkh8FGtu1w1FGbEC31if4VrNVMuKTgcAAnQRY= -golang.org/x/oauth2 v0.34.0 h1:hqK/t4AKgbqWkdkcAeI8XLmbK+4m4G5YeQRrmiotGlw= -golang.org/x/oauth2 v0.34.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA= +golang.org/x/net v0.58.0 h1:ynWG7rqYi4ccpTEuPZ2QGWHktVEM9DMCj9yzDE0Q7To= +golang.org/x/net v0.58.0/go.mod h1:YwCddHnFlT7eLQqVprV19OnhLGtc5xOKgE0RyqgfWAU= +golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs= +golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -750,8 +783,8 @@ golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= -golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4= -golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= +golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek= +golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -764,7 +797,6 @@ golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220615213510-4f61da869c0c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -774,8 +806,8 @@ golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.40.0 h1:DBZZqJ2Rkml6QMQsZywtnjnnGvHza6BTfYFWY9kjEWQ= -golang.org/x/sys v0.40.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= +golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -786,8 +818,8 @@ golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM= -golang.org/x/term v0.38.0 h1:PQ5pkm/rLO6HnxFR7N2lJHOZX6Kez5Y1gDSJla6jo7Q= -golang.org/x/term v0.38.0/go.mod h1:bSEAKrOT1W+VSu9TSCMtoGEOUcKxOKgl3LE5QEF/xVg= +golang.org/x/term v0.45.0 h1:NwWyBmoJCbfTHpxrWoZ9C6/VxOf7ic219I8xZZFdrf0= +golang.org/x/term v0.45.0/go.mod h1:9aqxs0blBcrm/n0L9QW0aRVD+ktan8ssZromtqJC43w= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= @@ -800,11 +832,11 @@ golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= -golang.org/x/text v0.32.0 h1:ZD01bjUt1FQ9WJ0ClOL5vxgxOI/sVCNgX1YtKwcY0mU= -golang.org/x/text v0.32.0/go.mod h1:o/rUWzghvpD5TXrTIBuJU77MTaN0ljMWE47kxGJQ7jY= +golang.org/x/text v0.41.0 h1:vz/seA0lnX87Othu2f/0L24RcgrXD9/YFTSuGjj3rH8= +golang.org/x/text v0.41.0/go.mod h1:jvf1O8ajNzZqhSrQBPbutR/EB83Cc0CFrezNQIwbb5M= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI= -golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4= +golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= +golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190829051458-42f498d34c4d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= @@ -813,20 +845,20 @@ golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= -golang.org/x/tools v0.39.0 h1:ik4ho21kwuQln40uelmciQPp9SipgNDdrafrYA4TmQQ= -golang.org/x/tools v0.39.0/go.mod h1:JnefbkDPyD8UU2kI5fuf8ZX4/yUeh9W877ZeBONxUqQ= +golang.org/x/tools v0.48.0 h1:3+hClM1aLL5mjMKm5ovokw9epgRXPuu2tILgismM6RE= +golang.org/x/tools v0.48.0/go.mod h1:08xX0orndb/F7jJxGDicx061tyd5pcMto75YMAXr6lk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= -gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= -google.golang.org/api v0.259.0 h1:90TaGVIxScrh1Vn/XI2426kRpBqHwWIzVBzJsVZ5XrQ= -google.golang.org/api v0.259.0/go.mod h1:LC2ISWGWbRoyQVpxGntWwLWN/vLNxxKBK9KuJRI8Te4= +gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= +gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= +google.golang.org/api v0.279.0 h1:hsx2M2OaRcaKtVYK6vXEUnQvdjnend7ZYES+lYaot74= +google.golang.org/api v0.279.0/go.mod h1:B9TqLBwJqVjp1mtt7WeoQwWRwvu/400y5lETOql+giQ= google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM= google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds= -google.golang.org/genproto/googleapis/rpc v0.0.0-20251222181119-0a764e51fe1b h1:Mv8VFug0MP9e5vUxfBcE3vUkV6CImK3cMNMIDFjmzxU= -google.golang.org/genproto/googleapis/rpc v0.0.0-20251222181119-0a764e51fe1b/go.mod h1:j9x/tPzZkyxcgEFkiKEEGxfvyumM01BEtsW8xzOahRQ= -google.golang.org/grpc v1.78.0 h1:K1XZG/yGDJnzMdd/uZHAkVqJE+xIDOcmdSFZkBUicNc= -google.golang.org/grpc v1.78.0/go.mod h1:I47qjTo4OKbMkjA/aOOwxDIiPSBofUtQUI5EfpWvW7U= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260715232425-e75dac1f907d h1:Jkpk39hlTZOIp3RbfvNX9R8Hv+Sw0X89nlU/xFOErsc= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260715232425-e75dac1f907d/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= +google.golang.org/grpc v1.85.0-dev h1:HxkDyKIIZPpFnroC56tQv5gNuKTmVvi0t7TzOf5zt7g= +google.golang.org/grpc v1.85.0-dev/go.mod h1:ljCht0DrxQrXBDRTZp52Qxh3Ffk8CdYm2sj4O2QN2C0= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= @@ -840,26 +872,24 @@ gopkg.in/go-jose/go-jose.v2 v2.6.3 h1:nt80fvSDlhKWQgSWyHyy5CfmlQr+asih51R8PTWNKK gopkg.in/go-jose/go-jose.v2 v2.6.3/go.mod h1:zzZDPkNNw/c9IE7Z9jr11mBZQhKQTMzoEEIoEdZlFBI= gopkg.in/ldap.v3 v3.1.0 h1:DIDWEjI7vQWREh0S8X5/NFPCZ3MCVd55LmXKPW4XLGE= gopkg.in/ldap.v3 v3.1.0/go.mod h1:dQjCc0R0kfyFjIlWNMH1DORwUASZyDxo2Ry1B51dXaQ= -gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8= -gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= +gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc= +gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.7/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gorm.io/driver/mysql v1.5.7 h1:MndhOPYOfEp2rHKgkZIhJ16eVUIRf2HmzgoPmh7FCWo= -gorm.io/driver/mysql v1.5.7/go.mod h1:sEtPWMiqiN1N1cMXoXmBbd8C6/l+TESwriotuRRpkDM= -gorm.io/driver/postgres v1.5.9 h1:DkegyItji119OlcaLjqN11kHoUgZ/j13E0jkJZgD6A8= -gorm.io/driver/postgres v1.5.9/go.mod h1:DX3GReXH+3FPWGrrgffdvCk3DQ1dwDPdmbenSkweRGI= +gorm.io/driver/mysql v1.6.0 h1:eNbLmNTpPpTOVZi8MMxCi2aaIm0ZpInbORNXDwyLGvg= +gorm.io/driver/mysql v1.6.0/go.mod h1:D/oCC2GWK3M/dqoLxnOlaNKmXz8WNTfcS9y5ovaSqKo= +gorm.io/driver/postgres v1.6.2 h1:BvXQ/cNUg63q5TFNg672DmDcowZSFrNLkkA3Xe6GXq4= +gorm.io/driver/postgres v1.6.2/go.mod h1:0c4fQA44XhOklXDkgtuKqysHCycTa5i9e3EIpDGCwXk= gorm.io/driver/sqlite v1.6.0 h1:WHRRrIiulaPiPFmDcod6prc4l2VGVWHz80KspNsxSfQ= gorm.io/driver/sqlite v1.6.0/go.mod h1:AO9V1qIQddBESngQUKWL9yoH93HIeA1X6V633rBwyT8= -gorm.io/gorm v1.25.7/go.mod h1:hbnx/Oo0ChWMn1BIhpy1oYozzpM15i4YPuHDmfYtwg8= -gorm.io/gorm v1.30.0 h1:qbT5aPv1UH8gI99OsRlvDToLxW5zR7FzS9acZDOZcgs= -gorm.io/gorm v1.30.0/go.mod h1:8Z33v652h4//uMA76KjeDH8mJXPm1QNCYrMeatR0DOE= +gorm.io/gorm v1.31.2 h1:3o8FXNo9v9S858gil+3LlZA1LkCOzgb4g5BL64FgaCo= +gorm.io/gorm v1.31.2/go.mod h1:XyQVbO2k6YkOis7C2437jSit3SsDK72s7n7rsSHd+Gs= lukechampine.com/blake3 v1.1.7 h1:GgRMhmdsuK8+ii6UZFDL8Nb+VyMwadAgcJyfYHxG6n0= lukechampine.com/blake3 v1.1.7/go.mod h1:tkKEOtDkNtklkXtLNEOGNq5tcV90tJiA1vAA12R78LA= modernc.org/libc v1.22.5 h1:91BNch/e5B0uPbJFgqbxXuOnxBQjlS//icfQEGmvyjE= @@ -870,7 +900,8 @@ modernc.org/memory v1.5.0 h1:N+/8c5rE6EqugZwHii4IFsaJ7MUhoWX07J5tC/iI5Ds= modernc.org/memory v1.5.0/go.mod h1:PkUhL0Mugw21sHPeskwZW4D6VscE/GQJOnIpCnW6pSU= modernc.org/sqlite v1.23.1 h1:nrSBg4aRQQwq59JpvGEQ15tNxoO5pX/kUjcRNwSAGQM= modernc.org/sqlite v1.23.1/go.mod h1:OrDj17Mggn6MhE+iPbBNf7RGKODDE9NFT0f3EwDzJqk= -nullprogram.com/x/optparse v1.0.0/go.mod h1:KdyPE+Igbe0jQUrVfMqDMeJQIJZEuyV7pjYmp6pbG50= +moul.io/http2curl/v2 v2.3.0 h1:9r3JfDzWPcbIklMOs2TnIFzDYvfAZvjeavG6EzP7jYs= +moul.io/http2curl/v2 v2.3.0/go.mod h1:RW4hyBjTWSYDOxapodpNEtX0g5Eb16sxklBqmd2RHcE= resty.dev/v3 v3.0.0-beta.2 h1:xu4mGAdbCLuc3kbk7eddWfWm4JfhwDtdapwss5nCjnQ= resty.dev/v3 v3.0.0-beta.2/go.mod h1:OgkqiPvTDtOuV4MGZuUDhwOpkY8enjOsjjMzeOHefy4= sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= diff --git a/internal/authn/authn.go b/internal/authn/authn.go index a57823d1..b9445499 100644 --- a/internal/authn/authn.go +++ b/internal/authn/authn.go @@ -6,13 +6,12 @@ import ( "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/setting" - "github.com/OpenListTeam/OpenList/v4/server/common" "github.com/gin-gonic/gin" "github.com/go-webauthn/webauthn/webauthn" ) func NewAuthnInstance(c *gin.Context) (*webauthn.WebAuthn, error) { - siteUrl, err := url.Parse(common.GetApiUrl(c.Request.Context())) + siteUrl, err := url.Parse(conf.GetApiUrl(c.Request.Context())) if err != nil { return nil, err } diff --git a/internal/bootstrap/data/setting.go b/internal/bootstrap/data/setting.go index d3542ea2..bf52f3d0 100644 --- a/internal/bootstrap/data/setting.go +++ b/internal/bootstrap/data/setting.go @@ -211,6 +211,7 @@ func InitialSettings() []model.SettingItem { {Key: conf.SSODefaultDir, Value: "/", Type: conf.TypeString, Group: model.SSO, Flag: model.PRIVATE}, {Key: conf.SSODefaultPermission, Value: "0", Type: conf.TypeNumber, Group: model.SSO, Flag: model.PRIVATE}, {Key: conf.SSOCompatibilityMode, Value: "false", Type: conf.TypeBool, Group: model.SSO, Flag: model.PUBLIC}, + {Key: conf.SSOPostMessageOrigin, Value: "", Type: conf.TypeString, Group: model.SSO, Flag: model.PUBLIC}, // ldap settings {Key: conf.LdapLoginEnabled, Value: "false", Type: conf.TypeBool, Group: model.LDAP, Flag: model.PUBLIC}, @@ -243,6 +244,7 @@ func InitialSettings() []model.SettingItem { {Key: conf.TaskOfflineDownloadTransferThreadsNum, Value: strconv.Itoa(conf.Conf.Tasks.Transfer.Workers), Type: conf.TypeNumber, Group: model.TRAFFIC, Flag: model.PRIVATE}, {Key: conf.TaskUploadThreadsNum, Value: strconv.Itoa(conf.Conf.Tasks.Upload.Workers), Type: conf.TypeNumber, Group: model.TRAFFIC, Flag: model.PRIVATE}, {Key: conf.TaskCopyThreadsNum, Value: strconv.Itoa(conf.Conf.Tasks.Copy.Workers), Type: conf.TypeNumber, Group: model.TRAFFIC, Flag: model.PRIVATE}, + {Key: conf.TaskMoveThreadsNum, Value: strconv.Itoa(conf.Conf.Tasks.Move.Workers), Type: conf.TypeNumber, Group: model.TRAFFIC, Flag: model.PRIVATE}, {Key: conf.TaskDecompressDownloadThreadsNum, Value: strconv.Itoa(conf.Conf.Tasks.Decompress.Workers), Type: conf.TypeNumber, Group: model.TRAFFIC, Flag: model.PRIVATE}, {Key: conf.TaskDecompressUploadThreadsNum, Value: strconv.Itoa(conf.Conf.Tasks.DecompressUpload.Workers), Type: conf.TypeNumber, Group: model.TRAFFIC, Flag: model.PRIVATE}, {Key: conf.StreamMaxClientDownloadSpeed, Value: "-1", Type: conf.TypeNumber, Group: model.TRAFFIC, Flag: model.PRIVATE}, @@ -322,6 +324,10 @@ videotoolbox = macOS 硬件加速`}, Help: `chunk 多久没被访问就 kill ffmpeg 释放资源（默认60秒）`}, {Key: conf.TranscodeChunkPrefetch, Value: "true", Type: conf.TypeBool, Group: model.TRANSCODE, Flag: model.PRIVATE, Help: `当前 chunk 转码进度过半时，是否预启动下一个 chunk（默认开启，可让顺序播放无缝衔接）`}, + + // multipart upload settings (from main) + {Key: conf.MultipartEnabled, Value: "true", Type: conf.TypeBool, Group: model.TRAFFIC, Flag: model.PUBLIC}, + {Key: conf.MultipartChunkSize, Value: "10", Type: conf.TypeNumber, Group: model.TRAFFIC, Flag: model.PUBLIC, Help: `chunk size of multipart upload in MB (positive integer), keep it under your CDN's request body limit; each active session buffers up to 8 chunks on the server's disk`}, } additionalSettingItems := tool.Tools.Items() // 固定顺序 diff --git a/internal/bootstrap/data/user.go b/internal/bootstrap/data/user.go index b1b1c77f..44f25440 100644 --- a/internal/bootstrap/data/user.go +++ b/internal/bootstrap/data/user.go @@ -15,18 +15,28 @@ import ( ) func initUser() { - admin, err := op.GetAdmin() - adminPassword := random.String(8) - envpass := os.Getenv("OPENLIST_ADMIN_PASSWORD") - if flags.Dev { - adminPassword = "admin" - } else if len(envpass) > 0 { - adminPassword = envpass - } + initAdmin() + initGuest() +} + +func initAdmin() { + _, err := op.GetAdmin() if err != nil { if errors.Is(err, gorm.ErrRecordNotFound) { + // 系统尚未初始化：仅在开发模式或显式配置 OPENLIST_ADMIN_PASSWORD 时 + // 自动创建管理员；否则交由 Web 安装向导（POST /api/public/init/setup）完成。 + adminPassword := "admin" + envpass := os.Getenv("OPENLIST_ADMIN_PASSWORD") + if flags.Dev { + adminPassword = "admin" + } else if len(envpass) > 0 { + adminPassword = envpass + } else { + // 未初始化：不自动创建管理员，等待 Web 安装向导。 + return + } salt := random.String(16) - admin = &model.User{ + admin := &model.User{ Username: "admin", Salt: salt, PwdHash: model.TwoHashPwd(adminPassword, salt), @@ -47,7 +57,10 @@ func initUser() { utils.Log.Fatalf("[init user] Failed to get admin user: %v", err) } } - _, err = op.GetGuest() +} + +func initGuest() { + _, err := op.GetGuest() if err != nil { if errors.Is(err, gorm.ErrRecordNotFound) { salt := random.String(16) diff --git a/internal/conf/config.go b/internal/conf/config.go index 0f667611..f8423d59 100644 --- a/internal/conf/config.go +++ b/internal/conf/config.go @@ -83,9 +83,10 @@ type Cors struct { } type S3 struct { - Enable bool `json:"enable" env:"ENABLE"` - Port int `json:"port" env:"PORT"` - SSL bool `json:"ssl" env:"SSL"` + Enable bool `json:"enable" env:"ENABLE"` + Port int `json:"port" env:"PORT"` + SSL bool `json:"ssl" env:"SSL"` + MultipartTTL string `json:"multipart_ttl" env:"MULTIPART_TTL"` } type FTP struct { @@ -229,9 +230,10 @@ func DefaultConfig(dataDir string) *Config { AllowHeaders: []string{"*"}, }, S3: S3{ - Enable: false, - Port: 5246, - SSL: false, + Enable: false, + Port: 5246, + SSL: false, + MultipartTTL: "24h", }, FTP: FTP{ Enable: false, diff --git a/internal/conf/const.go b/internal/conf/const.go index e43de5f0..c049c764 100644 --- a/internal/conf/const.go +++ b/internal/conf/const.go @@ -95,6 +95,9 @@ const ( // thunder_browser ThunderBrowserTempDir = "thunder_browser_temp_dir" + // guangyapan + GuangYaPanTempDir = "guangyapan_temp_dir" + // single Token = "token" IndexProgress = "index_progress" @@ -114,6 +117,7 @@ const ( SSODefaultDir = "sso_default_dir" SSODefaultPermission = "sso_default_permission" SSOCompatibilityMode = "sso_compatibility_mode" + SSOPostMessageOrigin = "sso_postmessage_origin" // ldap LdapLoginEnabled = "ldap_login_enabled" @@ -200,6 +204,10 @@ const ( TranscodeMaxChunkConcurrency = "transcode_max_chunk_concurrency" // 单个 Job 最多同时运行的 chunk 数（默认2） TranscodeChunkIdleSec = "transcode_chunk_idle_sec" // chunk 多久没被访问就 kill ffmpeg 释放资源（默认60秒） TranscodeChunkPrefetch = "transcode_chunk_prefetch" // 当前 chunk 转码进度过半时，是否预启动下一个 chunk（默认true） + + // multipart upload (from main) + MultipartEnabled = "multipart_enabled" + MultipartChunkSize = "multipart_chunk_size" ) const ( @@ -230,4 +238,6 @@ const ( PathKey SharingIDKey SkipHookKey + DirectUploadContentTypeKey + SkipNoOverwriteKey ) diff --git a/internal/conf/context.go b/internal/conf/context.go new file mode 100644 index 00000000..5b921d90 --- /dev/null +++ b/internal/conf/context.go @@ -0,0 +1,8 @@ +package conf + +import "context" + +func GetApiUrl(ctx context.Context) string { + api, _ := ctx.Value(ApiUrlKey).(string) + return api +} diff --git a/internal/conf/context_test.go b/internal/conf/context_test.go new file mode 100644 index 00000000..9d92c318 --- /dev/null +++ b/internal/conf/context_test.go @@ -0,0 +1,29 @@ +package conf_test + +import ( + "context" + "testing" + + "github.com/OpenListTeam/OpenList/v4/internal/conf" +) + +func TestGetApiUrl(t *testing.T) { + const want = "https://openlist.example" + tests := []struct { + name string + ctx context.Context + want string + }{ + {name: "present", ctx: context.WithValue(context.Background(), conf.ApiUrlKey, want), want: want}, + {name: "absent", ctx: context.Background()}, + {name: "wrong type", ctx: context.WithValue(context.Background(), conf.ApiUrlKey, 1)}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := conf.GetApiUrl(tt.ctx); got != tt.want { + t.Fatalf("origin = %q, want %q", got, tt.want) + } + }) + } +} diff --git a/internal/db/db.go b/internal/db/db.go index 6945a9a1..a14c775f 100644 --- a/internal/db/db.go +++ b/internal/db/db.go @@ -12,10 +12,8 @@ var db *gorm.DB func Init(d *gorm.DB) { db = d - // 迁移前处理：处理 media_items 的存储语义变更，避免旧数据与新代码不一致 - // 语义变更：folder_path 现为文件实际父目录（之前为扫描根路径），旧数据需重扫 migrateMediaItems() - err := AutoMigrate(new(model.Storage), new(model.User), new(model.Meta), new(model.SettingItem), new(model.SearchNode), new(model.TaskItem), new(model.SSHPublicKey), new(model.SharingDB), new(model.MediaItem), new(model.MediaConfig), new(model.MediaScanPath)) + err := AutoMigrate(new(model.Storage), new(model.User), new(model.Meta), new(model.SettingItem), new(model.SearchNode), new(model.TaskItem), new(model.SSHPublicKey), new(model.SharingDB), new(model.WebDAVProperty), new(model.MediaItem), new(model.MediaConfig), new(model.MediaScanPath)) if err != nil { log.Fatalf("failed migrate database: %s", err.Error()) } diff --git a/internal/db/meta.go b/internal/db/meta.go index 32eec2c3..b609261a 100644 --- a/internal/db/meta.go +++ b/internal/db/meta.go @@ -13,6 +13,14 @@ func GetMetaByPath(path string) (*model.Meta, error) { return &meta, nil } +func GetAllMetas() ([]model.Meta, error) { + var metas []model.Meta + if err := db.Find(&metas).Error; err != nil { + return nil, errors.Wrapf(err, "failed get all metas") + } + return metas, nil +} + func GetMetaById(id uint) (*model.Meta, error) { var u model.Meta if err := db.First(&u, id).Error; err != nil { diff --git a/internal/db/searchnode.go b/internal/db/searchnode.go index 11d345a7..18aa9305 100644 --- a/internal/db/searchnode.go +++ b/internal/db/searchnode.go @@ -35,7 +35,7 @@ func DeleteSearchNodesByParent(path string) error { if err != nil { return err } - dir, name := stdpath.Split(path) + dir, name := stdpath.Dir(path), stdpath.Base(path) return db.Where(fmt.Sprintf("%s = ? AND %s = ?", columnName("parent"), columnName("name")), dir, name).Delete(&model.SearchNode{}).Error diff --git a/internal/errs/errors.go b/internal/errs/errors.go index fdf7f218..44f7ff56 100644 --- a/internal/errs/errors.go +++ b/internal/errs/errors.go @@ -18,6 +18,7 @@ var ( StorageNotInit = errors.New("storage not init") StreamIncomplete = errors.New("upload/download stream incomplete, possible network issue") StreamPeekFail = errors.New("StreamPeekFail") + TemporaryCapacity = errors.New("temporary capacity unavailable") UnknownArchiveFormat = errors.New("unknown archive format") WrongArchivePassword = errors.New("wrong archive password") diff --git a/internal/fs/archive.go b/internal/fs/archive.go index 784ba587..02198356 100644 --- a/internal/fs/archive.go +++ b/internal/fs/archive.go @@ -21,7 +21,6 @@ import ( "github.com/OpenListTeam/OpenList/v4/internal/task" "github.com/OpenListTeam/OpenList/v4/internal/task_group" "github.com/OpenListTeam/OpenList/v4/pkg/utils" - "github.com/OpenListTeam/OpenList/v4/server/common" "github.com/OpenListTeam/tache" "github.com/pkg/errors" log "github.com/sirupsen/logrus" @@ -415,7 +414,7 @@ func archiveDecompress(ctx context.Context, srcObjPath, dstDirPath string, args return nil, err } else { tsk.Creator, _ = ctx.Value(conf.UserKey).(*model.User) - tsk.ApiUrl = common.GetApiUrl(ctx) + tsk.ApiUrl = conf.GetApiUrl(ctx) ArchiveDownloadTaskManager.Add(tsk) return tsk, nil } diff --git a/internal/fs/copy_move.go b/internal/fs/copy_move.go index e78fc9be..a1b34ebf 100644 --- a/internal/fs/copy_move.go +++ b/internal/fs/copy_move.go @@ -7,6 +7,7 @@ import ( "time" "github.com/OpenListTeam/OpenList/v4/internal/conf" + "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/op" @@ -14,9 +15,10 @@ import ( "github.com/OpenListTeam/OpenList/v4/internal/task" "github.com/OpenListTeam/OpenList/v4/internal/task_group" "github.com/OpenListTeam/OpenList/v4/pkg/utils" - "github.com/OpenListTeam/OpenList/v4/server/common" "github.com/OpenListTeam/tache" + "github.com/google/uuid" "github.com/pkg/errors" + log "github.com/sirupsen/logrus" ) type taskType uint8 @@ -100,7 +102,79 @@ func (t *FileTransferTask) SetRetry(retry int, maxRetry int) { } } -func transfer(ctx context.Context, taskType taskType, srcObjPath, dstDirPath string, skipHook ...bool) (task.TaskExtensionInfo, error) { +func supportsNamedNativeCopy(storage driver.Driver) bool { + _, hasMkdir := storage.(driver.Mkdir) + if !hasMkdir { + _, hasMkdir = storage.(driver.MkdirResult) + } + _, hasCopy := storage.(driver.Copy) + if !hasCopy { + _, hasCopy = storage.(driver.CopyResult) + } + _, hasMove := storage.(driver.Move) + if !hasMove { + _, hasMove = storage.(driver.MoveResult) + } + _, hasRename := storage.(driver.Rename) + if !hasRename { + _, hasRename = storage.(driver.RenameResult) + } + _, hasRemove := storage.(driver.Remove) + return hasMkdir && hasCopy && hasMove && hasRename && hasRemove +} + +func namedCopyStageObjectPath(stageDirPath, srcPath, dstName string) string { + name := stdpath.Base(srcPath) + if dstName != "" { + name = dstName + } + return stdpath.Join(stageDirPath, name) +} + +func copyNamedInStorage(ctx context.Context, storage driver.Driver, srcPath, dstDirPath, dstName string) error { + stageDirPath := stdpath.Join(dstDirPath, ".openlist-copy-"+uuid.NewString()) + stageCtx := context.WithValue(ctx, conf.SkipHookKey, struct{}{}) + if err := op.MakeDir(stageCtx, storage, stageDirPath); err != nil { + return errors.WithMessage(err, "failed create copy staging directory") + } + stageCreated := true + removeStage := func() error { + return op.Remove(stageCtx, storage, stageDirPath) + } + defer func() { + if !stageCreated { + return + } + if err := removeStage(); err != nil { + log.WithError(err).Warnf("copy staging directory remains and requires cleanup: %s", stageDirPath) + } + }() + + if err := op.Copy(stageCtx, storage, srcPath, stageDirPath); err != nil { + return errors.WithMessage(err, "failed copy object into staging directory") + } + srcName := stdpath.Base(srcPath) + stageObjPath := namedCopyStageObjectPath(stageDirPath, srcPath, "") + if srcName != dstName { + if err := op.Rename(stageCtx, storage, stageObjPath, dstName); err != nil { + return errors.WithMessage(err, "failed rename object in staging directory") + } + stageObjPath = namedCopyStageObjectPath(stageDirPath, srcPath, dstName) + } + // Staging operations suppress hooks so intermediate paths stay invisible. + // The final move intentionally uses the caller context to publish the target + // and trigger the normal cache/update hooks exactly once. + if err := op.Move(ctx, storage, stageObjPath, dstDirPath); err != nil { + return errors.WithMessage(err, "failed move copied object from staging directory") + } + if err := removeStage(); err != nil { + return errors.WithMessagef(err, "failed remove copy staging directory %s; cleanup will be retried", stageDirPath) + } + stageCreated = false + return nil +} + +func transfer(ctx context.Context, taskType taskType, srcObjPath, dstDirPath, dstName string, skipHook ...bool) (task.TaskExtensionInfo, error) { srcStorage, srcObjActualPath, err := op.GetStorageAndActualPath(srcObjPath) if err != nil { return nil, errors.WithMessage(err, "failed get src storage") @@ -115,9 +189,18 @@ func transfer(ctx context.Context, taskType taskType, srcObjPath, dstDirPath str ctx = context.WithValue(ctx, conf.SkipHookKey, struct{}{}) } if taskType == copy || taskType == merge { - err = op.Copy(ctx, srcStorage, srcObjActualPath, dstDirActualPath) - if !errors.Is(err, errs.NotImplement) && !errors.Is(err, errs.NotSupport) { - return nil, err + if dstName != "" { + if supportsNamedNativeCopy(srcStorage) { + err = copyNamedInStorage(ctx, srcStorage, srcObjActualPath, dstDirActualPath, dstName) + if !errors.Is(err, errs.NotImplement) && !errors.Is(err, errs.NotSupport) { + return nil, err + } + } + } else { + err = op.Copy(ctx, srcStorage, srcObjActualPath, dstDirActualPath) + if !errors.Is(err, errs.NotImplement) && !errors.Is(err, errs.NotSupport) { + return nil, err + } } } else { err = op.Move(ctx, srcStorage, srcObjActualPath, dstDirActualPath) @@ -127,13 +210,15 @@ func transfer(ctx context.Context, taskType taskType, srcObjPath, dstDirPath str } } - // not in the same storage + // Use the transfer path when native copy cannot preserve the destination + // name or the source and destination storages differ. t := &FileTransferTask{ TaskData: TaskData{ SrcStorage: srcStorage, DstStorage: dstStorage, SrcActualPath: srcObjActualPath, DstActualPath: dstDirActualPath, + DstName: dstName, SrcStorageMp: srcStorage.GetStorage().MountPath, DstStorageMp: dstStorage.GetStorage().MountPath, }, @@ -166,7 +251,7 @@ func transfer(ctx context.Context, taskType taskType, srcObjPath, dstDirPath str } t.Creator, _ = ctx.Value(conf.UserKey).(*model.User) - t.ApiUrl = common.GetApiUrl(ctx) + t.ApiUrl = conf.GetApiUrl(ctx) if taskType == copy || taskType == merge { CopyTaskManager.Add(t) } else { @@ -189,9 +274,14 @@ func (t *FileTransferTask) RunWithNextTaskCallback(f func(nextTask *FileTransfer if err != nil { return errors.WithMessagef(err, "failed list src [%s] objs", t.SrcActualPath) } - dstActualPath := stdpath.Join(t.DstActualPath, srcObj.GetName()) - task_group.TransferCoordinator.AppendPayload(t.groupID, task_group.DstPathToHook(dstActualPath)) - + dstName := srcObj.GetName() + if t.DstName != "" { + dstName = t.DstName + } + dstActualPath := stdpath.Join(t.DstActualPath, dstName) + if err := op.MakeDir(t.Ctx(), t.DstStorage, dstActualPath); err != nil { + return errors.WithMessagef(err, "failed create dst dir [%s]", dstActualPath) + } existedObjs := make(map[string]bool) if t.TaskType == merge { dstObjs, err := op.List(t.Ctx(), t.DstStorage, dstActualPath, model.ListArgs{}) @@ -250,8 +340,12 @@ func (t *FileTransferTask) RunWithNextTaskCallback(f func(nextTask *FileTransfer return errors.WithMessagef(err, "failed get [%s] link", t.SrcActualPath) } // any link provided is seekable + streamObj := srcObj + if t.DstName != "" { + streamObj = &model.ObjWrapName{Name: t.DstName, Obj: srcObj} + } ss, err := stream.NewSeekableStream(&stream.FileStream{ - Obj: srcObj, + Obj: streamObj, Ctx: t.Ctx(), }, link) if err != nil { diff --git a/internal/fs/copy_move_test.go b/internal/fs/copy_move_test.go new file mode 100644 index 00000000..95f65751 --- /dev/null +++ b/internal/fs/copy_move_test.go @@ -0,0 +1,23 @@ +package fs + +import "testing" + +func TestNamedCopyStageObjectPath(t *testing.T) { + tests := []struct { + name string + stageDir string + srcPath string + dstName string + want string + }{ + {name: "source basename", stageDir: "/dst/.stage", srcPath: "/src/foo", want: "/dst/.stage/foo"}, + {name: "requested name", stageDir: "/dst/.stage", srcPath: "/src/foo", dstName: "bar", want: "/dst/.stage/bar"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := namedCopyStageObjectPath(tt.stageDir, tt.srcPath, tt.dstName); got != tt.want { + t.Fatalf("namedCopyStageObjectPath(%q, %q, %q) = %q, want %q", tt.stageDir, tt.srcPath, tt.dstName, got, tt.want) + } + }) + } +} diff --git a/internal/fs/copy_queue_persistence_test.go b/internal/fs/copy_queue_persistence_test.go new file mode 100644 index 00000000..d3c559b0 --- /dev/null +++ b/internal/fs/copy_queue_persistence_test.go @@ -0,0 +1,65 @@ +package fs + +import ( + "encoding/json" + "testing" + + "github.com/OpenListTeam/OpenList/v4/internal/conf" + "github.com/OpenListTeam/tache" +) + +func TestMigratedCopyTaskRecoversNativeFields(t *testing.T) { + previousConf := conf.Conf + conf.Conf = &conf.Config{} + t.Cleanup(func() { conf.Conf = previousConf }) + + raw := []byte(`{ + "id":"task-one", + "state":0, + "retry":0, + "max_retry":0, + "Creator":{"id":1,"username":"admin","password":"","base_path":"/","role":2,"disabled":false,"permission":511,"sso_id":"","allow_ldap":true}, + "start_time":"2026-08-11T01:00:00Z", + "end_time":"2026-08-11T01:01:00Z", + "TotalBytes":42, + "ApiUrl":"http://openlist.test:5244", + "src_path":"/folder/file", + "dst_path":"/backup", + "src_storage_mp":"/source", + "dst_storage_mp":"/target", + "TaskType":0 + }`) + + var task FileTransferTask + if err := json.Unmarshal(raw, &task); err != nil { + t.Fatalf("unmarshal migrated task: %v", err) + } + if task.GetID() != "task-one" || task.GetState() != tache.StatePending { + t.Fatalf("unexpected base fields: id=%q state=%d", task.GetID(), task.GetState()) + } + if task.GetCreator() == nil || task.GetCreator().Username != "admin" { + t.Fatal("creator was not recovered") + } + if task.GetStartTime() == nil || task.GetEndTime() == nil { + t.Fatal("task timestamps were not recovered") + } + if task.TaskType != copy { + t.Fatalf("unexpected task type: %d", task.TaskType) + } + if task.SrcActualPath != "/folder/file" || task.DstActualPath != "/backup" { + t.Fatal("copy paths were not recovered") + } + + _, maxRetry := task.GetRetry() + if maxRetry != 0 { + t.Fatalf("migration must defer retry initialization, got %d", maxRetry) + } + task.SetRetry(0, 2) + _, maxRetry = task.GetRetry() + if maxRetry != 2 { + t.Fatalf("retry initialization failed, got %d", maxRetry) + } + if task.groupID != "/target/backup" { + t.Fatalf("task group was not rebuilt: %q", task.groupID) + } +} diff --git a/internal/fs/fs.go b/internal/fs/fs.go index 67a1ac06..b7f064ac 100644 --- a/internal/fs/fs.go +++ b/internal/fs/fs.go @@ -69,7 +69,7 @@ func MakeDir(ctx context.Context, path string) error { } func Move(ctx context.Context, srcPath, dstDirPath string, skipHook ...bool) (task.TaskExtensionInfo, error) { - req, err := transfer(ctx, move, srcPath, dstDirPath, skipHook...) + req, err := transfer(ctx, move, srcPath, dstDirPath, "", skipHook...) if err != nil { log.Errorf("failed move %s to %s: %+v", srcPath, dstDirPath, err) } @@ -77,15 +77,24 @@ func Move(ctx context.Context, srcPath, dstDirPath string, skipHook ...bool) (ta } func Copy(ctx context.Context, srcObjPath, dstDirPath string, skipHook ...bool) (task.TaskExtensionInfo, error) { - res, err := transfer(ctx, copy, srcObjPath, dstDirPath, skipHook...) + res, err := transfer(ctx, copy, srcObjPath, dstDirPath, "", skipHook...) if err != nil { log.Errorf("failed copy %s to %s: %+v", srcObjPath, dstDirPath, err) } return res, err } +// CopyTo copies a file or directory to dstDirPath using dstName as its name. +func CopyTo(ctx context.Context, srcObjPath, dstDirPath, dstName string, skipHook ...bool) (task.TaskExtensionInfo, error) { + res, err := transfer(ctx, copy, srcObjPath, dstDirPath, dstName, skipHook...) + if err != nil { + log.Errorf("failed copy %s to %s as %s: %+v", srcObjPath, dstDirPath, dstName, err) + } + return res, err +} + func Merge(ctx context.Context, srcObjPath, dstDirPath string, skipHook ...bool) (task.TaskExtensionInfo, error) { - res, err := transfer(ctx, merge, srcObjPath, dstDirPath, skipHook...) + res, err := transfer(ctx, merge, srcObjPath, dstDirPath, "", skipHook...) if err != nil { log.Errorf("failed merge %s to %s: %+v", srcObjPath, dstDirPath, err) } diff --git a/internal/fs/link.go b/internal/fs/link.go index d84090cf..8e1dedfd 100644 --- a/internal/fs/link.go +++ b/internal/fs/link.go @@ -4,9 +4,9 @@ import ( "context" "strings" + "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/op" - "github.com/OpenListTeam/OpenList/v4/server/common" "github.com/pkg/errors" ) @@ -20,7 +20,7 @@ func link(ctx context.Context, path string, args model.LinkArgs) (*model.Link, m return nil, nil, errors.WithMessage(err, "failed link") } if l.URL != "" && !strings.HasPrefix(l.URL, "http://") && !strings.HasPrefix(l.URL, "https://") { - l.URL = common.GetApiUrl(ctx) + l.URL + l.URL = conf.GetApiUrl(ctx) + l.URL } return l, obj, nil } diff --git a/internal/fs/other.go b/internal/fs/other.go index a23beb73..a74eff41 100644 --- a/internal/fs/other.go +++ b/internal/fs/other.go @@ -53,6 +53,7 @@ type TaskData struct { Status string `json:"-"` //don't save status to save space SrcActualPath string `json:"src_path"` DstActualPath string `json:"dst_path"` + DstName string `json:"dst_name,omitempty"` SrcStorage driver.Driver `json:"-"` DstStorage driver.Driver `json:"-"` SrcStorageMp string `json:"src_storage_mp"` diff --git a/internal/fs/put.go b/internal/fs/put.go index 0b905be0..9042ae88 100644 --- a/internal/fs/put.go +++ b/internal/fs/put.go @@ -7,7 +7,6 @@ import ( "time" "github.com/OpenListTeam/OpenList/v4/pkg/utils" - "github.com/OpenListTeam/OpenList/v4/server/common" "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/driver" @@ -81,7 +80,7 @@ func putAsTask(ctx context.Context, dstDirPath string, file model.FileStreamer) t := &UploadTask{ TaskExtension: task.TaskExtension{ Creator: taskCreator, - ApiUrl: common.GetApiUrl(ctx), + ApiUrl: conf.GetApiUrl(ctx), }, storage: storage, dstDirActualPath: dstDirActualPath, diff --git a/internal/fuse/fs.go b/internal/fuse/fs.go deleted file mode 100644 index 7783b169..00000000 --- a/internal/fuse/fs.go +++ /dev/null @@ -1,170 +0,0 @@ -package fuse - -import "github.com/winfsp/cgofuse/fuse" - -type Fs struct { - RootFolder string - fuse.FileSystemBase -} - -func (fs *Fs) Init() { - //TODO implement me - panic("implement me") -} - -func (fs *Fs) Destroy() { - //TODO implement me - panic("implement me") -} - -func (fs *Fs) Statfs(path string, stat *fuse.Statfs_t) int { - //TODO implement me - panic("implement me") -} - -func (fs *Fs) Mknod(path string, mode uint32, dev uint64) int { - //TODO implement me - panic("implement me") -} - -func (fs *Fs) Mkdir(path string, mode uint32) int { - //TODO implement me - panic("implement me") -} - -func (fs *Fs) Unlink(path string) int { - //TODO implement me - panic("implement me") -} - -func (fs *Fs) Rmdir(path string) int { - //TODO implement me - panic("implement me") -} - -func (fs *Fs) Link(oldpath string, newpath string) int { - //TODO implement me - panic("implement me") -} - -func (fs *Fs) Symlink(target string, newpath string) int { - //TODO implement me - panic("implement me") -} - -func (fs *Fs) Readlink(path string) (int, string) { - //TODO implement me - panic("implement me") -} - -func (fs *Fs) Rename(oldpath string, newpath string) int { - //TODO implement me - panic("implement me") -} - -func (fs *Fs) Chmod(path string, mode uint32) int { - //TODO implement me - panic("implement me") -} - -func (fs *Fs) Chown(path string, uid uint32, gid uint32) int { - //TODO implement me - panic("implement me") -} - -func (fs *Fs) Utimens(path string, tmsp []fuse.Timespec) int { - //TODO implement me - panic("implement me") -} - -func (fs *Fs) Access(path string, mask uint32) int { - //TODO implement me - panic("implement me") -} - -func (fs *Fs) Create(path string, flags int, mode uint32) (int, uint64) { - //TODO implement me - panic("implement me") -} - -func (fs *Fs) Open(path string, flags int) (int, uint64) { - //TODO implement me - panic("implement me") -} - -func (fs *Fs) Getattr(path string, stat *fuse.Stat_t, fh uint64) int { - //TODO implement me - panic("implement me") -} - -func (fs *Fs) Truncate(path string, size int64, fh uint64) int { - //TODO implement me - panic("implement me") -} - -func (fs *Fs) Read(path string, buff []byte, ofst int64, fh uint64) int { - //TODO implement me - panic("implement me") -} - -func (fs *Fs) Write(path string, buff []byte, ofst int64, fh uint64) int { - //TODO implement me - panic("implement me") -} - -func (fs *Fs) Flush(path string, fh uint64) int { - //TODO implement me - panic("implement me") -} - -func (fs *Fs) Release(path string, fh uint64) int { - //TODO implement me - panic("implement me") -} - -func (fs *Fs) Fsync(path string, datasync bool, fh uint64) int { - //TODO implement me - panic("implement me") -} - -func (fs *Fs) Opendir(path string) (int, uint64) { - //TODO implement me - panic("implement me") -} - -func (fs *Fs) Readdir(path string, fill func(name string, stat *fuse.Stat_t, ofst int64) bool, ofst int64, fh uint64) int { - //TODO implement me - panic("implement me") -} - -func (fs *Fs) Releasedir(path string, fh uint64) int { - //TODO implement me - panic("implement me") -} - -func (fs *Fs) Fsyncdir(path string, datasync bool, fh uint64) int { - //TODO implement me - panic("implement me") -} - -func (fs *Fs) Setxattr(path string, name string, value []byte, flags int) int { - //TODO implement me - panic("implement me") -} - -func (fs *Fs) Getxattr(path string, name string) (int, []byte) { - //TODO implement me - panic("implement me") -} - -func (fs *Fs) Removexattr(path string, name string) int { - //TODO implement me - panic("implement me") -} - -func (fs *Fs) Listxattr(path string, fill func(name string) bool) int { - //TODO implement me - panic("implement me") -} - -var _ fuse.FileSystemInterface = (*Fs)(nil) diff --git a/internal/fuse/mount.go b/internal/fuse/mount.go deleted file mode 100644 index 30e3d546..00000000 --- a/internal/fuse/mount.go +++ /dev/null @@ -1,9 +0,0 @@ -package fuse - -import "github.com/winfsp/cgofuse/fuse" - -func Mount(mountSrc, mountDst string, opts []string) { - fs := &Fs{RootFolder: mountSrc} - host := fuse.NewFileSystemHost(fs) - go host.Mount(mountDst, opts) -} diff --git a/internal/model/args.go b/internal/model/args.go index 393a716f..16a5c172 100644 --- a/internal/model/args.go +++ b/internal/model/args.go @@ -30,7 +30,7 @@ type Link struct { Header http.Header `json:"header"` // needed header (for url) RangeReader RangeReaderIF `json:"-"` // recommended way if can't use URL - Expiration *time.Duration // local cache expire Duration + Expiration *time.Duration // local cache expiration; not transferred by Clone //for accelerating request, use multi-thread downloading Concurrency int `json:"concurrency"` @@ -42,12 +42,12 @@ type Link struct { RequireReference bool `json:"-"` } +// Clone transfers ownership of l without inheriting its cache expiration. func (l *Link) Clone() *Link { return &Link{ URL: l.URL, Header: l.Header, RangeReader: l.RangeReader, - Expiration: l.Expiration, Concurrency: l.Concurrency, PartSize: l.PartSize, ContentLength: l.ContentLength, @@ -118,21 +118,3 @@ type SharingLinkArgs struct { type RangeReaderIF interface { RangeRead(ctx context.Context, httpRange http_range.Range) (io.ReadCloser, error) } - -type RangeReadCloserIF interface { - RangeReaderIF - utils.ClosersIF -} - -var _ RangeReadCloserIF = (*RangeReadCloser)(nil) - -type RangeReadCloser struct { - RangeReader RangeReaderIF - utils.Closers -} - -func (r *RangeReadCloser) RangeRead(ctx context.Context, httpRange http_range.Range) (io.ReadCloser, error) { - rc, err := r.RangeReader.RangeRead(ctx, httpRange) - r.Add(rc) - return rc, err -} diff --git a/internal/model/args_test.go b/internal/model/args_test.go new file mode 100644 index 00000000..a82ae44b --- /dev/null +++ b/internal/model/args_test.go @@ -0,0 +1,25 @@ +package model + +import ( + "testing" + "time" +) + +func TestLinkCloneTransfersOwnershipWithoutCachePolicy(t *testing.T) { + ttl := time.Minute + source := &Link{URL: "https://example.test/file", Expiration: &ttl} + + clone := source.Clone() + if clone.URL != source.URL { + t.Fatal("clone did not preserve transport data") + } + if clone.Expiration != nil { + t.Fatal("clone inherited source cache policy") + } + if err := clone.Close(); err != nil { + t.Fatal(err) + } + if !source.Expired() { + t.Fatal("closing clone did not release its source") + } +} diff --git a/internal/model/obj.go b/internal/model/obj.go index ba7467f7..1269b5b7 100644 --- a/internal/model/obj.go +++ b/internal/model/obj.go @@ -19,6 +19,10 @@ type ObjUnwrap interface { Unwrap() Obj } +type ObjDisplayName interface { + GetDisplayName() string +} + type Obj interface { GetSize() int64 GetName() string @@ -132,13 +136,17 @@ func ExtractFolder(objs []Obj, extractFolder string) { }) } -func WrapObjName(objs Obj) Obj { - return &ObjWrapName{Name: utils.MappingName(objs.GetName()), Obj: objs} +func WrapObjName(obj Obj) Obj { + name := obj.GetName() + if displayName, ok := obj.(ObjDisplayName); ok { + name = displayName.GetDisplayName() + } + return &ObjWrapName{Name: utils.MappingName(name), Obj: obj} } func WrapObjsName(objs []Obj) { for i := range objs { - objs[i] = &ObjWrapName{Name: utils.MappingName(objs[i].GetName()), Obj: objs[i]} + objs[i] = WrapObjName(objs[i]) } } diff --git a/internal/model/webdav_property.go b/internal/model/webdav_property.go new file mode 100644 index 00000000..cdfedb9e --- /dev/null +++ b/internal/model/webdav_property.go @@ -0,0 +1,11 @@ +package model + +// WebDAVProperty stores a dead WebDAV property for a resource. +type WebDAVProperty struct { + ID uint `json:"id" gorm:"primaryKey"` + Path string `json:"path" gorm:"uniqueIndex:idx_webdav_property"` + Namespace string `json:"namespace" gorm:"uniqueIndex:idx_webdav_property"` + Name string `json:"name" gorm:"uniqueIndex:idx_webdav_property"` + Lang string `json:"lang"` + InnerXML []byte `json:"inner_xml"` +} diff --git a/internal/multipart/session.go b/internal/multipart/session.go new file mode 100644 index 00000000..e25d1d1c --- /dev/null +++ b/internal/multipart/session.go @@ -0,0 +1,607 @@ +package multipart + +import ( + "context" + "errors" + "fmt" + "io" + "math" + "os" + stdpath "path" + "path/filepath" + "sync" + "sync/atomic" + "time" + + "github.com/OpenListTeam/OpenList/v4/internal/conf" + "github.com/OpenListTeam/OpenList/v4/internal/driver" + "github.com/OpenListTeam/OpenList/v4/internal/errs" + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/internal/op" + "github.com/OpenListTeam/OpenList/v4/internal/stream" + "github.com/OpenListTeam/OpenList/v4/pkg/utils" + "github.com/google/uuid" +) + +type State string + +const ( + StateReceiving State = "receiving" + StateCompleted State = "completed" + StateFailedRetriable State = "failed_retriable" + StateFailedPermanent State = "failed_permanent" + StateAborted State = "aborted" +) + +// WindowSlots bounds the per-session disk footprint to WindowSlots*ChunkSize. +var WindowSlots = 8 + +const ( + // defaultSessionTTL is the sliding inactivity timeout; it also serves as the + // grace period during which finished sessions remain queryable. + defaultSessionTTL = 30 * time.Minute + gcInterval = time.Minute +) + +var ( + ErrSessionNotFound = errors.New("multipart upload session not found") + ErrNotOwner = errors.New("multipart upload session belongs to another user") + // errAborted wraps context.Canceled so a driver blocked on the stream sees + // the abort as a canceled request and runs its cancellation cleanup. + errAborted = fmt.Errorf("multipart upload aborted: %w", context.Canceled) +) + +// putFile is the pipeline tail: resolve the storage and run the regular upload +// path. It mirrors the checks of fs.putDirectly (internal/fs/put.go) but calls +// op.Put directly so the driver's progress callback can be observed. +// It is a variable so session tests can stub the storage layer out. +var putFile = func(ctx context.Context, dstDirPath string, fs *stream.FileStream, up driver.UpdateProgress) error { + storage, dstDirActualPath, err := op.GetStorageAndActualPath(dstDirPath) + if err != nil { + return err + } + if storage.Config().NoUpload { + return errs.UploadNotSupported + } + return op.Put(ctx, storage, dstDirActualPath, fs, up) +} + +// Session is one multipart upload: metadata survives pipeline attempts, the +// window (chunk data) does not. +type Session struct { + ID string + Path string // full destination path (dir + name), already user-joined + DstDir string + Name string + Size int64 + ChunkSize int64 + Total int + Mimetype string + Modified time.Time + Hashes map[*utils.HashType]string + Creator *model.User + + mu sync.Mutex + state State + err error + attempt int + win *Window + done chan struct{} + cancel context.CancelFunc + prevCRCs []uint32 + prevSet []bool + + storagePct atomic.Uint64 // math.Float64bits of the driver progress (0-100) + lastActive atomic.Int64 // unix nano +} + +func (s *Session) touch() { s.lastActive.Store(time.Now().UnixNano()) } + +func (s *Session) setStoragePct(p float64) { s.storagePct.Store(math.Float64bits(p)) } + +// Snapshot is the wire representation of a session used by all endpoints. +type SessionSnapshot struct { + ID string `json:"upload_id"` + State State `json:"state"` + Attempt int `json:"attempt"` + Path string `json:"path"` + Size int64 `json:"size"` + ChunkSize int64 `json:"chunk_size"` + TotalChunks int `json:"total_chunks"` + Received [][2]int `json:"received"` + ReceivedBytes int64 `json:"received_bytes"` + Frontier int `json:"frontier"` + StorageProgress float64 `json:"storage_progress"` + Error string `json:"error,omitempty"` +} + +func (s *Session) snapshotLocked() SessionSnapshot { + snap := SessionSnapshot{ + ID: s.ID, + State: s.state, + Attempt: s.attempt, + Path: s.Path, + Size: s.Size, + ChunkSize: s.ChunkSize, + TotalChunks: s.Total, + Received: [][2]int{}, + StorageProgress: math.Float64frombits(s.storagePct.Load()), + } + if s.err != nil { + snap.Error = s.err.Error() + } + switch { + case s.state == StateCompleted: + snap.Received = [][2]int{{0, s.Total - 1}} + snap.ReceivedBytes = s.Size + snap.Frontier = s.Total + snap.StorageProgress = 100 + case s.win != nil: + ws := s.win.Snapshot() + snap.Received = ws.Received + snap.ReceivedBytes = ws.ReceivedBytes + snap.Frontier = ws.Frontier + } + return snap +} + +func (s *Session) Snapshot() SessionSnapshot { + s.mu.Lock() + defer s.mu.Unlock() + return s.snapshotLocked() +} + +// InitReq carries everything the handler parsed from the init request. +type InitReq struct { + User *model.User + Path string // full destination path, already user-joined + Size int64 + ChunkSize int64 // final chunk size in bytes, already clamped by the handler + Mimetype string + Modified time.Time + Hashes map[*utils.HashType]string +} + +// Manager owns all live sessions. Sessions are in-memory only (aligned with +// upload tasks not being persisted); a restart drops them and the ring files +// are swept on the next start. +type Manager struct { + mu sync.Mutex + byID map[string]*Session + byKey map[string]string + gcOnce sync.Once + ttl time.Duration // 0 means defaultSessionTTL; tests shrink it per instance +} + +func (m *Manager) sessionTTL() time.Duration { + if m.ttl > 0 { + return m.ttl + } + return defaultSessionTTL +} + +var DefaultManager = &Manager{ + byID: make(map[string]*Session), + byKey: make(map[string]string), +} + +func (m *Manager) dir() string { + return filepath.Join(conf.Conf.TempDir, "multipart") +} + +func sessionKey(userID uint, path string, size int64) string { + return fmt.Sprintf("%d|%s|%d", userID, path, size) +} + +// hashesQualifyResume reports whether two hash sets prove the client is +// re-uploading the same file: they must share at least one hash type and +// agree on every shared one. Path+size alone is NOT enough to resume into a +// receiving session — buffered chunks of a different same-sized file would be +// silently mixed into the result. +func hashesQualifyResume(old, new map[*utils.HashType]string) bool { + shared := false + for t, ov := range old { + if nv, ok := new[t]; ok { + if ov != nv { + return false + } + shared = true + } + } + return shared +} + +// StartGC sweeps ring files orphaned by a previous run and starts the expiry +// loop. It is called at server startup so orphans are reclaimed even if no +// multipart upload ever happens again; Init also calls it, so embedders that +// skip the server wiring still get GC lazily. +func (m *Manager) StartGC() { + m.ensureGC() +} + +func (m *Manager) ensureGC() { + m.gcOnce.Do(func() { + // sweep ring files orphaned by a previous run; bootstrap's CleanTempDir + // only runs when no transfer tasks are pending, so do not rely on it + _ = os.RemoveAll(m.dir()) + go func() { + ticker := time.NewTicker(gcInterval) + for range ticker.C { + m.gc() + } + }() + }) +} + +func (m *Manager) gc() { + deadline := time.Now().Add(-m.sessionTTL()).UnixNano() + m.mu.Lock() + var expired []*Session + for _, s := range m.byID { + if s.lastActive.Load() < deadline { + expired = append(expired, s) + } + } + m.mu.Unlock() + for _, s := range expired { + m.terminate(s, errors.New("multipart upload session expired")) + } +} + +// terminate aborts a session (if still receiving) and drops it from the maps. +func (m *Manager) terminate(s *Session, cause error) { + s.mu.Lock() + if s.state == StateReceiving { + s.state = StateAborted + s.err = cause + } + s.killAttemptLocked() + s.mu.Unlock() + m.remove(s) +} + +// killAttemptLocked stops the running pipeline attempt: the context cancel +// interrupts drivers blocked on network I/O, and closing the window wakes a +// driver blocked in Read (context cancellation cannot interrupt cond.Wait). +// The caller must hold s.mu and must have set the final state first. +func (s *Session) killAttemptLocked() { + if s.cancel != nil { + s.cancel() + } + if s.win != nil { + _ = s.win.CloseWithError(errAborted) + } +} + +func (m *Manager) remove(s *Session) { + m.mu.Lock() + defer m.mu.Unlock() + delete(m.byID, s.ID) + key := sessionKey(s.Creator.ID, s.Path, s.Size) + if m.byKey[key] == s.ID { + delete(m.byKey, key) + } +} + +func (m *Manager) get(user *model.User, id string) (*Session, error) { + m.mu.Lock() + s, ok := m.byID[id] + m.mu.Unlock() + if !ok { + return nil, ErrSessionNotFound + } + if s.Creator.ID != user.ID { + return nil, ErrNotOwner + } + return s, nil +} + +// Init creates a session and starts its pipeline, or returns the live session +// for the same (user, path, size) so an interrupted client resumes implicitly. +func (m *Manager) Init(req InitReq) (SessionSnapshot, bool, error) { + m.ensureGC() + if req.Size <= 0 { + return SessionSnapshot{}, false, fmt.Errorf("multipart upload requires a positive X-File-Size, got %d", req.Size) + } + if req.ChunkSize <= 0 { + return SessionSnapshot{}, false, fmt.Errorf("invalid chunk size %d", req.ChunkSize) + } + key := sessionKey(req.User.ID, req.Path, req.Size) + + m.mu.Lock() + if id, ok := m.byKey[key]; ok { + if s, ok := m.byID[id]; ok { + s.mu.Lock() + st := s.state + s.mu.Unlock() + // failed_retriable resumes unconditionally: nothing of the old + // attempt survives except CRCs, and the re-fill CRC check catches + // a changed file. A receiving session still holds data, so it only + // resumes when hashes prove it is the same file. + if st == StateFailedRetriable || + (st == StateReceiving && hashesQualifyResume(s.Hashes, req.Hashes)) { + m.mu.Unlock() + s.touch() + return s.Snapshot(), true, nil + } + // finished session, or same path+size without proof of identity: + // drop the old session and start fresh + m.mu.Unlock() + m.terminate(s, errors.New("superseded by a new upload of the same path and size")) + m.mu.Lock() + } + } + m.mu.Unlock() + + dstDir, name := stdpath.Split(req.Path) + s := &Session{ + ID: uuid.NewString(), + Path: req.Path, + DstDir: dstDir, + Name: name, + Size: req.Size, + ChunkSize: req.ChunkSize, + Mimetype: req.Mimetype, + Modified: req.Modified, + Hashes: req.Hashes, + Creator: req.User, + state: StateReceiving, + } + s.touch() + + s.mu.Lock() + if err := m.startAttemptLocked(s); err != nil { + s.mu.Unlock() + return SessionSnapshot{}, false, err + } + s.Total = s.win.TotalChunks() + snap := s.snapshotLocked() + s.mu.Unlock() + + m.mu.Lock() + m.byID[s.ID] = s + m.byKey[key] = s.ID + m.mu.Unlock() + return snap, false, nil +} + +// startAttemptLocked builds a fresh window and spawns the pipeline goroutine. +// The caller must hold s.mu. +func (m *Manager) startAttemptLocked(s *Session) error { + win, err := NewWindow(m.dir(), fmt.Sprintf("%s.%d", s.ID, s.attempt), s.ChunkSize, s.Size, WindowSlots) + if err != nil { + return err + } + ctx, cancel := context.WithCancel(context.WithValue(context.Background(), conf.UserKey, s.Creator)) + done := make(chan struct{}) + s.win = win + s.cancel = cancel + s.done = done + s.state = StateReceiving + s.err = nil + s.setStoragePct(0) + + fileStream := &stream.FileStream{ + Obj: &model.Object{ + Name: s.Name, + Size: s.Size, + Modified: s.Modified, + HashInfo: utils.NewHashInfoByMap(s.Hashes), + }, + Reader: win, + Mimetype: s.Mimetype, + } + fileStream.Add(win) + + dstDir := s.DstDir + put := putFile // capture: the seam must not be read after spawn + go func() { + err := put(ctx, dstDir, fileStream, s.setStoragePct) + s.finishAttempt(win, err) + close(done) + }() + return nil +} + +// finishAttempt records the pipeline outcome and harvests the CRC table for +// re-fill verification. The window's data is gone at this point (op.Put closed +// it); only metadata survives. +func (s *Session) finishAttempt(win *Window, err error) { + crcs, set := win.CRCs() + _ = win.Close() // op.Put already closed it; make sure the ring file is gone anyway + + s.mu.Lock() + defer s.mu.Unlock() + s.touch() + if s.win == win { + s.win = nil + } + s.prevCRCs, s.prevSet = crcs, set + switch { + case err == nil: + s.state = StateCompleted + s.err = nil + s.setStoragePct(100) + case s.state != StateReceiving: + // Abort/expiry already labeled this attempt; keep that state. + if s.err == nil { + s.err = err + } + case isPermanentPutError(err): + s.state = StateFailedPermanent + s.err = err + default: + s.state = StateFailedRetriable + s.err = err + s.attempt++ + } +} + +func isPermanentPutError(err error) bool { + if errors.Is(err, context.Canceled) { + return false + } + for _, target := range []error{ + errs.UploadNotSupported, + errs.PermissionDenied, + errs.StorageNotFound, + errs.ObjectAlreadyExists, + errs.RelativePath, + errs.IgnoredSystemFile, + } { + if errors.Is(err, target) { + return true + } + } + return false +} + +// Chunk feeds one chunk into the session. Re-sending chunk 0 to a +// failed_retriable session re-fills it: a fresh window and pipeline attempt. +func (m *Manager) Chunk(user *model.User, id string, idx int, body io.Reader) (SessionSnapshot, error) { + s, err := m.get(user, id) + if err != nil { + return SessionSnapshot{}, err + } + s.touch() + + s.mu.Lock() + switch s.state { + case StateReceiving: + case StateFailedRetriable: + if idx != 0 { + snap := s.snapshotLocked() + s.mu.Unlock() + return snap, fmt.Errorf("upload attempt failed, resend from chunk 0 to retry: %w", s.err) + } + if err := m.startAttemptLocked(s); err != nil { + snap := s.snapshotLocked() + s.mu.Unlock() + return snap, err + } + case StateCompleted: + snap := s.snapshotLocked() + s.mu.Unlock() + return snap, nil // idempotent: stragglers after rapid-upload/finish succeed + default: + snap := s.snapshotLocked() + s.mu.Unlock() + return snap, fmt.Errorf("session is %s: %w", s.state, s.err) + } + win := s.win + var prevCRC uint32 + hasPrev := false + if idx < len(s.prevSet) && s.prevSet[idx] { + prevCRC, hasPrev = s.prevCRCs[idx], true + } + s.mu.Unlock() + + crc, err := win.WriteChunk(idx, body) + if err != nil { + // A closed window means the pipeline ended while this chunk was in + // flight — rapid upload makes this the NORMAL case: the driver + // succeeds off the hash alone with chunks still arriving. The window + // closes (op.Put's defer) moments before the verdict is recorded, so + // wait for the verdict instead of racing it, then absorb the chunk + // idempotently if the upload in fact succeeded. + if errors.Is(err, ErrClosed) || errors.Is(err, context.Canceled) { + s.mu.Lock() + done := s.done + s.mu.Unlock() + select { + case <-done: + case <-time.After(10 * time.Second): // pipeline teardown is µs-scale; never expected + } + } + s.mu.Lock() + completed := s.state == StateCompleted + snap := s.snapshotLocked() + s.mu.Unlock() + if completed { + return snap, nil + } + return snap, err + } + if hasPrev && crc != prevCRC { + err := fmt.Errorf("chunk %d content changed between attempts, aborting", idx) + s.mu.Lock() + s.state = StateFailedPermanent + s.err = err + s.killAttemptLocked() + s.mu.Unlock() + return s.Snapshot(), err + } + return s.Snapshot(), nil +} + +// Complete waits for the pipeline outcome. It refuses to block while chunks +// are still missing, so a buggy client cannot park a connection for the TTL. +func (m *Manager) Complete(ctx context.Context, user *model.User, id string) (SessionSnapshot, error) { + s, err := m.get(user, id) + if err != nil { + return SessionSnapshot{}, err + } + s.touch() + for { + s.mu.Lock() + st := s.state + done := s.done + if st == StateReceiving && s.win != nil { + if ws := s.win.Snapshot(); ws.ReceivedBytes < s.Size { + snap := s.snapshotLocked() + s.mu.Unlock() + return snap, fmt.Errorf("cannot complete: %d of %d bytes received", ws.ReceivedBytes, s.Size) + } + } + s.mu.Unlock() + if st != StateReceiving { + break + } + select { + case <-done: + case <-ctx.Done(): + return s.Snapshot(), ctx.Err() + } + } + s.touch() + s.mu.Lock() + st, serr := s.state, s.err + snap := s.snapshotLocked() + s.mu.Unlock() + if st == StateCompleted { + m.remove(s) // served its purpose; frees the key for future uploads + return snap, nil + } + return snap, fmt.Errorf("upload failed (%s): %w", st, serr) +} + +// Status looks a session up by id. +func (m *Manager) Status(user *model.User, id string) (SessionSnapshot, error) { + s, err := m.get(user, id) + if err != nil { + return SessionSnapshot{}, err + } + s.touch() + return s.Snapshot(), nil +} + +// Find looks a live session up by destination path and size, for resume discovery. +func (m *Manager) Find(user *model.User, path string, size int64) (SessionSnapshot, error) { + m.mu.Lock() + id, ok := m.byKey[sessionKey(user.ID, path, size)] + m.mu.Unlock() + if !ok { + return SessionSnapshot{}, ErrSessionNotFound + } + return m.Status(user, id) +} + +// Abort cancels the pipeline and forgets the session immediately. +func (m *Manager) Abort(user *model.User, id string) error { + s, err := m.get(user, id) + if err != nil { + return err + } + m.terminate(s, errors.New("multipart upload aborted by client")) + return nil +} diff --git a/internal/multipart/session_test.go b/internal/multipart/session_test.go new file mode 100644 index 00000000..daec7f80 --- /dev/null +++ b/internal/multipart/session_test.go @@ -0,0 +1,669 @@ +package multipart + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "os" + "sync" + "testing" + "time" + + "github.com/OpenListTeam/OpenList/v4/internal/conf" + "github.com/OpenListTeam/OpenList/v4/internal/driver" + "github.com/OpenListTeam/OpenList/v4/internal/errs" + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/internal/stream" + "github.com/OpenListTeam/OpenList/v4/pkg/utils" +) + +// Tests in this file share the putFile seam and must not run in parallel. + +func setupSessionTest(t *testing.T) *Manager { + t.Helper() + oldConf := conf.Conf + conf.Conf = conf.DefaultConfig(t.TempDir()) + t.Cleanup(func() { conf.Conf = oldConf }) + return &Manager{byID: make(map[string]*Session), byKey: make(map[string]string)} +} + +func stubPut(t *testing.T, fn func(ctx context.Context, dst string, fs *stream.FileStream, up driver.UpdateProgress) error) { + t.Helper() + orig := putFile + putFile = fn + t.Cleanup(func() { putFile = orig }) +} + +func testUser() *model.User { return &model.User{ID: 7, Username: "tester"} } + +func initReq(user *model.User, size, chunkSize int64) InitReq { + return InitReq{ + User: user, + Path: "/local/test.bin", + Size: size, + ChunkSize: chunkSize, + Mimetype: "application/octet-stream", + Modified: time.Unix(1700000000, 0), + } +} + +func sendChunk(t *testing.T, m *Manager, user *model.User, id string, data []byte, idx int, chunkSize int64) SessionSnapshot { + t.Helper() + snap, err := m.Chunk(user, id, idx, bytes.NewReader(chunkOf(data, idx, chunkSize))) + if err != nil { + t.Fatalf("Chunk(%d): %v", idx, err) + } + return snap +} + +func waitState(t *testing.T, m *Manager, user *model.User, id string, want State) SessionSnapshot { + t.Helper() + deadline := time.Now().Add(10 * time.Second) + for { + snap, err := m.Status(user, id) + if err != nil { + t.Fatalf("Status: %v", err) + } + if snap.State == want { + return snap + } + if time.Now().After(deadline) { + t.Fatalf("session state = %s, want %s (err: %s)", snap.State, want, snap.Error) + } + time.Sleep(2 * time.Millisecond) + } +} + +func TestSessionHappyPath(t *testing.T) { + m := setupSessionTest(t) + const chunkSize = 1024 + totalSize := int64(3*chunkSize + 300) + data := genData(totalSize) + user := testUser() + + got := make(chan []byte, 1) + stubPut(t, func(ctx context.Context, dst string, fs *stream.FileStream, up driver.UpdateProgress) error { + defer fs.Close() + if dst != "/local/" { + return fmt.Errorf("unexpected dst dir %q", dst) + } + if fs.GetName() != "test.bin" || fs.GetSize() != totalSize { + return fmt.Errorf("unexpected stream meta %s/%d", fs.GetName(), fs.GetSize()) + } + b, err := io.ReadAll(fs) + if err != nil { + return err + } + up(100) + got <- b + return nil + }) + + snap, resumed, err := m.Init(initReq(user, totalSize, chunkSize)) + if err != nil || resumed { + t.Fatalf("Init = (resumed=%v, err=%v)", resumed, err) + } + if snap.TotalChunks != 4 || snap.State != StateReceiving { + t.Fatalf("init snapshot = %+v", snap) + } + for _, idx := range []int{1, 0, 3, 2} { // out of order on purpose + sendChunk(t, m, user, snap.ID, data, idx, chunkSize) + } + final, err := m.Complete(context.Background(), user, snap.ID) + if err != nil { + t.Fatalf("Complete: %v", err) + } + if final.State != StateCompleted || final.StorageProgress != 100 { + t.Fatalf("final snapshot = %+v", final) + } + if !bytes.Equal(<-got, data) { + t.Fatal("driver received different bytes") + } + if _, err := m.Status(user, snap.ID); !errors.Is(err, ErrSessionNotFound) { + t.Fatalf("session should be removed after Complete, got %v", err) + } +} + +func TestSessionRetriableRefill(t *testing.T) { + m := setupSessionTest(t) + const chunkSize = 1024 + totalSize := int64(2 * chunkSize) + data := genData(totalSize) + user := testUser() + + attempts := 0 + got := make(chan []byte, 1) + stubPut(t, func(ctx context.Context, dst string, fs *stream.FileStream, up driver.UpdateProgress) error { + defer fs.Close() + attempts++ + if attempts == 1 { + buf := make([]byte, chunkSize) + if _, err := io.ReadFull(fs, buf); err != nil { + return err + } + return errors.New("transient storage hiccup") + } + b, err := io.ReadAll(fs) + if err != nil { + return err + } + got <- b + return nil + }) + + snap, _, err := m.Init(initReq(user, totalSize, chunkSize)) + if err != nil { + t.Fatalf("Init: %v", err) + } + sendChunk(t, m, user, snap.ID, data, 0, chunkSize) + failed := waitState(t, m, user, snap.ID, StateFailedRetriable) + if failed.Attempt != 1 { + t.Fatalf("attempt = %d, want 1", failed.Attempt) + } + if len(failed.Received) != 0 { + t.Fatalf("failed session must report nothing received, got %v", failed.Received) + } + + // chunks other than 0 are rejected until the client restarts the fill + if _, err := m.Chunk(user, snap.ID, 1, bytes.NewReader(chunkOf(data, 1, chunkSize))); err == nil { + t.Fatal("chunk 1 on failed_retriable session: expected error") + } + // re-fill from chunk 0 respawns the pipeline + sendChunk(t, m, user, snap.ID, data, 0, chunkSize) + sendChunk(t, m, user, snap.ID, data, 1, chunkSize) + final, err := m.Complete(context.Background(), user, snap.ID) + if err != nil { + t.Fatalf("Complete after refill: %v", err) + } + if final.State != StateCompleted || attempts != 2 { + t.Fatalf("state=%s attempts=%d, want completed/2", final.State, attempts) + } + if !bytes.Equal(<-got, data) { + t.Fatal("driver received different bytes after refill") + } +} + +func TestSessionPermanentFailure(t *testing.T) { + m := setupSessionTest(t) + const chunkSize = 1024 + data := genData(chunkSize) + user := testUser() + + stubPut(t, func(ctx context.Context, dst string, fs *stream.FileStream, up driver.UpdateProgress) error { + defer fs.Close() + return fmt.Errorf("denied: %w", errs.PermissionDenied) + }) + + snap, _, err := m.Init(initReq(user, chunkSize, chunkSize)) + if err != nil { + t.Fatalf("Init: %v", err) + } + waitState(t, m, user, snap.ID, StateFailedPermanent) + if _, err := m.Chunk(user, snap.ID, 0, bytes.NewReader(data)); err == nil { + t.Fatal("chunk on failed_permanent session: expected error") + } + if _, err := m.Complete(context.Background(), user, snap.ID); err == nil { + t.Fatal("Complete on failed_permanent session: expected error") + } +} + +func TestRefillCRCMismatch(t *testing.T) { + m := setupSessionTest(t) + const chunkSize = 1024 + totalSize := int64(2 * chunkSize) + data := genData(totalSize) + user := testUser() + + attempts := 0 + stubPut(t, func(ctx context.Context, dst string, fs *stream.FileStream, up driver.UpdateProgress) error { + defer fs.Close() + attempts++ + if attempts == 1 { + buf := make([]byte, chunkSize) + if _, err := io.ReadFull(fs, buf); err != nil { + return err + } + return errors.New("transient") + } + _, err := io.ReadAll(fs) + return err + }) + + snap, _, err := m.Init(initReq(user, totalSize, chunkSize)) + if err != nil { + t.Fatalf("Init: %v", err) + } + sendChunk(t, m, user, snap.ID, data, 0, chunkSize) + waitState(t, m, user, snap.ID, StateFailedRetriable) + + tampered := genData(chunkSize + 5)[:chunkSize] // different content, same length + if _, err := m.Chunk(user, snap.ID, 0, bytes.NewReader(tampered)); err == nil { + t.Fatal("re-fill with changed content: expected error") + } + waitState(t, m, user, snap.ID, StateFailedPermanent) +} + +func TestCompleteRefusesIncomplete(t *testing.T) { + m := setupSessionTest(t) + const chunkSize = 1024 + totalSize := int64(3 * chunkSize) + data := genData(totalSize) + user := testUser() + + stubPut(t, func(ctx context.Context, dst string, fs *stream.FileStream, up driver.UpdateProgress) error { + defer fs.Close() + _, err := io.ReadAll(fs) + return err + }) + + snap, _, err := m.Init(initReq(user, totalSize, chunkSize)) + if err != nil { + t.Fatalf("Init: %v", err) + } + sendChunk(t, m, user, snap.ID, data, 0, chunkSize) + if _, err := m.Complete(context.Background(), user, snap.ID); err == nil { + t.Fatal("Complete with missing chunks: expected error") + } + // unblock the pipeline goroutine before the test tears down + if err := m.Abort(user, snap.ID); err != nil { + t.Fatalf("Abort: %v", err) + } +} + +func TestAbortAndOwnership(t *testing.T) { + m := setupSessionTest(t) + const chunkSize = 1024 + totalSize := int64(4 * chunkSize) + data := genData(totalSize) + user := testUser() + + stubPut(t, func(ctx context.Context, dst string, fs *stream.FileStream, up driver.UpdateProgress) error { + defer fs.Close() + _, err := io.ReadAll(fs) + return err + }) + + snap, _, err := m.Init(initReq(user, totalSize, chunkSize)) + if err != nil { + t.Fatalf("Init: %v", err) + } + sendChunk(t, m, user, snap.ID, data, 0, chunkSize) + + stranger := &model.User{ID: 99, Username: "stranger"} + if _, err := m.Status(stranger, snap.ID); !errors.Is(err, ErrNotOwner) { + t.Fatalf("stranger Status err = %v, want ErrNotOwner", err) + } + if err := m.Abort(stranger, snap.ID); !errors.Is(err, ErrNotOwner) { + t.Fatalf("stranger Abort err = %v, want ErrNotOwner", err) + } + if err := m.Abort(user, snap.ID); err != nil { + t.Fatalf("Abort: %v", err) + } + if _, err := m.Status(user, snap.ID); !errors.Is(err, ErrSessionNotFound) { + t.Fatalf("Status after Abort err = %v, want ErrSessionNotFound", err) + } +} + +func TestExpiry(t *testing.T) { + m := setupSessionTest(t) + m.ttl = 30 * time.Millisecond + + const chunkSize = 1024 + user := testUser() + stubPut(t, func(ctx context.Context, dst string, fs *stream.FileStream, up driver.UpdateProgress) error { + defer fs.Close() + _, err := io.ReadAll(fs) + return err + }) + + snap, _, err := m.Init(initReq(user, 4*chunkSize, chunkSize)) + if err != nil { + t.Fatalf("Init: %v", err) + } + time.Sleep(60 * time.Millisecond) + m.gc() + if _, err := m.Status(user, snap.ID); !errors.Is(err, ErrSessionNotFound) { + t.Fatalf("Status after expiry err = %v, want ErrSessionNotFound", err) + } + // the terminated pipeline must release the ring file + deadline := time.Now().Add(5 * time.Second) + for { + entries, _ := os.ReadDir(m.dir()) + if len(entries) == 0 { + break + } + if time.Now().After(deadline) { + names := make([]string, 0, len(entries)) + for _, e := range entries { + names = append(names, e.Name()) + } + t.Fatalf("ring files not cleaned up after expiry: %v", names) + } + time.Sleep(5 * time.Millisecond) + } +} + +func TestInitResume(t *testing.T) { + m := setupSessionTest(t) + const chunkSize = 1024 + totalSize := int64(4 * chunkSize) + data := genData(totalSize) + user := testUser() + + stubPut(t, func(ctx context.Context, dst string, fs *stream.FileStream, up driver.UpdateProgress) error { + defer fs.Close() + _, err := io.ReadAll(fs) + return err + }) + + hashed := initReq(user, totalSize, chunkSize) + hashed.Hashes = map[*utils.HashType]string{utils.MD5: "0123456789abcdef0123456789abcdef"} + snap, _, err := m.Init(hashed) + if err != nil { + t.Fatalf("Init: %v", err) + } + m.mu.Lock() + firstSession := m.byID[snap.ID] + m.mu.Unlock() + sessions := []*Session{firstSession} + t.Cleanup(func() { + for _, session := range sessions { + if session == nil { + continue + } + m.terminate(session, context.Canceled) + <-session.done + } + }) + sendChunk(t, m, user, snap.ID, data, 2, chunkSize) + + // same hashes prove the same file: resume with buffered chunks skippable + again, resumed, err := m.Init(hashed) + if err != nil || !resumed { + t.Fatalf("hashed re-Init = (resumed=%v, err=%v), want resumed", resumed, err) + } + if again.ID != snap.ID { + t.Fatalf("resumed session id = %s, want %s", again.ID, snap.ID) + } + if len(again.Received) != 1 || again.Received[0] != [2]int{2, 2} { + t.Fatalf("resumed received = %v, want [[2,2]]", again.Received) + } + + // a different size is a different upload + other, resumed, err := m.Init(initReq(user, totalSize+1, chunkSize)) + if err != nil || resumed || other.ID == snap.ID { + t.Fatalf("different-size Init = (id=%s, resumed=%v, err=%v)", other.ID, resumed, err) + } + m.mu.Lock() + sessions = append(sessions, m.byID[other.ID]) + m.mu.Unlock() + + if _, err := m.Find(user, "/local/test.bin", totalSize); err != nil { + t.Fatalf("Find: %v", err) + } + if _, err := m.Find(user, "/local/nope.bin", totalSize); !errors.Is(err, ErrSessionNotFound) { + t.Fatalf("Find miss err = %v, want ErrSessionNotFound", err) + } + + // without hashes, path+size cannot prove identity against a receiving + // session holding buffered data — the old session must be dropped, or a + // same-sized different file would be silently mixed into the result + bare := initReq(user, totalSize, chunkSize) + fresh, resumed, err := m.Init(bare) + if err != nil || resumed || fresh.ID == snap.ID { + t.Fatalf("bare re-Init = (id=%s, resumed=%v, err=%v), want a fresh session", fresh.ID, resumed, err) + } + m.mu.Lock() + sessions = append(sessions, m.byID[fresh.ID]) + m.mu.Unlock() + if _, err := m.Status(user, snap.ID); !errors.Is(err, ErrSessionNotFound) { + t.Fatalf("superseded session err = %v, want ErrSessionNotFound", err) + } +} + +func TestResumeFailedRetriableWithoutHash(t *testing.T) { + m := setupSessionTest(t) + const chunkSize = 1024 + totalSize := int64(2 * chunkSize) + data := genData(totalSize) + user := testUser() + + attempts := 0 + stubPut(t, func(ctx context.Context, dst string, fs *stream.FileStream, up driver.UpdateProgress) error { + defer fs.Close() + attempts++ + if attempts == 1 { + buf := make([]byte, chunkSize) + if _, err := io.ReadFull(fs, buf); err != nil { + return err + } + return errors.New("transient") + } + _, err := io.ReadAll(fs) + return err + }) + + snap, _, err := m.Init(initReq(user, totalSize, chunkSize)) + if err != nil { + t.Fatalf("Init: %v", err) + } + sendChunk(t, m, user, snap.ID, data, 0, chunkSize) + waitState(t, m, user, snap.ID, StateFailedRetriable) + + // a failed_retriable session keeps no chunk data, so identity proof is not + // required to resume: the retry-button flow works without rapid hashing, + // and the re-fill CRC check still catches a changed file + again, resumed, err := m.Init(initReq(user, totalSize, chunkSize)) + if err != nil || !resumed || again.ID != snap.ID { + t.Fatalf("re-Init on failed_retriable = (id=%s, resumed=%v, err=%v), want resumed same session", again.ID, resumed, err) + } + sendChunk(t, m, user, snap.ID, data, 0, chunkSize) + sendChunk(t, m, user, snap.ID, data, 1, chunkSize) + if _, err := m.Complete(context.Background(), user, snap.ID); err != nil { + t.Fatalf("Complete after hashless refill: %v", err) + } +} + +func TestRapidUploadShortCircuit(t *testing.T) { + m := setupSessionTest(t) + const chunkSize = 1024 + totalSize := int64(6 * chunkSize) + data := genData(totalSize) + user := testUser() + + wantMD5 := "0123456789abcdef0123456789abcdef" + wantSHA1 := "da39a3ee5e6b4b0d3255bfef95601890afd80709" + stubPut(t, func(ctx context.Context, dst string, fs *stream.FileStream, up driver.UpdateProgress) error { + defer fs.Close() + // the hash provided at init must reach the driver through the stream, + // that is what lets PutRapid-style drivers skip the transfer entirely + if got := fs.GetHash().GetHash(utils.MD5); got != wantMD5 { + return fmt.Errorf("md5 not propagated: %q", got) + } + if got := fs.GetHash().GetHash(utils.SHA1); got != wantSHA1 { + return fmt.Errorf("sha1 not propagated: %q", got) + } + time.Sleep(50 * time.Millisecond) // simulated rapid-upload API round trip + up(100) + return nil // rapid upload hit: succeed without reading the stream + }) + + req := initReq(user, totalSize, chunkSize) + req.Hashes = map[*utils.HashType]string{utils.MD5: wantMD5, utils.SHA1: wantSHA1} + snap, _, err := m.Init(req) + if err != nil { + t.Fatalf("Init: %v", err) + } + + // spam chunks across the completion moment: some land while receiving, + // some race the window close, some arrive after completion — with the + // completed-session absorption none of them may surface an error + var wg sync.WaitGroup + errCh := make(chan error, 64) + for w := 0; w < 4; w++ { + wg.Add(1) + go func(w int) { + defer wg.Done() + for round := 0; round < 8; round++ { + idx := (w*8 + round) % 6 + _, err := m.Chunk(user, snap.ID, idx, bytes.NewReader(chunkOf(data, idx, chunkSize))) + // flow-control signals are part of the protocol, not failures + if err != nil && !errors.Is(err, ErrChunkInFlight) && !errors.Is(err, ErrOutOfWindow) { + errCh <- fmt.Errorf("worker %d chunk %d: %w", w, idx, err) + return + } + time.Sleep(5 * time.Millisecond) + } + }(w) + } + wg.Wait() + close(errCh) + for err := range errCh { + t.Error(err) + } + + final, err := m.Complete(context.Background(), user, snap.ID) + if err != nil { + t.Fatalf("Complete: %v", err) + } + if final.State != StateCompleted || final.StorageProgress != 100 { + t.Fatalf("final snapshot = %+v", final) + } +} + +// TestChunkRacesRapidCompletion pins the exact race the completed-session +// absorption exists for: a chunk request grabs the live window, stalls while +// receiving its body, the pipeline completes off the hash alone (rapid +// upload) and closes the window — the stalled chunk must then succeed +// idempotently instead of surfacing "window closed" to a client whose upload +// in fact just finished. +func TestChunkRacesRapidCompletion(t *testing.T) { + m := setupSessionTest(t) + const chunkSize = 1024 + totalSize := int64(2 * chunkSize) + data := genData(totalSize) + user := testUser() + + proceed := make(chan struct{}) + stubPut(t, func(ctx context.Context, dst string, fs *stream.FileStream, up driver.UpdateProgress) error { + defer fs.Close() + <-proceed // the rapid-upload verdict arrives when the test says so + return nil + }) + + snap, _, err := m.Init(initReq(user, totalSize, chunkSize)) + if err != nil { + t.Fatalf("Init: %v", err) + } + + gate := make(chan struct{}) + type result struct { + snap SessionSnapshot + err error + } + resCh := make(chan result, 1) + go func() { + s, e := m.Chunk(user, snap.ID, 0, + &gatedReader{release: gate, inner: bytes.NewReader(chunkOf(data, 0, chunkSize))}) + resCh <- result{s, e} + }() + + // wait until the chunk writer holds slot 0 in the filling state + m.mu.Lock() + sess := m.byID[snap.ID] + m.mu.Unlock() + sess.mu.Lock() + win := sess.win + sess.mu.Unlock() + deadline := time.Now().Add(5 * time.Second) + for { + win.mu.Lock() + filling := win.slotState[0] == slotFilling + win.mu.Unlock() + if filling { + break + } + if time.Now().After(deadline) { + t.Fatal("chunk writer never reached the filling state") + } + time.Sleep(time.Millisecond) + } + + close(proceed) // rapid upload succeeds, window closes + waitState(t, m, user, snap.ID, StateCompleted) // completion recorded + close(gate) // stalled chunk finishes against the closed window + + res := <-resCh + if res.err != nil { + t.Fatalf("in-flight chunk across rapid completion must be absorbed, got: %v", res.err) + } + if res.snap.State != StateCompleted { + t.Fatalf("absorbed chunk snapshot state = %s, want completed", res.snap.State) + } +} + +// TestChunkDuringCompletionGap covers the moment between op.Put closing the +// window (its defer) and the verdict being recorded: a chunk hitting the +// closed window inside that gap must wait for the verdict and be absorbed, +// not bounce a "window closed" error at a client whose upload just succeeded. +func TestChunkDuringCompletionGap(t *testing.T) { + m := setupSessionTest(t) + const chunkSize = 1024 + totalSize := int64(2 * chunkSize) + data := genData(totalSize) + user := testUser() + + windowClosed := make(chan struct{}) + allowVerdict := make(chan struct{}) + stubPut(t, func(ctx context.Context, dst string, fs *stream.FileStream, up driver.UpdateProgress) error { + _ = fs.Close() // what op.Put's defer does before Put returns + close(windowClosed) + <-allowVerdict // hold the pipeline return open: this IS the gap + return nil + }) + + snap, _, err := m.Init(initReq(user, totalSize, chunkSize)) + if err != nil { + t.Fatalf("Init: %v", err) + } + <-windowClosed + + type result struct { + snap SessionSnapshot + err error + } + resCh := make(chan result, 1) + go func() { + s, e := m.Chunk(user, snap.ID, 0, bytes.NewReader(chunkOf(data, 0, chunkSize))) + resCh <- result{s, e} + }() + + select { + case r := <-resCh: + t.Fatalf("chunk inside the gap returned early with (%s, %v); it must wait for the verdict", r.snap.State, r.err) + case <-time.After(150 * time.Millisecond): + // still waiting on the verdict, as designed + } + + close(allowVerdict) + select { + case r := <-resCh: + if r.err != nil { + t.Fatalf("gap chunk must be absorbed after completion, got: %v", r.err) + } + if r.snap.State != StateCompleted { + t.Fatalf("gap chunk snapshot state = %s, want completed", r.snap.State) + } + case <-time.After(5 * time.Second): + t.Fatal("gap chunk never returned after the verdict") + } +} + +func TestInitRejectsBadSize(t *testing.T) { + m := setupSessionTest(t) + if _, _, err := m.Init(initReq(testUser(), 0, 1024)); err == nil { + t.Fatal("Init with size 0: expected error") + } +} diff --git a/internal/multipart/window.go b/internal/multipart/window.go new file mode 100644 index 00000000..01e72670 --- /dev/null +++ b/internal/multipart/window.go @@ -0,0 +1,377 @@ +package multipart + +import ( + "errors" + "fmt" + "hash/crc32" + "io" + "os" + "path/filepath" + "sync" + "time" + + "github.com/OpenListTeam/OpenList/v4/pkg/utils" +) + +const ( + slotFree uint8 = iota + slotFilling + slotReady +) + +var ( + // ErrClosed is the sticky error after Close; all pending and future reads/writes fail with it. + ErrClosed = errors.New("multipart upload window closed") + // ErrChunkInFlight means another request is uploading the same chunk right now. + ErrChunkInFlight = errors.New("chunk is being uploaded by another request") + // ErrOutOfWindow means the chunk is still too far ahead of the consumption + // frontier after waiting WindowWaitTimeout; the client should back off and + // resend it later (flow control, not a failure). + ErrOutOfWindow = errors.New("chunk is out of the receiving window") +) + +// WindowWaitTimeout bounds how long WriteChunk blocks waiting for its slot. +// Browsers cannot reliably read responses sent before the request body is +// consumed (they report a network error), so under backpressure it is far +// better to hold the request until a slot frees — the wait must just stay +// well below CDN request deadlines (Cloudflare: ~100s). Tests shrink this. +var WindowWaitTimeout = 10 * time.Second + +// Window reassembles concurrently uploaded chunks into a sequential stream. +// Chunks land in a ring file of slots*chunkSize bytes (chunk i -> slot i%slots), +// and Read serves bytes in order, blocking until the next needed chunk arrives. +// A chunk slot is released as soon as the reader crosses its boundary, so the +// disk footprint is bounded by slots*chunkSize regardless of the file size. +// +// WriteChunk is safe for concurrent use; Read must be called from a single +// goroutine (the same contract as the FileStreamer it backs). +type Window struct { + mu sync.Mutex + cond *sync.Cond + + f *os.File + path string + + chunkSize int64 + totalSize int64 + total int + slots int + + slotState []uint8 + slotChunk []int + readPos int64 + + crcs []uint32 + crcSet []bool + + err error +} + +// Snapshot describes the receiving state, used for status responses and resume. +type Snapshot struct { + // Frontier is the next chunk index to be consumed (== TotalChunks when the stream is fully consumed). + Frontier int + // ReadPos is the number of bytes already consumed by the pipeline. + ReadPos int64 + // ReceivedBytes is the number of payload bytes received from the client (consumed + buffered). + ReceivedBytes int64 + // Received holds inclusive ranges of chunk indexes the client does not need to resend. + Received [][2]int +} + +func NewWindow(dir, id string, chunkSize, totalSize int64, slots int) (*Window, error) { + if chunkSize <= 0 || totalSize <= 0 || slots <= 0 { + return nil, fmt.Errorf("invalid window params: chunkSize=%d totalSize=%d slots=%d", chunkSize, totalSize, slots) + } + if err := os.MkdirAll(dir, 0o700); err != nil { + return nil, err + } + path := filepath.Join(dir, id+".ring") + f, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE|os.O_EXCL, 0o600) + if err != nil { + return nil, err + } + total := int((totalSize + chunkSize - 1) / chunkSize) + w := &Window{ + f: f, + path: path, + chunkSize: chunkSize, + totalSize: totalSize, + total: total, + slots: slots, + slotState: make([]uint8, slots), + slotChunk: make([]int, slots), + crcs: make([]uint32, total), + crcSet: make([]bool, total), + } + for i := range w.slotChunk { + w.slotChunk[i] = -1 + } + w.cond = sync.NewCond(&w.mu) + return w, nil +} + +func (w *Window) TotalChunks() int { return w.total } + +// ChunkLen returns the payload length of chunk idx (the last chunk may be short). +func (w *Window) ChunkLen(idx int) int64 { + if idx == w.total-1 { + return w.totalSize - int64(idx)*w.chunkSize + } + return w.chunkSize +} + +func stopTimer(t *time.Timer) { + if t != nil { + t.Stop() + } +} + +// frontier returns the next chunk index to be consumed. Callers must hold mu. +func (w *Window) frontier() int { + if w.readPos >= w.totalSize { + return w.total + } + return int(w.readPos / w.chunkSize) +} + +// WriteChunk reads exactly the chunk payload from r into the ring and returns its CRC32 (IEEE). +// Re-sending an already buffered or consumed chunk succeeds immediately without touching data. +func (w *Window) WriteChunk(idx int, r io.Reader) (uint32, error) { + w.mu.Lock() + if w.err != nil { + w.mu.Unlock() + return 0, w.err + } + if idx < 0 || idx >= w.total { + w.mu.Unlock() + return 0, fmt.Errorf("chunk index %d out of range [0,%d)", idx, w.total) + } + length := w.ChunkLen(idx) + slot := idx % w.slots + // Admission control with a bounded wait: instead of bouncing a chunk the + // moment its slot is busy, park the request until the reader frees the + // slot. Rejecting fast would answer before the request body is read, which + // browsers surface as a network error — so under backpressure, waiting IS + // the flow control. sync.Cond has no timed wait; a timer broadcast wakes + // the loop at the deadline. + deadline := time.Now().Add(WindowWaitTimeout) + var timer *time.Timer + for { + if w.err != nil { + w.mu.Unlock() + stopTimer(timer) + return 0, w.err + } + if int64(idx)*w.chunkSize+length <= w.readPos { + // already fully consumed + crc := w.crcs[idx] + w.mu.Unlock() + stopTimer(timer) + return crc, nil + } + if w.slotChunk[slot] == idx { + if w.slotState[slot] == slotReady { + crc := w.crcs[idx] + w.mu.Unlock() + stopTimer(timer) + return crc, nil + } + if w.slotState[slot] == slotFilling { + w.mu.Unlock() + stopTimer(timer) + return 0, ErrChunkInFlight + } + } + if w.slotState[slot] == slotFree && idx < w.frontier()+w.slots { + break // admissible + } + if !time.Now().Before(deadline) { + w.mu.Unlock() + stopTimer(timer) + return 0, ErrOutOfWindow + } + if timer == nil { + timer = time.AfterFunc(time.Until(deadline), func() { + w.mu.Lock() + w.cond.Broadcast() + w.mu.Unlock() + }) + } + w.cond.Wait() + } + stopTimer(timer) + w.slotState[slot] = slotFilling + w.slotChunk[slot] = idx + f := w.f + w.mu.Unlock() + + h := crc32.NewIEEE() + n, err := utils.CopyWithBufferN(io.NewOffsetWriter(f, int64(slot)*w.chunkSize), io.TeeReader(r, h), length) + if err == nil { + // the body must contain exactly one chunk + var b [1]byte + if m, _ := io.ReadFull(r, b[:]); m > 0 { + err = fmt.Errorf("chunk %d larger than expected %d bytes", idx, length) + } + } else { + err = fmt.Errorf("incomplete chunk %d: got %d of %d bytes: %w", idx, n, length, err) + } + + w.mu.Lock() + defer w.mu.Unlock() + if w.slotState[slot] != slotFilling || w.slotChunk[slot] != idx { + // the window was closed and reset the slot while we were writing + if w.err != nil { + return 0, w.err + } + return 0, ErrClosed + } + if err == nil && w.err != nil { + err = w.err + } + if err != nil { + w.slotState[slot] = slotFree + w.slotChunk[slot] = -1 + return 0, err + } + w.slotState[slot] = slotReady + w.crcs[idx] = h.Sum32() + w.crcSet[idx] = true + w.cond.Broadcast() + return w.crcs[idx], nil +} + +// Read serves the reassembled stream in order, blocking until the next chunk +// is available, the window is closed, or the stream ends (io.EOF). +func (w *Window) Read(p []byte) (int, error) { + if len(p) == 0 { + return 0, nil + } + w.mu.Lock() + for { + if w.err != nil { + w.mu.Unlock() + return 0, w.err + } + if w.readPos >= w.totalSize { + w.mu.Unlock() + return 0, io.EOF + } + cur := int(w.readPos / w.chunkSize) + slot := cur % w.slots + if w.slotState[slot] == slotReady && w.slotChunk[slot] == cur { + chunkStart := int64(cur) * w.chunkSize + chunkEnd := chunkStart + w.ChunkLen(cur) + n := int64(len(p)) + if avail := chunkEnd - w.readPos; n > avail { + n = avail + } + off := int64(slot)*w.chunkSize + (w.readPos - chunkStart) + f := w.f + w.mu.Unlock() + + read, err := f.ReadAt(p[:n], off) + + w.mu.Lock() + if read > 0 { + w.readPos += int64(read) + if w.readPos >= chunkEnd { + w.slotState[slot] = slotFree + w.slotChunk[slot] = -1 + w.cond.Broadcast() // writers may be parked waiting for this slot + } + } + sticky := w.err + w.mu.Unlock() + if read > 0 { + return read, nil + } + if sticky != nil { + return 0, sticky + } + if err == nil { + err = io.ErrUnexpectedEOF + } + return 0, err + } + w.cond.Wait() + } +} + +// Close makes all pending and future operations fail with ErrClosed and removes +// the ring file. It is invoked by op.Put via FileStream.Closers when the +// pipeline ends, and is safe to call multiple times. +func (w *Window) Close() error { + return w.CloseWithError(ErrClosed) +} + +// CloseWithError is Close with a caller-chosen sticky error. The session's +// abort path passes an error wrapping context.Canceled so that a driver woken +// up from a blocked Read treats the abort exactly like a canceled request +// (e.g. the local driver only removes partially written files in that case). +func (w *Window) CloseWithError(sticky error) error { + w.mu.Lock() + if w.err == nil { + w.err = sticky + } + f := w.f + w.f = nil + for i := range w.slotState { + w.slotState[i] = slotFree + w.slotChunk[i] = -1 + } + w.cond.Broadcast() + w.mu.Unlock() + if f == nil { + return nil + } + err := f.Close() + if rmErr := os.Remove(w.path); rmErr != nil && err == nil { + err = rmErr + } + return err +} + +// Snapshot reports the receiving state for status responses and resume discovery. +func (w *Window) Snapshot() Snapshot { + w.mu.Lock() + defer w.mu.Unlock() + snap := Snapshot{Frontier: w.frontier(), ReadPos: w.readPos, ReceivedBytes: w.readPos} + var ranges [][2]int + if snap.Frontier > 0 { + ranges = append(ranges, [2]int{0, snap.Frontier - 1}) + } + for idx := snap.Frontier; idx < snap.Frontier+w.slots && idx < w.total; idx++ { + slot := idx % w.slots + if w.slotState[slot] != slotReady || w.slotChunk[slot] != idx { + continue + } + start := int64(idx) * w.chunkSize + var consumed int64 + if w.readPos > start { + consumed = w.readPos - start + } + snap.ReceivedBytes += w.ChunkLen(idx) - consumed + if len(ranges) > 0 && ranges[len(ranges)-1][1] == idx-1 { + ranges[len(ranges)-1][1] = idx + } else { + ranges = append(ranges, [2]int{idx, idx}) + } + } + snap.Received = ranges + return snap +} + +// CRCs returns a copy of the per-chunk CRC32 table and which entries are set. +// It remains readable after Close, so the session can compare re-filled chunks +// against a previous attempt. +func (w *Window) CRCs() ([]uint32, []bool) { + w.mu.Lock() + defer w.mu.Unlock() + crcs := make([]uint32, len(w.crcs)) + set := make([]bool, len(w.crcSet)) + copy(crcs, w.crcs) + copy(set, w.crcSet) + return crcs, set +} diff --git a/internal/multipart/window_test.go b/internal/multipart/window_test.go new file mode 100644 index 00000000..d368672c --- /dev/null +++ b/internal/multipart/window_test.go @@ -0,0 +1,527 @@ +package multipart + +import ( + "bytes" + "crypto/sha256" + "errors" + "hash/crc32" + "io" + "math/rand" + "sync" + "sync/atomic" + "testing" + "time" +) + +func genData(size int64) []byte { + data := make([]byte, size) + rnd := rand.New(rand.NewSource(size*7919 + 13)) + rnd.Read(data) + return data +} + +func newTestWindow(t *testing.T, chunkSize, totalSize int64, slots int) *Window { + t.Helper() + w, err := NewWindow(t.TempDir(), "test", chunkSize, totalSize, slots) + if err != nil { + t.Fatalf("NewWindow: %v", err) + } + t.Cleanup(func() { _ = w.Close() }) + return w +} + +func chunkOf(data []byte, idx int, chunkSize int64) []byte { + start := int64(idx) * chunkSize + end := start + chunkSize + if end > int64(len(data)) { + end = int64(len(data)) + } + return data[start:end] +} + +func writeChunkOK(t *testing.T, w *Window, data []byte, idx int) uint32 { + t.Helper() + crc, err := w.WriteChunk(idx, bytes.NewReader(chunkOf(data, idx, w.chunkSize))) + if err != nil { + t.Fatalf("WriteChunk(%d): %v", idx, err) + } + if want := crc32.ChecksumIEEE(chunkOf(data, idx, w.chunkSize)); crc != want { + t.Fatalf("WriteChunk(%d) crc = %08x, want %08x", idx, crc, want) + } + return crc +} + +// readAllWithin reads the whole stream in a goroutine and fails the test on timeout, +// so a reassembly bug cannot hang the suite. +func readAllWithin(t *testing.T, w *Window, timeout time.Duration) []byte { + t.Helper() + type result struct { + data []byte + err error + } + ch := make(chan result, 1) + go func() { + data, err := io.ReadAll(w) + ch <- result{data, err} + }() + select { + case res := <-ch: + if res.err != nil { + t.Fatalf("ReadAll: %v", res.err) + } + return res.data + case <-time.After(timeout): + t.Fatal("ReadAll timed out") + return nil + } +} + +func TestSequentialReadWrite(t *testing.T) { + const chunkSize = 64 * 1024 + totalSize := int64(4*chunkSize + 32*1024) // last chunk is short + data := genData(totalSize) + w := newTestWindow(t, chunkSize, totalSize, 8) + + go func() { + for i := 0; i < w.TotalChunks(); i++ { + if _, err := w.WriteChunk(i, bytes.NewReader(chunkOf(data, i, chunkSize))); err != nil { + t.Errorf("WriteChunk(%d): %v", i, err) + return + } + time.Sleep(time.Millisecond) + } + }() + + got := readAllWithin(t, w, 10*time.Second) + if !bytes.Equal(got, data) { + t.Fatalf("reassembled stream differs: got %d bytes, want %d", len(got), len(data)) + } +} + +func TestOutOfOrderWrites(t *testing.T) { + const chunkSize = 16 * 1024 + totalSize := int64(5*chunkSize - 100) + data := genData(totalSize) + w := newTestWindow(t, chunkSize, totalSize, 8) + + for _, idx := range []int{3, 0, 4, 2, 1} { + writeChunkOK(t, w, data, idx) + } + got := readAllWithin(t, w, 10*time.Second) + if !bytes.Equal(got, data) { + t.Fatal("reassembled stream differs after out-of-order writes") + } +} + +func TestConcurrentWriters(t *testing.T) { + const chunkSize = 32 * 1024 + totalSize := int64(8 * chunkSize) + data := genData(totalSize) + w := newTestWindow(t, chunkSize, totalSize, 8) + + var wg sync.WaitGroup + for i := 0; i < w.TotalChunks(); i++ { + wg.Add(1) + go func(idx int) { + defer wg.Done() + if _, err := w.WriteChunk(idx, bytes.NewReader(chunkOf(data, idx, chunkSize))); err != nil { + t.Errorf("WriteChunk(%d): %v", idx, err) + } + }(i) + } + got := readAllWithin(t, w, 10*time.Second) + wg.Wait() + if !bytes.Equal(got, data) { + t.Fatal("reassembled stream differs after concurrent writes") + } +} + +func setWaitTimeout(t *testing.T, d time.Duration) { + t.Helper() + old := WindowWaitTimeout + WindowWaitTimeout = d + t.Cleanup(func() { WindowWaitTimeout = old }) +} + +func TestBackpressure(t *testing.T) { + setWaitTimeout(t, 50*time.Millisecond) // assert the post-deadline rejection + const chunkSize = 1024 + totalSize := int64(5 * chunkSize) + data := genData(totalSize) + w := newTestWindow(t, chunkSize, totalSize, 2) + + if _, err := w.WriteChunk(2, bytes.NewReader(chunkOf(data, 2, chunkSize))); !errors.Is(err, ErrOutOfWindow) { + t.Fatalf("chunk 2 with frontier 0: err = %v, want ErrOutOfWindow", err) + } + writeChunkOK(t, w, data, 0) + writeChunkOK(t, w, data, 1) + // both slots occupied, frontier still 0 + if _, err := w.WriteChunk(2, bytes.NewReader(chunkOf(data, 2, chunkSize))); !errors.Is(err, ErrOutOfWindow) { + t.Fatalf("chunk 2 with full window: err = %v, want ErrOutOfWindow", err) + } + // consume chunk 0 -> slot released, frontier advances + buf := make([]byte, chunkSize) + if _, err := io.ReadFull(w, buf); err != nil { + t.Fatalf("ReadFull chunk 0: %v", err) + } + if !bytes.Equal(buf, chunkOf(data, 0, chunkSize)) { + t.Fatal("chunk 0 content differs") + } + writeChunkOK(t, w, data, 2) + // chunk 3 maps to the slot still holding buffered chunk 1 + if _, err := w.WriteChunk(3, bytes.NewReader(chunkOf(data, 3, chunkSize))); !errors.Is(err, ErrOutOfWindow) { + t.Fatalf("chunk 3 with occupied slot: err = %v, want ErrOutOfWindow", err) + } +} + +// TestWriteChunkWaitsForSlot pins the browser-friendly flow control: a chunk +// whose slot is occupied parks until the reader frees it instead of bouncing +// with an immediate rejection (early responses read as network errors in +// browsers). +func TestWriteChunkWaitsForSlot(t *testing.T) { + setWaitTimeout(t, 5*time.Second) + const chunkSize = 1024 + totalSize := int64(5 * chunkSize) + data := genData(totalSize) + w := newTestWindow(t, chunkSize, totalSize, 2) + + writeChunkOK(t, w, data, 0) + writeChunkOK(t, w, data, 1) + + done := make(chan error, 1) + go func() { + _, err := w.WriteChunk(2, bytes.NewReader(chunkOf(data, 2, chunkSize))) + done <- err + }() + select { + case err := <-done: + t.Fatalf("chunk 2 should be parked while the window is full, returned %v", err) + case <-time.After(100 * time.Millisecond): + // parked, as designed + } + + // consuming chunk 0 frees its slot and must wake the parked writer + buf := make([]byte, chunkSize) + if _, err := io.ReadFull(w, buf); err != nil { + t.Fatalf("ReadFull: %v", err) + } + select { + case err := <-done: + if err != nil { + t.Fatalf("parked chunk after slot freed: %v", err) + } + case <-time.After(5 * time.Second): + t.Fatal("parked chunk never admitted after its slot freed") + } +} + +func TestIdempotentResend(t *testing.T) { + const chunkSize = 1024 + totalSize := int64(2 * chunkSize) + data := genData(totalSize) + w := newTestWindow(t, chunkSize, totalSize, 4) + + first := writeChunkOK(t, w, data, 0) + again := writeChunkOK(t, w, data, 0) // buffered, not yet consumed + if first != again { + t.Fatalf("resend crc = %08x, want %08x", again, first) + } + buf := make([]byte, chunkSize) + if _, err := io.ReadFull(w, buf); err != nil { + t.Fatalf("ReadFull: %v", err) + } + consumed := writeChunkOK(t, w, data, 0) // already consumed + if consumed != first { + t.Fatalf("post-consume resend crc = %08x, want %08x", consumed, first) + } +} + +// gatedReader blocks the first Read until released, to hold a chunk in the filling state. +type gatedReader struct { + release <-chan struct{} + inner io.Reader +} + +func (g *gatedReader) Read(p []byte) (int, error) { + <-g.release + return g.inner.Read(p) +} + +func TestInFlightConflict(t *testing.T) { + const chunkSize = 1024 + totalSize := int64(2 * chunkSize) + data := genData(totalSize) + w := newTestWindow(t, chunkSize, totalSize, 4) + + release := make(chan struct{}) + done := make(chan error, 1) + go func() { + _, err := w.WriteChunk(0, &gatedReader{release: release, inner: bytes.NewReader(chunkOf(data, 0, chunkSize))}) + done <- err + }() + + // wait until the writer marked the slot as filling + deadline := time.Now().Add(5 * time.Second) + for { + w.mu.Lock() + filling := w.slotState[0] == slotFilling + w.mu.Unlock() + if filling { + break + } + if time.Now().After(deadline) { + t.Fatal("writer never reached filling state") + } + time.Sleep(time.Millisecond) + } + + if _, err := w.WriteChunk(0, bytes.NewReader(chunkOf(data, 0, chunkSize))); !errors.Is(err, ErrChunkInFlight) { + t.Fatalf("concurrent same-chunk write: err = %v, want ErrChunkInFlight", err) + } + close(release) + if err := <-done; err != nil { + t.Fatalf("gated WriteChunk: %v", err) + } + writeChunkOK(t, w, data, 0) // idempotent after settle +} + +func TestShortBodyRecovers(t *testing.T) { + const chunkSize = 1024 + totalSize := int64(2 * chunkSize) + data := genData(totalSize) + w := newTestWindow(t, chunkSize, totalSize, 4) + + if _, err := w.WriteChunk(0, bytes.NewReader(chunkOf(data, 0, chunkSize)[:100])); err == nil { + t.Fatal("short body: expected error") + } + writeChunkOK(t, w, data, 0) // slot must have been recycled +} + +func TestOversizeBodyRejected(t *testing.T) { + const chunkSize = 1024 + totalSize := int64(2 * chunkSize) + data := genData(totalSize) + w := newTestWindow(t, chunkSize, totalSize, 4) + + oversize := append(append([]byte{}, chunkOf(data, 0, chunkSize)...), 0xFF) + if _, err := w.WriteChunk(0, bytes.NewReader(oversize)); err == nil { + t.Fatal("oversize body: expected error") + } + writeChunkOK(t, w, data, 0) + + // the short last chunk must also reject a full-size body + last := w.TotalChunks() - 1 + if last == 0 { + t.Fatal("test needs at least 2 chunks") + } +} + +func TestLastChunkShortStrict(t *testing.T) { + const chunkSize = 1024 + totalSize := int64(chunkSize + 100) + data := genData(totalSize) + w := newTestWindow(t, chunkSize, totalSize, 4) + + if _, err := w.WriteChunk(1, bytes.NewReader(genData(chunkSize))); err == nil { + t.Fatal("full-size body for short last chunk: expected error") + } + writeChunkOK(t, w, data, 1) + writeChunkOK(t, w, data, 0) + got := readAllWithin(t, w, 10*time.Second) + if !bytes.Equal(got, data) { + t.Fatal("reassembled stream differs") + } +} + +func TestCloseUnblocksReader(t *testing.T) { + w := newTestWindow(t, 1024, 4096, 4) + errCh := make(chan error, 1) + go func() { + buf := make([]byte, 16) + _, err := w.Read(buf) + errCh <- err + }() + time.Sleep(20 * time.Millisecond) + _ = w.Close() + select { + case err := <-errCh: + if !errors.Is(err, ErrClosed) { + t.Fatalf("blocked Read after Close: err = %v, want ErrClosed", err) + } + case <-time.After(5 * time.Second): + t.Fatal("Read still blocked after Close") + } +} + +func TestCloseWithErrorPropagatesSticky(t *testing.T) { + w := newTestWindow(t, 1024, 4096, 4) + cause := errors.New("aborted for a reason") + errCh := make(chan error, 1) + go func() { + buf := make([]byte, 16) + _, err := w.Read(buf) + errCh <- err + }() + time.Sleep(20 * time.Millisecond) + _ = w.CloseWithError(cause) + select { + case err := <-errCh: + if !errors.Is(err, cause) { + t.Fatalf("blocked Read after CloseWithError: err = %v, want %v", err, cause) + } + case <-time.After(5 * time.Second): + t.Fatal("Read still blocked after CloseWithError") + } + if _, err := w.WriteChunk(0, bytes.NewReader(make([]byte, 1024))); !errors.Is(err, cause) { + t.Fatalf("WriteChunk after CloseWithError: err = %v, want %v", err, cause) + } +} + +func TestWriteAfterClose(t *testing.T) { + w := newTestWindow(t, 1024, 4096, 4) + _ = w.Close() + if _, err := w.WriteChunk(0, bytes.NewReader(make([]byte, 1024))); !errors.Is(err, ErrClosed) { + t.Fatalf("WriteChunk after Close: err = %v, want ErrClosed", err) + } +} + +func TestCloseUnblocksInFlightWriter(t *testing.T) { + const chunkSize = 1024 + data := genData(2 * chunkSize) + w := newTestWindow(t, chunkSize, 2*chunkSize, 4) + + release := make(chan struct{}) + done := make(chan error, 1) + go func() { + _, err := w.WriteChunk(0, &gatedReader{release: release, inner: bytes.NewReader(chunkOf(data, 0, chunkSize))}) + done <- err + }() + time.Sleep(20 * time.Millisecond) + _ = w.Close() + close(release) + if err := <-done; !errors.Is(err, ErrClosed) { + t.Fatalf("in-flight WriteChunk across Close: err = %v, want ErrClosed", err) + } +} + +func TestEOFExact(t *testing.T) { + const chunkSize = 1024 + totalSize := int64(chunkSize + 5) + data := genData(totalSize) + w := newTestWindow(t, chunkSize, totalSize, 4) + writeChunkOK(t, w, data, 0) + writeChunkOK(t, w, data, 1) + + got := readAllWithin(t, w, 10*time.Second) + if !bytes.Equal(got, data) { + t.Fatal("reassembled stream differs") + } + buf := make([]byte, 1) + if n, err := w.Read(buf); n != 0 || err != io.EOF { + t.Fatalf("Read at EOF = (%d, %v), want (0, io.EOF)", n, err) + } +} + +func TestManyLapsSmallWindow(t *testing.T) { + const chunkSize = 8 * 1024 + const chunks = 64 + totalSize := int64(chunks*chunkSize - 777) + data := genData(totalSize) + w := newTestWindow(t, chunkSize, totalSize, 3) + + var next atomic.Int64 + var wg sync.WaitGroup + for i := 0; i < 3; i++ { + wg.Add(1) + go func() { + defer wg.Done() + for { + idx := int(next.Add(1) - 1) + if idx >= w.TotalChunks() { + return + } + for { + _, err := w.WriteChunk(idx, bytes.NewReader(chunkOf(data, idx, chunkSize))) + if err == nil { + break + } + if errors.Is(err, ErrOutOfWindow) || errors.Is(err, ErrChunkInFlight) { + time.Sleep(200 * time.Microsecond) + continue + } + t.Errorf("WriteChunk(%d): %v", idx, err) + return + } + } + }() + } + + got := readAllWithin(t, w, 30*time.Second) + wg.Wait() + if wantSum, gotSum := sha256.Sum256(data), sha256.Sum256(got); wantSum != gotSum { + t.Fatalf("reassembled stream differs: got %d bytes, want %d", len(got), len(data)) + } +} + +func TestSnapshot(t *testing.T) { + const chunkSize = 1024 + totalSize := int64(5*chunkSize + 512) + data := genData(totalSize) + w := newTestWindow(t, chunkSize, totalSize, 4) + + writeChunkOK(t, w, data, 0) + writeChunkOK(t, w, data, 1) + writeChunkOK(t, w, data, 3) + + snap := w.Snapshot() + if snap.Frontier != 0 || snap.ReadPos != 0 { + t.Fatalf("snapshot frontier/readPos = %d/%d, want 0/0", snap.Frontier, snap.ReadPos) + } + wantRanges := [][2]int{{0, 1}, {3, 3}} + if len(snap.Received) != len(wantRanges) || snap.Received[0] != wantRanges[0] || snap.Received[1] != wantRanges[1] { + t.Fatalf("snapshot received = %v, want %v", snap.Received, wantRanges) + } + if snap.ReceivedBytes != 3*chunkSize { + t.Fatalf("snapshot receivedBytes = %d, want %d", snap.ReceivedBytes, 3*chunkSize) + } + + buf := make([]byte, chunkSize) + if _, err := io.ReadFull(w, buf); err != nil { + t.Fatalf("ReadFull: %v", err) + } + snap = w.Snapshot() + if snap.Frontier != 1 || snap.ReadPos != chunkSize { + t.Fatalf("snapshot frontier/readPos = %d/%d, want 1/%d", snap.Frontier, snap.ReadPos, chunkSize) + } + if len(snap.Received) != 2 || snap.Received[0] != [2]int{0, 1} || snap.Received[1] != [2]int{3, 3} { + t.Fatalf("snapshot received = %v, want [[0,1],[3,3]]", snap.Received) + } + if snap.ReceivedBytes != 3*chunkSize { + t.Fatalf("snapshot receivedBytes = %d, want %d", snap.ReceivedBytes, 3*chunkSize) + } +} + +func TestIndexOutOfRange(t *testing.T) { + w := newTestWindow(t, 1024, 4096, 4) + if _, err := w.WriteChunk(-1, bytes.NewReader(nil)); err == nil { + t.Fatal("negative index: expected error") + } + if _, err := w.WriteChunk(4, bytes.NewReader(nil)); err == nil { + t.Fatal("index == total: expected error") + } +} + +func TestCRCsSurviveClose(t *testing.T) { + const chunkSize = 1024 + data := genData(2 * chunkSize) + w := newTestWindow(t, chunkSize, 2*chunkSize, 4) + want := writeChunkOK(t, w, data, 0) + _ = w.Close() + crcs, set := w.CRCs() + if !set[0] || crcs[0] != want { + t.Fatalf("CRCs after Close = (%08x, %v), want (%08x, true)", crcs[0], set[0], want) + } + if set[1] { + t.Fatal("chunk 1 crc should not be set") + } +} diff --git a/internal/net/oss_test.go b/internal/net/oss_test.go index 9001cd39..ee901ff8 100644 --- a/internal/net/oss_test.go +++ b/internal/net/oss_test.go @@ -18,7 +18,7 @@ func TestNewOSSClientUsesEnvironmentHTTPSProxy(t *testing.T) { t.Setenv("HTTP_PROXY", "") t.Setenv("http_proxy", "") t.Setenv("HTTPS_PROXY", "http://127.0.0.1:7890") - t.Setenv("https_proxy", "") + t.Setenv("https_proxy", "http://127.0.0.1:7890") t.Setenv("NO_PROXY", "") t.Setenv("no_proxy", "") @@ -31,9 +31,13 @@ func TestNewOSSClientUsesEnvironmentHTTPSProxy(t *testing.T) { t.Fatal("expected OSS client to use a custom HTTP client") } - transport, ok := client.HTTPClient.Transport.(*http.Transport) + safe, ok := client.HTTPClient.Transport.(*safeTransport) if !ok { - t.Fatalf("expected *http.Transport, got %T", client.HTTPClient.Transport) + t.Fatalf("expected *safeTransport, got %T", client.HTTPClient.Transport) + } + transport, ok := safe.base.(*http.Transport) + if !ok { + t.Fatalf("expected safe transport to wrap *http.Transport, got %T", safe.base) } if transport.Proxy == nil { diff --git a/internal/net/preconditions.go b/internal/net/preconditions.go new file mode 100644 index 00000000..dc472713 --- /dev/null +++ b/internal/net/preconditions.go @@ -0,0 +1,271 @@ +package net + +import ( + "net/http" + "net/textproto" + "strings" + "time" +) + +// scanETag determines if a syntactically valid ETag is present at s. If so, +// the ETag and remaining text after consuming ETag is returned. Otherwise, +// it returns "", "". +func scanETag(s string) (etag string, remain string) { + s = textproto.TrimString(s) + start := 0 + if strings.HasPrefix(s, "W/") { + start = 2 + } + if len(s[start:]) < 2 || s[start] != '"' { + return "", "" + } + // ETag is either W/"text" or "text". + // See RFC 7232 2.3. + for i := start + 1; i < len(s); i++ { + c := s[i] + switch { + // Character values allowed in ETags. + case c == 0x21 || c >= 0x23 && c <= 0x7E || c >= 0x80: + case c == '"': + return s[:i+1], s[i+1:] + default: + return "", "" + } + } + return "", "" +} + +// etagStrongMatch reports whether a and b match using strong ETag comparison. +// Assumes a and b are valid ETags. +func etagStrongMatch(a, b string) bool { + return a == b && a != "" && a[0] == '"' +} + +// etagWeakMatch reports whether a and b match using weak ETag comparison. +// Assumes a and b are valid ETags. +func etagWeakMatch(a, b string) bool { + return strings.TrimPrefix(a, "W/") == strings.TrimPrefix(b, "W/") +} + +// condResult is the result of an HTTP request precondition check. +// See https://tools.ietf.org/html/rfc7232 section 3. +type condResult int + +const ( + condNone condResult = iota + condTrue + condFalse +) + +func checkIfMatch(w http.ResponseWriter, r *http.Request, exists bool) condResult { + values := r.Header.Values("If-Match") + if len(values) == 0 { + return condNone + } + im := strings.Join(values, ",") + r.Header.Del("If-Match") + if !exists { + return condFalse + } + for { + im = textproto.TrimString(im) + if len(im) == 0 { + break + } + if im[0] == ',' { + im = im[1:] + continue + } + if im[0] == '*' { + return condTrue + } + etag, remain := scanETag(im) + if etag == "" { + break + } + if etagStrongMatch(etag, w.Header().Get("Etag")) { + return condTrue + } + im = remain + } + + return condFalse +} + +func checkIfUnmodifiedSince(r *http.Request, modtime time.Time) condResult { + ius := r.Header.Get("If-Unmodified-Since") + if ius == "" { + return condNone + } + r.Header.Del("If-Unmodified-Since") + if isZeroTime(modtime) { + return condNone + } + t, err := http.ParseTime(ius) + if err != nil { + return condNone + } + + // The Last-Modified header truncates sub-second precision so + // the modtime needs to be truncated too. + modtime = modtime.Truncate(time.Second) + if ret := modtime.Compare(t); ret <= 0 { + return condTrue + } + return condFalse +} + +func checkIfNoneMatch(w http.ResponseWriter, r *http.Request, exists bool) condResult { + values := r.Header.Values("If-None-Match") + if len(values) == 0 { + return condNone + } + inm := strings.Join(values, ",") + r.Header.Del("If-None-Match") + if !exists { + return condTrue + } + buf := inm + for { + buf = textproto.TrimString(buf) + if len(buf) == 0 { + break + } + if buf[0] == ',' { + buf = buf[1:] + continue + } + if buf[0] == '*' { + return condFalse + } + etag, remain := scanETag(buf) + if etag == "" { + break + } + if etagWeakMatch(etag, w.Header().Get("Etag")) { + return condFalse + } + buf = remain + } + return condTrue +} + +func checkIfModifiedSince(r *http.Request, modtime time.Time) condResult { + if r.Method != "GET" && r.Method != "HEAD" { + return condNone + } + ims := r.Header.Get("If-Modified-Since") + if ims == "" { + return condNone + } + r.Header.Del("If-Modified-Since") + if isZeroTime(modtime) { + return condNone + } + t, err := http.ParseTime(ims) + if err != nil { + return condNone + } + // The Last-Modified header truncates sub-second precision so + // the modtime needs to be truncated too. + modtime = modtime.Truncate(time.Second) + if ret := modtime.Compare(t); ret <= 0 { + return condFalse + } + return condTrue +} + +func checkIfRange(w http.ResponseWriter, r *http.Request, modtime time.Time) condResult { + if r.Method != "GET" && r.Method != "HEAD" { + return condNone + } + ir := r.Header.Get("If-Range") + if ir == "" { + return condNone + } + r.Header.Del("If-Range") + etag, _ := scanETag(ir) + if etag != "" { + if etagStrongMatch(etag, w.Header().Get("Etag")) { + return condTrue + } + return condFalse + } + // The If-Range value is typically the ETag value, but it may also be + // the modtime date. See golang.org/issue/8367. + if modtime.IsZero() { + return condFalse + } + t, err := http.ParseTime(ir) + if err != nil { + return condFalse + } + if t.Unix() == modtime.Unix() { + return condTrue + } + return condFalse +} + +var unixEpochTime = time.Unix(0, 0) + +// isZeroTime reports whether t is obviously unspecified (either zero or Unix()=0). +func isZeroTime(t time.Time) bool { + return t.IsZero() || t.Equal(unixEpochTime) +} + +func setLastModified(w http.ResponseWriter, modtime time.Time) { + if !isZeroTime(modtime) { + w.Header().Set("Last-Modified", modtime.UTC().Format(http.TimeFormat)) + } +} + +func writeNotModified(w http.ResponseWriter) { + // RFC 7232 section 4.1: + // a sender SHOULD NOT generate representation metadata other than the + // above listed fields unless said metadata exists for the purpose of + // guiding cache updates (e.g., Last-Modified might be useful if the + // response does not have an ETag field). + h := w.Header() + delete(h, "Content-Type") + delete(h, "Content-Length") + delete(h, "Content-Encoding") + if h.Get("Etag") != "" { + delete(h, "Last-Modified") + } + w.WriteHeader(http.StatusNotModified) +} + +// CheckPreconditions evaluates request preconditions and reports whether a precondition +// resulted in sending StatusNotModified or StatusPreconditionFailed. The caller must +// set the current ETag response header and indicate whether the representation exists. +func CheckPreconditions(w http.ResponseWriter, r *http.Request, modtime time.Time, exists bool) (done bool, rangeHeader string) { + // Evaluate preconditions in the order specified by RFC 9110 section 13.2.2. + ch := checkIfMatch(w, r, exists) + if ch == condNone { + ch = checkIfUnmodifiedSince(r, modtime) + } + if ch == condFalse { + w.WriteHeader(http.StatusPreconditionFailed) + return true, "" + } + switch checkIfNoneMatch(w, r, exists) { + case condFalse: + if r.Method == "GET" || r.Method == "HEAD" { + writeNotModified(w) + return true, "" + } + w.WriteHeader(http.StatusPreconditionFailed) + return true, "" + case condNone: + if checkIfModifiedSince(r, modtime) == condFalse { + writeNotModified(w) + return true, "" + } + } + + rangeHeader = r.Header.Get("Range") + if rangeHeader != "" && checkIfRange(w, r, modtime) == condFalse { + rangeHeader = "" + } + return false, rangeHeader +} diff --git a/internal/net/preconditions_test.go b/internal/net/preconditions_test.go new file mode 100644 index 00000000..f1ab6966 --- /dev/null +++ b/internal/net/preconditions_test.go @@ -0,0 +1,114 @@ +package net + +import ( + "net/http" + "net/http/httptest" + "testing" + "time" +) + +func TestCheckPreconditions(t *testing.T) { + modified := time.Date(2026, time.January, 2, 3, 4, 5, 123456789, time.UTC) + date := modified.Format(http.TimeFormat) + older := modified.Add(-time.Second).Format(http.TimeFormat) + for _, tt := range []struct { + name string + method string + exists bool + etag string + header http.Header + status int + }{ + {"unconditional create", "PUT", false, "", nil, 0}, + {"unconditional overwrite", "PUT", true, `"current"`, nil, 0}, + {"create only missing", "PUT", false, "", http.Header{"If-None-Match": {"*"}}, 0}, + {"create only existing", "PUT", true, `"current"`, http.Header{"If-None-Match": {"*"}}, 412}, + {"create only existing without etag", "PUT", true, "", http.Header{"If-None-Match": {"*"}}, 412}, + {"match missing", "PUT", false, "", http.Header{"If-Match": {"*"}}, 412}, + {"match existing", "PUT", true, `"current"`, http.Header{"If-Match": {"*"}}, 0}, + {"match existing without etag", "PUT", true, "", http.Header{"If-Match": {"*"}}, 0}, + {"match current", "PUT", true, `"current"`, http.Header{"If-Match": {`"current"`}}, 0}, + {"match stale", "PUT", true, `"current"`, http.Header{"If-Match": {`"stale"`}}, 412}, + {"match missing tag", "PUT", false, "", http.Header{"If-Match": {`"current"`}}, 412}, + {"match weak request", "PUT", true, `"current"`, http.Header{"If-Match": {`W/"current"`}}, 412}, + {"match weak representation", "PUT", true, `W/"current"`, http.Header{"If-Match": {`"current"`}}, 412}, + {"match list", "PUT", true, `"current"`, http.Header{"If-Match": {`"stale", "current"`}}, 0}, + {"match multiple lines", "PUT", true, `"current"`, http.Header{"If-Match": {`"stale"`, `"current"`}}, 0}, + {"match empty list members", "PUT", true, `"current"`, http.Header{"If-Match": {`, , "current",`}}, 0}, + {"match quoted comma", "PUT", true, `"one,two"`, http.Header{"If-Match": {`"stale", "one,two"`}}, 0}, + {"malformed match", "PUT", true, `"current"`, http.Header{"If-Match": {"current"}}, 412}, + {"empty match", "PUT", true, `"current"`, http.Header{"If-Match": {""}}, 412}, + {"none match current", "PUT", true, `"current"`, http.Header{"If-None-Match": {`"current"`}}, 412}, + {"none match stale", "PUT", true, `"current"`, http.Header{"If-None-Match": {`"stale"`}}, 0}, + {"none match missing tag", "PUT", false, "", http.Header{"If-None-Match": {`"current"`}}, 0}, + {"none match weak request", "PUT", true, `"current"`, http.Header{"If-None-Match": {`W/"current"`}}, 412}, + {"none match weak representation", "PUT", true, `W/"current"`, http.Header{"If-None-Match": {`"current"`}}, 412}, + {"none match multiple lines", "PUT", true, `"current"`, http.Header{"If-None-Match": {`"stale"`, `"current"`}}, 412}, + {"none match quoted comma", "PUT", true, `"one,two"`, http.Header{"If-None-Match": {`"stale", "one,two"`}}, 412}, + {"both conditions must pass", "PUT", true, `"current"`, http.Header{"If-Match": {`"current"`}, "If-None-Match": {`"current"`}}, 412}, + {"both conditions pass", "PUT", true, `"current"`, http.Header{"If-Match": {`"current"`}, "If-None-Match": {`"stale"`}}, 0}, + {"both wildcards missing", "PUT", false, "", http.Header{"If-Match": {"*"}, "If-None-Match": {"*"}}, 412}, + {"unmodified since older", "PUT", true, `"current"`, http.Header{"If-Unmodified-Since": {older}}, 412}, + {"unmodified since same second", "PUT", true, `"current"`, http.Header{"If-Unmodified-Since": {date}}, 0}, + {"unmodified since missing", "PUT", false, "", http.Header{"If-Unmodified-Since": {older}}, 0}, + {"invalid unmodified since", "PUT", true, `"current"`, http.Header{"If-Unmodified-Since": {"invalid"}}, 0}, + {"match overrides unmodified since", "PUT", true, `"current"`, http.Header{"If-Match": {`"current"`}, "If-Unmodified-Since": {older}}, 0}, + {"ignore modified since on put", "PUT", true, `"current"`, http.Header{"If-Modified-Since": {date}}, 0}, + {"get not modified", "GET", true, `"current"`, http.Header{"If-None-Match": {`W/"current"`}}, 304}, + {"head not modified", "HEAD", true, `"current"`, http.Header{"If-None-Match": {"*"}}, 304}, + {"get stale match", "GET", true, `"current"`, http.Header{"If-Match": {`"stale"`}}, 412}, + {"get modified since", "GET", true, `"current"`, http.Header{"If-Modified-Since": {date}}, 304}, + {"none match overrides modified since", "GET", true, `"current"`, http.Header{"If-None-Match": {`"stale"`}, "If-Modified-Since": {date}}, 0}, + {"empty none match overrides modified since", "GET", true, `"current"`, http.Header{"If-None-Match": {""}, "If-Modified-Since": {date}}, 0}, + } { + t.Run(tt.name, func(t *testing.T) { + r := httptest.NewRequest(tt.method, "/file.txt", nil) + for name, values := range tt.header { + for _, value := range values { + r.Header.Add(name, value) + } + } + w := httptest.NewRecorder() + if tt.etag != "" { + w.Header().Set("Etag", tt.etag) + } + var modTime time.Time + if tt.exists { + modTime = modified + } + done, _ := CheckPreconditions(w, r, modTime, tt.exists) + if done != (tt.status != 0) { + t.Fatalf("done = %v, want status %d", done, tt.status) + } + if done && w.Code != tt.status { + t.Errorf("status = %d, want %d", w.Code, tt.status) + } + if w.Body.Len() != 0 { + t.Errorf("unexpected response body: %q", w.Body.String()) + } + }) + } +} + +func TestCheckPreconditionsIfRange(t *testing.T) { + for _, tt := range []struct { + etag string + want string + }{ + {`"current"`, "bytes=0-3"}, + {`"stale"`, ""}, + {`W/"current"`, ""}, + } { + t.Run(tt.etag, func(t *testing.T) { + r := httptest.NewRequest(http.MethodGet, "/file.txt", nil) + r.Header.Set("Range", "bytes=0-3") + r.Header.Set("If-Range", tt.etag) + w := httptest.NewRecorder() + w.Header().Set("Etag", `"current"`) + done, got := CheckPreconditions(w, r, time.Time{}, true) + if done || got != tt.want { + t.Fatalf("CheckPreconditions() = (%v, %q), want (false, %q)", done, got, tt.want) + } + }) + } +} diff --git a/internal/net/request.go b/internal/net/request.go index 0cfa7942..8c5794e2 100644 --- a/internal/net/request.go +++ b/internal/net/request.go @@ -206,7 +206,8 @@ func (d *downloader) download() (io.ReadCloser, error) { if err != nil { d.cancel(err) d.cfg.ConcurrencyLimit.Release() - return nil, d.interrupt() + _ = d.interrupt() + return nil, err } d.mu.Lock() @@ -268,10 +269,6 @@ func (d *downloader) sendChunkTask(newConcurrency bool) (err error) { if err != nil { return err // 分片算法错误或者下载中断 } - if newConcurrency { - go d.downloadPart() - d.concurrency-- - } ch := chunk{ start: d.pos, size: finalSize, @@ -286,6 +283,11 @@ func (d *downloader) sendChunkTask(newConcurrency bool) (err error) { case <-d.ctx.Done(): return context.Cause(d.ctx) case d.chunkCh <- ch: + if newConcurrency { + // The worker owns the acquired slot only after its chunk is queued. + go d.downloadPart() + d.concurrency-- + } return nil } } diff --git a/internal/net/request_cancel_test.go b/internal/net/request_cancel_test.go new file mode 100644 index 00000000..83ff187a --- /dev/null +++ b/internal/net/request_cancel_test.go @@ -0,0 +1,88 @@ +package net + +import ( + "context" + "errors" + "net/http" + "testing" + "time" + + "github.com/OpenListTeam/OpenList/v4/pkg/http_range" +) + +func TestDownloadCancelledAcquisitionReturnsErrorAndReleasesLimit(t *testing.T) { + const attempts = 32 + limits := make([]*ConcurrencyLimit, 0, attempts) + for range attempts { + limit := &ConcurrencyLimit{Limit: 1} + limits = append(limits, limit) + d := NewDownloader(func(d *Downloader) { + d.Concurrency = 2 + d.PartSize = 4 + d.ConcurrencyLimit = limit + d.HttpClient = func(ctx context.Context, _ *HttpRequestParams) (*http.Response, error) { + return nil, ctx.Err() + } + }) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + reader, err := d.Download(ctx, &HttpRequestParams{Size: 16, Range: http_range.Range{Length: -1}}) + if reader == nil && err == nil { + t.Error("cancelled download returned a nil reader and nil error") + } + if reader != nil { + _ = reader.Close() + } else if !errors.Is(err, context.Canceled) { + t.Errorf("cancelled download error = %v, want context.Canceled", err) + } + } + time.Sleep(50 * time.Millisecond) // allow any started workers to release their slots + for i, limit := range limits { + limit.mu.Lock() + got := limit.Limit + limit.mu.Unlock() + if got != 1 { + t.Errorf("attempt %d remaining concurrency = %d, want 1", i, got) + } + } +} + +func TestDownloadSinglePartFailureReleasesLimit(t *testing.T) { + upstreamErr := errors.New("upstream failure") + for _, tc := range []struct { + name string + ctx context.Context + want error + }{ + {name: "cancelled", ctx: func() context.Context { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + return ctx + }(), want: context.Canceled}, + {name: "upstream failure", ctx: context.Background(), want: upstreamErr}, + } { + t.Run(tc.name, func(t *testing.T) { + limit := &ConcurrencyLimit{Limit: 1} + d := NewDownloader(func(d *Downloader) { + d.PartSize = 32 + d.ConcurrencyLimit = limit + d.HttpClient = func(ctx context.Context, _ *HttpRequestParams) (*http.Response, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + return nil, upstreamErr + } + }) + reader, err := d.Download(tc.ctx, &HttpRequestParams{Size: 16, Range: http_range.Range{Length: -1}}) + if reader != nil || !errors.Is(err, tc.want) { + t.Fatalf("single-part failed download = %v, %v; want nil, %v", reader, err, tc.want) + } + limit.mu.Lock() + got := limit.Limit + limit.mu.Unlock() + if got != 1 { + t.Errorf("remaining concurrency = %d, want 1", got) + } + }) + } +} diff --git a/internal/net/serve.go b/internal/net/serve.go index 89a209d8..53604c3e 100644 --- a/internal/net/serve.go +++ b/internal/net/serve.go @@ -4,6 +4,7 @@ import ( "compress/gzip" "context" "crypto/tls" + stderrors "errors" "fmt" "io" "mime/multipart" @@ -15,7 +16,6 @@ import ( "time" "github.com/OpenListTeam/OpenList/v4/internal/conf" - "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/pkg/http_range" "github.com/OpenListTeam/OpenList/v4/pkg/utils" @@ -25,12 +25,8 @@ import ( //this file is inspired by GO_SDK net.http.ServeContent -//type RangeReadCloser struct { -// GetReaderForRange RangeReaderFunc -//} - // ServeHTTP replies to the request using the content in the -// provided RangeReadCloser. The main benefit of ServeHTTP over io.Copy +// provided range reader. The main benefit of ServeHTTP over io.Copy // is that it handles Range requests properly, sets the MIME type, and // handles If-Match, If-Unmodified-Since, If-None-Match, If-Modified-Since, // and If-Range requests. @@ -47,15 +43,13 @@ import ( // request includes an If-Modified-Since header, ServeHTTP uses // modtime to decide whether the content needs to be sent at all. // -// The content's RangeReadCloser method must work: ServeHTTP gives a range, -// caller will give the reader for that Range. +// The content's RangeRead method must return a reader for the requested range. // // If the caller has set w's ETag header formatted per RFC 7232, section 2.3, // ServeHTTP uses it to handle requests using If-Match, If-None-Match, or If-Range. -func ServeHTTP(w http.ResponseWriter, r *http.Request, name string, modTime time.Time, size int64, RangeReadCloser model.RangeReadCloserIF) error { - defer RangeReadCloser.Close() +func ServeHTTP(w http.ResponseWriter, r *http.Request, name string, modTime time.Time, size int64, rangeReader model.RangeReaderIF) (err error) { setLastModified(w, modTime) - done, rangeReq := checkPreconditions(w, r, modTime) + done, rangeReq := CheckPreconditions(w, r, modTime, true) if done { return nil } @@ -113,10 +107,11 @@ func ServeHTTP(w http.ResponseWriter, r *http.Request, name string, modTime time ctx := r.Context() switch { case len(ranges) == 0: - reader, err := RangeReadCloser.RangeRead(ctx, http_range.Range{Length: -1}) + reader, err := openRange(ctx, rangeReader, http_range.Range{Length: -1}) if err != nil { code = http.StatusRequestedRangeNotSatisfiable - if statusCode, ok := errs.UnwrapOrSelf(err).(HttpStatusCodeError); ok { + var statusCode HttpStatusCodeError + if errors.As(err, &statusCode) { code = int(statusCode) } http.Error(w, err.Error(), code) @@ -136,10 +131,11 @@ func ServeHTTP(w http.ResponseWriter, r *http.Request, name string, modTime time // does not request multiple parts might not support // multipart responses." ra := ranges[0] - sendContent, err = RangeReadCloser.RangeRead(ctx, ra) + sendContent, err = openRange(ctx, rangeReader, ra) if err != nil { code = http.StatusRequestedRangeNotSatisfiable - if statusCode, ok := errs.UnwrapOrSelf(err).(HttpStatusCodeError); ok { + var statusCode HttpStatusCodeError + if errors.As(err, &statusCode) { code = int(statusCode) } http.Error(w, err.Error(), code) @@ -159,7 +155,6 @@ func ServeHTTP(w http.ResponseWriter, r *http.Request, name string, modTime time mw := multipart.NewWriter(pw) w.Header().Set("Content-Type", "multipart/byteranges; boundary="+mw.Boundary()) sendContent = pr - defer pr.Close() // cause writing goroutine to fail and exit if CopyN doesn't finish. go func() { for _, ra := range ranges { part, err := mw.CreatePart(ra.MimeHeader(contentType, size)) @@ -167,21 +162,18 @@ func ServeHTTP(w http.ResponseWriter, r *http.Request, name string, modTime time pw.CloseWithError(err) return } - reader, err := RangeReadCloser.RangeRead(ctx, ra) - if err != nil { - pw.CloseWithError(err) - return - } - if _, err := utils.CopyWithBufferN(part, reader, ra.Length); err != nil { + if err := copyRange(ctx, part, rangeReader, ra); err != nil { pw.CloseWithError(err) return } } - mw.Close() - pw.Close() + _ = pw.CloseWithError(mw.Close()) }() } + defer func() { + err = closeWithError(err, sendContent) + }() w.Header().Set("Accept-Ranges", "bytes") if w.Header().Get("Content-Encoding") == "" { @@ -201,7 +193,8 @@ func ServeHTTP(w http.ResponseWriter, r *http.Request, name string, modTime time log.Warnf("Maybe size incorrect or reader not giving correct/full data, or connection closed before finish. written bytes: %d ,sendSize:%d, ", written, sendSize) } code = http.StatusInternalServerError - if statusCode, ok := errs.UnwrapOrSelf(err).(HttpStatusCodeError); ok { + var statusCode HttpStatusCodeError + if errors.As(err, &statusCode) { code = int(statusCode) } w.WriteHeader(code) @@ -210,16 +203,85 @@ func ServeHTTP(w http.ResponseWriter, r *http.Request, name string, modTime time } return nil } + +func copyRange(ctx context.Context, dst io.Writer, rangeReader model.RangeReaderIF, requested http_range.Range) (err error) { + reader, err := openRange(ctx, rangeReader, requested) + if err != nil { + return err + } + defer func() { + err = closeWithError(err, reader) + }() + _, err = utils.CopyWithBufferN(dst, reader, requested.Length) + return err +} + +func openRange(ctx context.Context, rangeReader model.RangeReaderIF, requested http_range.Range) (io.ReadCloser, error) { + reader, err := rangeReader.RangeRead(ctx, requested) + if err != nil { + if reader != nil { + err = closeWithError(err, reader) + } + return nil, err + } + if reader == nil { + return nil, errors.New("range reader returned a nil body") + } + return reader, nil +} + +func closeWithError(err error, closer io.Closer) error { + closeErr := closer.Close() + if err == nil { + return closeErr + } + if closeErr == nil { + return err + } + return stderrors.Join(err, closeErr) +} + +// unsafeProxyHeaders are never forwarded from the client request to the +// upstream storage, regardless of the proxy_ignore_headers setting. They either +// carry the caller's credentials, describe the hop to this server rather than +// the hop to upstream, or let the caller influence how upstream routes and +// authenticates the request. +var unsafeProxyHeaders = map[string]struct{}{ + "authorization": {}, + "cookie": {}, + "proxy-authorization": {}, + "www-authenticate": {}, + "host": {}, + "referer": {}, + "origin": {}, + "connection": {}, + "keep-alive": {}, + "proxy-connection": {}, + "te": {}, + "trailer": {}, + "transfer-encoding": {}, + "upgrade": {}, + "forwarded": {}, + "x-forwarded-for": {}, + "x-forwarded-host": {}, + "x-forwarded-proto": {}, + "x-real-ip": {}, +} + func ProcessHeader(origin, override http.Header) http.Header { result := http.Header{} // client header for h, val := range origin { - if utils.SliceContains(conf.SlicesMap[conf.ProxyIgnoreHeaders], strings.ToLower(h)) { + lower := strings.ToLower(h) + if _, unsafe := unsafeProxyHeaders[lower]; unsafe { + continue + } + if utils.SliceContains(conf.SlicesMap[conf.ProxyIgnoreHeaders], lower) { continue } result[h] = val } - // needed header + // needed header, produced by the storage driver rather than the client for h, val := range override { result[h] = val } @@ -239,20 +301,18 @@ func RequestHttp(ctx context.Context, httpMethod string, headerOverride http.Hea } // TODO clean header with blocklist or passlist res.Header.Del("set-cookie") - var reader io.Reader if res.StatusCode >= 400 { - // 根据 Content-Encoding 判断 Body 是否压缩 - switch res.Header.Get("Content-Encoding") { - case "gzip": - // 使用gzip.NewReader解压缩 - reader, _ = gzip.NewReader(res.Body) - defer reader.(*gzip.Reader).Close() - default: - // 没有Content-Encoding，直接读取 - reader = res.Body + defer res.Body.Close() + var reader io.Reader = res.Body + if res.Header.Get("Content-Encoding") == "gzip" { + gzipReader, err := gzip.NewReader(res.Body) + if err != nil { + return nil, fmt.Errorf("http request [%s] failure,status: %w decode gzip response: %v", URL, HttpStatusCodeError(res.StatusCode), err) + } + defer gzipReader.Close() + reader = gzipReader } all, _ := io.ReadAll(reader) - _ = res.Body.Close() msg := string(all) log.Debugln(msg) return nil, fmt.Errorf("http request [%s] failure,status: %w response:%s", URL, HttpStatusCodeError(res.StatusCode), msg) @@ -284,9 +344,13 @@ func HttpClient() *http.Client { } func NewHttpClient() *http.Client { + insecureSkipVerify := false + if conf.Conf != nil { + insecureSkipVerify = conf.Conf.TlsInsecureSkipVerify + } transport := &http.Transport{ Proxy: http.ProxyFromEnvironment, - TLSClientConfig: &tls.Config{InsecureSkipVerify: conf.Conf.TlsInsecureSkipVerify}, + TLSClientConfig: &tls.Config{InsecureSkipVerify: insecureSkipVerify}, } SetProxyIfConfigured(transport) diff --git a/internal/net/serve_header_test.go b/internal/net/serve_header_test.go new file mode 100644 index 00000000..d1df43c1 --- /dev/null +++ b/internal/net/serve_header_test.go @@ -0,0 +1,67 @@ +package net + +import ( + "net/http" + "testing" + + "github.com/OpenListTeam/OpenList/v4/internal/conf" +) + +// The client must not be able to smuggle credential or routing headers into the +// request that this server makes to the upstream storage, even when the +// proxy_ignore_headers setting has been emptied. +func TestProcessHeaderDropsUnsafeClientHeaders(t *testing.T) { + conf.SlicesMap[conf.ProxyIgnoreHeaders] = nil + + origin := http.Header{} + origin.Set("Authorization", "Bearer victim-token") + origin.Set("Cookie", "session=victim") + origin.Set("X-Forwarded-For", "127.0.0.1") + origin.Set("Host", "internal.example") + origin.Set("Range", "bytes=0-1023") + + result := ProcessHeader(origin, nil) + + for _, h := range []string{"Authorization", "Cookie", "X-Forwarded-For", "Host"} { + if got := result.Get(h); got != "" { + t.Errorf("header %q must not be forwarded upstream, got %q", h, got) + } + } + if got := result.Get("Range"); got != "bytes=0-1023" { + t.Errorf("Range must be preserved, got %q", got) + } +} + +// Headers supplied by the storage driver still win, since they carry the +// credentials needed to reach upstream. +func TestProcessHeaderOverrideWins(t *testing.T) { + conf.SlicesMap[conf.ProxyIgnoreHeaders] = nil + + origin := http.Header{} + origin.Set("Authorization", "Bearer victim-token") + + override := http.Header{} + override.Set("Authorization", "Bearer driver-token") + + result := ProcessHeader(origin, override) + if got := result.Get("Authorization"); got != "Bearer driver-token" { + t.Errorf("driver header must be used, got %q", got) + } +} + +func TestProcessHeaderStillHonoursIgnoreSetting(t *testing.T) { + conf.SlicesMap[conf.ProxyIgnoreHeaders] = []string{"x-custom"} + t.Cleanup(func() { conf.SlicesMap[conf.ProxyIgnoreHeaders] = nil }) + + origin := http.Header{} + origin.Set("X-Custom", "drop-me") + origin.Set("X-Keep", "keep-me") + + result := ProcessHeader(origin, nil) + if got := result.Get("X-Custom"); got != "" { + t.Errorf("configured ignore header must be dropped, got %q", got) + } + if got := result.Get("X-Keep"); got != "keep-me" { + t.Errorf("unrelated header must be preserved, got %q", got) + } +} diff --git a/internal/net/serve_test.go b/internal/net/serve_test.go new file mode 100644 index 00000000..d28d2ed1 --- /dev/null +++ b/internal/net/serve_test.go @@ -0,0 +1,236 @@ +package net + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "mime" + "mime/multipart" + "net/http" + "net/http/httptest" + "reflect" + "sync" + "testing" + "time" + + "github.com/OpenListTeam/OpenList/v4/pkg/http_range" +) + +func TestServeHTTPClosesMultipartRangeBeforeOpeningNext(t *testing.T) { + source := newSequentialRangeSource("abc") + ctx, cancel := context.WithTimeout(t.Context(), 500*time.Millisecond) + defer cancel() + request := httptest.NewRequest(http.MethodGet, "/file", nil).WithContext(ctx) + request.Header.Set("Range", "bytes=0-0,2-2") + recorder := httptest.NewRecorder() + + if err := ServeHTTP(recorder, request, "file.txt", time.Time{}, 3, source); err != nil { + t.Fatalf("ServeHTTP() error = %v", err) + } + response := recorder.Result() + defer response.Body.Close() + if response.StatusCode != http.StatusPartialContent { + t.Fatalf("status = %d, want %d", response.StatusCode, http.StatusPartialContent) + } + + mediaType, params, err := mime.ParseMediaType(response.Header.Get("Content-Type")) + if err != nil { + t.Fatalf("parse Content-Type: %v", err) + } + if mediaType != "multipart/byteranges" { + t.Fatalf("Content-Type = %q, want multipart/byteranges", mediaType) + } + multipartReader := multipart.NewReader(response.Body, params["boundary"]) + var parts []string + for { + part, err := multipartReader.NextPart() + if errors.Is(err, io.EOF) { + break + } + if err != nil { + t.Fatalf("read multipart part: %v", err) + } + body, err := io.ReadAll(part) + if err != nil { + t.Fatalf("read multipart body: %v", err) + } + parts = append(parts, string(body)) + } + if want := []string{"a", "c"}; !reflect.DeepEqual(parts, want) { + t.Fatalf("multipart parts = %q, want %q", parts, want) + } + assertRangeLifecycle(t, source, []string{"open:0", "close:0", "open:2", "close:2"}, []int{1, 1}) +} + +func TestServeHTTPClosesSelectedRangeBody(t *testing.T) { + tests := []struct { + name string + method string + rangeValue string + wantStatus int + wantEvents []string + }{ + {name: "full", method: http.MethodGet, wantStatus: http.StatusOK, wantEvents: []string{"open:0", "close:0"}}, + {name: "single range", method: http.MethodGet, rangeValue: "bytes=1-1", wantStatus: http.StatusPartialContent, wantEvents: []string{"open:1", "close:1"}}, + {name: "head", method: http.MethodHead, wantStatus: http.StatusOK, wantEvents: []string{"open:0", "close:0"}}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + source := newSequentialRangeSource("abc") + request := httptest.NewRequest(test.method, "/file", nil) + if test.rangeValue != "" { + request.Header.Set("Range", test.rangeValue) + } + recorder := httptest.NewRecorder() + + if err := ServeHTTP(recorder, request, "file.txt", time.Time{}, 3, source); err != nil { + t.Fatalf("ServeHTTP() error = %v", err) + } + if recorder.Code != test.wantStatus { + t.Fatalf("status = %d, want %d", recorder.Code, test.wantStatus) + } + assertRangeLifecycle(t, source, test.wantEvents, []int{1}) + }) + } +} + +func TestServeHTTPClosesRangeAfterWriteFailure(t *testing.T) { + writeErr := errors.New("write failed") + source := newSequentialRangeSource("abc") + request := httptest.NewRequest(http.MethodGet, "/file", nil) + writer := &failingResponseWriter{header: make(http.Header), err: writeErr} + + err := ServeHTTP(writer, request, "file.txt", time.Time{}, 3, source) + if !errors.Is(err, writeErr) { + t.Fatalf("ServeHTTP() error = %v, want %v", err, writeErr) + } + assertRangeLifecycle(t, source, []string{"open:0", "close:0"}, []int{1}) +} + +func TestServeHTTPClosesBodyReturnedWithOpenError(t *testing.T) { + source := newSequentialRangeSource("abc") + source.openErr = HttpStatusCodeError(http.StatusServiceUnavailable) + source.closeErr = errors.New("close failed") + request := httptest.NewRequest(http.MethodGet, "/file", nil) + recorder := httptest.NewRecorder() + + if err := ServeHTTP(recorder, request, "file.txt", time.Time{}, 3, source); err != nil { + t.Fatalf("ServeHTTP() error = %v", err) + } + if recorder.Code != http.StatusServiceUnavailable { + t.Fatalf("status = %d, want %d", recorder.Code, http.StatusServiceUnavailable) + } + assertRangeLifecycle(t, source, []string{"open:0", "close:0"}, []int{1}) +} + +func TestServeHTTPStopsMultipartAfterRangeCloseFailure(t *testing.T) { + closeErr := errors.New("close failed") + source := newSequentialRangeSource("abc") + source.closeErr = closeErr + request := httptest.NewRequest(http.MethodGet, "/file", nil) + request.Header.Set("Range", "bytes=0-0,2-2") + recorder := httptest.NewRecorder() + + err := ServeHTTP(recorder, request, "file.txt", time.Time{}, 3, source) + if !errors.Is(err, closeErr) { + t.Fatalf("ServeHTTP() error = %v, want %v", err, closeErr) + } + assertRangeLifecycle(t, source, []string{"open:0", "close:0"}, []int{1}) +} + +type sequentialRangeSource struct { + content []byte + permit chan struct{} + + mu sync.Mutex + events []string + closeCounts []int + closeErr error + openErr error +} + +func newSequentialRangeSource(content string) *sequentialRangeSource { + return &sequentialRangeSource{ + content: []byte(content), + permit: make(chan struct{}, 1), + } +} + +func (s *sequentialRangeSource) RangeRead(ctx context.Context, requested http_range.Range) (io.ReadCloser, error) { + select { + case s.permit <- struct{}{}: + case <-ctx.Done(): + return nil, ctx.Err() + } + + start := int(requested.Start) + length := int(requested.Length) + if length < 0 || start+length > len(s.content) { + length = len(s.content) - start + } + end := start + length + s.mu.Lock() + index := len(s.closeCounts) + s.events = append(s.events, fmt.Sprintf("open:%d", requested.Start)) + s.closeCounts = append(s.closeCounts, 0) + s.mu.Unlock() + return &testReadCloser{ + Reader: bytes.NewReader(s.content[start:end]), + close: func() error { + s.mu.Lock() + s.closeCounts[index]++ + closeCalls := s.closeCounts[index] + if closeCalls == 1 { + s.events = append(s.events, fmt.Sprintf("close:%d", requested.Start)) + } + s.mu.Unlock() + if closeCalls != 1 { + return fmt.Errorf("body closed %d times", closeCalls) + } + <-s.permit + return s.closeErr + }, + }, s.openErr +} + +func (s *sequentialRangeSource) eventsSnapshot() []string { + s.mu.Lock() + defer s.mu.Unlock() + return append([]string(nil), s.events...) +} + +func (s *sequentialRangeSource) closeCountsSnapshot() []int { + s.mu.Lock() + defer s.mu.Unlock() + return append([]int(nil), s.closeCounts...) +} + +func assertRangeLifecycle(t *testing.T, source *sequentialRangeSource, wantEvents []string, wantCloseCounts []int) { + t.Helper() + if got := source.eventsSnapshot(); !reflect.DeepEqual(got, wantEvents) { + t.Fatalf("range lifecycle = %v, want %v", got, wantEvents) + } + if got := source.closeCountsSnapshot(); !reflect.DeepEqual(got, wantCloseCounts) { + t.Fatalf("close counts = %v, want %v", got, wantCloseCounts) + } +} + +type failingResponseWriter struct { + header http.Header + err error +} + +func (w *failingResponseWriter) Header() http.Header { return w.header } +func (*failingResponseWriter) WriteHeader(int) {} +func (w *failingResponseWriter) Write([]byte) (int, error) { + return 0, w.err +} + +type testReadCloser struct { + io.Reader + close func() error +} + +func (b *testReadCloser) Close() error { return b.close() } diff --git a/internal/net/util.go b/internal/net/util.go index 4bf2ba6c..6bb31612 100644 --- a/internal/net/util.go +++ b/internal/net/util.go @@ -4,10 +4,7 @@ import ( "io" "mime/multipart" "net/http" - "net/textproto" "net/url" - "strings" - "time" "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/pkg/utils" @@ -18,260 +15,6 @@ import ( log "github.com/sirupsen/logrus" ) -// scanETag determines if a syntactically valid ETag is present at s. If so, -// the ETag and remaining text after consuming ETag is returned. Otherwise, -// it returns "", "". -func scanETag(s string) (etag string, remain string) { - s = textproto.TrimString(s) - start := 0 - if strings.HasPrefix(s, "W/") { - start = 2 - } - if len(s[start:]) < 2 || s[start] != '"' { - return "", "" - } - // ETag is either W/"text" or "text". - // See RFC 7232 2.3. - for i := start + 1; i < len(s); i++ { - c := s[i] - switch { - // Character values allowed in ETags. - case c == 0x21 || c >= 0x23 && c <= 0x7E || c >= 0x80: - case c == '"': - return s[:i+1], s[i+1:] - default: - return "", "" - } - } - return "", "" -} - -// etagStrongMatch reports whether a and b match using strong ETag comparison. -// Assumes a and b are valid ETags. -func etagStrongMatch(a, b string) bool { - return a == b && a != "" && a[0] == '"' -} - -// etagWeakMatch reports whether a and b match using weak ETag comparison. -// Assumes a and b are valid ETags. -func etagWeakMatch(a, b string) bool { - return strings.TrimPrefix(a, "W/") == strings.TrimPrefix(b, "W/") -} - -// condResult is the result of an HTTP request precondition check. -// See https://tools.ietf.org/html/rfc7232 section 3. -type condResult int - -const ( - condNone condResult = iota - condTrue - condFalse -) - -func checkIfMatch(w http.ResponseWriter, r *http.Request) condResult { - im := r.Header.Get("If-Match") - if im == "" { - return condNone - } - r.Header.Del("If-Match") - for { - im = textproto.TrimString(im) - if len(im) == 0 { - break - } - if im[0] == ',' { - im = im[1:] - continue - } - if im[0] == '*' { - return condTrue - } - etag, remain := scanETag(im) - if etag == "" { - break - } - if etagStrongMatch(etag, w.Header().Get("Etag")) { - return condTrue - } - im = remain - } - - return condFalse -} - -func checkIfUnmodifiedSince(r *http.Request, modtime time.Time) condResult { - ius := r.Header.Get("If-Unmodified-Since") - if ius == "" { - return condNone - } - r.Header.Del("If-Unmodified-Since") - if isZeroTime(modtime) { - return condNone - } - t, err := http.ParseTime(ius) - if err != nil { - return condNone - } - - // The Last-Modified header truncates sub-second precision so - // the modtime needs to be truncated too. - modtime = modtime.Truncate(time.Second) - if ret := modtime.Compare(t); ret <= 0 { - return condTrue - } - return condFalse -} - -func checkIfNoneMatch(w http.ResponseWriter, r *http.Request) condResult { - inm := r.Header.Get("If-None-Match") - if inm == "" { - return condNone - } - r.Header.Del("If-None-Match") - buf := inm - for { - buf = textproto.TrimString(buf) - if len(buf) == 0 { - break - } - if buf[0] == ',' { - buf = buf[1:] - continue - } - if buf[0] == '*' { - return condFalse - } - etag, remain := scanETag(buf) - if etag == "" { - break - } - if etagWeakMatch(etag, w.Header().Get("Etag")) { - return condFalse - } - buf = remain - } - return condTrue -} - -func checkIfModifiedSince(r *http.Request, modtime time.Time) condResult { - if r.Method != "GET" && r.Method != "HEAD" { - return condNone - } - ims := r.Header.Get("If-Modified-Since") - if ims == "" { - return condNone - } - r.Header.Del("If-Modified-Since") - if isZeroTime(modtime) { - return condNone - } - t, err := http.ParseTime(ims) - if err != nil { - return condNone - } - // The Last-Modified header truncates sub-second precision so - // the modtime needs to be truncated too. - modtime = modtime.Truncate(time.Second) - if ret := modtime.Compare(t); ret <= 0 { - return condFalse - } - return condTrue -} - -func checkIfRange(w http.ResponseWriter, r *http.Request, modtime time.Time) condResult { - if r.Method != "GET" && r.Method != "HEAD" { - return condNone - } - ir := r.Header.Get("If-Range") - if ir == "" { - return condNone - } - r.Header.Del("If-Range") - etag, _ := scanETag(ir) - if etag != "" { - if etagStrongMatch(etag, w.Header().Get("Etag")) { - return condTrue - } - return condFalse - } - // The If-Range value is typically the ETag value, but it may also be - // the modtime date. See golang.org/issue/8367. - if modtime.IsZero() { - return condFalse - } - t, err := http.ParseTime(ir) - if err != nil { - return condFalse - } - if t.Unix() == modtime.Unix() { - return condTrue - } - return condFalse -} - -var unixEpochTime = time.Unix(0, 0) - -// isZeroTime reports whether t is obviously unspecified (either zero or Unix()=0). -func isZeroTime(t time.Time) bool { - return t.IsZero() || t.Equal(unixEpochTime) -} - -func setLastModified(w http.ResponseWriter, modtime time.Time) { - if !isZeroTime(modtime) { - w.Header().Set("Last-Modified", modtime.UTC().Format(http.TimeFormat)) - } -} - -func writeNotModified(w http.ResponseWriter) { - // RFC 7232 section 4.1: - // a sender SHOULD NOT generate representation metadata other than the - // above listed fields unless said metadata exists for the purpose of - // guiding cache updates (e.g., Last-Modified might be useful if the - // response does not have an ETag field). - h := w.Header() - delete(h, "Content-Type") - delete(h, "Content-Length") - delete(h, "Content-Encoding") - if h.Get("Etag") != "" { - delete(h, "Last-Modified") - } - w.WriteHeader(http.StatusNotModified) -} - -// checkPreconditions evaluates request preconditions and reports whether a precondition -// resulted in sending StatusNotModified or StatusPreconditionFailed. -func checkPreconditions(w http.ResponseWriter, r *http.Request, modtime time.Time) (done bool, rangeHeader string) { - // This function carefully follows RFC 7232 section 6. - ch := checkIfMatch(w, r) - if ch == condNone { - ch = checkIfUnmodifiedSince(r, modtime) - } - if ch == condFalse { - w.WriteHeader(http.StatusPreconditionFailed) - return true, "" - } - switch checkIfNoneMatch(w, r) { - case condFalse: - if r.Method == "GET" || r.Method == "HEAD" { - writeNotModified(w) - return true, "" - } - w.WriteHeader(http.StatusPreconditionFailed) - return true, "" - case condNone: - if checkIfModifiedSince(r, modtime) == condFalse { - writeNotModified(w) - return true, "" - } - } - - rangeHeader = r.Header.Get("Range") - if rangeHeader != "" && checkIfRange(w, r, modtime) == condFalse { - rangeHeader = "" - } - return false, rangeHeader -} - func sumRangesSize(ranges []http_range.Range) (size int64) { for _, ra := range ranges { size += ra.Length @@ -326,7 +69,7 @@ func GetRangedHttpReader(readCloser io.ReadCloser, offset, length int64) (io.Rea // SetProxyIfConfigured sets proxy for HTTP Transport if configured func SetProxyIfConfigured(transport *http.Transport) { // If proxy address is configured, override environment variable settings - if conf.Conf.ProxyAddress != "" { + if conf.Conf != nil && conf.Conf.ProxyAddress != "" { if proxyURL, err := url.Parse(conf.Conf.ProxyAddress); err == nil { transport.Proxy = http.ProxyURL(proxyURL) } @@ -336,7 +79,7 @@ func SetProxyIfConfigured(transport *http.Transport) { // SetRestyProxyIfConfigured sets proxy for Resty client if configured func SetRestyProxyIfConfigured(client *resty.Client) { // If proxy address is configured, override environment variable settings - if conf.Conf.ProxyAddress != "" { + if conf.Conf != nil && conf.Conf.ProxyAddress != "" { if proxyURL, err := url.Parse(conf.Conf.ProxyAddress); err == nil { client.SetProxy(proxyURL.String()) } diff --git a/internal/offline_download/115/client.go b/internal/offline_download/115/client.go index d31971d7..c026ee7e 100644 --- a/internal/offline_download/115/client.go +++ b/internal/offline_download/115/client.go @@ -125,7 +125,7 @@ func (p *Cloud115) Status(task *tool.DownloadTask) (*tool.Status, error) { s.Completed = t.IsDone() s.TotalBytes = t.Size if t.IsFailed() { - s.Err = fmt.Errorf(t.GetStatus()) + s.Err = fmt.Errorf("%s", t.GetStatus()) } return s, nil } diff --git a/internal/offline_download/115_open/client.go b/internal/offline_download/115_open/client.go index 6f4c7fd0..dd179f31 100644 --- a/internal/offline_download/115_open/client.go +++ b/internal/offline_download/115_open/client.go @@ -122,7 +122,7 @@ func (o *Open115) Status(task *tool.DownloadTask) (*tool.Status, error) { s.Completed = t.IsDone() s.TotalBytes = t.Size if t.IsFailed() { - s.Err = fmt.Errorf(t.GetStatus()) + s.Err = fmt.Errorf("%s", t.GetStatus()) } return s, nil } diff --git a/internal/offline_download/all.go b/internal/offline_download/all.go index 4fcbdb9c..7c9b9dca 100644 --- a/internal/offline_download/all.go +++ b/internal/offline_download/all.go @@ -6,6 +6,7 @@ import ( _ "github.com/OpenListTeam/OpenList/v4/internal/offline_download/123" _ "github.com/OpenListTeam/OpenList/v4/internal/offline_download/123_open" _ "github.com/OpenListTeam/OpenList/v4/internal/offline_download/aria2" + _ "github.com/OpenListTeam/OpenList/v4/internal/offline_download/guangyapan" _ "github.com/OpenListTeam/OpenList/v4/internal/offline_download/http" _ "github.com/OpenListTeam/OpenList/v4/internal/offline_download/pikpak" _ "github.com/OpenListTeam/OpenList/v4/internal/offline_download/qbit" diff --git a/internal/offline_download/guangyapan/guangyapan.go b/internal/offline_download/guangyapan/guangyapan.go new file mode 100644 index 00000000..e28843ee --- /dev/null +++ b/internal/offline_download/guangyapan/guangyapan.go @@ -0,0 +1,132 @@ +package guangyapan + +import ( + "context" + "errors" + "fmt" + "sync/atomic" + + guangyapandriver "github.com/OpenListTeam/OpenList/v4/drivers/guangyapan" + "github.com/OpenListTeam/OpenList/v4/internal/conf" + "github.com/OpenListTeam/OpenList/v4/internal/errs" + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/internal/offline_download/tool" + "github.com/OpenListTeam/OpenList/v4/internal/op" + "github.com/OpenListTeam/OpenList/v4/internal/setting" +) + +type GuangYaPan struct { + refreshTaskCache atomic.Bool +} + +func (g *GuangYaPan) Name() string { + return "GuangYaPan" +} + +func (g *GuangYaPan) Items() []model.SettingItem { + return nil +} + +func (g *GuangYaPan) Run(task *tool.DownloadTask) error { + return errs.NotSupport +} + +func (g *GuangYaPan) Init() (string, error) { + g.refreshTaskCache.Store(false) + return "ok", nil +} + +func (g *GuangYaPan) IsReady() bool { + tempDir := setting.GetStr(conf.GuangYaPanTempDir) + if tempDir == "" { + return false + } + storage, _, err := op.GetStorageAndActualPath(tempDir) + if err != nil { + return false + } + if _, ok := storage.(*guangyapandriver.GuangYaPan); !ok { + return false + } + return true +} + +func (g *GuangYaPan) AddURL(args *tool.AddUrlArgs) (string, error) { + g.refreshTaskCache.Store(true) + storage, actualPath, err := op.GetStorageAndActualPath(args.TempDir) + if err != nil { + return "", err + } + driver, ok := storage.(*guangyapandriver.GuangYaPan) + if !ok { + return "", errors.New("GuangYaPan offline download only supports GuangYaPan destination storage") + } + + ctx := context.Background() + if err := op.MakeDir(ctx, storage, actualPath); err != nil { + return "", err + } + parentDir, err := op.GetUnwrap(ctx, storage, actualPath) + if err != nil { + return "", err + } + task, err := driver.OfflineDownload(ctx, args.Url, parentDir, "") + if err != nil { + return "", fmt.Errorf("failed to add offline download task: %w", err) + } + return task.TaskID, nil +} + +func (g *GuangYaPan) Remove(task *tool.DownloadTask) error { + storage, _, err := op.GetStorageAndActualPath(task.TempDir) + if err != nil { + return err + } + driver, ok := storage.(*guangyapandriver.GuangYaPan) + if !ok { + return errors.New("GuangYaPan offline download only supports GuangYaPan destination storage") + } + ctx := context.Background() + if err := driver.DeleteOfflineTasks(ctx, []string{task.GID}); err != nil { + return err + } + g.DelTaskCache(driver, task.GID) + return nil +} + +func (g *GuangYaPan) Status(task *tool.DownloadTask) (*tool.Status, error) { + storage, _, err := op.GetStorageAndActualPath(task.TempDir) + if err != nil { + return nil, err + } + driver, ok := storage.(*guangyapandriver.GuangYaPan) + if !ok { + return nil, errors.New("GuangYaPan offline download only supports GuangYaPan destination storage") + } + tasks, err := g.GetTasks(driver, task.GID) + if err != nil { + return nil, err + } + status := &tool.Status{ + Status: "the task has been deleted", + } + for _, t := range tasks { + if t.TaskID != task.GID { + continue + } + status.Progress = float64(t.Progress) + status.TotalBytes = t.TotalSize + status.Completed = t.Status == offlineStatusCompleted || t.Status == offlineStatusPartiallyCompleted + status.Status = taskStatusText(t) + if t.Status == offlineStatusFailed || t.Status == offlineStatusCanceled { + status.Err = errors.New(status.Status) + } + return status, nil + } + status.Err = errors.New("the task has been deleted") + return status, nil +} + +func init() { + tool.Tools.Add(&GuangYaPan{}) +} diff --git a/internal/offline_download/guangyapan/util.go b/internal/offline_download/guangyapan/util.go new file mode 100644 index 00000000..a696c594 --- /dev/null +++ b/internal/offline_download/guangyapan/util.go @@ -0,0 +1,77 @@ +package guangyapan + +import ( + "context" + "fmt" + "time" + + guangyapandriver "github.com/OpenListTeam/OpenList/v4/drivers/guangyapan" + "github.com/OpenListTeam/OpenList/v4/internal/op" + "github.com/OpenListTeam/OpenList/v4/pkg/singleflight" + "github.com/OpenListTeam/go-cache" +) + +const ( + offlineStatusQueued = 0 + offlineStatusRunning = 1 + offlineStatusCompleted = 2 + offlineStatusFailed = 3 + offlineStatusCanceled = 4 + offlineStatusPartiallyCompleted = 5 +) + +var taskCache = cache.NewMemCache(cache.WithShards[[]guangyapandriver.OfflineTask](16)) +var taskG singleflight.Group[[]guangyapandriver.OfflineTask] + +func (g *GuangYaPan) GetTasks(driver *guangyapandriver.GuangYaPan, taskID string) ([]guangyapandriver.OfflineTask, error) { + key := op.Key(driver, "/cloudcollection/v1/list_task/"+taskID) + if !g.refreshTaskCache.Load() { + if tasks, ok := taskCache.Get(key); ok { + return tasks, nil + } + } + g.refreshTaskCache.Store(false) + tasks, err, _ := taskG.Do(key, func() ([]guangyapandriver.OfflineTask, error) { + ctx := context.Background() + tasks, err := driver.OfflineList(ctx, []string{taskID}, nil, "", 0) + if err != nil { + return nil, err + } + if len(tasks) > 0 { + taskCache.Set(key, tasks, cache.WithEx[[]guangyapandriver.OfflineTask](time.Second*10)) + } else { + taskCache.Del(key) + } + return tasks, nil + }) + if err != nil { + return nil, err + } + return tasks, nil +} + +func (g *GuangYaPan) DelTaskCache(driver *guangyapandriver.GuangYaPan, taskID string) { + taskCache.Del(op.Key(driver, "/cloudcollection/v1/list_task/"+taskID)) +} + +func taskStatusText(task guangyapandriver.OfflineTask) string { + switch task.Status { + case offlineStatusQueued: + return "queued" + case offlineStatusRunning: + if task.Progress > 0 { + return fmt.Sprintf("running (%d%%)", task.Progress) + } + return "running" + case offlineStatusCompleted: + return "completed" + case offlineStatusFailed: + return "failed" + case offlineStatusCanceled: + return "canceled" + case offlineStatusPartiallyCompleted: + return "partially completed" + default: + return fmt.Sprintf("unknown status %d", task.Status) + } +} diff --git a/internal/offline_download/pikpak/pikpak.go b/internal/offline_download/pikpak/pikpak.go index 8a669309..e223937e 100644 --- a/internal/offline_download/pikpak/pikpak.go +++ b/internal/offline_download/pikpak/pikpak.go @@ -127,7 +127,7 @@ func (p *PikPak) Status(task *tool.DownloadTask) (*tool.Status, error) { s.TotalBytes = 0 } if t.Phase == "PHASE_TYPE_ERROR" { - s.Err = fmt.Errorf(t.Message) + s.Err = fmt.Errorf("%s", t.Message) } return s, nil } diff --git a/internal/offline_download/tool/add.go b/internal/offline_download/tool/add.go index 33128ccc..68a94f5f 100644 --- a/internal/offline_download/tool/add.go +++ b/internal/offline_download/tool/add.go @@ -12,18 +12,19 @@ import ( _115_open "github.com/OpenListTeam/OpenList/v4/drivers/115_open" _123 "github.com/OpenListTeam/OpenList/v4/drivers/123" _123_open "github.com/OpenListTeam/OpenList/v4/drivers/123_open" + "github.com/OpenListTeam/OpenList/v4/drivers/guangyapan" "github.com/OpenListTeam/OpenList/v4/drivers/pikpak" "github.com/OpenListTeam/OpenList/v4/drivers/thunder" "github.com/OpenListTeam/OpenList/v4/drivers/thunder_browser" "github.com/OpenListTeam/OpenList/v4/drivers/thunderx" "github.com/OpenListTeam/OpenList/v4/internal/conf" + "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/fs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/OpenListTeam/OpenList/v4/internal/setting" "github.com/OpenListTeam/OpenList/v4/internal/task" - "github.com/OpenListTeam/OpenList/v4/server/common" "github.com/google/uuid" "github.com/pkg/errors" ) @@ -68,7 +69,7 @@ func AddURL(ctx context.Context, args *AddURLArgs) (task.TaskExtensionInfo, erro } } // try putting url - if args.Tool == "SimpleHttp" { + if args.Tool == "SimpleHttp" && !isEd2kURL(args.URL) { if isSimpleHttpSchemeUnsupported(args.URL) { return nil, fmt.Errorf("SimpleHttp tool does not support this URL scheme, please use aria2 or other tools for magnet/ed2k links") } @@ -82,13 +83,17 @@ func AddURL(ctx context.Context, args *AddURLArgs) (task.TaskExtensionInfo, erro // ed2k 链接自动路由：如果当前工具不支持 ed2k，自动尝试使用迅雷系工具 if isEd2kURL(args.URL) { if !isEd2kCapableTool(args.Tool) { - // 尝试找到一个可用的支持 ed2k 的工具 - fallbackTool, fallbackName := findEd2kCapableTool() - if fallbackTool != nil { - // 使用找到的迅雷工具替代 - args.Tool = fallbackName + if storageTool := ed2kToolForStorage(storage); storageTool != "" { + // Prefer the matching native tool when the destination storage supports ed2k. + args.Tool = storageTool } else { - return nil, fmt.Errorf("ed2k protocol is not supported by %s. Please configure and use Thunder/ThunderX/ThunderBrowser for ed2k links", args.Tool) + // Otherwise, try to find an available Thunder-family tool. + fallbackTool, fallbackName := findEd2kCapableTool() + if fallbackTool != nil { + args.Tool = fallbackName + } else { + return nil, fmt.Errorf("ed2k protocol is not supported by %s. Please configure and use Thunder/ThunderX/ThunderBrowser for ed2k links", args.Tool) + } } } } @@ -162,13 +167,23 @@ func AddURL(ctx context.Context, args *AddURLArgs) (task.TaskExtensionInfo, erro } else { tempDir = filepath.Join(setting.GetStr(conf.ThunderXTempDir), uid) } + case "GuangYaPan": + if _, ok := storage.(*guangyapan.GuangYaPan); ok { + tempDir = args.DstDirPath + } else { + tempBase := setting.GetStr(conf.GuangYaPanTempDir) + if tempBase == "" { + return nil, errors.New("GuangYaPan temp dir is not set") + } + tempDir = filepath.Join(tempBase, uid) + } } taskCreator, _ := ctx.Value(conf.UserKey).(*model.User) // taskCreator is nil when convert failed t := &DownloadTask{ TaskExtension: task.TaskExtension{ Creator: taskCreator, - ApiUrl: common.GetApiUrl(ctx), + ApiUrl: conf.GetApiUrl(ctx), }, Url: args.URL, DstDirPath: args.DstDirPath, @@ -206,8 +221,47 @@ func isEd2kURL(urlStr string) bool { return strings.HasPrefix(strings.ToLower(urlStr), "ed2k://") } -// ed2kCapableTools 支持 ed2k 协议的工具列表（迅雷系） -var ed2kCapableTools = []string{"Thunder", "ThunderX", "ThunderBrowser"} +func ed2kToolForStorage(storage driver.Driver) string { + name := NativeToolName(storage) + switch name { + case "115 Cloud", "115 Open": + return name + default: + return "" + } +} + +// NativeToolName returns the offline-download tool implemented by storage. +func NativeToolName(storage driver.Driver) string { + switch storage.(type) { + case *_115.Pan115: + return "115 Cloud" + case *_115_open.Open115: + return "115 Open" + case *_123.Pan123: + return "123Pan" + case *_123_open.Open123: + return "123 Open" + case *guangyapan.GuangYaPan: + return "GuangYaPan" + case *pikpak.PikPak: + return "PikPak" + case *thunder.Thunder: + return "Thunder" + case *thunderx.ThunderX: + return "ThunderX" + case *thunder_browser.ThunderBrowser, *thunder_browser.ThunderBrowserExpert: + return "ThunderBrowser" + default: + return "" + } +} + +// ed2kCapableTools 支持 ed2k 协议的工具列表 +var ed2kCapableTools = []string{"115 Cloud", "115 Open", "Thunder", "ThunderX", "ThunderBrowser"} + +// ed2kFallbackTools 是当前可用于自动接管 ed2k 任务的工具列表。 +var ed2kFallbackTools = []string{"Thunder", "ThunderX", "ThunderBrowser"} // isEd2kCapableTool 检查工具是否支持 ed2k 协议 func isEd2kCapableTool(toolName string) bool { @@ -221,7 +275,7 @@ func isEd2kCapableTool(toolName string) bool { // findEd2kCapableTool 查找一个可用的支持 ed2k 的工具 func findEd2kCapableTool() (Tool, string) { - for _, name := range ed2kCapableTools { + for _, name := range ed2kFallbackTools { t, err := Tools.Get(name) if err != nil { continue diff --git a/internal/offline_download/tool/add_test.go b/internal/offline_download/tool/add_test.go new file mode 100644 index 00000000..e538ea2f --- /dev/null +++ b/internal/offline_download/tool/add_test.go @@ -0,0 +1,86 @@ +package tool + +import ( + "testing" + + _115 "github.com/OpenListTeam/OpenList/v4/drivers/115" + _115_open "github.com/OpenListTeam/OpenList/v4/drivers/115_open" + _123 "github.com/OpenListTeam/OpenList/v4/drivers/123" + _123_open "github.com/OpenListTeam/OpenList/v4/drivers/123_open" + "github.com/OpenListTeam/OpenList/v4/drivers/guangyapan" + "github.com/OpenListTeam/OpenList/v4/drivers/pikpak" + "github.com/OpenListTeam/OpenList/v4/drivers/thunder" + "github.com/OpenListTeam/OpenList/v4/drivers/thunder_browser" + "github.com/OpenListTeam/OpenList/v4/drivers/thunderx" + "github.com/OpenListTeam/OpenList/v4/internal/driver" +) + +func TestIsEd2kCapableTool(t *testing.T) { + tests := []struct { + name string + want bool + }{ + {name: "115 Cloud", want: true}, + {name: "115 Open", want: true}, + {name: "Thunder", want: true}, + {name: "ThunderX", want: true}, + {name: "ThunderBrowser", want: true}, + {name: "aria2", want: false}, + {name: "SimpleHttp", want: false}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := isEd2kCapableTool(tt.name); got != tt.want { + t.Fatalf("isEd2kCapableTool(%q) = %v, want %v", tt.name, got, tt.want) + } + }) + } +} + +func TestEd2kToolForStorage(t *testing.T) { + tests := []struct { + name string + storage driver.Driver + want string + }{ + {name: "115 Cloud", storage: &_115.Pan115{}, want: "115 Cloud"}, + {name: "115 Open", storage: &_115_open.Open115{}, want: "115 Open"}, + {name: "other", storage: nil, want: ""}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := ed2kToolForStorage(tt.storage); got != tt.want { + t.Fatalf("ed2kToolForStorage(%T) = %q, want %q", tt.storage, got, tt.want) + } + }) + } +} + +func TestNativeToolName(t *testing.T) { + tests := []struct { + name string + storage driver.Driver + want string + }{ + {name: "115 Cloud", storage: &_115.Pan115{}, want: "115 Cloud"}, + {name: "115 Open", storage: &_115_open.Open115{}, want: "115 Open"}, + {name: "123Pan", storage: &_123.Pan123{}, want: "123Pan"}, + {name: "123 Open", storage: &_123_open.Open123{}, want: "123 Open"}, + {name: "GuangYaPan", storage: &guangyapan.GuangYaPan{}, want: "GuangYaPan"}, + {name: "PikPak", storage: &pikpak.PikPak{}, want: "PikPak"}, + {name: "Thunder", storage: &thunder.Thunder{}, want: "Thunder"}, + {name: "ThunderX", storage: &thunderx.ThunderX{}, want: "ThunderX"}, + {name: "ThunderBrowser", storage: &thunder_browser.ThunderBrowser{}, want: "ThunderBrowser"}, + {name: "other", storage: nil, want: ""}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := NativeToolName(tt.storage); got != tt.want { + t.Fatalf("NativeToolName(%T) = %q, want %q", tt.storage, got, tt.want) + } + }) + } +} diff --git a/internal/offline_download/tool/download.go b/internal/offline_download/tool/download.go index b7f7a1a9..ca877683 100644 --- a/internal/offline_download/tool/download.go +++ b/internal/offline_download/tool/download.go @@ -98,6 +98,9 @@ outer: if t.tool.Name() == "ThunderX" { return nil } + if t.tool.Name() == "GuangYaPan" { + return nil + } if t.tool.Name() == "115 Cloud" { // hack for 115 <-time.After(time.Second * 1) @@ -176,7 +179,7 @@ func (t *DownloadTask) Update() (bool, error) { func (t *DownloadTask) Transfer() error { toolName := t.tool.Name() - if toolName == "115 Cloud" || toolName == "115 Open" || toolName == "123 Open" || toolName == "123Pan" || toolName == "PikPak" || toolName == "Thunder" || toolName == "ThunderX" || toolName == "ThunderBrowser" { + if toolName == "115 Cloud" || toolName == "115 Open" || toolName == "123 Open" || toolName == "123Pan" || toolName == "PikPak" || toolName == "Thunder" || toolName == "ThunderX" || toolName == "ThunderBrowser" || toolName == "GuangYaPan" { // 如果不是直接下载到目标路径，则进行转存 if t.TempDir != t.DstDirPath { return transferObj(t.Ctx(), t.TempDir, t.DstDirPath, t.DeletePolicy) diff --git a/internal/offline_download/tool/tools.go b/internal/offline_download/tool/tools.go index d5a90f50..57fccd06 100644 --- a/internal/offline_download/tool/tools.go +++ b/internal/offline_download/tool/tools.go @@ -5,6 +5,7 @@ import ( "sort" "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/internal/op" ) var ( @@ -35,6 +36,31 @@ func (t ToolsManager) Names() []string { return names } +// NamesForPath returns ready tools and the native tool for the destination storage. +// Native tools can write directly to their own storage even without a temporary path setting. +func (t ToolsManager) NamesForPath(path string) []string { + names := t.Names() + storage, _, err := op.GetStorageAndActualPath(path) + if err != nil { + return names + } + + name := NativeToolName(storage) + if name == "" { + return names + } + for _, existing := range names { + if existing == name { + return names + } + } + if _, ok := t[name]; ok { + names = append(names, name) + sort.Strings(names) + } + return names +} + func (t ToolsManager) Items() []model.SettingItem { var items []model.SettingItem for _, tool := range t { diff --git a/internal/offline_download/tool/transfer.go b/internal/offline_download/tool/transfer.go index 7109669e..7c5bd164 100644 --- a/internal/offline_download/tool/transfer.go +++ b/internal/offline_download/tool/transfer.go @@ -20,7 +20,6 @@ import ( "github.com/OpenListTeam/OpenList/v4/pkg/http_range" "github.com/OpenListTeam/OpenList/v4/pkg/torrent" "github.com/OpenListTeam/OpenList/v4/pkg/utils" - "github.com/OpenListTeam/OpenList/v4/server/common" "github.com/OpenListTeam/tache" "github.com/pkg/errors" log "github.com/sirupsen/logrus" @@ -140,7 +139,7 @@ func transferStd(ctx context.Context, tempDir, dstDirPath string, deletePolicy D TaskData: fs.TaskData{ TaskExtension: task.TaskExtension{ Creator: taskCreator, - ApiUrl: common.GetApiUrl(ctx), + ApiUrl: conf.GetApiUrl(ctx), }, SrcActualPath: stdpath.Join(tempDir, entry.Name()), DstActualPath: dstDirActualPath, @@ -276,7 +275,7 @@ func transferObj(ctx context.Context, tempDir, dstDirPath string, deletePolicy D TaskData: fs.TaskData{ TaskExtension: task.TaskExtension{ Creator: taskCreator, - ApiUrl: common.GetApiUrl(ctx), + ApiUrl: conf.GetApiUrl(ctx), }, SrcActualPath: stdpath.Join(srcObjActualPath, obj.GetName()), DstActualPath: dstDirActualPath, diff --git a/internal/op/archive.go b/internal/op/archive.go index d0a53a91..bb3de11a 100644 --- a/internal/op/archive.go +++ b/internal/op/archive.go @@ -390,8 +390,9 @@ func ArchiveGet(ctx context.Context, storage driver.Driver, path string, args mo } type objWithLink struct { - link *model.Link - obj model.Obj + link *model.Link + obj model.Obj + policy linkCachePolicy } var ( @@ -405,7 +406,7 @@ func DriverExtract(ctx context.Context, storage driver.Driver, path string, args } key := stdpath.Join(Key(storage, path), args.InnerPath) if ol, ok := extractCache.Get(key); ok { - if ol.link.Expiration != nil || ol.link.SyncClosers.AcquireReference() || !ol.link.RequireReference { + if ol.acquire() { return ol.link, ol.obj, nil } } @@ -415,8 +416,8 @@ func DriverExtract(ctx context.Context, storage driver.Driver, path string, args if err != nil { return nil, errors.Wrapf(err, "failed extract archive") } - if ol.link.Expiration != nil { - extractCache.SetWithTTL(key, ol, *ol.link.Expiration) + if ol.policy.expiration != nil { + extractCache.SetWithTTL(key, ol, *ol.policy.expiration) } else { extractCache.SetWithExpirable(key, ol, &ol.link.SyncClosers) } @@ -428,7 +429,7 @@ func DriverExtract(ctx context.Context, storage driver.Driver, path string, args if err != nil { return nil, nil, err } - if ol.link.SyncClosers.AcquireReference() || !ol.link.RequireReference { + if ol.acquire() { return ol.link, ol.obj, nil } } @@ -450,7 +451,10 @@ func driverExtract(ctx context.Context, storage driver.Driver, path string, args return nil, errors.WithStack(errs.NotFile) } link, err := storageAr.Extract(ctx, archiveFile, args) - return &objWithLink{link: link, obj: extracted}, err + if err != nil { + return nil, err + } + return admitLink(link, extracted) } type streamWithParent struct { diff --git a/internal/op/fs.go b/internal/op/fs.go index f82a3ca8..308e6236 100644 --- a/internal/op/fs.go +++ b/internal/op/fs.go @@ -233,7 +233,10 @@ func Link(ctx context.Context, storage driver.Driver, path string, args model.Li if mode == -1 { mode = storage.(driver.LinkCacheModeResolver).ResolveLinkCacheMode(path) } - typeKey := args.Type + typeKey := "proxy/" + args.Type + if args.Redirect { + typeKey = "redirect/" + args.Type + } if mode&driver.LinkCacheIP != 0 { typeKey += "/" + args.IP } @@ -242,8 +245,7 @@ func Link(ctx context.Context, storage driver.Driver, path string, args model.Li } key := Key(storage, path) if ol, exists := Cache.linkCache.GetType(key, typeKey); exists { - if ol.link.Expiration != nil || - ol.link.SyncClosers.AcquireReference() || !ol.link.RequireReference { + if ol.acquire() { return ol.link, ol.obj, nil } } @@ -261,9 +263,12 @@ func Link(ctx context.Context, storage driver.Driver, path string, args model.Li if err != nil { return nil, errors.Wrapf(err, "failed get link") } - ol := &objWithLink{link: link, obj: file} - if link.Expiration != nil { - Cache.linkCache.SetTypeWithTTL(key, typeKey, ol, *link.Expiration) + ol, err := admitLink(link, file) + if err != nil { + return nil, err + } + if ol.policy.expiration != nil { + Cache.linkCache.SetTypeWithTTL(key, typeKey, ol, *ol.policy.expiration) } else { Cache.linkCache.SetTypeWithExpirable(key, typeKey, ol, &link.SyncClosers) } @@ -274,7 +279,7 @@ func Link(ctx context.Context, storage driver.Driver, path string, args model.Li if err != nil { return nil, nil, err } - if ol.link.SyncClosers.AcquireReference() || !ol.link.RequireReference { + if ol.acquire() { return ol.link, ol.obj, nil } } @@ -514,7 +519,7 @@ func Copy(ctx context.Context, storage driver.Driver, srcPath, dstDirPath string srcPath = utils.FixAndCleanPath(srcPath) dstDirPath = utils.FixAndCleanPath(dstDirPath) if dstDirPath == stdpath.Dir(srcPath) { - return errors.New("copy in place") + return errors.WithStack(errs.NotImplement) } srcRawObj, err := Get(ctx, storage, srcPath, true) if err != nil { @@ -632,7 +637,7 @@ func Put(ctx context.Context, storage driver.Driver, dstDirPath string, file mod if err != nil { return errors.WithMessagef(err, "while uploading, failed remove existing file which size = 0") } - } else if storage.Config().NoOverwriteUpload { + } else if storage.Config().NoOverwriteUpload && ctx.Value(conf.SkipNoOverwriteKey) == nil { // try to rename old obj err = Rename(ctx, storage, dstPath, tempName) if err != nil { diff --git a/internal/op/hook.go b/internal/op/hook.go index 5cf01730..3d8530f9 100644 --- a/internal/op/hook.go +++ b/internal/op/hook.go @@ -2,7 +2,9 @@ package op import ( "context" + "fmt" "regexp" + "strconv" "strings" "github.com/OpenListTeam/OpenList/v4/internal/conf" @@ -83,6 +85,17 @@ var settingItemHooks = map[string]SettingItemHook{ conf.SlicesMap[conf.IgnoreDirectLinkParams] = strings.Split(item.Value, ",") return nil }, + conf.MultipartChunkSize: func(item *model.SettingItem) error { + size, err := strconv.Atoi(strings.TrimSpace(item.Value)) + if err != nil || size < 1 { + // deliberately a plain error: SaveSettings formats hook errors + // with %+v, which would dump a full stack trace into the UI + // notification for stack-carrying errors + return fmt.Errorf("multipart chunk size must be a positive integer (MB), got %q", item.Value) + } + item.Value = strconv.Itoa(size) + return nil + }, } func RegisterSettingItemHook(key string, hook SettingItemHook) { diff --git a/internal/op/link_cache_test.go b/internal/op/link_cache_test.go new file mode 100644 index 00000000..6924bf83 --- /dev/null +++ b/internal/op/link_cache_test.go @@ -0,0 +1,71 @@ +package op + +import ( + "context" + "io" + "strings" + "testing" + "time" + + "github.com/OpenListTeam/OpenList/v4/internal/driver" + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/internal/stream" + "github.com/OpenListTeam/OpenList/v4/pkg/http_range" +) + +type linkModeDriver struct { + driver.Driver + storage model.Storage + calls int +} + +func (d *linkModeDriver) Config() driver.Config { return driver.Config{} } + +func (d *linkModeDriver) GetStorage() *model.Storage { return &d.storage } + +func (d *linkModeDriver) Get(context.Context, string) (model.Obj, error) { + return &model.Object{Name: "file"}, nil +} + +func (d *linkModeDriver) Link(_ context.Context, _ model.Obj, args model.LinkArgs) (*model.Link, error) { + d.calls++ + expiration := time.Minute + if args.Redirect { + return &model.Link{URL: "https://example.com/file", Expiration: &expiration}, nil + } + return &model.Link{ + RangeReader: stream.RangeReaderFunc(func(context.Context, http_range.Range) (io.ReadCloser, error) { + return io.NopCloser(strings.NewReader("file")), nil + }), + Expiration: &expiration, + }, nil +} + +func TestLinkCacheSeparatesRedirectAndProxy(t *testing.T) { + for _, tc := range []struct { + name string + firstRedirect bool + }{ + {name: "redirect then proxy", firstRedirect: true}, + {name: "proxy then redirect", firstRedirect: false}, + } { + t.Run(tc.name, func(t *testing.T) { + d := &linkModeDriver{storage: model.Storage{MountPath: "/" + t.Name()}} + for _, redirect := range []bool{tc.firstRedirect, !tc.firstRedirect, tc.firstRedirect, !tc.firstRedirect} { + link, _, err := Link(context.Background(), d, "/file", model.LinkArgs{Redirect: redirect}) + if err != nil { + t.Fatal(err) + } + if redirect && (link.URL == "" || link.RangeReader != nil) { + t.Fatalf("redirect link has wrong shape: %+v", link) + } + if !redirect && (link.URL != "" || link.RangeReader == nil) { + t.Fatalf("proxy link has wrong shape: %+v", link) + } + } + if d.calls != 2 { + t.Fatalf("expected one driver call per mode, got %d", d.calls) + } + }) + } +} diff --git a/internal/op/link_lifecycle.go b/internal/op/link_lifecycle.go new file mode 100644 index 00000000..71491c97 --- /dev/null +++ b/internal/op/link_lifecycle.go @@ -0,0 +1,34 @@ +package op + +import ( + "errors" + "time" + + "github.com/OpenListTeam/OpenList/v4/internal/model" +) + +var errConflictingLinkLifecycle = errors.New("invalid link lifecycle: expiration cannot be combined with owned closers or RequireReference") + +type linkCachePolicy struct { + expiration *time.Duration + requireReference bool +} + +func admitLink(link *model.Link, obj model.Obj) (*objWithLink, error) { + if link.Expiration != nil && (link.RequireReference || link.SyncClosers.Length() > 0) { + return nil, errors.Join(errConflictingLinkLifecycle, link.Close()) + } + return &objWithLink{ + link: link, + obj: obj, + policy: linkCachePolicy{ + expiration: link.Expiration, + requireReference: link.RequireReference, + }, + }, nil +} + +func (ol *objWithLink) acquire() bool { + return ol.policy.expiration != nil || + ol.link.SyncClosers.AcquireReference() || !ol.policy.requireReference +} diff --git a/internal/op/link_lifecycle_test.go b/internal/op/link_lifecycle_test.go new file mode 100644 index 00000000..53bb0baf --- /dev/null +++ b/internal/op/link_lifecycle_test.go @@ -0,0 +1,143 @@ +package op + +import ( + "context" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/OpenListTeam/OpenList/v4/internal/driver" + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/pkg/singleflight" + "github.com/OpenListTeam/OpenList/v4/pkg/utils" +) + +type linkLifecycleDriver struct { + model.Storage + links func() *model.Link + calls atomic.Int32 +} + +func (d *linkLifecycleDriver) Config() driver.Config { return driver.Config{} } +func (d *linkLifecycleDriver) GetAddition() driver.Additional { return nil } +func (d *linkLifecycleDriver) Init(context.Context) error { return nil } +func (d *linkLifecycleDriver) Drop(context.Context) error { return nil } +func (d *linkLifecycleDriver) List(context.Context, model.Obj, model.ListArgs) ([]model.Obj, error) { + return nil, nil +} +func (d *linkLifecycleDriver) Get(context.Context, string) (model.Obj, error) { + return &model.Object{Name: "file", Path: "/file"}, nil +} +func (d *linkLifecycleDriver) Link(context.Context, model.Obj, model.LinkArgs) (*model.Link, error) { + d.calls.Add(1) + return d.links(), nil +} + +func resetLinkLifecycleState(t *testing.T) { + t.Helper() + oldCache := Cache + Cache, linkG = NewCacheManager(), singleflight.Group[*objWithLink]{} + t.Cleanup(func() { Cache, linkG = oldCache, singleflight.Group[*objWithLink]{} }) +} + +func acquireTestLink(t *testing.T, d *linkLifecycleDriver) *model.Link { + t.Helper() + link, _, err := Link(context.Background(), d, "/file", model.LinkArgs{}) + if err != nil { + t.Fatal(err) + } + return link +} + +func TestLinkLifecycleModes(t *testing.T) { + t.Run("TTL descriptor remains reusable after close", func(t *testing.T) { + resetLinkLifecycleState(t) + ttl := time.Minute + d := &linkLifecycleDriver{ + Storage: model.Storage{MountPath: "/ttl"}, + links: func() *model.Link { return &model.Link{URL: "https://example.test/file", Expiration: &ttl} }, + } + + first := acquireTestLink(t, d) + _ = first.Close() + second := acquireTestLink(t, d) + if second.URL != first.URL || d.calls.Load() != 1 { + t.Fatalf("TTL link was not reused: calls=%d", d.calls.Load()) + } + _ = second.Close() + }) + + t.Run("references keep shared resources alive until final close", func(t *testing.T) { + resetLinkLifecycleState(t) + var closes atomic.Int32 + d := &linkLifecycleDriver{ + Storage: model.Storage{MountPath: "/reference"}, + links: func() *model.Link { + return &model.Link{ + URL: "https://example.test/file", + SyncClosers: utils.NewSyncClosers(utils.CloseFunc(func() error { closes.Add(1); return nil })), + RequireReference: true, + } + }, + } + + first := acquireTestLink(t, d) + second := acquireTestLink(t, d) + _ = first.Close() + if closes.Load() != 0 { + t.Fatal("shared resource closed while another reference was active") + } + _ = second.Close() + if closes.Load() != 1 { + t.Fatalf("final close count = %d, want 1", closes.Load()) + } + third := acquireTestLink(t, d) + _ = third.Close() + if d.calls.Load() != 2 || closes.Load() != 2 { + t.Fatalf("stale link was not replaced: calls=%d closes=%d", d.calls.Load(), closes.Load()) + } + }) + + t.Run("close-invalidated link is reacquired", func(t *testing.T) { + resetLinkLifecycleState(t) + d := &linkLifecycleDriver{ + Storage: model.Storage{MountPath: "/close-invalidated"}, + links: func() *model.Link { + return &model.Link{SyncClosers: utils.NewSyncClosers(utils.CloseFunc(func() error { return nil }))} + }, + } + + first := acquireTestLink(t, d) + _ = first.Close() + second := acquireTestLink(t, d) + _ = second.Close() + if d.calls.Load() != 2 { + t.Fatalf("driver calls = %d, want 2", d.calls.Load()) + } + }) + + t.Run("TTL with owned resources is rejected and released", func(t *testing.T) { + resetLinkLifecycleState(t) + ttl := time.Minute + var closes atomic.Int32 + d := &linkLifecycleDriver{ + Storage: model.Storage{MountPath: "/conflict"}, + links: func() *model.Link { + return &model.Link{ + Expiration: &ttl, + SyncClosers: utils.NewSyncClosers(utils.CloseFunc(func() error { closes.Add(1); return nil })), + RequireReference: true, + } + }, + } + + _, _, err := Link(context.Background(), d, "/file", model.LinkArgs{}) + if err == nil || !strings.Contains(err.Error(), "expiration cannot be combined") { + t.Fatalf("unexpected error: %v", err) + } + if closes.Load() != 1 { + t.Fatalf("rejected link close count = %d, want 1", closes.Load()) + } + }) +} diff --git a/internal/op/meta.go b/internal/op/meta.go index b7d86730..526332b2 100644 --- a/internal/op/meta.go +++ b/internal/op/meta.go @@ -2,6 +2,8 @@ package op import ( stdpath "path" + "strings" + "sync" "time" "github.com/OpenListTeam/OpenList/v4/internal/db" @@ -15,6 +17,9 @@ import ( ) var metaCache = cache.NewMemCache(cache.WithShards[*model.Meta](2)) +var metaSnapshotMu sync.Mutex +var metaSnapshot []model.Meta +var metaSnapshotLoaded bool // metaG maybe not needed var metaG singleflight.Group[*model.Meta] @@ -23,17 +28,56 @@ func GetNearestMeta(path string) (*model.Meta, error) { return getNearestMeta(utils.FixAndCleanPath(path)) } func getNearestMeta(path string) (*model.Meta, error) { - meta, err := GetMetaByPath(path) - if err == nil { - return meta, nil + var metas []model.Meta + loaded := false + for { + meta, err := GetMetaByPath(path) + if err == nil { + return meta, nil + } + if errors.Cause(err) != errs.MetaNotFound { + return nil, err + } + if !loaded { + metas, err = getMetaSnapshot() + if err != nil { + return nil, err + } + loaded = true + } + var matched *model.Meta + for i := range metas { + if !strings.EqualFold(utils.FixAndCleanPath(metas[i].Path), path) { + continue + } + if matched != nil { + return nil, errors.Errorf("multiple metas match %q case-insensitively", path) + } + matched = &metas[i] + } + if matched != nil { + return matched, nil + } + if path == "/" { + return nil, errs.MetaNotFound + } + path = stdpath.Dir(path) } - if errors.Cause(err) != errs.MetaNotFound { +} + +func getMetaSnapshot() ([]model.Meta, error) { + metaSnapshotMu.Lock() + defer metaSnapshotMu.Unlock() + if metaSnapshotLoaded { + return metaSnapshot, nil + } + metas, err := db.GetAllMetas() + if err != nil { return nil, err } - if path == "/" { - return nil, errs.MetaNotFound - } - return getNearestMeta(stdpath.Dir(path)) + metaSnapshot = metas + metaSnapshotLoaded = true + return metaSnapshot, nil } func GetMetaByPath(path string) (*model.Meta, error) { @@ -68,7 +112,12 @@ func DeleteMetaById(id uint) error { return err } metaCache.Del(old.Path) - return db.DeleteMetaById(id) + metaSnapshotMu.Lock() + defer metaSnapshotMu.Unlock() + err = db.DeleteMetaById(id) + metaSnapshot = nil + metaSnapshotLoaded = false + return err } func UpdateMeta(u *model.Meta) error { @@ -79,13 +128,23 @@ func UpdateMeta(u *model.Meta) error { } metaCache.Del(old.Path) metaCache.Del(u.Path) - return db.UpdateMeta(u) + metaSnapshotMu.Lock() + defer metaSnapshotMu.Unlock() + err = db.UpdateMeta(u) + metaSnapshot = nil + metaSnapshotLoaded = false + return err } func CreateMeta(u *model.Meta) error { u.Path = utils.FixAndCleanPath(u.Path) metaCache.Del(u.Path) - return db.CreateMeta(u) + metaSnapshotMu.Lock() + defer metaSnapshotMu.Unlock() + err := db.CreateMeta(u) + metaSnapshot = nil + metaSnapshotLoaded = false + return err } func GetMetaById(id uint) (*model.Meta, error) { diff --git a/internal/op/path.go b/internal/op/path.go index 294261e8..9157e2f2 100644 --- a/internal/op/path.go +++ b/internal/op/path.go @@ -30,6 +30,20 @@ func GetStorageAndActualPath(rawPath string) (storage driver.Driver, actualPath return } +// GetStorageVirtualMountPath returns the deterministic virtual mount path +// without advancing the balanced-storage counter. +func GetStorageVirtualMountPath(rawPath string) (string, error) { + rawPath = utils.FixAndCleanPath(rawPath) + storages := getStoragesByPath(rawPath) + if len(storages) == 0 { + if rawPath == "/" { + return "", errs.NewErr(errs.StorageNotFound, "please add a storage first") + } + return "", errs.NewErr(errs.StorageNotFound, "rawPath: %s", rawPath) + } + return utils.FixAndCleanPath(utils.GetActualMountPath(storages[0].GetStorage().MountPath)), nil +} + // urlTreeSplitLineFormPath 分割path中分割真实路径和UrlTree定义字符串 func urlTreeSplitLineFormPath(path string) (pp string, file string) { // url.PathUnescape 会移除 // ，手动加回去 diff --git a/internal/op/sharing.go b/internal/op/sharing.go index 5be0f8e6..234ca415 100644 --- a/internal/op/sharing.go +++ b/internal/op/sharing.go @@ -91,6 +91,16 @@ func GetSharingUnwrapPath(sharing *model.Sharing, path string) (unwrapPath strin if len(sharing.Files) == 0 { return "", errors.New("cannot get actual path of an invalid sharing") } + // Re-validate that the shared paths are still within the creator's current + // BasePath. This prevents access to files that fell out-of-scope after the + // creator's BasePath was changed by an admin. + if sharing.Creator != nil && !sharing.Creator.IsAdmin() { + for _, f := range sharing.Files { + if !utils.IsSubPath(sharing.Creator.BasePath, f) { + return "", errors.Errorf("sharing path [%s] is outside the creator's base path", f) + } + } + } if len(sharing.Files) == 1 { return stdpath.Join(sharing.Files[0], path), nil } diff --git a/internal/op/storage.go b/internal/op/storage.go index da4c84e3..42b8c9d8 100644 --- a/internal/op/storage.go +++ b/internal/op/storage.go @@ -237,14 +237,16 @@ func UpdateStorage(ctx context.Context, storage model.Storage) error { return nil } storageDriver, err := GetStorageByMountPath(oldStorage.MountPath) + if err != nil { + return errors.WithMessage(err, "failed get storage driver") + } + // Storage settings may change which objects are exposed at the same mount + // path, so cached entries must be discarded before reinitializing the driver. + Cache.DeleteDirectoryTree(storageDriver, "/") + Cache.InvalidateStorageDetails(storageDriver) if oldStorage.MountPath != storage.MountPath { // mount path renamed, need to drop the storage storagesMap.Delete(oldStorage.MountPath) - Cache.DeleteDirectoryTree(storageDriver, "/") - Cache.InvalidateStorageDetails(storageDriver) - } - if err != nil { - return errors.WithMessage(err, "failed get storage driver") } err = storageDriver.Drop(ctx) if err != nil { diff --git a/internal/search/bleve/search.go b/internal/search/bleve/search.go index e95e82df..ed151871 100644 --- a/internal/search/bleve/search.go +++ b/internal/search/bleve/search.go @@ -26,18 +26,9 @@ func (b *Bleve) Config() searcher.Config { } func (b *Bleve) Search(ctx context.Context, req model.SearchReq) ([]model.SearchNode, int64, error) { - var queries []query2.Query - query := bleve.NewMatchQuery(req.Keywords) - query.SetField("name") - queries = append(queries, query) - if req.Scope != 0 { - isDir := req.Scope == 1 - isDirQuery := bleve.NewBoolFieldQuery(isDir) - queries = append(queries, isDirQuery) - } - reqQuery := bleve.NewConjunctionQuery(queries...) + reqQuery := buildQuery(req) search := bleve.NewSearchRequest(reqQuery) - search.SortBy([]string{"name"}) + search.SortBy([]string{"name", "_id"}) search.From = (req.Page - 1) * req.PerPage search.Size = req.PerPage search.Fields = []string{"*"} @@ -47,14 +38,74 @@ func (b *Bleve) Search(ctx context.Context, req model.SearchReq) ([]model.Search return nil, 0, err } res, err := utils.SliceConvert(searchResults.Hits, func(src *search2.DocumentMatch) (model.SearchNode, error) { - return model.SearchNode{ - Parent: src.Fields["parent"].(string), - Name: src.Fields["name"].(string), - IsDir: src.Fields["is_dir"].(bool), - Size: int64(src.Fields["size"].(float64)), - }, nil + return searchNodeFromHit(src), nil }) - return res, int64(searchResults.Total), nil + return res, int64(searchResults.Total), err +} + +const searchBatchSize = 1000 + +func (b *Bleve) SearchFiltered(ctx context.Context, req model.SearchReq, filter searcher.Filter) ([]model.SearchNode, int64, error) { + reqQuery := buildQuery(req) + from := int64(req.Page-1) * int64(req.PerPage) + to := from + int64(req.PerPage) + var ( + result []model.SearchNode + total int64 + searchAfter []string + ) + for { + search := bleve.NewSearchRequest(reqQuery) + search.SortBy([]string{"name", "_id"}) + search.Size = searchBatchSize + search.Fields = []string{"*"} + if searchAfter != nil { + search.SetSearchAfter(searchAfter) + } + searchResults, err := b.BIndex.Search(search) + if err != nil { + log.Errorf("search error: %+v", err) + return nil, 0, err + } + for _, hit := range searchResults.Hits { + node := searchNodeFromHit(hit) + if !utils.IsSubPath(req.Parent, node.Parent) || filter != nil && !filter(node) { + continue + } + if total >= from && total < to { + result = append(result, node) + } + total++ + } + if len(searchResults.Hits) < searchBatchSize { + break + } + last := searchResults.Hits[len(searchResults.Hits)-1] + searchAfter = append(searchAfter[:0], last.Sort...) + } + return result, total, nil +} + +func buildQuery(req model.SearchReq) query2.Query { + var queries []query2.Query + query := bleve.NewMatchQuery(req.Keywords) + query.SetField("name") + queries = append(queries, query) + if req.Scope != 0 { + isDir := req.Scope == 1 + isDirQuery := bleve.NewBoolFieldQuery(isDir) + queries = append(queries, isDirQuery) + } + return bleve.NewConjunctionQuery(queries...) +} + +func searchNodeFromHit(src *search2.DocumentMatch) model.SearchNode { + return model.SearchNode{ + Parent: src.Fields["parent"].(string), + Name: src.Fields["name"].(string), + IsDir: src.Fields["is_dir"].(bool), + Size: int64(src.Fields["size"].(float64)), + } } func (b *Bleve) Index(ctx context.Context, node model.SearchNode) error { @@ -103,3 +154,4 @@ func (b *Bleve) Clear(ctx context.Context) error { } var _ searcher.Searcher = (*Bleve)(nil) +var _ searcher.FilteredSearcher = (*Bleve)(nil) diff --git a/internal/search/bleve/search_test.go b/internal/search/bleve/search_test.go new file mode 100644 index 00000000..5b4ca703 --- /dev/null +++ b/internal/search/bleve/search_test.go @@ -0,0 +1,52 @@ +package bleve + +import ( + "context" + "fmt" + "testing" + + "github.com/OpenListTeam/OpenList/v4/internal/model" + blevelib "github.com/blevesearch/bleve/v2" +) + +func TestSearchFilteredKeepsDuplicateSortValuesAcrossBatches(t *testing.T) { + indexMapping := blevelib.NewIndexMapping() + searchNodeMapping := blevelib.NewDocumentMapping() + searchNodeMapping.AddFieldMappingsAt("is_dir", blevelib.NewBooleanFieldMapping()) + searchNodeMapping.AddFieldMappingsAt("parent", blevelib.NewTextFieldMapping()) + searchNodeMapping.AddFieldMappingsAt("name", blevelib.NewKeywordFieldMapping()) + indexMapping.AddDocumentMapping("SearchNode", searchNodeMapping) + index, err := blevelib.NewMemOnly(indexMapping) + if err != nil { + t.Fatalf("NewMemOnly() error = %v", err) + } + t.Cleanup(func() { _ = index.Close() }) + + batch := index.NewBatch() + for i := 0; i < searchBatchSize+1; i++ { + batch.Index(fmt.Sprintf("allowed-%04d", i), model.SearchNode{ + Parent: "/base", + Name: "duplicate", + }) + } + batch.Index("denied", model.SearchNode{Parent: "/base2", Name: "duplicate"}) + if err := index.Batch(batch); err != nil { + t.Fatalf("Batch() error = %v", err) + } + + b := &Bleve{BIndex: index} + nodes, total, err := b.SearchFiltered(context.Background(), model.SearchReq{ + Parent: "/base", + Keywords: "duplicate", + PageReq: model.PageReq{Page: 1, PerPage: searchBatchSize + 1}, + }, nil) + if err != nil { + t.Fatalf("SearchFiltered() error = %v", err) + } + if total != searchBatchSize+1 { + t.Fatalf("SearchFiltered() total = %d, want %d", total, searchBatchSize+1) + } + if len(nodes) != searchBatchSize+1 { + t.Fatalf("SearchFiltered() returned %d nodes, want %d", len(nodes), searchBatchSize+1) + } +} diff --git a/internal/search/build.go b/internal/search/build.go index bbed1f85..8dbe7d49 100644 --- a/internal/search/build.go +++ b/internal/search/build.go @@ -225,6 +225,9 @@ func Update(ctx context.Context, parent string, objs []model.Obj) { return } + unlock := lockUpdate(parent) + defer unlock() + nodes, err := instance.Get(ctx, parent) if err != nil { log.Errorf("update search index error while get nodes: %+v", err) diff --git a/internal/search/build_test.go b/internal/search/build_test.go new file mode 100644 index 00000000..fed2e155 --- /dev/null +++ b/internal/search/build_test.go @@ -0,0 +1,58 @@ +package search + +import ( + "testing" + "time" +) + +func TestLockUpdateSerializesSameParent(t *testing.T) { + unlockFirst := lockUpdate("/same-parent") + secondStarted := make(chan struct{}) + secondAcquired := make(chan struct{}) + secondReleased := make(chan struct{}) + go func() { + close(secondStarted) + unlockSecond := lockUpdate("/same-parent") + close(secondAcquired) + unlockSecond() + close(secondReleased) + }() + <-secondStarted + + select { + case <-secondAcquired: + t.Fatal("second update acquired the same parent lock") + case <-time.After(20 * time.Millisecond): + } + + unlockFirst() + select { + case <-secondReleased: + case <-time.After(time.Second): + t.Fatal("second update did not acquire the released parent lock") + } + + updateLocksMu.Lock() + defer updateLocksMu.Unlock() + if len(updateLocks) != 0 { + t.Fatalf("update locks were not cleaned up: %d", len(updateLocks)) + } +} + +func TestLockUpdateAllowsDifferentParents(t *testing.T) { + unlockFirst := lockUpdate("/first-parent") + defer unlockFirst() + + secondAcquired := make(chan struct{}) + go func() { + unlockSecond := lockUpdate("/second-parent") + unlockSecond() + close(secondAcquired) + }() + + select { + case <-secondAcquired: + case <-time.After(time.Second): + t.Fatal("update for a different parent was blocked") + } +} diff --git a/internal/search/search.go b/internal/search/search.go index 85be8495..0ca1e241 100644 --- a/internal/search/search.go +++ b/internal/search/search.go @@ -51,6 +51,46 @@ func Search(ctx context.Context, req model.SearchReq) ([]model.SearchNode, int64 return instance.Search(ctx, req) } +const searchBatchSize = 1000 + +func SearchFiltered(ctx context.Context, req model.SearchReq, filter searcher.Filter) ([]model.SearchNode, int64, error) { + if filteredSearcher, ok := instance.(searcher.FilteredSearcher); ok { + return filteredSearcher.SearchFiltered(ctx, req, filter) + } + + batchReq := req + batchReq.Page = 1 + batchReq.PerPage = searchBatchSize + from := int64(req.Page-1) * int64(req.PerPage) + to := from + int64(req.PerPage) + var ( + result []model.SearchNode + filteredTotal int64 + processedTotal int64 + ) + for { + nodes, total, err := instance.Search(ctx, batchReq) + if err != nil { + return nil, 0, err + } + for _, node := range nodes { + if filter != nil && !filter(node) { + continue + } + if filteredTotal >= from && filteredTotal < to { + result = append(result, node) + } + filteredTotal++ + } + processedTotal += int64(len(nodes)) + if len(nodes) == 0 || processedTotal >= total { + break + } + batchReq.Page++ + } + return result, filteredTotal, nil +} + func Index(ctx context.Context, parent string, obj model.Obj) error { if instance == nil { return errs.SearchNotAvailable diff --git a/internal/search/search_test.go b/internal/search/search_test.go new file mode 100644 index 00000000..0b1d75d4 --- /dev/null +++ b/internal/search/search_test.go @@ -0,0 +1,61 @@ +package search + +import ( + "context" + "testing" + + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/internal/search/searcher" +) + +type filteredSearchStub struct { + nodes []model.SearchNode +} + +func (s *filteredSearchStub) Config() searcher.Config { + return searcher.Config{Name: "stub"} +} + +func (s *filteredSearchStub) Search(_ context.Context, req model.SearchReq) ([]model.SearchNode, int64, error) { + from := (req.Page - 1) * req.PerPage + if from >= len(s.nodes) { + return nil, int64(len(s.nodes)), nil + } + to := min(from+req.PerPage, len(s.nodes)) + return s.nodes[from:to], int64(len(s.nodes)), nil +} + +func (s *filteredSearchStub) Index(context.Context, model.SearchNode) error { return nil } +func (s *filteredSearchStub) BatchIndex(context.Context, []model.SearchNode) error { return nil } +func (s *filteredSearchStub) Get(context.Context, string) ([]model.SearchNode, error) { + return nil, nil +} +func (s *filteredSearchStub) Del(context.Context, string) error { return nil } +func (s *filteredSearchStub) Release(context.Context) error { return nil } +func (s *filteredSearchStub) Clear(context.Context) error { return nil } + +func TestSearchFilteredFiltersBeforePagination(t *testing.T) { + previous := instance + instance = &filteredSearchStub{nodes: []model.SearchNode{ + {Name: "denied-1"}, + {Name: "allowed-1"}, + {Name: "denied-2"}, + {Name: "allowed-2"}, + }} + t.Cleanup(func() { instance = previous }) + + nodes, total, err := SearchFiltered(context.Background(), model.SearchReq{ + PageReq: model.PageReq{Page: 2, PerPage: 1}, + }, func(node model.SearchNode) bool { + return node.Name == "allowed-1" || node.Name == "allowed-2" + }) + if err != nil { + t.Fatalf("SearchFiltered() error = %v", err) + } + if total != 2 { + t.Fatalf("SearchFiltered() total = %d, want 2", total) + } + if len(nodes) != 1 || nodes[0].Name != "allowed-2" { + t.Fatalf("SearchFiltered() nodes = %#v, want allowed-2", nodes) + } +} diff --git a/internal/search/searcher/searcher.go b/internal/search/searcher/searcher.go index e7b7ba1e..aaecdf53 100644 --- a/internal/search/searcher/searcher.go +++ b/internal/search/searcher/searcher.go @@ -11,6 +11,13 @@ type Config struct { AutoUpdate bool } +type Filter func(node model.SearchNode) bool + +// FilteredSearcher applies filtering before calculating pagination and totals. +type FilteredSearcher interface { + SearchFiltered(ctx context.Context, req model.SearchReq, filter Filter) ([]model.SearchNode, int64, error) +} + type Searcher interface { // Config of the searcher Config() Config diff --git a/internal/search/update_lock.go b/internal/search/update_lock.go new file mode 100644 index 00000000..7ca8d28d --- /dev/null +++ b/internal/search/update_lock.go @@ -0,0 +1,38 @@ +package search + +import "sync" + +var ( + updateLocksMu sync.Mutex + updateLocks = make(map[string]*updateLock) +) + +type updateLock struct { + mu sync.Mutex + refs uint +} + +// lockUpdate serializes index updates for the same parent while allowing +// unrelated directories to update concurrently. +func lockUpdate(parent string) func() { + updateLocksMu.Lock() + lock, ok := updateLocks[parent] + if !ok { + lock = &updateLock{} + updateLocks[parent] = lock + } + lock.refs++ + updateLocksMu.Unlock() + + lock.mu.Lock() + return func() { + lock.mu.Unlock() + + updateLocksMu.Lock() + lock.refs-- + if lock.refs == 0 { + delete(updateLocks, parent) + } + updateLocksMu.Unlock() + } +} diff --git a/internal/sharing/link.go b/internal/sharing/link.go index 32ae6b83..e021f3c2 100644 --- a/internal/sharing/link.go +++ b/internal/sharing/link.go @@ -4,11 +4,11 @@ import ( "context" "strings" + "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/OpenListTeam/OpenList/v4/pkg/utils" - "github.com/OpenListTeam/OpenList/v4/server/common" "github.com/pkg/errors" ) @@ -38,7 +38,7 @@ func link(ctx context.Context, sid, path string, args *LinkArgs) (*model.Sharing return nil, nil, nil, errors.WithMessage(err, "failed get sharing link") } if l.URL != "" && !strings.HasPrefix(l.URL, "http://") && !strings.HasPrefix(l.URL, "https://") { - l.URL = common.GetApiUrl(ctx) + l.URL + l.URL = conf.GetApiUrl(ctx) + l.URL } return sharing, l, obj, nil } diff --git a/internal/stream/readat_test.go b/internal/stream/readat_test.go new file mode 100644 index 00000000..3698052c --- /dev/null +++ b/internal/stream/readat_test.go @@ -0,0 +1,179 @@ +package stream_test + +import ( + "bytes" + "context" + "io" + "math/rand" + "sync/atomic" + "testing" + + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/internal/stream" + "github.com/OpenListTeam/OpenList/v4/pkg/http_range" +) + +// maxReuseGap mirrors the internal continuation-reuse window (4*utils.MB). +const maxReuseGap = 4 * 1024 * 1024 + +// newMockSeekableStream builds a SeekableStream whose range reads are served +// from data, counting every upstream range request in gets. +func newMockSeekableStream(t *testing.T, data []byte, gets *atomic.Int64) *stream.SeekableStream { + t.Helper() + rr := stream.RangeReaderFunc(func(ctx context.Context, r http_range.Range) (io.ReadCloser, error) { + gets.Add(1) + if r.Length < 0 || r.Start+r.Length > int64(len(data)) { + r.Length = int64(len(data)) - r.Start + } + return io.NopCloser(io.NewSectionReader(bytes.NewReader(data), r.Start, r.Length)), nil + }) + ss, err := stream.NewSeekableStream(&stream.FileStream{ + Obj: &model.Object{Size: int64(len(data))}, + Ctx: context.Background(), + }, &model.Link{ + RangeReader: rr, + ContentLength: int64(len(data)), + }) + if err != nil { + t.Fatalf("NewSeekableStream() error = %v", err) + } + return ss +} + +// readAtFull reads len(p) bytes at off and fails the test on mismatch. +func readAtFull(t *testing.T, ra io.ReaderAt, data []byte, off int64, p []byte) { + t.Helper() + n, err := ra.ReadAt(p, off) + if err != nil { + t.Fatalf("ReadAt(off=%d) error = %v", off, err) + } + if !bytes.Equal(p, data[off:off+int64(n)]) { + t.Fatalf("ReadAt(off=%d) content mismatch", off) + } +} + +func randomData(size int) []byte { + data := make([]byte, size) + x := uint64(42) + for i := range data { + x = x*6364136223846793005 + 1 + data[i] = byte(x >> 33) + } + return data +} + +// Sequential reads must reuse a single upstream range request. +func TestReadAtSeekerSequentialReuse(t *testing.T) { + data := randomData(16 * 1024 * 1024) + var gets atomic.Int64 + ss := newMockSeekableStream(t, data, &gets) + ra, err := stream.NewReadAtSeeker(ss, 0, true) + if err != nil { + t.Fatalf("NewReadAtSeeker() error = %v", err) + } + buf := make([]byte, 128*1024) + for off := 0; off < len(data); off += len(buf) { + readAtFull(t, ra, data, int64(off), buf) + } + if n := gets.Load(); n != 1 { + t.Fatalf("sequential read issued %d range requests, want 1", n) + } +} + +// A read landing up to maxReuseGap bytes past a parked reader must be served +// by advancing that reader, without a new range request. +func TestReadAtSeekerSkipsAheadWithinWindow(t *testing.T) { + data := randomData(16 * 1024 * 1024) + var gets atomic.Int64 + ss := newMockSeekableStream(t, data, &gets) + ra, err := stream.NewReadAtSeeker(ss, 0, true) + if err != nil { + t.Fatalf("NewReadAtSeeker() error = %v", err) + } + // Park a continuation reader right after reading the first 2 MiB. + chunk := make([]byte, 256*1024) + for off := 0; off < 2*1024*1024; off += len(chunk) { + readAtFull(t, ra, data, int64(off), chunk) + } + skip := 512 * 1024 + off := int64(2*1024*1024 + skip) + readAtFull(t, ra, data, off, chunk) + if n := gets.Load(); n != 1 { + t.Fatalf("window skip issued %d range requests, want 1", n) + } + // A second skip deeper inside the window must also be free. + off = int64(4*1024*1024) - 128*1024 + readAtFull(t, ra, data, off, chunk) + if n := gets.Load(); n != 1 { + t.Fatalf("second window skip issued %d range requests, want 1", n) + } +} + +// A forward jump beyond the reuse window must open a new range request but +// keep the parked reader available for later window hits. +func TestReadAtSeekerFarJumpOpensNewRequest(t *testing.T) { + data := randomData(16 * 1024 * 1024) + var gets atomic.Int64 + ss := newMockSeekableStream(t, data, &gets) + ra, err := stream.NewReadAtSeeker(ss, 0, true) + if err != nil { + t.Fatalf("NewReadAtSeeker() error = %v", err) + } + buf := make([]byte, 256*1024) + for off := 0; off < 2*1024*1024; off += len(buf) { + readAtFull(t, ra, data, int64(off), buf) + } + // 2 MiB -> 10 MiB is beyond the 4 MiB reuse window. + off := int64(10 * 1024 * 1024) + readAtFull(t, ra, data, off, buf) + if n := gets.Load(); n != 2 { + t.Fatalf("far jump issued %d range requests, want 2", n) + } + // Back within the window of the 10 MiB chain: free reuse again. + readAtFull(t, ra, data, off+maxReuseGap, buf) + if n := gets.Load(); n != 2 { + t.Fatalf("jump inside new window issued %d range requests, want 2", n) + } +} + +// Backward reads can never reuse a parked continuation and must open a new +// range request. +func TestReadAtSeekerBackwardJumpOpensNewRequest(t *testing.T) { + data := randomData(8 * 1024 * 1024) + var gets atomic.Int64 + ss := newMockSeekableStream(t, data, &gets) + ra, err := stream.NewReadAtSeeker(ss, 0, true) + if err != nil { + t.Fatalf("NewReadAtSeeker() error = %v", err) + } + buf := make([]byte, 256*1024) + for off := 0; off < 2*1024*1024; off += len(buf) { + readAtFull(t, ra, data, int64(off), buf) + } + readAtFull(t, ra, data, int64(1024*1024), buf) + if n := gets.Load(); n != 2 { + t.Fatalf("backward jump issued %d range requests, want 2", n) + } +} + +// Random reads must return correct data and keep upstream requests bounded: +// each read is either a window hit or a fresh request, never more than one. +func TestReadAtSeekerRandomReads(t *testing.T) { + data := randomData(32 * 1024 * 1024) + var gets atomic.Int64 + ss := newMockSeekableStream(t, data, &gets) + ra, err := stream.NewReadAtSeeker(ss, 0, true) + if err != nil { + t.Fatalf("NewReadAtSeeker() error = %v", err) + } + const chunk = 8 * 1024 + buf := make([]byte, chunk) + r := rand.New(rand.NewSource(7)) + for i := 0; i < 200; i++ { + off := r.Int63n(int64(len(data)) - chunk) + readAtFull(t, ra, data, off, buf) + } + if n := gets.Load(); n > 200 { + t.Fatalf("random reads issued %d range requests, want <= 200", n) + } +} diff --git a/internal/stream/stream.go b/internal/stream/stream.go index 9ae8313f..09db2ea2 100644 --- a/internal/stream/stream.go +++ b/internal/stream/stream.go @@ -8,6 +8,7 @@ import ( "io" "math" "os" + "sort" "sync" "github.com/OpenListTeam/OpenList/v4/internal/conf" @@ -358,10 +359,72 @@ func (r *ReaderUpdatingProgress) Close() error { type RangeReadReadAtSeeker struct { ss *SeekableStream masterOff int64 - readerMap sync.Map + readers orderedReaders headCache *headCache } +type orderedReaders struct { + mu sync.Mutex + m map[int64]io.Reader + keys []int64 +} + +func (o *orderedReaders) store(off int64, r io.Reader) { + o.mu.Lock() + defer o.mu.Unlock() + if _, ok := o.m[off]; ok { + o.m[off] = r + return + } + if o.m == nil { + o.m = make(map[int64]io.Reader) + } + i := sort.Search(len(o.keys), func(i int) bool { return o.keys[i] >= off }) + o.keys = append(o.keys, 0) + copy(o.keys[i+1:], o.keys[i:]) + o.keys[i] = off + o.m[off] = r +} + +func (o *orderedReaders) takeExact(off int64) (io.Reader, bool) { + o.mu.Lock() + defer o.mu.Unlock() + r, ok := o.m[off] + if ok { + delete(o.m, off) + o.removeKey(off) + } + return r, ok +} + +func (o *orderedReaders) takeBest(off int64) (io.Reader, int64, bool) { + o.mu.Lock() + defer o.mu.Unlock() + if r, ok := o.m[off]; ok { + delete(o.m, off) + o.removeKey(off) + return r, off, true + } + i := sort.Search(len(o.keys), func(i int) bool { return o.keys[i] >= off }) + if i == 0 { + return nil, 0, false + } + k := o.keys[i-1] + if off-k > 4*utils.MB { + return nil, 0, false + } + r := o.m[k] + delete(o.m, k) + o.removeKey(k) + return r, k, true +} + +func (o *orderedReaders) removeKey(k int64) { + i := sort.Search(len(o.keys), func(i int) bool { return o.keys[i] >= k }) + copy(o.keys[i:], o.keys[i+1:]) + o.keys = o.keys[:len(o.keys)-1] +} + type headCache struct { reader io.Reader bufs [][]byte @@ -396,7 +459,7 @@ func (r *headCache) Close() error { func (r *RangeReadReadAtSeeker) InitHeadCache() { if r.masterOff == 0 { - value, _ := r.readerMap.LoadAndDelete(int64(0)) + value, _ := r.readers.takeExact(0) r.headCache = &headCache{reader: value.(io.Reader)} r.ss.Closers.Add(r.headCache) } @@ -422,9 +485,9 @@ func NewReadAtSeeker(ss *SeekableStream, offset int64, forceRange ...bool) (mode if err != nil { return nil, err } - r.readerMap.Store(int64(offset), reader) + r.readers.store(offset, reader) } else { - r.readerMap.Store(int64(offset), ss) + r.readers.store(0, ss) } return r, nil } @@ -442,41 +505,15 @@ func NewMultiReaderAt(ss []*SeekableStream) (readerutil.SizeReaderAt, error) { } func (r *RangeReadReadAtSeeker) getReaderAtOffset(off int64) (io.Reader, error) { - for { - var cur int64 = -1 - r.readerMap.Range(func(key, value any) bool { - k := key.(int64) - if off == k { - cur = k - return false - } - if off > k && off-k <= 4*utils.MB && k > cur { - cur = k - } - return true - }) - if cur < 0 { - break - } - v, ok := r.readerMap.LoadAndDelete(int64(cur)) - if !ok { - continue - } - rr := v.(io.Reader) - if off == int64(cur) { - // logrus.Debugf("getReaderAtOffset match_%d", off) + if rr, cur, ok := r.readers.takeBest(off); ok { + if cur == off { return rr, nil } n, _ := utils.CopyWithBufferN(io.Discard, rr, off-cur) - cur += n - if cur == off { - // logrus.Debugf("getReaderAtOffset old_%d", off) + if cur+n == off { return rr, nil } - break } - - // logrus.Debugf("getReaderAtOffset new_%d", off) reader, err := r.ss.RangeRead(http_range.Range{Start: off, Length: -1}) if err != nil { return nil, err @@ -501,7 +538,7 @@ func (r *RangeReadReadAtSeeker) ReadAt(p []byte, off int64) (n int, err error) { off += int64(n) switch err { case nil: - r.readerMap.Store(int64(off), rr) + r.readers.store(off, rr) case io.ErrUnexpectedEOF: err = io.EOF } @@ -519,7 +556,7 @@ func (r *RangeReadReadAtSeeker) Seek(offset int64, whence int) (int64, error) { default: return 0, errors.New("Seek: invalid whence") } - if offset < 0 || offset > r.ss.GetSize() { + if offset < 0 { return 0, errors.New("Seek: invalid offset") } r.masterOff = offset diff --git a/internal/task/base.go b/internal/task/base.go index f969e6c2..6e778dd9 100644 --- a/internal/task/base.go +++ b/internal/task/base.go @@ -12,8 +12,8 @@ import ( type TaskExtension struct { tache.Base Creator *model.User - startTime *time.Time - endTime *time.Time + StartTime *time.Time `json:"start_time,omitempty"` + EndTime *time.Time `json:"end_time,omitempty"` TotalBytes int64 ApiUrl string } @@ -38,23 +38,23 @@ func (t *TaskExtension) GetCreator() *model.User { } func (t *TaskExtension) SetStartTime(startTime time.Time) { - t.startTime = &startTime + t.StartTime = &startTime } func (t *TaskExtension) GetStartTime() *time.Time { - return t.startTime + return t.StartTime } func (t *TaskExtension) SetEndTime(endTime time.Time) { - t.endTime = &endTime + t.EndTime = &endTime } func (t *TaskExtension) GetEndTime() *time.Time { - return t.endTime + return t.EndTime } func (t *TaskExtension) ClearEndTime() { - t.endTime = nil + t.EndTime = nil } func (t *TaskExtension) SetTotalBytes(totalBytes int64) { diff --git a/internal/task/base_test.go b/internal/task/base_test.go new file mode 100644 index 00000000..51ddc675 --- /dev/null +++ b/internal/task/base_test.go @@ -0,0 +1,20 @@ +package task_test + +import ( + "context" + "testing" + + "github.com/OpenListTeam/OpenList/v4/internal/conf" + "github.com/OpenListTeam/OpenList/v4/internal/task" +) + +func TestTaskExtensionRestoresAPIURL(t *testing.T) { + const want = "https://openlist.example" + extension := task.TaskExtension{ApiUrl: want} + + extension.SetCtx(context.Background()) + + if got := conf.GetApiUrl(extension.Ctx()); got != want { + t.Fatalf("restored origin = %q, want %q", got, want) + } +} diff --git a/pkg/buffer/pipe.go b/pkg/buffer/pipe.go index 194fd58c..05e13e24 100644 --- a/pkg/buffer/pipe.go +++ b/pkg/buffer/pipe.go @@ -13,6 +13,7 @@ type PipeBuffer struct { offR int offW int rw sync.Mutex + ioWg sync.WaitGroup block Block readSignal chan struct{} @@ -73,6 +74,8 @@ func (br *PipeBuffer) Read(p []byte) (int, error) { off := br.offR block := br.block + br.ioWg.Add(1) + defer br.ioWg.Done() br.rw.Unlock() n, err := block.ReadAt(p[:min(len(p), canRead)], int64(off)) @@ -109,6 +112,8 @@ func (br *PipeBuffer) Write(p []byte) (int, error) { off := br.offW block := br.block + br.ioWg.Add(1) + defer br.ioWg.Done() br.rw.Unlock() n, err := block.WriteAt(p[:min(canWrite, len(p))], int64(off)) @@ -131,6 +136,7 @@ func (br *PipeBuffer) Write(p []byte) (int, error) { } func (br *PipeBuffer) Reset(limit int) error { + br.ioWg.Wait() br.rw.Lock() defer br.rw.Unlock() if br.block == nil { @@ -147,11 +153,12 @@ func (br *PipeBuffer) Reset(limit int) error { func (br *PipeBuffer) Close() error { br.rw.Lock() - defer br.rw.Unlock() if br.block != nil { br.block = nil br.readPending = false close(br.readSignal) } + br.rw.Unlock() + br.ioWg.Wait() return nil } diff --git a/pkg/buffer/pipe_test.go b/pkg/buffer/pipe_test.go new file mode 100644 index 00000000..2c44d8b4 --- /dev/null +++ b/pkg/buffer/pipe_test.go @@ -0,0 +1,131 @@ +package buffer + +import ( + "context" + "errors" + "io" + "testing" + "time" +) + +type blockingBlock struct { + data []byte + blockOn string + started chan struct{} + release chan struct{} +} + +func newBlockingBlock(blockOn string) *blockingBlock { + return &blockingBlock{ + data: make([]byte, 1), + blockOn: blockOn, + started: make(chan struct{}), + release: make(chan struct{}), + } +} + +func (b *blockingBlock) Size() int64 { + return int64(len(b.data)) +} + +func (b *blockingBlock) ReadAt(p []byte, off int64) (int, error) { + if b.blockOn == "read" { + close(b.started) + <-b.release + } + n := copy(p, b.data[off:]) + if n < len(p) { + return n, io.EOF + } + return n, nil +} + +func (b *blockingBlock) WriteAt(p []byte, off int64) (int, error) { + if b.blockOn == "write" { + close(b.started) + <-b.release + } + n := copy(b.data[off:], p) + if n < len(p) { + return n, io.ErrShortWrite + } + return n, nil +} + +func TestPipeBufferCloseWaitsForActiveIO(t *testing.T) { + for _, operation := range []string{"read", "write"} { + t.Run(operation, func(t *testing.T) { + block := newBlockingBlock(operation) + buf := NewPipeBuffer(context.Background(), block) + if operation == "read" { + if _, err := buf.Write([]byte{1}); err != nil { + t.Fatalf("prepare read: %v", err) + } + } + + ioDone := make(chan error, 1) + go func() { + var err error + if operation == "read" { + _, err = buf.Read(make([]byte, 1)) + } else { + _, err = buf.Write([]byte{1}) + } + ioDone <- err + }() + + select { + case <-block.started: + case <-time.After(time.Second): + t.Fatal("I/O did not start") + } + + closeDone := make(chan error, 1) + go func() { + closeDone <- buf.Close() + }() + + deadline := time.Now().Add(time.Second) + for { + buf.rw.Lock() + closed := buf.block == nil + buf.rw.Unlock() + if closed { + break + } + if time.Now().After(deadline) { + t.Fatal("buffer did not enter the closed state") + } + time.Sleep(time.Millisecond) + } + + select { + case err := <-closeDone: + t.Fatalf("Close returned before active %s completed: %v", operation, err) + default: + } + + close(block.release) + select { + case err := <-ioDone: + if err != nil { + t.Fatalf("active %s failed: %v", operation, err) + } + case <-time.After(time.Second): + t.Fatalf("active %s did not complete", operation) + } + select { + case err := <-closeDone: + if err != nil { + t.Fatalf("Close failed: %v", err) + } + case <-time.After(time.Second): + t.Fatal("Close did not wait for active I/O") + } + + if _, err := buf.Write([]byte{1}); !errors.Is(err, io.ErrClosedPipe) { + t.Fatalf("write after Close error = %v, want %v", err, io.ErrClosedPipe) + } + }) + } +} diff --git a/pkg/buffer/utils.go b/pkg/buffer/utils.go index 4783f4b7..cf494ce1 100644 --- a/pkg/buffer/utils.go +++ b/pkg/buffer/utils.go @@ -79,7 +79,7 @@ func (r *DynamicReadAtSeeker) Seek(offset int64, whence int) (int64, error) { return 0, errors.New("Seek: invalid whence") } - if offset < 0 || offset > r.block.Size() { + if offset < 0 { return 0, errors.New("Seek: invalid offset") } r.offset = offset diff --git a/renovate.json b/renovate.json index 39a2b6e9..d73a6619 100644 --- a/renovate.json +++ b/renovate.json @@ -1,6 +1,13 @@ { "$schema": "https://docs.renovatebot.com/renovate-schema.json", "extends": [ - "config:base" + "config:best-practices" + ], + "packageRules": [ + { + "matchManagers": ["dockerfile", "docker-compose"], + "matchPackageNames": ["alpine", "ghcr.io/openlistteam/openlist-base-image", "openlistteam/openlist:latest"], + "enabled": false + } ] } diff --git a/server/common/base.go b/server/common/base.go index 8aa669e3..53ef633b 100644 --- a/server/common/base.go +++ b/server/common/base.go @@ -31,6 +31,5 @@ func GetApiUrlFromRequest(r *http.Request) string { } func GetApiUrl(ctx context.Context) string { - api, _ := ctx.Value(conf.ApiUrlKey).(string) - return api + return conf.GetApiUrl(ctx) } diff --git a/server/common/check.go b/server/common/check.go index 2c8d3bc8..72889701 100644 --- a/server/common/check.go +++ b/server/common/check.go @@ -44,7 +44,10 @@ func CanWriteContentBypassUserPerms(meta *model.Meta, path string) bool { if meta == nil || !meta.Write { return false } - return MetaCoversPath(meta.Path, path, meta.WSub) + if utils.PathEqual(meta.Path, path) { + return true + } + return utils.IsSubPath(meta.Path, path) && meta.WSub } func CanAccess(user *model.User, meta *model.Meta, reqPath string, password string) bool { @@ -78,10 +81,21 @@ func CanAccess(user *model.User, meta *model.Meta, reqPath string, password stri } func MetaCoversPath(metaPath, reqPath string, applyToSubFolder bool) bool { - if utils.PathEqual(metaPath, reqPath) { + metaPath = utils.FixAndCleanPath(metaPath) + reqPath = utils.FixAndCleanPath(reqPath) + if strings.EqualFold(metaPath, reqPath) { return true } - return utils.IsSubPath(metaPath, reqPath) && applyToSubFolder + if !applyToSubFolder { + return false + } + for reqPath != "/" { + reqPath = path.Dir(reqPath) + if strings.EqualFold(metaPath, reqPath) { + return true + } + } + return false } // ShouldProxy TODO need optimize diff --git a/server/common/check_test.go b/server/common/check_test.go index 18abca8e..631cf266 100644 --- a/server/common/check_test.go +++ b/server/common/check_test.go @@ -84,6 +84,27 @@ func TestCoversPath(t *testing.T) { applySub: true, want: false, }, + { + name: "case-insensitive exact path match", + metaPath: "/Folder", + reqPath: "/folder", + applySub: false, + want: true, + }, + { + name: "case-insensitive sub path match", + metaPath: "/Folder", + reqPath: "/folder/Subfolder", + applySub: true, + want: true, + }, + { + name: "case-insensitive sibling prefix does not match", + metaPath: "/Folder", + reqPath: "/folder-name", + applySub: true, + want: false, + }, } for _, tt := range tests { @@ -166,6 +187,17 @@ func TestCanWriteContentIgnoringUserPerms(t *testing.T) { want: false, reason: "non-sub path should deny write even with WSub=true", }, + { + name: "case-insensitive match does not broaden write bypass", + meta: &model.Meta{ + Path: "/Folder", + Write: true, + WSub: true, + }, + path: "/folder/subfolder", + want: false, + reason: "case-insensitive matching should only enforce restrictions, not grant write bypass", + }, } for _, tt := range tests { @@ -579,6 +611,23 @@ func TestCanAccessWithReadPermissions(t *testing.T) { want: false, reason: "user not in ReadUsers list should be denied", }, + { + name: "case-insensitive exact path still requires password", + user: &model.User{ + ID: 1, + Role: model.GENERAL, + Permission: 0, + }, + meta: &model.Meta{ + Path: "/Folder", + Password: "secret", + PSub: false, + }, + reqPath: "/folder", + password: "wrong", + want: false, + reason: "changing only path casing must not bypass an exact-path password", + }, } for _, tt := range tests { diff --git a/server/common/etag_test.go b/server/common/etag_test.go new file mode 100644 index 00000000..d4784727 --- /dev/null +++ b/server/common/etag_test.go @@ -0,0 +1,42 @@ +package common + +import ( + "testing" + "time" + + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/pkg/utils" +) + +func TestGetEtagStableAcrossCacheRefresh(t *testing.T) { + uploaded := &model.Object{Size: 4, Modified: time.Unix(1700000000, 123456789)} + before := GetEtag(uploaded, uploaded.Size) + refreshed := &model.Object{Size: uploaded.Size, Modified: time.Unix(1700000000, 0)} + if got := GetEtag(refreshed, refreshed.Size); got != before { + t.Fatalf("refreshed file ETag = %q, want %q", got, before) + } + refreshed.Modified = refreshed.Modified.Add(time.Second) + if got := GetEtag(refreshed, refreshed.Size); got == before { + t.Errorf("changed modification second retained ETag %q", got) + } + refreshed.Modified = uploaded.Modified + if got := GetEtag(refreshed, refreshed.Size+1); got == before { + t.Errorf("changed size retained ETag %q", got) + } +} + +func TestGetEtagContentHash(t *testing.T) { + file := &model.Object{ + Size: 4, + HashInfo: utils.NewHashInfo(utils.SHA256, utils.HashData(utils.SHA256, []byte("data"))), + } + before := GetEtag(file, file.Size) + file.Modified = time.Unix(1700000000, 100) + if got := GetEtag(file, file.Size); got != before { + t.Errorf("unchanged content ETag = %q, want %q", got, before) + } + file.HashInfo = utils.NewHashInfo(utils.SHA256, utils.HashData(utils.SHA256, []byte("next"))) + if got := GetEtag(file, file.Size); got == before { + t.Errorf("changed content retained ETag %q", got) + } +} diff --git a/server/common/proxy.go b/server/common/proxy.go index c7c975d2..cc3ea8bb 100644 --- a/server/common/proxy.go +++ b/server/common/proxy.go @@ -34,9 +34,7 @@ func Proxy(w http.ResponseWriter, r *http.Request, link *model.Link, file model. if link.RangeReader == nil { r = r.WithContext(context.WithValue(r.Context(), conf.RequestHeaderKey, r.Header)) } - return net.ServeHTTP(w, r, file.GetName(), file.ModTime(), size, &model.RangeReadCloser{ - RangeReader: rrf, - }) + return net.ServeHTTP(w, r, file.GetName(), file.ModTime(), size, rrf) } if link.RangeReader != nil { @@ -45,9 +43,7 @@ func Proxy(w http.ResponseWriter, r *http.Request, link *model.Link, file model. if size <= 0 { size = file.GetSize() } - return net.ServeHTTP(w, r, file.GetName(), file.ModTime(), size, &model.RangeReadCloser{ - RangeReader: link.RangeReader, - }) + return net.ServeHTTP(w, r, file.GetName(), file.ModTime(), size, link.RangeReader) } //transparent proxy @@ -59,6 +55,7 @@ func Proxy(w http.ResponseWriter, r *http.Request, link *model.Link, file model. defer res.Body.Close() maps.Copy(w.Header(), res.Header) + w.Header().Set("Content-Disposition", utils.GenerateContentDisposition(file.GetName())) w.WriteHeader(res.StatusCode) if r.Method == http.MethodHead { return nil @@ -96,7 +93,8 @@ func GetEtag(file model.Obj, size int64) string { if len(hash) > 0 { return fmt.Sprintf(`"%s"`, hash) } - // 参考nginx + // Storage backends may not preserve sub-second timestamps across cache + // refreshes, so only use the reproducible second precision here. return fmt.Sprintf(`"%x-%x"`, file.ModTime().Unix(), size) } diff --git a/server/common/proxy_cancel_test.go b/server/common/proxy_cancel_test.go new file mode 100644 index 00000000..04f37401 --- /dev/null +++ b/server/common/proxy_cancel_test.go @@ -0,0 +1,49 @@ +package common + +import ( + "bytes" + "context" + "io" + "net/http" + "net/http/httptest" + "testing" + + "github.com/OpenListTeam/OpenList/v4/internal/conf" + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/internal/stream" + "github.com/OpenListTeam/OpenList/v4/pkg/http_range" +) + +func TestProxyCancelledPartitionedReaderDoesNotPanic(t *testing.T) { + oldConf := conf.Conf + conf.Conf = conf.DefaultConfig("data") + t.Cleanup(func() { conf.Conf = oldConf }) + link := &model.Link{ + Concurrency: 2, + PartSize: 4, + RangeReader: stream.RangeReaderFunc(func(ctx context.Context, requested http_range.Range) (io.ReadCloser, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + return io.NopCloser(bytes.NewReader([]byte("0123456789abcdef")[requested.Start : requested.Start+requested.Length])), nil + }), + } + file := &model.Object{Name: "fixture.bin", Size: 16} + for range 32 { + func() { + defer func() { + if recovered := recover(); recovered != nil { + t.Errorf("Proxy panicked on cancelled partitioned read: %v", recovered) + } + }() + r := httptest.NewRequest(http.MethodGet, "/proxy/fixture.bin", nil) + ctx, cancel := context.WithCancel(r.Context()) + cancel() + w := httptest.NewRecorder() + _ = Proxy(w, r.WithContext(ctx), link, file) + if bytes.Contains(w.Body.Bytes(), []byte("0123456789abcdef")) { + t.Errorf("cancelled response contained file contents: %q", w.Body.String()) + } + }() + } +} diff --git a/server/common/proxy_test.go b/server/common/proxy_test.go new file mode 100644 index 00000000..314763b2 --- /dev/null +++ b/server/common/proxy_test.go @@ -0,0 +1,54 @@ +package common + +import ( + "io" + "net/http" + "net/http/httptest" + "testing" + + "github.com/OpenListTeam/OpenList/v4/internal/conf" + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/pkg/utils" +) + +func TestProxyOverridesUpstreamContentDisposition(t *testing.T) { + previousConfig := conf.Conf + conf.Conf = conf.DefaultConfig("data") + t.Cleanup(func() { + conf.Conf = previousConfig + }) + + const content = "archive content" + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Disposition", `attachment; filename="download"`) + w.Header().Set("Content-Type", "application/x-rar-compressed") + w.WriteHeader(http.StatusOK) + _, _ = io.WriteString(w, content) + })) + t.Cleanup(upstream.Close) + + file := &model.Object{ + Name: "测试文件.rar", + Size: int64(len(content)), + } + link := &model.Link{URL: upstream.URL} + recorder := httptest.NewRecorder() + request := httptest.NewRequest(http.MethodGet, "/sd/example", nil) + + err := Proxy(recorder, request, link, file) + if err != nil { + t.Fatalf("Proxy() error = %v", err) + } + if got, want := recorder.Code, http.StatusOK; got != want { + t.Fatalf("status code = %d, want %d", got, want) + } + if got, want := recorder.Header().Get("Content-Disposition"), utils.GenerateContentDisposition(file.GetName()); got != want { + t.Errorf("Content-Disposition = %q, want %q", got, want) + } + if got, want := recorder.Header().Get("Content-Type"), "application/x-rar-compressed"; got != want { + t.Errorf("Content-Type = %q, want %q", got, want) + } + if got, want := recorder.Body.String(), content; got != want { + t.Errorf("body = %q, want %q", got, want) + } +} diff --git a/server/ftp/fsup.go b/server/ftp/fsup.go index 7a96a4f6..1894fc0e 100644 --- a/server/ftp/fsup.go +++ b/server/ftp/fsup.go @@ -94,10 +94,11 @@ func (f *FileUploadProxy) Close() error { return err } arr := make([]byte, 512) - if _, err := f.buffer.Read(arr); err != nil { + n, err := f.buffer.Read(arr) + if err != nil && err != io.EOF { return err } - contentType := http.DetectContentType(arr) + contentType := http.DetectContentType(arr[:n]) if _, err := f.buffer.Seek(0, io.SeekStart); err != nil { return err } diff --git a/server/handles/auth.go b/server/handles/auth.go index 78006909..2013a5c2 100644 --- a/server/handles/auth.go +++ b/server/handles/auth.go @@ -3,14 +3,17 @@ package handles import ( "bytes" "encoding/base64" + "errors" "image/png" "github.com/OpenListTeam/OpenList/v4/internal/conf" + "github.com/OpenListTeam/OpenList/v4/internal/db" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/OpenListTeam/OpenList/v4/server/common" "github.com/gin-gonic/gin" "github.com/pquerna/otp/totp" + "gorm.io/gorm" ) type LoginReq struct { @@ -111,11 +114,29 @@ func UpdateCurrent(c *gin.Context) { common.ErrorStrResp(c, model.GuestCannotUpdateProfile, 403) return } + ssoID := req.SsoID + if req.SsoID != "" && req.SsoID != user.SsoID { + claims, err := parseSSOBindingToken(c, req.SsoID, ssoBindingProofPurpose) + if err != nil { + common.ErrorStrResp(c, "invalid or expired SSO binding proof", 400) + return + } + boundUser, err := db.GetUserBySSOID(claims.SsoID) + if err == nil && boundUser.ID != user.ID { + common.ErrorStrResp(c, "SSO account is already bound to another user", 409) + return + } + if err != nil && !errors.Is(err, gorm.ErrRecordNotFound) { + common.ErrorResp(c, err, 500) + return + } + ssoID = claims.SsoID + } user.Username = req.Username if req.Password != "" { user.SetPassword(req.Password) } - user.SsoID = req.SsoID + user.SsoID = ssoID if err := op.UpdateUser(user); err != nil { common.ErrorResp(c, err, 500) } else { diff --git a/server/handles/direct_upload.go b/server/handles/direct_upload.go index d77c3044..221b1afa 100644 --- a/server/handles/direct_upload.go +++ b/server/handles/direct_upload.go @@ -1,6 +1,8 @@ package handles import ( + "context" + "mime" "net/url" stdpath "path" @@ -8,15 +10,18 @@ import ( "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/fs" "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/OpenListTeam/OpenList/v4/server/common" "github.com/gin-gonic/gin" + "github.com/pkg/errors" ) type FsGetDirectUploadInfoReq struct { - Path string `json:"path" form:"path"` - FileName string `json:"file_name" form:"file_name"` - FileSize int64 `json:"file_size" form:"file_size"` - Tool string `json:"tool" form:"tool"` + Path string `json:"path" form:"path"` + FileName string `json:"file_name" form:"file_name"` + FileSize int64 `json:"file_size" form:"file_size"` + ContentType string `json:"content_type" form:"content_type"` + Tool string `json:"tool" form:"tool"` } // FsGetDirectUploadInfo returns the direct upload info if supported by the driver @@ -44,8 +49,40 @@ func FsGetDirectUploadInfo(c *gin.Context) { common.ErrorResp(c, err, 403) return } - overwrite := c.GetHeader("Overwrite") != "false" + // Resolve the destination once so permission checks and the issued upload + // capability cannot target different paths. dstPath := stdpath.Join(path, req.FileName) + path = stdpath.Dir(dstPath) + req.FileName = stdpath.Base(dstPath) + parentMeta, err := op.GetNearestMeta(path) + if err != nil && !errors.Is(errors.Cause(err), errs.MetaNotFound) { + common.ErrorResp(c, err, 500, true) + return + } + if !user.CanWriteContent() && !common.CanWriteContentBypassUserPerms(parentMeta, path) { + common.ErrorResp(c, errs.PermissionDenied, 403) + return + } + if !common.CanWrite(user, parentMeta, path) { + common.ErrorResp(c, errs.PermissionDenied, 403) + return + } + // A single-segment file name can still name a nested mount point. + parentMountPath, err := op.GetStorageVirtualMountPath(path) + if err != nil { + common.ErrorResp(c, err, 500) + return + } + targetMountPath, err := op.GetStorageVirtualMountPath(dstPath) + if err != nil { + common.ErrorResp(c, err, 500) + return + } + if parentMountPath != targetMountPath { + common.ErrorResp(c, errs.PermissionDenied, 403) + return + } + overwrite := c.GetHeader("Overwrite") != "false" if !overwrite { res, err := fs.Get(c.Request.Context(), dstPath, &fs.GetArgs{NoLog: true}) if err != nil && !errs.IsObjectNotFound(err) { @@ -57,7 +94,14 @@ func FsGetDirectUploadInfo(c *gin.Context) { return } } - directUploadInfo, err := fs.GetDirectUploadInfo(c, req.Tool, path, req.FileName, req.FileSize, overwrite) + if req.ContentType != "" { + if _, _, err := mime.ParseMediaType(req.ContentType); err != nil { + common.ErrorResp(c, err, 400) + return + } + } + ctx := context.WithValue(c, conf.DirectUploadContentTypeKey, req.ContentType) + directUploadInfo, err := fs.GetDirectUploadInfo(ctx, req.Tool, path, req.FileName, req.FileSize, overwrite) if err != nil { if !overwrite && errs.IsObjectAlreadyExists(err) { common.ErrorStrResp(c, "file exists", 403) diff --git a/server/handles/fsbatch.go b/server/handles/fsbatch.go index e672165c..e0f98284 100644 --- a/server/handles/fsbatch.go +++ b/server/handles/fsbatch.go @@ -188,6 +188,11 @@ func FsBatchRename(c *gin.Context) { if renameObject.SrcName == "" || renameObject.NewName == "" { continue } + err = checkRelativePath(renameObject.SrcName) + if err != nil { + common.ErrorResp(c, err, 403) + return + } err = checkRelativePath(renameObject.NewName) if err != nil { common.ErrorResp(c, err, 403) diff --git a/server/handles/fsmanage.go b/server/handles/fsmanage.go index d97d36d2..11e904be 100644 --- a/server/handles/fsmanage.go +++ b/server/handles/fsmanage.go @@ -113,6 +113,10 @@ func FsMove(c *gin.Context) { srcDir += "/" } for i, name := range req.Names { + if err := checkRelativePath(name); err != nil { + common.ErrorResp(c, err, 403) + return + } // ensure req.Names is not a relative path srcPath := stdpath.Join(srcDir, name) if !strings.HasPrefix(srcPath+"/", srcDir) { @@ -216,6 +220,10 @@ func FsCopy(c *gin.Context) { srcDir += "/" } for i, name := range req.Names { + if err := checkRelativePath(name); err != nil { + common.ErrorResp(c, err, 403) + return + } // ensure req.Names is not a relative path srcPath := stdpath.Join(srcDir, name) if !strings.HasPrefix(srcPath+"/", srcDir) { @@ -373,6 +381,10 @@ func FsRemove(c *gin.Context) { reqPath += "/" } for i, name := range req.Names { + if err := checkRelativePath(name); err != nil { + common.ErrorResp(c, err, 403) + return + } fullPath := stdpath.Join(reqPath, name) if !strings.HasPrefix(fullPath+"/", reqPath) { req.Names[i] = "" diff --git a/server/handles/fsmanage_test.go b/server/handles/fsmanage_test.go new file mode 100644 index 00000000..37096e97 --- /dev/null +++ b/server/handles/fsmanage_test.go @@ -0,0 +1,126 @@ +package handles + +import ( + "bytes" + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + + _ "github.com/OpenListTeam/OpenList/v4/drivers/local" + "github.com/OpenListTeam/OpenList/v4/internal/conf" + "github.com/OpenListTeam/OpenList/v4/internal/db" + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/internal/op" + "github.com/OpenListTeam/OpenList/v4/pkg/utils" + "github.com/gin-gonic/gin" + "github.com/glebarez/sqlite" + "gorm.io/gorm" +) + +func setupBackslashTraversalTest(t *testing.T, root string, permission int32) *model.User { + t.Helper() + database, err := gorm.Open(sqlite.Open("file:"+t.Name()+"?mode=memory&cache=shared"), &gorm.Config{}) + if err != nil { + t.Fatal(err) + } + conf.Conf = conf.DefaultConfig(t.TempDir()) + db.Init(database) + addition, err := utils.Json.MarshalToString(map[string]string{"root_folder_path": root}) + if err != nil { + t.Fatal(err) + } + if _, err = op.CreateStorage(context.Background(), model.Storage{ + Driver: "Local", MountPath: "/", Addition: addition, + }); err != nil { + t.Fatal(err) + } + return &model.User{ + Username: "restricted-user", BasePath: "/team/a", Role: model.GENERAL, + Permission: permission, + } +} + +func prepareBackslashTraversalFs(t *testing.T) (root string, secretPath string) { + t.Helper() + root = t.TempDir() + if err := os.MkdirAll(filepath.Join(root, "team", "a", "writable"), 0o700); err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(filepath.Join(root, "team", "ab"), 0o700); err != nil { + t.Fatal(err) + } + secretPath = filepath.Join(root, "team", "ab", "secret.txt") + if err := os.WriteFile(secretPath, []byte("synthetic-secret"), 0o600); err != nil { + t.Fatal(err) + } + return root, secretPath +} + +func invokeHandler(t *testing.T, user *model.User, payload any, handler gin.HandlerFunc) *httptest.ResponseRecorder { + t.Helper() + body, err := json.Marshal(payload) + if err != nil { + t.Fatal(err) + } + recorder := httptest.NewRecorder() + ctx, _ := gin.CreateTestContext(recorder) + req := httptest.NewRequest(http.MethodPost, "/api/fs/remove", bytes.NewReader(body)) + req.Header.Set("Content-Type", "application/json") + req = req.WithContext(context.WithValue(req.Context(), conf.UserKey, user)) + ctx.Request = req + handler(ctx) + return recorder +} + +func TestFsRemoveRejectsBackslashTraversal(t *testing.T) { + gin.SetMode(gin.TestMode) + root, secretPath := prepareBackslashTraversalFs(t) + user := setupBackslashTraversalTest(t, root, 1<<3|1<<7) + + for _, name := range []string{"../../ab/secret.txt", `..\..\ab\secret.txt`} { + recorder := invokeHandler(t, user, map[string]any{"dir": "/writable", "names": []string{name}}, FsRemove) + if recorder.Code != http.StatusOK || !strings.Contains(recorder.Body.String(), `"code":403`) { + t.Fatalf("payload %q: got status=%d body=%s, want 403", name, recorder.Code, recorder.Body.String()) + } + if _, err := os.Stat(secretPath); err != nil { + t.Fatalf("payload %q deleted sibling file: %v", name, err) + } + } +} + +func TestFsMoveRejectsBackslashTraversal(t *testing.T) { + gin.SetMode(gin.TestMode) + root, secretPath := prepareBackslashTraversalFs(t) + user := setupBackslashTraversalTest(t, root, 1<<3|1<<5) + + recorder := invokeHandler(t, user, map[string]any{ + "src_dir": "/writable", "dst_dir": "/", "names": []string{`..\..\ab\secret.txt`}, + }, FsMove) + if recorder.Code != http.StatusOK || !strings.Contains(recorder.Body.String(), `"code":403`) { + t.Fatalf("got status=%d body=%s, want 403", recorder.Code, recorder.Body.String()) + } + if _, err := os.Stat(secretPath); err != nil { + t.Fatalf("backslash traversal moved sibling file: %v", err) + } +} + +func TestFsCopyRejectsBackslashTraversal(t *testing.T) { + gin.SetMode(gin.TestMode) + root, secretPath := prepareBackslashTraversalFs(t) + user := setupBackslashTraversalTest(t, root, 1<<3|1<<6) + + recorder := invokeHandler(t, user, map[string]any{ + "src_dir": "/writable", "dst_dir": "/", "names": []string{`..\..\ab\secret.txt`}, + }, FsCopy) + if recorder.Code != http.StatusOK || !strings.Contains(recorder.Body.String(), `"code":403`) { + t.Fatalf("got status=%d body=%s, want 403", recorder.Code, recorder.Body.String()) + } + if _, err := os.Stat(secretPath); err != nil { + t.Fatalf("backslash traversal affected sibling file: %v", err) + } +} diff --git a/server/handles/fsread.go b/server/handles/fsread.go index 856f46e5..841cb5af 100644 --- a/server/handles/fsread.go +++ b/server/handles/fsread.go @@ -347,11 +347,13 @@ func FsGet(c *gin.Context, req *FsGetReq, user *model.User) { } } } - var related []model.Obj parentPath := stdpath.Dir(reqPath) - sameLevelFiles, err := fs.List(c.Request.Context(), parentPath, &fs.ListArgs{}) - if err == nil { - related = filterRelated(sameLevelFiles, obj) + var related []model.Obj + if !obj.IsDir() && utils.GetFileType(obj.GetName()) == conf.VIDEO { + sameLevelFiles, err := fs.List(c.Request.Context(), parentPath, &fs.ListArgs{}) + if err == nil { + related = filterRelated(sameLevelFiles, obj) + } } parentMeta, _ := op.GetNearestMeta(parentPath) thumb, _ := model.GetThumb(obj) @@ -366,7 +368,7 @@ func FsGet(c *gin.Context, req *FsGetReq, user *model.User) { HashInfoStr: obj.GetHash().String(), HashInfo: obj.GetHash().Export(), Sign: common.Sign(obj, parentPath, isEncrypt(meta, reqPath)), - Type: utils.GetFileType(obj.GetName()), + Type: utils.GetObjType(obj.GetName(), obj.IsDir()), Thumb: thumb, MountDetails: mountDetails, }, diff --git a/server/handles/multipart.go b/server/handles/multipart.go new file mode 100644 index 00000000..991a102b --- /dev/null +++ b/server/handles/multipart.go @@ -0,0 +1,232 @@ +package handles + +import ( + "errors" + "io" + "net/url" + stdpath "path" + "strconv" + + "github.com/OpenListTeam/OpenList/v4/internal/conf" + "github.com/OpenListTeam/OpenList/v4/internal/errs" + "github.com/OpenListTeam/OpenList/v4/internal/fs" + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/internal/multipart" + "github.com/OpenListTeam/OpenList/v4/internal/op" + "github.com/OpenListTeam/OpenList/v4/internal/setting" + "github.com/OpenListTeam/OpenList/v4/pkg/utils" + "github.com/OpenListTeam/OpenList/v4/server/common" + "github.com/gin-gonic/gin" +) + +const multipartMinChunkSize = int64(1) << 20 // 1MB + +// multipartChunkSize resolves the effective chunk size. The admin setting is +// the ceiling: a client may suggest a smaller chunk via X-Chunk-Size but never +// a larger one — the server buffers a window of several chunks per session, so +// an unbounded client suggestion would translate directly into server-side +// disk usage. +func multipartChunkSize(requested int64) int64 { + ceiling := int64(setting.GetInt(conf.MultipartChunkSize, 10)) << 20 + if ceiling < multipartMinChunkSize { + ceiling = multipartMinChunkSize + } + size := ceiling + if requested > 0 && requested < ceiling { + size = max(requested, multipartMinChunkSize) + } + return size +} + +type MultipartInitResp struct { + multipart.SessionSnapshot + Resumed bool `json:"resumed"` +} + +// MultipartInit creates (or resumes) a multipart upload session and starts its +// upload pipeline. Headers mirror FsStream (fsup.go). +func MultipartInit(c *gin.Context) { + if !setting.GetBool(conf.MultipartEnabled) { + common.ErrorStrResp(c, "multipart upload is disabled", 403) + return + } + path := c.GetHeader("File-Path") + path, err := url.PathUnescape(path) + if err != nil { + common.ErrorResp(c, err, 400) + return + } + user := c.Request.Context().Value(conf.UserKey).(*model.User) + path, err = user.JoinPath(path) + if err != nil { + common.ErrorResp(c, err, 403) + return + } + size, err := strconv.ParseInt(c.GetHeader("X-File-Size"), 10, 64) + if err != nil { + common.ErrorStrResp(c, "multipart upload requires a valid X-File-Size header", 400) + return + } + if size <= 0 { + common.ErrorStrResp(c, "multipart upload requires a positive X-File-Size; upload empty files via /fs/put", 400) + return + } + var requestedChunkSize int64 + if v := c.GetHeader("X-Chunk-Size"); v != "" { + requestedChunkSize, err = strconv.ParseInt(v, 10, 64) + if err != nil { + common.ErrorResp(c, err, 400) + return + } + } + overwrite := c.GetHeader("Overwrite") != "false" + if !overwrite { + if res, _ := fs.Get(c.Request.Context(), path, &fs.GetArgs{NoLog: true}); res != nil { + common.ErrorStrResp(c, "file exists", 403) + return + } + } + dir, name := stdpath.Split(path) + if shouldIgnoreSystemFile(name) { + common.ErrorStrResp(c, errs.IgnoredSystemFile.Error(), 403) + return + } + // fail fast on unusable destinations instead of letting the pipeline discover it + storage, _, err := op.GetStorageAndActualPath(dir) + if err != nil { + common.ErrorResp(c, err, 500) + return + } + if storage.Config().NoUpload { + common.ErrorResp(c, errs.UploadNotSupported, 405) + return + } + h := make(map[*utils.HashType]string) + if md5 := c.GetHeader("X-File-Md5"); md5 != "" { + h[utils.MD5] = md5 + } + if sha1 := c.GetHeader("X-File-Sha1"); sha1 != "" { + h[utils.SHA1] = sha1 + } + if sha256 := c.GetHeader("X-File-Sha256"); sha256 != "" { + h[utils.SHA256] = sha256 + } + mimetype := c.GetHeader("Content-Type") + if len(mimetype) == 0 { + mimetype = utils.GetMimeType(name) + } + snap, resumed, err := multipart.DefaultManager.Init(multipart.InitReq{ + User: user, + Path: path, + Size: size, + ChunkSize: multipartChunkSize(requestedChunkSize), + Mimetype: mimetype, + Modified: getLastModified(c), + Hashes: h, + }) + if err != nil { + common.ErrorResp(c, err, 500) + return + } + common.SuccessResp(c, MultipartInitResp{SessionSnapshot: snap, Resumed: resumed}) +} + +// MultipartChunk ingests one chunk. Chunks are idempotent and may be sent +// concurrently and out of order within the receiving window. +// code 429 = window full (flow control, retry after a short delay), +// code 409 = the same chunk is already in flight on another connection. +func MultipartChunk(c *gin.Context) { + user := c.Request.Context().Value(conf.UserKey).(*model.User) + id := c.GetHeader("X-Upload-Id") + idx, err := strconv.Atoi(c.GetHeader("X-Chunk-Index")) + if err != nil { + common.ErrorStrResp(c, "invalid X-Chunk-Index header", 400) + return + } + snap, err := multipart.DefaultManager.Chunk(user, id, idx, c.Request.Body) + // Answer only after the request body is consumed — on EVERY path. Flow + // control (429), absorbed chunks and validation errors would otherwise + // respond while the browser is still streaming the body, which it reports + // as a network error and which poisons its connection pool. A rejected + // chunk gets resent anyway, so draining costs no extra round trip. The + // drain is bounded so a malformed request cannot pin the handler. + limit := multipartChunkSize(0) + 64*1024 + if snap.ChunkSize > 0 { + limit = snap.ChunkSize + 64*1024 + } + _, _ = utils.CopyWithBuffer(io.Discard, io.LimitReader(c.Request.Body, limit)) + if err != nil { + common.ErrorWithDataResp(c, err, multipartErrCode(err), snap) + return + } + common.SuccessResp(c, snap) +} + +// MultipartComplete waits for the pipeline outcome and reports it, mirroring +// how /fs/put only responds once the driver upload finished. +func MultipartComplete(c *gin.Context) { + user := c.Request.Context().Value(conf.UserKey).(*model.User) + id := c.GetHeader("X-Upload-Id") + snap, err := multipart.DefaultManager.Complete(c.Request.Context(), user, id) + if err != nil { + common.ErrorWithDataResp(c, err, multipartErrCode(err), snap) + return + } + common.SuccessResp(c, snap) +} + +// MultipartStatus looks a session up by upload_id, or by path+size so an +// interrupted client can discover a resumable session. +func MultipartStatus(c *gin.Context) { + user := c.Request.Context().Value(conf.UserKey).(*model.User) + if id := c.Query("upload_id"); id != "" { + snap, err := multipart.DefaultManager.Status(user, id) + if err != nil { + common.ErrorResp(c, err, multipartErrCode(err)) + return + } + common.SuccessResp(c, snap) + return + } + path, err := user.JoinPath(c.Query("path")) + if err != nil { + common.ErrorResp(c, err, 403) + return + } + size, err := strconv.ParseInt(c.Query("size"), 10, 64) + if err != nil { + common.ErrorStrResp(c, "status lookup requires upload_id, or path and size", 400) + return + } + snap, err := multipart.DefaultManager.Find(user, path, size) + if err != nil { + common.ErrorResp(c, err, multipartErrCode(err)) + return + } + common.SuccessResp(c, snap) +} + +// MultipartAbort cancels the pipeline and discards the session. +func MultipartAbort(c *gin.Context) { + user := c.Request.Context().Value(conf.UserKey).(*model.User) + if err := multipart.DefaultManager.Abort(user, c.GetHeader("X-Upload-Id")); err != nil { + common.ErrorResp(c, err, multipartErrCode(err)) + return + } + common.SuccessResp(c) +} + +func multipartErrCode(err error) int { + switch { + case errors.Is(err, multipart.ErrOutOfWindow): + return 429 + case errors.Is(err, multipart.ErrChunkInFlight): + return 409 + case errors.Is(err, multipart.ErrSessionNotFound): + return 404 + case errors.Is(err, multipart.ErrNotOwner): + return 403 + default: + return 400 + } +} diff --git a/server/handles/offline_download.go b/server/handles/offline_download.go index 32fa64a4..53f30662 100644 --- a/server/handles/offline_download.go +++ b/server/handles/offline_download.go @@ -3,14 +3,6 @@ package handles import ( "strings" - _115 "github.com/OpenListTeam/OpenList/v4/drivers/115" - _115_open "github.com/OpenListTeam/OpenList/v4/drivers/115_open" - _123 "github.com/OpenListTeam/OpenList/v4/drivers/123" - _123_open "github.com/OpenListTeam/OpenList/v4/drivers/123_open" - "github.com/OpenListTeam/OpenList/v4/drivers/pikpak" - "github.com/OpenListTeam/OpenList/v4/drivers/thunder" - "github.com/OpenListTeam/OpenList/v4/drivers/thunder_browser" - "github.com/OpenListTeam/OpenList/v4/drivers/thunderx" "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" @@ -22,6 +14,44 @@ import ( "github.com/pkg/errors" ) +func saveAndInitOfflineDownloadTool(c *gin.Context, name string, items []model.SettingItem) (string, bool) { + if err := op.SaveSettingItems(items); err != nil { + common.ErrorResp(c, err, 500) + return "", false + } + downloadTool, err := tool.Tools.Get(name) + if err != nil { + common.ErrorResp(c, err, 500) + return "", false + } + version, err := downloadTool.Init() + if err != nil { + common.ErrorResp(c, err, 500) + return "", false + } + return version, true +} + +func validateOfflineDownloadStorage(c *gin.Context, tempDir, nativeTool string) bool { + if tempDir == "" { + return true + } + storage, _, err := op.GetStorageAndActualPath(tempDir) + if err != nil { + common.ErrorStrResp(c, "storage does not exists", 400) + return false + } + if storage.Config().CheckStatus && storage.GetStorage().Status != op.WORK { + common.ErrorStrResp(c, "storage not init: "+storage.GetStorage().Status, 400) + return false + } + if tool.NativeToolName(storage) != nativeTool { + common.ErrorStrResp(c, "unsupported storage driver for offline download, only "+nativeTool+" is supported", 400) + return false + } + return true +} + type SetAria2Req struct { Uri string `json:"uri" form:"uri"` Secret string `json:"secret" form:"secret"` @@ -37,18 +67,8 @@ func SetAria2(c *gin.Context) { {Key: conf.Aria2Uri, Value: req.Uri, Type: conf.TypeString, Group: model.OFFLINE_DOWNLOAD, Flag: model.PRIVATE}, {Key: conf.Aria2Secret, Value: req.Secret, Type: conf.TypeString, Group: model.OFFLINE_DOWNLOAD, Flag: model.PRIVATE}, } - if err := op.SaveSettingItems(items); err != nil { - common.ErrorResp(c, err, 500) - return - } - _tool, err := tool.Tools.Get("aria2") - if err != nil { - common.ErrorResp(c, err, 500) - return - } - version, err := _tool.Init() - if err != nil { - common.ErrorResp(c, err, 500) + version, ok := saveAndInitOfflineDownloadTool(c, "aria2", items) + if !ok { return } common.SuccessResp(c, version) @@ -69,17 +89,7 @@ func SetQbittorrent(c *gin.Context) { {Key: conf.QbittorrentUrl, Value: req.Url, Type: conf.TypeString, Group: model.OFFLINE_DOWNLOAD, Flag: model.PRIVATE}, {Key: conf.QbittorrentSeedtime, Value: req.Seedtime, Type: conf.TypeNumber, Group: model.OFFLINE_DOWNLOAD, Flag: model.PRIVATE}, } - if err := op.SaveSettingItems(items); err != nil { - common.ErrorResp(c, err, 500) - return - } - _tool, err := tool.Tools.Get("qBittorrent") - if err != nil { - common.ErrorResp(c, err, 500) - return - } - if _, err := _tool.Init(); err != nil { - common.ErrorResp(c, err, 500) + if _, ok := saveAndInitOfflineDownloadTool(c, "qBittorrent", items); !ok { return } common.SuccessResp(c, "ok") @@ -100,17 +110,7 @@ func SetTransmission(c *gin.Context) { {Key: conf.TransmissionUri, Value: req.Uri, Type: conf.TypeString, Group: model.OFFLINE_DOWNLOAD, Flag: model.PRIVATE}, {Key: conf.TransmissionSeedtime, Value: req.Seedtime, Type: conf.TypeNumber, Group: model.OFFLINE_DOWNLOAD, Flag: model.PRIVATE}, } - if err := op.SaveSettingItems(items); err != nil { - common.ErrorResp(c, err, 500) - return - } - _tool, err := tool.Tools.Get("Transmission") - if err != nil { - common.ErrorResp(c, err, 500) - return - } - if _, err := _tool.Init(); err != nil { - common.ErrorResp(c, err, 500) + if _, ok := saveAndInitOfflineDownloadTool(c, "Transmission", items); !ok { return } common.SuccessResp(c, "ok") @@ -126,35 +126,13 @@ func Set115(c *gin.Context) { common.ErrorResp(c, err, 400) return } - if req.TempDir != "" { - storage, _, err := op.GetStorageAndActualPath(req.TempDir) - if err != nil { - common.ErrorStrResp(c, "storage does not exists", 400) - return - } - if storage.Config().CheckStatus && storage.GetStorage().Status != op.WORK { - common.ErrorStrResp(c, "storage not init: "+storage.GetStorage().Status, 400) - return - } - if _, ok := storage.(*_115.Pan115); !ok { - common.ErrorStrResp(c, "unsupported storage driver for offline download, only 115 Cloud is supported", 400) - return - } + if !validateOfflineDownloadStorage(c, req.TempDir, "115 Cloud") { + return } items := []model.SettingItem{ {Key: conf.Pan115TempDir, Value: req.TempDir, Type: conf.TypeString, Group: model.OFFLINE_DOWNLOAD, Flag: model.PRIVATE}, } - if err := op.SaveSettingItems(items); err != nil { - common.ErrorResp(c, err, 500) - return - } - _tool, err := tool.Tools.Get("115 Cloud") - if err != nil { - common.ErrorResp(c, err, 500) - return - } - if _, err := _tool.Init(); err != nil { - common.ErrorResp(c, err, 500) + if _, ok := saveAndInitOfflineDownloadTool(c, "115 Cloud", items); !ok { return } common.SuccessResp(c, "ok") @@ -170,35 +148,13 @@ func Set115Open(c *gin.Context) { common.ErrorResp(c, err, 400) return } - if req.TempDir != "" { - storage, _, err := op.GetStorageAndActualPath(req.TempDir) - if err != nil { - common.ErrorStrResp(c, "storage does not exists", 400) - return - } - if storage.Config().CheckStatus && storage.GetStorage().Status != op.WORK { - common.ErrorStrResp(c, "storage not init: "+storage.GetStorage().Status, 400) - return - } - if _, ok := storage.(*_115_open.Open115); !ok { - common.ErrorStrResp(c, "unsupported storage driver for offline download, only 115 Open is supported", 400) - return - } + if !validateOfflineDownloadStorage(c, req.TempDir, "115 Open") { + return } items := []model.SettingItem{ {Key: conf.Pan115OpenTempDir, Value: req.TempDir, Type: conf.TypeString, Group: model.OFFLINE_DOWNLOAD, Flag: model.PRIVATE}, } - if err := op.SaveSettingItems(items); err != nil { - common.ErrorResp(c, err, 500) - return - } - _tool, err := tool.Tools.Get("115 Open") - if err != nil { - common.ErrorResp(c, err, 500) - return - } - if _, err := _tool.Init(); err != nil { - common.ErrorResp(c, err, 500) + if _, ok := saveAndInitOfflineDownloadTool(c, "115 Open", items); !ok { return } common.SuccessResp(c, "ok") @@ -214,35 +170,13 @@ func Set123Pan(c *gin.Context) { common.ErrorResp(c, err, 400) return } - if req.TempDir != "" { - storage, _, err := op.GetStorageAndActualPath(req.TempDir) - if err != nil { - common.ErrorStrResp(c, "storage does not exists", 400) - return - } - if storage.Config().CheckStatus && storage.GetStorage().Status != op.WORK { - common.ErrorStrResp(c, "storage not init: "+storage.GetStorage().Status, 400) - return - } - if _, ok := storage.(*_123.Pan123); !ok { - common.ErrorStrResp(c, "unsupported storage driver for offline download, only 123Pan is supported", 400) - return - } + if !validateOfflineDownloadStorage(c, req.TempDir, "123Pan") { + return } items := []model.SettingItem{ {Key: conf.Pan123TempDir, Value: req.TempDir, Type: conf.TypeString, Group: model.OFFLINE_DOWNLOAD, Flag: model.PRIVATE}, } - if err := op.SaveSettingItems(items); err != nil { - common.ErrorResp(c, err, 500) - return - } - _tool, err := tool.Tools.Get("123Pan") - if err != nil { - common.ErrorResp(c, err, 500) - return - } - if _, err := _tool.Init(); err != nil { - common.ErrorResp(c, err, 500) + if _, ok := saveAndInitOfflineDownloadTool(c, "123Pan", items); !ok { return } common.SuccessResp(c, "ok") @@ -259,36 +193,14 @@ func Set123Open(c *gin.Context) { common.ErrorResp(c, err, 400) return } - if req.TempDir != "" { - storage, _, err := op.GetStorageAndActualPath(req.TempDir) - if err != nil { - common.ErrorStrResp(c, "storage does not exists", 400) - return - } - if storage.Config().CheckStatus && storage.GetStorage().Status != op.WORK { - common.ErrorStrResp(c, "storage not init: "+storage.GetStorage().Status, 400) - return - } - if _, ok := storage.(*_123_open.Open123); !ok { - common.ErrorStrResp(c, "unsupported storage driver for offline download, only 123 Open is supported", 400) - return - } + if !validateOfflineDownloadStorage(c, req.TempDir, "123 Open") { + return } items := []model.SettingItem{ {Key: conf.Pan123OpenTempDir, Value: req.TempDir, Type: conf.TypeString, Group: model.OFFLINE_DOWNLOAD, Flag: model.PRIVATE}, {Key: conf.Pan123OpenOfflineDownloadCallbackUrl, Value: req.CallbackUrl, Type: conf.TypeString, Group: model.OFFLINE_DOWNLOAD, Flag: model.PRIVATE}, } - if err := op.SaveSettingItems(items); err != nil { - common.ErrorResp(c, err, 500) - return - } - _tool, err := tool.Tools.Get("123 Open") - if err != nil { - common.ErrorResp(c, err, 500) - return - } - if _, err := _tool.Init(); err != nil { - common.ErrorResp(c, err, 500) + if _, ok := saveAndInitOfflineDownloadTool(c, "123 Open", items); !ok { return } common.SuccessResp(c, "ok") @@ -304,35 +216,13 @@ func SetPikPak(c *gin.Context) { common.ErrorResp(c, err, 400) return } - if req.TempDir != "" { - storage, _, err := op.GetStorageAndActualPath(req.TempDir) - if err != nil { - common.ErrorStrResp(c, "storage does not exists", 400) - return - } - if storage.Config().CheckStatus && storage.GetStorage().Status != op.WORK { - common.ErrorStrResp(c, "storage not init: "+storage.GetStorage().Status, 400) - return - } - if _, ok := storage.(*pikpak.PikPak); !ok { - common.ErrorStrResp(c, "unsupported storage driver for offline download, only PikPak is supported", 400) - return - } + if !validateOfflineDownloadStorage(c, req.TempDir, "PikPak") { + return } items := []model.SettingItem{ {Key: conf.PikPakTempDir, Value: req.TempDir, Type: conf.TypeString, Group: model.OFFLINE_DOWNLOAD, Flag: model.PRIVATE}, } - if err := op.SaveSettingItems(items); err != nil { - common.ErrorResp(c, err, 500) - return - } - _tool, err := tool.Tools.Get("PikPak") - if err != nil { - common.ErrorResp(c, err, 500) - return - } - if _, err := _tool.Init(); err != nil { - common.ErrorResp(c, err, 500) + if _, ok := saveAndInitOfflineDownloadTool(c, "PikPak", items); !ok { return } common.SuccessResp(c, "ok") @@ -348,35 +238,13 @@ func SetThunder(c *gin.Context) { common.ErrorResp(c, err, 400) return } - if req.TempDir != "" { - storage, _, err := op.GetStorageAndActualPath(req.TempDir) - if err != nil { - common.ErrorStrResp(c, "storage does not exists", 400) - return - } - if storage.Config().CheckStatus && storage.GetStorage().Status != op.WORK { - common.ErrorStrResp(c, "storage not init: "+storage.GetStorage().Status, 400) - return - } - if _, ok := storage.(*thunder.Thunder); !ok { - common.ErrorStrResp(c, "unsupported storage driver for offline download, only Thunder is supported", 400) - return - } + if !validateOfflineDownloadStorage(c, req.TempDir, "Thunder") { + return } items := []model.SettingItem{ {Key: conf.ThunderTempDir, Value: req.TempDir, Type: conf.TypeString, Group: model.OFFLINE_DOWNLOAD, Flag: model.PRIVATE}, } - if err := op.SaveSettingItems(items); err != nil { - common.ErrorResp(c, err, 500) - return - } - _tool, err := tool.Tools.Get("Thunder") - if err != nil { - common.ErrorResp(c, err, 500) - return - } - if _, err := _tool.Init(); err != nil { - common.ErrorResp(c, err, 500) + if _, ok := saveAndInitOfflineDownloadTool(c, "Thunder", items); !ok { return } common.SuccessResp(c, "ok") @@ -392,35 +260,13 @@ func SetThunderX(c *gin.Context) { common.ErrorResp(c, err, 400) return } - if req.TempDir != "" { - storage, _, err := op.GetStorageAndActualPath(req.TempDir) - if err != nil { - common.ErrorStrResp(c, "storage does not exists", 400) - return - } - if storage.Config().CheckStatus && storage.GetStorage().Status != op.WORK { - common.ErrorStrResp(c, "storage not init: "+storage.GetStorage().Status, 400) - return - } - if _, ok := storage.(*thunderx.ThunderX); !ok { - common.ErrorStrResp(c, "unsupported storage driver for offline download, only ThunderX is supported", 400) - return - } + if !validateOfflineDownloadStorage(c, req.TempDir, "ThunderX") { + return } items := []model.SettingItem{ {Key: conf.ThunderXTempDir, Value: req.TempDir, Type: conf.TypeString, Group: model.OFFLINE_DOWNLOAD, Flag: model.PRIVATE}, } - if err := op.SaveSettingItems(items); err != nil { - common.ErrorResp(c, err, 500) - return - } - _tool, err := tool.Tools.Get("ThunderX") - if err != nil { - common.ErrorResp(c, err, 500) - return - } - if _, err := _tool.Init(); err != nil { - common.ErrorResp(c, err, 500) + if _, ok := saveAndInitOfflineDownloadTool(c, "ThunderX", items); !ok { return } common.SuccessResp(c, "ok") @@ -436,44 +282,42 @@ func SetThunderBrowser(c *gin.Context) { common.ErrorResp(c, err, 400) return } - if req.TempDir != "" { - storage, _, err := op.GetStorageAndActualPath(req.TempDir) - if err != nil { - common.ErrorStrResp(c, "storage does not exists", 400) - return - } - if storage.Config().CheckStatus && storage.GetStorage().Status != op.WORK { - common.ErrorStrResp(c, "storage not init: "+storage.GetStorage().Status, 400) - return - } - switch storage.(type) { - case *thunder_browser.ThunderBrowser, *thunder_browser.ThunderBrowserExpert: - default: - common.ErrorStrResp(c, "unsupported storage driver for offline download, only ThunderBrowser is supported", 400) - return - } + if !validateOfflineDownloadStorage(c, req.TempDir, "ThunderBrowser") { + return } items := []model.SettingItem{ {Key: conf.ThunderBrowserTempDir, Value: req.TempDir, Type: conf.TypeString, Group: model.OFFLINE_DOWNLOAD, Flag: model.PRIVATE}, } - if err := op.SaveSettingItems(items); err != nil { - common.ErrorResp(c, err, 500) + if _, ok := saveAndInitOfflineDownloadTool(c, "ThunderBrowser", items); !ok { return } - _tool, err := tool.Tools.Get("ThunderBrowser") - if err != nil { - common.ErrorResp(c, err, 500) + common.SuccessResp(c, "ok") +} + +type SetGuangYaPanReq struct { + TempDir string `json:"temp_dir" form:"temp_dir"` +} + +func SetGuangYaPan(c *gin.Context) { + var req SetGuangYaPanReq + if err := c.ShouldBind(&req); err != nil { + common.ErrorResp(c, err, 400) return } - if _, err := _tool.Init(); err != nil { - common.ErrorResp(c, err, 500) + if !validateOfflineDownloadStorage(c, req.TempDir, "GuangYaPan") { + return + } + items := []model.SettingItem{ + {Key: conf.GuangYaPanTempDir, Value: req.TempDir, Type: conf.TypeString, Group: model.OFFLINE_DOWNLOAD, Flag: model.PRIVATE}, + } + if _, ok := saveAndInitOfflineDownloadTool(c, "GuangYaPan", items); !ok { return } common.SuccessResp(c, "ok") } func OfflineDownloadTools(c *gin.Context) { - tools := tool.Tools.Names() + tools := tool.Tools.NamesForPath(c.Query("path")) common.SuccessResp(c, tools) } diff --git a/server/handles/offline_download_test.go b/server/handles/offline_download_test.go new file mode 100644 index 00000000..6abecbee --- /dev/null +++ b/server/handles/offline_download_test.go @@ -0,0 +1,155 @@ +package handles + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "os" + "strings" + "testing" + + _ "github.com/OpenListTeam/OpenList/v4/drivers/local" + "github.com/OpenListTeam/OpenList/v4/internal/conf" + "github.com/OpenListTeam/OpenList/v4/internal/db" + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/internal/offline_download/tool" + "github.com/OpenListTeam/OpenList/v4/internal/op" + "github.com/OpenListTeam/OpenList/v4/server/common" + "github.com/gin-gonic/gin" + "github.com/glebarez/sqlite" + "gorm.io/gorm" +) + +func init() { + dataDir, err := os.MkdirTemp("", "openlist-handles-*") + if err != nil { + panic(err) + } + conf.Conf = conf.DefaultConfig(dataDir) + database, err := gorm.Open(sqlite.Open("file:handles?mode=memory&cache=shared"), &gorm.Config{}) + if err != nil { + panic(err) + } + db.Init(database) +} + +type settingsTool struct { + name string + version string + initCalls int +} + +func (t *settingsTool) Name() string { return t.name } +func (*settingsTool) Items() []model.SettingItem { return nil } +func (t *settingsTool) Init() (string, error) { t.initCalls++; return t.version, nil } +func (*settingsTool) IsReady() bool { return true } +func (*settingsTool) AddURL(*tool.AddUrlArgs) (string, error) { return "", nil } +func (*settingsTool) Remove(*tool.DownloadTask) error { return nil } +func (*settingsTool) Status(*tool.DownloadTask) (*tool.Status, error) { + return &tool.Status{}, nil +} +func (*settingsTool) Run(*tool.DownloadTask) error { return nil } + +func TestOfflineDownloadSettingsPreserveSuccessPayloads(t *testing.T) { + tests := []struct { + name string + toolName string + version string + body string + handler gin.HandlerFunc + wantData string + settingKey string + }{ + { + name: "aria2 returns version", + toolName: "aria2", + version: "v-test", + body: `{"uri":"http://aria2","secret":"secret"}`, + handler: SetAria2, + wantData: "v-test", + settingKey: conf.Aria2Uri, + }, + { + name: "qBittorrent returns ok", + toolName: "qBittorrent", + version: "ignored", + body: `{"url":"http://qbit","seedtime":"1"}`, + handler: SetQbittorrent, + wantData: "ok", + settingKey: conf.QbittorrentUrl, + }, + } + gin.SetMode(gin.TestMode) + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + fake := &settingsTool{name: test.toolName, version: test.version} + previous, existed := tool.Tools[test.toolName] + tool.Tools[test.toolName] = fake + t.Cleanup(func() { + if existed { + tool.Tools[test.toolName] = previous + } else { + delete(tool.Tools, test.toolName) + } + _ = db.DeleteSettingItemByKey(test.settingKey) + op.SettingCacheUpdate() + }) + + response := httptest.NewRecorder() + ctx, _ := gin.CreateTestContext(response) + ctx.Request = httptest.NewRequest(http.MethodPost, "/", strings.NewReader(test.body)) + ctx.Request.Header.Set("Content-Type", "application/json") + test.handler(ctx) + + var result common.Resp[string] + if err := json.Unmarshal(response.Body.Bytes(), &result); err != nil { + t.Fatal(err) + } + if result.Code != 200 || result.Data != test.wantData { + t.Fatalf("response = %#v, want code 200 and data %q", result, test.wantData) + } + if fake.initCalls != 1 { + t.Fatalf("Init calls = %d, want 1", fake.initCalls) + } + }) + } +} + +func TestValidateOfflineDownloadStorageRejectsWrongNativeTool(t *testing.T) { + root := t.TempDir() + addition, err := json.Marshal(struct { + RootFolderPath string `json:"root_folder_path"` + }{RootFolderPath: root}) + if err != nil { + t.Fatal(err) + } + mount := "/" + strings.ReplaceAll(t.Name(), "/", "_") + storageID, err := op.CreateStorage(context.Background(), model.Storage{ + Driver: "Local", + MountPath: mount, + Addition: string(addition), + }) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + if err := op.DeleteStorageById(context.Background(), storageID); err != nil { + t.Errorf("delete fixture storage: %v", err) + } + }) + + response := httptest.NewRecorder() + ctx, _ := gin.CreateTestContext(response) + if validateOfflineDownloadStorage(ctx, mount, "Thunder") { + t.Fatal("Local storage unexpectedly accepted as Thunder") + } + var result common.Resp[any] + if err := json.Unmarshal(response.Body.Bytes(), &result); err != nil { + t.Fatal(err) + } + want := "unsupported storage driver for offline download, only Thunder is supported" + if result.Code != 400 || result.Message != want { + t.Fatalf("response = %#v, want code 400 and message %q", result, want) + } +} diff --git a/server/handles/search.go b/server/handles/search.go index 32e4e955..e59e2597 100644 --- a/server/handles/search.go +++ b/server/handles/search.go @@ -2,7 +2,6 @@ package handles import ( "path" - "strings" "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/errs" @@ -44,31 +43,35 @@ func Search(c *gin.Context) { common.ErrorResp(c, err, 400) return } - nodes, total, err := search.Search(c, req.SearchReq) + nodes, total, err := search.SearchFiltered(c, req.SearchReq, func(node model.SearchNode) bool { + return isSearchNodeAccessible(user, node, req.Password, op.GetNearestMeta) + }) if err != nil { common.ErrorResp(c, err, 500) return } - var filteredNodes []model.SearchNode - for _, node := range nodes { - if !strings.HasPrefix(node.Parent, user.BasePath) { - continue - } - meta, err := op.GetNearestMeta(node.Parent) - if err != nil && !errors.Is(errors.Cause(err), errs.MetaNotFound) { - continue - } - if !common.CanAccess(user, meta, path.Join(node.Parent, node.Name), req.Password) { - continue - } - filteredNodes = append(filteredNodes, node) - } common.SuccessResp(c, common.PageResp{ - Content: utils.MustSliceConvert(filteredNodes, nodeToSearchResp), + Content: utils.MustSliceConvert(nodes, nodeToSearchResp), Total: total, }) } +func isSearchNodeAccessible(user *model.User, node model.SearchNode, password string, resolveMeta func(string) (*model.Meta, error)) bool { + if !utils.IsSubPath(user.BasePath, node.Parent) { + return false + } + nodePath := path.Join(node.Parent, node.Name) + metaPath := node.Parent + if node.IsDir { + metaPath = nodePath + } + meta, err := resolveMeta(metaPath) + if err != nil && !errors.Is(errors.Cause(err), errs.MetaNotFound) { + return false + } + return common.CanAccess(user, meta, nodePath, password) +} + func nodeToSearchResp(node model.SearchNode) SearchResp { return SearchResp{ SearchNode: node, diff --git a/server/handles/search_test.go b/server/handles/search_test.go new file mode 100644 index 00000000..1faa34c9 --- /dev/null +++ b/server/handles/search_test.go @@ -0,0 +1,84 @@ +package handles + +import ( + "path" + "testing" + + "github.com/OpenListTeam/OpenList/v4/internal/errs" + "github.com/OpenListTeam/OpenList/v4/internal/model" +) + +func fakeResolveMeta(metas map[string]*model.Meta) func(string) (*model.Meta, error) { + return func(p string) (*model.Meta, error) { + for { + if meta, ok := metas[p]; ok { + return meta, nil + } + if p == "/" { + return nil, errs.MetaNotFound + } + p = path.Dir(p) + } + } +} + +func TestIsSearchNodeAccessible(t *testing.T) { + tests := []struct { + name string + basePath string + metas map[string]*model.Meta + node model.SearchNode + want bool + wantMetaPath string + }{ + { + name: "restricted directory", + metas: map[string]*model.Meta{"/private": {Path: "/private", ReadUsers: []uint{1}}}, + node: model.SearchNode{Parent: "/", Name: "private", IsDir: true}, + want: false, + wantMetaPath: "/private", + }, + { + name: "restricted sub directory", + metas: map[string]*model.Meta{"/private": {Path: "/private", ReadUsers: []uint{1}, ReadUsersSub: true}}, + node: model.SearchNode{Parent: "/private", Name: "sub", IsDir: true}, + want: false, + wantMetaPath: "/private/sub", + }, + { + name: "file keeps parent scope", + metas: map[string]*model.Meta{"/private": {Path: "/private", ReadUsers: []uint{1}}}, + node: model.SearchNode{Parent: "/private", Name: "a.txt", IsDir: false}, + want: true, + wantMetaPath: "/private", + }, + { + name: "outside base path", + basePath: "/team", + node: model.SearchNode{Parent: "/other", Name: "private", IsDir: true}, + want: false, + wantMetaPath: "", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + basePath := tt.basePath + if basePath == "" { + basePath = "/" + } + user := &model.User{ID: 2, BasePath: basePath} + resolve := fakeResolveMeta(tt.metas) + var gotMetaPath string + spy := func(p string) (*model.Meta, error) { + gotMetaPath = p + return resolve(p) + } + if got := isSearchNodeAccessible(user, tt.node, "", spy); got != tt.want { + t.Fatalf("isSearchNodeAccessible() = %v, want %v", got, tt.want) + } + if tt.wantMetaPath != "" && gotMetaPath != tt.wantMetaPath { + t.Fatalf("meta resolved at %q, want %q", gotMetaPath, tt.wantMetaPath) + } + }) + } +} diff --git a/server/handles/setup.go b/server/handles/setup.go new file mode 100644 index 00000000..584d3c9d --- /dev/null +++ b/server/handles/setup.go @@ -0,0 +1,82 @@ +package handles + +import ( + "github.com/OpenListTeam/OpenList/v4/internal/conf" + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/internal/op" + "github.com/OpenListTeam/OpenList/v4/server/common" + "github.com/gin-gonic/gin" + "github.com/pkg/errors" + "gorm.io/gorm" +) + +// InitStatus 返回系统是否已完成初始化（即是否已存在管理员账号）。 +// 前端据此判断是否跳转到安装向导。 +func InitStatus(c *gin.Context) { + if _, err := op.GetAdmin(); err != nil { + if errors.Is(err, gorm.ErrRecordNotFound) { + common.SuccessResp(c, gin.H{"initialized": false}) + return + } + common.ErrorResp(c, err, 500, true) + return + } + common.SuccessResp(c, gin.H{"initialized": true}) +} + +// InitSetupReq 系统初始化请求体 +type InitSetupReq struct { + Username string `json:"username" binding:"required"` + Password string `json:"password" binding:"required"` + SiteTitle string `json:"site_title"` +} + +// InitSetup 执行系统初始化：创建管理员账号并设置站点名称等初始参数。 +// 仅在系统尚未初始化时允许调用；已初始化则返回错误。 +func InitSetup(c *gin.Context) { + var req InitSetupReq + if err := c.ShouldBind(&req); err != nil { + common.ErrorResp(c, err, 400) + return + } + // 已初始化则拒绝 + if _, err := op.GetAdmin(); err == nil { + common.ErrorStrResp(c, "system has already been initialized", 400) + return + } else if !errors.Is(err, gorm.ErrRecordNotFound) { + common.ErrorResp(c, err, 500, true) + return + } + // 密码最小长度校验 + if len(req.Password) < 4 { + common.ErrorStrResp(c, "password must be at least 4 characters", 400) + return + } + admin := &model.User{ + Username: req.Username, + Role: model.ADMIN, + BasePath: "/", + Authn: "[]", + Permission: 0x71FF, + } + admin.SetPassword(req.Password) + if err := op.CreateUser(admin); err != nil { + common.ErrorResp(c, err, 500, true) + return + } + // 设置站点名称（如果提供） + if req.SiteTitle != "" { + item := model.SettingItem{ + Key: conf.SiteTitle, + Value: req.SiteTitle, + Type: conf.TypeString, + Group: model.SITE, + Flag: model.PUBLIC, + } + if err := op.SaveSettingItem(&item); err != nil { + common.ErrorResp(c, err, 500, true) + return + } + } + common.SuccessResp(c) +} diff --git a/server/handles/sharing.go b/server/handles/sharing.go index 6090fcbd..221bc127 100644 --- a/server/handles/sharing.go +++ b/server/handles/sharing.go @@ -54,7 +54,7 @@ func SharingGet(c *gin.Context, req *FsGetReq) { HashInfoStr: obj.GetHash().String(), HashInfo: obj.GetHash().Export(), Sign: "", - Type: utils.GetFileType(obj.GetName()), + Type: utils.GetObjType(obj.GetName(), obj.IsDir()), Thumb: thumb, }, RawURL: url, @@ -461,7 +461,7 @@ func UpdateSharing(c *gin.Context) { for i, s := range req.Files { s = utils.FixAndCleanPath(s) req.Files[i] = s - if !reqUser.IsAdmin() && !strings.HasPrefix(s, user.BasePath) { + if !reqUser.IsAdmin() && !utils.IsSubPath(user.BasePath, s) { common.ErrorStrResp(c, fmt.Sprintf("permission denied to share path [%s]", s), 500) return } @@ -545,7 +545,7 @@ func CreateSharing(c *gin.Context) { for i, s := range req.Files { s = utils.FixAndCleanPath(s) req.Files[i] = s - if !reqUser.IsAdmin() && !strings.HasPrefix(s, user.BasePath) { + if !reqUser.IsAdmin() && !utils.IsSubPath(user.BasePath, s) { common.ErrorStrResp(c, fmt.Sprintf("permission denied to share path [%s]", s), 500) return } diff --git a/server/handles/ssologin.go b/server/handles/ssologin.go index 4baabf6c..3b152505 100644 --- a/server/handles/ssologin.go +++ b/server/handles/ssologin.go @@ -1,6 +1,7 @@ package handles import ( + "crypto/sha256" "encoding/base64" "errors" "fmt" @@ -22,12 +23,25 @@ import ( "github.com/coreos/go-oidc" "github.com/gin-gonic/gin" "github.com/go-resty/resty/v2" + "github.com/golang-jwt/jwt/v4" "golang.org/x/oauth2" "gorm.io/gorm" ) const stateLength = 16 const stateExpire = time.Minute * 5 +const ssoBindingExpire = time.Minute * 5 +const ssoBindingCookie = "openlist_sso_binding" +const ssoBindingStatePurpose = "sso_binding_state" +const ssoBindingProofPurpose = "sso_binding_proof" + +type ssoBindingClaims struct { + Purpose string `json:"purpose"` + Method string `json:"method"` + SessionDigest string `json:"session_digest"` + SsoID string `json:"sso_id,omitempty"` + jwt.RegisteredClaims +} var stateCache = cache.NewMemCache[string](cache.WithShards[string](stateLength)) @@ -46,6 +60,115 @@ func verifyState(clientID, ip, state string) bool { return ok && value == ip } +func ssoBindingSessionDigest(session string) string { + digest := sha256.Sum256([]byte(session)) + return base64.RawURLEncoding.EncodeToString(digest[:]) +} + +func parseSSOBindingToken(c *gin.Context, rawToken, purpose string) (*ssoBindingClaims, error) { + claims := &ssoBindingClaims{} + token, err := jwt.ParseWithClaims(rawToken, claims, func(token *jwt.Token) (interface{}, error) { + if token.Method != jwt.SigningMethodHS256 { + return nil, errors.New("invalid SSO binding token algorithm") + } + return common.SecretKey, nil + }) + if err != nil || !token.Valid || claims.ExpiresAt == nil || claims.Purpose != purpose || + claims.Method != "get_sso_id" || len(claims.SessionDigest) != 43 { + return nil, errors.New("invalid or expired SSO binding token") + } + if (purpose == ssoBindingStatePurpose && claims.SsoID != "") || + (purpose == ssoBindingProofPurpose && claims.SsoID == "") { + return nil, errors.New("invalid SSO binding token payload") + } + session, err := c.Cookie(ssoBindingCookie) + if err != nil || ssoBindingSessionDigest(session) != claims.SessionDigest { + return nil, errors.New("invalid SSO binding session") + } + return claims, nil +} + +func generateSSOBindingToken(c *gin.Context, purpose, ssoID string) (string, error) { + session, err := c.Cookie(ssoBindingCookie) + expire := ssoBindingExpire + if purpose == ssoBindingStatePurpose { + session = random.String(32) + expire = stateExpire + c.SetSameSite(http.SameSiteLaxMode) + c.SetCookie( + ssoBindingCookie, + session, + int((stateExpire+ssoBindingExpire).Seconds()), + path.Join(conf.URL.Path, "/api"), + "", + strings.HasPrefix(common.GetApiUrl(c), "https://"), + true, + ) + } else if err != nil { + return "", errors.New("missing SSO binding session") + } + now := time.Now() + claims := ssoBindingClaims{ + Purpose: purpose, + Method: "get_sso_id", + SessionDigest: ssoBindingSessionDigest(session), + SsoID: ssoID, + RegisteredClaims: jwt.RegisteredClaims{ + ExpiresAt: jwt.NewNumericDate(now.Add(expire)), + IssuedAt: jwt.NewNumericDate(now), + NotBefore: jwt.NewNumericDate(now), + }, + } + return jwt.NewWithClaims(jwt.SigningMethodHS256, claims).SignedString(common.SecretKey) +} + +// ssoTargetOrigin returns the origin that is allowed to receive the SSO result +// via postMessage. It honours the operator-configured sso_postmessage_origin so +// a frontend served from a different origin than the API can still receive the +// result; otherwise it falls back to the API origin, or "/" to restrict +// delivery to same-origin openers when that cannot be resolved. +func ssoTargetOrigin(c *gin.Context) string { + if configured := setting.GetStr(conf.SSOPostMessageOrigin); configured != "" { + if u, err := url.Parse(configured); err == nil && + (u.Scheme == "http" || u.Scheme == "https") && + u.Host != "" && u.User == nil && + (u.Path == "" || u.Path == "/") && + u.RawQuery == "" && u.Fragment == "" { + return u.Scheme + "://" + u.Host + } + } + u, err := url.Parse(common.GetApiUrl(c)) + if err != nil || u.Scheme == "" || u.Host == "" { + return "/" + } + return u.Scheme + "://" + u.Host +} + +// ssoPostMessage hands the SSO result back to the window that started the login. +// The target origin is pinned so that an arbitrary page cannot open the SSO +// endpoint in a popup and read the payload out of the message event. +func ssoPostMessage(c *gin.Context, payload map[string]string) { + data, err := utils.Json.MarshalToString(payload) + if err != nil { + common.ErrorResp(c, err, 500) + return + } + origin, err := utils.Json.MarshalToString(ssoTargetOrigin(c)) + if err != nil { + common.ErrorResp(c, err, 500) + return + } + html := fmt.Sprintf(` + + + + `, data, origin) + c.Data(200, "text/html; charset=utf-8", []byte(html)) +} + func ssoRedirectUri(c *gin.Context, useCompatibility bool, method string) string { if useCompatibility { return common.GetApiUrl(c) + "/api/auth/" + method @@ -74,6 +197,16 @@ func SSOLoginRedirect(c *gin.Context) { urlValues.Add("response_type", "code") urlValues.Add("redirect_uri", redirectUri) urlValues.Add("client_id", clientId) + bindingState := "" + var err error + if method == "get_sso_id" { + bindingState, err = generateSSOBindingToken(c, ssoBindingStatePurpose, "") + if err != nil { + common.ErrorResp(c, err, 500) + return + } + urlValues.Add("state", bindingState) + } switch platform { case "Github": rUrl = "https://github.com/login/oauth/authorize?" @@ -94,15 +227,19 @@ func SSOLoginRedirect(c *gin.Context) { endpoint := strings.TrimSuffix(setting.GetStr(conf.SSOEndpointName), "/") rUrl = endpoint + "/login/oauth/authorize?" urlValues.Add("scope", "profile") - urlValues.Add("state", endpoint) + if bindingState == "" { + urlValues.Add("state", endpoint) + } case "OIDC": oauth2Config, err := GetOIDCClient(c, useCompatibility, redirectUri, method) if err != nil { common.ErrorStrResp(c, err.Error(), 400) return } - state := generateState(clientId, c.ClientIP()) - c.Redirect(http.StatusFound, oauth2Config.AuthCodeURL(state)) + if bindingState == "" { + bindingState = generateState(clientId, c.ClientIP()) + } + c.Redirect(http.StatusFound, oauth2Config.AuthCodeURL(bindingState)) return default: common.ErrorStrResp(c, "invalid platform", 400) @@ -201,11 +338,15 @@ func OIDCLoginCallback(c *gin.Context) { common.ErrorResp(c, err, 400) return } - if !verifyState(clientId, c.ClientIP(), c.Query("state")) { + if method == "get_sso_id" { + if _, err := parseSSOBindingToken(c, c.Query("state"), ssoBindingStatePurpose); err != nil { + common.ErrorStrResp(c, "incorrect or expired state parameter", 400) + return + } + } else if !verifyState(clientId, c.ClientIP(), c.Query("state")) { common.ErrorStrResp(c, "incorrect or expired state parameter", 400) return } - oauth2Token, err := oauth2Config.Exchange(c, c.Query("code")) if err != nil { common.ErrorResp(c, err, 400) @@ -235,19 +376,16 @@ func OIDCLoginCallback(c *gin.Context) { return } if method == "get_sso_id" { - if useCompatibility { - c.Redirect(302, common.GetApiUrl(c)+"/@manage?sso_id="+userID) + bindingProof, err := generateSSOBindingToken(c, ssoBindingProofPurpose, userID) + if err != nil { + common.ErrorResp(c, err, 500) return } - html := fmt.Sprintf(` - - - - `, userID) - c.Data(200, "text/html; charset=utf-8", []byte(html)) + if useCompatibility { + c.Redirect(302, common.GetApiUrl(c)+"/@manage?sso_id="+bindingProof) + return + } + ssoPostMessage(c, map[string]string{"sso_id": bindingProof}) return } if method == "sso_get_token" { @@ -268,15 +406,7 @@ func OIDCLoginCallback(c *gin.Context) { c.Redirect(302, common.GetApiUrl(c)+"/@login?token="+token) return } - html := fmt.Sprintf(` - - - - `, token) - c.Data(200, "text/html; charset=utf-8", []byte(html)) + ssoPostMessage(c, map[string]string{"token": token}) return } } @@ -352,6 +482,12 @@ func SSOLoginCallback(c *gin.Context) { common.ErrorStrResp(c, "No code provided", 400) return } + if argument == "get_sso_id" { + if _, err := parseSSOBindingToken(c, c.Query("state"), ssoBindingStatePurpose); err != nil { + common.ErrorStrResp(c, "incorrect or expired state parameter", 400) + return + } + } var resp *resty.Response var err error if platform == "Dingtalk" { @@ -402,19 +538,16 @@ func SSOLoginCallback(c *gin.Context) { return } if argument == "get_sso_id" { - if usecompatibility { - c.Redirect(302, common.GetApiUrl(c)+"/@manage?sso_id="+userID) + bindingProof, err := generateSSOBindingToken(c, ssoBindingProofPurpose, userID) + if err != nil { + common.ErrorResp(c, err, 500) return } - html := fmt.Sprintf(` - - - - `, userID) - c.Data(200, "text/html; charset=utf-8", []byte(html)) + if usecompatibility { + c.Redirect(302, common.GetApiUrl(c)+"/@manage?sso_id="+bindingProof) + return + } + ssoPostMessage(c, map[string]string{"sso_id": bindingProof}) return } username := utils.Json.Get(resp.Body(), usernameField).ToString() @@ -435,13 +568,5 @@ func SSOLoginCallback(c *gin.Context) { c.Redirect(302, common.GetApiUrl(c)+"/@login?token="+token) return } - html := fmt.Sprintf(` - - - - `, token) - c.Data(200, "text/html; charset=utf-8", []byte(html)) + ssoPostMessage(c, map[string]string{"token": token}) } diff --git a/server/handles/ssologin_origin_test.go b/server/handles/ssologin_origin_test.go new file mode 100644 index 00000000..388ebc74 --- /dev/null +++ b/server/handles/ssologin_origin_test.go @@ -0,0 +1,102 @@ +package handles + +import ( + "context" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/OpenListTeam/OpenList/v4/internal/conf" + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/internal/op" + "github.com/gin-gonic/gin" +) + +func ssoTestContext(apiUrl string) (*gin.Context, *httptest.ResponseRecorder) { + gin.SetMode(gin.TestMode) + rec := httptest.NewRecorder() + c, engine := gin.CreateTestContext(rec) + // Matches server.Init, which is what lets GetApiUrl reach the value the + // middleware stored on the request context. + engine.ContextWithFallback = true + req := httptest.NewRequest(http.MethodGet, "/api/auth/sso?method=sso_get_token", nil) + if apiUrl != "" { + req = req.WithContext(context.WithValue(req.Context(), conf.ApiUrlKey, apiUrl)) + } + c.Request = req + // Keep setting lookups off the (uninitialised) database: ssoTargetOrigin + // reads sso_postmessage_origin through the setting cache. + op.Cache.SetSetting(conf.SSOPostMessageOrigin, &model.SettingItem{ + Key: conf.SSOPostMessageOrigin, + Value: "", + }) + return c, rec +} + +// A page that opens the SSO endpoint in a popup must not be able to read the +// token: the postMessage target origin has to name the site, never "*". +func TestSSOPostMessagePinsTargetOrigin(t *testing.T) { + c, rec := ssoTestContext("https://openlist.example.com/base") + ssoPostMessage(c, map[string]string{"token": "secret-token"}) + + body := rec.Body.String() + if strings.Contains(body, `"*"`) || strings.Contains(body, `, '*'`) { + t.Fatalf("wildcard target origin present in response:\n%s", body) + } + if !strings.Contains(body, `"https://openlist.example.com"`) { + t.Errorf("expected the site origin as target, got:\n%s", body) + } + if !strings.Contains(body, "secret-token") { + t.Errorf("payload should still reach a legitimate opener, got:\n%s", body) + } +} + +// A frontend served from a different origin than the API needs the operator to +// be able to point the target at the frontend origin. The configured origin +// must win over the API origin. +func TestSSOPostMessageUsesConfiguredOrigin(t *testing.T) { + c, rec := ssoTestContext("https://api.example.com/base") + op.Cache.SetSetting(conf.SSOPostMessageOrigin, &model.SettingItem{ + Key: conf.SSOPostMessageOrigin, + Value: "https://frontend.example.com", + }) + defer op.Cache.ClearAll() + + ssoPostMessage(c, map[string]string{"token": "secret-token"}) + + body := rec.Body.String() + if !strings.Contains(body, `"https://frontend.example.com"`) { + t.Errorf("expected the configured origin as target, got:\n%s", body) + } +} + +// If the site URL cannot be resolved the fallback must tighten delivery to +// same-origin openers, not widen it back to every origin. +func TestSSOPostMessageFallsBackToSameOrigin(t *testing.T) { + c, rec := ssoTestContext("") + ssoPostMessage(c, map[string]string{"token": "secret-token"}) + + body := rec.Body.String() + if strings.Contains(body, `"*"`) { + t.Fatalf("fallback must not be a wildcard origin:\n%s", body) + } + if !strings.Contains(body, `"/"`) { + t.Errorf(`expected "/" fallback origin, got:\n%s`, body) + } +} + +// userID comes from the identity provider, so it must be encoded rather than +// interpolated into the JS string literal it used to land in. +func TestSSOPostMessageEscapesProviderControlledValue(t *testing.T) { + c, rec := ssoTestContext("https://openlist.example.com") + ssoPostMessage(c, map[string]string{"sso_id": `"});alert(document.domain);//`}) + + body := rec.Body.String() + if strings.Contains(body, `alert(document.domain)`) && !strings.Contains(body, `\"`) { + t.Fatalf("provider value was not escaped:\n%s", body) + } + if !strings.Contains(body, `\"});alert`) { + t.Errorf("expected the injected quote to be escaped, got:\n%s", body) + } +} diff --git a/server/handles/webauthn.go b/server/handles/webauthn.go index 6848020f..dee94496 100644 --- a/server/handles/webauthn.go +++ b/server/handles/webauthn.go @@ -95,7 +95,7 @@ func FinishAuthnLogin(c *gin.Context) { } else { // client-side discoverable login _, err = authnInstance.FinishDiscoverableLogin(func(_, userHandle []byte) (webauthn.User, error) { // first param `rawID` in this callback function is equal to ID in webauthn.Credential, - // but it's unnnecessary to check it. + // but it's unnecessary to check it. // userHandle param is equal to (User).WebAuthnID(). userID := uint(binary.LittleEndian.Uint64(userHandle)) user, err = db.GetUserById(userID) diff --git a/server/mcp/fs_get.go b/server/mcp/fs_get.go index 550cc036..c975695f 100644 --- a/server/mcp/fs_get.go +++ b/server/mcp/fs_get.go @@ -68,9 +68,11 @@ func (s *Server) callFSGet(c *gin.Context, raw json.RawMessage) (any, *rpcError) parentPath := stdpath.Dir(reqPath) var related []model.Obj - sameLevelFiles, err := fs.List(ctx, parentPath, &fs.ListArgs{}) - if err == nil { - related = filterRelatedObjs(sameLevelFiles, obj) + if !obj.IsDir() && utils.GetFileType(obj.GetName()) == conf.VIDEO { + sameLevelFiles, err := fs.List(ctx, parentPath, &fs.ListArgs{}) + if err == nil { + related = filterRelatedObjs(sameLevelFiles, obj) + } } parentMeta, _ := op.GetNearestMeta(parentPath) @@ -85,7 +87,7 @@ func (s *Server) callFSGet(c *gin.Context, raw json.RawMessage) (any, *rpcError) Created: obj.CreateTime(), Sign: common.Sign(obj, parentPath, isEncrypt(meta, reqPath)), Thumb: thumb, - Type: utils.GetFileType(obj.GetName()), + Type: utils.GetObjType(obj.GetName(), obj.IsDir()), HashInfoStr: obj.GetHash().String(), HashInfo: obj.GetHash().Export(), MountDetails: mountDetails, diff --git a/server/mcp/handler_test.go b/server/mcp/handler_test.go index d3369576..a7515773 100644 --- a/server/mcp/handler_test.go +++ b/server/mcp/handler_test.go @@ -200,8 +200,10 @@ func TestInitializeNegotiatesUnsupportedProtocolVersion(t *testing.T) { func TestCreateSessionPrunesUserSessionLimit(t *testing.T) { srv := newTestServer(nil) var firstSessionID string + baseTime := time.Now() for i := range maxUserSessions { currentSession := srv.createSession(1) + currentSession.lastUsedAt = baseTime.Add(time.Duration(i)) if i == 0 { firstSessionID = currentSession.id } @@ -223,8 +225,10 @@ func TestCreateSessionPrunesUserSessionLimit(t *testing.T) { func TestCreateSessionPrunesGlobalSessionLimit(t *testing.T) { srv := newTestServer(nil) var firstSessionID string + baseTime := time.Now() for i := range maxSessions { currentSession := srv.createSession(uint(i + 1)) + currentSession.lastUsedAt = baseTime.Add(time.Duration(i)) if i == 0 { firstSessionID = currentSession.id } diff --git a/server/middlewares/check_test.go b/server/middlewares/check_test.go new file mode 100644 index 00000000..a9213637 --- /dev/null +++ b/server/middlewares/check_test.go @@ -0,0 +1,62 @@ +package middlewares + +import ( + "net/http" + "net/http/httptest" + "testing" + + "github.com/OpenListTeam/OpenList/v4/internal/conf" + "github.com/gin-gonic/gin" +) + +func TestStoragesLoadedAdmitsRequestOrigin(t *testing.T) { + originalMode := gin.Mode() + gin.SetMode(gin.TestMode) + originalConf := conf.Conf + originalLoaded := conf.StoragesLoaded + t.Cleanup(func() { + gin.SetMode(originalMode) + conf.Conf = originalConf + conf.StoragesLoaded = originalLoaded + }) + conf.StoragesLoaded = true + + router := gin.New() + router.Use(StoragesLoaded) + router.GET("/", func(c *gin.Context) { + c.String(http.StatusOK, conf.GetApiUrl(c.Request.Context())) + }) + + assertOrigin := func(name, siteURL, target string, header http.Header, want string) { + t.Run(name, func(t *testing.T) { + conf.Conf = &conf.Config{SiteURL: siteURL} + + req := httptest.NewRequest(http.MethodGet, target, nil) + req.Header = header + rec := httptest.NewRecorder() + router.ServeHTTP(rec, req) + + if got := rec.Body.String(); got != want { + t.Fatalf("origin = %q, want %q", got, want) + } + }) + } + + assertOrigin( + "configured site URL", + "https://openlist.example/base/", + "http://ignored.example/", + nil, + "https://openlist.example/base", + ) + assertOrigin( + "forwarded request", + "", + "http://internal.example/", + http.Header{ + "X-Forwarded-Proto": {"https"}, + "X-Forwarded-Host": {"public.example"}, + }, + "https://public.example", + ) +} diff --git a/server/middlewares/down.go b/server/middlewares/down.go index d71be00b..787936d6 100644 --- a/server/middlewares/down.go +++ b/server/middlewares/down.go @@ -60,7 +60,7 @@ func needSign(meta *model.Meta, path string) bool { if meta == nil || meta.Password == "" { return false } - if !meta.PSub && path != meta.Path { + if !meta.PSub && !common.MetaCoversPath(meta.Path, path, false) { return false } return true diff --git a/server/router.go b/server/router.go index 9dc66443..4c215969 100644 --- a/server/router.go +++ b/server/router.go @@ -4,6 +4,7 @@ import ( "github.com/OpenListTeam/OpenList/v4/cmd/flags" "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/message" + multipartPkg "github.com/OpenListTeam/OpenList/v4/internal/multipart" "github.com/OpenListTeam/OpenList/v4/internal/sign" "github.com/OpenListTeam/OpenList/v4/internal/stream" "github.com/OpenListTeam/OpenList/v4/pkg/utils" @@ -106,6 +107,8 @@ func Init(e *gin.Engine) { public.Any("/settings", handles.PublicSettings) public.Any("/offline_download_tools", handles.OfflineDownloadTools) public.Any("/archive_extensions", handles.ArchiveExtensions) + public.Any("/init_status", handles.InitStatus) + public.POST("/init/setup", handles.InitSetup) // 转码 Worker 协议端点（远程 Worker 节点对接，由共享密钥/token 鉴权，无需登录） transcodeWorker := api.Group("/transcode/worker") @@ -180,6 +183,7 @@ func admin(g *gin.RouterGroup) { setting.POST("/set_thunder", handles.SetThunder) setting.POST("/set_thunderx", handles.SetThunderX) setting.POST("/set_thunder_browser", handles.SetThunderBrowser) + setting.POST("/set_guangyapan", handles.SetGuangYaPan) // retain /admin/task API to ensure compatibility with legacy automation scripts _task(g.Group("/task")) @@ -267,6 +271,13 @@ func _fs(g *gin.RouterGroup) { uploadLimiter := middlewares.UploadRateLimiter(stream.ClientUploadLimit) g.PUT("/put", middlewares.FsUp, uploadLimiter, handles.FsStream) g.PUT("/form", middlewares.FsUp, uploadLimiter, handles.FsForm) + multipartPkg.DefaultManager.StartGC() // reclaim ring files orphaned by a previous run + multipart := g.Group("/multipart") + multipart.POST("/init", middlewares.FsUp, handles.MultipartInit) + multipart.PUT("/chunk", uploadLimiter, handles.MultipartChunk) + multipart.POST("/complete", handles.MultipartComplete) + multipart.GET("/status", handles.MultipartStatus) + multipart.POST("/abort", handles.MultipartAbort) g.POST("/link", middlewares.AuthAdmin, handles.Link) // g.POST("/add_aria2", handles.AddOfflineDownload) // g.POST("/add_qbit", handles.AddQbittorrent) @@ -279,7 +290,7 @@ func _fs(g *gin.RouterGroup) { g.POST("/torrent/rapid_upload", handles.TorrentRapidUpload) g.POST("/torrent/generate", handles.GenerateTorrentForPath) // Direct upload (client-side upload to storage) - g.POST("/get_direct_upload_info", middlewares.FsUp, handles.FsGetDirectUploadInfo) + g.POST("/get_direct_upload_info", handles.FsGetDirectUploadInfo) } func _task(g *gin.RouterGroup) { diff --git a/server/s3/backend.go b/server/s3/backend.go index f00d2149..de6413ce 100644 --- a/server/s3/backend.go +++ b/server/s3/backend.go @@ -23,7 +23,7 @@ import ( "github.com/OpenListTeam/OpenList/v4/internal/stream" "github.com/OpenListTeam/OpenList/v4/pkg/http_range" "github.com/OpenListTeam/OpenList/v4/pkg/utils" - "github.com/itsHenry35/gofakes3" + "github.com/OpenListTeam/gofakes3" "github.com/ncw/swift/v2" log "github.com/sirupsen/logrus" ) @@ -33,17 +33,25 @@ var ( timeFormat = "Mon, 2 Jan 2006 15:04:05 GMT" ) -// s3Backend implements the gofacess3.Backend interface to make an S3 -// backend for gofakes3 +// s3Backend implements the gofakes3.Backend interface to make an S3 +// backend for gofakes3. It also implements gofakes3.MultipartBackend so that +// multipart uploads are streamed to local temp files part-by-part and +// assembled into storage on completion, instead of being buffered in memory. type s3Backend struct { - meta *sync.Map + meta *sync.Map + listDir func(context.Context, string) ([]model.Obj, error) + uploads *sync.Map // map[gofakes3.UploadID]*multipartState } // newBackend creates a new SimpleBucketBackend. func newBackend() gofakes3.Backend { - return &s3Backend{ - meta: new(sync.Map), + b := &s3Backend{ + meta: new(sync.Map), + uploads: new(sync.Map), + listDir: getDirEntries, } + b.startReaper() + return b } // ListBuckets always returns the default bucket. @@ -84,10 +92,9 @@ func (b *s3Backend) ListBucket(ctx context.Context, bucketName string, prefix *g prefix.HasDelimiter = false } - response := gofakes3.NewObjectList() path, remaining := prefixParser(prefix) - err = b.entryListR(bucketPath, path, remaining, prefix.HasDelimiter, response) + response, err := b.listPage(ctx, bucketPath, path, remaining, prefix.HasDelimiter, page) if err == gofakes3.ErrNoSuchKey { // AWS just returns an empty list response = gofakes3.NewObjectList() @@ -95,7 +102,7 @@ func (b *s3Backend) ListBucket(ctx context.Context, bucketName string, prefix *g return nil, err } - return b.pager(response, page) + return response, nil } // HeadObject returns the fileinfo for the given object name. @@ -145,6 +152,8 @@ func (b *s3Backend) HeadObject(ctx context.Context, bucketName, objectName strin // GetObject fetchs the object from the filesystem. func (b *s3Backend) GetObject(ctx context.Context, bucketName, objectName string, rangeRequest *gofakes3.ObjectRangeRequest) (s3Obj *gofakes3.Object, err error) { + defer func() { err = mapBackendError(err) }() + bucket, err := getBucketByName(bucketName) if err != nil { return nil, err @@ -186,7 +195,7 @@ func (b *s3Backend) GetObject(ctx context.Context, bucketName, objectName string return nil, fmt.Errorf("the remote storage driver need to be enhanced to support s3") } - var rd io.Reader + var rd io.ReadCloser if rnge != nil { rd, err = rrf.RangeRead(ctx, http_range.Range(*rnge)) } else { @@ -208,6 +217,7 @@ func (b *s3Backend) GetObject(ctx context.Context, bucketName, objectName string meta[k] = v } } + closers := utils.NewClosers(rd, link) return &gofakes3.Object{ // Name: gofakes3.URLEncode(objectName), @@ -216,7 +226,7 @@ func (b *s3Backend) GetObject(ctx context.Context, bucketName, objectName string Metadata: meta, Size: size, Range: rnge, - Contents: utils.ReadCloser{Reader: rd, Closer: link}, + Contents: utils.ReadCloser{Reader: rd, Closer: &closers}, }, nil } @@ -232,9 +242,20 @@ func (b *s3Backend) PutObject( meta map[string]string, input io.Reader, size int64, ) (result gofakes3.PutObjectResult, err error) { + return result, b.putStream(ctx, bucketName, objectName, meta, input, size) +} + +// putStream stores the given object into the underlying storage. It is shared +// by PutObject and the multipart-upload Complete step so both paths apply the +// same directory creation, metadata and ignore rules. +func (b *s3Backend) putStream( + ctx context.Context, bucketName, objectName string, + meta map[string]string, + input io.Reader, size int64, +) error { bucket, err := getBucketByName(bucketName) if err != nil { - return result, err + return err } bucketPath := bucket.Path @@ -260,15 +281,15 @@ func (b *s3Backend) PutObject( log.Debugf("reqPath: %s not found and objectName contains /, need to makeDir", reqPath) err = fs.MakeDir(ctx, reqPath) if err != nil { - return result, errors.WithMessagef(err, "failed to makeDir, reqPath: %s", reqPath) + return errors.WithMessagef(err, "failed to makeDir, reqPath: %s", reqPath) } } else { - return result, gofakes3.KeyNotFound(objectName) + return gofakes3.KeyNotFound(objectName) } } if isDir { - return result, nil + return nil } var ti time.Time @@ -294,28 +315,25 @@ func (b *s3Backend) PutObject( } // Check if system file should be ignored if setting.GetBool(conf.IgnoreSystemFiles) && utils.IsSystemFile(obj.Name) { - return result, errs.IgnoredSystemFile + return errs.IgnoredSystemFile } stream := &stream.FileStream{ Obj: &obj, Reader: input, Mimetype: meta["Content-Type"], } + if stream.Mimetype == "" { + stream.Mimetype = "application/octet-stream" + } err = fs.PutDirectly(ctx, reqPath, stream) if err != nil { - return result, err + return err } - // if err := stream.Close(); err != nil { - // // remove file when close error occurred (FsPutErr) - // _ = fs.Remove(ctx, fp) - // return result, err - // } - b.meta.Store(fp, meta) - return result, nil + return nil } // DeleteMulti deletes multiple objects in a single request. diff --git a/server/s3/errors_test.go b/server/s3/errors_test.go new file mode 100644 index 00000000..85df7fe8 --- /dev/null +++ b/server/s3/errors_test.go @@ -0,0 +1,66 @@ +package s3 + +import ( + "context" + "encoding/xml" + "errors" + "net/http" + "net/http/httptest" + "testing" + + "github.com/OpenListTeam/OpenList/v4/internal/errs" + "github.com/OpenListTeam/gofakes3" + "github.com/OpenListTeam/gofakes3/s3mem" +) + +func TestMapBackendErrorMapsOnlyTemporaryCapacity(t *testing.T) { + capacity := errs.NewErr(errs.TemporaryCapacity, "callback admission timed out") + if got := mapBackendError(capacity); got != gofakes3.ErrSlowDown { + t.Fatalf("capacity error mapped to %v, want %v", got, gofakes3.ErrSlowDown) + } + + permanent := errors.New("permission denied") + if got := mapBackendError(permanent); got != permanent { + t.Fatalf("permanent error mapped to %v, want original error", got) + } + if got := mapBackendError(nil); got != nil { + t.Fatalf("nil error mapped to %v", got) + } +} + +type capacityBackend struct { + gofakes3.Backend +} + +func (b capacityBackend) GetObject(context.Context, string, string, *gofakes3.ObjectRangeRequest) (*gofakes3.Object, error) { + return nil, mapBackendError(errs.NewErr(errs.TemporaryCapacity, "callback admission timed out")) +} + +func TestTemporaryCapacityProducesS3SlowDownResponse(t *testing.T) { + memory := s3mem.New() + if err := memory.CreateBucket(t.Context(), "bucket"); err != nil { + t.Fatal(err) + } + server := httptest.NewServer(gofakes3.New(capacityBackend{Backend: memory}).Server()) + defer server.Close() + + request, err := http.NewRequestWithContext(t.Context(), http.MethodGet, server.URL+"/bucket/object", nil) + if err != nil { + t.Fatal(err) + } + response, err := server.Client().Do(request) + if err != nil { + t.Fatal(err) + } + defer response.Body.Close() + if response.StatusCode != http.StatusServiceUnavailable { + t.Fatalf("status = %d, want %d", response.StatusCode, http.StatusServiceUnavailable) + } + var result gofakes3.ErrorResult + if err := xml.NewDecoder(response.Body).Decode(&result); err != nil { + t.Fatal(err) + } + if result.Code != gofakes3.ErrSlowDown || result.Message != gofakes3.ErrSlowDown.Message() { + t.Fatalf("S3 error = %#v, want SlowDown with standard message", result) + } +} diff --git a/server/s3/get_object_close_test.go b/server/s3/get_object_close_test.go new file mode 100644 index 00000000..00ef3420 --- /dev/null +++ b/server/s3/get_object_close_test.go @@ -0,0 +1,132 @@ +package s3 + +import ( + "context" + "encoding/json" + "errors" + "io" + "os" + "path/filepath" + "slices" + "strings" + "testing" + + "github.com/OpenListTeam/OpenList/v4/drivers/local" + "github.com/OpenListTeam/OpenList/v4/internal/conf" + "github.com/OpenListTeam/OpenList/v4/internal/db" + "github.com/OpenListTeam/OpenList/v4/internal/driver" + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/internal/op" + "github.com/OpenListTeam/OpenList/v4/internal/stream" + "github.com/OpenListTeam/OpenList/v4/pkg/http_range" + "github.com/OpenListTeam/OpenList/v4/pkg/utils" + "gorm.io/gorm" +) + +const closeTrackingDriverName = "S3CloseTrackingLocal" + +type closeTrackingDriver struct { + local.Local + closed *[]string +} + +func (d *closeTrackingDriver) Config() driver.Config { + c := d.Local.Config() + c.Name = closeTrackingDriverName + return c +} + +func (d *closeTrackingDriver) Link(context.Context, model.Obj, model.LinkArgs) (*model.Link, error) { + link := &model.Link{ + ContentLength: 4, + RangeReader: stream.RangeReaderFunc(func(context.Context, http_range.Range) (io.ReadCloser, error) { + return utils.NewReadCloser(strings.NewReader("body"), func() error { + *d.closed = append(*d.closed, "body") + return nil + }), nil + }), + RequireReference: true, + } + link.SyncClosers.Add(utils.CloseFunc(func() error { + *d.closed = append(*d.closed, "link") + return nil + })) + return link, nil +} + +func TestGetObjectClosesRangeBodyBeforeLink(t *testing.T) { + ctx := context.Background() + var closed []string + op.RegisterDriver(func() driver.Driver { + return &closeTrackingDriver{closed: &closed} + }) + + root := t.TempDir() + if err := os.WriteFile(filepath.Join(root, "fixture.txt"), []byte("body"), 0o600); err != nil { + t.Fatal(err) + } + addition, err := json.Marshal(struct { + RootFolderPath string `json:"root_folder_path"` + }{RootFolderPath: root}) + if err != nil { + t.Fatal(err) + } + mount := "/" + sanitizeTestName(t.Name()) + storageID, err := op.CreateStorage(ctx, model.Storage{ + Driver: closeTrackingDriverName, + MountPath: mount, + Addition: string(addition), + }) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + if err := op.DeleteStorageById(ctx, storageID); err != nil { + t.Errorf("delete fixture storage: %v", err) + } + }) + + previousBuckets, previousBucketsErr := op.GetSettingItemByKey(conf.S3Buckets) + if previousBucketsErr != nil && !errors.Is(previousBucketsErr, gorm.ErrRecordNotFound) { + t.Fatal(previousBucketsErr) + } + if err := op.SaveSettingItem(&model.SettingItem{ + Key: conf.S3Buckets, + Value: `[{"name":"close","path":"` + mount + `"}]`, + }); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + if previousBucketsErr == nil { + if err := op.SaveSettingItem(previousBuckets); err != nil { + t.Errorf("restore S3 buckets: %v", err) + } + return + } + if err := db.DeleteSettingItemByKey(conf.S3Buckets); err != nil { + t.Errorf("delete fixture S3 buckets: %v", err) + } + op.SettingCacheUpdate() + }) + + object, err := newBackend().(*s3Backend).GetObject(ctx, "close", "fixture.txt", nil) + if err != nil { + t.Fatal(err) + } + contents, err := io.ReadAll(object.Contents) + if err != nil { + t.Fatal(err) + } + if string(contents) != "body" { + t.Fatalf("contents = %q, want body", contents) + } + if err := object.Contents.Close(); err != nil { + t.Fatal(err) + } + if err := object.Contents.Close(); err != nil { + t.Fatal(err) + } + if !slices.Equal(closed, []string{"body", "link"}) { + t.Fatalf("close order = %v, want [body link] exactly once", closed) + } +} diff --git a/server/s3/list.go b/server/s3/list.go index d783099a..73a8bcaa 100644 --- a/server/s3/list.go +++ b/server/s3/list.go @@ -3,24 +3,119 @@ package s3 import ( + "context" "path" + "slices" + "sort" "strings" "time" - "github.com/itsHenry35/gofakes3" + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/gofakes3" log "github.com/sirupsen/logrus" ) -func (b *s3Backend) entryListR(bucket, fdPath, name string, addPrefix bool, response *gofakes3.ObjectList) error { - fp := path.Join(bucket, fdPath) +// S3 ListObjects responses contain at most 1,000 keys. +const maxListPageKeys int64 = 1000 - dirEntries, err := getDirEntries(fp) +type objectPage struct { + result *gofakes3.ObjectList + marker string + maxKeys int64 + count int64 + lastKey string +} + +func newObjectPage(page gofakes3.ListBucketPage) *objectPage { + maxKeys := page.MaxKeys + if maxKeys <= 0 || maxKeys > maxListPageKeys { + maxKeys = maxListPageKeys + } + marker := "" + if page.HasMarker { + marker = page.Marker + } + return &objectPage{ + result: gofakes3.NewObjectList(), + marker: marker, + maxKeys: maxKeys, + } +} + +func (p *objectPage) addContent(item *gofakes3.Content) bool { + if item.Key <= p.marker { + return true + } + if p.count >= p.maxKeys { + p.result.IsTruncated = true + return false + } + p.result.Add(item) + p.count++ + p.lastKey = item.Key + return true +} + +func (p *objectPage) addPrefix(prefix string) bool { + if prefix <= p.marker { + return true + } + if p.count >= p.maxKeys { + p.result.IsTruncated = true + return false + } + p.result.AddPrefix(prefix) + p.count++ + p.lastKey = prefix + return true +} + +func (p *objectPage) finish() *gofakes3.ObjectList { + if p.result.IsTruncated { + p.result.NextMarker = p.lastKey + } + return p.result +} + +func (b *s3Backend) listPage( + ctx context.Context, + bucket, fdPath, name string, + addPrefix bool, + page gofakes3.ListBucketPage, +) (*gofakes3.ObjectList, error) { + result := newObjectPage(page) + _, err := b.walkPage(ctx, bucket, fdPath, name, addPrefix, result) if err != nil { - return err + return nil, err + } + return result.finish(), nil +} + +// walkPage returns false after finding an entry beyond the requested page. +func (b *s3Backend) walkPage( + ctx context.Context, + bucket, fdPath, name string, + addPrefix bool, + page *objectPage, +) (bool, error) { + if err := ctx.Err(); err != nil { + return false, err + } + + fp := path.Join(bucket, fdPath) + dirEntries, err := b.listDir(ctx, fp) + if err != nil { + return false, err + } + if err := ctx.Err(); err != nil { + return false, err } // workaround as s3 can't have empty files in directories, useful in deletions if len(dirEntries) == 0 { + if !strings.HasPrefix(emptyObjectName, name) { + return true, nil + } item := &gofakes3.Content{ // Key: gofakes3.URLEncode(path.Join(fdPath, emptyObjectName)), Key: path.Join(fdPath, emptyObjectName), @@ -29,11 +124,19 @@ func (b *s3Backend) entryListR(bucket, fdPath, name string, addPrefix bool, resp Size: 0, StorageClass: gofakes3.StorageStandard, } - response.Add(item) log.Debugf("Adding empty object %s to response", item.Key) - return nil + return page.addContent(item), nil } + // Emit entries in the order their keys have in a flat S3 keyspace. A + // directory sorts as if it carried its trailing slash, so "b.txt" comes + // before "b/..." as it does in a real bucket. Without this a listing + // resumed from a marker could silently skip or repeat keys. + dirEntries = slices.Clone(dirEntries) + sort.Slice(dirEntries, func(i, j int) bool { + return listSortKey(dirEntries[i]) < listSortKey(dirEntries[j]) + }) + for _, entry := range dirEntries { object := entry.GetName() @@ -45,14 +148,23 @@ func (b *s3Backend) entryListR(bucket, fdPath, name string, addPrefix bool, resp } if entry.IsDir() { + // S3 common prefixes include the delimiter, e.g. "base/20261009T020008/". + // Clients such as barman-cloud ignore prefixes without it. + subtreePrefix := objectPath + "/" if addPrefix { - // response.AddPrefix(gofakes3.URLEncode(objectPath)) - response.AddPrefix(objectPath) + // response.AddPrefix(gofakes3.URLEncode(subtreePrefix)) + if !page.addPrefix(subtreePrefix) { + return false, nil + } continue } - err := b.entryListR(bucket, path.Join(fdPath, object), "", false, response) - if err != nil { - return err + // A marker beyond this subtree lets us avoid an upstream directory read. + if subtreePrefix <= page.marker && !strings.HasPrefix(page.marker, subtreePrefix) { + continue + } + keepGoing, err := b.walkPage(ctx, bucket, objectPath, "", false, page) + if err != nil || !keepGoing { + return keepGoing, err } } else { item := &gofakes3.Content{ @@ -63,8 +175,19 @@ func (b *s3Backend) entryListR(bucket, fdPath, name string, addPrefix bool, resp Size: entry.GetSize(), StorageClass: gofakes3.StorageStandard, } - response.Add(item) + if !page.addContent(item) { + return false, nil + } } } - return nil + return true, nil +} + +// listSortKey returns the name used to order directory entries so that the +// emitted keys follow S3 lexicographic order. +func listSortKey(entry model.Obj) string { + if entry.IsDir() { + return entry.GetName() + "/" + } + return entry.GetName() } diff --git a/server/s3/list_test.go b/server/s3/list_test.go new file mode 100644 index 00000000..fbfb2bc0 --- /dev/null +++ b/server/s3/list_test.go @@ -0,0 +1,346 @@ +package s3 + +import ( + "context" + "errors" + "fmt" + "slices" + "sort" + "strings" + "testing" + + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/gofakes3" +) + +func TestListPageBoundsRecursiveTraversal(t *testing.T) { + listCalls := 0 + b := &s3Backend{ + listDir: func(_ context.Context, dir string) ([]model.Obj, error) { + listCalls++ + if dir == "bucket/data" { + entries := make([]model.Obj, 256) + for i := range entries { + entries[i] = &model.Object{Name: fmt.Sprintf("%02x", i), IsFolder: true} + } + return entries, nil + } + return []model.Obj{&model.Object{Name: "pack", Size: 1}}, nil + }, + } + + got, err := b.listPage(context.Background(), "bucket", "data", "", false, gofakes3.ListBucketPage{MaxKeys: 3}) + if err != nil { + t.Fatalf("listPage() error = %v", err) + } + + wantKeys := []string{"data/00/pack", "data/01/pack", "data/02/pack"} + if len(got.Contents) != len(wantKeys) { + t.Fatalf("len(Contents) = %d, want %d", len(got.Contents), len(wantKeys)) + } + for i, want := range wantKeys { + if got.Contents[i].Key != want { + t.Errorf("Contents[%d].Key = %q, want %q", i, got.Contents[i].Key, want) + } + } + if !got.IsTruncated { + t.Error("IsTruncated = false, want true") + } + if got.NextMarker != wantKeys[len(wantKeys)-1] { + t.Errorf("NextMarker = %q, want %q", got.NextMarker, wantKeys[len(wantKeys)-1]) + } + if listCalls > 5 { + t.Errorf("list calls = %d, want at most 5 for a three-key page", listCalls) + } +} + +func TestListPageContinuesWithoutGapsOrDuplicates(t *testing.T) { + b := &s3Backend{ + listDir: func(_ context.Context, dir string) ([]model.Obj, error) { + if dir == "bucket/data" { + entries := make([]model.Obj, 16) + for i := range entries { + entries[i] = &model.Object{Name: fmt.Sprintf("%02x", 15-i), IsFolder: true} + } + return entries, nil + } + entries := make([]model.Obj, 20) + for i := range entries { + entries[i] = &model.Object{Name: fmt.Sprintf("pack-%02d", 19-i), Size: 1} + } + return entries, nil + }, + } + + var gotKeys []string + page := gofakes3.ListBucketPage{MaxKeys: 37} + for pageNumber := 0; pageNumber < 10; pageNumber++ { + got, err := b.listPage(context.Background(), "bucket", "data", "", false, page) + if err != nil { + t.Fatalf("listPage() page %d error = %v", pageNumber, err) + } + for _, item := range got.Contents { + gotKeys = append(gotKeys, item.Key) + } + if !got.IsTruncated { + break + } + page.Marker = got.NextMarker + page.HasMarker = true + } + + wantKeys := make([]string, 0, 16*20) + for shard := 0; shard < 16; shard++ { + for pack := 0; pack < 20; pack++ { + wantKeys = append(wantKeys, fmt.Sprintf("data/%02x/pack-%02d", shard, pack)) + } + } + sort.Strings(wantKeys) + if len(gotKeys) != len(wantKeys) { + t.Fatalf("listed %d keys, want %d", len(gotKeys), len(wantKeys)) + } + for i, want := range wantKeys { + if gotKeys[i] != want { + t.Fatalf("key %d = %q, want %q", i, gotKeys[i], want) + } + } +} + +func TestListPagePaginatesContentsAndCommonPrefixesTogether(t *testing.T) { + b := &s3Backend{ + listDir: func(_ context.Context, _ string) ([]model.Obj, error) { + return []model.Obj{ + &model.Object{Name: "d", IsFolder: true}, + &model.Object{Name: "c", Size: 1}, + &model.Object{Name: "b", IsFolder: true}, + &model.Object{Name: "a", Size: 1}, + }, nil + }, + } + + first, err := b.listPage(context.Background(), "bucket", "data", "", true, gofakes3.ListBucketPage{MaxKeys: 2}) + if err != nil { + t.Fatalf("first listPage() error = %v", err) + } + if len(first.Contents) != 1 || first.Contents[0].Key != "data/a" { + t.Fatalf("first contents = %#v, want data/a", first.Contents) + } + if len(first.CommonPrefixes) != 1 || first.CommonPrefixes[0].Prefix != "data/b/" { + t.Fatalf("first prefixes = %#v, want data/b/", first.CommonPrefixes) + } + if !first.IsTruncated || first.NextMarker != "data/b/" { + t.Fatalf("first page truncated = %v, next marker = %q", first.IsTruncated, first.NextMarker) + } + + second, err := b.listPage(context.Background(), "bucket", "data", "", true, gofakes3.ListBucketPage{ + Marker: first.NextMarker, + HasMarker: true, + MaxKeys: 2, + }) + if err != nil { + t.Fatalf("second listPage() error = %v", err) + } + if len(second.Contents) != 1 || second.Contents[0].Key != "data/c" { + t.Fatalf("second contents = %#v, want data/c", second.Contents) + } + if len(second.CommonPrefixes) != 1 || second.CommonPrefixes[0].Prefix != "data/d/" { + t.Fatalf("second prefixes = %#v, want data/d/", second.CommonPrefixes) + } + if second.IsTruncated || second.NextMarker != "" { + t.Fatalf("second page truncated = %v, next marker = %q", second.IsTruncated, second.NextMarker) + } +} + +func TestListPageSkipsCompletedSubtrees(t *testing.T) { + var calls []string + b := &s3Backend{ + listDir: func(_ context.Context, dir string) ([]model.Obj, error) { + calls = append(calls, dir) + if dir == "bucket/data" { + return []model.Obj{ + &model.Object{Name: "c", IsFolder: true}, + &model.Object{Name: "b", IsFolder: true}, + &model.Object{Name: "a", IsFolder: true}, + }, nil + } + return []model.Obj{&model.Object{Name: "pack", Size: 1}}, nil + }, + } + + got, err := b.listPage(context.Background(), "bucket", "data", "", false, gofakes3.ListBucketPage{ + Marker: "data/b/pack", + HasMarker: true, + MaxKeys: 1, + }) + if err != nil { + t.Fatalf("listPage() error = %v", err) + } + if len(got.Contents) != 1 || got.Contents[0].Key != "data/c/pack" { + t.Fatalf("contents = %#v, want data/c/pack", got.Contents) + } + for _, call := range calls { + if call == "bucket/data/a" { + t.Fatalf("completed subtree was listed: calls = %v", calls) + } + } +} + +func TestListPageUsesS3MaximumAndProbesForTruncation(t *testing.T) { + b := &s3Backend{ + listDir: func(_ context.Context, _ string) ([]model.Obj, error) { + entries := make([]model.Obj, maxListPageKeys+1) + for i := range entries { + entries[i] = &model.Object{Name: fmt.Sprintf("pack-%04d", i), Size: 1} + } + return entries, nil + }, + } + + got, err := b.listPage(context.Background(), "bucket", "data", "", false, gofakes3.ListBucketPage{MaxKeys: 5000}) + if err != nil { + t.Fatalf("listPage() error = %v", err) + } + if len(got.Contents) != int(maxListPageKeys) { + t.Fatalf("len(Contents) = %d, want %d", len(got.Contents), maxListPageKeys) + } + if !got.IsTruncated || got.NextMarker != "data/pack-0999" { + t.Fatalf("truncated = %v, next marker = %q", got.IsTruncated, got.NextMarker) + } +} + +func TestListPageDoesNotReturnEmptyDirectoryMarkerForDifferentPrefix(t *testing.T) { + b := &s3Backend{ + listDir: func(_ context.Context, _ string) ([]model.Obj, error) { + return nil, nil + }, + } + + got, err := b.listPage(context.Background(), "bucket", "empty", "not-the-marker", false, gofakes3.ListBucketPage{MaxKeys: 10}) + if err != nil { + t.Fatalf("listPage() error = %v", err) + } + if len(got.Contents) != 0 { + t.Fatalf("contents = %#v, want empty", got.Contents) + } +} + +func TestListPageHonorsCanceledContext(t *testing.T) { + listCalls := 0 + b := &s3Backend{ + listDir: func(_ context.Context, _ string) ([]model.Obj, error) { + listCalls++ + return nil, nil + }, + } + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + _, err := b.listPage(ctx, "bucket", "data", "", false, gofakes3.ListBucketPage{MaxKeys: 10}) + if !errors.Is(err, context.Canceled) { + t.Fatalf("listPage() error = %v, want context.Canceled", err) + } + if listCalls != 0 { + t.Fatalf("list calls = %d, want 0", listCalls) + } +} + +func TestListPageStopsWhenDirectoryReadCancelsContext(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + b := &s3Backend{ + listDir: func(_ context.Context, _ string) ([]model.Obj, error) { + cancel() + return []model.Obj{&model.Object{Name: "pack", Size: 1}}, nil + }, + } + + _, err := b.listPage(ctx, "bucket", "data", "", false, gofakes3.ListBucketPage{MaxKeys: 10}) + if !errors.Is(err, context.Canceled) { + t.Fatalf("listPage() error = %v, want context.Canceled", err) + } +} + +func TestListPageDoesNotReorderDirectoryCacheEntries(t *testing.T) { + entries := []model.Obj{ + &model.Object{Name: "c", Size: 1}, + &model.Object{Name: "a", Size: 1}, + &model.Object{Name: "b", Size: 1}, + } + b := &s3Backend{ + listDir: func(_ context.Context, _ string) ([]model.Obj, error) { + return entries, nil + }, + } + + if _, err := b.listPage(context.Background(), "bucket", "data", "", false, gofakes3.ListBucketPage{MaxKeys: 10}); err != nil { + t.Fatalf("listPage() error = %v", err) + } + wantOrder := []string{"c", "a", "b"} + for i, want := range wantOrder { + if entries[i].GetName() != want { + t.Fatalf("entry %d = %q, want original order %q", i, entries[i].GetName(), want) + } + } +} + +func TestListPageMixedDirectoryAndFilePaginatesInKeyOrder(t *testing.T) { + tree := map[string][]model.Obj{ + "bucket/data": { + &model.Object{Name: "c", Size: 1}, + &model.Object{Name: "b", IsFolder: true}, + &model.Object{Name: "b.txt", Size: 1}, + &model.Object{Name: "b-c", IsFolder: true}, + &model.Object{Name: "a", Size: 1}, + }, + "bucket/data/b": {&model.Object{Name: "y", Size: 1}, &model.Object{Name: "x", Size: 1}}, + "bucket/data/b-c": {&model.Object{Name: "k", Size: 1}}, + } + b := &s3Backend{ + listDir: func(_ context.Context, dir string) ([]model.Obj, error) { + return tree[dir], nil + }, + } + tests := []struct { + name string + delimiter bool + want []string + }{ + {"delimiter", true, []string{"data/a", "data/b-c/", "data/b.txt", "data/b/", "data/c"}}, + {"recursive", false, []string{"data/a", "data/b-c/k", "data/b.txt", "data/b/x", "data/b/y", "data/c"}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if !sort.StringsAreSorted(tt.want) { + t.Fatalf("expectation is not in key order: %v", tt.want) + } + for maxKeys := int64(1); maxKeys <= int64(len(tt.want))+1; maxKeys++ { + var got []string + page := gofakes3.ListBucketPage{MaxKeys: maxKeys} + for i := 0; i <= len(tt.want); i++ { + list, err := b.listPage(context.Background(), "bucket", "data", "", tt.delimiter, page) + if err != nil { + t.Fatalf("MaxKeys=%d: listPage() error = %v", maxKeys, err) + } + var keys []string + for _, item := range list.Contents { + keys = append(keys, item.Key) + } + for _, prefix := range list.CommonPrefixes { + if !strings.HasSuffix(prefix.Prefix, "/") { + t.Fatalf("common prefix %q does not end with /", prefix.Prefix) + } + keys = append(keys, prefix.Prefix) + } + sort.Strings(keys) + got = append(got, keys...) + if !list.IsTruncated { + break + } + page.Marker, page.HasMarker = list.NextMarker, true + } + if !slices.Equal(got, tt.want) { + t.Fatalf("MaxKeys=%d: listing = %v, want %v", maxKeys, got, tt.want) + } + } + }) + } +} diff --git a/server/s3/logger.go b/server/s3/logger.go index daf0aec2..1715a400 100644 --- a/server/s3/logger.go +++ b/server/s3/logger.go @@ -6,7 +6,7 @@ import ( "fmt" "github.com/OpenListTeam/OpenList/v4/pkg/utils" - "github.com/itsHenry35/gofakes3" + "github.com/OpenListTeam/gofakes3" ) // logger output formatted message diff --git a/server/s3/multipart.go b/server/s3/multipart.go new file mode 100644 index 00000000..7261ea36 --- /dev/null +++ b/server/s3/multipart.go @@ -0,0 +1,383 @@ +// Credits: https://pkg.go.dev/github.com/rclone/rclone@v1.65.2/cmd/serve/s3 +// Package s3 implements a fake s3 server for openlist +package s3 + +import ( + "context" + "crypto/md5" + "encoding/hex" + "fmt" + "io" + "os" + "path/filepath" + "strings" + "sync" + "time" + + "github.com/google/uuid" + + "github.com/OpenListTeam/OpenList/v4/internal/conf" + "github.com/OpenListTeam/OpenList/v4/pkg/utils" + "github.com/OpenListTeam/gofakes3" + log "github.com/sirupsen/logrus" +) + +// Compile-time assertions that s3Backend implements both the base Backend and +// the optional MultipartBackend interface from gofakes3. +var ( + _ gofakes3.Backend = (*s3Backend)(nil) + _ gofakes3.MultipartBackend = (*s3Backend)(nil) +) + +// multipartPart records a single uploaded part on disk. +type multipartPart struct { + path string + size int64 + md5hex string // unquoted lowercase hex + updated time.Time +} + +// multipartState tracks one in-progress multipart upload. +// +// Concurrency: gofakes3 does not serialize operations for the same uploadID, +// so concurrent UploadPart/Complete/Abort calls may overlap. The parts map is +// protected by mu. Each part is written to its own file inside dir, so +// concurrent UploadPart calls for different part numbers are safe without +// additional locking. lastActivity is updated under mu on create and on each +// part upload so the reaper can make a consistent expiry decision. +type multipartState struct { + bucket string + object string + meta map[string]string + dir string + created time.Time + lastActivity time.Time // updated under mu on create and each part upload + + mu sync.Mutex + parts map[int]*multipartPart +} + +// CreateMultipartUpload begins a new multipart upload. Parts are streamed to +// local temp files so that large uploads do not need to be buffered in memory. +// +// It implements gofakes3.MultipartBackend. +func (b *s3Backend) CreateMultipartUpload(ctx context.Context, bucket, object string, meta map[string]string) (gofakes3.UploadID, error) { + if _, err := getBucketByName(bucket); err != nil { + return "", err + } + + tempDir := conf.Conf.TempDir + if tempDir == "" { + tempDir = os.TempDir() + } + dir, err := os.MkdirTemp(tempDir, "s3-multipart-*") + if err != nil { + return "", fmt.Errorf("create multipart upload dir: %w", err) + } + + uploadID := gofakes3.UploadID(strings.ReplaceAll(uuid.NewString(), "-", "")) + now := time.Now() + state := &multipartState{ + bucket: bucket, + object: object, + meta: meta, + dir: dir, + created: now, + lastActivity: now, + parts: map[int]*multipartPart{}, + } + + b.uploads.Store(uploadID, state) + log.Debugf("s3 multipart: created upload %s for %s/%s", uploadID, bucket, object) + return uploadID, nil +} + +// UploadPart writes a single part to disk and returns its (quoted) MD5 etag. +// +// It implements gofakes3.MultipartBackend. The body must contain exactly +// contentLength bytes; a short read is reported as ErrIncompleteBody so a +// truncated client request is never silently stored. +func (b *s3Backend) UploadPart(ctx context.Context, bucket, object string, uploadID gofakes3.UploadID, partNumber int, contentLength int64, body io.Reader) (string, error) { + if partNumber <= 0 || partNumber > gofakes3.MaxUploadPartNumber { + return "", gofakes3.ErrInvalidPart + } + + val, ok := b.uploads.Load(uploadID) + if !ok { + return "", gofakes3.ErrNoSuchUpload + } + state := val.(*multipartState) + + partPath := filepath.Join(state.dir, fmt.Sprintf("part-%05d", partNumber)) + f, err := os.Create(partPath) + if err != nil { + return "", fmt.Errorf("create part file: %w", err) + } + // Remove a half-written file on any failure path. + partFailed := true + defer func() { + if partFailed { + _ = f.Close() + _ = os.Remove(partPath) + } + }() + + hash := md5.New() + // io.TeeReader feeds the hasher while the part is streamed to disk, so the + // etag costs no extra pass over the data. + n, err := utils.CopyWithBuffer(io.MultiWriter(f, hash), io.LimitReader(body, contentLength)) + if err != nil { + return "", fmt.Errorf("write part %d: %w", partNumber, err) + } + if err := f.Close(); err != nil { + return "", fmt.Errorf("close part %d: %w", partNumber, err) + } + if n != contentLength { + // The client under-delivered (truncated/aborted request). gofakes3 does + // not validate this for streaming backends, so we must. + return "", gofakes3.ErrIncompleteBody + } + + md5hex := hex.EncodeToString(hash.Sum(nil)) + etag := fmt.Sprintf("%q", md5hex) + + now := time.Now() + state.mu.Lock() + if old := state.parts[partNumber]; old != nil && old.path != partPath { + _ = os.Remove(old.path) + } + state.parts[partNumber] = &multipartPart{ + path: partPath, + size: n, + md5hex: md5hex, + updated: now, + } + state.lastActivity = now + state.mu.Unlock() + + partFailed = false + log.Debugf("s3 multipart: stored part %d for %s (%d bytes)", partNumber, uploadID, n) + return etag, nil +} + +// CompleteMultipartUpload assembles the uploaded parts in ascending part-number +// order and streams the result into storage via the shared putStream path. +// +// It implements gofakes3.MultipartBackend. Part ordering and etags are +// validated against the parts actually received. +func (b *s3Backend) CompleteMultipartUpload(ctx context.Context, bucket, object string, uploadID gofakes3.UploadID, input *gofakes3.CompleteMultipartUploadRequest) (gofakes3.VersionID, string, error) { + val, ok := b.uploads.Load(uploadID) + if !ok { + return "", "", gofakes3.ErrNoSuchUpload + } + state := val.(*multipartState) + + if input == nil || len(input.Parts) == 0 { + return "", "", gofakes3.ErrorMessagef(gofakes3.ErrMalformedXML, "complete multipart upload has no parts") + } + // S3 requires the parts in a CompleteMultipartUpload request to be listed + // in ascending part-number order. + for i := 1; i < len(input.Parts); i++ { + if input.Parts[i].PartNumber <= input.Parts[i-1].PartNumber { + return "", "", gofakes3.ErrInvalidPartOrder + } + } + + // Validate every requested part exists with a matching etag, and collect + // them in the order requested by the client (which is sorted ascending). + state.mu.Lock() + ordered := make([]*multipartPart, 0, len(input.Parts)) + var concat []byte + for _, p := range input.Parts { + stored := state.parts[p.PartNumber] + if stored == nil { + state.mu.Unlock() + return "", "", gofakes3.ErrorMessagef(gofakes3.ErrInvalidPart, "unexpected part number %d in complete request", p.PartNumber) + } + if strings.Trim(p.ETag, "\"") != stored.md5hex { + state.mu.Unlock() + return "", "", gofakes3.ErrorMessagef(gofakes3.ErrInvalidPart, "unexpected part etag for number %d in complete request", p.PartNumber) + } + ordered = append(ordered, stored) + // S3 multipart etag = hex(md5(concat(part_md5_digests)))-N + concat = append(concat, stored.md5Bytes()...) + } + // Hold the lock until the part files are opened so an abort racing with + // complete cannot delete them out from under us. + readers := make([]io.Reader, 0, len(ordered)) + closers := make([]io.Closer, 0, len(ordered)) + var total int64 + for _, part := range ordered { + f, err := os.Open(part.path) + if err != nil { + for _, c := range closers { + _ = c.Close() + } + state.mu.Unlock() + return "", "", fmt.Errorf("open part %s: %w", part.path, err) + } + readers = append(readers, f) + closers = append(closers, f) + total += part.size + } + state.mu.Unlock() + + combined := utils.NewReadCloser(io.MultiReader(readers...), func() error { + var firstErr error + for _, c := range closers { + if err := c.Close(); err != nil && firstErr == nil { + firstErr = err + } + } + return firstErr + }) + + defer combined.Close() + + err := b.putStream(ctx, bucket, object, state.meta, combined, total) + if err != nil { + // Leave the upload in place so the client may retry completion, per + // the gofakes3 MultipartBackend contract. + return "", "", err + } + + // Success: drop bookkeeping and clean up part files. + b.removeUpload(uploadID) + + sum := md5.Sum(concat) + etag := fmt.Sprintf("%q", fmt.Sprintf("%s-%d", hex.EncodeToString(sum[:]), len(ordered))) + log.Debugf("s3 multipart: completed upload %s -> %s/%s (%d bytes)", uploadID, bucket, object, total) + return "", etag, nil +} + +// AbortMultipartUpload discards an in-progress upload and its parts. +// +// It implements gofakes3.MultipartBackend and is idempotent: aborting an +// unknown upload succeeds so retries do not fail. +func (b *s3Backend) AbortMultipartUpload(ctx context.Context, bucket, object string, uploadID gofakes3.UploadID) error { + b.removeUpload(uploadID) + return nil +} + +// removeUpload deletes the upload's temp directory and drops its bookkeeping. +// Missing uploads are ignored to keep abort/complete idempotent. +func (b *s3Backend) removeUpload(uploadID gofakes3.UploadID) { + val, ok := b.uploads.LoadAndDelete(uploadID) + if !ok { + return + } + state := val.(*multipartState) + if state.dir != "" { + if err := os.RemoveAll(state.dir); err != nil { + log.Warnf("s3 multipart: failed to clean up %s: %v", state.dir, err) + } + } +} + +// md5Bytes returns the raw 16-byte MD5 digest of the part. +func (p *multipartPart) md5Bytes() []byte { + b, _ := hex.DecodeString(p.md5hex) + return b +} + +// Defaults for reaping abandoned multipart uploads. A client that never sends +// CompleteMultipartUpload or AbortMultipartUpload would otherwise leave part +// files on disk forever; the reaper drops uploads inactive for longer than the +// TTL. +const ( + defaultMultipartTTL = 24 * time.Hour + multipartDirPrefix = "s3-multipart-" +) + +// multipartTTL returns the configured max idle time for an upload before the +// reaper reclaims it. It parses conf.Conf.S3.MultipartTTL as a Go duration +// (e.g. "24h", "30m"); an empty or invalid value falls back to the default. +func multipartTTL() time.Duration { + if v := conf.Conf.S3.MultipartTTL; v != "" { + if d, err := time.ParseDuration(v); err == nil && d > 0 { + return d + } + } + return defaultMultipartTTL +} + +// reapInterval derives the reaper tick interval from the TTL: a quarter of the +// TTL, clamped to [10s, 1h]. +func reapInterval(ttl time.Duration) time.Duration { + d := ttl / 4 + if d < 10*time.Second { + d = 10 * time.Second + } + if d > time.Hour { + d = time.Hour + } + return d +} + +// startReaper removes leftover part directories from a previous process crash +// and then launches a background goroutine that periodically reclaims uploads +// inactive for longer than the TTL. The goroutine runs for the lifetime of the +// process; NewServer is called once at startup, so there is one reaper per +// backend instance. +func (b *s3Backend) startReaper() { + b.cleanupStaleDirs(time.Now(), multipartTTL()) + interval := reapInterval(multipartTTL()) + go func() { + ticker := time.NewTicker(interval) + defer ticker.Stop() + for now := range ticker.C { + b.reapExpired(now, multipartTTL()) + } + }() +} + +// reapExpired removes uploads whose lastActivity is older than ttl. It is safe +// to call concurrently with UploadPart/Complete/Abort: each candidate is +// re-checked under its own lock and removed atomically via removeUpload. +func (b *s3Backend) reapExpired(now time.Time, ttl time.Duration) { + b.uploads.Range(func(key, val any) bool { + state := val.(*multipartState) + state.mu.Lock() + expired := now.Sub(state.lastActivity) > ttl + state.mu.Unlock() + if !expired { + return true + } + b.removeUpload(key.(gofakes3.UploadID)) + log.Infof("s3 multipart: reaped abandoned upload %s (%s/%s)", key, state.bucket, state.object) + return true + }) +} + +// cleanupStaleDirs removes s3-multipart-* directories under TempDir that are +// older than ttl. This reclaims part files left behind by a previous process +// crash; dirs younger than ttl are left alone so a concurrently-starting +// sibling backend instance is never disturbed. +func (b *s3Backend) cleanupStaleDirs(now time.Time, ttl time.Duration) { + tempDir := conf.Conf.TempDir + if tempDir == "" { + tempDir = os.TempDir() + } + entries, err := os.ReadDir(tempDir) + if err != nil { + return + } + cutoff := now.Add(-ttl) + for _, e := range entries { + if !e.IsDir() || !strings.HasPrefix(e.Name(), multipartDirPrefix) { + continue + } + info, err := e.Info() + if err != nil { + continue + } + if info.ModTime().After(cutoff) { + continue + } + if err := os.RemoveAll(filepath.Join(tempDir, e.Name())); err != nil { + log.Warnf("s3 multipart: failed to clean up stale dir %s: %v", e.Name(), err) + } else { + log.Infof("s3 multipart: removed stale multipart dir %s", e.Name()) + } + } +} diff --git a/server/s3/multipart_test.go b/server/s3/multipart_test.go new file mode 100644 index 00000000..a7349583 --- /dev/null +++ b/server/s3/multipart_test.go @@ -0,0 +1,332 @@ +package s3 + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "os" + "path/filepath" + "strings" + "testing" + "time" + + _ "github.com/OpenListTeam/OpenList/v4/drivers/local" + "github.com/OpenListTeam/OpenList/v4/internal/conf" + "github.com/OpenListTeam/OpenList/v4/internal/db" + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/internal/op" + + "github.com/OpenListTeam/gofakes3" + "github.com/glebarez/sqlite" + "gorm.io/gorm" +) + +func init() { + dataDir, err := os.MkdirTemp("", "openlist-s3-mp-*") + if err != nil { + panic(err) + } + conf.Conf = conf.DefaultConfig(dataDir) + if err := os.MkdirAll(conf.Conf.TempDir, 0o755); err != nil { + panic("mkdir temp dir: " + err.Error()) + } + dB, err := gorm.Open(sqlite.Open("file::memory:?cache=shared"), &gorm.Config{}) + if err != nil { + panic("failed to connect database: " + err.Error()) + } + db.Init(dB) +} + +// s3ErrorCode extracts the gofakes3 ErrorCode from an error returned by the +// MultipartBackend methods. +func s3ErrorCode(err error) gofakes3.ErrorCode { + if err == nil { + return gofakes3.ErrNone + } + var s3err interface{ ErrorCode() gofakes3.ErrorCode } + if errors.As(err, &s3err) { + return s3err.ErrorCode() + } + return gofakes3.ErrNone +} + +// setupMultipartBackend prepares a Local storage mounted at /mpbucket and an +// s3Backend with an "mp" bucket pointing at it. It returns the backend, the +// local root directory on disk, and a cleanup function. +func setupMultipartBackend(t *testing.T) (*s3Backend, string) { + t.Helper() + ctx := context.Background() + + // Unique mount path and bucket per test: the in-memory sqlite is shared + // across tests in this package, so a fixed mount path would clash. + mount := "/" + sanitizeTestName(t.Name()) + bucket := "mp" + + localRoot, err := os.MkdirTemp("", "openlist-s3-local-*") + if err != nil { + t.Fatalf("mkdir local root: %v", err) + } + t.Cleanup(func() { _ = os.RemoveAll(localRoot) }) + addition, err := json.Marshal(struct { + RootFolderPath string `json:"root_folder_path"` + Thumbnail bool `json:"thumbnail"` + }{ + RootFolderPath: localRoot, + Thumbnail: false, + }) + if err != nil { + t.Fatalf("marshal local storage addition: %v", err) + } + + _, err = op.CreateStorage(ctx, model.Storage{ + Driver: "Local", + MountPath: mount, + Addition: string(addition), + }) + if err != nil { + t.Fatalf("create local storage: %+v", err) + } + + if err := op.SaveSettingItem(&model.SettingItem{ + Key: conf.S3Buckets, + Value: `[{"name":"` + bucket + `","path":"` + mount + `"}]`, + }); err != nil { + t.Fatalf("save s3 buckets setting: %+v", err) + } + + return newBackend().(*s3Backend), localRoot +} + +func sanitizeTestName(name string) string { + r := strings.NewReplacer("/", "_", " ", "_") + return r.Replace(name) +} + +func TestMultipartUploadEndToEnd(t *testing.T) { + ctx := context.Background() + b, localRoot := setupMultipartBackend(t) + + meta := map[string]string{"Content-Type": "text/plain"} + uploadID, err := b.CreateMultipartUpload(ctx, "mp", "dir/hello.txt", meta) + if err != nil { + t.Fatalf("CreateMultipartUpload: %+v", err) + } + if uploadID == "" { + t.Fatal("empty upload id") + } + + part := func(n int, body string) string { + t.Helper() + etag, err := b.UploadPart(ctx, "mp", "dir/hello.txt", uploadID, n, int64(len(body)), strings.NewReader(body)) + if err != nil { + t.Fatalf("UploadPart %d: %+v", n, err) + } + return etag + } + + etag1 := part(1, "Hello, ") + etag2 := part(2, "multipart ") + etag3 := part(3, "world!") + + // Re-uploading the same part number overwrites it and returns a fresh etag. + if e := part(2, "multipart "); e != etag2 { + t.Fatalf("re-upload part 2 etag = %q, want %q", e, etag2) + } + + // Short read (body smaller than declared Content-Length) must fail. + _, shortErr := b.UploadPart(ctx, "mp", "dir/hello.txt", uploadID, 4, 10, strings.NewReader("abc")) + shortErr = s3ErrorCode(shortErr) + if shortErr != gofakes3.ErrIncompleteBody { + t.Fatalf("short read error = %v, want IncompleteBody", shortErr) + } + + // Unknown upload id. + _, uerr := b.UploadPart(ctx, "mp", "dir/hello.txt", "does-not-exist", 1, 1, strings.NewReader("x")) + if code := s3ErrorCode(uerr); code != gofakes3.ErrNoSuchUpload { + t.Fatalf("unknown upload error = %v, want NoSuchUpload", code) + } + // Out-of-range part number. + _, perr := b.UploadPart(ctx, "mp", "dir/hello.txt", uploadID, 0, 1, strings.NewReader("x")) + if code := s3ErrorCode(perr); code != gofakes3.ErrInvalidPart { + t.Fatalf("part 0 error = %v, want InvalidPart", code) + } + + // Parts out of order. + _, _, err = b.CompleteMultipartUpload(ctx, "mp", "dir/hello.txt", uploadID, &gofakes3.CompleteMultipartUploadRequest{ + Parts: []gofakes3.CompletedPart{ + {PartNumber: 2, ETag: etag2}, + {PartNumber: 1, ETag: etag1}, + }, + }) + if code := s3ErrorCode(err); code != gofakes3.ErrInvalidPartOrder { + t.Fatalf("out-of-order complete error = %v, want InvalidPartOrder", code) + } + + // Wrong etag. + _, _, err = b.CompleteMultipartUpload(ctx, "mp", "dir/hello.txt", uploadID, &gofakes3.CompleteMultipartUploadRequest{ + Parts: []gofakes3.CompletedPart{ + {PartNumber: 1, ETag: etag1}, + {PartNumber: 2, ETag: `"deadbeef"`}, + {PartNumber: 3, ETag: etag3}, + }, + }) + if code := s3ErrorCode(err); code != gofakes3.ErrInvalidPart { + t.Fatalf("wrong etag complete error = %v, want InvalidPart", code) + } + + // Missing part number. + _, _, err = b.CompleteMultipartUpload(ctx, "mp", "dir/hello.txt", uploadID, &gofakes3.CompleteMultipartUploadRequest{ + Parts: []gofakes3.CompletedPart{ + {PartNumber: 1, ETag: etag1}, + {PartNumber: 99, ETag: etag3}, + }, + }) + if code := s3ErrorCode(err); code != gofakes3.ErrInvalidPart { + t.Fatalf("missing part complete error = %v, want InvalidPart", code) + } + + // The failed completes must leave the upload available for retry. + if _, ok := b.uploads.Load(uploadID); !ok { + t.Fatal("upload was removed after a failed complete") + } + + // Successful complete. + _, etag, err := b.CompleteMultipartUpload(ctx, "mp", "dir/hello.txt", uploadID, &gofakes3.CompleteMultipartUploadRequest{ + Parts: []gofakes3.CompletedPart{ + {PartNumber: 1, ETag: etag1}, + {PartNumber: 2, ETag: etag2}, + {PartNumber: 3, ETag: etag3}, + }, + }) + if err != nil { + t.Fatalf("CompleteMultipartUpload: %+v", err) + } + if !strings.HasSuffix(etag, `-3"`) || !strings.HasPrefix(etag, `"`) { + t.Fatalf("complete etag = %q, want a quoted \"-3\" multipart etag", etag) + } + + // The object must exist on disk with the concatenated content. + got, err := os.ReadFile(filepath.Join(localRoot, "dir", "hello.txt")) + if err != nil { + t.Fatalf("read resulting file: %+v", err) + } + want := []byte("Hello, multipart world!") + if !bytes.Equal(got, want) { + t.Fatalf("resulting file content = %q, want %q", got, want) + } + + // Bookkeeping and temp files must be cleaned up on success. + if _, ok := b.uploads.Load(uploadID); ok { + t.Fatal("upload still tracked after successful complete") + } +} + +func TestMultipartAbort(t *testing.T) { + ctx := context.Background() + b, _ := setupMultipartBackend(t) + + uploadID, err := b.CreateMultipartUpload(ctx, "mp", "abort.txt", nil) + if err != nil { + t.Fatalf("CreateMultipartUpload: %+v", err) + } + if _, err := b.UploadPart(ctx, "mp", "abort.txt", uploadID, 1, 3, strings.NewReader("abc")); err != nil { + t.Fatalf("UploadPart: %+v", err) + } + + state, _ := b.uploads.Load(uploadID) + dir := state.(*multipartState).dir + if _, err := os.Stat(dir); err != nil { + t.Fatalf("temp dir missing before abort: %+v", err) + } + + if err := b.AbortMultipartUpload(ctx, "mp", "abort.txt", uploadID); err != nil { + t.Fatalf("AbortMultipartUpload: %+v", err) + } + if _, ok := b.uploads.Load(uploadID); ok { + t.Fatal("upload still tracked after abort") + } + if _, err := os.Stat(dir); !os.IsNotExist(err) { + t.Fatalf("temp dir still exists after abort (err=%v)", err) + } + + // Aborting an unknown upload must be idempotent. + if err := b.AbortMultipartUpload(ctx, "mp", "abort.txt", "nope"); err != nil { + t.Fatalf("abort unknown upload returned error: %+v", err) + } +} + +func TestMultipartReapExpired(t *testing.T) { + ctx := context.Background() + b, _ := setupMultipartBackend(t) + + // An active upload (fresh lastActivity) must be kept. + freshID, err := b.CreateMultipartUpload(ctx, "mp", "fresh.txt", nil) + if err != nil { + t.Fatalf("create fresh upload: %+v", err) + } + + // An abandoned upload (stale lastActivity) must be reaped. + staleID, err := b.CreateMultipartUpload(ctx, "mp", "stale.txt", nil) + if err != nil { + t.Fatalf("create stale upload: %+v", err) + } + if _, err := b.UploadPart(ctx, "mp", "stale.txt", staleID, 1, 3, strings.NewReader("abc")); err != nil { + t.Fatalf("upload stale part: %+v", err) + } + staleState, _ := b.uploads.Load(staleID) + staleDir := staleState.(*multipartState).dir + if _, err := os.Stat(staleDir); err != nil { + t.Fatalf("stale temp dir missing: %+v", err) + } + + // Force the stale upload's lastActivity well into the past. + ttl := 30 * time.Minute + now := time.Now() + staleState.(*multipartState).mu.Lock() + staleState.(*multipartState).lastActivity = now.Add(-2 * ttl) + staleState.(*multipartState).mu.Unlock() + + b.reapExpired(now, ttl) + + if _, ok := b.uploads.Load(staleID); ok { + t.Fatal("stale upload still tracked after reap") + } + if _, err := os.Stat(staleDir); !os.IsNotExist(err) { + t.Fatalf("stale temp dir still exists after reap (err=%v)", err) + } + if _, ok := b.uploads.Load(freshID); !ok { + t.Fatal("fresh upload was reaped, should have been kept") + } +} + +func TestMultipartCleanupStaleDirs(t *testing.T) { + b, _ := setupMultipartBackend(t) + + tempDir := conf.Conf.TempDir + staleDir, err := os.MkdirTemp(tempDir, multipartDirPrefix+"*") + if err != nil { + t.Fatalf("mkdir stale dir: %v", err) + } + freshDir, err := os.MkdirTemp(tempDir, multipartDirPrefix+"*") + if err != nil { + t.Fatalf("mkdir fresh dir: %v", err) + } + // Age the stale dir beyond the TTL; leave the fresh dir young. + ttl := 30 * time.Minute + now := time.Now() + past := now.Add(-2 * ttl) + if err := os.Chtimes(staleDir, past, past); err != nil { + t.Fatalf("chtimes stale dir: %v", err) + } + + b.cleanupStaleDirs(now, ttl) + + if _, err := os.Stat(staleDir); !os.IsNotExist(err) { + t.Fatalf("stale dir should have been removed (err=%v)", err) + } + if _, err := os.Stat(freshDir); err != nil { + t.Fatalf("fresh dir should have been kept (err=%v)", err) + } + _ = os.RemoveAll(freshDir) +} diff --git a/server/s3/pager.go b/server/s3/pager.go deleted file mode 100644 index 26824297..00000000 --- a/server/s3/pager.go +++ /dev/null @@ -1,67 +0,0 @@ -// Credits: https://pkg.go.dev/github.com/rclone/rclone@v1.65.2/cmd/serve/s3 -// Package s3 implements a fake s3 server for openlist -package s3 - -import ( - "sort" - - "github.com/itsHenry35/gofakes3" -) - -// pager splits the object list into smulitply pages. -func (db *s3Backend) pager(list *gofakes3.ObjectList, page gofakes3.ListBucketPage) (*gofakes3.ObjectList, error) { - // sort by alphabet - sort.Slice(list.CommonPrefixes, func(i, j int) bool { - return list.CommonPrefixes[i].Prefix < list.CommonPrefixes[j].Prefix - }) - // sort by modtime - sort.Slice(list.Contents, func(i, j int) bool { - return list.Contents[i].LastModified.Before(list.Contents[j].LastModified.Time) - }) - tokens := page.MaxKeys - if tokens == 0 { - tokens = 1000 - } - if page.HasMarker { - for i, obj := range list.Contents { - if obj.Key == page.Marker { - list.Contents = list.Contents[i+1:] - break - } - } - for i, obj := range list.CommonPrefixes { - if obj.Prefix == page.Marker { - list.CommonPrefixes = list.CommonPrefixes[i+1:] - break - } - } - } - - response := gofakes3.NewObjectList() - for _, obj := range list.CommonPrefixes { - if tokens <= 0 { - break - } - response.AddPrefix(obj.Prefix) - tokens-- - } - - for _, obj := range list.Contents { - if tokens <= 0 { - break - } - response.Add(obj) - tokens-- - } - - if len(list.CommonPrefixes)+len(list.Contents) > int(page.MaxKeys) { - response.IsTruncated = true - if len(response.Contents) > 0 { - response.NextMarker = response.Contents[len(response.Contents)-1].Key - } else { - response.NextMarker = response.CommonPrefixes[len(response.CommonPrefixes)-1].Prefix - } - } - - return response, nil -} diff --git a/server/s3/range.go b/server/s3/range.go new file mode 100644 index 00000000..7cc3f1cb --- /dev/null +++ b/server/s3/range.go @@ -0,0 +1,47 @@ +package s3 + +import "net/http" + +// gofakes3 sets Content-Range without selecting the partial-content status. +func rangeStatusHandler(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet || r.Header.Get("Range") == "" { + next.ServeHTTP(w, r) + return + } + rw := &rangeResponseWriter{ResponseWriter: w} + next.ServeHTTP(rw, r) + if !rw.wroteHeader { + rw.WriteHeader(http.StatusOK) + } + }) +} + +type rangeResponseWriter struct { + http.ResponseWriter + wroteHeader bool +} + +func (w *rangeResponseWriter) WriteHeader(status int) { + if w.wroteHeader { + return + } + if status >= 200 { + w.wroteHeader = true + } + if status == http.StatusOK && w.Header().Get("Content-Range") != "" { + status = http.StatusPartialContent + } + w.ResponseWriter.WriteHeader(status) +} + +func (w *rangeResponseWriter) Write(p []byte) (int, error) { + if !w.wroteHeader { + w.WriteHeader(http.StatusOK) + } + return w.ResponseWriter.Write(p) +} + +func (w *rangeResponseWriter) Unwrap() http.ResponseWriter { + return w.ResponseWriter +} diff --git a/server/s3/range_test.go b/server/s3/range_test.go new file mode 100644 index 00000000..9d1a7683 --- /dev/null +++ b/server/s3/range_test.go @@ -0,0 +1,55 @@ +package s3 + +import ( + "io" + "net/http" + "net/http/httptest" + "testing" +) + +func TestRangeStatusHandler(t *testing.T) { + for _, tt := range []struct { + name string + method string + rangeHeader string + contentRange string + status int + body string + want int + }{ + {"partial", "GET", "bytes=2-5", "bytes 2-5/16", 0, "2345", 206}, + {"explicit OK", "GET", "bytes=2-5", "bytes 2-5/16", 200, "2345", 206}, + {"already partial", "GET", "bytes=2-5", "bytes 2-5/16", 206, "2345", 206}, + {"full", "GET", "", "", 0, "0123456789abcdef", 200}, + {"range ignored", "GET", "bytes=2-5", "", 0, "0123456789abcdef", 200}, + {"not modified", "GET", "bytes=2-5", "", 304, "", 304}, + {"invalid range", "GET", "bytes=20-", "bytes */16", 416, "error", 416}, + {"forbidden", "GET", "bytes=2-5", "", 403, "error", 403}, + {"empty body", "GET", "bytes=2-5", "bytes 2-5/16", 0, "", 206}, + {"head", "HEAD", "bytes=2-5", "", 0, "", 200}, + } { + t.Run(tt.name, func(t *testing.T) { + next := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if tt.contentRange != "" { + w.Header().Set("Content-Range", tt.contentRange) + } + if tt.status != 0 { + w.WriteHeader(tt.status) + } + if tt.body != "" { + _, _ = io.WriteString(w, tt.body) + } + }) + r := httptest.NewRequest(tt.method, "/bucket/object", nil) + r.Header.Set("Range", tt.rangeHeader) + w := httptest.NewRecorder() + rangeStatusHandler(next).ServeHTTP(w, r) + if w.Code != tt.want { + t.Fatalf("status = %d, want %d", w.Code, tt.want) + } + if w.Body.String() != tt.body || w.Header().Get("Content-Range") != tt.contentRange { + t.Fatalf("response changed: body=%q, range=%q", w.Body.String(), w.Header().Get("Content-Range")) + } + }) + } +} diff --git a/server/s3/redirect.go b/server/s3/redirect.go index 4031e171..6d8d430b 100644 --- a/server/s3/redirect.go +++ b/server/s3/redirect.go @@ -14,7 +14,7 @@ import ( "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/OpenListTeam/OpenList/v4/server/common" - "github.com/itsHenry35/gofakes3/signature" + "github.com/OpenListTeam/gofakes3/signature" ) func redirectHandler(next http.Handler, authPairs map[string]string) http.Handler { diff --git a/server/s3/server.go b/server/s3/server.go index d33b63dd..f1ed5ba3 100644 --- a/server/s3/server.go +++ b/server/s3/server.go @@ -7,7 +7,7 @@ import ( "math/rand" "net/http" - "github.com/itsHenry35/gofakes3" + "github.com/OpenListTeam/gofakes3" ) // Make a new S3 Server to serve the remote @@ -24,5 +24,5 @@ func NewServer(ctx context.Context) (h http.Handler, err error) { gofakes3.WithIntegrityCheck(true), // Check Content-MD5 if supplied ) - return redirectHandler(faker.Server(), authPairs), nil + return redirectHandler(rangeStatusHandler(faker.Server()), authPairs), nil } diff --git a/server/s3/utils.go b/server/s3/utils.go index eb3dd568..6624a07f 100644 --- a/server/s3/utils.go +++ b/server/s3/utils.go @@ -5,6 +5,7 @@ package s3 import ( "context" "encoding/json" + stderrors "errors" "strings" "github.com/OpenListTeam/OpenList/v4/internal/conf" @@ -13,7 +14,7 @@ import ( "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/OpenListTeam/OpenList/v4/internal/setting" - "github.com/itsHenry35/gofakes3" + "github.com/OpenListTeam/gofakes3" ) type Bucket struct { @@ -21,6 +22,13 @@ type Bucket struct { Path string `json:"path"` } +func mapBackendError(err error) error { + if stderrors.Is(err, errs.TemporaryCapacity) { + return gofakes3.ErrSlowDown + } + return err +} + const emptyObjectName = "ThisIsAnEmptyFolderInTheS3Bucket" func getAndParseBuckets() ([]Bucket, error) { @@ -42,10 +50,12 @@ func getBucketByName(name string) (Bucket, error) { return Bucket{}, gofakes3.BucketNotFound(name) } -func getDirEntries(path string) ([]model.Obj, error) { - ctx := context.Background() +func getDirEntries(ctx context.Context, path string) ([]model.Obj, error) { meta, _ := op.GetNearestMeta(path) fi, err := fs.Get(context.WithValue(ctx, conf.MetaKey, meta), path, &fs.GetArgs{}) + if ctxErr := ctx.Err(); ctxErr != nil { + return nil, ctxErr + } if errs.IsNotFoundError(err) { return nil, gofakes3.ErrNoSuchKey } else if err != nil { @@ -57,6 +67,9 @@ func getDirEntries(path string) ([]model.Obj, error) { } dirEntries, err := fs.List(context.WithValue(ctx, conf.MetaKey, meta), path, &fs.ListArgs{}) + if ctxErr := ctx.Err(); ctxErr != nil { + return nil, ctxErr + } if err != nil { return nil, err } diff --git a/server/webdav/file.go b/server/webdav/file.go index ea609973..ed4c3a70 100644 --- a/server/webdav/file.go +++ b/server/webdav/file.go @@ -6,9 +6,12 @@ package webdav import ( "context" + stderrors "errors" + "fmt" "net/http" "path" "path/filepath" + "strings" "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/errs" @@ -16,7 +19,10 @@ import ( "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/OpenListTeam/OpenList/v4/server/common" + "github.com/google/uuid" "github.com/pkg/errors" + log "github.com/sirupsen/logrus" + "gorm.io/gorm" ) // slashClean is equivalent to but slightly more efficient than @@ -28,6 +34,271 @@ func slashClean(name string) string { return path.Clean(name) } +func moveResource(ctx context.Context, src, dstDir, srcDir, srcName, dstName string) error { + if srcDir == dstDir { + return fs.Rename(ctx, src, dstName) + } + if _, err := fs.Move(context.WithValue(ctx, conf.NoTaskKey, struct{}{}), src, dstDir); err != nil { + return err + } + if srcName != dstName { + return fs.Rename(ctx, path.Join(dstDir, srcName), dstName) + } + return nil +} + +type fileMutation struct { + rollback func() error + finalize func() + backupPaths []string +} + +type fileBackup struct { + originalPath string + backupPath string + renamed bool +} + +func stagingContext(ctx context.Context) context.Context { + ctx = context.WithValue(ctx, conf.NoTaskKey, struct{}{}) + return context.WithValue(ctx, conf.SkipHookKey, struct{}{}) +} + +func removeIfExists(ctx context.Context, name string) error { + if _, err := fs.Get(ctx, name, &fs.GetArgs{}); err != nil { + if errs.IsObjectNotFound(err) { + return nil + } + return err + } + return fs.Remove(ctx, name) +} + +func cleanupAfterFailure(ctx context.Context, name string, cause error) error { + if err := removeIfExists(stagingContext(context.WithoutCancel(ctx)), name); err != nil { + log.WithError(err).Errorf("failed to clean up WebDAV staging path %s", name) + return fmt.Errorf("%w; failed to clean up %s: %w", cause, name, err) + } + return cause +} + +func newFileBackup(ctx context.Context, originalPath string, keepOriginal bool) (*fileBackup, error) { + backupPath := path.Join(path.Dir(originalPath), ".openlist-backup-"+uuid.NewString()) + backup := &fileBackup{originalPath: originalPath, backupPath: backupPath} + if !keepOriginal { + err := fs.Rename(stagingContext(ctx), originalPath, path.Base(backupPath)) + if err == nil { + backup.renamed = true + return backup, nil + } + if !errs.IsNotImplementError(err) && !errs.IsNotSupportError(err) { + return nil, errors.WithMessage(err, "failed to rename object into rollback backup") + } + } + if _, err := fs.CopyTo(stagingContext(ctx), originalPath, path.Dir(originalPath), path.Base(backupPath)); err != nil { + return nil, cleanupAfterFailure(ctx, backupPath, errors.WithMessage(err, "failed to copy object into rollback backup")) + } + return backup, nil +} + +func (b *fileBackup) restore(ctx context.Context) error { + if b == nil { + return nil + } + ctx = context.WithoutCancel(ctx) + if err := removeIfExists(ctx, b.originalPath); err != nil { + return err + } + if b.renamed { + if err := fs.Rename(ctx, b.backupPath, path.Base(b.originalPath)); err != nil { + return errors.WithMessagef(err, "failed to restore %s from %s", b.originalPath, b.backupPath) + } + return nil + } + if _, err := fs.CopyTo(context.WithValue(ctx, conf.NoTaskKey, struct{}{}), b.backupPath, path.Dir(b.originalPath), path.Base(b.originalPath)); err != nil { + return errors.WithMessagef(err, "failed to restore %s from %s", b.originalPath, b.backupPath) + } + if err := fs.Remove(stagingContext(ctx), b.backupPath); err != nil { + if !errs.IsObjectNotFound(err) { + log.Warnf("failed to remove restored WebDAV backup %s: %v", b.backupPath, err) + } + } + return nil +} + +func (b *fileBackup) discard(ctx context.Context) { + if b == nil { + return + } + if err := removeIfExists(stagingContext(context.WithoutCancel(ctx)), b.backupPath); err != nil { + log.Warnf("failed to remove WebDAV backup %s: %v", b.backupPath, err) + } +} + +func installStagedObject(ctx context.Context, stagePath, dstDir, dstName string) error { + if err := fs.Rename(ctx, stagePath, dstName); err == nil { + return nil + } else if !errs.IsNotImplementError(err) && !errs.IsNotSupportError(err) { + return err + } + copyCtx := context.WithValue(ctx, conf.NoTaskKey, struct{}{}) + if _, err := fs.CopyTo(copyCtx, stagePath, dstDir, dstName); err != nil { + return err + } + if err := fs.Remove(stagingContext(context.WithoutCancel(ctx)), stagePath); err != nil { + log.Warnf("failed to remove installed WebDAV staging object %s: %v", stagePath, err) + } + return nil +} + +func copyWithRollback(ctx context.Context, src, dstDir, dstName string, dstExisted, shallow bool) (*fileMutation, error) { + stagePath := path.Join(dstDir, ".openlist-copy-"+uuid.NewString()) + var err error + if shallow { + err = fs.MakeDir(stagingContext(ctx), stagePath) + } else { + _, err = fs.CopyTo(stagingContext(ctx), src, dstDir, path.Base(stagePath)) + } + if err != nil { + return nil, cleanupAfterFailure(ctx, stagePath, err) + } + dstPath := path.Join(dstDir, dstName) + var dstBackup *fileBackup + if dstExisted { + var err error + dstBackup, err = newFileBackup(ctx, dstPath, false) + if err != nil { + return nil, cleanupAfterFailure(ctx, stagePath, err) + } + if !dstBackup.renamed { + if err := removeIfExists(ctx, dstPath); err != nil { + mutation := copyMutation(ctx, dstPath, stagePath, dstBackup) + return mutation, errors.WithMessage(err, "failed to remove destination after creating rollback copy") + } + } + } + mutation := copyMutation(ctx, dstPath, stagePath, dstBackup) + if err := installStagedObject(ctx, stagePath, dstDir, dstName); err != nil { + return mutation, errors.WithMessage(err, "failed to install staged COPY") + } + return mutation, nil +} + +func copyMutation(ctx context.Context, dstPath, stagePath string, dstBackup *fileBackup) *fileMutation { + paths := []string{stagePath} + if dstBackup != nil { + paths = append(paths, dstBackup.backupPath) + } + return &fileMutation{ + backupPaths: paths, + rollback: func() error { + var rollbackErrs []error + if err := removeIfExists(context.WithoutCancel(ctx), dstPath); err != nil { + rollbackErrs = append(rollbackErrs, err) + } + if err := dstBackup.restore(ctx); err != nil { + rollbackErrs = append(rollbackErrs, err) + } + if err := removeIfExists(stagingContext(context.WithoutCancel(ctx)), stagePath); err != nil { + rollbackErrs = append(rollbackErrs, err) + } + return stderrors.Join(rollbackErrs...) + }, + finalize: func() { + if err := removeIfExists(stagingContext(context.WithoutCancel(ctx)), stagePath); err != nil { + log.Warnf("failed to remove WebDAV COPY staging object %s: %v", stagePath, err) + } + dstBackup.discard(ctx) + }, + } +} + +func moveWithRollback(ctx context.Context, src, dst, dstDir, srcDir, srcName, dstName string, dstExisted bool) (*fileMutation, error) { + // Keep a source copy so a partially completed cross-storage MOVE can be undone. + srcBackup, err := newFileBackup(ctx, src, true) + if err != nil { + return nil, err + } + var dstBackup *fileBackup + if dstExisted { + dstBackup, err = newFileBackup(ctx, dst, false) + if err != nil { + srcBackup.discard(ctx) + return nil, err + } + if !dstBackup.renamed { + if err := removeIfExists(ctx, dst); err != nil { + mutation := moveMutation(ctx, dst, dstDir, srcName, dstName, srcBackup, dstBackup) + return mutation, errors.WithMessage(err, "failed to remove destination after creating rollback copy") + } + } + } + mutation := moveMutation(ctx, dst, dstDir, srcName, dstName, srcBackup, dstBackup) + if err := moveResource(ctx, src, dstDir, srcDir, srcName, dstName); err != nil { + return mutation, err + } + return mutation, nil +} + +func moveMutation(ctx context.Context, dst, dstDir, srcName, dstName string, srcBackup, dstBackup *fileBackup) *fileMutation { + paths := []string{srcBackup.backupPath} + if dstBackup != nil { + paths = append(paths, dstBackup.backupPath) + } + return &fileMutation{ + backupPaths: paths, + rollback: func() error { + rollbackCtx := context.WithoutCancel(ctx) + var rollbackErrs []error + candidates := []string{dst} + if srcName != dstName { + candidates = append(candidates, path.Join(dstDir, srcName)) + } + for _, candidate := range candidates { + if err := removeIfExists(rollbackCtx, candidate); err != nil { + rollbackErrs = append(rollbackErrs, err) + } + } + if err := srcBackup.restore(rollbackCtx); err != nil { + rollbackErrs = append(rollbackErrs, err) + } + if err := dstBackup.restore(rollbackCtx); err != nil { + rollbackErrs = append(rollbackErrs, err) + } + return stderrors.Join(rollbackErrs...) + }, + finalize: func() { + srcBackup.discard(ctx) + dstBackup.discard(ctx) + }, + } +} + +func runDeadPropsFileMutation(ctx context.Context, update func(*gorm.DB) error, apply func() (*fileMutation, error)) error { + var mutation *fileMutation + err := withDeadPropsTransaction(ctx, func(tx *gorm.DB) error { + if err := update(tx); err != nil { + return err + } + var err error + mutation, err = apply() + return err + }) + if err != nil { + if mutation != nil && mutation.rollback != nil { + if rollbackErr := mutation.rollback(); rollbackErr != nil { + log.WithError(rollbackErr).Errorf("failed to compensate WebDAV file operation; backups may remain at %s", strings.Join(mutation.backupPaths, ", ")) + return fmt.Errorf("%w; filesystem compensation failed: %w", err, rollbackErr) + } + } + return err + } + if mutation != nil && mutation.finalize != nil { + mutation.finalize() + } + return nil +} + // moveFiles moves files and/or directories from src to dst. // Individual item permission checks are skipped for performance reasons. // @@ -55,21 +326,41 @@ func moveFiles(ctx context.Context, src, dst string, overwrite bool) (status int if !common.CanWrite(user, srcMeta, srcDir) || !common.CanWrite(user, dstMeta, dstDir) { return http.StatusForbidden, nil } - if srcDir == dstDir { - err = fs.Rename(ctx, src, dstName) - } else { - _, err = fs.Move(context.WithValue(ctx, conf.NoTaskKey, struct{}{}), src, dstDir) - if err != nil { - return http.StatusInternalServerError, err - } - if srcName != dstName { - err = fs.Rename(ctx, path.Join(dstDir, srcName), dstName) + if _, err = fs.Get(ctx, src, &fs.GetArgs{}); err != nil { + if errs.IsObjectNotFound(err) { + return http.StatusNotFound, err } + return http.StatusInternalServerError, err } + if dstDirInfo, err := fs.Get(ctx, dstDir, &fs.GetArgs{}); err != nil { + if errs.IsObjectNotFound(err) { + return http.StatusConflict, err + } + return http.StatusMethodNotAllowed, err + } else if !dstDirInfo.IsDir() { + return http.StatusConflict, nil + } + dstExisted := false + if _, err = fs.Get(ctx, dst, &fs.GetArgs{}); err == nil { + dstExisted = true + if !overwrite { + return http.StatusPreconditionFailed, nil + } + } else if !errs.IsObjectNotFound(err) { + return http.StatusInternalServerError, err + } + err = runDeadPropsFileMutation(ctx, + func(tx *gorm.DB) error { return moveDeadProps(tx, src, dst) }, + func() (*fileMutation, error) { + return moveWithRollback(ctx, src, dst, dstDir, srcDir, srcName, dstName, dstExisted && overwrite) + }, + ) if err != nil { return http.StatusInternalServerError, err } - // TODO if there are no files copy, should return 204 + if dstExisted { + return http.StatusNoContent, nil + } return http.StatusCreated, nil } @@ -77,9 +368,10 @@ func moveFiles(ctx context.Context, src, dst string, overwrite bool) (status int // Individual item permission checks are skipped for performance reasons. // // See section 9.8.5 for when various HTTP status codes apply. -func copyFiles(ctx context.Context, src, dst string, overwrite bool) (status int, err error) { +func copyFiles(ctx context.Context, src, dst string, overwrite bool, depth int) (status int, err error) { srcDir := path.Dir(src) dstDir := path.Dir(dst) + dstName := path.Base(dst) user := ctx.Value(conf.UserKey).(*model.User) if !user.CanCopy() { return http.StatusForbidden, nil @@ -98,11 +390,44 @@ func copyFiles(ctx context.Context, src, dst string, overwrite bool) (status int if !common.CanWrite(user, dstMeta, dstDir) { return http.StatusForbidden, nil } - _, err = fs.Copy(context.WithValue(ctx, conf.NoTaskKey, struct{}{}), src, dstDir) + srcInfo, err := fs.Get(ctx, src, &fs.GetArgs{}) + if err != nil { + if errs.IsObjectNotFound(err) { + return http.StatusNotFound, err + } + return http.StatusInternalServerError, err + } + shallow := depth == 0 && srcInfo.IsDir() + if dstDirInfo, err := fs.Get(ctx, dstDir, &fs.GetArgs{}); err != nil { + if errs.IsObjectNotFound(err) { + return http.StatusConflict, err + } + return http.StatusMethodNotAllowed, err + } else if !dstDirInfo.IsDir() { + return http.StatusConflict, nil + } + dstExisted := false + if _, err = fs.Get(ctx, dst, &fs.GetArgs{}); err == nil { + dstExisted = true + if !overwrite { + return http.StatusPreconditionFailed, nil + } + } else if !errs.IsObjectNotFound(err) { + return http.StatusInternalServerError, err + } + + err = runDeadPropsFileMutation(ctx, + func(tx *gorm.DB) error { return copyDeadProps(tx, src, dst) }, + func() (*fileMutation, error) { + return copyWithRollback(ctx, src, dstDir, dstName, dstExisted && overwrite, shallow) + }, + ) if err != nil { return http.StatusInternalServerError, err } - // TODO if there are no files copy, should return 204 + if dstExisted { + return http.StatusNoContent, nil + } return http.StatusCreated, nil } diff --git a/server/webdav/lock.go b/server/webdav/lock.go index 344ac5ce..0d279e06 100644 --- a/server/webdav/lock.go +++ b/server/webdav/lock.go @@ -109,13 +109,15 @@ type LockDetails struct { // ZeroDepth is whether the lock has zero depth. If it does not have zero // depth, it has infinite depth. ZeroDepth bool + // Shared is whether the lock may coexist with other shared locks on Root. + Shared bool } // NewMemLS returns a new in-memory LockSystem. func NewMemLS() LockSystem { return &memLS{ byName: make(map[string]*memLSNode), - byToken: make(map[string]*memLSNode), + byToken: make(map[string]*memLSLock), gen: uint64(time.Now().Unix()), } } @@ -123,7 +125,7 @@ func NewMemLS() LockSystem { type memLS struct { mu sync.Mutex byName map[string]*memLSNode - byToken map[string]*memLSNode + byToken map[string]*memLSLock gen uint64 // byExpiry only contains those nodes whose LockDetails have a finite // Duration and are yet to expire. @@ -149,7 +151,7 @@ func (m *memLS) Confirm(now time.Time, name0, name1 string, conditions ...Condit defer m.mu.Unlock() m.collectExpiredNodes(now) - var n0, n1 *memLSNode + var n0, n1 *memLSLock if name0 != "" { if n0 = m.lookup(slashClean(name0), conditions...); n0 == nil { return nil, ErrConfirmationFailed @@ -184,48 +186,43 @@ func (m *memLS) Confirm(now time.Time, name0, name1 string, conditions ...Condit }, nil } -// lookup returns the node n that locks the named resource, provided that n -// matches at least one of the given conditions and that lock isn't held by -// another party. Otherwise, it returns nil. -// -// n may be a parent of the named resource, if n is an infinite depth lock. -func (m *memLS) lookup(name string, conditions ...Condition) (n *memLSNode) { +func (m *memLS) lookup(name string, conditions ...Condition) *memLSLock { // TODO: support Condition.Not and Condition.ETag. for _, c := range conditions { - n = m.byToken[c.Token] - if n == nil || n.held { + lock := m.byToken[c.Token] + if lock == nil || lock.held { continue } - if name == n.details.Root { - return n + if name == lock.details.Root { + return lock } - if n.details.ZeroDepth { + if lock.details.ZeroDepth { continue } - if n.details.Root == "/" || strings.HasPrefix(name, n.details.Root+"/") { - return n + if lock.details.Root == "/" || strings.HasPrefix(name, lock.details.Root+"/") { + return lock } } return nil } -func (m *memLS) hold(n *memLSNode) { - if n.held { +func (m *memLS) hold(lock *memLSLock) { + if lock.held { panic("webdav: memLS inconsistent held state") } - n.held = true - if n.details.Duration >= 0 && n.byExpiryIndex >= 0 { - heap.Remove(&m.byExpiry, n.byExpiryIndex) + lock.held = true + if lock.details.Duration >= 0 && lock.byExpiryIndex >= 0 { + heap.Remove(&m.byExpiry, lock.byExpiryIndex) } } -func (m *memLS) unhold(n *memLSNode) { - if !n.held { +func (m *memLS) unhold(lock *memLSLock) { + if !lock.held { panic("webdav: memLS inconsistent held state") } - n.held = false - if n.details.Duration >= 0 { - heap.Push(&m.byExpiry, n) + lock.held = false + if lock.details.Duration >= 0 { + heap.Push(&m.byExpiry, lock) } } @@ -235,18 +232,23 @@ func (m *memLS) Create(now time.Time, details LockDetails) (string, error) { m.collectExpiredNodes(now) details.Root = slashClean(details.Root) - if !m.canCreate(details.Root, details.ZeroDepth) { + if !m.canCreate(details.Root, details.ZeroDepth, details.Shared) { return "", ErrLocked } - n := m.create(details.Root) - n.token = m.nextToken() - m.byToken[n.token] = n - n.details = details - if n.details.Duration >= 0 { - n.expiry = now.Add(n.details.Duration) - heap.Push(&m.byExpiry, n) + node := m.create(details.Root) + lock := &memLSLock{ + token: m.nextToken(), + details: details, + node: node, + byExpiryIndex: -1, } - return n.token, nil + node.locks[lock.token] = lock + m.byToken[lock.token] = lock + if lock.details.Duration >= 0 { + lock.expiry = now.Add(lock.details.Duration) + heap.Push(&m.byExpiry, lock) + } + return lock.token, nil } func (m *memLS) Refresh(now time.Time, token string, duration time.Duration) (LockDetails, error) { @@ -254,22 +256,22 @@ func (m *memLS) Refresh(now time.Time, token string, duration time.Duration) (Lo defer m.mu.Unlock() m.collectExpiredNodes(now) - n := m.byToken[token] - if n == nil { + lock := m.byToken[token] + if lock == nil { return LockDetails{}, ErrNoSuchLock } - if n.held { + if lock.held { return LockDetails{}, ErrLocked } - if n.byExpiryIndex >= 0 { - heap.Remove(&m.byExpiry, n.byExpiryIndex) + if lock.byExpiryIndex >= 0 { + heap.Remove(&m.byExpiry, lock.byExpiryIndex) } - n.details.Duration = duration - if n.details.Duration >= 0 { - n.expiry = now.Add(n.details.Duration) - heap.Push(&m.byExpiry, n) + lock.details.Duration = duration + if lock.details.Duration >= 0 { + lock.expiry = now.Add(lock.details.Duration) + heap.Push(&m.byExpiry, lock) } - return n.details, nil + return lock.details, nil } func (m *memLS) Unlock(now time.Time, token string) error { @@ -277,34 +279,42 @@ func (m *memLS) Unlock(now time.Time, token string) error { defer m.mu.Unlock() m.collectExpiredNodes(now) - n := m.byToken[token] - if n == nil { + lock := m.byToken[token] + if lock == nil { return ErrNoSuchLock } - if n.held { + if lock.held { return ErrLocked } - m.remove(n) + m.remove(lock) return nil } -func (m *memLS) canCreate(name string, zeroDepth bool) bool { +func (m *memLS) canCreate(name string, zeroDepth bool, shared ...bool) bool { + wantShared := len(shared) > 0 && shared[0] return walkToRoot(name, func(name0 string, first bool) bool { n := m.byName[name0] if n == nil { return true } if first { - if n.token != "" { - // The target node is already locked. - return false + if len(n.locks) != 0 { + if !wantShared || (!zeroDepth && n.refCount != len(n.locks)) { + return false + } + for _, lock := range n.locks { + if !lock.details.Shared || lock.details.ZeroDepth != zeroDepth { + return false + } + } + return true } - if !zeroDepth { + if !zeroDepth && n.refCount > 0 { // The requested lock depth is infinite, and the fact that n exists // (n != nil) means that a descendent of the target node is locked. return false } - } else if n.token != "" && !n.details.ZeroDepth { + } else if n.hasInfiniteLock() { // An ancestor of the target node is locked with infinite depth. return false } @@ -317,10 +327,8 @@ func (m *memLS) create(name string) (ret *memLSNode) { n := m.byName[name0] if n == nil { n = &memLSNode{ - details: LockDetails{ - Root: name0, - }, - byExpiryIndex: -1, + details: LockDetails{Root: name0}, + locks: make(map[string]*memLSLock), } m.byName[name0] = n } @@ -333,10 +341,10 @@ func (m *memLS) create(name string) (ret *memLSNode) { return ret } -func (m *memLS) remove(n *memLSNode) { - delete(m.byToken, n.token) - n.token = "" - walkToRoot(n.details.Root, func(name0 string, first bool) bool { +func (m *memLS) remove(lock *memLSLock) { + delete(m.byToken, lock.token) + delete(lock.node.locks, lock.token) + walkToRoot(lock.details.Root, func(name0 string, first bool) bool { x := m.byName[name0] x.refCount-- if x.refCount == 0 { @@ -344,8 +352,8 @@ func (m *memLS) remove(n *memLSNode) { } return true }) - if n.byExpiryIndex >= 0 { - heap.Remove(&m.byExpiry, n.byExpiryIndex) + if lock.byExpiryIndex >= 0 { + heap.Remove(&m.byExpiry, lock.byExpiryIndex) } } @@ -366,24 +374,33 @@ func walkToRoot(name string, f func(name0 string, first bool) bool) bool { } type memLSNode struct { - // details are the lock metadata. Even if this node's name is not explicitly locked, - // details.Root will still equal the node's name. + // details identifies the resource path represented by this node. details LockDetails - // token is the unique identifier for this node's lock. An empty token means that - // this node is not explicitly locked. - token string - // refCount is the number of self-or-descendent nodes that are explicitly locked. + // locks contains the independent lock instances rooted at this resource. + locks map[string]*memLSLock + // refCount is the number of self-or-descendent lock instances. refCount int - // expiry is when this node's lock expires. - expiry time.Time - // byExpiryIndex is the index of this node in memLS.byExpiry. It is -1 - // if this node does not expire, or has expired. - byExpiryIndex int - // held is whether this node's lock is actively held by a Confirm call. - held bool } -type byExpiry []*memLSNode +type memLSLock struct { + token string + details LockDetails + node *memLSNode + expiry time.Time + byExpiryIndex int + held bool +} + +func (n *memLSNode) hasInfiniteLock() bool { + for _, lock := range n.locks { + if !lock.details.ZeroDepth { + return true + } + } + return false +} + +type byExpiry []*memLSLock func (b *byExpiry) Len() int { return len(*b) @@ -400,18 +417,18 @@ func (b *byExpiry) Swap(i, j int) { } func (b *byExpiry) Push(x interface{}) { - n := x.(*memLSNode) - n.byExpiryIndex = len(*b) - *b = append(*b, n) + lock := x.(*memLSLock) + lock.byExpiryIndex = len(*b) + *b = append(*b, lock) } func (b *byExpiry) Pop() interface{} { i := len(*b) - 1 - n := (*b)[i] + lock := (*b)[i] (*b)[i] = nil - n.byExpiryIndex = -1 + lock.byExpiryIndex = -1 *b = (*b)[:i] - return n + return lock } const infiniteTimeout = -1 diff --git a/server/webdav/lock_test.go b/server/webdav/lock_test.go index e7fe9706..8353c158 100644 --- a/server/webdav/lock_test.go +++ b/server/webdav/lock_test.go @@ -182,7 +182,10 @@ func TestMemLSLookup(t *testing.T) { goodToken := "" base := m.byName[baseName] if base != nil && (suffix == "" || !lockTestZeroDepth(baseName)) { - goodToken = base.token + for token := range base.locks { + goodToken = token + break + } } for _, token := range []string{badToken, goodToken} { @@ -191,9 +194,9 @@ func TestMemLSLookup(t *testing.T) { } got := m.lookup(name, Condition{Token: token}) - want := base - if token == badToken { - want = nil + var want *memLSLock + if token != badToken { + want = m.byToken[token] } if got != want { t.Errorf("name=%-20qtoken=%q (bad=%t): got %p, want %p", @@ -449,6 +452,46 @@ func TestMemLSExpiry(t *testing.T) { } } +func TestMemLSSharedLockExpiry(t *testing.T) { + m := NewMemLS().(*memLS) + now := time.Unix(0, 0) + first, err := m.Create(now, LockDetails{Root: "/shared", Duration: infiniteTimeout, ZeroDepth: true, Shared: true}) + if err != nil { + t.Fatal(err) + } + second, err := m.Create(now, LockDetails{Root: "/shared", Duration: 2 * time.Second, ZeroDepth: true, Shared: true}) + if err != nil { + t.Fatal(err) + } + third, err := m.Create(now, LockDetails{Root: "/shared", Duration: 4 * time.Second, ZeroDepth: true, Shared: true}) + if err != nil { + t.Fatal(err) + } + + if got := m.byToken[first].details.Duration; got != infiniteTimeout { + t.Fatalf("first lock duration = %v, want infinite", got) + } + if got := m.byToken[second].details.Duration; got != 2*time.Second { + t.Fatalf("second lock duration = %v, want 2s", got) + } + if got := m.byToken[third].details.Duration; got != 4*time.Second { + t.Fatalf("third lock duration = %v, want 4s", got) + } + + m.mu.Lock() + m.collectExpiredNodes(now.Add(2 * time.Second)) + m.mu.Unlock() + if m.byToken[second] != nil { + t.Fatal("second shared lock did not expire independently") + } + if m.byToken[first] == nil || m.byToken[third] == nil { + t.Fatal("independent shared lock expired with second lock") + } + if err := m.consistent(); err != nil { + t.Fatal(err) + } +} + func TestMemLS(t *testing.T) { now := time.Unix(0, 0) m := NewMemLS().(*memLS) @@ -578,8 +621,10 @@ func (m *memLS) consistent() error { // so strings.HasPrefix is equivalent to self-or-descendent name match. // We don't have to worry about "/foo/bar" being a false positive match // for "/foo/b". - if strings.HasPrefix(name0, name) && n0.token != "" { - list = append(list, name0) + if strings.HasPrefix(name0, name) { + for range n0.locks { + list = append(list, name0) + } } } if n.refCount != len(list) { @@ -588,50 +633,34 @@ func (m *memLS) consistent() error { name, n.refCount, list, len(list)) } - // A node n is in m.byToken if it has a non-empty token. - if n.token != "" { - if _, ok := m.byToken[n.token]; !ok { - return fmt.Errorf("node at name %q has token %q but not in m.byToken", name, n.token) - } - } - - // A node n is in m.byExpiry if it has a non-negative byExpiryIndex. - if n.byExpiryIndex >= 0 { - if n.byExpiryIndex >= len(m.byExpiry) { - return fmt.Errorf("node at name %q has byExpiryIndex %d but m.byExpiry has length %d", name, n.byExpiryIndex, len(m.byExpiry)) - } - if n != m.byExpiry[n.byExpiryIndex] { - return fmt.Errorf("node at name %q has byExpiryIndex %d but that indexes a different node", name, n.byExpiryIndex) + for token, lock := range n.locks { + if lock.token != token || lock.node != n || m.byToken[token] != lock { + return fmt.Errorf("lock %q at node %q is inconsistent", token, name) } } } - for token, n := range m.byToken { - // The map keys should be consistent with the node's copy of the key. - if n.token != token { - return fmt.Errorf("node token %q != byToken map key %q", n.token, token) + for token, lock := range m.byToken { + if lock.token != token { + return fmt.Errorf("lock token %q != byToken map key %q", lock.token, token) } - - // Every node in m.byToken is in m.byName. - if _, ok := m.byName[n.details.Root]; !ok { - return fmt.Errorf("node at name %q in m.byToken but not in m.byName", n.details.Root) + if lock.node == nil || lock.node.locks[token] != lock { + return fmt.Errorf("lock %q is missing from its node", token) + } + if m.byName[lock.details.Root] != lock.node { + return fmt.Errorf("lock %q has inconsistent root %q", token, lock.details.Root) } } - for i, n := range m.byExpiry { - // The slice indices should be consistent with the node's copy of the index. - if n.byExpiryIndex != i { - return fmt.Errorf("node byExpiryIndex %d != byExpiry slice index %d", n.byExpiryIndex, i) + for i, lock := range m.byExpiry { + if lock.byExpiryIndex != i { + return fmt.Errorf("lock byExpiryIndex %d != byExpiry slice index %d", lock.byExpiryIndex, i) } - - // Every node in m.byExpiry is in m.byName. - if _, ok := m.byName[n.details.Root]; !ok { - return fmt.Errorf("node at name %q in m.byExpiry but not in m.byName", n.details.Root) + if m.byToken[lock.token] != lock { + return fmt.Errorf("lock %q is missing from byToken", lock.token) } - - // No node in m.byExpiry should be held. - if n.held { - return fmt.Errorf("node at name %q in m.byExpiry is held", n.details.Root) + if lock.held { + return fmt.Errorf("lock at name %q is held", lock.details.Root) } } return nil @@ -733,3 +762,22 @@ func TestParseTimeout(t *testing.T) { } } } + +func TestHasNegatedTokenCondition(t *testing.T) { + tests := []struct { + name string + conditions []Condition + want bool + }{ + {name: "positive token", conditions: []Condition{{Token: "DAV:no-lock"}}}, + {name: "negated token", conditions: []Condition{{Not: true, Token: "DAV:no-lock"}}, want: true}, + {name: "negated etag", conditions: []Condition{{Not: true, ETag: `"etag"`}}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := hasNegatedTokenCondition(tt.conditions); got != tt.want { + t.Fatalf("hasNegatedTokenCondition(%v) = %t, want %t", tt.conditions, got, tt.want) + } + }) + } +} diff --git a/server/webdav/prop.go b/server/webdav/prop.go index 5c888934..8fc0bae4 100644 --- a/server/webdav/prop.go +++ b/server/webdav/prop.go @@ -16,9 +16,11 @@ import ( "time" "github.com/OpenListTeam/OpenList/v4/internal/conf" + "github.com/OpenListTeam/OpenList/v4/internal/db" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/OpenListTeam/OpenList/v4/server/common" + "gorm.io/gorm" ) // Proppatch describes a property update instruction as defined in RFC 4918. @@ -170,79 +172,39 @@ var liveProps = map[xml.Name]struct { // TODO(nigeltao) merge props and allprop? // Props returns the status of the properties named pnames for resource name. -// -// Each Propstat has a unique status and each property name will only be part -// of one Propstat element. -func props(ctx context.Context, ls LockSystem, fi model.Obj, pnames []xml.Name) ([]Propstat, error) { - //f, err := fs.OpenFile(ctx, name, os.O_RDONLY, 0) - //if err != nil { - // return nil, err - //} - //defer f.Close() - //fi, err := f.Stat() - //if err != nil { - // return nil, err - //} +func props(tx *gorm.DB, ctx context.Context, ls LockSystem, name string, fi model.Obj, pnames []xml.Name) ([]Propstat, error) { isDir := fi.IsDir() - - var deadProps map[xml.Name]Property - // ??? what is this for? - //if dph, ok := f.(DeadPropsHolder); ok { - // deadProps, err = dph.DeadProps() - // if err != nil { - // return nil, err - // } - //} - + deadProps, err := getDeadProps(tx, name) + if err != nil { + return nil, err + } pstatOK := Propstat{Status: http.StatusOK} pstatNotFound := Propstat{Status: http.StatusNotFound} for _, pn := range pnames { - // If this file has dead properties, check if they contain pn. if dp, ok := deadProps[pn]; ok { pstatOK.Props = append(pstatOK.Props, dp) continue } - // Otherwise, it must either be a live property or we don't know it. if prop := liveProps[pn]; prop.findFn != nil && (prop.dir || !isDir) { innerXML, err := prop.findFn(ctx, ls, fi.GetName(), fi) if err != nil { return nil, err } - pstatOK.Props = append(pstatOK.Props, Property{ - XMLName: pn, - InnerXML: []byte(innerXML), - }) + pstatOK.Props = append(pstatOK.Props, Property{XMLName: pn, InnerXML: []byte(innerXML)}) } else { - pstatNotFound.Props = append(pstatNotFound.Props, Property{ - XMLName: pn, - }) + pstatNotFound.Props = append(pstatNotFound.Props, Property{XMLName: pn}) } } return makePropstats(pstatOK, pstatNotFound), nil } // Propnames returns the property names defined for resource name. -func propnames(ctx context.Context, ls LockSystem, fi model.Obj) ([]xml.Name, error) { - //f, err := fs.OpenFile(ctx, name, os.O_RDONLY, 0) - //if err != nil { - // return nil, err - //} - //defer f.Close() - //fi, err := f.Stat() - //if err != nil { - // return nil, err - //} +func propnames(tx *gorm.DB, name string, fi model.Obj) ([]xml.Name, error) { isDir := fi.IsDir() - - var deadProps map[xml.Name]Property - // ??? what is this for? - //if dph, ok := f.(DeadPropsHolder); ok { - // deadProps, err = dph.DeadProps() - // if err != nil { - // return nil, err - // } - //} - + deadProps, err := getDeadProps(tx, name) + if err != nil { + return nil, err + } pnames := make([]xml.Name, 0, len(liveProps)+len(deadProps)) for pn, prop := range liveProps { if prop.findFn != nil && (prop.dir || !isDir) { @@ -255,20 +217,12 @@ func propnames(ctx context.Context, ls LockSystem, fi model.Obj) ([]xml.Name, er return pnames, nil } -// Allprop returns the properties defined for resource name and the properties -// named in include. -// -// Note that RFC 4918 defines 'allprop' to return the DAV: properties defined -// within the RFC plus dead properties. Other live properties should only be -// returned if they are named in 'include'. -// -// See http://www.webdav.org/specs/rfc4918.html#METHOD_PROPFIND -func allprop(ctx context.Context, ls LockSystem, fi model.Obj, include []xml.Name) ([]Propstat, error) { - pnames, err := propnames(ctx, ls, fi) +// Allprop returns the properties defined for resource name and the properties named in include. +func allprop(tx *gorm.DB, ctx context.Context, ls LockSystem, name string, fi model.Obj, include []xml.Name) ([]Propstat, error) { + pnames, err := propnames(tx, name, fi) if err != nil { return nil, err } - // Add names from include if they are not already covered in pnames. nameset := make(map[xml.Name]bool) for _, pn := range pnames { nameset[pn] = true @@ -278,11 +232,10 @@ func allprop(ctx context.Context, ls LockSystem, fi model.Obj, include []xml.Nam pnames = append(pnames, pn) } } - return props(ctx, ls, fi, pnames) + return props(tx, ctx, ls, name, fi, pnames) } -// Patch patches the properties of resource name. The return values are -// constrained in the same manner as DeadPropsHolder.Patch. +// Patch patches the properties of resource name. func patch(ctx context.Context, ls LockSystem, name string, patches []Proppatch) ([]Propstat, error) { conflict := false loop: @@ -314,53 +267,93 @@ loop: return makePropstats(pstatForbidden, pstatFailedDep), nil } - // ------------------------------------------------------------ - //f, err := fs.OpenFile(ctx, name, os.O_RDWR, 0) - //if err != nil { - // return nil, err - //} - //defer f.Close() - //if dph, ok := f.(DeadPropsHolder); ok { - // ret, err := dph.Patch(patches) - // if err != nil { - // return nil, err - // } - // // http://www.webdav.org/specs/rfc4918.html#ELEMENT_propstat says that - // // "The contents of the prop XML element must only list the names of - // // properties to which the result in the status element applies." - // for _, pstat := range ret { - // for i, p := range pstat.Props { - // pstat.Props[i] = Property{XMLName: p.XMLName} - // } - // } - // return ret, nil - //} - // ------------------------------------------------------------ - - // The file doesn't implement the optional DeadPropsHolder interface, so - // all patches are forbidden. - pstat := Propstat{Status: http.StatusForbidden} - for _, patch := range patches { - for _, p := range patch.Props { - pstat.Props = append(pstat.Props, Property{XMLName: p.XMLName}) + pstat := Propstat{Status: http.StatusOK} + var err error + for attempt := 0; attempt < 10; attempt++ { + pstat.Props = nil + err = withDeadPropsTransaction(ctx, func(tx *gorm.DB) error { + for _, patch := range patches { + for _, prop := range patch.Props { + if patch.Remove { + if err := tx.Where("path = ? AND namespace = ? AND name = ?", name, prop.XMLName.Space, prop.XMLName.Local).Delete(&model.WebDAVProperty{}).Error; err != nil { + return err + } + } else { + row := model.WebDAVProperty{ + Path: name, + Namespace: prop.XMLName.Space, + Name: prop.XMLName.Local, + } + if err := tx.Where("path = ? AND namespace = ? AND name = ?", row.Path, row.Namespace, row.Name). + Assign(model.WebDAVProperty{Lang: prop.Lang, InnerXML: prop.InnerXML}). + FirstOrCreate(&row).Error; err != nil { + return err + } + } + pstat.Props = append(pstat.Props, Property{XMLName: prop.XMLName}) + } + } + return nil + }) + if err == nil || !strings.Contains(err.Error(), "database is locked") { + break } + time.Sleep(time.Duration(attempt+1) * 20 * time.Millisecond) + } + if err != nil { + return nil, err } return []Propstat{pstat}, nil } +func getDeadProps(tx *gorm.DB, path string) (map[xml.Name]Property, error) { + var rows []model.WebDAVProperty + if tx == nil { + return nil, errors.New("webdav property transaction is not initialized") + } + if err := tx.Where("path = ?", path).Find(&rows).Error; err != nil { + return nil, err + } + props := make(map[xml.Name]Property, len(rows)) + for _, row := range rows { + props[xml.Name{Space: row.Namespace, Local: row.Name}] = Property{ + XMLName: xml.Name{Space: row.Namespace, Local: row.Name}, + Lang: row.Lang, + InnerXML: row.InnerXML, + } + } + return props, nil +} + +func withDeadPropsTransaction(ctx context.Context, fn func(*gorm.DB) error) error { + database := db.GetDb() + if database == nil { + return errors.New("webdav property database is not initialized") + } + var transaction *gorm.DB + err := database.WithContext(ctx).Transaction(func(tx *gorm.DB) error { + transaction = tx + var probe int + if err := tx.Raw("SELECT 1").Scan(&probe).Error; err != nil { + return err + } + return fn(tx) + }) + if err != nil && transaction != nil { + _ = transaction.Rollback().Error + } + return err +} + func escapeXML(s string) string { for i := 0; i < len(s); i++ { - // As an optimization, if s contains only ASCII letters, digits or a - // few special characters, the escaped value is s itself and we don't - // need to allocate a buffer and convert between string and []byte. switch c := s[i]; { case c == ' ' || c == '_' || - ('+' <= c && c <= '9') || // Digits as well as + , - . and / + ('+' <= c && c <= '9') || ('A' <= c && c <= 'Z') || ('a' <= c && c <= 'z'): continue } - // Otherwise, go through the full escaping process. var buf bytes.Buffer xml.EscapeText(&buf, []byte(s)) return buf.String() @@ -368,6 +361,72 @@ func escapeXML(s string) string { return s } +func deadPropsLikePath(path string) string { + return strings.ReplaceAll(strings.ReplaceAll(strings.ReplaceAll(path, "\\", "\\\\"), "%", "\\%"), "_", "\\_") +} + +func deadPropsSubtree(tx *gorm.DB, path string) *gorm.DB { + return tx.Where("path = ? OR path LIKE ? ESCAPE '\\'", path, deadPropsLikePath(path)+"/%") +} + +func deleteDeadProps(tx *gorm.DB, path string) error { + return deadPropsSubtree(tx, path).Delete(&model.WebDAVProperty{}).Error +} + +func removeResourceWithDeadProps(ctx context.Context, path string, remove func() error) (removeErr, transactionErr error) { + transactionErr = withDeadPropsTransaction(ctx, func(tx *gorm.DB) error { + if err := deleteDeadProps(tx, path); err != nil { + return err + } + removeErr = remove() + return removeErr + }) + return removeErr, transactionErr +} + +func copyDeadProps(tx *gorm.DB, src, dst string) error { + var rows []model.WebDAVProperty + if err := deadPropsSubtree(tx, src).Find(&rows).Error; err != nil { + return err + } + if err := deadPropsSubtree(tx, dst).Delete(&model.WebDAVProperty{}).Error; err != nil { + return err + } + for _, row := range rows { + copy := row + copy.ID = 0 + copy.Path = dst + strings.TrimPrefix(row.Path, src) + if err := tx.Create(©).Error; err != nil { + return err + } + } + return nil +} + +func moveDeadProps(tx *gorm.DB, src, dst string) error { + var rows []model.WebDAVProperty + if err := deadPropsSubtree(tx, src).Find(&rows).Error; err != nil { + return err + } + // Clear destination only after the source rows are staged in memory. + if err := deadPropsSubtree(tx, dst).Delete(&model.WebDAVProperty{}).Error; err != nil { + return err + } + for _, row := range rows { + newPath := dst + strings.TrimPrefix(row.Path, src) + copy := row + copy.ID = 0 + copy.Path = newPath + if err := tx.Create(©).Error; err != nil { + return err + } + if err := tx.Delete(&row).Error; err != nil { + return err + } + } + return nil +} + func findResourceType(ctx context.Context, ls LockSystem, name string, fi model.Obj) (string, error) { if fi.IsDir() { return ``, nil diff --git a/server/webdav/webdav.go b/server/webdav/webdav.go index 06d1431a..d2d87400 100644 --- a/server/webdav/webdav.go +++ b/server/webdav/webdav.go @@ -29,6 +29,7 @@ import ( "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/OpenListTeam/OpenList/v4/server/common" + "gorm.io/gorm" ) type Handler struct { @@ -82,10 +83,6 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { status, err = h.handleUnlock(brw, r) case "PROPFIND": status, err = h.handlePropfind(brw, r) - // if there is a error for PROPFIND, we should be as an empty folder to the client - if err != nil { - status = http.StatusNotFound - } case "PROPPATCH": status, err = h.handleProppatch(brw, r) } @@ -119,14 +116,64 @@ func (h *Handler) lock(now time.Time, root string) (token string, status int, er return token, 0, nil } +func ifETagConditionsMatch(ctx context.Context, ls LockSystem, name string, conditions []Condition) (bool, error) { + var obj model.Obj + objLoaded, objMissing := false, false + for _, condition := range conditions { + if condition.ETag == "" { + continue + } + if !objLoaded { + var err error + obj, err = fs.Get(ctx, name, &fs.GetArgs{}) + objLoaded = true + if err != nil { + if !errs.IsObjectNotFound(err) { + return false, err + } + objMissing = true + } + } + matches := false + if !objMissing { + etag, err := findETag(ctx, ls, name, obj) + if err != nil { + return false, err + } + matches = etag == condition.ETag + } + if condition.Not { + matches = !matches + } + if !matches { + return false, nil + } + } + return true, nil +} + +func positiveTokenConditions(conditions []Condition) []Condition { + ret := make([]Condition, 0, len(conditions)) + for _, condition := range conditions { + if condition.Token != "" && !condition.Not { + ret = append(ret, condition) + } + } + return ret +} + +func hasNegatedTokenCondition(conditions []Condition) bool { + for _, condition := range conditions { + if condition.Token != "" && condition.Not { + return true + } + } + return false +} + func (h *Handler) confirmLocks(r *http.Request, src, dst string) (release func(), status int, err error) { hdr := r.Header.Get("If") if hdr == "" { - // An empty If header means that the client hasn't previously created locks. - // Even if this client doesn't care about locks, we still need to check that - // the resources aren't locked by another client, so we create temporary - // locks that would conflict with another client's locks. These temporary - // locks are unlocked at the end of the HTTP request. now, srcToken, dstToken := time.Now(), "", "" if src != "" { srcToken, status, err = h.lock(now, src) @@ -143,7 +190,6 @@ func (h *Handler) confirmLocks(r *http.Request, src, dst string) (release func() return nil, status, err } } - return func() { if dstToken != "" { h.LockSystem.Unlock(now, dstToken) @@ -158,7 +204,9 @@ func (h *Handler) confirmLocks(r *http.Request, src, dst string) (release func() if !ok { return nil, http.StatusBadRequest, errInvalidIfHeader } - // ih is a disjunction (OR) of ifLists, so any ifList will do. + ctx := r.Context() + user, _ := ctx.Value(conf.UserKey).(*model.User) + etagMismatch, hasNegatedToken := false, false for _, l := range ih.lists { lsrc := l.resourceTag if lsrc == "" { @@ -175,8 +223,23 @@ func (h *Handler) confirmLocks(r *http.Request, src, dst string) (release func() if err != nil { return nil, status, err } + if user != nil { + lsrc, err = user.JoinPath(lsrc) + if err != nil { + return nil, http.StatusForbidden, err + } + } } - release, err = h.LockSystem.Confirm(time.Now(), lsrc, dst, l.conditions...) + matches, err := ifETagConditionsMatch(ctx, h.LockSystem, lsrc, l.conditions) + if err != nil { + return nil, http.StatusInternalServerError, err + } + if !matches { + etagMismatch = true + continue + } + hasNegatedToken = hasNegatedToken || hasNegatedTokenCondition(l.conditions) + release, err = h.LockSystem.Confirm(time.Now(), lsrc, dst, positiveTokenConditions(l.conditions)...) if err == ErrConfirmationFailed { continue } @@ -185,10 +248,28 @@ func (h *Handler) confirmLocks(r *http.Request, src, dst string) (release func() } return release, 0, nil } - // Section 10.4.1 says that "If this header is evaluated and all state lists - // fail, then the request must fail with a 412 (Precondition Failed) status." - // We follow the spec even though the cond_put_corrupt_token test case from - // the litmus test warns on seeing a 412 instead of a 423 (Locked). + if etagMismatch || !hasNegatedToken { + return nil, http.StatusPreconditionFailed, ErrLocked + } + + // Litmus expects a corrupt token paired with Not on an + // actively locked resource to report the lock conflict. + now := time.Now() + for _, name := range []string{src, dst} { + if name == "" { + continue + } + token, lockStatus, lockErr := h.lock(now, name) + if lockErr == ErrLocked { + return nil, lockStatus, lockErr + } + if lockErr != nil { + return nil, lockStatus, lockErr + } + if unlockErr := h.LockSystem.Unlock(now, token); unlockErr != nil { + return nil, http.StatusInternalServerError, unlockErr + } + } return nil, http.StatusPreconditionFailed, ErrLocked } @@ -212,9 +293,7 @@ func (h *Handler) handleOptions(w http.ResponseWriter, r *http.Request) (status } } w.Header().Set("Allow", allow) - // http://www.webdav.org/specs/rfc4918.html#dav.compliance.classes w.Header().Set("DAV", "1, 2") - // http://msdn.microsoft.com/en-au/library/cc250217.aspx w.Header().Set("MS-Author-Via", "DAV") return 0, nil } @@ -295,12 +374,6 @@ func (h *Handler) handleDelete(w http.ResponseWriter, r *http.Request) (status i if err != nil { return status, err } - release, status, err := h.confirmLocks(r, reqPath, "") - if err != nil { - return status, err - } - defer release() - ctx := r.Context() user := ctx.Value(conf.UserKey).(*model.User) if !user.CanRemove() { @@ -310,18 +383,29 @@ func (h *Handler) handleDelete(w http.ResponseWriter, r *http.Request) (status i if err != nil { return http.StatusForbidden, err } + release, status, err := h.confirmLocks(r, reqPath, "") + if err != nil { + return status, err + } + defer release() // TODO: return MultiStatus where appropriate. // "godoc os RemoveAll" says that "If the path does not exist, RemoveAll // returns nil (no error)." WebDAV semantics are that it should return a // "404 Not Found". We therefore have to Stat before we RemoveAll. + parentPath := path.Dir(reqPath) + if _, err := fs.List(ctx, parentPath, &fs.ListArgs{Refresh: true, NoLog: true}); err != nil { + if errs.IsNotFoundError(err) { + return http.StatusNotFound, err + } + return http.StatusMethodNotAllowed, err + } if _, err := fs.Get(ctx, reqPath, &fs.GetArgs{}); err != nil { - if errs.IsObjectNotFound(err) { + if errs.IsNotFoundError(err) { return http.StatusNotFound, err } return http.StatusMethodNotAllowed, err } - parentPath := path.Dir(reqPath) parentMeta, err := op.GetNearestMeta(parentPath) if err != nil && !errors.Is(errors.Cause(err), errs.MetaNotFound) { return http.StatusInternalServerError, err @@ -329,8 +413,14 @@ func (h *Handler) handleDelete(w http.ResponseWriter, r *http.Request) (status i if !common.CanWrite(user, parentMeta, parentPath) { return http.StatusForbidden, errs.PermissionDenied } - if err := fs.Remove(ctx, reqPath); err != nil { - return http.StatusMethodNotAllowed, err + removeErr, transactionErr := removeResourceWithDeadProps(ctx, reqPath, func() error { + return fs.Remove(ctx, reqPath) + }) + if removeErr != nil { + return http.StatusMethodNotAllowed, removeErr + } + if transactionErr != nil { + return http.StatusInternalServerError, transactionErr } //fs.ClearCache(path.Dir(reqPath)) return http.StatusNoContent, nil @@ -350,11 +440,6 @@ func (h *Handler) handlePut(w http.ResponseWriter, r *http.Request) (status int, if reqPath == "" { return http.StatusMethodNotAllowed, nil } - release, status, err := h.confirmLocks(r, reqPath, "") - if err != nil { - return status, err - } - defer release() // TODO(rost): Support the If-Match, If-None-Match headers? See bradfitz' // comments in http.checkEtag. ctx := r.Context() @@ -363,6 +448,11 @@ func (h *Handler) handlePut(w http.ResponseWriter, r *http.Request) (status int, if err != nil { return http.StatusForbidden, err } + release, status, err := h.confirmLocks(r, reqPath, "") + if err != nil { + return status, err + } + defer release() size := r.ContentLength if size < 0 { sizeStr := r.Header.Get("X-File-Size") @@ -384,6 +474,16 @@ func (h *Handler) handlePut(w http.ResponseWriter, r *http.Request) (status int, return http.StatusForbidden, errs.IgnoredSystemFile } parentPath := path.Dir(reqPath) + parent, err := fs.Get(ctx, parentPath, &fs.GetArgs{}) + if err != nil { + if errs.IsNotFoundError(err) { + return http.StatusConflict, err + } + return http.StatusInternalServerError, err + } + if !parent.IsDir() { + return http.StatusConflict, errs.NotFolder + } parentMeta, err := op.GetNearestMeta(parentPath) if err != nil && !errors.Is(errors.Cause(err), errs.MetaNotFound) { return http.StatusInternalServerError, err @@ -394,6 +494,30 @@ func (h *Handler) handlePut(w http.ResponseWriter, r *http.Request) (status int, if !common.CanWrite(user, parentMeta, parentPath) { return http.StatusForbidden, errs.PermissionDenied } + fi, err := fs.Get(ctx, reqPath, &fs.GetArgs{}) + exists := err == nil + if err != nil && !errs.IsObjectNotFound(err) { + if errs.IsNotFoundError(err) { + return http.StatusNotFound, err + } + return http.StatusInternalServerError, err + } + var modTime time.Time + if exists { + if fi.IsDir() { + return http.StatusMethodNotAllowed, nil + } + etag, err := findETag(ctx, h.LockSystem, reqPath, fi) + if err != nil { + return http.StatusInternalServerError, err + } + w.Header().Set("Etag", etag) + modTime = fi.ModTime() + } + if done, _ := net.CheckPreconditions(w, r, modTime, exists); done { + return 0, nil + } + w.Header().Del("Etag") fsStream := &stream.FileStream{ Obj: &obj, Reader: r.Body, @@ -411,16 +535,21 @@ func (h *Handler) handlePut(w http.ResponseWriter, r *http.Request) (status int, if err != nil { return http.StatusMethodNotAllowed, err } - fi, err := fs.Get(ctx, reqPath, &fs.GetArgs{}) + fi, err = fs.Get(ctx, reqPath, &fs.GetArgs{}) if err != nil { - fi = &obj + return http.StatusInternalServerError, err } etag, err := findETag(ctx, h.LockSystem, reqPath, fi) if err != nil { return http.StatusInternalServerError, err } w.Header().Set("Etag", etag) - return http.StatusCreated, nil + if exists { + w.WriteHeader(http.StatusNoContent) + } else { + w.WriteHeader(http.StatusCreated) + } + return 0, nil } func (h *Handler) handleMkcol(w http.ResponseWriter, r *http.Request) (status int, err error) { @@ -428,18 +557,17 @@ func (h *Handler) handleMkcol(w http.ResponseWriter, r *http.Request) (status in if err != nil { return status, err } - release, status, err := h.confirmLocks(r, reqPath, "") - if err != nil { - return status, err - } - defer release() - ctx := r.Context() user := ctx.Value(conf.UserKey).(*model.User) reqPath, err = user.JoinPath(reqPath) if err != nil { return http.StatusForbidden, err } + release, status, err := h.confirmLocks(r, reqPath, "") + if err != nil { + return status, err + } + defer release() if r.ContentLength > 0 { return http.StatusUnsupportedMediaType, nil @@ -449,15 +577,21 @@ func (h *Handler) handleMkcol(w http.ResponseWriter, r *http.Request) (status in //405 (Method Not Allowed) - MKCOL can only be executed on an unmapped URL if _, err := fs.Get(ctx, reqPath, &fs.GetArgs{}); err == nil { return http.StatusMethodNotAllowed, err + } else if !errs.IsNotFoundError(err) { + return http.StatusInternalServerError, err } // RFC 4918 9.3.1 // 409 (Conflict) The server MUST NOT create those intermediate collections automatically. parentPath := path.Dir(reqPath) - if _, err := fs.Get(ctx, parentPath, &fs.GetArgs{}); err != nil { - if errs.IsObjectNotFound(err) { + parent, err := fs.Get(ctx, parentPath, &fs.GetArgs{}) + if err != nil { + if errs.IsNotFoundError(err) { return http.StatusConflict, err } - return http.StatusMethodNotAllowed, err + return http.StatusInternalServerError, err + } + if !parent.IsDir() { + return http.StatusConflict, errs.NotFolder } parentMeta, err := op.GetNearestMeta(parentPath) if err != nil && !errors.Is(errors.Cause(err), errs.MetaNotFound) { @@ -542,7 +676,7 @@ func (h *Handler) handleCopyMove(w http.ResponseWriter, r *http.Request) (status return http.StatusBadRequest, errInvalidDepth } } - return copyFiles(ctx, src, dst, r.Header.Get("Overwrite") != "F") + return copyFiles(ctx, src, dst, r.Header.Get("Overwrite") != "F", depth) } release, status, err := h.confirmLocks(r, src, dst) @@ -622,11 +756,18 @@ func (h *Handler) handleLock(w http.ResponseWriter, r *http.Request) (retStatus if !common.CanWrite(user, meta, reqPath) { return http.StatusForbidden, errs.PermissionDenied } + if _, err := fs.Get(ctx, reqPath, &fs.GetArgs{}); err != nil { + if !errs.IsObjectNotFound(err) { + return http.StatusInternalServerError, err + } + created = true + } ld = LockDetails{ Root: reqPath, Duration: duration, OwnerXML: li.Owner.InnerXML, ZeroDepth: depth == 0, + Shared: li.Shared != nil, } token, err = h.LockSystem.Create(now, ld) if err != nil { @@ -752,43 +893,44 @@ func (h *Handler) handlePropfind(w http.ResponseWriter, r *http.Request) (status mw := multistatusWriter{w: w} - walkFn := func(reqPath string, info model.Obj, err error) error { - if err != nil { - return err - } - var pstats []Propstat - if pf.Propname != nil { - pnames, err := propnames(ctx, h.LockSystem, info) - if err != nil { - return err + err = withDeadPropsTransaction(ctx, func(tx *gorm.DB) error { + walkFn := func(currentPath string, info model.Obj, walkErr error) error { + if walkErr != nil { + return walkErr } - pstat := Propstat{Status: http.StatusOK} - for _, xmlname := range pnames { - pstat.Props = append(pstat.Props, Property{XMLName: xmlname}) + var pstats []Propstat + if pf.Propname != nil { + pnames, err := propnames(tx, currentPath, info) + if err != nil { + return err + } + pstat := Propstat{Status: http.StatusOK} + for _, xmlname := range pnames { + pstat.Props = append(pstat.Props, Property{XMLName: xmlname}) + } + pstats = append(pstats, pstat) + } else if pf.Allprop != nil { + pstats, walkErr = allprop(tx, ctx, h.LockSystem, currentPath, info, pf.Prop) + } else { + pstats, walkErr = props(tx, ctx, h.LockSystem, currentPath, info, pf.Prop) } - pstats = append(pstats, pstat) - } else if pf.Allprop != nil { - pstats, err = allprop(ctx, h.LockSystem, info, pf.Prop) - } else { - pstats, err = props(ctx, h.LockSystem, info, pf.Prop) + if walkErr != nil { + return walkErr + } + href := path.Join(h.Prefix, strings.TrimPrefix(currentPath, user.BasePath)) + if href != "/" && info.IsDir() { + href += "/" + } + return mw.write(makePropstatResponse(href, pstats)) } - if err != nil { - return err - } - href := path.Join(h.Prefix, strings.TrimPrefix(reqPath, user.BasePath)) - if href != "/" && info.IsDir() { - href += "/" - } - return mw.write(makePropstatResponse(href, pstats)) - } - walkErr := walkFS(ctx, depth, reqPath, fi, walkFn) - closeErr := mw.close() - if walkErr != nil { - return http.StatusInternalServerError, walkErr - } - if closeErr != nil { - return http.StatusInternalServerError, closeErr + if walkErr := walkFS(ctx, depth, reqPath, fi, walkFn); walkErr != nil { + return walkErr + } + return mw.close() + }) + if err != nil { + return http.StatusInternalServerError, err } return 0, nil } @@ -798,18 +940,17 @@ func (h *Handler) handleProppatch(w http.ResponseWriter, r *http.Request) (statu if err != nil { return status, err } - release, status, err := h.confirmLocks(r, reqPath, "") - if err != nil { - return status, err - } - defer release() - ctx := r.Context() user := ctx.Value(conf.UserKey).(*model.User) reqPath, err = user.JoinPath(reqPath) if err != nil { return http.StatusForbidden, err } + release, status, err := h.confirmLocks(r, reqPath, "") + if err != nil { + return status, err + } + defer release() meta, err := op.GetNearestMeta(reqPath) if err != nil && !errors.Is(errors.Cause(err), errs.MetaNotFound) { return http.StatusInternalServerError, err diff --git a/server/webdav/xml.go b/server/webdav/xml.go index c9ec61df..d87eb863 100644 --- a/server/webdav/xml.go +++ b/server/webdav/xml.go @@ -62,9 +62,7 @@ func readLockInfo(r io.Reader) (li lockInfo, status int, err error) { } return lockInfo{}, http.StatusBadRequest, err } - // We only support exclusive (non-shared) write locks. In practice, these are - // the only types of locks that seem to matter. - if li.Exclusive == nil || li.Shared != nil || li.Write == nil { + if (li.Exclusive == nil) == (li.Shared == nil) || li.Write == nil { return lockInfo{}, http.StatusNotImplemented, errUnsupportedLockInfo } return li, 0, nil @@ -86,18 +84,22 @@ func writeLockInfo(w io.Writer, token string, ld LockDetails) (int, error) { if ld.ZeroDepth { depth = "0" } + scope := "exclusive" + if ld.Shared { + scope = "shared" + } timeout := ld.Duration / time.Second return fmt.Fprintf(w, "\n"+ "\n"+ - " \n"+ - " \n"+ - " %s\n"+ - " %s\n"+ - " Second-%d\n"+ - " %s\n"+ - " %s\n"+ + "\t\n"+ + "\t\n"+ + "\t%s\n"+ + "\t%s\n"+ + "\tSecond-%d\n"+ + "\t%s\n"+ + "\t%s\n"+ "", - depth, ld.OwnerXML, timeout, escape(token), escape(ld.Root), + scope, depth, ld.OwnerXML, timeout, escape(token), escape(ld.Root), ) } @@ -176,19 +178,31 @@ type propfind struct { } func readPropfind(r io.Reader) (pf propfind, status int, err error) { - c := countingReader{r: r} - if err = ixml.NewDecoder(&c).Decode(&pf); err != nil { + // 64KB is more than enough for a well-formed PROPFIND body. + const maxBody = 64 << 10 + body, err := io.ReadAll(io.LimitReader(r, maxBody)) + if err != nil { + return propfind{}, http.StatusBadRequest, err + } + // If the limit was reached, the body was too large. + if len(body) >= maxBody { + // Drain any remaining bytes so the connection stays usable. + _, _ = io.Copy(io.Discard, r) + return propfind{}, http.StatusRequestEntityTooLarge, nil + } + if len(body) == 0 { + // An empty body means to propfind allprop. + return propfind{Allprop: new(struct{})}, 0, nil + } + if hasEmptyNamespacePrefix(body) { + return propfind{}, http.StatusBadRequest, errInvalidPropfind + } + if err = ixml.NewDecoder(bytes.NewReader(body)).Decode(&pf); err != nil { if err == io.EOF { - if c.n == 0 { - // An empty body means to propfind allprop. - // http://www.webdav.org/specs/rfc4918.html#METHOD_PROPFIND - return propfind{Allprop: new(struct{})}, 0, nil - } err = errInvalidPropfind } return propfind{}, http.StatusBadRequest, err } - if pf.Allprop == nil && pf.Include != nil { return propfind{}, http.StatusBadRequest, errInvalidPropfind } @@ -204,6 +218,33 @@ func readPropfind(r io.Reader) (pf propfind, status int, err error) { return pf, 0, nil } +func hasEmptyNamespacePrefix(body []byte) bool { + for offset := 0; ; { + i := bytes.Index(body[offset:], []byte("xmlns:")) + if i < 0 { + return false + } + i += offset + len("xmlns:") + j := i + for j < len(body) && body[j] != '=' && body[j] != '>' && body[j] != '/' && body[j] != ' ' && body[j] != '\t' && body[j] != '\n' && body[j] != '\r' { + j++ + } + for j < len(body) && (body[j] == ' ' || body[j] == '\t' || body[j] == '\n' || body[j] == '\r') { + j++ + } + if j < len(body) && body[j] == '=' { + j++ + for j < len(body) && (body[j] == ' ' || body[j] == '\t' || body[j] == '\n' || body[j] == '\r') { + j++ + } + if j+1 < len(body) && (body[j] == '\'' || body[j] == '"') && body[j+1] == body[j] { + return true + } + } + offset = i + } +} + // Property represents a single DAV resource property as defined in RFC 4918. // See http://www.webdav.org/specs/rfc4918.html#data.model.for.resource.properties type Property struct {