From f33a2c780dfb0eb0aed6e29d51cb4931fdd2bb06 Mon Sep 17 00:00:00 2001 From: SenkjM Date: Mon, 26 Oct 2026 22:19:08 +0800 Subject: [PATCH] feat(drivers): add CtFile OpenAPI driver (#3214) - Add the CtFile OpenAPI storage driver with session authentication and common file operations. - Reuse cached streams and stream multipart uploads to avoid large-file memory duplication. - Fix copy cache handling when the API does not return a destination ID. - Guard upload caching when no progress callback is provided. Co-authored-by: Grok Bot Co-authored-by: Claude Co-authored-by: Suyunmeng Co-authored-by: LunelleEverdawn --- drivers/all.go | 1 + drivers/ctfile/driver.go | 252 +++++++++++++++ drivers/ctfile/meta.go | 44 +++ drivers/ctfile/types.go | 51 +++ drivers/ctfile/util.go | 598 ++++++++++++++++++++++++++++++++++++ drivers/ctfile/util_test.go | 352 +++++++++++++++++++++ 6 files changed, 1298 insertions(+) create mode 100644 drivers/ctfile/driver.go create mode 100644 drivers/ctfile/meta.go create mode 100644 drivers/ctfile/types.go create mode 100644 drivers/ctfile/util.go create mode 100644 drivers/ctfile/util_test.go diff --git a/drivers/all.go b/drivers/all.go index 601c7bfc..ea4fbd77 100644 --- a/drivers/all.go +++ b/drivers/all.go @@ -30,6 +30,7 @@ import ( _ "github.com/OpenListTeam/OpenList/v4/drivers/cloudreve_v4" _ "github.com/OpenListTeam/OpenList/v4/drivers/cnb_releases" _ "github.com/OpenListTeam/OpenList/v4/drivers/crypt" + _ "github.com/OpenListTeam/OpenList/v4/drivers/ctfile" _ "github.com/OpenListTeam/OpenList/v4/drivers/degoo" _ "github.com/OpenListTeam/OpenList/v4/drivers/doubao" _ "github.com/OpenListTeam/OpenList/v4/drivers/doubao_new" diff --git a/drivers/ctfile/driver.go b/drivers/ctfile/driver.go new file mode 100644 index 00000000..92464354 --- /dev/null +++ b/drivers/ctfile/driver.go @@ -0,0 +1,252 @@ +package ctfile + +import ( + "context" + "fmt" + "io" + "strings" + "time" + + "github.com/OpenListTeam/OpenList/v4/internal/driver" + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/internal/stream" + "github.com/OpenListTeam/OpenList/v4/pkg/utils" +) + +// CtFile implements driver.Driver for 城通网盘 (CtFile) via the OpenAPI +// session-based authentication. No email/password/SMS login is used. +type CtFile struct { + model.Storage + Addition +} + +func (d *CtFile) Config() driver.Config { + return config +} + +func (d *CtFile) GetAddition() driver.Additional { + return &d.Addition +} + +func (d *CtFile) Init(ctx context.Context) error { + if strings.TrimSpace(d.Session) == "" { + return fmt.Errorf("[CtFile] session is required; generate one in the CtFile backend under 「开放接口登录密钥管理」") + } + return d.profile(ctx) +} + +func (d *CtFile) Drop(ctx context.Context) error { + return nil +} + +// List returns the contents of dir (both sub-folders and files). +func (d *CtFile) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { + folderID := dir.GetID() + if folderID == "" { + folderID = rootFolderID + } + + folders, err := d.listFolders(ctx, folderID) + if err != nil { + return nil, err + } + files, err := d.listFiles(ctx, folderID) + if err != nil { + return nil, err + } + + var objs []model.Obj + for _, f := range folders { + objs = append(objs, &model.Object{ + ID: f.Key, + Name: f.Name, + IsFolder: true, + Modified: parseDate(f.Date), + Ctime: parseDate(f.Date), + }) + } + for _, f := range files { + objs = append(objs, &model.Object{ + ID: f.Key, + Name: f.Name, + Size: f.Size, + IsFolder: false, + Modified: parseDate(f.Date), + Ctime: parseDate(f.Date), + }) + } + return objs, nil +} + +// Link fetches the CDN download URL for file via fetch_url. +func (d *CtFile) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { + url, err := d.fetchURL(ctx, file.GetID()) + if err != nil { + return nil, err + } + return &model.Link{ + URL: url, + Header: map[string][]string{ + "User-Agent": {d.ua()}, + }, + }, nil +} + +// MakeDir creates a sub-folder under parentDir. +func (d *CtFile) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) (model.Obj, error) { + parentID := parentDir.GetID() + id, err := d.mkdir(ctx, parentID, dirName) + if err != nil { + return nil, err + } + return &model.Object{ + ID: id, + Name: dirName, + IsFolder: true, + Modified: time.Now(), + Ctime: time.Now(), + }, nil +} + +// Move relocates srcObj into dstDir. +func (d *CtFile) Move(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { + err := d.move(ctx, []string{srcObj.GetID()}, dstDir.GetID()) + if err != nil { + return nil, err + } + return srcObj, nil +} + +// Rename changes the name of srcObj to newName. +func (d *CtFile) Rename(ctx context.Context, srcObj model.Obj, newName string) (model.Obj, error) { + var err error + if srcObj.IsDir() { + err = d.renameFolder(ctx, srcObj.GetID(), newName) + } else { + err = d.renameFile(ctx, srcObj.GetID(), newName) + } + if err != nil { + return nil, err + } + return &model.Object{ + ID: srcObj.GetID(), + Name: newName, + Size: srcObj.GetSize(), + IsFolder: srcObj.IsDir(), + Modified: srcObj.ModTime(), + Ctime: srcObj.CreateTime(), + }, nil +} + +// Copy copies srcObj into dstDir. +// The spec explicitly requires implementation; the Worker's "unimplemented" +// note is overridden here. The API endpoint is POST /{ns}/file/copy. +func (d *CtFile) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { + return d.copyFiles(ctx, []string{srcObj.GetID()}, dstDir.GetID()) +} + +// Remove deletes obj (moves to trash). +func (d *CtFile) Remove(ctx context.Context, obj model.Obj) error { + return d.remove(ctx, []string{obj.GetID()}) +} + +// Put uploads a file to dstDir. The stream is cached so the checksum and the +// subsequent multipart upload can reuse it without keeping the entire file in memory. +func (d *CtFile) Put(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress) error { + md5hex := file.GetHash().GetHash(utils.MD5) + tmpFile := file.GetFile() + var err error + var cacheProgress *model.UpdateProgress + if up != nil { + cacheProgress = &up + } + + if len(md5hex) != utils.MD5.Width { + // Cache the full stream and compute MD5 simultaneously. + tmpFile, md5hex, err = stream.CacheFullAndHash(file, cacheProgress, utils.MD5) + } else if tmpFile == nil { + // A pre-computed MD5 still needs a seekable source for the upload. + tmpFile, err = file.CacheFullAndWriter(cacheProgress, nil) + } + if err != nil { + return err + } + if tmpFile == nil { + return fmt.Errorf("[CtFile] failed to cache upload stream") + } + if _, err := tmpFile.Seek(0, io.SeekStart); err != nil { + return err + } + + size := file.GetSize() + if size < 0 { + size, err = tmpFile.Seek(0, io.SeekEnd) + if err != nil { + return err + } + if _, err := tmpFile.Seek(0, io.SeekStart); err != nil { + return err + } + } + + if up != nil { + up(50) + } + + parentID := dstDir.GetID() + err = d.uploadFile(ctx, parentID, file.GetName(), tmpFile, size, md5hex) + if err != nil { + return err + } + + if up != nil { + up(100) + } + return nil +} + +// parseDate converts a CtFile date value (unix seconds or "YYYY-MM-DD HH:mm:ss") +// to time.Time, falling back to time.Now() on failure. +func parseDate(v any) time.Time { + if v == nil { + return time.Now() + } + switch tv := v.(type) { + case float64: + if tv > 0 { + return time.Unix(int64(tv), 0) + } + case string: + if tv == "" { + return time.Now() + } + // Try unix seconds as string + if n, err := parseInt64(tv); err == nil && n > 0 { + return time.Unix(n, 0) + } + // Try "YYYY-MM-DD HH:mm:ss" + layouts := []string{"2006-01-02 15:04:05", "2006-01-02T15:04:05Z"} + for _, layout := range layouts { + if t, err := time.Parse(layout, tv); err == nil { + return t + } + } + } + return time.Now() +} + +func parseInt64(s string) (int64, error) { + var n int64 + _, err := fmt.Sscanf(s, "%d", &n) + return n, err +} + +var ( + _ driver.Driver = (*CtFile)(nil) + _ driver.MkdirResult = (*CtFile)(nil) + _ driver.MoveResult = (*CtFile)(nil) + _ driver.RenameResult = (*CtFile)(nil) + _ driver.Copy = (*CtFile)(nil) + _ driver.Remove = (*CtFile)(nil) + _ driver.Put = (*CtFile)(nil) +) diff --git a/drivers/ctfile/meta.go b/drivers/ctfile/meta.go new file mode 100644 index 00000000..39bdd6ba --- /dev/null +++ b/drivers/ctfile/meta.go @@ -0,0 +1,44 @@ +package ctfile + +import ( + "github.com/OpenListTeam/OpenList/v4/internal/driver" + "github.com/OpenListTeam/OpenList/v4/internal/op" +) + +// Addition holds all user-configurable fields for the CtFile driver. +type Addition struct { + // Session is the OpenAPI token generated in the CtFile backend under + // 「开放接口登录密钥管理」. Required. + Session string `json:"session" required:"true" secret:"true" help:"OpenAPI session token from 开放接口登录密钥管理"` + + // CloudType selects namespace: "public" (default) or "private". + CloudType string `json:"cloud_type" type:"select" options:"public,private" default:"public"` + + // BaseURL is the API root, defaulting to https://rest.ctfile.com/v1. + BaseURL string `json:"base_url" default:"https://rest.ctfile.com/v1" help:"API base URL"` + + // OrderBy controls server-side sort field. + OrderBy string `json:"order_by" type:"select" options:"name,size,time" default:"name"` + + // OrderDirection is the sort direction. + OrderDirection string `json:"order_direction" type:"select" options:"asc,desc" default:"asc"` + + // UserAgent overrides the default Chrome-like UA sent with every request. + UserAgent string `json:"user_agent" default:"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/142.0.0.0 Safari/537.36"` + + driver.RootID +} + +var config = driver.Config{ + Name: "CtFile", + DefaultRoot: "0", + LocalSort: true, + NoUpload: false, + OnlyProxy: false, +} + +func init() { + op.RegisterDriver(func() driver.Driver { + return &CtFile{} + }) +} diff --git a/drivers/ctfile/types.go b/drivers/ctfile/types.go new file mode 100644 index 00000000..c65a2641 --- /dev/null +++ b/drivers/ctfile/types.go @@ -0,0 +1,51 @@ +package ctfile + +// CtFile (城通网盘) OpenAPI response structs. +// API doc: https://openapi.ctfile.com/ (REST API v1, unified POST JSON + session auth). +// All business endpoints return a unified wrapper { code, message?, ...business fields }. +// `code == 200` means success; `401` means session expired; `403` with folder_id on mkdir +// means the folder already exists. + +// Resp is the unified response wrapper. Business fields are decoded via +// utils.Json to stay tolerant of the server's loosely-typed payloads (numbers +// may arrive as strings and vice-versa). +type Resp struct { + Code int `json:"code"` + Message string `json:"message"` + // folder_id / file_id / download_url / upload_url / exists / results / ... + // are accessed through utils.Json, so we keep the raw bytes here. +} + +// profileResp holds the fields returned by POST /user/info/profile. +type profileResp struct { + UserID int64 `json:"userid"` + UserName string `json:"username"` + NickName string `json:"nick_name"` + GroupName string `json:"group_name"` + MaxFileSz int64 `json:"max_filesize"` +} + +// folderEntry is a single folder in folder/list results. +type folderEntry struct { + Key string `json:"key"` + Name string `json:"name"` + Icon string `json:"icon"` + Date any `json:"date"` // unix seconds or "YYYY-MM-DD HH:mm:ss" +} + +// fileEntry is a single file in file/list results. +type fileEntry struct { + Key string `json:"key"` + Name string `json:"name"` + Icon string `json:"icon"` + ImgSrc string `json:"imgsrc"` + Size int64 `json:"size"` + Date any `json:"date"` + Status int `json:"status"` // 1=complete, 2=incomplete (not downloadable) +} + +// uploadTicket is the POST /{ns}/file/upload response. +type uploadTicket struct { + UploadURL string `json:"upload_url"` + Exists int `json:"exists"` // >0 => instant upload hit +} diff --git a/drivers/ctfile/util.go b/drivers/ctfile/util.go new file mode 100644 index 00000000..1a7164b1 --- /dev/null +++ b/drivers/ctfile/util.go @@ -0,0 +1,598 @@ +package ctfile + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "mime/multipart" + "net/http" + "regexp" + "strconv" + "strings" + "time" +) + +const ( + defaultBaseURL = "https://rest.ctfile.com/v1" + defaultUserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/142.0.0.0 Safari/537.36" + rootFolderID = "d0" + maxListPages = 500 +) + +// sessionExpiredMsg is the human-readable error for 401 responses. +const sessionExpiredMsg = "[CtFile] Session expired (401). Please regenerate your session token in the CtFile backend under 「开放接口登录密钥管理」" + +// stripIDPrefix removes a leading d or f prefix from an ID when the next +// character is a digit. E.g. "d123" → "123", "f456" → "456", "abc" → "abc". +var stripPrefixRe = regexp.MustCompile(`^([df])(\d)`) + +func stripIDPrefix(id string) string { + if m := stripPrefixRe.FindStringSubmatch(id); m != nil { + return id[1:] // drop the leading letter + } + return id +} + +func (d *CtFile) ns() string { + if d.CloudType == "private" { + return "private" + } + return "public" +} + +func (d *CtFile) baseURL() string { + if u := strings.TrimRight(d.BaseURL, "/"); u != "" { + return u + } + return defaultBaseURL +} + +func (d *CtFile) ua() string { + if d.UserAgent != "" { + return d.UserAgent + } + return defaultUserAgent +} + +// orderby maps the user fields to CtFile's server-side sort tokens. +func (d *CtFile) orderby() string { + keys := map[string]string{"name": "az", "size": "big", "time": "new"} + rev := map[string]string{"az": "za", "big": "small", "new": "old"} + base, ok := keys[d.OrderBy] + if !ok { + base = "az" + } + if d.OrderDirection == "desc" { + return rev[base] + } + return base +} + +// post is the raw transport layer: POST JSON, include session, retry on +// 429/5xx, map 401 to a clear error. Returns decoded JSON map or error. +func (d *CtFile) post(ctx context.Context, path string, body map[string]any) (map[string]any, error) { + session := strings.TrimSpace(d.Session) + if session == "" { + return nil, fmt.Errorf("[CtFile] session is not configured; please fill in the session token from 开放接口登录密钥管理") + } + + payload := make(map[string]any, len(body)+1) + for k, v := range body { + payload[k] = v + } + payload["session"] = session + + data, err := json.Marshal(payload) + if err != nil { + return nil, fmt.Errorf("[CtFile] json marshal: %w", err) + } + + url := d.baseURL() + path + var lastErr error + for attempt := 0; attempt < 3; attempt++ { + if attempt > 0 { + select { + case <-ctx.Done(): + return nil, ctx.Err() + case <-time.After(time.Duration(300*attempt) * time.Millisecond): + } + } + + req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(data)) + if err != nil { + return nil, err + } + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Accept", "application/json") + req.Header.Set("User-Agent", d.ua()) + + resp, err := http.DefaultClient.Do(req) + if err != nil { + lastErr = err + continue + } + + // Retry on 429 / 5xx + if resp.StatusCode == 429 || resp.StatusCode >= 500 { + _ = resp.Body.Close() + lastErr = fmt.Errorf("HTTP %d", resp.StatusCode) + continue + } + + body2, err := io.ReadAll(resp.Body) + _ = resp.Body.Close() + if err != nil { + return nil, err + } + + var result map[string]any + if err := json.Unmarshal(body2, &result); err != nil { + return nil, fmt.Errorf("[CtFile] %s: non-JSON response (HTTP %d): %.200s", path, resp.StatusCode, body2) + } + + code := int(toInt64(result["code"])) + if code == 401 { + return nil, fmt.Errorf("%s", sessionExpiredMsg) + } + if code == 429 { + lastErr = fmt.Errorf("[CtFile] rate limited (code 429)") + continue + } + return result, nil + } + if lastErr != nil { + return nil, fmt.Errorf("[CtFile] %s failed after retries: %w", path, lastErr) + } + return nil, fmt.Errorf("[CtFile] %s failed after retries", path) +} + +// call wraps post and checks code == 200. +func (d *CtFile) call(ctx context.Context, path string, body map[string]any) (map[string]any, error) { + result, err := d.post(ctx, path, body) + if err != nil { + return nil, err + } + // Missing / zero code is treated as success (Worker: code === undefined → ok). + if _, hasCode := result["code"]; hasCode { + code := int(toInt64(result["code"])) + if code != 200 { + return result, &apiError{code: code, message: toString(result["message"]), raw: result} + } + } + return result, nil +} + +// callWithId calls an endpoint that takes a single id parameter, implementing +// the dual-try strategy (prefer one form; on 400/404 retry the other). +// +// prefer = "raw" → try id as-is first, fallback to stripped +// prefer = "stripped" → try stripped first, fallback to raw +func (d *CtFile) callWithId(ctx context.Context, path, idKey, id string, extra map[string]any, prefer string) (map[string]any, error) { + stripped := stripIDPrefix(id) + var primary, secondary string + if prefer == "stripped" { + primary, secondary = stripped, id + } else { + primary, secondary = id, stripped + } + + body := make(map[string]any, len(extra)+1) + for k, v := range extra { + body[k] = v + } + + body[idKey] = primary + result, err := d.call(ctx, path, body) + if err != nil { + var ae *apiError + if asAPIError(err, &ae) && (ae.code == 400 || ae.code == 404) && primary != secondary { + body[idKey] = secondary + return d.call(ctx, path, body) + } + return nil, err + } + return result, nil +} + +// callWithIds calls an endpoint that takes a list of ids; tries comma-string +// first, on 400/404/405 retries JSON array. +func (d *CtFile) callWithIds(ctx context.Context, path string, ids []string, extra map[string]any) (map[string]any, error) { + body := make(map[string]any, len(extra)+2) + for k, v := range extra { + body[k] = v + } + + // First attempt: comma-separated string + body["ids"] = strings.Join(ids, ",") + result, err := d.call(ctx, path, body) + if err != nil { + var ae *apiError + if asAPIError(err, &ae) && (ae.code == 400 || ae.code == 404 || ae.code == 405) { + // Second attempt: JSON array + body["ids"] = ids + return d.call(ctx, path, body) + } + return nil, err + } + return result, nil +} + +// -------------------------------------------------------------------------- +// High-level API helpers +// -------------------------------------------------------------------------- + +// profile fetches the user's profile; used to validate the session in Init. +func (d *CtFile) profile(ctx context.Context) error { + _, err := d.call(ctx, "/user/info/profile", nil) + return err +} + +// listFolders returns all sub-folders for a given parent folder ID. +func (d *CtFile) listFolders(ctx context.Context, folderID string) ([]folderEntry, error) { + if folderID == "" || folderID == "0" { + folderID = rootFolderID + } + result, err := d.call(ctx, "/"+d.ns()+"/folder/list", map[string]any{ + "folder_id": folderID, + "orderby": d.orderby(), + }) + if err != nil { + return nil, err + } + items := toSlice(result["results"]) + out := make([]folderEntry, 0, len(items)) + for _, raw := range items { + m, ok := raw.(map[string]any) + if !ok { + continue + } + out = append(out, folderEntry{ + Key: toString(m["key"]), + Name: toString(m["name"]), + Icon: toString(m["icon"]), + Date: m["date"], + }) + } + return out, nil +} + +// listFiles returns all files in a folder, paginating until empty or duplicate. +func (d *CtFile) listFiles(ctx context.Context, folderID string) ([]fileEntry, error) { + if folderID == "" || folderID == "0" { + folderID = rootFolderID + } + var all []fileEntry + start := 0 + prevFirstKey := "" + + for page := 0; page < maxListPages; page++ { + result, err := d.call(ctx, "/"+d.ns()+"/file/list", map[string]any{ + "folder_id": folderID, + "start": start, + "orderby": d.orderby(), + }) + if err != nil { + return nil, err + } + items := toSlice(result["results"]) + if len(items) == 0 { + break + } + + // Guard against server ignoring `start` and looping forever + firstKey := "" + if m, ok := items[0].(map[string]any); ok { + firstKey = toString(m["key"]) + } + if page > 0 && firstKey == prevFirstKey { + break + } + prevFirstKey = firstKey + + for _, raw := range items { + m, ok := raw.(map[string]any) + if !ok { + continue + } + all = append(all, fileEntry{ + Key: toString(m["key"]), + Name: toString(m["name"]), + Icon: toString(m["icon"]), + ImgSrc: toString(m["imgsrc"]), + Size: toInt64(m["size"]), + Date: m["date"], + Status: int(toInt64(m["status"])), + }) + } + start += len(items) + } + return all, nil +} + +// mkdir creates a folder and returns its ID. 403 with an existing folder_id is +// treated as success (folder already exists). +func (d *CtFile) mkdir(ctx context.Context, parentID, name string) (string, error) { + if parentID == "" || parentID == "0" { + parentID = rootFolderID + } + result, err := d.call(ctx, "/"+d.ns()+"/folder/create", map[string]any{ + "folder_id": parentID, + "name": name, + "description": "", + "is_hidden": false, + }) + if err != nil { + var ae *apiError + if asAPIError(err, &ae) && ae.code == 403 { + // Folder already exists; the response still carries folder_id + if id := toString(ae.raw["folder_id"]); id != "" { + return id, nil + } + } + return "", err + } + id := toString(result["folder_id"]) + if id == "" { + id = toString(result["path"]) + } + return id, nil +} + +// fetchURL returns the CDN download URL for a file. +func (d *CtFile) fetchURL(ctx context.Context, fileID string) (string, error) { + result, err := d.callWithId(ctx, "/"+d.ns()+"/file/fetch_url", "file_id", fileID, nil, "stripped") + if err != nil { + return "", err + } + url := toString(result["download_url"]) + if url == "" { + return "", fmt.Errorf("[CtFile] fetch_url returned empty download_url") + } + return url, nil +} + +// renameFile renames a file. +func (d *CtFile) renameFile(ctx context.Context, fileID, newName string) error { + _, err := d.callWithId(ctx, "/"+d.ns()+"/file/modify_meta", "file_id", fileID, map[string]any{"name": newName}, "stripped") + return err +} + +// renameFolder renames a folder. +func (d *CtFile) renameFolder(ctx context.Context, folderID, newName string) error { + _, err := d.callWithId(ctx, "/"+d.ns()+"/folder/modify_meta", "folder_id", folderID, map[string]any{ + "name": newName, + "is_rename": true, + }, "raw") + return err +} + +// remove moves files/folders to trash. +func (d *CtFile) remove(ctx context.Context, ids []string) error { + if len(ids) == 0 { + return nil + } + _, err := d.callWithIds(ctx, "/"+d.ns()+"/file/delete", ids, nil) + return err +} + +// move moves files/folders to another folder. +func (d *CtFile) move(ctx context.Context, ids []string, targetFolderID string) error { + if len(ids) == 0 { + return nil + } + if targetFolderID == "" || targetFolderID == "0" { + targetFolderID = rootFolderID + } + _, err := d.callWithIds(ctx, "/"+d.ns()+"/file/move", ids, map[string]any{"folder_id": targetFolderID}) + return err +} + +// copyFiles copies files to a target folder. +func (d *CtFile) copyFiles(ctx context.Context, ids []string, targetFolderID string) error { + if len(ids) == 0 { + return nil + } + if targetFolderID == "" || targetFolderID == "0" { + targetFolderID = rootFolderID + } + _, err := d.callWithIds(ctx, "/"+d.ns()+"/file/copy", ids, map[string]any{"folder_id": targetFolderID}) + return err +} + +// uploadFile uploads a stream to CtFile using the two-step ticket+multipart flow. +// checksum format: md5hex + "-" + size +func (d *CtFile) uploadFile(ctx context.Context, folderID, fileName string, content io.Reader, size int64, md5hex string) error { + if folderID == "" || folderID == "0" { + folderID = rootFolderID + } + + checksum := fmt.Sprintf("%s-%d", md5hex, size) + + // Step 1: get upload ticket + result, err := d.call(ctx, "/"+d.ns()+"/file/upload", map[string]any{ + "folder_id": folderID, + "checksum": checksum, + "size": size, + "name": fileName, + }) + if err != nil { + return err + } + + exists := toInt64(result["exists"]) + if exists > 0 { + // Instant upload / deduplication + return nil + } + + uploadURL := toString(result["upload_url"]) + if uploadURL == "" { + return fmt.Errorf("[CtFile] upload ticket returned no upload_url") + } + + // Step 2: multipart upload to the presigned URL + return d.multipartUpload(ctx, uploadURL, fileName, content, size) +} + +// countingWriter counts bytes without retaining the written data. +type countingWriter int64 + +func (w *countingWriter) Write(p []byte) (int, error) { + *w += countingWriter(len(p)) + return len(p), nil +} + +func writeMultipartFields(mw *multipart.Writer, fileName string, size int64) (io.Writer, error) { + if err := mw.WriteField("name", fileName); err != nil { + return nil, err + } + if err := mw.WriteField("filesize", strconv.FormatInt(size, 10)); err != nil { + return nil, err + } + return mw.CreateFormFile("file", fileName) +} + +func multipartLayout(fileName string, size int64) (int64, string, error) { + var count countingWriter + mw := multipart.NewWriter(&count) + if _, err := writeMultipartFields(mw, fileName, size); err != nil { + return 0, "", err + } + if err := mw.Close(); err != nil { + return 0, "", err + } + return int64(count) + size, mw.Boundary(), nil +} + +// multipartUpload POSTs name+filesize+file to a presigned upload URL without +// retaining the complete multipart body in memory. +func (d *CtFile) multipartUpload(ctx context.Context, uploadURL, fileName string, content io.Reader, size int64) error { + contentLength, boundary, err := multipartLayout(fileName, size) + if err != nil { + return err + } + + pr, pw := io.Pipe() + req, err := http.NewRequestWithContext(ctx, http.MethodPost, uploadURL, pr) + if err != nil { + return err + } + req.ContentLength = contentLength + req.Header.Set("Content-Type", "multipart/form-data; boundary="+boundary) + req.Header.Set("User-Agent", d.ua()) + + go func() { + mw := multipart.NewWriter(pw) + if err := mw.SetBoundary(boundary); err != nil { + _ = pw.CloseWithError(err) + return + } + fw, err := writeMultipartFields(mw, fileName, size) + if err != nil { + _ = pw.CloseWithError(err) + return + } + if _, err := io.Copy(fw, content); err != nil { + _ = pw.CloseWithError(err) + return + } + if err := mw.Close(); err != nil { + _ = pw.CloseWithError(err) + return + } + _ = pw.Close() + }() + + resp, err := http.DefaultClient.Do(req) + if err != nil { + _ = pr.CloseWithError(err) + return err + } + defer resp.Body.Close() + + body, err := io.ReadAll(resp.Body) + if err != nil { + return err + } + + if resp.StatusCode != 200 { + return fmt.Errorf("[CtFile] multipart upload HTTP %d: %.200s", resp.StatusCode, body) + } + + var result map[string]any + if err := json.Unmarshal(body, &result); err != nil { + return fmt.Errorf("[CtFile] multipart upload non-JSON: %.200s", body) + } + code := int(toInt64(result["code"])) + if code != 200 { + return fmt.Errorf("[CtFile] multipart upload failed (code=%d): %s", code, toString(result["message"])) + } + return nil +} + +// -------------------------------------------------------------------------- +// apiError carries a structured error with code + raw body for retry checks. +// -------------------------------------------------------------------------- + +type apiError struct { + code int + message string + raw map[string]any +} + +func (e *apiError) Error() string { + return fmt.Sprintf("[CtFile] API error (code=%d): %s", e.code, e.message) +} + +// asAPIError sets dst and returns true when err is an *apiError. +func asAPIError(err error, dst **apiError) bool { + if ae, ok := err.(*apiError); ok { + *dst = ae + return true + } + return false +} + +// -------------------------------------------------------------------------- +// Type-coercion helpers for loosely-typed JSON payloads. +// -------------------------------------------------------------------------- + +func toInt64(v any) int64 { + if v == nil { + return 0 + } + switch n := v.(type) { + case float64: + return int64(n) + case int64: + return n + case int: + return int64(n) + case string: + i, _ := strconv.ParseInt(n, 10, 64) + return i + default: + return 0 + } +} + +func toString(v any) string { + if v == nil { + return "" + } + switch s := v.(type) { + case string: + return s + default: + return fmt.Sprintf("%v", v) + } +} + +func toSlice(v any) []any { + if s, ok := v.([]any); ok { + return s + } + return nil +} diff --git a/drivers/ctfile/util_test.go b/drivers/ctfile/util_test.go new file mode 100644 index 00000000..c8e03c51 --- /dev/null +++ b/drivers/ctfile/util_test.go @@ -0,0 +1,352 @@ +package ctfile + +import ( + "context" + "crypto/md5" + "fmt" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" +) + +// --------------------------------------------------------------------------- +// Helpers / mock +// --------------------------------------------------------------------------- + +// buildServer creates a test HTTP server that returns pre-scripted responses in +// sequence. Each response is a JSON string (already complete). The variadic +// responses are consumed one at a time; the last one is reused for all +// subsequent requests. +func buildServer(t *testing.T, responses ...string) (*httptest.Server, *[]string) { + t.Helper() + calls := new([]string) + idx := 0 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body, _ := io.ReadAll(r.Body) + *calls = append(*calls, string(body)) + + resp := responses[idx] + if idx < len(responses)-1 { + idx++ + } + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(200) + _, _ = w.Write([]byte(resp)) + })) + return srv, calls +} + +// driverWithBase returns a minimal CtFile driver pointing at the test server. +func driverWithBase(baseURL string) *CtFile { + d := &CtFile{} + d.Session = "test-session" + d.BaseURL = baseURL + d.CloudType = "public" + d.OrderBy = "name" + d.OrderDirection = "asc" + return d +} + +// --------------------------------------------------------------------------- +// StripIDPrefix +// --------------------------------------------------------------------------- + +func TestStripIDPrefix(t *testing.T) { + cases := []struct { + in string + out string + }{ + {"d123", "123"}, + {"f456", "456"}, + {"abc", "abc"}, + {"d", "d"}, // no digit follows → keep as-is + {"f", "f"}, // no digit follows → keep as-is + {"d0", "0"}, + {"", ""}, + {"123", "123"}, + } + for _, tc := range cases { + got := stripIDPrefix(tc.in) + if got != tc.out { + t.Errorf("stripIDPrefix(%q) = %q, want %q", tc.in, got, tc.out) + } + } +} + +// --------------------------------------------------------------------------- +// callWithId dual-try: 400 → retry with alternate id form +// --------------------------------------------------------------------------- + +func TestCallWithId_DualTry(t *testing.T) { + // First call returns code 400; second returns code 200 with a name field. + srv, calls := buildServer(t, + `{"code":400,"message":"not found"}`, + `{"code":200,"name":"Photos"}`, + ) + defer srv.Close() + + d := driverWithBase(srv.URL) + ctx := context.Background() + + // prefer = "raw" → first attempt with "d123", fallback with "123" + result, err := d.callWithId(ctx, "/public/folder/meta", "folder_id", "d123", nil, "raw") + if err != nil { + t.Fatalf("callWithId returned error: %v", err) + } + if toString(result["name"]) != "Photos" { + t.Errorf("expected name=Photos, got %v", result["name"]) + } + if len(*calls) != 2 { + t.Fatalf("expected 2 HTTP calls, got %d", len(*calls)) + } + if !strings.Contains((*calls)[0], `"d123"`) { + t.Errorf("first call should use raw id d123, got: %s", (*calls)[0]) + } + if !strings.Contains((*calls)[1], `"123"`) { + t.Errorf("second call should use stripped id 123, got: %s", (*calls)[1]) + } +} + +// --------------------------------------------------------------------------- +// callWithId: prefer stripped → file/fetch_url pattern +// --------------------------------------------------------------------------- + +func TestCallWithId_PreferStripped(t *testing.T) { + // First call (stripped "456") returns 400; second (raw "f456") returns 200. + srv, calls := buildServer(t, + `{"code":400,"message":"param error"}`, + `{"code":200,"download_url":"https://cdn.example.com/file"}`, + ) + defer srv.Close() + + d := driverWithBase(srv.URL) + ctx := context.Background() + + result, err := d.callWithId(ctx, "/public/file/fetch_url", "file_id", "f456", nil, "stripped") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if toString(result["download_url"]) != "https://cdn.example.com/file" { + t.Errorf("unexpected download_url: %v", result["download_url"]) + } + if len(*calls) != 2 { + t.Fatalf("expected 2 calls, got %d", len(*calls)) + } + if !strings.Contains((*calls)[0], `"456"`) { + t.Errorf("first call should use stripped 456: %s", (*calls)[0]) + } + if !strings.Contains((*calls)[1], `"f456"`) { + t.Errorf("second call should use raw f456: %s", (*calls)[1]) + } +} + +// --------------------------------------------------------------------------- +// callWithIds: comma-string → JSON array fallback on 400 +// --------------------------------------------------------------------------- + +func TestCallWithIds_DualTry(t *testing.T) { + srv, calls := buildServer(t, + `{"code":400,"message":"param error"}`, + `{"code":200,"message":"deleted"}`, + ) + defer srv.Close() + + d := driverWithBase(srv.URL) + ctx := context.Background() + + err := d.remove(ctx, []string{"f1", "f2"}) + if err != nil { + t.Fatalf("remove returned error: %v", err) + } + if len(*calls) != 2 { + t.Fatalf("expected 2 calls, got %d", len(*calls)) + } + // First attempt: ids is comma-string + if !strings.Contains((*calls)[0], `"f1,f2"`) { + t.Errorf("first call should have comma-string ids: %s", (*calls)[0]) + } + // Second attempt: ids is JSON array + if !strings.Contains((*calls)[1], `["f1","f2"]`) { + t.Errorf("second call should have JSON array ids: %s", (*calls)[1]) + } +} + +// --------------------------------------------------------------------------- +// Checksum format: md5hex-size +// --------------------------------------------------------------------------- + +func TestChecksumFormat(t *testing.T) { + content := []byte("hello") + sum := md5.Sum(content) + checksum := fmt.Sprintf("%x-%d", sum, len(content)) + // md5("hello") = 5d41402abc4b2a76b9719d911017c592 + want := "5d41402abc4b2a76b9719d911017c592-5" + if checksum != want { + t.Errorf("checksum = %q, want %q", checksum, want) + } +} + +// --------------------------------------------------------------------------- +// 401 → session expired message +// --------------------------------------------------------------------------- + +func TestSession401Error(t *testing.T) { + srv, _ := buildServer(t, `{"code":401,"message":"not logged in"}`) + defer srv.Close() + + d := driverWithBase(srv.URL) + ctx := context.Background() + + err := d.profile(ctx) + if err == nil { + t.Fatal("expected error for 401, got nil") + } + msg := err.Error() + if !strings.Contains(msg, "401") && !strings.Contains(msg, "expired") && !strings.Contains(msg, "Session") { + t.Errorf("error message should mention session/expired/401; got: %s", msg) + } + if !strings.Contains(msg, "开放接口登录密钥管理") { + t.Errorf("error message should mention 开放接口登录密钥管理; got: %s", msg) + } +} + +// --------------------------------------------------------------------------- +// mkdir: 403 with folder_id → treat as success +// --------------------------------------------------------------------------- + +func TestMkdir403Success(t *testing.T) { + // Server returns 403 with an existing folder_id + srv, _ := buildServer(t, `{"code":403,"message":"folder exists","folder_id":"d999"}`) + defer srv.Close() + + d := driverWithBase(srv.URL) + ctx := context.Background() + + id, err := d.mkdir(ctx, "d0", "MyFolder") + if err != nil { + t.Fatalf("mkdir should succeed on 403 with folder_id, got error: %v", err) + } + if id != "d999" { + t.Errorf("expected id=d999, got %q", id) + } +} + +// --------------------------------------------------------------------------- +// list files: pagination stops on empty page +// --------------------------------------------------------------------------- + +func TestListFiles_Pagination(t *testing.T) { + // Page 1: two files; page 2: empty + srv, calls := buildServer(t, + `{"code":200,"results":[{"key":"f1","name":"a.txt","size":100},{"key":"f2","name":"b.txt","size":200}]}`, + `{"code":200,"results":[]}`, + ) + defer srv.Close() + + d := driverWithBase(srv.URL) + ctx := context.Background() + + files, err := d.listFiles(ctx, "d0") + if err != nil { + t.Fatalf("listFiles: %v", err) + } + if len(files) != 2 { + t.Errorf("expected 2 files, got %d", len(files)) + } + if len(*calls) != 2 { + t.Errorf("expected 2 HTTP calls (2 pages), got %d", len(*calls)) + } +} + +// --------------------------------------------------------------------------- +// list files: duplicate first key terminates pagination (anti-loop guard) +// --------------------------------------------------------------------------- + +func TestListFiles_DuplicateKeyStops(t *testing.T) { + sameResp := `{"code":200,"results":[{"key":"f1","name":"a.txt","size":1},{"key":"f2","name":"b.txt","size":2}]}` + srv, calls := buildServer(t, sameResp, sameResp, sameResp) + defer srv.Close() + + d := driverWithBase(srv.URL) + ctx := context.Background() + + files, err := d.listFiles(ctx, "d0") + if err != nil { + t.Fatalf("listFiles: %v", err) + } + // Should stop after 2 calls (first page + duplicate detection on second) + if len(*calls) != 2 { + t.Errorf("expected 2 calls (duplicate guard), got %d", len(*calls)) + } + if len(files) != 2 { + t.Errorf("expected 2 files, got %d", len(files)) + } +} + +// --------------------------------------------------------------------------- +// namespace: private cloud uses /private/ prefix +// --------------------------------------------------------------------------- + +func TestNamespace_Private(t *testing.T) { + srv, calls := buildServer(t, `{"code":200,"results":[]}`) + defer srv.Close() + + d := driverWithBase(srv.URL) + d.CloudType = "private" + ctx := context.Background() + + _, _ = d.listFolders(ctx, "d0") + if len(*calls) < 1 { + t.Fatal("expected at least 1 call") + } + // The request URL path should contain /private/ + // We can't directly read the URL from body, but we can check the server + // received a body referencing private — instead check via a separate handler. + // We verify indirectly by checking the test server was hit (calls recorded). +} + +func TestNamespace_PrivateURL(t *testing.T) { + var gotPath string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotPath = r.URL.Path + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(200) + _, _ = w.Write([]byte(`{"code":200,"results":[]}`)) + })) + defer srv.Close() + + d := driverWithBase(srv.URL) + d.CloudType = "private" + ctx := context.Background() + + _, _ = d.listFolders(ctx, "d0") + if !strings.Contains(gotPath, "/private/") { + t.Errorf("expected /private/ in path, got %q", gotPath) + } +} + +// --------------------------------------------------------------------------- +// session missing → clear error +// --------------------------------------------------------------------------- + +func TestSessionMissing(t *testing.T) { + d := &CtFile{} + d.Session = "" + d.BaseURL = "http://localhost:9" + ctx := context.Background() + + err := d.profile(ctx) + if err == nil { + t.Fatal("expected error for missing session") + } + if !strings.Contains(err.Error(), "session") { + t.Errorf("error should mention session, got: %v", err) + } +} + +// --------------------------------------------------------------------------- +// helper: background context for tests (unused now — kept as reference) +// --------------------------------------------------------------------------- +