diff --git a/docker/.env b/docker/.env index a98ff61fe9..ace2a8ba3a 100644 --- a/docker/.env +++ b/docker/.env @@ -352,10 +352,12 @@ TZ=Asia/Shanghai # Uncomment the following line if your operating system is MacOS: # MACOS=1 -# The maximum file size limit (in bytes) for each upload to your dataset or RAGFlow's File system. -# To change the 1GB file size limit, uncomment the line below and update as needed. -# MAX_CONTENT_LENGTH=1073741824 -# After updating, ensure `client_max_body_size` in nginx/nginx.conf is updated accordingly. +# Maximum total request-body size for multipart uploads (default 1 GiB). +# MAX_MULTIPART_CONTENT_LENGTH=1073741824 +# MAX_CONTENT_LENGTH=1073741824 # fallback name when MAX_MULTIPART_CONTENT_LENGTH is unset +# Maximum size in bytes for one dataset document upload (default 128 MiB). +# MAX_DOCUMENT_UPLOAD_SIZE=134217728 +# When raising the multipart limit, ensure `client_max_body_size` in nginx/nginx.conf is updated accordingly. # Note that neither `MAX_CONTENT_LENGTH` nor `client_max_body_size` sets the maximum size for files uploaded to an agent. # See https://ragflow.io/docs/dev/begin_component for details. @@ -557,4 +559,3 @@ OAUTH_AUTO_REGISTER=true # RAGFLOW_MCP_PORT=9382 # RAGFLOW_MCP_LAUNCH_MODE=self-host # RAGFLOW_MCP_HOST_API_KEY= - diff --git a/docs/faq.mdx b/docs/faq.mdx index 21d053efe8..cf902a8d09 100644 --- a/docs/faq.mdx +++ b/docs/faq.mdx @@ -328,7 +328,7 @@ If your model is not currently supported but has APIs compatible with those of O The Go API limits non-multipart request bodies to 1 MiB by default. Set `MAX_JSON_CONTENT_LENGTH` to change that limit in bytes. Multipart upload requests share a 1 GiB default total-body limit; set `MAX_MULTIPART_CONTENT_LENGTH` to change it in bytes. Upload endpoints can enforce smaller per-file or per-request limits. -The file-manager per-file limit follows `MAX_MULTIPART_CONTENT_LENGTH` (1 GiB by default). Dataset documents allow 128 MiB per file, and document-info uploads allow 100 MiB per file; increasing the shared multipart limit does not raise those per-file limits. +The file-manager per-file limit follows `MAX_MULTIPART_CONTENT_LENGTH` (1 GiB by default). Dataset documents allow 128 MiB per file by default; set `MAX_DOCUMENT_UPLOAD_SIZE` to change that limit in bytes. The total multipart request limit must also be large enough for the file and multipart overhead. Document-info uploads allow 100 MiB per file, and increasing the shared multipart limit does not raise that limit. If you raise the multipart limit above 1 GiB, also increase `client_max_body_size` in **docker/nginx/nginx.conf**. The Go image contains its own copy of this file, so make the change effective by either rebuilding `RAGFLOW_IMAGE` or enabling the `./nginx/nginx.conf:/etc/nginx/nginx.conf` bind mount for the deployed service in **docker/docker-compose.yml**, then recreate that service. diff --git a/internal/common/environments.go b/internal/common/environments.go index 2d5e673656..40120f83c9 100644 --- a/internal/common/environments.go +++ b/internal/common/environments.go @@ -89,6 +89,7 @@ const ( EnvMaxContentLength = "MAX_CONTENT_LENGTH" EnvMaxJSONContentLength = "MAX_JSON_CONTENT_LENGTH" EnvMaxMultipartContentLength = "MAX_MULTIPART_CONTENT_LENGTH" + EnvMaxDocumentUploadSize = "MAX_DOCUMENT_UPLOAD_SIZE" EnvRAGFlowDictPath = "RAGFLOW_DICT_PATH" EnvDefaultSuperuserEmail = "DEFAULT_SUPERUSER_EMAIL" EnvDefaultSuperuserNickname = "DEFAULT_SUPERUSER_NICKNAME" diff --git a/internal/common/http.go b/internal/common/http.go index d106e7d58a..d05c291bd2 100644 --- a/internal/common/http.go +++ b/internal/common/http.go @@ -43,6 +43,8 @@ const ( DefaultMaxJSONRequestBodyBytes int64 = 1 << 20 // DefaultMaxMultipartRequestBodyBytes is the default total limit for multipart uploads. DefaultMaxMultipartRequestBodyBytes int64 = 1 << 30 + // DefaultMaxDocumentUploadBytes is the default per-file limit for dataset document uploads. + DefaultMaxDocumentUploadBytes int64 = 128 << 20 // MaxWebhookRequestBodyBytes is the default and maximum request body size for agent webhooks. MaxWebhookRequestBodyBytes int64 = 10 * 1_000_000 ) @@ -50,17 +52,23 @@ const ( // GetMaxJSONRequestBodyBytes returns the configured non-multipart request-body // limit in bytes, defaulting to 1 MiB. func GetMaxJSONRequestBodyBytes() int64 { - return getMaxRequestBodyBytes(EnvMaxJSONContentLength, "", DefaultMaxJSONRequestBodyBytes) + return getMaxConfiguredBytes(EnvMaxJSONContentLength, "", DefaultMaxJSONRequestBodyBytes) } // GetMaxMultipartRequestBodyBytes returns the configured multipart upload // request-body limit in bytes, defaulting to 1 GiB. MAX_CONTENT_LENGTH remains // a fallback for existing deployments when the new variable is unset. func GetMaxMultipartRequestBodyBytes() int64 { - return getMaxRequestBodyBytes(EnvMaxMultipartContentLength, EnvMaxContentLength, DefaultMaxMultipartRequestBodyBytes) + return getMaxConfiguredBytes(EnvMaxMultipartContentLength, EnvMaxContentLength, DefaultMaxMultipartRequestBodyBytes) } -func getMaxRequestBodyBytes(envName, fallbackEnvName string, defaultBytes int64) int64 { +// GetMaxDocumentUploadBytes returns the configured per-file limit for dataset document uploads, +// defaulting to 128 MiB. +func GetMaxDocumentUploadBytes() int64 { + return getMaxConfiguredBytes(EnvMaxDocumentUploadSize, "", DefaultMaxDocumentUploadBytes) +} + +func getMaxConfiguredBytes(envName, fallbackEnvName string, defaultBytes int64) int64 { raw := strings.TrimSpace(GetEnv(envName)) if raw == "" && fallbackEnvName != "" { raw = strings.TrimSpace(GetEnv(fallbackEnvName)) diff --git a/internal/common/http_test.go b/internal/common/http_test.go index e72c94b527..b2e1fa9280 100644 --- a/internal/common/http_test.go +++ b/internal/common/http_test.go @@ -72,6 +72,27 @@ func (f roundTripFunc) RoundTrip(req *http.Request) (*http.Response, error) { return f(req) } +func TestGetMaxDocumentUploadBytes(t *testing.T) { + tests := []struct { + name string + value string + want int64 + }{ + {name: "default", want: DefaultMaxDocumentUploadBytes}, + {name: "configured", value: "4096", want: 4096}, + {name: "invalid uses default", value: "invalid", want: DefaultMaxDocumentUploadBytes}, + {name: "nonpositive uses default", value: "0", want: DefaultMaxDocumentUploadBytes}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Setenv(EnvMaxDocumentUploadSize, tt.value) + if got := GetMaxDocumentUploadBytes(); got != tt.want { + t.Fatalf("GetMaxDocumentUploadBytes() = %d, want %d", got, tt.want) + } + }) + } +} + func TestProviderLoggingRecordsTimingsAndTruncatesVectors(t *testing.T) { core, logs := observer.New(zapcore.InfoLevel) previousLogger := Logger diff --git a/internal/service/document/document.go b/internal/service/document/document.go index 61361adfc5..f23c6f27d8 100644 --- a/internal/service/document/document.go +++ b/internal/service/document/document.go @@ -267,10 +267,6 @@ type valueInfo struct { // that holds per-dataset subfolders, mirroring Python KNOWLEDGEBASE_FOLDER_NAME. const knowledgebaseFolderName = ".knowledgebase" -// maxUploadDocSize bounds a single uploaded file held in memory, mirroring the -// Python DOC_MAXIMUM_SIZE default (128 MiB; overridable there via MAX_CONTENT_LENGTH). -const maxUploadDocSize = 128 * 1024 * 1024 - // MetadataUpdate is one update item: set key to value. type MetadataUpdate struct { Key string `json:"key"` diff --git a/internal/service/document/document_test.go b/internal/service/document/document_test.go index e72a56222a..ba0e731038 100644 --- a/internal/service/document/document_test.go +++ b/internal/service/document/document_test.go @@ -628,6 +628,23 @@ func makeTestFileHeader(t *testing.T, field, filename string, content []byte) *m return fhs[0] } +func TestReadFileHeaderBytesUsesConfiguredMax(t *testing.T) { + t.Setenv(common.EnvMaxDocumentUploadSize, "4") + + blob, err := readFileHeaderBytes(makeTestFileHeader(t, "file", "small.txt", []byte("1234"))) + if err != nil { + t.Fatalf("readFileHeaderBytes() error = %v, want success at configured limit", err) + } + if string(blob) != "1234" { + t.Fatalf("readFileHeaderBytes() = %q, want %q", blob, "1234") + } + + _, err = readFileHeaderBytes(makeTestFileHeader(t, "file", "large.txt", []byte("12345"))) + if err == nil || !strings.Contains(err.Error(), "4 bytes") { + t.Fatalf("readFileHeaderBytes() error = %v, want configured 4-byte limit", err) + } +} + // sptr returns a pointer to the given string. func sptr(s string) *string { return &s } diff --git a/internal/service/document/document_upload.go b/internal/service/document/document_upload.go index 2eb781cb83..6b9e58a9e4 100644 --- a/internal/service/document/document_upload.go +++ b/internal/service/document/document_upload.go @@ -242,7 +242,7 @@ func (s *DocumentService) UploadWebDocument(ctx context.Context, kb *entity.Know return nil, common.CodeServerError, err } - blob, headers, _, err := utility.FetchRemoteFileSafely(ctx, url, maxUploadDocSize) + blob, headers, _, err := utility.FetchRemoteFileSafely(ctx, url, common.GetMaxDocumentUploadBytes()) if err != nil { return nil, common.CodeDataError, err } @@ -385,20 +385,21 @@ func docToRawMap(doc *entity.Document) map[string]interface{} { } func readFileHeaderBytes(fh *multipart.FileHeader) ([]byte, error) { - if fh.Size > maxUploadDocSize { - return nil, fmt.Errorf("file exceeds the maximum allowed size of %d bytes", maxUploadDocSize) + maxBytes := common.GetMaxDocumentUploadBytes() + if fh.Size > maxBytes { + return nil, fmt.Errorf("file exceeds the maximum allowed size of %d bytes", maxBytes) } src, err := fh.Open() if err != nil { return nil, err } defer src.Close() - blob, err := io.ReadAll(io.LimitReader(src, maxUploadDocSize+1)) + blob, err := io.ReadAll(io.LimitReader(src, maxBytes+1)) if err != nil { return nil, err } - if len(blob) > maxUploadDocSize { - return nil, fmt.Errorf("file exceeds the maximum allowed size of %d bytes", maxUploadDocSize) + if int64(len(blob)) > maxBytes { + return nil, fmt.Errorf("file exceeds the maximum allowed size of %d bytes", maxBytes) } return blob, nil }