feat: configure dataset document upload size (#21517)

docker/.env

@@ -352,10 +352,12 @@ TZ=Asia/Shanghai
 # Uncomment the following line if your operating system is MacOS:
 # MACOS=1
 
-# The maximum file size limit (in bytes) for each upload to your dataset or RAGFlow's File system.
-# To change the 1GB file size limit, uncomment the line below and update as needed.
-# MAX_CONTENT_LENGTH=1073741824
-# After updating, ensure `client_max_body_size` in nginx/nginx.conf is updated accordingly.
+# Maximum total request-body size for multipart uploads (default 1 GiB).
+# MAX_MULTIPART_CONTENT_LENGTH=1073741824
+# MAX_CONTENT_LENGTH=1073741824  # fallback name when MAX_MULTIPART_CONTENT_LENGTH is unset
+# Maximum size in bytes for one dataset document upload (default 128 MiB).
+# MAX_DOCUMENT_UPLOAD_SIZE=134217728
+# When raising the multipart limit, ensure `client_max_body_size` in nginx/nginx.conf is updated accordingly.
 # Note that neither `MAX_CONTENT_LENGTH` nor `client_max_body_size` sets the maximum size for files uploaded to an agent.
 # See https://ragflow.io/docs/dev/begin_component for details.
 
@@ -557,4 +559,3 @@ OAUTH_AUTO_REGISTER=true
 # RAGFLOW_MCP_PORT=9382
 # RAGFLOW_MCP_LAUNCH_MODE=self-host
 # RAGFLOW_MCP_HOST_API_KEY=
-

docs/faq.mdx

@@ -328,7 +328,7 @@ If your model is not currently supported but has APIs compatible with those of O
 
 The Go API limits non-multipart request bodies to 1 MiB by default. Set `MAX_JSON_CONTENT_LENGTH` to change that limit in bytes. Multipart upload requests share a 1 GiB default total-body limit; set `MAX_MULTIPART_CONTENT_LENGTH` to change it in bytes. Upload endpoints can enforce smaller per-file or per-request limits.
 
-The file-manager per-file limit follows `MAX_MULTIPART_CONTENT_LENGTH` (1 GiB by default). Dataset documents allow 128 MiB per file, and document-info uploads allow 100 MiB per file; increasing the shared multipart limit does not raise those per-file limits.
+The file-manager per-file limit follows `MAX_MULTIPART_CONTENT_LENGTH` (1 GiB by default). Dataset documents allow 128 MiB per file by default; set `MAX_DOCUMENT_UPLOAD_SIZE` to change that limit in bytes. The total multipart request limit must also be large enough for the file and multipart overhead. Document-info uploads allow 100 MiB per file, and increasing the shared multipart limit does not raise that limit.
 
 If you raise the multipart limit above 1 GiB, also increase `client_max_body_size` in **docker/nginx/nginx.conf**. The Go image contains its own copy of this file, so make the change effective by either rebuilding `RAGFLOW_IMAGE` or enabling the `./nginx/nginx.conf:/etc/nginx/nginx.conf` bind mount for the deployed service in **docker/docker-compose.yml**, then recreate that service.
 

internal/common/environments.go

@@ -89,6 +89,7 @@ const (
 	EnvMaxContentLength                  = "MAX_CONTENT_LENGTH"
 	EnvMaxJSONContentLength              = "MAX_JSON_CONTENT_LENGTH"
 	EnvMaxMultipartContentLength         = "MAX_MULTIPART_CONTENT_LENGTH"
+	EnvMaxDocumentUploadSize             = "MAX_DOCUMENT_UPLOAD_SIZE"
 	EnvRAGFlowDictPath                   = "RAGFLOW_DICT_PATH"
 	EnvDefaultSuperuserEmail             = "DEFAULT_SUPERUSER_EMAIL"
 	EnvDefaultSuperuserNickname          = "DEFAULT_SUPERUSER_NICKNAME"

internal/common/http.go

@@ -43,6 +43,8 @@ const (
 	DefaultMaxJSONRequestBodyBytes int64 = 1 << 20
 	// DefaultMaxMultipartRequestBodyBytes is the default total limit for multipart uploads.
 	DefaultMaxMultipartRequestBodyBytes int64 = 1 << 30
+	// DefaultMaxDocumentUploadBytes is the default per-file limit for dataset document uploads.
+	DefaultMaxDocumentUploadBytes int64 = 128 << 20
 	// MaxWebhookRequestBodyBytes is the default and maximum request body size for agent webhooks.
 	MaxWebhookRequestBodyBytes int64 = 10 * 1_000_000
 )
@@ -50,17 +52,23 @@ const (
 // GetMaxJSONRequestBodyBytes returns the configured non-multipart request-body
 // limit in bytes, defaulting to 1 MiB.
 func GetMaxJSONRequestBodyBytes() int64 {
-	return getMaxRequestBodyBytes(EnvMaxJSONContentLength, "", DefaultMaxJSONRequestBodyBytes)
+	return getMaxConfiguredBytes(EnvMaxJSONContentLength, "", DefaultMaxJSONRequestBodyBytes)
 }
 
 // GetMaxMultipartRequestBodyBytes returns the configured multipart upload
 // request-body limit in bytes, defaulting to 1 GiB. MAX_CONTENT_LENGTH remains
 // a fallback for existing deployments when the new variable is unset.
 func GetMaxMultipartRequestBodyBytes() int64 {
-	return getMaxRequestBodyBytes(EnvMaxMultipartContentLength, EnvMaxContentLength, DefaultMaxMultipartRequestBodyBytes)
+	return getMaxConfiguredBytes(EnvMaxMultipartContentLength, EnvMaxContentLength, DefaultMaxMultipartRequestBodyBytes)
 }
 
-func getMaxRequestBodyBytes(envName, fallbackEnvName string, defaultBytes int64) int64 {
+// GetMaxDocumentUploadBytes returns the configured per-file limit for dataset document uploads,
+// defaulting to 128 MiB.
+func GetMaxDocumentUploadBytes() int64 {
+	return getMaxConfiguredBytes(EnvMaxDocumentUploadSize, "", DefaultMaxDocumentUploadBytes)
+}
+
+func getMaxConfiguredBytes(envName, fallbackEnvName string, defaultBytes int64) int64 {
 	raw := strings.TrimSpace(GetEnv(envName))
 	if raw == "" && fallbackEnvName != "" {
 		raw = strings.TrimSpace(GetEnv(fallbackEnvName))

internal/common/http_test.go

@@ -72,6 +72,27 @@ func (f roundTripFunc) RoundTrip(req *http.Request) (*http.Response, error) {
 	return f(req)
 }
 
+func TestGetMaxDocumentUploadBytes(t *testing.T) {
+	tests := []struct {
+		name  string
+		value string
+		want  int64
+	}{
+		{name: "default", want: DefaultMaxDocumentUploadBytes},
+		{name: "configured", value: "4096", want: 4096},
+		{name: "invalid uses default", value: "invalid", want: DefaultMaxDocumentUploadBytes},
+		{name: "nonpositive uses default", value: "0", want: DefaultMaxDocumentUploadBytes},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			t.Setenv(EnvMaxDocumentUploadSize, tt.value)
+			if got := GetMaxDocumentUploadBytes(); got != tt.want {
+				t.Fatalf("GetMaxDocumentUploadBytes() = %d, want %d", got, tt.want)
+			}
+		})
+	}
+}
+
 func TestProviderLoggingRecordsTimingsAndTruncatesVectors(t *testing.T) {
 	core, logs := observer.New(zapcore.InfoLevel)
 	previousLogger := Logger

internal/service/document/document.go

@@ -267,10 +267,6 @@ type valueInfo struct {
 // that holds per-dataset subfolders, mirroring Python KNOWLEDGEBASE_FOLDER_NAME.
 const knowledgebaseFolderName = ".knowledgebase"
 
-// maxUploadDocSize bounds a single uploaded file held in memory, mirroring the
-// Python DOC_MAXIMUM_SIZE default (128 MiB; overridable there via MAX_CONTENT_LENGTH).
-const maxUploadDocSize = 128 * 1024 * 1024
-
 // MetadataUpdate is one update item: set key to value.
 type MetadataUpdate struct {
 	Key       string      `json:"key"`

internal/service/document/document_test.go

@@ -628,6 +628,23 @@ func makeTestFileHeader(t *testing.T, field, filename string, content []byte) *m
 	return fhs[0]
 }
 
+func TestReadFileHeaderBytesUsesConfiguredMax(t *testing.T) {
+	t.Setenv(common.EnvMaxDocumentUploadSize, "4")
+
+	blob, err := readFileHeaderBytes(makeTestFileHeader(t, "file", "small.txt", []byte("1234")))
+	if err != nil {
+		t.Fatalf("readFileHeaderBytes() error = %v, want success at configured limit", err)
+	}
+	if string(blob) != "1234" {
+		t.Fatalf("readFileHeaderBytes() = %q, want %q", blob, "1234")
+	}
+
+	_, err = readFileHeaderBytes(makeTestFileHeader(t, "file", "large.txt", []byte("12345")))
+	if err == nil || !strings.Contains(err.Error(), "4 bytes") {
+		t.Fatalf("readFileHeaderBytes() error = %v, want configured 4-byte limit", err)
+	}
+}
+
 // sptr returns a pointer to the given string.
 func sptr(s string) *string { return &s }
 

internal/service/document/document_upload.go

@@ -242,7 +242,7 @@ func (s *DocumentService) UploadWebDocument(ctx context.Context, kb *entity.Know
 		return nil, common.CodeServerError, err
 	}
 
-	blob, headers, _, err := utility.FetchRemoteFileSafely(ctx, url, maxUploadDocSize)
+	blob, headers, _, err := utility.FetchRemoteFileSafely(ctx, url, common.GetMaxDocumentUploadBytes())
 	if err != nil {
 		return nil, common.CodeDataError, err
 	}
@@ -385,20 +385,21 @@ func docToRawMap(doc *entity.Document) map[string]interface{} {
 }
 
 func readFileHeaderBytes(fh *multipart.FileHeader) ([]byte, error) {
-	if fh.Size > maxUploadDocSize {
-		return nil, fmt.Errorf("file exceeds the maximum allowed size of %d bytes", maxUploadDocSize)
+	maxBytes := common.GetMaxDocumentUploadBytes()
+	if fh.Size > maxBytes {
+		return nil, fmt.Errorf("file exceeds the maximum allowed size of %d bytes", maxBytes)
 	}
 	src, err := fh.Open()
 	if err != nil {
 		return nil, err
 	}
 	defer src.Close()
-	blob, err := io.ReadAll(io.LimitReader(src, maxUploadDocSize+1))
+	blob, err := io.ReadAll(io.LimitReader(src, maxBytes+1))
 	if err != nil {
 		return nil, err
 	}
-	if len(blob) > maxUploadDocSize {
-		return nil, fmt.Errorf("file exceeds the maximum allowed size of %d bytes", maxUploadDocSize)
+	if int64(len(blob)) > maxBytes {
+		return nil, fmt.Errorf("file exceeds the maximum allowed size of %d bytes", maxBytes)
 	}
 	return blob, nil
 }