fix(parser): detect Office content before selecting Go parsers (#21909)

This commit is contained in:
jay77721 authored and GitHub committed 2026-11-03 17:38:56 +08:00
1 parent 1d5ae546d3
commit 5dc7683aa4
37 files changed
+707 -502

No files matched your search

+1 -1
View File
@@ -62,6 +62,7 @@ require (
github.com/pkg/errors v0.9.1
github.com/pkoukk/tiktoken-go v0.1.8
github.com/redis/go-redis/v9 v9.18.0
github.com/richardlehane/mscfb v1.0.8
github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d
github.com/shirou/gopsutil/v3 v3.24.5
github.com/signintech/gopdf v0.36.1
@@ -208,7 +209,6 @@ require (
github.com/quic-go/qpack v0.6.0 // indirect
github.com/quic-go/quic-go v0.59.1 // indirect
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
github.com/richardlehane/mscfb v1.0.8 // indirect
github.com/richardlehane/msoleps v1.0.6 // indirect
github.com/rs/xid v1.6.0 // indirect
github.com/sagikazarmark/locafero v0.4.0 // indirect
+7 -5
View File
@@ -3034,15 +3034,17 @@ func (c *CLI) APIParseLocalFileCommand(commandCount int, cmd *Command) (Response
}
fileType := utility.GetFileType(filename)
fileParser, err := parser.GetTextOnlyParser(fileType)
if err != nil {
return nil, err
}
fileContent, err := os.ReadFile(filename)
if err != nil {
return nil, fmt.Errorf("failed to read dsl file: %w", err)
}
fileParser, actual, err := parser.GetTextOnlyParser(fileType, fileContent)
if err != nil {
return nil, err
}
if actual != fileType {
fmt.Fprintf(os.Stderr, "WARNING: file %q: requested format %s; detected %s; continuing with the %s parser\n", filepath.Base(filename), fileType, actual, actual)
}
ctx := context.Background()
parseResult := fileParser.ParseWithResult(ctx, filename, fileContent)
+49
View File
@@ -0,0 +1,49 @@
//go:build cgo
package cli
import (
"io"
"os"
"path/filepath"
"strings"
"testing"
)
func TestParseLocalFileCorrectsOfficeFormat(t *testing.T) {
data, err := os.ReadFile("../parser/parser/testdata/docx_table_image.docx")
if err != nil {
t.Fatal(err)
}
filename := filepath.Join(t.TempDir(), "renamed.doc")
if err := os.WriteFile(filename, data, 0600); err != nil {
t.Fatal(err)
}
r, w, err := os.Pipe()
if err != nil {
t.Fatal(err)
}
defer r.Close()
defer w.Close()
previous := os.Stderr
os.Stderr = w
t.Cleanup(func() { os.Stderr = previous })
c := &CLI{Config: &CommandLineConfig{CLIMode: APIMode}}
response, err := c.APIParseLocalFileCommand(1, &Command{Params: map[string]interface{}{"filename": filename}})
if err != nil {
t.Fatal(err)
}
if result, ok := response.(*SimpleResponse); !ok || result.Code != 0 {
t.Fatalf("unexpected response: %#v", response)
}
if err := w.Close(); err != nil {
t.Fatal(err)
}
warning, err := io.ReadAll(r)
if err != nil {
t.Fatal(err)
}
if !strings.Contains(string(warning), "WARNING:") || !strings.Contains(string(warning), "docx") {
t.Fatalf("missing format correction warning: %q", warning)
}
}
@@ -120,7 +120,7 @@ func TestImageUploadDecorator_MarkdownMediaRefMaterializes(t *testing.T) {
}
cache := &chunkerMediaCache{}
ctx := medialib.WithCache(t.Context(), cache)
p, err := parserlib.GetParser(utility.FileTypeMarkdown)
p, _, err := parserlib.GetParser(utility.FileTypeMarkdown, nil)
if err != nil {
t.Fatalf("GetParser(FileTypeMarkdown): %v", err)
}
@@ -174,7 +174,7 @@ func TestImageOCRSelectedTenantModel(t *testing.T) {
data := picturePNG(t)
setups := imageSwitchSetups(t, true)
setups["image"]["ocr_model_id"] = "model-b"
res, handled, err := maybeDispatchImage(t.Context(), db, utility.FileTypeVISUAL,
res, handled, err := maybeDispatchImage(t.Context(), parser.NewPictureParser(), db, utility.FileTypeVISUAL,
"invoice.png", data, map[string]any{"tenant_id": "tenant-1"}, setups, false, visionSettings{})
if err != nil || !handled {
t.Fatalf("handled=%v error=%v", handled, err)
@@ -238,7 +238,7 @@ func TestImageOCRReleasesMediaAdmissionBeforeRequest(t *testing.T) {
}
setups := imageSwitchSetups(t, true)
setups["image"]["ocr_model_id"] = "model-b"
result, _, err := maybeDispatchImage(t.Context(), db, utility.FileTypeVISUAL,
result, _, err := maybeDispatchImage(t.Context(), parser.NewPictureParser(), db, utility.FileTypeVISUAL,
"invoice.png", picturePNG(t), map[string]any{"tenant_id": "tenant-1"}, setups, false, visionSettings{})
if err != nil || len(result.JSON) != 1 {
t.Fatalf("err=%v result=%+v", err, result)
@@ -275,7 +275,7 @@ func TestImageOCRFailureRetainsAttachment(t *testing.T) {
db, requests := imageOCRFixture(t, provider, true)
setups := imageSwitchSetups(t, true)
setups["image"]["ocr_model_id"] = "model-b"
res, _, err := maybeDispatchImage(t.Context(), db, utility.FileTypeVISUAL,
res, _, err := maybeDispatchImage(t.Context(), parser.NewPictureParser(), db, utility.FileTypeVISUAL,
"invoice.png", picturePNG(t), map[string]any{"tenant_id": "tenant-1"}, setups, false, visionSettings{})
if err != nil {
t.Fatal(err)
@@ -321,7 +321,7 @@ func TestImageOCRInvalidSelectionNeverFallsBack(t *testing.T) {
}
setups := imageSwitchSetups(t, true)
setups["image"]["ocr_model_id"] = tc.ref
res, _, err := maybeDispatchImage(t.Context(), db, utility.FileTypeVISUAL,
res, _, err := maybeDispatchImage(t.Context(), parser.NewPictureParser(), db, utility.FileTypeVISUAL,
"invoice.png", picturePNG(t), map[string]any{"tenant_id": tc.tenant}, setups, false, visionSettings{})
if err != nil {
t.Fatal(err)
@@ -351,7 +351,7 @@ func TestImageOCRSelectionRespectsSwitchAndSVG(t *testing.T) {
t.Run(tc.name, func(t *testing.T) {
setups := imageSwitchSetups(t, tc.ocr)
setups["image"]["ocr_model_id"] = "model-b"
res, _, err := maybeDispatchImage(t.Context(), db, utility.FileTypeVISUAL,
res, _, err := maybeDispatchImage(t.Context(), parser.NewPictureParser(), db, utility.FileTypeVISUAL,
tc.filename, tc.data, map[string]any{"tenant_id": "tenant-1"}, setups, false, visionSettings{})
if err != nil || len(res.JSON) != 1 {
t.Fatalf("err=%v result=%+v", err, res)
@@ -81,6 +81,7 @@ func maybeDispatchVideo(
// preserves the original image and optionally appends global vision enhancement.
func maybeDispatchImage(
ctx context.Context,
p parser.ParseResultProducer,
db *gorm.DB,
fileType utility.FileType,
filename string,
@@ -140,7 +141,7 @@ func maybeDispatchImage(
}
}
release()
parsed := dispatchParse(ctx, fileType, filename, binary, setups, progress.Window{})
parsed := dispatchParse(ctx, p, fileType, filename, binary, setups, progress.Window{})
if parsed.Err != nil {
return parsed, true, parsed.Err
}
@@ -61,7 +61,7 @@ func assertNativeOCRTextAndImage(t *testing.T, result parser.ParseResult) {
// reading of canvases saved before the ocr_enabled switch existed.
func TestMaybeDispatchImageNativeOCRWithoutVision(t *testing.T) {
registerNativeOCRModels(t)
result, handled, err := maybeDispatchImage(t.Context(), nil, utility.FileTypeVISUAL,
result, handled, err := maybeDispatchImage(t.Context(), parser.NewPictureParser(), nil, utility.FileTypeVISUAL,
"picture_ocr.png", pictureOCRFixture, nil, defaultSetups(), false, visionSettings{})
if err != nil {
t.Fatal(err)
@@ -77,7 +77,7 @@ func TestMaybeDispatchImageNativeOCRWithoutVision(t *testing.T) {
// would only ever be exercised against a fake analyzer.
func TestMaybeDispatchImageNativeOCRWithSwitch(t *testing.T) {
registerNativeOCRModels(t)
result, handled, err := maybeDispatchImage(t.Context(), nil, utility.FileTypeVISUAL,
result, handled, err := maybeDispatchImage(t.Context(), parser.NewPictureParser(), nil, utility.FileTypeVISUAL,
"picture_ocr.png", pictureOCRFixture, nil, imageSetupsWithSwitch(true), false, visionSettings{})
if err != nil {
t.Fatal(err)
@@ -94,7 +94,7 @@ func TestMaybeDispatchImageNativeOCRWithSwitch(t *testing.T) {
// dispatch says so instead of failing silently.
func TestMaybeDispatchImageNativeSwitchOffRunsNoOCR(t *testing.T) {
registerNativeOCRModels(t)
result, handled, err := maybeDispatchImage(t.Context(), nil, utility.FileTypeVISUAL,
result, handled, err := maybeDispatchImage(t.Context(), parser.NewPictureParser(), nil, utility.FileTypeVISUAL,
"picture_ocr.png", pictureOCRFixture, nil, imageSetupsWithSwitch(false), false, visionSettings{})
if err != nil || result.Err != nil {
t.Fatalf("degraded config must not error: err = %v, result.Err = %v", err, result.Err)
@@ -176,8 +176,7 @@ func TestMaybeDispatchImage_ParseMethodModelDoesNotOverrideGlobalEnhancement(t *
setups := defaultSetups()
setups["image"]["parse_method"] = "custom-vlm@provider"
res, dispatched, err := maybeDispatchImage(
t.Context(),
res, dispatched, err := maybeDispatchImage(t.Context(), parser.NewPictureParser(),
dao.DB,
utility.FileTypeVISUAL,
"photo.png",
@@ -235,7 +234,7 @@ func TestMaybeDispatchImage_OCRWithoutEnhancementSkipsVision(t *testing.T) {
setups := defaultSetups()
setups["image"]["parse_method"] = "ocr"
res, _, err := maybeDispatchImage(t.Context(), dao.DB, utility.FileTypeVISUAL, "photo.png", picturePNG(t), map[string]any{"tenant_id": "t1"}, setups, false, visionSettings{})
res, _, err := maybeDispatchImage(t.Context(), parser.NewPictureParser(), dao.DB, utility.FileTypeVISUAL, "photo.png", picturePNG(t), map[string]any{"tenant_id": "t1"}, setups, false, visionSettings{})
if err != nil {
t.Fatalf("maybeDispatchImage: %v", err)
}
@@ -325,8 +324,7 @@ func TestMaybeDispatchImage_UsesSystemPrompt(t *testing.T) {
setups["image"]["system_prompt"] = "自定义视觉提示"
ctx := t.Context()
res, dispatched, err := maybeDispatchImage(
ctx,
res, dispatched, err := maybeDispatchImage(ctx, parser.NewPictureParser(),
dao.DB,
utility.FileTypeVISUAL,
"test.png",
@@ -377,8 +375,7 @@ func TestMaybeDispatchImage_DefaultPromptUsesDatasetLanguage(t *testing.T) {
setups["image"]["lang"] = "Chinese"
setups["image"]["system_prompt"] = ""
_, _, err := maybeDispatchImage(
t.Context(),
_, _, err := maybeDispatchImage(t.Context(), parser.NewPictureParser(),
dao.DB,
utility.FileTypeVISUAL,
"test.png",
@@ -448,8 +445,7 @@ func TestMaybeDispatchImage_GlobalVisionPromptDrivesImage(t *testing.T) {
setups := defaultSetups()
setups["image"]["lang"] = tc.familyLang
setups["image"]["system_prompt"] = tc.familyPrompt
_, _, err := maybeDispatchImage(
t.Context(), dao.DB, utility.FileTypeVISUAL, "test.png", picturePNG(t),
_, _, err := maybeDispatchImage(t.Context(), parser.NewPictureParser(), dao.DB, utility.FileTypeVISUAL, "test.png", picturePNG(t),
tc.inputs, setups, true,
visionSettings{systemPrompt: tc.globalPrompt},
)
@@ -484,8 +480,7 @@ func TestMaybeDispatchImage_ReturnsJSONWithImage(t *testing.T) {
setups := defaultSetups()
ctx := t.Context()
res, dispatched, err := maybeDispatchImage(
ctx,
res, dispatched, err := maybeDispatchImage(ctx, parser.NewPictureParser(),
dao.DB,
utility.FileTypeVISUAL,
"test.png",
@@ -533,8 +528,7 @@ func TestMaybeDispatchImage_HardcodesJSONOutput(t *testing.T) {
setups := defaultSetups()
ctx := t.Context()
res, _, err := maybeDispatchImage(
ctx,
res, _, err := maybeDispatchImage(ctx, parser.NewPictureParser(),
dao.DB,
utility.FileTypeVISUAL,
"test.png",
@@ -670,8 +664,7 @@ func TestMaybeDispatchImage_UsesConfiguredVLMModel(t *testing.T) {
setups := defaultSetups()
setups["image"]["parse_method"] = "custom-vlm@provider"
_, dispatched, err := maybeDispatchImage(
t.Context(),
_, dispatched, err := maybeDispatchImage(t.Context(), parser.NewPictureParser(),
dao.DB,
utility.FileTypeVISUAL,
"test.png",
@@ -805,7 +798,7 @@ func TestMaybeDispatchImageWithoutOCRTextKeepsImage(t *testing.T) {
for _, method := range []string{"ocr", ""} {
setups := defaultSetups()
setups["image"]["parse_method"] = method
result, handled, err := maybeDispatchImage(t.Context(), nil, utility.FileTypeVISUAL, "photo.png", picturePNG(t), nil, setups, false, visionSettings{})
result, handled, err := maybeDispatchImage(t.Context(), parser.NewPictureParser(), nil, utility.FileTypeVISUAL, "photo.png", picturePNG(t), nil, setups, false, visionSettings{})
if err != nil {
t.Fatal(err)
}
@@ -829,7 +822,7 @@ func TestMaybeDispatchImageUnhealthyOCRKeepsImage(t *testing.T) {
deepdoctype.NativeDocAnalyzerFactory = func() (deepdoctype.DocAnalyzer, bool) {
return &unhealthyPictureOCRAnalyzer{}, true
}
result, handled, err := maybeDispatchImage(t.Context(), nil, utility.FileTypeVISUAL, "photo.png", picturePNG(t), nil, defaultSetups(), false, visionSettings{})
result, handled, err := maybeDispatchImage(t.Context(), parser.NewPictureParser(), nil, utility.FileTypeVISUAL, "photo.png", picturePNG(t), nil, defaultSetups(), false, visionSettings{})
if err != nil || result.Err != nil {
t.Fatalf("unhealthy OCR should degrade without an error: err = %v, result.Err = %v", err, result.Err)
}
@@ -845,7 +838,7 @@ func TestMaybeDispatchImageRejectsUndecodableBytes(t *testing.T) {
for _, method := range []string{"ocr", "custom-vlm"} {
setups := defaultSetups()
setups["image"]["parse_method"] = method
_, _, err := maybeDispatchImage(t.Context(), nil, utility.FileTypeVISUAL, "photo.png", []byte("bad image"), nil, setups, false, visionSettings{})
_, _, err := maybeDispatchImage(t.Context(), parser.NewPictureParser(), nil, utility.FileTypeVISUAL, "photo.png", []byte("bad image"), nil, setups, false, visionSettings{})
if err == nil || !strings.Contains(err.Error(), "decode") {
t.Fatalf("method %q: error = %v, want decode error", method, err)
}
@@ -865,7 +858,7 @@ func TestMaybeDispatchImageReleasesAdmissionAfterPanic(t *testing.T) {
t.Error("expected OCR analyzer panic")
}
}()
_, _, _ = maybeDispatchImage(t.Context(), nil, utility.FileTypeVISUAL, "photo.png", data, nil, defaultSetups(), false, visionSettings{})
_, _, _ = maybeDispatchImage(t.Context(), parser.NewPictureParser(), nil, utility.FileTypeVISUAL, "photo.png", data, nil, defaultSetups(), false, visionSettings{})
}()
ctx, cancel := context.WithTimeout(t.Context(), time.Second)
defer cancel()
@@ -892,7 +885,7 @@ func TestMaybeDispatchImageDecodesRasterOnlyForOCR(t *testing.T) {
t.Run(method, func(t *testing.T) {
setups := defaultSetups()
setups["image"]["parse_method"] = method
result, handled, err := maybeDispatchImage(t.Context(), nil, utility.FileTypeVISUAL,
result, handled, err := maybeDispatchImage(t.Context(), parser.NewPictureParser(), nil, utility.FileTypeVISUAL,
"photo.png", []byte(magic), map[string]any{"tenant_id": "t1"}, setups, true, visionSettings{})
if method == "ocr" {
if err == nil || !strings.Contains(err.Error(), "raster decoding requested") {
@@ -935,7 +928,7 @@ func TestMaybeDispatchImageOCRSwitch(t *testing.T) {
}
t.Run("ocr-on", func(t *testing.T) {
res, handled, err := maybeDispatchImage(t.Context(), nil, utility.FileTypeVISUAL, "photo.png", picturePNG(t),
res, handled, err := maybeDispatchImage(t.Context(), parser.NewPictureParser(), nil, utility.FileTypeVISUAL, "photo.png", picturePNG(t),
map[string]any{"tenant_id": "t1"}, ocrSetups(true), false, visionSettings{})
if err != nil || !handled {
t.Fatalf("err = %v, handled = %v", err, handled)
@@ -945,7 +938,7 @@ func TestMaybeDispatchImageOCRSwitch(t *testing.T) {
}
})
t.Run("ocr-off-no-enhancement-keeps-empty", func(t *testing.T) {
res, handled, err := maybeDispatchImage(t.Context(), nil, utility.FileTypeVISUAL, "photo.png", picturePNG(t),
res, handled, err := maybeDispatchImage(t.Context(), parser.NewPictureParser(), nil, utility.FileTypeVISUAL, "photo.png", picturePNG(t),
map[string]any{"tenant_id": "t1"}, ocrSetups(false), false, visionSettings{})
if err != nil || !handled {
t.Fatalf("err = %v, handled = %v", err, handled)
@@ -961,7 +954,7 @@ func TestMaybeDispatchImageOCRSwitch(t *testing.T) {
}
})
t.Run("ocr-off-vision-only", func(t *testing.T) {
res, handled, err := maybeDispatchImage(t.Context(), nil, utility.FileTypeVISUAL, "photo.png", picturePNG(t),
res, handled, err := maybeDispatchImage(t.Context(), parser.NewPictureParser(), nil, utility.FileTypeVISUAL, "photo.png", picturePNG(t),
map[string]any{"tenant_id": "t1"}, ocrSetups(false), true, visionSettings{})
if err != nil || !handled {
t.Fatalf("err = %v, handled = %v", err, handled)
@@ -973,7 +966,7 @@ func TestMaybeDispatchImageOCRSwitch(t *testing.T) {
t.Run("switch-beats-stale-parse-method", func(t *testing.T) {
setups := defaultSetups() // image still carries parse_method:"ocr"
setups["image"]["ocr_enabled"] = false
res, _, err := maybeDispatchImage(t.Context(), nil, utility.FileTypeVISUAL, "photo.png", picturePNG(t),
res, _, err := maybeDispatchImage(t.Context(), parser.NewPictureParser(), nil, utility.FileTypeVISUAL, "photo.png", picturePNG(t),
map[string]any{"tenant_id": "t1"}, setups, false, visionSettings{})
if err != nil {
t.Fatal(err)
@@ -1003,7 +996,7 @@ func TestMaybeDispatchImageBlankVisionDescriptionWarns(t *testing.T) {
return blankAnswerVisionDriver{}, "vision", &modelModule.APIConfig{}, 0, nil
}
res, handled, err := maybeDispatchImage(t.Context(), nil, utility.FileTypeVISUAL, "photo.png", picturePNG(t),
res, handled, err := maybeDispatchImage(t.Context(), parser.NewPictureParser(), nil, utility.FileTypeVISUAL, "photo.png", picturePNG(t),
map[string]any{"tenant_id": "t1"}, imageSwitchSetups(t, false), true, visionSettings{})
if err != nil || !handled {
t.Fatalf("err = %v, handled = %v", err, handled)
+12 -3
View File
@@ -495,7 +495,13 @@ func (c *ParserComponent) Invoke(ctx context.Context, db *gorm.DB, inputs map[st
// "docx", ...). Used by parser.GetParser, whose switch
// arms are keyed off the utility constants.
//
fileTypeExt := fileTypeFromInputs(inputs)
requestedType := fileTypeFromInputs(inputs)
fileParser, fileTypeExt, selectionErr := parser.GetParser(requestedType, binary)
if selectionErr == nil && fileTypeExt != requestedType {
warning := fmt.Sprintf("file %q: requested format %s does not match detected format %s; continuing with the %s parser", filename, requestedType, fileTypeExt, fileTypeExt)
common.Warn(warning)
reportParserWarnings(ctx, []string{warning})
}
parsePagesOn := pdfPageProgressSupported(fileTypeExt, setups)
enhanceOn := c.enableVisionEnhancement
split := progress.NewSplit(parsePagesOn, enhanceOn)
@@ -519,7 +525,7 @@ func (c *ParserComponent) Invoke(ctx context.Context, db *gorm.DB, inputs map[st
var handledImage bool
if !handledVision && !handledMedia {
// Image dispatch: OCR with independently controlled VLM enhancement.
dispatched, handledImage, visionErr = maybeDispatchImage(ctx, db, fileTypeExt, filename, binary, inputs, setups, c.enableVisionEnhancement, c.vision)
dispatched, handledImage, visionErr = maybeDispatchImage(ctx, fileParser, db, fileTypeExt, filename, binary, inputs, setups, c.enableVisionEnhancement, c.vision)
if visionErr != nil {
return nil, visionErr
}
@@ -534,7 +540,10 @@ func (c *ParserComponent) Invoke(ctx context.Context, db *gorm.DB, inputs map[st
}
}
if !handledVision && !handledMedia && !handledImage && !handledAudio {
dispatched = dispatchParse(ctx, fileTypeExt, filename, binary, setups, pageWin)
if selectionErr != nil && fileTypeExt != utility.FileTypeOTHER {
return nil, fmt.Errorf("parser: resolve %q: %w", fileTypeExt, selectionErr)
}
dispatched = dispatchParse(ctx, fileParser, fileTypeExt, filename, binary, setups, pageWin)
if dispatched.Err == nil && ctx.Err() == nil {
pageWin.Report(ctx, 1.0)
}
@@ -58,8 +58,8 @@ func configureParserFromSetups(p any, fileType utility.FileType, setups map[stri
cfg.ConfigureFromSetup(map[string]any(setup))
}
// dispatchParse resolves the parser for the given fileType and invokes
// its structured ParseWithResult contract.
// dispatchParse configures the selected parser and invokes its structured
// ParseWithResult contract using the effective file type.
//
// The function NEVER returns a partial result. On error the result
// is the zero value (Err != nil, no JSON and no File). Callers detect the
@@ -74,7 +74,7 @@ func configureParserFromSetups(p any, fileType utility.FileType, setups map[stri
// re-reading setups. lib_type is no longer threaded through: the
// Python dispatcher picks a single backend per family and the Go
// constructors mirror that.
func dispatchParse(ctx context.Context, fileType utility.FileType, filename string, data []byte, setups map[string]schema.ParserSetup, window progress.Window) parser.ParseResult {
func dispatchParse(ctx context.Context, p parser.ParseResultProducer, fileType utility.FileType, filename string, data []byte, setups map[string]schema.ParserSetup, window progress.Window) parser.ParseResult {
if fileType == utility.FileTypeOTHER {
// Unknown / unset family. The component treats the bytes
// as text pages; splitIntoPages handles it. We return no
@@ -89,10 +89,6 @@ func dispatchParse(ctx context.Context, fileType utility.FileType, filename stri
}
}
p, err := parser.GetParser(fileType)
if err != nil {
return parser.ParseResult{Err: fmt.Errorf("parser: resolve %q: %w", fileType, err)}
}
configureParserFromSetups(p, fileType, setups)
attachPDFPageProgress(ctx, p, window)
@@ -0,0 +1,156 @@
package component
import (
"archive/zip"
"bytes"
"encoding/json"
"net/http"
"net/http/httptest"
"strings"
"testing"
"github.com/xuri/excelize/v2"
"go.uber.org/zap"
"go.uber.org/zap/zaptest/observer"
"ragflow/internal/agent/runtime"
"ragflow/internal/common"
)
func TestParserCorrectionUsesTargetSetupForRemoteParsing(t *testing.T) {
withSSRFBypass(t)
f := excelize.NewFile()
defer f.Close()
if err := f.SetCellValue("Sheet1", "A1", "Workbook"); err != nil {
t.Fatal(err)
}
data, err := f.WriteToBuffer()
if err != nil {
t.Fatal(err)
}
payload := tcadpZipFixtureForComponent(t)
var receivedType string
var server *httptest.Server
server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch r.URL.Path {
case "/reconstruct_document":
var request struct {
FileType string `json:"file_type"`
}
if err := json.NewDecoder(r.Body).Decode(&request); err != nil {
t.Error(err)
}
receivedType = request.FileType
_, _ = w.Write([]byte(`{"DocumentRecognizeResultUrl":"` + server.URL + `/download.zip"}`))
case "/download.zip":
_, _ = w.Write(payload)
default:
http.NotFound(w, r)
}
}))
defer server.Close()
setups := defaultSetups()
setups["spreadsheet"]["parse_method"] = "tcadp"
setups["spreadsheet"]["tcadp_apiserver"] = server.URL
c := &ParserComponent{setups: setups}
out, err := c.Invoke(t.Context(), nil, map[string]any{"name": "renamed.doc", "binary": data.Bytes()})
if err != nil {
t.Fatal(err)
}
if receivedType != "XLSX" {
t.Fatalf("remote file_type=%q, want XLSX", receivedType)
}
if out["file_type"] != "xlsx" {
t.Fatalf("output file_type=%v", out["file_type"])
}
requireJSONText(t, out, "Hello TCADP")
}
func TestParserComponentCorrectsDOCXUploadedAsDOC(t *testing.T) {
var buf bytes.Buffer
w := zip.NewWriter(&buf)
for name, content := range map[string]string{
"[Content_Types].xml": `<Types xmlns="http://schemas.openxmlformats.org/package/2006/content-types"><Default Extension="rels" ContentType="application/vnd.openxmlformats-package.relationships+xml"/><Default Extension="xml" ContentType="application/xml"/><Override PartName="/word/document.xml" ContentType="application/vnd.openxmlformats-officedocument.wordprocessingml.document.main+xml"/></Types>`,
"_rels/.rels": `<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships"><Relationship Id="rId1" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/officeDocument" Target="word/document.xml"/></Relationships>`,
"word/document.xml": `<w:document xmlns:w="http://schemas.openxmlformats.org/wordprocessingml/2006/main"><w:body><w:p><w:r><w:t>Recognized document</w:t></w:r></w:p></w:body></w:document>`,
} {
f, err := w.Create(name)
if err != nil {
t.Fatal(err)
}
if _, err := f.Write([]byte(content)); err != nil {
t.Fatal(err)
}
}
if err := w.Close(); err != nil {
t.Fatal(err)
}
c, err := NewParserComponent(map[string]any{})
if err != nil {
t.Fatal(err)
}
core, logs := observer.New(zap.WarnLevel)
originalLogger := common.Logger
common.Logger = zap.New(core)
t.Cleanup(func() { common.Logger = originalLogger })
for _, suffix := range []string{"doc", "xls", "ppt", "wps", "docx"} {
t.Run(suffix, func(t *testing.T) {
var messages []string
ctx := runtime.WithProgressMessageCallback(t.Context(), func(_, message string) { messages = append(messages, message) })
logs.TakeAll()
name := "renamed." + suffix
out, err := c.Invoke(ctx, nil, map[string]any{"name": name, "binary": buf.Bytes()})
if err != nil {
t.Fatal(err)
}
if out["file_type"] != "docx" || out["name"] != name {
t.Fatalf("incorrect output metadata: type=%v name=%v", out["file_type"], out["name"])
}
requireJSONText(t, out, "Recognized document")
if suffix == "docx" {
if len(messages) != 0 || logs.Len() != 0 {
t.Fatal("matching format should not warn")
}
return
}
if len(messages) != 1 || !strings.Contains(messages[0], "WARNING:") || !strings.Contains(messages[0], "docx") {
t.Fatalf("expected correction warning, got %v", messages)
}
if logs.Len() != 1 {
t.Fatalf("expected one server warning, got %d", logs.Len())
}
})
}
}
func TestParserComponentKeepsUnrecognizedFormat(t *testing.T) {
c, err := NewParserComponent(map[string]any{})
if err != nil {
t.Fatal(err)
}
var messages []string
ctx := runtime.WithProgressMessageCallback(t.Context(), func(_, message string) { messages = append(messages, message) })
_, err = c.Invoke(ctx, nil, map[string]any{"name": "broken.doc", "binary": []byte("unrecognized content")})
if err == nil || !strings.Contains(err.Error(), `"doc"`) {
t.Fatalf("expected original DOC parser error, got %v", err)
}
if len(messages) != 0 {
t.Fatalf("unknown content should not trigger a correction warning: %v", messages)
}
}
func TestParserComponentSkipsUnavailableTypeCheck(t *testing.T) {
c, err := NewParserComponent(map[string]any{})
if err != nil {
t.Fatal(err)
}
var messages []string
ctx := runtime.WithProgressMessageCallback(t.Context(), func(_, message string) { messages = append(messages, message) })
out, err := c.Invoke(ctx, nil, map[string]any{"name": "notes.txt", "binary": []byte("Unchanged text")})
if err != nil {
t.Fatal(err)
}
if out["file_type"] != "txt" || len(messages) != 0 {
t.Fatalf("unexpected type correction: type=%v messages=%v", out["file_type"], messages)
}
requireJSONText(t, out, "Unchanged text")
}
@@ -189,7 +189,7 @@ func TestMaybeDispatchImageSVGWarnings(t *testing.T) {
}
t.Cleanup(func() { resolveTenantModelByType = original })
textless := []byte(`<svg xmlns="http://www.w3.org/2000/svg"><rect width="1" height="1"/></svg>`)
result, handled, err := maybeDispatchImage(t.Context(), nil, utility.FileTypeVISUAL, "shape.SVG", textless, map[string]any{"tenant_id": "t1"}, defaultSetups(), true, visionSettings{})
result, handled, err := maybeDispatchImage(t.Context(), parser.NewPictureParser(), nil, utility.FileTypeVISUAL, "shape.SVG", textless, map[string]any{"tenant_id": "t1"}, defaultSetups(), true, visionSettings{})
if err != nil {
t.Fatal(err)
}
@@ -211,11 +211,11 @@ func TestMaybeDispatchImageRejectsInvalidSVG(t *testing.T) {
for _, ocr := range []bool{true, false} {
setups := defaultSetups()
setups["image"]["ocr_enabled"] = ocr
_, handled, err := maybeDispatchImage(t.Context(), nil, utility.FileTypeVISUAL, "broken.svg", []byte(`<svg><text>a</svg>`), nil, setups, false, visionSettings{})
_, handled, err := maybeDispatchImage(t.Context(), parser.NewPictureParser(), nil, utility.FileTypeVISUAL, "broken.svg", []byte(`<svg><text>a</svg>`), nil, setups, false, visionSettings{})
if !handled || err == nil || !strings.Contains(err.Error(), "parser: decode SVG") {
t.Fatalf("ocr %v: handled = %v, error = %v, want SVG decode error", ocr, handled, err)
}
_, _, err = maybeDispatchImage(t.Context(), nil, utility.FileTypeVISUAL, "empty.svg", nil, nil, setups, false, visionSettings{})
_, _, err = maybeDispatchImage(t.Context(), parser.NewPictureParser(), nil, utility.FileTypeVISUAL, "empty.svg", nil, nil, setups, false, visionSettings{})
if err == nil || !strings.Contains(err.Error(), "size limits") {
t.Fatalf("ocr %v: error = %v, want size limit error", ocr, err)
}
@@ -0,0 +1,51 @@
package parser
import (
"fmt"
"testing"
)
// TestDOCXParser_JSONEmptyIRReturnsError locks Finding B: when the native
// backend opens the document but returns an empty structured IR, the JSON
// output path must fail loud (it is built entirely from the IR, which the
// Markdown path can fall back from plain text). This guards against silently
// emitting empty JSON sections.
func TestDOCXParser_JSONEmptyIRReturnsError(t *testing.T) {
ctx := t.Context()
p := NewDOCXParser()
orig := docxExtract
defer func() { docxExtract = orig }()
docxExtract = func(data []byte, format string) (string, string, string, error, error) {
// Open succeeds, but the IR view is empty.
return "", "", "", nil, nil
}
res := p.ParseWithResult(ctx, "empty.docx", []byte("PK\x03\x04"))
if res.Err == nil {
t.Fatal("expected error for JSON output with empty IR, got nil")
}
}
// TestDOCXParser_AllViewsEmptyReturnsError locks the engine-level contract
// at the parser boundary: when the native backend opens the document but
// every view (IR / Markdown / PlainText) is empty, OpenAndExtract reports a
// joined error, and the parser must fail loud rather than silently emit
// empty JSON. (office_oxide always returns a structured IR for an openable
// doc, so this all-empty error is simulated via the seam rather than a real
// fixture.)
func TestDOCXParser_AllViewsEmptyReturnsError(t *testing.T) {
ctx := t.Context()
p := NewDOCXParser()
orig := docxExtract
defer func() { docxExtract = orig }()
docxExtract = func(data []byte, format string) (string, string, string, error, error) {
return "", "", "", nil, fmt.Errorf("office_oxide docx: all text views failed")
}
res := p.ParseWithResult(ctx, "empty.docx", []byte("PK\x03\x04"))
if res.Err == nil {
t.Fatal("expected error for a document with no extractable text in any view, got nil")
}
}
+5 -28
View File
@@ -24,14 +24,11 @@ import (
)
// docxExtract is a test seam for the native office_oxide extraction. It
// defaults to the deepdoc/docx backend (which is the only place that depends
// on office_oxide) and is overridden in tests to capture the effective
// container format passed to the engine.
// defaults to the deepdoc/office backend, which owns the native dependency.
var docxExtract = deepdocoffice.OpenAndExtract
// DOCXParser is the DOCX/DOC parser. It is a CGO-free facade: magic-byte
// sniffing and post-processing live here, while the native extraction lives
// in internal/deepdoc/parser/office. The IR data model and postprocessing live
// DOCXParser parses DOCX documents. Native extraction lives in
// internal/deepdoc/parser/office. The IR data model and postprocessing live
// in cgo-free files (docx_ir.go, docx_postprocess.go, office_table_render.go)
// so they compile and test without native libraries.
type DOCXParser struct {
@@ -62,28 +59,8 @@ func (p *DOCXParser) ConfigureFromSetup(setup map[string]any) {
// ParseWithResult produces structured JSON items from a docx document.
// Embedded images are stored in the run media cache and emitted as media_ref;
// text-only parsers skip them instead.
//
// File["format"] in the returned ParseResult reflects the effective
// container format after magic-byte sniffing (e.g. "doc" for an OLE
// payload uploaded as .docx), not merely the file extension. See
// office_detect.go:officeContainer for the detection contract.
func (p *DOCXParser) ParseWithResult(ctx context.Context, filename string, data []byte) ParseResult {
// office_oxide's OpenFromBytes takes the container format as given
// and does no magic-byte detection, so a legacy .doc uploaded with a
// .docx extension would be parsed as ZIP/OOXML and fail. Sniff the
// real container and pass the matching format so mislabeled files
// still parse (mirrors the magic-byte correction office_oxide::Open
// performs on file paths).
format := "docx"
if officeContainer(data) == "ole" {
format = "doc"
}
// Note: when format == "doc" (OLE fallback), header/footer and TOC
// post-processing below degrades gracefully: extractDOCXHeaderFooterTexts
// opens the payload as ZIP and fails closed, so RemoveHeaderFooter becomes
// a no-op for legacy OLE inputs. This is acceptable — legacy DOC has no
// OOXML header/footer parts to strip.
irJSON, _, _, _, err := docxExtract(data, format)
irJSON, _, _, _, err := docxExtract(data, "docx")
if err != nil {
if errors.Is(err, deepdocoffice.ErrOfficeCGORequired) {
return ParseResult{Err: fmt.Errorf("%w: %s", ErrOfficeCGORequired, filename)}
@@ -93,7 +70,7 @@ func (p *DOCXParser) ParseWithResult(ctx context.Context, filename string, data
fileMeta := map[string]any{
"name": filename,
"format": format,
"format": "docx",
}
var sink *officeMediaSink
+5 -2
View File
@@ -606,7 +606,7 @@ func recoverParse(fn func() ParseResult) (res ParseResult, panicked bool) {
// document.
//
// KNOWN LIMITATION: re-chunk uses the text-only, default-config parser
// returned by GetTextOnlyParser(ft) (and, for a nested email, only the
// returned by GetTextOnlyParser (and, for a nested email, only the
// top-level p.fields — no tenant/setup language, parse_method, or OCR/VLM
// model). The extracted attachment text may therefore differ from ingesting
// the same file as a standalone document through the pipeline's
@@ -688,10 +688,13 @@ func (p *EmailParser) rechunkEmailAttachments(ctx context.Context, content map[s
return ep.parseEmail(ctx, fn, []byte(payload), depth+1)
})
} else {
np, err := GetTextOnlyParser(ft)
np, actual, err := GetTextOnlyParser(ft, []byte(payload))
if err != nil {
continue
}
if actual != ft {
common.Warn(fmt.Sprintf("email attachment %q: requested format %s; detected %s; continuing with the %s parser", fn, ft, actual, actual))
}
res, panicked = recoverParse(func() ParseResult {
return np.ParseWithResult(ctx, fn, []byte(payload))
})
@@ -299,7 +299,7 @@ func TestHTMLTextParserSkipsRemoteImages(t *testing.T) {
var hits atomic.Int32
server := imageTestServer(t, "remote-bytes", &hits)
p, err := GetTextOnlyParser(utility.FileTypeHTML)
p, _, err := GetTextOnlyParser(utility.FileTypeHTML, nil)
if err != nil {
t.Fatalf("GetTextOnlyParser: %v", err)
}
@@ -36,7 +36,7 @@ const markdownMediaDataURI = "data:image/png;base64,aGVsbG8="
func markdownIngestionParser(t *testing.T) ParseResultProducer {
t.Helper()
p, err := GetParser(utility.FileTypeMarkdown)
p, _, err := GetParser(utility.FileTypeMarkdown, nil)
if err != nil {
t.Fatalf("GetParser(FileTypeMarkdown): %v", err)
}
@@ -592,7 +592,7 @@ func TestMarkdownTextParserSkipsImages(t *testing.T) {
var hits atomic.Int32
server := imageTestServer(t, "\x89PNG\r\n\x1a\nremote", &hits)
p, err := GetTextOnlyParser(utility.FileTypeMarkdown)
p, _, err := GetTextOnlyParser(utility.FileTypeMarkdown, nil)
if err != nil {
t.Fatalf("GetTextOnlyParser(FileTypeMarkdown): %v", err)
}
-29
View File
@@ -1,29 +0,0 @@
package parser
// officeContainer reports the on-disk container format of an Office
// document by sniffing its leading magic bytes. It returns "ooxml" for
// the ZIP-based OOXML family (docx/xlsx/pptx), "ole" for the legacy
// binary OLE family (doc/xls/ppt), or "" when the container cannot be
// determined from the first bytes. The return values are internal
// markers — callers map them to the concrete office_oxide format strings
// ("docx"→"doc", "pptx"→"ppt"); this avoids coupling the detector to a
// single family and keeps it reusable for xlsx/xls if needed later.
//
// The office_oxide backend's OpenFromBytes requires the caller to name
// the container format and does no magic-byte detection of its own, so
// a legacy file renamed to an OOXML extension (e.g. a .doc uploaded as
// .docx) is otherwise parsed as ZIP and fails with OFFICE_ERR_PARSE.
// Sniffing here lets the cgo parsers pass the real container format.
func officeContainer(data []byte) string {
if len(data) >= 4 && data[0] == 0x50 && data[1] == 0x4B && data[2] == 0x03 && data[3] == 0x04 {
return "ooxml"
}
// OLE Compound Document signature: D0 CF 11 E0 A1 B1 1A E1. Require
// the full eight bytes to avoid misclassifying truncated input
// (a four-byte prefix could also be garbage).
if len(data) >= 8 && data[0] == 0xD0 && data[1] == 0xCF && data[2] == 0x11 && data[3] == 0xE0 &&
data[4] == 0xA1 && data[5] == 0xB1 && data[6] == 0x1A && data[7] == 0xE1 {
return "ole"
}
return ""
}
@@ -1,228 +0,0 @@
package parser
import (
"fmt"
"testing"
)
// TestOfficeContainer verifies the magic-byte sniffing that lets the
// office_oxide parsers recover from a mislabeled extension (e.g. a
// legacy .doc uploaded as .docx). office_oxide::OpenFromBytes takes the
// container format on trust and does no detection of its own.
func TestOfficeContainer(t *testing.T) {
tests := []struct {
name string
data []byte
want string
}{
{"ooxml", []byte{0x50, 0x4B, 0x03, 0x04, 0x00, 0x00}, "ooxml"},
{"ole", []byte{0xD0, 0xCF, 0x11, 0xE0, 0xA1, 0xB1, 0x1A, 0xE1}, "ole"},
{"ole_with_trailing", []byte{0xD0, 0xCF, 0x11, 0xE0, 0xA1, 0xB1, 0x1A, 0xE1, 0x00, 0xFF}, "ole"},
{"ole_short_header", []byte{0xD0, 0xCF, 0x11, 0xE0}, ""},
{"ole_truncated_7", []byte{0xD0, 0xCF, 0x11, 0xE0, 0xA1, 0xB1, 0x1A}, ""},
{"empty", []byte{}, ""},
{"too_short", []byte{0x50, 0x4B}, ""},
{"ooxml_truncated_3", []byte{0x50, 0x4B, 0x03}, ""},
{"garbage", []byte{0x00, 0x11, 0x22, 0x33}, ""},
{"ole_prefix_mismatch", []byte{0xD0, 0xCF, 0x11, 0xE0, 0xA1, 0xB1, 0x1A, 0x00}, ""},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := officeContainer(tt.data); got != tt.want {
t.Errorf("officeContainer(% x) = %q, want %q", tt.data, got, tt.want)
}
})
}
}
// TestDOCXParser_FallsBackToDOCForOLEHeader ensures a legacy OLE .doc
// payload routed to the DOCXParser (the .doc-as-.docx case) is opened
// with the "doc" container format rather than "docx". A minimal OLE
// header is not a valid document, so office_oxide still errors — but
// the error must come from the DOC path, proving the fallback fired.
// The test uses a seam to capture the effective format without needing
// a full valid OLE document fixture.
func TestDOCXParser_FallsBackToDOCForOLEHeader(t *testing.T) {
ctx := t.Context()
p := NewDOCXParser()
orig := docxExtract
defer func() { docxExtract = orig }()
var gotFormat string
docxExtract = func(data []byte, format string) (string, string, string, error, error) {
gotFormat = format
return "", "", "", nil, fmt.Errorf("stub doc open: %s", format)
}
data := []byte{0xD0, 0xCF, 0x11, 0xE0, 0xA1, 0xB1, 0x1A, 0xE1}
res := p.ParseWithResult(ctx, "renamed.docx", data)
if res.Err == nil {
t.Fatal("expected parse error for a non-document OLE header, got nil")
}
if gotFormat != "doc" {
t.Fatalf("docxExtract format = %q, want %q (OLE fallback not triggered)", gotFormat, "doc")
}
}
// TestDOCXParser_JSONEmptyIRReturnsError locks Finding B: when the native
// backend opens the document but returns an empty structured IR, the JSON
// output path must fail loud (it is built entirely from the IR, which the
// Markdown path can fall back from plain text). This guards against silently
// emitting empty JSON sections.
func TestDOCXParser_JSONEmptyIRReturnsError(t *testing.T) {
ctx := t.Context()
p := NewDOCXParser()
orig := docxExtract
defer func() { docxExtract = orig }()
docxExtract = func(data []byte, format string) (string, string, string, error, error) {
// Open succeeds, but the IR view is empty.
return "", "", "", nil, nil
}
res := p.ParseWithResult(ctx, "empty.docx", []byte("PK\x03\x04"))
if res.Err == nil {
t.Fatal("expected error for JSON output with empty IR, got nil")
}
}
// TestDOCXParser_AllViewsEmptyReturnsError locks the engine-level contract
// at the parser boundary: when the native backend opens the document but
// every view (IR / Markdown / PlainText) is empty, OpenAndExtract reports a
// joined error, and the parser must fail loud rather than silently emit
// empty JSON. (office_oxide always returns a structured IR for an openable
// doc, so this all-empty error is simulated via the seam rather than a real
// fixture.)
func TestDOCXParser_AllViewsEmptyReturnsError(t *testing.T) {
ctx := t.Context()
p := NewDOCXParser()
orig := docxExtract
defer func() { docxExtract = orig }()
docxExtract = func(data []byte, format string) (string, string, string, error, error) {
return "", "", "", nil, fmt.Errorf("office_oxide docx: all text views failed")
}
res := p.ParseWithResult(ctx, "empty.docx", []byte("PK\x03\x04"))
if res.Err == nil {
t.Fatal("expected error for a document with no extractable text in any view, got nil")
}
}
// TestDOCXParser_TruncatedOLENotFallback ensures a truncated OLE prefix
// (only 4 bytes) does not trigger the DOC fallback — it must stay "docx"
// so truncated garbage is not misclassified as OLE.
func TestDOCXParser_TruncatedOLENotFallback(t *testing.T) {
ctx := t.Context()
p := NewDOCXParser()
orig := docxExtract
defer func() { docxExtract = orig }()
var gotFormat string
docxExtract = func(data []byte, format string) (string, string, string, error, error) {
gotFormat = format
return "", "", "", nil, fmt.Errorf("stub")
}
data := []byte{0xD0, 0xCF, 0x11, 0xE0}
res := p.ParseWithResult(ctx, "truncated.docx", data)
if res.Err == nil {
t.Fatal("expected error, got nil")
}
if gotFormat != "docx" {
t.Fatalf("format = %q, want %q for truncated OLE (should not fallback)", gotFormat, "docx")
}
}
// TestPPTXParser_FallsBackToPPTForOLEHeader ensures a legacy OLE .ppt
// payload routed to PPTXParser is opened with "ppt" rather than "pptx".
func TestPPTXParser_FallsBackToPPTForOLEHeader(t *testing.T) {
ctx := t.Context()
p := NewPPTXParser()
orig := pptxExtract
defer func() { pptxExtract = orig }()
var gotFormat string
pptxExtract = func(data []byte, format string) (string, string, string, error, error) {
gotFormat = format
return "", "", "", nil, fmt.Errorf("stub ppt open: %s", format)
}
data := []byte{0xD0, 0xCF, 0x11, 0xE0, 0xA1, 0xB1, 0x1A, 0xE1}
res := p.ParseWithResult(ctx, "renamed.pptx", data)
if res.Err == nil {
t.Fatal("expected parse error for OLE header, got nil")
}
if gotFormat != "ppt" {
t.Fatalf("pptxExtract format = %q, want %q", gotFormat, "ppt")
}
if p.format != "pptx" {
t.Fatalf("PPTXParser.format mutated to %q, want %q (must use local effFormat)", p.format, "pptx")
}
}
// TestPPTParser_FallsBackToPPTXForOOXMLHeader ensures an OOXML .pptx
// payload uploaded as .ppt ( PPTParser with p.format == "ppt") is
// correctly routed to "pptx" via the bidirectional container sniffing.
func TestPPTParser_FallsBackToPPTXForOOXMLHeader(t *testing.T) {
ctx := t.Context()
p := NewPPTParser() // underlying PPTXParser{format:"ppt"}
orig := pptxExtract
defer func() { pptxExtract = orig }()
var gotFormat string
pptxExtract = func(data []byte, format string) (string, string, string, error, error) {
gotFormat = format
return "", "", "", nil, fmt.Errorf("stub ppt open: %s", format)
}
// Minimal OOXML ZIP local file header.
data := []byte{0x50, 0x4B, 0x03, 0x04, 0x00, 0x00}
res := p.ParseWithResult(ctx, "renamed.ppt", data)
if res.Err == nil {
t.Fatal("expected parse error for OOXML header stub, got nil")
}
if gotFormat != "pptx" {
t.Fatalf("pptxExtract format = %q, want %q (OOXML fallback ppt->pptx)", gotFormat, "pptx")
}
// Underlying PPTXParser must not be mutated permanently.
if p.pptx.format != "ppt" {
t.Fatalf("PPTParser.pptx.format mutated to %q, want %q (must use local effFormat)", p.pptx.format, "ppt")
}
}
// TestPPTXParser_NoStatePollution verifies that after an OLE file mutates
// the effective format for that call, a subsequent OOXML file still uses
// "pptx" — i.e., the parser instance is not permanently polluted.
func TestPPTXParser_NoStatePollution(t *testing.T) {
ctx := t.Context()
p := NewPPTXParser()
orig := pptxExtract
defer func() { pptxExtract = orig }()
calls := []string{}
pptxExtract = func(data []byte, format string) (string, string, string, error, error) {
calls = append(calls, format)
return "", "", "", nil, fmt.Errorf("stub")
}
oleData := []byte{0xD0, 0xCF, 0x11, 0xE0, 0xA1, 0xB1, 0x1A, 0xE1}
_ = p.ParseWithResult(ctx, "a.pptx", oleData)
ooxmlData := []byte{0x50, 0x4B, 0x03, 0x04, 0x00, 0x00}
_ = p.ParseWithResult(ctx, "b.pptx", ooxmlData)
if len(calls) != 2 {
t.Fatalf("calls = %v, want 2", calls)
}
if calls[0] != "ppt" {
t.Errorf("first call format = %q, want %q", calls[0], "ppt")
}
if calls[1] != "pptx" {
t.Errorf("second call format = %q, want %q (state polluted)", calls[1], "pptx")
}
}
+2 -2
View File
@@ -373,7 +373,7 @@ func TestDOCXTextParserSkipsImagesWithoutCache(t *testing.T) {
return irJSON, "", "", nil, nil
}
p, err := GetTextOnlyParser(utility.FileTypeDOCX)
p, _, err := GetTextOnlyParser(utility.FileTypeDOCX, nil)
if err != nil {
t.Fatal(err)
}
@@ -418,7 +418,7 @@ func TestPPTXTextParserSkipsImagesWithoutCache(t *testing.T) {
return irJSON, "", "", nil, nil
}
p, err := GetTextOnlyParser(utility.FileTypePPTX)
p, _, err := GetTextOnlyParser(utility.FileTypePPTX, nil)
if err != nil {
t.Fatal(err)
}
@@ -22,7 +22,7 @@ func TestGetParser_RoutesOfficeFamilies(t *testing.T) {
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
p, err := GetParser(tc.fileType)
p, _, err := GetParser(tc.fileType, nil)
if err != nil {
t.Fatalf("GetParser(%q): %v", tc.fileType, err)
}
@@ -47,13 +47,13 @@ func TestGetTextOnlyParserRoutesOfficeFamilies(t *testing.T) {
return ok && parser.textOnly
}},
{utility.FileTypeXLS, func(p ParseResultProducer) bool {
parser, ok := p.(*XLSParser)
return ok && parser.textOnly
_, ok := p.(*XLSParser)
return ok
}},
}
for _, tc := range tests {
t.Run(string(tc.fileType), func(t *testing.T) {
p, err := GetTextOnlyParser(tc.fileType)
p, _, err := GetTextOnlyParser(tc.fileType, nil)
if err != nil {
t.Fatalf("GetTextOnlyParser: %v", err)
}
@@ -0,0 +1,85 @@
package parser
import (
"archive/zip"
"bytes"
"github.com/richardlehane/mscfb"
"ragflow/internal/utility"
)
func (*DOCParser) TryParse(data []byte) utility.FileType { return tryParseOffice(data) }
func (*DOCXParser) TryParse(data []byte) utility.FileType { return tryParseOffice(data) }
func (*XLSParser) TryParse(data []byte) utility.FileType { return tryParseOffice(data) }
func (*XLSXParser) TryParse(data []byte) utility.FileType { return tryParseOffice(data) }
func (*PPTParser) TryParse(data []byte) utility.FileType { return tryParseOffice(data) }
func (*PPTXParser) TryParse(data []byte) utility.FileType { return tryParseOffice(data) }
// tryParseOffice reads container directories without extracting document bodies.
// A container signature alone is insufficient to identify an Office format.
func tryParseOffice(data []byte) utility.FileType {
actual := utility.FileTypeOTHER
accept := func(candidate utility.FileType) bool {
if actual != utility.FileTypeOTHER && actual != candidate {
return false
}
actual = candidate
return true
}
switch {
case bytes.HasPrefix(data, []byte{'P', 'K', 3, 4}):
r, err := zip.NewReader(bytes.NewReader(data), int64(len(data)))
if err != nil {
return utility.FileTypeOTHER
}
hasContentTypes := false
for _, f := range r.File {
switch f.Name {
case "[Content_Types].xml":
hasContentTypes = true
case "word/document.xml":
if !accept(utility.FileTypeDOCX) {
return utility.FileTypeOTHER
}
case "xl/workbook.xml":
if !accept(utility.FileTypeXLSX) {
return utility.FileTypeOTHER
}
case "ppt/presentation.xml":
if !accept(utility.FileTypePPTX) {
return utility.FileTypeOTHER
}
}
}
if !hasContentTypes {
return utility.FileTypeOTHER
}
case bytes.HasPrefix(data, CompoundDocumentSignature):
r, err := mscfb.New(bytes.NewReader(data))
if err != nil {
return utility.FileTypeOTHER
}
for _, f := range r.File {
if len(f.Path) != 0 || f.FileInfo().IsDir() {
continue
}
switch f.Name {
case "EncryptedPackage", "EncryptionInfo":
return utility.FileTypeOTHER
case "WordDocument":
if !accept(utility.FileTypeDOC) {
return utility.FileTypeOTHER
}
case "Workbook", "Book":
if !accept(utility.FileTypeXLS) {
return utility.FileTypeOTHER
}
case "PowerPoint Document":
if !accept(utility.FileTypePPT) {
return utility.FileTypeOTHER
}
}
}
}
return actual
}
@@ -0,0 +1,76 @@
package parser
import (
"archive/zip"
"bytes"
"os"
"testing"
"ragflow/internal/utility"
)
func TestOfficeTryParse(t *testing.T) {
docx := officeProbeZIP(t, "[Content_Types].xml", "word/document.xml")
xlsx := officeProbeZIP(t, "[Content_Types].xml", "xl/workbook.xml")
pptx := officeProbeZIP(t, "[Content_Types].xml", "ppt/presentation.xml")
ordinaryZIP := officeProbeZIP(t, "notes.txt")
missingContentTypes := officeProbeZIP(t, "word/document.xml")
ambiguousZIP := officeProbeZIP(t, "[Content_Types].xml", "word/document.xml", "xl/workbook.xml")
doc, err := os.ReadFile("testdata/table.doc")
if err != nil {
t.Fatal(err)
}
for _, tc := range []struct {
name string
data []byte
want utility.FileType
}{
{"docx", docx, utility.FileTypeDOCX},
{"xlsx", xlsx, utility.FileTypeXLSX},
{"pptx", pptx, utility.FileTypePPTX},
{"doc", doc, utility.FileTypeDOC},
{"ordinary_zip", ordinaryZIP, utility.FileTypeOTHER},
{"missing_content_types", missingContentTypes, utility.FileTypeOTHER},
{"ambiguous_zip", ambiguousZIP, utility.FileTypeOTHER},
{"truncated_zip", []byte("PK\x03\x04"), utility.FileTypeOTHER},
{"truncated_ole", CompoundDocumentSignature, utility.FileTypeOTHER},
{"unknown", []byte("plain text"), utility.FileTypeOTHER},
} {
t.Run(tc.name, func(t *testing.T) {
if got := tryParseOffice(tc.data); got != tc.want {
t.Fatalf("got %q, want %q", got, tc.want)
}
})
}
}
func officeProbeZIP(t *testing.T, names ...string) []byte {
t.Helper()
var buf bytes.Buffer
w := zip.NewWriter(&buf)
for _, name := range names {
f, err := w.Create(name)
if err != nil {
t.Fatal(err)
}
if _, err := f.Write([]byte("<document/>")); err != nil {
t.Fatal(err)
}
}
if err := w.Close(); err != nil {
t.Fatal(err)
}
return buf.Bytes()
}
func TestOfficeParsersSupportTryParse(t *testing.T) {
for _, ft := range []utility.FileType{utility.FileTypeDOC, utility.FileTypeDOCX, utility.FileTypeXLS, utility.FileTypeXLSX, utility.FileTypePPT, utility.FileTypePPTX} {
p, err := newParser(ft, false)
if err != nil {
t.Fatal(err)
}
if _, ok := p.(TryParser); !ok {
t.Fatalf("%s parser does not support TryParse", ft)
}
}
}
+1 -8
View File
@@ -49,14 +49,7 @@ type ParseResult struct {
// Nil when the backend does not produce metadata. It is not an augmented
// copy of the upstream file descriptor.
//
// For the office family (docx/doc, pptx/ppt), File["format"]
// reflects the real container format detected via magic-byte
// sniffing (officeContainer), not just the file extension.
// E.g. a legacy OLE .doc uploaded as .docx yields
// File["format"] == "doc" so downstream consumers see the
// effective format. This is an intentional contract change
// from the pre-fallback behaviour where File["format"] always
// echoed the requested extension.
// Office parsers report their selected format in File["format"].
File map[string]any
// JSON is the structured payload when available, as typed DocItems.
@@ -289,7 +289,7 @@ func TestHTMLParser_ParseWithResult_SkipsScriptAndStyle(t *testing.T) {
// `utility.FileTypeTXT` resolves to a TextParser that satisfies
// ParseResultProducer.
func TestGetParser_RoutesTextAndCode(t *testing.T) {
p, err := GetParser(utility.FileTypeTXT)
p, _, err := GetParser(utility.FileTypeTXT, nil)
if err != nil {
t.Fatalf("GetParser(FileTypeTXT): %v", err)
}
@@ -0,0 +1,112 @@
//go:build cgo
package parser
import (
"encoding/base64"
"os"
"testing"
"github.com/xuri/excelize/v2"
"ragflow/internal/utility"
)
func TestParserSelectionCorrectsFormat(t *testing.T) {
type selector func(utility.FileType, []byte) (ParseResultProducer, utility.FileType, error)
doc, err := os.ReadFile("testdata/table.doc")
if err != nil {
t.Fatal(err)
}
xlsx := newTestXLSX(t, func(f *excelize.File) { mustSetCell(t, f, "Sheet1", "A1", "Selected workbook") })
xls, err := base64.StdEncoding.DecodeString(genuineBIFF8FixtureBase64)
if err != nil {
t.Fatal(err)
}
for mode, selectParser := range map[string]selector{"ingestion": GetParser, "text": GetTextOnlyParser} {
t.Run(mode, func(t *testing.T) {
for _, tc := range []struct {
name string
requested, want utility.FileType
data []byte
}{
{"doc_as_docx", utility.FileTypeDOCX, utility.FileTypeDOC, doc},
{"docx_as_doc", utility.FileTypeDOC, utility.FileTypeDOCX, minimalDOCX(t, "Selected document")},
{"xlsx_as_xls", utility.FileTypeXLS, utility.FileTypeXLSX, xlsx},
{"xls_as_xlsx", utility.FileTypeXLSX, utility.FileTypeXLS, xls},
{"pptx_as_ppt", utility.FileTypePPT, utility.FileTypePPTX, buildPPTX(t, "Selected presentation")},
{"unknown", utility.FileTypeDOC, utility.FileTypeDOC, []byte("unrecognized")},
{"unchecked_text", utility.FileTypeTXT, utility.FileTypeTXT, []byte("plain text")},
} {
t.Run(tc.name, func(t *testing.T) {
p, actual, err := selectParser(tc.requested, tc.data)
if err != nil {
t.Fatal(err)
}
if actual != tc.want {
t.Fatalf("format=%q, want %q", actual, tc.want)
}
res := p.ParseWithResult(t.Context(), "original."+string(tc.requested), tc.data)
if tc.name == "unknown" {
if res.Err == nil {
t.Fatal("unrecognized document should still fail in the original parser")
}
return
}
if res.Err != nil {
t.Fatal(res.Err)
}
if !res.Produced() {
t.Fatal("selected parser produced no result")
}
if actual != utility.FileTypeTXT && res.File["format"] != string(actual) {
t.Fatalf("result format=%v, selected type=%s", res.File["format"], actual)
}
})
}
})
}
}
func TestOfficeParsersDoNotChangeFormat(t *testing.T) {
doc, err := os.ReadFile("testdata/table.doc")
if err != nil {
t.Fatal(err)
}
if res := NewDOCXParser().ParseWithResult(t.Context(), "renamed.docx", doc); res.Err == nil {
t.Fatal("DOCX parser silently accepted DOC")
}
xlsx := newTestXLSX(t, func(f *excelize.File) { mustSetCell(t, f, "Sheet1", "A1", "workbook") })
p, err := NewXLSParser("")
if err != nil {
t.Fatal(err)
}
if res := p.ParseWithResult(t.Context(), "renamed.xls", xlsx); res.Err == nil {
t.Fatal("XLS parser silently accepted XLSX")
}
}
func TestCorrectedTextOnlyParserSkipsMedia(t *testing.T) {
data, err := os.ReadFile("testdata/docx_table_image.docx")
if err != nil {
t.Fatal(err)
}
p, actual, err := GetTextOnlyParser(utility.FileTypeDOC, data)
if err != nil {
t.Fatal(err)
}
if actual != utility.FileTypeDOCX {
t.Fatalf("actual=%q", actual)
}
res := p.ParseWithResult(t.Context(), "renamed.doc", data)
if res.Err != nil {
t.Fatal(res.Err)
}
if len(res.JSON) == 0 {
t.Fatal("no extracted contents")
}
for _, item := range res.JSON {
if item.ToMap()["doc_type_kwd"] == "image" {
t.Fatal("text-only selection extracted an image")
}
}
}
+32 -15
View File
@@ -22,17 +22,39 @@ import (
"ragflow/internal/utility"
)
// GetTextOnlyParser returns a parser for readable-text extraction: previews,
// attachments, agent uploads and CLI parsing. Embedded media is skipped even
// when the extraction is nested inside an ingestion run (email attachments).
// These paths also work without an injected run media cache.
func GetTextOnlyParser(fileType utility.FileType) (ParseResultProducer, error) {
return newParser(fileType, true)
// TryParser optionally identifies a format before full parsing by inspecting
// recognition structures. FileTypeOTHER means unknown: keep the requested type.
type TryParser interface {
TryParse(data []byte) utility.FileType
}
// GetParser returns the ingestion parser for the given file type.
func GetParser(fileType utility.FileType) (ParseResultProducer, error) {
return newParser(fileType, false)
// GetTextOnlyParser selects a parser and effective file type for readable-text
// extraction: previews, attachments, agent uploads and CLI parsing.
// Embedded media is skipped even when extraction is nested inside an ingestion
// run (email attachments).
// These paths also work without an injected run media cache.
func GetTextOnlyParser(fileType utility.FileType, data []byte) (ParseResultProducer, utility.FileType, error) {
return selectParser(fileType, data, true)
}
// GetParser selects the ingestion parser and effective file type from the
// requested type and contents. Unrecognized contents keep the requested type.
func GetParser(fileType utility.FileType, data []byte) (ParseResultProducer, utility.FileType, error) {
return selectParser(fileType, data, false)
}
func selectParser(fileType utility.FileType, data []byte, textOnly bool) (ParseResultProducer, utility.FileType, error) {
p, err := newParser(fileType, textOnly)
if err != nil {
return nil, fileType, err
}
if probe, ok := p.(TryParser); ok {
if actual := probe.TryParse(data); actual != "" && actual != utility.FileTypeOTHER && actual != fileType {
p, err = newParser(actual, textOnly)
return p, actual, err
}
}
return p, fileType, nil
}
// newParser owns the format routes for both ingestion and text extraction.
@@ -54,12 +76,7 @@ func newParser(fileType utility.FileType, textOnly bool) (ParseResultProducer, e
p.textOnly = textOnly
return p, nil
case utility.FileTypeXLS:
p, err := NewXLSParser("")
if err != nil {
return nil, err
}
p.textOnly = textOnly
return p, nil
return NewXLSParser("")
case utility.FileTypeCSV:
return NewCSVParser(), nil
case utility.FileTypeDOCX:
@@ -132,24 +132,6 @@ func TestPPTXParser_ParseWithResult_EmptySlide_CGO(t *testing.T) {
}
}
// TestPPTParser_ParseWithResult_CGO verifies that PPTParser
// delegates correctly to PPTXParser{format:"ppt"} and, via the
// bidirectional magic-byte fallback, correctly opens OOXML content
// through the PPTX path. A .pptx payload uploaded as .ppt should
// succeed and report File["format"] == "pptx" (real container).
func TestPPTParser_ParseWithResult_CGO(t *testing.T) {
ctx := t.Context()
p := NewPPTParser()
data := buildPPTX(t, "Hello")
res := p.ParseWithResult(ctx, "test.ppt", data)
if res.Err != nil {
t.Fatalf("ParseWithResult: %v", res.Err)
}
if got := res.File["format"]; got != "pptx" {
t.Errorf("File[format] = %v, want %q (OOXML fallback ppt->pptx)", got, "pptx")
}
}
// TestPPTParser_TCADPFileType verifies that when a .ppt file is routed
// through PPTParser with parse_method="tcadp", the underlying
// PPTXParser must pass "PPT" as the file_type to TCADP, not "PPTX".
+3 -29
View File
@@ -91,12 +91,6 @@ func (p *PPTXParser) ParseWithResult(ctx context.Context, filename string, data
method := strings.ToLower(strings.TrimSpace(p.ParseMethod))
switch method {
case "tcadp":
// TCADP file_type is intentionally derived from the parser's
// configured family (p.format == "pptx" → "PPTX", "ppt" → "PPT"),
// not from magic-byte sniffing. The pipeline routes purely on
// extension → parser family, and TCADP reconstructs by family.
// The OLE fallback below therefore only applies to the local
// office_oxide path.
return parseWithTCADP(ctx,
filename, data, strings.ToUpper(p.format),
p.TCADPAPIServer, p.TCADPAPIKey,
@@ -109,27 +103,7 @@ func (p *PPTXParser) ParseWithResult(ctx context.Context, filename string, data
// meaningless for PPTX; treat as default path.
}
// office_oxide's OpenFromBytes takes the container format as given
// and does no magic-byte detection, so a legacy .ppt uploaded with a
// .pptx extension would be parsed as ZIP/OOXML and fail. Sniff the
// real container and pass the matching format so mislabeled files
// still parse (mirrors the magic-byte correction office_oxide::Open
// performs on file paths).
// File["format"] in the success case reflects effFormat, i.e. the
// real container ("ppt" for OLE fallback, "pptx" for OOXML fallback),
// not just the extension.
effFormat := p.format
switch officeContainer(data) {
case "ole":
if effFormat == "pptx" {
effFormat = "ppt"
}
case "ooxml":
if effFormat == "ppt" {
effFormat = "pptx"
}
}
irJSON, _, plainText, _, err := pptxExtract(data, effFormat)
irJSON, _, plainText, _, err := pptxExtract(data, p.format)
if err != nil {
if errors.Is(err, deepdocoffice.ErrOfficeCGORequired) {
return ParseResult{Err: fmt.Errorf("%w: %s", ErrOfficeCGORequired, filename)}
@@ -148,7 +122,7 @@ func (p *PPTXParser) ParseWithResult(ctx context.Context, filename string, data
return ParseResult{Err: fmt.Errorf("presentation ir-json: empty result")}
}
return ParseResult{
File: map[string]any{"name": filename, "format": effFormat},
File: map[string]any{"name": filename, "format": p.format},
JSON: MapsToItems(itemsFromPlainText(plainText)),
}
}
@@ -179,7 +153,7 @@ func (p *PPTXParser) ParseWithResult(ctx context.Context, filename string, data
}
return ParseResult{
File: map[string]any{"name": filename, "format": effFormat},
File: map[string]any{"name": filename, "format": p.format},
JSON: MapsToItems(items),
Warnings: sink.warnings(),
}
+3 -25
View File
@@ -33,7 +33,6 @@ import (
var CompoundDocumentSignature = []byte{0xd0, 0xcf, 0x11, 0xe0, 0xa1, 0xb1, 0x1a, 0xe1}
type XLSParser struct {
textOnly bool
ParseMethod string
TCADPAPIServer string
TCADPAPIKey string
@@ -92,27 +91,7 @@ func (p *XLSParser) ParseWithResult(ctx context.Context, filename string, data [
}
}
var (
items []map[string]any
warnings []string
sheetsCount int
err error
)
if bytes.HasPrefix(data, CompoundDocumentSignature) {
// BIFF8 extraction reads cell text only; the drawing layer is not
// parsed, so no image can reach the media cache from this branch.
items, sheetsCount, err = parseBIFF8Bytes(ctx, data)
} else {
var sink *officeMediaSink
if !p.textOnly {
sink = newOfficeMediaSink(ctx, officeMediaFormatXLSX)
}
items, warnings, sheetsCount, err = parseXLSXBytes(data, sink)
if err == nil && sink != nil && sink.err != nil {
return ParseResult{Err: sink.err}
}
warnings = append(warnings, sink.warnings()...)
}
items, sheetsCount, err := parseBIFF8Bytes(ctx, data)
if err := ctx.Err(); err != nil {
return ParseResult{Err: err}
}
@@ -121,9 +100,8 @@ func (p *XLSParser) ParseWithResult(ctx context.Context, filename string, data [
}
return ParseResult{
File: map[string]any{"name": filename, "format": "xls", "sheets": sheetsCount},
JSON: MapsToItems(items),
Warnings: warnings,
File: map[string]any{"name": filename, "format": "xls", "sheets": sheetsCount},
JSON: MapsToItems(items),
}
}
+8 -52
View File
@@ -205,50 +205,6 @@ func TestXLSParser_GenuineBIFF8SpreadsheetJSONOutput(t *testing.T) {
}
}
func TestXLSParser_SpreadsheetJSONOutput(t *testing.T) {
data := createTestExcelBytes(t)
p, err := NewXLSParser("excelize")
if err != nil {
t.Fatalf("NewXLSParser failed: %v", err)
}
res := p.ParseWithResult(context.Background(), "test.xls", data)
if res.Err != nil {
t.Fatalf("ParseWithResult failed: %v", res.Err)
}
// Default output format for XLSParser is "json"
if !res.Produced() {
t.Errorf("res.produced = %t, want true", res.Produced())
}
if len(res.JSON) == 0 {
t.Fatalf("res.JSON should have structured table items")
}
item := res.JSON[0].ToMap()
if item["doc_type_kwd"] != "table" {
t.Errorf("table item doc_type_kwd = %v, want table", item["doc_type_kwd"])
}
text := tableItemHTML(t, item)
if !strings.Contains(text, "<tr><th>Header1</th><th>Header2</th></tr>") ||
!strings.Contains(text, "<tr><td>Val1</td><td>Val2</td></tr>") {
t.Errorf("table markup = %q, want captioned header and data row", text)
}
if item["doc_type_kwd"] != "table" {
t.Errorf("doc_type_kwd = %v, want 'table'", item["doc_type_kwd"])
}
resJSON := p.ParseWithResult(context.Background(), "test.xls", data)
if resJSON.Err != nil {
t.Fatalf("ParseWithResult failed: %v", resJSON.Err)
}
if !resJSON.Produced() {
t.Errorf("resJSON.produced = %t, want true", resJSON.Produced())
}
if len(resJSON.JSON) == 0 {
t.Errorf("resJSON.JSON should not be empty")
}
}
func TestXLSXParser_JSONOutput(t *testing.T) {
data := createTestExcelBytes(t)
p, err := NewXLSXParser("excelize")
@@ -272,7 +228,7 @@ func TestXLSXParser_JSONOutput(t *testing.T) {
}
}
func TestXLSParser_ImagesDoNotRunOCR(t *testing.T) {
func TestXLSXParser_ImagesDoNotRunOCR(t *testing.T) {
var imageData bytes.Buffer
if err := png.Encode(&imageData, image.NewRGBA(image.Rect(0, 0, 2, 2))); err != nil {
t.Fatalf("encode fixture: %v", err)
@@ -298,12 +254,12 @@ func TestXLSParser_ImagesDoNotRunOCR(t *testing.T) {
return &deepdocpdf.MockDocAnalyzer{Healthy: true, OCRDetectErr: errors.New("xls OCR sentinel")}, true
})
p, err := NewXLSParser("excelize")
p, err := NewXLSXParser("excelize")
if err != nil {
t.Fatalf("NewXLSParser: %v", err)
t.Fatalf("NewXLSXParser: %v", err)
}
ctx, cache := officeMediaTestContext()
result := p.ParseWithResult(ctx, "with-image.xls", workbook.Bytes())
result := p.ParseWithResult(ctx, "with-image.xlsx", workbook.Bytes())
if result.Err != nil {
t.Fatalf("ParseWithResult: %v", result.Err)
}
@@ -329,7 +285,7 @@ func TestXLSParser_ImagesDoNotRunOCR(t *testing.T) {
}
}
func TestXLSParser_CapsEmbeddedImageCount(t *testing.T) {
func TestXLSXParser_CapsEmbeddedImageCount(t *testing.T) {
const limit = 256
t.Setenv("RAGFLOW_MEDIA_MAX_IMAGES_PER_RUN", "256")
f := excelize.NewFile()
@@ -353,12 +309,12 @@ func TestXLSParser_CapsEmbeddedImageCount(t *testing.T) {
deepdoctype.SetNativeDocAnalyzerFactory(func() (deepdoctype.DocAnalyzer, bool) {
return &deepdocpdf.MockDocAnalyzer{Healthy: true}, true
})
p, err := NewXLSParser("excelize")
p, err := NewXLSXParser("excelize")
if err != nil {
t.Fatalf("NewXLSParser: %v", err)
t.Fatalf("NewXLSXParser: %v", err)
}
ctx, _ := officeMediaTestContext()
result := p.ParseWithResult(ctx, "many-images.xls", workbook.Bytes())
result := p.ParseWithResult(ctx, "many-images.xlsx", workbook.Bytes())
if result.Err != nil {
t.Fatalf("ParseWithResult: %v", result.Err)
}
+1 -1
View File
@@ -56,7 +56,7 @@ func TestXLSXTextParserSkipsImagesWithoutCache(t *testing.T) {
t.Fatalf("AddPictureFromBytes: %v", err)
}
})
p, err := GetTextOnlyParser(utility.FileTypeXLSX)
p, _, err := GetTextOnlyParser(utility.FileTypeXLSX, nil)
if err != nil {
t.Fatal(err)
}
@@ -319,10 +319,13 @@ func (s *DocumentService) GetDocumentPreview(ctx context.Context, docID string)
// document via the office_oxide-backed DOCParser. The parser only emits JSON
// payloads, so the preview text is flattened from the JSON items.
func extractDOCPreviewText(ctx context.Context, filename string, data []byte) (string, error) {
p, err := parser.GetTextOnlyParser(utility.FileTypeDOC)
p, actual, err := parser.GetTextOnlyParser(utility.FileTypeDOC, data)
if err != nil {
return "", err
}
if actual != utility.FileTypeDOC {
common.Warn(fmt.Sprintf("file %q: requested format doc; detected %s; continuing with the %s parser", filename, actual, actual))
}
res := p.ParseWithResult(ctx, filename, data)
if res.Err != nil {
return "", res.Err
@@ -0,0 +1,23 @@
//go:build cgo
package document
import (
"os"
"strings"
"testing"
)
func TestDOCPreviewCorrectsDOCXContent(t *testing.T) {
data, err := os.ReadFile("../../parser/parser/testdata/docx_table_image.docx")
if err != nil {
t.Fatal(err)
}
text, err := extractDOCPreviewText(t.Context(), "renamed.doc", data)
if err != nil {
t.Fatal(err)
}
if strings.TrimSpace(text) == "" || text == string(data) {
t.Fatal("preview did not extract corrected DOCX content")
}
}
+9 -2
View File
@@ -5,6 +5,7 @@ import (
"encoding/base64"
"encoding/json"
"fmt"
"ragflow/internal/common"
"ragflow/internal/dao"
"ragflow/internal/entity"
"ragflow/internal/parser/parser"
@@ -217,10 +218,13 @@ func parseAgentUploadContent(ctx context.Context, filename string, data []byte,
content := string(data)
fileType := utility.GetFileType(filename)
if fileType != utility.FileTypeOTHER {
fp, err := parser.GetTextOnlyParser(fileType)
fp, actual, err := parser.GetTextOnlyParser(fileType, data)
if err != nil {
return "", err
}
if actual != fileType {
common.Warn(fmt.Sprintf("file %q: requested format %s; detected %s; continuing with the %s parser", filename, fileType, actual, actual))
}
if configurable, ok := fp.(interface{ ConfigureFromSetup(map[string]any) }); ok {
configurable.ConfigureFromSetup(map[string]any{"layout_recognize": layoutRecognize})
}
@@ -269,10 +273,13 @@ func parseFileContent(ctx context.Context, filename string, data []byte) string
if fileType == utility.FileTypeOTHER {
return string(data)
}
fp, err := parser.GetTextOnlyParser(fileType)
fp, actual, err := parser.GetTextOnlyParser(fileType, data)
if err != nil {
return string(data)
}
if actual != fileType {
common.Warn(fmt.Sprintf("file %q: requested format %s; detected %s; continuing with the %s parser", filename, fileType, actual, actual))
}
res := fp.ParseWithResult(ctx, filename, data)
if res.Err != nil {
return string(data)
@@ -82,6 +82,25 @@ func TestParseFileContent_DOCXReadsJSONItems(t *testing.T) {
}
}
func TestMisnamedOfficeContentExtraction(t *testing.T) {
data := minimalDOCX(t, "Corrected document content")
for _, name := range []string{"renamed.doc", "renamed.wps", "renamed.xls"} {
t.Run(name, func(t *testing.T) {
content := parseFileContent(t.Context(), name, data)
if !strings.Contains(content, "Corrected document content") || content == string(data) {
t.Fatal("file extraction did not use the corrected parser")
}
content, err := parseAgentUploadContent(t.Context(), name, data, "")
if err != nil {
t.Fatal(err)
}
if !strings.Contains(content, "Corrected document content") {
t.Fatal("agent extraction did not use the corrected parser")
}
})
}
}
// TestOfficeContentExtractionWithoutMediaCache pins that previews, uploads and
// other non-ingestion callers use the text-only Office parsers: an
// image-bearing .docx must yield readable text instead of failing because no