refactor(deepdoc): make model filenames, threshold, and post-processing overridable at runtime (#21275)

internal/deepdoc/native/det_core.go

@@ -24,7 +24,6 @@ import (
 const (
 	detLimitSideLen  = 960
 	detThresh        = 0.3
-	detBoxThresh     = 0.5
 	detMaxCandidates = 1000
 	detUnclipRatio   = 1.5
 	detMinSize       = 3
@@ -33,6 +32,18 @@ const (
 	detStd0, detStd1, detStd2    = 0.229, 0.224, 0.225
 )
 
+// detBoxThresh is the detection box confidence threshold. It is a package-level
+// variable (not a constant) so a downstream build variant can override it.
+// Override seam: a downstream build variant assigns this at init/startup,
+// before any inference runs. It is set once and never mutated during serving,
+// so concurrent inference stays race-free.
+var detBoxThresh float32 = 0.5
+
+// modelFileDet is the detection model filename. Override seam: a downstream
+// build variant assigns this at init/startup, before any inference runs. Set
+// once, never mutated during serving.
+var modelFileDet = "det.ort"
+
 // DetBox is one detected text region as a 4-point quad in original-image
 // coordinates, clockwise from top-left.
 type DetBox struct {
@@ -97,7 +108,7 @@ func getDetSession(ctx context.Context, modelPath string, rh, rw int64) (*sessio
 // extraction uses the pure-Go connected-components backend.
 func RunDet(ctx context.Context, modelDir string, img *Image) (DetResult, error) {
 	blob, rh, rw, sh, sw := detPreprocess(img)
-	sess, release, e := getDetSession(ctx, filepath.Join(modelDir, "det.ort"), int64(rh), int64(rw))
+	sess, release, e := getDetSession(ctx, filepath.Join(modelDir, modelFileDet), int64(rh), int64(rw))
 	if e != nil {
 		return DetResult{}, e
 	}

internal/deepdoc/native/dla.go

@@ -54,10 +54,20 @@ var (
 	}
 )
 
+// modelFileLayout is the layout model filename. Override seam: a downstream
+// build variant assigns this at init/startup, before any inference runs. Set
+// once, never mutated during serving.
+var modelFileLayout = "layout.ort"
+
+// dlaPostprocessFn is the DLA post-processing function. Override seam: a
+// downstream build variant assigns this at init/startup, before any inference
+// runs. Defaults to dlaPostprocess; set once, never mutated during serving.
+var dlaPostprocessFn = dlaPostprocess
+
 // RunDLA runs layout detection on a page image.
 func RunDLA(ctx context.Context, modelDir string, img *Image) (DLAResult, error) {
 	blob, sf := dlaPreprocess(img)
-	sess, release, err := getModelSession(ctx, filepath.Join(modelDir, "layout.ort"), "images",
+	sess, release, err := getModelSession(ctx, filepath.Join(modelDir, modelFileLayout), "images",
 		[]int64{1, 3, dlaInputSize, dlaInputSize}, "output0")
 	if err != nil {
 		return DLAResult{}, err
@@ -71,7 +81,7 @@ func RunDLA(ctx context.Context, modelDir string, img *Image) (DLAResult, error)
 	if err := checkOutputLength("dla", len(out), dlaMaxBoxes*6); err != nil {
 		return DLAResult{}, err
 	}
-	res := dlaPostprocess(out, sf)
+	res := dlaPostprocessFn(out, sf)
 	res.W, res.H = img.W, img.H
 	return res, nil
 }

internal/deepdoc/native/model_file_wiring_test.go

@@ -0,0 +1,42 @@
+//go:build cgo && integration
+
+package native
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+)
+
+// TestModelFileDetWiredToPipeline guards that the modelFileDet override seam is
+// actually consumed by RunDet: overriding it to a bogus filename makes RunDet
+// fail, proving the seam is wired into the detection pipeline rather than being
+// dead. No new model is required — we reuse the existing MODEL_DIR (which holds
+// the real det.ort) but point at a name that does not exist there.
+//
+// The wiring proof is the error itself: the real det.ort loads fine in every
+// other integration test, so if modelFileDet were not wired (RunDet hardcoding
+// "det.ort") this call would succeed (err == nil) and the test would fail. We
+// assert failure rather than matching the filename in the error, because ORT's
+// load error ("Error creating C session from file: Invalid fd was supplied: -1")
+// does not echo the path.
+func TestModelFileDetWiredToPipeline(t *testing.T) {
+	skipIfNoModels(t)
+
+	imgPath := filepath.Join("testdata", "page0.png")
+	img, err := Decode(imgPath)
+	if err != nil {
+		t.Fatalf("decode: %v", err)
+	}
+
+	const bogus = "does-not-exist-det.ort"
+	saved := modelFileDet
+	defer func() { modelFileDet = saved }()
+	modelFileDet = bogus
+
+	_, err = RunDet(t.Context(), os.Getenv("MODEL_DIR"), img)
+	if err == nil {
+		t.Fatalf("expected RunDet to fail loading the overridden (bogus) model file %q, got nil error — the modelFileDet seam may not be wired into RunDet", bogus)
+	}
+	t.Logf("RunDet failed as expected for bogus modelFileDet %q: %v", bogus, err)
+}

internal/deepdoc/native/ocr_rec.go

@@ -29,6 +29,11 @@ const (
 	recMaxBatch = 1
 )
 
+// modelFileRec is the recognition model filename. Override seam: a downstream
+// build variant assigns this at init/startup, before any inference runs. Set
+// once, never mutated during serving.
+var modelFileRec = "rec.ort"
+
 // OCRRecResult is the recognized text for one cropped line.
 type OCRRecResult struct {
 	Text  string
@@ -116,7 +121,7 @@ func RunOCRRecBatchReal(ctx context.Context, modelDir string, imgs []*Image) ([]
 		copy(batch[i*lineStride:(i+1)*lineStride], b)
 	}
 
-	sess, release, err := getRecSession(ctx, filepath.Join(modelDir, "rec.ort"), "x",
+	sess, release, err := getRecSession(ctx, filepath.Join(modelDir, modelFileRec), "x",
 		[]int64{int64(n), 3, recH, int64(imgW)}, "softmax_11.tmp_0")
 	if err != nil {
 		return nil, err
@@ -157,7 +162,7 @@ func recognizeLine(ctx context.Context, modelDir string, img *Image, maxWhRatio
 		resizedW = imgW
 	}
 	blob := ocrRecPreprocess(img, resizedW, imgW)
-	sess, release, err := getRecSession(ctx, filepath.Join(modelDir, "rec.ort"), "x",
+	sess, release, err := getRecSession(ctx, filepath.Join(modelDir, modelFileRec), "x",
 		[]int64{recMaxBatch, 3, recH, int64(imgW)}, "softmax_11.tmp_0")
 	if err != nil {
 		return OCRRecResult{}, err

internal/deepdoc/native/overridable_config_test.go

@@ -0,0 +1,57 @@
+package native
+
+import "testing"
+
+// TestModelConfigOverridable guards the contract that DeepDoc model filenames,
+// the detection box threshold, and the DLA post-processing indirection are
+// package-level variables (override seams assigned by a downstream build variant
+// at init/startup, before any inference runs, set once and never mutated during
+// serving).
+//
+// If any of these are still literals or constants, this test fails to compile,
+// which is the intended red signal before the refactor lands.
+func TestModelConfigOverridable(t *testing.T) {
+	type fileVar struct {
+		name string
+		get  func() string
+		set  func(string)
+		want string
+	}
+	files := []fileVar{
+		{"det", func() string { return modelFileDet }, func(s string) { modelFileDet = s }, "det.ort"},
+		{"rec", func() string { return modelFileRec }, func(s string) { modelFileRec = s }, "rec.ort"},
+		{"layout", func() string { return modelFileLayout }, func(s string) { modelFileLayout = s }, "layout.ort"},
+		{"tsr", func() string { return modelFileTsr }, func(s string) { modelFileTsr = s }, "tsr.ort"},
+	}
+	for _, f := range files {
+		if got := f.get(); got != f.want {
+			t.Fatalf("%s default = %q, want %q", f.name, got, f.want)
+		}
+		saved := f.get()
+		f.set("OVERRIDE-" + f.name)
+		if got := f.get(); got != "OVERRIDE-"+f.name {
+			t.Fatalf("%s override = %q, want OVERRIDE-%s", f.name, got, f.name)
+		}
+		f.set(saved)
+		if got := f.get(); got != f.want {
+			t.Fatalf("%s restore = %q, want %q", f.name, got, f.want)
+		}
+	}
+
+	if detBoxThresh != 0.5 {
+		t.Fatalf("detBoxThresh default = %v, want 0.5", detBoxThresh)
+	}
+	savedThresh := detBoxThresh
+	detBoxThresh = 0.99
+	if detBoxThresh != 0.99 {
+		t.Fatalf("detBoxThresh override = %v, want 0.99", detBoxThresh)
+	}
+	detBoxThresh = savedThresh
+	if detBoxThresh != 0.5 {
+		t.Fatalf("detBoxThresh restore = %v, want 0.5", detBoxThresh)
+	}
+
+	if dlaPostprocessFn == nil {
+		t.Fatal("dlaPostprocessFn must be set by default")
+	}
+}

internal/deepdoc/native/tsr.go

@@ -20,6 +20,11 @@ import (
 const tsrInputSize = 640
 const tsrCandidates = 8400
 
+// modelFileTsr is the table-structure recognition model filename. Override
+// seam: a downstream build variant assigns this at init/startup, before any
+// inference runs. Set once, never mutated during serving.
+var modelFileTsr = "tsr.ort"
+
 // tsrLabels mirrors TableStructureRecognizer.labels (also tsr_adapter.TSR_CLASS_MAP keys).
 var tsrLabels = []string{
 	"table", "table column", "table row",
@@ -44,7 +49,7 @@ type TSRResult struct {
 // RunTSR runs table-structure recognition on a cropped table image.
 func RunTSR(ctx context.Context, modelDir string, img *Image) (TSRResult, error) {
 	blob, sf := tsrPreprocess(img)
-	sess, release, err := getModelSession(ctx, filepath.Join(modelDir, "tsr.ort"), "images",
+	sess, release, err := getModelSession(ctx, filepath.Join(modelDir, modelFileTsr), "images",
 		[]int64{1, 3, tsrInputSize, tsrInputSize}, "output0")
 	if err != nil {
 		return TSRResult{}, err

internal/deepdoc/parser/pdf/inference/native_analyzer/native_analyzer.go

@@ -136,16 +136,11 @@ func (a *NativeAnalyzer) DLA(ctx context.Context, img image.Image) ([]deepdoctyp
 	return out, nil
 }
 
-// TSR recognises table structure from a cropped image.
-func (a *NativeAnalyzer) TSR(ctx context.Context, img image.Image) ([]deepdoctype.TSRCell, error) {
-	ni, err := native.FromImage(img)
-	if err != nil {
-		return nil, err
-	}
-	res, err := native.RunTSR(ctx, a.modelDir, ni)
-	if err != nil {
-		return nil, err
-	}
+// tsrCellsFn maps a native TSR result to the DeepDoc TSR cells. Override seam:
+// a downstream build variant assigns this at init/startup, before any inference
+// runs. Defaults to the inlined clamp-and-convert mapping; set once, never
+// mutated during serving.
+var tsrCellsFn = func(res native.TSRResult) []deepdoctype.TSRCell {
 	out := make([]deepdoctype.TSRCell, 0, len(res.Boxes))
 	// Clamp into the source image bounds (mirrors tsr_adapter.py): this
 	// backend replaced the HTTP service whose adapter clamped every
@@ -157,7 +152,20 @@ func (a *NativeAnalyzer) TSR(ctx context.Context, img image.Image) ([]deepdoctyp
 			Label: b.Label,
 		})
 	}
-	return out, nil
+	return out
+}
+
+// TSR recognises table structure from a cropped image.
+func (a *NativeAnalyzer) TSR(ctx context.Context, img image.Image) ([]deepdoctype.TSRCell, error) {
+	ni, err := native.FromImage(img)
+	if err != nil {
+		return nil, err
+	}
+	res, err := native.RunTSR(ctx, a.modelDir, ni)
+	if err != nil {
+		return nil, err
+	}
+	return tsrCellsFn(res), nil
 }
 
 // OCRDetect detects text regions (quad boxes) in a cropped image.

internal/deepdoc/parser/pdf/inference/native_analyzer/overridable_config_test.go

@@ -0,0 +1,63 @@
+package infnative
+
+import (
+	"testing"
+
+	"ragflow/internal/deepdoc/native"
+	deepdoctype "ragflow/internal/deepdoc/parser/type"
+)
+
+// TestTSRCellsFnOverridable guards two contracts of the TSR mapping seam:
+//
+//  1. tsrCellsFn is a reassignable package-level variable (the override seam);
+//     a downstream build variant can swap the mapping at init/startup.
+//  2. The default mapping clamps out-of-bounds coordinates into the source
+//     image bounds [0, W] x [0, H] — parity-critical behaviour that must hold
+//     without loading a model.
+func TestTSRCellsFnOverridable(t *testing.T) {
+	// (1) The seam must be reassignable. Swap in a stub, confirm it is invoked,
+	// then restore the default so other tests see the real mapping.
+	saved := tsrCellsFn
+	defer func() { tsrCellsFn = saved }()
+
+	stubCalled := false
+	tsrCellsFn = func(res native.TSRResult) []deepdoctype.TSRCell {
+		stubCalled = true
+		return nil
+	}
+	if tsrCellsFn == nil {
+		t.Fatal("tsrCellsFn must remain assignable")
+	}
+	_ = tsrCellsFn(native.TSRResult{})
+	if !stubCalled {
+		t.Fatal("reassigned tsrCellsFn was not invoked")
+	}
+
+	// (2) The default mapping clamps each coordinate independently into bounds.
+	W, H := 100, 200
+	res := native.TSRResult{
+		W: W, H: H,
+		Boxes: []native.TSRBox{{
+			Label:  "table",
+			Score:  0.9,
+			X0:     -5,
+			X1:     float32(W) + 10,
+			Top:    -3,
+			Bottom: float32(H) + 7,
+		}},
+	}
+	cells := saved(res)
+	if len(cells) != 1 {
+		t.Fatalf("expected 1 cell from default mapping, got %d", len(cells))
+	}
+	c := cells[0]
+	if c.X0 != 0 || c.Y0 != 0 {
+		t.Errorf("low coordinates must clamp to 0, got X0=%v Y0=%v", c.X0, c.Y0)
+	}
+	if c.X1 != float64(W) || c.Y1 != float64(H) {
+		t.Errorf("high coordinates must clamp to bounds, got X1=%v Y1=%v", c.X1, c.Y1)
+	}
+	if c.Label != "table" {
+		t.Errorf("unexpected label %q", c.Label)
+	}
+}