fix(deepdoc): harden ORT build guard, OCR test contract (#19082)

This commit is contained in:
xugangqiang authored and GitHub committed 2026-09-02 12:13:19 +08:00
1 parent 61947f9eeb
commit 0d1338d63e
20 files changed
+627 -69

No files matched your search

+22
View File
@@ -396,6 +396,17 @@ jobs:
./build.sh --test -- $PKGS
fi
- name: Run ORT deps-download regression guard
if: env.API_PROXY_SCHEME == 'go'
# Pure-Python, local, needs no API keys or model downloads; guards the
# ONNX Runtime extraction logic the in-process (Go) DeepDoc backend build
# depends on (notably the version-bump path C2 hardened). Run it in
# isolation so the test/testcases conftest (which requires
# SILICONFLOW_API_KEY) cannot fail the collection.
run: |
set -euo pipefail
uv sync --python 3.13 --group test --frozen
uv run pytest ragflow_deps/test_download_go_deps.py -q
- name: Build ragflow:nightly
run: |
@@ -1016,6 +1027,17 @@ jobs:
./build.sh --test -- $PKGS
fi
- name: Run ORT deps-download regression guard
if: env.API_PROXY_SCHEME == 'go'
# Pure-Python, local, needs no API keys or model downloads; guards the
# ONNX Runtime extraction logic the in-process (Go) DeepDoc backend build
# depends on (notably the version-bump path C2 hardened). Run it in
# isolation so the test/testcases conftest (which requires
# SILICONFLOW_API_KEY) cannot fail the collection.
run: |
set -euo pipefail
uv sync --python 3.13 --group test --frozen
uv run pytest ragflow_deps/test_download_go_deps.py -q
- name: Build ragflow:nightly
run: |
+10
View File
@@ -30,6 +30,16 @@ RUN --mount=type=bind,from=infiniflow/ragflow_deps:latest,source=/huggingface.co
ln -s layout.onnx /ragflow/rag/res/deepdoc/layout.manual.onnx && \
ln -s layout.onnx /ragflow/rag/res/deepdoc/layout.paper.onnx
# Copy the cl100k_base BPE table used by the Go tokenizer (tiktoken-go
# cl100k_base). The deps image ships it at its root; the Go image previously
# mounted only /huggingface.co and omitted this file, so NumTokensFromString
# silently returned 0. localBpeLoader resolves it from <workdir>/ragflow_deps/,
# so dropping it here makes the table load offline at startup (see
# InitCL100KEncoder fail-fast guard).
RUN --mount=type=bind,from=infiniflow/ragflow_deps:latest,source=/cl100k_base.tiktoken,target=/tmp/cl100k_base.tiktoken \
mkdir -p /ragflow/ragflow_deps && \
cp /tmp/cl100k_base.tiktoken /ragflow/ragflow_deps/cl100k_base.tiktoken
ENV DEBIAN_FRONTEND=noninteractive
# Setup apt
+29 -1
View File
@@ -41,7 +41,8 @@ PDF_OXIDE_VERSION="0.3.73"
# (Go) DeepDoc backend. libonnxruntime*.a is linked into the server binary
# (--whole-archive + --export-dynamic); OrtGetApiBase is then resolved via
# dlopen(self), so no libonnxruntime.so is needed at runtime. Downloaded by
# ragflow_deps/download_deps.py into onnxruntime/static_lib.
# ragflow_deps/download_go_deps.py (and ragflow_deps/download_deps.py) into
# onnxruntime/static_lib.
ONNXRUNTIME_STATIC_PREFIX="${HOME}/ragflow-native-libs/onnxruntime/static_lib"
# Copy a dependency from the system pre-seed directory to the user cache.
@@ -370,6 +371,33 @@ build_go() {
setup_cgo_env
# The in-process (Go) DeepDoc backend is statically linked against ONNX
# Runtime (--whole-archive + -Wl,--export-dynamic, see setup_cgo_env). The
# forked onnxruntime_go binding resolves OrtGetApiBase only at RUNTIME via
# dlopen(NULL), so a missing ORT archive still lets `go build` SUCCEED and
# yields a server binary that FAILS AT STARTUP with a fatal
# "no in-process DeepDoc backend serving". That is exactly the breakage
# colleagues hit when ORT was not present at build time and setup_cgo_env
# silently skipped it. Fail the build HERE instead of deferring the breakage
# to runtime: a server binary without ORT is unusable, not a degraded one.
if ! printf '%s' "$CGO_LDFLAGS" | grep -q 'libonnxruntime'; then
echo -e "${RED}Error: ONNX Runtime static libraries are not linked.${NC}" >&2
echo " The in-process DeepDoc backend requires libonnxruntime.a to be" >&2
echo " statically linked into ragflow_server (--whole-archive +" >&2
echo " -Wl,--export-dynamic). Without it the binary compiles but dies" >&2
echo " at startup with a fatal 'no in-process DeepDoc backend serving'." >&2
echo " Fetch the static libs with:" >&2
echo " uv run python3 ragflow_deps/download_go_deps.py" >&2
echo " or pre-seed them at /opt/ragflow-native-libs/onnxruntime (CI image)." >&2
echo " This production binary must statically include ORT — there is no" >&2
echo " ORT-free build path. ORT ends up unlinked only via one of:" >&2
echo " - the ORT static_lib dir was never seeded: run" >&2
echo " 'uv run python3 ragflow_deps/download_go_deps.py', or pre-seed" >&2
echo " /opt/ragflow-native-libs/onnxruntime as the CI image does;" >&2
echo " - setup_cgo_env did not add libonnxruntime to CGO_LDFLAGS." >&2
return 1
fi
local strip_flags=()
[ -n "$STRIP_SYMBOLS" ] && strip_flags=(-ldflags="-s -w")
+12
View File
@@ -568,6 +568,12 @@ func runIngestor(ctx context.Context, cancel context.CancelFunc, args *serverArg
}
defer tokenizer.Close()
// Fail fast if the cl100k_base BPE table is missing: without it
// NumTokensFromString silently returns 0, corrupting every token budget.
if err := tokenizer.InitCL100KEncoder(); err != nil {
common.Fatal("Failed to initialize cl100k_base tokenizer", zap.Error(err))
}
// The dataset-level post-processing consumer cluster (§11) is owned and run by
// the Ingestor: it is started inside ingestor.Start() and joined inside
// ingestor.Stop(), so its lifecycle matches the ingestor. The configured
@@ -712,6 +718,12 @@ func runAPI(ctx context.Context, args *serverArgs) error {
}
defer tokenizer.Close()
// Fail fast if the cl100k_base BPE table is missing: without it
// NumTokensFromString silently returns 0, corrupting every token budget.
if err := tokenizer.InitCL100KEncoder(); err != nil {
common.Fatal("Failed to initialize cl100k_base tokenizer", zap.Error(err))
}
// Initialize global QueryBuilder using tokenizer's DictPath
// This ensures the Synonym uses the same wordnet directory as tokenizer
if err := nlp.InitQueryBuilderFromTokenizer(tokenizerCfg.DictPath); err != nil {
+13 -7
View File
@@ -254,17 +254,23 @@ func HasModelFiles(dir string) bool {
}
// DeepDocORTVersion is the onnxruntime native release the in-process (Go)
// DeepDoc backend is built and tested against (e.g. "1.23.2"). It is the
// single source for the download URL and extracted dir name used across Go
// and Python. The Go binding (github.com/yalue/onnxruntime_go, forked to
// DeepDoc backend is built and tested against (e.g. "1.23.2"). It is ONE OF
// THREE raw version declarations that must stay equal (the other two are
// ORT_VERSION in ragflow_deps/download_go_deps.py and ragflow_deps/download_deps.py)
// — NOT a single source of truth. The download URL and extracted dir name are
// built from those ORT_VERSION constants, not from this one. The Go binding
// (github.com/yalue/onnxruntime_go, forked to
// github.com/xugangqiang/onnxruntime_go) and the pip onnxruntime== pin must
// track this MINOR version: the binding uses its own release numbering
// (v1.23.0 <-> ORT 1.23.x) but is ABI-compatible with this native release on
// the same minor line. ONNX Runtime is linked statically (libonnxruntime.a),
// so there is no .so / SONAME at runtime.
//
// To bump ORT: update DeepDocORTVersion (Go) AND ORT_VERSION in
// ragflow_deps/download_deps.py AND the onnxruntime/onnxruntime-gpu pins in
// pyproject.toml + .github/workflows/deepdoc-drift.yml, and refresh the
// onnxruntime_go binding minor in go.mod.
// To bump ORT, ALL of the following must change together (drift breaks the
// static link or the runtime OrtGetApiBase lookup):
// - DeepDocORTVersion (here, Go) AND ORT_VERSION in BOTH
// ragflow_deps/download_go_deps.py and ragflow_deps/download_deps.py;
// - the onnxruntime== pin in pyproject.toml and the onnxruntime /
// onnxruntime-gpu pins in .github/workflows/deepdoc-drift.yml;
// - the onnxruntime_go binding minor in go.mod.
const DeepDocORTVersion = "1.23.2"
+10 -22
View File
@@ -36,7 +36,7 @@ type OCRRecResult struct {
}
// RunOCRRec recognizes a single cropped text-line image. It is equivalent to a
// one-line batch (see RunOCRRecBatch): the line is resized against its own
// one-line batch (see RunOCRRecBatchReal): the line is resized against its own
// wh_ratio floored at 320/48, never the batch-max, so a caller recognizing
// lines independently gets the same result as a standalone Python
// TextRecognizer call.
@@ -55,28 +55,16 @@ func RunOCRRec(ctx context.Context, modelDir string, img *Image) (OCRRecResult,
return recognizeLine(ctx, modelDir, img, maxWhRatio, chars)
}
// RunOCRRecBatch recognizes a batch of cropped text-line images the way
// deepdoc's TextRecognizer.__call__ does: every line in the batch is resized to
// the SAME width derived from the batch's maximum wh_ratio (floored at 320/48),
// so a narrow line inside a wide batch is widened to the batch max rather than
// its own ratio. This makes Go match production batch semantics, where a page's
// lines are recognized together. Lines are recognized independently after the
// shared resize, so order is preserved and results are deterministic per batch.
//
// DEPRECATED semantic-shape note: this variant still runs one ONNX Run PER
// line (recMaxBatch=1). Prefer RunOCRRecBatchReal, which concatenates every
// line's preprocessed blob into a single {N,3,48,imgW} tensor and performs ONE
// ONNX Run — identical numerically (each line uses the same shared batch width)
// but with the real throughput benefit of batched matmul. RunOCRRecBatchReal is
// the production path.
// RunOCRRecBatchReal recognizes a batch of cropped text-line images with a
// SINGLE ONNX Run, mirroring deepdoc's TextRecognizer.__call__ exactly: every
// line is resized to the batch's shared width (imgW = 48*max_wh_ratio, floored
// at 320/48) and zero-padded on the right, all blobs are concatenated into one
// {N,3,48,imgW} tensor, and the model runs once. The output is split back into
// per-line sequences and CTC-decoded in order, so the result is numerically
// identical to calling RunOCRRec on each line (each line sees the same shared
// batch width), but amortized over one forward pass instead of N.
// SINGLE ONNX Run, mirroring deepdoc's TextRecognizer.__call__: each line is
// resized to its own proportional width (recH * that line's wh_ratio), capped
// by the batch-shared imgW (imgW = recH * max_wh_ratio, with max_wh_ratio
// floored at 320/48), and zero-padded on the right out to imgW; all blobs are
// concatenated into one {N,3,48,imgW} tensor, and the model runs once. The
// output is split back into per-line sequences and CTC-decoded in order, so
// the result is numerically identical to calling RunOCRRec on each line (each
// line sees the same shared batch width), but amortized over one forward pass
// instead of N.
//
// The shared batch width means a line is resized against the batch max wh_ratio,
// not its own — exactly what deepdoc does inside a batch. A standalone call to
+4 -2
View File
@@ -2,8 +2,9 @@
// This file is compiled only with -tags fetch_testdata. Its init() downloads
// and symlinks the external DeepDoc testdata for this package (pkg = "native")
// so the consuming tests can run. The download is best-effort: if it fails, the
// guard in testdata_skip.go skips the tests instead of failing.
// so the consuming tests can run. A fetch failure is fatal under this build tag:
// it sets testdataFetchFailed so TestMain fails the package instead of silently
// skipping — a missing fixture must never be a green CI run.
package native
import (
@@ -17,6 +18,7 @@ import (
func init() {
if err := fetchTestdata(); err != nil {
fmt.Fprintf(os.Stderr, "fetch_deepdoc_testdata: not fetched (%v)\n", err)
testdataFetchFailed = true
return
}
testdataFetchAttempted = true
+8
View File
@@ -8,3 +8,11 @@ package native
// imports native, and _test.go files are excluded from dependency builds). In the
// default build it stays false, so TestMain skips the package.
var testdataFetchAttempted bool
// testdataFetchFailed is set to true by testdata_fetch.go (compiled only with
// -tags fetch_testdata) when the external testdata download fails. It is
// declared here (always-compiled) so TestMain can read it. Under the
// fetch_testdata build tag a failed fetch must FAIL the package rather than
// silently skip — a missing fixture must never produce a green CI run. In the
// default build this stays false, so the absence of fixtures is a benign skip.
var testdataFetchFailed bool
@@ -1,9 +1,10 @@
// This file is always compiled (no build constraint) and provides the package
// TestMain that guards the whole test binary behind the presence of the
// package-level testdata directory. When the external testdata has not been
// fetched (e.g. -tags fetch_testdata was not supplied, or the download failed),
// the tests are skipped rather than failed — keeping the default `go test ./...`
// and `go test -tags cgo` runs green without the assets.
// fetched (e.g. -tags fetch_testdata was not supplied), the tests are skipped
// rather than failed — keeping the default `go test ./...` and `go test -tags cgo`
// runs green without the assets. A fetch ATTEMPTED but FAILED (under
// -tags fetch_testdata) is fatal: a missing fixture must never be a green run.
package native
import (
@@ -18,6 +19,10 @@ import (
// assets — which also makes a stale/partial leftover testdata directory on disk
// harmless: we skip regardless of what (if anything) is present.
func TestMain(m *testing.M) {
if testdataFetchFailed {
fmt.Fprintln(os.Stderr, "FAIL: DeepDoc native testdata fetch failed (see the fetch_deepdoc_testdata error above). CI must not pass with missing fixtures; run with -tags fetch_testdata and fix the download.")
os.Exit(1)
}
if !testdataFetchAttempted {
fmt.Println("SKIP: DeepDoc native testdata not fetched; run with -tags fetch_testdata to fetch and run it")
os.Exit(0)
@@ -3,9 +3,10 @@
// This file is compiled only with -tags fetch_testdata. Its init() downloads
// and symlinks the external DeepDoc testdata. The native_analyzer tests read
// their fixtures from the sibling internal/deepdoc/native/testdata directory
// (via a relative path), so we fetch with pkg = "native". The download is
// best-effort: if it fails, the guard in testdata_skip.go skips the tests
// instead of failing.
// (via a relative path), so we fetch with pkg = "native". A fetch failure is
// fatal under this build tag: it sets testdataFetchFailed so TestMain fails the
// package instead of silently skipping — a missing fixture must never be a green
// CI run.
package infnative
import (
@@ -19,6 +20,7 @@ import (
func init() {
if err := fetchTestdata(); err != nil {
fmt.Fprintf(os.Stderr, "fetch_deepdoc_testdata: not fetched (%v)\n", err)
testdataFetchFailed = true
return
}
testdataFetchAttempted = true
@@ -7,3 +7,11 @@ package infnative
// this package is built as a dependency of another package's test binary. In the
// default build it stays false, so TestMain skips the package.
var testdataFetchAttempted bool
// testdataFetchFailed is set to true by testdata_fetch.go (compiled only with
// -tags fetch_testdata) when the external testdata download fails. Declared here
// (always-compiled) so TestMain can read it. Under the fetch_testdata build tag a
// failed fetch must FAIL the package rather than silently skip — a missing
// fixture must never produce a green CI run. In the default build this stays
// false, so the absence of fixtures is a benign skip.
var testdataFetchFailed bool
@@ -3,9 +3,10 @@
// DeepDoc testdata directory. The native_analyzer tests read their fixtures
// from the sibling internal/deepdoc/native/testdata directory (via a relative
// path), so that is what we check here. When the external testdata has not been
// fetched (e.g. -tags fetch_testdata was not supplied, or the download failed),
// the tests are skipped rather than failed — keeping the default `go test ./...`
// and `go test -tags cgo` runs green without the assets.
// fetched (e.g. -tags fetch_testdata was not supplied), the tests are skipped
// rather than failed — keeping the default `go test ./...` and `go test -tags cgo`
// runs green without the assets. A fetch ATTEMPTED but FAILED (under
// -tags fetch_testdata) is fatal: a missing fixture must never be a green run.
package infnative
import (
@@ -20,6 +21,10 @@ import (
// assets — which also makes a stale/partial leftover testdata directory on disk
// harmless: we skip regardless of what (if anything) is present.
func TestMain(m *testing.M) {
if testdataFetchFailed {
fmt.Fprintln(os.Stderr, "FAIL: DeepDoc native testdata fetch failed (see the fetch_deepdoc_testdata error above). CI must not pass with missing fixtures; run with -tags fetch_testdata and fix the download.")
os.Exit(1)
}
if !testdataFetchAttempted {
fmt.Println("SKIP: DeepDoc native testdata not fetched; run with -tags fetch_testdata to fetch and run it")
os.Exit(0)
+29 -16
View File
@@ -105,24 +105,37 @@ func (p *Parser) ocrDetectAndRecognize(ctx context.Context, pageImg image.Image,
allTexts := make([][]pdf.OCRText, len(cropAcc))
if p.docSupportsBatchOCR(doc) {
batch, berr := p.inferOCRRecognizeBatch(ctx, doc, cropAcc)
if berr != nil {
slog.Warn(logLabel+" OCR batch recognize failed", "page", pageNum, "err", berr)
switch {
case berr != nil:
// A batch error must not abort the whole page: the canonical
// per-crop path below still produces correct results.
slog.Warn(logLabel+" OCR batch recognize failed; falling back to per-crop", "page", pageNum, "err", berr)
case len(batch) != len(cropAcc):
// Defensive: a count mismatch (or a nil result) would corrupt the
// per-box indexing further down. Fall back to per-crop instead of
// indexing out of range.
slog.Warn(logLabel+" OCR batch recognize returned unexpected count; falling back to per-crop", "page", pageNum, "got", len(batch), "want", len(cropAcc))
default:
allTexts = batch
}
}
// Fill any crop the batch path did not cover (unsupported doc, batch error,
// or count mismatch) with the per-crop canonical path. Stamping the
// detect-box index lets a replay DocAnalyzer route the recognition back to
// the Python-dumped text for the same box; the production analyzer ignores
// the key.
for ci := range cropAcc {
if allTexts[ci] != nil {
continue
}
c := cropAcc[ci]
recCtx := context.WithValue(ctx, ocrBoxIdxCtxKey, cropBoxIdx[ci])
texts, rerr := p.ocrRecognizeWithRotation(recCtx, doc, c)
if rerr != nil {
slog.Warn(logLabel+" OCR recognize failed", "page", pageNum, "err", rerr)
return nil
}
allTexts = batch
} else {
for ci, c := range cropAcc {
// Stamp the detect-box index so a replay DocAnalyzer routes this
// recognition back to the Python-dumped text for the same box. The
// production analyzer ignores the key.
recCtx := context.WithValue(ctx, ocrBoxIdxCtxKey, cropBoxIdx[ci])
texts, rerr := p.ocrRecognizeWithRotation(recCtx, doc, c)
if rerr != nil {
slog.Warn(logLabel+" OCR recognize failed", "page", pageNum, "err", rerr)
return nil
}
allTexts[ci] = texts
}
allTexts[ci] = texts
}
// Per box: pick the highest-confidence candidate (layer-2 winner) and
+68 -2
View File
@@ -39,14 +39,33 @@ go version
### 1.4 Install dependent library
```shell
sudo apt install libpcre2-dev
# Native libs for the Go build (office_oxide, pdfium, pdf_oxide, onnxruntime).
# download_go_deps.py is the Go-end script: it now fetches the ONNX Runtime
# static lib too, so this single command covers every native dependency the
# Go server binary needs (incl. the in-process DeepDoc backend).
python3 ragflow_deps/download_go_deps.py
# Shared (Go + Python) deps. Retains ONNX Runtime for the ragflow_deps image /
# backward-compat; also pulls Python-side artifacts (models, nltk, tika, ...).
uv run python3 ragflow_deps/download_deps.py
```
> **Note**: If you use IDEs like GoLand to run/debug directly (via Run/Debug buttons), or run `go build` / `go run` from command line, set these CGO environment variables:
>
> ```bash
> RAGFLOW_DEPS="${HOME}/ragflow-native-libs" # created by uv run ragflow_deps/download_deps.py
> RAGFLOW_DEPS="${HOME}/ragflow-native-libs" # created by download_go_deps.py + download_deps.py
> PLATFORM="linux_amd64" # or darwin_amd64, linux_arm64, darwin_arm64
> # NOTE: the ONNX Runtime static lib fetched by download_go_deps.py is
> # linux-x64 ONLY (onnxruntime-linux-x64-static_lib-*). On darwin_* / non-amd64
> # PLATFORM values the production DeepDoc backend cannot be linked, so those
> # PLATFORM examples cover office_oxide/pdfium/pdf_oxide only — ORT is a
> # Linux-amd64 link dependency here.
>
> # Resolve the version-stamped ORT archive path FIRST, in its own unquoted
> # assignment: the shell does not expand `*` inside the double-quoted
> # CGO_LDFLAGS below, and `--whole-archive` must stay a separate argument from
> # the archive path. If this lists more than one match, delete the stale
> # version dir — build.sh refuses to link two ORT versions.
> ORT_A="$(ls ${RAGFLOW_DEPS}/onnxruntime/static_lib/*/lib/libonnxruntime.a)"
>
> export CGO_CFLAGS="-I${RAGFLOW_DEPS}/office_oxide/include/office_oxide_c"
> export CGO_LDFLAGS="\
@@ -55,11 +74,29 @@ python3 ragflow_deps/download_go_deps.py
> ${RAGFLOW_DEPS}/pdfium-static/lib/libc++.a \
> ${RAGFLOW_DEPS}/pdfium-static/lib/libc++abi.a \
> ${RAGFLOW_DEPS}/pdf_oxide/lib/${PLATFORM}/libpdf_oxide.a \
> -Wl,--export-dynamic -Wl,--whole-archive ${ORT_A} -Wl,--no-whole-archive -lstdc++ \
> -fuse-ld=lld \
> -lm -lpthread -ldl -lrt -lgcc_s -lutil -lc"
> ```
>
> All three native libraries are statically linked — no `LD_LIBRARY_PATH` or `-Wl,-rpath` needed.
> All four native libraries are statically linked — no `LD_LIBRARY_PATH` or `-Wl,-rpath` needed.
>
> **ONNX Runtime is mandatory for the production binary.** The in-process (Go)
> DeepDoc backend is statically linked against `libonnxruntime.a` via
> `--whole-archive -Wl,--export-dynamic`, and `OrtGetApiBase` is resolved at
> runtime through `dlopen(NULL)`. The forked `onnxruntime_go` binding only
> needs `-ldl` to *compile*, so a binary built **without** ORT links
> successfully but dies at startup with:
> `Error looking up OrtGetApiBase in statically-linked ONNX Runtime` → fatal
> `no in-process DeepDoc backend serving`.
> Since `build.sh` (`build_go`) now **fails fast** when ORT is absent from
> `CGO_LDFLAGS`, this breakage surfaces at build time instead of at runtime. If
> you see `Error: ONNX Runtime static libraries are not linked`, run
> `uv run python3 ragflow_deps/download_go_deps.py` (or pre-seed
> `/opt/ragflow-native-libs/onnxruntime` as the CI runner image does). There is
> no ORT-free production build path — if ORT is absent the binary fails at
> startup, so the remedy is always to seed the static lib above, never to build
> without it.
### 1.5 Build RAGFlow
@@ -77,6 +114,35 @@ python3 ragflow_deps/download_go_deps.py
./build.sh -s --go
```
### 1.6 In-process (Go) DeepDoc backend
The in-process DeepDoc backend is statically linked against ONNX Runtime
(see §1.4). After a successful `./build.sh -s --go`, the `ragflow_server`
binary carries it and registers the backend at startup.
- **Confirm it is serving** — the server logs, at startup:
`in-process DeepDoc backend registered (production backend)`
If you instead see a fatal `no in-process DeepDoc backend serving`, ORT was
not linked into the binary. Re-run `uv run python3 ragflow_deps/download_go_deps.py`
and rebuild (§1.4 explains why; `build.sh` fails fast with
`Error: ONNX Runtime static libraries are not linked` before this happens).
- **Run the binary directly (local dev)** — `./bin/ragflow_server --api`
(start `--admin` first, see §2) launches the Go server and registers the
backend. No extra environment variable is required.
- **Run the Go Docker image** — the container entrypoint only starts the Go
server (`bin/ragflow_server --api/--ingestor/--admin`) when
`API_PROXY_SCHEME` is `go` or `hybrid`. With the variable unset (or `python`)
the Go server does NOT start, so the in-process backend is absent and the
image looks like a Python-only build. Start it with:
```bash
docker run -e API_PROXY_SCHEME=go infiniflow/ragflow:go-test-1
```
- ORT is resolved via `dlopen(NULL)` at runtime, so no `LD_LIBRARY_PATH` /
`-rpath` is needed and no `libonnxruntime.so` ships with the image.
## 2. Start RAGFlow
- Start dependencies
+20 -1
View File
@@ -138,7 +138,7 @@ func bpeCandidatePaths(bpeURL string) []string {
}
}
for _, root := range searchRoots() {
for _, root := range bpeSearchRoots() {
// Same layout the Dockerfile creates: the table sits in the
// installation root under its sha1 name.
add(filepath.Join(root, cacheName))
@@ -172,6 +172,25 @@ func searchRoots() []string {
return roots
}
// bpeSearchRoots is the directory set the loader walks for the table. It
// defaults to searchRoots (working + executable dirs and their ancestors) so
// the production image finds the table dropped into the install root. Tests
// override it with SetBpeSearchRootsForTest to isolate from the host
// filesystem — without that, a tiktoken cache leaked into /tmp (an ancestor of
// any temp working dir) would be picked up and make "missing table" tests
// non-deterministic.
var bpeSearchRoots = searchRoots
// SetBpeSearchRootsForTest replaces the directory set the loader searches.
// Pass nil to restore the default. Test-only; it mutates package state.
func SetBpeSearchRootsForTest(roots []string) {
if roots == nil {
bpeSearchRoots = searchRoots
return
}
bpeSearchRoots = func() []string { return roots }
}
func startingDirs() []string {
var dirs []string
if wd, err := os.Getwd(); err == nil {
+10 -2
View File
@@ -41,8 +41,12 @@ func writeBpeTable(t *testing.T, path string, marker int) {
}
// Register this synthetic table's digest so the SHA-1 integrity gate in
// bpe_loader.go accepts it. This also exercises the verification path
// instead of disabling it.
// instead of disabling it. Restore the prior value afterwards so the
// mutation does not leak into sibling tests (which would otherwise read a
// stale digest for the same URL).
prev := expectedBpeHashes[testBpeURL]
expectedBpeHashes[testBpeURL] = fmt.Sprintf("%x", sha1.Sum([]byte(line)))
t.Cleanup(func() { expectedBpeHashes[testBpeURL] = prev })
}
func cacheFileName(url string) string {
@@ -52,13 +56,17 @@ func cacheFileName(url string) string {
// isolate moves the process into an empty directory and clears both cache
// environment variables, so a test sees only the files it creates itself.
// Without this the real repository checkout — which does ship the table — would
// satisfy every lookup and hide ordering bugs.
// satisfy every lookup and hide ordering bugs. It also scopes the loader's
// search roots to this directory so a tiktoken cache leaked into an ancestor
// (e.g. /tmp) can't be picked up and make "missing table" tests flaky.
func isolate(t *testing.T) string {
t.Helper()
dir := t.TempDir()
t.Setenv("TIKTOKEN_CACHE_DIR", "")
t.Setenv("DATA_GYM_CACHE_DIR", "")
t.Chdir(dir)
SetBpeSearchRootsForTest([]string{dir})
t.Cleanup(func() { SetBpeSearchRootsForTest(nil) })
return dir
}
+147
View File
@@ -0,0 +1,147 @@
//
// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
package tokenizer
import (
"io"
"os"
"path/filepath"
"strings"
"testing"
)
// TestInitCL100KEncoder_FailFast pins the contract that InitCL100KEncoder fails
// fast on a missing cl100k_base table, and — the part the regression actually
// guards — that a PRESENT table yields a working encoder (NumTokensFromString > 0)
// rather than a silent 0.
//
// Background: NumTokensFromString swallows the loader error and returns 0, so a
// Go image that forgot to bake cl100k_base.tiktoken silently zeroed every token
// count while content_ltks (a separate C++ tokenizer) kept working. The fix is
// to fail fast at startup via InitCL100KEncoder; this test must exercise BOTH
// branches, otherwise it only re-asserts "missing → error" and leaves the
// "present → counts" behaviour — the thing that regressed — uncovered.
//
// Both branches reset the encoder cache and scope bpeSearchRoots to a temp dir,
// so neither depends on the host (no leaked tiktoken cache in an ancestor, no
// success cached by a sibling test, no order dependence). "absent" runs first so
// that the table "present" later copies under /tmp is not discoverable by it via
// searchRoots() walking up to the shared /tmp ancestor.
func TestInitCL100KEncoder_FailFast(t *testing.T) {
// fail-fast: an empty scoped dir must surface a hard error, not a silent 0.
t.Run("absent", func(t *testing.T) {
resetCL100KEncoderForTest()
dir := t.TempDir()
t.Setenv("TIKTOKEN_CACHE_DIR", "")
t.Setenv("DATA_GYM_CACHE_DIR", "")
SetBpeSearchRootsForTest([]string{dir})
t.Cleanup(func() { SetBpeSearchRootsForTest(nil) })
err := InitCL100KEncoder()
if err == nil {
t.Fatalf("InitCL100KEncoder returned nil with no table present; expected a fail-fast error")
}
if !strings.Contains(err.Error(), "cl100k") {
t.Fatalf("InitCL100KEncoder error does not mention cl100k: %v", err)
}
})
// present: copy a real table into an isolated dir (via TIKTOKEN_CACHE_DIR so
// the loader finds it without walking ancestors) and require that startup
// succeeds AND the encoder actually counts tokens (not just "loads").
t.Run("present", func(t *testing.T) {
src := findRealBpeTable(t)
if src == "" {
t.Skip("no cl100k_base.tiktoken on disk to exercise the present branch")
}
resetCL100KEncoderForTest()
dir := t.TempDir()
t.Setenv("TIKTOKEN_CACHE_DIR", dir)
t.Setenv("DATA_GYM_CACHE_DIR", "")
dst := filepath.Join(dir, cacheFileName(testBpeURL))
copyFile(t, src, dst)
SetBpeSearchRootsForTest([]string{dir})
t.Cleanup(func() { SetBpeSearchRootsForTest(nil) })
if err := InitCL100KEncoder(); err != nil {
t.Fatalf("InitCL100KEncoder with table present: unexpected error: %v", err)
}
if got := NumTokensFromString("hello world"); got <= 0 {
t.Fatalf("InitCL100KEncoder succeeded but NumTokensFromString(%q) = %d, want > 0 (silent-zero regression)", "hello world", got)
}
})
}
// findRealBpeTable returns a path to a cl100k_base.tiktoken on disk, or "" if
// none is available. The loader's integrity gate already knows this table's
// digest (expectedBpeHashes), so copying it into a scoped dir is enough.
func findRealBpeTable(t *testing.T) string {
t.Helper()
// 1. Explicit cache dirs (tiktoken-go / data-gym layouts).
for _, env := range []string{"TIKTOKEN_CACHE_DIR", "DATA_GYM_CACHE_DIR"} {
if dir := strings.TrimSpace(os.Getenv(env)); dir != "" {
for _, name := range []string{cacheFileName(testBpeURL), "cl100k_base.tiktoken"} {
if p := filepath.Join(dir, name); fileExists(p) {
return p
}
}
}
}
// 2. repo-local ragflow_deps/cl100k_base.tiktoken (provisioned by download
// scripts / CI), walking up from the package dir.
pkgDir, err := os.Getwd()
if err != nil {
return ""
}
for dir := pkgDir; ; {
p := filepath.Join(dir, "ragflow_deps", "cl100k_base.tiktoken")
if fileExists(p) {
return p
}
parent := filepath.Dir(dir)
if parent == dir {
break
}
dir = parent
}
return ""
}
func fileExists(p string) bool {
info, err := os.Stat(p)
return err == nil && !info.IsDir()
}
func copyFile(t *testing.T, src, dst string) {
t.Helper()
if err := os.MkdirAll(filepath.Dir(dst), 0o755); err != nil {
t.Fatalf("mkdir %s: %v", filepath.Dir(dst), err)
}
in, err := os.Open(src)
if err != nil {
t.Fatalf("open %s: %v", src, err)
}
defer in.Close()
out, err := os.Create(dst)
if err != nil {
t.Fatalf("create %s: %v", dst, err)
}
defer out.Close()
if _, err := io.Copy(out, in); err != nil {
t.Fatalf("copy %s -> %s: %v", src, dst, err)
}
}
+40
View File
@@ -560,9 +560,49 @@ func getCL100KEncoder() (*tiktoken.Tiktoken, error) {
return cl100kEncoder.enc, cl100kEncoder.err
}
// resetCL100KEncoderForTest clears the cl100k encoder cache so a test can force a
// fresh load under a different search-root scope. Test-only; it mutates package
// state. A plain assignment replaces the embedded sync.Once with an un-fired one
// and wipes any cached result, so the next getCL100KEncoder / InitCL100KEncoder
// re-runs the loader rather than returning a result cached by a sibling test.
func resetCL100KEncoderForTest() {
cl100kEncoder = struct {
sync.Once
enc *tiktoken.Tiktoken
err error
}{}
}
// InitCL100KEncoder loads the cl100k_base BPE encoder once and returns an
// error if the table is unavailable. Call it during server startup (before any
// request reaches NumTokensFromString) so a missing table fails fast instead
// of silently zeroing every token count.
//
// Why fail-fast and not a retry/warning: NumTokensFromString intentionally
// returns 0 on encoder error to stay cheap on the hot path, which means a
// missing cl100k_base.tiktoken (the Go image used to omit it) degraded every
// token budget to 0 while content_ltks — a separate offline C++ tokenizer —
// kept working, and the divergence went unnoticed. Catching it at startup turns
// that silent data corruption into a hard, loud failure.
func InitCL100KEncoder() error {
enc, err := getCL100KEncoder()
if err != nil {
return fmt.Errorf("cl100k_base BPE table unavailable (NumTokensFromString would silently return 0): %w", err)
}
if enc == nil {
return fmt.Errorf("cl100k_base encoder unavailable: GetEncoding returned a nil encoder without error")
}
return nil
}
// NumTokensFromString returns the number of tokens in s using the cl100k_base
// BPE encoding. Mirrors Python's num_tokens_from_string (common/token_utils.py):
// returns 0 on encoder error.
//
// Callers must ensure InitCL100KEncoder has succeeded at startup; otherwise a
// missing table makes this return 0 for every non-empty string. The startup
// check is the fail-fast guard — this cheap hot-path function stays silent by
// design so it can be called per-token without error plumbing.
func NumTokensFromString(s string) int {
if s == "" {
return 0
+105 -7
View File
@@ -33,10 +33,69 @@
import argparse
import os
import shutil
import sys
import zipfile
import requests
# Mirrors internal/common.DeepDocORTVersion (Go in-process backend). ONE OF
# THREE places (with that Go constant and the ORT_VERSION in
# ragflow_deps/download_deps.py) that must carry the same ONNX Runtime native
# release for the statically-linked Go DeepDoc backend. There is no single
# source of truth — keep all three equal. The pip onnxruntime== pin
# (pyproject.toml) and the onnxruntime_go binding minor (go.mod) must stay on
# the same minor line.
ORT_VERSION = "1.23.2"
def prune_stale_onnxruntime(static_lib_dir, version):
"""Remove ONNX Runtime version dirs under static_lib that do NOT match
`version`. Without this, a version bump leaves the stale dir next to
the new one and build.sh's `find ... -name '*.a'` links BOTH (duplicate
symbols / wrong version, silently)."""
if not os.path.isdir(static_lib_dir):
return
expected = f"onnxruntime-linux-x64-static_lib-{version}-glibc2_28"
for name in os.listdir(static_lib_dir):
if not name.startswith("onnxruntime-linux-x64-static_lib-"):
continue
if name == expected:
continue
stale = os.path.join(static_lib_dir, name)
print(f" Removing stale ONNX Runtime dir: {stale}")
shutil.rmtree(stale)
def has_static_archives(directory):
"""True when `directory` holds at least one static archive (.a)."""
return any(f.endswith(".a") for _, _, files in os.walk(directory) for f in files)
def extract_onnxruntime(static_lib_dir, archive_path, version):
"""Ensure the ONNX Runtime static archives for `version` sit under
`static_lib_dir`. Returns True when that version is available afterwards
(extracted now or already present), False when the archive is missing.
ORT ships a version-stamped top-level dir inside the zip
(onnxruntime-linux-x64-static_lib-<version>-glibc2_28/), so a present
`static_lib_dir` is NOT evidence that THIS version is extracted: after a
version bump the stale dir is pruned and the new one must be extracted.
"""
if not os.path.isfile(archive_path):
print(f" Skipping extraction: {os.path.basename(archive_path)} not found")
return False
prune_stale_onnxruntime(static_lib_dir, version)
version_dir = os.path.join(static_lib_dir, f"onnxruntime-linux-x64-static_lib-{version}-glibc2_28")
if os.path.isdir(version_dir) and has_static_archives(version_dir):
print(f" ✓ onnxruntime/static_lib ({version}) already extracted to {version_dir}")
return True
os.makedirs(static_lib_dir, exist_ok=True)
print(f" Extracting {os.path.basename(archive_path)} → {static_lib_dir}")
with zipfile.ZipFile(archive_path) as zf:
zf.extractall(static_lib_dir)
return True
def get_urls(use_china_mirrors=False) -> list[str | list[str]]:
if use_china_mirrors:
@@ -57,12 +116,16 @@ def get_urls(use_china_mirrors=False) -> list[str | list[str]]:
# compatibility contract.
"https://gh-proxy.com/https://github.com/browserbase/stagehand/releases/download/stagehand-server-v3/v3.7.2/stagehand-server-v3-linux-x64",
"https://gh-proxy.com/https://github.com/browserbase/stagehand/releases/download/stagehand-server-v3/v3.7.2/stagehand-server-v3-linux-arm64",
# Native static libraries for Go build (pdfium, pdf_oxide, office_oxide)
# Used by build.sh's check_*_deps functions — pre-downloaded to avoid
# network access during CI.
# Native static libraries for Go build (pdfium, pdf_oxide,
# office_oxide, onnxruntime). Used by build.sh's check_*_deps
# functions — pre-downloaded to avoid network access during CI.
["https://gh-proxy.com/https://github.com/kognitos/pdfium-static/releases/download/chromium%2F7809/pdfium-linux-x64-static.tgz", "pdfium-linux-x64-static.tgz"],
["https://gh-proxy.com/https://github.com/yfedoseev/pdf_oxide/releases/download/v0.3.73/pdf_oxide-go-ffi-linux-amd64.tar.gz", "pdf_oxide-go-ffi-linux-amd64.tar.gz"],
["https://gh-proxy.com/https://github.com/yfedoseev/office_oxide/releases/download/v0.1.9/native-linux-x86_64.tar.gz", "office_oxide-linux-x86_64.tar.gz"],
[
f"https://gh-proxy.com/https://github.com/csukuangfj/onnxruntime-libs/releases/download/v{ORT_VERSION}/onnxruntime-linux-x64-static_lib-{ORT_VERSION}-glibc2_28.zip",
f"onnxruntime-linux-x64-static_lib-{ORT_VERSION}-glibc2_28.zip",
],
]
else:
return [
@@ -82,12 +145,16 @@ def get_urls(use_china_mirrors=False) -> list[str | list[str]]:
# compatibility contract.
"https://github.com/browserbase/stagehand/releases/download/stagehand-server-v3/v3.7.2/stagehand-server-v3-linux-x64",
"https://github.com/browserbase/stagehand/releases/download/stagehand-server-v3/v3.7.2/stagehand-server-v3-linux-arm64",
# Native static libraries for Go build (pdfium, pdf_oxide, office_oxide)
# Used by build.sh's check_*_deps functions — pre-downloaded to avoid
# network access during CI.
# Native static libraries for Go build (pdfium, pdf_oxide,
# office_oxide, onnxruntime). Used by build.sh's check_*_deps
# functions — pre-downloaded to avoid network access during CI.
["https://github.com/kognitos/pdfium-static/releases/download/chromium%2F7809/pdfium-linux-x64-static.tgz", "pdfium-linux-x64-static.tgz"],
["https://github.com/yfedoseev/pdf_oxide/releases/download/v0.3.73/pdf_oxide-go-ffi-linux-amd64.tar.gz", "pdf_oxide-go-ffi-linux-amd64.tar.gz"],
["https://github.com/yfedoseev/office_oxide/releases/download/v0.1.9/native-linux-x86_64.tar.gz", "office_oxide-linux-x86_64.tar.gz"],
[
f"https://github.com/csukuangfj/onnxruntime-libs/releases/download/v{ORT_VERSION}/onnxruntime-linux-x64-static_lib-{ORT_VERSION}-glibc2_28.zip",
f"onnxruntime-linux-x64-static_lib-{ORT_VERSION}-glibc2_28.zip",
],
]
@@ -139,12 +206,13 @@ if __name__ == "__main__":
# Extract native static libraries to ~/ragflow-native-libs for Go build.
# Ensures build.sh can find them without network access.
native_deps_dir = os.path.expanduser("~/ragflow-native-libs")
import tarfile
extractions = [
("pdfium-linux-x64-static.tgz", "pdfium-static"),
("pdf_oxide-go-ffi-linux-amd64.tar.gz", "pdf_oxide"),
("office_oxide-linux-x86_64.tar.gz", "office_oxide"),
]
import tarfile
for archive, subdir in extractions:
archive_path = os.path.join(os.getcwd(), archive)
@@ -159,3 +227,33 @@ if __name__ == "__main__":
print(f" Extracting {archive} → {target}")
with tarfile.open(archive_path) as tf:
tf.extractall(target)
if not extract_onnxruntime(
os.path.join(native_deps_dir, "onnxruntime", "static_lib"),
os.path.join(os.getcwd(), f"onnxruntime-linux-x64-static_lib-{ORT_VERSION}-glibc2_28.zip"),
ORT_VERSION,
):
# The archive was not downloaded or failed to extract, so no .a landed.
# Fail loud instead of exiting 0: build.sh's ORT guard would otherwise
# reject the build later with a less actionable message, and a missing
# .a left here is exactly the "silent green" this PR is meant to prevent.
print(
f" ERROR: ONNX Runtime static archives for {ORT_VERSION} were not "
f"extracted to {os.path.join(native_deps_dir, 'onnxruntime', 'static_lib')}. "
f"Check the download above; build.sh will refuse to link without them.",
file=sys.stderr,
)
sys.exit(1)
# ONNX Runtime is statically linked into the server binary, so there is no
# runtime .so to surface. Log where build.sh (ONNXRUNTIME_STATIC_PREFIX) will
# find the archives — the .a files live under
# ~/ragflow-native-libs/onnxruntime/static_lib. The in-process backend
# resolves OrtGetApiBase via dlopen(NULL); there is no dynamic .so fallback.
ort_static_dir = os.path.join(native_deps_dir, "onnxruntime", "static_lib")
ort_a_files = [os.path.join(root, f) for root, _, files in os.walk(ort_static_dir) for f in files if f.endswith(".a")]
if ort_a_files:
print(f" ✓ onnxruntime static archives ready: {len(ort_a_files)} .a under {ort_static_dir}")
else:
print(f" ERROR: ONNX Runtime .a files still missing under {ort_static_dir} after extraction; build.sh will refuse to link.", file=sys.stderr)
sys.exit(1)
+71
View File
@@ -0,0 +1,71 @@
# Tests for ragflow_deps/download_go_deps.py ONNX Runtime extraction.
#
# build.sh's build_go() fails fast when libonnxruntime.a is not linked, so these
# tests must guarantee the archive is really landed on disk — above all after an
# ORT version bump, where the zip's version-stamped top-level dir changes but
# static_lib/ still exists from the previous run.
import os
import zipfile
from download_go_deps import extract_onnxruntime, has_static_archives
def make_ort_zip(path, version):
"""Build a zip shaped like the upstream ORT release: a version-stamped
top-level dir holding lib/libonnxruntime.a."""
member = f"onnxruntime-linux-x64-static_lib-{version}-glibc2_28/lib/libonnxruntime.a"
with zipfile.ZipFile(path, "w") as zf:
zf.writestr(member, b"!<arch>\nort-payload")
return path
def test_extracts_on_first_run(tmp_path):
# static_lib/ does not exist yet: the plain first-run path.
static_lib = tmp_path / "onnxruntime" / "static_lib"
version = "1.23.2"
archive = make_ort_zip(tmp_path / f"onnxruntime-linux-x64-static_lib-{version}-glibc2_28.zip", version)
assert extract_onnxruntime(str(static_lib), str(archive), version) is True
version_dir = static_lib / f"onnxruntime-linux-x64-static_lib-{version}-glibc2_28"
assert version_dir.is_dir()
assert has_static_archives(str(version_dir))
def test_extracts_after_version_bump_when_static_lib_exists(tmp_path):
"""Regression: a static_lib/ left over from a previous version must not
suppress extraction of the bumped version. Pruning the stale dir leaves
static_lib/ in place; if extraction is then skipped no .a is ever landed and
build.sh's ORT guard rejects the build."""
static_lib = tmp_path / "onnxruntime" / "static_lib"
static_lib.mkdir(parents=True)
stale = static_lib / "onnxruntime-linux-x64-static_lib-1.23.1-glibc2_28"
(stale / "lib").mkdir(parents=True)
(stale / "lib" / "libonnxruntime.a").write_bytes(b"old-ort")
version = "1.23.2"
archive = make_ort_zip(tmp_path / f"onnxruntime-linux-x64-static_lib-{version}-glibc2_28.zip", version)
assert extract_onnxruntime(str(static_lib), str(archive), version) is True
assert not stale.exists(), "stale ORT version should be pruned"
version_dir = static_lib / f"onnxruntime-linux-x64-static_lib-{version}-glibc2_28"
assert version_dir.is_dir(), "bumped ORT version was silently not extracted"
assert has_static_archives(str(version_dir)), "bumped ORT version landed no .a"
def test_skips_when_archive_missing(tmp_path):
static_lib = tmp_path / "onnxruntime" / "static_lib"
assert extract_onnxruntime(str(static_lib), str(tmp_path / "missing.zip"), "1.23.2") is False
def test_idempotent_when_matching_version_already_present(tmp_path):
static_lib = tmp_path / "onnxruntime" / "static_lib"
version = "1.23.2"
archive = make_ort_zip(tmp_path / f"onnxruntime-linux-x64-static_lib-{version}-glibc2_28.zip", version)
assert extract_onnxruntime(str(static_lib), str(archive), version) is True
before = sorted(os.listdir(static_lib))
assert extract_onnxruntime(str(static_lib), str(archive), version) is True
assert sorted(os.listdir(static_lib)) == before