diff --git a/.github/workflows/sep-tests.yml b/.github/workflows/sep-tests.yml index a32faca5a6..b4f4db16bb 100644 --- a/.github/workflows/sep-tests.yml +++ b/.github/workflows/sep-tests.yml @@ -396,6 +396,17 @@ jobs: ./build.sh --test -- $PKGS fi + - name: Run ORT deps-download regression guard + if: env.API_PROXY_SCHEME == 'go' + # Pure-Python, local, needs no API keys or model downloads; guards the + # ONNX Runtime extraction logic the in-process (Go) DeepDoc backend build + # depends on (notably the version-bump path C2 hardened). Run it in + # isolation so the test/testcases conftest (which requires + # SILICONFLOW_API_KEY) cannot fail the collection. + run: | + set -euo pipefail + uv sync --python 3.13 --group test --frozen + uv run pytest ragflow_deps/test_download_go_deps.py -q - name: Build ragflow:nightly run: | @@ -1016,6 +1027,17 @@ jobs: ./build.sh --test -- $PKGS fi + - name: Run ORT deps-download regression guard + if: env.API_PROXY_SCHEME == 'go' + # Pure-Python, local, needs no API keys or model downloads; guards the + # ONNX Runtime extraction logic the in-process (Go) DeepDoc backend build + # depends on (notably the version-bump path C2 hardened). Run it in + # isolation so the test/testcases conftest (which requires + # SILICONFLOW_API_KEY) cannot fail the collection. + run: | + set -euo pipefail + uv sync --python 3.13 --group test --frozen + uv run pytest ragflow_deps/test_download_go_deps.py -q - name: Build ragflow:nightly run: | diff --git a/Dockerfile_go b/Dockerfile_go index 1c0e89f838..75e2ff3a95 100644 --- a/Dockerfile_go +++ b/Dockerfile_go @@ -30,6 +30,16 @@ RUN --mount=type=bind,from=infiniflow/ragflow_deps:latest,source=/huggingface.co ln -s layout.onnx /ragflow/rag/res/deepdoc/layout.manual.onnx && \ ln -s layout.onnx /ragflow/rag/res/deepdoc/layout.paper.onnx +# Copy the cl100k_base BPE table used by the Go tokenizer (tiktoken-go +# cl100k_base). The deps image ships it at its root; the Go image previously +# mounted only /huggingface.co and omitted this file, so NumTokensFromString +# silently returned 0. localBpeLoader resolves it from /ragflow_deps/, +# so dropping it here makes the table load offline at startup (see +# InitCL100KEncoder fail-fast guard). +RUN --mount=type=bind,from=infiniflow/ragflow_deps:latest,source=/cl100k_base.tiktoken,target=/tmp/cl100k_base.tiktoken \ + mkdir -p /ragflow/ragflow_deps && \ + cp /tmp/cl100k_base.tiktoken /ragflow/ragflow_deps/cl100k_base.tiktoken + ENV DEBIAN_FRONTEND=noninteractive # Setup apt diff --git a/build.sh b/build.sh index e88f92e090..888c8eb365 100755 --- a/build.sh +++ b/build.sh @@ -41,7 +41,8 @@ PDF_OXIDE_VERSION="0.3.73" # (Go) DeepDoc backend. libonnxruntime*.a is linked into the server binary # (--whole-archive + --export-dynamic); OrtGetApiBase is then resolved via # dlopen(self), so no libonnxruntime.so is needed at runtime. Downloaded by -# ragflow_deps/download_deps.py into onnxruntime/static_lib. +# ragflow_deps/download_go_deps.py (and ragflow_deps/download_deps.py) into +# onnxruntime/static_lib. ONNXRUNTIME_STATIC_PREFIX="${HOME}/ragflow-native-libs/onnxruntime/static_lib" # Copy a dependency from the system pre-seed directory to the user cache. @@ -370,6 +371,33 @@ build_go() { setup_cgo_env + # The in-process (Go) DeepDoc backend is statically linked against ONNX + # Runtime (--whole-archive + -Wl,--export-dynamic, see setup_cgo_env). The + # forked onnxruntime_go binding resolves OrtGetApiBase only at RUNTIME via + # dlopen(NULL), so a missing ORT archive still lets `go build` SUCCEED and + # yields a server binary that FAILS AT STARTUP with a fatal + # "no in-process DeepDoc backend serving". That is exactly the breakage + # colleagues hit when ORT was not present at build time and setup_cgo_env + # silently skipped it. Fail the build HERE instead of deferring the breakage + # to runtime: a server binary without ORT is unusable, not a degraded one. + if ! printf '%s' "$CGO_LDFLAGS" | grep -q 'libonnxruntime'; then + echo -e "${RED}Error: ONNX Runtime static libraries are not linked.${NC}" >&2 + echo " The in-process DeepDoc backend requires libonnxruntime.a to be" >&2 + echo " statically linked into ragflow_server (--whole-archive +" >&2 + echo " -Wl,--export-dynamic). Without it the binary compiles but dies" >&2 + echo " at startup with a fatal 'no in-process DeepDoc backend serving'." >&2 + echo " Fetch the static libs with:" >&2 + echo " uv run python3 ragflow_deps/download_go_deps.py" >&2 + echo " or pre-seed them at /opt/ragflow-native-libs/onnxruntime (CI image)." >&2 + echo " This production binary must statically include ORT — there is no" >&2 + echo " ORT-free build path. ORT ends up unlinked only via one of:" >&2 + echo " - the ORT static_lib dir was never seeded: run" >&2 + echo " 'uv run python3 ragflow_deps/download_go_deps.py', or pre-seed" >&2 + echo " /opt/ragflow-native-libs/onnxruntime as the CI image does;" >&2 + echo " - setup_cgo_env did not add libonnxruntime to CGO_LDFLAGS." >&2 + return 1 + fi + local strip_flags=() [ -n "$STRIP_SYMBOLS" ] && strip_flags=(-ldflags="-s -w") diff --git a/cmd/ragflow_server.go b/cmd/ragflow_server.go index 01e316c5e1..1cd60c8074 100644 --- a/cmd/ragflow_server.go +++ b/cmd/ragflow_server.go @@ -568,6 +568,12 @@ func runIngestor(ctx context.Context, cancel context.CancelFunc, args *serverArg } defer tokenizer.Close() + // Fail fast if the cl100k_base BPE table is missing: without it + // NumTokensFromString silently returns 0, corrupting every token budget. + if err := tokenizer.InitCL100KEncoder(); err != nil { + common.Fatal("Failed to initialize cl100k_base tokenizer", zap.Error(err)) + } + // The dataset-level post-processing consumer cluster (§11) is owned and run by // the Ingestor: it is started inside ingestor.Start() and joined inside // ingestor.Stop(), so its lifecycle matches the ingestor. The configured @@ -712,6 +718,12 @@ func runAPI(ctx context.Context, args *serverArgs) error { } defer tokenizer.Close() + // Fail fast if the cl100k_base BPE table is missing: without it + // NumTokensFromString silently returns 0, corrupting every token budget. + if err := tokenizer.InitCL100KEncoder(); err != nil { + common.Fatal("Failed to initialize cl100k_base tokenizer", zap.Error(err)) + } + // Initialize global QueryBuilder using tokenizer's DictPath // This ensures the Synonym uses the same wordnet directory as tokenizer if err := nlp.InitQueryBuilderFromTokenizer(tokenizerCfg.DictPath); err != nil { diff --git a/internal/common/environments.go b/internal/common/environments.go index 64111abd11..f461377289 100644 --- a/internal/common/environments.go +++ b/internal/common/environments.go @@ -254,17 +254,23 @@ func HasModelFiles(dir string) bool { } // DeepDocORTVersion is the onnxruntime native release the in-process (Go) -// DeepDoc backend is built and tested against (e.g. "1.23.2"). It is the -// single source for the download URL and extracted dir name used across Go -// and Python. The Go binding (github.com/yalue/onnxruntime_go, forked to +// DeepDoc backend is built and tested against (e.g. "1.23.2"). It is ONE OF +// THREE raw version declarations that must stay equal (the other two are +// ORT_VERSION in ragflow_deps/download_go_deps.py and ragflow_deps/download_deps.py) +// — NOT a single source of truth. The download URL and extracted dir name are +// built from those ORT_VERSION constants, not from this one. The Go binding +// (github.com/yalue/onnxruntime_go, forked to // github.com/xugangqiang/onnxruntime_go) and the pip onnxruntime== pin must // track this MINOR version: the binding uses its own release numbering // (v1.23.0 <-> ORT 1.23.x) but is ABI-compatible with this native release on // the same minor line. ONNX Runtime is linked statically (libonnxruntime.a), // so there is no .so / SONAME at runtime. // -// To bump ORT: update DeepDocORTVersion (Go) AND ORT_VERSION in -// ragflow_deps/download_deps.py AND the onnxruntime/onnxruntime-gpu pins in -// pyproject.toml + .github/workflows/deepdoc-drift.yml, and refresh the -// onnxruntime_go binding minor in go.mod. +// To bump ORT, ALL of the following must change together (drift breaks the +// static link or the runtime OrtGetApiBase lookup): +// - DeepDocORTVersion (here, Go) AND ORT_VERSION in BOTH +// ragflow_deps/download_go_deps.py and ragflow_deps/download_deps.py; +// - the onnxruntime== pin in pyproject.toml and the onnxruntime / +// onnxruntime-gpu pins in .github/workflows/deepdoc-drift.yml; +// - the onnxruntime_go binding minor in go.mod. const DeepDocORTVersion = "1.23.2" diff --git a/internal/deepdoc/native/ocr_rec.go b/internal/deepdoc/native/ocr_rec.go index a21f51abdb..e1e0b16c58 100644 --- a/internal/deepdoc/native/ocr_rec.go +++ b/internal/deepdoc/native/ocr_rec.go @@ -36,7 +36,7 @@ type OCRRecResult struct { } // RunOCRRec recognizes a single cropped text-line image. It is equivalent to a -// one-line batch (see RunOCRRecBatch): the line is resized against its own +// one-line batch (see RunOCRRecBatchReal): the line is resized against its own // wh_ratio floored at 320/48, never the batch-max, so a caller recognizing // lines independently gets the same result as a standalone Python // TextRecognizer call. @@ -55,28 +55,16 @@ func RunOCRRec(ctx context.Context, modelDir string, img *Image) (OCRRecResult, return recognizeLine(ctx, modelDir, img, maxWhRatio, chars) } -// RunOCRRecBatch recognizes a batch of cropped text-line images the way -// deepdoc's TextRecognizer.__call__ does: every line in the batch is resized to -// the SAME width derived from the batch's maximum wh_ratio (floored at 320/48), -// so a narrow line inside a wide batch is widened to the batch max rather than -// its own ratio. This makes Go match production batch semantics, where a page's -// lines are recognized together. Lines are recognized independently after the -// shared resize, so order is preserved and results are deterministic per batch. -// -// DEPRECATED semantic-shape note: this variant still runs one ONNX Run PER -// line (recMaxBatch=1). Prefer RunOCRRecBatchReal, which concatenates every -// line's preprocessed blob into a single {N,3,48,imgW} tensor and performs ONE -// ONNX Run — identical numerically (each line uses the same shared batch width) -// but with the real throughput benefit of batched matmul. RunOCRRecBatchReal is -// the production path. // RunOCRRecBatchReal recognizes a batch of cropped text-line images with a -// SINGLE ONNX Run, mirroring deepdoc's TextRecognizer.__call__ exactly: every -// line is resized to the batch's shared width (imgW = 48*max_wh_ratio, floored -// at 320/48) and zero-padded on the right, all blobs are concatenated into one -// {N,3,48,imgW} tensor, and the model runs once. The output is split back into -// per-line sequences and CTC-decoded in order, so the result is numerically -// identical to calling RunOCRRec on each line (each line sees the same shared -// batch width), but amortized over one forward pass instead of N. +// SINGLE ONNX Run, mirroring deepdoc's TextRecognizer.__call__: each line is +// resized to its own proportional width (recH * that line's wh_ratio), capped +// by the batch-shared imgW (imgW = recH * max_wh_ratio, with max_wh_ratio +// floored at 320/48), and zero-padded on the right out to imgW; all blobs are +// concatenated into one {N,3,48,imgW} tensor, and the model runs once. The +// output is split back into per-line sequences and CTC-decoded in order, so +// the result is numerically identical to calling RunOCRRec on each line (each +// line sees the same shared batch width), but amortized over one forward pass +// instead of N. // // The shared batch width means a line is resized against the batch max wh_ratio, // not its own — exactly what deepdoc does inside a batch. A standalone call to diff --git a/internal/deepdoc/native/testdata_fetch.go b/internal/deepdoc/native/testdata_fetch.go index b3d0a922ec..37fd37c1f6 100644 --- a/internal/deepdoc/native/testdata_fetch.go +++ b/internal/deepdoc/native/testdata_fetch.go @@ -2,8 +2,9 @@ // This file is compiled only with -tags fetch_testdata. Its init() downloads // and symlinks the external DeepDoc testdata for this package (pkg = "native") -// so the consuming tests can run. The download is best-effort: if it fails, the -// guard in testdata_skip.go skips the tests instead of failing. +// so the consuming tests can run. A fetch failure is fatal under this build tag: +// it sets testdataFetchFailed so TestMain fails the package instead of silently +// skipping — a missing fixture must never be a green CI run. package native import ( @@ -17,6 +18,7 @@ import ( func init() { if err := fetchTestdata(); err != nil { fmt.Fprintf(os.Stderr, "fetch_deepdoc_testdata: not fetched (%v)\n", err) + testdataFetchFailed = true return } testdataFetchAttempted = true diff --git a/internal/deepdoc/native/testdata_flag.go b/internal/deepdoc/native/testdata_flag.go index 5f329abe5d..c01461a8ed 100644 --- a/internal/deepdoc/native/testdata_flag.go +++ b/internal/deepdoc/native/testdata_flag.go @@ -8,3 +8,11 @@ package native // imports native, and _test.go files are excluded from dependency builds). In the // default build it stays false, so TestMain skips the package. var testdataFetchAttempted bool + +// testdataFetchFailed is set to true by testdata_fetch.go (compiled only with +// -tags fetch_testdata) when the external testdata download fails. It is +// declared here (always-compiled) so TestMain can read it. Under the +// fetch_testdata build tag a failed fetch must FAIL the package rather than +// silently skip — a missing fixture must never produce a green CI run. In the +// default build this stays false, so the absence of fixtures is a benign skip. +var testdataFetchFailed bool diff --git a/internal/deepdoc/native/testdata_skip_test.go b/internal/deepdoc/native/testdata_skip_test.go index 278455ef49..7126ec9c2f 100644 --- a/internal/deepdoc/native/testdata_skip_test.go +++ b/internal/deepdoc/native/testdata_skip_test.go @@ -1,9 +1,10 @@ // This file is always compiled (no build constraint) and provides the package // TestMain that guards the whole test binary behind the presence of the // package-level testdata directory. When the external testdata has not been -// fetched (e.g. -tags fetch_testdata was not supplied, or the download failed), -// the tests are skipped rather than failed — keeping the default `go test ./...` -// and `go test -tags cgo` runs green without the assets. +// fetched (e.g. -tags fetch_testdata was not supplied), the tests are skipped +// rather than failed — keeping the default `go test ./...` and `go test -tags cgo` +// runs green without the assets. A fetch ATTEMPTED but FAILED (under +// -tags fetch_testdata) is fatal: a missing fixture must never be a green run. package native import ( @@ -18,6 +19,10 @@ import ( // assets — which also makes a stale/partial leftover testdata directory on disk // harmless: we skip regardless of what (if anything) is present. func TestMain(m *testing.M) { + if testdataFetchFailed { + fmt.Fprintln(os.Stderr, "FAIL: DeepDoc native testdata fetch failed (see the fetch_deepdoc_testdata error above). CI must not pass with missing fixtures; run with -tags fetch_testdata and fix the download.") + os.Exit(1) + } if !testdataFetchAttempted { fmt.Println("SKIP: DeepDoc native testdata not fetched; run with -tags fetch_testdata to fetch and run it") os.Exit(0) diff --git a/internal/deepdoc/parser/pdf/inference/native_analyzer/testdata_fetch.go b/internal/deepdoc/parser/pdf/inference/native_analyzer/testdata_fetch.go index cbd04e1345..0c2d9a8902 100644 --- a/internal/deepdoc/parser/pdf/inference/native_analyzer/testdata_fetch.go +++ b/internal/deepdoc/parser/pdf/inference/native_analyzer/testdata_fetch.go @@ -3,9 +3,10 @@ // This file is compiled only with -tags fetch_testdata. Its init() downloads // and symlinks the external DeepDoc testdata. The native_analyzer tests read // their fixtures from the sibling internal/deepdoc/native/testdata directory -// (via a relative path), so we fetch with pkg = "native". The download is -// best-effort: if it fails, the guard in testdata_skip.go skips the tests -// instead of failing. +// (via a relative path), so we fetch with pkg = "native". A fetch failure is +// fatal under this build tag: it sets testdataFetchFailed so TestMain fails the +// package instead of silently skipping — a missing fixture must never be a green +// CI run. package infnative import ( @@ -19,6 +20,7 @@ import ( func init() { if err := fetchTestdata(); err != nil { fmt.Fprintf(os.Stderr, "fetch_deepdoc_testdata: not fetched (%v)\n", err) + testdataFetchFailed = true return } testdataFetchAttempted = true diff --git a/internal/deepdoc/parser/pdf/inference/native_analyzer/testdata_flag.go b/internal/deepdoc/parser/pdf/inference/native_analyzer/testdata_flag.go index 24a85df01b..4ba72471c7 100644 --- a/internal/deepdoc/parser/pdf/inference/native_analyzer/testdata_flag.go +++ b/internal/deepdoc/parser/pdf/inference/native_analyzer/testdata_flag.go @@ -7,3 +7,11 @@ package infnative // this package is built as a dependency of another package's test binary. In the // default build it stays false, so TestMain skips the package. var testdataFetchAttempted bool + +// testdataFetchFailed is set to true by testdata_fetch.go (compiled only with +// -tags fetch_testdata) when the external testdata download fails. Declared here +// (always-compiled) so TestMain can read it. Under the fetch_testdata build tag a +// failed fetch must FAIL the package rather than silently skip — a missing +// fixture must never produce a green CI run. In the default build this stays +// false, so the absence of fixtures is a benign skip. +var testdataFetchFailed bool diff --git a/internal/deepdoc/parser/pdf/inference/native_analyzer/testdata_skip_test.go b/internal/deepdoc/parser/pdf/inference/native_analyzer/testdata_skip_test.go index 7d37f03ac1..7ec26e5dcd 100644 --- a/internal/deepdoc/parser/pdf/inference/native_analyzer/testdata_skip_test.go +++ b/internal/deepdoc/parser/pdf/inference/native_analyzer/testdata_skip_test.go @@ -3,9 +3,10 @@ // DeepDoc testdata directory. The native_analyzer tests read their fixtures // from the sibling internal/deepdoc/native/testdata directory (via a relative // path), so that is what we check here. When the external testdata has not been -// fetched (e.g. -tags fetch_testdata was not supplied, or the download failed), -// the tests are skipped rather than failed — keeping the default `go test ./...` -// and `go test -tags cgo` runs green without the assets. +// fetched (e.g. -tags fetch_testdata was not supplied), the tests are skipped +// rather than failed — keeping the default `go test ./...` and `go test -tags cgo` +// runs green without the assets. A fetch ATTEMPTED but FAILED (under +// -tags fetch_testdata) is fatal: a missing fixture must never be a green run. package infnative import ( @@ -20,6 +21,10 @@ import ( // assets — which also makes a stale/partial leftover testdata directory on disk // harmless: we skip regardless of what (if anything) is present. func TestMain(m *testing.M) { + if testdataFetchFailed { + fmt.Fprintln(os.Stderr, "FAIL: DeepDoc native testdata fetch failed (see the fetch_deepdoc_testdata error above). CI must not pass with missing fixtures; run with -tags fetch_testdata and fix the download.") + os.Exit(1) + } if !testdataFetchAttempted { fmt.Println("SKIP: DeepDoc native testdata not fetched; run with -tags fetch_testdata to fetch and run it") os.Exit(0) diff --git a/internal/deepdoc/parser/pdf/parser_ocr.go b/internal/deepdoc/parser/pdf/parser_ocr.go index 79ab8c4ead..7a137a85e3 100644 --- a/internal/deepdoc/parser/pdf/parser_ocr.go +++ b/internal/deepdoc/parser/pdf/parser_ocr.go @@ -105,24 +105,37 @@ func (p *Parser) ocrDetectAndRecognize(ctx context.Context, pageImg image.Image, allTexts := make([][]pdf.OCRText, len(cropAcc)) if p.docSupportsBatchOCR(doc) { batch, berr := p.inferOCRRecognizeBatch(ctx, doc, cropAcc) - if berr != nil { - slog.Warn(logLabel+" OCR batch recognize failed", "page", pageNum, "err", berr) + switch { + case berr != nil: + // A batch error must not abort the whole page: the canonical + // per-crop path below still produces correct results. + slog.Warn(logLabel+" OCR batch recognize failed; falling back to per-crop", "page", pageNum, "err", berr) + case len(batch) != len(cropAcc): + // Defensive: a count mismatch (or a nil result) would corrupt the + // per-box indexing further down. Fall back to per-crop instead of + // indexing out of range. + slog.Warn(logLabel+" OCR batch recognize returned unexpected count; falling back to per-crop", "page", pageNum, "got", len(batch), "want", len(cropAcc)) + default: + allTexts = batch + } + } + // Fill any crop the batch path did not cover (unsupported doc, batch error, + // or count mismatch) with the per-crop canonical path. Stamping the + // detect-box index lets a replay DocAnalyzer route the recognition back to + // the Python-dumped text for the same box; the production analyzer ignores + // the key. + for ci := range cropAcc { + if allTexts[ci] != nil { + continue + } + c := cropAcc[ci] + recCtx := context.WithValue(ctx, ocrBoxIdxCtxKey, cropBoxIdx[ci]) + texts, rerr := p.ocrRecognizeWithRotation(recCtx, doc, c) + if rerr != nil { + slog.Warn(logLabel+" OCR recognize failed", "page", pageNum, "err", rerr) return nil } - allTexts = batch - } else { - for ci, c := range cropAcc { - // Stamp the detect-box index so a replay DocAnalyzer routes this - // recognition back to the Python-dumped text for the same box. The - // production analyzer ignores the key. - recCtx := context.WithValue(ctx, ocrBoxIdxCtxKey, cropBoxIdx[ci]) - texts, rerr := p.ocrRecognizeWithRotation(recCtx, doc, c) - if rerr != nil { - slog.Warn(logLabel+" OCR recognize failed", "page", pageNum, "err", rerr) - return nil - } - allTexts[ci] = texts - } + allTexts[ci] = texts } // Per box: pick the highest-confidence candidate (layer-2 winner) and diff --git a/internal/development.md b/internal/development.md index bfd506fa40..a0226c2146 100644 --- a/internal/development.md +++ b/internal/development.md @@ -39,14 +39,33 @@ go version ### 1.4 Install dependent library ```shell sudo apt install libpcre2-dev +# Native libs for the Go build (office_oxide, pdfium, pdf_oxide, onnxruntime). +# download_go_deps.py is the Go-end script: it now fetches the ONNX Runtime +# static lib too, so this single command covers every native dependency the +# Go server binary needs (incl. the in-process DeepDoc backend). python3 ragflow_deps/download_go_deps.py +# Shared (Go + Python) deps. Retains ONNX Runtime for the ragflow_deps image / +# backward-compat; also pulls Python-side artifacts (models, nltk, tika, ...). +uv run python3 ragflow_deps/download_deps.py ``` > **Note**: If you use IDEs like GoLand to run/debug directly (via Run/Debug buttons), or run `go build` / `go run` from command line, set these CGO environment variables: > > ```bash -> RAGFLOW_DEPS="${HOME}/ragflow-native-libs" # created by uv run ragflow_deps/download_deps.py +> RAGFLOW_DEPS="${HOME}/ragflow-native-libs" # created by download_go_deps.py + download_deps.py > PLATFORM="linux_amd64" # or darwin_amd64, linux_arm64, darwin_arm64 +> # NOTE: the ONNX Runtime static lib fetched by download_go_deps.py is +> # linux-x64 ONLY (onnxruntime-linux-x64-static_lib-*). On darwin_* / non-amd64 +> # PLATFORM values the production DeepDoc backend cannot be linked, so those +> # PLATFORM examples cover office_oxide/pdfium/pdf_oxide only — ORT is a +> # Linux-amd64 link dependency here. +> +> # Resolve the version-stamped ORT archive path FIRST, in its own unquoted +> # assignment: the shell does not expand `*` inside the double-quoted +> # CGO_LDFLAGS below, and `--whole-archive` must stay a separate argument from +> # the archive path. If this lists more than one match, delete the stale +> # version dir — build.sh refuses to link two ORT versions. +> ORT_A="$(ls ${RAGFLOW_DEPS}/onnxruntime/static_lib/*/lib/libonnxruntime.a)" > > export CGO_CFLAGS="-I${RAGFLOW_DEPS}/office_oxide/include/office_oxide_c" > export CGO_LDFLAGS="\ @@ -55,11 +74,29 @@ python3 ragflow_deps/download_go_deps.py > ${RAGFLOW_DEPS}/pdfium-static/lib/libc++.a \ > ${RAGFLOW_DEPS}/pdfium-static/lib/libc++abi.a \ > ${RAGFLOW_DEPS}/pdf_oxide/lib/${PLATFORM}/libpdf_oxide.a \ +> -Wl,--export-dynamic -Wl,--whole-archive ${ORT_A} -Wl,--no-whole-archive -lstdc++ \ > -fuse-ld=lld \ > -lm -lpthread -ldl -lrt -lgcc_s -lutil -lc" > ``` > -> All three native libraries are statically linked — no `LD_LIBRARY_PATH` or `-Wl,-rpath` needed. +> All four native libraries are statically linked — no `LD_LIBRARY_PATH` or `-Wl,-rpath` needed. +> +> **ONNX Runtime is mandatory for the production binary.** The in-process (Go) +> DeepDoc backend is statically linked against `libonnxruntime.a` via +> `--whole-archive -Wl,--export-dynamic`, and `OrtGetApiBase` is resolved at +> runtime through `dlopen(NULL)`. The forked `onnxruntime_go` binding only +> needs `-ldl` to *compile*, so a binary built **without** ORT links +> successfully but dies at startup with: +> `Error looking up OrtGetApiBase in statically-linked ONNX Runtime` → fatal +> `no in-process DeepDoc backend serving`. +> Since `build.sh` (`build_go`) now **fails fast** when ORT is absent from +> `CGO_LDFLAGS`, this breakage surfaces at build time instead of at runtime. If +> you see `Error: ONNX Runtime static libraries are not linked`, run +> `uv run python3 ragflow_deps/download_go_deps.py` (or pre-seed +> `/opt/ragflow-native-libs/onnxruntime` as the CI runner image does). There is +> no ORT-free production build path — if ORT is absent the binary fails at +> startup, so the remedy is always to seed the static lib above, never to build +> without it. ### 1.5 Build RAGFlow @@ -77,6 +114,35 @@ python3 ragflow_deps/download_go_deps.py ./build.sh -s --go ``` +### 1.6 In-process (Go) DeepDoc backend + +The in-process DeepDoc backend is statically linked against ONNX Runtime +(see §1.4). After a successful `./build.sh -s --go`, the `ragflow_server` +binary carries it and registers the backend at startup. + +- **Confirm it is serving** — the server logs, at startup: + `in-process DeepDoc backend registered (production backend)` + If you instead see a fatal `no in-process DeepDoc backend serving`, ORT was + not linked into the binary. Re-run `uv run python3 ragflow_deps/download_go_deps.py` + and rebuild (§1.4 explains why; `build.sh` fails fast with + `Error: ONNX Runtime static libraries are not linked` before this happens). + +- **Run the binary directly (local dev)** — `./bin/ragflow_server --api` + (start `--admin` first, see §2) launches the Go server and registers the + backend. No extra environment variable is required. + +- **Run the Go Docker image** — the container entrypoint only starts the Go + server (`bin/ragflow_server --api/--ingestor/--admin`) when + `API_PROXY_SCHEME` is `go` or `hybrid`. With the variable unset (or `python`) + the Go server does NOT start, so the in-process backend is absent and the + image looks like a Python-only build. Start it with: + ```bash + docker run -e API_PROXY_SCHEME=go infiniflow/ragflow:go-test-1 + ``` + +- ORT is resolved via `dlopen(NULL)` at runtime, so no `LD_LIBRARY_PATH` / + `-rpath` is needed and no `libonnxruntime.so` ships with the image. + ## 2. Start RAGFlow - Start dependencies diff --git a/internal/tokenizer/bpe_loader.go b/internal/tokenizer/bpe_loader.go index 396a4e5732..80ad8eeb85 100644 --- a/internal/tokenizer/bpe_loader.go +++ b/internal/tokenizer/bpe_loader.go @@ -138,7 +138,7 @@ func bpeCandidatePaths(bpeURL string) []string { } } - for _, root := range searchRoots() { + for _, root := range bpeSearchRoots() { // Same layout the Dockerfile creates: the table sits in the // installation root under its sha1 name. add(filepath.Join(root, cacheName)) @@ -172,6 +172,25 @@ func searchRoots() []string { return roots } +// bpeSearchRoots is the directory set the loader walks for the table. It +// defaults to searchRoots (working + executable dirs and their ancestors) so +// the production image finds the table dropped into the install root. Tests +// override it with SetBpeSearchRootsForTest to isolate from the host +// filesystem — without that, a tiktoken cache leaked into /tmp (an ancestor of +// any temp working dir) would be picked up and make "missing table" tests +// non-deterministic. +var bpeSearchRoots = searchRoots + +// SetBpeSearchRootsForTest replaces the directory set the loader searches. +// Pass nil to restore the default. Test-only; it mutates package state. +func SetBpeSearchRootsForTest(roots []string) { + if roots == nil { + bpeSearchRoots = searchRoots + return + } + bpeSearchRoots = func() []string { return roots } +} + func startingDirs() []string { var dirs []string if wd, err := os.Getwd(); err == nil { diff --git a/internal/tokenizer/bpe_loader_test.go b/internal/tokenizer/bpe_loader_test.go index 324bb38aae..cb00102155 100644 --- a/internal/tokenizer/bpe_loader_test.go +++ b/internal/tokenizer/bpe_loader_test.go @@ -41,8 +41,12 @@ func writeBpeTable(t *testing.T, path string, marker int) { } // Register this synthetic table's digest so the SHA-1 integrity gate in // bpe_loader.go accepts it. This also exercises the verification path - // instead of disabling it. + // instead of disabling it. Restore the prior value afterwards so the + // mutation does not leak into sibling tests (which would otherwise read a + // stale digest for the same URL). + prev := expectedBpeHashes[testBpeURL] expectedBpeHashes[testBpeURL] = fmt.Sprintf("%x", sha1.Sum([]byte(line))) + t.Cleanup(func() { expectedBpeHashes[testBpeURL] = prev }) } func cacheFileName(url string) string { @@ -52,13 +56,17 @@ func cacheFileName(url string) string { // isolate moves the process into an empty directory and clears both cache // environment variables, so a test sees only the files it creates itself. // Without this the real repository checkout — which does ship the table — would -// satisfy every lookup and hide ordering bugs. +// satisfy every lookup and hide ordering bugs. It also scopes the loader's +// search roots to this directory so a tiktoken cache leaked into an ancestor +// (e.g. /tmp) can't be picked up and make "missing table" tests flaky. func isolate(t *testing.T) string { t.Helper() dir := t.TempDir() t.Setenv("TIKTOKEN_CACHE_DIR", "") t.Setenv("DATA_GYM_CACHE_DIR", "") t.Chdir(dir) + SetBpeSearchRootsForTest([]string{dir}) + t.Cleanup(func() { SetBpeSearchRootsForTest(nil) }) return dir } diff --git a/internal/tokenizer/failfast_test.go b/internal/tokenizer/failfast_test.go new file mode 100644 index 0000000000..4567c66487 --- /dev/null +++ b/internal/tokenizer/failfast_test.go @@ -0,0 +1,147 @@ +// +// Copyright 2026 The InfiniFlow Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package tokenizer + +import ( + "io" + "os" + "path/filepath" + "strings" + "testing" +) + +// TestInitCL100KEncoder_FailFast pins the contract that InitCL100KEncoder fails +// fast on a missing cl100k_base table, and — the part the regression actually +// guards — that a PRESENT table yields a working encoder (NumTokensFromString > 0) +// rather than a silent 0. +// +// Background: NumTokensFromString swallows the loader error and returns 0, so a +// Go image that forgot to bake cl100k_base.tiktoken silently zeroed every token +// count while content_ltks (a separate C++ tokenizer) kept working. The fix is +// to fail fast at startup via InitCL100KEncoder; this test must exercise BOTH +// branches, otherwise it only re-asserts "missing → error" and leaves the +// "present → counts" behaviour — the thing that regressed — uncovered. +// +// Both branches reset the encoder cache and scope bpeSearchRoots to a temp dir, +// so neither depends on the host (no leaked tiktoken cache in an ancestor, no +// success cached by a sibling test, no order dependence). "absent" runs first so +// that the table "present" later copies under /tmp is not discoverable by it via +// searchRoots() walking up to the shared /tmp ancestor. +func TestInitCL100KEncoder_FailFast(t *testing.T) { + // fail-fast: an empty scoped dir must surface a hard error, not a silent 0. + t.Run("absent", func(t *testing.T) { + resetCL100KEncoderForTest() + dir := t.TempDir() + t.Setenv("TIKTOKEN_CACHE_DIR", "") + t.Setenv("DATA_GYM_CACHE_DIR", "") + SetBpeSearchRootsForTest([]string{dir}) + t.Cleanup(func() { SetBpeSearchRootsForTest(nil) }) + + err := InitCL100KEncoder() + if err == nil { + t.Fatalf("InitCL100KEncoder returned nil with no table present; expected a fail-fast error") + } + if !strings.Contains(err.Error(), "cl100k") { + t.Fatalf("InitCL100KEncoder error does not mention cl100k: %v", err) + } + }) + + // present: copy a real table into an isolated dir (via TIKTOKEN_CACHE_DIR so + // the loader finds it without walking ancestors) and require that startup + // succeeds AND the encoder actually counts tokens (not just "loads"). + t.Run("present", func(t *testing.T) { + src := findRealBpeTable(t) + if src == "" { + t.Skip("no cl100k_base.tiktoken on disk to exercise the present branch") + } + resetCL100KEncoderForTest() + dir := t.TempDir() + t.Setenv("TIKTOKEN_CACHE_DIR", dir) + t.Setenv("DATA_GYM_CACHE_DIR", "") + dst := filepath.Join(dir, cacheFileName(testBpeURL)) + copyFile(t, src, dst) + SetBpeSearchRootsForTest([]string{dir}) + t.Cleanup(func() { SetBpeSearchRootsForTest(nil) }) + + if err := InitCL100KEncoder(); err != nil { + t.Fatalf("InitCL100KEncoder with table present: unexpected error: %v", err) + } + if got := NumTokensFromString("hello world"); got <= 0 { + t.Fatalf("InitCL100KEncoder succeeded but NumTokensFromString(%q) = %d, want > 0 (silent-zero regression)", "hello world", got) + } + }) +} + +// findRealBpeTable returns a path to a cl100k_base.tiktoken on disk, or "" if +// none is available. The loader's integrity gate already knows this table's +// digest (expectedBpeHashes), so copying it into a scoped dir is enough. +func findRealBpeTable(t *testing.T) string { + t.Helper() + // 1. Explicit cache dirs (tiktoken-go / data-gym layouts). + for _, env := range []string{"TIKTOKEN_CACHE_DIR", "DATA_GYM_CACHE_DIR"} { + if dir := strings.TrimSpace(os.Getenv(env)); dir != "" { + for _, name := range []string{cacheFileName(testBpeURL), "cl100k_base.tiktoken"} { + if p := filepath.Join(dir, name); fileExists(p) { + return p + } + } + } + } + // 2. repo-local ragflow_deps/cl100k_base.tiktoken (provisioned by download + // scripts / CI), walking up from the package dir. + pkgDir, err := os.Getwd() + if err != nil { + return "" + } + for dir := pkgDir; ; { + p := filepath.Join(dir, "ragflow_deps", "cl100k_base.tiktoken") + if fileExists(p) { + return p + } + parent := filepath.Dir(dir) + if parent == dir { + break + } + dir = parent + } + return "" +} + +func fileExists(p string) bool { + info, err := os.Stat(p) + return err == nil && !info.IsDir() +} + +func copyFile(t *testing.T, src, dst string) { + t.Helper() + if err := os.MkdirAll(filepath.Dir(dst), 0o755); err != nil { + t.Fatalf("mkdir %s: %v", filepath.Dir(dst), err) + } + in, err := os.Open(src) + if err != nil { + t.Fatalf("open %s: %v", src, err) + } + defer in.Close() + out, err := os.Create(dst) + if err != nil { + t.Fatalf("create %s: %v", dst, err) + } + defer out.Close() + if _, err := io.Copy(out, in); err != nil { + t.Fatalf("copy %s -> %s: %v", src, dst, err) + } +} diff --git a/internal/tokenizer/tokenizer.go b/internal/tokenizer/tokenizer.go index fcf9acdd66..6e6fc51b52 100644 --- a/internal/tokenizer/tokenizer.go +++ b/internal/tokenizer/tokenizer.go @@ -560,9 +560,49 @@ func getCL100KEncoder() (*tiktoken.Tiktoken, error) { return cl100kEncoder.enc, cl100kEncoder.err } +// resetCL100KEncoderForTest clears the cl100k encoder cache so a test can force a +// fresh load under a different search-root scope. Test-only; it mutates package +// state. A plain assignment replaces the embedded sync.Once with an un-fired one +// and wipes any cached result, so the next getCL100KEncoder / InitCL100KEncoder +// re-runs the loader rather than returning a result cached by a sibling test. +func resetCL100KEncoderForTest() { + cl100kEncoder = struct { + sync.Once + enc *tiktoken.Tiktoken + err error + }{} +} + +// InitCL100KEncoder loads the cl100k_base BPE encoder once and returns an +// error if the table is unavailable. Call it during server startup (before any +// request reaches NumTokensFromString) so a missing table fails fast instead +// of silently zeroing every token count. +// +// Why fail-fast and not a retry/warning: NumTokensFromString intentionally +// returns 0 on encoder error to stay cheap on the hot path, which means a +// missing cl100k_base.tiktoken (the Go image used to omit it) degraded every +// token budget to 0 while content_ltks — a separate offline C++ tokenizer — +// kept working, and the divergence went unnoticed. Catching it at startup turns +// that silent data corruption into a hard, loud failure. +func InitCL100KEncoder() error { + enc, err := getCL100KEncoder() + if err != nil { + return fmt.Errorf("cl100k_base BPE table unavailable (NumTokensFromString would silently return 0): %w", err) + } + if enc == nil { + return fmt.Errorf("cl100k_base encoder unavailable: GetEncoding returned a nil encoder without error") + } + return nil +} + // NumTokensFromString returns the number of tokens in s using the cl100k_base // BPE encoding. Mirrors Python's num_tokens_from_string (common/token_utils.py): // returns 0 on encoder error. +// +// Callers must ensure InitCL100KEncoder has succeeded at startup; otherwise a +// missing table makes this return 0 for every non-empty string. The startup +// check is the fail-fast guard — this cheap hot-path function stays silent by +// design so it can be called per-token without error plumbing. func NumTokensFromString(s string) int { if s == "" { return 0 diff --git a/ragflow_deps/download_go_deps.py b/ragflow_deps/download_go_deps.py index b6ef0267b2..3819f6bc8c 100644 --- a/ragflow_deps/download_go_deps.py +++ b/ragflow_deps/download_go_deps.py @@ -33,10 +33,69 @@ import argparse import os +import shutil import sys +import zipfile import requests +# Mirrors internal/common.DeepDocORTVersion (Go in-process backend). ONE OF +# THREE places (with that Go constant and the ORT_VERSION in +# ragflow_deps/download_deps.py) that must carry the same ONNX Runtime native +# release for the statically-linked Go DeepDoc backend. There is no single +# source of truth — keep all three equal. The pip onnxruntime== pin +# (pyproject.toml) and the onnxruntime_go binding minor (go.mod) must stay on +# the same minor line. +ORT_VERSION = "1.23.2" + + +def prune_stale_onnxruntime(static_lib_dir, version): + """Remove ONNX Runtime version dirs under static_lib that do NOT match + `version`. Without this, a version bump leaves the stale dir next to + the new one and build.sh's `find ... -name '*.a'` links BOTH (duplicate + symbols / wrong version, silently).""" + if not os.path.isdir(static_lib_dir): + return + expected = f"onnxruntime-linux-x64-static_lib-{version}-glibc2_28" + for name in os.listdir(static_lib_dir): + if not name.startswith("onnxruntime-linux-x64-static_lib-"): + continue + if name == expected: + continue + stale = os.path.join(static_lib_dir, name) + print(f" Removing stale ONNX Runtime dir: {stale}") + shutil.rmtree(stale) + + +def has_static_archives(directory): + """True when `directory` holds at least one static archive (.a).""" + return any(f.endswith(".a") for _, _, files in os.walk(directory) for f in files) + + +def extract_onnxruntime(static_lib_dir, archive_path, version): + """Ensure the ONNX Runtime static archives for `version` sit under + `static_lib_dir`. Returns True when that version is available afterwards + (extracted now or already present), False when the archive is missing. + + ORT ships a version-stamped top-level dir inside the zip + (onnxruntime-linux-x64-static_lib--glibc2_28/), so a present + `static_lib_dir` is NOT evidence that THIS version is extracted: after a + version bump the stale dir is pruned and the new one must be extracted. + """ + if not os.path.isfile(archive_path): + print(f" Skipping extraction: {os.path.basename(archive_path)} not found") + return False + prune_stale_onnxruntime(static_lib_dir, version) + version_dir = os.path.join(static_lib_dir, f"onnxruntime-linux-x64-static_lib-{version}-glibc2_28") + if os.path.isdir(version_dir) and has_static_archives(version_dir): + print(f" ✓ onnxruntime/static_lib ({version}) already extracted to {version_dir}") + return True + os.makedirs(static_lib_dir, exist_ok=True) + print(f" Extracting {os.path.basename(archive_path)} → {static_lib_dir}") + with zipfile.ZipFile(archive_path) as zf: + zf.extractall(static_lib_dir) + return True + def get_urls(use_china_mirrors=False) -> list[str | list[str]]: if use_china_mirrors: @@ -57,12 +116,16 @@ def get_urls(use_china_mirrors=False) -> list[str | list[str]]: # compatibility contract. "https://gh-proxy.com/https://github.com/browserbase/stagehand/releases/download/stagehand-server-v3/v3.7.2/stagehand-server-v3-linux-x64", "https://gh-proxy.com/https://github.com/browserbase/stagehand/releases/download/stagehand-server-v3/v3.7.2/stagehand-server-v3-linux-arm64", - # Native static libraries for Go build (pdfium, pdf_oxide, office_oxide) - # Used by build.sh's check_*_deps functions — pre-downloaded to avoid - # network access during CI. + # Native static libraries for Go build (pdfium, pdf_oxide, + # office_oxide, onnxruntime). Used by build.sh's check_*_deps + # functions — pre-downloaded to avoid network access during CI. ["https://gh-proxy.com/https://github.com/kognitos/pdfium-static/releases/download/chromium%2F7809/pdfium-linux-x64-static.tgz", "pdfium-linux-x64-static.tgz"], ["https://gh-proxy.com/https://github.com/yfedoseev/pdf_oxide/releases/download/v0.3.73/pdf_oxide-go-ffi-linux-amd64.tar.gz", "pdf_oxide-go-ffi-linux-amd64.tar.gz"], ["https://gh-proxy.com/https://github.com/yfedoseev/office_oxide/releases/download/v0.1.9/native-linux-x86_64.tar.gz", "office_oxide-linux-x86_64.tar.gz"], + [ + f"https://gh-proxy.com/https://github.com/csukuangfj/onnxruntime-libs/releases/download/v{ORT_VERSION}/onnxruntime-linux-x64-static_lib-{ORT_VERSION}-glibc2_28.zip", + f"onnxruntime-linux-x64-static_lib-{ORT_VERSION}-glibc2_28.zip", + ], ] else: return [ @@ -82,12 +145,16 @@ def get_urls(use_china_mirrors=False) -> list[str | list[str]]: # compatibility contract. "https://github.com/browserbase/stagehand/releases/download/stagehand-server-v3/v3.7.2/stagehand-server-v3-linux-x64", "https://github.com/browserbase/stagehand/releases/download/stagehand-server-v3/v3.7.2/stagehand-server-v3-linux-arm64", - # Native static libraries for Go build (pdfium, pdf_oxide, office_oxide) - # Used by build.sh's check_*_deps functions — pre-downloaded to avoid - # network access during CI. + # Native static libraries for Go build (pdfium, pdf_oxide, + # office_oxide, onnxruntime). Used by build.sh's check_*_deps + # functions — pre-downloaded to avoid network access during CI. ["https://github.com/kognitos/pdfium-static/releases/download/chromium%2F7809/pdfium-linux-x64-static.tgz", "pdfium-linux-x64-static.tgz"], ["https://github.com/yfedoseev/pdf_oxide/releases/download/v0.3.73/pdf_oxide-go-ffi-linux-amd64.tar.gz", "pdf_oxide-go-ffi-linux-amd64.tar.gz"], ["https://github.com/yfedoseev/office_oxide/releases/download/v0.1.9/native-linux-x86_64.tar.gz", "office_oxide-linux-x86_64.tar.gz"], + [ + f"https://github.com/csukuangfj/onnxruntime-libs/releases/download/v{ORT_VERSION}/onnxruntime-linux-x64-static_lib-{ORT_VERSION}-glibc2_28.zip", + f"onnxruntime-linux-x64-static_lib-{ORT_VERSION}-glibc2_28.zip", + ], ] @@ -139,12 +206,13 @@ if __name__ == "__main__": # Extract native static libraries to ~/ragflow-native-libs for Go build. # Ensures build.sh can find them without network access. native_deps_dir = os.path.expanduser("~/ragflow-native-libs") + import tarfile + extractions = [ ("pdfium-linux-x64-static.tgz", "pdfium-static"), ("pdf_oxide-go-ffi-linux-amd64.tar.gz", "pdf_oxide"), ("office_oxide-linux-x86_64.tar.gz", "office_oxide"), ] - import tarfile for archive, subdir in extractions: archive_path = os.path.join(os.getcwd(), archive) @@ -159,3 +227,33 @@ if __name__ == "__main__": print(f" Extracting {archive} → {target}") with tarfile.open(archive_path) as tf: tf.extractall(target) + + if not extract_onnxruntime( + os.path.join(native_deps_dir, "onnxruntime", "static_lib"), + os.path.join(os.getcwd(), f"onnxruntime-linux-x64-static_lib-{ORT_VERSION}-glibc2_28.zip"), + ORT_VERSION, + ): + # The archive was not downloaded or failed to extract, so no .a landed. + # Fail loud instead of exiting 0: build.sh's ORT guard would otherwise + # reject the build later with a less actionable message, and a missing + # .a left here is exactly the "silent green" this PR is meant to prevent. + print( + f" ERROR: ONNX Runtime static archives for {ORT_VERSION} were not " + f"extracted to {os.path.join(native_deps_dir, 'onnxruntime', 'static_lib')}. " + f"Check the download above; build.sh will refuse to link without them.", + file=sys.stderr, + ) + sys.exit(1) + + # ONNX Runtime is statically linked into the server binary, so there is no + # runtime .so to surface. Log where build.sh (ONNXRUNTIME_STATIC_PREFIX) will + # find the archives — the .a files live under + # ~/ragflow-native-libs/onnxruntime/static_lib. The in-process backend + # resolves OrtGetApiBase via dlopen(NULL); there is no dynamic .so fallback. + ort_static_dir = os.path.join(native_deps_dir, "onnxruntime", "static_lib") + ort_a_files = [os.path.join(root, f) for root, _, files in os.walk(ort_static_dir) for f in files if f.endswith(".a")] + if ort_a_files: + print(f" ✓ onnxruntime static archives ready: {len(ort_a_files)} .a under {ort_static_dir}") + else: + print(f" ERROR: ONNX Runtime .a files still missing under {ort_static_dir} after extraction; build.sh will refuse to link.", file=sys.stderr) + sys.exit(1) diff --git a/ragflow_deps/test_download_go_deps.py b/ragflow_deps/test_download_go_deps.py new file mode 100644 index 0000000000..6660797236 --- /dev/null +++ b/ragflow_deps/test_download_go_deps.py @@ -0,0 +1,71 @@ +# Tests for ragflow_deps/download_go_deps.py ONNX Runtime extraction. +# +# build.sh's build_go() fails fast when libonnxruntime.a is not linked, so these +# tests must guarantee the archive is really landed on disk — above all after an +# ORT version bump, where the zip's version-stamped top-level dir changes but +# static_lib/ still exists from the previous run. + +import os +import zipfile + +from download_go_deps import extract_onnxruntime, has_static_archives + + +def make_ort_zip(path, version): + """Build a zip shaped like the upstream ORT release: a version-stamped + top-level dir holding lib/libonnxruntime.a.""" + member = f"onnxruntime-linux-x64-static_lib-{version}-glibc2_28/lib/libonnxruntime.a" + with zipfile.ZipFile(path, "w") as zf: + zf.writestr(member, b"!\nort-payload") + return path + + +def test_extracts_on_first_run(tmp_path): + # static_lib/ does not exist yet: the plain first-run path. + static_lib = tmp_path / "onnxruntime" / "static_lib" + version = "1.23.2" + archive = make_ort_zip(tmp_path / f"onnxruntime-linux-x64-static_lib-{version}-glibc2_28.zip", version) + + assert extract_onnxruntime(str(static_lib), str(archive), version) is True + + version_dir = static_lib / f"onnxruntime-linux-x64-static_lib-{version}-glibc2_28" + assert version_dir.is_dir() + assert has_static_archives(str(version_dir)) + + +def test_extracts_after_version_bump_when_static_lib_exists(tmp_path): + """Regression: a static_lib/ left over from a previous version must not + suppress extraction of the bumped version. Pruning the stale dir leaves + static_lib/ in place; if extraction is then skipped no .a is ever landed and + build.sh's ORT guard rejects the build.""" + static_lib = tmp_path / "onnxruntime" / "static_lib" + static_lib.mkdir(parents=True) + stale = static_lib / "onnxruntime-linux-x64-static_lib-1.23.1-glibc2_28" + (stale / "lib").mkdir(parents=True) + (stale / "lib" / "libonnxruntime.a").write_bytes(b"old-ort") + + version = "1.23.2" + archive = make_ort_zip(tmp_path / f"onnxruntime-linux-x64-static_lib-{version}-glibc2_28.zip", version) + + assert extract_onnxruntime(str(static_lib), str(archive), version) is True + + assert not stale.exists(), "stale ORT version should be pruned" + version_dir = static_lib / f"onnxruntime-linux-x64-static_lib-{version}-glibc2_28" + assert version_dir.is_dir(), "bumped ORT version was silently not extracted" + assert has_static_archives(str(version_dir)), "bumped ORT version landed no .a" + + +def test_skips_when_archive_missing(tmp_path): + static_lib = tmp_path / "onnxruntime" / "static_lib" + assert extract_onnxruntime(str(static_lib), str(tmp_path / "missing.zip"), "1.23.2") is False + + +def test_idempotent_when_matching_version_already_present(tmp_path): + static_lib = tmp_path / "onnxruntime" / "static_lib" + version = "1.23.2" + archive = make_ort_zip(tmp_path / f"onnxruntime-linux-x64-static_lib-{version}-glibc2_28.zip", version) + + assert extract_onnxruntime(str(static_lib), str(archive), version) is True + before = sorted(os.listdir(static_lib)) + assert extract_onnxruntime(str(static_lib), str(archive), version) is True + assert sorted(os.listdir(static_lib)) == before