mirror of
https://github.com/infiniflow/ragflow.git
synced 2026-11-10 21:32:43 +08:00
fix(deepdoc): harden ORT build guard, OCR test contract (#19082)
This commit is contained in:
1 parent
61947f9eeb
commit
0d1338d63e
20 files changed
+627
-69
No files matched your search
@@ -396,6 +396,17 @@ jobs:
|
||||
./build.sh --test -- $PKGS
|
||||
fi
|
||||
|
||||
- name: Run ORT deps-download regression guard
|
||||
if: env.API_PROXY_SCHEME == 'go'
|
||||
# Pure-Python, local, needs no API keys or model downloads; guards the
|
||||
# ONNX Runtime extraction logic the in-process (Go) DeepDoc backend build
|
||||
# depends on (notably the version-bump path C2 hardened). Run it in
|
||||
# isolation so the test/testcases conftest (which requires
|
||||
# SILICONFLOW_API_KEY) cannot fail the collection.
|
||||
run: |
|
||||
set -euo pipefail
|
||||
uv sync --python 3.13 --group test --frozen
|
||||
uv run pytest ragflow_deps/test_download_go_deps.py -q
|
||||
|
||||
- name: Build ragflow:nightly
|
||||
run: |
|
||||
@@ -1016,6 +1027,17 @@ jobs:
|
||||
./build.sh --test -- $PKGS
|
||||
fi
|
||||
|
||||
- name: Run ORT deps-download regression guard
|
||||
if: env.API_PROXY_SCHEME == 'go'
|
||||
# Pure-Python, local, needs no API keys or model downloads; guards the
|
||||
# ONNX Runtime extraction logic the in-process (Go) DeepDoc backend build
|
||||
# depends on (notably the version-bump path C2 hardened). Run it in
|
||||
# isolation so the test/testcases conftest (which requires
|
||||
# SILICONFLOW_API_KEY) cannot fail the collection.
|
||||
run: |
|
||||
set -euo pipefail
|
||||
uv sync --python 3.13 --group test --frozen
|
||||
uv run pytest ragflow_deps/test_download_go_deps.py -q
|
||||
|
||||
- name: Build ragflow:nightly
|
||||
run: |
|
||||
|
||||
@@ -30,6 +30,16 @@ RUN --mount=type=bind,from=infiniflow/ragflow_deps:latest,source=/huggingface.co
|
||||
ln -s layout.onnx /ragflow/rag/res/deepdoc/layout.manual.onnx && \
|
||||
ln -s layout.onnx /ragflow/rag/res/deepdoc/layout.paper.onnx
|
||||
|
||||
# Copy the cl100k_base BPE table used by the Go tokenizer (tiktoken-go
|
||||
# cl100k_base). The deps image ships it at its root; the Go image previously
|
||||
# mounted only /huggingface.co and omitted this file, so NumTokensFromString
|
||||
# silently returned 0. localBpeLoader resolves it from <workdir>/ragflow_deps/,
|
||||
# so dropping it here makes the table load offline at startup (see
|
||||
# InitCL100KEncoder fail-fast guard).
|
||||
RUN --mount=type=bind,from=infiniflow/ragflow_deps:latest,source=/cl100k_base.tiktoken,target=/tmp/cl100k_base.tiktoken \
|
||||
mkdir -p /ragflow/ragflow_deps && \
|
||||
cp /tmp/cl100k_base.tiktoken /ragflow/ragflow_deps/cl100k_base.tiktoken
|
||||
|
||||
ENV DEBIAN_FRONTEND=noninteractive
|
||||
|
||||
# Setup apt
|
||||
|
||||
@@ -41,7 +41,8 @@ PDF_OXIDE_VERSION="0.3.73"
|
||||
# (Go) DeepDoc backend. libonnxruntime*.a is linked into the server binary
|
||||
# (--whole-archive + --export-dynamic); OrtGetApiBase is then resolved via
|
||||
# dlopen(self), so no libonnxruntime.so is needed at runtime. Downloaded by
|
||||
# ragflow_deps/download_deps.py into onnxruntime/static_lib.
|
||||
# ragflow_deps/download_go_deps.py (and ragflow_deps/download_deps.py) into
|
||||
# onnxruntime/static_lib.
|
||||
ONNXRUNTIME_STATIC_PREFIX="${HOME}/ragflow-native-libs/onnxruntime/static_lib"
|
||||
|
||||
# Copy a dependency from the system pre-seed directory to the user cache.
|
||||
@@ -370,6 +371,33 @@ build_go() {
|
||||
|
||||
setup_cgo_env
|
||||
|
||||
# The in-process (Go) DeepDoc backend is statically linked against ONNX
|
||||
# Runtime (--whole-archive + -Wl,--export-dynamic, see setup_cgo_env). The
|
||||
# forked onnxruntime_go binding resolves OrtGetApiBase only at RUNTIME via
|
||||
# dlopen(NULL), so a missing ORT archive still lets `go build` SUCCEED and
|
||||
# yields a server binary that FAILS AT STARTUP with a fatal
|
||||
# "no in-process DeepDoc backend serving". That is exactly the breakage
|
||||
# colleagues hit when ORT was not present at build time and setup_cgo_env
|
||||
# silently skipped it. Fail the build HERE instead of deferring the breakage
|
||||
# to runtime: a server binary without ORT is unusable, not a degraded one.
|
||||
if ! printf '%s' "$CGO_LDFLAGS" | grep -q 'libonnxruntime'; then
|
||||
echo -e "${RED}Error: ONNX Runtime static libraries are not linked.${NC}" >&2
|
||||
echo " The in-process DeepDoc backend requires libonnxruntime.a to be" >&2
|
||||
echo " statically linked into ragflow_server (--whole-archive +" >&2
|
||||
echo " -Wl,--export-dynamic). Without it the binary compiles but dies" >&2
|
||||
echo " at startup with a fatal 'no in-process DeepDoc backend serving'." >&2
|
||||
echo " Fetch the static libs with:" >&2
|
||||
echo " uv run python3 ragflow_deps/download_go_deps.py" >&2
|
||||
echo " or pre-seed them at /opt/ragflow-native-libs/onnxruntime (CI image)." >&2
|
||||
echo " This production binary must statically include ORT — there is no" >&2
|
||||
echo " ORT-free build path. ORT ends up unlinked only via one of:" >&2
|
||||
echo " - the ORT static_lib dir was never seeded: run" >&2
|
||||
echo " 'uv run python3 ragflow_deps/download_go_deps.py', or pre-seed" >&2
|
||||
echo " /opt/ragflow-native-libs/onnxruntime as the CI image does;" >&2
|
||||
echo " - setup_cgo_env did not add libonnxruntime to CGO_LDFLAGS." >&2
|
||||
return 1
|
||||
fi
|
||||
|
||||
local strip_flags=()
|
||||
[ -n "$STRIP_SYMBOLS" ] && strip_flags=(-ldflags="-s -w")
|
||||
|
||||
|
||||
@@ -568,6 +568,12 @@ func runIngestor(ctx context.Context, cancel context.CancelFunc, args *serverArg
|
||||
}
|
||||
defer tokenizer.Close()
|
||||
|
||||
// Fail fast if the cl100k_base BPE table is missing: without it
|
||||
// NumTokensFromString silently returns 0, corrupting every token budget.
|
||||
if err := tokenizer.InitCL100KEncoder(); err != nil {
|
||||
common.Fatal("Failed to initialize cl100k_base tokenizer", zap.Error(err))
|
||||
}
|
||||
|
||||
// The dataset-level post-processing consumer cluster (§11) is owned and run by
|
||||
// the Ingestor: it is started inside ingestor.Start() and joined inside
|
||||
// ingestor.Stop(), so its lifecycle matches the ingestor. The configured
|
||||
@@ -712,6 +718,12 @@ func runAPI(ctx context.Context, args *serverArgs) error {
|
||||
}
|
||||
defer tokenizer.Close()
|
||||
|
||||
// Fail fast if the cl100k_base BPE table is missing: without it
|
||||
// NumTokensFromString silently returns 0, corrupting every token budget.
|
||||
if err := tokenizer.InitCL100KEncoder(); err != nil {
|
||||
common.Fatal("Failed to initialize cl100k_base tokenizer", zap.Error(err))
|
||||
}
|
||||
|
||||
// Initialize global QueryBuilder using tokenizer's DictPath
|
||||
// This ensures the Synonym uses the same wordnet directory as tokenizer
|
||||
if err := nlp.InitQueryBuilderFromTokenizer(tokenizerCfg.DictPath); err != nil {
|
||||
|
||||
@@ -254,17 +254,23 @@ func HasModelFiles(dir string) bool {
|
||||
}
|
||||
|
||||
// DeepDocORTVersion is the onnxruntime native release the in-process (Go)
|
||||
// DeepDoc backend is built and tested against (e.g. "1.23.2"). It is the
|
||||
// single source for the download URL and extracted dir name used across Go
|
||||
// and Python. The Go binding (github.com/yalue/onnxruntime_go, forked to
|
||||
// DeepDoc backend is built and tested against (e.g. "1.23.2"). It is ONE OF
|
||||
// THREE raw version declarations that must stay equal (the other two are
|
||||
// ORT_VERSION in ragflow_deps/download_go_deps.py and ragflow_deps/download_deps.py)
|
||||
// — NOT a single source of truth. The download URL and extracted dir name are
|
||||
// built from those ORT_VERSION constants, not from this one. The Go binding
|
||||
// (github.com/yalue/onnxruntime_go, forked to
|
||||
// github.com/xugangqiang/onnxruntime_go) and the pip onnxruntime== pin must
|
||||
// track this MINOR version: the binding uses its own release numbering
|
||||
// (v1.23.0 <-> ORT 1.23.x) but is ABI-compatible with this native release on
|
||||
// the same minor line. ONNX Runtime is linked statically (libonnxruntime.a),
|
||||
// so there is no .so / SONAME at runtime.
|
||||
//
|
||||
// To bump ORT: update DeepDocORTVersion (Go) AND ORT_VERSION in
|
||||
// ragflow_deps/download_deps.py AND the onnxruntime/onnxruntime-gpu pins in
|
||||
// pyproject.toml + .github/workflows/deepdoc-drift.yml, and refresh the
|
||||
// onnxruntime_go binding minor in go.mod.
|
||||
// To bump ORT, ALL of the following must change together (drift breaks the
|
||||
// static link or the runtime OrtGetApiBase lookup):
|
||||
// - DeepDocORTVersion (here, Go) AND ORT_VERSION in BOTH
|
||||
// ragflow_deps/download_go_deps.py and ragflow_deps/download_deps.py;
|
||||
// - the onnxruntime== pin in pyproject.toml and the onnxruntime /
|
||||
// onnxruntime-gpu pins in .github/workflows/deepdoc-drift.yml;
|
||||
// - the onnxruntime_go binding minor in go.mod.
|
||||
const DeepDocORTVersion = "1.23.2"
|
||||
@@ -36,7 +36,7 @@ type OCRRecResult struct {
|
||||
}
|
||||
|
||||
// RunOCRRec recognizes a single cropped text-line image. It is equivalent to a
|
||||
// one-line batch (see RunOCRRecBatch): the line is resized against its own
|
||||
// one-line batch (see RunOCRRecBatchReal): the line is resized against its own
|
||||
// wh_ratio floored at 320/48, never the batch-max, so a caller recognizing
|
||||
// lines independently gets the same result as a standalone Python
|
||||
// TextRecognizer call.
|
||||
@@ -55,28 +55,16 @@ func RunOCRRec(ctx context.Context, modelDir string, img *Image) (OCRRecResult,
|
||||
return recognizeLine(ctx, modelDir, img, maxWhRatio, chars)
|
||||
}
|
||||
|
||||
// RunOCRRecBatch recognizes a batch of cropped text-line images the way
|
||||
// deepdoc's TextRecognizer.__call__ does: every line in the batch is resized to
|
||||
// the SAME width derived from the batch's maximum wh_ratio (floored at 320/48),
|
||||
// so a narrow line inside a wide batch is widened to the batch max rather than
|
||||
// its own ratio. This makes Go match production batch semantics, where a page's
|
||||
// lines are recognized together. Lines are recognized independently after the
|
||||
// shared resize, so order is preserved and results are deterministic per batch.
|
||||
//
|
||||
// DEPRECATED semantic-shape note: this variant still runs one ONNX Run PER
|
||||
// line (recMaxBatch=1). Prefer RunOCRRecBatchReal, which concatenates every
|
||||
// line's preprocessed blob into a single {N,3,48,imgW} tensor and performs ONE
|
||||
// ONNX Run — identical numerically (each line uses the same shared batch width)
|
||||
// but with the real throughput benefit of batched matmul. RunOCRRecBatchReal is
|
||||
// the production path.
|
||||
// RunOCRRecBatchReal recognizes a batch of cropped text-line images with a
|
||||
// SINGLE ONNX Run, mirroring deepdoc's TextRecognizer.__call__ exactly: every
|
||||
// line is resized to the batch's shared width (imgW = 48*max_wh_ratio, floored
|
||||
// at 320/48) and zero-padded on the right, all blobs are concatenated into one
|
||||
// {N,3,48,imgW} tensor, and the model runs once. The output is split back into
|
||||
// per-line sequences and CTC-decoded in order, so the result is numerically
|
||||
// identical to calling RunOCRRec on each line (each line sees the same shared
|
||||
// batch width), but amortized over one forward pass instead of N.
|
||||
// SINGLE ONNX Run, mirroring deepdoc's TextRecognizer.__call__: each line is
|
||||
// resized to its own proportional width (recH * that line's wh_ratio), capped
|
||||
// by the batch-shared imgW (imgW = recH * max_wh_ratio, with max_wh_ratio
|
||||
// floored at 320/48), and zero-padded on the right out to imgW; all blobs are
|
||||
// concatenated into one {N,3,48,imgW} tensor, and the model runs once. The
|
||||
// output is split back into per-line sequences and CTC-decoded in order, so
|
||||
// the result is numerically identical to calling RunOCRRec on each line (each
|
||||
// line sees the same shared batch width), but amortized over one forward pass
|
||||
// instead of N.
|
||||
//
|
||||
// The shared batch width means a line is resized against the batch max wh_ratio,
|
||||
// not its own — exactly what deepdoc does inside a batch. A standalone call to
|
||||
|
||||
@@ -2,8 +2,9 @@
|
||||
|
||||
// This file is compiled only with -tags fetch_testdata. Its init() downloads
|
||||
// and symlinks the external DeepDoc testdata for this package (pkg = "native")
|
||||
// so the consuming tests can run. The download is best-effort: if it fails, the
|
||||
// guard in testdata_skip.go skips the tests instead of failing.
|
||||
// so the consuming tests can run. A fetch failure is fatal under this build tag:
|
||||
// it sets testdataFetchFailed so TestMain fails the package instead of silently
|
||||
// skipping — a missing fixture must never be a green CI run.
|
||||
package native
|
||||
|
||||
import (
|
||||
@@ -17,6 +18,7 @@ import (
|
||||
func init() {
|
||||
if err := fetchTestdata(); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "fetch_deepdoc_testdata: not fetched (%v)\n", err)
|
||||
testdataFetchFailed = true
|
||||
return
|
||||
}
|
||||
testdataFetchAttempted = true
|
||||
|
||||
@@ -8,3 +8,11 @@ package native
|
||||
// imports native, and _test.go files are excluded from dependency builds). In the
|
||||
// default build it stays false, so TestMain skips the package.
|
||||
var testdataFetchAttempted bool
|
||||
|
||||
// testdataFetchFailed is set to true by testdata_fetch.go (compiled only with
|
||||
// -tags fetch_testdata) when the external testdata download fails. It is
|
||||
// declared here (always-compiled) so TestMain can read it. Under the
|
||||
// fetch_testdata build tag a failed fetch must FAIL the package rather than
|
||||
// silently skip — a missing fixture must never produce a green CI run. In the
|
||||
// default build this stays false, so the absence of fixtures is a benign skip.
|
||||
var testdataFetchFailed bool
|
||||
@@ -1,9 +1,10 @@
|
||||
// This file is always compiled (no build constraint) and provides the package
|
||||
// TestMain that guards the whole test binary behind the presence of the
|
||||
// package-level testdata directory. When the external testdata has not been
|
||||
// fetched (e.g. -tags fetch_testdata was not supplied, or the download failed),
|
||||
// the tests are skipped rather than failed — keeping the default `go test ./...`
|
||||
// and `go test -tags cgo` runs green without the assets.
|
||||
// fetched (e.g. -tags fetch_testdata was not supplied), the tests are skipped
|
||||
// rather than failed — keeping the default `go test ./...` and `go test -tags cgo`
|
||||
// runs green without the assets. A fetch ATTEMPTED but FAILED (under
|
||||
// -tags fetch_testdata) is fatal: a missing fixture must never be a green run.
|
||||
package native
|
||||
|
||||
import (
|
||||
@@ -18,6 +19,10 @@ import (
|
||||
// assets — which also makes a stale/partial leftover testdata directory on disk
|
||||
// harmless: we skip regardless of what (if anything) is present.
|
||||
func TestMain(m *testing.M) {
|
||||
if testdataFetchFailed {
|
||||
fmt.Fprintln(os.Stderr, "FAIL: DeepDoc native testdata fetch failed (see the fetch_deepdoc_testdata error above). CI must not pass with missing fixtures; run with -tags fetch_testdata and fix the download.")
|
||||
os.Exit(1)
|
||||
}
|
||||
if !testdataFetchAttempted {
|
||||
fmt.Println("SKIP: DeepDoc native testdata not fetched; run with -tags fetch_testdata to fetch and run it")
|
||||
os.Exit(0)
|
||||
|
||||
@@ -3,9 +3,10 @@
|
||||
// This file is compiled only with -tags fetch_testdata. Its init() downloads
|
||||
// and symlinks the external DeepDoc testdata. The native_analyzer tests read
|
||||
// their fixtures from the sibling internal/deepdoc/native/testdata directory
|
||||
// (via a relative path), so we fetch with pkg = "native". The download is
|
||||
// best-effort: if it fails, the guard in testdata_skip.go skips the tests
|
||||
// instead of failing.
|
||||
// (via a relative path), so we fetch with pkg = "native". A fetch failure is
|
||||
// fatal under this build tag: it sets testdataFetchFailed so TestMain fails the
|
||||
// package instead of silently skipping — a missing fixture must never be a green
|
||||
// CI run.
|
||||
package infnative
|
||||
|
||||
import (
|
||||
@@ -19,6 +20,7 @@ import (
|
||||
func init() {
|
||||
if err := fetchTestdata(); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "fetch_deepdoc_testdata: not fetched (%v)\n", err)
|
||||
testdataFetchFailed = true
|
||||
return
|
||||
}
|
||||
testdataFetchAttempted = true
|
||||
|
||||
@@ -7,3 +7,11 @@ package infnative
|
||||
// this package is built as a dependency of another package's test binary. In the
|
||||
// default build it stays false, so TestMain skips the package.
|
||||
var testdataFetchAttempted bool
|
||||
|
||||
// testdataFetchFailed is set to true by testdata_fetch.go (compiled only with
|
||||
// -tags fetch_testdata) when the external testdata download fails. Declared here
|
||||
// (always-compiled) so TestMain can read it. Under the fetch_testdata build tag a
|
||||
// failed fetch must FAIL the package rather than silently skip — a missing
|
||||
// fixture must never produce a green CI run. In the default build this stays
|
||||
// false, so the absence of fixtures is a benign skip.
|
||||
var testdataFetchFailed bool
|
||||
@@ -3,9 +3,10 @@
|
||||
// DeepDoc testdata directory. The native_analyzer tests read their fixtures
|
||||
// from the sibling internal/deepdoc/native/testdata directory (via a relative
|
||||
// path), so that is what we check here. When the external testdata has not been
|
||||
// fetched (e.g. -tags fetch_testdata was not supplied, or the download failed),
|
||||
// the tests are skipped rather than failed — keeping the default `go test ./...`
|
||||
// and `go test -tags cgo` runs green without the assets.
|
||||
// fetched (e.g. -tags fetch_testdata was not supplied), the tests are skipped
|
||||
// rather than failed — keeping the default `go test ./...` and `go test -tags cgo`
|
||||
// runs green without the assets. A fetch ATTEMPTED but FAILED (under
|
||||
// -tags fetch_testdata) is fatal: a missing fixture must never be a green run.
|
||||
package infnative
|
||||
|
||||
import (
|
||||
@@ -20,6 +21,10 @@ import (
|
||||
// assets — which also makes a stale/partial leftover testdata directory on disk
|
||||
// harmless: we skip regardless of what (if anything) is present.
|
||||
func TestMain(m *testing.M) {
|
||||
if testdataFetchFailed {
|
||||
fmt.Fprintln(os.Stderr, "FAIL: DeepDoc native testdata fetch failed (see the fetch_deepdoc_testdata error above). CI must not pass with missing fixtures; run with -tags fetch_testdata and fix the download.")
|
||||
os.Exit(1)
|
||||
}
|
||||
if !testdataFetchAttempted {
|
||||
fmt.Println("SKIP: DeepDoc native testdata not fetched; run with -tags fetch_testdata to fetch and run it")
|
||||
os.Exit(0)
|
||||
|
||||
@@ -105,24 +105,37 @@ func (p *Parser) ocrDetectAndRecognize(ctx context.Context, pageImg image.Image,
|
||||
allTexts := make([][]pdf.OCRText, len(cropAcc))
|
||||
if p.docSupportsBatchOCR(doc) {
|
||||
batch, berr := p.inferOCRRecognizeBatch(ctx, doc, cropAcc)
|
||||
if berr != nil {
|
||||
slog.Warn(logLabel+" OCR batch recognize failed", "page", pageNum, "err", berr)
|
||||
switch {
|
||||
case berr != nil:
|
||||
// A batch error must not abort the whole page: the canonical
|
||||
// per-crop path below still produces correct results.
|
||||
slog.Warn(logLabel+" OCR batch recognize failed; falling back to per-crop", "page", pageNum, "err", berr)
|
||||
case len(batch) != len(cropAcc):
|
||||
// Defensive: a count mismatch (or a nil result) would corrupt the
|
||||
// per-box indexing further down. Fall back to per-crop instead of
|
||||
// indexing out of range.
|
||||
slog.Warn(logLabel+" OCR batch recognize returned unexpected count; falling back to per-crop", "page", pageNum, "got", len(batch), "want", len(cropAcc))
|
||||
default:
|
||||
allTexts = batch
|
||||
}
|
||||
}
|
||||
// Fill any crop the batch path did not cover (unsupported doc, batch error,
|
||||
// or count mismatch) with the per-crop canonical path. Stamping the
|
||||
// detect-box index lets a replay DocAnalyzer route the recognition back to
|
||||
// the Python-dumped text for the same box; the production analyzer ignores
|
||||
// the key.
|
||||
for ci := range cropAcc {
|
||||
if allTexts[ci] != nil {
|
||||
continue
|
||||
}
|
||||
c := cropAcc[ci]
|
||||
recCtx := context.WithValue(ctx, ocrBoxIdxCtxKey, cropBoxIdx[ci])
|
||||
texts, rerr := p.ocrRecognizeWithRotation(recCtx, doc, c)
|
||||
if rerr != nil {
|
||||
slog.Warn(logLabel+" OCR recognize failed", "page", pageNum, "err", rerr)
|
||||
return nil
|
||||
}
|
||||
allTexts = batch
|
||||
} else {
|
||||
for ci, c := range cropAcc {
|
||||
// Stamp the detect-box index so a replay DocAnalyzer routes this
|
||||
// recognition back to the Python-dumped text for the same box. The
|
||||
// production analyzer ignores the key.
|
||||
recCtx := context.WithValue(ctx, ocrBoxIdxCtxKey, cropBoxIdx[ci])
|
||||
texts, rerr := p.ocrRecognizeWithRotation(recCtx, doc, c)
|
||||
if rerr != nil {
|
||||
slog.Warn(logLabel+" OCR recognize failed", "page", pageNum, "err", rerr)
|
||||
return nil
|
||||
}
|
||||
allTexts[ci] = texts
|
||||
}
|
||||
allTexts[ci] = texts
|
||||
}
|
||||
|
||||
// Per box: pick the highest-confidence candidate (layer-2 winner) and
|
||||
|
||||
+68
-2
@@ -39,14 +39,33 @@ go version
|
||||
### 1.4 Install dependent library
|
||||
```shell
|
||||
sudo apt install libpcre2-dev
|
||||
# Native libs for the Go build (office_oxide, pdfium, pdf_oxide, onnxruntime).
|
||||
# download_go_deps.py is the Go-end script: it now fetches the ONNX Runtime
|
||||
# static lib too, so this single command covers every native dependency the
|
||||
# Go server binary needs (incl. the in-process DeepDoc backend).
|
||||
python3 ragflow_deps/download_go_deps.py
|
||||
# Shared (Go + Python) deps. Retains ONNX Runtime for the ragflow_deps image /
|
||||
# backward-compat; also pulls Python-side artifacts (models, nltk, tika, ...).
|
||||
uv run python3 ragflow_deps/download_deps.py
|
||||
```
|
||||
|
||||
> **Note**: If you use IDEs like GoLand to run/debug directly (via Run/Debug buttons), or run `go build` / `go run` from command line, set these CGO environment variables:
|
||||
>
|
||||
> ```bash
|
||||
> RAGFLOW_DEPS="${HOME}/ragflow-native-libs" # created by uv run ragflow_deps/download_deps.py
|
||||
> RAGFLOW_DEPS="${HOME}/ragflow-native-libs" # created by download_go_deps.py + download_deps.py
|
||||
> PLATFORM="linux_amd64" # or darwin_amd64, linux_arm64, darwin_arm64
|
||||
> # NOTE: the ONNX Runtime static lib fetched by download_go_deps.py is
|
||||
> # linux-x64 ONLY (onnxruntime-linux-x64-static_lib-*). On darwin_* / non-amd64
|
||||
> # PLATFORM values the production DeepDoc backend cannot be linked, so those
|
||||
> # PLATFORM examples cover office_oxide/pdfium/pdf_oxide only — ORT is a
|
||||
> # Linux-amd64 link dependency here.
|
||||
>
|
||||
> # Resolve the version-stamped ORT archive path FIRST, in its own unquoted
|
||||
> # assignment: the shell does not expand `*` inside the double-quoted
|
||||
> # CGO_LDFLAGS below, and `--whole-archive` must stay a separate argument from
|
||||
> # the archive path. If this lists more than one match, delete the stale
|
||||
> # version dir — build.sh refuses to link two ORT versions.
|
||||
> ORT_A="$(ls ${RAGFLOW_DEPS}/onnxruntime/static_lib/*/lib/libonnxruntime.a)"
|
||||
>
|
||||
> export CGO_CFLAGS="-I${RAGFLOW_DEPS}/office_oxide/include/office_oxide_c"
|
||||
> export CGO_LDFLAGS="\
|
||||
@@ -55,11 +74,29 @@ python3 ragflow_deps/download_go_deps.py
|
||||
> ${RAGFLOW_DEPS}/pdfium-static/lib/libc++.a \
|
||||
> ${RAGFLOW_DEPS}/pdfium-static/lib/libc++abi.a \
|
||||
> ${RAGFLOW_DEPS}/pdf_oxide/lib/${PLATFORM}/libpdf_oxide.a \
|
||||
> -Wl,--export-dynamic -Wl,--whole-archive ${ORT_A} -Wl,--no-whole-archive -lstdc++ \
|
||||
> -fuse-ld=lld \
|
||||
> -lm -lpthread -ldl -lrt -lgcc_s -lutil -lc"
|
||||
> ```
|
||||
>
|
||||
> All three native libraries are statically linked — no `LD_LIBRARY_PATH` or `-Wl,-rpath` needed.
|
||||
> All four native libraries are statically linked — no `LD_LIBRARY_PATH` or `-Wl,-rpath` needed.
|
||||
>
|
||||
> **ONNX Runtime is mandatory for the production binary.** The in-process (Go)
|
||||
> DeepDoc backend is statically linked against `libonnxruntime.a` via
|
||||
> `--whole-archive -Wl,--export-dynamic`, and `OrtGetApiBase` is resolved at
|
||||
> runtime through `dlopen(NULL)`. The forked `onnxruntime_go` binding only
|
||||
> needs `-ldl` to *compile*, so a binary built **without** ORT links
|
||||
> successfully but dies at startup with:
|
||||
> `Error looking up OrtGetApiBase in statically-linked ONNX Runtime` → fatal
|
||||
> `no in-process DeepDoc backend serving`.
|
||||
> Since `build.sh` (`build_go`) now **fails fast** when ORT is absent from
|
||||
> `CGO_LDFLAGS`, this breakage surfaces at build time instead of at runtime. If
|
||||
> you see `Error: ONNX Runtime static libraries are not linked`, run
|
||||
> `uv run python3 ragflow_deps/download_go_deps.py` (or pre-seed
|
||||
> `/opt/ragflow-native-libs/onnxruntime` as the CI runner image does). There is
|
||||
> no ORT-free production build path — if ORT is absent the binary fails at
|
||||
> startup, so the remedy is always to seed the static lib above, never to build
|
||||
> without it.
|
||||
|
||||
|
||||
### 1.5 Build RAGFlow
|
||||
@@ -77,6 +114,35 @@ python3 ragflow_deps/download_go_deps.py
|
||||
./build.sh -s --go
|
||||
```
|
||||
|
||||
### 1.6 In-process (Go) DeepDoc backend
|
||||
|
||||
The in-process DeepDoc backend is statically linked against ONNX Runtime
|
||||
(see §1.4). After a successful `./build.sh -s --go`, the `ragflow_server`
|
||||
binary carries it and registers the backend at startup.
|
||||
|
||||
- **Confirm it is serving** — the server logs, at startup:
|
||||
`in-process DeepDoc backend registered (production backend)`
|
||||
If you instead see a fatal `no in-process DeepDoc backend serving`, ORT was
|
||||
not linked into the binary. Re-run `uv run python3 ragflow_deps/download_go_deps.py`
|
||||
and rebuild (§1.4 explains why; `build.sh` fails fast with
|
||||
`Error: ONNX Runtime static libraries are not linked` before this happens).
|
||||
|
||||
- **Run the binary directly (local dev)** — `./bin/ragflow_server --api`
|
||||
(start `--admin` first, see §2) launches the Go server and registers the
|
||||
backend. No extra environment variable is required.
|
||||
|
||||
- **Run the Go Docker image** — the container entrypoint only starts the Go
|
||||
server (`bin/ragflow_server --api/--ingestor/--admin`) when
|
||||
`API_PROXY_SCHEME` is `go` or `hybrid`. With the variable unset (or `python`)
|
||||
the Go server does NOT start, so the in-process backend is absent and the
|
||||
image looks like a Python-only build. Start it with:
|
||||
```bash
|
||||
docker run -e API_PROXY_SCHEME=go infiniflow/ragflow:go-test-1
|
||||
```
|
||||
|
||||
- ORT is resolved via `dlopen(NULL)` at runtime, so no `LD_LIBRARY_PATH` /
|
||||
`-rpath` is needed and no `libonnxruntime.so` ships with the image.
|
||||
|
||||
## 2. Start RAGFlow
|
||||
|
||||
- Start dependencies
|
||||
|
||||
@@ -138,7 +138,7 @@ func bpeCandidatePaths(bpeURL string) []string {
|
||||
}
|
||||
}
|
||||
|
||||
for _, root := range searchRoots() {
|
||||
for _, root := range bpeSearchRoots() {
|
||||
// Same layout the Dockerfile creates: the table sits in the
|
||||
// installation root under its sha1 name.
|
||||
add(filepath.Join(root, cacheName))
|
||||
@@ -172,6 +172,25 @@ func searchRoots() []string {
|
||||
return roots
|
||||
}
|
||||
|
||||
// bpeSearchRoots is the directory set the loader walks for the table. It
|
||||
// defaults to searchRoots (working + executable dirs and their ancestors) so
|
||||
// the production image finds the table dropped into the install root. Tests
|
||||
// override it with SetBpeSearchRootsForTest to isolate from the host
|
||||
// filesystem — without that, a tiktoken cache leaked into /tmp (an ancestor of
|
||||
// any temp working dir) would be picked up and make "missing table" tests
|
||||
// non-deterministic.
|
||||
var bpeSearchRoots = searchRoots
|
||||
|
||||
// SetBpeSearchRootsForTest replaces the directory set the loader searches.
|
||||
// Pass nil to restore the default. Test-only; it mutates package state.
|
||||
func SetBpeSearchRootsForTest(roots []string) {
|
||||
if roots == nil {
|
||||
bpeSearchRoots = searchRoots
|
||||
return
|
||||
}
|
||||
bpeSearchRoots = func() []string { return roots }
|
||||
}
|
||||
|
||||
func startingDirs() []string {
|
||||
var dirs []string
|
||||
if wd, err := os.Getwd(); err == nil {
|
||||
|
||||
@@ -41,8 +41,12 @@ func writeBpeTable(t *testing.T, path string, marker int) {
|
||||
}
|
||||
// Register this synthetic table's digest so the SHA-1 integrity gate in
|
||||
// bpe_loader.go accepts it. This also exercises the verification path
|
||||
// instead of disabling it.
|
||||
// instead of disabling it. Restore the prior value afterwards so the
|
||||
// mutation does not leak into sibling tests (which would otherwise read a
|
||||
// stale digest for the same URL).
|
||||
prev := expectedBpeHashes[testBpeURL]
|
||||
expectedBpeHashes[testBpeURL] = fmt.Sprintf("%x", sha1.Sum([]byte(line)))
|
||||
t.Cleanup(func() { expectedBpeHashes[testBpeURL] = prev })
|
||||
}
|
||||
|
||||
func cacheFileName(url string) string {
|
||||
@@ -52,13 +56,17 @@ func cacheFileName(url string) string {
|
||||
// isolate moves the process into an empty directory and clears both cache
|
||||
// environment variables, so a test sees only the files it creates itself.
|
||||
// Without this the real repository checkout — which does ship the table — would
|
||||
// satisfy every lookup and hide ordering bugs.
|
||||
// satisfy every lookup and hide ordering bugs. It also scopes the loader's
|
||||
// search roots to this directory so a tiktoken cache leaked into an ancestor
|
||||
// (e.g. /tmp) can't be picked up and make "missing table" tests flaky.
|
||||
func isolate(t *testing.T) string {
|
||||
t.Helper()
|
||||
dir := t.TempDir()
|
||||
t.Setenv("TIKTOKEN_CACHE_DIR", "")
|
||||
t.Setenv("DATA_GYM_CACHE_DIR", "")
|
||||
t.Chdir(dir)
|
||||
SetBpeSearchRootsForTest([]string{dir})
|
||||
t.Cleanup(func() { SetBpeSearchRootsForTest(nil) })
|
||||
return dir
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,147 @@
|
||||
//
|
||||
// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//
|
||||
|
||||
package tokenizer
|
||||
|
||||
import (
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// TestInitCL100KEncoder_FailFast pins the contract that InitCL100KEncoder fails
|
||||
// fast on a missing cl100k_base table, and — the part the regression actually
|
||||
// guards — that a PRESENT table yields a working encoder (NumTokensFromString > 0)
|
||||
// rather than a silent 0.
|
||||
//
|
||||
// Background: NumTokensFromString swallows the loader error and returns 0, so a
|
||||
// Go image that forgot to bake cl100k_base.tiktoken silently zeroed every token
|
||||
// count while content_ltks (a separate C++ tokenizer) kept working. The fix is
|
||||
// to fail fast at startup via InitCL100KEncoder; this test must exercise BOTH
|
||||
// branches, otherwise it only re-asserts "missing → error" and leaves the
|
||||
// "present → counts" behaviour — the thing that regressed — uncovered.
|
||||
//
|
||||
// Both branches reset the encoder cache and scope bpeSearchRoots to a temp dir,
|
||||
// so neither depends on the host (no leaked tiktoken cache in an ancestor, no
|
||||
// success cached by a sibling test, no order dependence). "absent" runs first so
|
||||
// that the table "present" later copies under /tmp is not discoverable by it via
|
||||
// searchRoots() walking up to the shared /tmp ancestor.
|
||||
func TestInitCL100KEncoder_FailFast(t *testing.T) {
|
||||
// fail-fast: an empty scoped dir must surface a hard error, not a silent 0.
|
||||
t.Run("absent", func(t *testing.T) {
|
||||
resetCL100KEncoderForTest()
|
||||
dir := t.TempDir()
|
||||
t.Setenv("TIKTOKEN_CACHE_DIR", "")
|
||||
t.Setenv("DATA_GYM_CACHE_DIR", "")
|
||||
SetBpeSearchRootsForTest([]string{dir})
|
||||
t.Cleanup(func() { SetBpeSearchRootsForTest(nil) })
|
||||
|
||||
err := InitCL100KEncoder()
|
||||
if err == nil {
|
||||
t.Fatalf("InitCL100KEncoder returned nil with no table present; expected a fail-fast error")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "cl100k") {
|
||||
t.Fatalf("InitCL100KEncoder error does not mention cl100k: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
// present: copy a real table into an isolated dir (via TIKTOKEN_CACHE_DIR so
|
||||
// the loader finds it without walking ancestors) and require that startup
|
||||
// succeeds AND the encoder actually counts tokens (not just "loads").
|
||||
t.Run("present", func(t *testing.T) {
|
||||
src := findRealBpeTable(t)
|
||||
if src == "" {
|
||||
t.Skip("no cl100k_base.tiktoken on disk to exercise the present branch")
|
||||
}
|
||||
resetCL100KEncoderForTest()
|
||||
dir := t.TempDir()
|
||||
t.Setenv("TIKTOKEN_CACHE_DIR", dir)
|
||||
t.Setenv("DATA_GYM_CACHE_DIR", "")
|
||||
dst := filepath.Join(dir, cacheFileName(testBpeURL))
|
||||
copyFile(t, src, dst)
|
||||
SetBpeSearchRootsForTest([]string{dir})
|
||||
t.Cleanup(func() { SetBpeSearchRootsForTest(nil) })
|
||||
|
||||
if err := InitCL100KEncoder(); err != nil {
|
||||
t.Fatalf("InitCL100KEncoder with table present: unexpected error: %v", err)
|
||||
}
|
||||
if got := NumTokensFromString("hello world"); got <= 0 {
|
||||
t.Fatalf("InitCL100KEncoder succeeded but NumTokensFromString(%q) = %d, want > 0 (silent-zero regression)", "hello world", got)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// findRealBpeTable returns a path to a cl100k_base.tiktoken on disk, or "" if
|
||||
// none is available. The loader's integrity gate already knows this table's
|
||||
// digest (expectedBpeHashes), so copying it into a scoped dir is enough.
|
||||
func findRealBpeTable(t *testing.T) string {
|
||||
t.Helper()
|
||||
// 1. Explicit cache dirs (tiktoken-go / data-gym layouts).
|
||||
for _, env := range []string{"TIKTOKEN_CACHE_DIR", "DATA_GYM_CACHE_DIR"} {
|
||||
if dir := strings.TrimSpace(os.Getenv(env)); dir != "" {
|
||||
for _, name := range []string{cacheFileName(testBpeURL), "cl100k_base.tiktoken"} {
|
||||
if p := filepath.Join(dir, name); fileExists(p) {
|
||||
return p
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
// 2. repo-local ragflow_deps/cl100k_base.tiktoken (provisioned by download
|
||||
// scripts / CI), walking up from the package dir.
|
||||
pkgDir, err := os.Getwd()
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
for dir := pkgDir; ; {
|
||||
p := filepath.Join(dir, "ragflow_deps", "cl100k_base.tiktoken")
|
||||
if fileExists(p) {
|
||||
return p
|
||||
}
|
||||
parent := filepath.Dir(dir)
|
||||
if parent == dir {
|
||||
break
|
||||
}
|
||||
dir = parent
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func fileExists(p string) bool {
|
||||
info, err := os.Stat(p)
|
||||
return err == nil && !info.IsDir()
|
||||
}
|
||||
|
||||
func copyFile(t *testing.T, src, dst string) {
|
||||
t.Helper()
|
||||
if err := os.MkdirAll(filepath.Dir(dst), 0o755); err != nil {
|
||||
t.Fatalf("mkdir %s: %v", filepath.Dir(dst), err)
|
||||
}
|
||||
in, err := os.Open(src)
|
||||
if err != nil {
|
||||
t.Fatalf("open %s: %v", src, err)
|
||||
}
|
||||
defer in.Close()
|
||||
out, err := os.Create(dst)
|
||||
if err != nil {
|
||||
t.Fatalf("create %s: %v", dst, err)
|
||||
}
|
||||
defer out.Close()
|
||||
if _, err := io.Copy(out, in); err != nil {
|
||||
t.Fatalf("copy %s -> %s: %v", src, dst, err)
|
||||
}
|
||||
}
|
||||
@@ -560,9 +560,49 @@ func getCL100KEncoder() (*tiktoken.Tiktoken, error) {
|
||||
return cl100kEncoder.enc, cl100kEncoder.err
|
||||
}
|
||||
|
||||
// resetCL100KEncoderForTest clears the cl100k encoder cache so a test can force a
|
||||
// fresh load under a different search-root scope. Test-only; it mutates package
|
||||
// state. A plain assignment replaces the embedded sync.Once with an un-fired one
|
||||
// and wipes any cached result, so the next getCL100KEncoder / InitCL100KEncoder
|
||||
// re-runs the loader rather than returning a result cached by a sibling test.
|
||||
func resetCL100KEncoderForTest() {
|
||||
cl100kEncoder = struct {
|
||||
sync.Once
|
||||
enc *tiktoken.Tiktoken
|
||||
err error
|
||||
}{}
|
||||
}
|
||||
|
||||
// InitCL100KEncoder loads the cl100k_base BPE encoder once and returns an
|
||||
// error if the table is unavailable. Call it during server startup (before any
|
||||
// request reaches NumTokensFromString) so a missing table fails fast instead
|
||||
// of silently zeroing every token count.
|
||||
//
|
||||
// Why fail-fast and not a retry/warning: NumTokensFromString intentionally
|
||||
// returns 0 on encoder error to stay cheap on the hot path, which means a
|
||||
// missing cl100k_base.tiktoken (the Go image used to omit it) degraded every
|
||||
// token budget to 0 while content_ltks — a separate offline C++ tokenizer —
|
||||
// kept working, and the divergence went unnoticed. Catching it at startup turns
|
||||
// that silent data corruption into a hard, loud failure.
|
||||
func InitCL100KEncoder() error {
|
||||
enc, err := getCL100KEncoder()
|
||||
if err != nil {
|
||||
return fmt.Errorf("cl100k_base BPE table unavailable (NumTokensFromString would silently return 0): %w", err)
|
||||
}
|
||||
if enc == nil {
|
||||
return fmt.Errorf("cl100k_base encoder unavailable: GetEncoding returned a nil encoder without error")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// NumTokensFromString returns the number of tokens in s using the cl100k_base
|
||||
// BPE encoding. Mirrors Python's num_tokens_from_string (common/token_utils.py):
|
||||
// returns 0 on encoder error.
|
||||
//
|
||||
// Callers must ensure InitCL100KEncoder has succeeded at startup; otherwise a
|
||||
// missing table makes this return 0 for every non-empty string. The startup
|
||||
// check is the fail-fast guard — this cheap hot-path function stays silent by
|
||||
// design so it can be called per-token without error plumbing.
|
||||
func NumTokensFromString(s string) int {
|
||||
if s == "" {
|
||||
return 0
|
||||
|
||||
@@ -33,10 +33,69 @@
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import shutil
|
||||
import sys
|
||||
import zipfile
|
||||
|
||||
import requests
|
||||
|
||||
# Mirrors internal/common.DeepDocORTVersion (Go in-process backend). ONE OF
|
||||
# THREE places (with that Go constant and the ORT_VERSION in
|
||||
# ragflow_deps/download_deps.py) that must carry the same ONNX Runtime native
|
||||
# release for the statically-linked Go DeepDoc backend. There is no single
|
||||
# source of truth — keep all three equal. The pip onnxruntime== pin
|
||||
# (pyproject.toml) and the onnxruntime_go binding minor (go.mod) must stay on
|
||||
# the same minor line.
|
||||
ORT_VERSION = "1.23.2"
|
||||
|
||||
|
||||
def prune_stale_onnxruntime(static_lib_dir, version):
|
||||
"""Remove ONNX Runtime version dirs under static_lib that do NOT match
|
||||
`version`. Without this, a version bump leaves the stale dir next to
|
||||
the new one and build.sh's `find ... -name '*.a'` links BOTH (duplicate
|
||||
symbols / wrong version, silently)."""
|
||||
if not os.path.isdir(static_lib_dir):
|
||||
return
|
||||
expected = f"onnxruntime-linux-x64-static_lib-{version}-glibc2_28"
|
||||
for name in os.listdir(static_lib_dir):
|
||||
if not name.startswith("onnxruntime-linux-x64-static_lib-"):
|
||||
continue
|
||||
if name == expected:
|
||||
continue
|
||||
stale = os.path.join(static_lib_dir, name)
|
||||
print(f" Removing stale ONNX Runtime dir: {stale}")
|
||||
shutil.rmtree(stale)
|
||||
|
||||
|
||||
def has_static_archives(directory):
|
||||
"""True when `directory` holds at least one static archive (.a)."""
|
||||
return any(f.endswith(".a") for _, _, files in os.walk(directory) for f in files)
|
||||
|
||||
|
||||
def extract_onnxruntime(static_lib_dir, archive_path, version):
|
||||
"""Ensure the ONNX Runtime static archives for `version` sit under
|
||||
`static_lib_dir`. Returns True when that version is available afterwards
|
||||
(extracted now or already present), False when the archive is missing.
|
||||
|
||||
ORT ships a version-stamped top-level dir inside the zip
|
||||
(onnxruntime-linux-x64-static_lib-<version>-glibc2_28/), so a present
|
||||
`static_lib_dir` is NOT evidence that THIS version is extracted: after a
|
||||
version bump the stale dir is pruned and the new one must be extracted.
|
||||
"""
|
||||
if not os.path.isfile(archive_path):
|
||||
print(f" Skipping extraction: {os.path.basename(archive_path)} not found")
|
||||
return False
|
||||
prune_stale_onnxruntime(static_lib_dir, version)
|
||||
version_dir = os.path.join(static_lib_dir, f"onnxruntime-linux-x64-static_lib-{version}-glibc2_28")
|
||||
if os.path.isdir(version_dir) and has_static_archives(version_dir):
|
||||
print(f" ✓ onnxruntime/static_lib ({version}) already extracted to {version_dir}")
|
||||
return True
|
||||
os.makedirs(static_lib_dir, exist_ok=True)
|
||||
print(f" Extracting {os.path.basename(archive_path)} → {static_lib_dir}")
|
||||
with zipfile.ZipFile(archive_path) as zf:
|
||||
zf.extractall(static_lib_dir)
|
||||
return True
|
||||
|
||||
|
||||
def get_urls(use_china_mirrors=False) -> list[str | list[str]]:
|
||||
if use_china_mirrors:
|
||||
@@ -57,12 +116,16 @@ def get_urls(use_china_mirrors=False) -> list[str | list[str]]:
|
||||
# compatibility contract.
|
||||
"https://gh-proxy.com/https://github.com/browserbase/stagehand/releases/download/stagehand-server-v3/v3.7.2/stagehand-server-v3-linux-x64",
|
||||
"https://gh-proxy.com/https://github.com/browserbase/stagehand/releases/download/stagehand-server-v3/v3.7.2/stagehand-server-v3-linux-arm64",
|
||||
# Native static libraries for Go build (pdfium, pdf_oxide, office_oxide)
|
||||
# Used by build.sh's check_*_deps functions — pre-downloaded to avoid
|
||||
# network access during CI.
|
||||
# Native static libraries for Go build (pdfium, pdf_oxide,
|
||||
# office_oxide, onnxruntime). Used by build.sh's check_*_deps
|
||||
# functions — pre-downloaded to avoid network access during CI.
|
||||
["https://gh-proxy.com/https://github.com/kognitos/pdfium-static/releases/download/chromium%2F7809/pdfium-linux-x64-static.tgz", "pdfium-linux-x64-static.tgz"],
|
||||
["https://gh-proxy.com/https://github.com/yfedoseev/pdf_oxide/releases/download/v0.3.73/pdf_oxide-go-ffi-linux-amd64.tar.gz", "pdf_oxide-go-ffi-linux-amd64.tar.gz"],
|
||||
["https://gh-proxy.com/https://github.com/yfedoseev/office_oxide/releases/download/v0.1.9/native-linux-x86_64.tar.gz", "office_oxide-linux-x86_64.tar.gz"],
|
||||
[
|
||||
f"https://gh-proxy.com/https://github.com/csukuangfj/onnxruntime-libs/releases/download/v{ORT_VERSION}/onnxruntime-linux-x64-static_lib-{ORT_VERSION}-glibc2_28.zip",
|
||||
f"onnxruntime-linux-x64-static_lib-{ORT_VERSION}-glibc2_28.zip",
|
||||
],
|
||||
]
|
||||
else:
|
||||
return [
|
||||
@@ -82,12 +145,16 @@ def get_urls(use_china_mirrors=False) -> list[str | list[str]]:
|
||||
# compatibility contract.
|
||||
"https://github.com/browserbase/stagehand/releases/download/stagehand-server-v3/v3.7.2/stagehand-server-v3-linux-x64",
|
||||
"https://github.com/browserbase/stagehand/releases/download/stagehand-server-v3/v3.7.2/stagehand-server-v3-linux-arm64",
|
||||
# Native static libraries for Go build (pdfium, pdf_oxide, office_oxide)
|
||||
# Used by build.sh's check_*_deps functions — pre-downloaded to avoid
|
||||
# network access during CI.
|
||||
# Native static libraries for Go build (pdfium, pdf_oxide,
|
||||
# office_oxide, onnxruntime). Used by build.sh's check_*_deps
|
||||
# functions — pre-downloaded to avoid network access during CI.
|
||||
["https://github.com/kognitos/pdfium-static/releases/download/chromium%2F7809/pdfium-linux-x64-static.tgz", "pdfium-linux-x64-static.tgz"],
|
||||
["https://github.com/yfedoseev/pdf_oxide/releases/download/v0.3.73/pdf_oxide-go-ffi-linux-amd64.tar.gz", "pdf_oxide-go-ffi-linux-amd64.tar.gz"],
|
||||
["https://github.com/yfedoseev/office_oxide/releases/download/v0.1.9/native-linux-x86_64.tar.gz", "office_oxide-linux-x86_64.tar.gz"],
|
||||
[
|
||||
f"https://github.com/csukuangfj/onnxruntime-libs/releases/download/v{ORT_VERSION}/onnxruntime-linux-x64-static_lib-{ORT_VERSION}-glibc2_28.zip",
|
||||
f"onnxruntime-linux-x64-static_lib-{ORT_VERSION}-glibc2_28.zip",
|
||||
],
|
||||
]
|
||||
|
||||
|
||||
@@ -139,12 +206,13 @@ if __name__ == "__main__":
|
||||
# Extract native static libraries to ~/ragflow-native-libs for Go build.
|
||||
# Ensures build.sh can find them without network access.
|
||||
native_deps_dir = os.path.expanduser("~/ragflow-native-libs")
|
||||
import tarfile
|
||||
|
||||
extractions = [
|
||||
("pdfium-linux-x64-static.tgz", "pdfium-static"),
|
||||
("pdf_oxide-go-ffi-linux-amd64.tar.gz", "pdf_oxide"),
|
||||
("office_oxide-linux-x86_64.tar.gz", "office_oxide"),
|
||||
]
|
||||
import tarfile
|
||||
|
||||
for archive, subdir in extractions:
|
||||
archive_path = os.path.join(os.getcwd(), archive)
|
||||
@@ -159,3 +227,33 @@ if __name__ == "__main__":
|
||||
print(f" Extracting {archive} → {target}")
|
||||
with tarfile.open(archive_path) as tf:
|
||||
tf.extractall(target)
|
||||
|
||||
if not extract_onnxruntime(
|
||||
os.path.join(native_deps_dir, "onnxruntime", "static_lib"),
|
||||
os.path.join(os.getcwd(), f"onnxruntime-linux-x64-static_lib-{ORT_VERSION}-glibc2_28.zip"),
|
||||
ORT_VERSION,
|
||||
):
|
||||
# The archive was not downloaded or failed to extract, so no .a landed.
|
||||
# Fail loud instead of exiting 0: build.sh's ORT guard would otherwise
|
||||
# reject the build later with a less actionable message, and a missing
|
||||
# .a left here is exactly the "silent green" this PR is meant to prevent.
|
||||
print(
|
||||
f" ERROR: ONNX Runtime static archives for {ORT_VERSION} were not "
|
||||
f"extracted to {os.path.join(native_deps_dir, 'onnxruntime', 'static_lib')}. "
|
||||
f"Check the download above; build.sh will refuse to link without them.",
|
||||
file=sys.stderr,
|
||||
)
|
||||
sys.exit(1)
|
||||
|
||||
# ONNX Runtime is statically linked into the server binary, so there is no
|
||||
# runtime .so to surface. Log where build.sh (ONNXRUNTIME_STATIC_PREFIX) will
|
||||
# find the archives — the .a files live under
|
||||
# ~/ragflow-native-libs/onnxruntime/static_lib. The in-process backend
|
||||
# resolves OrtGetApiBase via dlopen(NULL); there is no dynamic .so fallback.
|
||||
ort_static_dir = os.path.join(native_deps_dir, "onnxruntime", "static_lib")
|
||||
ort_a_files = [os.path.join(root, f) for root, _, files in os.walk(ort_static_dir) for f in files if f.endswith(".a")]
|
||||
if ort_a_files:
|
||||
print(f" ✓ onnxruntime static archives ready: {len(ort_a_files)} .a under {ort_static_dir}")
|
||||
else:
|
||||
print(f" ERROR: ONNX Runtime .a files still missing under {ort_static_dir} after extraction; build.sh will refuse to link.", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
@@ -0,0 +1,71 @@
|
||||
# Tests for ragflow_deps/download_go_deps.py ONNX Runtime extraction.
|
||||
#
|
||||
# build.sh's build_go() fails fast when libonnxruntime.a is not linked, so these
|
||||
# tests must guarantee the archive is really landed on disk — above all after an
|
||||
# ORT version bump, where the zip's version-stamped top-level dir changes but
|
||||
# static_lib/ still exists from the previous run.
|
||||
|
||||
import os
|
||||
import zipfile
|
||||
|
||||
from download_go_deps import extract_onnxruntime, has_static_archives
|
||||
|
||||
|
||||
def make_ort_zip(path, version):
|
||||
"""Build a zip shaped like the upstream ORT release: a version-stamped
|
||||
top-level dir holding lib/libonnxruntime.a."""
|
||||
member = f"onnxruntime-linux-x64-static_lib-{version}-glibc2_28/lib/libonnxruntime.a"
|
||||
with zipfile.ZipFile(path, "w") as zf:
|
||||
zf.writestr(member, b"!<arch>\nort-payload")
|
||||
return path
|
||||
|
||||
|
||||
def test_extracts_on_first_run(tmp_path):
|
||||
# static_lib/ does not exist yet: the plain first-run path.
|
||||
static_lib = tmp_path / "onnxruntime" / "static_lib"
|
||||
version = "1.23.2"
|
||||
archive = make_ort_zip(tmp_path / f"onnxruntime-linux-x64-static_lib-{version}-glibc2_28.zip", version)
|
||||
|
||||
assert extract_onnxruntime(str(static_lib), str(archive), version) is True
|
||||
|
||||
version_dir = static_lib / f"onnxruntime-linux-x64-static_lib-{version}-glibc2_28"
|
||||
assert version_dir.is_dir()
|
||||
assert has_static_archives(str(version_dir))
|
||||
|
||||
|
||||
def test_extracts_after_version_bump_when_static_lib_exists(tmp_path):
|
||||
"""Regression: a static_lib/ left over from a previous version must not
|
||||
suppress extraction of the bumped version. Pruning the stale dir leaves
|
||||
static_lib/ in place; if extraction is then skipped no .a is ever landed and
|
||||
build.sh's ORT guard rejects the build."""
|
||||
static_lib = tmp_path / "onnxruntime" / "static_lib"
|
||||
static_lib.mkdir(parents=True)
|
||||
stale = static_lib / "onnxruntime-linux-x64-static_lib-1.23.1-glibc2_28"
|
||||
(stale / "lib").mkdir(parents=True)
|
||||
(stale / "lib" / "libonnxruntime.a").write_bytes(b"old-ort")
|
||||
|
||||
version = "1.23.2"
|
||||
archive = make_ort_zip(tmp_path / f"onnxruntime-linux-x64-static_lib-{version}-glibc2_28.zip", version)
|
||||
|
||||
assert extract_onnxruntime(str(static_lib), str(archive), version) is True
|
||||
|
||||
assert not stale.exists(), "stale ORT version should be pruned"
|
||||
version_dir = static_lib / f"onnxruntime-linux-x64-static_lib-{version}-glibc2_28"
|
||||
assert version_dir.is_dir(), "bumped ORT version was silently not extracted"
|
||||
assert has_static_archives(str(version_dir)), "bumped ORT version landed no .a"
|
||||
|
||||
|
||||
def test_skips_when_archive_missing(tmp_path):
|
||||
static_lib = tmp_path / "onnxruntime" / "static_lib"
|
||||
assert extract_onnxruntime(str(static_lib), str(tmp_path / "missing.zip"), "1.23.2") is False
|
||||
|
||||
|
||||
def test_idempotent_when_matching_version_already_present(tmp_path):
|
||||
static_lib = tmp_path / "onnxruntime" / "static_lib"
|
||||
version = "1.23.2"
|
||||
archive = make_ort_zip(tmp_path / f"onnxruntime-linux-x64-static_lib-{version}-glibc2_28.zip", version)
|
||||
|
||||
assert extract_onnxruntime(str(static_lib), str(archive), version) is True
|
||||
before = sorted(os.listdir(static_lib))
|
||||
assert extract_onnxruntime(str(static_lib), str(archive), version) is True
|
||||
assert sorted(os.listdir(static_lib)) == before
|
||||
Reference in new issue
Block a user