Provision native libs from versioned user cache (offline CI, drop /opt pre-bake) (#22023)

.dockerignore

@@ -38,9 +38,10 @@ target/
 # regardless of which resources are prepared for the dependency image.
 # The deps image is built from this directory with:
 #   cd deps && docker build -f Dockerfile -t infiniflow/ragflow_deps .
-# The native downloader tools/scripts/download_dev_deps.py is copied by the
-# main Dockerfile (not excluded here) so build.sh can read its ORT version pin
-# when running `./build.sh --go`.
+# tools/scripts is NOT excluded here: the main Dockerfile COPYs it so build.sh
+# can `source tools/scripts/native_lib_version.sh` (it does so at the top of the
+# script) and reference tools/scripts/download_dev_deps.py in its remediation
+# messages while building `./build.sh --go`.
 deps/*
 
 # ── IDE and editor config ──────────────────────────────────────────────────

.github/workflows/sep-tests.yml

@@ -243,6 +243,16 @@ jobs:
             export GOCACHE="${RUNNER_WORKSPACE_PREFIX}/gobuild"
           fi
 
+          # Fail fast if the native-lib user cache is not mounted: a missing or
+          # empty cache would otherwise surface as a cryptic cgo link error later.
+          NATIVE_LIBS_DIR="${RAGFLOW_NATIVE_LIBS_DIR:-${HOME}/ragflow-native-libs}"
+          for lib in office_oxide pdfium-static pdf_oxide onnxruntime; do
+            if [ ! -d "${NATIVE_LIBS_DIR}/${lib}" ]; then
+              echo "ERROR: ${NATIVE_LIBS_DIR}/${lib} missing — native-lib cache not mounted on this runner." >&2
+              exit 1
+            fi
+          done
+
           BUILDER_CONTAINER=ragflow_build_${GITHUB_RUN_ID}_${DOC_ENGINE}_$(od -An -N4 -tx4 /dev/urandom | tr -d ' ')
           cleanup_builder() {
             if [[ -n "${BUILDER_CONTAINER:-}" ]]; then
@@ -252,11 +262,21 @@ jobs:
           trap cleanup_builder EXIT
 
           TZ=${TZ:-$(readlink -f /etc/localtime | awk -F '/zoneinfo/' '{print $2}')}
+          # Mount the native static-lib user cache into the builder container.
+          # build.sh resolves the libs from RAGFLOW_NATIVE_LIBS_DIR (default
+          # ~/ragflow-native-libs); this pins the same root for the container and
+          # bind-mounts it read-only so the cgo link finds the versioned
+          # v<version>/ dirs without an /opt pre-bake. CI mounts the cache from
+          # the build host (populated by download_dev_deps.py); a missing mount is
+          # caught by the assert below, not by a cryptic cgo link error.
+          export RAGFLOW_NATIVE_LIBS_DIR="${RAGFLOW_NATIVE_LIBS_DIR:-${HOME}/ragflow-native-libs}"
           sudo docker run --privileged -d --name "${BUILDER_CONTAINER}" \
             -e TZ="${TZ}" \
             -e UV_INDEX=https://mirrors.aliyun.com/pypi/simple \
+            -e "RAGFLOW_NATIVE_LIBS_DIR=${RAGFLOW_NATIVE_LIBS_DIR}" \
             -v "${PWD}:/ragflow" \
             -v "${PWD}/internal/binding/cpp/resource:/usr/share/infinity/resource" \
+            -v "${RAGFLOW_NATIVE_LIBS_DIR}:${RAGFLOW_NATIVE_LIBS_DIR}:ro" \
             infiniflow/infinity_builder:ubuntu22_clang20
           sudo docker exec "${BUILDER_CONTAINER}" bash -c 'git config --global safe.directory "*" && cd /ragflow && ./build.sh --cpp'
           ./build.sh --go
@@ -810,6 +830,16 @@ jobs:
             export GOCACHE="${RUNNER_WORKSPACE_PREFIX}/gobuild"
           fi
 
+          # Fail fast if the native-lib user cache is not mounted: a missing or
+          # empty cache would otherwise surface as a cryptic cgo link error later.
+          NATIVE_LIBS_DIR="${RAGFLOW_NATIVE_LIBS_DIR:-${HOME}/ragflow-native-libs}"
+          for lib in office_oxide pdfium-static pdf_oxide onnxruntime; do
+            if [ ! -d "${NATIVE_LIBS_DIR}/${lib}" ]; then
+              echo "ERROR: ${NATIVE_LIBS_DIR}/${lib} missing — native-lib cache not mounted on this runner." >&2
+              exit 1
+            fi
+          done
+
           BUILDER_CONTAINER=ragflow_build_${GITHUB_RUN_ID}_${DOC_ENGINE}_$(od -An -N4 -tx4 /dev/urandom | tr -d ' ')
           cleanup_builder() {
             if [[ -n "${BUILDER_CONTAINER:-}" ]]; then
@@ -819,11 +849,21 @@ jobs:
           trap cleanup_builder EXIT
 
           TZ=${TZ:-$(readlink -f /etc/localtime | awk -F '/zoneinfo/' '{print $2}')}
+          # Mount the native static-lib user cache into the builder container.
+          # build.sh resolves the libs from RAGFLOW_NATIVE_LIBS_DIR (default
+          # ~/ragflow-native-libs); this pins the same root for the container and
+          # bind-mounts it read-only so the cgo link finds the versioned
+          # v<version>/ dirs without an /opt pre-bake. CI mounts the cache from
+          # the build host (populated by download_dev_deps.py); a missing mount is
+          # caught by the assert below, not by a cryptic cgo link error.
+          export RAGFLOW_NATIVE_LIBS_DIR="${RAGFLOW_NATIVE_LIBS_DIR:-${HOME}/ragflow-native-libs}"
           sudo docker run --privileged -d --name "${BUILDER_CONTAINER}" \
             -e TZ="${TZ}" \
             -e UV_INDEX=https://mirrors.aliyun.com/pypi/simple \
+            -e "RAGFLOW_NATIVE_LIBS_DIR=${RAGFLOW_NATIVE_LIBS_DIR}" \
             -v "${PWD}:/ragflow" \
             -v "${PWD}/internal/binding/cpp/resource:/usr/share/infinity/resource" \
+            -v "${RAGFLOW_NATIVE_LIBS_DIR}:${RAGFLOW_NATIVE_LIBS_DIR}:ro" \
             infiniflow/infinity_builder:ubuntu22_clang20
           sudo docker exec "${BUILDER_CONTAINER}" bash -c 'git config --global safe.directory "*" && cd /ragflow && ./build.sh --cpp'
           ./build.sh --go
@@ -1386,6 +1426,16 @@ jobs:
         # integration tests; the per-engine go jobs build the full server.
         run: |
           set -euo pipefail
+          # Fail fast if the native-lib user cache is not mounted: a missing or
+          # empty cache would otherwise surface as a cryptic cgo link error later.
+          NATIVE_LIBS_DIR="${RAGFLOW_NATIVE_LIBS_DIR:-${HOME}/ragflow-native-libs}"
+          for lib in office_oxide pdfium-static pdf_oxide onnxruntime; do
+            if [ ! -d "${NATIVE_LIBS_DIR}/${lib}" ]; then
+              echo "ERROR: ${NATIVE_LIBS_DIR}/${lib} missing — native-lib cache not mounted on this runner." >&2
+              exit 1
+            fi
+          done
+
           BUILDER_CONTAINER=ragflow_build_${GITHUB_RUN_ID}_native_$(od -An -N4 -tx4 /dev/urandom | tr -d ' ')
           cleanup_builder() {
             if [[ -n "${BUILDER_CONTAINER:-}" ]]; then
@@ -1395,11 +1445,21 @@ jobs:
           trap cleanup_builder EXIT
 
           TZ=${TZ:-$(readlink -f /etc/localtime | awk -F '/zoneinfo/' '{print $2}')}
+          # Mount the native static-lib user cache into the builder container.
+          # build.sh resolves the libs from RAGFLOW_NATIVE_LIBS_DIR (default
+          # ~/ragflow-native-libs); this pins the same root for the container and
+          # bind-mounts it read-only so the cgo link finds the versioned
+          # v<version>/ dirs without an /opt pre-bake. CI mounts the cache from
+          # the build host (populated by download_dev_deps.py); a missing mount is
+          # caught by the assert below, not by a cryptic cgo link error.
+          export RAGFLOW_NATIVE_LIBS_DIR="${RAGFLOW_NATIVE_LIBS_DIR:-${HOME}/ragflow-native-libs}"
           sudo docker run --privileged -d --name "${BUILDER_CONTAINER}" \
             -e TZ="${TZ}" \
             -e UV_INDEX=https://mirrors.aliyun.com/pypi/simple \
+            -e "RAGFLOW_NATIVE_LIBS_DIR=${RAGFLOW_NATIVE_LIBS_DIR}" \
             -v "${PWD}:/ragflow" \
             -v "${PWD}/internal/binding/cpp/resource:/usr/share/infinity/resource" \
+            -v "${RAGFLOW_NATIVE_LIBS_DIR}:${RAGFLOW_NATIVE_LIBS_DIR}:ro" \
             infiniflow/infinity_builder:ubuntu22_clang20
           sudo docker exec "${BUILDER_CONTAINER}" bash -c 'git config --global safe.directory "*" && cd /ragflow && ./build.sh --cpp'
 
@@ -1486,6 +1546,16 @@ jobs:
         # starts only the dependency services, not the ragflow-cpu container.
         run: |
           set -euo pipefail
+          # Fail fast if the native-lib user cache is not mounted: a missing or
+          # empty cache would otherwise surface as a cryptic cgo link error later.
+          NATIVE_LIBS_DIR="${RAGFLOW_NATIVE_LIBS_DIR:-${HOME}/ragflow-native-libs}"
+          for lib in office_oxide pdfium-static pdf_oxide onnxruntime; do
+            if [ ! -d "${NATIVE_LIBS_DIR}/${lib}" ]; then
+              echo "ERROR: ${NATIVE_LIBS_DIR}/${lib} missing — native-lib cache not mounted on this runner." >&2
+              exit 1
+            fi
+          done
+
           BUILDER_CONTAINER=ragflow_build_${GITHUB_RUN_ID}_go_int_$(od -An -N4 -tx4 /dev/urandom | tr -d ' ')
           cleanup_builder() {
             if [[ -n "${BUILDER_CONTAINER:-}" ]]; then
@@ -1494,11 +1564,21 @@ jobs:
           }
           trap cleanup_builder EXIT
           TZ=${TZ:-$(readlink -f /etc/localtime | awk -F '/zoneinfo/' '{print $2}')}
+          # Mount the native static-lib user cache into the builder container.
+          # build.sh resolves the libs from RAGFLOW_NATIVE_LIBS_DIR (default
+          # ~/ragflow-native-libs); this pins the same root for the container and
+          # bind-mounts it read-only so the cgo link finds the versioned
+          # v<version>/ dirs without an /opt pre-bake. CI mounts the cache from
+          # the build host (populated by download_dev_deps.py); a missing mount is
+          # caught by the assert below, not by a cryptic cgo link error.
+          export RAGFLOW_NATIVE_LIBS_DIR="${RAGFLOW_NATIVE_LIBS_DIR:-${HOME}/ragflow-native-libs}"
           sudo docker run --privileged -d --name "${BUILDER_CONTAINER}" \
             -e TZ="${TZ}" \
             -e UV_INDEX=https://mirrors.aliyun.com/pypi/simple \
+            -e "RAGFLOW_NATIVE_LIBS_DIR=${RAGFLOW_NATIVE_LIBS_DIR}" \
             -v "${PWD}:/ragflow" \
             -v "${PWD}/internal/binding/cpp/resource:/usr/share/infinity/resource" \
+            -v "${RAGFLOW_NATIVE_LIBS_DIR}:${RAGFLOW_NATIVE_LIBS_DIR}:ro" \
             infiniflow/infinity_builder:ubuntu22_clang20
           sudo docker exec "${BUILDER_CONTAINER}" bash -c 'git config --global safe.directory "*" && cd /ragflow && ./build.sh --cpp'
 

Dockerfile

@@ -227,19 +227,21 @@ RUN --mount=type=cache,id=ragflow_gomod,target=/root/.cache/gomod,sharing=locked
 COPY internal internal
 COPY cmd cmd
 COPY build.sh ./
+COPY tools/scripts tools/scripts
 COPY Dockerfile ./
 
-# ONNX Runtime static archives: build.sh's _seed_from_system copies the ORT
-# static libs from the runner's pre-baked /opt/ragflow-native-libs/onnxruntime
-# (arch-correct per TARGETARCH) into the user-cache path ONNXRUNTIME_STATIC_PREFIX
-# before linking. These .a files are consumed at link time only and do NOT enter
-# the final image (only the compiled /ragflow/bin is COPY --from=go-builder).
-# The Dockerfile no longer pre-copies them here: pre-creating the user-cache dir
-# would make _seed_from_system treat it as "already cached" and skip the copy on
-# the non-matching architecture (e.g. the arm64 runner has no x64 dir), so we
-# rely on build.sh to seed ORT for both amd64 and arm64. ARG ORT_VERSION stays
-# because build.sh's check_ort_version_consistency greps it to cross-check the
-# pinned ORT release against internal/common/environments.go.
+# ONNX Runtime static archives are provisioned by build.sh from the versioned
+# user cache at ONNXRUNTIME_STATIC_PREFIX (${RAGFLOW_NATIVE_LIBS_DIR:-$HOME/
+# ragflow-native-libs}/onnxruntime/v<version>). There is NO /opt pre-bake: on CI
+# the host cache is mounted in at build time, and for the product image build
+# (where no host cache is mounted) build.sh falls back to _ensure_native_libs_installed,
+# which runs `uv run tools/scripts/download_dev_deps.py`. That downloader fetches
+# the native archives on demand and self-creates its deps/ workdir when absent
+# (see download_dev_deps.py::ensure_deps_dir), so it no longer crashes on a bare
+# go-builder checkout. These .a files are consumed at link time only and do NOT
+# enter the final image (only the compiled /ragflow/bin is COPY --from=go-builder).
+# ARG ORT_VERSION stays because build.sh's check_ort_version_consistency greps it
+# to cross-check the pinned ORT release against internal/common/environments.go.
 ARG ORT_VERSION=1.29.0
 
 RUN git config --global safe.directory "*" && \

build.sh

@@ -15,6 +15,18 @@ PROJECT_ROOT="$SCRIPT_DIR"
 # untouched and merge conflicts against the EE branch are limited to this
 # single line plus the guarded hook call inside run_native_integration_tests.
 [[ -f "${SCRIPT_DIR}/build_ee.sh" ]] && source "${SCRIPT_DIR}/build_ee.sh"
+# Version-keyed native-lib resolution: each native lib sits under a
+# v<version>/ subdir of the user cache (~/ragflow-native-libs/<lib>/v<version>/),
+# so a single mounted cache holds several versions at once. The version numbers
+# come from go.mod (office_oxide / pdf_oxide), environments.go (ORT) and
+# native_lib_version.sh (pdfium) — see that file for the single source of truth.
+source "${SCRIPT_DIR}/tools/scripts/native_lib_version.sh"
+
+# Native static-lib user-cache root. Overridable via RAGFLOW_NATIVE_LIBS_DIR so
+# CI can mount the cache at a fixed host path (sep-tests.yml Slice 3); defaults
+# to ~/ragflow-native-libs. Captured once here and reused by every <lib>/v<version>
+# prefix below so the whole script agrees on a single cache root.
+NATIVE_LIBS_DIR="$(ragflow_native_libs_dir)"
 
 # Build directories
 CPP_DIR="$PROJECT_ROOT/internal/binding/cpp"
@@ -43,31 +55,42 @@ if [ -n "$EXTRA_GO_TAGS" ]; then
 fi
 
 # Native static library settings. These are the user-cache paths (~/ragflow-native-libs/).
-# If /opt/ragflow-native-libs/ exists (pre-seeded in CI runner image), it takes priority
-# and skips the network (download_dev_deps.py) fallback.
-SYSTEM_DEPS="/opt/ragflow-native-libs"
+# Each lib lives under a version-keyed subdir (v<version>/) placed there by
+# tools/scripts/download_dev_deps.py (idempotent, version-pinned) — run once by a
+# developer, or mounted in from the build host on CI. There is NO /opt pre-bake and
+# NO flat (version-less) fallback: a missing version dir means the cache is
+# incomplete and build.sh triggers the downloader itself (see setup_cgo_env).
 
-# office_oxide native library settings — static linking
-OFFICE_OXIDE_PREFIX="${HOME}/ragflow-native-libs/office_oxide"
-# Derived from go.mod (single source of truth: the Go module version). Matches
-# tools/scripts/download_dev_deps.py's office_oxide_url derivation. The pipeline exits 0
-# with an empty value when go.mod has no matching entry (no pipefail here), so
-# the empty case needs its own check: an empty version would slip past the
-# stale-lib comparison below and produce a broken .../lib/v link path.
-OFFICE_OXIDE_VERSION="$(grep -m1 -E '^[[:space:]]*github.com/yfedoseev/office_oxide/go[[:space:]]+v' "$PROJECT_ROOT/go.mod" | sed -E 's/.*v([0-9][^[:space:]]*).*/\1/')"
+# office_oxide native library settings — static linking. Version is derived from
+# go.mod (single source of truth, exactly as tools/scripts/download_dev_deps.py
+# derives it). An empty value (module absent from go.mod) would produce a broken
+# .../lib/v link path, so it is fatal. The prefix is set after the version, since
+# it embeds the version.
+OFFICE_OXIDE_VERSION="$(go_module_version "github.com/yfedoseev/office_oxide/go" "$PROJECT_ROOT")"
 if [ -z "$OFFICE_OXIDE_VERSION" ]; then
     echo -e "${RED}Error: could not read office_oxide version from go.mod${NC}" >&2; exit 1
 fi
+OFFICE_OXIDE_PREFIX="${NATIVE_LIBS_DIR}/office_oxide/v${OFFICE_OXIDE_VERSION}"
 
-# pdfium native library settings — static linking (kognitos/pdfium-static)
-# Pinned here (not grepped from the dev downloader) so build.sh stays tied to the
-# checked-out source tree and the Docker build needs no extra COPY of the script.
-PDFIUM_STATIC_PREFIX="${HOME}/ragflow-native-libs/pdfium-static"
-PDFIUM_STATIC_VERSION="7809"
+# pdfium native library settings — static linking (kognitos/pdfium-static).
+# Version is read from the single source of truth tools/scripts/native_lib_version.sh
+# (pdfium_static_version), exactly as download_dev_deps.py does, so the bare build
+# number lives in exactly one place. An empty value is fatal.
+PDFIUM_STATIC_VERSION="$(pdfium_static_version "$PROJECT_ROOT")"
+if [ -z "$PDFIUM_STATIC_VERSION" ]; then
+    echo -e "${RED}Error: could not read pdfium version from native_lib_version.sh${NC}" >&2; exit 1
+fi
+PDFIUM_STATIC_PREFIX="${NATIVE_LIBS_DIR}/pdfium-static/v${PDFIUM_STATIC_VERSION}"
 
-# pdf_oxide native library settings — static linking (go-ffi tarball)
-PDF_OXIDE_PREFIX="${HOME}/ragflow-native-libs/pdf_oxide"
-PDF_OXIDE_VERSION="0.3.73"
+# pdf_oxide native library settings — static linking (go-ffi tarball). Version
+# is read from go.mod (single source of truth) instead of a hardcoded constant,
+# so bumping the dependency needs no build.sh edit. The empty case is fatal for
+# the same reason as office_oxide. The prefix is set after the version.
+PDF_OXIDE_VERSION="$(go_module_version "github.com/yfedoseev/pdf_oxide/go" "$PROJECT_ROOT")"
+if [ -z "$PDF_OXIDE_VERSION" ]; then
+    echo -e "${RED}Error: could not read pdf_oxide version from go.mod${NC}" >&2; exit 1
+fi
+PDF_OXIDE_PREFIX="${NATIVE_LIBS_DIR}/pdf_oxide/v${PDF_OXIDE_VERSION}"
 
 # onnxruntime native library settings — static linking for the in-process
 # (Go) DeepDoc backend. libonnxruntime*.a is linked into the server binary
@@ -75,7 +98,7 @@ PDF_OXIDE_VERSION="0.3.73"
 # is then resolved via dlopen(NULL) (the process-global symbol table, not the
 # executable's own path), so no libonnxruntime.so is needed at runtime.
 # Downloaded by tools/scripts/download_dev_deps.py into onnxruntime/static_lib.
-ONNXRUNTIME_STATIC_PREFIX="${HOME}/ragflow-native-libs/onnxruntime/static_lib"
+ONNXRUNTIME_STATIC_PREFIX="${NATIVE_LIBS_DIR}/onnxruntime/static_lib"
 
 # ── target platform detection ──────────────────────────────────────────
 # Defaults to the build host (uname), overridable with RAGFLOW_TARGET_OS /
@@ -155,27 +178,53 @@ ort_static_lib_prefix() {
     esac
 }
 
-# Copy a dependency from the system pre-seed directory to the user cache.
-# Returns 0 if the dep was copied or already exists in cache, 1 otherwise.
-_seed_from_system() {
-    local dep_name="$1"  # e.g. "pdfium-static", "pdf_oxide", "office_oxide"
-    local dep_dir="${HOME}/ragflow-native-libs/${dep_name}"
-    local sys_dir="${SYSTEM_DEPS}/${dep_name}"
+# ── native static library provisioning ──────────────────────────────────
+# The native .a files live in the version-keyed user cache
+# (~/ragflow-native-libs/<lib>/v<version>/), placed there by
+# tools/scripts/download_dev_deps.py — idempotently and version-pinned — which a
+# developer runs once, and which CI mounts in from the build host. There is NO
+# /opt pre-bake and NO flat (version-less) fallback any more: a missing version
+# dir means the cache is incomplete, so build.sh triggers the downloader itself
+# (controlled auto-download) rather than silently linking a stale or wrong lib.
 
-    echo "check if dep ${dep_name} exists in ${dep_dir} or ${sys_dir}"
+# True when every native lib the Go build links is already in the versioned user
+# cache for the target platform. Used to skip the network entirely on an
+# already-provisioned host.
+_native_libs_present() {
+    [ -f "${OFFICE_OXIDE_PREFIX}/lib/liboffice_oxide.a" ] || return 1
+    [ -f "${OFFICE_OXIDE_PREFIX}/include/office_oxide_c/office_oxide.h" ] || return 1
+    [ -f "${PDFIUM_STATIC_PREFIX}/lib/libpdfium.a" ] || return 1
+    [ -f "${PDF_OXIDE_PREFIX}/lib/${TARGET_OS}_${TARGET_ARCH}/libpdf_oxide.a" ] || return 1
+    # ORT is version+platform keyed under static_lib; any real .a there is
+    # authoritative (the version-stamped symlink dir created later holds only
+    # symlinks, so -type f skips them and matches the real archives).
+    [ -n "$(find "$ONNXRUNTIME_STATIC_PREFIX" -type f -name '*.a' 2>/dev/null | head -1)" ] || return 1
+    return 0
+}
 
-    if [ -d "$dep_dir" ]; then
-        echo "  ${dep_name} → ${dep_dir} (user cache)"
-        return 0  # already cached
-    fi
-    if [ -d "$sys_dir" ]; then
-        echo "  ${dep_name} → ${sys_dir} (system)"
-        mkdir -p "$(dirname "$dep_dir")"
-        cp -r "$sys_dir" "$dep_dir"
+# Controlled auto-download: if the versioned native libs are not yet in the user
+# cache, run the one-stop downloader ONCE. It is idempotent and version-pinned,
+# so it can only land the exact pinned .a — never a wrong version. Fail loud only
+# when the cache is still incomplete afterwards (a locked-down host with no
+# network and no pre-placed libs).
+_ensure_native_libs_installed() {
+    _native_libs_present && return 0
+    echo "One or more native static libraries are missing from the user cache;"
+    echo "  running downloader: uv run tools/scripts/download_dev_deps.py"
+    ( cd "$PROJECT_ROOT" && uv run tools/scripts/download_dev_deps.py ) || {
+        echo -e "${RED}Error: native library downloader failed.${NC}" >&2
+        exit 1
+    }
+    if _native_libs_present; then
         return 0
     fi
-    echo "  ${dep_name} not found in system or user cache"
-    return 1
+    echo -e "${RED}Error: native static libraries still missing after download.${NC}" >&2
+    echo "  Pre-place the version-keyed dirs manually on a locked-down host:" >&2
+    echo "    ${OFFICE_OXIDE_PREFIX}" >&2
+    echo "    ${PDFIUM_STATIC_PREFIX}" >&2
+    echo "    ${PDF_OXIDE_PREFIX}" >&2
+    echo "    ${ONNXRUNTIME_STATIC_PREFIX}" >&2
+    exit 1
 }
 
 # ── cl100k_base BPE table ─────────────────────────────────────────────
@@ -400,6 +449,11 @@ check_onnxruntime_deps() {
     # another OS/arch, and the Python side legitimately keeps another ORT
     # version in the same prefix. Matching both is why coexisting version
     # dirs need no separate fail-fast guard here.
+    # The ORT static archives are resolved from the version-keyed user cache
+    # (~/ragflow-native-libs/onnxruntime/static_lib, which the downloader
+    # populates and CI mounts in). There is no /opt pre-bake to reseed from; an
+    # incomplete cache is filled by the controlled auto-download in setup_cgo_env.
+
     local ort_dir_prefix
     ort_dir_prefix="$(ort_static_lib_prefix)"
     while IFS= read -r f; do
@@ -432,7 +486,7 @@ check_onnxruntime_deps() {
         echo "  available: ${avail:-<none>}" >&2
         echo "  DeepDocORTVersion=${ort_version}; fetch it with:" >&2
         echo "    uv run tools/scripts/download_dev_deps.py" >&2
-        echo "  or pre-seed /opt/ragflow-native-libs/onnxruntime (CI image)." >&2
+        echo "  or pre-seed ${NATIVE_LIBS_DIR}/onnxruntime (CI mounts the cache from the build host)." >&2
         return 1
     fi
 
@@ -495,7 +549,9 @@ check_onnxruntime_deps() {
 # Check office_oxide native library
 check_office_oxide_deps() {
     print_section "Checking office_oxide native library"
-    _seed_from_system "office_oxide" || true
+    # office_oxide is resolved from the version-keyed user cache
+    # (${OFFICE_OXIDE_PREFIX}). An incomplete cache is filled by the controlled
+    # auto-download in setup_cgo_env; there is no /opt pre-bake to recover from.
 
     local lib_file="liboffice_oxide.a"
     local lib_path="${OFFICE_OXIDE_PREFIX}/lib/${lib_file}"
@@ -516,28 +572,12 @@ check_office_oxide_deps() {
     if ! strings "$lib_path" 2>/dev/null | grep -Fxq "$OFFICE_OXIDE_VERSION"; then
         local found_version
         found_version=$(strings "$lib_path" 2>/dev/null | grep -E "^0\.[0-9]+\.[0-9]+$" | head -1)
-        echo -e "${YELLOW}office_oxide native lib version mismatch (required v${OFFICE_OXIDE_VERSION}; found: ${found_version:-unknown}); refreshing from download${NC}"
-        rm -rf "${OFFICE_OXIDE_PREFIX}" "${PROJECT_ROOT}/deps/$(office_oxide_asset)"
-        if ! (cd "${PROJECT_ROOT}" && uv run tools/scripts/download_dev_deps.py); then
-            echo -e "${RED}Error: office_oxide native lib version mismatch${NC}"
-            echo "  Required: v${OFFICE_OXIDE_VERSION}; found: ${found_version:-unknown}"
-            echo "  A stale lib silently loses PPT97 (.ppt) slide content. Refresh:"
-            echo "    rm -rf ~/ragflow-native-libs/office_oxide deps/$(office_oxide_asset)"
-            echo "    uv run tools/scripts/download_dev_deps.py"
-            exit 1
-        fi
-        lib_path="${OFFICE_OXIDE_PREFIX}/lib/${lib_file}"
-        header_path="${OFFICE_OXIDE_PREFIX}/include/office_oxide_c/office_oxide.h"
-        if [ ! -f "$lib_path" ] || [ ! -f "$header_path" ]; then
-            echo -e "${RED}Error: office_oxide native library not found after refresh${NC}"
-            exit 1
-        fi
-        if ! strings "$lib_path" 2>/dev/null | grep -Fxq "$OFFICE_OXIDE_VERSION"; then
-            found_version=$(strings "$lib_path" 2>/dev/null | grep -E "^0\.[0-9]+\.[0-9]+$" | head -1)
-            echo -e "${RED}Error: office_oxide native lib version mismatch after refresh${NC}"
-            echo "  Required: v${OFFICE_OXIDE_VERSION}; found: ${found_version:-unknown}"
-            exit 1
-        fi
+        echo -e "${RED}Error: office_oxide native lib version mismatch${NC}"
+        echo "  Required: v${OFFICE_OXIDE_VERSION}; found: ${found_version:-unknown}"
+        echo "  A stale lib silently loses PPT97 (.ppt) slide content. Refresh:"
+        echo "    rm -rf ~/ragflow-native-libs/office_oxide deps/$(office_oxide_asset)"
+        echo "    uv run tools/scripts/download_dev_deps.py"
+        exit 1
     fi
 
     echo "✓ office_oxide v${OFFICE_OXIDE_VERSION} native library found at ${OFFICE_OXIDE_PREFIX}"
@@ -546,7 +586,9 @@ check_office_oxide_deps() {
 
 # Check pdfium static library.
 check_pdfium_deps() {
-    _seed_from_system "pdfium-static" || true
+    # pdfium is resolved from the version-keyed user cache
+    # (${PDFIUM_STATIC_PREFIX}). An incomplete cache is filled by the controlled
+    # auto-download in setup_cgo_env; there is no /opt pre-bake to recover from.
     local lib_path="${PDFIUM_STATIC_PREFIX}/lib/libpdfium.a"
 
     if [ -f "$lib_path" ]; then
@@ -604,7 +646,9 @@ pdf_oxide_validate_version() {
 }
 
 check_pdf_oxide_deps() {
-    _seed_from_system "pdf_oxide" || true
+    # pdf_oxide is resolved from the version-keyed user cache
+    # (${PDF_OXIDE_PREFIX}). An incomplete cache is filled by the controlled
+    # auto-download in setup_cgo_env; there is no /opt pre-bake to recover from.
     # Map platform to tarball-internal subdirectory (lib/<os>_<arch>/). Reuses
     # TARGET_OS/TARGET_ARCH set by detect_target_platform instead of re-deriving
     # from uname, so it stays consistent with the other check_*_deps helpers.
@@ -617,10 +661,10 @@ check_pdf_oxide_deps() {
     local lib_path="${PDF_OXIDE_PREFIX}/lib/${platform_subdir}/libpdf_oxide.a"
 
     if [ -f "$lib_path" ]; then
-        # Verify the on-disk lib matches the pinned version. _seed_from_system
-        # accepts an existing user-cache directory without inspecting it, so a
-        # lib left over from an earlier pin is reused silently and the upgrade
-        # becomes a no-op.
+        # Verify the on-disk lib matches the pinned version. The cache is
+        # version-keyed (${PDF_OXIDE_PREFIX}), so an earlier pin lives under a
+        # different v<version>/ dir and is never silently reused; this check is a
+        # defense-in-depth against a stale .a that somehow landed in this dir.
         #
         # The version marker "pdf_oxide <version>" is merged into Rust's string
         # constant pool (unlike office_oxide's standalone "0.1.9" line), so we
@@ -798,12 +842,12 @@ build_go() {
         echo "  at startup with a fatal 'no in-process DeepDoc backend serving'." >&2
         echo "  Fetch the static libs with:" >&2
         echo "    uv run tools/scripts/download_dev_deps.py" >&2
-        echo "  or pre-seed them at /opt/ragflow-native-libs/onnxruntime (CI image)." >&2
+        echo "  or pre-seed them at ${NATIVE_LIBS_DIR}/onnxruntime (CI mounts the cache from the build host)." >&2
         echo "  This production binary must statically include ORT — there is no" >&2
         echo "  ORT-free build path. ORT ends up unlinked only via one of:" >&2
         echo "    - the ORT static_lib dir was never seeded: run" >&2
         echo "      'uv run tools/scripts/download_dev_deps.py', or pre-seed" >&2
-        echo "      /opt/ragflow-native-libs/onnxruntime as the CI image does;" >&2
+        echo "      ${NATIVE_LIBS_DIR}/onnxruntime as the CI mount does;" >&2
         echo "    - setup_cgo_env did not add libonnxruntime to CGO_LDFLAGS." >&2
         return 1
     fi
@@ -837,25 +881,22 @@ build_go() {
 setup_cgo_env() {
     detect_target_platform
 
+    # Controlled auto-download: fill any missing version-keyed native lib from
+    # the user cache before resolving individual libs. On an already-provisioned
+    # host (the common CI case, where the cache is mounted in) this is a no-op;
+    # on a fresh host it runs the one-stop downloader once. There is no /opt
+    # pre-bake and no flat fallback to recover from.
+    _ensure_native_libs_installed
+
     # ── office_oxide ──────────────────────────────────────────────────
     check_office_oxide_deps
 
-    # Go's build cache keys CGO_LDFLAGS as a string — it does NOT hash the
-    # file content of referenced .a archives. So swapping the .a in-place
-    # (same path) does NOT invalidate the cache, and `go build` silently
-    # reuses a stale binary linked against the old .a.
-    #
-    # Create a version-stamped symlink directory so the flag string
-    # includes the actual linked version. When the .a is upgraded, the
-    # path changes → Go cache key changes → automatic relink.
-    local office_oxide_lib_dir="${OFFICE_OXIDE_PREFIX}/lib"
-    local versioned_lib_dir="${office_oxide_lib_dir}/v${OFFICE_OXIDE_VERSION}"
-    mkdir -p "$versioned_lib_dir"
-    ln -sf "${office_oxide_lib_dir}/liboffice_oxide.a" \
-        "${versioned_lib_dir}/liboffice_oxide.a"
-
+    # The office_oxide prefix is already version-keyed
+    # (${OFFICE_OXIDE_PREFIX} = .../office_oxide/v<version>), so the linked path
+    # carries the version and Go's build cache (which keys CGO_LDFLAGS as a
+    # string, not the .a contents) automatically relinks when the version bumps.
     export CGO_CFLAGS="-I${OFFICE_OXIDE_PREFIX}/include/office_oxide_c${CGO_CFLAGS:+ $CGO_CFLAGS}"
-    export CGO_LDFLAGS="${versioned_lib_dir}/liboffice_oxide.a"
+    export CGO_LDFLAGS="${OFFICE_OXIDE_PREFIX}/lib/liboffice_oxide.a"
 
     # ── pdfium ────────────────────────────────────────────────────────
     check_pdfium_deps || return 1
@@ -894,16 +935,11 @@ setup_cgo_env() {
         linux_amd64|linux_arm64|darwin_amd64|darwin_arm64) ;;
         *) echo "pdf_oxide: unsupported platform: $pdf_oxide_subdir"; return 1 ;;
     esac
-    # Version-stamp the archive path so an in-place .a upgrade invalidates
-    # Go's build cache. See the office_oxide block above for why the raw path
-    # is not enough: the cache key hashes CGO_LDFLAGS as a string, not the
-    # contents of the archives it names.
+    # The pdf_oxide prefix is already version-keyed
+    # (${PDF_OXIDE_PREFIX} = .../pdf_oxide/v<version>), so the linked path
+    # carries the version and Go's build cache relinks on a version bump.
     local pdf_oxide_lib_dir="${PDF_OXIDE_PREFIX}/lib/${pdf_oxide_subdir}"
-    local pdf_oxide_versioned_dir="${pdf_oxide_lib_dir}/v${PDF_OXIDE_VERSION}"
-    mkdir -p "$pdf_oxide_versioned_dir"
-    ln -sf "${pdf_oxide_lib_dir}/libpdf_oxide.a" \
-        "${pdf_oxide_versioned_dir}/libpdf_oxide.a"
-    export CGO_LDFLAGS="$CGO_LDFLAGS ${pdf_oxide_versioned_dir}/libpdf_oxide.a"
+    export CGO_LDFLAGS="$CGO_LDFLAGS ${pdf_oxide_lib_dir}/libpdf_oxide.a"
 
     # ── onnxruntime (static, resolved via dlopen(NULL)) ────────────────
     # Statically link libonnxruntime*.a into the binary. The org Go binding
@@ -930,11 +966,10 @@ setup_cgo_env() {
     #     all unused objects and missing transitive dependencies), and exports
     #     it via -export_dynamic so runtime dlopen(NULL) resolves it.
     #
-    # Seed the static ORT archives from the system pre-bake (/opt, laid down
-    # by the CI runner image) into the user cache before the link check
-    # below. Mirrors the existing _seed_from_system calls for the other
-    # native libs so CI never downloads ORT at build/test time.
-    _seed_from_system "onnxruntime" || true
+    # The static ORT archives come from the version-keyed user cache (populated
+    # by download_dev_deps.py and mounted in on CI) — there is no /opt pre-bake.
+    # An incomplete cache is filled by the controlled auto-download at the top
+    # of setup_cgo_env.
     check_onnxruntime_deps || return 1
 
     # Version-stamp the archive paths so an in-place .a upgrade invalidates

internal/development.md

@@ -214,11 +214,11 @@ The image contains Go DeepDoc weights, tokenizer assets, and the BPE table. Deep
 > Since `build.sh` (`build_go`) now **fails fast** when ORT is absent from
 > `CGO_LDFLAGS`, this breakage surfaces at build time instead of at runtime. If
 > you see `Error: ONNX Runtime static libraries are not linked`, run
-> `uv run tools/scripts/download_dev_deps.py` (or pre-seed
-> `/opt/ragflow-native-libs/onnxruntime` as the CI runner image does). There is
-> no ORT-free production build path — if ORT is absent the binary fails at
-> startup, so the remedy is always to seed the static lib above, never to build
-> without it.
+> `uv run tools/scripts/download_dev_deps.py` (or pre-seed the versioned user
+> cache at `${RAGFLOW_NATIVE_LIBS_DIR:-$HOME/ragflow-native-libs}/onnxruntime/
+> v<version>`; CI mounts the host cache into the builder). There is no ORT-free
+> production build path — if ORT is absent the binary fails at startup, so the
+> remedy is always to seed the static lib above, never to build without it.
 
 > **Note**: The ONNX Runtime native version is pinned in a few places that must
 > stay in sync. Bumping it in one spot and not the others fails the build with

tools/scripts/download_dev_deps.py

@@ -57,7 +57,9 @@ import hashlib
 import os
 import platform
 import re
+import shlex
 import shutil
+import subprocess
 import sys
 import tarfile
 import tempfile
@@ -66,11 +68,17 @@ import zipfile
 import requests
 
 # ── Dependency versions & URLs (edit here) ────────────────────────────────
-# This script's own copy of the native static-library versions and download URLs
-# it fetches. build.sh does NOT grep these (it pins PDFIUM/PDF_OXIDE itself and
-# reads ORT from internal/common/environments.go + the Dockerfile ARG, so it
-# stays tied to the checked-out source tree and the Docker build needs no COPY
-# of this script) — keep the two in sync when bumping a version.
+# The native-library VERSIONS are NOT hardcoded in this script. They are derived
+# from a single source of truth so build.sh and this downloader can never drift:
+#   - office_oxide / pdf_oxide  -> go.mod (go_module_version, below)
+#   - ORT                       -> internal/common/environments.go DeepDocORTVersion
+#                                  (deepdoc_ort_version, below)
+#   - pdfium                    -> tools/scripts/native_lib_version.sh
+#                                  pdfium_static_version (a shared const; pdfium
+#                                  is not a Go module so go.mod does not apply)
+# ORT_VERSION / PDFIUM_VERSION are bound to those derivations right after
+# REPO_ROOT is resolved (see the block following _repo_root()), so bumping a
+# version touches exactly one place upstream.
 
 # ONNX Runtime (statically linked into the Go in-process DeepDoc backend).
 #
@@ -83,7 +91,6 @@ import requests
 # patched content (e.g. the build that exports SessionGetInitializer*); the
 # downloader detects that via a `{asset}.sha256` sidecar and
 # re-downloads/re-extracts, so a stale local copy never silently lingers.
-ORT_VERSION = "1.29.0"
 
 # Release tag on infiniflow/ragflow-build. v1.29.0's Linux x86_64 minimal build
 # reduced operators to required_operators.config, which omitted Clip@13 — so
@@ -101,9 +108,9 @@ ORT_RELEASE_TAG = "onnxruntime-v1.29.0-patch2"
 PDF_OXIDE_VERSION = "0.3.73"
 
 # pdfium release, as the numeric build number after "chromium%2F" (the "v" and
-# the URL-encoded slash are added by pdfium_url). Matches build.sh's
-# PDFIUM_STATIC_VERSION.
-PDFIUM_VERSION = "7809"
+# the URL-encoded slash are added by pdfium_url). Its bare value is bound to
+# pdfium_static_version(REPO_ROOT) below (the single shared const, since pdfium
+# is not a Go module).
 
 
 def host_platform(target_os=None, target_arch=None):
@@ -185,6 +192,48 @@ def go_module_version(module: str, repo_root: str) -> str:
     return match.group(1)
 
 
+def deepdoc_ort_version(repo_root: str) -> str:
+    """Read the ONNX Runtime version the in-process (Go) DeepDoc backend links
+    from internal/common/environments.go's DeepDocORTVersion const.
+
+    ORT is the one native lib whose version is owned by Go source (not go.mod),
+    so both build.sh and this script parse the SAME const — no second hardcoded
+    copy to drift."""
+    path = os.path.join(repo_root, "internal", "common", "environments.go")
+    if not os.path.isfile(path):
+        raise RuntimeError(f"environments.go not found at {path}")
+    with open(path, encoding="utf-8") as file:
+        match = re.search(r'DeepDocORTVersion\s*=\s*"([^"]+)"', file.read())
+    if match is None:
+        raise RuntimeError(f"DeepDocORTVersion const not found in {path}")
+    return match.group(1)
+
+
+def pdfium_static_version(repo_root: str) -> str:
+    """Return the pinned pdfium build number from the single shared bash source
+    of truth tools/scripts/native_lib_version.sh.
+
+    pdfium (kognitos/pdfium-static) is not a Go module, so unlike
+    office_oxide / pdf_oxide it cannot be derived from go.mod; its bare numeric
+    build number is the one constant exported by pdfium_static_version() and
+    sourced by both this script and build.sh."""
+    script = os.path.join(repo_root, "tools", "scripts", "native_lib_version.sh")
+    if not os.path.isfile(script):
+        raise RuntimeError(f"native_lib_version.sh not found at {script}")
+    result = subprocess.run(
+        ["bash", "-c", f"source {shlex.quote(script)} >/dev/null 2>&1; pdfium_static_version"],
+        capture_output=True,
+        text=True,
+        check=False,
+    )
+    if result.returncode != 0:
+        raise RuntimeError(f"pdfium_static_version failed: {result.stderr.strip()}")
+    version = result.stdout.strip()
+    if not version:
+        raise RuntimeError("pdfium_static_version produced no output")
+    return version
+
+
 # ── URL builders ──────────────────────────────────────────────────────────
 # Every download URL is assembled from the constants above via one of these
 # helpers, so a version bump touches exactly one constant. Each takes the target
@@ -269,6 +318,15 @@ def _repo_root() -> str:
 
 
 REPO_ROOT = _repo_root()
+
+# Bind the native-library versions to their single source of truth (see the
+# version block above). Keeping these as module-level constants means every
+# downstream URL/dir builder still references ORT_VERSION / PDFIUM_VERSION, but
+# the bare values now live in exactly one place upstream (environments.go and
+# native_lib_version.sh) — bumping a version needs no edit here.
+ORT_VERSION = deepdoc_ort_version(REPO_ROOT)
+PDFIUM_VERSION = pdfium_static_version(REPO_ROOT)
+
 # Downloaded archives are anchored under deps/ (ignored by .gitignore
 # via *.tgz / deps/*.zip) regardless of the caller's working directory,
 # so a move of this script does not scatter large files into tools/scripts/.
@@ -276,6 +334,28 @@ RAGFLOW_DEPS_DIR = os.path.join(REPO_ROOT, "deps")
 NATIVE_LIBS_DIR = os.path.expanduser("~/ragflow-native-libs")
 
 
+def native_libs_dir():
+    """Resolve where native static libraries are written/read.
+
+    Honours RAGFLOW_NATIVE_LIBS_DIR so a CI builder can mount a pre-populated
+    user cache (e.g. ~/ragflow-native-libs) at a configurable path and the
+    downloader agrees with build.sh on that location. Falls back to the
+    in-home default when the variable is unset or empty.
+    """
+    return os.environ.get("RAGFLOW_NATIVE_LIBS_DIR") or NATIVE_LIBS_DIR
+
+
+def ensure_deps_dir():
+    """Make sure RAGFLOW_DEPS_DIR exists before any chdir into it.
+
+    In a product image build the go-builder stage does not carry a deps/
+    checkout, so a bare os.chdir(RAGFLOW_DEPS_DIR) would raise
+    FileNotFoundError before the auto-download fallback could run. Creating it
+    here lets the downloader proceed and fetch archives on demand.
+    """
+    os.makedirs(RAGFLOW_DEPS_DIR, exist_ok=True)
+
+
 def _sha256_of(path):
     """sha256 of a file, streamed in chunks so large archives don't blow memory."""
     h = hashlib.sha256()
@@ -342,10 +422,30 @@ def _valid_download(filename):
     return True
 
 
-def _extract_native_archive(archive_path, target, required_files, version=None):
+def _stamp_present(data, version, stamp_format):
+    """True when the .a archive DATA carries VERSION as a known stamp form.
+
+    Different native libs embed their version stamp differently:
+      - office_oxide embeds a null-delimited token (b"\\x00"+version+b"\\x00");
+      - pdf_oxide embeds a plain-text token (b"pdf_oxide "+version).
+    Accepting both keeps the extractor library-agnostic while still rejecting a
+    .a that does not actually contain the expected version."""
+    needle = version.encode()
+    if stamp_format == "null":
+        return b"\x00" + needle + b"\x00" in data
+    if stamp_format == "text":
+        return (b"pdf_oxide " + needle in data) or (b"\x00" + needle + b"\x00" in data)
+    return needle in data
+
+
+def _extract_native_archive(archive_path, target, required_files, version=None, stamp_format="null"):
     """Extract ARCHIVE_PATH into TARGET, verifying REQUIRED_FILES (and, for .a
-    files, an optional null-delimited VERSION stamp) before promoting the
-    staged tree.
+    files, an optional VERSION stamp) before promoting the staged tree.
+
+    STAMP_FORMAT selects how the version is embedded in the .a:
+      - "null": office_oxide's b"\\x00"+version+b"\\x00" token;
+      - "text": pdf_oxide's plain b"pdf_oxide "+version token (also accepts a
+        null-delimited token, so a format switch does not silently break).
 
     Path-traversal / absolute / non-regular members are rejected before
     extraction, and an incomplete archive raises instead of replacing a
@@ -358,8 +458,9 @@ def _extract_native_archive(archive_path, target, required_files, version=None):
                 return False
             if version is not None and name.endswith(".a"):
                 with open(path, "rb") as file:
-                    if b"\x00" + version.encode() + b"\x00" not in file.read():
-                        return False
+                    data = file.read()
+                if not _stamp_present(data, version, stamp_format):
+                    return False
         return True
 
     if complete(target):
@@ -388,6 +489,37 @@ def _extract_native_archive(archive_path, target, required_files, version=None):
     print(f"  ✓ extracted {os.path.basename(archive_path)} to {target}")
 
 
+def extract_native_lib(archive_path, subdir, dir_version, required_files, stamp_version=None, stamp_format="null"):
+    """Extract a native static archive into the version-keyed cache dir
+    ~/ragflow-native-libs/<subdir>/<dir_version>/.
+
+    The version-keyed dir (e.g. v7809) — NOT a flat layout — is what lets a
+    single mounted user cache hold several native-lib versions at once, so PRs
+    pinned to different go.mod versions all build offline. Idempotent: when the
+    target dir already holds the required files (matching stamp_version for
+    office_oxide) the extraction is skipped, so re-running the downloader is a
+    no-op for an already-provisioned cache.
+
+    STAMP_FORMAT is forwarded to _extract_native_archive (see its docstring):
+    "null" for office_oxide, "text" for pdf_oxide. pdfium passes stamp_version
+    None and is not stamp-checked.
+
+    Returns True when extraction ran or was skipped-successfully (the dir is in
+    place afterwards), False when the archive is absent (caller must then
+    download it). Re-raises the underlying tar/validation error after purging a
+    corrupt archive."""
+    if not os.path.isfile(archive_path):
+        print(f"  Skipping extraction: {os.path.basename(archive_path)} not found")
+        return False
+    target = os.path.join(native_libs_dir(), subdir, dir_version)
+    try:
+        _extract_native_archive(archive_path, target, required_files, stamp_version, stamp_format)
+    except (tarfile.TarError, EOFError, RuntimeError):
+        os.unlink(archive_path)
+        raise
+    return True
+
+
 def prune_stale_onnxruntime(static_lib_dir, expected_dir):
     """Remove ONNX Runtime version dirs under static_lib that do NOT match
     `expected_dir`. Without this, a version bump (or a foreign-arch run on a
@@ -495,7 +627,7 @@ def _download_ort_archive(archive_path, goos, goarch):
     if not _valid_download(archive_path):
         os.unlink(archive_path)
         raise RuntimeError(f"Invalid downloaded dependency: {os.path.basename(archive_path)}")
-    version_dir = os.path.join(NATIVE_LIBS_DIR, "onnxruntime", "static_lib", ort_normalized_dir(ORT_VERSION, goos, goarch))
+    version_dir = os.path.join(native_libs_dir(), "onnxruntime", "static_lib", ort_normalized_dir(ORT_VERSION, goos, goarch))
     if os.path.isdir(version_dir):
         print(f"  Removing stale extracted ONNX Runtime dir: {version_dir}")
         shutil.rmtree(version_dir)
@@ -701,6 +833,7 @@ def main():
 
     # Anchor the downloaded archives to deps/ regardless of the
     # caller's working directory (build.sh invokes this from the repo root).
+    ensure_deps_dir()
     os.chdir(RAGFLOW_DEPS_DIR)
 
     # ONNX Runtime archive (+ checksum sidecar), then the other native archives.
@@ -721,28 +854,22 @@ def main():
             raise RuntimeError(f"Invalid downloaded dependency: {filename}")
 
     # Extract native static libraries to ~/ragflow-native-libs for the Go build.
-    # Ensures build.sh can find them without network access.
+    # Ensures build.sh can find them without network access. Each lands under a
+    # version-keyed subdir (v<version>/) so a mounted cache can hold several
+    # versions at once; see extract_native_lib().
     office_version = go_module_version("github.com/yfedoseev/office_oxide/go", REPO_ROOT)
     extractions = [
-        (PDFIUM_STATIC_ASSETS[(goos, goarch)], "pdfium-static", ["lib/libpdfium.a", "lib/libc++.a", "lib/libc++abi.a", "include/fpdfview.h"], None),
-        (PDF_OXIDE_ASSETS[(goos, goarch)], "pdf_oxide", [f"lib/{goos}_{goarch}/libpdf_oxide.a", "include/pdf_oxide.h"], None),
-        (OFFICE_OXIDE_ASSETS[(goos, goarch)], "office_oxide", ["lib/liboffice_oxide.a", "include/office_oxide_c/office_oxide.h"], office_version),
+        ("pdfium-static", PDFIUM_STATIC_ASSETS[(goos, goarch)], "v" + PDFIUM_VERSION, ["lib/libpdfium.a", "lib/libc++.a", "lib/libc++abi.a", "include/fpdfview.h"], None, "null"),
+        ("pdf_oxide", PDF_OXIDE_ASSETS[(goos, goarch)], "v" + PDF_OXIDE_VERSION, [f"lib/{goos}_{goarch}/libpdf_oxide.a", "include/pdf_oxide.h"], PDF_OXIDE_VERSION, "text"),
+        ("office_oxide", OFFICE_OXIDE_ASSETS[(goos, goarch)], "v" + office_version, ["lib/liboffice_oxide.a", "include/office_oxide_c/office_oxide.h"], office_version, "null"),
     ]
 
-    for archive, subdir, required_files, version in extractions:
+    for subdir, archive, dir_version, required_files, stamp_version, stamp_format in extractions:
         archive_path = os.path.join(RAGFLOW_DEPS_DIR, archive)
-        if not os.path.isfile(archive_path):
-            print(f"  Skipping extraction: {archive} not found")
-            continue
-        target = os.path.join(NATIVE_LIBS_DIR, subdir)
-        try:
-            _extract_native_archive(archive_path, target, required_files, version)
-        except (tarfile.TarError, EOFError, RuntimeError):
-            os.unlink(archive_path)
-            raise
+        extract_native_lib(archive_path, subdir, dir_version, required_files, stamp_version, stamp_format)
 
     if not extract_onnxruntime(
-        os.path.join(NATIVE_LIBS_DIR, "onnxruntime", "static_lib"),
+        os.path.join(native_libs_dir(), "onnxruntime", "static_lib"),
         os.path.join(RAGFLOW_DEPS_DIR, ort_zip),
         ort_dir,
     ):
@@ -752,7 +879,7 @@ def main():
         # .a left here is exactly the "silent green" this split must prevent.
         print(
             f"  ERROR: ONNX Runtime static archives for {ort_dir} were not "
-            f"extracted to {os.path.join(NATIVE_LIBS_DIR, 'onnxruntime', 'static_lib')}. "
+            f"extracted to {os.path.join(native_libs_dir(), 'onnxruntime', 'static_lib')}. "
             f"Check the download above; build.sh will refuse to link without them.",
             file=sys.stderr,
         )
@@ -763,7 +890,7 @@ def main():
     # find the archives — the .a files live under
     # ~/ragflow-native-libs/onnxruntime/static_lib. The in-process backend
     # resolves OrtGetApiBase via dlopen(NULL); there is no dynamic .so fallback.
-    ort_static_dir = os.path.join(NATIVE_LIBS_DIR, "onnxruntime", "static_lib")
+    ort_static_dir = os.path.join(native_libs_dir(), "onnxruntime", "static_lib")
     ort_a_files = [os.path.join(root, f) for root, _, files in os.walk(ort_static_dir) for f in files if f.endswith(".a")]
     if ort_a_files:
         print(f"  ✓ onnxruntime static archives ready: {len(ort_a_files)} .a under {ort_static_dir}")
@@ -784,7 +911,7 @@ def main():
     download_tokenizer_assets(args.china_mirrors)
 
     print()
-    print(f"✓ Native build libraries ready under {NATIVE_LIBS_DIR}/")
+    print(f"✓ Native build libraries ready under {native_libs_dir()}/")
     print("  Run: bash build.sh (it will find them without network access).")
     print("  Runtime resources (cl100k / DeepDoc .ort / tokenizer) are also in place,")
     print("  so the local Go server is ready to build, run, parse and count tokens.")

tools/scripts/native_lib_version.sh

@@ -0,0 +1,58 @@
+#!/usr/bin/env bash
+# Single source of truth for the native static-library versions build.sh and
+# tools/scripts/download_dev_deps.py agree on. Source this file from build.sh;
+# it defines only functions (no top-level side effects) so it is safe to source.
+#
+# Each native lib resolves its version from exactly one place:
+#   - office_oxide / pdf_oxide  -> go.mod       (go_module_version)
+#   - ORT                       -> internal/common/environments.go
+#                                   DeepDocORTVersion (read by both build.sh and
+#                                   download_dev_deps.py directly — it is Go-owned)
+#   - pdfium                    -> pdfium_static_version (below; pdfium is not a
+#                                   Go module, so go.mod does not apply)
+#
+# There is NO /opt pre-bake and NO flat (version-less) cache layout any more: the
+# libs land under ~/ragflow-native-libs/<lib>/v<version>/ (see download_dev_deps.py
+# and build.sh), so a single mounted cache carries several versions at once.
+
+# ragflow_native_libs_dir
+# Root of the version-keyed native static-lib user cache
+# (~/ragflow-native-libs by default). Overridable via RAGFLOW_NATIVE_LIBS_DIR so
+# CI can mount the cache at a fixed host path (e.g. a RUNNER_WORKSPACE_PREFIX
+# bind mount) instead of the build user's $HOME — see sep-tests.yml Slice 3. An
+# EMPTY override falls back to the $HOME default (the ${VAR:-default} expansion),
+# so a half-configured runner never resolves to an empty cache root. Echoes the
+# resolved path and returns 0.
+ragflow_native_libs_dir() {
+    printf '%s\n' "${RAGFLOW_NATIVE_LIBS_DIR:-${HOME}/ragflow-native-libs}"
+}
+
+# go_module_version <module> <repo_root>
+# Reads the pinned version (the "vX.Y.Z" after the require line) of a Go module
+# from <repo_root>/go.mod so build.sh and tools/scripts/download_dev_deps.py agree
+# on the version (single source of truth). The module may appear either as a bare
+# require line (`\t<module> vX.Y.Z`) inside a require block or on a single-line
+# `require <module> vX.Y.Z`; an optional leading `require` keyword is allowed.
+# Echoes the bare version (no "v" prefix) and returns 0, or returns 1 with empty
+# output when go.mod is missing or the module is not required.
+go_module_version() {
+    local module="$1" repo_root="$2"
+    local go_mod="${repo_root}/go.mod"
+    [ -f "$go_mod" ] || return 1
+    local version
+    version="$(grep -m1 -E "^[[:space:]]*(require[[:space:]]+)?${module}[[:space:]]+v" "$go_mod" \
+        | sed -E 's/.*v([0-9][^[:space:]]*).*/\1/')"
+    [ -n "$version" ] || return 1
+    printf '%s\n' "$version"
+}
+
+# pdfium_static_version
+# pdfium (kognitos/pdfium-static) is NOT a Go module, so unlike office_oxide /
+# pdf_oxide its version cannot be derived from go.mod. It is pinned here as the
+# single shared constant; download_dev_deps.py and build.sh both source this
+# function so the bare numeric build number lives in exactly one place. Echoes
+# the constant and returns 0.
+pdfium_static_version() {
+    # Single source of truth for the pdfium static build number. Bump here only.
+    printf '%s\n' "7809"
+}

tools/scripts/test_download_dev_deps.py

@@ -22,6 +22,7 @@
 import importlib.util
 import io
 import os
+import subprocess
 import tarfile
 import threading
 import zipfile
@@ -315,6 +316,118 @@ def test_native_archive_rejects_unsafe_members(tmp_path, kind):
     assert sorted(path.name for path in tmp_path.iterdir()) == ["native", "outside", "unsafe.tar.gz"]
 
 
+# ── Version-stamped extraction (Slice 2): native .a must land under a
+# version-keyed subdir (v<version>/) of the user cache, never a flat layout, so a
+# single mounted cache can hold several versions at once (different PRs pinned to
+# different go.mod versions all build offline). extract_native_lib() is the
+# single extraction seam main() calls; it is idempotent (skips when the version
+# dir already holds the required files). ─────────────────────────────────────
+
+
+def _make_pdfium_archive(path, names):
+    with tarfile.open(path, "w:gz") as tf:
+        for name in names:
+            data = b"pdfium-payload-1234"  # non-empty so complete() accepts it
+            info = tarfile.TarInfo(name)
+            info.size = len(data)
+            tf.addfile(info, io.BytesIO(data))
+
+
+def test_extracts_pdfium_into_versioned_dir(tmp_path, monkeypatch):
+    monkeypatch.setattr(native, "NATIVE_LIBS_DIR", str(tmp_path / "native-libs"))
+    archive = tmp_path / "pdfium-linux-x64-static.tgz"
+    _make_pdfium_archive(archive, ["lib/libpdfium.a", "lib/libc++.a", "lib/libc++abi.a", "include/fpdfview.h"])
+    native.extract_native_lib(
+        str(archive),
+        "pdfium-static",
+        "v7809",
+        ["lib/libpdfium.a", "lib/libc++.a", "lib/libc++abi.a", "include/fpdfview.h"],
+        None,
+    )
+    target = tmp_path / "native-libs" / "pdfium-static" / "v7809"
+    assert (target / "lib" / "libpdfium.a").is_file()
+    assert (target / "include" / "fpdfview.h").is_file()
+
+
+def test_extract_native_lib_idempotent_on_rerun(tmp_path, monkeypatch):
+    monkeypatch.setattr(native, "NATIVE_LIBS_DIR", str(tmp_path / "native-libs"))
+    archive = tmp_path / "pdfium-linux-x64-static.tgz"
+    _make_pdfium_archive(archive, ["lib/libpdfium.a"])
+    native.extract_native_lib(str(archive), "pdfium-static", "v7809", ["lib/libpdfium.a"], None)
+    before = sorted(os.listdir(tmp_path / "native-libs" / "pdfium-static" / "v7809" / "lib"))
+    native.extract_native_lib(str(archive), "pdfium-static", "v7809", ["lib/libpdfium.a"], None)
+    after = sorted(os.listdir(tmp_path / "native-libs" / "pdfium-static" / "v7809" / "lib"))
+    assert before == after
+
+
+def _make_stamped_archive(path, lib_name, stamp, header_name):
+    """Build a tar.gz with the .a (carrying STAMP, a version-bearing byte
+    string) plus an include header, so complete()'s file-list AND stamp checks
+    can both be exercised per library format."""
+    with tarfile.open(path, "w:gz") as tf:
+        info = tarfile.TarInfo(lib_name)
+        payload = b"!<arch>\n" + stamp
+        info.size = len(payload)
+        tf.addfile(info, io.BytesIO(payload))
+        header = tarfile.TarInfo(header_name)
+        header.size = len(b"// header")
+        tf.addfile(header, io.BytesIO(b"// header"))
+
+
+def test_pdf_oxide_stamp_text_accepts_matching_version(tmp_path, monkeypatch):
+    # pdf_oxide embeds a plain-text `pdf_oxide <version>` token (not the
+    # null-delimited token office_oxide uses). complete() must accept it when
+    # stamp_format="text", or auto-download would reject a good archive and
+    # build.sh's controlled download path would fail closed.
+    monkeypatch.setattr(native, "NATIVE_LIBS_DIR", str(tmp_path / "native-libs"))
+    archive = tmp_path / "pdf_oxide-go-ffi-linux-amd64.tar.gz"
+    _make_stamped_archive(archive, "lib/linux_amd64/libpdf_oxide.a", b"pdf_oxide 0.3.73", "include/pdf_oxide.h")
+    native.extract_native_lib(
+        str(archive),
+        "pdf_oxide",
+        "v0.3.73",
+        ["lib/linux_amd64/libpdf_oxide.a", "include/pdf_oxide.h"],
+        "0.3.73",
+        "text",
+    )
+    assert (tmp_path / "native-libs" / "pdf_oxide" / "v0.3.73" / "lib" / "linux_amd64" / "libpdf_oxide.a").is_file()
+
+
+def test_pdf_oxide_stamp_text_rejects_wrong_version(tmp_path, monkeypatch):
+    # A pdf_oxide .a stamped with a different version must be rejected, not
+    # silently accepted (the whole point of the stamp check is to catch a
+    # mismatched/wrong native build).
+    monkeypatch.setattr(native, "NATIVE_LIBS_DIR", str(tmp_path / "native-libs"))
+    archive = tmp_path / "pdf_oxide-go-ffi-linux-amd64.tar.gz"
+    _make_stamped_archive(archive, "lib/linux_amd64/libpdf_oxide.a", b"pdf_oxide 0.9.9", "include/pdf_oxide.h")
+    with pytest.raises(RuntimeError, match="missing required native files or version 0.3.73"):
+        native.extract_native_lib(
+            str(archive),
+            "pdf_oxide",
+            "v0.3.73",
+            ["lib/linux_amd64/libpdf_oxide.a", "include/pdf_oxide.h"],
+            "0.3.73",
+            "text",
+        )
+
+
+def test_office_oxide_stamp_null_accepts_matching_version(tmp_path, monkeypatch):
+    # Regression guard: office_oxide keeps its null-delimited stamp token; the
+    # "text" vs "null" split must not break it.
+    monkeypatch.setattr(native, "NATIVE_LIBS_DIR", str(tmp_path / "native-libs"))
+    archive = tmp_path / "native-linux-x86_64.tar.gz"
+    _make_stamped_archive(archive, "lib/liboffice_oxide.a", b"\x000.1.16\x00", "include/office_oxide_c/office_oxide.h")
+    native.extract_native_lib(
+        str(archive),
+        "office_oxide",
+        "v0.1.16",
+        ["lib/liboffice_oxide.a", "include/office_oxide_c/office_oxide.h"],
+        "0.1.16",
+        "null",
+    )
+    assert (tmp_path / "native-libs" / "office_oxide" / "v0.1.16" / "lib" / "liboffice_oxide.a").is_file()
+
+
 # ── Local-dev one-stop: the dev downloader must also fetch the runtime
 # resources (cl100k / DeepDoc .ort / tokenizer assets) so a developer runs THIS
 # script ONCE and the Go server is ready to build, run, parse and count. ─────
@@ -560,3 +673,145 @@ def test_arm64_docs_section_warns_that_lld18_links_but_hangs():
     section = _arm64_docs_section()
     assert "lld-18" in section, "lld-18 is no longer called out as insufficient"
     assert "hang" in section.lower() or "spin" in section.lower(), "lld-18 startup-hang failure mode missing"
+
+
+# ── Single source of truth for native versions (Slice 1) ───────────────────────
+# build.sh and download_dev_deps.py must agree on every native version. The
+# authoritative values live in tools/scripts/native_lib_version.sh (office_oxide
+# / pdf_oxide from go.mod, ORT from environments.go, pdfium as one shared const).
+# These tests pin that contract: no second hardcoded copy may drift.
+
+
+def test_deepdoc_ort_version_reads_environments_go(tmp_path):
+    # Fixture environments.go with a deliberately non-default version.
+    common = tmp_path / "internal" / "common"
+    common.mkdir(parents=True)
+    (common / "environments.go").write_text('package common\n\n// DeepDocORTVersion is ...\nconst DeepDocORTVersion = "9.9.9"\n')
+    assert native.deepdoc_ort_version(str(tmp_path)) == "9.9.9"
+
+
+def test_deepdoc_ort_version_against_real_repo():
+    # Sanity: parses the checked-out repo's actual pin.
+    assert native.deepdoc_ort_version(native.REPO_ROOT) == "1.29.0"
+
+
+def test_pdfium_static_version_reads_single_source(tmp_path):
+    # Fixture native_lib_version.sh exporting a sentinel (NOT the real 7809).
+    scripts = tmp_path / "tools" / "scripts"
+    scripts.mkdir(parents=True)
+    (scripts / "native_lib_version.sh").write_text("pdfium_static_version() { echo 1234; }\n")
+    assert native.pdfium_static_version(str(tmp_path)) == "1234"
+
+
+def test_pdfium_static_version_against_real_repo():
+    # Sanity: the shared const on the checked-out repo.
+    assert native.pdfium_static_version(native.REPO_ROOT) == "7809"
+
+
+# ── Slice 3: CI mounts the native-lib user cache (RAGFLOW_NATIVE_LIBS_DIR) ──
+# build.sh resolves the native static libs from a single cache root
+# (default ~/ragflow-native-libs, overridable via RAGFLOW_NATIVE_LIBS_DIR so CI
+# can mount the cache at a fixed host path instead of the build user's $HOME).
+# These tests pin that resolution contract (native_lib_version.sh) and the
+# sep-tests.yml wiring that mounts the cache into every BUILDER_CONTAINER.
+
+
+def _sh_eval(func_call, env):
+    """Source the real native_lib_version.sh and call FUNC_CALL in a throwaway
+    bash, returning the CompletedProcess. ENV is the full env (HOME/PATH must be
+    supplied), mirroring how build.sh sources the file."""
+    sh = os.path.join(native.REPO_ROOT, "tools", "scripts", "native_lib_version.sh")
+    assert os.path.isfile(sh), f"native_lib_version.sh not found at {sh}"
+    cmd = f'source "{sh}" && {func_call}'
+    return subprocess.run(["bash", "-c", cmd], env=env, capture_output=True, text=True, check=False)
+
+
+def _env_with(home, override=None):
+    env = {"HOME": home, "PATH": os.environ.get("PATH", "")}
+    if override is not None:
+        env["RAGFLOW_NATIVE_LIBS_DIR"] = override
+    return env
+
+
+def test_native_libs_dir_defaults_to_home():
+    # No override: resolve to $HOME/ragflow-native-libs.
+    res = _sh_eval("ragflow_native_libs_dir", _env_with("/home/alice"))
+    assert res.returncode == 0, res.stderr
+    assert res.stdout.strip() == "/home/alice/ragflow-native-libs"
+
+
+def test_native_libs_dir_honors_override():
+    # Override set: CI mounts the cache at a fixed host path; build.sh must use it.
+    res = _sh_eval("ragflow_native_libs_dir", _env_with("/home/alice", "/mnt/cache"))
+    assert res.returncode == 0, res.stderr
+    assert res.stdout.strip() == "/mnt/cache"
+
+
+def test_native_libs_dir_empty_override_falls_back():
+    # Adversarial: an explicitly EMPTY override must fall back to the HOME default
+    # (the ${VAR:-default} expansion), never resolve to an empty cache root.
+    res = _sh_eval("ragflow_native_libs_dir", _env_with("/home/alice", ""))
+    assert res.returncode == 0, res.stderr
+    assert res.stdout.strip() == "/home/alice/ragflow-native-libs"
+
+
+SEP_TESTS_YML = os.path.join(native.REPO_ROOT, ".github", "workflows", "sep-tests.yml")
+
+
+def test_sep_tests_mounts_native_libs_cache():
+    # Slice 3 contract: the native static-lib user cache is mounted into every
+    # BUILDER_CONTAINER (4 docker runs) and exposed via RAGFLOW_NATIVE_LIBS_DIR,
+    # replacing the old /opt pre-bake. The YAML must also stay parseable.
+    assert os.path.isfile(SEP_TESTS_YML), f"sep-tests.yml not found at {SEP_TESTS_YML}"
+    with open(SEP_TESTS_YML, encoding="utf-8") as f:
+        content = f.read()
+    # YAML must still parse (guards against a malformed edit); skip when pyyaml
+    # is unavailable in the test env (the CI runner has it).
+    try:
+        import yaml
+
+        yaml.safe_load(content)
+    except ImportError:
+        pass
+    # The cache root is declared as a job env so runner and builder container agree.
+    assert "RAGFLOW_NATIVE_LIBS_DIR" in content
+    # Each of the 4 BUILDER_CONTAINER docker runs bind-mounts the cache.
+    assert content.count("${RAGFLOW_NATIVE_LIBS_DIR}:${RAGFLOW_NATIVE_LIBS_DIR}") >= 4
+    # A fail-fast guard catches a missing/partial mount before a cryptic cgo link error.
+    assert "native-lib cache not mounted" in content
+
+
+# ── downloader root resolution must honour RAGFLOW_NATIVE_LIBS_DIR (P1#2) ─────
+# build.sh resolves the cache root through native_lib_version.ragflow_native_libs_dir()
+# (${RAGFLOW_NATIVE_LIBS_DIR:-~/ragflow-native-libs}); the downloader must agree so a
+# runner that overrides the mount path does not silently download into the default.
+
+
+def test_python_native_libs_dir_default(monkeypatch):
+    monkeypatch.delenv("RAGFLOW_NATIVE_LIBS_DIR", raising=False)
+    assert native.native_libs_dir() == os.path.expanduser("~/ragflow-native-libs")
+
+
+def test_python_native_libs_dir_honors_override(monkeypatch):
+    # Adversarial: a custom mount path must be returned verbatim, NOT the default.
+    monkeypatch.setenv("RAGFLOW_NATIVE_LIBS_DIR", "/srv/custom/native-libs")
+    assert native.native_libs_dir() == "/srv/custom/native-libs"
+
+
+def test_python_native_libs_dir_empty_override_falls_back(monkeypatch):
+    # Adversarial: an empty override must fall back to the default, not resolve to "".
+    monkeypatch.setenv("RAGFLOW_NATIVE_LIBS_DIR", "")
+    assert native.native_libs_dir() == os.path.expanduser("~/ragflow-native-libs")
+
+
+# ── downloader must not crash when deps/ is absent (P1#3) ─────────────────────
+
+
+def test_ensure_deps_dir_creates_missing(monkeypatch, tmp_path):
+    # In the Docker go-builder stage nothing COPYs deps/, so the downloader's
+    # os.chdir(deps) used to raise FileNotFoundError and abort the image build.
+    # ensure_deps_dir() must create it first.
+    missing = tmp_path / "no_deps_dir"
+    monkeypatch.setattr(native, "RAGFLOW_DEPS_DIR", str(missing))
+    native.ensure_deps_dir()
+    assert missing.is_dir()