Provision native libs from versioned user cache (offline CI, drop /opt pre-bake) (#22099)

This commit is contained in:
xugangqiang authored and GitHub committed 2026-11-09 18:07:31 +08:00
1 parent 2b4d628d51
commit e5a51b680b
11 files changed
+1523 -187

No files matched your search

+4 -3
View File
@@ -38,9 +38,10 @@ target/
# regardless of which resources are prepared for the dependency image.
# The deps image is built from this directory with:
# cd deps && docker build -f Dockerfile -t infiniflow/ragflow_deps .
# The native downloader tools/scripts/download_dev_deps.py is copied by the
# main Dockerfile (not excluded here) so build.sh can read its ORT version pin
# when running `./build.sh --go`.
# tools/scripts is NOT excluded here: the main Dockerfile COPYs it so build.sh
# can `source tools/scripts/native_lib_version.sh` (it does so at the top of the
# script) and reference tools/scripts/download_dev_deps.py in its remediation
# messages while building `./build.sh --go`.
deps/*
# ── IDE and editor config ──────────────────────────────────────────────────
+86
View File
@@ -37,6 +37,12 @@ concurrency:
# backend to build against github.com/infiniflow/onnxruntime_go.
env:
GOPRIVATE: github.com/infiniflow/onnxruntime_go
# Native static-lib cache location on infiniflow's self-hosted runners.
# build.sh resolves the libs from this dir (default ~/ragflow-native-libs);
# the offline-first contract keeps the version-keyed cache here so the
# fail-fast guard and the builder bind-mount both find it without an /opt
# pre-bake. Overridable per environment.
RAGFLOW_NATIVE_LIBS_DIR: /home/infiniflow/runners_work/runners_data/ragflow-native-libs
jobs:
ragflow_preflight:
@@ -243,6 +249,16 @@ jobs:
export GOCACHE="${RUNNER_WORKSPACE_PREFIX}/gobuild"
fi
# Fail fast if the native-lib user cache is not mounted: a missing
# cache would otherwise surface as a cryptic cgo link error later.
NATIVE_LIBS_DIR="${RAGFLOW_NATIVE_LIBS_DIR:-${HOME}/ragflow-native-libs}"
for lib in office_oxide pdfium-static pdf_oxide onnxruntime; do
if [ ! -d "${NATIVE_LIBS_DIR}/${lib}" ]; then
echo "ERROR: ${NATIVE_LIBS_DIR}/${lib} missing — native-lib cache not mounted on this runner." >&2
exit 1
fi
done
BUILDER_CONTAINER=ragflow_build_${GITHUB_RUN_ID}_${DOC_ENGINE}_$(od -An -N4 -tx4 /dev/urandom | tr -d ' ')
cleanup_builder() {
if [[ -n "${BUILDER_CONTAINER:-}" ]]; then
@@ -252,11 +268,21 @@ jobs:
trap cleanup_builder EXIT
TZ=${TZ:-$(readlink -f /etc/localtime | awk -F '/zoneinfo/' '{print $2}')}
# Mount the native static-lib user cache into the builder container.
# build.sh resolves the libs from RAGFLOW_NATIVE_LIBS_DIR (default
# ~/ragflow-native-libs); this pins the same root for the container and
# bind-mounts it read-only so the cgo link finds the versioned
# v<version>/ dirs without an /opt pre-bake. CI mounts the cache from
# the build host (populated by download_dev_deps.py); a missing mount is
# caught by the assert below, not by a cryptic cgo link error.
export RAGFLOW_NATIVE_LIBS_DIR="${RAGFLOW_NATIVE_LIBS_DIR:-${HOME}/ragflow-native-libs}"
sudo docker run --privileged -d --name "${BUILDER_CONTAINER}" \
-e TZ="${TZ}" \
-e UV_INDEX=https://mirrors.aliyun.com/pypi/simple \
-e "RAGFLOW_NATIVE_LIBS_DIR=${RAGFLOW_NATIVE_LIBS_DIR}" \
-v "${PWD}:/ragflow" \
-v "${PWD}/internal/binding/cpp/resource:/usr/share/infinity/resource" \
-v "${RAGFLOW_NATIVE_LIBS_DIR}:${RAGFLOW_NATIVE_LIBS_DIR}:ro" \
infiniflow/infinity_builder:ubuntu22_clang20
sudo docker exec "${BUILDER_CONTAINER}" bash -c 'git config --global safe.directory "*" && cd /ragflow && ./build.sh --cpp'
./build.sh --go
@@ -810,6 +836,16 @@ jobs:
export GOCACHE="${RUNNER_WORKSPACE_PREFIX}/gobuild"
fi
# Fail fast if the native-lib user cache is not mounted: a missing
# cache would otherwise surface as a cryptic cgo link error later.
NATIVE_LIBS_DIR="${RAGFLOW_NATIVE_LIBS_DIR:-${HOME}/ragflow-native-libs}"
for lib in office_oxide pdfium-static pdf_oxide onnxruntime; do
if [ ! -d "${NATIVE_LIBS_DIR}/${lib}" ]; then
echo "ERROR: ${NATIVE_LIBS_DIR}/${lib} missing — native-lib cache not mounted on this runner." >&2
exit 1
fi
done
BUILDER_CONTAINER=ragflow_build_${GITHUB_RUN_ID}_${DOC_ENGINE}_$(od -An -N4 -tx4 /dev/urandom | tr -d ' ')
cleanup_builder() {
if [[ -n "${BUILDER_CONTAINER:-}" ]]; then
@@ -819,11 +855,21 @@ jobs:
trap cleanup_builder EXIT
TZ=${TZ:-$(readlink -f /etc/localtime | awk -F '/zoneinfo/' '{print $2}')}
# Mount the native static-lib user cache into the builder container.
# build.sh resolves the libs from RAGFLOW_NATIVE_LIBS_DIR (default
# ~/ragflow-native-libs); this pins the same root for the container and
# bind-mounts it read-only so the cgo link finds the versioned
# v<version>/ dirs without an /opt pre-bake. CI mounts the cache from
# the build host (populated by download_dev_deps.py); a missing mount is
# caught by the assert below, not by a cryptic cgo link error.
export RAGFLOW_NATIVE_LIBS_DIR="${RAGFLOW_NATIVE_LIBS_DIR:-${HOME}/ragflow-native-libs}"
sudo docker run --privileged -d --name "${BUILDER_CONTAINER}" \
-e TZ="${TZ}" \
-e UV_INDEX=https://mirrors.aliyun.com/pypi/simple \
-e "RAGFLOW_NATIVE_LIBS_DIR=${RAGFLOW_NATIVE_LIBS_DIR}" \
-v "${PWD}:/ragflow" \
-v "${PWD}/internal/binding/cpp/resource:/usr/share/infinity/resource" \
-v "${RAGFLOW_NATIVE_LIBS_DIR}:${RAGFLOW_NATIVE_LIBS_DIR}:ro" \
infiniflow/infinity_builder:ubuntu22_clang20
sudo docker exec "${BUILDER_CONTAINER}" bash -c 'git config --global safe.directory "*" && cd /ragflow && ./build.sh --cpp'
./build.sh --go
@@ -1386,6 +1432,16 @@ jobs:
# integration tests; the per-engine go jobs build the full server.
run: |
set -euo pipefail
# Fail fast if the native-lib user cache is not mounted: a missing
# cache would otherwise surface as a cryptic cgo link error later.
NATIVE_LIBS_DIR="${RAGFLOW_NATIVE_LIBS_DIR:-${HOME}/ragflow-native-libs}"
for lib in office_oxide pdfium-static pdf_oxide onnxruntime; do
if [ ! -d "${NATIVE_LIBS_DIR}/${lib}" ]; then
echo "ERROR: ${NATIVE_LIBS_DIR}/${lib} missing — native-lib cache not mounted on this runner." >&2
exit 1
fi
done
BUILDER_CONTAINER=ragflow_build_${GITHUB_RUN_ID}_native_$(od -An -N4 -tx4 /dev/urandom | tr -d ' ')
cleanup_builder() {
if [[ -n "${BUILDER_CONTAINER:-}" ]]; then
@@ -1395,11 +1451,21 @@ jobs:
trap cleanup_builder EXIT
TZ=${TZ:-$(readlink -f /etc/localtime | awk -F '/zoneinfo/' '{print $2}')}
# Mount the native static-lib user cache into the builder container.
# build.sh resolves the libs from RAGFLOW_NATIVE_LIBS_DIR (default
# ~/ragflow-native-libs); this pins the same root for the container and
# bind-mounts it read-only so the cgo link finds the versioned
# v<version>/ dirs without an /opt pre-bake. CI mounts the cache from
# the build host (populated by download_dev_deps.py); a missing mount is
# caught by the assert below, not by a cryptic cgo link error.
export RAGFLOW_NATIVE_LIBS_DIR="${RAGFLOW_NATIVE_LIBS_DIR:-${HOME}/ragflow-native-libs}"
sudo docker run --privileged -d --name "${BUILDER_CONTAINER}" \
-e TZ="${TZ}" \
-e UV_INDEX=https://mirrors.aliyun.com/pypi/simple \
-e "RAGFLOW_NATIVE_LIBS_DIR=${RAGFLOW_NATIVE_LIBS_DIR}" \
-v "${PWD}:/ragflow" \
-v "${PWD}/internal/binding/cpp/resource:/usr/share/infinity/resource" \
-v "${RAGFLOW_NATIVE_LIBS_DIR}:${RAGFLOW_NATIVE_LIBS_DIR}:ro" \
infiniflow/infinity_builder:ubuntu22_clang20
sudo docker exec "${BUILDER_CONTAINER}" bash -c 'git config --global safe.directory "*" && cd /ragflow && ./build.sh --cpp'
@@ -1486,6 +1552,16 @@ jobs:
# starts only the dependency services, not the ragflow-cpu container.
run: |
set -euo pipefail
# Fail fast if the native-lib user cache is not mounted: a missing
# cache would otherwise surface as a cryptic cgo link error later.
NATIVE_LIBS_DIR="${RAGFLOW_NATIVE_LIBS_DIR:-${HOME}/ragflow-native-libs}"
for lib in office_oxide pdfium-static pdf_oxide onnxruntime; do
if [ ! -d "${NATIVE_LIBS_DIR}/${lib}" ]; then
echo "ERROR: ${NATIVE_LIBS_DIR}/${lib} missing — native-lib cache not mounted on this runner." >&2
exit 1
fi
done
BUILDER_CONTAINER=ragflow_build_${GITHUB_RUN_ID}_go_int_$(od -An -N4 -tx4 /dev/urandom | tr -d ' ')
cleanup_builder() {
if [[ -n "${BUILDER_CONTAINER:-}" ]]; then
@@ -1494,11 +1570,21 @@ jobs:
}
trap cleanup_builder EXIT
TZ=${TZ:-$(readlink -f /etc/localtime | awk -F '/zoneinfo/' '{print $2}')}
# Mount the native static-lib user cache into the builder container.
# build.sh resolves the libs from RAGFLOW_NATIVE_LIBS_DIR (default
# ~/ragflow-native-libs); this pins the same root for the container and
# bind-mounts it read-only so the cgo link finds the versioned
# v<version>/ dirs without an /opt pre-bake. CI mounts the cache from
# the build host (populated by download_dev_deps.py); a missing mount is
# caught by the assert below, not by a cryptic cgo link error.
export RAGFLOW_NATIVE_LIBS_DIR="${RAGFLOW_NATIVE_LIBS_DIR:-${HOME}/ragflow-native-libs}"
sudo docker run --privileged -d --name "${BUILDER_CONTAINER}" \
-e TZ="${TZ}" \
-e UV_INDEX=https://mirrors.aliyun.com/pypi/simple \
-e "RAGFLOW_NATIVE_LIBS_DIR=${RAGFLOW_NATIVE_LIBS_DIR}" \
-v "${PWD}:/ragflow" \
-v "${PWD}/internal/binding/cpp/resource:/usr/share/infinity/resource" \
-v "${RAGFLOW_NATIVE_LIBS_DIR}:${RAGFLOW_NATIVE_LIBS_DIR}:ro" \
infiniflow/infinity_builder:ubuntu22_clang20
sudo docker exec "${BUILDER_CONTAINER}" bash -c 'git config --global safe.directory "*" && cd /ragflow && ./build.sh --cpp'
+13 -11
View File
@@ -227,19 +227,21 @@ RUN --mount=type=cache,id=ragflow_gomod,target=/root/.cache/gomod,sharing=locked
COPY internal internal
COPY cmd cmd
COPY build.sh ./
COPY tools/scripts tools/scripts
COPY Dockerfile ./
# ONNX Runtime static archives: build.sh's _seed_from_system copies the ORT
# static libs from the runner's pre-baked /opt/ragflow-native-libs/onnxruntime
# (arch-correct per TARGETARCH) into the user-cache path ONNXRUNTIME_STATIC_PREFIX
# before linking. These .a files are consumed at link time only and do NOT enter
# the final image (only the compiled /ragflow/bin is COPY --from=go-builder).
# The Dockerfile no longer pre-copies them here: pre-creating the user-cache dir
# would make _seed_from_system treat it as "already cached" and skip the copy on
# the non-matching architecture (e.g. the arm64 runner has no x64 dir), so we
# rely on build.sh to seed ORT for both amd64 and arm64. ARG ORT_VERSION stays
# because build.sh's check_ort_version_consistency greps it to cross-check the
# pinned ORT release against internal/common/environments.go.
# ONNX Runtime static archives are provisioned by build.sh from the versioned
# user cache at ONNXRUNTIME_STATIC_PREFIX (${RAGFLOW_NATIVE_LIBS_DIR:-$HOME/
# ragflow-native-libs}/onnxruntime/static_lib). There is NO /opt pre-bake: on CI
# the host cache is mounted in at build time, and for the product image build
# (where no host cache is mounted) build.sh falls back to _ensure_native_libs_installed,
# which runs `uv run tools/scripts/download_dev_deps.py`. That downloader fetches
# the native archives on demand and self-creates its deps/ workdir when absent
# (see download_dev_deps.py::ensure_deps_dir), so it no longer crashes on a bare
# go-builder checkout. These .a files are consumed at link time only and do NOT
# enter the final image (only the compiled /ragflow/bin is COPY --from=go-builder).
# ARG ORT_VERSION stays because build.sh's check_ort_version_consistency greps it
# to cross-check the pinned ORT release against internal/common/environments.go.
ARG ORT_VERSION=1.29.0
RUN git config --global safe.directory "*" && \
+165 -110
View File
@@ -15,7 +15,25 @@ PROJECT_ROOT="$SCRIPT_DIR"
# untouched and merge conflicts against the EE branch are limited to this
# single line plus the guarded hook call inside run_native_integration_tests.
[[ -f "${SCRIPT_DIR}/build_ee.sh" ]] && source "${SCRIPT_DIR}/build_ee.sh"
# Version-keyed native-lib resolution: each native lib sits under a
# v<version>/ subdir of the user cache (~/ragflow-native-libs/<lib>/v<version>/),
# so a single mounted cache holds several versions at once. The version numbers
# come from go.mod (office_oxide / pdf_oxide), environments.go (ORT) and
# native_lib_version.sh (pdfium) — see that file for the single source of truth.
source "${SCRIPT_DIR}/tools/scripts/native_lib_version.sh"
# ort_stamp.sh: pure helper that computes the relative, read-only-safe ORT
# version-stamp symlink target (Fix #1/#2).
source "${SCRIPT_DIR}/tools/scripts/ort_stamp.sh"
# ort_present.sh: pure helper that decides whether the required ORT static lib
# is already in the version+platform-keyed user cache (Fix: version/platform
# aware presence gate).
source "${SCRIPT_DIR}/tools/scripts/ort_present.sh"
# Native static-lib user-cache root. Overridable via RAGFLOW_NATIVE_LIBS_DIR so
# CI can mount the cache at a fixed host path (sep-tests.yml Slice 3); defaults
# to ~/ragflow-native-libs. Captured once here and reused by every <lib>/v<version>
# prefix below so the whole script agrees on a single cache root.
NATIVE_LIBS_DIR="$(ragflow_native_libs_dir)"
# Build directories
CPP_DIR="$PROJECT_ROOT/internal/binding/cpp"
BUILD_DIR="$CPP_DIR/cmake-build-release"
@@ -43,31 +61,42 @@ if [ -n "$EXTRA_GO_TAGS" ]; then
fi
# Native static library settings. These are the user-cache paths (~/ragflow-native-libs/).
# If /opt/ragflow-native-libs/ exists (pre-seeded in CI runner image), it takes priority
# and skips the network (download_dev_deps.py) fallback.
SYSTEM_DEPS="/opt/ragflow-native-libs"
# Each lib lives under a version-keyed subdir (v<version>/) placed there by
# tools/scripts/download_dev_deps.py (idempotent, version-pinned) — run once by a
# developer, or mounted in from the build host on CI. There is NO /opt pre-bake and
# NO flat (version-less) fallback: a missing version dir means the cache is
# incomplete and build.sh triggers the downloader itself (see setup_cgo_env).
# office_oxide native library settings — static linking
OFFICE_OXIDE_PREFIX="${HOME}/ragflow-native-libs/office_oxide"
# Derived from go.mod (single source of truth: the Go module version). Matches
# tools/scripts/download_dev_deps.py's office_oxide_url derivation. The pipeline exits 0
# with an empty value when go.mod has no matching entry (no pipefail here), so
# the empty case needs its own check: an empty version would slip past the
# stale-lib comparison below and produce a broken .../lib/v link path.
OFFICE_OXIDE_VERSION="$(grep -m1 -E '^[[:space:]]*github.com/yfedoseev/office_oxide/go[[:space:]]+v' "$PROJECT_ROOT/go.mod" | sed -E 's/.*v([0-9][^[:space:]]*).*/\1/')"
# office_oxide native library settings — static linking. Version is derived from
# go.mod (single source of truth, exactly as tools/scripts/download_dev_deps.py
# derives it). An empty value (module absent from go.mod) would produce a broken
# .../lib/v link path, so it is fatal. The prefix is set after the version, since
# it embeds the version.
OFFICE_OXIDE_VERSION="$(go_module_version "github.com/yfedoseev/office_oxide/go" "$PROJECT_ROOT")"
if [ -z "$OFFICE_OXIDE_VERSION" ]; then
echo -e "${RED}Error: could not read office_oxide version from go.mod${NC}" >&2; exit 1
fi
OFFICE_OXIDE_PREFIX="${NATIVE_LIBS_DIR}/office_oxide/v${OFFICE_OXIDE_VERSION}"
# pdfium native library settings — static linking (kognitos/pdfium-static)
# Pinned here (not grepped from the dev downloader) so build.sh stays tied to the
# checked-out source tree and the Docker build needs no extra COPY of the script.
PDFIUM_STATIC_PREFIX="${HOME}/ragflow-native-libs/pdfium-static"
PDFIUM_STATIC_VERSION="7809"
# pdfium native library settings — static linking (kognitos/pdfium-static).
# Version is read from the single source of truth tools/scripts/native_lib_version.sh
# (pdfium_static_version), exactly as download_dev_deps.py does, so the bare build
# number lives in exactly one place. An empty value is fatal.
PDFIUM_STATIC_VERSION="$(pdfium_static_version "$PROJECT_ROOT")"
if [ -z "$PDFIUM_STATIC_VERSION" ]; then
echo -e "${RED}Error: could not read pdfium version from native_lib_version.sh${NC}" >&2; exit 1
fi
PDFIUM_STATIC_PREFIX="${NATIVE_LIBS_DIR}/pdfium-static/v${PDFIUM_STATIC_VERSION}"
# pdf_oxide native library settings — static linking (go-ffi tarball)
PDF_OXIDE_PREFIX="${HOME}/ragflow-native-libs/pdf_oxide"
PDF_OXIDE_VERSION="0.3.73"
# pdf_oxide native library settings — static linking (go-ffi tarball). Version
# is read from go.mod (single source of truth) instead of a hardcoded constant,
# so bumping the dependency needs no build.sh edit. The empty case is fatal for
# the same reason as office_oxide. The prefix is set after the version.
PDF_OXIDE_VERSION="$(go_module_version "github.com/yfedoseev/pdf_oxide/go" "$PROJECT_ROOT")"
if [ -z "$PDF_OXIDE_VERSION" ]; then
echo -e "${RED}Error: could not read pdf_oxide version from go.mod${NC}" >&2; exit 1
fi
PDF_OXIDE_PREFIX="${NATIVE_LIBS_DIR}/pdf_oxide/v${PDF_OXIDE_VERSION}"
# onnxruntime native library settings — static linking for the in-process
# (Go) DeepDoc backend. libonnxruntime*.a is linked into the server binary
@@ -75,7 +104,7 @@ PDF_OXIDE_VERSION="0.3.73"
# is then resolved via dlopen(NULL) (the process-global symbol table, not the
# executable's own path), so no libonnxruntime.so is needed at runtime.
# Downloaded by tools/scripts/download_dev_deps.py into onnxruntime/static_lib.
ONNXRUNTIME_STATIC_PREFIX="${HOME}/ragflow-native-libs/onnxruntime/static_lib"
ONNXRUNTIME_STATIC_PREFIX="${NATIVE_LIBS_DIR}/onnxruntime/static_lib"
# ── target platform detection ──────────────────────────────────────────
# Defaults to the build host (uname), overridable with RAGFLOW_TARGET_OS /
@@ -155,27 +184,62 @@ ort_static_lib_prefix() {
esac
}
# Copy a dependency from the system pre-seed directory to the user cache.
# Returns 0 if the dep was copied or already exists in cache, 1 otherwise.
_seed_from_system() {
local dep_name="$1" # e.g. "pdfium-static", "pdf_oxide", "office_oxide"
local dep_dir="${HOME}/ragflow-native-libs/${dep_name}"
local sys_dir="${SYSTEM_DEPS}/${dep_name}"
# ── native static library provisioning ──────────────────────────────────
# The native .a files live in the version-keyed user cache
# (~/ragflow-native-libs/<lib>/v<version>/), placed there by
# tools/scripts/download_dev_deps.py — idempotently and version-pinned — which a
# developer runs once, and which CI mounts in from the build host. There is NO
# /opt pre-bake and NO flat (version-less) fallback any more: a missing version
# dir means the cache is incomplete, so build.sh triggers the downloader itself
# (controlled auto-download) rather than silently linking a stale or wrong lib.
echo "check if dep ${dep_name} exists in ${dep_dir} or ${sys_dir}"
if [ -d "$dep_dir" ]; then
echo " ${dep_name} → ${dep_dir} (user cache)"
return 0 # already cached
# True when every native lib the Go build links is already in the versioned user
# cache for the target platform. Used to skip the network entirely on an
# already-provisioned host.
_native_libs_present() {
[ -f "${OFFICE_OXIDE_PREFIX}/lib/liboffice_oxide.a" ] || return 1
[ -f "${OFFICE_OXIDE_PREFIX}/include/office_oxide_c/office_oxide.h" ] || return 1
[ -f "${PDFIUM_STATIC_PREFIX}/lib/libpdfium.a" ] || return 1
[ -f "${PDF_OXIDE_PREFIX}/lib/${TARGET_OS}_${TARGET_ARCH}/libpdf_oxide.a" ] || return 1
# ORT must match the SAME version+platform-keyed dir that check_onnxruntime_deps
# links (*/${ort_dir_prefix}${ort_version}*/lib/*.a). A shared cache holding
# only an OLD version or a DIFFERENT platform's ORT must be reported missing so
# the controlled auto-downloader runs, instead of being silently treated as
# present and failing only at link time.
if [ -z "$_TARGET_PLATFORM_DETECTED" ]; then
detect_target_platform
_TARGET_PLATFORM_DETECTED=1
fi
if [ -d "$sys_dir" ]; then
echo " ${dep_name} → ${sys_dir} (system)"
mkdir -p "$(dirname "$dep_dir")"
cp -r "$sys_dir" "$dep_dir"
local ort_version ort_dir_prefix
ort_version="$(deepdoc_ort_version "$PROJECT_ROOT")" || return 1
ort_dir_prefix="$(ort_static_lib_prefix)"
ragflow_ort_present "$ONNXRUNTIME_STATIC_PREFIX" "$ort_dir_prefix" "$ort_version" || return 1
return 0
}
# Controlled auto-download: if the versioned native libs are not yet in the user
# cache, run the one-stop downloader ONCE. It is idempotent and version-pinned,
# so it can only land the exact pinned .a — never a wrong version. Fail loud only
# when the cache is still incomplete afterwards (a locked-down host with no
# network and no pre-placed libs).
_ensure_native_libs_installed() {
_native_libs_present && return 0
echo "One or more native static libraries are missing from the user cache;"
echo " running downloader: uv run tools/scripts/download_dev_deps.py"
( cd "$PROJECT_ROOT" && uv run tools/scripts/download_dev_deps.py ) || {
echo -e "${RED}Error: native library downloader failed.${NC}" >&2
exit 1
}
if _native_libs_present; then
return 0
fi
echo " ${dep_name} not found in system or user cache"
return 1
echo -e "${RED}Error: native static libraries still missing after download.${NC}" >&2
echo " Pre-place the version-keyed dirs manually on a locked-down host:" >&2
echo " ${OFFICE_OXIDE_PREFIX}" >&2
echo " ${PDFIUM_STATIC_PREFIX}" >&2
echo " ${PDF_OXIDE_PREFIX}" >&2
echo " ${ONNXRUNTIME_STATIC_PREFIX}" >&2
exit 1
}
# ── cl100k_base BPE table ─────────────────────────────────────────────
@@ -333,7 +397,7 @@ check_ort_version_consistency() {
print_section "Checking ONNX Runtime version consistency"
local env_go dockerfile
env_go="$(grep -m1 -E 'DeepDocORTVersion[[:space:]]*=[[:space:]]*"' "${PROJECT_ROOT}/internal/common/environments.go" | sed -E 's/.*"([^"]+)".*/\1/')"
env_go="$(deepdoc_ort_version "${PROJECT_ROOT}")"
dockerfile="$(grep -m1 -E 'ARG[[:space:]]+ORT_VERSION=' "${PROJECT_ROOT}/Dockerfile" | sed -E 's/.*ORT_VERSION=([0-9][^"[:space:]]*).*/\1/')"
if [ -z "$env_go" ] || [ -z "$dockerfile" ]; then
@@ -366,8 +430,9 @@ check_ort_version_consistency() {
# that lacks those custom slots links successfully and only fails at runtime.
# The check below catches that by comparing the local release ZIP against the
# published {asset}.sha256 sidecar (same source of truth the download scripts
# use). Devs who ran a download script have the zip under deps/; CI
# seeds from /opt and has no zip, where the bake is authoritative.
# use). Devs who ran a download script have the zip under deps/; on CI the
# version-keyed cache is mounted from the build host and no zip is present, so
# the sha256 check is skipped there and the mounted cache seed is authoritative.
check_onnxruntime_deps() {
# Ensure TARGET_OS/TARGET_ARCH are resolved. ort_static_lib_prefix below
# depends on them; without this guard an early call (before any other path
@@ -380,9 +445,7 @@ check_onnxruntime_deps() {
fi
local ort_version
ort_version="$(grep -m1 -E 'DeepDocORTVersion[[:space:]]*=[[:space:]]*"' \
"${PROJECT_ROOT}/internal/common/environments.go" \
| sed -E 's/.*"([^"]+)".*/\1/')"
ort_version="$(deepdoc_ort_version "${PROJECT_ROOT}")"
if [ -z "$ort_version" ]; then
echo -e "${RED}Error: cannot parse DeepDocORTVersion from internal/common/environments.go${NC}" >&2
return 1
@@ -400,6 +463,11 @@ check_onnxruntime_deps() {
# another OS/arch, and the Python side legitimately keeps another ORT
# version in the same prefix. Matching both is why coexisting version
# dirs need no separate fail-fast guard here.
# The ORT static archives are resolved from the version-keyed user cache
# (~/ragflow-native-libs/onnxruntime/static_lib, which the downloader
# populates and CI mounts in). There is no /opt pre-bake to reseed from; an
# incomplete cache is filled by the controlled auto-download in setup_cgo_env.
local ort_dir_prefix
ort_dir_prefix="$(ort_static_lib_prefix)"
while IFS= read -r f; do
@@ -432,7 +500,7 @@ check_onnxruntime_deps() {
echo " available: ${avail:-<none>}" >&2
echo " DeepDocORTVersion=${ort_version}; fetch it with:" >&2
echo " uv run tools/scripts/download_dev_deps.py" >&2
echo " or pre-seed /opt/ragflow-native-libs/onnxruntime (CI image)." >&2
echo " or pre-seed ${NATIVE_LIBS_DIR}/onnxruntime (CI mounts the cache from the build host)." >&2
return 1
fi
@@ -495,7 +563,9 @@ check_onnxruntime_deps() {
# Check office_oxide native library
check_office_oxide_deps() {
print_section "Checking office_oxide native library"
_seed_from_system "office_oxide" || true
# office_oxide is resolved from the version-keyed user cache
# (${OFFICE_OXIDE_PREFIX}). An incomplete cache is filled by the controlled
# auto-download in setup_cgo_env; there is no /opt pre-bake to recover from.
local lib_file="liboffice_oxide.a"
local lib_path="${OFFICE_OXIDE_PREFIX}/lib/${lib_file}"
@@ -516,28 +586,11 @@ check_office_oxide_deps() {
if ! strings "$lib_path" 2>/dev/null | grep -Fxq "$OFFICE_OXIDE_VERSION"; then
local found_version
found_version=$(strings "$lib_path" 2>/dev/null | grep -E "^0\.[0-9]+\.[0-9]+$" | head -1)
echo -e "${YELLOW}office_oxide native lib version mismatch (required v${OFFICE_OXIDE_VERSION}; found: ${found_version:-unknown}); refreshing from download${NC}"
rm -rf "${OFFICE_OXIDE_PREFIX}" "${PROJECT_ROOT}/deps/$(office_oxide_asset)"
if ! (cd "${PROJECT_ROOT}" && uv run tools/scripts/download_dev_deps.py); then
echo -e "${RED}Error: office_oxide native lib version mismatch${NC}"
echo " Required: v${OFFICE_OXIDE_VERSION}; found: ${found_version:-unknown}"
echo " A stale lib silently loses PPT97 (.ppt) slide content. Refresh:"
echo " rm -rf ~/ragflow-native-libs/office_oxide deps/$(office_oxide_asset)"
echo " uv run tools/scripts/download_dev_deps.py"
exit 1
fi
lib_path="${OFFICE_OXIDE_PREFIX}/lib/${lib_file}"
header_path="${OFFICE_OXIDE_PREFIX}/include/office_oxide_c/office_oxide.h"
if [ ! -f "$lib_path" ] || [ ! -f "$header_path" ]; then
echo -e "${RED}Error: office_oxide native library not found after refresh${NC}"
exit 1
fi
if ! strings "$lib_path" 2>/dev/null | grep -Fxq "$OFFICE_OXIDE_VERSION"; then
found_version=$(strings "$lib_path" 2>/dev/null | grep -E "^0\.[0-9]+\.[0-9]+$" | head -1)
echo -e "${RED}Error: office_oxide native lib version mismatch after refresh${NC}"
echo " Required: v${OFFICE_OXIDE_VERSION}; found: ${found_version:-unknown}"
exit 1
fi
echo -e "${RED}Error: office_oxide native lib version mismatch${NC}"
echo " Required: v${OFFICE_OXIDE_VERSION}; found: ${found_version:-unknown}"
echo " A stale lib silently loses PPT97 (.ppt) slide content. Refresh:"
echo " uv run tools/scripts/download_dev_deps.py # re-extracts into ${OFFICE_OXIDE_PREFIX}"
exit 1
fi
echo "✓ office_oxide v${OFFICE_OXIDE_VERSION} native library found at ${OFFICE_OXIDE_PREFIX}"
@@ -546,7 +599,9 @@ check_office_oxide_deps() {
# Check pdfium static library.
check_pdfium_deps() {
_seed_from_system "pdfium-static" || true
# pdfium is resolved from the version-keyed user cache
# (${PDFIUM_STATIC_PREFIX}). An incomplete cache is filled by the controlled
# auto-download in setup_cgo_env; there is no /opt pre-bake to recover from.
local lib_path="${PDFIUM_STATIC_PREFIX}/lib/libpdfium.a"
if [ -f "$lib_path" ]; then
@@ -604,7 +659,9 @@ pdf_oxide_validate_version() {
}
check_pdf_oxide_deps() {
_seed_from_system "pdf_oxide" || true
# pdf_oxide is resolved from the version-keyed user cache
# (${PDF_OXIDE_PREFIX}). An incomplete cache is filled by the controlled
# auto-download in setup_cgo_env; there is no /opt pre-bake to recover from.
# Map platform to tarball-internal subdirectory (lib/<os>_<arch>/). Reuses
# TARGET_OS/TARGET_ARCH set by detect_target_platform instead of re-deriving
# from uname, so it stays consistent with the other check_*_deps helpers.
@@ -617,10 +674,10 @@ check_pdf_oxide_deps() {
local lib_path="${PDF_OXIDE_PREFIX}/lib/${platform_subdir}/libpdf_oxide.a"
if [ -f "$lib_path" ]; then
# Verify the on-disk lib matches the pinned version. _seed_from_system
# accepts an existing user-cache directory without inspecting it, so a
# lib left over from an earlier pin is reused silently and the upgrade
# becomes a no-op.
# Verify the on-disk lib matches the pinned version. The cache is
# version-keyed (${PDF_OXIDE_PREFIX}), so an earlier pin lives under a
# different v<version>/ dir and is never silently reused; this check is a
# defense-in-depth against a stale .a that somehow landed in this dir.
#
# The version marker "pdf_oxide <version>" is merged into Rust's string
# constant pool (unlike office_oxide's standalone "0.1.9" line), so we
@@ -798,12 +855,12 @@ build_go() {
echo " at startup with a fatal 'no in-process DeepDoc backend serving'." >&2
echo " Fetch the static libs with:" >&2
echo " uv run tools/scripts/download_dev_deps.py" >&2
echo " or pre-seed them at /opt/ragflow-native-libs/onnxruntime (CI image)." >&2
echo " or pre-seed them at ${NATIVE_LIBS_DIR}/onnxruntime (CI mounts the cache from the build host)." >&2
echo " This production binary must statically include ORT — there is no" >&2
echo " ORT-free build path. ORT ends up unlinked only via one of:" >&2
echo " - the ORT static_lib dir was never seeded: run" >&2
echo " 'uv run tools/scripts/download_dev_deps.py', or pre-seed" >&2
echo " /opt/ragflow-native-libs/onnxruntime as the CI image does;" >&2
echo " ${NATIVE_LIBS_DIR}/onnxruntime as the CI mount does;" >&2
echo " - setup_cgo_env did not add libonnxruntime to CGO_LDFLAGS." >&2
return 1
fi
@@ -837,25 +894,22 @@ build_go() {
setup_cgo_env() {
detect_target_platform
# Controlled auto-download: fill any missing version-keyed native lib from
# the user cache before resolving individual libs. On an already-provisioned
# host (the common CI case, where the cache is mounted in) this is a no-op;
# on a fresh host it runs the one-stop downloader once. There is no /opt
# pre-bake and no flat fallback to recover from.
_ensure_native_libs_installed
# ── office_oxide ──────────────────────────────────────────────────
check_office_oxide_deps
# Go's build cache keys CGO_LDFLAGS as a string — it does NOT hash the
# file content of referenced .a archives. So swapping the .a in-place
# (same path) does NOT invalidate the cache, and `go build` silently
# reuses a stale binary linked against the old .a.
#
# Create a version-stamped symlink directory so the flag string
# includes the actual linked version. When the .a is upgraded, the
# path changes → Go cache key changes → automatic relink.
local office_oxide_lib_dir="${OFFICE_OXIDE_PREFIX}/lib"
local versioned_lib_dir="${office_oxide_lib_dir}/v${OFFICE_OXIDE_VERSION}"
mkdir -p "$versioned_lib_dir"
ln -sf "${office_oxide_lib_dir}/liboffice_oxide.a" \
"${versioned_lib_dir}/liboffice_oxide.a"
# The office_oxide prefix is already version-keyed
# (${OFFICE_OXIDE_PREFIX} = .../office_oxide/v<version>), so the linked path
# carries the version and Go's build cache (which keys CGO_LDFLAGS as a
# string, not the .a contents) automatically relinks when the version bumps.
export CGO_CFLAGS="-I${OFFICE_OXIDE_PREFIX}/include/office_oxide_c${CGO_CFLAGS:+ $CGO_CFLAGS}"
export CGO_LDFLAGS="${versioned_lib_dir}/liboffice_oxide.a"
export CGO_LDFLAGS="${OFFICE_OXIDE_PREFIX}/lib/liboffice_oxide.a"
# ── pdfium ────────────────────────────────────────────────────────
check_pdfium_deps || return 1
@@ -894,16 +948,11 @@ setup_cgo_env() {
linux_amd64|linux_arm64|darwin_amd64|darwin_arm64) ;;
*) echo "pdf_oxide: unsupported platform: $pdf_oxide_subdir"; return 1 ;;
esac
# Version-stamp the archive path so an in-place .a upgrade invalidates
# Go's build cache. See the office_oxide block above for why the raw path
# is not enough: the cache key hashes CGO_LDFLAGS as a string, not the
# contents of the archives it names.
# The pdf_oxide prefix is already version-keyed
# (${PDF_OXIDE_PREFIX} = .../pdf_oxide/v<version>), so the linked path
# carries the version and Go's build cache relinks on a version bump.
local pdf_oxide_lib_dir="${PDF_OXIDE_PREFIX}/lib/${pdf_oxide_subdir}"
local pdf_oxide_versioned_dir="${pdf_oxide_lib_dir}/v${PDF_OXIDE_VERSION}"
mkdir -p "$pdf_oxide_versioned_dir"
ln -sf "${pdf_oxide_lib_dir}/libpdf_oxide.a" \
"${pdf_oxide_versioned_dir}/libpdf_oxide.a"
export CGO_LDFLAGS="$CGO_LDFLAGS ${pdf_oxide_versioned_dir}/libpdf_oxide.a"
export CGO_LDFLAGS="$CGO_LDFLAGS ${pdf_oxide_lib_dir}/libpdf_oxide.a"
# ── onnxruntime (static, resolved via dlopen(NULL)) ────────────────
# Statically link libonnxruntime*.a into the binary. The org Go binding
@@ -930,11 +979,10 @@ setup_cgo_env() {
# all unused objects and missing transitive dependencies), and exports
# it via -export_dynamic so runtime dlopen(NULL) resolves it.
#
# Seed the static ORT archives from the system pre-bake (/opt, laid down
# by the CI runner image) into the user cache before the link check
# below. Mirrors the existing _seed_from_system calls for the other
# native libs so CI never downloads ORT at build/test time.
_seed_from_system "onnxruntime" || true
# The static ORT archives come from the version-keyed user cache (populated
# by download_dev_deps.py and mounted in on CI) — there is no /opt pre-bake.
# An incomplete cache is filled by the controlled auto-download at the top
# of setup_cgo_env.
check_onnxruntime_deps || return 1
# Version-stamp the archive paths so an in-place .a upgrade invalidates
@@ -944,15 +992,22 @@ setup_cgo_env() {
# linked binary. Stamp with the primary archive's sha256 so the flag string
# changes when the content changes → automatic relink.
local ort_version
ort_version="$(grep -m1 -E 'DeepDocORTVersion[[:space:]]*=[[:space:]]*"' \
"${PROJECT_ROOT}/internal/common/environments.go" \
| sed -E 's/.*"([^"]+)".*/\1/')"
ort_version="$(deepdoc_ort_version "${PROJECT_ROOT}")"
local ort_versioned_dir="${ONNXRUNTIME_STATIC_PREFIX}/v${ort_version}-${ORT_A_SHA256:0:16}"
mkdir -p "$ort_versioned_dir"
local ort_a_versioned=""
local f
local f stamp needs_write want_target
for f in $ORT_A_FILES; do
ln -sf "$f" "$ort_versioned_dir/$(basename "$f")"
# Relative, read-only-safe stamp (Fix #1/#2): the symlink target is
# relative to the versioned dir so it resolves under any cache mount, and
# the write is skipped when an existing link already points at the same
# archive (a read-only CI mount is then a no-op).
stamp="$(ragflow_ort_stamp_target "$ort_versioned_dir" "$f")"
needs_write="${stamp%%|*}"
want_target="${stamp#*|}"
if [ "$needs_write" = "yes" ]; then
mkdir -p "$ort_versioned_dir"
ln -sf "$want_target" "$ort_versioned_dir/$(basename "$f")"
fi
ort_a_versioned="$ort_a_versioned $ort_versioned_dir/$(basename "$f")"
done
+51 -19
View File
@@ -183,13 +183,40 @@ The image contains Go DeepDoc weights, tokenizer assets, and the BPE table. Deep
> SYS_LIBS="-fuse-ld=lld -lm -lpthread -ldl -lrt -lgcc_s -lutil -lc"
> fi
>
> export CGO_CFLAGS="-I${RAGFLOW_DEPS}/office_oxide/include/office_oxide_c"
> # Native libs are version-keyed under <lib>/v<version>/ (one mounted cache can
> # hold several versions at once), matching the layout download_dev_deps.py and
> # build.sh resolve. Resolve each version from its single source of truth — NOT
> # by glob + `head -1`, which silently pins the lexicographically-oldest version
> # in the cache (e.g. v0.1.12 over the pinned v0.1.16, dropping PPT97 slide
> # content). build.sh does exactly this in setup_cgo_env(); mirror it here.
> # Run this snippet from the ragflow repo root (where go.mod lives).
> source tools/scripts/native_lib_version.sh
> REPO_ROOT="$(pwd)"
> OFFICE_OXIDE_VERSION="$(go_module_version "github.com/yfedoseev/office_oxide/go" "$REPO_ROOT")"
> PDFIUM_STATIC_VERSION="$(pdfium_static_version "$REPO_ROOT")"
> PDF_OXIDE_VERSION="$(go_module_version "github.com/yfedoseev/pdf_oxide/go" "$REPO_ROOT")"
>
> OFFICE_OXIDE_PREFIX="${RAGFLOW_DEPS}/office_oxide/v${OFFICE_OXIDE_VERSION}"
> PDFIUM_PREFIX="${RAGFLOW_DEPS}/pdfium-static/v${PDFIUM_STATIC_VERSION}"
> PDF_OXIDE_PREFIX="${RAGFLOW_DEPS}/pdf_oxide/v${PDF_OXIDE_VERSION}"
>
> OFFICE_OXIDE_A="${OFFICE_OXIDE_PREFIX}/lib/liboffice_oxide.a"
> PDFIUM_A="${PDFIUM_PREFIX}/lib/libpdfium.a"
> PDFIUM_CXX_A="${PDFIUM_PREFIX}/lib/libc++.a"
> PDFIUM_CXXABI_A="${PDFIUM_PREFIX}/lib/libc++abi.a"
> PDF_OXIDE_A="${PDF_OXIDE_PREFIX}/lib/${PLATFORM}/libpdf_oxide.a"
>
> # Derive the include dir from the resolved office_oxide archive: the shell does
> # not expand globs inside double quotes, so a literal
> # ".../office_oxide/v*/include/..." passed to CGO_CFLAGS would never resolve and
> # cgo would fail to find office_oxide.h on the first native compile.
> export CGO_CFLAGS="-I$(dirname "$(dirname "$OFFICE_OXIDE_A")")/include/office_oxide_c"
> export CGO_LDFLAGS="\
> ${RAGFLOW_DEPS}/office_oxide/lib/liboffice_oxide.a \
> ${RAGFLOW_DEPS}/pdfium-static/lib/libpdfium.a \
> ${RAGFLOW_DEPS}/pdfium-static/lib/libc++.a \
> ${RAGFLOW_DEPS}/pdfium-static/lib/libc++abi.a \
> ${RAGFLOW_DEPS}/pdf_oxide/lib/${PLATFORM}/libpdf_oxide.a \
> ${OFFICE_OXIDE_A} \
> ${PDFIUM_A} \
> ${PDFIUM_CXX_A} \
> ${PDFIUM_CXXABI_A} \
> ${PDF_OXIDE_A} \
> ${ORT_LINK} \
> ${SYS_LIBS}"
> ```
@@ -214,23 +241,27 @@ The image contains Go DeepDoc weights, tokenizer assets, and the BPE table. Deep
> Since `build.sh` (`build_go`) now **fails fast** when ORT is absent from
> `CGO_LDFLAGS`, this breakage surfaces at build time instead of at runtime. If
> you see `Error: ONNX Runtime static libraries are not linked`, run
> `uv run tools/scripts/download_dev_deps.py` (or pre-seed
> `/opt/ragflow-native-libs/onnxruntime` as the CI runner image does). There is
> no ORT-free production build path — if ORT is absent the binary fails at
> startup, so the remedy is always to seed the static lib above, never to build
> without it.
> `uv run tools/scripts/download_dev_deps.py` (or pre-seed the versioned user
> cache at `${RAGFLOW_NATIVE_LIBS_DIR:-$HOME/ragflow-native-libs}/onnxruntime/static_lib
> `; CI mounts the host cache into the builder). There is no ORT-free
> production build path — if ORT is absent the binary fails at startup, so the
> remedy is always to seed the static lib above, never to build without it.
> **Note**: The ONNX Runtime native version is pinned in a few places that must
> stay in sync. Bumping it in one spot and not the others fails the build with
> **Note**: The ONNX Runtime native version is pinned in two places that must
> stay in sync. Bumping one and not the other fails the build with
> `Error: ONNX Runtime version is inconsistent`:
> - `internal/common/environments.go` — `DeepDocORTVersion`
> - `Dockerfile` — `ARG ORT_VERSION`
> - `tools/scripts/download_dev_deps.py` — `ORT_VERSION` (used by the dev downloader to fetch the matching release; build.sh does not grep it, but it must match what gets linked)
>
> `build.sh` and `tools/scripts/download_dev_deps.py` derive the ORT version at
> build/download time from `DeepDocORTVersion` in `environments.go` (the Go const
> is the single source of truth), so the downloader has no separate `ORT_VERSION`
> pin to edit. To upgrade ORT, edit the two entries above to the same version,
> then run the check.
>
> `build.sh` runs this consistency check automatically before the Go build
> (through `check_go_deps`) and fails fast on any mismatch. Run it on demand
> with `./build.sh --check-ort-version`. To upgrade ORT, edit every entry above
> to the same version, then run the check. The Python pip `onnxruntime==` pin in
> with `./build.sh --check-ort-version`. The Python pip `onnxruntime==` pin in
> `pyproject.toml` is versioned independently and is intentionally not part of
> this check.
@@ -243,9 +274,10 @@ The image contains Go DeepDoc weights, tokenizer assets, and the BPE table. Deep
> published `onnxruntime-v<ver>-<target>.zip.sha256` sidecar. A local ZIP
> mismatch only emits a warning with the expected/actual SHA-256 and the
> refresh command (`rm -f <zip>` + `uv run tools/scripts/download_dev_deps.py`);
> the build continues checking and linking the extracted static libraries. CI seeds
> ORT from `/opt` and has no local zip, so the check is skipped there (the bake
> is authoritative). This matters because the `onnxruntime_go` binding reaches
> the build continues checking and linking the extracted static libraries. On CI
> the version-keyed cache is mounted from the build host and no local ZIP is
> present, so the sha256 check is skipped there and the mounted cache seed is
> treated as authoritative. This matters because the `onnxruntime_go` binding reaches
> ORT only through the OrtApi function-pointer table
> (`ort_api->SessionGetInitializer*`), so a `.a` lacking a custom slot links
> successfully and only crashes at runtime — not at link time.
+308
View File
@@ -0,0 +1,308 @@
// Package docguard holds regression guards for repository documentation and
// build glue that the native-lib versioned-cache change (PR #22099) must keep
// correct. These are plain unit tests (no build tag) so they run in the default
// `bash build.sh --test` tier and in CI.
//
// The guards exist because the IDE/`go build` CGO snippet in
// internal/development.md is the one piece of build glue that is NOT executed by
// CI: a broken snippet only fails for the developer who copy-pastes it. These
// tests make the P1 review findings (review 5821944375) falsifiable — an
// un-expanded `v*` include glob, a glob+`head -1` version resolution that pins
// the oldest cache version, and stale `/opt` pre-bake prose — fail a test rather
// than relying on prose review alone.
package docguard
import (
"os"
"os/exec"
"path/filepath"
"runtime"
"strings"
"testing"
)
// repoRoot walks up from this source file to the directory containing go.mod.
func repoRoot(t *testing.T) string {
t.Helper()
_, thisFile, _, ok := runtime.Caller(0)
if !ok {
t.Fatal("runtime.Caller failed")
}
dir := filepath.Dir(thisFile)
for i := 0; i < 12; i++ {
if _, err := os.Stat(filepath.Join(dir, "go.mod")); err == nil {
return dir
}
parent := filepath.Dir(dir)
if parent == dir {
break
}
dir = parent
}
t.Fatal("could not locate repo root (go.mod)")
return ""
}
func readFile(t *testing.T, path string) string {
t.Helper()
b, err := os.ReadFile(path)
if err != nil {
t.Fatalf("read %s: %v", path, err)
}
return string(b)
}
// TestDevelopmentDocResolvesNativeLibVersions guards the P1 finding from review
// 5821944375: the IDE/`go build` CGO snippet must resolve each native lib from
// its single source of truth (the version-pinned resolvers) instead of a glob +
// `head -1`, and the include path must be derived from the resolved archive
// rather than a literal `v*` glob that the shell never expands inside quotes.
func TestDevelopmentDocResolvesNativeLibVersions(t *testing.T) {
doc := readFile(t, filepath.Join(repoRoot(t), "internal", "development.md"))
// The include path must no longer be a literal `v*` glob inside the
// double-quoted CGO_CFLAGS assignment — that glob is never expanded by the
// shell and breaks the first cgo compile.
if strings.Contains(doc, "office_oxide/v*/include/office_oxide_c") {
t.Error("development.md still has the un-expanded `v*` include glob in CGO_CFLAGS; derive the include from the resolved archive instead")
}
// The `.a` archives must no longer be resolved by `ls …/v*/… | head -1`,
// which silently pins the lexicographically-oldest version in a cache that
// holds several versions at once.
if strings.Contains(doc, "v*/lib/liboffice_oxide.a | head -1") {
t.Error("development.md still resolves office_oxide via `ls …/v*… | head -1`; use the version-pinned resolver")
}
if strings.Contains(doc, "v*/lib/libpdfium.a | head -1") {
t.Error("development.md still resolves pdfium via `ls …/v*… | head -1`; use the version-pinned resolver")
}
// Positive guard: the snippet must resolve versions from the single source
// of truth and derive the include from the resolved archive variable.
if !strings.Contains(doc, "native_lib_version.sh") {
t.Error("development.md should source tools/scripts/native_lib_version.sh to resolve versions from the single source of truth")
}
if !strings.Contains(doc, "OFFICE_OXIDE_A") {
t.Error("development.md should resolve the office_oxide archive into OFFICE_OXIDE_A and derive CGO_CFLAGS from it")
}
}
// normalizeDoc strips markdown blockquote prefixes ('>') and collapses
// newlines so multi-line blockquote sentences can be matched as a single
// string. The native-lib notes in development.md are long blockquotes that wrap
// across lines, so a raw substring search misses the stale phrases.
func normalizeDoc(s string) string {
s = strings.ReplaceAll(s, "\n", " ")
s = strings.ReplaceAll(s, ">", " ")
return strings.Join(strings.Fields(s), " ")
}
// TestDevelopmentDocNoStaleOptBake guards against the superseded /opt pre-bake
// description that the PR deleted from build.sh but that the doc still asserts.
func TestDevelopmentDocNoStaleOptBake(t *testing.T) {
doc := normalizeDoc(readFile(t, filepath.Join(repoRoot(t), "internal", "development.md")))
if strings.Contains(doc, "seeds ORT from") {
t.Error("development.md still claims CI seeds ORT from /opt; the /opt pre-bake was removed")
}
if strings.Contains(doc, "bake is authoritative") {
t.Error("development.md still references the /opt pre-bake as authoritative; the pre-bake was removed")
}
}
// TestBuildShNoStaleOptBake guards the stale comment inside
// check_onnxruntime_deps that contradicts the new "There is no /opt pre-bake"
// comment a few lines below it.
func TestBuildShNoStaleOptBake(t *testing.T) {
script := readFile(t, filepath.Join(repoRoot(t), "build.sh"))
if strings.Contains(script, "seeds from /opt") {
t.Error("build.sh still claims CI seeds ORT from /opt inside check_onnxruntime_deps; the /opt pre-bake was removed")
}
if strings.Contains(script, "bake is authoritative") {
t.Error("build.sh still references the /opt pre-bake as authoritative; the /opt pre-bake was removed")
}
}
// ortVersionNote extracts the "pinned in a few places" note block from
// development.md so the ORT pin-list guards can assert on that note alone.
func ortVersionNote(t *testing.T) string {
t.Helper()
doc := readFile(t, filepath.Join(repoRoot(t), "internal", "development.md"))
start := strings.Index(doc, "ONNX Runtime native version is pinned")
if start < 0 {
t.Fatal("development.md has no ORT version-pinning note")
}
endMarker := "build.sh` runs this consistency check"
end := strings.Index(doc[start:], endMarker)
if end < 0 {
t.Fatal("development.md ORT version note has no recognizable end")
}
return normalizeDoc(doc[start : start+end])
}
// TestDevelopmentDocOrtPinsAreNotMisleading guards P1 sub-finding 3a: the note
// used to list tools/scripts/download_dev_deps.py's ORT_VERSION as a THIRD pin
// to edit when bumping ORT. That constant is derived at runtime from
// DeepDocORTVersion (environments.go) — check_ort_version_consistency compares
// only environments.go and the Dockerfile — so telling developers to edit the
// downloader sends them to the wrong place. The note must (a) not present the
// downloader as a pin bullet, and (b) still list the two real pins.
func TestDevelopmentDocOrtPinsAreNotMisleading(t *testing.T) {
note := ortVersionNote(t)
// (a) No bullet that presents the downloader's ORT_VERSION as a pin to edit.
if strings.Contains(note, "- `tools/scripts/download_dev_deps.py`") {
t.Error("development.md still lists download_dev_deps.py ORT_VERSION as a pin to edit; it is derived from DeepDocORTVersion, not a pin")
}
// (a) The downloader derives its version from the Go const — no second copy.
if !strings.Contains(note, "derive") || !strings.Contains(note, "DeepDocORTVersion") {
t.Error("development.md should explain that build.sh and download_dev_deps.py derive ORT_VERSION from DeepDocORTVersion, so there is no third pin to edit")
}
// (b) The two real pin locations must remain documented.
if !strings.Contains(note, "internal/common/environments.go") || !strings.Contains(note, "Dockerfile") {
t.Error("development.md must still list the two real ORT pins (environments.go:DeepDocORTVersion and Dockerfile:ARG ORT_VERSION)")
}
}
// TestNativeLibVersionPinnedNotGlob is the adversarial behavioural guard for the
// config-pinning requirement: with two versions present in the cache, the
// resolver must return the version pinned in go.mod (pinned != latest), whereas
// the naive `ls …/v*… | head -1` would silently pick the lexicographically
// oldest. This is exactly the failure mode the doc snippet used to have.
func TestNativeLibVersionPinnedNotGlob(t *testing.T) {
root := repoRoot(t)
versionSh := filepath.Join(root, "tools", "scripts", "native_lib_version.sh")
if _, err := os.Stat(versionSh); err != nil {
t.Fatalf("native_lib_version.sh missing: %v", err)
}
// Build a fake repo root with go.mod pinning office_oxide v0.1.16 and a
// cache that also carries an OLDER v0.1.12 — the adversarial case where a
// glob+head -1 would wrongly select the stale version.
tmp := t.TempDir()
goMod := "module example.com/ragflow\n\nrequire github.com/yfedoseev/office_oxide/go v0.1.16\n"
if err := os.WriteFile(filepath.Join(tmp, "go.mod"), []byte(goMod), 0o644); err != nil {
t.Fatal(err)
}
cache := filepath.Join(tmp, "ragflow-native-libs", "office_oxide")
for _, v := range []string{"v0.1.12", "v0.1.16"} {
dir := filepath.Join(cache, v, "lib")
if err := os.MkdirAll(dir, 0o755); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(dir, "liboffice_oxide.a"), []byte("fake"), 0o644); err != nil {
t.Fatal(err)
}
}
// Resolve the pinned version through the shared resolver.
out, err := exec.Command("bash", "-c",
sourceCmd(versionSh)+" && go_module_version \"github.com/yfedoseev/office_oxide/go\" "+tmp,
).Output()
if err != nil {
t.Fatalf("go_module_version failed: %v (%s)", err, out)
}
got := strings.TrimSpace(string(out))
if got != "0.1.16" {
t.Fatalf("go_module_version returned %q, want pinned 0.1.16", got)
}
// Demonstrate the bug the fix prevents: the naive glob+head -1 picks the
// lexicographically-oldest version, not the pinned one.
naive, err := exec.Command("bash", "-c",
"ls "+filepath.Join(cache, "v*", "lib", "liboffice_oxide.a")+" | head -1",
).Output()
if err != nil {
t.Fatalf("naive glob failed: %v (%s)", err, naive)
}
if !strings.Contains(string(naive), "v0.1.12") {
t.Fatalf("naive glob+head -1 did not select the oldest version; got %q", naive)
}
}
// TestDocumentedCgoCflagsResolvesPinnedInclude is the end-to-end adversarial
// guard for the blocking P1 finding: it extracts the ACTUAL `export CGO_CFLAGS=`
// line from development.md, executes it against a fake cache holding two
// versions, and asserts the resulting include path (a) resolves to a real header
// under the pinned v0.1.16, not the oldest v0.1.12, and (b) is not a literal
// un-expanded `v*` glob. This catches both the "glob never expands in quotes"
// bug and the "include must follow the pinned version" requirement.
func TestDocumentedCgoCflagsResolvesPinnedInclude(t *testing.T) {
root := repoRoot(t)
doc := readFile(t, filepath.Join(root, "internal", "development.md"))
// Extract the documented CGO_CFLAGS export line from the fenced bash block,
// stripping the markdown blockquote prefix ("> "). Executing the real line
// (rather than a copy) means a future edit that reintroduces a bug fails here.
var cflagsLine string
for _, line := range strings.Split(doc, "\n") {
trimmed := strings.TrimSpace(strings.TrimPrefix(strings.TrimSpace(line), ">"))
if strings.HasPrefix(trimmed, "export CGO_CFLAGS=") {
cflagsLine = trimmed
break
}
}
if cflagsLine == "" {
t.Fatal("development.md has no `export CGO_CFLAGS=` line to verify")
}
// Fake cache: two versions, each with a header. The pinned version
// (v0.1.16) is NOT the lexicographically-oldest (v0.1.12), so a glob+head -1
// or a literal `v*` would both be wrong.
tmp := t.TempDir()
cache := filepath.Join(tmp, "office_oxide")
for _, v := range []string{"v0.1.12", "v0.1.16"} {
inc := filepath.Join(cache, v, "include", "office_oxide_c")
lib := filepath.Join(cache, v, "lib")
if err := os.MkdirAll(inc, 0o755); err != nil {
t.Fatal(err)
}
if err := os.MkdirAll(lib, 0o755); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(inc, "office_oxide.h"), []byte("header"), 0o644); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(lib, "liboffice_oxide.a"), []byte("archive"), 0o644); err != nil {
t.Fatal(err)
}
}
pinnedArchive := filepath.Join(cache, "v0.1.16", "lib", "liboffice_oxide.a")
// Execute the documented line with OFFICE_OXIDE_A bound to the pinned
// archive, then emit the resulting CGO_CFLAGS value.
cmd := exec.Command("bash", "-c", cflagsLine+` ; printf '%s' "$CGO_CFLAGS"`)
cmd.Env = append(os.Environ(), "OFFICE_OXIDE_A="+pinnedArchive)
out, err := cmd.Output()
if err != nil {
t.Fatalf("documented CGO_CFLAGS line failed to evaluate: %v (%s)", err, out)
}
cflags := strings.TrimSpace(string(out))
if !strings.HasPrefix(cflags, "-I") {
t.Fatalf("CGO_CFLAGS is not an -I include flag: %q", cflags)
}
includeDir := strings.TrimPrefix(cflags, "-I")
// (a) It must be derived from the PINNED archive, not the oldest version.
if !strings.Contains(includeDir, "v0.1.16") || strings.Contains(includeDir, "v0.1.12") {
t.Errorf("include %q not derived from pinned v0.1.16 archive", includeDir)
}
// (a) It must resolve to a real header — the whole point of the fix.
if _, err := os.Stat(filepath.Join(includeDir, "office_oxide.h")); err != nil {
t.Errorf("derived include %q has no office_oxide.h: %v", includeDir, err)
}
// (b) It must NOT be an un-expanded literal `v*` glob (the original bug:
// the shell does not expand globs inside double-quoted CGO_CFLAGS).
if strings.Contains(includeDir, "v*/") || strings.Contains(includeDir, "v*") {
t.Errorf("include %q still contains an un-expanded v* glob", includeDir)
}
// Adversarial proof of the original defect: a literal `v*` inside a quoted
// path never matches a real file, so the old line could not find the header.
literalGlob := filepath.Join(cache, "v*", "include", "office_oxide_c", "office_oxide.h")
if _, err := os.Stat(literalGlob); err == nil {
t.Errorf("a literal v* path unexpectedly resolved to a header; the un-expanded-glob guard is vacuous")
}
}
func sourceCmd(path string) string {
return "set -e; source " + path
}
+201 -41
View File
@@ -57,7 +57,9 @@ import hashlib
import os
import platform
import re
import shlex
import shutil
import subprocess
import sys
import tarfile
import tempfile
@@ -66,11 +68,17 @@ import zipfile
import requests
# ── Dependency versions & URLs (edit here) ────────────────────────────────
# This script's own copy of the native static-library versions and download URLs
# it fetches. build.sh does NOT grep these (it pins PDFIUM/PDF_OXIDE itself and
# reads ORT from internal/common/environments.go + the Dockerfile ARG, so it
# stays tied to the checked-out source tree and the Docker build needs no COPY
# of this script) — keep the two in sync when bumping a version.
# The native-library VERSIONS are NOT hardcoded in this script. They are derived
# from a single source of truth so build.sh and this downloader can never drift:
# - office_oxide / pdf_oxide -> go.mod (go_module_version, below)
# - ORT -> internal/common/environments.go DeepDocORTVersion
# (deepdoc_ort_version, below)
# - pdfium -> tools/scripts/native_lib_version.sh
# pdfium_static_version (a shared const; pdfium
# is not a Go module so go.mod does not apply)
# ORT_VERSION / PDFIUM_VERSION are bound to those derivations right after
# REPO_ROOT is resolved (see the block following _repo_root()), so bumping a
# version touches exactly one place upstream.
# ONNX Runtime (statically linked into the Go in-process DeepDoc backend).
#
@@ -83,7 +91,6 @@ import requests
# patched content (e.g. the build that exports SessionGetInitializer*); the
# downloader detects that via a `{asset}.sha256` sidecar and
# re-downloads/re-extracts, so a stale local copy never silently lingers.
ORT_VERSION = "1.29.0"
# Release tag on infiniflow/ragflow-build. v1.29.0's Linux x86_64 minimal build
# reduced operators to required_operators.config, which omitted Clip@13 — so
@@ -95,15 +102,15 @@ ORT_VERSION = "1.29.0"
# build ships Clip@13).
ORT_RELEASE_TAG = "onnxruntime-v1.29.0-patch2"
# pdf_oxide go-ffi release. Bare module version (the "v" is added by
# pdf_oxide_url); matches the value build.sh uses to validate the embedded
# .a stamp.
PDF_OXIDE_VERSION = "0.3.73"
# pdf_oxide go-ffi release. The bare module version is derived from go.mod at
# download/extract time (see pdf_oxide_url / the extraction block below) via
# go_module_version(), so there is no second hardcoded copy to drift from what
# the Go build actually links (mirrors office_oxide).
# pdfium release, as the numeric build number after "chromium%2F" (the "v" and
# the URL-encoded slash are added by pdfium_url). Matches build.sh's
# PDFIUM_STATIC_VERSION.
PDFIUM_VERSION = "7809"
# the URL-encoded slash are added by pdfium_url). Its bare value is bound to
# pdfium_static_version(REPO_ROOT) below (the single shared const, since pdfium
# is not a Go module).
def host_platform(target_os=None, target_arch=None):
@@ -185,6 +192,48 @@ def go_module_version(module: str, repo_root: str) -> str:
return match.group(1)
def deepdoc_ort_version(repo_root: str) -> str:
"""Read the ONNX Runtime version the in-process (Go) DeepDoc backend links
from internal/common/environments.go's DeepDocORTVersion const.
ORT is the one native lib whose version is owned by Go source (not go.mod),
so both build.sh and this script parse the SAME const — no second hardcoded
copy to drift."""
path = os.path.join(repo_root, "internal", "common", "environments.go")
if not os.path.isfile(path):
raise RuntimeError(f"environments.go not found at {path}")
with open(path, encoding="utf-8") as file:
match = re.search(r'DeepDocORTVersion\s*=\s*"([^"]+)"', file.read())
if match is None:
raise RuntimeError(f"DeepDocORTVersion const not found in {path}")
return match.group(1)
def pdfium_static_version(repo_root: str) -> str:
"""Return the pinned pdfium build number from the single shared bash source
of truth tools/scripts/native_lib_version.sh.
pdfium (kognitos/pdfium-static) is not a Go module, so unlike
office_oxide / pdf_oxide it cannot be derived from go.mod; its bare numeric
build number is the one constant exported by pdfium_static_version() and
sourced by both this script and build.sh."""
script = os.path.join(repo_root, "tools", "scripts", "native_lib_version.sh")
if not os.path.isfile(script):
raise RuntimeError(f"native_lib_version.sh not found at {script}")
result = subprocess.run(
["bash", "-c", f"source {shlex.quote(script)} >/dev/null 2>&1; pdfium_static_version"],
capture_output=True,
text=True,
check=False,
)
if result.returncode != 0:
raise RuntimeError(f"pdfium_static_version failed: {result.stderr.strip()}")
version = result.stdout.strip()
if not version:
raise RuntimeError("pdfium_static_version produced no output")
return version
# ── URL builders ──────────────────────────────────────────────────────────
# Every download URL is assembled from the constants above via one of these
# helpers, so a version bump touches exactly one constant. Each takes the target
@@ -192,9 +241,12 @@ def go_module_version(module: str, repo_root: str) -> str:
# maps above.
def pdf_oxide_url(goos: str, goarch: str) -> str:
"""pdf_oxide go-ffi tarball for the target platform."""
return f"{gh_proxy()}https://github.com/yfedoseev/pdf_oxide/releases/download/v{PDF_OXIDE_VERSION}/{PDF_OXIDE_ASSETS[(goos, goarch)]}"
def pdf_oxide_url(repo_root: str, goos: str, goarch: str) -> str:
"""pdf_oxide go-ffi tarball for the target platform. The version is derived
from go.mod so it always matches what the Go build actually links (no second
hardcoded copy to drift -- mirrors office_oxide_url)."""
version = go_module_version("github.com/yfedoseev/pdf_oxide/go", repo_root)
return f"{gh_proxy()}https://github.com/yfedoseev/pdf_oxide/releases/download/v{version}/{PDF_OXIDE_ASSETS[(goos, goarch)]}"
def pdfium_url(goos: str, goarch: str) -> str:
@@ -269,6 +321,15 @@ def _repo_root() -> str:
REPO_ROOT = _repo_root()
# Bind the native-library versions to their single source of truth (see the
# version block above). Keeping these as module-level constants means every
# downstream URL/dir builder still references ORT_VERSION / PDFIUM_VERSION, but
# the bare values now live in exactly one place upstream (environments.go and
# native_lib_version.sh) — bumping a version needs no edit here.
ORT_VERSION = deepdoc_ort_version(REPO_ROOT)
PDFIUM_VERSION = pdfium_static_version(REPO_ROOT)
# Downloaded archives are anchored under deps/ (ignored by .gitignore
# via *.tgz / deps/*.zip) regardless of the caller's working directory,
# so a move of this script does not scatter large files into tools/scripts/.
@@ -276,6 +337,28 @@ RAGFLOW_DEPS_DIR = os.path.join(REPO_ROOT, "deps")
NATIVE_LIBS_DIR = os.path.expanduser("~/ragflow-native-libs")
def native_libs_dir():
"""Resolve where native static libraries are written/read.
Honours RAGFLOW_NATIVE_LIBS_DIR so a CI builder can mount a pre-populated
user cache (e.g. ~/ragflow-native-libs) at a configurable path and the
downloader agrees with build.sh on that location. Falls back to the
in-home default when the variable is unset or empty.
"""
return os.environ.get("RAGFLOW_NATIVE_LIBS_DIR") or NATIVE_LIBS_DIR
def ensure_deps_dir():
"""Make sure RAGFLOW_DEPS_DIR exists before any chdir into it.
In a product image build the go-builder stage does not carry a deps/
checkout, so a bare os.chdir(RAGFLOW_DEPS_DIR) would raise
FileNotFoundError before the auto-download fallback could run. Creating it
here lets the downloader proceed and fetch archives on demand.
"""
os.makedirs(RAGFLOW_DEPS_DIR, exist_ok=True)
def _sha256_of(path):
"""sha256 of a file, streamed in chunks so large archives don't blow memory."""
h = hashlib.sha256()
@@ -342,10 +425,30 @@ def _valid_download(filename):
return True
def _extract_native_archive(archive_path, target, required_files, version=None):
def _stamp_present(data, version, stamp_format):
"""True when the .a archive DATA carries VERSION as a known stamp form.
Different native libs embed their version stamp differently:
- office_oxide embeds a null-delimited token (b"\\x00"+version+b"\\x00");
- pdf_oxide embeds a plain-text token (b"pdf_oxide "+version).
Accepting both keeps the extractor library-agnostic while still rejecting a
.a that does not actually contain the expected version."""
needle = version.encode()
if stamp_format == "null":
return b"\x00" + needle + b"\x00" in data
if stamp_format == "text":
return (b"pdf_oxide " + needle in data) or (b"\x00" + needle + b"\x00" in data)
return needle in data
def _extract_native_archive(archive_path, target, required_files, version=None, stamp_format="null"):
"""Extract ARCHIVE_PATH into TARGET, verifying REQUIRED_FILES (and, for .a
files, an optional null-delimited VERSION stamp) before promoting the
staged tree.
files, an optional VERSION stamp) before promoting the staged tree.
STAMP_FORMAT selects how the version is embedded in the .a:
- "null": office_oxide's b"\\x00"+version+b"\\x00" token;
- "text": pdf_oxide's plain b"pdf_oxide "+version token (also accepts a
null-delimited token, so a format switch does not silently break).
Path-traversal / absolute / non-regular members are rejected before
extraction, and an incomplete archive raises instead of replacing a
@@ -358,8 +461,9 @@ def _extract_native_archive(archive_path, target, required_files, version=None):
return False
if version is not None and name.endswith(".a"):
with open(path, "rb") as file:
if b"\x00" + version.encode() + b"\x00" not in file.read():
return False
data = file.read()
if not _stamp_present(data, version, stamp_format):
return False
return True
if complete(target):
@@ -388,6 +492,37 @@ def _extract_native_archive(archive_path, target, required_files, version=None):
print(f" ✓ extracted {os.path.basename(archive_path)} to {target}")
def extract_native_lib(archive_path, subdir, dir_version, required_files, stamp_version=None, stamp_format="null"):
"""Extract a native static archive into the version-keyed cache dir
~/ragflow-native-libs/<subdir>/<dir_version>/.
The version-keyed dir (e.g. v7809) — NOT a flat layout — is what lets a
single mounted user cache hold several native-lib versions at once, so PRs
pinned to different go.mod versions all build offline. Idempotent: when the
target dir already holds the required files (matching stamp_version for
office_oxide) the extraction is skipped, so re-running the downloader is a
no-op for an already-provisioned cache.
STAMP_FORMAT is forwarded to _extract_native_archive (see its docstring):
"null" for office_oxide, "text" for pdf_oxide. pdfium passes stamp_version
None and is not stamp-checked.
Returns True when extraction ran or was skipped-successfully (the dir is in
place afterwards), False when the archive is absent (caller must then
download it). Re-raises the underlying tar/validation error after purging a
corrupt archive."""
if not os.path.isfile(archive_path):
print(f" Skipping extraction: {os.path.basename(archive_path)} not found")
return False
target = os.path.join(native_libs_dir(), subdir, dir_version)
try:
_extract_native_archive(archive_path, target, required_files, stamp_version, stamp_format)
except (tarfile.TarError, EOFError, RuntimeError):
os.unlink(archive_path)
raise
return True
def prune_stale_onnxruntime(static_lib_dir, expected_dir):
"""Remove ONNX Runtime version dirs under static_lib that do NOT match
`expected_dir`. Without this, a version bump (or a foreign-arch run on a
@@ -428,6 +563,7 @@ def extract_onnxruntime(static_lib_dir, archive_path, expected_dir):
expected_path = os.path.join(static_lib_dir, expected_dir)
if os.path.isdir(expected_path) and has_static_archives(expected_path):
print(f" ✓ onnxruntime/static_lib ({expected_dir}) already extracted to {expected_path}")
stamp_onnxruntime_version(static_lib_dir, expected_dir, ORT_VERSION)
return True
os.makedirs(static_lib_dir, exist_ok=True)
print(f" Extracting {os.path.basename(archive_path)} → {static_lib_dir}")
@@ -445,9 +581,37 @@ def extract_onnxruntime(static_lib_dir, archive_path, expected_dir):
shutil.rmtree(normalized)
print(f" Renaming {os.path.basename(extracted)} → {os.path.basename(normalized)}")
os.rename(extracted, normalized)
# Pre-create the version-stamp symlink (Fix #2) while the cache is still
# writable, so a read-only CI mount already has it and build.sh's stamp step
# is a no-op there.
stamp_onnxruntime_version(static_lib_dir, expected_dir, ORT_VERSION)
return True
def stamp_onnxruntime_version(static_lib_dir, normalized_dir, version):
"""Pre-create the version-stamp ORT symlink inside the native cache.
build.sh keys its ORT link flag on onnxruntime/static_lib/v<version>-<sha16>/
(the sha16 disambiguates a re-issued archive of the same version). That dir
is created HERE, at population time when the cache is still writable, with a
RELATIVE symlink to the real libonnxruntime.a so it resolves under any mount
point. Idempotent: an existing correct symlink is left untouched, which keeps
a read-only CI mount a no-op (build.sh's stamp step then skips the write)."""
primary = os.path.join(static_lib_dir, normalized_dir, "lib", "libonnxruntime.a")
if not os.path.isfile(primary):
return
sha16 = _sha256_of(primary)[:16]
stamp_dir = os.path.join(static_lib_dir, f"v{version}-{sha16}")
stamp_link = os.path.join(stamp_dir, "libonnxruntime.a")
rel = os.path.relpath(primary, stamp_dir)
if os.path.islink(stamp_link) and os.path.realpath(stamp_link) == os.path.realpath(primary):
return
os.makedirs(stamp_dir, exist_ok=True)
if os.path.lexists(stamp_link):
os.unlink(stamp_link)
os.symlink(rel, stamp_link)
def _fetch_ort_sidecar_sha(url):
"""Fetch the published `<asset>.sha256` sidecar for the ORT archive.
@@ -495,7 +659,7 @@ def _download_ort_archive(archive_path, goos, goarch):
if not _valid_download(archive_path):
os.unlink(archive_path)
raise RuntimeError(f"Invalid downloaded dependency: {os.path.basename(archive_path)}")
version_dir = os.path.join(NATIVE_LIBS_DIR, "onnxruntime", "static_lib", ort_normalized_dir(ORT_VERSION, goos, goarch))
version_dir = os.path.join(native_libs_dir(), "onnxruntime", "static_lib", ort_normalized_dir(ORT_VERSION, goos, goarch))
if os.path.isdir(version_dir):
print(f" Removing stale extracted ONNX Runtime dir: {version_dir}")
shutil.rmtree(version_dir)
@@ -701,13 +865,14 @@ def main():
# Anchor the downloaded archives to deps/ regardless of the
# caller's working directory (build.sh invokes this from the repo root).
ensure_deps_dir()
os.chdir(RAGFLOW_DEPS_DIR)
# ONNX Runtime archive (+ checksum sidecar), then the other native archives.
_download_ort_archive(os.path.join(RAGFLOW_DEPS_DIR, ort_zip), goos, goarch)
archives = [
(pdfium_url(goos, goarch), PDFIUM_STATIC_ASSETS[(goos, goarch)]),
(pdf_oxide_url(goos, goarch), PDF_OXIDE_ASSETS[(goos, goarch)]),
(pdf_oxide_url(REPO_ROOT, goos, goarch), PDF_OXIDE_ASSETS[(goos, goarch)]),
(office_oxide_url(REPO_ROOT, goos, goarch), OFFICE_OXIDE_ASSETS[(goos, goarch)]),
]
for url, filename in archives:
@@ -721,28 +886,23 @@ def main():
raise RuntimeError(f"Invalid downloaded dependency: {filename}")
# Extract native static libraries to ~/ragflow-native-libs for the Go build.
# Ensures build.sh can find them without network access.
# Ensures build.sh can find them without network access. Each lands under a
# version-keyed subdir (v<version>/) so a mounted cache can hold several
# versions at once; see extract_native_lib().
office_version = go_module_version("github.com/yfedoseev/office_oxide/go", REPO_ROOT)
pdf_oxide_version = go_module_version("github.com/yfedoseev/pdf_oxide/go", REPO_ROOT)
extractions = [
(PDFIUM_STATIC_ASSETS[(goos, goarch)], "pdfium-static", ["lib/libpdfium.a", "lib/libc++.a", "lib/libc++abi.a", "include/fpdfview.h"], None),
(PDF_OXIDE_ASSETS[(goos, goarch)], "pdf_oxide", [f"lib/{goos}_{goarch}/libpdf_oxide.a", "include/pdf_oxide.h"], None),
(OFFICE_OXIDE_ASSETS[(goos, goarch)], "office_oxide", ["lib/liboffice_oxide.a", "include/office_oxide_c/office_oxide.h"], office_version),
("pdfium-static", PDFIUM_STATIC_ASSETS[(goos, goarch)], "v" + PDFIUM_VERSION, ["lib/libpdfium.a", "lib/libc++.a", "lib/libc++abi.a", "include/fpdfview.h"], None, "null"),
("pdf_oxide", PDF_OXIDE_ASSETS[(goos, goarch)], "v" + pdf_oxide_version, [f"lib/{goos}_{goarch}/libpdf_oxide.a", "include/pdf_oxide.h"], pdf_oxide_version, "text"),
("office_oxide", OFFICE_OXIDE_ASSETS[(goos, goarch)], "v" + office_version, ["lib/liboffice_oxide.a", "include/office_oxide_c/office_oxide.h"], office_version, "null"),
]
for archive, subdir, required_files, version in extractions:
for subdir, archive, dir_version, required_files, stamp_version, stamp_format in extractions:
archive_path = os.path.join(RAGFLOW_DEPS_DIR, archive)
if not os.path.isfile(archive_path):
print(f" Skipping extraction: {archive} not found")
continue
target = os.path.join(NATIVE_LIBS_DIR, subdir)
try:
_extract_native_archive(archive_path, target, required_files, version)
except (tarfile.TarError, EOFError, RuntimeError):
os.unlink(archive_path)
raise
extract_native_lib(archive_path, subdir, dir_version, required_files, stamp_version, stamp_format)
if not extract_onnxruntime(
os.path.join(NATIVE_LIBS_DIR, "onnxruntime", "static_lib"),
os.path.join(native_libs_dir(), "onnxruntime", "static_lib"),
os.path.join(RAGFLOW_DEPS_DIR, ort_zip),
ort_dir,
):
@@ -752,7 +912,7 @@ def main():
# .a left here is exactly the "silent green" this split must prevent.
print(
f" ERROR: ONNX Runtime static archives for {ort_dir} were not "
f"extracted to {os.path.join(NATIVE_LIBS_DIR, 'onnxruntime', 'static_lib')}. "
f"extracted to {os.path.join(native_libs_dir(), 'onnxruntime', 'static_lib')}. "
f"Check the download above; build.sh will refuse to link without them.",
file=sys.stderr,
)
@@ -763,7 +923,7 @@ def main():
# find the archives — the .a files live under
# ~/ragflow-native-libs/onnxruntime/static_lib. The in-process backend
# resolves OrtGetApiBase via dlopen(NULL); there is no dynamic .so fallback.
ort_static_dir = os.path.join(NATIVE_LIBS_DIR, "onnxruntime", "static_lib")
ort_static_dir = os.path.join(native_libs_dir(), "onnxruntime", "static_lib")
ort_a_files = [os.path.join(root, f) for root, _, files in os.walk(ort_static_dir) for f in files if f.endswith(".a")]
if ort_a_files:
print(f" ✓ onnxruntime static archives ready: {len(ort_a_files)} .a under {ort_static_dir}")
@@ -784,7 +944,7 @@ def main():
download_tokenizer_assets(args.china_mirrors)
print()
print(f"✓ Native build libraries ready under {NATIVE_LIBS_DIR}/")
print(f"✓ Native build libraries ready under {native_libs_dir()}/")
print(" Run: bash build.sh (it will find them without network access).")
print(" Runtime resources (cl100k / DeepDoc .ort / tokenizer) are also in place,")
print(" so the local Go server is ready to build, run, parse and count tokens.")
+82
View File
@@ -0,0 +1,82 @@
#!/usr/bin/env bash
# Single source of truth for the native static-library versions build.sh and
# tools/scripts/download_dev_deps.py agree on. Source this file from build.sh;
# it defines only functions (no top-level side effects) so it is safe to source.
#
# Each native lib resolves its version from exactly one place:
# - office_oxide / pdf_oxide -> go.mod (go_module_version)
# - ORT -> internal/common/environments.go
# DeepDocORTVersion (read by both build.sh via
# deepdoc_ort_version() below and
# download_dev_deps.deepdoc_ort_version() — it is
# Go-owned, so the const itself stays in Go)
# - pdfium -> pdfium_static_version (below; pdfium is not a
# Go module, so go.mod does not apply)
#
# There is NO /opt pre-bake and NO flat (version-less) cache layout any more: the
# libs land under ~/ragflow-native-libs/<lib>/v<version>/ (see download_dev_deps.py
# and build.sh), so a single mounted cache carries several versions at once.
# ragflow_native_libs_dir
# Root of the version-keyed native static-lib user cache
# (~/ragflow-native-libs by default). Overridable via RAGFLOW_NATIVE_LIBS_DIR so
# CI can mount the cache at a fixed host path (e.g. a RUNNER_WORKSPACE_PREFIX
# bind mount) instead of the build user's $HOME — see sep-tests.yml Slice 3. An
# EMPTY override falls back to the $HOME default (the ${VAR:-default} expansion),
# so a half-configured runner never resolves to an empty cache root. Echoes the
# resolved path and returns 0.
ragflow_native_libs_dir() {
printf '%s\n' "${RAGFLOW_NATIVE_LIBS_DIR:-${HOME}/ragflow-native-libs}"
}
# go_module_version <module> <repo_root>
# Reads the pinned version (the "vX.Y.Z" after the require line) of a Go module
# from <repo_root>/go.mod so build.sh and tools/scripts/download_dev_deps.py agree
# on the version (single source of truth). The module may appear either as a bare
# require line (`\t<module> vX.Y.Z`) inside a require block or on a single-line
# `require <module> vX.Y.Z`; an optional leading `require` keyword is allowed.
# Echoes the bare version (no "v" prefix) and returns 0, or returns 1 with empty
# output when go.mod is missing or the module is not required.
go_module_version() {
local module="$1" repo_root="$2"
local go_mod="${repo_root}/go.mod"
[ -f "$go_mod" ] || return 1
local version
version="$(grep -m1 -E "^[[:space:]]*(require[[:space:]]+)?${module}[[:space:]]+v" "$go_mod" \
| sed -E 's/.*v([0-9][^[:space:]]*).*/\1/')"
[ -n "$version" ] || return 1
printf '%s\n' "$version"
}
# pdfium_static_version
# pdfium (kognitos/pdfium-static) is NOT a Go module, so unlike office_oxide /
# pdf_oxide its version cannot be derived from go.mod. It is pinned here as the
# single shared constant; download_dev_deps.py and build.sh both source this
# function so the bare numeric build number lives in exactly one place. Echoes
# the constant and returns 0.
pdfium_static_version() {
# Single source of truth for the pdfium static build number. Bump here only.
printf '%s\n' "7809"
}
# deepdoc_ort_version <repo_root>
# Reads the ONNX Runtime version the in-process (Go) DeepDoc backend links from
# <repo_root>/internal/common/environments.go's DeepDocORTVersion const.
#
# ORT is the one native lib whose version is owned by Go source (not go.mod), so
# build.sh and download_dev_deps.deepdoc_ort_version() both parse the SAME const
# through this helper — no second hardcoded copy to drift. Echoes the bare
# version and returns 0, or returns 1 with empty output when environments.go is
# missing or the const is absent (so callers fail loud, mirroring the Python
# RuntimeError). build.sh used to inline this grep at every call site; collapsing
# it here keeps a single bash source of truth alongside the Python one.
deepdoc_ort_version() {
local repo_root="$1"
local env_go="${repo_root}/internal/common/environments.go"
[ -f "$env_go" ] || return 1
local version
version="$(grep -m1 -E 'DeepDocORTVersion[[:space:]]*=[[:space:]]*"' "$env_go" \
| sed -E 's/.*"([^"]+)".*/\1/')"
[ -n "$version" ] || return 1
printf '%s\n' "$version"
}
+34
View File
@@ -0,0 +1,34 @@
#!/usr/bin/env bash
# Pure helper for build.sh: decide whether the required ONNX Runtime static lib
# is already present in the version+platform-keyed user cache.
#
# $1 = ONNXRUNTIME_STATIC_PREFIX root holding the extracted ORT dirs
# $2 = ort_dir_prefix platform top-level dir prefix
# (see build.sh ort_static_lib_prefix)
# $3 = ort_version DeepDocORTVersion (internal/common/environments.go)
#
# Exits 0 (present) when at least one real base libonnxruntime*.a lives under a
# dir matching BOTH the target platform and the required version —
# */${ort_dir_prefix}${ort_version}*/lib/*.a — and 1 (missing) otherwise. This is
# the SAME dir check_onnxruntime_deps links, so a shared cache holding only an
# OLD version or a DIFFERENT platform's ORT is reported missing (and the
# controlled auto-downloader runs) instead of being silently treated as present
# and failing only at link time.
#
# The lib/ segment is required so a stray .a sitting in the prefix root (not
# under a versioned .../lib/ dir) is ignored. Provider-specific archives
# (*cuda*/*tensorrt*/*coreml*/*dml*/*migraphx*) are excluded, matching the
# link-time selection in check_onnxruntime_deps.
ragflow_ort_present() {
local prefix="$1" ort_dir_prefix="$2" ort_version="$3"
local f
while IFS= read -r f; do
case "$(basename "$f")" in
*cuda*|*tensorrt*|*coreml*|*dml*|*migraphx*) continue ;;
esac
case "$f" in
*/${ort_dir_prefix}${ort_version}*/lib/*.a) return 0 ;;
esac
done < <(find "$prefix" -type f -name '*.a' 2>/dev/null)
return 1
}
+25
View File
@@ -0,0 +1,25 @@
#!/usr/bin/env bash
# Pure helper for build.sh: decide how to version-stamp the ORT static archive
# with a symlink. Echoes "<needs_write>|<relative_target>".
#
# $1 = versioned_dir the v<version>-<sha16> stamp dir (under the native cache)
# $2 = archive_file absolute path to the real libonnxruntime.a
#
# The stamp target is RELATIVE (never absolute) so it resolves under ANY mount
# point of the cache (build host, runner, CI builder container). `needs_write`
# is "no" when an existing symlink already resolves to the same archive, so a
# read-only cache mount is a no-op instead of a fatal `ln` on a read-only fs.
ragflow_ort_stamp_target() {
local versioned_dir="$1" archive_file="$2"
local link_name link_target want
link_name="$(basename "$archive_file")"
link_target="${versioned_dir}/${link_name}"
# -m: compute the relative path without requiring either path to exist yet.
want="$(realpath -m --relative-to="$versioned_dir" "$archive_file")"
local needs_write="yes"
if [ -L "$link_target" ] && \
[ "$(readlink -f "$link_target" 2>/dev/null)" = "$(readlink -f "$archive_file" 2>/dev/null)" ]; then
needs_write="no"
fi
printf '%s|%s\n' "$needs_write" "$want"
}
+554 -3
View File
@@ -22,6 +22,7 @@
import importlib.util
import io
import os
import subprocess
import tarfile
import threading
import zipfile
@@ -201,8 +202,8 @@ def test_url_builders_select_platform_assets(monkeypatch):
assert native.pdfium_url("linux", "amd64").endswith("/pdfium-linux-x64-static.tgz")
assert native.pdfium_url("linux", "arm64").endswith("/pdfium-linux-arm64-static.tgz")
assert native.pdfium_url("darwin", "arm64").endswith("/pdfium-mac-arm64-static.tgz")
assert native.pdf_oxide_url("linux", "amd64").endswith("/pdf_oxide-go-ffi-linux-amd64.tar.gz")
assert native.pdf_oxide_url("darwin", "amd64").endswith("/pdf_oxide-go-ffi-darwin-amd64.tar.gz")
assert native.pdf_oxide_url(native.REPO_ROOT, "linux", "amd64").endswith("/pdf_oxide-go-ffi-linux-amd64.tar.gz")
assert native.pdf_oxide_url(native.REPO_ROOT, "darwin", "amd64").endswith("/pdf_oxide-go-ffi-darwin-amd64.tar.gz")
assert native.ort_url("1.29.0", native.ort_target("darwin", "amd64")).endswith("/onnxruntime-v1.29.0-osx-universal.zip")
@@ -212,10 +213,33 @@ def test_office_oxide_url_selects_platform_asset(tmp_path):
assert native.office_oxide_url(str(tmp_path), "darwin", "arm64").endswith("/v0.1.12/native-macos-aarch64.tar.gz")
def test_pdf_oxide_url_derives_from_go_mod(tmp_path):
# pdf_oxide version MUST come from go.mod (the single source of truth
# build.sh also reads), never a hardcoded constant. Otherwise build.sh and
# the downloader silently drift apart. A fixture go.mod pins a non-default
# version; the produced URL must reflect exactly that version.
(tmp_path / "go.mod").write_text("module fixture\n\nrequire github.com/yfedoseev/pdf_oxide/go v0.4.1\n")
url = native.pdf_oxide_url(str(tmp_path), "linux", "amd64")
assert "/v0.4.1/pdf_oxide-go-ffi-linux-amd64.tar.gz" in url
def test_pdf_oxide_url_reads_its_own_module_line(tmp_path):
# Adversarial: a go.mod that pins office_oxide and pdf_oxide to DIFFERENT
# versions. pdf_oxide_url must read the pdf_oxide line, not the
# office_oxide one (a naive "read the first version in go.mod" would pick
# the wrong library). office_oxide_url must likewise stay on its own line.
(tmp_path / "go.mod").write_text("module fixture\n\nrequire github.com/yfedoseev/office_oxide/go v0.1.12\nrequire github.com/yfedoseev/pdf_oxide/go v0.4.1\n")
pdf_url = native.pdf_oxide_url(str(tmp_path), "linux", "amd64")
office_url = native.office_oxide_url(str(tmp_path), "linux", "amd64")
assert "/v0.4.1/" in pdf_url # pdf_oxide's own version
assert "/v0.1.12/" in office_url # office_oxide's own version
assert "/v0.1.12/" not in pdf_url # not coupled to office_oxide's line
def test_url_builders_honor_gh_proxy(monkeypatch):
monkeypatch.setenv("GH_PROXY", "https://gh-proxy.com/")
assert native.pdfium_url("linux", "amd64").startswith("https://gh-proxy.com/https://github.com/")
assert native.pdf_oxide_url("linux", "arm64").startswith("https://gh-proxy.com/https://github.com/")
assert native.pdf_oxide_url(native.REPO_ROOT, "linux", "arm64").startswith("https://gh-proxy.com/https://github.com/")
def test_dev_downloader_rejects_untrusted_gh_proxy(monkeypatch):
@@ -315,6 +339,121 @@ def test_native_archive_rejects_unsafe_members(tmp_path, kind):
assert sorted(path.name for path in tmp_path.iterdir()) == ["native", "outside", "unsafe.tar.gz"]
# ── Version-stamped extraction (Slice 2): native .a must land under a
# version-keyed subdir (v<version>/) of the user cache, never a flat layout, so a
# single mounted cache can hold several versions at once (different PRs pinned to
# different go.mod versions all build offline). extract_native_lib() is the
# single extraction seam main() calls; it is idempotent (skips when the version
# dir already holds the required files). ─────────────────────────────────────
def _make_pdfium_archive(path, names):
with tarfile.open(path, "w:gz") as tf:
for name in names:
data = b"pdfium-payload-1234" # non-empty so complete() accepts it
info = tarfile.TarInfo(name)
info.size = len(data)
tf.addfile(info, io.BytesIO(data))
def test_extracts_pdfium_into_versioned_dir(tmp_path, monkeypatch):
monkeypatch.setenv("RAGFLOW_NATIVE_LIBS_DIR", str(tmp_path / "native-libs"))
archive = tmp_path / "pdfium-linux-x64-static.tgz"
_make_pdfium_archive(archive, ["lib/libpdfium.a", "lib/libc++.a", "lib/libc++abi.a", "include/fpdfview.h"])
native.extract_native_lib(
str(archive),
"pdfium-static",
"v7809",
["lib/libpdfium.a", "lib/libc++.a", "lib/libc++abi.a", "include/fpdfview.h"],
None,
)
target = tmp_path / "native-libs" / "pdfium-static" / "v7809"
assert (target / "lib" / "libpdfium.a").is_file()
assert (target / "include" / "fpdfview.h").is_file()
def test_extract_native_lib_idempotent_on_rerun(tmp_path, monkeypatch):
monkeypatch.setenv("RAGFLOW_NATIVE_LIBS_DIR", str(tmp_path / "native-libs"))
archive = tmp_path / "pdfium-linux-x64-static.tgz"
_make_pdfium_archive(archive, ["lib/libpdfium.a"])
native.extract_native_lib(str(archive), "pdfium-static", "v7809", ["lib/libpdfium.a"], None)
before = sorted(os.listdir(tmp_path / "native-libs" / "pdfium-static" / "v7809" / "lib"))
native.extract_native_lib(str(archive), "pdfium-static", "v7809", ["lib/libpdfium.a"], None)
after = sorted(os.listdir(tmp_path / "native-libs" / "pdfium-static" / "v7809" / "lib"))
assert before == after
def _make_stamped_archive(path, lib_name, stamp, header_name):
"""Build a tar.gz with the .a (carrying STAMP, a version-bearing byte
string) plus an include header, so complete()'s file-list AND stamp checks
can both be exercised per library format."""
with tarfile.open(path, "w:gz") as tf:
info = tarfile.TarInfo(lib_name)
payload = b"!<arch>\n" + stamp
info.size = len(payload)
tf.addfile(info, io.BytesIO(payload))
header = tarfile.TarInfo(header_name)
header.size = len(b"// header")
tf.addfile(header, io.BytesIO(b"// header"))
def test_pdf_oxide_stamp_text_accepts_matching_version(tmp_path, monkeypatch):
# pdf_oxide embeds a plain-text `pdf_oxide <version>` token (not the
# null-delimited token office_oxide uses). complete() must accept it when
# stamp_format="text", or auto-download would reject a good archive and
# build.sh's controlled download path would fail closed.
version = native.go_module_version("github.com/yfedoseev/pdf_oxide/go", native.REPO_ROOT)
monkeypatch.setenv("RAGFLOW_NATIVE_LIBS_DIR", str(tmp_path / "native-libs"))
archive = tmp_path / "pdf_oxide-go-ffi-linux-amd64.tar.gz"
_make_stamped_archive(archive, "lib/linux_amd64/libpdf_oxide.a", f"pdf_oxide {version}".encode(), "include/pdf_oxide.h")
native.extract_native_lib(
str(archive),
"pdf_oxide",
f"v{version}",
["lib/linux_amd64/libpdf_oxide.a", "include/pdf_oxide.h"],
version,
"text",
)
assert (tmp_path / "native-libs" / "pdf_oxide" / f"v{version}" / "lib" / "linux_amd64" / "libpdf_oxide.a").is_file()
def test_pdf_oxide_stamp_text_rejects_wrong_version(tmp_path, monkeypatch):
# A pdf_oxide .a stamped with a different version must be rejected, not
# silently accepted (the whole point of the stamp check is to catch a
# mismatched/wrong native build).
version = native.go_module_version("github.com/yfedoseev/pdf_oxide/go", native.REPO_ROOT)
monkeypatch.setenv("RAGFLOW_NATIVE_LIBS_DIR", str(tmp_path / "native-libs"))
archive = tmp_path / "pdf_oxide-go-ffi-linux-amd64.tar.gz"
_make_stamped_archive(archive, "lib/linux_amd64/libpdf_oxide.a", b"pdf_oxide 0.9.9", "include/pdf_oxide.h")
with pytest.raises(RuntimeError, match=f"missing required native files or version {version}"):
native.extract_native_lib(
str(archive),
"pdf_oxide",
f"v{version}",
["lib/linux_amd64/libpdf_oxide.a", "include/pdf_oxide.h"],
version,
"text",
)
def test_office_oxide_stamp_null_accepts_matching_version(tmp_path, monkeypatch):
# Regression guard: office_oxide keeps its null-delimited stamp token; the
# "text" vs "null" split must not break it.
version = native.go_module_version("github.com/yfedoseev/office_oxide/go", native.REPO_ROOT)
monkeypatch.setenv("RAGFLOW_NATIVE_LIBS_DIR", str(tmp_path / "native-libs"))
archive = tmp_path / "native-linux-x86_64.tar.gz"
_make_stamped_archive(archive, "lib/liboffice_oxide.a", f"\x00{version}\x00".encode(), "include/office_oxide_c/office_oxide.h")
native.extract_native_lib(
str(archive),
"office_oxide",
f"v{version}",
["lib/liboffice_oxide.a", "include/office_oxide_c/office_oxide.h"],
version,
"null",
)
assert (tmp_path / "native-libs" / "office_oxide" / f"v{version}" / "lib" / "liboffice_oxide.a").is_file()
# ── Local-dev one-stop: the dev downloader must also fetch the runtime
# resources (cl100k / DeepDoc .ort / tokenizer assets) so a developer runs THIS
# script ONCE and the Go server is ready to build, run, parse and count. ─────
@@ -560,3 +699,415 @@ def test_arm64_docs_section_warns_that_lld18_links_but_hangs():
section = _arm64_docs_section()
assert "lld-18" in section, "lld-18 is no longer called out as insufficient"
assert "hang" in section.lower() or "spin" in section.lower(), "lld-18 startup-hang failure mode missing"
# ── Single source of truth for native versions (Slice 1) ───────────────────────
# build.sh and download_dev_deps.py must agree on every native version. The
# authoritative values live in tools/scripts/native_lib_version.sh (office_oxide
# / pdf_oxide from go.mod, ORT from environments.go, pdfium as one shared const).
# These tests pin that contract: no second hardcoded copy may drift.
def test_deepdoc_ort_version_reads_environments_go(tmp_path):
# Fixture environments.go with a deliberately non-default version.
common = tmp_path / "internal" / "common"
common.mkdir(parents=True)
(common / "environments.go").write_text('package common\n\n// DeepDocORTVersion is ...\nconst DeepDocORTVersion = "9.9.9"\n')
assert native.deepdoc_ort_version(str(tmp_path)) == "9.9.9"
def test_deepdoc_ort_version_against_real_repo():
# Sanity: parses the checked-out repo's actual pin. Assert the shape (semver)
# rather than a hardcoded literal so the test tracks environments.go on bumps.
import re
version = native.deepdoc_ort_version(native.REPO_ROOT)
assert re.match(r"^\d+\.\d+\.\d+", version), version
def test_pdfium_static_version_reads_single_source(tmp_path):
# Fixture native_lib_version.sh exporting a sentinel (NOT the real 7809).
scripts = tmp_path / "tools" / "scripts"
scripts.mkdir(parents=True)
(scripts / "native_lib_version.sh").write_text("pdfium_static_version() { echo 1234; }\n")
assert native.pdfium_static_version(str(tmp_path)) == "1234"
def test_pdfium_static_version_against_real_repo():
# Sanity: the shared const on the checked-out repo.
assert native.pdfium_static_version(native.REPO_ROOT) == "7809"
# ── deepdoc_ort_version (bash) single source ───────────────────────────────
# build.sh must read DeepDocORTVersion from environments.go through the SAME
# native_lib_version.sh helper the Python downloader mirrors (deepdoc_ort_version
# in download_dev_deps.py), not an inline grep repeated at every call site. These
# tests pin the bash function contract so the 4 inline greps in build.sh can be
# collapsed onto one source.
def test_deepdoc_ort_version_bash_reads_environments_go(tmp_path):
# Fixture environments.go with a deliberately non-default version.
common = tmp_path / "internal" / "common"
common.mkdir(parents=True)
(common / "environments.go").write_text('package common\n\n// DeepDocORTVersion is ...\nconst DeepDocORTVersion = "9.9.9"\n')
res = _sh_eval(f"deepdoc_ort_version {tmp_path}", _env_with("/home/alice"))
assert res.returncode == 0, res.stderr
assert res.stdout.strip() == "9.9.9"
def test_deepdoc_ort_version_bash_against_real_repo():
# Sanity: parses the checked-out repo's actual pin. Assert the shape (semver)
# rather than a hardcoded literal so the test tracks environments.go on bumps.
import re
res = _sh_eval(f"deepdoc_ort_version {native.REPO_ROOT}", _env_with("/home/alice"))
assert res.returncode == 0, res.stderr
assert re.match(r"^\d+\.\d+\.\d+", res.stdout.strip()), res.stdout
def test_deepdoc_ort_version_bash_missing_file(tmp_path):
# Adversarial: an empty repo root with no environments.go must NOT echo a
# version and must signal failure (so callers fail loud, mirroring the
# Python RuntimeError), not silently return an empty string with rc 0.
res = _sh_eval(f"deepdoc_ort_version {tmp_path}", _env_with("/home/alice"))
assert res.returncode == 1, res.stdout
assert res.stdout.strip() == ""
# ── Slice 3: CI mounts the native-lib user cache (RAGFLOW_NATIVE_LIBS_DIR) ──
# build.sh resolves the native static libs from a single cache root
# (default ~/ragflow-native-libs, overridable via RAGFLOW_NATIVE_LIBS_DIR so CI
# can mount the cache at a fixed host path instead of the build user's $HOME).
# These tests pin that resolution contract (native_lib_version.sh) and the
# sep-tests.yml wiring that mounts the cache into every BUILDER_CONTAINER.
def _sh_eval(func_call, env):
"""Source the real native_lib_version.sh and call FUNC_CALL in a throwaway
bash, returning the CompletedProcess. ENV is the full env (HOME/PATH must be
supplied), mirroring how build.sh sources the file."""
sh = os.path.join(native.REPO_ROOT, "tools", "scripts", "native_lib_version.sh")
assert os.path.isfile(sh), f"native_lib_version.sh not found at {sh}"
cmd = f'source "{sh}" && {func_call}'
return subprocess.run(["bash", "-c", cmd], env=env, capture_output=True, text=True, check=False)
def _env_with(home, override=None):
env = {"HOME": home, "PATH": os.environ.get("PATH", "")}
if override is not None:
env["RAGFLOW_NATIVE_LIBS_DIR"] = override
return env
def test_native_libs_dir_defaults_to_home():
# No override: resolve to $HOME/ragflow-native-libs.
res = _sh_eval("ragflow_native_libs_dir", _env_with("/home/alice"))
assert res.returncode == 0, res.stderr
assert res.stdout.strip() == "/home/alice/ragflow-native-libs"
def test_native_libs_dir_honors_override():
# Override set: CI mounts the cache at a fixed host path; build.sh must use it.
res = _sh_eval("ragflow_native_libs_dir", _env_with("/home/alice", "/mnt/cache"))
assert res.returncode == 0, res.stderr
assert res.stdout.strip() == "/mnt/cache"
def test_native_libs_dir_empty_override_falls_back():
# Adversarial: an explicitly EMPTY override must fall back to the HOME default
# (the ${VAR:-default} expansion), never resolve to an empty cache root.
res = _sh_eval("ragflow_native_libs_dir", _env_with("/home/alice", ""))
assert res.returncode == 0, res.stderr
assert res.stdout.strip() == "/home/alice/ragflow-native-libs"
SEP_TESTS_YML = os.path.join(native.REPO_ROOT, ".github", "workflows", "sep-tests.yml")
def test_sep_tests_mounts_native_libs_cache():
# Slice 3 contract: the native static-lib user cache is mounted into every
# BUILDER_CONTAINER (4 docker runs) and exposed via RAGFLOW_NATIVE_LIBS_DIR,
# replacing the old /opt pre-bake. The YAML must also stay parseable.
assert os.path.isfile(SEP_TESTS_YML), f"sep-tests.yml not found at {SEP_TESTS_YML}"
with open(SEP_TESTS_YML, encoding="utf-8") as f:
content = f.read()
# YAML must still parse (guards against a malformed edit); skip when pyyaml
# is unavailable in the test env (the CI runner has it).
try:
import yaml
yaml.safe_load(content)
except ImportError:
pass
# The cache root is declared as a job env so runner and builder container agree.
assert "RAGFLOW_NATIVE_LIBS_DIR" in content
# Each of the 4 BUILDER_CONTAINER docker runs bind-mounts the cache.
assert content.count("${RAGFLOW_NATIVE_LIBS_DIR}:${RAGFLOW_NATIVE_LIBS_DIR}") >= 4
# A fail-fast guard catches a missing/partial mount before a cryptic cgo link error.
assert "native-lib cache not mounted" in content
# ── downloader root resolution must honour RAGFLOW_NATIVE_LIBS_DIR (P1#2) ─────
# build.sh resolves the cache root through native_lib_version.ragflow_native_libs_dir()
# (${RAGFLOW_NATIVE_LIBS_DIR:-~/ragflow-native-libs}); the downloader must agree so a
# runner that overrides the mount path does not silently download into the default.
def test_python_native_libs_dir_default(monkeypatch):
monkeypatch.delenv("RAGFLOW_NATIVE_LIBS_DIR", raising=False)
assert native.native_libs_dir() == os.path.expanduser("~/ragflow-native-libs")
def test_python_native_libs_dir_honors_override(monkeypatch):
# Adversarial: a custom mount path must be returned verbatim, NOT the default.
monkeypatch.setenv("RAGFLOW_NATIVE_LIBS_DIR", "/srv/custom/native-libs")
assert native.native_libs_dir() == "/srv/custom/native-libs"
def test_python_native_libs_dir_empty_override_falls_back(monkeypatch):
# Adversarial: an empty override must fall back to the default, not resolve to "".
monkeypatch.setenv("RAGFLOW_NATIVE_LIBS_DIR", "")
assert native.native_libs_dir() == os.path.expanduser("~/ragflow-native-libs")
# ── downloader must not crash when deps/ is absent (P1#3) ─────────────────────
def test_ensure_deps_dir_creates_missing(monkeypatch, tmp_path):
# In the Docker go-builder stage nothing COPYs deps/, so the downloader's
# os.chdir(deps) used to raise FileNotFoundError and abort the image build.
# ensure_deps_dir() must create it first.
missing = tmp_path / "no_deps_dir"
monkeypatch.setattr(native, "RAGFLOW_DEPS_DIR", str(missing))
native.ensure_deps_dir()
assert missing.is_dir()
# ── ORT version-stamp symlink must be RELATIVE and read-only-safe (Fix #1/#2) ──
# build.sh stamps the ORT archive with a v<version>-<sha16> symlink so an
# in-place .a upgrade invalidates Go's build cache. The symlink MUST be relative
# (resolves under any cache mount: build host, runner, CI builder) and build.sh
# MUST skip the write when an existing link already resolves to the same archive
# (so a read-only CI mount is a no-op instead of a fatal `ln` on a read-only fs).
_ORT_STAMP_SH = os.path.join(native.REPO_ROOT, "tools", "scripts", "ort_stamp.sh")
def _stamp_eval(versioned_dir, archive_file):
assert os.path.isfile(_ORT_STAMP_SH), f"ort_stamp.sh not found at {_ORT_STAMP_SH}"
cmd = f'source "{_ORT_STAMP_SH}" && ragflow_ort_stamp_target "$@"'
return subprocess.run(
["bash", "-c", cmd, "_", versioned_dir, archive_file],
capture_output=True,
text=True,
check=False,
)
def test_ort_stamp_target_is_relative(tmp_path):
versioned = tmp_path / "cache" / "onnxruntime" / "static_lib" / "v1.29.0-abc123"
versioned.mkdir(parents=True)
archive = tmp_path / "cache" / "onnxruntime" / "static_lib" / "onnxruntime-linux-x64-static_lib-1.29.0-glibc2_28" / "lib" / "libonnxruntime.a"
archive.parent.mkdir(parents=True)
archive.write_bytes(b"ort")
res = _stamp_eval(str(versioned), str(archive))
assert res.returncode == 0, res.stderr
needs, target = res.stdout.strip().split("|")
assert needs == "yes"
assert not os.path.isabs(target), f"stamp target must be relative, got {target!r}"
assert (versioned / target).resolve() == archive.resolve()
def test_ort_stamp_target_skips_when_existing_link_correct(tmp_path):
# ro-safe: an existing symlink already pointing at the archive means no write.
versioned = tmp_path / "cache" / "onnxruntime" / "static_lib" / "v1.29.0-abc123"
versioned.mkdir(parents=True)
archive = tmp_path / "cache" / "onnxruntime" / "static_lib" / "norm" / "lib" / "libonnxruntime.a"
archive.parent.mkdir(parents=True)
archive.write_bytes(b"ort")
rel = os.path.relpath(archive, versioned)
os.symlink(rel, versioned / "libonnxruntime.a")
res = _stamp_eval(str(versioned), str(archive))
assert res.returncode == 0, res.stderr
needs, target = res.stdout.strip().split("|")
assert needs == "no", "existing correct symlink must need no write (ro-safe)"
assert target == rel
def test_ort_stamp_target_writes_when_link_missing(tmp_path):
versioned = tmp_path / "v1.29.0-abc123"
versioned.mkdir(parents=True)
archive = tmp_path / "norm" / "lib" / "libonnxruntime.a"
archive.parent.mkdir(parents=True)
archive.write_bytes(b"ort")
res = _stamp_eval(str(versioned), str(archive))
assert res.returncode == 0, res.stderr
needs, _ = res.stdout.strip().split("|")
assert needs == "yes"
# ── download_dev_deps.py pre-creates the ORT stamp dir during population ──────
# build.sh's stamp step is a no-op on a read-only CI mount ONLY if the stamp dir
# already exists in the cache. The downloader must therefore create the
# v<version>-<sha16> dir (relative symlink) while it still has write access.
def test_stamp_onnxruntime_version_creates_relative_symlink(tmp_path):
static_lib = tmp_path / "onnxruntime" / "static_lib"
static_lib.mkdir(parents=True)
norm = static_lib / "onnxruntime-linux-x64-static_lib-1.29.0-glibc2_28"
(norm / "lib").mkdir(parents=True)
(norm / "lib" / "libonnxruntime.a").write_bytes(b"!<arch>\nort")
native.stamp_onnxruntime_version(
str(static_lib),
"onnxruntime-linux-x64-static_lib-1.29.0-glibc2_28",
"1.29.0",
)
stamp_dirs = list(static_lib.glob("v1.29.0-*"))
assert len(stamp_dirs) == 1, f"expected exactly one stamp dir, found {stamp_dirs}"
link = stamp_dirs[0] / "libonnxruntime.a"
assert link.is_symlink(), "stamp must be a symlink"
assert not os.path.isabs(os.readlink(link)), "stamp symlink must be relative"
assert link.resolve() == (norm / "lib" / "libonnxruntime.a").resolve()
def test_stamp_onnxruntime_version_idempotent(tmp_path):
static_lib = tmp_path / "onnxruntime" / "static_lib"
static_lib.mkdir(parents=True)
norm_dir = "onnxruntime-linux-x64-static_lib-1.29.0-glibc2_28"
norm = static_lib / norm_dir
(norm / "lib").mkdir(parents=True)
(norm / "lib" / "libonnxruntime.a").write_bytes(b"!<arch>\nort")
native.stamp_onnxruntime_version(str(static_lib), norm_dir, "1.29.0")
before = sorted(p.name for p in static_lib.iterdir())
native.stamp_onnxruntime_version(str(static_lib), norm_dir, "1.29.0")
after = sorted(p.name for p in static_lib.iterdir())
assert before == after
# ── _native_libs_present ORT gate must be version+platform aware ──────────────
# build.sh's _native_libs_present decides whether the controlled auto-downloader
# runs. Its ORT check historically matched ANY .a under the static_lib prefix
# (find ... -name '*.a'), so a shared cache holding only an OLD version or a
# DIFFERENT platform's ORT was reported "present" and the downloader was skipped
# — the missing lib then failed only at link time. The gate must match the SAME
# <platform-prefix><version> dir that check_onnxruntime_deps links
# (*/${ort_dir_prefix}${ort_version}*/lib/*.a). ragflow_ort_present is that pure
# decision so it is unit-testable in isolation.
_ORT_PRESENT_SH = os.path.join(native.REPO_ROOT, "tools", "scripts", "ort_present.sh")
def _present_eval(prefix, ort_dir_prefix, ort_version):
"""Source the real ort_present.sh and call ragflow_ort_present."""
assert os.path.isfile(_ORT_PRESENT_SH), f"ort_present.sh not found at {_ORT_PRESENT_SH}"
cmd = f'source "{_ORT_PRESENT_SH}" && ragflow_ort_present "$@"'
return subprocess.run(
["bash", "-c", cmd, "_", prefix, ort_dir_prefix, ort_version],
capture_output=True,
text=True,
check=False,
)
def _seed_ort(prefix, dir_name):
"""Create a version-keyed ORT dir with a real libonnxruntime.a inside lib/."""
lib = prefix / dir_name / "lib"
lib.mkdir(parents=True)
(lib / "libonnxruntime.a").write_bytes(b"!<arch>\nort")
return prefix / dir_name
def test_ort_present_happy_path_matches_version_and_platform(tmp_path):
prefix = tmp_path / "static_lib"
_seed_ort(prefix, "onnxruntime-linux-x64-static_lib-1.29.0-glibc2_28")
res = _present_eval(str(prefix), "onnxruntime-linux-x64-static_lib-", "1.29.0")
assert res.returncode == 0, res.stderr
def test_ort_present_rejects_old_version(tmp_path):
# Adversarial: cache holds ONLY an older ORT version; required version differs.
# The old any-.a check wrongly returned present and skipped the downloader.
prefix = tmp_path / "static_lib"
_seed_ort(prefix, "onnxruntime-linux-x64-static_lib-1.23.2-glibc2_28")
res = _present_eval(str(prefix), "onnxruntime-linux-x64-static_lib-", "1.29.0")
assert res.returncode == 1, "old-version ORT must NOT satisfy the required version"
assert res.stderr == ""
def test_ort_present_rejects_wrong_platform(tmp_path):
# Adversarial: cache holds ORT for aarch64 but the target is amd64.
prefix = tmp_path / "static_lib"
_seed_ort(prefix, "onnxruntime-linux-aarch64-static_lib-1.29.0-glibc2_28")
res = _present_eval(str(prefix), "onnxruntime-linux-x64-static_lib-", "1.29.0")
assert res.returncode == 1, "other-platform ORT must NOT satisfy the target platform"
def test_ort_present_rejects_empty_cache(tmp_path):
prefix = tmp_path / "static_lib"
prefix.mkdir()
res = _present_eval(str(prefix), "onnxruntime-linux-x64-static_lib-", "1.29.0")
assert res.returncode == 1
def test_ort_present_rejects_stray_top_level_archive(tmp_path):
# Adversarial: a .a sitting directly in the prefix root (not under a
# versioned .../lib/ dir) must NOT count as present.
prefix = tmp_path / "static_lib"
prefix.mkdir()
(prefix / "libonnxruntime.a").write_bytes(b"!<arch>\nort")
res = _present_eval(str(prefix), "onnxruntime-linux-x64-static_lib-", "1.29.0")
assert res.returncode == 1, "stray top-level .a must not satisfy the versioned gate"
def test_ort_present_ignores_provider_specific_archives(tmp_path):
# Adversarial: a cuda provider archive inside the correct version dir must be
# excluded (it is not the base libonnxruntime.a), matching the link-time
# selection. Only the provider lib present => not present.
prefix = tmp_path / "static_lib"
lib = prefix / "onnxruntime-linux-x64-static_lib-1.29.0-glibc2_28" / "lib"
lib.mkdir(parents=True)
(lib / "libonnxruntime_providers_cuda.a").write_bytes(b"!<arch>\nort")
res = _present_eval(str(prefix), "onnxruntime-linux-x64-static_lib-", "1.29.0")
assert res.returncode == 1, "provider-specific archive must not satisfy the base lib gate"
# ── Documentation contract: IDE CGO snippet uses the versioned layout ─────────
# The native libs live under <lib>/v<version>/ (one mounted cache holds several
# versions). The IDE CGO snippet must point there, or a developer following the
# doc links a .a that does not exist at the legacy flat path.
def test_development_doc_ide_cgo_uses_versioned_layout():
doc = os.path.join(native.REPO_ROOT, "internal", "development.md")
with open(doc, encoding="utf-8") as file:
text = file.read()
# Version-keyed paths must appear for every native lib the IDE snippet links
# (resolved via v*/ glob, matching download_dev_deps.py / build.sh).
assert "office_oxide/v" in text, "office_oxide must use v<version> layout"
assert "pdfium-static/v" in text, "pdfium-static must use v<version> layout"
assert "pdf_oxide/v" in text, "pdf_oxide must use v<version> layout"
# The legacy flat layout must be gone so it cannot mislead.
assert "office_oxide/lib/liboffice_oxide.a" not in text
assert "pdfium-static/lib/libpdfium.a" not in text
# ── Versioned-extraction fixtures must isolate via env, not the module attr ───
# native_libs_dir() reads RAGFLOW_NATIVE_LIBS_DIR (env); monkeypatching the
# NATIVE_LIBS_DIR module attribute only isolates when that env is unset, so under
# CI (where the env is set to the real cache) the attribute stub is ignored and
# files leak into the real cache. The fixtures must isolate via env.
def test_extract_native_lib_honors_ragflow_native_libs_dir_env(tmp_path, monkeypatch):
# Adversarial: with the env set (as CI does) AND the module attribute stubbed
# to a different path, extraction must follow the env, not the attribute.
monkeypatch.setenv("RAGFLOW_NATIVE_LIBS_DIR", str(tmp_path / "isolated"))
monkeypatch.setattr(native, "NATIVE_LIBS_DIR", "/attribute-path-ignored")
archive = tmp_path / "pdfium-linux-x64-static.tgz"
_make_pdfium_archive(archive, ["lib/libpdfium.a"])
native.extract_native_lib(str(archive), "pdfium-static", "v7809", ["lib/libpdfium.a"], None)
assert (tmp_path / "isolated" / "pdfium-static" / "v7809" / "lib" / "libpdfium.a").is_file()
assert not os.path.exists("/attribute-path-ignored")