mirror of
https://github.com/infiniflow/ragflow.git
synced 2026-11-10 13:24:05 +08:00
Provision native libs from versioned user cache (offline CI, drop /opt pre-bake) (#22099)
This commit is contained in:
1 parent
2b4d628d51
commit
e5a51b680b
11 files changed
+1523
-187
No files matched your search
+4
-3
@@ -38,9 +38,10 @@ target/
|
||||
# regardless of which resources are prepared for the dependency image.
|
||||
# The deps image is built from this directory with:
|
||||
# cd deps && docker build -f Dockerfile -t infiniflow/ragflow_deps .
|
||||
# The native downloader tools/scripts/download_dev_deps.py is copied by the
|
||||
# main Dockerfile (not excluded here) so build.sh can read its ORT version pin
|
||||
# when running `./build.sh --go`.
|
||||
# tools/scripts is NOT excluded here: the main Dockerfile COPYs it so build.sh
|
||||
# can `source tools/scripts/native_lib_version.sh` (it does so at the top of the
|
||||
# script) and reference tools/scripts/download_dev_deps.py in its remediation
|
||||
# messages while building `./build.sh --go`.
|
||||
deps/*
|
||||
|
||||
# ── IDE and editor config ──────────────────────────────────────────────────
|
||||
|
||||
@@ -37,6 +37,12 @@ concurrency:
|
||||
# backend to build against github.com/infiniflow/onnxruntime_go.
|
||||
env:
|
||||
GOPRIVATE: github.com/infiniflow/onnxruntime_go
|
||||
# Native static-lib cache location on infiniflow's self-hosted runners.
|
||||
# build.sh resolves the libs from this dir (default ~/ragflow-native-libs);
|
||||
# the offline-first contract keeps the version-keyed cache here so the
|
||||
# fail-fast guard and the builder bind-mount both find it without an /opt
|
||||
# pre-bake. Overridable per environment.
|
||||
RAGFLOW_NATIVE_LIBS_DIR: /home/infiniflow/runners_work/runners_data/ragflow-native-libs
|
||||
|
||||
jobs:
|
||||
ragflow_preflight:
|
||||
@@ -243,6 +249,16 @@ jobs:
|
||||
export GOCACHE="${RUNNER_WORKSPACE_PREFIX}/gobuild"
|
||||
fi
|
||||
|
||||
# Fail fast if the native-lib user cache is not mounted: a missing
|
||||
# cache would otherwise surface as a cryptic cgo link error later.
|
||||
NATIVE_LIBS_DIR="${RAGFLOW_NATIVE_LIBS_DIR:-${HOME}/ragflow-native-libs}"
|
||||
for lib in office_oxide pdfium-static pdf_oxide onnxruntime; do
|
||||
if [ ! -d "${NATIVE_LIBS_DIR}/${lib}" ]; then
|
||||
echo "ERROR: ${NATIVE_LIBS_DIR}/${lib} missing — native-lib cache not mounted on this runner." >&2
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
|
||||
BUILDER_CONTAINER=ragflow_build_${GITHUB_RUN_ID}_${DOC_ENGINE}_$(od -An -N4 -tx4 /dev/urandom | tr -d ' ')
|
||||
cleanup_builder() {
|
||||
if [[ -n "${BUILDER_CONTAINER:-}" ]]; then
|
||||
@@ -252,11 +268,21 @@ jobs:
|
||||
trap cleanup_builder EXIT
|
||||
|
||||
TZ=${TZ:-$(readlink -f /etc/localtime | awk -F '/zoneinfo/' '{print $2}')}
|
||||
# Mount the native static-lib user cache into the builder container.
|
||||
# build.sh resolves the libs from RAGFLOW_NATIVE_LIBS_DIR (default
|
||||
# ~/ragflow-native-libs); this pins the same root for the container and
|
||||
# bind-mounts it read-only so the cgo link finds the versioned
|
||||
# v<version>/ dirs without an /opt pre-bake. CI mounts the cache from
|
||||
# the build host (populated by download_dev_deps.py); a missing mount is
|
||||
# caught by the assert below, not by a cryptic cgo link error.
|
||||
export RAGFLOW_NATIVE_LIBS_DIR="${RAGFLOW_NATIVE_LIBS_DIR:-${HOME}/ragflow-native-libs}"
|
||||
sudo docker run --privileged -d --name "${BUILDER_CONTAINER}" \
|
||||
-e TZ="${TZ}" \
|
||||
-e UV_INDEX=https://mirrors.aliyun.com/pypi/simple \
|
||||
-e "RAGFLOW_NATIVE_LIBS_DIR=${RAGFLOW_NATIVE_LIBS_DIR}" \
|
||||
-v "${PWD}:/ragflow" \
|
||||
-v "${PWD}/internal/binding/cpp/resource:/usr/share/infinity/resource" \
|
||||
-v "${RAGFLOW_NATIVE_LIBS_DIR}:${RAGFLOW_NATIVE_LIBS_DIR}:ro" \
|
||||
infiniflow/infinity_builder:ubuntu22_clang20
|
||||
sudo docker exec "${BUILDER_CONTAINER}" bash -c 'git config --global safe.directory "*" && cd /ragflow && ./build.sh --cpp'
|
||||
./build.sh --go
|
||||
@@ -810,6 +836,16 @@ jobs:
|
||||
export GOCACHE="${RUNNER_WORKSPACE_PREFIX}/gobuild"
|
||||
fi
|
||||
|
||||
# Fail fast if the native-lib user cache is not mounted: a missing
|
||||
# cache would otherwise surface as a cryptic cgo link error later.
|
||||
NATIVE_LIBS_DIR="${RAGFLOW_NATIVE_LIBS_DIR:-${HOME}/ragflow-native-libs}"
|
||||
for lib in office_oxide pdfium-static pdf_oxide onnxruntime; do
|
||||
if [ ! -d "${NATIVE_LIBS_DIR}/${lib}" ]; then
|
||||
echo "ERROR: ${NATIVE_LIBS_DIR}/${lib} missing — native-lib cache not mounted on this runner." >&2
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
|
||||
BUILDER_CONTAINER=ragflow_build_${GITHUB_RUN_ID}_${DOC_ENGINE}_$(od -An -N4 -tx4 /dev/urandom | tr -d ' ')
|
||||
cleanup_builder() {
|
||||
if [[ -n "${BUILDER_CONTAINER:-}" ]]; then
|
||||
@@ -819,11 +855,21 @@ jobs:
|
||||
trap cleanup_builder EXIT
|
||||
|
||||
TZ=${TZ:-$(readlink -f /etc/localtime | awk -F '/zoneinfo/' '{print $2}')}
|
||||
# Mount the native static-lib user cache into the builder container.
|
||||
# build.sh resolves the libs from RAGFLOW_NATIVE_LIBS_DIR (default
|
||||
# ~/ragflow-native-libs); this pins the same root for the container and
|
||||
# bind-mounts it read-only so the cgo link finds the versioned
|
||||
# v<version>/ dirs without an /opt pre-bake. CI mounts the cache from
|
||||
# the build host (populated by download_dev_deps.py); a missing mount is
|
||||
# caught by the assert below, not by a cryptic cgo link error.
|
||||
export RAGFLOW_NATIVE_LIBS_DIR="${RAGFLOW_NATIVE_LIBS_DIR:-${HOME}/ragflow-native-libs}"
|
||||
sudo docker run --privileged -d --name "${BUILDER_CONTAINER}" \
|
||||
-e TZ="${TZ}" \
|
||||
-e UV_INDEX=https://mirrors.aliyun.com/pypi/simple \
|
||||
-e "RAGFLOW_NATIVE_LIBS_DIR=${RAGFLOW_NATIVE_LIBS_DIR}" \
|
||||
-v "${PWD}:/ragflow" \
|
||||
-v "${PWD}/internal/binding/cpp/resource:/usr/share/infinity/resource" \
|
||||
-v "${RAGFLOW_NATIVE_LIBS_DIR}:${RAGFLOW_NATIVE_LIBS_DIR}:ro" \
|
||||
infiniflow/infinity_builder:ubuntu22_clang20
|
||||
sudo docker exec "${BUILDER_CONTAINER}" bash -c 'git config --global safe.directory "*" && cd /ragflow && ./build.sh --cpp'
|
||||
./build.sh --go
|
||||
@@ -1386,6 +1432,16 @@ jobs:
|
||||
# integration tests; the per-engine go jobs build the full server.
|
||||
run: |
|
||||
set -euo pipefail
|
||||
# Fail fast if the native-lib user cache is not mounted: a missing
|
||||
# cache would otherwise surface as a cryptic cgo link error later.
|
||||
NATIVE_LIBS_DIR="${RAGFLOW_NATIVE_LIBS_DIR:-${HOME}/ragflow-native-libs}"
|
||||
for lib in office_oxide pdfium-static pdf_oxide onnxruntime; do
|
||||
if [ ! -d "${NATIVE_LIBS_DIR}/${lib}" ]; then
|
||||
echo "ERROR: ${NATIVE_LIBS_DIR}/${lib} missing — native-lib cache not mounted on this runner." >&2
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
|
||||
BUILDER_CONTAINER=ragflow_build_${GITHUB_RUN_ID}_native_$(od -An -N4 -tx4 /dev/urandom | tr -d ' ')
|
||||
cleanup_builder() {
|
||||
if [[ -n "${BUILDER_CONTAINER:-}" ]]; then
|
||||
@@ -1395,11 +1451,21 @@ jobs:
|
||||
trap cleanup_builder EXIT
|
||||
|
||||
TZ=${TZ:-$(readlink -f /etc/localtime | awk -F '/zoneinfo/' '{print $2}')}
|
||||
# Mount the native static-lib user cache into the builder container.
|
||||
# build.sh resolves the libs from RAGFLOW_NATIVE_LIBS_DIR (default
|
||||
# ~/ragflow-native-libs); this pins the same root for the container and
|
||||
# bind-mounts it read-only so the cgo link finds the versioned
|
||||
# v<version>/ dirs without an /opt pre-bake. CI mounts the cache from
|
||||
# the build host (populated by download_dev_deps.py); a missing mount is
|
||||
# caught by the assert below, not by a cryptic cgo link error.
|
||||
export RAGFLOW_NATIVE_LIBS_DIR="${RAGFLOW_NATIVE_LIBS_DIR:-${HOME}/ragflow-native-libs}"
|
||||
sudo docker run --privileged -d --name "${BUILDER_CONTAINER}" \
|
||||
-e TZ="${TZ}" \
|
||||
-e UV_INDEX=https://mirrors.aliyun.com/pypi/simple \
|
||||
-e "RAGFLOW_NATIVE_LIBS_DIR=${RAGFLOW_NATIVE_LIBS_DIR}" \
|
||||
-v "${PWD}:/ragflow" \
|
||||
-v "${PWD}/internal/binding/cpp/resource:/usr/share/infinity/resource" \
|
||||
-v "${RAGFLOW_NATIVE_LIBS_DIR}:${RAGFLOW_NATIVE_LIBS_DIR}:ro" \
|
||||
infiniflow/infinity_builder:ubuntu22_clang20
|
||||
sudo docker exec "${BUILDER_CONTAINER}" bash -c 'git config --global safe.directory "*" && cd /ragflow && ./build.sh --cpp'
|
||||
|
||||
@@ -1486,6 +1552,16 @@ jobs:
|
||||
# starts only the dependency services, not the ragflow-cpu container.
|
||||
run: |
|
||||
set -euo pipefail
|
||||
# Fail fast if the native-lib user cache is not mounted: a missing
|
||||
# cache would otherwise surface as a cryptic cgo link error later.
|
||||
NATIVE_LIBS_DIR="${RAGFLOW_NATIVE_LIBS_DIR:-${HOME}/ragflow-native-libs}"
|
||||
for lib in office_oxide pdfium-static pdf_oxide onnxruntime; do
|
||||
if [ ! -d "${NATIVE_LIBS_DIR}/${lib}" ]; then
|
||||
echo "ERROR: ${NATIVE_LIBS_DIR}/${lib} missing — native-lib cache not mounted on this runner." >&2
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
|
||||
BUILDER_CONTAINER=ragflow_build_${GITHUB_RUN_ID}_go_int_$(od -An -N4 -tx4 /dev/urandom | tr -d ' ')
|
||||
cleanup_builder() {
|
||||
if [[ -n "${BUILDER_CONTAINER:-}" ]]; then
|
||||
@@ -1494,11 +1570,21 @@ jobs:
|
||||
}
|
||||
trap cleanup_builder EXIT
|
||||
TZ=${TZ:-$(readlink -f /etc/localtime | awk -F '/zoneinfo/' '{print $2}')}
|
||||
# Mount the native static-lib user cache into the builder container.
|
||||
# build.sh resolves the libs from RAGFLOW_NATIVE_LIBS_DIR (default
|
||||
# ~/ragflow-native-libs); this pins the same root for the container and
|
||||
# bind-mounts it read-only so the cgo link finds the versioned
|
||||
# v<version>/ dirs without an /opt pre-bake. CI mounts the cache from
|
||||
# the build host (populated by download_dev_deps.py); a missing mount is
|
||||
# caught by the assert below, not by a cryptic cgo link error.
|
||||
export RAGFLOW_NATIVE_LIBS_DIR="${RAGFLOW_NATIVE_LIBS_DIR:-${HOME}/ragflow-native-libs}"
|
||||
sudo docker run --privileged -d --name "${BUILDER_CONTAINER}" \
|
||||
-e TZ="${TZ}" \
|
||||
-e UV_INDEX=https://mirrors.aliyun.com/pypi/simple \
|
||||
-e "RAGFLOW_NATIVE_LIBS_DIR=${RAGFLOW_NATIVE_LIBS_DIR}" \
|
||||
-v "${PWD}:/ragflow" \
|
||||
-v "${PWD}/internal/binding/cpp/resource:/usr/share/infinity/resource" \
|
||||
-v "${RAGFLOW_NATIVE_LIBS_DIR}:${RAGFLOW_NATIVE_LIBS_DIR}:ro" \
|
||||
infiniflow/infinity_builder:ubuntu22_clang20
|
||||
sudo docker exec "${BUILDER_CONTAINER}" bash -c 'git config --global safe.directory "*" && cd /ragflow && ./build.sh --cpp'
|
||||
|
||||
|
||||
+13
-11
@@ -227,19 +227,21 @@ RUN --mount=type=cache,id=ragflow_gomod,target=/root/.cache/gomod,sharing=locked
|
||||
COPY internal internal
|
||||
COPY cmd cmd
|
||||
COPY build.sh ./
|
||||
COPY tools/scripts tools/scripts
|
||||
COPY Dockerfile ./
|
||||
|
||||
# ONNX Runtime static archives: build.sh's _seed_from_system copies the ORT
|
||||
# static libs from the runner's pre-baked /opt/ragflow-native-libs/onnxruntime
|
||||
# (arch-correct per TARGETARCH) into the user-cache path ONNXRUNTIME_STATIC_PREFIX
|
||||
# before linking. These .a files are consumed at link time only and do NOT enter
|
||||
# the final image (only the compiled /ragflow/bin is COPY --from=go-builder).
|
||||
# The Dockerfile no longer pre-copies them here: pre-creating the user-cache dir
|
||||
# would make _seed_from_system treat it as "already cached" and skip the copy on
|
||||
# the non-matching architecture (e.g. the arm64 runner has no x64 dir), so we
|
||||
# rely on build.sh to seed ORT for both amd64 and arm64. ARG ORT_VERSION stays
|
||||
# because build.sh's check_ort_version_consistency greps it to cross-check the
|
||||
# pinned ORT release against internal/common/environments.go.
|
||||
# ONNX Runtime static archives are provisioned by build.sh from the versioned
|
||||
# user cache at ONNXRUNTIME_STATIC_PREFIX (${RAGFLOW_NATIVE_LIBS_DIR:-$HOME/
|
||||
# ragflow-native-libs}/onnxruntime/static_lib). There is NO /opt pre-bake: on CI
|
||||
# the host cache is mounted in at build time, and for the product image build
|
||||
# (where no host cache is mounted) build.sh falls back to _ensure_native_libs_installed,
|
||||
# which runs `uv run tools/scripts/download_dev_deps.py`. That downloader fetches
|
||||
# the native archives on demand and self-creates its deps/ workdir when absent
|
||||
# (see download_dev_deps.py::ensure_deps_dir), so it no longer crashes on a bare
|
||||
# go-builder checkout. These .a files are consumed at link time only and do NOT
|
||||
# enter the final image (only the compiled /ragflow/bin is COPY --from=go-builder).
|
||||
# ARG ORT_VERSION stays because build.sh's check_ort_version_consistency greps it
|
||||
# to cross-check the pinned ORT release against internal/common/environments.go.
|
||||
ARG ORT_VERSION=1.29.0
|
||||
|
||||
RUN git config --global safe.directory "*" && \
|
||||
|
||||
@@ -15,7 +15,25 @@ PROJECT_ROOT="$SCRIPT_DIR"
|
||||
# untouched and merge conflicts against the EE branch are limited to this
|
||||
# single line plus the guarded hook call inside run_native_integration_tests.
|
||||
[[ -f "${SCRIPT_DIR}/build_ee.sh" ]] && source "${SCRIPT_DIR}/build_ee.sh"
|
||||
# Version-keyed native-lib resolution: each native lib sits under a
|
||||
# v<version>/ subdir of the user cache (~/ragflow-native-libs/<lib>/v<version>/),
|
||||
# so a single mounted cache holds several versions at once. The version numbers
|
||||
# come from go.mod (office_oxide / pdf_oxide), environments.go (ORT) and
|
||||
# native_lib_version.sh (pdfium) — see that file for the single source of truth.
|
||||
source "${SCRIPT_DIR}/tools/scripts/native_lib_version.sh"
|
||||
# ort_stamp.sh: pure helper that computes the relative, read-only-safe ORT
|
||||
# version-stamp symlink target (Fix #1/#2).
|
||||
source "${SCRIPT_DIR}/tools/scripts/ort_stamp.sh"
|
||||
# ort_present.sh: pure helper that decides whether the required ORT static lib
|
||||
# is already in the version+platform-keyed user cache (Fix: version/platform
|
||||
# aware presence gate).
|
||||
source "${SCRIPT_DIR}/tools/scripts/ort_present.sh"
|
||||
|
||||
# Native static-lib user-cache root. Overridable via RAGFLOW_NATIVE_LIBS_DIR so
|
||||
# CI can mount the cache at a fixed host path (sep-tests.yml Slice 3); defaults
|
||||
# to ~/ragflow-native-libs. Captured once here and reused by every <lib>/v<version>
|
||||
# prefix below so the whole script agrees on a single cache root.
|
||||
NATIVE_LIBS_DIR="$(ragflow_native_libs_dir)"
|
||||
# Build directories
|
||||
CPP_DIR="$PROJECT_ROOT/internal/binding/cpp"
|
||||
BUILD_DIR="$CPP_DIR/cmake-build-release"
|
||||
@@ -43,31 +61,42 @@ if [ -n "$EXTRA_GO_TAGS" ]; then
|
||||
fi
|
||||
|
||||
# Native static library settings. These are the user-cache paths (~/ragflow-native-libs/).
|
||||
# If /opt/ragflow-native-libs/ exists (pre-seeded in CI runner image), it takes priority
|
||||
# and skips the network (download_dev_deps.py) fallback.
|
||||
SYSTEM_DEPS="/opt/ragflow-native-libs"
|
||||
# Each lib lives under a version-keyed subdir (v<version>/) placed there by
|
||||
# tools/scripts/download_dev_deps.py (idempotent, version-pinned) — run once by a
|
||||
# developer, or mounted in from the build host on CI. There is NO /opt pre-bake and
|
||||
# NO flat (version-less) fallback: a missing version dir means the cache is
|
||||
# incomplete and build.sh triggers the downloader itself (see setup_cgo_env).
|
||||
|
||||
# office_oxide native library settings — static linking
|
||||
OFFICE_OXIDE_PREFIX="${HOME}/ragflow-native-libs/office_oxide"
|
||||
# Derived from go.mod (single source of truth: the Go module version). Matches
|
||||
# tools/scripts/download_dev_deps.py's office_oxide_url derivation. The pipeline exits 0
|
||||
# with an empty value when go.mod has no matching entry (no pipefail here), so
|
||||
# the empty case needs its own check: an empty version would slip past the
|
||||
# stale-lib comparison below and produce a broken .../lib/v link path.
|
||||
OFFICE_OXIDE_VERSION="$(grep -m1 -E '^[[:space:]]*github.com/yfedoseev/office_oxide/go[[:space:]]+v' "$PROJECT_ROOT/go.mod" | sed -E 's/.*v([0-9][^[:space:]]*).*/\1/')"
|
||||
# office_oxide native library settings — static linking. Version is derived from
|
||||
# go.mod (single source of truth, exactly as tools/scripts/download_dev_deps.py
|
||||
# derives it). An empty value (module absent from go.mod) would produce a broken
|
||||
# .../lib/v link path, so it is fatal. The prefix is set after the version, since
|
||||
# it embeds the version.
|
||||
OFFICE_OXIDE_VERSION="$(go_module_version "github.com/yfedoseev/office_oxide/go" "$PROJECT_ROOT")"
|
||||
if [ -z "$OFFICE_OXIDE_VERSION" ]; then
|
||||
echo -e "${RED}Error: could not read office_oxide version from go.mod${NC}" >&2; exit 1
|
||||
fi
|
||||
OFFICE_OXIDE_PREFIX="${NATIVE_LIBS_DIR}/office_oxide/v${OFFICE_OXIDE_VERSION}"
|
||||
|
||||
# pdfium native library settings — static linking (kognitos/pdfium-static)
|
||||
# Pinned here (not grepped from the dev downloader) so build.sh stays tied to the
|
||||
# checked-out source tree and the Docker build needs no extra COPY of the script.
|
||||
PDFIUM_STATIC_PREFIX="${HOME}/ragflow-native-libs/pdfium-static"
|
||||
PDFIUM_STATIC_VERSION="7809"
|
||||
# pdfium native library settings — static linking (kognitos/pdfium-static).
|
||||
# Version is read from the single source of truth tools/scripts/native_lib_version.sh
|
||||
# (pdfium_static_version), exactly as download_dev_deps.py does, so the bare build
|
||||
# number lives in exactly one place. An empty value is fatal.
|
||||
PDFIUM_STATIC_VERSION="$(pdfium_static_version "$PROJECT_ROOT")"
|
||||
if [ -z "$PDFIUM_STATIC_VERSION" ]; then
|
||||
echo -e "${RED}Error: could not read pdfium version from native_lib_version.sh${NC}" >&2; exit 1
|
||||
fi
|
||||
PDFIUM_STATIC_PREFIX="${NATIVE_LIBS_DIR}/pdfium-static/v${PDFIUM_STATIC_VERSION}"
|
||||
|
||||
# pdf_oxide native library settings — static linking (go-ffi tarball)
|
||||
PDF_OXIDE_PREFIX="${HOME}/ragflow-native-libs/pdf_oxide"
|
||||
PDF_OXIDE_VERSION="0.3.73"
|
||||
# pdf_oxide native library settings — static linking (go-ffi tarball). Version
|
||||
# is read from go.mod (single source of truth) instead of a hardcoded constant,
|
||||
# so bumping the dependency needs no build.sh edit. The empty case is fatal for
|
||||
# the same reason as office_oxide. The prefix is set after the version.
|
||||
PDF_OXIDE_VERSION="$(go_module_version "github.com/yfedoseev/pdf_oxide/go" "$PROJECT_ROOT")"
|
||||
if [ -z "$PDF_OXIDE_VERSION" ]; then
|
||||
echo -e "${RED}Error: could not read pdf_oxide version from go.mod${NC}" >&2; exit 1
|
||||
fi
|
||||
PDF_OXIDE_PREFIX="${NATIVE_LIBS_DIR}/pdf_oxide/v${PDF_OXIDE_VERSION}"
|
||||
|
||||
# onnxruntime native library settings — static linking for the in-process
|
||||
# (Go) DeepDoc backend. libonnxruntime*.a is linked into the server binary
|
||||
@@ -75,7 +104,7 @@ PDF_OXIDE_VERSION="0.3.73"
|
||||
# is then resolved via dlopen(NULL) (the process-global symbol table, not the
|
||||
# executable's own path), so no libonnxruntime.so is needed at runtime.
|
||||
# Downloaded by tools/scripts/download_dev_deps.py into onnxruntime/static_lib.
|
||||
ONNXRUNTIME_STATIC_PREFIX="${HOME}/ragflow-native-libs/onnxruntime/static_lib"
|
||||
ONNXRUNTIME_STATIC_PREFIX="${NATIVE_LIBS_DIR}/onnxruntime/static_lib"
|
||||
|
||||
# ── target platform detection ──────────────────────────────────────────
|
||||
# Defaults to the build host (uname), overridable with RAGFLOW_TARGET_OS /
|
||||
@@ -155,27 +184,62 @@ ort_static_lib_prefix() {
|
||||
esac
|
||||
}
|
||||
|
||||
# Copy a dependency from the system pre-seed directory to the user cache.
|
||||
# Returns 0 if the dep was copied or already exists in cache, 1 otherwise.
|
||||
_seed_from_system() {
|
||||
local dep_name="$1" # e.g. "pdfium-static", "pdf_oxide", "office_oxide"
|
||||
local dep_dir="${HOME}/ragflow-native-libs/${dep_name}"
|
||||
local sys_dir="${SYSTEM_DEPS}/${dep_name}"
|
||||
# ── native static library provisioning ──────────────────────────────────
|
||||
# The native .a files live in the version-keyed user cache
|
||||
# (~/ragflow-native-libs/<lib>/v<version>/), placed there by
|
||||
# tools/scripts/download_dev_deps.py — idempotently and version-pinned — which a
|
||||
# developer runs once, and which CI mounts in from the build host. There is NO
|
||||
# /opt pre-bake and NO flat (version-less) fallback any more: a missing version
|
||||
# dir means the cache is incomplete, so build.sh triggers the downloader itself
|
||||
# (controlled auto-download) rather than silently linking a stale or wrong lib.
|
||||
|
||||
echo "check if dep ${dep_name} exists in ${dep_dir} or ${sys_dir}"
|
||||
|
||||
if [ -d "$dep_dir" ]; then
|
||||
echo " ${dep_name} → ${dep_dir} (user cache)"
|
||||
return 0 # already cached
|
||||
# True when every native lib the Go build links is already in the versioned user
|
||||
# cache for the target platform. Used to skip the network entirely on an
|
||||
# already-provisioned host.
|
||||
_native_libs_present() {
|
||||
[ -f "${OFFICE_OXIDE_PREFIX}/lib/liboffice_oxide.a" ] || return 1
|
||||
[ -f "${OFFICE_OXIDE_PREFIX}/include/office_oxide_c/office_oxide.h" ] || return 1
|
||||
[ -f "${PDFIUM_STATIC_PREFIX}/lib/libpdfium.a" ] || return 1
|
||||
[ -f "${PDF_OXIDE_PREFIX}/lib/${TARGET_OS}_${TARGET_ARCH}/libpdf_oxide.a" ] || return 1
|
||||
# ORT must match the SAME version+platform-keyed dir that check_onnxruntime_deps
|
||||
# links (*/${ort_dir_prefix}${ort_version}*/lib/*.a). A shared cache holding
|
||||
# only an OLD version or a DIFFERENT platform's ORT must be reported missing so
|
||||
# the controlled auto-downloader runs, instead of being silently treated as
|
||||
# present and failing only at link time.
|
||||
if [ -z "$_TARGET_PLATFORM_DETECTED" ]; then
|
||||
detect_target_platform
|
||||
_TARGET_PLATFORM_DETECTED=1
|
||||
fi
|
||||
if [ -d "$sys_dir" ]; then
|
||||
echo " ${dep_name} → ${sys_dir} (system)"
|
||||
mkdir -p "$(dirname "$dep_dir")"
|
||||
cp -r "$sys_dir" "$dep_dir"
|
||||
local ort_version ort_dir_prefix
|
||||
ort_version="$(deepdoc_ort_version "$PROJECT_ROOT")" || return 1
|
||||
ort_dir_prefix="$(ort_static_lib_prefix)"
|
||||
ragflow_ort_present "$ONNXRUNTIME_STATIC_PREFIX" "$ort_dir_prefix" "$ort_version" || return 1
|
||||
return 0
|
||||
}
|
||||
|
||||
# Controlled auto-download: if the versioned native libs are not yet in the user
|
||||
# cache, run the one-stop downloader ONCE. It is idempotent and version-pinned,
|
||||
# so it can only land the exact pinned .a — never a wrong version. Fail loud only
|
||||
# when the cache is still incomplete afterwards (a locked-down host with no
|
||||
# network and no pre-placed libs).
|
||||
_ensure_native_libs_installed() {
|
||||
_native_libs_present && return 0
|
||||
echo "One or more native static libraries are missing from the user cache;"
|
||||
echo " running downloader: uv run tools/scripts/download_dev_deps.py"
|
||||
( cd "$PROJECT_ROOT" && uv run tools/scripts/download_dev_deps.py ) || {
|
||||
echo -e "${RED}Error: native library downloader failed.${NC}" >&2
|
||||
exit 1
|
||||
}
|
||||
if _native_libs_present; then
|
||||
return 0
|
||||
fi
|
||||
echo " ${dep_name} not found in system or user cache"
|
||||
return 1
|
||||
echo -e "${RED}Error: native static libraries still missing after download.${NC}" >&2
|
||||
echo " Pre-place the version-keyed dirs manually on a locked-down host:" >&2
|
||||
echo " ${OFFICE_OXIDE_PREFIX}" >&2
|
||||
echo " ${PDFIUM_STATIC_PREFIX}" >&2
|
||||
echo " ${PDF_OXIDE_PREFIX}" >&2
|
||||
echo " ${ONNXRUNTIME_STATIC_PREFIX}" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
# ── cl100k_base BPE table ─────────────────────────────────────────────
|
||||
@@ -333,7 +397,7 @@ check_ort_version_consistency() {
|
||||
print_section "Checking ONNX Runtime version consistency"
|
||||
|
||||
local env_go dockerfile
|
||||
env_go="$(grep -m1 -E 'DeepDocORTVersion[[:space:]]*=[[:space:]]*"' "${PROJECT_ROOT}/internal/common/environments.go" | sed -E 's/.*"([^"]+)".*/\1/')"
|
||||
env_go="$(deepdoc_ort_version "${PROJECT_ROOT}")"
|
||||
dockerfile="$(grep -m1 -E 'ARG[[:space:]]+ORT_VERSION=' "${PROJECT_ROOT}/Dockerfile" | sed -E 's/.*ORT_VERSION=([0-9][^"[:space:]]*).*/\1/')"
|
||||
|
||||
if [ -z "$env_go" ] || [ -z "$dockerfile" ]; then
|
||||
@@ -366,8 +430,9 @@ check_ort_version_consistency() {
|
||||
# that lacks those custom slots links successfully and only fails at runtime.
|
||||
# The check below catches that by comparing the local release ZIP against the
|
||||
# published {asset}.sha256 sidecar (same source of truth the download scripts
|
||||
# use). Devs who ran a download script have the zip under deps/; CI
|
||||
# seeds from /opt and has no zip, where the bake is authoritative.
|
||||
# use). Devs who ran a download script have the zip under deps/; on CI the
|
||||
# version-keyed cache is mounted from the build host and no zip is present, so
|
||||
# the sha256 check is skipped there and the mounted cache seed is authoritative.
|
||||
check_onnxruntime_deps() {
|
||||
# Ensure TARGET_OS/TARGET_ARCH are resolved. ort_static_lib_prefix below
|
||||
# depends on them; without this guard an early call (before any other path
|
||||
@@ -380,9 +445,7 @@ check_onnxruntime_deps() {
|
||||
fi
|
||||
|
||||
local ort_version
|
||||
ort_version="$(grep -m1 -E 'DeepDocORTVersion[[:space:]]*=[[:space:]]*"' \
|
||||
"${PROJECT_ROOT}/internal/common/environments.go" \
|
||||
| sed -E 's/.*"([^"]+)".*/\1/')"
|
||||
ort_version="$(deepdoc_ort_version "${PROJECT_ROOT}")"
|
||||
if [ -z "$ort_version" ]; then
|
||||
echo -e "${RED}Error: cannot parse DeepDocORTVersion from internal/common/environments.go${NC}" >&2
|
||||
return 1
|
||||
@@ -400,6 +463,11 @@ check_onnxruntime_deps() {
|
||||
# another OS/arch, and the Python side legitimately keeps another ORT
|
||||
# version in the same prefix. Matching both is why coexisting version
|
||||
# dirs need no separate fail-fast guard here.
|
||||
# The ORT static archives are resolved from the version-keyed user cache
|
||||
# (~/ragflow-native-libs/onnxruntime/static_lib, which the downloader
|
||||
# populates and CI mounts in). There is no /opt pre-bake to reseed from; an
|
||||
# incomplete cache is filled by the controlled auto-download in setup_cgo_env.
|
||||
|
||||
local ort_dir_prefix
|
||||
ort_dir_prefix="$(ort_static_lib_prefix)"
|
||||
while IFS= read -r f; do
|
||||
@@ -432,7 +500,7 @@ check_onnxruntime_deps() {
|
||||
echo " available: ${avail:-<none>}" >&2
|
||||
echo " DeepDocORTVersion=${ort_version}; fetch it with:" >&2
|
||||
echo " uv run tools/scripts/download_dev_deps.py" >&2
|
||||
echo " or pre-seed /opt/ragflow-native-libs/onnxruntime (CI image)." >&2
|
||||
echo " or pre-seed ${NATIVE_LIBS_DIR}/onnxruntime (CI mounts the cache from the build host)." >&2
|
||||
return 1
|
||||
fi
|
||||
|
||||
@@ -495,7 +563,9 @@ check_onnxruntime_deps() {
|
||||
# Check office_oxide native library
|
||||
check_office_oxide_deps() {
|
||||
print_section "Checking office_oxide native library"
|
||||
_seed_from_system "office_oxide" || true
|
||||
# office_oxide is resolved from the version-keyed user cache
|
||||
# (${OFFICE_OXIDE_PREFIX}). An incomplete cache is filled by the controlled
|
||||
# auto-download in setup_cgo_env; there is no /opt pre-bake to recover from.
|
||||
|
||||
local lib_file="liboffice_oxide.a"
|
||||
local lib_path="${OFFICE_OXIDE_PREFIX}/lib/${lib_file}"
|
||||
@@ -516,28 +586,11 @@ check_office_oxide_deps() {
|
||||
if ! strings "$lib_path" 2>/dev/null | grep -Fxq "$OFFICE_OXIDE_VERSION"; then
|
||||
local found_version
|
||||
found_version=$(strings "$lib_path" 2>/dev/null | grep -E "^0\.[0-9]+\.[0-9]+$" | head -1)
|
||||
echo -e "${YELLOW}office_oxide native lib version mismatch (required v${OFFICE_OXIDE_VERSION}; found: ${found_version:-unknown}); refreshing from download${NC}"
|
||||
rm -rf "${OFFICE_OXIDE_PREFIX}" "${PROJECT_ROOT}/deps/$(office_oxide_asset)"
|
||||
if ! (cd "${PROJECT_ROOT}" && uv run tools/scripts/download_dev_deps.py); then
|
||||
echo -e "${RED}Error: office_oxide native lib version mismatch${NC}"
|
||||
echo " Required: v${OFFICE_OXIDE_VERSION}; found: ${found_version:-unknown}"
|
||||
echo " A stale lib silently loses PPT97 (.ppt) slide content. Refresh:"
|
||||
echo " rm -rf ~/ragflow-native-libs/office_oxide deps/$(office_oxide_asset)"
|
||||
echo " uv run tools/scripts/download_dev_deps.py"
|
||||
exit 1
|
||||
fi
|
||||
lib_path="${OFFICE_OXIDE_PREFIX}/lib/${lib_file}"
|
||||
header_path="${OFFICE_OXIDE_PREFIX}/include/office_oxide_c/office_oxide.h"
|
||||
if [ ! -f "$lib_path" ] || [ ! -f "$header_path" ]; then
|
||||
echo -e "${RED}Error: office_oxide native library not found after refresh${NC}"
|
||||
exit 1
|
||||
fi
|
||||
if ! strings "$lib_path" 2>/dev/null | grep -Fxq "$OFFICE_OXIDE_VERSION"; then
|
||||
found_version=$(strings "$lib_path" 2>/dev/null | grep -E "^0\.[0-9]+\.[0-9]+$" | head -1)
|
||||
echo -e "${RED}Error: office_oxide native lib version mismatch after refresh${NC}"
|
||||
echo " Required: v${OFFICE_OXIDE_VERSION}; found: ${found_version:-unknown}"
|
||||
exit 1
|
||||
fi
|
||||
echo -e "${RED}Error: office_oxide native lib version mismatch${NC}"
|
||||
echo " Required: v${OFFICE_OXIDE_VERSION}; found: ${found_version:-unknown}"
|
||||
echo " A stale lib silently loses PPT97 (.ppt) slide content. Refresh:"
|
||||
echo " uv run tools/scripts/download_dev_deps.py # re-extracts into ${OFFICE_OXIDE_PREFIX}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "✓ office_oxide v${OFFICE_OXIDE_VERSION} native library found at ${OFFICE_OXIDE_PREFIX}"
|
||||
@@ -546,7 +599,9 @@ check_office_oxide_deps() {
|
||||
|
||||
# Check pdfium static library.
|
||||
check_pdfium_deps() {
|
||||
_seed_from_system "pdfium-static" || true
|
||||
# pdfium is resolved from the version-keyed user cache
|
||||
# (${PDFIUM_STATIC_PREFIX}). An incomplete cache is filled by the controlled
|
||||
# auto-download in setup_cgo_env; there is no /opt pre-bake to recover from.
|
||||
local lib_path="${PDFIUM_STATIC_PREFIX}/lib/libpdfium.a"
|
||||
|
||||
if [ -f "$lib_path" ]; then
|
||||
@@ -604,7 +659,9 @@ pdf_oxide_validate_version() {
|
||||
}
|
||||
|
||||
check_pdf_oxide_deps() {
|
||||
_seed_from_system "pdf_oxide" || true
|
||||
# pdf_oxide is resolved from the version-keyed user cache
|
||||
# (${PDF_OXIDE_PREFIX}). An incomplete cache is filled by the controlled
|
||||
# auto-download in setup_cgo_env; there is no /opt pre-bake to recover from.
|
||||
# Map platform to tarball-internal subdirectory (lib/<os>_<arch>/). Reuses
|
||||
# TARGET_OS/TARGET_ARCH set by detect_target_platform instead of re-deriving
|
||||
# from uname, so it stays consistent with the other check_*_deps helpers.
|
||||
@@ -617,10 +674,10 @@ check_pdf_oxide_deps() {
|
||||
local lib_path="${PDF_OXIDE_PREFIX}/lib/${platform_subdir}/libpdf_oxide.a"
|
||||
|
||||
if [ -f "$lib_path" ]; then
|
||||
# Verify the on-disk lib matches the pinned version. _seed_from_system
|
||||
# accepts an existing user-cache directory without inspecting it, so a
|
||||
# lib left over from an earlier pin is reused silently and the upgrade
|
||||
# becomes a no-op.
|
||||
# Verify the on-disk lib matches the pinned version. The cache is
|
||||
# version-keyed (${PDF_OXIDE_PREFIX}), so an earlier pin lives under a
|
||||
# different v<version>/ dir and is never silently reused; this check is a
|
||||
# defense-in-depth against a stale .a that somehow landed in this dir.
|
||||
#
|
||||
# The version marker "pdf_oxide <version>" is merged into Rust's string
|
||||
# constant pool (unlike office_oxide's standalone "0.1.9" line), so we
|
||||
@@ -798,12 +855,12 @@ build_go() {
|
||||
echo " at startup with a fatal 'no in-process DeepDoc backend serving'." >&2
|
||||
echo " Fetch the static libs with:" >&2
|
||||
echo " uv run tools/scripts/download_dev_deps.py" >&2
|
||||
echo " or pre-seed them at /opt/ragflow-native-libs/onnxruntime (CI image)." >&2
|
||||
echo " or pre-seed them at ${NATIVE_LIBS_DIR}/onnxruntime (CI mounts the cache from the build host)." >&2
|
||||
echo " This production binary must statically include ORT — there is no" >&2
|
||||
echo " ORT-free build path. ORT ends up unlinked only via one of:" >&2
|
||||
echo " - the ORT static_lib dir was never seeded: run" >&2
|
||||
echo " 'uv run tools/scripts/download_dev_deps.py', or pre-seed" >&2
|
||||
echo " /opt/ragflow-native-libs/onnxruntime as the CI image does;" >&2
|
||||
echo " ${NATIVE_LIBS_DIR}/onnxruntime as the CI mount does;" >&2
|
||||
echo " - setup_cgo_env did not add libonnxruntime to CGO_LDFLAGS." >&2
|
||||
return 1
|
||||
fi
|
||||
@@ -837,25 +894,22 @@ build_go() {
|
||||
setup_cgo_env() {
|
||||
detect_target_platform
|
||||
|
||||
# Controlled auto-download: fill any missing version-keyed native lib from
|
||||
# the user cache before resolving individual libs. On an already-provisioned
|
||||
# host (the common CI case, where the cache is mounted in) this is a no-op;
|
||||
# on a fresh host it runs the one-stop downloader once. There is no /opt
|
||||
# pre-bake and no flat fallback to recover from.
|
||||
_ensure_native_libs_installed
|
||||
|
||||
# ── office_oxide ──────────────────────────────────────────────────
|
||||
check_office_oxide_deps
|
||||
|
||||
# Go's build cache keys CGO_LDFLAGS as a string — it does NOT hash the
|
||||
# file content of referenced .a archives. So swapping the .a in-place
|
||||
# (same path) does NOT invalidate the cache, and `go build` silently
|
||||
# reuses a stale binary linked against the old .a.
|
||||
#
|
||||
# Create a version-stamped symlink directory so the flag string
|
||||
# includes the actual linked version. When the .a is upgraded, the
|
||||
# path changes → Go cache key changes → automatic relink.
|
||||
local office_oxide_lib_dir="${OFFICE_OXIDE_PREFIX}/lib"
|
||||
local versioned_lib_dir="${office_oxide_lib_dir}/v${OFFICE_OXIDE_VERSION}"
|
||||
mkdir -p "$versioned_lib_dir"
|
||||
ln -sf "${office_oxide_lib_dir}/liboffice_oxide.a" \
|
||||
"${versioned_lib_dir}/liboffice_oxide.a"
|
||||
|
||||
# The office_oxide prefix is already version-keyed
|
||||
# (${OFFICE_OXIDE_PREFIX} = .../office_oxide/v<version>), so the linked path
|
||||
# carries the version and Go's build cache (which keys CGO_LDFLAGS as a
|
||||
# string, not the .a contents) automatically relinks when the version bumps.
|
||||
export CGO_CFLAGS="-I${OFFICE_OXIDE_PREFIX}/include/office_oxide_c${CGO_CFLAGS:+ $CGO_CFLAGS}"
|
||||
export CGO_LDFLAGS="${versioned_lib_dir}/liboffice_oxide.a"
|
||||
export CGO_LDFLAGS="${OFFICE_OXIDE_PREFIX}/lib/liboffice_oxide.a"
|
||||
|
||||
# ── pdfium ────────────────────────────────────────────────────────
|
||||
check_pdfium_deps || return 1
|
||||
@@ -894,16 +948,11 @@ setup_cgo_env() {
|
||||
linux_amd64|linux_arm64|darwin_amd64|darwin_arm64) ;;
|
||||
*) echo "pdf_oxide: unsupported platform: $pdf_oxide_subdir"; return 1 ;;
|
||||
esac
|
||||
# Version-stamp the archive path so an in-place .a upgrade invalidates
|
||||
# Go's build cache. See the office_oxide block above for why the raw path
|
||||
# is not enough: the cache key hashes CGO_LDFLAGS as a string, not the
|
||||
# contents of the archives it names.
|
||||
# The pdf_oxide prefix is already version-keyed
|
||||
# (${PDF_OXIDE_PREFIX} = .../pdf_oxide/v<version>), so the linked path
|
||||
# carries the version and Go's build cache relinks on a version bump.
|
||||
local pdf_oxide_lib_dir="${PDF_OXIDE_PREFIX}/lib/${pdf_oxide_subdir}"
|
||||
local pdf_oxide_versioned_dir="${pdf_oxide_lib_dir}/v${PDF_OXIDE_VERSION}"
|
||||
mkdir -p "$pdf_oxide_versioned_dir"
|
||||
ln -sf "${pdf_oxide_lib_dir}/libpdf_oxide.a" \
|
||||
"${pdf_oxide_versioned_dir}/libpdf_oxide.a"
|
||||
export CGO_LDFLAGS="$CGO_LDFLAGS ${pdf_oxide_versioned_dir}/libpdf_oxide.a"
|
||||
export CGO_LDFLAGS="$CGO_LDFLAGS ${pdf_oxide_lib_dir}/libpdf_oxide.a"
|
||||
|
||||
# ── onnxruntime (static, resolved via dlopen(NULL)) ────────────────
|
||||
# Statically link libonnxruntime*.a into the binary. The org Go binding
|
||||
@@ -930,11 +979,10 @@ setup_cgo_env() {
|
||||
# all unused objects and missing transitive dependencies), and exports
|
||||
# it via -export_dynamic so runtime dlopen(NULL) resolves it.
|
||||
#
|
||||
# Seed the static ORT archives from the system pre-bake (/opt, laid down
|
||||
# by the CI runner image) into the user cache before the link check
|
||||
# below. Mirrors the existing _seed_from_system calls for the other
|
||||
# native libs so CI never downloads ORT at build/test time.
|
||||
_seed_from_system "onnxruntime" || true
|
||||
# The static ORT archives come from the version-keyed user cache (populated
|
||||
# by download_dev_deps.py and mounted in on CI) — there is no /opt pre-bake.
|
||||
# An incomplete cache is filled by the controlled auto-download at the top
|
||||
# of setup_cgo_env.
|
||||
check_onnxruntime_deps || return 1
|
||||
|
||||
# Version-stamp the archive paths so an in-place .a upgrade invalidates
|
||||
@@ -944,15 +992,22 @@ setup_cgo_env() {
|
||||
# linked binary. Stamp with the primary archive's sha256 so the flag string
|
||||
# changes when the content changes → automatic relink.
|
||||
local ort_version
|
||||
ort_version="$(grep -m1 -E 'DeepDocORTVersion[[:space:]]*=[[:space:]]*"' \
|
||||
"${PROJECT_ROOT}/internal/common/environments.go" \
|
||||
| sed -E 's/.*"([^"]+)".*/\1/')"
|
||||
ort_version="$(deepdoc_ort_version "${PROJECT_ROOT}")"
|
||||
local ort_versioned_dir="${ONNXRUNTIME_STATIC_PREFIX}/v${ort_version}-${ORT_A_SHA256:0:16}"
|
||||
mkdir -p "$ort_versioned_dir"
|
||||
local ort_a_versioned=""
|
||||
local f
|
||||
local f stamp needs_write want_target
|
||||
for f in $ORT_A_FILES; do
|
||||
ln -sf "$f" "$ort_versioned_dir/$(basename "$f")"
|
||||
# Relative, read-only-safe stamp (Fix #1/#2): the symlink target is
|
||||
# relative to the versioned dir so it resolves under any cache mount, and
|
||||
# the write is skipped when an existing link already points at the same
|
||||
# archive (a read-only CI mount is then a no-op).
|
||||
stamp="$(ragflow_ort_stamp_target "$ort_versioned_dir" "$f")"
|
||||
needs_write="${stamp%%|*}"
|
||||
want_target="${stamp#*|}"
|
||||
if [ "$needs_write" = "yes" ]; then
|
||||
mkdir -p "$ort_versioned_dir"
|
||||
ln -sf "$want_target" "$ort_versioned_dir/$(basename "$f")"
|
||||
fi
|
||||
ort_a_versioned="$ort_a_versioned $ort_versioned_dir/$(basename "$f")"
|
||||
done
|
||||
|
||||
|
||||
+51
-19
@@ -183,13 +183,40 @@ The image contains Go DeepDoc weights, tokenizer assets, and the BPE table. Deep
|
||||
> SYS_LIBS="-fuse-ld=lld -lm -lpthread -ldl -lrt -lgcc_s -lutil -lc"
|
||||
> fi
|
||||
>
|
||||
> export CGO_CFLAGS="-I${RAGFLOW_DEPS}/office_oxide/include/office_oxide_c"
|
||||
> # Native libs are version-keyed under <lib>/v<version>/ (one mounted cache can
|
||||
> # hold several versions at once), matching the layout download_dev_deps.py and
|
||||
> # build.sh resolve. Resolve each version from its single source of truth — NOT
|
||||
> # by glob + `head -1`, which silently pins the lexicographically-oldest version
|
||||
> # in the cache (e.g. v0.1.12 over the pinned v0.1.16, dropping PPT97 slide
|
||||
> # content). build.sh does exactly this in setup_cgo_env(); mirror it here.
|
||||
> # Run this snippet from the ragflow repo root (where go.mod lives).
|
||||
> source tools/scripts/native_lib_version.sh
|
||||
> REPO_ROOT="$(pwd)"
|
||||
> OFFICE_OXIDE_VERSION="$(go_module_version "github.com/yfedoseev/office_oxide/go" "$REPO_ROOT")"
|
||||
> PDFIUM_STATIC_VERSION="$(pdfium_static_version "$REPO_ROOT")"
|
||||
> PDF_OXIDE_VERSION="$(go_module_version "github.com/yfedoseev/pdf_oxide/go" "$REPO_ROOT")"
|
||||
>
|
||||
> OFFICE_OXIDE_PREFIX="${RAGFLOW_DEPS}/office_oxide/v${OFFICE_OXIDE_VERSION}"
|
||||
> PDFIUM_PREFIX="${RAGFLOW_DEPS}/pdfium-static/v${PDFIUM_STATIC_VERSION}"
|
||||
> PDF_OXIDE_PREFIX="${RAGFLOW_DEPS}/pdf_oxide/v${PDF_OXIDE_VERSION}"
|
||||
>
|
||||
> OFFICE_OXIDE_A="${OFFICE_OXIDE_PREFIX}/lib/liboffice_oxide.a"
|
||||
> PDFIUM_A="${PDFIUM_PREFIX}/lib/libpdfium.a"
|
||||
> PDFIUM_CXX_A="${PDFIUM_PREFIX}/lib/libc++.a"
|
||||
> PDFIUM_CXXABI_A="${PDFIUM_PREFIX}/lib/libc++abi.a"
|
||||
> PDF_OXIDE_A="${PDF_OXIDE_PREFIX}/lib/${PLATFORM}/libpdf_oxide.a"
|
||||
>
|
||||
> # Derive the include dir from the resolved office_oxide archive: the shell does
|
||||
> # not expand globs inside double quotes, so a literal
|
||||
> # ".../office_oxide/v*/include/..." passed to CGO_CFLAGS would never resolve and
|
||||
> # cgo would fail to find office_oxide.h on the first native compile.
|
||||
> export CGO_CFLAGS="-I$(dirname "$(dirname "$OFFICE_OXIDE_A")")/include/office_oxide_c"
|
||||
> export CGO_LDFLAGS="\
|
||||
> ${RAGFLOW_DEPS}/office_oxide/lib/liboffice_oxide.a \
|
||||
> ${RAGFLOW_DEPS}/pdfium-static/lib/libpdfium.a \
|
||||
> ${RAGFLOW_DEPS}/pdfium-static/lib/libc++.a \
|
||||
> ${RAGFLOW_DEPS}/pdfium-static/lib/libc++abi.a \
|
||||
> ${RAGFLOW_DEPS}/pdf_oxide/lib/${PLATFORM}/libpdf_oxide.a \
|
||||
> ${OFFICE_OXIDE_A} \
|
||||
> ${PDFIUM_A} \
|
||||
> ${PDFIUM_CXX_A} \
|
||||
> ${PDFIUM_CXXABI_A} \
|
||||
> ${PDF_OXIDE_A} \
|
||||
> ${ORT_LINK} \
|
||||
> ${SYS_LIBS}"
|
||||
> ```
|
||||
@@ -214,23 +241,27 @@ The image contains Go DeepDoc weights, tokenizer assets, and the BPE table. Deep
|
||||
> Since `build.sh` (`build_go`) now **fails fast** when ORT is absent from
|
||||
> `CGO_LDFLAGS`, this breakage surfaces at build time instead of at runtime. If
|
||||
> you see `Error: ONNX Runtime static libraries are not linked`, run
|
||||
> `uv run tools/scripts/download_dev_deps.py` (or pre-seed
|
||||
> `/opt/ragflow-native-libs/onnxruntime` as the CI runner image does). There is
|
||||
> no ORT-free production build path — if ORT is absent the binary fails at
|
||||
> startup, so the remedy is always to seed the static lib above, never to build
|
||||
> without it.
|
||||
> `uv run tools/scripts/download_dev_deps.py` (or pre-seed the versioned user
|
||||
> cache at `${RAGFLOW_NATIVE_LIBS_DIR:-$HOME/ragflow-native-libs}/onnxruntime/static_lib
|
||||
> `; CI mounts the host cache into the builder). There is no ORT-free
|
||||
> production build path — if ORT is absent the binary fails at startup, so the
|
||||
> remedy is always to seed the static lib above, never to build without it.
|
||||
|
||||
> **Note**: The ONNX Runtime native version is pinned in a few places that must
|
||||
> stay in sync. Bumping it in one spot and not the others fails the build with
|
||||
> **Note**: The ONNX Runtime native version is pinned in two places that must
|
||||
> stay in sync. Bumping one and not the other fails the build with
|
||||
> `Error: ONNX Runtime version is inconsistent`:
|
||||
> - `internal/common/environments.go` — `DeepDocORTVersion`
|
||||
> - `Dockerfile` — `ARG ORT_VERSION`
|
||||
> - `tools/scripts/download_dev_deps.py` — `ORT_VERSION` (used by the dev downloader to fetch the matching release; build.sh does not grep it, but it must match what gets linked)
|
||||
>
|
||||
> `build.sh` and `tools/scripts/download_dev_deps.py` derive the ORT version at
|
||||
> build/download time from `DeepDocORTVersion` in `environments.go` (the Go const
|
||||
> is the single source of truth), so the downloader has no separate `ORT_VERSION`
|
||||
> pin to edit. To upgrade ORT, edit the two entries above to the same version,
|
||||
> then run the check.
|
||||
>
|
||||
> `build.sh` runs this consistency check automatically before the Go build
|
||||
> (through `check_go_deps`) and fails fast on any mismatch. Run it on demand
|
||||
> with `./build.sh --check-ort-version`. To upgrade ORT, edit every entry above
|
||||
> to the same version, then run the check. The Python pip `onnxruntime==` pin in
|
||||
> with `./build.sh --check-ort-version`. The Python pip `onnxruntime==` pin in
|
||||
> `pyproject.toml` is versioned independently and is intentionally not part of
|
||||
> this check.
|
||||
|
||||
@@ -243,9 +274,10 @@ The image contains Go DeepDoc weights, tokenizer assets, and the BPE table. Deep
|
||||
> published `onnxruntime-v<ver>-<target>.zip.sha256` sidecar. A local ZIP
|
||||
> mismatch only emits a warning with the expected/actual SHA-256 and the
|
||||
> refresh command (`rm -f <zip>` + `uv run tools/scripts/download_dev_deps.py`);
|
||||
> the build continues checking and linking the extracted static libraries. CI seeds
|
||||
> ORT from `/opt` and has no local zip, so the check is skipped there (the bake
|
||||
> is authoritative). This matters because the `onnxruntime_go` binding reaches
|
||||
> the build continues checking and linking the extracted static libraries. On CI
|
||||
> the version-keyed cache is mounted from the build host and no local ZIP is
|
||||
> present, so the sha256 check is skipped there and the mounted cache seed is
|
||||
> treated as authoritative. This matters because the `onnxruntime_go` binding reaches
|
||||
> ORT only through the OrtApi function-pointer table
|
||||
> (`ort_api->SessionGetInitializer*`), so a `.a` lacking a custom slot links
|
||||
> successfully and only crashes at runtime — not at link time.
|
||||
|
||||
@@ -0,0 +1,308 @@
|
||||
// Package docguard holds regression guards for repository documentation and
|
||||
// build glue that the native-lib versioned-cache change (PR #22099) must keep
|
||||
// correct. These are plain unit tests (no build tag) so they run in the default
|
||||
// `bash build.sh --test` tier and in CI.
|
||||
//
|
||||
// The guards exist because the IDE/`go build` CGO snippet in
|
||||
// internal/development.md is the one piece of build glue that is NOT executed by
|
||||
// CI: a broken snippet only fails for the developer who copy-pastes it. These
|
||||
// tests make the P1 review findings (review 5821944375) falsifiable — an
|
||||
// un-expanded `v*` include glob, a glob+`head -1` version resolution that pins
|
||||
// the oldest cache version, and stale `/opt` pre-bake prose — fail a test rather
|
||||
// than relying on prose review alone.
|
||||
package docguard
|
||||
|
||||
import (
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// repoRoot walks up from this source file to the directory containing go.mod.
|
||||
func repoRoot(t *testing.T) string {
|
||||
t.Helper()
|
||||
_, thisFile, _, ok := runtime.Caller(0)
|
||||
if !ok {
|
||||
t.Fatal("runtime.Caller failed")
|
||||
}
|
||||
dir := filepath.Dir(thisFile)
|
||||
for i := 0; i < 12; i++ {
|
||||
if _, err := os.Stat(filepath.Join(dir, "go.mod")); err == nil {
|
||||
return dir
|
||||
}
|
||||
parent := filepath.Dir(dir)
|
||||
if parent == dir {
|
||||
break
|
||||
}
|
||||
dir = parent
|
||||
}
|
||||
t.Fatal("could not locate repo root (go.mod)")
|
||||
return ""
|
||||
}
|
||||
|
||||
func readFile(t *testing.T, path string) string {
|
||||
t.Helper()
|
||||
b, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatalf("read %s: %v", path, err)
|
||||
}
|
||||
return string(b)
|
||||
}
|
||||
|
||||
// TestDevelopmentDocResolvesNativeLibVersions guards the P1 finding from review
|
||||
// 5821944375: the IDE/`go build` CGO snippet must resolve each native lib from
|
||||
// its single source of truth (the version-pinned resolvers) instead of a glob +
|
||||
// `head -1`, and the include path must be derived from the resolved archive
|
||||
// rather than a literal `v*` glob that the shell never expands inside quotes.
|
||||
func TestDevelopmentDocResolvesNativeLibVersions(t *testing.T) {
|
||||
doc := readFile(t, filepath.Join(repoRoot(t), "internal", "development.md"))
|
||||
|
||||
// The include path must no longer be a literal `v*` glob inside the
|
||||
// double-quoted CGO_CFLAGS assignment — that glob is never expanded by the
|
||||
// shell and breaks the first cgo compile.
|
||||
if strings.Contains(doc, "office_oxide/v*/include/office_oxide_c") {
|
||||
t.Error("development.md still has the un-expanded `v*` include glob in CGO_CFLAGS; derive the include from the resolved archive instead")
|
||||
}
|
||||
|
||||
// The `.a` archives must no longer be resolved by `ls …/v*/… | head -1`,
|
||||
// which silently pins the lexicographically-oldest version in a cache that
|
||||
// holds several versions at once.
|
||||
if strings.Contains(doc, "v*/lib/liboffice_oxide.a | head -1") {
|
||||
t.Error("development.md still resolves office_oxide via `ls …/v*… | head -1`; use the version-pinned resolver")
|
||||
}
|
||||
if strings.Contains(doc, "v*/lib/libpdfium.a | head -1") {
|
||||
t.Error("development.md still resolves pdfium via `ls …/v*… | head -1`; use the version-pinned resolver")
|
||||
}
|
||||
|
||||
// Positive guard: the snippet must resolve versions from the single source
|
||||
// of truth and derive the include from the resolved archive variable.
|
||||
if !strings.Contains(doc, "native_lib_version.sh") {
|
||||
t.Error("development.md should source tools/scripts/native_lib_version.sh to resolve versions from the single source of truth")
|
||||
}
|
||||
if !strings.Contains(doc, "OFFICE_OXIDE_A") {
|
||||
t.Error("development.md should resolve the office_oxide archive into OFFICE_OXIDE_A and derive CGO_CFLAGS from it")
|
||||
}
|
||||
}
|
||||
|
||||
// normalizeDoc strips markdown blockquote prefixes ('>') and collapses
|
||||
// newlines so multi-line blockquote sentences can be matched as a single
|
||||
// string. The native-lib notes in development.md are long blockquotes that wrap
|
||||
// across lines, so a raw substring search misses the stale phrases.
|
||||
func normalizeDoc(s string) string {
|
||||
s = strings.ReplaceAll(s, "\n", " ")
|
||||
s = strings.ReplaceAll(s, ">", " ")
|
||||
return strings.Join(strings.Fields(s), " ")
|
||||
}
|
||||
|
||||
// TestDevelopmentDocNoStaleOptBake guards against the superseded /opt pre-bake
|
||||
// description that the PR deleted from build.sh but that the doc still asserts.
|
||||
func TestDevelopmentDocNoStaleOptBake(t *testing.T) {
|
||||
doc := normalizeDoc(readFile(t, filepath.Join(repoRoot(t), "internal", "development.md")))
|
||||
if strings.Contains(doc, "seeds ORT from") {
|
||||
t.Error("development.md still claims CI seeds ORT from /opt; the /opt pre-bake was removed")
|
||||
}
|
||||
if strings.Contains(doc, "bake is authoritative") {
|
||||
t.Error("development.md still references the /opt pre-bake as authoritative; the pre-bake was removed")
|
||||
}
|
||||
}
|
||||
|
||||
// TestBuildShNoStaleOptBake guards the stale comment inside
|
||||
// check_onnxruntime_deps that contradicts the new "There is no /opt pre-bake"
|
||||
// comment a few lines below it.
|
||||
func TestBuildShNoStaleOptBake(t *testing.T) {
|
||||
script := readFile(t, filepath.Join(repoRoot(t), "build.sh"))
|
||||
if strings.Contains(script, "seeds from /opt") {
|
||||
t.Error("build.sh still claims CI seeds ORT from /opt inside check_onnxruntime_deps; the /opt pre-bake was removed")
|
||||
}
|
||||
if strings.Contains(script, "bake is authoritative") {
|
||||
t.Error("build.sh still references the /opt pre-bake as authoritative; the /opt pre-bake was removed")
|
||||
}
|
||||
}
|
||||
|
||||
// ortVersionNote extracts the "pinned in a few places" note block from
|
||||
// development.md so the ORT pin-list guards can assert on that note alone.
|
||||
func ortVersionNote(t *testing.T) string {
|
||||
t.Helper()
|
||||
doc := readFile(t, filepath.Join(repoRoot(t), "internal", "development.md"))
|
||||
start := strings.Index(doc, "ONNX Runtime native version is pinned")
|
||||
if start < 0 {
|
||||
t.Fatal("development.md has no ORT version-pinning note")
|
||||
}
|
||||
endMarker := "build.sh` runs this consistency check"
|
||||
end := strings.Index(doc[start:], endMarker)
|
||||
if end < 0 {
|
||||
t.Fatal("development.md ORT version note has no recognizable end")
|
||||
}
|
||||
return normalizeDoc(doc[start : start+end])
|
||||
}
|
||||
|
||||
// TestDevelopmentDocOrtPinsAreNotMisleading guards P1 sub-finding 3a: the note
|
||||
// used to list tools/scripts/download_dev_deps.py's ORT_VERSION as a THIRD pin
|
||||
// to edit when bumping ORT. That constant is derived at runtime from
|
||||
// DeepDocORTVersion (environments.go) — check_ort_version_consistency compares
|
||||
// only environments.go and the Dockerfile — so telling developers to edit the
|
||||
// downloader sends them to the wrong place. The note must (a) not present the
|
||||
// downloader as a pin bullet, and (b) still list the two real pins.
|
||||
func TestDevelopmentDocOrtPinsAreNotMisleading(t *testing.T) {
|
||||
note := ortVersionNote(t)
|
||||
// (a) No bullet that presents the downloader's ORT_VERSION as a pin to edit.
|
||||
if strings.Contains(note, "- `tools/scripts/download_dev_deps.py`") {
|
||||
t.Error("development.md still lists download_dev_deps.py ORT_VERSION as a pin to edit; it is derived from DeepDocORTVersion, not a pin")
|
||||
}
|
||||
// (a) The downloader derives its version from the Go const — no second copy.
|
||||
if !strings.Contains(note, "derive") || !strings.Contains(note, "DeepDocORTVersion") {
|
||||
t.Error("development.md should explain that build.sh and download_dev_deps.py derive ORT_VERSION from DeepDocORTVersion, so there is no third pin to edit")
|
||||
}
|
||||
// (b) The two real pin locations must remain documented.
|
||||
if !strings.Contains(note, "internal/common/environments.go") || !strings.Contains(note, "Dockerfile") {
|
||||
t.Error("development.md must still list the two real ORT pins (environments.go:DeepDocORTVersion and Dockerfile:ARG ORT_VERSION)")
|
||||
}
|
||||
}
|
||||
|
||||
// TestNativeLibVersionPinnedNotGlob is the adversarial behavioural guard for the
|
||||
// config-pinning requirement: with two versions present in the cache, the
|
||||
// resolver must return the version pinned in go.mod (pinned != latest), whereas
|
||||
// the naive `ls …/v*… | head -1` would silently pick the lexicographically
|
||||
// oldest. This is exactly the failure mode the doc snippet used to have.
|
||||
func TestNativeLibVersionPinnedNotGlob(t *testing.T) {
|
||||
root := repoRoot(t)
|
||||
versionSh := filepath.Join(root, "tools", "scripts", "native_lib_version.sh")
|
||||
if _, err := os.Stat(versionSh); err != nil {
|
||||
t.Fatalf("native_lib_version.sh missing: %v", err)
|
||||
}
|
||||
|
||||
// Build a fake repo root with go.mod pinning office_oxide v0.1.16 and a
|
||||
// cache that also carries an OLDER v0.1.12 — the adversarial case where a
|
||||
// glob+head -1 would wrongly select the stale version.
|
||||
tmp := t.TempDir()
|
||||
goMod := "module example.com/ragflow\n\nrequire github.com/yfedoseev/office_oxide/go v0.1.16\n"
|
||||
if err := os.WriteFile(filepath.Join(tmp, "go.mod"), []byte(goMod), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cache := filepath.Join(tmp, "ragflow-native-libs", "office_oxide")
|
||||
for _, v := range []string{"v0.1.12", "v0.1.16"} {
|
||||
dir := filepath.Join(cache, v, "lib")
|
||||
if err := os.MkdirAll(dir, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(dir, "liboffice_oxide.a"), []byte("fake"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// Resolve the pinned version through the shared resolver.
|
||||
out, err := exec.Command("bash", "-c",
|
||||
sourceCmd(versionSh)+" && go_module_version \"github.com/yfedoseev/office_oxide/go\" "+tmp,
|
||||
).Output()
|
||||
if err != nil {
|
||||
t.Fatalf("go_module_version failed: %v (%s)", err, out)
|
||||
}
|
||||
got := strings.TrimSpace(string(out))
|
||||
if got != "0.1.16" {
|
||||
t.Fatalf("go_module_version returned %q, want pinned 0.1.16", got)
|
||||
}
|
||||
|
||||
// Demonstrate the bug the fix prevents: the naive glob+head -1 picks the
|
||||
// lexicographically-oldest version, not the pinned one.
|
||||
naive, err := exec.Command("bash", "-c",
|
||||
"ls "+filepath.Join(cache, "v*", "lib", "liboffice_oxide.a")+" | head -1",
|
||||
).Output()
|
||||
if err != nil {
|
||||
t.Fatalf("naive glob failed: %v (%s)", err, naive)
|
||||
}
|
||||
if !strings.Contains(string(naive), "v0.1.12") {
|
||||
t.Fatalf("naive glob+head -1 did not select the oldest version; got %q", naive)
|
||||
}
|
||||
}
|
||||
|
||||
// TestDocumentedCgoCflagsResolvesPinnedInclude is the end-to-end adversarial
|
||||
// guard for the blocking P1 finding: it extracts the ACTUAL `export CGO_CFLAGS=`
|
||||
// line from development.md, executes it against a fake cache holding two
|
||||
// versions, and asserts the resulting include path (a) resolves to a real header
|
||||
// under the pinned v0.1.16, not the oldest v0.1.12, and (b) is not a literal
|
||||
// un-expanded `v*` glob. This catches both the "glob never expands in quotes"
|
||||
// bug and the "include must follow the pinned version" requirement.
|
||||
func TestDocumentedCgoCflagsResolvesPinnedInclude(t *testing.T) {
|
||||
root := repoRoot(t)
|
||||
doc := readFile(t, filepath.Join(root, "internal", "development.md"))
|
||||
|
||||
// Extract the documented CGO_CFLAGS export line from the fenced bash block,
|
||||
// stripping the markdown blockquote prefix ("> "). Executing the real line
|
||||
// (rather than a copy) means a future edit that reintroduces a bug fails here.
|
||||
var cflagsLine string
|
||||
for _, line := range strings.Split(doc, "\n") {
|
||||
trimmed := strings.TrimSpace(strings.TrimPrefix(strings.TrimSpace(line), ">"))
|
||||
if strings.HasPrefix(trimmed, "export CGO_CFLAGS=") {
|
||||
cflagsLine = trimmed
|
||||
break
|
||||
}
|
||||
}
|
||||
if cflagsLine == "" {
|
||||
t.Fatal("development.md has no `export CGO_CFLAGS=` line to verify")
|
||||
}
|
||||
|
||||
// Fake cache: two versions, each with a header. The pinned version
|
||||
// (v0.1.16) is NOT the lexicographically-oldest (v0.1.12), so a glob+head -1
|
||||
// or a literal `v*` would both be wrong.
|
||||
tmp := t.TempDir()
|
||||
cache := filepath.Join(tmp, "office_oxide")
|
||||
for _, v := range []string{"v0.1.12", "v0.1.16"} {
|
||||
inc := filepath.Join(cache, v, "include", "office_oxide_c")
|
||||
lib := filepath.Join(cache, v, "lib")
|
||||
if err := os.MkdirAll(inc, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.MkdirAll(lib, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(inc, "office_oxide.h"), []byte("header"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(lib, "liboffice_oxide.a"), []byte("archive"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
pinnedArchive := filepath.Join(cache, "v0.1.16", "lib", "liboffice_oxide.a")
|
||||
|
||||
// Execute the documented line with OFFICE_OXIDE_A bound to the pinned
|
||||
// archive, then emit the resulting CGO_CFLAGS value.
|
||||
cmd := exec.Command("bash", "-c", cflagsLine+` ; printf '%s' "$CGO_CFLAGS"`)
|
||||
cmd.Env = append(os.Environ(), "OFFICE_OXIDE_A="+pinnedArchive)
|
||||
out, err := cmd.Output()
|
||||
if err != nil {
|
||||
t.Fatalf("documented CGO_CFLAGS line failed to evaluate: %v (%s)", err, out)
|
||||
}
|
||||
cflags := strings.TrimSpace(string(out))
|
||||
if !strings.HasPrefix(cflags, "-I") {
|
||||
t.Fatalf("CGO_CFLAGS is not an -I include flag: %q", cflags)
|
||||
}
|
||||
includeDir := strings.TrimPrefix(cflags, "-I")
|
||||
|
||||
// (a) It must be derived from the PINNED archive, not the oldest version.
|
||||
if !strings.Contains(includeDir, "v0.1.16") || strings.Contains(includeDir, "v0.1.12") {
|
||||
t.Errorf("include %q not derived from pinned v0.1.16 archive", includeDir)
|
||||
}
|
||||
// (a) It must resolve to a real header — the whole point of the fix.
|
||||
if _, err := os.Stat(filepath.Join(includeDir, "office_oxide.h")); err != nil {
|
||||
t.Errorf("derived include %q has no office_oxide.h: %v", includeDir, err)
|
||||
}
|
||||
// (b) It must NOT be an un-expanded literal `v*` glob (the original bug:
|
||||
// the shell does not expand globs inside double-quoted CGO_CFLAGS).
|
||||
if strings.Contains(includeDir, "v*/") || strings.Contains(includeDir, "v*") {
|
||||
t.Errorf("include %q still contains an un-expanded v* glob", includeDir)
|
||||
}
|
||||
|
||||
// Adversarial proof of the original defect: a literal `v*` inside a quoted
|
||||
// path never matches a real file, so the old line could not find the header.
|
||||
literalGlob := filepath.Join(cache, "v*", "include", "office_oxide_c", "office_oxide.h")
|
||||
if _, err := os.Stat(literalGlob); err == nil {
|
||||
t.Errorf("a literal v* path unexpectedly resolved to a header; the un-expanded-glob guard is vacuous")
|
||||
}
|
||||
}
|
||||
|
||||
func sourceCmd(path string) string {
|
||||
return "set -e; source " + path
|
||||
}
|
||||
@@ -57,7 +57,9 @@ import hashlib
|
||||
import os
|
||||
import platform
|
||||
import re
|
||||
import shlex
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import tarfile
|
||||
import tempfile
|
||||
@@ -66,11 +68,17 @@ import zipfile
|
||||
import requests
|
||||
|
||||
# ── Dependency versions & URLs (edit here) ────────────────────────────────
|
||||
# This script's own copy of the native static-library versions and download URLs
|
||||
# it fetches. build.sh does NOT grep these (it pins PDFIUM/PDF_OXIDE itself and
|
||||
# reads ORT from internal/common/environments.go + the Dockerfile ARG, so it
|
||||
# stays tied to the checked-out source tree and the Docker build needs no COPY
|
||||
# of this script) — keep the two in sync when bumping a version.
|
||||
# The native-library VERSIONS are NOT hardcoded in this script. They are derived
|
||||
# from a single source of truth so build.sh and this downloader can never drift:
|
||||
# - office_oxide / pdf_oxide -> go.mod (go_module_version, below)
|
||||
# - ORT -> internal/common/environments.go DeepDocORTVersion
|
||||
# (deepdoc_ort_version, below)
|
||||
# - pdfium -> tools/scripts/native_lib_version.sh
|
||||
# pdfium_static_version (a shared const; pdfium
|
||||
# is not a Go module so go.mod does not apply)
|
||||
# ORT_VERSION / PDFIUM_VERSION are bound to those derivations right after
|
||||
# REPO_ROOT is resolved (see the block following _repo_root()), so bumping a
|
||||
# version touches exactly one place upstream.
|
||||
|
||||
# ONNX Runtime (statically linked into the Go in-process DeepDoc backend).
|
||||
#
|
||||
@@ -83,7 +91,6 @@ import requests
|
||||
# patched content (e.g. the build that exports SessionGetInitializer*); the
|
||||
# downloader detects that via a `{asset}.sha256` sidecar and
|
||||
# re-downloads/re-extracts, so a stale local copy never silently lingers.
|
||||
ORT_VERSION = "1.29.0"
|
||||
|
||||
# Release tag on infiniflow/ragflow-build. v1.29.0's Linux x86_64 minimal build
|
||||
# reduced operators to required_operators.config, which omitted Clip@13 — so
|
||||
@@ -95,15 +102,15 @@ ORT_VERSION = "1.29.0"
|
||||
# build ships Clip@13).
|
||||
ORT_RELEASE_TAG = "onnxruntime-v1.29.0-patch2"
|
||||
|
||||
# pdf_oxide go-ffi release. Bare module version (the "v" is added by
|
||||
# pdf_oxide_url); matches the value build.sh uses to validate the embedded
|
||||
# .a stamp.
|
||||
PDF_OXIDE_VERSION = "0.3.73"
|
||||
# pdf_oxide go-ffi release. The bare module version is derived from go.mod at
|
||||
# download/extract time (see pdf_oxide_url / the extraction block below) via
|
||||
# go_module_version(), so there is no second hardcoded copy to drift from what
|
||||
# the Go build actually links (mirrors office_oxide).
|
||||
|
||||
# pdfium release, as the numeric build number after "chromium%2F" (the "v" and
|
||||
# the URL-encoded slash are added by pdfium_url). Matches build.sh's
|
||||
# PDFIUM_STATIC_VERSION.
|
||||
PDFIUM_VERSION = "7809"
|
||||
# the URL-encoded slash are added by pdfium_url). Its bare value is bound to
|
||||
# pdfium_static_version(REPO_ROOT) below (the single shared const, since pdfium
|
||||
# is not a Go module).
|
||||
|
||||
|
||||
def host_platform(target_os=None, target_arch=None):
|
||||
@@ -185,6 +192,48 @@ def go_module_version(module: str, repo_root: str) -> str:
|
||||
return match.group(1)
|
||||
|
||||
|
||||
def deepdoc_ort_version(repo_root: str) -> str:
|
||||
"""Read the ONNX Runtime version the in-process (Go) DeepDoc backend links
|
||||
from internal/common/environments.go's DeepDocORTVersion const.
|
||||
|
||||
ORT is the one native lib whose version is owned by Go source (not go.mod),
|
||||
so both build.sh and this script parse the SAME const — no second hardcoded
|
||||
copy to drift."""
|
||||
path = os.path.join(repo_root, "internal", "common", "environments.go")
|
||||
if not os.path.isfile(path):
|
||||
raise RuntimeError(f"environments.go not found at {path}")
|
||||
with open(path, encoding="utf-8") as file:
|
||||
match = re.search(r'DeepDocORTVersion\s*=\s*"([^"]+)"', file.read())
|
||||
if match is None:
|
||||
raise RuntimeError(f"DeepDocORTVersion const not found in {path}")
|
||||
return match.group(1)
|
||||
|
||||
|
||||
def pdfium_static_version(repo_root: str) -> str:
|
||||
"""Return the pinned pdfium build number from the single shared bash source
|
||||
of truth tools/scripts/native_lib_version.sh.
|
||||
|
||||
pdfium (kognitos/pdfium-static) is not a Go module, so unlike
|
||||
office_oxide / pdf_oxide it cannot be derived from go.mod; its bare numeric
|
||||
build number is the one constant exported by pdfium_static_version() and
|
||||
sourced by both this script and build.sh."""
|
||||
script = os.path.join(repo_root, "tools", "scripts", "native_lib_version.sh")
|
||||
if not os.path.isfile(script):
|
||||
raise RuntimeError(f"native_lib_version.sh not found at {script}")
|
||||
result = subprocess.run(
|
||||
["bash", "-c", f"source {shlex.quote(script)} >/dev/null 2>&1; pdfium_static_version"],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False,
|
||||
)
|
||||
if result.returncode != 0:
|
||||
raise RuntimeError(f"pdfium_static_version failed: {result.stderr.strip()}")
|
||||
version = result.stdout.strip()
|
||||
if not version:
|
||||
raise RuntimeError("pdfium_static_version produced no output")
|
||||
return version
|
||||
|
||||
|
||||
# ── URL builders ──────────────────────────────────────────────────────────
|
||||
# Every download URL is assembled from the constants above via one of these
|
||||
# helpers, so a version bump touches exactly one constant. Each takes the target
|
||||
@@ -192,9 +241,12 @@ def go_module_version(module: str, repo_root: str) -> str:
|
||||
# maps above.
|
||||
|
||||
|
||||
def pdf_oxide_url(goos: str, goarch: str) -> str:
|
||||
"""pdf_oxide go-ffi tarball for the target platform."""
|
||||
return f"{gh_proxy()}https://github.com/yfedoseev/pdf_oxide/releases/download/v{PDF_OXIDE_VERSION}/{PDF_OXIDE_ASSETS[(goos, goarch)]}"
|
||||
def pdf_oxide_url(repo_root: str, goos: str, goarch: str) -> str:
|
||||
"""pdf_oxide go-ffi tarball for the target platform. The version is derived
|
||||
from go.mod so it always matches what the Go build actually links (no second
|
||||
hardcoded copy to drift -- mirrors office_oxide_url)."""
|
||||
version = go_module_version("github.com/yfedoseev/pdf_oxide/go", repo_root)
|
||||
return f"{gh_proxy()}https://github.com/yfedoseev/pdf_oxide/releases/download/v{version}/{PDF_OXIDE_ASSETS[(goos, goarch)]}"
|
||||
|
||||
|
||||
def pdfium_url(goos: str, goarch: str) -> str:
|
||||
@@ -269,6 +321,15 @@ def _repo_root() -> str:
|
||||
|
||||
|
||||
REPO_ROOT = _repo_root()
|
||||
|
||||
# Bind the native-library versions to their single source of truth (see the
|
||||
# version block above). Keeping these as module-level constants means every
|
||||
# downstream URL/dir builder still references ORT_VERSION / PDFIUM_VERSION, but
|
||||
# the bare values now live in exactly one place upstream (environments.go and
|
||||
# native_lib_version.sh) — bumping a version needs no edit here.
|
||||
ORT_VERSION = deepdoc_ort_version(REPO_ROOT)
|
||||
PDFIUM_VERSION = pdfium_static_version(REPO_ROOT)
|
||||
|
||||
# Downloaded archives are anchored under deps/ (ignored by .gitignore
|
||||
# via *.tgz / deps/*.zip) regardless of the caller's working directory,
|
||||
# so a move of this script does not scatter large files into tools/scripts/.
|
||||
@@ -276,6 +337,28 @@ RAGFLOW_DEPS_DIR = os.path.join(REPO_ROOT, "deps")
|
||||
NATIVE_LIBS_DIR = os.path.expanduser("~/ragflow-native-libs")
|
||||
|
||||
|
||||
def native_libs_dir():
|
||||
"""Resolve where native static libraries are written/read.
|
||||
|
||||
Honours RAGFLOW_NATIVE_LIBS_DIR so a CI builder can mount a pre-populated
|
||||
user cache (e.g. ~/ragflow-native-libs) at a configurable path and the
|
||||
downloader agrees with build.sh on that location. Falls back to the
|
||||
in-home default when the variable is unset or empty.
|
||||
"""
|
||||
return os.environ.get("RAGFLOW_NATIVE_LIBS_DIR") or NATIVE_LIBS_DIR
|
||||
|
||||
|
||||
def ensure_deps_dir():
|
||||
"""Make sure RAGFLOW_DEPS_DIR exists before any chdir into it.
|
||||
|
||||
In a product image build the go-builder stage does not carry a deps/
|
||||
checkout, so a bare os.chdir(RAGFLOW_DEPS_DIR) would raise
|
||||
FileNotFoundError before the auto-download fallback could run. Creating it
|
||||
here lets the downloader proceed and fetch archives on demand.
|
||||
"""
|
||||
os.makedirs(RAGFLOW_DEPS_DIR, exist_ok=True)
|
||||
|
||||
|
||||
def _sha256_of(path):
|
||||
"""sha256 of a file, streamed in chunks so large archives don't blow memory."""
|
||||
h = hashlib.sha256()
|
||||
@@ -342,10 +425,30 @@ def _valid_download(filename):
|
||||
return True
|
||||
|
||||
|
||||
def _extract_native_archive(archive_path, target, required_files, version=None):
|
||||
def _stamp_present(data, version, stamp_format):
|
||||
"""True when the .a archive DATA carries VERSION as a known stamp form.
|
||||
|
||||
Different native libs embed their version stamp differently:
|
||||
- office_oxide embeds a null-delimited token (b"\\x00"+version+b"\\x00");
|
||||
- pdf_oxide embeds a plain-text token (b"pdf_oxide "+version).
|
||||
Accepting both keeps the extractor library-agnostic while still rejecting a
|
||||
.a that does not actually contain the expected version."""
|
||||
needle = version.encode()
|
||||
if stamp_format == "null":
|
||||
return b"\x00" + needle + b"\x00" in data
|
||||
if stamp_format == "text":
|
||||
return (b"pdf_oxide " + needle in data) or (b"\x00" + needle + b"\x00" in data)
|
||||
return needle in data
|
||||
|
||||
|
||||
def _extract_native_archive(archive_path, target, required_files, version=None, stamp_format="null"):
|
||||
"""Extract ARCHIVE_PATH into TARGET, verifying REQUIRED_FILES (and, for .a
|
||||
files, an optional null-delimited VERSION stamp) before promoting the
|
||||
staged tree.
|
||||
files, an optional VERSION stamp) before promoting the staged tree.
|
||||
|
||||
STAMP_FORMAT selects how the version is embedded in the .a:
|
||||
- "null": office_oxide's b"\\x00"+version+b"\\x00" token;
|
||||
- "text": pdf_oxide's plain b"pdf_oxide "+version token (also accepts a
|
||||
null-delimited token, so a format switch does not silently break).
|
||||
|
||||
Path-traversal / absolute / non-regular members are rejected before
|
||||
extraction, and an incomplete archive raises instead of replacing a
|
||||
@@ -358,8 +461,9 @@ def _extract_native_archive(archive_path, target, required_files, version=None):
|
||||
return False
|
||||
if version is not None and name.endswith(".a"):
|
||||
with open(path, "rb") as file:
|
||||
if b"\x00" + version.encode() + b"\x00" not in file.read():
|
||||
return False
|
||||
data = file.read()
|
||||
if not _stamp_present(data, version, stamp_format):
|
||||
return False
|
||||
return True
|
||||
|
||||
if complete(target):
|
||||
@@ -388,6 +492,37 @@ def _extract_native_archive(archive_path, target, required_files, version=None):
|
||||
print(f" ✓ extracted {os.path.basename(archive_path)} to {target}")
|
||||
|
||||
|
||||
def extract_native_lib(archive_path, subdir, dir_version, required_files, stamp_version=None, stamp_format="null"):
|
||||
"""Extract a native static archive into the version-keyed cache dir
|
||||
~/ragflow-native-libs/<subdir>/<dir_version>/.
|
||||
|
||||
The version-keyed dir (e.g. v7809) — NOT a flat layout — is what lets a
|
||||
single mounted user cache hold several native-lib versions at once, so PRs
|
||||
pinned to different go.mod versions all build offline. Idempotent: when the
|
||||
target dir already holds the required files (matching stamp_version for
|
||||
office_oxide) the extraction is skipped, so re-running the downloader is a
|
||||
no-op for an already-provisioned cache.
|
||||
|
||||
STAMP_FORMAT is forwarded to _extract_native_archive (see its docstring):
|
||||
"null" for office_oxide, "text" for pdf_oxide. pdfium passes stamp_version
|
||||
None and is not stamp-checked.
|
||||
|
||||
Returns True when extraction ran or was skipped-successfully (the dir is in
|
||||
place afterwards), False when the archive is absent (caller must then
|
||||
download it). Re-raises the underlying tar/validation error after purging a
|
||||
corrupt archive."""
|
||||
if not os.path.isfile(archive_path):
|
||||
print(f" Skipping extraction: {os.path.basename(archive_path)} not found")
|
||||
return False
|
||||
target = os.path.join(native_libs_dir(), subdir, dir_version)
|
||||
try:
|
||||
_extract_native_archive(archive_path, target, required_files, stamp_version, stamp_format)
|
||||
except (tarfile.TarError, EOFError, RuntimeError):
|
||||
os.unlink(archive_path)
|
||||
raise
|
||||
return True
|
||||
|
||||
|
||||
def prune_stale_onnxruntime(static_lib_dir, expected_dir):
|
||||
"""Remove ONNX Runtime version dirs under static_lib that do NOT match
|
||||
`expected_dir`. Without this, a version bump (or a foreign-arch run on a
|
||||
@@ -428,6 +563,7 @@ def extract_onnxruntime(static_lib_dir, archive_path, expected_dir):
|
||||
expected_path = os.path.join(static_lib_dir, expected_dir)
|
||||
if os.path.isdir(expected_path) and has_static_archives(expected_path):
|
||||
print(f" ✓ onnxruntime/static_lib ({expected_dir}) already extracted to {expected_path}")
|
||||
stamp_onnxruntime_version(static_lib_dir, expected_dir, ORT_VERSION)
|
||||
return True
|
||||
os.makedirs(static_lib_dir, exist_ok=True)
|
||||
print(f" Extracting {os.path.basename(archive_path)} → {static_lib_dir}")
|
||||
@@ -445,9 +581,37 @@ def extract_onnxruntime(static_lib_dir, archive_path, expected_dir):
|
||||
shutil.rmtree(normalized)
|
||||
print(f" Renaming {os.path.basename(extracted)} → {os.path.basename(normalized)}")
|
||||
os.rename(extracted, normalized)
|
||||
# Pre-create the version-stamp symlink (Fix #2) while the cache is still
|
||||
# writable, so a read-only CI mount already has it and build.sh's stamp step
|
||||
# is a no-op there.
|
||||
stamp_onnxruntime_version(static_lib_dir, expected_dir, ORT_VERSION)
|
||||
return True
|
||||
|
||||
|
||||
def stamp_onnxruntime_version(static_lib_dir, normalized_dir, version):
|
||||
"""Pre-create the version-stamp ORT symlink inside the native cache.
|
||||
|
||||
build.sh keys its ORT link flag on onnxruntime/static_lib/v<version>-<sha16>/
|
||||
(the sha16 disambiguates a re-issued archive of the same version). That dir
|
||||
is created HERE, at population time when the cache is still writable, with a
|
||||
RELATIVE symlink to the real libonnxruntime.a so it resolves under any mount
|
||||
point. Idempotent: an existing correct symlink is left untouched, which keeps
|
||||
a read-only CI mount a no-op (build.sh's stamp step then skips the write)."""
|
||||
primary = os.path.join(static_lib_dir, normalized_dir, "lib", "libonnxruntime.a")
|
||||
if not os.path.isfile(primary):
|
||||
return
|
||||
sha16 = _sha256_of(primary)[:16]
|
||||
stamp_dir = os.path.join(static_lib_dir, f"v{version}-{sha16}")
|
||||
stamp_link = os.path.join(stamp_dir, "libonnxruntime.a")
|
||||
rel = os.path.relpath(primary, stamp_dir)
|
||||
if os.path.islink(stamp_link) and os.path.realpath(stamp_link) == os.path.realpath(primary):
|
||||
return
|
||||
os.makedirs(stamp_dir, exist_ok=True)
|
||||
if os.path.lexists(stamp_link):
|
||||
os.unlink(stamp_link)
|
||||
os.symlink(rel, stamp_link)
|
||||
|
||||
|
||||
def _fetch_ort_sidecar_sha(url):
|
||||
"""Fetch the published `<asset>.sha256` sidecar for the ORT archive.
|
||||
|
||||
@@ -495,7 +659,7 @@ def _download_ort_archive(archive_path, goos, goarch):
|
||||
if not _valid_download(archive_path):
|
||||
os.unlink(archive_path)
|
||||
raise RuntimeError(f"Invalid downloaded dependency: {os.path.basename(archive_path)}")
|
||||
version_dir = os.path.join(NATIVE_LIBS_DIR, "onnxruntime", "static_lib", ort_normalized_dir(ORT_VERSION, goos, goarch))
|
||||
version_dir = os.path.join(native_libs_dir(), "onnxruntime", "static_lib", ort_normalized_dir(ORT_VERSION, goos, goarch))
|
||||
if os.path.isdir(version_dir):
|
||||
print(f" Removing stale extracted ONNX Runtime dir: {version_dir}")
|
||||
shutil.rmtree(version_dir)
|
||||
@@ -701,13 +865,14 @@ def main():
|
||||
|
||||
# Anchor the downloaded archives to deps/ regardless of the
|
||||
# caller's working directory (build.sh invokes this from the repo root).
|
||||
ensure_deps_dir()
|
||||
os.chdir(RAGFLOW_DEPS_DIR)
|
||||
|
||||
# ONNX Runtime archive (+ checksum sidecar), then the other native archives.
|
||||
_download_ort_archive(os.path.join(RAGFLOW_DEPS_DIR, ort_zip), goos, goarch)
|
||||
archives = [
|
||||
(pdfium_url(goos, goarch), PDFIUM_STATIC_ASSETS[(goos, goarch)]),
|
||||
(pdf_oxide_url(goos, goarch), PDF_OXIDE_ASSETS[(goos, goarch)]),
|
||||
(pdf_oxide_url(REPO_ROOT, goos, goarch), PDF_OXIDE_ASSETS[(goos, goarch)]),
|
||||
(office_oxide_url(REPO_ROOT, goos, goarch), OFFICE_OXIDE_ASSETS[(goos, goarch)]),
|
||||
]
|
||||
for url, filename in archives:
|
||||
@@ -721,28 +886,23 @@ def main():
|
||||
raise RuntimeError(f"Invalid downloaded dependency: {filename}")
|
||||
|
||||
# Extract native static libraries to ~/ragflow-native-libs for the Go build.
|
||||
# Ensures build.sh can find them without network access.
|
||||
# Ensures build.sh can find them without network access. Each lands under a
|
||||
# version-keyed subdir (v<version>/) so a mounted cache can hold several
|
||||
# versions at once; see extract_native_lib().
|
||||
office_version = go_module_version("github.com/yfedoseev/office_oxide/go", REPO_ROOT)
|
||||
pdf_oxide_version = go_module_version("github.com/yfedoseev/pdf_oxide/go", REPO_ROOT)
|
||||
extractions = [
|
||||
(PDFIUM_STATIC_ASSETS[(goos, goarch)], "pdfium-static", ["lib/libpdfium.a", "lib/libc++.a", "lib/libc++abi.a", "include/fpdfview.h"], None),
|
||||
(PDF_OXIDE_ASSETS[(goos, goarch)], "pdf_oxide", [f"lib/{goos}_{goarch}/libpdf_oxide.a", "include/pdf_oxide.h"], None),
|
||||
(OFFICE_OXIDE_ASSETS[(goos, goarch)], "office_oxide", ["lib/liboffice_oxide.a", "include/office_oxide_c/office_oxide.h"], office_version),
|
||||
("pdfium-static", PDFIUM_STATIC_ASSETS[(goos, goarch)], "v" + PDFIUM_VERSION, ["lib/libpdfium.a", "lib/libc++.a", "lib/libc++abi.a", "include/fpdfview.h"], None, "null"),
|
||||
("pdf_oxide", PDF_OXIDE_ASSETS[(goos, goarch)], "v" + pdf_oxide_version, [f"lib/{goos}_{goarch}/libpdf_oxide.a", "include/pdf_oxide.h"], pdf_oxide_version, "text"),
|
||||
("office_oxide", OFFICE_OXIDE_ASSETS[(goos, goarch)], "v" + office_version, ["lib/liboffice_oxide.a", "include/office_oxide_c/office_oxide.h"], office_version, "null"),
|
||||
]
|
||||
|
||||
for archive, subdir, required_files, version in extractions:
|
||||
for subdir, archive, dir_version, required_files, stamp_version, stamp_format in extractions:
|
||||
archive_path = os.path.join(RAGFLOW_DEPS_DIR, archive)
|
||||
if not os.path.isfile(archive_path):
|
||||
print(f" Skipping extraction: {archive} not found")
|
||||
continue
|
||||
target = os.path.join(NATIVE_LIBS_DIR, subdir)
|
||||
try:
|
||||
_extract_native_archive(archive_path, target, required_files, version)
|
||||
except (tarfile.TarError, EOFError, RuntimeError):
|
||||
os.unlink(archive_path)
|
||||
raise
|
||||
extract_native_lib(archive_path, subdir, dir_version, required_files, stamp_version, stamp_format)
|
||||
|
||||
if not extract_onnxruntime(
|
||||
os.path.join(NATIVE_LIBS_DIR, "onnxruntime", "static_lib"),
|
||||
os.path.join(native_libs_dir(), "onnxruntime", "static_lib"),
|
||||
os.path.join(RAGFLOW_DEPS_DIR, ort_zip),
|
||||
ort_dir,
|
||||
):
|
||||
@@ -752,7 +912,7 @@ def main():
|
||||
# .a left here is exactly the "silent green" this split must prevent.
|
||||
print(
|
||||
f" ERROR: ONNX Runtime static archives for {ort_dir} were not "
|
||||
f"extracted to {os.path.join(NATIVE_LIBS_DIR, 'onnxruntime', 'static_lib')}. "
|
||||
f"extracted to {os.path.join(native_libs_dir(), 'onnxruntime', 'static_lib')}. "
|
||||
f"Check the download above; build.sh will refuse to link without them.",
|
||||
file=sys.stderr,
|
||||
)
|
||||
@@ -763,7 +923,7 @@ def main():
|
||||
# find the archives — the .a files live under
|
||||
# ~/ragflow-native-libs/onnxruntime/static_lib. The in-process backend
|
||||
# resolves OrtGetApiBase via dlopen(NULL); there is no dynamic .so fallback.
|
||||
ort_static_dir = os.path.join(NATIVE_LIBS_DIR, "onnxruntime", "static_lib")
|
||||
ort_static_dir = os.path.join(native_libs_dir(), "onnxruntime", "static_lib")
|
||||
ort_a_files = [os.path.join(root, f) for root, _, files in os.walk(ort_static_dir) for f in files if f.endswith(".a")]
|
||||
if ort_a_files:
|
||||
print(f" ✓ onnxruntime static archives ready: {len(ort_a_files)} .a under {ort_static_dir}")
|
||||
@@ -784,7 +944,7 @@ def main():
|
||||
download_tokenizer_assets(args.china_mirrors)
|
||||
|
||||
print()
|
||||
print(f"✓ Native build libraries ready under {NATIVE_LIBS_DIR}/")
|
||||
print(f"✓ Native build libraries ready under {native_libs_dir()}/")
|
||||
print(" Run: bash build.sh (it will find them without network access).")
|
||||
print(" Runtime resources (cl100k / DeepDoc .ort / tokenizer) are also in place,")
|
||||
print(" so the local Go server is ready to build, run, parse and count tokens.")
|
||||
|
||||
@@ -0,0 +1,82 @@
|
||||
#!/usr/bin/env bash
|
||||
# Single source of truth for the native static-library versions build.sh and
|
||||
# tools/scripts/download_dev_deps.py agree on. Source this file from build.sh;
|
||||
# it defines only functions (no top-level side effects) so it is safe to source.
|
||||
#
|
||||
# Each native lib resolves its version from exactly one place:
|
||||
# - office_oxide / pdf_oxide -> go.mod (go_module_version)
|
||||
# - ORT -> internal/common/environments.go
|
||||
# DeepDocORTVersion (read by both build.sh via
|
||||
# deepdoc_ort_version() below and
|
||||
# download_dev_deps.deepdoc_ort_version() — it is
|
||||
# Go-owned, so the const itself stays in Go)
|
||||
# - pdfium -> pdfium_static_version (below; pdfium is not a
|
||||
# Go module, so go.mod does not apply)
|
||||
#
|
||||
# There is NO /opt pre-bake and NO flat (version-less) cache layout any more: the
|
||||
# libs land under ~/ragflow-native-libs/<lib>/v<version>/ (see download_dev_deps.py
|
||||
# and build.sh), so a single mounted cache carries several versions at once.
|
||||
|
||||
# ragflow_native_libs_dir
|
||||
# Root of the version-keyed native static-lib user cache
|
||||
# (~/ragflow-native-libs by default). Overridable via RAGFLOW_NATIVE_LIBS_DIR so
|
||||
# CI can mount the cache at a fixed host path (e.g. a RUNNER_WORKSPACE_PREFIX
|
||||
# bind mount) instead of the build user's $HOME — see sep-tests.yml Slice 3. An
|
||||
# EMPTY override falls back to the $HOME default (the ${VAR:-default} expansion),
|
||||
# so a half-configured runner never resolves to an empty cache root. Echoes the
|
||||
# resolved path and returns 0.
|
||||
ragflow_native_libs_dir() {
|
||||
printf '%s\n' "${RAGFLOW_NATIVE_LIBS_DIR:-${HOME}/ragflow-native-libs}"
|
||||
}
|
||||
|
||||
# go_module_version <module> <repo_root>
|
||||
# Reads the pinned version (the "vX.Y.Z" after the require line) of a Go module
|
||||
# from <repo_root>/go.mod so build.sh and tools/scripts/download_dev_deps.py agree
|
||||
# on the version (single source of truth). The module may appear either as a bare
|
||||
# require line (`\t<module> vX.Y.Z`) inside a require block or on a single-line
|
||||
# `require <module> vX.Y.Z`; an optional leading `require` keyword is allowed.
|
||||
# Echoes the bare version (no "v" prefix) and returns 0, or returns 1 with empty
|
||||
# output when go.mod is missing or the module is not required.
|
||||
go_module_version() {
|
||||
local module="$1" repo_root="$2"
|
||||
local go_mod="${repo_root}/go.mod"
|
||||
[ -f "$go_mod" ] || return 1
|
||||
local version
|
||||
version="$(grep -m1 -E "^[[:space:]]*(require[[:space:]]+)?${module}[[:space:]]+v" "$go_mod" \
|
||||
| sed -E 's/.*v([0-9][^[:space:]]*).*/\1/')"
|
||||
[ -n "$version" ] || return 1
|
||||
printf '%s\n' "$version"
|
||||
}
|
||||
|
||||
# pdfium_static_version
|
||||
# pdfium (kognitos/pdfium-static) is NOT a Go module, so unlike office_oxide /
|
||||
# pdf_oxide its version cannot be derived from go.mod. It is pinned here as the
|
||||
# single shared constant; download_dev_deps.py and build.sh both source this
|
||||
# function so the bare numeric build number lives in exactly one place. Echoes
|
||||
# the constant and returns 0.
|
||||
pdfium_static_version() {
|
||||
# Single source of truth for the pdfium static build number. Bump here only.
|
||||
printf '%s\n' "7809"
|
||||
}
|
||||
|
||||
# deepdoc_ort_version <repo_root>
|
||||
# Reads the ONNX Runtime version the in-process (Go) DeepDoc backend links from
|
||||
# <repo_root>/internal/common/environments.go's DeepDocORTVersion const.
|
||||
#
|
||||
# ORT is the one native lib whose version is owned by Go source (not go.mod), so
|
||||
# build.sh and download_dev_deps.deepdoc_ort_version() both parse the SAME const
|
||||
# through this helper — no second hardcoded copy to drift. Echoes the bare
|
||||
# version and returns 0, or returns 1 with empty output when environments.go is
|
||||
# missing or the const is absent (so callers fail loud, mirroring the Python
|
||||
# RuntimeError). build.sh used to inline this grep at every call site; collapsing
|
||||
# it here keeps a single bash source of truth alongside the Python one.
|
||||
deepdoc_ort_version() {
|
||||
local repo_root="$1"
|
||||
local env_go="${repo_root}/internal/common/environments.go"
|
||||
[ -f "$env_go" ] || return 1
|
||||
local version
|
||||
version="$(grep -m1 -E 'DeepDocORTVersion[[:space:]]*=[[:space:]]*"' "$env_go" \
|
||||
| sed -E 's/.*"([^"]+)".*/\1/')"
|
||||
[ -n "$version" ] || return 1
|
||||
printf '%s\n' "$version"
|
||||
}
|
||||
@@ -0,0 +1,34 @@
|
||||
#!/usr/bin/env bash
|
||||
# Pure helper for build.sh: decide whether the required ONNX Runtime static lib
|
||||
# is already present in the version+platform-keyed user cache.
|
||||
#
|
||||
# $1 = ONNXRUNTIME_STATIC_PREFIX root holding the extracted ORT dirs
|
||||
# $2 = ort_dir_prefix platform top-level dir prefix
|
||||
# (see build.sh ort_static_lib_prefix)
|
||||
# $3 = ort_version DeepDocORTVersion (internal/common/environments.go)
|
||||
#
|
||||
# Exits 0 (present) when at least one real base libonnxruntime*.a lives under a
|
||||
# dir matching BOTH the target platform and the required version —
|
||||
# */${ort_dir_prefix}${ort_version}*/lib/*.a — and 1 (missing) otherwise. This is
|
||||
# the SAME dir check_onnxruntime_deps links, so a shared cache holding only an
|
||||
# OLD version or a DIFFERENT platform's ORT is reported missing (and the
|
||||
# controlled auto-downloader runs) instead of being silently treated as present
|
||||
# and failing only at link time.
|
||||
#
|
||||
# The lib/ segment is required so a stray .a sitting in the prefix root (not
|
||||
# under a versioned .../lib/ dir) is ignored. Provider-specific archives
|
||||
# (*cuda*/*tensorrt*/*coreml*/*dml*/*migraphx*) are excluded, matching the
|
||||
# link-time selection in check_onnxruntime_deps.
|
||||
ragflow_ort_present() {
|
||||
local prefix="$1" ort_dir_prefix="$2" ort_version="$3"
|
||||
local f
|
||||
while IFS= read -r f; do
|
||||
case "$(basename "$f")" in
|
||||
*cuda*|*tensorrt*|*coreml*|*dml*|*migraphx*) continue ;;
|
||||
esac
|
||||
case "$f" in
|
||||
*/${ort_dir_prefix}${ort_version}*/lib/*.a) return 0 ;;
|
||||
esac
|
||||
done < <(find "$prefix" -type f -name '*.a' 2>/dev/null)
|
||||
return 1
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
#!/usr/bin/env bash
|
||||
# Pure helper for build.sh: decide how to version-stamp the ORT static archive
|
||||
# with a symlink. Echoes "<needs_write>|<relative_target>".
|
||||
#
|
||||
# $1 = versioned_dir the v<version>-<sha16> stamp dir (under the native cache)
|
||||
# $2 = archive_file absolute path to the real libonnxruntime.a
|
||||
#
|
||||
# The stamp target is RELATIVE (never absolute) so it resolves under ANY mount
|
||||
# point of the cache (build host, runner, CI builder container). `needs_write`
|
||||
# is "no" when an existing symlink already resolves to the same archive, so a
|
||||
# read-only cache mount is a no-op instead of a fatal `ln` on a read-only fs.
|
||||
ragflow_ort_stamp_target() {
|
||||
local versioned_dir="$1" archive_file="$2"
|
||||
local link_name link_target want
|
||||
link_name="$(basename "$archive_file")"
|
||||
link_target="${versioned_dir}/${link_name}"
|
||||
# -m: compute the relative path without requiring either path to exist yet.
|
||||
want="$(realpath -m --relative-to="$versioned_dir" "$archive_file")"
|
||||
local needs_write="yes"
|
||||
if [ -L "$link_target" ] && \
|
||||
[ "$(readlink -f "$link_target" 2>/dev/null)" = "$(readlink -f "$archive_file" 2>/dev/null)" ]; then
|
||||
needs_write="no"
|
||||
fi
|
||||
printf '%s|%s\n' "$needs_write" "$want"
|
||||
}
|
||||
@@ -22,6 +22,7 @@
|
||||
import importlib.util
|
||||
import io
|
||||
import os
|
||||
import subprocess
|
||||
import tarfile
|
||||
import threading
|
||||
import zipfile
|
||||
@@ -201,8 +202,8 @@ def test_url_builders_select_platform_assets(monkeypatch):
|
||||
assert native.pdfium_url("linux", "amd64").endswith("/pdfium-linux-x64-static.tgz")
|
||||
assert native.pdfium_url("linux", "arm64").endswith("/pdfium-linux-arm64-static.tgz")
|
||||
assert native.pdfium_url("darwin", "arm64").endswith("/pdfium-mac-arm64-static.tgz")
|
||||
assert native.pdf_oxide_url("linux", "amd64").endswith("/pdf_oxide-go-ffi-linux-amd64.tar.gz")
|
||||
assert native.pdf_oxide_url("darwin", "amd64").endswith("/pdf_oxide-go-ffi-darwin-amd64.tar.gz")
|
||||
assert native.pdf_oxide_url(native.REPO_ROOT, "linux", "amd64").endswith("/pdf_oxide-go-ffi-linux-amd64.tar.gz")
|
||||
assert native.pdf_oxide_url(native.REPO_ROOT, "darwin", "amd64").endswith("/pdf_oxide-go-ffi-darwin-amd64.tar.gz")
|
||||
assert native.ort_url("1.29.0", native.ort_target("darwin", "amd64")).endswith("/onnxruntime-v1.29.0-osx-universal.zip")
|
||||
|
||||
|
||||
@@ -212,10 +213,33 @@ def test_office_oxide_url_selects_platform_asset(tmp_path):
|
||||
assert native.office_oxide_url(str(tmp_path), "darwin", "arm64").endswith("/v0.1.12/native-macos-aarch64.tar.gz")
|
||||
|
||||
|
||||
def test_pdf_oxide_url_derives_from_go_mod(tmp_path):
|
||||
# pdf_oxide version MUST come from go.mod (the single source of truth
|
||||
# build.sh also reads), never a hardcoded constant. Otherwise build.sh and
|
||||
# the downloader silently drift apart. A fixture go.mod pins a non-default
|
||||
# version; the produced URL must reflect exactly that version.
|
||||
(tmp_path / "go.mod").write_text("module fixture\n\nrequire github.com/yfedoseev/pdf_oxide/go v0.4.1\n")
|
||||
url = native.pdf_oxide_url(str(tmp_path), "linux", "amd64")
|
||||
assert "/v0.4.1/pdf_oxide-go-ffi-linux-amd64.tar.gz" in url
|
||||
|
||||
|
||||
def test_pdf_oxide_url_reads_its_own_module_line(tmp_path):
|
||||
# Adversarial: a go.mod that pins office_oxide and pdf_oxide to DIFFERENT
|
||||
# versions. pdf_oxide_url must read the pdf_oxide line, not the
|
||||
# office_oxide one (a naive "read the first version in go.mod" would pick
|
||||
# the wrong library). office_oxide_url must likewise stay on its own line.
|
||||
(tmp_path / "go.mod").write_text("module fixture\n\nrequire github.com/yfedoseev/office_oxide/go v0.1.12\nrequire github.com/yfedoseev/pdf_oxide/go v0.4.1\n")
|
||||
pdf_url = native.pdf_oxide_url(str(tmp_path), "linux", "amd64")
|
||||
office_url = native.office_oxide_url(str(tmp_path), "linux", "amd64")
|
||||
assert "/v0.4.1/" in pdf_url # pdf_oxide's own version
|
||||
assert "/v0.1.12/" in office_url # office_oxide's own version
|
||||
assert "/v0.1.12/" not in pdf_url # not coupled to office_oxide's line
|
||||
|
||||
|
||||
def test_url_builders_honor_gh_proxy(monkeypatch):
|
||||
monkeypatch.setenv("GH_PROXY", "https://gh-proxy.com/")
|
||||
assert native.pdfium_url("linux", "amd64").startswith("https://gh-proxy.com/https://github.com/")
|
||||
assert native.pdf_oxide_url("linux", "arm64").startswith("https://gh-proxy.com/https://github.com/")
|
||||
assert native.pdf_oxide_url(native.REPO_ROOT, "linux", "arm64").startswith("https://gh-proxy.com/https://github.com/")
|
||||
|
||||
|
||||
def test_dev_downloader_rejects_untrusted_gh_proxy(monkeypatch):
|
||||
@@ -315,6 +339,121 @@ def test_native_archive_rejects_unsafe_members(tmp_path, kind):
|
||||
assert sorted(path.name for path in tmp_path.iterdir()) == ["native", "outside", "unsafe.tar.gz"]
|
||||
|
||||
|
||||
# ── Version-stamped extraction (Slice 2): native .a must land under a
|
||||
# version-keyed subdir (v<version>/) of the user cache, never a flat layout, so a
|
||||
# single mounted cache can hold several versions at once (different PRs pinned to
|
||||
# different go.mod versions all build offline). extract_native_lib() is the
|
||||
# single extraction seam main() calls; it is idempotent (skips when the version
|
||||
# dir already holds the required files). ─────────────────────────────────────
|
||||
|
||||
|
||||
def _make_pdfium_archive(path, names):
|
||||
with tarfile.open(path, "w:gz") as tf:
|
||||
for name in names:
|
||||
data = b"pdfium-payload-1234" # non-empty so complete() accepts it
|
||||
info = tarfile.TarInfo(name)
|
||||
info.size = len(data)
|
||||
tf.addfile(info, io.BytesIO(data))
|
||||
|
||||
|
||||
def test_extracts_pdfium_into_versioned_dir(tmp_path, monkeypatch):
|
||||
monkeypatch.setenv("RAGFLOW_NATIVE_LIBS_DIR", str(tmp_path / "native-libs"))
|
||||
archive = tmp_path / "pdfium-linux-x64-static.tgz"
|
||||
_make_pdfium_archive(archive, ["lib/libpdfium.a", "lib/libc++.a", "lib/libc++abi.a", "include/fpdfview.h"])
|
||||
native.extract_native_lib(
|
||||
str(archive),
|
||||
"pdfium-static",
|
||||
"v7809",
|
||||
["lib/libpdfium.a", "lib/libc++.a", "lib/libc++abi.a", "include/fpdfview.h"],
|
||||
None,
|
||||
)
|
||||
target = tmp_path / "native-libs" / "pdfium-static" / "v7809"
|
||||
assert (target / "lib" / "libpdfium.a").is_file()
|
||||
assert (target / "include" / "fpdfview.h").is_file()
|
||||
|
||||
|
||||
def test_extract_native_lib_idempotent_on_rerun(tmp_path, monkeypatch):
|
||||
monkeypatch.setenv("RAGFLOW_NATIVE_LIBS_DIR", str(tmp_path / "native-libs"))
|
||||
archive = tmp_path / "pdfium-linux-x64-static.tgz"
|
||||
_make_pdfium_archive(archive, ["lib/libpdfium.a"])
|
||||
native.extract_native_lib(str(archive), "pdfium-static", "v7809", ["lib/libpdfium.a"], None)
|
||||
before = sorted(os.listdir(tmp_path / "native-libs" / "pdfium-static" / "v7809" / "lib"))
|
||||
native.extract_native_lib(str(archive), "pdfium-static", "v7809", ["lib/libpdfium.a"], None)
|
||||
after = sorted(os.listdir(tmp_path / "native-libs" / "pdfium-static" / "v7809" / "lib"))
|
||||
assert before == after
|
||||
|
||||
|
||||
def _make_stamped_archive(path, lib_name, stamp, header_name):
|
||||
"""Build a tar.gz with the .a (carrying STAMP, a version-bearing byte
|
||||
string) plus an include header, so complete()'s file-list AND stamp checks
|
||||
can both be exercised per library format."""
|
||||
with tarfile.open(path, "w:gz") as tf:
|
||||
info = tarfile.TarInfo(lib_name)
|
||||
payload = b"!<arch>\n" + stamp
|
||||
info.size = len(payload)
|
||||
tf.addfile(info, io.BytesIO(payload))
|
||||
header = tarfile.TarInfo(header_name)
|
||||
header.size = len(b"// header")
|
||||
tf.addfile(header, io.BytesIO(b"// header"))
|
||||
|
||||
|
||||
def test_pdf_oxide_stamp_text_accepts_matching_version(tmp_path, monkeypatch):
|
||||
# pdf_oxide embeds a plain-text `pdf_oxide <version>` token (not the
|
||||
# null-delimited token office_oxide uses). complete() must accept it when
|
||||
# stamp_format="text", or auto-download would reject a good archive and
|
||||
# build.sh's controlled download path would fail closed.
|
||||
version = native.go_module_version("github.com/yfedoseev/pdf_oxide/go", native.REPO_ROOT)
|
||||
monkeypatch.setenv("RAGFLOW_NATIVE_LIBS_DIR", str(tmp_path / "native-libs"))
|
||||
archive = tmp_path / "pdf_oxide-go-ffi-linux-amd64.tar.gz"
|
||||
_make_stamped_archive(archive, "lib/linux_amd64/libpdf_oxide.a", f"pdf_oxide {version}".encode(), "include/pdf_oxide.h")
|
||||
native.extract_native_lib(
|
||||
str(archive),
|
||||
"pdf_oxide",
|
||||
f"v{version}",
|
||||
["lib/linux_amd64/libpdf_oxide.a", "include/pdf_oxide.h"],
|
||||
version,
|
||||
"text",
|
||||
)
|
||||
assert (tmp_path / "native-libs" / "pdf_oxide" / f"v{version}" / "lib" / "linux_amd64" / "libpdf_oxide.a").is_file()
|
||||
|
||||
|
||||
def test_pdf_oxide_stamp_text_rejects_wrong_version(tmp_path, monkeypatch):
|
||||
# A pdf_oxide .a stamped with a different version must be rejected, not
|
||||
# silently accepted (the whole point of the stamp check is to catch a
|
||||
# mismatched/wrong native build).
|
||||
version = native.go_module_version("github.com/yfedoseev/pdf_oxide/go", native.REPO_ROOT)
|
||||
monkeypatch.setenv("RAGFLOW_NATIVE_LIBS_DIR", str(tmp_path / "native-libs"))
|
||||
archive = tmp_path / "pdf_oxide-go-ffi-linux-amd64.tar.gz"
|
||||
_make_stamped_archive(archive, "lib/linux_amd64/libpdf_oxide.a", b"pdf_oxide 0.9.9", "include/pdf_oxide.h")
|
||||
with pytest.raises(RuntimeError, match=f"missing required native files or version {version}"):
|
||||
native.extract_native_lib(
|
||||
str(archive),
|
||||
"pdf_oxide",
|
||||
f"v{version}",
|
||||
["lib/linux_amd64/libpdf_oxide.a", "include/pdf_oxide.h"],
|
||||
version,
|
||||
"text",
|
||||
)
|
||||
|
||||
|
||||
def test_office_oxide_stamp_null_accepts_matching_version(tmp_path, monkeypatch):
|
||||
# Regression guard: office_oxide keeps its null-delimited stamp token; the
|
||||
# "text" vs "null" split must not break it.
|
||||
version = native.go_module_version("github.com/yfedoseev/office_oxide/go", native.REPO_ROOT)
|
||||
monkeypatch.setenv("RAGFLOW_NATIVE_LIBS_DIR", str(tmp_path / "native-libs"))
|
||||
archive = tmp_path / "native-linux-x86_64.tar.gz"
|
||||
_make_stamped_archive(archive, "lib/liboffice_oxide.a", f"\x00{version}\x00".encode(), "include/office_oxide_c/office_oxide.h")
|
||||
native.extract_native_lib(
|
||||
str(archive),
|
||||
"office_oxide",
|
||||
f"v{version}",
|
||||
["lib/liboffice_oxide.a", "include/office_oxide_c/office_oxide.h"],
|
||||
version,
|
||||
"null",
|
||||
)
|
||||
assert (tmp_path / "native-libs" / "office_oxide" / f"v{version}" / "lib" / "liboffice_oxide.a").is_file()
|
||||
|
||||
|
||||
# ── Local-dev one-stop: the dev downloader must also fetch the runtime
|
||||
# resources (cl100k / DeepDoc .ort / tokenizer assets) so a developer runs THIS
|
||||
# script ONCE and the Go server is ready to build, run, parse and count. ─────
|
||||
@@ -560,3 +699,415 @@ def test_arm64_docs_section_warns_that_lld18_links_but_hangs():
|
||||
section = _arm64_docs_section()
|
||||
assert "lld-18" in section, "lld-18 is no longer called out as insufficient"
|
||||
assert "hang" in section.lower() or "spin" in section.lower(), "lld-18 startup-hang failure mode missing"
|
||||
|
||||
|
||||
# ── Single source of truth for native versions (Slice 1) ───────────────────────
|
||||
# build.sh and download_dev_deps.py must agree on every native version. The
|
||||
# authoritative values live in tools/scripts/native_lib_version.sh (office_oxide
|
||||
# / pdf_oxide from go.mod, ORT from environments.go, pdfium as one shared const).
|
||||
# These tests pin that contract: no second hardcoded copy may drift.
|
||||
|
||||
|
||||
def test_deepdoc_ort_version_reads_environments_go(tmp_path):
|
||||
# Fixture environments.go with a deliberately non-default version.
|
||||
common = tmp_path / "internal" / "common"
|
||||
common.mkdir(parents=True)
|
||||
(common / "environments.go").write_text('package common\n\n// DeepDocORTVersion is ...\nconst DeepDocORTVersion = "9.9.9"\n')
|
||||
assert native.deepdoc_ort_version(str(tmp_path)) == "9.9.9"
|
||||
|
||||
|
||||
def test_deepdoc_ort_version_against_real_repo():
|
||||
# Sanity: parses the checked-out repo's actual pin. Assert the shape (semver)
|
||||
# rather than a hardcoded literal so the test tracks environments.go on bumps.
|
||||
import re
|
||||
|
||||
version = native.deepdoc_ort_version(native.REPO_ROOT)
|
||||
assert re.match(r"^\d+\.\d+\.\d+", version), version
|
||||
|
||||
|
||||
def test_pdfium_static_version_reads_single_source(tmp_path):
|
||||
# Fixture native_lib_version.sh exporting a sentinel (NOT the real 7809).
|
||||
scripts = tmp_path / "tools" / "scripts"
|
||||
scripts.mkdir(parents=True)
|
||||
(scripts / "native_lib_version.sh").write_text("pdfium_static_version() { echo 1234; }\n")
|
||||
assert native.pdfium_static_version(str(tmp_path)) == "1234"
|
||||
|
||||
|
||||
def test_pdfium_static_version_against_real_repo():
|
||||
# Sanity: the shared const on the checked-out repo.
|
||||
assert native.pdfium_static_version(native.REPO_ROOT) == "7809"
|
||||
|
||||
|
||||
# ── deepdoc_ort_version (bash) single source ───────────────────────────────
|
||||
# build.sh must read DeepDocORTVersion from environments.go through the SAME
|
||||
# native_lib_version.sh helper the Python downloader mirrors (deepdoc_ort_version
|
||||
# in download_dev_deps.py), not an inline grep repeated at every call site. These
|
||||
# tests pin the bash function contract so the 4 inline greps in build.sh can be
|
||||
# collapsed onto one source.
|
||||
|
||||
|
||||
def test_deepdoc_ort_version_bash_reads_environments_go(tmp_path):
|
||||
# Fixture environments.go with a deliberately non-default version.
|
||||
common = tmp_path / "internal" / "common"
|
||||
common.mkdir(parents=True)
|
||||
(common / "environments.go").write_text('package common\n\n// DeepDocORTVersion is ...\nconst DeepDocORTVersion = "9.9.9"\n')
|
||||
res = _sh_eval(f"deepdoc_ort_version {tmp_path}", _env_with("/home/alice"))
|
||||
assert res.returncode == 0, res.stderr
|
||||
assert res.stdout.strip() == "9.9.9"
|
||||
|
||||
|
||||
def test_deepdoc_ort_version_bash_against_real_repo():
|
||||
# Sanity: parses the checked-out repo's actual pin. Assert the shape (semver)
|
||||
# rather than a hardcoded literal so the test tracks environments.go on bumps.
|
||||
import re
|
||||
|
||||
res = _sh_eval(f"deepdoc_ort_version {native.REPO_ROOT}", _env_with("/home/alice"))
|
||||
assert res.returncode == 0, res.stderr
|
||||
assert re.match(r"^\d+\.\d+\.\d+", res.stdout.strip()), res.stdout
|
||||
|
||||
|
||||
def test_deepdoc_ort_version_bash_missing_file(tmp_path):
|
||||
# Adversarial: an empty repo root with no environments.go must NOT echo a
|
||||
# version and must signal failure (so callers fail loud, mirroring the
|
||||
# Python RuntimeError), not silently return an empty string with rc 0.
|
||||
res = _sh_eval(f"deepdoc_ort_version {tmp_path}", _env_with("/home/alice"))
|
||||
assert res.returncode == 1, res.stdout
|
||||
assert res.stdout.strip() == ""
|
||||
|
||||
|
||||
# ── Slice 3: CI mounts the native-lib user cache (RAGFLOW_NATIVE_LIBS_DIR) ──
|
||||
# build.sh resolves the native static libs from a single cache root
|
||||
# (default ~/ragflow-native-libs, overridable via RAGFLOW_NATIVE_LIBS_DIR so CI
|
||||
# can mount the cache at a fixed host path instead of the build user's $HOME).
|
||||
# These tests pin that resolution contract (native_lib_version.sh) and the
|
||||
# sep-tests.yml wiring that mounts the cache into every BUILDER_CONTAINER.
|
||||
|
||||
|
||||
def _sh_eval(func_call, env):
|
||||
"""Source the real native_lib_version.sh and call FUNC_CALL in a throwaway
|
||||
bash, returning the CompletedProcess. ENV is the full env (HOME/PATH must be
|
||||
supplied), mirroring how build.sh sources the file."""
|
||||
sh = os.path.join(native.REPO_ROOT, "tools", "scripts", "native_lib_version.sh")
|
||||
assert os.path.isfile(sh), f"native_lib_version.sh not found at {sh}"
|
||||
cmd = f'source "{sh}" && {func_call}'
|
||||
return subprocess.run(["bash", "-c", cmd], env=env, capture_output=True, text=True, check=False)
|
||||
|
||||
|
||||
def _env_with(home, override=None):
|
||||
env = {"HOME": home, "PATH": os.environ.get("PATH", "")}
|
||||
if override is not None:
|
||||
env["RAGFLOW_NATIVE_LIBS_DIR"] = override
|
||||
return env
|
||||
|
||||
|
||||
def test_native_libs_dir_defaults_to_home():
|
||||
# No override: resolve to $HOME/ragflow-native-libs.
|
||||
res = _sh_eval("ragflow_native_libs_dir", _env_with("/home/alice"))
|
||||
assert res.returncode == 0, res.stderr
|
||||
assert res.stdout.strip() == "/home/alice/ragflow-native-libs"
|
||||
|
||||
|
||||
def test_native_libs_dir_honors_override():
|
||||
# Override set: CI mounts the cache at a fixed host path; build.sh must use it.
|
||||
res = _sh_eval("ragflow_native_libs_dir", _env_with("/home/alice", "/mnt/cache"))
|
||||
assert res.returncode == 0, res.stderr
|
||||
assert res.stdout.strip() == "/mnt/cache"
|
||||
|
||||
|
||||
def test_native_libs_dir_empty_override_falls_back():
|
||||
# Adversarial: an explicitly EMPTY override must fall back to the HOME default
|
||||
# (the ${VAR:-default} expansion), never resolve to an empty cache root.
|
||||
res = _sh_eval("ragflow_native_libs_dir", _env_with("/home/alice", ""))
|
||||
assert res.returncode == 0, res.stderr
|
||||
assert res.stdout.strip() == "/home/alice/ragflow-native-libs"
|
||||
|
||||
|
||||
SEP_TESTS_YML = os.path.join(native.REPO_ROOT, ".github", "workflows", "sep-tests.yml")
|
||||
|
||||
|
||||
def test_sep_tests_mounts_native_libs_cache():
|
||||
# Slice 3 contract: the native static-lib user cache is mounted into every
|
||||
# BUILDER_CONTAINER (4 docker runs) and exposed via RAGFLOW_NATIVE_LIBS_DIR,
|
||||
# replacing the old /opt pre-bake. The YAML must also stay parseable.
|
||||
assert os.path.isfile(SEP_TESTS_YML), f"sep-tests.yml not found at {SEP_TESTS_YML}"
|
||||
with open(SEP_TESTS_YML, encoding="utf-8") as f:
|
||||
content = f.read()
|
||||
# YAML must still parse (guards against a malformed edit); skip when pyyaml
|
||||
# is unavailable in the test env (the CI runner has it).
|
||||
try:
|
||||
import yaml
|
||||
|
||||
yaml.safe_load(content)
|
||||
except ImportError:
|
||||
pass
|
||||
# The cache root is declared as a job env so runner and builder container agree.
|
||||
assert "RAGFLOW_NATIVE_LIBS_DIR" in content
|
||||
# Each of the 4 BUILDER_CONTAINER docker runs bind-mounts the cache.
|
||||
assert content.count("${RAGFLOW_NATIVE_LIBS_DIR}:${RAGFLOW_NATIVE_LIBS_DIR}") >= 4
|
||||
# A fail-fast guard catches a missing/partial mount before a cryptic cgo link error.
|
||||
assert "native-lib cache not mounted" in content
|
||||
|
||||
|
||||
# ── downloader root resolution must honour RAGFLOW_NATIVE_LIBS_DIR (P1#2) ─────
|
||||
# build.sh resolves the cache root through native_lib_version.ragflow_native_libs_dir()
|
||||
# (${RAGFLOW_NATIVE_LIBS_DIR:-~/ragflow-native-libs}); the downloader must agree so a
|
||||
# runner that overrides the mount path does not silently download into the default.
|
||||
|
||||
|
||||
def test_python_native_libs_dir_default(monkeypatch):
|
||||
monkeypatch.delenv("RAGFLOW_NATIVE_LIBS_DIR", raising=False)
|
||||
assert native.native_libs_dir() == os.path.expanduser("~/ragflow-native-libs")
|
||||
|
||||
|
||||
def test_python_native_libs_dir_honors_override(monkeypatch):
|
||||
# Adversarial: a custom mount path must be returned verbatim, NOT the default.
|
||||
monkeypatch.setenv("RAGFLOW_NATIVE_LIBS_DIR", "/srv/custom/native-libs")
|
||||
assert native.native_libs_dir() == "/srv/custom/native-libs"
|
||||
|
||||
|
||||
def test_python_native_libs_dir_empty_override_falls_back(monkeypatch):
|
||||
# Adversarial: an empty override must fall back to the default, not resolve to "".
|
||||
monkeypatch.setenv("RAGFLOW_NATIVE_LIBS_DIR", "")
|
||||
assert native.native_libs_dir() == os.path.expanduser("~/ragflow-native-libs")
|
||||
|
||||
|
||||
# ── downloader must not crash when deps/ is absent (P1#3) ─────────────────────
|
||||
|
||||
|
||||
def test_ensure_deps_dir_creates_missing(monkeypatch, tmp_path):
|
||||
# In the Docker go-builder stage nothing COPYs deps/, so the downloader's
|
||||
# os.chdir(deps) used to raise FileNotFoundError and abort the image build.
|
||||
# ensure_deps_dir() must create it first.
|
||||
missing = tmp_path / "no_deps_dir"
|
||||
monkeypatch.setattr(native, "RAGFLOW_DEPS_DIR", str(missing))
|
||||
native.ensure_deps_dir()
|
||||
assert missing.is_dir()
|
||||
|
||||
|
||||
# ── ORT version-stamp symlink must be RELATIVE and read-only-safe (Fix #1/#2) ──
|
||||
# build.sh stamps the ORT archive with a v<version>-<sha16> symlink so an
|
||||
# in-place .a upgrade invalidates Go's build cache. The symlink MUST be relative
|
||||
# (resolves under any cache mount: build host, runner, CI builder) and build.sh
|
||||
# MUST skip the write when an existing link already resolves to the same archive
|
||||
# (so a read-only CI mount is a no-op instead of a fatal `ln` on a read-only fs).
|
||||
|
||||
|
||||
_ORT_STAMP_SH = os.path.join(native.REPO_ROOT, "tools", "scripts", "ort_stamp.sh")
|
||||
|
||||
|
||||
def _stamp_eval(versioned_dir, archive_file):
|
||||
assert os.path.isfile(_ORT_STAMP_SH), f"ort_stamp.sh not found at {_ORT_STAMP_SH}"
|
||||
cmd = f'source "{_ORT_STAMP_SH}" && ragflow_ort_stamp_target "$@"'
|
||||
return subprocess.run(
|
||||
["bash", "-c", cmd, "_", versioned_dir, archive_file],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False,
|
||||
)
|
||||
|
||||
|
||||
def test_ort_stamp_target_is_relative(tmp_path):
|
||||
versioned = tmp_path / "cache" / "onnxruntime" / "static_lib" / "v1.29.0-abc123"
|
||||
versioned.mkdir(parents=True)
|
||||
archive = tmp_path / "cache" / "onnxruntime" / "static_lib" / "onnxruntime-linux-x64-static_lib-1.29.0-glibc2_28" / "lib" / "libonnxruntime.a"
|
||||
archive.parent.mkdir(parents=True)
|
||||
archive.write_bytes(b"ort")
|
||||
res = _stamp_eval(str(versioned), str(archive))
|
||||
assert res.returncode == 0, res.stderr
|
||||
needs, target = res.stdout.strip().split("|")
|
||||
assert needs == "yes"
|
||||
assert not os.path.isabs(target), f"stamp target must be relative, got {target!r}"
|
||||
assert (versioned / target).resolve() == archive.resolve()
|
||||
|
||||
|
||||
def test_ort_stamp_target_skips_when_existing_link_correct(tmp_path):
|
||||
# ro-safe: an existing symlink already pointing at the archive means no write.
|
||||
versioned = tmp_path / "cache" / "onnxruntime" / "static_lib" / "v1.29.0-abc123"
|
||||
versioned.mkdir(parents=True)
|
||||
archive = tmp_path / "cache" / "onnxruntime" / "static_lib" / "norm" / "lib" / "libonnxruntime.a"
|
||||
archive.parent.mkdir(parents=True)
|
||||
archive.write_bytes(b"ort")
|
||||
rel = os.path.relpath(archive, versioned)
|
||||
os.symlink(rel, versioned / "libonnxruntime.a")
|
||||
res = _stamp_eval(str(versioned), str(archive))
|
||||
assert res.returncode == 0, res.stderr
|
||||
needs, target = res.stdout.strip().split("|")
|
||||
assert needs == "no", "existing correct symlink must need no write (ro-safe)"
|
||||
assert target == rel
|
||||
|
||||
|
||||
def test_ort_stamp_target_writes_when_link_missing(tmp_path):
|
||||
versioned = tmp_path / "v1.29.0-abc123"
|
||||
versioned.mkdir(parents=True)
|
||||
archive = tmp_path / "norm" / "lib" / "libonnxruntime.a"
|
||||
archive.parent.mkdir(parents=True)
|
||||
archive.write_bytes(b"ort")
|
||||
res = _stamp_eval(str(versioned), str(archive))
|
||||
assert res.returncode == 0, res.stderr
|
||||
needs, _ = res.stdout.strip().split("|")
|
||||
assert needs == "yes"
|
||||
|
||||
|
||||
# ── download_dev_deps.py pre-creates the ORT stamp dir during population ──────
|
||||
# build.sh's stamp step is a no-op on a read-only CI mount ONLY if the stamp dir
|
||||
# already exists in the cache. The downloader must therefore create the
|
||||
# v<version>-<sha16> dir (relative symlink) while it still has write access.
|
||||
|
||||
|
||||
def test_stamp_onnxruntime_version_creates_relative_symlink(tmp_path):
|
||||
static_lib = tmp_path / "onnxruntime" / "static_lib"
|
||||
static_lib.mkdir(parents=True)
|
||||
norm = static_lib / "onnxruntime-linux-x64-static_lib-1.29.0-glibc2_28"
|
||||
(norm / "lib").mkdir(parents=True)
|
||||
(norm / "lib" / "libonnxruntime.a").write_bytes(b"!<arch>\nort")
|
||||
native.stamp_onnxruntime_version(
|
||||
str(static_lib),
|
||||
"onnxruntime-linux-x64-static_lib-1.29.0-glibc2_28",
|
||||
"1.29.0",
|
||||
)
|
||||
stamp_dirs = list(static_lib.glob("v1.29.0-*"))
|
||||
assert len(stamp_dirs) == 1, f"expected exactly one stamp dir, found {stamp_dirs}"
|
||||
link = stamp_dirs[0] / "libonnxruntime.a"
|
||||
assert link.is_symlink(), "stamp must be a symlink"
|
||||
assert not os.path.isabs(os.readlink(link)), "stamp symlink must be relative"
|
||||
assert link.resolve() == (norm / "lib" / "libonnxruntime.a").resolve()
|
||||
|
||||
|
||||
def test_stamp_onnxruntime_version_idempotent(tmp_path):
|
||||
static_lib = tmp_path / "onnxruntime" / "static_lib"
|
||||
static_lib.mkdir(parents=True)
|
||||
norm_dir = "onnxruntime-linux-x64-static_lib-1.29.0-glibc2_28"
|
||||
norm = static_lib / norm_dir
|
||||
(norm / "lib").mkdir(parents=True)
|
||||
(norm / "lib" / "libonnxruntime.a").write_bytes(b"!<arch>\nort")
|
||||
native.stamp_onnxruntime_version(str(static_lib), norm_dir, "1.29.0")
|
||||
before = sorted(p.name for p in static_lib.iterdir())
|
||||
native.stamp_onnxruntime_version(str(static_lib), norm_dir, "1.29.0")
|
||||
after = sorted(p.name for p in static_lib.iterdir())
|
||||
assert before == after
|
||||
|
||||
|
||||
# ── _native_libs_present ORT gate must be version+platform aware ──────────────
|
||||
# build.sh's _native_libs_present decides whether the controlled auto-downloader
|
||||
# runs. Its ORT check historically matched ANY .a under the static_lib prefix
|
||||
# (find ... -name '*.a'), so a shared cache holding only an OLD version or a
|
||||
# DIFFERENT platform's ORT was reported "present" and the downloader was skipped
|
||||
# — the missing lib then failed only at link time. The gate must match the SAME
|
||||
# <platform-prefix><version> dir that check_onnxruntime_deps links
|
||||
# (*/${ort_dir_prefix}${ort_version}*/lib/*.a). ragflow_ort_present is that pure
|
||||
# decision so it is unit-testable in isolation.
|
||||
|
||||
_ORT_PRESENT_SH = os.path.join(native.REPO_ROOT, "tools", "scripts", "ort_present.sh")
|
||||
|
||||
|
||||
def _present_eval(prefix, ort_dir_prefix, ort_version):
|
||||
"""Source the real ort_present.sh and call ragflow_ort_present."""
|
||||
assert os.path.isfile(_ORT_PRESENT_SH), f"ort_present.sh not found at {_ORT_PRESENT_SH}"
|
||||
cmd = f'source "{_ORT_PRESENT_SH}" && ragflow_ort_present "$@"'
|
||||
return subprocess.run(
|
||||
["bash", "-c", cmd, "_", prefix, ort_dir_prefix, ort_version],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False,
|
||||
)
|
||||
|
||||
|
||||
def _seed_ort(prefix, dir_name):
|
||||
"""Create a version-keyed ORT dir with a real libonnxruntime.a inside lib/."""
|
||||
lib = prefix / dir_name / "lib"
|
||||
lib.mkdir(parents=True)
|
||||
(lib / "libonnxruntime.a").write_bytes(b"!<arch>\nort")
|
||||
return prefix / dir_name
|
||||
|
||||
|
||||
def test_ort_present_happy_path_matches_version_and_platform(tmp_path):
|
||||
prefix = tmp_path / "static_lib"
|
||||
_seed_ort(prefix, "onnxruntime-linux-x64-static_lib-1.29.0-glibc2_28")
|
||||
res = _present_eval(str(prefix), "onnxruntime-linux-x64-static_lib-", "1.29.0")
|
||||
assert res.returncode == 0, res.stderr
|
||||
|
||||
|
||||
def test_ort_present_rejects_old_version(tmp_path):
|
||||
# Adversarial: cache holds ONLY an older ORT version; required version differs.
|
||||
# The old any-.a check wrongly returned present and skipped the downloader.
|
||||
prefix = tmp_path / "static_lib"
|
||||
_seed_ort(prefix, "onnxruntime-linux-x64-static_lib-1.23.2-glibc2_28")
|
||||
res = _present_eval(str(prefix), "onnxruntime-linux-x64-static_lib-", "1.29.0")
|
||||
assert res.returncode == 1, "old-version ORT must NOT satisfy the required version"
|
||||
assert res.stderr == ""
|
||||
|
||||
|
||||
def test_ort_present_rejects_wrong_platform(tmp_path):
|
||||
# Adversarial: cache holds ORT for aarch64 but the target is amd64.
|
||||
prefix = tmp_path / "static_lib"
|
||||
_seed_ort(prefix, "onnxruntime-linux-aarch64-static_lib-1.29.0-glibc2_28")
|
||||
res = _present_eval(str(prefix), "onnxruntime-linux-x64-static_lib-", "1.29.0")
|
||||
assert res.returncode == 1, "other-platform ORT must NOT satisfy the target platform"
|
||||
|
||||
|
||||
def test_ort_present_rejects_empty_cache(tmp_path):
|
||||
prefix = tmp_path / "static_lib"
|
||||
prefix.mkdir()
|
||||
res = _present_eval(str(prefix), "onnxruntime-linux-x64-static_lib-", "1.29.0")
|
||||
assert res.returncode == 1
|
||||
|
||||
|
||||
def test_ort_present_rejects_stray_top_level_archive(tmp_path):
|
||||
# Adversarial: a .a sitting directly in the prefix root (not under a
|
||||
# versioned .../lib/ dir) must NOT count as present.
|
||||
prefix = tmp_path / "static_lib"
|
||||
prefix.mkdir()
|
||||
(prefix / "libonnxruntime.a").write_bytes(b"!<arch>\nort")
|
||||
res = _present_eval(str(prefix), "onnxruntime-linux-x64-static_lib-", "1.29.0")
|
||||
assert res.returncode == 1, "stray top-level .a must not satisfy the versioned gate"
|
||||
|
||||
|
||||
def test_ort_present_ignores_provider_specific_archives(tmp_path):
|
||||
# Adversarial: a cuda provider archive inside the correct version dir must be
|
||||
# excluded (it is not the base libonnxruntime.a), matching the link-time
|
||||
# selection. Only the provider lib present => not present.
|
||||
prefix = tmp_path / "static_lib"
|
||||
lib = prefix / "onnxruntime-linux-x64-static_lib-1.29.0-glibc2_28" / "lib"
|
||||
lib.mkdir(parents=True)
|
||||
(lib / "libonnxruntime_providers_cuda.a").write_bytes(b"!<arch>\nort")
|
||||
res = _present_eval(str(prefix), "onnxruntime-linux-x64-static_lib-", "1.29.0")
|
||||
assert res.returncode == 1, "provider-specific archive must not satisfy the base lib gate"
|
||||
|
||||
|
||||
# ── Documentation contract: IDE CGO snippet uses the versioned layout ─────────
|
||||
# The native libs live under <lib>/v<version>/ (one mounted cache holds several
|
||||
# versions). The IDE CGO snippet must point there, or a developer following the
|
||||
# doc links a .a that does not exist at the legacy flat path.
|
||||
|
||||
|
||||
def test_development_doc_ide_cgo_uses_versioned_layout():
|
||||
doc = os.path.join(native.REPO_ROOT, "internal", "development.md")
|
||||
with open(doc, encoding="utf-8") as file:
|
||||
text = file.read()
|
||||
# Version-keyed paths must appear for every native lib the IDE snippet links
|
||||
# (resolved via v*/ glob, matching download_dev_deps.py / build.sh).
|
||||
assert "office_oxide/v" in text, "office_oxide must use v<version> layout"
|
||||
assert "pdfium-static/v" in text, "pdfium-static must use v<version> layout"
|
||||
assert "pdf_oxide/v" in text, "pdf_oxide must use v<version> layout"
|
||||
# The legacy flat layout must be gone so it cannot mislead.
|
||||
assert "office_oxide/lib/liboffice_oxide.a" not in text
|
||||
assert "pdfium-static/lib/libpdfium.a" not in text
|
||||
|
||||
|
||||
# ── Versioned-extraction fixtures must isolate via env, not the module attr ───
|
||||
# native_libs_dir() reads RAGFLOW_NATIVE_LIBS_DIR (env); monkeypatching the
|
||||
# NATIVE_LIBS_DIR module attribute only isolates when that env is unset, so under
|
||||
# CI (where the env is set to the real cache) the attribute stub is ignored and
|
||||
# files leak into the real cache. The fixtures must isolate via env.
|
||||
|
||||
|
||||
def test_extract_native_lib_honors_ragflow_native_libs_dir_env(tmp_path, monkeypatch):
|
||||
# Adversarial: with the env set (as CI does) AND the module attribute stubbed
|
||||
# to a different path, extraction must follow the env, not the attribute.
|
||||
monkeypatch.setenv("RAGFLOW_NATIVE_LIBS_DIR", str(tmp_path / "isolated"))
|
||||
monkeypatch.setattr(native, "NATIVE_LIBS_DIR", "/attribute-path-ignored")
|
||||
archive = tmp_path / "pdfium-linux-x64-static.tgz"
|
||||
_make_pdfium_archive(archive, ["lib/libpdfium.a"])
|
||||
native.extract_native_lib(str(archive), "pdfium-static", "v7809", ["lib/libpdfium.a"], None)
|
||||
assert (tmp_path / "isolated" / "pdfium-static" / "v7809" / "lib" / "libpdfium.a").is_file()
|
||||
assert not os.path.exists("/attribute-path-ignored")
|
||||
Reference in new issue
Block a user