mirror of
https://github.com/infiniflow/ragflow.git
synced 2026-11-10 21:32:43 +08:00
fix(deps): restore kept ragflow_deps tag and CI path after rename
The rename sed rewrote two names that the rename intentionally kept unchanged: - the CI install path /opt/ragflow_deps (sep-tests.yml read /opt/deps, breaking the offline cl100k fallback) - the published image tag infiniflow/ragflow_deps (left as bare infiniflow/deps in sep-tests.yml, .dockerignore, and Dockerfile/Dockerfile_base error messages) Restore the intentional names in those files. Add tools/scripts/test_dep_rename_invariants.py as a regression guard asserting the two exceptions stay intact so the blanket rename cannot silently regress them.
This commit is contained in:
1 parent
f83a3fe6b7
commit
2f6857095d
5 files changed
+107
-18
No files matched your search
+2
-2
@@ -1,7 +1,7 @@
|
||||
# RAGFlow .dockerignore
|
||||
# Reduces Docker build context sent to the daemon.
|
||||
# All excluded items are either rebuilt inside Docker, mounted from
|
||||
# infiniflow/deps, or are local-only artifacts.
|
||||
# infiniflow/ragflow_deps, or are local-only artifacts.
|
||||
|
||||
# ── Python virtual environments ─────────────────────────────────────────────
|
||||
.venv/
|
||||
@@ -37,7 +37,7 @@ target/
|
||||
# Excluding the directory contents keeps the main build context small
|
||||
# regardless of which resources are prepared for the dependency image.
|
||||
# The deps image is built from this directory with:
|
||||
# cd deps && docker build -f Dockerfile -t infiniflow/deps .
|
||||
# cd deps && docker build -f Dockerfile -t infiniflow/ragflow_deps .
|
||||
# The native downloader tools/scripts/download_dev_deps.py is copied by the
|
||||
# main Dockerfile (not excluded here) so build.sh can read its ORT version pin
|
||||
# when running `./build.sh --go`.
|
||||
|
||||
@@ -286,14 +286,14 @@ jobs:
|
||||
# openai blob first (small ~1.6MB, reachable from the CI runner — the
|
||||
# pre-PR code downloaded it on the fly, so the network path is known
|
||||
# good). If the direct fetch fails (e.g. GFW), fall back to extracting
|
||||
# it from the local infiniflow/deps image *if already present*
|
||||
# it from the local infiniflow/ragflow_deps image *if already present*
|
||||
# on the runner (no `docker pull`, since that huge image is first
|
||||
# fetched at the later "Build ragflow:nightly" step). Best-effort: if
|
||||
# neither source works, the loader still fails loudly (no silent
|
||||
# 0-token degradation).
|
||||
# tiktoken is already installed in runner
|
||||
mkdir -p deps
|
||||
cp /opt/deps/cl100k_base.tiktoken deps/cl100k_base.tiktoken || true
|
||||
cp /opt/ragflow_deps/cl100k_base.tiktoken deps/cl100k_base.tiktoken || true
|
||||
if [ ! -f deps/cl100k_base.tiktoken ]; then
|
||||
mkdir -p deps
|
||||
if command -v curl >/dev/null 2>&1; then
|
||||
@@ -308,7 +308,7 @@ jobs:
|
||||
CID=$(docker create infiniflow/ragflow_deps:latest true) || true
|
||||
if [ -n "${CID:-}" ]; then
|
||||
if docker cp "$CID":/cl100k_base.tiktoken deps/cl100k_base.tiktoken 2>/dev/null; then
|
||||
echo "tiktoken: cl100k table copied from cached infiniflow/deps image"
|
||||
echo "tiktoken: cl100k table copied from cached infiniflow/ragflow_deps image"
|
||||
fi
|
||||
docker rm -f "$CID" >/dev/null 2>&1 || true
|
||||
fi
|
||||
@@ -322,7 +322,7 @@ jobs:
|
||||
|
||||
# Provide the tokenizer assets the four non-cl100k counters load
|
||||
# (deps/huggingface.co/<repo>/<file>). Their only no-network source
|
||||
# is the infiniflow/deps image — the runtime images read them from
|
||||
# is the infiniflow/ragflow_deps image — the runtime images read them from
|
||||
# exactly the same path — so this is the same "if the image is already
|
||||
# cached" fallback the table above uses (again, no `docker pull`). Without
|
||||
# them counter_properties_test / asset_dir_test silently drop their
|
||||
@@ -343,7 +343,7 @@ jobs:
|
||||
fi
|
||||
mkdir -p "$(dirname "$target")"
|
||||
if docker cp "$CID:/huggingface.co/$asset" "$target" 2>/dev/null; then
|
||||
echo "tokenizer: $asset copied from cached infiniflow/deps image"
|
||||
echo "tokenizer: $asset copied from cached infiniflow/ragflow_deps image"
|
||||
else
|
||||
echo "tokenizer: $asset absent from the cached image — its counter will skip"
|
||||
fi
|
||||
@@ -351,7 +351,7 @@ jobs:
|
||||
docker rm -f "$CID" >/dev/null 2>&1 || true
|
||||
fi
|
||||
else
|
||||
echo "tokenizer: infiniflow/deps not cached on this runner — non-cl100k counters will skip"
|
||||
echo "tokenizer: infiniflow/ragflow_deps not cached on this runner — non-cl100k counters will skip"
|
||||
fi
|
||||
|
||||
# Keep Go module/build caches OFF the runner container's writable layer
|
||||
@@ -832,14 +832,14 @@ jobs:
|
||||
# openai blob first (small ~1.6MB, reachable from the CI runner — the
|
||||
# pre-PR code downloaded it on the fly, so the network path is known
|
||||
# good). If the direct fetch fails (e.g. GFW), fall back to extracting
|
||||
# it from the local infiniflow/deps image *if already present*
|
||||
# it from the local infiniflow/ragflow_deps image *if already present*
|
||||
# on the runner (no `docker pull`, since that huge image is first
|
||||
# fetched at the later "Build ragflow:nightly" step). Best-effort: if
|
||||
# neither source works, the loader still fails loudly (no silent
|
||||
# 0-token degradation).
|
||||
# tiktoken is already installed in runner
|
||||
mkdir -p deps
|
||||
cp /opt/deps/cl100k_base.tiktoken deps/cl100k_base.tiktoken || true
|
||||
cp /opt/ragflow_deps/cl100k_base.tiktoken deps/cl100k_base.tiktoken || true
|
||||
if [ ! -f deps/cl100k_base.tiktoken ]; then
|
||||
mkdir -p deps
|
||||
if command -v curl >/dev/null 2>&1; then
|
||||
@@ -854,7 +854,7 @@ jobs:
|
||||
CID=$(docker create infiniflow/ragflow_deps:latest true) || true
|
||||
if [ -n "${CID:-}" ]; then
|
||||
if docker cp "$CID":/cl100k_base.tiktoken deps/cl100k_base.tiktoken 2>/dev/null; then
|
||||
echo "tiktoken: cl100k table copied from cached infiniflow/deps image"
|
||||
echo "tiktoken: cl100k table copied from cached infiniflow/ragflow_deps image"
|
||||
fi
|
||||
docker rm -f "$CID" >/dev/null 2>&1 || true
|
||||
fi
|
||||
@@ -868,7 +868,7 @@ jobs:
|
||||
|
||||
# Provide the tokenizer assets the four non-cl100k counters load
|
||||
# (deps/huggingface.co/<repo>/<file>). Their only no-network source
|
||||
# is the infiniflow/deps image — the runtime images read them from
|
||||
# is the infiniflow/ragflow_deps image — the runtime images read them from
|
||||
# exactly the same path — so this is the same "if the image is already
|
||||
# cached" fallback the table above uses (again, no `docker pull`). Without
|
||||
# them counter_properties_test / asset_dir_test silently drop their
|
||||
@@ -889,7 +889,7 @@ jobs:
|
||||
fi
|
||||
mkdir -p "$(dirname "$target")"
|
||||
if docker cp "$CID:/huggingface.co/$asset" "$target" 2>/dev/null; then
|
||||
echo "tokenizer: $asset copied from cached infiniflow/deps image"
|
||||
echo "tokenizer: $asset copied from cached infiniflow/ragflow_deps image"
|
||||
else
|
||||
echo "tokenizer: $asset absent from the cached image — its counter will skip"
|
||||
fi
|
||||
@@ -897,7 +897,7 @@ jobs:
|
||||
docker rm -f "$CID" >/dev/null 2>&1 || true
|
||||
fi
|
||||
else
|
||||
echo "tokenizer: infiniflow/deps not cached on this runner — non-cl100k counters will skip"
|
||||
echo "tokenizer: infiniflow/ragflow_deps not cached on this runner — non-cl100k counters will skip"
|
||||
fi
|
||||
|
||||
# Keep Go module/build caches OFF the runner container's writable layer
|
||||
@@ -1789,13 +1789,13 @@ jobs:
|
||||
- name: Provide cl100k_base tiktoken vocabulary
|
||||
# The Go integration tests (tokenizer / service / graph) panic without the
|
||||
# cl100k_base BPE table. The runner already bakes it at
|
||||
# /opt/deps/cl100k_base.tiktoken, so copy from there first; fall
|
||||
# /opt/ragflow_deps/cl100k_base.tiktoken, so copy from there first; fall
|
||||
# back to the upstream openai blob only if the local copy is missing
|
||||
# (e.g. a non-standard runner). Mirrors the ragflow_native_backend job.
|
||||
run: |
|
||||
set -euo pipefail
|
||||
mkdir -p deps
|
||||
cp /opt/deps/cl100k_base.tiktoken deps/cl100k_base.tiktoken || true
|
||||
cp /opt/ragflow_deps/cl100k_base.tiktoken deps/cl100k_base.tiktoken || true
|
||||
if [ ! -f deps/cl100k_base.tiktoken ] && command -v curl >/dev/null 2>&1; then
|
||||
curl -fsSL -o deps/cl100k_base.tiktoken \
|
||||
https://openaipublic.blob.core.windows.net/encodings/cl100k_base.tiktoken || \
|
||||
|
||||
+1
-1
@@ -78,7 +78,7 @@ RUN --mount=type=bind,from=infiniflow/ragflow_deps:latest,source=/huggingface.co
|
||||
mkdir -p "/ragflow/deps/huggingface.co/$(dirname "$asset")" && \
|
||||
cp "/huggingface.co/$asset" "/ragflow/deps/huggingface.co/$asset" ; \
|
||||
else \
|
||||
echo "ERROR: tokenizer asset $asset is missing from the infiniflow/deps image; this image would count with the calibrated estimate instead of the model's own tokenizer" >&2 ; \
|
||||
echo "ERROR: tokenizer asset $asset is missing from the infiniflow/ragflow_deps image; this image would count with the calibrated estimate instead of the model's own tokenizer" >&2 ; \
|
||||
exit 1 ; \
|
||||
fi ; \
|
||||
done
|
||||
|
||||
+1
-1
@@ -40,7 +40,7 @@ RUN --mount=type=bind,from=infiniflow/ragflow_deps:latest,source=/huggingface.co
|
||||
mkdir -p "/ragflow/deps/huggingface.co/$(dirname "$asset")" && \
|
||||
cp "/huggingface.co/$asset" "/ragflow/deps/huggingface.co/$asset" ; \
|
||||
else \
|
||||
echo "ERROR: tokenizer asset $asset is missing from the infiniflow/deps image; this image would count with the calibrated estimate instead of the model's own tokenizer" >&2 ; \
|
||||
echo "ERROR: tokenizer asset $asset is missing from the infiniflow/ragflow_deps image; this image would count with the calibrated estimate instead of the model's own tokenizer" >&2 ; \
|
||||
exit 1 ; \
|
||||
fi ; \
|
||||
done
|
||||
|
||||
@@ -0,0 +1,89 @@
|
||||
"""Guard the two intentional exceptions of the ragflow_deps -> deps rename.
|
||||
|
||||
The rename PR decided to keep (a) the published image tag `infiniflow/ragflow_deps:latest`
|
||||
and (b) the CI install path `/opt/ragflow_deps` unchanged. A blanket
|
||||
`sed s/ragflow_deps/deps/g` accidentally rewrote those too, breaking the CI
|
||||
offline fallback for the cl100k table and leaving misleading tag references.
|
||||
|
||||
This test fails while those exceptions are violated, and stays green once the
|
||||
intentional names are restored. Run standalone: `python3 tools/scripts/test_dep_rename_invariants.py`.
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
|
||||
REPO_ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
|
||||
SEP_TESTS = os.path.join(REPO_ROOT, ".github", "workflows", "sep-tests.yml")
|
||||
DOCKERIGNORE = os.path.join(REPO_ROOT, ".dockerignore")
|
||||
DOCKERFILE = os.path.join(REPO_ROOT, "Dockerfile")
|
||||
DOCKERFILE_BASE = os.path.join(REPO_ROOT, "Dockerfile_base")
|
||||
|
||||
# The bare, wrong forms that the over-broad rename introduced.
|
||||
BARE_OPT_DEPS = "/opt/deps" # must stay /opt/ragflow_deps (CI install path)
|
||||
BARE_IMAGE_TAG = "infiniflow/deps" # must stay infiniflow/ragflow_deps (image tag)
|
||||
|
||||
# The intentional, retained forms that must remain present.
|
||||
KEPT_OPT_DEPS = "/opt/ragflow_deps"
|
||||
KEPT_IMAGE_TAG = "infiniflow/ragflow_deps"
|
||||
|
||||
|
||||
def _read(path):
|
||||
with open(path, "r", encoding="utf-8") as f:
|
||||
return f.read()
|
||||
|
||||
|
||||
def test_sep_tests_keeps_opt_ragflow_deps_path():
|
||||
text = _read(SEP_TESTS)
|
||||
# The CI install path must NOT have been renamed to /opt/deps.
|
||||
assert BARE_OPT_DEPS not in text, f"{SEP_TESTS} still references '{BARE_OPT_DEPS}'; the CI install path must remain '{KEPT_OPT_DEPS}' (intentional rename exception)."
|
||||
# The retained path must actually be present as the cl100k source.
|
||||
assert KEPT_OPT_DEPS in text, f"{SEP_TESTS} is missing '{KEPT_OPT_DEPS}'; the CI cl100k source was lost."
|
||||
|
||||
|
||||
def test_sep_tests_keeps_ragflow_deps_image_tag():
|
||||
text = _read(SEP_TESTS)
|
||||
# Comments/echo strings must reference the real tag, not the bare wrong one.
|
||||
assert BARE_IMAGE_TAG not in text, f"{SEP_TESTS} still references bare '{BARE_IMAGE_TAG}'; the published image tag is '{KEPT_IMAGE_TAG}'."
|
||||
assert KEPT_IMAGE_TAG in text, f"{SEP_TESTS} is missing '{KEPT_IMAGE_TAG}'; the image tag reference was lost."
|
||||
|
||||
|
||||
def test_dockerignore_keeps_ragflow_deps_image_tag():
|
||||
text = _read(DOCKERIGNORE)
|
||||
assert BARE_IMAGE_TAG not in text, f"{DOCKERIGNORE} still references bare '{BARE_IMAGE_TAG}'; the published image tag is '{KEPT_IMAGE_TAG}'."
|
||||
assert KEPT_IMAGE_TAG in text, f"{DOCKERIGNORE} is missing '{KEPT_IMAGE_TAG}'; the image tag reference was lost."
|
||||
|
||||
|
||||
def test_dockerfiles_keep_ragflow_deps_image_tag():
|
||||
for path in (DOCKERFILE, DOCKERFILE_BASE):
|
||||
text = _read(path)
|
||||
# Error-message echo strings must reference the real tag, not the bare wrong one.
|
||||
assert BARE_IMAGE_TAG not in text, f"{path} still references bare '{BARE_IMAGE_TAG}'; the published image tag is '{KEPT_IMAGE_TAG}'."
|
||||
# The mount source (infiniflow/ragflow_deps:latest) must be present.
|
||||
assert KEPT_IMAGE_TAG in text, f"{path} is missing '{KEPT_IMAGE_TAG}'; the image tag reference was lost."
|
||||
|
||||
|
||||
def main():
|
||||
tests = [
|
||||
test_sep_tests_keeps_opt_ragflow_deps_path,
|
||||
test_sep_tests_keeps_ragflow_deps_image_tag,
|
||||
test_dockerignore_keeps_ragflow_deps_image_tag,
|
||||
test_dockerfiles_keep_ragflow_deps_image_tag,
|
||||
]
|
||||
failed = 0
|
||||
for t in tests:
|
||||
try:
|
||||
t()
|
||||
print(f"PASS {t.__name__}")
|
||||
except AssertionError as e:
|
||||
failed += 1
|
||||
print(f"FAIL {t.__name__}: {e}")
|
||||
if failed:
|
||||
print(f"\n{failed} test(s) failed.")
|
||||
return 1
|
||||
print("\nAll rename-invariant checks passed.")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Reference in new issue
Block a user