diff --git a/.dockerignore b/.dockerignore index 4a950666c3..028801cb52 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,7 +1,7 @@ # RAGFlow .dockerignore # Reduces Docker build context sent to the daemon. # All excluded items are either rebuilt inside Docker, mounted from -# infiniflow/deps, or are local-only artifacts. +# infiniflow/ragflow_deps, or are local-only artifacts. # ── Python virtual environments ───────────────────────────────────────────── .venv/ @@ -37,7 +37,7 @@ target/ # Excluding the directory contents keeps the main build context small # regardless of which resources are prepared for the dependency image. # The deps image is built from this directory with: -# cd deps && docker build -f Dockerfile -t infiniflow/deps . +# cd deps && docker build -f Dockerfile -t infiniflow/ragflow_deps . # The native downloader tools/scripts/download_dev_deps.py is copied by the # main Dockerfile (not excluded here) so build.sh can read its ORT version pin # when running `./build.sh --go`. diff --git a/.github/workflows/sep-tests.yml b/.github/workflows/sep-tests.yml index e71ae5a0cf..229af7d952 100644 --- a/.github/workflows/sep-tests.yml +++ b/.github/workflows/sep-tests.yml @@ -286,14 +286,14 @@ jobs: # openai blob first (small ~1.6MB, reachable from the CI runner — the # pre-PR code downloaded it on the fly, so the network path is known # good). If the direct fetch fails (e.g. GFW), fall back to extracting - # it from the local infiniflow/deps image *if already present* + # it from the local infiniflow/ragflow_deps image *if already present* # on the runner (no `docker pull`, since that huge image is first # fetched at the later "Build ragflow:nightly" step). Best-effort: if # neither source works, the loader still fails loudly (no silent # 0-token degradation). # tiktoken is already installed in runner mkdir -p deps - cp /opt/deps/cl100k_base.tiktoken deps/cl100k_base.tiktoken || true + cp /opt/ragflow_deps/cl100k_base.tiktoken deps/cl100k_base.tiktoken || true if [ ! -f deps/cl100k_base.tiktoken ]; then mkdir -p deps if command -v curl >/dev/null 2>&1; then @@ -308,7 +308,7 @@ jobs: CID=$(docker create infiniflow/ragflow_deps:latest true) || true if [ -n "${CID:-}" ]; then if docker cp "$CID":/cl100k_base.tiktoken deps/cl100k_base.tiktoken 2>/dev/null; then - echo "tiktoken: cl100k table copied from cached infiniflow/deps image" + echo "tiktoken: cl100k table copied from cached infiniflow/ragflow_deps image" fi docker rm -f "$CID" >/dev/null 2>&1 || true fi @@ -322,7 +322,7 @@ jobs: # Provide the tokenizer assets the four non-cl100k counters load # (deps/huggingface.co//). Their only no-network source - # is the infiniflow/deps image — the runtime images read them from + # is the infiniflow/ragflow_deps image — the runtime images read them from # exactly the same path — so this is the same "if the image is already # cached" fallback the table above uses (again, no `docker pull`). Without # them counter_properties_test / asset_dir_test silently drop their @@ -343,7 +343,7 @@ jobs: fi mkdir -p "$(dirname "$target")" if docker cp "$CID:/huggingface.co/$asset" "$target" 2>/dev/null; then - echo "tokenizer: $asset copied from cached infiniflow/deps image" + echo "tokenizer: $asset copied from cached infiniflow/ragflow_deps image" else echo "tokenizer: $asset absent from the cached image — its counter will skip" fi @@ -351,7 +351,7 @@ jobs: docker rm -f "$CID" >/dev/null 2>&1 || true fi else - echo "tokenizer: infiniflow/deps not cached on this runner — non-cl100k counters will skip" + echo "tokenizer: infiniflow/ragflow_deps not cached on this runner — non-cl100k counters will skip" fi # Keep Go module/build caches OFF the runner container's writable layer @@ -832,14 +832,14 @@ jobs: # openai blob first (small ~1.6MB, reachable from the CI runner — the # pre-PR code downloaded it on the fly, so the network path is known # good). If the direct fetch fails (e.g. GFW), fall back to extracting - # it from the local infiniflow/deps image *if already present* + # it from the local infiniflow/ragflow_deps image *if already present* # on the runner (no `docker pull`, since that huge image is first # fetched at the later "Build ragflow:nightly" step). Best-effort: if # neither source works, the loader still fails loudly (no silent # 0-token degradation). # tiktoken is already installed in runner mkdir -p deps - cp /opt/deps/cl100k_base.tiktoken deps/cl100k_base.tiktoken || true + cp /opt/ragflow_deps/cl100k_base.tiktoken deps/cl100k_base.tiktoken || true if [ ! -f deps/cl100k_base.tiktoken ]; then mkdir -p deps if command -v curl >/dev/null 2>&1; then @@ -854,7 +854,7 @@ jobs: CID=$(docker create infiniflow/ragflow_deps:latest true) || true if [ -n "${CID:-}" ]; then if docker cp "$CID":/cl100k_base.tiktoken deps/cl100k_base.tiktoken 2>/dev/null; then - echo "tiktoken: cl100k table copied from cached infiniflow/deps image" + echo "tiktoken: cl100k table copied from cached infiniflow/ragflow_deps image" fi docker rm -f "$CID" >/dev/null 2>&1 || true fi @@ -868,7 +868,7 @@ jobs: # Provide the tokenizer assets the four non-cl100k counters load # (deps/huggingface.co//). Their only no-network source - # is the infiniflow/deps image — the runtime images read them from + # is the infiniflow/ragflow_deps image — the runtime images read them from # exactly the same path — so this is the same "if the image is already # cached" fallback the table above uses (again, no `docker pull`). Without # them counter_properties_test / asset_dir_test silently drop their @@ -889,7 +889,7 @@ jobs: fi mkdir -p "$(dirname "$target")" if docker cp "$CID:/huggingface.co/$asset" "$target" 2>/dev/null; then - echo "tokenizer: $asset copied from cached infiniflow/deps image" + echo "tokenizer: $asset copied from cached infiniflow/ragflow_deps image" else echo "tokenizer: $asset absent from the cached image — its counter will skip" fi @@ -897,7 +897,7 @@ jobs: docker rm -f "$CID" >/dev/null 2>&1 || true fi else - echo "tokenizer: infiniflow/deps not cached on this runner — non-cl100k counters will skip" + echo "tokenizer: infiniflow/ragflow_deps not cached on this runner — non-cl100k counters will skip" fi # Keep Go module/build caches OFF the runner container's writable layer @@ -1789,13 +1789,13 @@ jobs: - name: Provide cl100k_base tiktoken vocabulary # The Go integration tests (tokenizer / service / graph) panic without the # cl100k_base BPE table. The runner already bakes it at - # /opt/deps/cl100k_base.tiktoken, so copy from there first; fall + # /opt/ragflow_deps/cl100k_base.tiktoken, so copy from there first; fall # back to the upstream openai blob only if the local copy is missing # (e.g. a non-standard runner). Mirrors the ragflow_native_backend job. run: | set -euo pipefail mkdir -p deps - cp /opt/deps/cl100k_base.tiktoken deps/cl100k_base.tiktoken || true + cp /opt/ragflow_deps/cl100k_base.tiktoken deps/cl100k_base.tiktoken || true if [ ! -f deps/cl100k_base.tiktoken ] && command -v curl >/dev/null 2>&1; then curl -fsSL -o deps/cl100k_base.tiktoken \ https://openaipublic.blob.core.windows.net/encodings/cl100k_base.tiktoken || \ diff --git a/Dockerfile b/Dockerfile index 51452df048..c4820f08f2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -78,7 +78,7 @@ RUN --mount=type=bind,from=infiniflow/ragflow_deps:latest,source=/huggingface.co mkdir -p "/ragflow/deps/huggingface.co/$(dirname "$asset")" && \ cp "/huggingface.co/$asset" "/ragflow/deps/huggingface.co/$asset" ; \ else \ - echo "ERROR: tokenizer asset $asset is missing from the infiniflow/deps image; this image would count with the calibrated estimate instead of the model's own tokenizer" >&2 ; \ + echo "ERROR: tokenizer asset $asset is missing from the infiniflow/ragflow_deps image; this image would count with the calibrated estimate instead of the model's own tokenizer" >&2 ; \ exit 1 ; \ fi ; \ done diff --git a/Dockerfile_base b/Dockerfile_base index 64b0a24068..a84ad82a55 100644 --- a/Dockerfile_base +++ b/Dockerfile_base @@ -40,7 +40,7 @@ RUN --mount=type=bind,from=infiniflow/ragflow_deps:latest,source=/huggingface.co mkdir -p "/ragflow/deps/huggingface.co/$(dirname "$asset")" && \ cp "/huggingface.co/$asset" "/ragflow/deps/huggingface.co/$asset" ; \ else \ - echo "ERROR: tokenizer asset $asset is missing from the infiniflow/deps image; this image would count with the calibrated estimate instead of the model's own tokenizer" >&2 ; \ + echo "ERROR: tokenizer asset $asset is missing from the infiniflow/ragflow_deps image; this image would count with the calibrated estimate instead of the model's own tokenizer" >&2 ; \ exit 1 ; \ fi ; \ done diff --git a/tools/scripts/test_dep_rename_invariants.py b/tools/scripts/test_dep_rename_invariants.py new file mode 100644 index 0000000000..b88fa58d74 --- /dev/null +++ b/tools/scripts/test_dep_rename_invariants.py @@ -0,0 +1,89 @@ +"""Guard the two intentional exceptions of the ragflow_deps -> deps rename. + +The rename PR decided to keep (a) the published image tag `infiniflow/ragflow_deps:latest` +and (b) the CI install path `/opt/ragflow_deps` unchanged. A blanket +`sed s/ragflow_deps/deps/g` accidentally rewrote those too, breaking the CI +offline fallback for the cl100k table and leaving misleading tag references. + +This test fails while those exceptions are violated, and stays green once the +intentional names are restored. Run standalone: `python3 tools/scripts/test_dep_rename_invariants.py`. +""" + +import os +import sys + +REPO_ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +SEP_TESTS = os.path.join(REPO_ROOT, ".github", "workflows", "sep-tests.yml") +DOCKERIGNORE = os.path.join(REPO_ROOT, ".dockerignore") +DOCKERFILE = os.path.join(REPO_ROOT, "Dockerfile") +DOCKERFILE_BASE = os.path.join(REPO_ROOT, "Dockerfile_base") + +# The bare, wrong forms that the over-broad rename introduced. +BARE_OPT_DEPS = "/opt/deps" # must stay /opt/ragflow_deps (CI install path) +BARE_IMAGE_TAG = "infiniflow/deps" # must stay infiniflow/ragflow_deps (image tag) + +# The intentional, retained forms that must remain present. +KEPT_OPT_DEPS = "/opt/ragflow_deps" +KEPT_IMAGE_TAG = "infiniflow/ragflow_deps" + + +def _read(path): + with open(path, "r", encoding="utf-8") as f: + return f.read() + + +def test_sep_tests_keeps_opt_ragflow_deps_path(): + text = _read(SEP_TESTS) + # The CI install path must NOT have been renamed to /opt/deps. + assert BARE_OPT_DEPS not in text, f"{SEP_TESTS} still references '{BARE_OPT_DEPS}'; the CI install path must remain '{KEPT_OPT_DEPS}' (intentional rename exception)." + # The retained path must actually be present as the cl100k source. + assert KEPT_OPT_DEPS in text, f"{SEP_TESTS} is missing '{KEPT_OPT_DEPS}'; the CI cl100k source was lost." + + +def test_sep_tests_keeps_ragflow_deps_image_tag(): + text = _read(SEP_TESTS) + # Comments/echo strings must reference the real tag, not the bare wrong one. + assert BARE_IMAGE_TAG not in text, f"{SEP_TESTS} still references bare '{BARE_IMAGE_TAG}'; the published image tag is '{KEPT_IMAGE_TAG}'." + assert KEPT_IMAGE_TAG in text, f"{SEP_TESTS} is missing '{KEPT_IMAGE_TAG}'; the image tag reference was lost." + + +def test_dockerignore_keeps_ragflow_deps_image_tag(): + text = _read(DOCKERIGNORE) + assert BARE_IMAGE_TAG not in text, f"{DOCKERIGNORE} still references bare '{BARE_IMAGE_TAG}'; the published image tag is '{KEPT_IMAGE_TAG}'." + assert KEPT_IMAGE_TAG in text, f"{DOCKERIGNORE} is missing '{KEPT_IMAGE_TAG}'; the image tag reference was lost." + + +def test_dockerfiles_keep_ragflow_deps_image_tag(): + for path in (DOCKERFILE, DOCKERFILE_BASE): + text = _read(path) + # Error-message echo strings must reference the real tag, not the bare wrong one. + assert BARE_IMAGE_TAG not in text, f"{path} still references bare '{BARE_IMAGE_TAG}'; the published image tag is '{KEPT_IMAGE_TAG}'." + # The mount source (infiniflow/ragflow_deps:latest) must be present. + assert KEPT_IMAGE_TAG in text, f"{path} is missing '{KEPT_IMAGE_TAG}'; the image tag reference was lost." + + +def main(): + tests = [ + test_sep_tests_keeps_opt_ragflow_deps_path, + test_sep_tests_keeps_ragflow_deps_image_tag, + test_dockerignore_keeps_ragflow_deps_image_tag, + test_dockerfiles_keep_ragflow_deps_image_tag, + ] + failed = 0 + for t in tests: + try: + t() + print(f"PASS {t.__name__}") + except AssertionError as e: + failed += 1 + print(f"FAIL {t.__name__}: {e}") + if failed: + print(f"\n{failed} test(s) failed.") + return 1 + print("\nAll rename-invariant checks passed.") + return 0 + + +if __name__ == "__main__": + sys.exit(main())