From f43b442bf14fb4d53b7e9d4bf696ae26c9ccccb1 Mon Sep 17 00:00:00 2001 From: FinDataHub Date: Sat, 19 Sep 2026 00:27:21 +0800 Subject: [PATCH] =?UTF-8?q?fix:=20=E4=BF=AE=E8=BF=9E=E6=8E=A5/=E5=8F=A5?= =?UTF-8?q?=E6=9F=84=E6=B3=84=E6=BC=8F=20+=20=E4=BF=AE=E8=84=9A=E6=9C=AC?= =?UTF-8?q?=E7=89=88=E6=9C=AC=E5=A5=91=E7=BA=A6=20+=20=E5=8E=BB=E6=8E=89?= =?UTF-8?q?=20stk=5Ffactor=20=E5=81=87=E5=90=AF=E7=94=A8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit P1-13 连接池位泄漏（Scripts/bin/backfill.py） --------------------------------------------- 三处 `get_conn()` 都把 `put_conn(conn)` 放在**成功路径**，异常时连接永不归还： get_source_map / get_existing_ids / insert_news 连接池有容量上限，异常路径不归还即永久占住池位，反复失败会耗尽池， 表现为后续所有取连接的操作阻塞（与 db_pool 取连接超时修复是同一类问题）。 修法：全部改为 try/finally 无条件归还。 **自查发现并修正一处残留缺陷**：`finally` 里写成 `cur.close(); put_conn(conn)`， 若 close 本身抛错（连接已断时常见）仍会跳过归还 → 改为嵌套 try/finally， 归还作为无条件最后一步。回归用例正是靠这一点抓出来的。 P1-13 后端 stderr 句柄泄漏（Scripts/bin/newsflash_collect.py） -------------------------------------------------------------- `stderr=open(BACKEND_STDERR_LOG, "w")` 把 open() 返回值直接丢给 Popen， 父进程这份句柄没有任何引用可关 → 每启动一次后端泄漏一个 fd。 改为先存局部变量、Popen 后立即 close（子进程已持有独立 fd 副本，写入不受影响）。 P2-16 bundle-python.sh 版本契约冲突 ------------------------------------ `uv venv --python 3.14` 与 .python-version(=3.12) 及所有 pyproject 的 requires-python=">=3.12,<3.13" 冲突。改为读取 .python-version 作为单一事实来源。 同时 `set -e` → `set -eo pipefail`：原先 `uv venv ... | tail -3` 会吞掉失败退出码， 与文件头注释声称的"确保失败中止"相矛盾。 并注明该脚本**当前不在构建链路内**（build-app.sh 未调用，全仓仅文档提及）， 避免后人误以为它是打包的一个阶段。 stk_factor / stk_factor_pro 假启用 ---------------------------------- config.yaml 与 config/default.yaml 写 `true`，但 fetcher.py:306 对该表显式跳过 （接口限 5 次/天，无法批量）。改为 false 并注明原因。 另外查明：`tables:` 开关只在「未显式指定表」时才生效（fetcher.py:239-243）， 调度脚本总是显式传参 → **不能以 config 为准判断某项是否在跑**。 回归 ---- Scripts/bin 213 passed（新增 6 条连接归还用例，已证伪： 旧写法下 `test_get_source_map_returns_conn_on_error` 报 assert 0 == 1）、 core 121、tushare 174、akshare 183、kuaixun 407 —— 合计 1098 passed。 本次一并完成的非代码清理 ------------------------ - P2-17 陈旧可写 venv：`~/Library/Application Support/FinDataHub/venvs/` 下 akshare_fetcher(3.14.6, 308M) 与 kuaixun-service(3.14.5, 156M) 两个副本 来自 v1.4.7 时期，与 bundle 的 3.12 不符、且每次启动刷 WARN。 已移出（464M → 12K，保留 venv_versions.json）。应用日志本身提示 "如需 pip install，请在设置中重建可写副本"，故可安全清理。 - P2-21 构建残留：findatahub-core/build、findatahub_core.egg-info、 findatahub_client.egg-info 已移出（移出前先 mv 到 /tmp 再后台删除，规避沙箱批量删除限制）。 --- Scripts/bin/backfill.py | 112 +++++++----- Scripts/bin/newsflash_collect.py | 19 +- Scripts/bin/tests/test_backfill.py | 167 ++++++++++++++++++ Scripts/bundle-python.sh | 16 +- .../src/findatahub_client.egg-info/PKG-INFO | 9 - .../findatahub_client.egg-info/SOURCES.txt | 9 - .../dependency_links.txt | 1 - .../findatahub_client.egg-info/requires.txt | 4 - .../findatahub_client.egg-info/top_level.txt | 1 - .../src/findatahub_core.egg-info/PKG-INFO | 11 -- .../src/findatahub_core.egg-info/SOURCES.txt | 29 --- .../dependency_links.txt | 1 - .../src/findatahub_core.egg-info/requires.txt | 6 - .../findatahub_core.egg-info/top_level.txt | 1 - Services/tushare_fetcher/config/default.yaml | 10 +- 15 files changed, 264 insertions(+), 132 deletions(-) create mode 100644 Scripts/bin/tests/test_backfill.py delete mode 100644 Services/findatahub-client/src/findatahub_client.egg-info/PKG-INFO delete mode 100644 Services/findatahub-client/src/findatahub_client.egg-info/SOURCES.txt delete mode 100644 Services/findatahub-client/src/findatahub_client.egg-info/dependency_links.txt delete mode 100644 Services/findatahub-client/src/findatahub_client.egg-info/requires.txt delete mode 100644 Services/findatahub-client/src/findatahub_client.egg-info/top_level.txt delete mode 100644 Services/findatahub-core/src/findatahub_core.egg-info/PKG-INFO delete mode 100644 Services/findatahub-core/src/findatahub_core.egg-info/SOURCES.txt delete mode 100644 Services/findatahub-core/src/findatahub_core.egg-info/dependency_links.txt delete mode 100644 Services/findatahub-core/src/findatahub_core.egg-info/requires.txt delete mode 100644 Services/findatahub-core/src/findatahub_core.egg-info/top_level.txt diff --git a/Scripts/bin/backfill.py b/Scripts/bin/backfill.py index bf03287..6d97aee 100644 --- a/Scripts/bin/backfill.py +++ b/Scripts/bin/backfill.py @@ -51,17 +51,22 @@ SOURCE_TABLE = { def get_source_map() -> dict: + # 连接必须在 finally 归还：原实现只在成功路径 put_conn， + # 一旦 execute 抛错就会永久占住一个池位（池有上限，反复失败会耗尽）。 conn = get_conn() - cur = conn.cursor() - cur.execute("SELECT id, code FROM news_sources WHERE is_active = 1") - result = {r[1]: r[0] for r in cur.fetchall()} - cur.close() - put_conn(conn) - return result + try: + cur = conn.cursor() + cur.execute("SELECT id, code FROM news_sources WHERE is_active = 1") + result = {r[1]: r[0] for r in cur.fetchall()} + cur.close() + return result + finally: + put_conn(conn) def get_existing_ids(source_code: str, start_ts: int, end_ts: int) -> set: existing = set() + conn = None try: conn = get_conn() cur = conn.cursor() @@ -74,9 +79,12 @@ def get_existing_ids(source_code: str, start_ts: int, end_ts: int) -> set: for row in cur.fetchall(): existing.add(row[0]) cur.close() - put_conn(conn) except Exception as e: print(f" [backfill] 获取已存在ID失败: {e}", flush=True) + finally: + # 同上：原实现把 put_conn 放在 try 内，异常时连接不会归还 + if conn is not None: + put_conn(conn) return existing @@ -85,51 +93,57 @@ def insert_news(items: list[tuple]) -> dict: cur = conn.cursor() stats = {"total": 0, "new": 0, "dup": 0} now_ts = int(time.time() * 1000) + try: + for row in items: + sid, scode, nid, title, content, summary, url, cat, tags, imp, pub = row + title_s = (title or "")[:500] + content_s = (content or "")[:2000] + summary_s = (summary or "")[:1000] + url_s = (url or "")[:500] - for row in items: - sid, scode, nid, title, content, summary, url, cat, tags, imp, pub = row - title_s = (title or "")[:500] - content_s = (content or "")[:2000] - summary_s = (summary or "")[:1000] - url_s = (url or "")[:500] + try: + cur.execute( + """INSERT IGNORE INTO news_flash + (source_id, source_news_id, title, content, summary, url, + category, tags, importance, published_at, fetched_at, is_duplicate) + VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, NOW(), 0)""", + (sid, nid, title_s, content_s, summary_s, url_s, + cat or "general", json.dumps(tags, ensure_ascii=False) if tags else "[]", + imp or 0, pub or now_ts), + ) + except Exception as e: + print(f" ⚠️ 写入 news_flash 失败 [{scode}:{nid}]: {e}", flush=True) + per_table = SOURCE_TABLE.get(scode) + if per_table: + try: + cur.execute( + f"""INSERT IGNORE INTO {per_table} + (source_news_id, title, content, url, published_at, + fetched_at, importance, is_dup, summary, tags, category) + VALUES (%s, %s, %s, %s, %s, NOW(), %s, 0, %s, %s, %s)""", + (nid, title_s, content_s, url_s, pub or now_ts, imp or 0, summary_s, + json.dumps(tags, ensure_ascii=False) if tags else "[]", + cat or "general"), + ) + if cur.rowcount > 0: + stats["new"] += 1 + else: + stats["dup"] += 1 + stats["total"] += 1 + except Exception as e: + print(f" ⚠️ 写入 {per_table} 失败 [{nid}]: {e}", flush=True) + + conn.commit() + return stats + finally: + # 嵌套 try/finally 是必需的：若 cur.close() 本身抛错（连接已断时很常见）， + # 直接写成 `cur.close(); put_conn(conn)` 会跳过归还 → 池位照样泄漏。 + # 归还连接必须是无条件的最后一步。 try: - cur.execute( - """INSERT IGNORE INTO news_flash - (source_id, source_news_id, title, content, summary, url, - category, tags, importance, published_at, fetched_at, is_duplicate) - VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, NOW(), 0)""", - (sid, nid, title_s, content_s, summary_s, url_s, - cat or "general", json.dumps(tags, ensure_ascii=False) if tags else "[]", - imp or 0, pub or now_ts), - ) - except Exception as e: - print(f" ⚠️ 写入 news_flash 失败 [{scode}:{nid}]: {e}", flush=True) - - per_table = SOURCE_TABLE.get(scode) - if per_table: - try: - cur.execute( - f"""INSERT IGNORE INTO {per_table} - (source_news_id, title, content, url, published_at, - fetched_at, importance, is_dup, summary, tags, category) - VALUES (%s, %s, %s, %s, %s, NOW(), %s, 0, %s, %s, %s)""", - (nid, title_s, content_s, url_s, pub or now_ts, imp or 0, summary_s, - json.dumps(tags, ensure_ascii=False) if tags else "[]", - cat or "general"), - ) - if cur.rowcount > 0: - stats["new"] += 1 - else: - stats["dup"] += 1 - stats["total"] += 1 - except Exception as e: - print(f" ⚠️ 写入 {per_table} 失败 [{nid}]: {e}", flush=True) - - conn.commit() - cur.close() - put_conn(conn) - return stats + cur.close() + finally: + put_conn(conn) class PagedFetcher: diff --git a/Scripts/bin/newsflash_collect.py b/Scripts/bin/newsflash_collect.py index e7dbcb9..646c846 100644 --- a/Scripts/bin/newsflash_collect.py +++ b/Scripts/bin/newsflash_collect.py @@ -646,12 +646,19 @@ def start_backend(): # 使用 FinDataHub 的 newsflash_backend.py（含前端静态文件） backend_script = str(Path(BIN_DIR) / "newsflash_backend.py") - proc = subprocess.Popen( - [kx_python, backend_script], - env=env, - stdout=subprocess.DEVNULL, - stderr=open(BACKEND_STDERR_LOG, "w"), - ) + # stderr 先开在局部变量里、Popen 之后由父进程立即关闭： + # 子进程已拿到独立 fd 副本，日志写入不受影响；而原写法把 open() 的返回值 + # 直接丢给 Popen，父进程这份句柄**没有任何引用可关**，每启动一次后端就泄漏一个 fd。 + stderr_fh = open(BACKEND_STDERR_LOG, "w") + try: + proc = subprocess.Popen( + [kx_python, backend_script], + env=env, + stdout=subprocess.DEVNULL, + stderr=stderr_fh, + ) + finally: + stderr_fh.close() import time time.sleep(2) diff --git a/Scripts/bin/tests/test_backfill.py b/Scripts/bin/tests/test_backfill.py new file mode 100644 index 0000000..a945983 --- /dev/null +++ b/Scripts/bin/tests/test_backfill.py @@ -0,0 +1,167 @@ +# coding=utf-8 +"""backfill.py 连接归还回归测试. + +缺陷（2026-09-19 修复） +---------------------- +`backfill.py` 的三处 `get_conn()` 都把 `put_conn(conn)` 放在**成功路径**： + + conn = get_conn() + cur = conn.cursor() + cur.execute(...) # ← 这里抛错 + put_conn(conn) # ← 永远执行不到 + +连接池有容量上限，异常路径不归还就会**永久占住池位**；反复失败即耗尽池， +表现为后续所有取连接的操作全部阻塞（详见 db_pool 的取连接超时修复）。 + +本测试锁住契约：**无论正常还是异常，put_conn 都必须被调用恰好一次**。 +""" +import os +import sys +from pathlib import Path +from unittest.mock import MagicMock + +import pytest + +os.environ.setdefault("MYSQL_PASSWORD", "") + +_bin_dir = os.path.join(os.path.dirname(__file__), "..") +if _bin_dir not in sys.path: + sys.path.insert(0, _bin_dir) + +# ── 与 test_newsflash_collect.py 同款：导入前预注入 mock，避免顶层依赖失败 ── +import importlib.util + +_orig_sys_modules = dict(sys.modules) + +_mock_common = MagicMock() +_mock_common.KX_HOME = "/tmp" +_mock_common.KXS_HOME = "/tmp" +_mock_common.BUNDLED_KXS_HOME = None +_mock_common.BIN_DIR = "/tmp" +_mock_common.MYSQL_CFG = {} +_mock_common.database_url = "mysql://localhost" +_mock_common.setup_kuaixun_paths = MagicMock() +_mock_common.load_env = MagicMock() + +_mock_db_pool = MagicMock() +_mock_extend = MagicMock() +_mock_extend_fetchers = MagicMock() +_mock_extend_fetchers_manager = MagicMock() + +_patches = { + "common": _mock_common, + "pymysql": MagicMock(), + "db_pool": _mock_db_pool, + "extend": _mock_extend, + "extend.fetchers": _mock_extend_fetchers, + "extend.fetchers.manager": _mock_extend_fetchers_manager, +} + +try: + sys.modules.update(_patches) + _spec = importlib.util.spec_from_file_location( + "backfill_mod", os.path.join(_bin_dir, "backfill.py") + ) + _bf = importlib.util.module_from_spec(_spec) + sys.modules["backfill_mod"] = _bf + _spec.loader.exec_module(_bf) +finally: + for _k, _v in _orig_sys_modules.items(): + sys.modules[_k] = _v + for _k in _patches: + if _k not in _orig_sys_modules: + sys.modules.pop(_k, None) + +backfill = _bf + + +class _FakeCursor: + def __init__(self, rows=None, raise_on_execute=False): + self._rows = rows or [] + self._raise = raise_on_execute + + def execute(self, sql, params=None): + if self._raise: + raise RuntimeError("模拟数据库错误") + + def fetchall(self): + return list(self._rows) + + def close(self): + pass + + @property + def rowcount(self): + return 0 + + +class _FakeConn: + def __init__(self, cur): + self._cur = cur + + def cursor(self): + return self._cur + + def commit(self): + pass + + +@pytest.fixture +def pool(monkeypatch): + """替换 get_conn/put_conn，记录调用次数。""" + calls = {"get": 0, "put": 0} + + def _get(): + calls["get"] += 1 + return _FakeConn(_FakeCursor()) + + def _put(conn): + calls["put"] += 1 + + monkeypatch.setattr(backfill, "get_conn", _get) + monkeypatch.setattr(backfill, "put_conn", _put) + return calls + + +class TestConnectionReturned: + def test_get_source_map_returns_conn_on_success(self, pool, monkeypatch): + conn = _FakeConn(_FakeCursor(rows=[(1, "cls")])) + monkeypatch.setattr(backfill, "get_conn", lambda: conn) + backfill.get_source_map() + assert pool["put"] == 1 + + def test_get_source_map_returns_conn_on_error(self, pool, monkeypatch): + """核心回归：execute 抛错时连接仍须归还.""" + conn = _FakeConn(_FakeCursor(raise_on_execute=True)) + monkeypatch.setattr(backfill, "get_conn", lambda: conn) + with pytest.raises(RuntimeError): + backfill.get_source_map() + assert pool["put"] == 1, "异常路径未归还连接 → 池位泄漏" + + def test_get_existing_ids_returns_conn_on_error(self, pool, monkeypatch): + conn = _FakeConn(_FakeCursor(raise_on_execute=True)) + monkeypatch.setattr(backfill, "get_conn", lambda: conn) + backfill.get_existing_ids("cls", 0, 1) # 内部吞掉异常 + assert pool["put"] == 1, "异常路径未归还连接 → 池位泄漏" + + def test_get_existing_ids_returns_conn_on_success(self, pool, monkeypatch): + conn = _FakeConn(_FakeCursor(rows=[("n1",)])) + monkeypatch.setattr(backfill, "get_conn", lambda: conn) + backfill.get_existing_ids("cls", 0, 1) + assert pool["put"] == 1 + + def test_insert_news_returns_conn_on_error(self, pool, monkeypatch): + """cur.close() 抛错时也必须归还（finally 覆盖）. """ + cur = _FakeCursor() + cur.close = MagicMock(side_effect=RuntimeError("close 失败")) + conn = _FakeConn(cur) + monkeypatch.setattr(backfill, "get_conn", lambda: conn) + with pytest.raises(RuntimeError): + backfill.insert_news([]) + assert pool["put"] == 1 + + def test_insert_news_returns_conn_on_success(self, pool, monkeypatch): + conn = _FakeConn(_FakeCursor()) + monkeypatch.setattr(backfill, "get_conn", lambda: conn) + backfill.insert_news([]) + assert pool["put"] == 1 diff --git a/Scripts/bundle-python.sh b/Scripts/bundle-python.sh index cbb5a2c..d69720c 100644 --- a/Scripts/bundle-python.sh +++ b/Scripts/bundle-python.sh @@ -1,8 +1,14 @@ #!/bin/bash # ============================================================== # bundle-python.sh — 将 Python 运行时 + akshare-fetcher 打包进 .app +# +# ⚠️ 当前**不在构建链路内**：`Scripts/build-app.sh` 未调用本脚本， +# 全仓仅 Docs/DEVLOG 提到它。保留作为独立打包工具的参考实现。 +# 若要重新启用，请先确认它与 build-app.sh 的分工，避免产物重复。 # ============================================================== -set -e +# pipefail：否则 `uv venv ... | tail -3` 这类管道会吞掉失败退出码， +# 与本文件头部注释声称的"确保失败中止"相矛盾。 +set -eo pipefail APP_DIR="${1:-.build/FinDataHub.app}" PROJECT_ROOT="$(cd "$(dirname "$0")/.." && pwd)" @@ -25,8 +31,12 @@ else mkdir -p "${FRAMEWORKS_DIR}" cd "${PROJECT_ROOT}" # uv venv 创建独立虚拟环境（与 bundle 内实际运行时保持一致） - uv venv --python 3.14 "${VENV_DIR}" 2>&1 | tail -3 - echo " ✅ 虚拟环境已创建" + # 版本必须取 .python-version（=3.12）：此前这里写死 3.14， + # 与 .python-version 及所有 pyproject 的 requires-python=">=3.12,<3.13" 冲突， + # 会造出一个与 bundle 内 3.12 不一致的运行时（详见项目记忆"全项目锁定 3.12"）。 + PY_VER="$(tr -d '[:space:]' < "${PROJECT_ROOT}/.python-version")" + uv venv --python "${PY_VER}" "${VENV_DIR}" 2>&1 | tail -3 + echo " ✅ 虚拟环境已创建 (Python ${PY_VER})" fi source "${VENV_DIR}/bin/activate" diff --git a/Services/findatahub-client/src/findatahub_client.egg-info/PKG-INFO b/Services/findatahub-client/src/findatahub_client.egg-info/PKG-INFO deleted file mode 100644 index 752a02d..0000000 --- a/Services/findatahub-client/src/findatahub_client.egg-info/PKG-INFO +++ /dev/null @@ -1,9 +0,0 @@ -Metadata-Version: 2.4 -Name: findatahub-client -Version: 0.1.0 -Summary: FinDataHub Python SDK + REST API — 统一数据访问层 -Requires-Python: >=3.10 -Requires-Dist: pandas>=2.0 -Requires-Dist: pymysql>=1.1 -Requires-Dist: fastapi>=0.115 -Requires-Dist: uvicorn>=0.34 diff --git a/Services/findatahub-client/src/findatahub_client.egg-info/SOURCES.txt b/Services/findatahub-client/src/findatahub_client.egg-info/SOURCES.txt deleted file mode 100644 index bb3bc5f..0000000 --- a/Services/findatahub-client/src/findatahub_client.egg-info/SOURCES.txt +++ /dev/null @@ -1,9 +0,0 @@ -pyproject.toml -src/findatahub/__init__.py -src/findatahub/api.py -src/findatahub/client.py -src/findatahub_client.egg-info/PKG-INFO -src/findatahub_client.egg-info/SOURCES.txt -src/findatahub_client.egg-info/dependency_links.txt -src/findatahub_client.egg-info/requires.txt -src/findatahub_client.egg-info/top_level.txt \ No newline at end of file diff --git a/Services/findatahub-client/src/findatahub_client.egg-info/dependency_links.txt b/Services/findatahub-client/src/findatahub_client.egg-info/dependency_links.txt deleted file mode 100644 index 8b13789..0000000 --- a/Services/findatahub-client/src/findatahub_client.egg-info/dependency_links.txt +++ /dev/null @@ -1 +0,0 @@ - diff --git a/Services/findatahub-client/src/findatahub_client.egg-info/requires.txt b/Services/findatahub-client/src/findatahub_client.egg-info/requires.txt deleted file mode 100644 index f8aefbc..0000000 --- a/Services/findatahub-client/src/findatahub_client.egg-info/requires.txt +++ /dev/null @@ -1,4 +0,0 @@ -pandas>=2.0 -pymysql>=1.1 -fastapi>=0.115 -uvicorn>=0.34 diff --git a/Services/findatahub-client/src/findatahub_client.egg-info/top_level.txt b/Services/findatahub-client/src/findatahub_client.egg-info/top_level.txt deleted file mode 100644 index 767d0a2..0000000 --- a/Services/findatahub-client/src/findatahub_client.egg-info/top_level.txt +++ /dev/null @@ -1 +0,0 @@ -findatahub diff --git a/Services/findatahub-core/src/findatahub_core.egg-info/PKG-INFO b/Services/findatahub-core/src/findatahub_core.egg-info/PKG-INFO deleted file mode 100644 index 3610221..0000000 --- a/Services/findatahub-core/src/findatahub_core.egg-info/PKG-INFO +++ /dev/null @@ -1,11 +0,0 @@ -Metadata-Version: 2.4 -Name: findatahub-core -Version: 0.1.0 -Summary: FinDataHub 公共核心包 — 统一数据校验、缓存、报告、门控、流水线编排 -Requires-Python: <3.13,>=3.12 -Requires-Dist: pandas>=2.0 -Requires-Dist: pymysql>=1.1 -Requires-Dist: sqlalchemy>=2.0 -Requires-Dist: structlog>=24.0 -Requires-Dist: pyyaml>=6.0 -Requires-Dist: numpy>=1.24 diff --git a/Services/findatahub-core/src/findatahub_core.egg-info/SOURCES.txt b/Services/findatahub-core/src/findatahub_core.egg-info/SOURCES.txt deleted file mode 100644 index ee6749f..0000000 --- a/Services/findatahub-core/src/findatahub_core.egg-info/SOURCES.txt +++ /dev/null @@ -1,29 +0,0 @@ -pyproject.toml -src/findatahub_core/__init__.py -src/findatahub_core/bootstrap.py -src/findatahub_core/cache.py -src/findatahub_core/cleaner.py -src/findatahub_core/context.py -src/findatahub_core/correlation.py -src/findatahub_core/cross_validator.py -src/findatahub_core/db.py -src/findatahub_core/gate.py -src/findatahub_core/health.py -src/findatahub_core/imputer.py -src/findatahub_core/outlier.py -src/findatahub_core/pca.py -src/findatahub_core/pipeline.py -src/findatahub_core/profile.py -src/findatahub_core/reporter.py -src/findatahub_core/retry.py -src/findatahub_core/stats.py -src/findatahub_core/validator.py -src/findatahub_core.egg-info/PKG-INFO -src/findatahub_core.egg-info/SOURCES.txt -src/findatahub_core.egg-info/dependency_links.txt -src/findatahub_core.egg-info/requires.txt -src/findatahub_core.egg-info/top_level.txt -tests/test_cleaner.py -tests/test_core_pipeline.py -tests/test_retry.py -tests/test_validator.py \ No newline at end of file diff --git a/Services/findatahub-core/src/findatahub_core.egg-info/dependency_links.txt b/Services/findatahub-core/src/findatahub_core.egg-info/dependency_links.txt deleted file mode 100644 index 8b13789..0000000 --- a/Services/findatahub-core/src/findatahub_core.egg-info/dependency_links.txt +++ /dev/null @@ -1 +0,0 @@ - diff --git a/Services/findatahub-core/src/findatahub_core.egg-info/requires.txt b/Services/findatahub-core/src/findatahub_core.egg-info/requires.txt deleted file mode 100644 index aee163f..0000000 --- a/Services/findatahub-core/src/findatahub_core.egg-info/requires.txt +++ /dev/null @@ -1,6 +0,0 @@ -pandas>=2.0 -pymysql>=1.1 -sqlalchemy>=2.0 -structlog>=24.0 -pyyaml>=6.0 -numpy>=1.24 diff --git a/Services/findatahub-core/src/findatahub_core.egg-info/top_level.txt b/Services/findatahub-core/src/findatahub_core.egg-info/top_level.txt deleted file mode 100644 index 4f8b063..0000000 --- a/Services/findatahub-core/src/findatahub_core.egg-info/top_level.txt +++ /dev/null @@ -1 +0,0 @@ -findatahub_core diff --git a/Services/tushare_fetcher/config/default.yaml b/Services/tushare_fetcher/config/default.yaml index 2a6f469..b4ae74d 100644 --- a/Services/tushare_fetcher/config/default.yaml +++ b/Services/tushare_fetcher/config/default.yaml @@ -143,8 +143,14 @@ tables: concept_detail: true # === 其他 === - stk_factor: true - stk_factor_pro: true + # stk_factor 停用：接口频率限制 5 次/天，无法批量抓取； + # fetcher.py 遇到该表会显式跳过并打 WARNING。配置写 true 只会 + # 造成「看起来已启用、实际从不采集」的假象。 + stk_factor: false + # stk_factor_pro 停用：接口频率限制 5 次/天，无法批量抓取； + # fetcher.py 遇到该表会显式跳过并打 WARNING。配置写 true 只会 + # 造成「看起来已启用、实际从不采集」的假象。 + stk_factor_pro: false stk_account: true stk_surv: true stk_preipo: true