#!/usr/bin/env python3 """ sync_bridge.py — 统一同步桥接层 统一入口: sync_bridge.py [workers] source: akshare | tushare type: daily | weekly | monthly tushare_fetcher 批量同步策略: daily → [daily, market_sentiment] — 日线、资金流、融资融券 weekly → [fundamental, static, event] — 财报、基础信息、事件 monthly → [index, extra, stock_driven] — 指数、补充数据、股东 """ import os, subprocess, sys, traceback from pathlib import Path HOME = Path.home() # 优先使用 App 内部路径,再回退到外部源码 def _find_project(name: str) -> Path: candidates = [] # 1. App 内部(通过环境变量传入,或从脚本位置推断) resources_dir = os.environ.get("APP_RESOURCES", "") if resources_dir: candidates.append(Path(resources_dir) / name) # 2. 从脚本位置推断(脚本在 Resources/bin/ 下) script_dir = Path(__file__).resolve().parent if script_dir.name == "bin" and script_dir.parent.name == "Resources": candidates.append(script_dir.parent / name) # 3. 外部源码 candidates.append(HOME / "Projects/项目" / name) candidates.append(HOME / "Projects" / name) candidates.append(HOME / name) for p in candidates: if (p / "pyproject.toml").exists(): return p return candidates[0] # 返回第一个作为默认 PROJ = { "akshare": _find_project("akshare_fetcher"), "tushare": _find_project("tushare_fetcher"), } # tushare 批量同步 — 每个 sync_type 对应一组 fetch type TUSHARE_BATCH = { "daily": ["daily", "market_sentiment"], "weekly": ["fundamental", "static", "event"], "monthly": ["index", "extra", "stock_driven"], } # 扩展的 fetch type → table list(含 CLI 里没有的 static/event/extra/stock_driven) TYPE_TABLES = { "daily": ["daily", "daily_basic", "weekly", "monthly", "adj_factor", "suspend"], "market_sentiment": ["moneyflow", "margin", "margin_detail", "top_list", "block_trade", "stk_holdernumber", "moneyflow_hsgt"], "fundamental": ["stock_basic", "income", "balancesheet", "cashflow", "fina_indicator", "fina_audit", "forecast", "express", "dividend"], "static": ["trade_cal", "index_basic", "index_classify", "stock_company", "bak_basic", "hs_const"], "event": ["new_share", "daily_info", "sz_daily_info", "fund_nav", "fund_share", "fund_manager"], "index": ["index_daily", "index_basic", "index_weight", "index_classify", "index_member_all", "index_dailybasic", "index_weekly", "index_monthly"], "extra": ["yearly", "stk_weekly_monthly", "stk_week_month_adj", "stk_limit", "report_rc", "cn_gdp", "cn_cpi", "cn_ppi", "cn_pmi", "cn_m"], "stock_driven": ["stk_rewards", "stk_holdernumber", "stk_holdertrade"], } def find_python(project: Path) -> str: for p in [project / ".venv/bin/python3", project / ".venv/bin/python", Path("/usr/bin/python3")]: if p.exists(): return str(p) raise FileNotFoundError(f"找不到 python3 (from {project})") def load_token(project: Path) -> str: env_file = project / ".env" if env_file.exists(): for line in env_file.read_text().splitlines(): if line.startswith("TUSHARE_TOKEN="): return line.split("=", 1)[1].strip().strip("'\"") return os.environ.get("TUSHARE_TOKEN", "") def _pump_stream(fd, sink): """实时转发子进程输出流(块级读取,兼容 tqdm 用 \r 原地刷新的进度行)。""" while True: try: chunk = os.read(fd, 8192) except OSError: break if not chunk: break sink.write(chunk) sink.flush() def run_akshare(project: Path, sync_type: str, workers: int, python: str) -> int: import threading env = os.environ.copy() env.update({"PYTHONPATH": f"{project}/src", "AKSHARE_HOME": str(project), "PYTHONUNBUFFERED": "1"}) cmd = [python, "-m", "akshare_fetcher.cli", "sync", "--type", sync_type, "--workers", str(workers)] print(f" {' '.join(cmd)}", flush=True) # 实时转发 stdout/stderr,不能 capture_output=True: # 子进程的 tqdm 进度写在 stderr,若被缓冲则 App 端收不到实时进度 proc = subprocess.Popen( cmd, cwd=str(project), env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE, ) t1 = threading.Thread(target=_pump_stream, args=(proc.stdout.fileno(), sys.stdout.buffer), daemon=True) t2 = threading.Thread(target=_pump_stream, args=(proc.stderr.fileno(), sys.stderr.buffer), daemon=True) t1.start(); t2.start() rc = proc.wait() t1.join(); t2.join() return rc def run_tushare(project: Path, sync_type: str, workers: int, python: str) -> int: """tushare 批量同步 — 按策略拉取多组表""" token = load_token(project) if not token: print(" ❌ TUSHARE_TOKEN 未配置", flush=True) return 1 fetch_types = TUSHARE_BATCH.get(sync_type, ["daily"]) overall_rc = 0 for ft in fetch_types: tables = TYPE_TABLES.get(ft, [ft]) if not tables: print(f" ⏭ {ft}: 无表定义", flush=True) continue print(f"\n 📡 [{ft}] → {len(tables)} 张表: {', '.join(tables[:5])}{'...' if len(tables) > 5 else ''}", flush=True) # 直接调用 DataFetcher,避免多次启动 Python 的开销 env = os.environ.copy() env.update({ "PYTHONPATH": f"{project}/src", "TUSHARE_HOME": str(project), "PYTHONUNBUFFERED": "1", "TUSHARE_TOKEN": token, }) code = f""" import sys sys.path.insert(0, "{project}/src") from tushare_fetcher.core.config import load_config from tushare_fetcher.core.fetcher import DataFetcher cfg = load_config() fetcher = DataFetcher(cfg, max_workers={workers}) try: fetcher.run({tables}) except KeyboardInterrupt: print("\\n⚠ 用户中断") finally: fetcher.db.dispose() """ cmd = [python, "-c", code] proc = subprocess.Popen( cmd, cwd=str(project), env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE, ) import threading t1 = threading.Thread(target=_pump_stream, args=(proc.stdout.fileno(), sys.stdout.buffer), daemon=True) t2 = threading.Thread(target=_pump_stream, args=(proc.stderr.fileno(), sys.stderr.buffer), daemon=True) t1.start(); t2.start() rc = proc.wait() t1.join(); t2.join() if rc != 0: print(f" ❌ [{ft}] 失败 (rc={rc})", flush=True) overall_rc = rc else: print(f" ✅ [{ft}] 完成", flush=True) return overall_rc if __name__ == "__main__": if len(sys.argv) < 3: print("用法: sync_bridge.py [workers]") sys.exit(2) source = sys.argv[1] sync_type = sys.argv[2] workers = int(sys.argv[3]) if len(sys.argv) > 3 else 4 proj = PROJ.get(source) if not proj or not proj.exists(): print(f"❌ {source}_fetcher 项目不存在: {proj}") sys.exit(1) python = find_python(proj) print(f" 项目: {proj}", flush=True) print(f" Python: {python}", flush=True) if source == "akshare": rc = run_akshare(proj, sync_type, workers, python) else: rc = run_tushare(proj, sync_type, workers, python) print(f"\n 退出码: {rc}", flush=True) sys.exit(rc)