FinDataHub
11d36d1c1e
问题 - 索引中该目录记为小写 `scripts/`，工作区实为 `Scripts/`。因 core.ignorecase=true， 本地 `git diff -- Scripts/bin/x.py` 会**静默返回空**（看似无改动）， 且在大小写敏感环境（Linux CI / Docker 挂载）会检出 `scripts/`， 与全仓所有 `Scripts/...` 引用不匹配。 - 该目录此前 10 个脚本（check_python_version.py、perf_snapshot.py、 install_collector_launchd.py、mysql_toggle.sh 等）从未入库， 而 CI 第 31 行正调用 `Scripts/bin/check_python_version.py --strict` —— 全新检出必然失败。 改动 - 索引路径统一为 `Scripts/`（工作区文件未移动，仅索引改名）。 - 补齐上述 10 个未跟踪脚本及其测试。 影响 - 修正前索引 35 条（全为小写路径），修正后 45 条。
335 lines
12 KiB
Python
335 lines
12 KiB
Python
# coding=utf-8
|
||
"""rss_fetcher.py RSS 采集器纯解析方法单元测试.
|
||
|
||
重点覆盖（全部为无网络/无 DB 的纯函数）：
|
||
- _gen_id：MD5 稳定 ID 生成
|
||
- _parse_pub_date：RFC 822 日期解析 + 多级 fallback
|
||
- _categorize：关键词分类（macro/commodity/tech/stock + 默认）
|
||
- _extract_tags：标题关键词标签提取
|
||
- _parse_rss_xml：RSS 2.0 XML 解析（多 item、缺字段、ParseError 容错）
|
||
"""
|
||
|
||
import os
|
||
import sys
|
||
import time
|
||
import types
|
||
from dataclasses import dataclass, field
|
||
from typing import List
|
||
from abc import ABC, abstractmethod
|
||
from xml.etree import ElementTree as ET
|
||
|
||
import pytest
|
||
|
||
# 将 Scripts/bin 加入 sys.path
|
||
_bin_dir = os.path.join(os.path.dirname(__file__), "..")
|
||
if _bin_dir not in sys.path:
|
||
sys.path.insert(0, _bin_dir)
|
||
|
||
# ── 注入 extend.fetchers mock（测试环境无 kuaixun-service） ──
|
||
_extend_mock = types.ModuleType("extend")
|
||
_fetchers_mock = types.ModuleType("extend.fetchers")
|
||
|
||
|
||
@dataclass
|
||
class _NewsItem:
|
||
source_id: str
|
||
source_news_id: str
|
||
title: str
|
||
content: str = ""
|
||
summary: str = ""
|
||
url: str = ""
|
||
published_at: int = 0
|
||
category: str = ""
|
||
tags: List[str] = field(default_factory=list)
|
||
importance: int = 0
|
||
|
||
|
||
class _BaseFetcher(ABC):
|
||
def __init__(self, source_name: str, source_code: str, fetch_interval: int = 30):
|
||
self.source_name = source_name
|
||
self.source_code = source_code
|
||
self.fetch_interval = fetch_interval
|
||
|
||
@abstractmethod
|
||
async def fetch(self):
|
||
...
|
||
|
||
@abstractmethod
|
||
def health_check(self) -> bool:
|
||
...
|
||
|
||
|
||
_fetchers_mock.NewsItem = _NewsItem
|
||
_fetchers_mock.BaseFetcher = _BaseFetcher
|
||
_extend_mock.fetchers = _fetchers_mock
|
||
sys.modules["extend"] = _extend_mock
|
||
sys.modules["extend.fetchers"] = _fetchers_mock
|
||
|
||
from rss_fetcher import RssFetcher
|
||
|
||
|
||
def _make_fetcher(source_code: str = "test"):
|
||
"""创建 RssFetcher 实例（不需要网络连接）."""
|
||
return RssFetcher("测试源", source_code, "/test", fetch_interval=60)
|
||
|
||
|
||
# ── _gen_id ───────────────────────────────────────────────
|
||
|
||
|
||
class TestGenId:
|
||
"""MD5 稳定 ID 生成测试."""
|
||
|
||
def test_deterministic_same_input(self):
|
||
"""相同输入产生相同 ID."""
|
||
f = _make_fetcher()
|
||
assert f._gen_id("https://example.com/news/1") == f._gen_id("https://example.com/news/1")
|
||
|
||
def test_different_inputs_different_ids(self):
|
||
"""不同输入产生不同 ID."""
|
||
f = _make_fetcher()
|
||
assert f._gen_id("title-a") != f._gen_id("title-b")
|
||
|
||
def test_id_length_is_16(self):
|
||
"""ID 截取为 16 字符."""
|
||
f = _make_fetcher()
|
||
assert len(f._gen_id("any text")) == 16
|
||
|
||
def test_empty_string_produces_valid_id(self):
|
||
"""空字符串也能产生合法 ID（不抛异常）."""
|
||
f = _make_fetcher()
|
||
result = f._gen_id("")
|
||
assert len(result) == 16
|
||
assert result == f._gen_id("") # 确定性
|
||
|
||
|
||
# ── _parse_pub_date ───────────────────────────────────────
|
||
|
||
|
||
class TestParsePubDate:
|
||
"""RFC 822 日期解析测试."""
|
||
|
||
def test_rfc822_with_timezone(self):
|
||
"""RFC 822 带时区格式正确解析为毫秒时间戳."""
|
||
f = _make_fetcher()
|
||
# Mon, 15 Jan 2024 10:30:00 +0800 → UTC 02:30:00
|
||
from datetime import datetime, timezone, timedelta
|
||
dt = datetime(2024, 1, 15, 10, 30, 0, tzinfo=timezone(timedelta(hours=8)))
|
||
expected = int(dt.timestamp() * 1000)
|
||
result = f._parse_pub_date("Mon, 15 Jan 2024 10:30:00 +0800")
|
||
assert result == expected
|
||
|
||
def test_rfc822_utc_timezone(self):
|
||
"""RFC 822 UTC (+0000) 格式正确解析."""
|
||
f = _make_fetcher()
|
||
from datetime import datetime, timezone
|
||
dt = datetime(2024, 1, 15, 10, 30, 0, tzinfo=timezone.utc)
|
||
expected = int(dt.timestamp() * 1000)
|
||
result = f._parse_pub_date("Mon, 15 Jan 2024 10:30:00 +0000")
|
||
assert result == expected
|
||
|
||
def test_iso_fallback_no_timezone(self):
|
||
"""非 RFC 822 格式回退到 ISO 解析，按 UTC 处理."""
|
||
f = _make_fetcher()
|
||
from datetime import datetime, timezone
|
||
dt = datetime(2024, 1, 15, 10, 30, 0, tzinfo=timezone.utc)
|
||
expected = int(dt.timestamp() * 1000)
|
||
result = f._parse_pub_date("2024-01-15 10:30:00")
|
||
assert result == expected
|
||
|
||
def test_empty_string_returns_current_time(self):
|
||
"""空字符串返回当前毫秒时间戳."""
|
||
f = _make_fetcher()
|
||
before = int(time.time() * 1000)
|
||
result = f._parse_pub_date("")
|
||
after = int(time.time() * 1000)
|
||
assert before <= result <= after
|
||
|
||
def test_invalid_format_returns_current_time(self):
|
||
"""完全无法解析的格式返回当前毫秒时间戳."""
|
||
f = _make_fetcher()
|
||
before = int(time.time() * 1000)
|
||
result = f._parse_pub_date("not-a-date-at-all!!!")
|
||
after = int(time.time() * 1000)
|
||
assert before <= result <= after
|
||
|
||
|
||
# ── _categorize ───────────────────────────────────────────
|
||
|
||
|
||
class TestCategorize:
|
||
"""关键词分类测试."""
|
||
|
||
@pytest.mark.parametrize("keyword", ["美联储", "央行", "CPI", "GDP", "非农", "利率", "MLF", "LPR"])
|
||
def test_macro_keywords(self, keyword):
|
||
"""宏观类关键词命中 → macro."""
|
||
f = _make_fetcher()
|
||
assert f._categorize(f"标题包含{keyword}", "") == "macro"
|
||
|
||
@pytest.mark.parametrize("keyword", ["原油", "黄金", "大宗", "铜", "铝", "铁矿石"])
|
||
def test_commodity_keywords(self, keyword):
|
||
"""商品类关键词命中 → commodity."""
|
||
f = _make_fetcher()
|
||
assert f._categorize(f"标题包含{keyword}", "") == "commodity"
|
||
|
||
@pytest.mark.parametrize("keyword", ["科技", "AI", "芯片", "半导体", "人工智能"])
|
||
def test_tech_keywords(self, keyword):
|
||
"""科技类关键词命中 → tech."""
|
||
f = _make_fetcher()
|
||
assert f._categorize(f"标题包含{keyword}", "") == "tech"
|
||
|
||
@pytest.mark.parametrize("keyword", ["股票", "A股", "涨停", "跌停", "IPO", "财报"])
|
||
def test_stock_keywords(self, keyword):
|
||
"""股票类关键词命中 → stock."""
|
||
f = _make_fetcher()
|
||
assert f._categorize(f"标题包含{keyword}", "") == "stock"
|
||
|
||
def test_no_keyword_defaults_to_macro(self):
|
||
"""无任何关键词命中时默认分类为 macro."""
|
||
f = _make_fetcher()
|
||
assert f._categorize("今日天气晴朗", "无相关内容") == "macro"
|
||
|
||
def test_content_also_searched(self):
|
||
"""关键词在 content 中出现也能命中."""
|
||
f = _make_fetcher()
|
||
assert f._categorize("普通标题", "美联储加息50基点") == "macro"
|
||
|
||
def test_priority_macro_over_commodity(self):
|
||
"""标题同时含宏观和商品关键词时，宏观优先（先匹配先返回）."""
|
||
f = _make_fetcher()
|
||
assert f._categorize("美联储原油", "") == "macro"
|
||
|
||
|
||
# ── _extract_tags ─────────────────────────────────────────
|
||
|
||
|
||
class TestExtractTags:
|
||
"""标题关键词标签提取测试."""
|
||
|
||
def test_multiple_tags(self):
|
||
"""标题含多个关键词 → 全部提取."""
|
||
f = _make_fetcher()
|
||
tags = f._extract_tags("美联储非农原油A股")
|
||
assert "美联储" in tags
|
||
assert "非农" in tags
|
||
assert "原油" in tags
|
||
assert "A股" in tags
|
||
assert len(tags) == 4
|
||
|
||
def test_no_tags(self):
|
||
"""标题无关键词 → 空列表."""
|
||
f = _make_fetcher()
|
||
assert f._extract_tags("今日天气晴朗") == []
|
||
|
||
def test_empty_title(self):
|
||
"""空标题 → 空列表."""
|
||
f = _make_fetcher()
|
||
assert f._extract_tags("") == []
|
||
|
||
|
||
# ── _parse_rss_xml ────────────────────────────────────────
|
||
|
||
|
||
class TestParseRssXml:
|
||
"""RSS 2.0 XML 解析测试."""
|
||
|
||
def test_valid_rss_multiple_items(self):
|
||
"""标准 RSS 2.0 含多个 item → 全部解析."""
|
||
f = _make_fetcher()
|
||
xml = """<?xml version="1.0" encoding="UTF-8"?>
|
||
<rss version="2.0">
|
||
<channel>
|
||
<title>测试频道</title>
|
||
<item>
|
||
<title>美联储宣布加息</title>
|
||
<link>https://example.com/1</link>
|
||
<description>美联储宣布加息50基点</description>
|
||
<pubDate>Mon, 15 Jan 2024 10:30:00 +0800</pubDate>
|
||
</item>
|
||
<item>
|
||
<title>原油价格下跌</title>
|
||
<link>https://example.com/2</link>
|
||
<description>WTI原油期货下跌3%</description>
|
||
<pubDate>Mon, 15 Jan 2024 11:00:00 +0800</pubDate>
|
||
</item>
|
||
</channel>
|
||
</rss>"""
|
||
items = f._parse_rss_xml(xml)
|
||
assert len(items) == 2
|
||
assert items[0].title == "美联储宣布加息"
|
||
assert items[0].source_id == "test"
|
||
assert items[0].category == "macro"
|
||
assert items[0].url == "https://example.com/1"
|
||
assert items[1].title == "原油价格下跌"
|
||
assert items[1].category == "commodity"
|
||
|
||
def test_missing_channel_returns_empty(self):
|
||
"""XML 无 channel 元素 → 返回空列表."""
|
||
f = _make_fetcher()
|
||
xml = '<?xml version="1.0"?><rss version="2.0"></rss>'
|
||
assert f._parse_rss_xml(xml) == []
|
||
|
||
def test_item_without_title_skipped(self):
|
||
"""item 缺少 title → 跳过该条目."""
|
||
f = _make_fetcher()
|
||
xml = """<?xml version="1.0"?>
|
||
<rss version="2.0"><channel>
|
||
<item>
|
||
<link>https://example.com/no-title</link>
|
||
<description>无标题的快讯</description>
|
||
</item>
|
||
<item>
|
||
<title>有标题</title>
|
||
<link>https://example.com/has-title</link>
|
||
</item>
|
||
</channel></rss>"""
|
||
items = f._parse_rss_xml(xml)
|
||
assert len(items) == 1
|
||
assert items[0].title == "有标题"
|
||
|
||
def test_empty_description_produces_empty_summary(self):
|
||
"""description 为空 → summary 也为空."""
|
||
f = _make_fetcher()
|
||
xml = """<?xml version="1.0"?>
|
||
<rss version="2.0"><channel>
|
||
<item>
|
||
<title>仅有标题</title>
|
||
<link>https://example.com/3</link>
|
||
</item>
|
||
</channel></rss>"""
|
||
items = f._parse_rss_xml(xml)
|
||
assert len(items) == 1
|
||
assert items[0].content == ""
|
||
assert items[0].summary == ""
|
||
|
||
def test_parse_error_returns_empty(self):
|
||
"""XML 格式错误 → 返回空列表不抛异常."""
|
||
f = _make_fetcher()
|
||
assert f._parse_rss_xml("not xml at all <broken") == []
|
||
|
||
def test_stable_id_from_link_or_title(self):
|
||
"""source_news_id 基于 link 或 title 的 MD5."""
|
||
f = _make_fetcher()
|
||
xml = """<?xml version="1.0"?>
|
||
<rss version="2.0"><channel>
|
||
<item>
|
||
<title>测试标题</title>
|
||
<link>https://example.com/unique</link>
|
||
</item>
|
||
</channel></rss>"""
|
||
items = f._parse_rss_xml(xml)
|
||
assert items[0].source_news_id == f._gen_id("https://example.com/unique")
|
||
|
||
def test_tags_extracted_from_title(self):
|
||
"""_parse_rss_xml 调用 _extract_tags 提取标签."""
|
||
f = _make_fetcher()
|
||
xml = """<?xml version="1.0"?>
|
||
<rss version="2.0"><channel>
|
||
<item>
|
||
<title>美联储加息影响A股</title>
|
||
<link>https://example.com/4</link>
|
||
</item>
|
||
</channel></rss>"""
|
||
items = f._parse_rss_xml(xml)
|
||
assert "美联储" in items[0].tags
|
||
assert "A股" in items[0].tags
|