FinDataHub/Scripts/bin/rss_fetcher.py

# coding=utf-8
"""
RSS 通用采集器 — 通过 china-finance-rss 项目获取财经快讯

支持数据源：
  - 东方财富 7×24 快讯 (eastmoney)
  - 同花顺 7×24 快讯 (10jqka)
  - 雪球用户动态 (xueqiu)

RSS 服务地址默认: http://localhost:1200
可通过环境变量 RSS_HUB_URL 覆盖
"""
import os
import hashlib
import time
from datetime import datetime, timezone
from xml.etree import ElementTree as ET
from typing import List

import httpx

from common import RSS_HUB_URL
# 复用 kuaixun-old 的 BaseFetcher 和 NewsItem
from extend.fetchers import BaseFetcher, NewsItem

# RSS 服务地址（china-finance-rss 本地部署）
DEFAULT_RSS_URL = RSS_HUB_URL


class RssFetcher(BaseFetcher):
    """RSS 通用采集器基类 — 解析 RSS 2.0 XML"""

    def __init__(self, source_name: str, source_code: str, rss_path: str, fetch_interval: int = 60):
        super().__init__(source_name, source_code, fetch_interval)
        self.rss_url = f"{DEFAULT_RSS_URL}{rss_path}"
        self.headers = {
            "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36",
        }

    async def fetch(self) -> List[NewsItem]:
        items = []
        try:
            async with httpx.AsyncClient(timeout=15) as client:
                resp = await client.get(self.rss_url, headers=self.headers)
                resp.raise_for_status()

            items = self._parse_rss_xml(resp.text)
            print(f"  [RSS:{self.source_code}] 获取 {len(items)} 条快讯", flush=True)
        except Exception as e:
            print(f"  [RSS:{self.source_code}] fetch error: {e}", flush=True)
        return items

    def _parse_rss_xml(self, xml_text: str) -> List[NewsItem]:
        """解析 RSS 2.0 XML，提取快讯条目"""
        items = []
        try:
            root = ET.fromstring(xml_text)
            channel = root.find("channel")
            if channel is None:
                return items

            for item in channel.findall("item"):
                title = (item.findtext("title") or "").strip()
                link = (item.findtext("link") or "").strip()
                description = (item.findtext("description") or "").strip()
                pub_date_str = (item.findtext("pubDate") or "").strip()

                if not title:
                    continue

                # 生成稳定的 source_news_id
                news_id = self._gen_id(link or title)

                # 解析发布时间
                published_at = self._parse_pub_date(pub_date_str)

                # 分类
                category = self._categorize(title, description)

                items.append(NewsItem(
                    source_id=self.source_code,
                    source_news_id=news_id,
                    title=title,
                    content=description,
                    summary=description[:200] if description else "",
                    url=link,
                    published_at=published_at,
                    category=category,
                    tags=self._extract_tags(title),
                ))
        except ET.ParseError as e:
            print(f"  [RSS:{self.source_code}] XML parse error: {e}", flush=True)
        return items

    def _gen_id(self, text: str) -> str:
        """根据 URL 或标题生成稳定 ID"""
        return hashlib.md5(text.encode("utf-8")).hexdigest()[:16]

    def _parse_pub_date(self, date_str: str) -> int:
        """解析 RFC 822 日期格式 → Unix 毫秒时间戳"""
        if not date_str:
            return int(time.time() * 1000)
        try:
            dt = datetime.strptime(date_str, "%a, %d %b %Y %H:%M:%S %z")
            return int(dt.timestamp() * 1000)
        except ValueError:
            try:
                dt = datetime.strptime(date_str, "%Y-%m-%d %H:%M:%S")
                return int(dt.replace(tzinfo=timezone.utc).timestamp() * 1000)
            except ValueError:
                return int(time.time() * 1000)

    def _categorize(self, title: str, content: str) -> str:
        text = title + content
        if any(kw in text for kw in ["美联储", "央行", "CPI", "GDP", "非农", "利率", "MLF", "LPR"]):
            return "macro"
        if any(kw in text for kw in ["原油", "黄金", "大宗", "铜", "铝", "铁矿石"]):
            return "commodity"
        if any(kw in text for kw in ["科技", "AI", "芯片", "半导体", "人工智能"]):
            return "tech"
        if any(kw in text for kw in ["股票", "A股", "涨停", "跌停", "IPO", "财报"]):
            return "stock"
        return "macro"

    def _extract_tags(self, title: str) -> list:
        tags = []
        for tag in ["美联储", "央行", "非农", "原油", "黄金", "科技", "芯片", "A股", "IPO"]:
            if tag in title:
                tags.append(tag)
        return tags

    def health_check(self) -> bool:
        try:
            resp = httpx.get(self.rss_url, headers=self.headers, timeout=10)
            return resp.status_code == 200
        except Exception:
            return False


# ═══════════════════════════════════════════════════════════
# 具体数据源实现
# ═══════════════════════════════════════════════════════════

class EastMoneyRssFetcher(RssFetcher):
    """东方财富 7×24 快讯 — 通过 RSS 服务"""

    def __init__(self):
        super().__init__(
            source_name="东方财富快讯",
            source_code="eastmoney",
            rss_path="/eastmoney",
            fetch_interval=60,
        )


class TonghuashunFetcher(RssFetcher):
    """同花顺 7×24 快讯 — 通过 RSS 服务"""

    def __init__(self):
        super().__init__(
            source_name="同花顺快讯",
            source_code="10jqka",
            rss_path="/10jqka",
            fetch_interval=60,
        )


class XueqiuFetcher(RssFetcher):
    """雪球用户动态 — 通过 RSS 服务"""

    def __init__(self):
        super().__init__(
            source_name="雪球动态",
            source_code="xueqiu",
            rss_path="/xueqiu",
            fetch_interval=120,
        )