AH-Oil-Trader/core/news.py

"""
core/news.py — News-Fetcher + Übersetzer
==========================================
Lädt RSS-Feeds zu WTI-Crude/Öl-News und übersetzt sie optional auf Deutsch.

  • NewsTranslator  — deep-translator (Google + MyMemory-Fallback)
  • NewsFetcher     — 4 RSS-Feeds, Sortierung nach Datum

Cache der Übersetzungen liegt in oil_widget_translations.json
(neben dem Hauptscript) und überlebt Neustarts.
"""

from __future__ import annotations
import json
import hashlib
import threading
import time
from pathlib import Path

from core.logger import get_logger

log_trans = get_logger("trans")
log_news  = get_logger("news")


# Cache-Pfad neben dem Hauptscript
TRANSLATION_CACHE_FILE = Path(__file__).parent.parent / "oil_widget_translations.json"
TRANSLATION_CACHE_MAX  = 1000          # FIFO-Trim ab 1200
TRANSLATION_TIMEOUT_S  = 8


# ══════════════════════════════════════════════
#  NEWS-ÜBERSETZUNG
# ══════════════════════════════════════════════
class NewsTranslator:
    """
    Übersetzt englische Nachrichten-Titel via deep-translator (Google).

      • Persistenter JSON-Cache (kein doppelter API-Call für gleiche Headlines)
      • Background-Threading: UI bleibt responsiv
      • Fallback-Provider (Google → MyMemory) falls primary fehlschlägt
      • Komplett lokal abschaltbar via [translation] enabled = false
    """

    def __init__(self, enabled: bool = True,
                 target: str = "de",
                 provider: str = "google",
                 cache_file: Path = TRANSLATION_CACHE_FILE):
        self.enabled    = enabled
        self.target     = target.strip().lower() or "de"
        self.provider   = provider.strip().lower() or "google"
        self.cache_file = cache_file
        self.cache      = {}
        self._cache_dirty = False
        self._lock      = threading.Lock()
        self.error      = None
        self.api_ok     = None
        self._load_cache()

    def _load_cache(self):
        if not self.cache_file.exists():
            return
        try:
            with open(self.cache_file, "r", encoding="utf-8") as f:
                self.cache = json.load(f)
            log_trans.info(f"Cache geladen: {len(self.cache)} Einträge")
        except Exception as e:
            log_trans.error(f"Cache-Lesefehler: {e}")
            self.cache = {}

    def _save_cache(self):
        if not self._cache_dirty:
            return
        try:
            if len(self.cache) > TRANSLATION_CACHE_MAX + 200:
                items = list(self.cache.items())
                self.cache = dict(items[-TRANSLATION_CACHE_MAX:])
            with open(self.cache_file, "w", encoding="utf-8") as f:
                json.dump(self.cache, f, ensure_ascii=False, indent=1)
            self._cache_dirty = False
        except Exception as e:
            log_trans.error(f"Cache-Schreibfehler: {e}")

    @staticmethod
    def _hash(text: str) -> str:
        return hashlib.sha1(text.encode("utf-8", errors="ignore")).hexdigest()[:16]

    def _translate_one(self, text: str) -> str | None:
        try:
            from deep_translator import GoogleTranslator
        except ImportError:
            self.error = "pip install deep-translator"
            return None

        try:
            if self.provider == "google":
                return GoogleTranslator(source="auto", target=self.target).translate(text)
        except Exception as e:
            log_trans.warning(f"Google-Fehler: {e}")

        try:
            from deep_translator import MyMemoryTranslator
            src_full = "en-GB"
            tgt_full = f"{self.target}-{self.target.upper()}"
            return MyMemoryTranslator(source=src_full, target=tgt_full).translate(text)
        except Exception as e:
            log_trans.warning(f"MyMemory-Fehler: {e}")

        return None

    def translate(self, text: str) -> str:
        if not self.enabled or not text:
            return text
        h = self._hash(text)
        with self._lock:
            cached = self.cache.get(h)
        if cached:
            return cached
        result = self._translate_one(text)
        if result and result.strip() and result.lower() != text.lower():
            with self._lock:
                self.cache[h] = result
                self._cache_dirty = True
                self.api_ok = True
            return result
        if result is None:
            self.api_ok = False
        return text

    def translate_batch(self, headlines: list) -> list:
        if not self.enabled:
            return headlines
        out = []
        for h in headlines:
            new = dict(h)
            orig = h.get("title", "")
            translated = self.translate(orig)
            if translated != orig:
                new["title_original"] = orig
                new["title"]          = translated
            out.append(new)
        if self._cache_dirty:
            self._save_cache()
        return out

    def shutdown(self):
        self._save_cache()


# ══════════════════════════════════════════════
#  NEWS-FETCHER
# ══════════════════════════════════════════════
class NewsFetcher:
    """
    Holt aktuelle WTI-Crude-News sowie geopolitische / makroökonomische News
    aus öffentlichen RSS-Feeds.

    Energie-Feeds (WTI / Rohstoff-spezifisch):
      OilPrice, EIA Press, Rigzone, ShaleMag, Offshore Energy,
      Oil&Gas 360, Guardian Oil

    Makro- / Geopolitik-Feeds (marktrelevant für WTI):
      Al Jazeera, BBC Business, BBC World, MarketWatch, The Hill Energy,
      DW World, FT Commodities, Middle East Eye, Hellenic Shipping News
    """

    # ── Energie / Rohstoffe — WTI-fokussiert ─────────────────────────────
    FEEDS_ENERGY = [
        ("OilPrice",        "https://oilprice.com/rss/main"),
        ("EIA Press",       "https://www.eia.gov/rss/press_rss.xml"),
        ("Rigzone",         "https://www.rigzone.com/news/rss/rigzone_latest.aspx"),
        ("ShaleMag",        "https://shalemag.com/feed/"),
        ("Offshore Energy", "https://www.offshore-energy.biz/feed/"),
        ("Oil&Gas 360",     "https://www.oilandgas360.com/feed/"),
        ("Guardian Oil",    "https://www.theguardian.com/business/oil/rss"),
    ]

    # ── Makro / Geopolitik (marktrelevant für WTI) ───────────────────────
    # Reuters-Feed wurde 2020 abgeschaltet — nicht mehr verwenden.
    FEEDS_GEO = [
        ("Al Jazeera",    "https://www.aljazeera.com/xml/rss/all.xml"),
        ("BBC Business",  "https://feeds.bbci.co.uk/news/business/rss.xml"),
        ("BBC World",     "https://feeds.bbci.co.uk/news/world/rss.xml"),
        ("MarketWatch",   "https://feeds.marketwatch.com/marketwatch/marketpulse/"),
        ("The Hill",      "https://thehill.com/policy/energy-environment/feed/"),
        ("DW World",      "https://rss.dw.com/rdf/rss-en-world"),
        ("FT Commodities","https://www.ft.com/commodities?format=rss"),
        ("Middle East Eye","https://www.middleeasteye.net/rss"),
        ("Hellenic Ship.", "https://www.hellenicshippingnews.com/feed/"),
    ]

    # Kombination beider Feed-Gruppen
    FEEDS = FEEDS_ENERGY + FEEDS_GEO

    # Schlüsselwörter zur Relevanz-Filterung (Geo-Feeds)
    GEO_KEYWORDS = {
        # Öl/Energie allgemein
        "oil", "crude", "wti", "brent", "petroleum", "energy",
        "opec", "iea", "eia", "barrel", "supply", "demand",
        "refin", "pipeline", "tanker", "stockpile", "inventory", "stocks",
        "spr", "gas", "fuel", "commodity", "commodities", "lng",
        # Schifffahrt / Tankerrouten — direkt preisrelevant
        "shipping", "vessel", "suez", "canal", "chokepoint",
        "bab-el-mandeb", "vlcc", "freight", "cargo",
        # WTI-spezifisch: US-Produktion & Infrastruktur
        "cushing", "permian", "bakken", "eagle ford", "marcellus",
        "keystone", "gulf of mexico", "nymex", "shale", "fracking",
        "us oil", "us energy", "us crude", "us production",
        "refinery capacity", "hurricane",
        # Sanktionen / Geopolitik
        "iran", "sanction", "embargo", "strait", "hormuz", "gulf",
        # Akteure
        "saudi", "russia", "moscow", "putin", "iraq",
        "venezuela", "libya", "nigeria", "kuwait", "uae", "qatar",
        "ukraine", "hamas", "houthi", "yemen", "israel", "hezbollah",
        "lebanon", "syria", "china", "beijing", "iran",
        # Ereignisse
        "war", "attack", "ceasefire", "truce", "drone", "missile",
        "embargo", "blockade", "strike", "shutdown", "outage",
        "disruption", "force majeure", "summit", "deal", "agreement",
        "tariff", "trade war", "recession", "inflation", "gdp",
    }

    # ── Vollständige Feed-Pools inkl. Alternativen (Reihenfolge = Priorität) ──
    # validate_feeds() testet alle Feeds parallel und ersetzt ausgefallene
    # primäre Feeds automatisch mit dem nächsten funktionierenden Pool-Eintrag.
    _ENERGY_POOL = [
        # Primär
        ("OilPrice",         "https://oilprice.com/rss/main"),
        ("EIA Press",        "https://www.eia.gov/rss/press_rss.xml"),
        ("Rigzone",          "https://www.rigzone.com/news/rss/rigzone_latest.aspx"),
        ("ShaleMag",         "https://shalemag.com/feed/"),
        ("Offshore Energy",  "https://www.offshore-energy.biz/feed/"),
        ("Oil&Gas 360",      "https://www.oilandgas360.com/feed/"),
        ("Guardian Oil",     "https://www.theguardian.com/business/oil/rss"),
        # Alternativen (automatischer Fallback)
        ("Energy Monitor",   "https://www.energymonitor.ai/feed/"),
        ("Natural Gas Int.", "https://www.naturalgasintel.com/feed/"),
    ]

    _GEO_POOL = [
        # Primär
        ("Al Jazeera",       "https://www.aljazeera.com/xml/rss/all.xml"),
        ("BBC Business",     "https://feeds.bbci.co.uk/news/business/rss.xml"),
        ("BBC World",        "https://feeds.bbci.co.uk/news/world/rss.xml"),
        ("MarketWatch",      "https://feeds.marketwatch.com/marketwatch/marketpulse/"),
        ("The Hill",         "https://thehill.com/policy/energy-environment/feed/"),
        ("DW World",         "https://rss.dw.com/rdf/rss-en-world"),
        ("FT Commodities",   "https://www.ft.com/commodities?format=rss"),
        ("Middle East Eye",  "https://www.middleeasteye.net/rss"),
        ("Hellenic Ship.",   "https://www.hellenicshippingnews.com/feed/"),
        # Alternativen (automatischer Fallback)
        ("New Arab",         "https://www.newarab.com/rss.xml"),
    ]

    # WTI-Direktseite (HTML-Scraper, keine RSS)
    OILPRICE_WTI_URL    = "https://oilprice.com/futures/wti/"
    # Globale Öl-Preistabelle (Breadth + WTI/Brent Spot)
    OILPRICE_CHARTS_URL = "https://oilprice.com/oil-price-charts/"

    # HTTP-Timeout pro Feed in Sekunden — verhindert dass langsame/tote
    # Feeds den ganzen Fetch-Thread blockieren.
    FETCH_TIMEOUT_S = 8

    def __init__(self, translator: 'NewsTranslator | None' = None):
        self.headlines    = []
        self.last_fetch   = None
        self.error        = None
        self.translator   = translator
        self.sentiment    = {"score": 0.0, "n_bull": 0, "n_bear": 0, "samples": []}
        self.price_data:  dict | None = None   # OilPrice Charts Breadth + Spot
        # Pro-Feed Status: {source: {"ok": bool, "n": int, "msg": str}}
        self.feed_status: dict = {}
        # Validierte, aktive Feed-Listen (None = noch nicht validiert → Klassenvariable)
        self._active_energy: list | None = None
        self._active_geo:    list | None = None
        self._lock        = threading.Lock()

    def _is_geo_relevant(self, title: str) -> bool:
        """Gibt True zurück, wenn ein Geo-Feed-Titel marktrelevante Keywords enthält."""
        low = title.lower()
        return any(kw in low for kw in self.GEO_KEYWORDS)

    # ══════════════════════════════════════════════
    #  FEED-VALIDIERUNG (einmalig beim Start)
    # ══════════════════════════════════════════════

    def _test_url(self, source: str, url: str) -> tuple[bool, int, str]:
        """Schneller Feed-Test ohne feed_status zu verändern. Gibt (ok, n_entries, msg) zurück."""
        try:
            import feedparser, requests
        except ImportError:
            return False, 0, "missing: feedparser/requests"
        try:
            resp = requests.get(
                url,
                timeout=(2, 6),  # kurze Timeouts für schnelle Validierung
                headers={
                    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
                                  "AppleWebKit/537.36 OilWidget/1.0",
                    "Accept": "application/rss+xml, application/xml, text/xml, */*",
                },
            )
            if resp.status_code != 200:
                return False, 0, f"HTTP {resp.status_code}"
            feed = feedparser.parse(resp.content)
            n = len(feed.entries)
            if n == 0:
                return False, 0, "0 Einträge"
            return True, n, "ok"
        except Exception as e:
            return False, 0, str(e)[:60]

    def validate_feeds(self) -> dict:
        """
        Testet alle Feeds aus _ENERGY_POOL / _GEO_POOL parallel.
        Ausgefallene Primär-Feeds werden automatisch durch den nächsten
        funktionierenden Pool-Eintrag ersetzt.
        Ergebnis wird in self._active_energy / self._active_geo gespeichert.

        Aufruf: einmalig beim Start in einem Daemon-Thread.
        """
        import concurrent.futures

        all_feeds = list({u: (s, u) for s, u in
                          self._ENERGY_POOL + self._GEO_POOL}.values())

        log_news.info(f"Feed-Validierung: teste {len(all_feeds)} Feeds …")

        # Parallel testen
        test_results: dict[str, tuple[bool, int, str]] = {}  # url → (ok, n, msg)
        with concurrent.futures.ThreadPoolExecutor(max_workers=12) as ex:
            fut_map = {ex.submit(self._test_url, s, u): (s, u) for s, u in all_feeds}
            try:
                for fut in concurrent.futures.as_completed(fut_map, timeout=25):
                    src, url = fut_map[fut]
                    try:
                        ok, n, msg = fut.result()
                    except Exception as e:
                        ok, n, msg = False, 0, str(e)[:50]
                    test_results[url] = (ok, n, msg)
                    status = "OK" if ok else "FAIL"
                    log_news.debug(f"  [{status}] {src}: {msg} ({n} Einträge)")
            except concurrent.futures.TimeoutError:
                log_news.warning("Feed-Validierung: Timeout nach 25 s")

        def build_active(primary: list, pool: list) -> tuple[list, list]:
            """
            Gibt (aktive_feeds, ersetzungen) zurück.
            Primäre Feeds haben Vorrang; ausgefallene werden mit dem
            nächsten noch nicht verwendeten Pool-Feed ersetzt.
            """
            primary_urls = {u for _, u in primary}
            alternatives = [(s, u) for s, u in pool if u not in primary_urls]

            active = []
            used_urls: set[str] = set()
            replacements = []

            for src, url in primary:
                ok, n, msg = test_results.get(url, (False, 0, "nicht getestet"))
                if ok and n > 0:
                    active.append((src, url))
                    used_urls.add(url)
                else:
                    log_news.warning(f"Feed ausgefallen: {src}  [{msg}]  → suche Ersatz")
                    replaced = False
                    for alt_src, alt_url in alternatives:
                        if alt_url in used_urls:
                            continue
                        alt_ok, alt_n, alt_msg = test_results.get(alt_url, (False, 0, "nicht getestet"))
                        if alt_ok and alt_n > 0:
                            active.append((alt_src, alt_url))
                            used_urls.add(alt_url)
                            replacements.append((src, alt_src))
                            log_news.info(f"  → Ersetzt: {src} → {alt_src}")
                            replaced = True
                            break
                    if not replaced:
                        log_news.warning(f"  → Kein Ersatz verfügbar für {src}")

            return active, replacements

        active_e, rep_e = build_active(self.FEEDS_ENERGY, self._ENERGY_POOL)
        active_g, rep_g = build_active(self.FEEDS_GEO,    self._GEO_POOL)

        with self._lock:
            self._active_energy = active_e
            self._active_geo    = active_g

        total_ok   = sum(1 for ok, _, _ in test_results.values() if ok)
        total_fail = len(test_results) - total_ok
        log_news.info(
            f"Feed-Validierung abgeschlossen: {total_ok} OK / {total_fail} ausgefallen  "
            f"| Energie: {len(active_e)} aktiv ({len(rep_e)} ersetzt)  "
            f"| Geo: {len(active_g)} aktiv ({len(rep_g)} ersetzt)"
        )
        return {
            "energy_active":  active_e,
            "geo_active":     active_g,
            "replacements":   rep_e + rep_g,
            "n_ok":           total_ok,
            "n_fail":         total_fail,
        }

    def _fetch_oilprice_wti_scrape(self) -> list:
        """
        Scrapet https://oilprice.com/futures/wti/ nach WTI-spezifischen Headlines.
        Nutzt BeautifulSoup wenn verfügbar, sonst Regex-Fallback.
        Gibt bis zu 8 Einträge im selben Format wie RSS-Feeds zurück.
        """
        try:
            import requests
        except ImportError:
            return []

        try:
            resp = requests.get(
                self.OILPRICE_WTI_URL,
                timeout=(3, self.FETCH_TIMEOUT_S),
                headers={
                    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
                                  "AppleWebKit/537.36 Chrome/120.0 OilWidget/1.0",
                    "Accept":     "text/html,application/xhtml+xml,*/*",
                    "Accept-Language": "en-US,en;q=0.9",
                },
            )
        except Exception as e:
            log_news.warning(f"OilPrice WTI scrape: {e}")
            self.feed_status["OilPrice WTI"] = {"ok": False, "n": 0, "msg": str(e)[:60]}
            return []

        if resp.status_code != 200:
            log_news.warning(f"OilPrice WTI: HTTP {resp.status_code}")
            self.feed_status["OilPrice WTI"] = {
                "ok": False, "n": 0, "msg": f"HTTP {resp.status_code}"}
            return []

        items = []
        now   = time.time()
        seen  = set()

        # Artikel-Pfade auf OilPrice.com: /Energy/... und /Latest-Energy-News/...
        _ARTICLE_PATHS = ("/Energy/", "/Latest-Energy-News/")

        try:
            from bs4 import BeautifulSoup
            soup = BeautifulSoup(resp.content, "html.parser")

            for tag in soup.find_all("a", href=True):
                href  = tag.get("href", "")
                title = tag.get_text(strip=True)
                # Nur Artikel-Pfade, keine Navigation/Werbung
                if not any(p in href for p in _ARTICLE_PATHS):
                    continue
                if len(title) < 25 or title in seen:
                    continue
                if not href.startswith("http"):
                    href = ("https://oilprice.com" + href
                            if href.startswith("/") else "")
                if not href:
                    continue
                seen.add(title)
                items.append({
                    "title":    title,
                    "link":     href,
                    "source":   "OilPrice WTI",
                    "ts":       now,
                    "category": "energy",
                })
                if len(items) >= 8:
                    break

        except ImportError:
            # Regex-Fallback: kein beautifulsoup4 installiert
            import re
            pat = re.compile(
                r'<a[^>]+href="(https?://oilprice\.com/'
                r'(?:Energy|Latest-Energy-News)/[^"#?]{10,})"[^>]*>'
                r'\s*([^<\n]{25,200})\s*</a>',
                re.IGNORECASE | re.DOTALL,
            )
            for href, raw in pat.findall(resp.text):
                title = re.sub(r'\s+', ' ', raw).strip()
                if not title or title in seen:
                    continue
                seen.add(title)
                items.append({
                    "title":    title,
                    "link":     href,
                    "source":   "OilPrice WTI",
                    "ts":       now,
                    "category": "energy",
                })
                if len(items) >= 8:
                    break
            if not items:
                log_news.info("OilPrice WTI: BeautifulSoup nicht installiert "
                              "(pip install beautifulsoup4 lxml) — Regex-Fallback aktiv")

        self.feed_status["OilPrice WTI"] = {
            "ok": len(items) > 0,
            "n":  len(items),
            "msg": "scrape ok" if items else "0 Headlines",
        }
        log_news.info(f"OilPrice WTI scrape: {len(items)} Headlines")
        return items

    def _fetch_oilprice_charts(self) -> dict | None:
        """
        Scrapet https://oilprice.com/oil-price-charts/ und extrahiert:
          • WTI Crude und Brent Crude: Preis, absolute Änderung, %Änderung
          • Market Breadth aus Tabelle 'Futures & Indexes':
            Anteil der Öl-Benchmarks mit positivem Tageschange
          • breadth_signal: -1.0 (alle runter) … +1.0 (alle rauf)

        Breadth > +0.5  → bullisher Marktkontext
        Breadth < -0.5  → bärischer Marktkontext
        """
        try:
            import requests
        except ImportError:
            return None

        try:
            resp = requests.get(
                self.OILPRICE_CHARTS_URL,
                timeout=(3, self.FETCH_TIMEOUT_S),
                headers={
                    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
                                  "AppleWebKit/537.36 Chrome/120.0 OilWidget/1.0",
                    "Accept":     "text/html,application/xhtml+xml,*/*",
                    "Accept-Language": "en-US,en;q=0.9",
                },
            )
        except Exception as e:
            log_news.warning(f"OilPrice Charts fetch: {e}")
            return None

        if resp.status_code != 200:
            log_news.warning(f"OilPrice Charts: HTTP {resp.status_code}")
            return None

        try:
            from bs4 import BeautifulSoup
        except ImportError:
            log_news.warning("OilPrice Charts: BeautifulSoup fehlt (pip install beautifulsoup4)")
            return None

        try:
            soup   = BeautifulSoup(resp.content, "html.parser")
            tables = soup.find_all("table", class_="oilprices__table")
            if not tables:
                log_news.warning("OilPrice Charts: Tabelle nicht gefunden")
                return None

            t0     = tables[0]   # "Futures & Indexes"
            n_up   = n_down = 0
            wti    = brent  = None

            for row in t0.find_all("tr"):
                cells = row.find_all(["th", "td"])
                if len(cells) < 4:
                    continue
                name_cell   = cells[1]
                price_cell  = cells[2]
                change_cell = cells[3]
                pct_cell    = cells[4] if len(cells) > 4 else None

                name = name_cell.get_text(strip=True)
                try:
                    price = float(price_cell.get_text(strip=True).replace(",", ""))
                except ValueError:
                    continue

                change_text    = change_cell.get_text(strip=True)
                change_classes = change_cell.get("class", [])
                try:
                    change = float(change_text.replace(",", ""))
                except ValueError:
                    change = 0.0

                is_up   = any("up"   in c for c in change_classes)
                is_down = any("down" in c for c in change_classes)
                if is_up:
                    n_up   += 1
                elif is_down:
                    n_down += 1

                if name in ("WTI Crude", "Brent Crude"):
                    pct = 0.0
                    if pct_cell:
                        raw_pct = pct_cell.get_text(strip=True).split("(")[0]
                        try:
                            pct = float(raw_pct.replace("%", "").replace(",", ""))
                        except ValueError:
                            pass
                    entry = {"price": price, "change": change, "pct": pct}
                    if name == "WTI Crude":
                        wti   = entry
                    else:
                        brent = entry

            n_total = n_up + n_down
            breadth_pct    = round(n_up / n_total * 100, 1) if n_total else 50.0
            breadth_signal = round((n_up - n_down) / n_total, 3) if n_total else 0.0

            result = {
                "wti":             wti,
                "brent":           brent,
                "n_up":            n_up,
                "n_down":          n_down,
                "n_total":         n_total,
                "breadth_pct":     breadth_pct,     # % der Futures die gestiegen sind
                "breadth_signal":  breadth_signal,  # -1..+1
            }

            wti_str   = f"WTI={wti['price']:.2f} ({wti['pct']:+.2f}%)"  if wti   else "WTI=?"
            brent_str = f"Brent={brent['price']:.2f} ({brent['pct']:+.2f}%)" if brent else "Brent=?"
            log_news.info(
                f"OilPrice Charts: {wti_str}  {brent_str}  "
                f"Breadth {breadth_pct:.0f}% ({n_up}up/{n_down}dn) signal={breadth_signal:+.2f}"
            )
            return result

        except Exception as e:
            log_news.warning(f"OilPrice Charts parse: {e}", exc_info=True)
            return None

    def price_data_snapshot(self) -> dict | None:
        """Thread-safe Kopie der letzten Preisdaten (OilPrice Charts)."""
        with self._lock:
            return dict(self.price_data) if self.price_data else None

    def _load_feed(self, source: str, url: str):
        """
        Lädt eine einzelne RSS-URL mit hardem HTTP-Timeout.
        Liefert die feedparser-Feed oder None bei Fehler.
        Setzt feed_status[source] für spätere Diagnose.
        """
        try:
            import feedparser
            import requests
        except ImportError as e:
            self.feed_status[source] = {"ok": False, "n": 0, "msg": f"missing: {e}"}
            return None

        try:
            resp = requests.get(
                url,
                timeout=(3, self.FETCH_TIMEOUT_S),
                headers={
                    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
                                  "AppleWebKit/537.36 OilWidget/1.0",
                    "Accept": "application/rss+xml, application/xml, text/xml, */*",
                },
            )
            if resp.status_code != 200:
                self.feed_status[source] = {
                    "ok": False, "n": 0,
                    "msg": f"HTTP {resp.status_code}",
                }
                log_news.warning(f"{source}: HTTP {resp.status_code}")
                return None
            feed = feedparser.parse(resp.content)
            if feed.bozo and not feed.entries:
                # bozo=1 mit Entries ist meist nur "namespace warning" → OK
                msg = str(feed.bozo_exception)[:80] if hasattr(feed, "bozo_exception") else "parse error"
                self.feed_status[source] = {"ok": False, "n": 0, "msg": msg}
                log_news.warning(f"{source}: {msg}")
                return None
            n = len(feed.entries)
            self.feed_status[source] = {"ok": True, "n": n, "msg": "ok"}
            return feed
        except Exception as e:
            self.feed_status[source] = {"ok": False, "n": 0, "msg": str(e)[:80]}
            log_news.warning(f"{source}: {e}")
            return None

    def fetch(self):
        try:
            import feedparser  # nur für Verfügbarkeits-Check
        except ImportError:
            with self._lock:
                self.error = "pip install feedparser requests"
            return

        # Per-Feed Status für diesen Run leeren
        self.feed_status = {}
        energy_items = []
        geo_items    = []

        # ── OilPrice WTI-Direktseite (Scraper, höchste Priorität) ──
        wti_items  = self._fetch_oilprice_wti_scrape()
        energy_items.extend(wti_items)

        # ── OilPrice Charts — Preis-Breadth ──
        charts_data = self._fetch_oilprice_charts()

        # Validierte Listen verwenden (nach validate_feeds()), sonst Klassen-Defaults
        with self._lock:
            energy_feeds = self._active_energy if self._active_energy is not None else self.FEEDS_ENERGY
            geo_feeds    = self._active_geo    if self._active_geo    is not None else self.FEEDS_GEO

        # ── Energie-Feeds: bis zu 5 Einträge je Feed, keine Filterung ──
        for source, url in energy_feeds:
            feed = self._load_feed(source, url)
            if feed is None:
                continue
            for entry in feed.entries[:5]:
                pub = entry.get("published_parsed") or time.gmtime()
                energy_items.append({
                    "title":    entry.get("title", "(no title)").strip(),
                    "link":     entry.get("link", ""),
                    "source":   source,
                    "ts":       time.mktime(pub),
                    "category": "energy",
                })

        # ── Geo-Feeds: bis zu 3 Einträge je Feed, nur relevante Titel ──
        for source, url in geo_feeds:
            feed = self._load_feed(source, url)
            if feed is None:
                continue
            count = 0
            for entry in feed.entries:
                if count >= 3:
                    break
                title = entry.get("title", "").strip()
                if not self._is_geo_relevant(title):
                    continue
                pub = entry.get("published_parsed") or time.gmtime()
                geo_items.append({
                    "title":    title,
                    "link":     entry.get("link", ""),
                    "source":   source,
                    "ts":       time.mktime(pub),
                    "category": "geo",
                })
                count += 1

        # Diagnose-Zeilen ins Log
        ok_count = sum(1 for s in self.feed_status.values() if s["ok"])
        fail_count = len(self.feed_status) - ok_count
        log_news.info(
            f"Feed-Status: {ok_count} OK, {fail_count} fehlgeschlagen"
        )
        for source, st in self.feed_status.items():
            if st["ok"]:
                log_news.debug(f"  ✓ {source}: {st['n']} Einträge")
            else:
                log_news.info(f"  ✗ {source}: {st['msg']}")

        # ── Zusammenführen: WTI-Scraper + RSS Energie + Geo ──
        # WTI-Scraper-Items haben ts=now → sortieren immer an erste Stelle wenn aktuell
        energy_items.sort(key=lambda x: x["ts"], reverse=True)
        geo_items.sort(key=lambda x: x["ts"], reverse=True)

        # WTI-Scraper bekommt bis zu 4 Slots im Energy-Bucket (von 7 gesamt)
        wti_out    = [i for i in energy_items if i.get("source") == "OilPrice WTI"][:4]
        rss_energy = [i for i in energy_items if i.get("source") != "OilPrice WTI"][:3]
        items = wti_out + rss_energy + geo_items[:4]
        items.sort(key=lambda x: x["ts"], reverse=True)
        items = items[:10]

        log_news.info(
            f"→ WTI-Scraper:{len(wti_out)} + Energie-RSS:{len(rss_energy)} + "
            f"Geo:{len(geo_items[:4])} = {len(items)} Headlines"
        )

        if self.translator and self.translator.enabled:
            try:
                items = self.translator.translate_batch(items)
            except Exception as e:
                log_news.warning(f"Übersetzung fehlgeschlagen: {e}")

        # Sentiment auf den (ggf. übersetzten) Headlines berechnen.
        # calc_news_sentiment() greift bevorzugt auf title_original (englisch) zurück.
        try:
            from core.analysis import calc_news_sentiment
            sentiment = calc_news_sentiment(items)
        except Exception as e:
            log_news.warning(f"Sentiment-Berechnung fehlgeschlagen: {e}")
            sentiment = {"score": 0.0, "n_bull": 0, "n_bear": 0, "samples": []}

        # ── Breadth-Signal in Sentiment einblenden (Gewicht 25%) ──────────────
        # Breadth misst ob die Mehrheit aller Öl-Benchmarks steigt/fällt.
        # Nur bei starkem Signal (|breadth| > 0.4) und ohne starke News-Gegenmeinung.
        if charts_data:
            bs = charts_data.get("breadth_signal", 0.0)
            if abs(bs) > 0.4:
                raw_score = sentiment["score"]
                blended   = round(raw_score * 0.75 + bs * 0.25, 3)
                blended   = max(-1.0, min(1.0, blended))
                sentiment  = dict(sentiment)
                sentiment["score"]          = blended
                sentiment["breadth_signal"] = bs
                sentiment["breadth_pct"]    = charts_data.get("breadth_pct", 50.0)
                log_news.info(
                    f"Breadth {bs:+.2f} → Sentiment {raw_score:+.2f} → {blended:+.2f}"
                )

        with self._lock:
            self.headlines  = items
            self.last_fetch = time.time()
            self.error      = None if items else "Keine News-Feeds erreichbar"
            self.sentiment  = sentiment
            self.price_data = charts_data

        log_news.info(f"News-Sentiment: {sentiment['score']:+.2f} "
                      f"(bull={sentiment['n_bull']}, bear={sentiment['n_bear']})")

    def snapshot(self):
        with self._lock:
            return list(self.headlines), self.last_fetch, self.error

    def sentiment_snapshot(self) -> dict:
        """Thread-safe Kopie des letzten Sentiment-Snapshots."""
        with self._lock:
            return dict(self.sentiment)

    def storm_state(self) -> dict:
        """News-Sturm-Indikator (REINE ANZEIGE, kein Signal): Headline-Rate der
        letzten Stunde vs. Basisrate des vorhandenen Feed-Fensters. Viel frische
        Öl-/Geo-Schlagzeilen auf einmal = Ereignis läuft → Vorsicht (Vola).
        normal · elevated (≥3 frisch & ≥2× Basis) · storm (≥5 frisch & ≥3× Basis)."""
        with self._lock:
            items = [h for h in self.headlines if h.get("ts")]
        now = time.time()
        if not items:
            return {"level": "normal", "fresh": 0, "base_per_h": 0.0}
        fresh = sum(1 for h in items if now - h["ts"] <= 3600)
        # Basis-Fenster min. 6 h verankern — sonst bläht ein frischer Burst die
        # Basisrate selbst auf und der Sturm erkennt sich nicht (Selbst-Normierung).
        span_h = min(48.0, max(6.0, (now - min(h["ts"] for h in items)) / 3600))
        base = len(items) / span_h                     # Ø Headlines/Stunde im Fenster
        level = "normal"
        if fresh >= 5 and fresh >= 3 * base:
            level = "storm"
        elif fresh >= 3 and fresh >= 2 * base:
            level = "elevated"
        return {"level": level, "fresh": fresh, "base_per_h": round(base, 1)}