Initial commit: Oil Trading Bot (MT5, WTI)
Headless FastAPI-Backend (server.py + core/engine.py) mit Mobile-PWA (web/), Strategie-/Backtest-Suite und Doku. Secrets, DB, Logs und Laufzeit-State sind via .gitignore ausgeschlossen; Config-Vorlage: oil_widget_config.ini.example. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Axel Hocks
+842
@@ -0,0 +1,842 @@
|
||||
"""
|
||||
core/news.py — News-Fetcher + Übersetzer
|
||||
==========================================
|
||||
Lädt RSS-Feeds zu WTI-Crude/Öl-News und übersetzt sie optional auf Deutsch.
|
||||
|
||||
• NewsTranslator — deep-translator (Google + MyMemory-Fallback)
|
||||
• NewsFetcher — 4 RSS-Feeds, Sortierung nach Datum
|
||||
|
||||
Cache der Übersetzungen liegt in oil_widget_translations.json
|
||||
(neben dem Hauptscript) und überlebt Neustarts.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
import json
|
||||
import hashlib
|
||||
import threading
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
from core.logger import get_logger
|
||||
|
||||
log_trans = get_logger("trans")
|
||||
log_news = get_logger("news")
|
||||
|
||||
|
||||
# Cache-Pfad neben dem Hauptscript
|
||||
TRANSLATION_CACHE_FILE = Path(__file__).parent.parent / "oil_widget_translations.json"
|
||||
TRANSLATION_CACHE_MAX = 1000 # FIFO-Trim ab 1200
|
||||
TRANSLATION_TIMEOUT_S = 8
|
||||
|
||||
|
||||
# ══════════════════════════════════════════════
|
||||
# NEWS-ÜBERSETZUNG
|
||||
# ══════════════════════════════════════════════
|
||||
class NewsTranslator:
|
||||
"""
|
||||
Übersetzt englische Nachrichten-Titel via deep-translator (Google).
|
||||
|
||||
• Persistenter JSON-Cache (kein doppelter API-Call für gleiche Headlines)
|
||||
• Background-Threading: UI bleibt responsiv
|
||||
• Fallback-Provider (Google → MyMemory) falls primary fehlschlägt
|
||||
• Komplett lokal abschaltbar via [translation] enabled = false
|
||||
"""
|
||||
|
||||
def __init__(self, enabled: bool = True,
|
||||
target: str = "de",
|
||||
provider: str = "google",
|
||||
cache_file: Path = TRANSLATION_CACHE_FILE):
|
||||
self.enabled = enabled
|
||||
self.target = target.strip().lower() or "de"
|
||||
self.provider = provider.strip().lower() or "google"
|
||||
self.cache_file = cache_file
|
||||
self.cache = {}
|
||||
self._cache_dirty = False
|
||||
self._lock = threading.Lock()
|
||||
self.error = None
|
||||
self.api_ok = None
|
||||
self._load_cache()
|
||||
|
||||
def _load_cache(self):
|
||||
if not self.cache_file.exists():
|
||||
return
|
||||
try:
|
||||
with open(self.cache_file, "r", encoding="utf-8") as f:
|
||||
self.cache = json.load(f)
|
||||
log_trans.info(f"Cache geladen: {len(self.cache)} Einträge")
|
||||
except Exception as e:
|
||||
log_trans.error(f"Cache-Lesefehler: {e}")
|
||||
self.cache = {}
|
||||
|
||||
def _save_cache(self):
|
||||
if not self._cache_dirty:
|
||||
return
|
||||
try:
|
||||
if len(self.cache) > TRANSLATION_CACHE_MAX + 200:
|
||||
items = list(self.cache.items())
|
||||
self.cache = dict(items[-TRANSLATION_CACHE_MAX:])
|
||||
with open(self.cache_file, "w", encoding="utf-8") as f:
|
||||
json.dump(self.cache, f, ensure_ascii=False, indent=1)
|
||||
self._cache_dirty = False
|
||||
except Exception as e:
|
||||
log_trans.error(f"Cache-Schreibfehler: {e}")
|
||||
|
||||
@staticmethod
|
||||
def _hash(text: str) -> str:
|
||||
return hashlib.sha1(text.encode("utf-8", errors="ignore")).hexdigest()[:16]
|
||||
|
||||
def _translate_one(self, text: str) -> str | None:
|
||||
try:
|
||||
from deep_translator import GoogleTranslator
|
||||
except ImportError:
|
||||
self.error = "pip install deep-translator"
|
||||
return None
|
||||
|
||||
try:
|
||||
if self.provider == "google":
|
||||
return GoogleTranslator(source="auto", target=self.target).translate(text)
|
||||
except Exception as e:
|
||||
log_trans.warning(f"Google-Fehler: {e}")
|
||||
|
||||
try:
|
||||
from deep_translator import MyMemoryTranslator
|
||||
src_full = "en-GB"
|
||||
tgt_full = f"{self.target}-{self.target.upper()}"
|
||||
return MyMemoryTranslator(source=src_full, target=tgt_full).translate(text)
|
||||
except Exception as e:
|
||||
log_trans.warning(f"MyMemory-Fehler: {e}")
|
||||
|
||||
return None
|
||||
|
||||
def translate(self, text: str) -> str:
|
||||
if not self.enabled or not text:
|
||||
return text
|
||||
h = self._hash(text)
|
||||
with self._lock:
|
||||
cached = self.cache.get(h)
|
||||
if cached:
|
||||
return cached
|
||||
result = self._translate_one(text)
|
||||
if result and result.strip() and result.lower() != text.lower():
|
||||
with self._lock:
|
||||
self.cache[h] = result
|
||||
self._cache_dirty = True
|
||||
self.api_ok = True
|
||||
return result
|
||||
if result is None:
|
||||
self.api_ok = False
|
||||
return text
|
||||
|
||||
def translate_batch(self, headlines: list) -> list:
|
||||
if not self.enabled:
|
||||
return headlines
|
||||
out = []
|
||||
for h in headlines:
|
||||
new = dict(h)
|
||||
orig = h.get("title", "")
|
||||
translated = self.translate(orig)
|
||||
if translated != orig:
|
||||
new["title_original"] = orig
|
||||
new["title"] = translated
|
||||
out.append(new)
|
||||
if self._cache_dirty:
|
||||
self._save_cache()
|
||||
return out
|
||||
|
||||
def shutdown(self):
|
||||
self._save_cache()
|
||||
|
||||
|
||||
# ══════════════════════════════════════════════
|
||||
# NEWS-FETCHER
|
||||
# ══════════════════════════════════════════════
|
||||
class NewsFetcher:
|
||||
"""
|
||||
Holt aktuelle WTI-Crude-News sowie geopolitische / makroökonomische News
|
||||
aus öffentlichen RSS-Feeds.
|
||||
|
||||
Energie-Feeds (WTI / Rohstoff-spezifisch):
|
||||
OilPrice, EIA Press, Rigzone, ShaleMag, Offshore Energy,
|
||||
Oil&Gas 360, Guardian Oil
|
||||
|
||||
Makro- / Geopolitik-Feeds (marktrelevant für WTI):
|
||||
Al Jazeera, BBC Business, BBC World, MarketWatch, The Hill Energy,
|
||||
DW World, FT Commodities, Middle East Eye, Hellenic Shipping News
|
||||
"""
|
||||
|
||||
# ── Energie / Rohstoffe — WTI-fokussiert ─────────────────────────────
|
||||
FEEDS_ENERGY = [
|
||||
("OilPrice", "https://oilprice.com/rss/main"),
|
||||
("EIA Press", "https://www.eia.gov/rss/press_rss.xml"),
|
||||
("Rigzone", "https://www.rigzone.com/news/rss/rigzone_latest.aspx"),
|
||||
("ShaleMag", "https://shalemag.com/feed/"),
|
||||
("Offshore Energy", "https://www.offshore-energy.biz/feed/"),
|
||||
("Oil&Gas 360", "https://www.oilandgas360.com/feed/"),
|
||||
("Guardian Oil", "https://www.theguardian.com/business/oil/rss"),
|
||||
]
|
||||
|
||||
# ── Makro / Geopolitik (marktrelevant für WTI) ───────────────────────
|
||||
# Reuters-Feed wurde 2020 abgeschaltet — nicht mehr verwenden.
|
||||
FEEDS_GEO = [
|
||||
("Al Jazeera", "https://www.aljazeera.com/xml/rss/all.xml"),
|
||||
("BBC Business", "https://feeds.bbci.co.uk/news/business/rss.xml"),
|
||||
("BBC World", "https://feeds.bbci.co.uk/news/world/rss.xml"),
|
||||
("MarketWatch", "https://feeds.marketwatch.com/marketwatch/marketpulse/"),
|
||||
("The Hill", "https://thehill.com/policy/energy-environment/feed/"),
|
||||
("DW World", "https://rss.dw.com/rdf/rss-en-world"),
|
||||
("FT Commodities","https://www.ft.com/commodities?format=rss"),
|
||||
("Middle East Eye","https://www.middleeasteye.net/rss"),
|
||||
("Hellenic Ship.", "https://www.hellenicshippingnews.com/feed/"),
|
||||
]
|
||||
|
||||
# Kombination beider Feed-Gruppen
|
||||
FEEDS = FEEDS_ENERGY + FEEDS_GEO
|
||||
|
||||
# Schlüsselwörter zur Relevanz-Filterung (Geo-Feeds)
|
||||
GEO_KEYWORDS = {
|
||||
# Öl/Energie allgemein
|
||||
"oil", "crude", "wti", "brent", "petroleum", "energy",
|
||||
"opec", "iea", "eia", "barrel", "supply", "demand",
|
||||
"refin", "pipeline", "tanker", "stockpile", "inventory", "stocks",
|
||||
"spr", "gas", "fuel", "commodity", "commodities", "lng",
|
||||
# Schifffahrt / Tankerrouten — direkt preisrelevant
|
||||
"shipping", "vessel", "suez", "canal", "chokepoint",
|
||||
"bab-el-mandeb", "vlcc", "freight", "cargo",
|
||||
# WTI-spezifisch: US-Produktion & Infrastruktur
|
||||
"cushing", "permian", "bakken", "eagle ford", "marcellus",
|
||||
"keystone", "gulf of mexico", "nymex", "shale", "fracking",
|
||||
"us oil", "us energy", "us crude", "us production",
|
||||
"refinery capacity", "hurricane",
|
||||
# Sanktionen / Geopolitik
|
||||
"iran", "sanction", "embargo", "strait", "hormuz", "gulf",
|
||||
# Akteure
|
||||
"saudi", "russia", "moscow", "putin", "iraq",
|
||||
"venezuela", "libya", "nigeria", "kuwait", "uae", "qatar",
|
||||
"ukraine", "hamas", "houthi", "yemen", "israel", "hezbollah",
|
||||
"lebanon", "syria", "china", "beijing", "iran",
|
||||
# Ereignisse
|
||||
"war", "attack", "ceasefire", "truce", "drone", "missile",
|
||||
"embargo", "blockade", "strike", "shutdown", "outage",
|
||||
"disruption", "force majeure", "summit", "deal", "agreement",
|
||||
"tariff", "trade war", "recession", "inflation", "gdp",
|
||||
}
|
||||
|
||||
# ── Vollständige Feed-Pools inkl. Alternativen (Reihenfolge = Priorität) ──
|
||||
# validate_feeds() testet alle Feeds parallel und ersetzt ausgefallene
|
||||
# primäre Feeds automatisch mit dem nächsten funktionierenden Pool-Eintrag.
|
||||
_ENERGY_POOL = [
|
||||
# Primär
|
||||
("OilPrice", "https://oilprice.com/rss/main"),
|
||||
("EIA Press", "https://www.eia.gov/rss/press_rss.xml"),
|
||||
("Rigzone", "https://www.rigzone.com/news/rss/rigzone_latest.aspx"),
|
||||
("ShaleMag", "https://shalemag.com/feed/"),
|
||||
("Offshore Energy", "https://www.offshore-energy.biz/feed/"),
|
||||
("Oil&Gas 360", "https://www.oilandgas360.com/feed/"),
|
||||
("Guardian Oil", "https://www.theguardian.com/business/oil/rss"),
|
||||
# Alternativen (automatischer Fallback)
|
||||
("Energy Monitor", "https://www.energymonitor.ai/feed/"),
|
||||
("Natural Gas Int.", "https://www.naturalgasintel.com/feed/"),
|
||||
]
|
||||
|
||||
_GEO_POOL = [
|
||||
# Primär
|
||||
("Al Jazeera", "https://www.aljazeera.com/xml/rss/all.xml"),
|
||||
("BBC Business", "https://feeds.bbci.co.uk/news/business/rss.xml"),
|
||||
("BBC World", "https://feeds.bbci.co.uk/news/world/rss.xml"),
|
||||
("MarketWatch", "https://feeds.marketwatch.com/marketwatch/marketpulse/"),
|
||||
("The Hill", "https://thehill.com/policy/energy-environment/feed/"),
|
||||
("DW World", "https://rss.dw.com/rdf/rss-en-world"),
|
||||
("FT Commodities", "https://www.ft.com/commodities?format=rss"),
|
||||
("Middle East Eye", "https://www.middleeasteye.net/rss"),
|
||||
("Hellenic Ship.", "https://www.hellenicshippingnews.com/feed/"),
|
||||
# Alternativen (automatischer Fallback)
|
||||
("New Arab", "https://www.newarab.com/rss.xml"),
|
||||
]
|
||||
|
||||
# WTI-Direktseite (HTML-Scraper, keine RSS)
|
||||
OILPRICE_WTI_URL = "https://oilprice.com/futures/wti/"
|
||||
# Globale Öl-Preistabelle (Breadth + WTI/Brent Spot)
|
||||
OILPRICE_CHARTS_URL = "https://oilprice.com/oil-price-charts/"
|
||||
|
||||
# HTTP-Timeout pro Feed in Sekunden — verhindert dass langsame/tote
|
||||
# Feeds den ganzen Fetch-Thread blockieren.
|
||||
FETCH_TIMEOUT_S = 8
|
||||
|
||||
def __init__(self, translator: 'NewsTranslator | None' = None):
|
||||
self.headlines = []
|
||||
self.last_fetch = None
|
||||
self.error = None
|
||||
self.translator = translator
|
||||
self.sentiment = {"score": 0.0, "n_bull": 0, "n_bear": 0, "samples": []}
|
||||
self.price_data: dict | None = None # OilPrice Charts Breadth + Spot
|
||||
# Pro-Feed Status: {source: {"ok": bool, "n": int, "msg": str}}
|
||||
self.feed_status: dict = {}
|
||||
# Validierte, aktive Feed-Listen (None = noch nicht validiert → Klassenvariable)
|
||||
self._active_energy: list | None = None
|
||||
self._active_geo: list | None = None
|
||||
self._lock = threading.Lock()
|
||||
|
||||
def _is_geo_relevant(self, title: str) -> bool:
|
||||
"""Gibt True zurück, wenn ein Geo-Feed-Titel marktrelevante Keywords enthält."""
|
||||
low = title.lower()
|
||||
return any(kw in low for kw in self.GEO_KEYWORDS)
|
||||
|
||||
# ══════════════════════════════════════════════
|
||||
# FEED-VALIDIERUNG (einmalig beim Start)
|
||||
# ══════════════════════════════════════════════
|
||||
|
||||
def _test_url(self, source: str, url: str) -> tuple[bool, int, str]:
|
||||
"""Schneller Feed-Test ohne feed_status zu verändern. Gibt (ok, n_entries, msg) zurück."""
|
||||
try:
|
||||
import feedparser, requests
|
||||
except ImportError:
|
||||
return False, 0, "missing: feedparser/requests"
|
||||
try:
|
||||
resp = requests.get(
|
||||
url,
|
||||
timeout=(2, 6), # kurze Timeouts für schnelle Validierung
|
||||
headers={
|
||||
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
|
||||
"AppleWebKit/537.36 OilWidget/1.0",
|
||||
"Accept": "application/rss+xml, application/xml, text/xml, */*",
|
||||
},
|
||||
)
|
||||
if resp.status_code != 200:
|
||||
return False, 0, f"HTTP {resp.status_code}"
|
||||
feed = feedparser.parse(resp.content)
|
||||
n = len(feed.entries)
|
||||
if n == 0:
|
||||
return False, 0, "0 Einträge"
|
||||
return True, n, "ok"
|
||||
except Exception as e:
|
||||
return False, 0, str(e)[:60]
|
||||
|
||||
def validate_feeds(self) -> dict:
|
||||
"""
|
||||
Testet alle Feeds aus _ENERGY_POOL / _GEO_POOL parallel.
|
||||
Ausgefallene Primär-Feeds werden automatisch durch den nächsten
|
||||
funktionierenden Pool-Eintrag ersetzt.
|
||||
Ergebnis wird in self._active_energy / self._active_geo gespeichert.
|
||||
|
||||
Aufruf: einmalig beim Start in einem Daemon-Thread.
|
||||
"""
|
||||
import concurrent.futures
|
||||
|
||||
all_feeds = list({u: (s, u) for s, u in
|
||||
self._ENERGY_POOL + self._GEO_POOL}.values())
|
||||
|
||||
log_news.info(f"Feed-Validierung: teste {len(all_feeds)} Feeds …")
|
||||
|
||||
# Parallel testen
|
||||
test_results: dict[str, tuple[bool, int, str]] = {} # url → (ok, n, msg)
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=12) as ex:
|
||||
fut_map = {ex.submit(self._test_url, s, u): (s, u) for s, u in all_feeds}
|
||||
try:
|
||||
for fut in concurrent.futures.as_completed(fut_map, timeout=25):
|
||||
src, url = fut_map[fut]
|
||||
try:
|
||||
ok, n, msg = fut.result()
|
||||
except Exception as e:
|
||||
ok, n, msg = False, 0, str(e)[:50]
|
||||
test_results[url] = (ok, n, msg)
|
||||
status = "OK" if ok else "FAIL"
|
||||
log_news.debug(f" [{status}] {src}: {msg} ({n} Einträge)")
|
||||
except concurrent.futures.TimeoutError:
|
||||
log_news.warning("Feed-Validierung: Timeout nach 25 s")
|
||||
|
||||
def build_active(primary: list, pool: list) -> tuple[list, list]:
|
||||
"""
|
||||
Gibt (aktive_feeds, ersetzungen) zurück.
|
||||
Primäre Feeds haben Vorrang; ausgefallene werden mit dem
|
||||
nächsten noch nicht verwendeten Pool-Feed ersetzt.
|
||||
"""
|
||||
primary_urls = {u for _, u in primary}
|
||||
alternatives = [(s, u) for s, u in pool if u not in primary_urls]
|
||||
|
||||
active = []
|
||||
used_urls: set[str] = set()
|
||||
replacements = []
|
||||
|
||||
for src, url in primary:
|
||||
ok, n, msg = test_results.get(url, (False, 0, "nicht getestet"))
|
||||
if ok and n > 0:
|
||||
active.append((src, url))
|
||||
used_urls.add(url)
|
||||
else:
|
||||
log_news.warning(f"Feed ausgefallen: {src} [{msg}] → suche Ersatz")
|
||||
replaced = False
|
||||
for alt_src, alt_url in alternatives:
|
||||
if alt_url in used_urls:
|
||||
continue
|
||||
alt_ok, alt_n, alt_msg = test_results.get(alt_url, (False, 0, "nicht getestet"))
|
||||
if alt_ok and alt_n > 0:
|
||||
active.append((alt_src, alt_url))
|
||||
used_urls.add(alt_url)
|
||||
replacements.append((src, alt_src))
|
||||
log_news.info(f" → Ersetzt: {src} → {alt_src}")
|
||||
replaced = True
|
||||
break
|
||||
if not replaced:
|
||||
log_news.warning(f" → Kein Ersatz verfügbar für {src}")
|
||||
|
||||
return active, replacements
|
||||
|
||||
active_e, rep_e = build_active(self.FEEDS_ENERGY, self._ENERGY_POOL)
|
||||
active_g, rep_g = build_active(self.FEEDS_GEO, self._GEO_POOL)
|
||||
|
||||
with self._lock:
|
||||
self._active_energy = active_e
|
||||
self._active_geo = active_g
|
||||
|
||||
total_ok = sum(1 for ok, _, _ in test_results.values() if ok)
|
||||
total_fail = len(test_results) - total_ok
|
||||
log_news.info(
|
||||
f"Feed-Validierung abgeschlossen: {total_ok} OK / {total_fail} ausgefallen "
|
||||
f"| Energie: {len(active_e)} aktiv ({len(rep_e)} ersetzt) "
|
||||
f"| Geo: {len(active_g)} aktiv ({len(rep_g)} ersetzt)"
|
||||
)
|
||||
return {
|
||||
"energy_active": active_e,
|
||||
"geo_active": active_g,
|
||||
"replacements": rep_e + rep_g,
|
||||
"n_ok": total_ok,
|
||||
"n_fail": total_fail,
|
||||
}
|
||||
|
||||
def _fetch_oilprice_wti_scrape(self) -> list:
|
||||
"""
|
||||
Scrapet https://oilprice.com/futures/wti/ nach WTI-spezifischen Headlines.
|
||||
Nutzt BeautifulSoup wenn verfügbar, sonst Regex-Fallback.
|
||||
Gibt bis zu 8 Einträge im selben Format wie RSS-Feeds zurück.
|
||||
"""
|
||||
try:
|
||||
import requests
|
||||
except ImportError:
|
||||
return []
|
||||
|
||||
try:
|
||||
resp = requests.get(
|
||||
self.OILPRICE_WTI_URL,
|
||||
timeout=(3, self.FETCH_TIMEOUT_S),
|
||||
headers={
|
||||
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
|
||||
"AppleWebKit/537.36 Chrome/120.0 OilWidget/1.0",
|
||||
"Accept": "text/html,application/xhtml+xml,*/*",
|
||||
"Accept-Language": "en-US,en;q=0.9",
|
||||
},
|
||||
)
|
||||
except Exception as e:
|
||||
log_news.warning(f"OilPrice WTI scrape: {e}")
|
||||
self.feed_status["OilPrice WTI"] = {"ok": False, "n": 0, "msg": str(e)[:60]}
|
||||
return []
|
||||
|
||||
if resp.status_code != 200:
|
||||
log_news.warning(f"OilPrice WTI: HTTP {resp.status_code}")
|
||||
self.feed_status["OilPrice WTI"] = {
|
||||
"ok": False, "n": 0, "msg": f"HTTP {resp.status_code}"}
|
||||
return []
|
||||
|
||||
items = []
|
||||
now = time.time()
|
||||
seen = set()
|
||||
|
||||
# Artikel-Pfade auf OilPrice.com: /Energy/... und /Latest-Energy-News/...
|
||||
_ARTICLE_PATHS = ("/Energy/", "/Latest-Energy-News/")
|
||||
|
||||
try:
|
||||
from bs4 import BeautifulSoup
|
||||
soup = BeautifulSoup(resp.content, "html.parser")
|
||||
|
||||
for tag in soup.find_all("a", href=True):
|
||||
href = tag.get("href", "")
|
||||
title = tag.get_text(strip=True)
|
||||
# Nur Artikel-Pfade, keine Navigation/Werbung
|
||||
if not any(p in href for p in _ARTICLE_PATHS):
|
||||
continue
|
||||
if len(title) < 25 or title in seen:
|
||||
continue
|
||||
if not href.startswith("http"):
|
||||
href = ("https://oilprice.com" + href
|
||||
if href.startswith("/") else "")
|
||||
if not href:
|
||||
continue
|
||||
seen.add(title)
|
||||
items.append({
|
||||
"title": title,
|
||||
"link": href,
|
||||
"source": "OilPrice WTI",
|
||||
"ts": now,
|
||||
"category": "energy",
|
||||
})
|
||||
if len(items) >= 8:
|
||||
break
|
||||
|
||||
except ImportError:
|
||||
# Regex-Fallback: kein beautifulsoup4 installiert
|
||||
import re
|
||||
pat = re.compile(
|
||||
r'<a[^>]+href="(https?://oilprice\.com/'
|
||||
r'(?:Energy|Latest-Energy-News)/[^"#?]{10,})"[^>]*>'
|
||||
r'\s*([^<\n]{25,200})\s*</a>',
|
||||
re.IGNORECASE | re.DOTALL,
|
||||
)
|
||||
for href, raw in pat.findall(resp.text):
|
||||
title = re.sub(r'\s+', ' ', raw).strip()
|
||||
if not title or title in seen:
|
||||
continue
|
||||
seen.add(title)
|
||||
items.append({
|
||||
"title": title,
|
||||
"link": href,
|
||||
"source": "OilPrice WTI",
|
||||
"ts": now,
|
||||
"category": "energy",
|
||||
})
|
||||
if len(items) >= 8:
|
||||
break
|
||||
if not items:
|
||||
log_news.info("OilPrice WTI: BeautifulSoup nicht installiert "
|
||||
"(pip install beautifulsoup4 lxml) — Regex-Fallback aktiv")
|
||||
|
||||
self.feed_status["OilPrice WTI"] = {
|
||||
"ok": len(items) > 0,
|
||||
"n": len(items),
|
||||
"msg": "scrape ok" if items else "0 Headlines",
|
||||
}
|
||||
log_news.info(f"OilPrice WTI scrape: {len(items)} Headlines")
|
||||
return items
|
||||
|
||||
def _fetch_oilprice_charts(self) -> dict | None:
|
||||
"""
|
||||
Scrapet https://oilprice.com/oil-price-charts/ und extrahiert:
|
||||
• WTI Crude und Brent Crude: Preis, absolute Änderung, %Änderung
|
||||
• Market Breadth aus Tabelle 'Futures & Indexes':
|
||||
Anteil der Öl-Benchmarks mit positivem Tageschange
|
||||
• breadth_signal: -1.0 (alle runter) … +1.0 (alle rauf)
|
||||
|
||||
Breadth > +0.5 → bullisher Marktkontext
|
||||
Breadth < -0.5 → bärischer Marktkontext
|
||||
"""
|
||||
try:
|
||||
import requests
|
||||
except ImportError:
|
||||
return None
|
||||
|
||||
try:
|
||||
resp = requests.get(
|
||||
self.OILPRICE_CHARTS_URL,
|
||||
timeout=(3, self.FETCH_TIMEOUT_S),
|
||||
headers={
|
||||
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
|
||||
"AppleWebKit/537.36 Chrome/120.0 OilWidget/1.0",
|
||||
"Accept": "text/html,application/xhtml+xml,*/*",
|
||||
"Accept-Language": "en-US,en;q=0.9",
|
||||
},
|
||||
)
|
||||
except Exception as e:
|
||||
log_news.warning(f"OilPrice Charts fetch: {e}")
|
||||
return None
|
||||
|
||||
if resp.status_code != 200:
|
||||
log_news.warning(f"OilPrice Charts: HTTP {resp.status_code}")
|
||||
return None
|
||||
|
||||
try:
|
||||
from bs4 import BeautifulSoup
|
||||
except ImportError:
|
||||
log_news.warning("OilPrice Charts: BeautifulSoup fehlt (pip install beautifulsoup4)")
|
||||
return None
|
||||
|
||||
try:
|
||||
soup = BeautifulSoup(resp.content, "html.parser")
|
||||
tables = soup.find_all("table", class_="oilprices__table")
|
||||
if not tables:
|
||||
log_news.warning("OilPrice Charts: Tabelle nicht gefunden")
|
||||
return None
|
||||
|
||||
t0 = tables[0] # "Futures & Indexes"
|
||||
n_up = n_down = 0
|
||||
wti = brent = None
|
||||
|
||||
for row in t0.find_all("tr"):
|
||||
cells = row.find_all(["th", "td"])
|
||||
if len(cells) < 4:
|
||||
continue
|
||||
name_cell = cells[1]
|
||||
price_cell = cells[2]
|
||||
change_cell = cells[3]
|
||||
pct_cell = cells[4] if len(cells) > 4 else None
|
||||
|
||||
name = name_cell.get_text(strip=True)
|
||||
try:
|
||||
price = float(price_cell.get_text(strip=True).replace(",", ""))
|
||||
except ValueError:
|
||||
continue
|
||||
|
||||
change_text = change_cell.get_text(strip=True)
|
||||
change_classes = change_cell.get("class", [])
|
||||
try:
|
||||
change = float(change_text.replace(",", ""))
|
||||
except ValueError:
|
||||
change = 0.0
|
||||
|
||||
is_up = any("up" in c for c in change_classes)
|
||||
is_down = any("down" in c for c in change_classes)
|
||||
if is_up:
|
||||
n_up += 1
|
||||
elif is_down:
|
||||
n_down += 1
|
||||
|
||||
if name in ("WTI Crude", "Brent Crude"):
|
||||
pct = 0.0
|
||||
if pct_cell:
|
||||
raw_pct = pct_cell.get_text(strip=True).split("(")[0]
|
||||
try:
|
||||
pct = float(raw_pct.replace("%", "").replace(",", ""))
|
||||
except ValueError:
|
||||
pass
|
||||
entry = {"price": price, "change": change, "pct": pct}
|
||||
if name == "WTI Crude":
|
||||
wti = entry
|
||||
else:
|
||||
brent = entry
|
||||
|
||||
n_total = n_up + n_down
|
||||
breadth_pct = round(n_up / n_total * 100, 1) if n_total else 50.0
|
||||
breadth_signal = round((n_up - n_down) / n_total, 3) if n_total else 0.0
|
||||
|
||||
result = {
|
||||
"wti": wti,
|
||||
"brent": brent,
|
||||
"n_up": n_up,
|
||||
"n_down": n_down,
|
||||
"n_total": n_total,
|
||||
"breadth_pct": breadth_pct, # % der Futures die gestiegen sind
|
||||
"breadth_signal": breadth_signal, # -1..+1
|
||||
}
|
||||
|
||||
wti_str = f"WTI={wti['price']:.2f} ({wti['pct']:+.2f}%)" if wti else "WTI=?"
|
||||
brent_str = f"Brent={brent['price']:.2f} ({brent['pct']:+.2f}%)" if brent else "Brent=?"
|
||||
log_news.info(
|
||||
f"OilPrice Charts: {wti_str} {brent_str} "
|
||||
f"Breadth {breadth_pct:.0f}% ({n_up}up/{n_down}dn) signal={breadth_signal:+.2f}"
|
||||
)
|
||||
return result
|
||||
|
||||
except Exception as e:
|
||||
log_news.warning(f"OilPrice Charts parse: {e}", exc_info=True)
|
||||
return None
|
||||
|
||||
def price_data_snapshot(self) -> dict | None:
|
||||
"""Thread-safe Kopie der letzten Preisdaten (OilPrice Charts)."""
|
||||
with self._lock:
|
||||
return dict(self.price_data) if self.price_data else None
|
||||
|
||||
def _load_feed(self, source: str, url: str):
|
||||
"""
|
||||
Lädt eine einzelne RSS-URL mit hardem HTTP-Timeout.
|
||||
Liefert die feedparser-Feed oder None bei Fehler.
|
||||
Setzt feed_status[source] für spätere Diagnose.
|
||||
"""
|
||||
try:
|
||||
import feedparser
|
||||
import requests
|
||||
except ImportError as e:
|
||||
self.feed_status[source] = {"ok": False, "n": 0, "msg": f"missing: {e}"}
|
||||
return None
|
||||
|
||||
try:
|
||||
resp = requests.get(
|
||||
url,
|
||||
timeout=(3, self.FETCH_TIMEOUT_S),
|
||||
headers={
|
||||
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
|
||||
"AppleWebKit/537.36 OilWidget/1.0",
|
||||
"Accept": "application/rss+xml, application/xml, text/xml, */*",
|
||||
},
|
||||
)
|
||||
if resp.status_code != 200:
|
||||
self.feed_status[source] = {
|
||||
"ok": False, "n": 0,
|
||||
"msg": f"HTTP {resp.status_code}",
|
||||
}
|
||||
log_news.warning(f"{source}: HTTP {resp.status_code}")
|
||||
return None
|
||||
feed = feedparser.parse(resp.content)
|
||||
if feed.bozo and not feed.entries:
|
||||
# bozo=1 mit Entries ist meist nur "namespace warning" → OK
|
||||
msg = str(feed.bozo_exception)[:80] if hasattr(feed, "bozo_exception") else "parse error"
|
||||
self.feed_status[source] = {"ok": False, "n": 0, "msg": msg}
|
||||
log_news.warning(f"{source}: {msg}")
|
||||
return None
|
||||
n = len(feed.entries)
|
||||
self.feed_status[source] = {"ok": True, "n": n, "msg": "ok"}
|
||||
return feed
|
||||
except Exception as e:
|
||||
self.feed_status[source] = {"ok": False, "n": 0, "msg": str(e)[:80]}
|
||||
log_news.warning(f"{source}: {e}")
|
||||
return None
|
||||
|
||||
def fetch(self):
|
||||
try:
|
||||
import feedparser # nur für Verfügbarkeits-Check
|
||||
except ImportError:
|
||||
with self._lock:
|
||||
self.error = "pip install feedparser requests"
|
||||
return
|
||||
|
||||
# Per-Feed Status für diesen Run leeren
|
||||
self.feed_status = {}
|
||||
energy_items = []
|
||||
geo_items = []
|
||||
|
||||
# ── OilPrice WTI-Direktseite (Scraper, höchste Priorität) ──
|
||||
wti_items = self._fetch_oilprice_wti_scrape()
|
||||
energy_items.extend(wti_items)
|
||||
|
||||
# ── OilPrice Charts — Preis-Breadth ──
|
||||
charts_data = self._fetch_oilprice_charts()
|
||||
|
||||
# Validierte Listen verwenden (nach validate_feeds()), sonst Klassen-Defaults
|
||||
with self._lock:
|
||||
energy_feeds = self._active_energy if self._active_energy is not None else self.FEEDS_ENERGY
|
||||
geo_feeds = self._active_geo if self._active_geo is not None else self.FEEDS_GEO
|
||||
|
||||
# ── Energie-Feeds: bis zu 5 Einträge je Feed, keine Filterung ──
|
||||
for source, url in energy_feeds:
|
||||
feed = self._load_feed(source, url)
|
||||
if feed is None:
|
||||
continue
|
||||
for entry in feed.entries[:5]:
|
||||
pub = entry.get("published_parsed") or time.gmtime()
|
||||
energy_items.append({
|
||||
"title": entry.get("title", "(no title)").strip(),
|
||||
"link": entry.get("link", ""),
|
||||
"source": source,
|
||||
"ts": time.mktime(pub),
|
||||
"category": "energy",
|
||||
})
|
||||
|
||||
# ── Geo-Feeds: bis zu 3 Einträge je Feed, nur relevante Titel ──
|
||||
for source, url in geo_feeds:
|
||||
feed = self._load_feed(source, url)
|
||||
if feed is None:
|
||||
continue
|
||||
count = 0
|
||||
for entry in feed.entries:
|
||||
if count >= 3:
|
||||
break
|
||||
title = entry.get("title", "").strip()
|
||||
if not self._is_geo_relevant(title):
|
||||
continue
|
||||
pub = entry.get("published_parsed") or time.gmtime()
|
||||
geo_items.append({
|
||||
"title": title,
|
||||
"link": entry.get("link", ""),
|
||||
"source": source,
|
||||
"ts": time.mktime(pub),
|
||||
"category": "geo",
|
||||
})
|
||||
count += 1
|
||||
|
||||
# Diagnose-Zeilen ins Log
|
||||
ok_count = sum(1 for s in self.feed_status.values() if s["ok"])
|
||||
fail_count = len(self.feed_status) - ok_count
|
||||
log_news.info(
|
||||
f"Feed-Status: {ok_count} OK, {fail_count} fehlgeschlagen"
|
||||
)
|
||||
for source, st in self.feed_status.items():
|
||||
if st["ok"]:
|
||||
log_news.debug(f" ✓ {source}: {st['n']} Einträge")
|
||||
else:
|
||||
log_news.info(f" ✗ {source}: {st['msg']}")
|
||||
|
||||
# ── Zusammenführen: WTI-Scraper + RSS Energie + Geo ──
|
||||
# WTI-Scraper-Items haben ts=now → sortieren immer an erste Stelle wenn aktuell
|
||||
energy_items.sort(key=lambda x: x["ts"], reverse=True)
|
||||
geo_items.sort(key=lambda x: x["ts"], reverse=True)
|
||||
|
||||
# WTI-Scraper bekommt bis zu 4 Slots im Energy-Bucket (von 7 gesamt)
|
||||
wti_out = [i for i in energy_items if i.get("source") == "OilPrice WTI"][:4]
|
||||
rss_energy = [i for i in energy_items if i.get("source") != "OilPrice WTI"][:3]
|
||||
items = wti_out + rss_energy + geo_items[:4]
|
||||
items.sort(key=lambda x: x["ts"], reverse=True)
|
||||
items = items[:10]
|
||||
|
||||
log_news.info(
|
||||
f"→ WTI-Scraper:{len(wti_out)} + Energie-RSS:{len(rss_energy)} + "
|
||||
f"Geo:{len(geo_items[:4])} = {len(items)} Headlines"
|
||||
)
|
||||
|
||||
if self.translator and self.translator.enabled:
|
||||
try:
|
||||
items = self.translator.translate_batch(items)
|
||||
except Exception as e:
|
||||
log_news.warning(f"Übersetzung fehlgeschlagen: {e}")
|
||||
|
||||
# Sentiment auf den (ggf. übersetzten) Headlines berechnen.
|
||||
# calc_news_sentiment() greift bevorzugt auf title_original (englisch) zurück.
|
||||
try:
|
||||
from core.analysis import calc_news_sentiment
|
||||
sentiment = calc_news_sentiment(items)
|
||||
except Exception as e:
|
||||
log_news.warning(f"Sentiment-Berechnung fehlgeschlagen: {e}")
|
||||
sentiment = {"score": 0.0, "n_bull": 0, "n_bear": 0, "samples": []}
|
||||
|
||||
# ── Breadth-Signal in Sentiment einblenden (Gewicht 25%) ──────────────
|
||||
# Breadth misst ob die Mehrheit aller Öl-Benchmarks steigt/fällt.
|
||||
# Nur bei starkem Signal (|breadth| > 0.4) und ohne starke News-Gegenmeinung.
|
||||
if charts_data:
|
||||
bs = charts_data.get("breadth_signal", 0.0)
|
||||
if abs(bs) > 0.4:
|
||||
raw_score = sentiment["score"]
|
||||
blended = round(raw_score * 0.75 + bs * 0.25, 3)
|
||||
blended = max(-1.0, min(1.0, blended))
|
||||
sentiment = dict(sentiment)
|
||||
sentiment["score"] = blended
|
||||
sentiment["breadth_signal"] = bs
|
||||
sentiment["breadth_pct"] = charts_data.get("breadth_pct", 50.0)
|
||||
log_news.info(
|
||||
f"Breadth {bs:+.2f} → Sentiment {raw_score:+.2f} → {blended:+.2f}"
|
||||
)
|
||||
|
||||
with self._lock:
|
||||
self.headlines = items
|
||||
self.last_fetch = time.time()
|
||||
self.error = None if items else "Keine News-Feeds erreichbar"
|
||||
self.sentiment = sentiment
|
||||
self.price_data = charts_data
|
||||
|
||||
log_news.info(f"News-Sentiment: {sentiment['score']:+.2f} "
|
||||
f"(bull={sentiment['n_bull']}, bear={sentiment['n_bear']})")
|
||||
|
||||
def snapshot(self):
|
||||
with self._lock:
|
||||
return list(self.headlines), self.last_fetch, self.error
|
||||
|
||||
def sentiment_snapshot(self) -> dict:
|
||||
"""Thread-safe Kopie des letzten Sentiment-Snapshots."""
|
||||
with self._lock:
|
||||
return dict(self.sentiment)
|
||||
|
||||
def storm_state(self) -> dict:
|
||||
"""News-Sturm-Indikator (REINE ANZEIGE, kein Signal): Headline-Rate der
|
||||
letzten Stunde vs. Basisrate des vorhandenen Feed-Fensters. Viel frische
|
||||
Öl-/Geo-Schlagzeilen auf einmal = Ereignis läuft → Vorsicht (Vola).
|
||||
normal · elevated (≥3 frisch & ≥2× Basis) · storm (≥5 frisch & ≥3× Basis)."""
|
||||
with self._lock:
|
||||
items = [h for h in self.headlines if h.get("ts")]
|
||||
now = time.time()
|
||||
if not items:
|
||||
return {"level": "normal", "fresh": 0, "base_per_h": 0.0}
|
||||
fresh = sum(1 for h in items if now - h["ts"] <= 3600)
|
||||
# Basis-Fenster min. 6 h verankern — sonst bläht ein frischer Burst die
|
||||
# Basisrate selbst auf und der Sturm erkennt sich nicht (Selbst-Normierung).
|
||||
span_h = min(48.0, max(6.0, (now - min(h["ts"] for h in items)) / 3600))
|
||||
base = len(items) / span_h # Ø Headlines/Stunde im Fenster
|
||||
level = "normal"
|
||||
if fresh >= 5 and fresh >= 3 * base:
|
||||
level = "storm"
|
||||
elif fresh >= 3 and fresh >= 2 * base:
|
||||
level = "elevated"
|
||||
return {"level": level, "fresh": fresh, "base_per_h": round(base, 1)}
|
||||
Reference in New Issue
Block a user