Initial commit: Trading Portal - FastAPI + React + PostgreSQL

backend/scripts/apply_top100.py

@@ -0,0 +1,71 @@
+#!/usr/bin/env python3
+"""Quick: just update is_trading flag from pre-computed top 100 base symbols."""
+import os
+import asyncio
+import asyncpg
+
+DB_URL = os.environ.get("DATABASE_URL", "postgresql://trading:***@db:5432/trading_portal")
+# Convert asyncpg URL to postgresql
+if "+asyncpg" in DB_URL:
+    DB_URL = DB_URL.replace("+asyncpg", "")
+
+TOP100_BASES = [
+    "BTC","ETH","SOL","XRP","USD1","DOGE","XLM","BNB","NEAR","ENA",
+    "WLD","ADA","SUI","RE","AAVE","TRX","TAO","LTC","XPL","LINK",
+    "AVAX","PYTH","BCH","DYDX","ORDI","UNI","HBAR","JTO","ONDO","DOT",
+    "TIA","TRUMP","INJ","APT","BIO","FIL","ARB","MANTA","MEGA","ACT",
+    "ZBT","ESP","ICP","ASTER","CHZ","OP","ID","ZRO","RENDER","HEI",
+    "VIRTUAL","OPG","ATOM","PENDLE","SEI","STRK","FF","ALLO","ALGO","LDO",
+    "EIGEN","CRV","BICO","APE","ETHFI","POL","KITE","ZAMA","ETC","AXS",
+    "STG","S","RESOLV","SAHARA","SKY","MORPHO","CHIP","IOTA","LRC","SAND",
+    "EGLD","BEL","GRT","CAKE","AR","PORTAL","TNSR","IO","ME","KAITO",
+    "HOME","SPK","MITO","AVNT","MMT","OPN","CFG","GENIUS","NEO","ONT",
+]
+
+
+async def main():
+    conn = await asyncpg.connect(DB_URL)
+    try:
+        async with conn.transaction():
+            await conn.execute("UPDATE symbols SET is_trading = false")
+            result = await conn.execute(
+                """
+                UPDATE symbols s
+                SET is_trading = true
+                FROM exchanges e
+                WHERE s.exchange_id = e.id
+                  AND s.base = ANY($1::text[])
+                  AND s.quote = 'USDT'
+                  AND s.symbol NOT LIKE '%:%'   -- exclude futures/options (USDT:USDT-...)
+                  AND s.is_active = true
+                  AND e.is_active = true
+                """,
+                TOP100_BASES,
+            )
+            print(f"UPDATE result: {result}")
+
+        verify = await conn.fetch("""
+            SELECT e.name as exchange, COUNT(*) as cnt
+            FROM symbols s
+            JOIN exchanges e ON e.id = s.exchange_id
+            WHERE s.is_trading = true
+            GROUP BY e.name
+            ORDER BY e.name
+        """)
+        total = 0
+        print("=== SYMBOLS PER EXCHANGE (is_trading=true) ===")
+        for row in verify:
+            print(f"  {row['exchange']}: {row['cnt']}")
+            total += row['cnt']
+        print(f"  TOTAL: {total}")
+
+        # Check missing bases
+        db_bases = await conn.fetch("SELECT DISTINCT base FROM symbols WHERE is_trading = true")
+        db_set = {r["base"] for r in db_bases}
+        missing = [b for b in TOP100_BASES if b not in db_set]
+        if missing:
+            print(f"WARNING: bases NOT in DB: {missing}")
+    finally:
+        await conn.close()
+
+asyncio.run(main())

backend/scripts/backtest.py

@@ -0,0 +1,458 @@
+"""Backtest script: replay historical candles with PnL + Win Rate simulation.
+
+Usage:
+  docker exec trading-backend python3 /opt/backtest.py --symbol BTC/USDT --timeframe 30m --days 30 --trade-size 10
+
+Or from host:
+  python3 /opt/data/trading-portal/backend/scripts/backtest.py --symbol BTC/USDT --timeframe 30m --days 30
+"""
+
+import asyncio
+import argparse
+from datetime import datetime, timedelta, timezone
+from decimal import Decimal
+from collections import defaultdict
+
+import sys
+sys.path.insert(0, "/app")
+
+from app.database import async_session_factory
+from app.models.candle import Candle
+from app.models.symbol import Symbol
+from app.models.exchange import Exchange
+from sqlalchemy import select, and_
+from app.services.indicator_service import (
+    bollinger_bands, rsi, sma, macd, supertrend,
+    volume_breakout, ichimoku, detect_divergence, market_structure,
+)
+from app.services.signal_service import (
+    _classify_signal_combined, _classify_signal_bb, _detect_squeeze,
+    SQUEEZE_ALERT, CAUTION_LONG, CAUTION_SHORT,
+    STRONG_BUY, BUY, STRONG_SELL, SELL,
+)
+
+
+TRADE_SIZE = Decimal("10")  # USDT per trade
+MAX_HOLD_CANDLES = 48        # Max candles before auto-close (48 * 30m = 24h for 30m)
+
+# P1-24: Realistic trading costs for backtest
+SPREAD_BPS = 5        # 0.05% spread (half-spread per side)
+FEE_BPS = 10          # 0.10% fee (maker 0.05% + taker 0.05%)
+SLIPPAGE_BPS = 5      # 0.05% slippage
+TOTAL_COST_PCT = (SPREAD_BPS + FEE_BPS + SLIPPAGE_BPS) / 10000.0
+
+
+def _net_pnl(entry: float, exit: float, qty: float, direction: str) -> tuple[float, float]:
+    """Compute PnL with spread, fee, and slippage costs deducted."""
+    notional = entry * qty
+    cost = notional * TOTAL_COST_PCT  # entry cost (spread+fee+slippage)
+    # Also pay cost on exit for realistic round-trip
+    exit_notional = exit * qty
+    exit_cost = exit_notional * TOTAL_COST_PCT
+
+    if direction == "LONG":
+        gross = (exit - entry) * qty
+    else:
+        gross = (entry - exit) * qty
+
+    net = gross - cost - exit_cost
+    net_pct = (net / notional) * 100 if notional > 0 else 0.0
+    return net, net_pct
+
+
+async def get_mtf_indicators(
+    prices: list[float],
+    candle_dicts: list[dict],
+    tf_multiplier: int,
+) -> dict:
+    """Resample candles to a different timeframe and compute indicators.
+
+    tf_multiplier: how many main-TF candles per MTF candle (e.g. 2 for 1h from 30m).
+    """
+    if len(candle_dicts) < tf_multiplier * 3:
+        return {}
+
+    # Aggregate candles into MTF candles
+    mtf_candles: list[dict] = []
+    for i in range(0, len(candle_dicts) - tf_multiplier + 1, tf_multiplier):
+        chunk = candle_dicts[i:i + tf_multiplier]
+        mtf_open = chunk[0]["open"]
+        mtf_high = max(c["high"] for c in chunk)
+        mtf_low = min(c["low"] for c in chunk)
+        mtf_close = chunk[-1]["close"]
+        mtf_volume = sum(c["volume"] for c in chunk)
+        mtf_candles.append({
+            "open": mtf_open, "high": mtf_high, "low": mtf_low,
+            "close": mtf_close, "volume": mtf_volume,
+        })
+
+    if len(mtf_candles) < 60:
+        return {}
+
+    mtf_prices = [c["close"] for c in mtf_candles]
+
+    return {
+        "bb": bollinger_bands(mtf_prices),
+        "rsi": rsi(mtf_prices),
+        "sma": sma(mtf_prices, 20),
+        "macd": macd(mtf_prices),
+        "st": supertrend(mtf_candles),
+        "vb": volume_breakout(mtf_candles),
+        "ichi": ichimoku(mtf_candles),
+        "smc": market_structure(mtf_candles),
+    }
+
+
+def classify_mtf(mtf_inds: dict, latest_close: float) -> tuple:
+    """Classify a single MTF timeframe and return a vote."""
+    if not mtf_inds or not mtf_inds.get("bb"):
+        return None, None
+    rsi_vals = mtf_inds.get("rsi", [])
+    mtf_rsi_div = detect_divergence(
+        [c["close"] for c in mtf_inds.get("_candles", [])],
+        rsi_vals,
+    )
+    macd_data = mtf_inds.get("macd", {})
+    macd_hist = macd_data.get("histogram", [None] * (rsi_vals and len(rsi_vals) or 0))
+    mtf_macd_div = detect_divergence(
+        [c["close"] for c in mtf_inds.get("_candles", [])],
+        macd_hist,
+    )
+    mtf_sig, mtf_str, _ = _classify_signal_combined(
+        latest_close,
+        mtf_inds["bb"], mtf_inds.get("rsi"), mtf_inds.get("sma"),
+        mtf_inds.get("macd"), mtf_inds.get("st"), mtf_inds.get("vb"),
+        mtf_inds.get("ichi"), mtf_rsi_div, mtf_macd_div, mtf_inds.get("smc"),
+    )
+    return mtf_sig, mtf_str
+
+
+async def run_backtest(symbol: str, exchange: str, timeframe: str, days: int):
+    print(f"\n{'='*65}")
+    print(f"📊 BACKTEST: {symbol} @ {exchange} [{timeframe}] — {days} ngày")
+    print(f"{'='*65}\n")
+
+    async with async_session_factory() as db:
+        # Get symbol
+        result = await db.execute(
+            select(Symbol)
+            .join(Exchange, Exchange.id == Symbol.exchange_id)
+            .where(and_(Exchange.name == exchange, Symbol.symbol == symbol))
+        )
+        db_symbol = result.scalar_one_or_none()
+        if not db_symbol:
+            print(f"❌ Symbol {symbol} not found on {exchange}")
+            return
+
+        # Fetch candles
+        cutoff = datetime.now(timezone.utc) - timedelta(days=days)
+        result = await db.execute(
+            select(Candle)
+            .where(and_(
+                Candle.symbol_id == db_symbol.id,
+                Candle.timeframe == timeframe,
+                Candle.timestamp >= cutoff,
+            ))
+            .order_by(Candle.timestamp.asc())
+        )
+        candles = result.scalars().all()
+        print(f"📈 {len(candles)} candles loaded")
+
+        if len(candles) < 60:
+            print(f"❌ Cần ít nhất 60 nến để backtest (chỉ có {len(candles)})")
+            return
+
+        # Determine MTF multipliers (relative to main TF)
+        tf_minutes = {"15m": 15, "30m": 30, "1h": 60, "4h": 240}
+        main_minutes = tf_minutes.get(timeframe, 30)
+        mtf_config = []
+        for mtf_tf, mtf_minutes in [("15m", 15), ("1h", 60), ("4h", 240)]:
+            if mtf_tf == timeframe:
+                continue
+            mult = mtf_minutes // main_minutes
+            if mult >= 1 and len(candles) >= mult * 60:
+                mtf_config.append((mtf_tf, mult, {"15m": 0.5, "1h": 1.5, "4h": 2.0}[mtf_tf]))
+            else:
+                print(f"  ℹ️  Bỏ qua {mtf_tf} (cần {mult * 60} candles, chỉ có {len(candles)})")
+
+        if mtf_config:
+            print(f"  🔄 Multi-TF: {', '.join(t for t, _, _ in mtf_config)}")
+        else:
+            print(f"  🔄 Multi-TF: không đủ dữ liệu")
+
+        # ── Simulation ──
+        all_signals: list[tuple[str, str, str, float]] = []  # (timestamp, sig, strength, price)
+        trades: list[dict] = []  # simulated trades
+        current_position: dict | None = None
+        PnL_signals: list[dict] = []  # signals used for PnL simulation
+
+        for i in range(60, len(candles)):
+            candle = candles[i]
+            window = candles[:i + 1]
+
+            candle_dicts = [
+                {"high": float(c.high), "low": float(c.low),
+                 "close": float(c.close), "volume": float(c.volume),
+                 "open": float(c.open)}
+                for c in window
+            ]
+            close_prices = [float(c.close) for c in window]
+            latest_close = float(candle.close)
+            timestamp = candle.timestamp.strftime("%m-%d %H:%M")
+
+            # Compute indicators
+            bb_data = bollinger_bands(close_prices)
+            rsi_data = rsi(close_prices)
+            sma_data = sma(close_prices, 20)
+            macd_data = macd(close_prices)
+            st_data = supertrend(candle_dicts)
+            vb_data = volume_breakout(candle_dicts)
+            ichi_data = ichimoku(candle_dicts)
+            smc_data = market_structure(candle_dicts)
+
+            # Divergence
+            rsi_vals = rsi_data if rsi_data else [None] * len(close_prices)
+            rsi_div = detect_divergence(close_prices, rsi_vals)
+            macd_hist = macd_data.get("histogram", [None] * len(close_prices)) if macd_data else [None] * len(close_prices)
+            macd_div = detect_divergence(close_prices, macd_hist)
+
+            # ── Multi-Timeframe votes ──
+            mtf_votes: list[tuple] = []
+            for mtf_name, mtf_mult, mtf_w in mtf_config:
+                mtf_inds = await get_mtf_indicators(close_prices, candle_dicts, mtf_mult)
+                mtf_sig, mtf_str = classify_mtf(mtf_inds, latest_close)
+                if mtf_sig and mtf_sig not in (CAUTION_LONG, CAUTION_SHORT, SQUEEZE_ALERT):
+                    mtf_votes.append((mtf_sig, mtf_str, mtf_w))
+
+            # Classify
+            signal_type, strength, _ = _classify_signal_combined(
+                latest_close, bb_data, rsi_data if rsi_data else None,
+                sma_data if sma_data else None,
+                macd_data, st_data, vb_data, ichi_data,
+                rsi_div, macd_div, smc_data, mtf_votes if mtf_votes else None,
+            )
+
+            if signal_type:
+                all_signals.append((timestamp, signal_type, strength or "", latest_close))
+
+                # ── PnL Simulation (only STRONG can reverse) ──
+                if signal_type in (STRONG_BUY, BUY):
+                    # BUY signal
+                    if current_position and current_position["direction"] == "SHORT":
+                        if signal_type == STRONG_BUY:
+                            # Only STRONG_BUY can close SHORT
+                            entry = current_position["entry_price"]
+                            qty = current_position["quantity"]
+                            pnl, pnl_pct = _net_pnl(entry, latest_close, float(qty), "SHORT")
+                            current_position["exit_price"] = latest_close
+                            current_position["exit_time"] = timestamp
+                            current_position["pnl"] = pnl
+                            current_position["pnl_pct"] = pnl_pct
+                            current_position["status"] = "CLOSED"
+                            current_position["exit_reason"] = "REVERSAL"
+                            trades.append(current_position)
+                            current_position = None
+                        else:
+                            # Regular BUY can't reverse SHORT
+                            continue
+
+                    if not current_position:
+                        qty = float(TRADE_SIZE) / latest_close
+                        current_position = {
+                            "direction": "LONG",
+                            "entry_price": latest_close,
+                            "entry_time": timestamp,
+                            "quantity": qty,
+                            "entry_signal": signal_type,
+                            "entry_index": i,
+                            "status": "OPEN",
+                        }
+
+                elif signal_type in (STRONG_SELL, SELL):
+                    # SELL signal
+                    if current_position and current_position["direction"] == "LONG":
+                        if signal_type == STRONG_SELL:
+                            # Only STRONG_SELL can close LONG
+                            entry = current_position["entry_price"]
+                            qty = current_position["quantity"]
+                            pnl, pnl_pct = _net_pnl(entry, latest_close, float(qty), "LONG")
+                            current_position["exit_price"] = latest_close
+                            current_position["exit_time"] = timestamp
+                            current_position["pnl"] = pnl
+                            current_position["pnl_pct"] = pnl_pct
+                            current_position["status"] = "CLOSED"
+                            current_position["exit_reason"] = "REVERSAL"
+                            trades.append(current_position)
+                            current_position = None
+                        else:
+                            # Regular SELL can't reverse LONG
+                            continue
+
+                    if not current_position:
+                        qty = float(TRADE_SIZE) / latest_close
+                        current_position = {
+                            "direction": "SHORT",
+                            "entry_price": latest_close,
+                            "entry_time": timestamp,
+                            "quantity": qty,
+                            "entry_signal": signal_type,
+                            "entry_index": i,
+                            "status": "OPEN",
+                        }
+
+            # ── Time limit check for open position ──
+            if current_position and current_position["status"] == "OPEN":
+                hold_candles = i - current_position["entry_index"]
+                if hold_candles >= MAX_HOLD_CANDLES:
+                    entry = current_position["entry_price"]
+                    qty = current_position["quantity"]
+                    pnl, pnl_pct = _net_pnl(entry, latest_close, float(qty), current_position["direction"])
+                    current_position["exit_price"] = latest_close
+                    current_position["exit_time"] = timestamp
+                    current_position["pnl"] = pnl
+                    current_position["pnl_pct"] = pnl_pct
+                    current_position["status"] = "CLOSED"
+                    current_position["exit_reason"] = "TIME_LIMIT"
+                    trades.append(current_position)
+                    current_position = None
+
+        # Close any remaining open position at the last candle
+        if current_position and current_position["status"] == "OPEN":
+            last_close = float(candles[-1].close)
+            entry = current_position["entry_price"]
+            qty = current_position["quantity"]
+            pnl, pnl_pct = _net_pnl(entry, last_close, float(qty), current_position["direction"])
+            current_position["exit_price"] = last_close
+            current_position["exit_time"] = candles[-1].timestamp.strftime("%m-%d %H:%M")
+            current_position["pnl"] = pnl
+            current_position["pnl_pct"] = pnl_pct
+            current_position["status"] = "CLOSED"
+            current_position["exit_reason"] = "END_OF_DATA"
+            trades.append(current_position)
+
+        # ── Print Results ──
+        print(f"\n{'─'*65}")
+        print(f"📊 KẾT QUẢ PHÂN TÍCH & BACKTEST")
+        print(f"{'─'*65}")
+
+        # Signal counts
+        counts = defaultdict(int)
+        for _, sig, _, _ in all_signals:
+            counts[sig] += 1
+
+        if not all_signals:
+            print("❌ Không có tín hiệu nào được tạo ra")
+            return
+
+        print(f"\n📋 Tổng tín hiệu: {len(all_signals)}")
+        print(f"  🚀 STRONG_BUY:   {counts.get('STRONG_BUY', 0)}")
+        print(f"  📈 BUY:           {counts.get('BUY', 0)}")
+        print(f"  📉 SELL:          {counts.get('SELL', 0)}")
+        print(f"  🔻 STRONG_SELL:  {counts.get('STRONG_SELL', 0)}")
+        warnings = counts.get(CAUTION_LONG, 0) + counts.get(CAUTION_SHORT, 0)
+        squeezes = counts.get(SQUEEZE_ALERT, 0)
+        if warnings:
+            print(f"  ⚠️  CAUTION:       {warnings}")
+        if squeezes:
+            print(f"  ⚡ SQUEEZE:       {squeezes}")
+
+        # ── PnL Summary ──
+        closed_trades = [t for t in trades if t.get("status") == "CLOSED"]
+        winning_trades = [t for t in closed_trades if t.get("pnl", 0) > 0]
+        losing_trades = [t for t in closed_trades if t.get("pnl", 0) <= 0]
+
+        total_pnl = sum(t.get("pnl", 0) for t in closed_trades)
+        total_pnl_pct = sum(t.get("pnl_pct", 0) for t in closed_trades)
+        gross_profit = sum(t.get("pnl", 0) for t in winning_trades)
+        gross_loss = sum(t.get("pnl", 0) for t in losing_trades)
+        win_rate = (len(winning_trades) / len(closed_trades) * 100) if closed_trades else 0
+        profit_factor = abs(gross_profit / gross_loss) if gross_loss != 0 else float("inf")
+
+        avg_win = gross_profit / len(winning_trades) if winning_trades else 0
+        avg_loss = gross_loss / len(losing_trades) if losing_trades else 0
+
+        print(f"\n💰 KẾT QUẢ GIAO DỊCH (Trade Size: {TRADE_SIZE} USDT)")
+        print(f"{'─'*40}")
+        print(f"  📊 Tổng giao dịch:     {len(closed_trades)}")
+        print(f"  ✅ Thắng:              {len(winning_trades)} ({win_rate:.1f}%)")
+        print(f"  ❌ Thua:               {len(losing_trades)} ({100 - win_rate:.1f}%)")
+        print(f"  💵 Tổng PnL:           ${total_pnl:+.2f}")
+        print(f"  📈 Tổng PnL %:         {total_pnl_pct:+.2f}%")
+        print(f"  🏆 Lợi nhuận gộp:      ${gross_profit:.2f}")
+        print(f"  💸 Thua lỗ gộp:        ${gross_loss:.2f}")
+        print(f"  ⚡ Profit Factor:      {profit_factor:.2f}x")
+        if avg_win and avg_loss:
+            print(f"  📈 Trung bình lãi:    ${avg_win:.2f}")
+            print(f"  📉 Trung bình lỗ:     ${avg_loss:.2f}")
+            if avg_loss != 0:
+                print(f"  🎯 Lãi/Lỗ ratio:     {abs(avg_win / avg_loss):.2f}x")
+
+        # Best / worst trades
+        if closed_trades:
+            best = max(closed_trades, key=lambda t: t.get("pnl", 0))
+            worst = min(closed_trades, key=lambda t: t.get("pnl", 0))
+            print(f"\n  🥇 Giao dịch tốt nhất:")
+            print(f"     {best.get('direction','')} {symbol} | Entry: ${best.get('entry_price',0):.4f} | "
+                  f"Exit: ${best.get('exit_price',0):.4f} | PnL: ${best.get('pnl',0):+.2f}")
+            print(f"  🥉 Giao dịch tệ nhất:")
+            print(f"     {worst.get('direction','')} {symbol} | Entry: ${worst.get('entry_price',0):.4f} | "
+                  f"Exit: ${worst.get('exit_price',0):.4f} | PnL: ${worst.get('pnl',0):+.2f}")
+
+        # Per-signal breakdown
+        print(f"\n📊 CHI TIẾT THEO TÍN HIỆU")
+        print(f"{'─'*50}")
+        for sig_name in [STRONG_BUY, BUY, SELL, STRONG_SELL]:
+            sig_count = counts.get(sig_name, 0)
+            if sig_count == 0:
+                continue
+            sig_trades = [t for t in closed_trades if t.get("entry_signal") == sig_name]
+            sig_wins = sum(1 for t in sig_trades if t.get("pnl", 0) > 0)
+            sig_pnl = sum(t.get("pnl", 0) for t in sig_trades)
+            sig_wr = (sig_wins / len(sig_trades) * 100) if sig_trades else 0
+            icon = {"STRONG_BUY": "🚀", "BUY": "📈", "SELL": "📉", "STRONG_SELL": "🔻"}.get(sig_name, "❓")
+            print(f"  {icon} {sig_name:14s}: {sig_count:3d} signals → {len(sig_trades):2d} trades, "
+                  f"{sig_wins:2d}W/{len(sig_trades)-sig_wins}L, WR={sig_wr:.0f}%, PnL=${sig_pnl:+.2f}")
+
+        # Print last 15 signals
+        print(f"\n📋 {min(15, len(all_signals))} tín hiệu gần nhất:")
+        print(f"  {'Thời gian':<14} {'Signal':<16} {'Giá':<12}")
+        print(f"  {'─'*42}")
+        for ts, sig, st, price in all_signals[-15:]:
+            icon = {"STRONG_BUY": "🚀", "BUY": "📈", "STRONG_SELL": "🔻", "SELL": "📉",
+                    CAUTION_LONG: "⚠️", CAUTION_SHORT: "⚠️", SQUEEZE_ALERT: "⚡"}.get(sig, "❓")
+            print(f"  {ts:<14} {icon} {sig:<13} {price:<10.4f}")
+
+        # Print last 10 trades
+        if closed_trades:
+            print(f"\n📋 {min(10, len(closed_trades))} giao dịch gần nhất:")
+            print(f"  {'Thời gian':<14} {'Loại':<6} {'Entry':<10} {'Exit':<10} {'PnL':<10}")
+            print(f"  {'─'*50}")
+            for t in closed_trades[-10:]:
+                pnl_str = f"${t.get('pnl',0):+.2f}"
+                print(f"  {t.get('entry_time',''):<14} {t.get('direction',''):<6} "
+                      f"{t.get('entry_price',0):<10.4f} {t.get('exit_price',0):<10.4f} "
+                      f"{pnl_str:<10}")
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Backtest signal algorithms with PnL + Win Rate")
+    parser.add_argument("--symbol", default="BTC/USDT", help="Symbol to test")
+    parser.add_argument("--exchange", default="mexc", help="Exchange")
+    parser.add_argument("--timeframe", default="30m", help="Timeframe")
+    parser.add_argument("--days", type=int, default=7, help="Days of history")
+    parser.add_argument("--trade-size", type=float, default=10, help="Trade size in USDT")
+    args = parser.parse_args()
+
+    global TRADE_SIZE
+    TRADE_SIZE = Decimal(str(args.trade_size))
+
+    asyncio.run(run_backtest(
+        symbol=args.symbol,
+        exchange=args.exchange,
+        timeframe=args.timeframe,
+        days=args.days,
+    ))
+
+
+if __name__ == "__main__":
+    main()

backend/scripts/backup_strategies.py

@@ -0,0 +1,115 @@
+"""P2-9: Strategy backup utility.
+
+Exports all enabled strategies to a JSON file for disaster recovery.
+Run manually or via cron.
+
+Usage:
+    python3 /app/scripts/backup_strategies.py
+    python3 /app/scripts/backup_strategies.py --restore backup.json
+"""
+
+import argparse
+import json
+import sys
+from datetime import datetime, timezone
+
+sys.path.insert(0, "/app")
+
+from app.database import async_session_factory
+from app.models import User
+from sqlalchemy import select
+
+
+BACKUP_DIR = "/app/backups"
+
+
+async def backup_strategies(output_path: str | None = None):
+    """Export all user strategy preferences."""
+    if output_path is None:
+        ts = datetime.now(timezone.utc).strftime("%Y%m%d_%H%M%S")
+        output_path = f"{BACKUP_DIR}/strategies_{ts}.json"
+
+    async with async_session_factory() as db:
+        result = await db.execute(select(User).where(User.is_active == True))
+        users = result.scalars().all()
+
+        backup = {
+            "exported_at": datetime.now(timezone.utc).isoformat(),
+            "version": "2.0",
+            "users": [],
+        }
+        for u in users:
+            prefs = u.preferences or {}
+            backup["users"].append({
+                "username": u.username,
+                "email": u.email,
+                "enabled_strategies": prefs.get("enabled_strategies", []),
+                "auto_trade": prefs.get("auto_trade", False),
+                "auto_trade_tokens": prefs.get("auto_trade_tokens", []),
+                "trade_size": prefs.get("trade_size", 10),
+                "timeframe": prefs.get("timeframe", "30m"),
+            })
+
+    # Ensure backup dir exists
+    import os
+    os.makedirs(BACKUP_DIR, exist_ok=True)
+
+    with open(output_path, "w") as f:
+        json.dump(backup, f, indent=2)
+
+    print(f"✅ Strategy backup saved to {output_path}")
+    print(f"   {len(backup['users'])} user(s) exported")
+
+
+async def restore_strategies(input_path: str, dry_run: bool = False):
+    """Restore strategy preferences from a backup file."""
+    with open(input_path) as f:
+        backup = json.load(f)
+
+    print(f"📥 Restoring from {input_path}")
+    print(f"   Exported at: {backup.get('exported_at', 'unknown')}")
+    print(f"   Users: {len(backup['users'])}")
+    if dry_run:
+        print("   🔍 DRY RUN — no changes made")
+    print()
+
+    async with async_session_factory() as db:
+        for bu in backup["users"]:
+            result = await db.execute(select(User).where(User.username == bu["username"]))
+            user = result.scalar_one_or_none()
+            if not user:
+                print(f"   ⚠️  User '{bu['username']}' not found — skipping")
+                continue
+
+            prefs = user.preferences or {}
+            prefs["enabled_strategies"] = bu["enabled_strategies"]
+            prefs["auto_trade"] = bu["auto_trade"]
+            prefs["auto_trade_tokens"] = bu["auto_trade_tokens"]
+            prefs["trade_size"] = bu["trade_size"]
+            prefs["timeframe"] = bu["timeframe"]
+            user.preferences = prefs
+            db.add(user)
+            print(f"   ✅ {bu['username']}: {len(bu['enabled_strategies'])} strategies, "
+                  f"auto_trade={'ON' if bu['auto_trade'] else 'OFF'}")
+
+        if not dry_run:
+            await db.commit()
+            print("\n✅ Restore complete")
+
+
+async def main():
+    parser = argparse.ArgumentParser(description="P2-9: Strategy backup/restore")
+    parser.add_argument("--restore", type=str, help="Restore from backup file")
+    parser.add_argument("--dry-run", action="store_true", help="Preview restore without changes")
+    parser.add_argument("--output", type=str, help="Custom output path for backup")
+    args = parser.parse_args()
+
+    if args.restore:
+        await restore_strategies(args.restore, dry_run=args.dry_run)
+    else:
+        await backup_strategies(args.output)
+
+
+if __name__ == "__main__":
+    import asyncio
+    asyncio.run(main())

backend/scripts/export_symbols.py

@@ -0,0 +1,60 @@
+#!/usr/bin/env python3
+"""Export all trading symbols (is_trading=true) to Excel."""
+import os
+import asyncio
+import asyncpg
+import pandas as pd
+from datetime import datetime
+
+DB_URL = os.environ.get("DATABASE_URL", "postgresql://trading:***@db:5432/trading_portal")
+if "+asyncpg" in DB_URL:
+    DB_URL = DB_URL.replace("+asyncpg", "")
+
+OUTPUT_DIR = "/opt/data"
+OUTPUT_FILE = f"{OUTPUT_DIR}/trading_symbols_{datetime.now().strftime('%Y%m%d_%H%M')}.xlsx"
+
+
+async def main():
+    conn = await asyncpg.connect(DB_URL)
+    try:
+        rows = await conn.fetch("""
+            SELECT e.name as exchange, s.symbol, s.base, s.quote, s.id
+            FROM symbols s
+            JOIN exchanges e ON e.id = s.exchange_id
+            WHERE s.is_trading = true
+            ORDER BY e.name, s.symbol
+        """)
+        data = [dict(r) for r in rows]
+        print(f"Got {len(data)} rows")
+
+        df = pd.DataFrame(data)
+
+        # Summary per exchange
+        summary = df.groupby("exchange").agg(
+            symbol_count=("symbol", "count"),
+            examples=("symbol", lambda x: ", ".join(x.head(5)))
+        ).reset_index()
+        summary.columns = ["Sàn", "Số lượng", "Ví dụ"]
+
+        # Full list
+        full = df[["exchange", "symbol", "base", "quote"]]
+        full.columns = ["Sàn", "Symbol", "Base", "Quote"]
+
+        # Write to Excel with multiple sheets
+        with pd.ExcelWriter(OUTPUT_FILE, engine="openpyxl") as writer:
+            summary.to_excel(writer, sheet_name="Tổng quan", index=False)
+            full.to_excel(writer, sheet_name="Chi tiết", index=False)
+
+            # Per exchange sheets
+            for ex in sorted(df["exchange"].unique()):
+                ex_df = df[df["exchange"] == ex][["symbol", "base", "quote"]]
+                ex_df.columns = ["Symbol", "Base", "Quote"]
+                sheet_name = ex.capitalize()[:31]
+                ex_df.to_excel(writer, sheet_name=sheet_name, index=False)
+
+        print(f"Exported to {OUTPUT_FILE}")
+    finally:
+        await conn.close()
+
+
+asyncio.run(main())

backend/scripts/fetch_candles_cron.py

@@ -0,0 +1,112 @@
+#!/usr/bin/env python3
+"""
+Cron job: fetch candles for watched symbols x ALL timeframes (except 1m, 5m).
+
+Gets watched symbols from API, adds default BTC/USDT, then fetches.
+Timeframes: 15m, 30m, 1h, 4h, 1d, 1w, 1M
+"""
+import urllib.request, urllib.error, json, sys
+
+BASE = "http://localhost:8001"
+TIMEFRAMES = ["15m", "30m", "1h", "4h", "1d", "1w", "1M"]
+LIMIT = 200
+TIMEOUT_PER = 25
+MAX_CALLS = 30  # max API calls total (3 symbols x 7 TFs = 21 + buffer)
+
+# --- 1. Get token ---
+# Use env vars if available, fallback to file-based credentials
+import os
+DEMO_USER = os.environ.get("DEMO_USER", "")
+DEMO_PASS = os.environ.get("DEMO_PASS", "")
+if not DEMO_USER or not DEMO_PASS:
+    # Fallback: read from .env file in project root
+    env_file = os.path.join(os.path.dirname(__file__), "..", ".env")
+    if os.path.exists(env_file):
+        with open(env_file) as f:
+            for line in f:
+                line = line.strip()
+                if line.startswith("DEMO_USER="):
+                    DEMO_USER = line.split("=", 1)[1]
+                elif line.startswith("DEMO_PASS="):
+                    DEMO_PASS = line.split("=", 1)[1]
+
+if not DEMO_USER or not DEMO_PASS:
+    print("FAIL: No credentials configured (set DEMO_USER/DEMO_PASS env vars or .env file)")
+    sys.exit(1)
+
+try:
+    body = json.dumps({"username": DEMO_USER, "password": DEMO_PASS}).encode()
+    req = urllib.request.Request(BASE + "/api/v1/auth/login",
+        data=body, headers={"Content-Type": "application/json"}, method="POST")
+    with urllib.request.urlopen(req, timeout=10) as r:
+        token = json.loads(r.read())["access_token"]
+except Exception as e:
+    print("FAIL: get token -", str(e))
+    sys.exit(1)
+
+auth = "Bearer " + token
+hdr = {"Content-Type": "application/json", "Authorization": auth}
+
+# --- 2. Get watched symbols ---
+watched = set()
+# Demo user watchlist
+try:
+    req = urllib.request.Request(BASE + "/api/v1/watchlist", headers=hdr)
+    with urllib.request.urlopen(req, timeout=10) as r:
+        items = json.loads(r.read())
+        for i in items if isinstance(items, list) else []:
+            if isinstance(i, dict) and i.get("symbol"):
+                watched.add(i["symbol"])
+except Exception:
+    pass
+# Also try admin endpoint for all users
+try:
+    req = urllib.request.Request(BASE + "/api/v1/watchlist", headers=hdr)
+    with urllib.request.urlopen(req, timeout=5) as r:
+        data = json.loads(r.read())
+        for i in data.get("items", []) if isinstance(data, dict) else data if isinstance(data, list) else []:
+            if isinstance(i, dict) and i.get("symbol"):
+                watched.add(i["symbol"])
+except Exception:
+    pass
+# Add defaults
+for d in ["BTC/USDT", "ETH/USDT"]:
+    watched.add(d)
+
+watched_list = sorted(watched)
+print("Symbols (" + str(len(watched_list)) + "): " + ", ".join(watched_list))
+print("Timeframes: " + ", ".join(TIMEFRAMES))
+print("Total calls: " + str(len(watched_list) * len(TIMEFRAMES)))
+print()
+
+# --- 3. Fetch candles (max MAX_CALLS total) ---
+ok = fail = 0
+errs = []
+calls = 0
+for sym in watched_list:
+    for tf in TIMEFRAMES:
+        if calls >= MAX_CALLS:
+            break
+        calls += 1
+        try:
+            payload = json.dumps({"symbol": sym, "timeframe": tf, "limit": LIMIT}).encode()
+            req3 = urllib.request.Request(BASE + "/api/v1/exchanges/1/fetch_candles",
+                data=payload, headers=hdr, method="POST")
+            with urllib.request.urlopen(req3, timeout=TIMEOUT_PER) as r:
+                data = json.loads(r.read())
+                cnt = len(data.get("candles", [])) if isinstance(data, dict) else (len(data) if isinstance(data, list) else 0)
+                if cnt > 0 or "message" in data:
+                    ok += 1
+                else:
+                    fail += 1
+                    errs.append(sym + "/" + tf + " (0 candles)")
+        except Exception as e:
+            fail += 1
+            errs.append(sym + "/" + tf + " (" + str(e)[:60] + ")")
+
+print("OK=" + str(ok) + " FAIL=" + str(fail))
+if errs:
+    for e in errs[:8]:
+        print("  " + e)
+    if len(errs) > 8:
+        print("  ... +" + str(len(errs) - 8) + " more")

backend/scripts/select_top100_symbols.py

@@ -0,0 +1,232 @@
+"""
+Scan CCXT 5 exchanges for all USDT pairs, filter by quality criteria,
+score by volume (60%) + exchange coverage (40%), and select top 100.
+
+Updates symbols.is_trading = true for the top 100 bases across ALL exchanges.
+"""
+import asyncio
+import logging
+from collections import defaultdict
+
+import ccxt.async_support as ccxt_async
+import asyncpg
+
+logging.basicConfig(level=logging.INFO, format="%(asctime)s %(message)s")
+logger = logging.getLogger(__name__)
+
+import os
+DB_URL = os.environ.get("DATABASE_URL", "postgresql://trading:trading_secret@db:5432/trading_portal").replace(
+    "asyncpg", "postgresql"
+).replace("+asyncpg", "")
+MAX_SYMBOLS = 100
+MIN_PRICE = 0.01
+VOLUME_WEIGHT = 0.6
+EXCHANGE_COVERAGE_WEIGHT = 0.4
+
+EXCHANGES = ["binance", "bybit", "mexc", "gate", "bingx"]
+
+STABLECOINS = {
+    "USDC", "BUSD", "DAI", "TUSD", "USDP", "USDD", "FDUSD", "USDE",
+    "USTC", "USDJ", "EURC", "USDS", "PYUSD", "CRVUSD", "GHO", "USR",
+}
+
+LEVERAGED_SUFFIXES = ("UP", "DOWN", "BULL", "BEAR")
+ALLOWED_W = {"WBTC", "WETH", "WBNB", "WAVAX", "WSOL", "WMATIC", "WTRX", "WLD"}
+
+
+def is_stablecoin(base: str) -> bool:
+    return base.upper() in STABLECOINS
+
+
+def is_leveraged(base: str) -> bool:
+    upper = base.upper()
+    for sfx in LEVERAGED_SUFFIXES:
+        if upper.endswith(f"{sfx}") or f"{sfx}USDT" in upper:
+            return True
+    return False
+
+
+def is_valid_symbol(base: str, quote: str, price: float) -> bool:
+    if quote.upper() != "USDT":
+        return False
+    if price is not None and price < MIN_PRICE:
+        return False
+    if is_stablecoin(base):
+        return False
+    if is_leveraged(base):
+        return False
+    if base.upper().startswith("W") and base.upper() not in ALLOWED_W:
+        return False
+    return True
+
+
+def safe_ticker_field(ticker: dict, field: str, default=None):
+    """Get a field from ticker, handling None values."""
+    val = ticker.get(field, default)
+    return val if val is not None else default
+
+
+async def fetch_exchange_tickers(exchange_id: str) -> list[dict]:
+    exchange_class = getattr(ccxt_async, exchange_id, None)
+    if exchange_class is None:
+        logger.warning("CCXT class not found for %s", exchange_id)
+        return []
+
+    exchange = exchange_class({"enableRateLimit": True, "timeout": 30000})
+    try:
+        logger.info("Fetching tickers from %s...", exchange_id)
+        tickers = await exchange.fetch_tickers()
+        logger.info("  %s: got %d tickers total", exchange_id, len(tickers))
+        results = []
+        for symbol_str, ticker in tickers.items():
+            # Parse symbol parts
+            if "/" in symbol_str:
+                parts = symbol_str.split("/")
+                base_raw = parts[0]
+                quote_raw = parts[1].split(":")[0]  # handle "USDT:USDC" etc
+            else:
+                base_raw = ticker.get("base", "")
+                quote_raw = ticker.get("quote", "")
+
+            # Safely extract numeric fields
+            last = safe_ticker_field(ticker, "last", 1.0)
+            if last is None:
+                last = 1.0
+
+            if not is_valid_symbol(base_raw, quote_raw, last):
+                continue
+
+            # Volume: prefer quoteVolume, fallback to baseVolume * last
+            quote_vol = safe_ticker_field(ticker, "quoteVolume", 0)
+            if quote_vol and quote_vol > 0:
+                volume = quote_vol
+            else:
+                base_vol = safe_ticker_field(ticker, "baseVolume", 0) or 0
+                volume = float(base_vol) * float(last) if base_vol else 0
+
+            results.append({
+                "base": base_raw.upper(),
+                "quote": "USDT",
+                "exchange": exchange_id,
+                "last_price": last,
+                "volume_24h": volume,
+                "symbol_str": symbol_str,
+            })
+
+        logger.info("  %s: %d valid USDT trading pairs", exchange_id, len(results))
+        return results
+    except Exception as e:
+        logger.error("Error fetching %s: %s", exchange_id, str(e))
+        return []
+    finally:
+        await exchange.close()
+
+
+async def main():
+    # 1. Fetch all tickers
+    all_tickers = []
+    for ex in EXCHANGES:
+        tickers = await fetch_exchange_tickers(ex)
+        all_tickers.extend(tickers)
+
+    logger.info("Total valid USDT pairs: %d", len(all_tickers))
+
+    # 2. Group by base, compute exchange coverage + total volume
+    by_base: dict[str, dict] = defaultdict(lambda: {
+        "exchanges": set(),
+        "total_volume": 0.0,
+        "prices": [],
+    })
+
+    for t in all_tickers:
+        base = t["base"]
+        by_base[base]["exchanges"].add(t["exchange"])
+        by_base[base]["total_volume"] += t["volume_24h"]
+        by_base[base]["prices"].append(t["last_price"])
+
+    # 3. Score
+    max_volume = max((v["total_volume"] for v in by_base.values()), default=1.0)
+    max_exchanges = max((len(v["exchanges"]) for v in by_base.values()), default=1)
+
+    scored = []
+    for base, data in by_base.items():
+        volume_score = data["total_volume"] / max_volume if max_volume > 0 else 0
+        exchange_score = len(data["exchanges"]) / max_exchanges if max_exchanges > 0 else 0
+        final_score = volume_score * VOLUME_WEIGHT + exchange_score * EXCHANGE_COVERAGE_WEIGHT
+        avg_price = sum(data["prices"]) / len(data["prices"]) if data["prices"] else 0
+        scored.append({
+            "base": base,
+            "exchanges": sorted(data["exchanges"]),
+            "exchange_count": len(data["exchanges"]),
+            "volume_24h": data["total_volume"],
+            "avg_price": avg_price,
+            "score": round(final_score, 4),
+        })
+
+    scored.sort(key=lambda x: x["score"], reverse=True)
+    top100 = scored[:MAX_SYMBOLS]
+
+    logger.info("=== TOP %d TRADING SYMBOLS ===", MAX_SYMBOLS)
+    for i, s in enumerate(top100, 1):
+        vol_str = f"${s['volume_24h']/1e6:.1f}M" if s['volume_24h'] < 1e9 else f"${s['volume_24h']/1e9:.2f}B"
+        logger.info(
+            "%3d. %-10s | %d sàn (%s) | %s | score=%.4f | $%.4f",
+            i, s["base"], s["exchange_count"],
+            ",".join(s["exchanges"]),
+            vol_str, s["score"], s["avg_price"],
+        )
+
+    # 4. Update DB
+    bases = [s["base"] for s in top100]
+    logger.info("Updating is_trading flag for %d bases...", len(bases))
+
+    conn = await asyncpg.connect(DB_URL)
+    try:
+        async with conn.transaction():
+            await conn.execute("UPDATE symbols SET is_trading = false")
+            result = await conn.execute(
+                """
+                UPDATE symbols s
+                SET is_trading = true
+                FROM exchanges e
+                WHERE s.exchange_id = e.id
+                  AND s.base = ANY($1::text[])
+                  AND s.is_active = true
+                  AND e.is_active = true
+                """,
+                bases,
+            )
+            # result is like "UPDATE N"
+            logger.info("DB UPDATE result: %s", result)
+
+        # Verify
+        verify = await conn.fetch("""
+            SELECT e.name as exchange, COUNT(*) as cnt
+            FROM symbols s
+            JOIN exchanges e ON e.id = s.exchange_id
+            WHERE s.is_trading = true
+            GROUP BY e.name
+            ORDER BY e.name
+        """)
+        logger.info("=== SYMBOLS PER EXCHANGE (is_trading=true) ===")
+        total = 0
+        for row in verify:
+            logger.info("  %s: %d", row["exchange"], row["cnt"])
+            total += row["cnt"]
+        logger.info("  TOTAL: %d symbols across all exchanges", total)
+
+        # Show non-matching bases
+        verify_bases = await conn.fetch("""
+            SELECT DISTINCT base FROM symbols WHERE is_trading = true ORDER BY base
+        """)
+        db_bases = {r["base"] for r in verify_bases}
+        missing = [b for b in bases if b.upper() not in db_bases]
+        if missing:
+            logger.warning("Bases NOT found in DB: %s", missing[:20])
+
+    finally:
+        await conn.close()
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

backend/scripts/sync_candles.py

@@ -0,0 +1,102 @@
+#!/usr/bin/env python3
+"""
+Fetch candles for tracked symbols/timeframes and cleanup old 1m data.
+Runs via cron every 30 minutes.
+
+Timeframes retained:
+  1M, 1W, 1D, 4H, 1H, 30m  -> unlimited (forever)
+  1m (minute)               -> max 7 days
+"""
+import subprocess, json, sys
+from datetime import datetime, timezone, timedelta
+
+# --- Config ---
+SYMBOLS = ["BTC/USDT", "ETH/USDT", "SOL/USDT"]
+TIMEFRAMES = [
+    # (tf, fetch_limit)  limit=0 means fetch all available
+    ("1M", 200),
+    ("1w", 600),
+    ("1d", 2000),
+    ("4h", 2000),
+    ("1h", 2000),
+    ("30m", 2000),
+    ("1m", 10000),   # ~7 days
+]
+API_BASE = "http://localhost:8001/api/v1"
+EXCHANGE_ID = 1
+MINUTE_RETENTION_DAYS = 7
+
+def login():
+    r = subprocess.run([
+        "curl", "-s", "-X", "POST",
+        f"{API_BASE}/auth/login",
+        "-H", "Content-Type: application/json",
+        "-d", '{"username":"demo","password":"demo1234"}'
+    ], capture_output=True, text=True, timeout=10)
+    try:
+        data = json.loads(r.stdout)
+        return data["access_token"]
+    except (json.JSONDecodeError, KeyError):
+        print("FAIL: Cannot login", r.stdout[:100])
+        sys.exit(1)
+
+def fetch_candles(token, symbol, tf, limit):
+    """Fetch candles for one symbol/timeframe."""
+    auth_h = "Authorization: Bearer " + token
+    payload = json.dumps({"symbol": symbol, "timeframe": tf, "limit": limit})
+    cmd = [
+        "curl", "-s", "-X", "POST",
+        f"{API_BASE}/exchanges/{EXCHANGE_ID}/fetch_candles",
+        "-H", auth_h,
+        "-H", "Content-Type: application/json",
+        "-d", payload
+    ]
+    r = subprocess.run(cmd, capture_output=True, text=True, timeout=120)
+    return r.stdout.strip()
+
+def cleanup_old_1m(token):
+    """Delete 1m candles older than MINUTE_RETENTION_DAYS."""
+    cutoff = (datetime.now(timezone.utc) - timedelta(days=MINUTE_RETENTION_DAYS)).isoformat()
+    # Delete via direct DB
+    cmd = [
+        "docker", "exec", "trading-db", "psql", "-U", "trading", "trading_portal",
+        "-c",
+        f"DELETE FROM candles WHERE timeframe='1m' AND timestamp < '{cutoff}'::timestamptz;"
+    ]
+    r = subprocess.run(cmd, capture_output=True, text=True, timeout=30)
+    out = r.stdout.strip()
+    # Extract count from "DELETE N"
+    if out.startswith("DELETE"):
+        count = out.split()[1] if len(out.split()) > 1 else "?"
+        if count != "0":
+            print(f"CLEANUP: Deleted {count} old 1m candles (>{MINUTE_RETENTION_DAYS} days)")
+    return out
+
+def main():
+    token = login()
+    total = 0
+    errors = 0
+
+    # 1. Fetch candles for all symbols x timeframes
+    for sym in SYMBOLS:
+        for tf, limit in TIMEFRAMES:
+            try:
+                result = fetch_candles(token, sym, tf, limit)
+                if '"message"' in result:
+                    total += 1
+                else:
+                    errors += 1
+                    print(f"ERR {sym:>9} {tf:>4}: {result[:80]}")
+            except Exception as e:
+                errors += 1
+                print(f"EXC {sym:>9} {tf:>4}: {e}")
+
+    # 2. Cleanup old 1m data
+    cleanup_out = cleanup_old_1m(token)
+
+    print(f"DONE: {total}/{total+errors} fetches OK, {errors} errors")
+    if errors == 0 and total > 0:
+        print("All timeframes synced successfully!")
+
+if __name__ == "__main__":
+    main()

backend/scripts/train_xgboost_model.py

@@ -0,0 +1,205 @@
+#!/usr/bin/env python3
+"""Train XGBoost model from collected algo_scores in signals.indicators_snapshot.
+
+Run by cron job on Wednesday to update the signal fusion model.
+Saves model to /opt/data/trading-portal/models/xgb_model.json
+"""
+
+import json
+import sys
+import os
+from decimal import Decimal
+from datetime import datetime, timezone
+
+# Add project to path
+sys.path.insert(0, '/opt/data/trading-portal/backend')
+
+try:
+    import xgboost as xgb
+    import numpy as np
+    from sqlalchemy import select, text
+    from app.database import async_session_factory
+    from app.models.signal import Signal
+except ImportError:
+    print("XGBoost not installed. Run: pip install xgboost")
+    sys.exit(1)
+
+
+STRATEGY_ORDER = [
+    "double_bb_rsi", "macd_crossover", "supertrend",
+    "volume_breakout", "ichimoku", "divergence",
+    "smc", "mtf", "obv", "stoch_rsi",
+    "mfi", "fvg", "candlestick",
+]
+
+
+async def collect_training_data():
+    """Fetch all signals with algo_scores + corresponding trade PnL."""
+    async with async_session_factory() as db:
+        result = await db.execute(
+            select(Signal)
+            .where(Signal.indicators_snapshot.isnot(None))
+            .order_by(Signal.created_at.desc())
+        )
+        signals = result.scalars().all()
+
+    features = []
+    labels = []
+    symbol_list = []
+    timestamps = []
+
+    for sig in signals:
+        try:
+            snap = json.loads(sig.indicators_snapshot)
+        except (json.JSONDecodeError, TypeError):
+            continue
+
+        scores = snap.get("algo_scores")
+        if not scores or not isinstance(scores, dict):
+            continue
+
+        # Build feature vector in STRATEGY_ORDER
+        feature_row = [scores.get(s, 0.0) for s in STRATEGY_ORDER]
+
+        # Add confidence
+        feature_row.append(snap.get("confidence", 0.5))
+
+        # P2-11: Enhanced feature engineering
+        # RSI value (momentum)
+        rsi_val = snap.get("rsi")
+        feature_row.append(float(rsi_val) / 100.0 if rsi_val is not None else 0.5)
+
+        # BB width (volatility indicator)
+        bb_upper = snap.get("bb_upper_2")
+        bb_lower = snap.get("bb_lower_2")
+        close_val = snap.get("close")
+        if bb_upper is not None and bb_lower is not None and close_val is not None and close_val > 0:
+            feature_row.append((float(bb_upper) - float(bb_lower)) / float(close_val))
+        else:
+            feature_row.append(0.0)
+
+        # Signal strength (numeric encoding)
+        strength_map = {"STRONG": 1.0, "MODERATE": 0.5, "WEAK": 0.2}
+        feature_row.append(strength_map.get(sig.strength, 0.5) if sig.strength else 0.5)
+
+        # One-hot encode market_regime
+        regime = snap.get("market_regime", "neutral")
+        for r in ["trending", "sideways", "volatile", "breakout", "squeeze", "neutral"]:
+            feature_row.append(1.0 if regime == r else 0.0)
+
+        # Label: use actual PnL from hypothetical_trades (joined by signal_id)
+        # Default to signal_type as fallback if no trade data exists
+        label = None
+        try:
+            from app.models.hypothetical_trade import HypotheticalTrade
+            trade_result = await db.execute(
+                select(HypotheticalTrade.pnl_percent)
+                .where(HypotheticalTrade.signal_id == sig.id)
+                .order_by(HypotheticalTrade.closed_at.desc().nulls_last())
+                .limit(1)
+            )
+            pnl_pct = trade_result.scalar()
+            if pnl_pct is not None:
+                label = 1.0 if float(pnl_pct) > 0 else 0.0
+        except Exception:
+            pass
+        if label is None:
+            # Fallback: signal_type as weak proxy
+            if sig.signal_type in ("STRONG_BUY", "BUY"):
+                label = 1.0
+            elif sig.signal_type in ("STRONG_SELL", "SELL"):
+                label = 0.0
+            else:
+                continue
+
+        features.append(feature_row)
+        labels.append(label)
+        symbol_list.append(sig.symbol)
+        timestamps.append(sig.created_at.isoformat() if sig.created_at else "")
+
+    return np.array(features), np.array(labels), symbol_list, timestamps
+
+
+FEATURE_NAMES = STRATEGY_ORDER + ["confidence", "rsi", "bb_width", "signal_strength"] + [
+    f"regime_{r}" for r in ["trending", "sideways", "volatile", "breakout", "squeeze", "neutral"]
+]
+
+
+def main():
+    import asyncio
+
+    print("=" * 60)
+    print("📊 XGBoost Signal Fusion — Training Pipeline")
+    print(f"   Date: {datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M UTC')}")
+    print("=" * 60)
+
+    # Collect data
+    X, y, symbols, timestamps = asyncio.run(collect_training_data())
+
+    if len(X) < 100:
+        print(f"\n❌ Not enough data: only {len(X)} samples (need ≥ 100)")
+        print("   Let the system run longer to accumulate features.")
+        sys.exit(1)
+
+    print(f"\n📈 Training data: {len(X)} samples")
+    print(f"   Features: {len(FEATURE_NAMES)} ({', '.join(FEATURE_NAMES[:5])}...)")
+    print(f"   Class balance: {y.mean():.1%} positive / {1-y.mean():.1%} negative")
+
+    # Split: 80% train, 20% test
+    split = int(len(X) * 0.8)
+    X_train, X_test = X[:split], X[split:]
+    y_train, y_test = y[:split], y[split:]
+
+    # Train XGBoost
+    model = xgb.XGBClassifier(
+        n_estimators=200,
+        max_depth=5,
+        learning_rate=0.05,
+        subsample=0.8,
+        colsample_bytree=0.8,
+        random_state=42,
+        eval_metric="logloss",
+        use_label_encoder=False,
+    )
+
+    model.fit(
+        X_train, y_train,
+        eval_set=[(X_test, y_test)],
+        verbose=False,
+    )
+
+    # Evaluate
+    train_acc = model.score(X_train, y_train)
+    test_acc = model.score(X_test, y_test)
+
+    from sklearn.metrics import classification_report
+    y_pred = model.predict(X_test)
+
+    print(f"\n🎯 Model Performance:")
+    print(f"   Train accuracy: {train_acc:.1%}")
+    print(f"   Test accuracy:  {test_acc:.1%}")
+    print(f"\n   Classification Report (test set):")
+    print(classification_report(y_test, y_pred, target_names=["SELL/WAIT", "BUY"], zero_division=0))
+
+    # Feature importance
+    importance = model.feature_importances_
+    top_idx = np.argsort(importance)[-5:][::-1]
+    print(f"\n🔑 Top 5 Features by Importance:")
+    for idx in top_idx:
+        print(f"   {FEATURE_NAMES[idx]}: {importance[idx]:.3f}")
+
+    # Save model
+    os.makedirs("/opt/data/trading-portal/models", exist_ok=True)
+    model_path = "/opt/data/trading-portal/models/xgb_model.json"
+    model.save_model(model_path)
+    print(f"\n💾 Model saved to: {model_path}")
+
+    # Summary
+    print(f"\n{'='*60}")
+    print(f"✅ Training complete! Model ready for inference.")
+    print(f"   Next step: Deploy model and replace linear voting in signal_service.py")
+    print(f"{'='*60}")
+
+
+if __name__ == "__main__":
+    main()

backend/scripts/weekly_report.py

@@ -0,0 +1,251 @@
+#!/usr/bin/env python3
+"""Weekly trading report — queries the last 7 days of signals and trades,
+formats a summary, and prints it to stdout for Telegram delivery.
+
+Usage:
+    python scripts/weekly_report.py
+
+Requires:
+    psycopg2-binary
+    Environment variables (or .env):
+        TRADING_DB_HOST, TRADING_DB_PORT, TRADING_DB_NAME,
+        TRADING_DB_USER, TRADING_DB_PASSWORD
+    Or relies on psycopg2 defaults / pgpass.
+
+Output: Markdown-formatted report printed to stdout.
+"""
+
+from __future__ import annotations
+
+import os
+import sys
+from collections import Counter
+from datetime import datetime, timedelta, timezone
+from decimal import Decimal
+from typing import Any
+
+try:
+    import psycopg2
+except ImportError:
+    print("❌ psycopg2 not installed. Run: pip install psycopg2-binary", file=sys.stderr)
+    sys.exit(1)
+
+
+# ---------------------------------------------------------------------------
+# Configuration
+# ---------------------------------------------------------------------------
+
+DB_CONFIG = {
+    "host": os.environ.get("TRADING_DB_HOST", "trading-db"),
+    "port": int(os.environ.get("TRADING_DB_PORT", "5432")),
+    "dbname": os.environ.get("TRADING_DB_NAME", "trading_portal"),
+    "user": os.environ.get("TRADING_DB_USER", "trading"),
+    "password": os.environ.get("TRADING_DB_PASSWORD", "trading_secret"),
+}
+
+# Fallback: also try standard trading-portal .env variable names
+if not os.environ.get("TRADING_DB_HOST"):
+    DB_CONFIG["host"] = os.environ.get("DB_HOST", "trading-db")
+    DB_CONFIG["port"] = int(os.environ.get("DB_PORT", "5432"))
+    DB_CONFIG["dbname"] = os.environ.get("DB_NAME", "trading_portal")
+    DB_CONFIG["user"] = os.environ.get("DB_USER", "trading")
+    DB_CONFIG["password"] = os.environ.get("DB_PASSWORD", "trading_secret")
+
+
+# ---------------------------------------------------------------------------
+# Helpers
+# ---------------------------------------------------------------------------
+
+
+def get_connection():
+    """Create and return a database connection."""
+    try:
+        conn = psycopg2.connect(**DB_CONFIG)
+        conn.autocommit = True
+        return conn
+    except psycopg2.OperationalError as e:
+        print(f"❌ Database connection failed: {e}", file=sys.stderr)
+        sys.exit(1)
+
+
+def query(conn, sql: str, params: tuple = ()) -> list[tuple]:
+    """Execute a query and return all rows."""
+    with conn.cursor() as cur:
+        cur.execute(sql, params)
+        return cur.fetchall()
+
+
+def format_pnl(value: Decimal | None) -> str:
+    """Format PnL with sign and $."""
+    if value is None:
+        return "$0.00"
+    v = float(value)
+    prefix = "+" if v >= 0 else ""
+    return f"{prefix}${v:.2f}"
+
+
+def format_pct(value: Decimal | None) -> str:
+    """Format percentage."""
+    if value is None:
+        return "0.0%"
+    return f"{float(value):.1f}%"
+
+
+# ---------------------------------------------------------------------------
+# Report generation
+# ---------------------------------------------------------------------------
+
+
+def generate_report() -> str:
+    """Query the database and build the weekly report string.
+
+    Returns
+    -------
+    str
+        Markdown-formatted report for Telegram.
+    """
+    conn = get_connection()
+    now = datetime.now(timezone.utc)
+    week_ago = now - timedelta(days=7)
+
+    start_str = week_ago.strftime("%d %b %H:%M UTC")
+    end_str = now.strftime("%d %b %H:%M UTC")
+
+    lines: list[str] = []
+    lines.append("📊 *Weekly Trading Report*")
+    lines.append(f"Period: {start_str} → {end_str}")
+    lines.append("")
+
+    # ------------------------------------------------------------------
+    # 1. Signal summary
+    # ------------------------------------------------------------------
+    sig_rows = query(
+        conn,
+        """
+        SELECT signal_type, COUNT(*)
+        FROM signals
+        WHERE created_at >= %s
+        GROUP BY signal_type
+        ORDER BY signal_type
+        """,
+        (week_ago,),
+    )
+
+    total_signals = 0
+    signal_counts: dict[str, int] = {}
+    for sig_type, cnt in sig_rows:
+        signal_counts[sig_type] = cnt
+        total_signals += cnt
+
+    # Count each type
+    strong_buy = signal_counts.get("STRONG_BUY", 0)
+    buy = signal_counts.get("BUY", 0)
+    strong_sell = signal_counts.get("STRONG_SELL", 0)
+    sell = signal_counts.get("SELL", 0)
+
+    lines.append(
+        f"📈 *Signals:* {total_signals} "
+        f"({strong_buy} SB / {buy} B / {sell} S / {strong_sell} SS)"
+    )
+
+    # ------------------------------------------------------------------
+    # 2. Top 5 symbols
+    # ------------------------------------------------------------------
+    top5_rows = query(
+        conn,
+        """
+        SELECT symbol, COUNT(*) as cnt
+        FROM signals
+        WHERE created_at >= %s
+        GROUP BY symbol
+        ORDER BY cnt DESC
+        LIMIT 5
+        """,
+        (week_ago,),
+    )
+
+    if top5_rows:
+        top5_str = ", ".join(f"{sym}({cnt})" for sym, cnt in top5_rows)
+        lines.append(f"🔥 *Hot symbols:* {top5_str}")
+    else:
+        lines.append("🔥 *Hot symbols:* (none)")
+
+    lines.append("")
+
+    # ------------------------------------------------------------------
+    # 3. Hypothetical trades summary
+    # ------------------------------------------------------------------
+    trade_rows = query(
+        conn,
+        """
+        SELECT
+            COUNT(*) as total,
+            SUM(CASE WHEN pnl > 0 THEN 1 ELSE 0 END) as wins,
+            SUM(CASE WHEN pnl < 0 THEN 1 ELSE 0 END) as losses,
+            SUM(pnl) as total_pnl
+        FROM hypothetical_trades
+        WHERE created_at >= %s
+          AND status = 'CLOSED'
+        """,
+        (week_ago,),
+    )
+
+    if trade_rows and trade_rows[0][0] and trade_rows[0][0] > 0:
+        total_trades = int(trade_rows[0][0])
+        wins = int(trade_rows[0][1] or 0)
+        losses = int(trade_rows[0][2] or 0)
+        total_pnl = trade_rows[0][3] or Decimal("0.00")
+
+        win_rate = (wins / total_trades * 100) if total_trades > 0 else 0.0
+
+        lines.append(
+            f"💰 *Trades:* {total_trades} ({wins}W/{losses}L) — WR: {win_rate:.1f}%"
+        )
+        lines.append(f"PnL: {format_pnl(total_pnl)}")
+    else:
+        lines.append("💰 *Trades:* 0 (0W/0L) — WR: 0.0%")
+        lines.append("PnL: $0.00")
+
+    # ------------------------------------------------------------------
+    # 4. Best / Worst symbol by PnL
+    # ------------------------------------------------------------------
+    best_worst_rows = query(
+        conn,
+        """
+        SELECT symbol, SUM(pnl) as total_pnl
+        FROM hypothetical_trades
+        WHERE created_at >= %s
+          AND status = 'CLOSED'
+        GROUP BY symbol
+        ORDER BY total_pnl DESC
+        """,
+        (week_ago,),
+    )
+
+    if best_worst_rows:
+        best_symbol, best_pnl = best_worst_rows[0]
+        worst_symbol, worst_pnl = best_worst_rows[-1]
+        lines.append(f"Best: {best_symbol} ({format_pnl(best_pnl)})")
+        lines.append(f"Worst: {worst_symbol} ({format_pnl(worst_pnl)})")
+    else:
+        lines.append("Best: — ($0.00)")
+        lines.append("Worst: — ($0.00)")
+
+    # ------------------------------------------------------------------
+    # 5. Footer
+    # ------------------------------------------------------------------
+    lines.append("")
+    lines.append("━━━━━━━━━━━━━━━")
+    lines.append(f"_Generated {now.strftime('%Y-%m-%d %H:%M UTC')}_")
+
+    conn.close()
+    return "\n".join(lines)
+
+
+# ---------------------------------------------------------------------------
+# Main
+# ---------------------------------------------------------------------------
+
+if __name__ == "__main__":
+    report = generate_report()
+    print(report)