diff --git a/backend/app/api/v1/backtest.py b/backend/app/api/v1/backtest.py index badb584..7ac8710 100755 --- a/backend/app/api/v1/backtest.py +++ b/backend/app/api/v1/backtest.py @@ -11,7 +11,12 @@ from app.database import get_db from app.models.candle import Candle from app.models.symbol import Symbol from app.models.exchange import Exchange -from app.services.backtest_engine import run_backtest as _run_backtest, MIN_CANDLES +from app.services.backtest_engine import ( + run_backtest as _run_backtest, + MIN_CANDLES, + DEFAULT_TAKER_FEE_PCT, + DEFAULT_SLIPPAGE_PCT, +) logger = logging.getLogger(__name__) router = APIRouter(prefix="/backtest", tags=["backtest"]) @@ -76,11 +81,13 @@ async def run_backtest( timeframe: str = Query("30m"), days: int = Query(7), trade_size: float = Query(10.0), + fee_pct: float = Query(DEFAULT_TAKER_FEE_PCT, ge=0, description="Round-trip-per-fill taker fee, e.g. 0.001 = 0.1%"), + slippage_pct: float = Query(DEFAULT_SLIPPAGE_PCT, ge=0, description="Adverse slippage per fill, e.g. 0.0005 = 0.05%"), db: AsyncSession = Depends(get_db), ): """Run backtest and return JSON results.""" result = await _run_backtest( - db, symbol, exchange, timeframe, days, Decimal(str(trade_size)) + db, symbol, exchange, timeframe, days, Decimal(str(trade_size)), fee_pct, slippage_pct, ) if "error" in result: raise HTTPException(status_code=400, detail=result["error"]) @@ -94,7 +101,9 @@ async def run_backtest_post( timeframe: str = Query("30m"), days: int = Query(7), trade_size: float = Query(10.0), + fee_pct: float = Query(DEFAULT_TAKER_FEE_PCT, ge=0), + slippage_pct: float = Query(DEFAULT_SLIPPAGE_PCT, ge=0), db: AsyncSession = Depends(get_db), ): """Alias for GET /backtest/run — supports POST method.""" - return await run_backtest(symbol, exchange, timeframe, days, trade_size, db) + return await run_backtest(symbol, exchange, timeframe, days, trade_size, fee_pct, slippage_pct, db) diff --git a/backend/app/api/v1/orders.py b/backend/app/api/v1/orders.py index aeec0ec..a41b55c 100755 --- a/backend/app/api/v1/orders.py +++ b/backend/app/api/v1/orders.py @@ -96,7 +96,11 @@ async def place_order( current_user.username, exchange_name, req.symbol, req.side, req.amount, ) - # Persist to real_trades + # Persist to real_trades. For market orders `req.price` is None (no + # limit price was ever set) — the actual execution price only comes + # back from the exchange as `order.average`/`order.price`. Without + # it, this row would have no price at all and PnL could never be + # computed for it later (see sync_real_trades' FIFO PnL matching). real_trade = RealTrade( user_id=current_user.id, exchange=exchange_name, @@ -104,7 +108,7 @@ async def place_order( side=req.side, order_type=req.order_type, amount=req.amount, - price=req.price, + price=order.average or order.price or req.price, filled_amount=order.filled, status=order.status, order_id=order.order_id, diff --git a/backend/app/api/v1/walk_forward.py b/backend/app/api/v1/walk_forward.py index 1e1e18d..4f84169 100644 --- a/backend/app/api/v1/walk_forward.py +++ b/backend/app/api/v1/walk_forward.py @@ -21,6 +21,7 @@ from app.database import get_db from app.core.deps import get_current_user from app.models.user import User as UserModel from app.services.walk_forward import run_walk_forward +from app.services.backtest_engine import DEFAULT_TAKER_FEE_PCT, DEFAULT_SLIPPAGE_PCT logger = logging.getLogger(__name__) router = APIRouter(prefix="/walk-forward", tags=["walk_forward"]) @@ -35,6 +36,8 @@ async def run( train_days: int = Query(270, ge=30, description="Train window size per fold, in days"), test_days: int = Query(90, ge=14, description="Held-out test window size per fold, in days"), trade_size: float = Query(10.0), + fee_pct: float = Query(DEFAULT_TAKER_FEE_PCT, ge=0, description="Round-trip-per-fill taker fee, e.g. 0.001 = 0.1%"), + slippage_pct: float = Query(DEFAULT_SLIPPAGE_PCT, ge=0, description="Adverse slippage per fill, e.g. 0.0005 = 0.05%"), db: AsyncSession = Depends(get_db), current_user: UserModel = Depends(get_current_user), ): @@ -43,6 +46,7 @@ async def run( db, symbol, exchange, timeframe, total_days=total_days, train_days=train_days, test_days=test_days, trade_size=Decimal(str(trade_size)), + fee_pct=fee_pct, slippage_pct=slippage_pct, ) if "error" in result: raise HTTPException(status_code=400, detail=result["error"]) diff --git a/backend/app/services/backtest_engine.py b/backend/app/services/backtest_engine.py index ab6215b..bf53f7d 100644 --- a/backend/app/services/backtest_engine.py +++ b/backend/app/services/backtest_engine.py @@ -37,6 +37,77 @@ from app.services.signal_scoring import ( # Min candles for warmup: BB(20) + RSI(14) + some room = 30 MIN_CANDLES = 30 +# Trading-cost defaults — previously the simulator priced every fill at the +# exact candle close with zero cost, which made every backtest/walk-forward +# report systematically more profitable than live trading could ever be. +# Taker fee: 0.1% per fill (0.2% round-trip) matches typical crypto spot +# taker fees without VIP/token discounts (Binance, MEXC). Slippage: 0.05% +# per fill is a conservative estimate for liquid majors on market orders — +# thinner symbols would see more in reality, but this at least stops +# results from assuming frictionless fills. +DEFAULT_TAKER_FEE_PCT = 0.001 +DEFAULT_SLIPPAGE_PCT = 0.0005 + + +def _fill_price(mark_price: float, direction: str, is_entry: bool, slippage_pct: float) -> float: + """Simulate a realistic market-order fill price — slippage always moves + the price against the trader, never in their favor. + + LONG entry / SHORT exit both buy (fill above mark price). + SHORT entry / LONG exit both sell (fill below mark price). + """ + buying = (direction == "LONG") == is_entry + return mark_price * (1 + slippage_pct) if buying else mark_price * (1 - slippage_pct) + + +def _open_position(direction: str, mark_price: float, timestamp, index: int, + trade_size: Decimal, entry_signal: str, + fee_pct: float, slippage_pct: float) -> dict: + """Build a new open position dict, applying entry slippage/fees. + + `quantity` is sized off the slipped fill price (not the raw mark price) + so that `entry_price * quantity == trade_size`, matching how a real + market order spends a fixed quote-currency amount and receives fewer + units when the fill is worse than the observed price. + """ + entry_price = _fill_price(mark_price, direction, True, slippage_pct) + quantity = float(trade_size) / entry_price + entry_fee = entry_price * quantity * fee_pct + return { + "direction": direction, "entry_price": entry_price, + "entry_time": timestamp, "quantity": quantity, + "entry_signal": entry_signal, "entry_index": index, "status": "OPEN", + "entry_fee": entry_fee, + } + + +def _close_position(position: dict, mark_price: float, timestamp, exit_reason: str, + fee_pct: float, slippage_pct: float) -> dict: + """Close an open position in-place, applying exit slippage/fees. + + `pnl` is the NET result (gross price movement minus round-trip fees) — + every caller downstream (_compute_stats, walk_forward's fold scoring) + reads `pnl` directly, so this is the only place cost needs to be + subtracted for it to flow through the whole system. + """ + direction = position["direction"] + entry_price = position["entry_price"] + quantity = position["quantity"] + exit_price = _fill_price(mark_price, direction, False, slippage_pct) + exit_fee = exit_price * quantity * fee_pct + if direction == "LONG": + gross_pnl = (exit_price - entry_price) * quantity + else: + gross_pnl = (entry_price - exit_price) * quantity + entry_fee = position.get("entry_fee", 0.0) + fees = entry_fee + exit_fee + position.update({ + "exit_price": exit_price, "exit_time": timestamp, + "pnl": gross_pnl - fees, "gross_pnl": gross_pnl, "fees": fees, + "status": "CLOSED", "exit_reason": exit_reason, + }) + return position + # Bounded trailing-window sizes used when replaying indicators per candle. # Every consumer in signal_scoring.py only ever reads the last 1-2 elements # of these arrays except the BB squeeze check (lookback=10), so windows @@ -433,12 +504,20 @@ def _simulate_from_scores( strong_threshold: float = 4.0, signal_threshold: float = 1.0, max_hold_candles: int = 48, + fee_pct: float = DEFAULT_TAKER_FEE_PCT, + slippage_pct: float = DEFAULT_SLIPPAGE_PCT, ) -> tuple[list[dict], list[dict]]: """Cheap half of simulation: turn a precomputed score series into signals + trades for one choice of thresholds. See `_compute_scores_series` for the expensive half — run once, reused across every threshold combination a grid search tries. + + Every fill (entry and exit) goes through `_fill_price`/`_open_position`/ + `_close_position` so simulated trades pay the same round-trip fee and + adverse slippage a live market order would, instead of pricing fills at + the exact candle close for free — see DEFAULT_TAKER_FEE_PCT/ + DEFAULT_SLIPPAGE_PCT above. """ all_signals: list[dict] = [] trades: list[dict] = [] @@ -466,77 +545,43 @@ def _simulate_from_scores( if signal_type in (STRONG_BUY, BUY): if current_position and current_position["direction"] == "SHORT": if signal_type == STRONG_BUY: - entry_price = current_position["entry_price"] - qty = current_position["quantity"] - pnl = (entry_price - latest_close) * qty - current_position.update({ - "exit_price": latest_close, "exit_time": timestamp, - "pnl": pnl, "status": "CLOSED", "exit_reason": "REVERSAL", - }) + _close_position(current_position, latest_close, timestamp, "REVERSAL", fee_pct, slippage_pct) trades.append(current_position) current_position = None else: continue if not current_position: - qty = float(trade_size) / latest_close - current_position = { - "direction": "LONG", "entry_price": latest_close, - "entry_time": timestamp, "quantity": qty, - "entry_signal": signal_type, "entry_index": i, "status": "OPEN", - } + current_position = _open_position( + "LONG", latest_close, timestamp, i, trade_size, signal_type, fee_pct, slippage_pct, + ) elif signal_type in (STRONG_SELL, SELL): if current_position and current_position["direction"] == "LONG": if signal_type == STRONG_SELL: - entry_price = current_position["entry_price"] - qty = current_position["quantity"] - pnl = (latest_close - entry_price) * qty - current_position.update({ - "exit_price": latest_close, "exit_time": timestamp, - "pnl": pnl, "status": "CLOSED", "exit_reason": "REVERSAL", - }) + _close_position(current_position, latest_close, timestamp, "REVERSAL", fee_pct, slippage_pct) trades.append(current_position) current_position = None else: continue if not current_position: - qty = float(trade_size) / latest_close - current_position = { - "direction": "SHORT", "entry_price": latest_close, - "entry_time": timestamp, "quantity": qty, - "entry_signal": signal_type, "entry_index": i, "status": "OPEN", - } + current_position = _open_position( + "SHORT", latest_close, timestamp, i, trade_size, signal_type, fee_pct, slippage_pct, + ) if current_position and current_position["status"] == "OPEN": hold = i - current_position["entry_index"] if hold >= max_hold_candles: - entry_price = current_position["entry_price"] - qty = current_position["quantity"] - if current_position["direction"] == "LONG": - pnl = (latest_close - entry_price) * qty - else: - pnl = (entry_price - latest_close) * qty - current_position.update({ - "exit_price": latest_close, "exit_time": timestamp, - "pnl": pnl, "status": "CLOSED", "exit_reason": "TIME_LIMIT", - }) + _close_position(current_position, latest_close, timestamp, "TIME_LIMIT", fee_pct, slippage_pct) trades.append(current_position) current_position = None # Close final position if current_position and current_position["status"] == "OPEN": last_close = float(candles[-1].close) - entry_price = current_position["entry_price"] - qty = current_position["quantity"] - if current_position["direction"] == "LONG": - pnl = (last_close - entry_price) * qty - else: - pnl = (entry_price - last_close) * qty - current_position.update({ - "exit_price": last_close, - "exit_time": candles[-1].timestamp.isoformat(), - "pnl": pnl, "status": "CLOSED", "exit_reason": "END_OF_DATA", - }) + _close_position( + current_position, last_close, candles[-1].timestamp.isoformat(), + "END_OF_DATA", fee_pct, slippage_pct, + ) trades.append(current_position) return all_signals, trades @@ -550,6 +595,8 @@ def _simulate_trades( signal_threshold: float = 1.0, max_hold_candles: int = 48, active_from_index: int = MIN_CANDLES, + fee_pct: float = DEFAULT_TAKER_FEE_PCT, + slippage_pct: float = DEFAULT_SLIPPAGE_PCT, ) -> tuple[list[dict], list[dict]]: """Convenience wrapper: compute scores then simulate with one set of thresholds. Callers trying many threshold combinations against the @@ -558,7 +605,10 @@ def _simulate_trades( combination instead — see walk_forward.py. """ scores_series = _compute_scores_series(candles, precomputed, active_from_index) - return _simulate_from_scores(candles, scores_series, trade_size, strong_threshold, signal_threshold, max_hold_candles) + return _simulate_from_scores( + candles, scores_series, trade_size, strong_threshold, signal_threshold, + max_hold_candles, fee_pct, slippage_pct, + ) def _compute_stats(all_signals: list[dict], trades: list[dict]) -> dict: @@ -578,6 +628,7 @@ def _compute_stats(all_signals: list[dict], trades: list[dict]) -> dict: profit_factor = round(abs(gross_profit / gross_loss), 2) if gross_loss != 0 else None avg_win = round(gross_profit / len(winning_trades), 2) if winning_trades else None avg_loss = round(gross_loss / len(losing_trades), 2) if losing_trades else None + total_fees = round(sum(t.get("fees", 0) for t in closed_trades), 2) best_trade = max(closed_trades, key=lambda t: t.get("pnl", 0)) if closed_trades else None worst_trade = min(closed_trades, key=lambda t: t.get("pnl", 0)) if closed_trades else None @@ -595,6 +646,7 @@ def _compute_stats(all_signals: list[dict], trades: list[dict]) -> dict: "profit_factor": profit_factor, "avg_win": avg_win, "avg_loss": avg_loss, + "total_fees": total_fees, "best_trade": { "direction": best_trade.get("direction"), "entry_price": round(best_trade["entry_price"], 4), @@ -622,6 +674,8 @@ async def run_backtest( timeframe: str = "30m", days: int = 7, trade_size: Decimal = Decimal("10"), + fee_pct: float = DEFAULT_TAKER_FEE_PCT, + slippage_pct: float = DEFAULT_SLIPPAGE_PCT, ) -> dict: """Run a single backtest over the last `days` days and return structured results.""" db_symbol = await _fetch_symbol(db, symbol, exchange) @@ -642,7 +696,9 @@ async def run_backtest( return {"error": f"No candle data found for {symbol} on {timeframe}. The exchange may not support this pair."} precomputed = _precompute_indicators(candles, timeframe) - all_signals, trades = _simulate_trades(candles, precomputed, trade_size) + all_signals, trades = _simulate_trades( + candles, precomputed, trade_size, fee_pct=fee_pct, slippage_pct=slippage_pct, + ) stats = _compute_stats(all_signals, trades) return { diff --git a/backend/app/services/trade_executor.py b/backend/app/services/trade_executor.py index 305f0c0..6b76695 100644 --- a/backend/app/services/trade_executor.py +++ b/backend/app/services/trade_executor.py @@ -8,6 +8,7 @@ from __future__ import annotations import json import logging +from collections import defaultdict from datetime import datetime, timezone, timedelta from decimal import Decimal from sqlalchemy import and_, desc, select @@ -351,55 +352,143 @@ async def execute_signal_trade( # ═══════════════════════════════════════════════════════════ -# Real Trade Sync — closes stale real trades +# Real Trade Sync — closes stale real trades, computes real PnL # ═══════════════════════════════════════════════════════════ -async def sync_real_trades() -> None: - """Sync real trades: close stale ones, calculate PnL for closed ones. +async def _recompute_realized_pnl(db: AsyncSession) -> int: + """Recompute realized PnL for every filled real trade via FIFO lot + matching, grouped by (user_id, symbol, exchange). - Called periodically (every 5 min) by the scheduler. - Fixes: real trades were never being closed or having PnL calculated. + `RealTrade` rows represent individual order fills (one buy or one + sell), not paired open/close positions like `HypotheticalTrade` — so a + single row's PnL can't be read off itself. This walks each group's + fills in chronological order, matching each buy against any open SHORT + lots first (then opening a LONG lot with anything left over), and each + sell against open LONG lots first (then opening a SHORT lot) — the + realized PnL from whatever portion closes an existing lot is assigned + to that fill's row. A fill that only opens a new position (nothing to + close yet) has no realized PnL and is left at pnl=None until a later + fill closes it. + + Recomputes from the full history every time rather than accumulating + incrementally, which is safe to rerun (idempotent) — the trade volume + this endpoint supports (manual/auto real trading through one connected + exchange per user, not HFT) makes a full recompute cheap enough to run + on every periodic sync. + + Rows with no recorded price (e.g. legacy market-order fills persisted + before `orders.py` started storing `order.average`) can't be matched + and are skipped — they neither consume nor produce a lot, so a FIFO + walk that includes them would silently misattribute PnL to whichever + fill happens to match next. + """ + result = await db.execute( + select(RealTrade) + .where(RealTrade.filled_amount > 0, RealTrade.price.isnot(None)) + .order_by(RealTrade.user_id, RealTrade.symbol, RealTrade.exchange, RealTrade.created_at.asc()) + ) + trades = result.scalars().all() + + groups: dict[tuple, list[RealTrade]] = defaultdict(list) + for t in trades: + groups[(t.user_id, t.symbol, t.exchange)].append(t) + + updated = 0 + for _key, group in groups.items(): + long_lots: list[list] = [] # each lot: [remaining_qty: Decimal, entry_price: Decimal] + short_lots: list[list] = [] + + for t in group: + side = (t.side or "").lower() + qty = Decimal(t.filled_amount) + price = Decimal(t.price) + if side not in ("buy", "sell") or qty <= 0 or price <= 0: + continue + + opposing = short_lots if side == "buy" else long_lots + same_side_lots = long_lots if side == "buy" else short_lots + + remaining = qty + realized = Decimal("0") + matched_cost = Decimal("0") + while remaining > 0 and opposing: + lot = opposing[0] + matched = min(remaining, lot[0]) + if side == "buy": + realized += (lot[1] - price) * matched # closing a short: profit if price fell + else: + realized += (price - lot[1]) * matched # closing a long: profit if price rose + matched_cost += lot[1] * matched + lot[0] -= matched + remaining -= matched + if lot[0] <= 0: + opposing.pop(0) + + if remaining > 0: + same_side_lots.append([remaining, price]) + + if matched_cost > 0: + pnl_percent = (realized / matched_cost) * Decimal("100") + if t.pnl != realized or t.pnl_percent != pnl_percent: + t.pnl = realized + t.pnl_percent = pnl_percent + updated += 1 + elif t.pnl is not None: + # Pure opening fill — nothing existed yet to realize against. + t.pnl = None + t.pnl_percent = None + updated += 1 + + return updated + + +async def sync_real_trades() -> None: + """Sync real trades: close stale open orders, then (re)compute realized + PnL for every filled trade via FIFO matching (see + `_recompute_realized_pnl`). + + Called periodically (every 5 min) by the scheduler. Previously this + force-set pnl=0 for every stale-closed AND every already-closed-but- + unpriced trade regardless of whether it actually filled — silently + reporting "break-even" for real-money trades that may have made or + lost real money. Now pnl is only ever 0/None for orders that genuinely + never filled; any order with a nonzero fill gets a real FIFO-matched + PnL once a later trade closes its position. """ async with async_session_factory() as db: - # 1. Fetch open real trades + # 1. Force-close real orders stuck "open" too long (exchange likely + # never filled them). Only the STATUS changes here for anything + # with a partial fill — its PnL (if any) comes from the FIFO + # recompute below once/if a later trade closes it out. result = await db.execute( select(RealTrade).where(RealTrade.status == "open") ) open_trades = result.scalars().all() - - if not open_trades: - return - now = datetime.now(timezone.utc) - + stale_closed = 0 for trade in open_trades: hold_duration = now - trade.created_at if hold_duration > timedelta(hours=24): trade.status = "closed" trade.closed_at = now - trade.pnl = Decimal("0") - trade.pnl_percent = Decimal("0") + if not trade.filled_amount or trade.filled_amount <= 0: + # Never filled at all — there is genuinely no PnL. + trade.pnl = Decimal("0") + trade.pnl_percent = Decimal("0") + stale_closed += 1 logger.info( "🔒 Real trade #%d CLOSED (time limit 24h): %s %s", trade.id, trade.side, trade.symbol, ) - # 2. Calculate PnL for closed trades missing it - result2 = await db.execute( - select(RealTrade).where( - and_(RealTrade.status.in_(["closed", "filled"]), - RealTrade.pnl.is_(None)) - ) - ) - closed_no_pnl = result2.scalars().all() - - for trade in closed_no_pnl: - trade.pnl = Decimal("0") - trade.pnl_percent = Decimal("0") + # 2. Recompute realized PnL for every filled trade (not just ones + # missing it — a new closing fill can retroactively realize PnL on + # an earlier opening fill that already had pnl=None). + updated = await _recompute_realized_pnl(db) await db.commit() - if open_trades or closed_no_pnl: + if stale_closed or updated: logger.info( - "Real trade sync: %d open checked, %d closed PnL fixed", - len(open_trades), len(closed_no_pnl), + "Real trade sync: %d stale trades closed, %d PnL rows recomputed", + stale_closed, updated, ) diff --git a/backend/app/services/walk_forward.py b/backend/app/services/walk_forward.py index 65abe4f..edceac8 100644 --- a/backend/app/services/walk_forward.py +++ b/backend/app/services/walk_forward.py @@ -26,6 +26,8 @@ from sqlalchemy.ext.asyncio import AsyncSession from app.services.backtest_engine import ( MIN_CANDLES, + DEFAULT_TAKER_FEE_PCT, + DEFAULT_SLIPPAGE_PCT, _fetch_symbol, _fetch_candles, _precompute_indicators, @@ -138,7 +140,7 @@ async def _prepare_window( return candles, precomputed, active_from_index -def _run_combo(candles, scores_series, trade_size, params): +def _run_combo(candles, scores_series, trade_size, params, fee_pct, slippage_pct): """Apply one parameter combination to an already-computed score series (see `_compute_scores_series`) — cheap, no indicator recomputation.""" return _simulate_from_scores( @@ -146,6 +148,8 @@ def _run_combo(candles, scores_series, trade_size, params): strong_threshold=params["strong_threshold"], signal_threshold=params["signal_threshold"], max_hold_candles=int(params["max_hold_candles"]), + fee_pct=fee_pct, + slippage_pct=slippage_pct, ) @@ -154,12 +158,20 @@ def _grid_search( scores_series: list[dict], param_grid: dict[str, list[float]], trade_size: Decimal, + fee_pct: float = DEFAULT_TAKER_FEE_PCT, + slippage_pct: float = DEFAULT_SLIPPAGE_PCT, ) -> tuple[dict[str, float], float, dict]: """Try every combination in param_grid against one precomputed score series, return the one that scores best on the train window by `_fold_score`. The 13-algorithm scoring itself already happened once to produce `scores_series` — this loop only replays cheap threshold - comparisons and trade bookkeeping per combination.""" + comparisons and trade bookkeeping per combination. + + Fees/slippage are applied here too (not just the final OOS report) so + the grid search picks parameters that hold up after trading costs, + instead of favoring high-frequency combos that only look good when + fills are free. + """ keys = list(param_grid.keys()) best_params: dict[str, float] | None = None best_score = float("-inf") @@ -167,7 +179,7 @@ def _grid_search( for combo in product(*(param_grid[k] for k in keys)): params = dict(zip(keys, combo)) - all_signals, trades = _run_combo(candles, scores_series, trade_size, params) + all_signals, trades = _run_combo(candles, scores_series, trade_size, params, fee_pct, slippage_pct) closed_trades = [t for t in trades if t.get("status") == "CLOSED"] score = _fold_score(closed_trades) if score > best_score: @@ -179,7 +191,7 @@ def _grid_search( # Every combo scored -inf (too few trades) — still report the # grid's first combination so the fold has *something* to show. best_params = {k: param_grid[k][0] for k in keys} - all_signals, trades = _run_combo(candles, scores_series, trade_size, best_params) + all_signals, trades = _run_combo(candles, scores_series, trade_size, best_params, fee_pct, slippage_pct) best_stats = _compute_stats(all_signals, trades) return best_params, best_score, best_stats @@ -209,6 +221,8 @@ async def run_walk_forward( test_days: int = 90, trade_size: Decimal = Decimal("10"), param_grid: dict[str, list[float]] | None = None, + fee_pct: float = DEFAULT_TAKER_FEE_PCT, + slippage_pct: float = DEFAULT_SLIPPAGE_PCT, ) -> dict: """Run a full walk-forward analysis: optimize params per fold on the train window, evaluate out-of-sample on the test window, then stitch @@ -234,7 +248,7 @@ async def run_walk_forward( train_scores = _compute_scores_series(train_candles, train_precomputed, train_active_from) best_params, _train_score, train_stats = _grid_search( - train_candles, train_scores, param_grid, trade_size, + train_candles, train_scores, param_grid, trade_size, fee_pct, slippage_pct, ) test_window = await _prepare_window(db, db_symbol.id, timeframe, spec["test_start"], spec["test_end"]) @@ -243,7 +257,7 @@ async def run_walk_forward( test_candles, test_precomputed, test_active_from = test_window test_scores = _compute_scores_series(test_candles, test_precomputed, test_active_from) - test_signals, test_trades = _run_combo(test_candles, test_scores, trade_size, best_params) + test_signals, test_trades = _run_combo(test_candles, test_scores, trade_size, best_params, fee_pct, slippage_pct) test_stats = _compute_stats(test_signals, test_trades) stitched_oos_trades.extend(t for t in test_trades if t.get("status") == "CLOSED") @@ -265,6 +279,7 @@ async def run_walk_forward( "win_rate": test_stats["trades"]["win_rate"], "total_pnl": test_stats["trades"]["total_pnl"], "profit_factor": test_stats["trades"]["profit_factor"], + "total_fees": test_stats["trades"]["total_fees"], }, }) @@ -282,6 +297,7 @@ async def run_walk_forward( oos_total_pnl = sum(t.get("pnl", 0) for t in stitched_oos_trades) oos_win_rate = round(len(oos_wins) / len(stitched_oos_trades) * 100, 1) if stitched_oos_trades else 0.0 oos_profit_factor = round(abs(oos_gross_profit / oos_gross_loss), 2) if oos_gross_loss != 0 else None + oos_total_fees = round(sum(t.get("fees", 0) for t in stitched_oos_trades), 2) equity_curve = [0.0] running = 0.0 @@ -297,6 +313,8 @@ async def run_walk_forward( "train_days": train_days, "test_days": test_days, "param_grid": param_grid, + "fee_pct": fee_pct, + "slippage_pct": slippage_pct, "folds": fold_results, "out_of_sample_summary": { "trades": len(stitched_oos_trades), @@ -305,6 +323,7 @@ async def run_walk_forward( "win_rate": oos_win_rate, "total_pnl": round(oos_total_pnl, 2), "profit_factor": oos_profit_factor, + "total_fees": oos_total_fees, "max_drawdown_pct": _max_drawdown_pct(equity_curve), "equity_curve": equity_curve, }, diff --git a/backend/tests/test_backtest_engine.py b/backend/tests/test_backtest_engine.py index 5f85866..ab67daf 100644 --- a/backend/tests/test_backtest_engine.py +++ b/backend/tests/test_backtest_engine.py @@ -196,6 +196,110 @@ async def test_simulate_trades_active_from_index_skips_warmup_region(monkeypatch assert all_signals == [] +class TestFeeAndSlippage: + """Regression tests for fee (o): backtest/walk-forward used to price + every fill at the exact candle close with zero cost, which made every + reported win rate/profit factor systematically more optimistic than + live trading could ever achieve. See DEFAULT_TAKER_FEE_PCT/ + DEFAULT_SLIPPAGE_PCT and _fill_price/_open_position/_close_position in + backtest_engine.py. + """ + + def test_fill_price_moves_against_the_trader(self): + mark = 100.0 + slip = 0.001 + # LONG entry buys -> fills higher than mark. + assert backtest_engine._fill_price(mark, "LONG", True, slip) == pytest.approx(100.1) + # LONG exit sells -> fills lower than mark. + assert backtest_engine._fill_price(mark, "LONG", False, slip) == pytest.approx(99.9) + # SHORT entry sells -> fills lower than mark. + assert backtest_engine._fill_price(mark, "SHORT", True, slip) == pytest.approx(99.9) + # SHORT exit buys -> fills higher than mark. + assert backtest_engine._fill_price(mark, "SHORT", False, slip) == pytest.approx(100.1) + + async def test_reversal_exit_pnl_is_net_of_fees_and_slippage(self, monkeypatch, db_session): + _, symbol = await _seed_symbol(db_session) + base = datetime.now(timezone.utc) - timedelta(hours=40) + await _seed_candles(db_session, symbol.id, "1h", base, 40, timedelta(hours=1), lambda i: i) + candles = await backtest_engine._fetch_candles(db_session, symbol.id, "1h", since=base) + precomputed = backtest_engine._precompute_indicators(candles, "1h") + + # STRONG_BUY at candle 30 (price 30), STRONG_SELL at candle 35 + # (price 35) reverses and closes it — a straightforward winning + # LONG before costs. The STRONG_SELL that closes it also opens a + # new SHORT in the same step (existing reversal behavior), which + # then rides to END_OF_DATA — only the first (LONG) trade matters + # for this assertion. + fake, _ = _make_fake_score_fn(buy_at={30}, sell_at={35}) + monkeypatch.setattr(backtest_engine, "_compute_adjusted_score", fake) + + _, trades = backtest_engine._simulate_trades(candles, precomputed, Decimal("10")) + assert len(trades) == 2 + trade = trades[0] + assert trade["direction"] == "LONG" + assert trade["exit_reason"] == "REVERSAL" + + fee_pct = backtest_engine.DEFAULT_TAKER_FEE_PCT + slip = backtest_engine.DEFAULT_SLIPPAGE_PCT + expected_entry = 30.0 * (1 + slip) + expected_qty = 10.0 / expected_entry + expected_exit = 35.0 * (1 - slip) + expected_gross = (expected_exit - expected_entry) * expected_qty + expected_fees = (expected_entry + expected_exit) * expected_qty * fee_pct + expected_net = expected_gross - expected_fees + + assert trade["entry_price"] == pytest.approx(expected_entry) + assert trade["exit_price"] == pytest.approx(expected_exit) + assert trade["gross_pnl"] == pytest.approx(expected_gross) + assert trade["fees"] == pytest.approx(expected_fees) + assert trade["pnl"] == pytest.approx(expected_net) + # The whole point of the fix: costs must actually eat into PnL. + assert trade["pnl"] < trade["gross_pnl"] + assert trade["fees"] > 0 + + async def test_zero_fee_and_slippage_matches_raw_price_pnl(self, monkeypatch, db_session): + """fee_pct=0/slippage_pct=0 must reduce to the old frictionless + behavior — fills at the exact close, no cost — so existing callers + that don't care about costs (or want to see raw signal quality) + aren't forced into a changed baseline.""" + _, symbol = await _seed_symbol(db_session) + base = datetime.now(timezone.utc) - timedelta(hours=40) + await _seed_candles(db_session, symbol.id, "1h", base, 40, timedelta(hours=1), lambda i: i) + candles = await backtest_engine._fetch_candles(db_session, symbol.id, "1h", since=base) + precomputed = backtest_engine._precompute_indicators(candles, "1h") + + fake, _ = _make_fake_score_fn(buy_at={30}, sell_at={35}) + monkeypatch.setattr(backtest_engine, "_compute_adjusted_score", fake) + + _, trades = backtest_engine._simulate_trades( + candles, precomputed, Decimal("10"), fee_pct=0.0, slippage_pct=0.0, + ) + trade = trades[0] + assert trade["direction"] == "LONG" + assert trade["entry_price"] == pytest.approx(30.0) + assert trade["exit_price"] == pytest.approx(35.0) + expected_qty = 10.0 / 30.0 + assert trade["pnl"] == pytest.approx((35.0 - 30.0) * expected_qty) + assert trade["fees"] == pytest.approx(0.0) + + async def test_compute_stats_reports_total_fees(self, monkeypatch, db_session): + _, symbol = await _seed_symbol(db_session) + base = datetime.now(timezone.utc) - timedelta(hours=40) + await _seed_candles(db_session, symbol.id, "1h", base, 40, timedelta(hours=1), lambda i: i) + candles = await backtest_engine._fetch_candles(db_session, symbol.id, "1h", since=base) + precomputed = backtest_engine._precompute_indicators(candles, "1h") + + fake, _ = _make_fake_score_fn(buy_at={30}, sell_at={35}) + monkeypatch.setattr(backtest_engine, "_compute_adjusted_score", fake) + + all_signals, trades = backtest_engine._simulate_trades(candles, precomputed, Decimal("10")) + stats = backtest_engine._compute_stats(all_signals, trades) + assert stats["trades"]["total_fees"] == pytest.approx( + sum(t["fees"] for t in trades), abs=0.01, + ) + assert stats["trades"]["total_fees"] > 0 + + def _build_candle_series(base, prices): return [ Candle( diff --git a/backend/tests/test_indicator_service.py b/backend/tests/test_indicator_service.py index bd77b24..520e60d 100644 --- a/backend/tests/test_indicator_service.py +++ b/backend/tests/test_indicator_service.py @@ -16,12 +16,15 @@ from app.services.indicator_service import ( detect_market_regime, ema, macd, + market_structure, mfi, obv, obv_signal, rsi, sma, vwap, + _find_pivot_highs, + _find_pivot_lows, ) @@ -264,3 +267,64 @@ class TestDetectMarketRegime: self._adx(22), bb, atr_pct=1.0, volume_data=None, ) assert regime == "neutral" + + +class TestPivotCausalConsistency: + """A pivot at index i is only knowable once `right` bars after it exist + (see `_find_pivot_highs`/`_find_pivot_lows`'s definition). Live trading + (`candle_service.get_indicators`) calls `detect_divergence`/ + `market_structure` on data "as of now" with no future bars — these + tests lock in that this is self-consistent (never claims a pivot it + can't yet know about) and never repaints (a pivot's status, once + confirmable, doesn't change as more future data arrives). This is the + same causal delay backtest_engine.py's explicit confirmed-pointer + bookkeeping was built to match (see its _PIVOT_LOOKBACK handling and + test_scores_are_causal_future_prices_dont_change_earlier_scores) — if + either side ever stopped honoring it, backtest and live would silently + diverge on how early Divergence/SMC signals fire. + """ + + def _wavy_prices(self, n: int, seed: int = 0) -> list[float]: + return [100 + math.sin((i + seed) / 3.0) * 10 + (i % 5) for i in range(n)] + + def test_pivot_highs_never_flag_the_last_right_bars(self): + prices = self._wavy_prices(40) + right = 3 + highs = _find_pivot_highs(prices, left=right, right=right) + assert all(v is None for v in highs[-right:]) + + def test_pivot_lows_never_flag_the_last_right_bars(self): + prices = self._wavy_prices(40, seed=2) + right = 3 + lows = _find_pivot_lows(prices, left=right, right=right) + assert all(v is None for v in lows[-right:]) + + def test_pivot_status_never_repaints_once_confirmable(self): + prices = self._wavy_prices(30) + right = 3 + highs_before = _find_pivot_highs(prices, right, right) + lows_before = _find_pivot_lows(prices, right, right) + + # More candles arrive — bars that already had enough future + # confirmation must keep the exact same pivot/non-pivot verdict. + extended = prices + [95.0, 130.0, 80.0, 140.0, 70.0, 150.0] + highs_after = _find_pivot_highs(extended, right, right) + lows_after = _find_pivot_lows(extended, right, right) + + confirmable = len(prices) - right + assert highs_after[:confirmable] == highs_before[:confirmable] + assert lows_after[:confirmable] == lows_before[:confirmable] + + def test_market_structure_swings_never_repaint_as_more_candles_arrive(self): + prices = self._wavy_prices(30, seed=1) + candles = [candle(p + 2, p - 2, p) for p in prices] + ms_before = market_structure(candles, pivot_lookback=3) + + more_candles = candles + [ + candle(122, 98, 120), candle(92, 68, 70), candle(142, 118, 140), + ] + ms_after = market_structure(more_candles, pivot_lookback=3) + + confirmable = len(candles) - 3 + assert ms_after["swing_highs"][:confirmable] == ms_before["swing_highs"][:confirmable] + assert ms_after["swing_lows"][:confirmable] == ms_before["swing_lows"][:confirmable] diff --git a/backend/tests/test_order_exchange_routing.py b/backend/tests/test_order_exchange_routing.py index e625a78..76c9b83 100644 --- a/backend/tests/test_order_exchange_routing.py +++ b/backend/tests/test_order_exchange_routing.py @@ -35,6 +35,8 @@ class StubAdapter: order_id="STUB-ORDER-1", filled=req.amount, status="closed", + average=None, + price=None, ) diff --git a/backend/tests/test_trade_executor.py b/backend/tests/test_trade_executor.py index fdb5bc6..a850782 100644 --- a/backend/tests/test_trade_executor.py +++ b/backend/tests/test_trade_executor.py @@ -22,12 +22,15 @@ from sqlalchemy import select from app.models import Exchange, HypotheticalTrade, Signal, Symbol, User from app.models.candle import Candle +from app.models.real_trade import RealTrade from app.services.trade_executor import ( MAX_OPEN_TRADES, STRONG_BUY, STRONG_SELL, _calculate_pnl, + _recompute_realized_pnl, execute_signal_trade, + sync_real_trades, ) @@ -296,3 +299,182 @@ async def test_hybrid_eviction_uses_each_trades_own_symbol_price_not_incoming_si assert closed_trades[0].exit_price == Decimal("50"), "exit price must come from REAL/USDT's own candle, not the signal's 1000" assert closed_trades[0].pnl == Decimal("-50") assert any(t.symbol == "NEW/USDT" and t.status == "OPEN" for t in all_trades) + + +def make_real_trade(user_id, symbol="BTC/USDT", side="buy", amount="1", price="100", + filled_amount=None, status="filled", created_at=None) -> RealTrade: + return RealTrade( + user_id=user_id, + exchange="mexc", + symbol=symbol, + side=side, + order_type="market", + amount=Decimal(amount), + price=Decimal(price) if price is not None else None, + filled_amount=Decimal(filled_amount if filled_amount is not None else amount), + status=status, + created_at=created_at or datetime.now(timezone.utc), + ) + + +class TestRecomputeRealizedPnl: + """Regression tests for the sync_real_trades() fix: `RealTrade` rows are + individual order fills (one buy or one sell), not paired open/close + positions — PnL was previously hardcoded to 0 for every trade needing + it, silently reporting "break-even" for real trades that may have won + or lost real money. `_recompute_realized_pnl` matches opposing fills + FIFO per (user, symbol, exchange) instead. + """ + + async def test_opening_fill_has_no_pnl_until_something_closes_it(self, db_session): + user = make_user() + db_session.add(user) + await db_session.flush() + buy = make_real_trade(user.id, side="buy", price="100") + db_session.add(buy) + await db_session.flush() + + updated = await _recompute_realized_pnl(db_session) + + assert updated == 0 + assert buy.pnl is None + + async def test_full_close_realizes_pnl_on_the_closing_fill(self, db_session): + user = make_user() + db_session.add(user) + await db_session.flush() + t0 = datetime.now(timezone.utc) - timedelta(minutes=10) + buy = make_real_trade(user.id, side="buy", amount="1", price="100", created_at=t0) + sell = make_real_trade(user.id, side="sell", amount="1", price="110", created_at=t0 + timedelta(minutes=5)) + db_session.add_all([buy, sell]) + await db_session.flush() + + updated = await _recompute_realized_pnl(db_session) + + assert updated == 1 + assert buy.pnl is None, "opening fill never realizes PnL on itself" + assert sell.pnl == Decimal("10") + assert sell.pnl_percent == Decimal("10") + + async def test_partial_close_realizes_pnl_only_on_matched_quantity(self, db_session): + user = make_user() + db_session.add(user) + await db_session.flush() + t0 = datetime.now(timezone.utc) - timedelta(minutes=10) + buy = make_real_trade(user.id, amount="2", price="100", side="buy", created_at=t0) + sell = make_real_trade(user.id, amount="1", price="110", side="sell", created_at=t0 + timedelta(minutes=5)) + db_session.add_all([buy, sell]) + await db_session.flush() + + await _recompute_realized_pnl(db_session) + + assert sell.pnl == Decimal("10"), "only the 1 matched unit realizes, not the full 2-unit lot" + assert sell.pnl_percent == Decimal("10") + + async def test_short_side_profits_when_price_falls(self, db_session): + user = make_user() + db_session.add(user) + await db_session.flush() + t0 = datetime.now(timezone.utc) - timedelta(minutes=10) + # sell-first (open short) then buy-to-cover lower -> profit + open_short = make_real_trade(user.id, amount="1", price="100", side="sell", created_at=t0) + cover = make_real_trade(user.id, amount="1", price="90", side="buy", created_at=t0 + timedelta(minutes=5)) + db_session.add_all([open_short, cover]) + await db_session.flush() + + await _recompute_realized_pnl(db_session) + + assert open_short.pnl is None + assert cover.pnl == Decimal("10") + + async def test_trades_without_price_are_skipped_not_matched(self, db_session): + """A market-order fill persisted before the orders.py fix (no + order.average recorded) has price=None — it must not be treated as + a zero-cost lot that corrupts FIFO matching for real, priced fills.""" + user = make_user() + db_session.add(user) + await db_session.flush() + t0 = datetime.now(timezone.utc) - timedelta(minutes=10) + unpriced_buy = make_real_trade(user.id, amount="1", price=None, side="buy", created_at=t0) + sell = make_real_trade(user.id, amount="1", price="110", side="sell", created_at=t0 + timedelta(minutes=5)) + db_session.add_all([unpriced_buy, sell]) + await db_session.flush() + + await _recompute_realized_pnl(db_session) + + assert unpriced_buy.pnl is None + assert sell.pnl is None, "sell opens a new SHORT lot since the unpriced buy couldn't be matched" + + async def test_different_symbols_do_not_cross_match(self, db_session): + user = make_user() + db_session.add(user) + await db_session.flush() + t0 = datetime.now(timezone.utc) - timedelta(minutes=10) + buy_btc = make_real_trade(user.id, symbol="BTC/USDT", amount="1", price="100", side="buy", created_at=t0) + sell_eth = make_real_trade(user.id, symbol="ETH/USDT", amount="1", price="110", side="sell", created_at=t0 + timedelta(minutes=5)) + db_session.add_all([buy_btc, sell_eth]) + await db_session.flush() + + await _recompute_realized_pnl(db_session) + + assert buy_btc.pnl is None + assert sell_eth.pnl is None, "ETH sell must open its own SHORT lot, not close the unrelated BTC buy" + + +class TestSyncRealTrades: + async def test_never_filled_stale_order_is_zeroed_out(self, session_factory, monkeypatch): + import app.services.trade_executor as trade_executor_module + monkeypatch.setattr(trade_executor_module, "async_session_factory", session_factory) + + async with session_factory() as db: + user = make_user() + db.add(user) + await db.flush() + stale = make_real_trade( + user.id, side="buy", amount="1", price=None, filled_amount="0", + status="open", created_at=datetime.now(timezone.utc) - timedelta(hours=25), + ) + db.add(stale) + await db.commit() + stale_id = stale.id + + await sync_real_trades() + + async with session_factory() as db: + refreshed = await db.get(RealTrade, stale_id) + assert refreshed.status == "closed" + assert refreshed.pnl == Decimal("0") + + async def test_stale_order_with_a_real_fill_gets_real_pnl_not_zero(self, session_factory, monkeypatch): + """The bug this fixes: a stale 'open' order that DID partially fill + used to be force-closed with pnl=0 regardless of what actually + happened. If a later trade already closed that fill's position, the + FIFO recompute (run every sync) must report the real PnL instead.""" + import app.services.trade_executor as trade_executor_module + monkeypatch.setattr(trade_executor_module, "async_session_factory", session_factory) + + t0 = datetime.now(timezone.utc) - timedelta(hours=25) + async with session_factory() as db: + user = make_user() + db.add(user) + await db.flush() + stale_buy = make_real_trade( + user.id, side="buy", amount="1", price="100", filled_amount="1", + status="open", created_at=t0, + ) + closing_sell = make_real_trade( + user.id, side="sell", amount="1", price="120", filled_amount="1", + status="filled", created_at=t0 + timedelta(minutes=1), + ) + db.add_all([stale_buy, closing_sell]) + await db.commit() + stale_id, sell_id = stale_buy.id, closing_sell.id + + await sync_real_trades() + + async with session_factory() as db: + stale_refreshed = await db.get(RealTrade, stale_id) + sell_refreshed = await db.get(RealTrade, sell_id) + assert stale_refreshed.status == "closed", "stuck-open order is still force-closed after 24h" + assert stale_refreshed.pnl is None, "opening fill itself never carries the realized PnL" + assert sell_refreshed.pnl == Decimal("20"), "the closing fill must show the real, non-zero PnL" diff --git a/backend/tests/test_walk_forward.py b/backend/tests/test_walk_forward.py index 119d784..845beef 100644 --- a/backend/tests/test_walk_forward.py +++ b/backend/tests/test_walk_forward.py @@ -89,7 +89,7 @@ async def test_grid_search_picks_the_best_scoring_combo(monkeypatch): best one and reports its stats.""" good_params = {"strong_threshold": 4.5, "signal_threshold": 1.5, "max_hold_candles": 96} - def fake_run_combo(candles, scores_series, trade_size, params): + def fake_run_combo(candles, scores_series, trade_size, params, fee_pct=None, slippage_pct=None): if params == good_params: trades = [{"pnl": 10.0, "status": "CLOSED", "entry_price": 100.0, "exit_price": 110.0} for _ in range(10)] else: @@ -113,7 +113,7 @@ async def test_grid_search_picks_the_best_scoring_combo(monkeypatch): @pytest.mark.asyncio async def test_grid_search_falls_back_when_every_combo_too_sparse(monkeypatch): - def fake_run_combo(candles, scores_series, trade_size, params): + def fake_run_combo(candles, scores_series, trade_size, params, fee_pct=None, slippage_pct=None): # 1 trade, below MIN_TRADES_PER_FOLD return [], [{"pnl": 1.0, "status": "CLOSED", "entry_price": 100.0, "exit_price": 101.0}] diff --git a/theo_doi_trading-portal_v10.md b/theo_doi_trading-portal_v10.md new file mode 100644 index 0000000..36430b1 --- /dev/null +++ b/theo_doi_trading-portal_v10.md @@ -0,0 +1,110 @@ +# Theo dõi đánh giá dự án Trading Portal — v10 + +> **Ngày đánh giá gốc:** 2026-07-03 +> **Cập nhật v1-v9:** xem các file `theo_doi_trading-portal_v1.md`…`v9.md` +> **Cập nhật v10 (lần này):** 2026-07-04 — audit toàn diện 13 thuật toán vote + toàn hệ thống risk/execution, sau đó triển khai 3 fix ưu tiên cao nhất: (1) mô phỏng phí giao dịch + slippage trong backtest/walk-forward, (2) tính PnL thật (FIFO) cho real trades thay vì gán cứng 0, (3) xác nhận độ trễ pivot của Divergence/SMC nhất quán giữa live và backtest (đã đúng từ trước, bổ sung test khoá lại) +> **Người thực hiện:** Claude (Sonnet 5), theo yêu cầu của tien.a.le@accenture.com +> **Quy ước đặt tên:** Mỗi lần có thay đổi lớn → tạo bản mới `theo_doi_trading-portal_v11.md`, ... giữ nguyên các bản cũ làm lịch sử. + +--- + +## 1. Tổng quan dự án + +(Không đổi — xem [theo_doi_trading-portal_v2.md](theo_doi_trading-portal_v2.md) mục 1.) + +--- + +## 2. Bối cảnh + +Người dùng yêu cầu rà soát lại 13 thuật toán trong hệ thống vote tín hiệu (`signal_scoring.py`) xem có gì cần cải thiện để nâng cao hiệu quả trade, sau đó rà soát tiếp toàn bộ hệ thống xem cần tinh chỉnh gì khác. Đã dùng 3 agent con đọc trực tiếp toàn bộ `indicator_service.py` (1499 dòng), `risk_manager.py`, `trade_executor.py`, `walk_forward.py`, `backtest_engine.py` để đưa ra đánh giá dựa trên code thực tế, không suy đoán. Người dùng chọn 3 vấn đề ưu tiên cao nhất để sửa ngay; các vấn đề còn lại (bên dưới, mục 4) để dành cho vòng tiếp theo. + +--- + +## 3. Các thay đổi trong v10 + +### 3.1 Mô phỏng phí giao dịch + slippage trong backtest/walk-forward + +**Vấn đề:** `backtest_engine.py`/`walk_forward.py` trước đây định giá mọi lệnh khớp đúng bằng giá đóng nến, không tính phí (taker fee) hay trượt giá (slippage) — khiến mọi con số win rate/profit factor/Sharpe trong báo cáo backtest và walk-forward đều lạc quan giả tạo so với live trading thực tế. + +**Đã sửa** (`backend/app/services/backtest_engine.py`): +- Thêm `DEFAULT_TAKER_FEE_PCT = 0.001` (0.1%/lượt khớp, tương đương phí taker phổ biến trên Binance/MEXC không có ưu đãi VIP/token) và `DEFAULT_SLIPPAGE_PCT = 0.0005` (0.05%/lượt khớp, ước lượng thận trọng cho cặp thanh khoản tốt). +- Hàm `_fill_price()` mới: mô phỏng giá khớp lệnh market thực tế — trượt giá luôn bất lợi cho trader (mua thì khớp giá cao hơn, bán thì khớp giá thấp hơn giá quan sát). +- Hàm `_open_position()`/`_close_position()` mới: tính `entry_price`/`exit_price` đã trượt giá, `quantity` tính theo giá đã trượt (để `entry_price × quantity == trade_size`, đúng bản chất lệnh market chi 1 số tiền cố định), và trừ phí round-trip (`entry_fee + exit_fee`) khỏi PnL. `pnl` trong mỗi trade giờ là **net** (đã trừ phí); có thêm field `gross_pnl` và `fees` để minh bạch. +- `_simulate_from_scores()` refactor để dùng 2 hàm trên cho cả 4 kiểu thoát lệnh (REVERSAL × 2, TIME_LIMIT, END_OF_DATA) thay vì tính PnL thô lặp lại 4 lần. +- `_compute_stats()` báo cáo thêm `total_fees` trong phần `trades`. +- `run_backtest()`, `_simulate_trades()` nhận thêm tham số `fee_pct`/`slippage_pct` (mặc định = 2 hằng số trên); truyền `fee_pct=0, slippage_pct=0` để tái tạo hành vi cũ (miễn phí) nếu cần so sánh. +- `walk_forward.py`: `_run_combo()`, `_grid_search()`, `run_walk_forward()` đều nhận và truyền `fee_pct`/`slippage_pct` — **áp dụng ngay trong grid search**, không chỉ ở báo cáo OOS cuối cùng, để việc chọn tham số tối ưu không thiên vị các combo tần suất giao dịch cao chỉ "đẹp" khi lệnh miễn phí. Kết quả OOS summary và từng fold đều có thêm `total_fees`. +- API: `GET/POST /backtest/run` và `POST /walk-forward/run` có thêm query param tùy chọn `fee_pct`/`slippage_pct` để người dùng nâng cao tự điều chỉnh giả định chi phí. + +**Test mới** (`tests/test_backtest_engine.py`, class `TestFeeAndSlippage`): xác nhận `_fill_price` trượt đúng hướng bất lợi cho cả 4 tổ hợp (LONG/SHORT × entry/exit); một lệnh REVERSAL thắng về giá vẫn bị trừ đúng phí+slippage tính toán được; `fee_pct=0, slippage_pct=0` tái tạo chính xác hành vi cũ (không phí, khớp đúng giá đóng nến); `_compute_stats` báo cáo đúng tổng phí. + +### 3.2 PnL thật cho Real Trade (FIFO) thay vì gán cứng 0 + +**Vấn đề:** `RealTrade` (bảng `real_trades`) lưu **từng lượt khớp lệnh** (1 buy hoặc 1 sell), không phải cặp mở/đóng vị thế như `HypotheticalTrade` — nên không thể đọc PnL trực tiếp từ 1 dòng. `sync_real_trades()` trước đây gán cứng `pnl = 0` cho mọi lệnh cần tính PnL (cả lệnh "open" quá hạn 24h lẫn lệnh "closed"/"filled" thiếu PnL), khiến báo cáo PnL cho tiền thật luôn sai (hiển thị "hòa vốn" bất kể thực tế lời/lỗ). + +**Nguyên nhân gốc phát hiện thêm:** với lệnh market (đa số lệnh đặt qua OrderPanel), `req.price` gửi lên luôn là `None` (không đặt giá limit), nên trước đây `RealTrade.price` cũng luôn `None` — hoàn toàn không có dữ liệu giá để tính PnL. Sàn trả về giá khớp thật qua `order.average`/`order.price`. + +**Đã sửa:** +- `backend/app/api/v1/orders.py`: lưu `order.average or order.price or req.price` thay vì chỉ `req.price` — lệnh market giờ có giá khớp thật để dùng cho việc tính PnL sau này. +- `backend/app/services/trade_executor.py`: thêm hàm `_recompute_realized_pnl(db)` — ghép lệnh theo kiểu FIFO (First-In-First-Out) trên từng nhóm `(user_id, symbol, exchange)`, theo thứ tự thời gian: mỗi lệnh buy khớp với các lô SHORT đang mở trước (đóng short), phần dư mở lô LONG mới; mỗi lệnh sell khớp với các lô LONG đang mở trước (đóng long), phần dư mở lô SHORT mới. PnL thực (`realized`) được gán vào đúng dòng lệnh "đóng" — lệnh chỉ "mở" (chưa có gì để đóng) giữ `pnl = None` cho đến khi có lệnh sau đóng nó. Bỏ qua an toàn các dòng thiếu `price` (dữ liệu cũ trước khi có fix trên) để không làm sai lệch matching FIFO của các dòng có giá hợp lệ. +- `sync_real_trades()` viết lại: bước 1 chỉ đổi **status** (đóng) cho lệnh "open" quá hạn 24h, chỉ gán `pnl=0` khi lệnh thực sự **chưa khớp gì** (`filled_amount == 0`) — lệnh có khớp một phần để `_recompute_realized_pnl` xử lý; bước 2 chạy `_recompute_realized_pnl` cho **mọi** lệnh có khớp (không chỉ lệnh đang thiếu PnL) vì một lệnh đóng mới có thể "hồi tố" PnL cho lệnh mở trước đó. + +**Test mới** (`tests/test_trade_executor.py`, class `TestRecomputeRealizedPnl` + `TestSyncRealTrades`): lệnh mở đơn lẻ không có PnL; buy rồi sell full khớp đúng PnL; đóng một phần khớp đúng theo số lượng khớp; mở SHORT trước rồi cover lãi khi giá giảm; lệnh thiếu giá bị bỏ qua khỏi FIFO (không làm hỏng khớp lệnh khác); symbol khác nhau không bị khớp chéo; lệnh "open" quá hạn chưa khớp gì → PnL=0 đúng; lệnh "open" quá hạn NHƯNG đã khớp và có lệnh đóng sau đó → PnL thật (không phải 0). + +### 3.3 Xác nhận độ trễ pivot của Divergence/SMC — đã đúng từ trước, bổ sung test khoá lại + +**Bối cảnh:** audit nghi ngờ `_find_pivot_highs`/`_find_pivot_lows` (dùng cho Divergence và SMC/BOS/CHoCH) có thể gây "repaint bias" — một pivot tại nến `i` chỉ được xác nhận sau khi biết `right` (3-5) nến tương lai, nên nếu live trading dùng trực tiếp mà không bù độ trễ, backtest có thể đánh giá tín hiệu sớm hơn thực tế có thể trade được. + +**Kết quả điều tra:** **không có bug.** Cả live (`candle_service.get_indicators` → `detect_divergence`/`market_structure`) lẫn backtest (`backtest_engine._compute_scores_series`, đã sửa ở v8/task #52) đều **tự động nhất quán**: hàm quét pivot (`_find_pivot_highs`/`_find_pivot_lows`/`_find_pivot_highs_levels`/`_find_pivot_lows_levels`) chỉ gán giá trị pivot cho vị trí `i` nếu mảng có đủ `right` phần tử **sau** `i` — khi gọi trên dữ liệu "tính đến hiện tại" (không có nến tương lai), `right` nến cuối cùng luôn tự động không thể là pivot (tự giới hạn theo đúng định nghĩa chỉ báo), giống hệt cơ chế `confirmed_swing_highs`/`confirmed_div_high_idx` mà backtest đã cài tường minh. Do đó live và backtest cùng chịu độ trễ như nhau, không có sai lệch. + +**Đã làm:** thêm class `TestPivotCausalConsistency` vào `tests/test_indicator_service.py` — khoá lại 2 tính chất: (1) `right` phần tử cuối của `_find_pivot_highs`/`_find_pivot_lows` luôn là `None`; (2) trạng thái pivot của một nến, một khi đã đủ dữ liệu xác nhận, **không đổi** khi có thêm nến tương lai xuất hiện (không repaint) — test cho cả hàm pivot thô lẫn `market_structure()` (dùng trực tiếp bởi live trading). Đây là lưới an toàn cho các lần refactor sau này không vô tình phá vỡ tính nhất quán live-vs-backtest. + +### 3.4 Kiểm chứng + +Toàn bộ 187 test backend pass (tăng từ 170 ở v9 lên 187 — thêm 4 test pivot causal-consistency, 7 test fee/slippage/fill-price, 8 test FIFO PnL, cộng điều chỉnh 2 test cũ bị ảnh hưởng gián tiếp bởi thay đổi chữ ký `_run_combo` và interface `order.average` trong stub test). Không đổi gì ở frontend trong đợt này. + +--- + +## 4. Danh sách vấn đề còn tồn đọng — ưu tiên cho vòng tiếp theo + +Từ audit toàn diện 13 thuật toán + risk/execution, các vấn đề dưới đây **chưa được sửa trong v10** (người dùng chọn 3 việc ở mục 3 trước), xếp theo mức độ ưu tiên ước lượng: + +### Ưu tiên CAO +| # | Vấn đề | Vị trí | +|---|---|---| +| hh | Kelly sizing không tính correlation giữa các vị thế mở đồng thời (tối đa 10 lệnh, nhiều coin crypto tương quan cao với BTC) | `risk_manager.py` `DynamicKellySizer`, `trade_executor.py:32` | +| ii | Backtest chỉ mô phỏng 1 vị thế/symbol — chưa từng kiểm chứng eviction đa vị thế (MAX_OPEN_TRADES=10) hay SL/TP theo ATR (`AdaptiveSLTPOptimizer`) bằng dữ liệu lịch sử | `backtest_engine.py` vs `trade_executor.py`, `risk_manager.py` | +| jj | Đa cộng tuyến RSI–StochRSI–MFI chưa xử lý đúng mức (StochRSI = hàm trực tiếp của RSI, tương quan thường >0.8) — dampening `1/sqrt(n)` hiện đối xử đồng đều bất kể mức tương quan thực | `signal_scoring.py:455-460`, `indicator_service.py` | +| kk | Ngưỡng `volatile: atr_pct > 5.0` cố định, không thích ứng theo từng coin/timeframe | `indicator_service.py:1186` | +| ll | Race condition tiềm ẩn trong eviction logic khi nhiều tín hiệu STRONG đến đồng thời cho cùng user | `trade_executor.py:110` | + +### Ưu tiên TRUNG BÌNH +| # | Vấn đề | Vị trí | +|---|---|---| +| mm | Kelly sizing dùng win-rate tổng hợp `__all__` thay vì theo symbol/strategy cụ thể | `trade_executor.py:272-273` | +| nn | `compute_partial_tp_levels` thiếu 40% khối lượng "trailing" như docstring hứa hẹn (chỉ trả về 60%) | `risk_manager.py:169-203` | +| oo | `AdaptiveSLTPOptimizer`/`compute_volatility_adjusted_size` có vẻ là code mồ côi, chưa wire vào `trade_executor.py` | `risk_manager.py` | +| pp | Ichimoku dùng nguyên bộ 9-26-52-26 của lịch giao dịch Nhật cổ, không có cơ sở cho crypto 24/7 | `indicator_service.py:583-599` | +| qq | FVG thiếu bộ lọc kích thước gap theo ATR — sinh nhiều tín hiệu nhiễu ở khung 15m | `indicator_service.py:1334-1377` | +| rr | `MIN_TRADES_PER_FOLD=5` hơi thấp cho ý nghĩa thống kê Sharpe (khuyến nghị quant thường ≥30) | `walk_forward.py:48` | +| ss | Volatility filter và Kelly sizing bọc trong `except: pass` quá rộng, lỗi bị nuốt âm thầm trong production | `trade_executor.py:183-184, 286-287` | + +### Ưu tiên THẤP +| # | Vấn đề | Vị trí | +|---|---|---| +| tt | Toàn bộ tham số kinh điển của 13 thuật toán (RSI-14, MACD 12-26-9, BB-20, SuperTrend 10/3.0...) là giá trị sách giáo khoa cho cổ phiếu/forex daily, chưa qua walk-forward riêng cho crypto | `indicator_service.py` (toàn bộ) | +| uu | BOS buffer 0.3% cố định nên đổi theo ATR thay vì % tuyệt đối | `indicator_service.py:760,762` | +| vv | Order Block detection quá đơn giản (thiếu điều kiện volume/imbalance) | `indicator_service.py:805-851` | +| ww | Ngưỡng candlestick pattern (Doji/Marubozu/Hammer...) chưa kiểm định theo winrate thực tế trên crypto | `indicator_service.py:1432-1497` | +| xx | Fallback grid search khi mọi combo đều `-inf` chọn tùy ý phần tử đầu grid thay vì combo "ít tệ nhất" | `walk_forward.py:178-183` | + +--- + +## 5. Lịch sử phiên bản + +| Phiên bản | Ngày | Thay đổi | +|---|---|---| +| v0-v7 | 2026-07-03 — 2026-07-04 | Xem file tương ứng | +| v8 | 2026-07-04 | Wire ADX/regime vào bộ lọc tín hiệu dùng chung, hợp nhất 2 hệ thống regime. 170 test pass. | +| v9 | 2026-07-04 | Sửa bug CSS toàn cục: reset viết tay không nằm trong `@layer` vô hiệu hóa toàn bộ margin/padding utility Tailwind app-wide (gg). Kèm 2 bug nhỏ: inline style đè max-width trên Login/Register (ee), BacktestPage thiếu wrapper mất canh giữa (ff). 170 test pass. | +| v10 | 2026-07-04 | Audit 13 thuật toán + toàn hệ thống risk/execution. Triển khai: mô phỏng phí+slippage trong backtest/WFO (fee_pct/slippage_pct, mặc định 0.1%/0.05% mỗi lượt khớp), FIFO PnL thật cho real trades (thay vì gán cứng 0) + fix orders.py lưu giá khớp thật cho lệnh market, xác nhận độ trễ pivot Divergence/SMC đã đúng từ trước (task #52) + test khoá lại. 187 test pass (+17). Danh sách 15 vấn đề còn tồn đọng (hh-xx) để dành vòng sau. \ No newline at end of file