Add walk-forward backtest optimization to mitigate signal overfitting (item m)

Rolling train/test folds over 3 years of data auto-optimize the three
cheap-to-tune trading parameters (STRONG/BUY score thresholds, max hold
time) via grid search on each fold's train window, then evaluate purely
on the held-out test window. Stitching all out-of-sample results gives
an honest performance estimate uninflated by tuning against the same
data used to score it.

Split signal_scoring.py's expensive 13-algorithm scoring from its cheap
final threshold classification so grid search can replay many parameter
combinations without recomputing indicators each time. Moved the
backtest engine (fetch/precompute/simulate) out of the API layer into
app/services/backtest_engine.py so both /backtest/run and the new
walk-forward optimizer share one implementation instead of drifting
copies — same rationale as the earlier signal_service.py split (item h).

Also merges two long-diverged Alembic migration heads discovered while
adding the walk_forward_results table, so `alembic upgrade head` has a
single target again.

New: POST/GET/DELETE /walk-forward/* endpoints, a Walk-Forward tab on
the Backtest page (fold table, out-of-sample equity curve, run history).
19 new backend tests (153 total, all passing).

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>

frontend/src/features/backtest/BacktestPage.tsx

@@ -1,6 +1,6 @@
 import { useState, useCallback, useEffect } from 'react';
 import { useT } from '../../translations';
-import { apiFetch } from '../api/apiService';
+import { apiFetch, ApiServiceError } from '../api/apiService';
 
 interface SignalCounts {
   [key: string]: number;
@@ -50,6 +50,65 @@ interface BacktestResult {
   };
 }
 
+// ── Walk-forward types ──
+interface WfBestParams {
+  strong_threshold: number;
+  signal_threshold: number;
+  max_hold_candles: number;
+}
+interface WfFoldMetrics {
+  trades: number;
+  win_rate: number;
+  total_pnl: number;
+  profit_factor: number | null;
+}
+interface WfFoldResult {
+  fold_index: number;
+  train_start: string;
+  train_end: string;
+  test_start: string;
+  test_end: string;
+  best_params: WfBestParams;
+  in_sample: WfFoldMetrics;
+  out_of_sample: WfFoldMetrics;
+}
+interface WalkForwardResult {
+  id?: string;
+  symbol: string;
+  exchange: string;
+  timeframe: string;
+  total_days: number;
+  train_days: number;
+  test_days: number;
+  folds: WfFoldResult[];
+  out_of_sample_summary: {
+    trades: number;
+    wins: number;
+    losses: number;
+    win_rate: number;
+    total_pnl: number;
+    profit_factor: number | null;
+    max_drawdown_pct: number;
+    equity_curve: number[];
+  };
+}
+interface WfHistoryItem {
+  id: string;
+  symbol: string;
+  exchange: string;
+  timeframe: string;
+  total_days: number;
+  train_days: number;
+  test_days: number;
+  folds_count: number;
+  oos_trades: number;
+  oos_win_rate: number | null;
+  oos_total_pnl: number | null;
+  oos_profit_factor: number | null;
+  oos_max_drawdown_pct: number | null;
+  created_at: string;
+}
+
 const SIGNAL_ICONS: Record<string, string> = {
   'STRONG_BUY': '🚀', 'BUY': '📈',
   'STRONG_SELL': '🔻', 'SELL': '📉',
@@ -59,25 +118,23 @@ const SIGNAL_ICONS: Record<string, string> = {
 
 const EXCHANGES = ['binance', 'bybit', 'mexc', 'gate', 'bingx'];
 const TIMEFRAMES = ['15m', '30m', '1h', '4h'];
+const WF_TIMEFRAMES = ['1h', '4h'];
 
 const selectClass = 'rounded-md border border-border-default bg-bg-surface px-2.5 py-1.5 text-sm text-text-primary';
 const thClass = 'px-3 py-2 text-left font-medium text-text-secondary';
 const tdClass = 'px-3 py-1.5';
 const cardClass = 'rounded-lg bg-bg-surface p-4';
+const modeBtnClass = 'min-h-9 touch-manipulation rounded-md border px-4 py-1.5 text-sm';
 
 export default function BacktestPage() {
   const { t } = useT();
+  const [mode, setMode] = useState<'single' | 'walk-forward'>('single');
   const [exchange, setExchange] = useState('binance');
   const [symbol, setSymbol] = useState('BTC/USDT');
   const [symbols, setSymbols] = useState<string[]>([]);
-  const [timeframe, setTimeframe] = useState('30m');
-  const [days, setDays] = useState(7);
   const [tradeSize, setTradeSize] = useState(10);
-  const [result, setResult] = useState<BacktestResult | null>(null);
-  const [loading, setLoading] = useState(false);
-  const [error, setError] = useState('');
 
-  // Load symbols when exchange changes
+  // Load symbols when exchange changes — shared between both modes
   useEffect(() => {
     let cancelled = false;
     async function loadSymbols() {
@@ -86,7 +143,6 @@ export default function BacktestPage() {
         if (cancelled) return;
         const names = (data.symbols || []).map((s: any) => s.symbol);
         setSymbols(names);
-        // Keep current symbol if in list, else pick first
         if (names.length > 0 && !names.includes(symbol)) {
           setSymbol(names[0]);
         }
@@ -96,6 +152,43 @@ export default function BacktestPage() {
     return () => { cancelled = true; };
   }, [exchange]);
 
+  return (
+    <div className="mx-auto max-w-[1000px] bg-bg-primary p-5 text-text-primary">
+      <h1 className="text-2xl font-bold text-text-heading">📊 {t('Backtest')}</h1>
+
+      <div className="mb-5 mt-3 flex gap-1">
+        <button onClick={() => setMode('single')} className={`${modeBtnClass} ${mode === 'single' ? 'border-accent-blue bg-accent-blue font-bold text-white' : 'border-border-default bg-bg-hover text-text-secondary'}`}>
+          ▶ Single Run
+        </button>
+        <button onClick={() => setMode('walk-forward')} className={`${modeBtnClass} ${mode === 'walk-forward' ? 'border-accent-blue bg-accent-blue font-bold text-white' : 'border-border-default bg-bg-hover text-text-secondary'}`}>
+          🧪 Walk-Forward
+        </button>
+      </div>
+
+      {mode === 'single' ? (
+        <SingleRunView exchange={exchange} setExchange={setExchange} symbol={symbol} setSymbol={setSymbol}
+          symbols={symbols} tradeSize={tradeSize} setTradeSize={setTradeSize} />
+      ) : (
+        <WalkForwardView exchange={exchange} setExchange={setExchange} symbol={symbol} setSymbol={setSymbol}
+          symbols={symbols} tradeSize={tradeSize} setTradeSize={setTradeSize} />
+      )}
+    </div>
+  );
+}
+
+// ═══════════════ SINGLE RUN ═══════════════
+function SingleRunView({ exchange, setExchange, symbol, setSymbol, symbols, tradeSize, setTradeSize }: {
+  exchange: string; setExchange: (v: string) => void;
+  symbol: string; setSymbol: (v: string) => void;
+  symbols: string[];
+  tradeSize: number; setTradeSize: (v: number) => void;
+}) {
+  const [timeframe, setTimeframe] = useState('30m');
+  const [days, setDays] = useState(7);
+  const [result, setResult] = useState<BacktestResult | null>(null);
+  const [loading, setLoading] = useState(false);
+  const [error, setError] = useState('');
+
   const runBacktest = useCallback(async () => {
     setLoading(true);
     setError('');
@@ -110,12 +203,10 @@ export default function BacktestPage() {
     } finally {
       setLoading(false);
     }
-  }, [symbol, timeframe, days, tradeSize]);
+  }, [symbol, exchange, timeframe, days, tradeSize]);
 
   return (
-    <div className="mx-auto max-w-[1000px] bg-bg-primary p-5 text-text-primary">
-      <h1 className="text-2xl font-bold text-text-heading">📊 {t('Backtest')}</h1>
-
+    <div>
       {/* Controls */}
       <div className="mb-5 flex flex-wrap items-end gap-3">
         <div>
@@ -135,7 +226,7 @@ export default function BacktestPage() {
         <div>
           <label className="text-xs text-text-secondary">Timeframe</label><br />
           <select value={timeframe} onChange={e => setTimeframe(e.target.value)} className={selectClass}>
-            {TIMEFRAMES.map(t => <option key={t} value={t}>{t}</option>)}
+            {TIMEFRAMES.map(tf => <option key={tf} value={tf}>{tf}</option>)}
           </select>
         </div>
         <div>
@@ -289,6 +380,266 @@ export default function BacktestPage() {
   );
 }
 
+// ═══════════════ WALK-FORWARD ═══════════════
+function EquityCurveSvg({ points }: { points: number[] }) {
+  if (points.length < 2) return <div className="text-xs text-text-dim">Not enough out-of-sample trades to chart.</div>;
+  const w = 600, h = 140, pad = 6;
+  const min = Math.min(...points), max = Math.max(...points);
+  const range = max - min || 1;
+  const stepX = (w - pad * 2) / (points.length - 1);
+  const toY = (v: number) => h - pad - ((v - min) / range) * (h - pad * 2);
+  const path = points.map((v, i) => `${i === 0 ? 'M' : 'L'} ${pad + i * stepX} ${toY(v)}`).join(' ');
+  const zeroY = toY(0);
+  const isPositive = points[points.length - 1] >= 0;
+  return (
+    <svg viewBox={`0 0 ${w} ${h}`} className="w-full" style={{ height: 140 }}>
+      <line x1={pad} y1={zeroY} x2={w - pad} y2={zeroY} stroke="var(--color-border-muted)" strokeDasharray="4 3" />
+      <path d={path} fill="none" stroke={isPositive ? 'var(--color-green)' : 'var(--color-red)'} strokeWidth={2} />
+    </svg>
+  );
+}
+
+function WalkForwardView({ exchange, setExchange, symbol, setSymbol, symbols, tradeSize, setTradeSize }: {
+  exchange: string; setExchange: (v: string) => void;
+  symbol: string; setSymbol: (v: string) => void;
+  symbols: string[];
+  tradeSize: number; setTradeSize: (v: number) => void;
+}) {
+  const [timeframe, setTimeframe] = useState('4h');
+  const [totalDays, setTotalDays] = useState(1095);
+  const [trainDays, setTrainDays] = useState(270);
+  const [testDays, setTestDays] = useState(90);
+  const [showAdvanced, setShowAdvanced] = useState(false);
+  const [result, setResult] = useState<WalkForwardResult | null>(null);
+  const [loading, setLoading] = useState(false);
+  const [error, setError] = useState('');
+  const [needsLogin, setNeedsLogin] = useState(false);
+  const [history, setHistory] = useState<WfHistoryItem[]>([]);
+  const [historyLoading, setHistoryLoading] = useState(true);
+
+  const loadHistory = useCallback(async () => {
+    setHistoryLoading(true);
+    try {
+      const data = await apiFetch<WfHistoryItem[]>('/walk-forward/history?limit=20');
+      setHistory(data);
+      setNeedsLogin(false);
+    } catch (e: any) {
+      if (e instanceof ApiServiceError && e.status === 401) setNeedsLogin(true);
+    } finally {
+      setHistoryLoading(false);
+    }
+  }, []);
+  useEffect(() => { loadHistory(); }, [loadHistory]);
+
+  const runWalkForward = useCallback(async () => {
+    setLoading(true);
+    setError('');
+    setResult(null);
+    try {
+      const data = await apiFetch<WalkForwardResult>(
+        `/walk-forward/run?symbol=${encodeURIComponent(symbol)}&exchange=${exchange}&timeframe=${timeframe}` +
+        `&total_days=${totalDays}&train_days=${trainDays}&test_days=${testDays}&trade_size=${tradeSize}`,
+        { method: 'POST' },
+      );
+      setResult(data);
+      setNeedsLogin(false);
+      loadHistory();
+    } catch (e: any) {
+      if (e instanceof ApiServiceError && e.status === 401) {
+        setNeedsLogin(true);
+      } else {
+        setError(e.message);
+      }
+    } finally {
+      setLoading(false);
+    }
+  }, [symbol, exchange, timeframe, totalDays, trainDays, testDays, tradeSize, loadHistory]);
+
+  const loadPastRun = useCallback(async (id: string) => {
+    setLoading(true);
+    setError('');
+    try {
+      const data = await apiFetch<WalkForwardResult>(`/walk-forward/${id}`);
+      setResult(data);
+    } catch (e: any) {
+      setError(e.message);
+    } finally {
+      setLoading(false);
+    }
+  }, []);
+
+  const deletePastRun = useCallback(async (id: string) => {
+    try {
+      await apiFetch(`/walk-forward/${id}`, { method: 'DELETE' });
+      loadHistory();
+    } catch {}
+  }, [loadHistory]);
+
+  return (
+    <div>
+      <div className="mb-4 rounded-lg border border-accent-blue/25 bg-accent-blue/10 p-3 text-xs text-text-secondary">
+        💡 Tối ưu tự động ngưỡng tín hiệu &amp; thời gian giữ lệnh trên từng cửa sổ dữ liệu quá khứ (train), rồi kiểm định trên dữ liệu chưa từng thấy (test) — tránh overfitting so với chạy 1 lần trên toàn bộ lịch sử.
+      </div>
+
+      {/* Controls */}
+      <div className="mb-3 flex flex-wrap items-end gap-3">
+        <div>
+          <label className="text-xs text-text-secondary">Exchange</label><br />
+          <select value={exchange} onChange={e => setExchange(e.target.value)} className={selectClass}>
+            {EXCHANGES.map(ex => <option key={ex} value={ex}>{ex.toUpperCase()}</option>)}
+          </select>
+        </div>
+        <div>
+          <label className="text-xs text-text-secondary">Symbol ({symbols.length})</label><br />
+          <select value={symbol} onChange={e => setSymbol(e.target.value)} className={`${selectClass} min-w-[140px]`}>
+            {symbols.length > 0
+              ? symbols.map(s => <option key={s} value={s}>{s}</option>)
+              : <option value="">Loading...</option>}
+          </select>
+        </div>
+        <div>
+          <label className="text-xs text-text-secondary">Timeframe</label><br />
+          <select value={timeframe} onChange={e => setTimeframe(e.target.value)} className={selectClass}>
+            {WF_TIMEFRAMES.map(tf => <option key={tf} value={tf}>{tf}</option>)}
+          </select>
+        </div>
+        <div>
+          <label className="text-xs text-text-secondary">Trade Size (USDT)</label><br />
+          <input type="number" value={tradeSize} onChange={e => setTradeSize(Number(e.target.value))} className={`${selectClass} w-20`} min={1} />
+        </div>
+        <button
+          onClick={runWalkForward}
+          disabled={loading}
+          className={`rounded-md border-none bg-accent-blue px-6 py-2 font-semibold text-white ${loading ? 'cursor-wait' : 'cursor-pointer'}`}
+        >
+          {loading ? '⏳ Đang chạy...' : '🧪 Run Walk-Forward'}
+        </button>
+      </div>
+
+      <button onClick={() => setShowAdvanced(!showAdvanced)} className="mb-3 text-xs text-accent underline">
+        {showAdvanced ? '▾ Ẩn tùy chọn nâng cao' : '▸ Tùy chọn nâng cao (window size)'}
+      </button>
+      {showAdvanced && (
+        <div className={`${cardClass} mb-4 flex flex-wrap gap-4`}>
+          <div>
+            <label className="text-xs text-text-secondary">Tổng dữ liệu (ngày)</label><br />
+            <input type="number" value={totalDays} onChange={e => setTotalDays(Number(e.target.value))} className={`${selectClass} w-24`} min={180} />
+          </div>
+          <div>
+            <label className="text-xs text-text-secondary">Train window (ngày)</label><br />
+            <input type="number" value={trainDays} onChange={e => setTrainDays(Number(e.target.value))} className={`${selectClass} w-24`} min={30} />
+          </div>
+          <div>
+            <label className="text-xs text-text-secondary">Test window (ngày)</label><br />
+            <input type="number" value={testDays} onChange={e => setTestDays(Number(e.target.value))} className={`${selectClass} w-24`} min={14} />
+          </div>
+        </div>
+      )}
+
+      {needsLogin && <div className="mb-4 rounded-md border border-yellow/25 bg-yellow/10 p-3 text-sm text-yellow">🔒 Đăng nhập để chạy và lưu Walk-Forward Analysis.</div>}
+      {error && <div className="mb-4 text-red">❌ {error}</div>}
+
+      {result && (
+        <>
+          <div className="mb-5 grid grid-cols-[repeat(auto-fit,minmax(140px,1fr))] gap-3">
+            <SummaryCard label="OOS Trades" value={result.out_of_sample_summary.trades.toString()} />
+            <SummaryCard label="OOS Win Rate" value={`${result.out_of_sample_summary.win_rate}%`}
+              color={result.out_of_sample_summary.win_rate >= 50 ? 'text-green' : 'text-red'} />
+            <SummaryCard label="OOS Total PnL" value={`$${result.out_of_sample_summary.total_pnl.toFixed(2)}`}
+              color={result.out_of_sample_summary.total_pnl >= 0 ? 'text-green' : 'text-red'} />
+            <SummaryCard label="OOS Profit Factor" value={result.out_of_sample_summary.profit_factor?.toFixed(2) ?? '∞'}
+              color={result.out_of_sample_summary.profit_factor && result.out_of_sample_summary.profit_factor >= 1 ? 'text-green' : 'text-red'} />
+            <SummaryCard label="Max Drawdown" value={`${result.out_of_sample_summary.max_drawdown_pct}%`} color="text-red" />
+            <SummaryCard label="Folds" value={result.folds.length.toString()} />
+          </div>
+
+          <div className={`${cardClass} mb-4`}>
+            <h3 className="mb-3 text-text-heading">📈 Out-of-Sample Equity Curve (đã ghép các fold)</h3>
+            <EquityCurveSvg points={result.out_of_sample_summary.equity_curve} />
+          </div>
+
+          <div className={cardClass}>
+            <h3 className="mb-1 text-text-heading">📋 Chi tiết từng Fold</h3>
+            <p className="mb-3 text-xs text-text-dim">So sánh In-Sample (train, đã tối ưu) với Out-of-Sample (test, chưa từng thấy) — chênh lệch càng lớn thì càng có dấu hiệu overfitting.</p>
+            <div className="overflow-x-auto">
+              <table className="w-full min-w-[720px] border-collapse text-[12px]">
+                <thead>
+                  <tr className="border-b border-border-default">
+                    <th className={thClass}>Fold</th>
+                    <th className={thClass}>Train</th>
+                    <th className={thClass}>Test</th>
+                    <th className={thClass}>Best Params</th>
+                    <th className={thClass}>In-Sample WR / PnL</th>
+                    <th className={thClass}>Out-of-Sample WR / PnL</th>
+                  </tr>
+                </thead>
+                <tbody>
+                  {result.folds.map(f => (
+                    <tr key={f.fold_index} className="border-b border-border-muted">
+                      <td className={tdClass}>#{f.fold_index + 1}</td>
+                      <td className={tdClass}>{f.train_start.slice(0, 10)} → {f.train_end.slice(0, 10)}</td>
+                      <td className={tdClass}>{f.test_start.slice(0, 10)} → {f.test_end.slice(0, 10)}</td>
+                      <td className={`${tdClass} text-text-dim`}>
+                        S≥{f.best_params.strong_threshold} / B≥{f.best_params.signal_threshold} / {f.best_params.max_hold_candles}c
+                      </td>
+                      <td className={tdClass}>{f.in_sample.win_rate}% / <span className={f.in_sample.total_pnl >= 0 ? 'text-green' : 'text-red'}>${f.in_sample.total_pnl.toFixed(2)}</span></td>
+                      <td className={tdClass}>{f.out_of_sample.win_rate}% / <span className={f.out_of_sample.total_pnl >= 0 ? 'text-green' : 'text-red'}>${f.out_of_sample.total_pnl.toFixed(2)}</span></td>
+                    </tr>
+                  ))}
+                </tbody>
+              </table>
+            </div>
+          </div>
+        </>
+      )}
+
+      {/* History */}
+      {!needsLogin && (
+        <div className={`${cardClass} mt-4`}>
+          <h3 className="mb-3 text-text-heading">🕓 Lịch sử Walk-Forward</h3>
+          {historyLoading ? (
+            <div className="text-xs text-text-dim">Đang tải...</div>
+          ) : history.length === 0 ? (
+            <div className="text-xs text-text-dim">Chưa có lần chạy nào được lưu.</div>
+          ) : (
+            <div className="overflow-x-auto">
+              <table className="w-full min-w-[600px] border-collapse text-[12px]">
+                <thead>
+                  <tr className="border-b border-border-default">
+                    <th className={thClass}>Symbol</th>
+                    <th className={thClass}>TF</th>
+                    <th className={thClass}>Folds</th>
+                    <th className={thClass}>OOS Win Rate</th>
+                    <th className={thClass}>OOS PnL</th>
+                    <th className={thClass}>Ngày</th>
+                    <th className={thClass}></th>
+                  </tr>
+                </thead>
+                <tbody>
+                  {history.map(h => (
+                    <tr key={h.id} className="border-b border-border-muted">
+                      <td className={tdClass}>{h.symbol} @ {h.exchange}</td>
+                      <td className={tdClass}>{h.timeframe}</td>
+                      <td className={tdClass}>{h.folds_count}</td>
+                      <td className={tdClass}>{h.oos_win_rate ?? '-'}%</td>
+                      <td className={`${tdClass} ${(h.oos_total_pnl ?? 0) >= 0 ? 'text-green' : 'text-red'}`}>${(h.oos_total_pnl ?? 0).toFixed(2)}</td>
+                      <td className={`${tdClass} text-text-dim`}>{h.created_at.slice(0, 10)}</td>
+                      <td className={tdClass}>
+                        <button onClick={() => loadPastRun(h.id)} className="mr-2 text-accent underline">Xem</button>
+                        <button onClick={() => deletePastRun(h.id)} className="text-red underline">Xóa</button>
+                      </td>
+                    </tr>
+                  ))}
+                </tbody>
+              </table>
+            </div>
+          )}
+        </div>
+      )}
+    </div>
+  );
+}
+
 function SummaryCard({ label, value, color }: { label: string; value: string; color?: string }) {
   return (
     <div className="rounded-lg border border-border-default bg-bg-surface p-3.5">