Fix O(n^2) blowup and look-ahead leak in SMC/divergence backtest scoring

market_structure() (SMC) and detect_divergence() were each precomputed
once over the ENTIRE multi-year backtest range and reused unchanged for
every candle, so every candle's score could see results derived from
years of future price data — a look-ahead bug that inflated both
single-run backtest and walk-forward results, undermining the very
overfitting check walk-forward exists to provide. A prior fix bounded
this to a per-candle trailing window, which closed most of the leak but
still rescanned pivots from scratch on every candle (O(window) per
candle), too slow to enable 15m/30m walk-forward runs.

The real fix: pivot detection is itself a bounded rolling-window scan
(each position only depends on a few bars on either side), so it can be
precomputed once for the whole dataset just like BB/RSI/MACD. Per candle,
_compute_scores_series now just advances a monotonic pointer over
already-known pivots to whatever is causally confirmable as of that
candle — O(1) amortized across the whole run instead of O(window) or
O(n) per candle. Added an optional precomputed_pivots param to
detect_divergence() (backward compatible) to reuse this for RSI/MACD
divergence too.

Net effect: 16,000 candles went from 16.1s to 1.7s (confirmed empirically,
on top of an earlier ~10x from fixing the raw O(n^2)), and scaling stays
linear at 32,000 candles (3.2s). Walk-forward's timeframe options are now
15m/30m/1h/4h/1d (up from 1h/4h/1d) since 15m at the 3-year default
lookback now costs roughly 30s instead of 5+ minutes. Also wired
walk_forward.py's grid search to actually reuse one computed score series
across all 27 parameter combinations per fold (it was recomputing full
classification for every combination despite the scoring/threshold split
added earlier). 156 backend tests passing (3 new: causal-score regression,
pivot-detection-runs-once, order-block-window-bounded).

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>

backend/tests/test_backtest_engine.py

@@ -104,48 +104,52 @@ async def test_run_backtest_returns_expected_shape(db_session):
     assert set(result["trades"].keys()) >= {"total", "wins", "losses", "win_rate", "total_pnl", "profit_factor"}
 
 
-def _make_fake_classifier(buy_at: set[int], sell_at: set[int]):
-    """Build a fake `_classify_signal_combined` that ignores indicator data
-    and instead signals BUY/SELL purely from `close_price`, which the real
-    test encodes as the candle index (so we can trigger deterministically).
-    Also records the strong_threshold/signal_threshold it was called with.
+def _make_fake_score_fn(buy_at: set[int], sell_at: set[int], score: float = 10.0):
+    """Build a fake `_compute_adjusted_score` that ignores indicator data
+    and instead reports a strong score purely from `close_price`, which the
+    real test encodes as the candle index (so triggers are deterministic).
+    `_compute_adjusted_score` is threshold-independent post-refactor — the
+    thresholds now live entirely in `_score_to_signal`/`_simulate_from_scores`.
     """
-    calls = []
+    calls: list[float] = []
 
     def fake(close_price, *args, **kwargs):
-        calls.append({
-            "strong_threshold": kwargs.get("strong_threshold"),
-            "signal_threshold": kwargs.get("signal_threshold"),
-        })
+        calls.append(close_price)
         idx = int(round(close_price))
         if idx in buy_at:
-            return backtest_engine.STRONG_BUY, "STRONG", 0.9, {}
+            return None, None, score, 0.9, {}
         if idx in sell_at:
-            return backtest_engine.STRONG_SELL, "STRONG", 0.9, {}
-        return None, None, 0.5, {}
+            return None, None, -score, 0.9, {}
+        return None, None, 0.0, 0.5, {}
 
     return fake, calls
 
 
-async def test_simulate_trades_passes_thresholds_to_classifier(monkeypatch, db_session):
+async def test_simulate_trades_thresholds_control_signal_strength(monkeypatch, db_session):
+    """Post-refactor, `_compute_adjusted_score` no longer takes thresholds
+    at all — `strong_threshold`/`signal_threshold` are applied afterwards
+    by `_score_to_signal`. This proves that pass-through still works: the
+    same fixed score classifies differently depending on the thresholds
+    given to `_simulate_trades`."""
     _, symbol = await _seed_symbol(db_session)
     base = datetime.now(timezone.utc) - timedelta(hours=40)
-    # close price == candle index, so the fake classifier can key off it
     await _seed_candles(db_session, symbol.id, "1h", base, 40, timedelta(hours=1), lambda i: i)
     candles = await backtest_engine._fetch_candles(db_session, symbol.id, "1h", since=base)
     precomputed = backtest_engine._precompute_indicators(candles, "1h")
 
-    fake, calls = _make_fake_classifier(buy_at={35}, sell_at=set())
-    monkeypatch.setattr(backtest_engine, "_classify_signal_combined", fake)
+    def fake_fixed_score(close_price, *args, **kwargs):
+        return None, None, 3.0, 0.9, {}
+    monkeypatch.setattr(backtest_engine, "_compute_adjusted_score", fake_fixed_score)
 
-    backtest_engine._simulate_trades(
-        candles, precomputed, Decimal("10"),
-        strong_threshold=2.5, signal_threshold=0.5,
+    signals_default, _ = backtest_engine._simulate_trades(candles, precomputed, Decimal("10"))
+    assert len(signals_default) > 0
+    assert all(s["signal"] == backtest_engine.BUY for s in signals_default)
+
+    signals_lowered, _ = backtest_engine._simulate_trades(
+        candles, precomputed, Decimal("10"), strong_threshold=2.5,
     )
-
-    assert len(calls) > 0
-    assert all(c["strong_threshold"] == 2.5 for c in calls)
-    assert all(c["signal_threshold"] == 0.5 for c in calls)
+    assert len(signals_lowered) > 0
+    assert all(s["signal"] == backtest_engine.STRONG_BUY for s in signals_lowered)
 
 
 async def test_simulate_trades_exits_on_max_hold_candles(monkeypatch, db_session):
@@ -157,8 +161,8 @@ async def test_simulate_trades_exits_on_max_hold_candles(monkeypatch, db_session
 
     # Open a LONG at index 30 (candle price 30) and never signal again —
     # it must be force-closed exactly `max_hold_candles` candles later.
-    fake, _ = _make_fake_classifier(buy_at={30}, sell_at=set())
-    monkeypatch.setattr(backtest_engine, "_classify_signal_combined", fake)
+    fake, _ = _make_fake_score_fn(buy_at={30}, sell_at=set())
+    monkeypatch.setattr(backtest_engine, "_compute_adjusted_score", fake)
 
     _, trades = backtest_engine._simulate_trades(
         candles, precomputed, Decimal("10"), max_hold_candles=5,
@@ -181,8 +185,8 @@ async def test_simulate_trades_active_from_index_skips_warmup_region(monkeypatch
 
     # A BUY signal planted inside the warmup region (index 32, but
     # active_from_index=35) must never open a trade.
-    fake, _ = _make_fake_classifier(buy_at={32}, sell_at=set())
-    monkeypatch.setattr(backtest_engine, "_classify_signal_combined", fake)
+    fake, _ = _make_fake_score_fn(buy_at={32}, sell_at=set())
+    monkeypatch.setattr(backtest_engine, "_compute_adjusted_score", fake)
 
     all_signals, trades = backtest_engine._simulate_trades(
         candles, precomputed, Decimal("10"), active_from_index=35,
@@ -190,3 +194,110 @@ async def test_simulate_trades_active_from_index_skips_warmup_region(monkeypatch
 
     assert trades == []
     assert all_signals == []
+
+
+def _build_candle_series(base, prices):
+    return [
+        Candle(
+            symbol_id=1, timeframe="1h", timestamp=base + timedelta(hours=i),
+            open=Decimal(str(p)), high=Decimal(str(p * 1.01)), low=Decimal(str(p * 0.99)),
+            close=Decimal(str(p)), volume=Decimal("1000"),
+        )
+        for i, p in enumerate(prices)
+    ]
+
+
+def test_scores_are_causal_future_prices_dont_change_earlier_scores():
+    """Regression test for the look-ahead fix: market_structure() (SMC) and
+    detect_divergence() used to be precomputed ONCE over the whole candle
+    range and reused unchanged for every candle, so a candle's score could
+    be influenced by prices that hadn't happened yet as of that candle.
+    This proves candle 90's score is identical whether candle 140 behaves
+    normally or crashes hard afterward — the future can no longer leak
+    backward into an earlier candle's classification.
+    """
+    base = datetime.now(timezone.utc) - timedelta(hours=150)
+    shared_prefix = [100 + 5 * math.sin(i / 4) + (i % 5) for i in range(100)]
+    calm_tail = [shared_prefix[-1] + (i % 3) for i in range(1, 51)]
+    crash_tail = [shared_prefix[-1] - i * 3 for i in range(1, 51)]
+
+    candles_calm = _build_candle_series(base, shared_prefix + calm_tail)
+    candles_crash = _build_candle_series(base, shared_prefix + crash_tail)
+
+    precomputed_calm = backtest_engine._precompute_indicators(candles_calm, "1h")
+    precomputed_crash = backtest_engine._precompute_indicators(candles_crash, "1h")
+
+    scores_calm = backtest_engine._compute_scores_series(candles_calm, precomputed_calm)
+    scores_crash = backtest_engine._compute_scores_series(candles_crash, precomputed_crash)
+
+    # Candle 90 is well before the two futures diverge at candle 100 —
+    # its score must not depend on which future it's embedded in.
+    entry_calm = next(s for s in scores_calm if s["index"] == 90)
+    entry_crash = next(s for s in scores_crash if s["index"] == 90)
+
+    assert entry_calm["adjusted_score"] == entry_crash["adjusted_score"]
+    assert entry_calm["override_signal"] == entry_crash["override_signal"]
+
+
+def test_pivot_detection_runs_once_not_per_candle(monkeypatch):
+    """Regression test for the O(n^2) fix: pivot detection
+    (_find_pivot_highs[_levels]/_find_pivot_lows[_levels]) must run ONCE
+    over the whole dataset in `_precompute_indicators`, not be rescanned
+    for every candle in `_compute_scores_series` — that per-candle
+    rescanning (on top of it also leaking future data — see the causal
+    test above) is what made 15m/30m walk-forward runs too slow.
+    """
+    call_counts = {"highs_levels": 0, "lows_levels": 0, "highs": 0, "lows": 0}
+    real_highs_levels = backtest_engine._find_pivot_highs_levels
+    real_lows_levels = backtest_engine._find_pivot_lows_levels
+    real_highs = backtest_engine._find_pivot_highs
+    real_lows = backtest_engine._find_pivot_lows
+
+    def spy(name, real_fn):
+        def wrapper(*args, **kwargs):
+            call_counts[name] += 1
+            return real_fn(*args, **kwargs)
+        return wrapper
+
+    monkeypatch.setattr(backtest_engine, "_find_pivot_highs_levels", spy("highs_levels", real_highs_levels))
+    monkeypatch.setattr(backtest_engine, "_find_pivot_lows_levels", spy("lows_levels", real_lows_levels))
+    monkeypatch.setattr(backtest_engine, "_find_pivot_highs", spy("highs", real_highs))
+    monkeypatch.setattr(backtest_engine, "_find_pivot_lows", spy("lows", real_lows))
+
+    base = datetime.now(timezone.utc) - timedelta(hours=500)
+    prices = [100 + 10 * math.sin(i / 7) for i in range(500)]
+    candles = _build_candle_series(base, prices)
+
+    precomputed = backtest_engine._precompute_indicators(candles, "1h")
+    scores = backtest_engine._compute_scores_series(candles, precomputed)
+
+    assert len(scores) > 400  # sanity: the loop actually ran over most of the 500 candles
+    # Called once for the main timeframe plus once per configured MTF
+    # sub-timeframe (a small constant, e.g. 2-3) — never once per candle
+    # (which would show call counts in the hundreds here).
+    for name, count in call_counts.items():
+        assert 1 <= count <= 4, f"{name} called {count} times — expected a small constant, not scaling with candle count"
+
+
+def test_order_block_detection_uses_bounded_window_regardless_of_dataset_size(monkeypatch):
+    """The other potentially-expensive-per-candle SMC piece: order block
+    detection must always be called with a small, bounded candle window
+    (~_ORDER_BLOCK_LOOKBACK), never one that grows with the candle index."""
+    seen_lengths: list[int] = []
+    real_detect_order_blocks = backtest_engine._detect_order_blocks
+
+    def spy(candles, *args, **kwargs):
+        seen_lengths.append(len(candles))
+        return real_detect_order_blocks(candles, *args, **kwargs)
+
+    monkeypatch.setattr(backtest_engine, "_detect_order_blocks", spy)
+
+    base = datetime.now(timezone.utc) - timedelta(hours=500)
+    prices = [100 + 10 * math.sin(i / 7) for i in range(500)]
+    candles = _build_candle_series(base, prices)
+
+    precomputed = backtest_engine._precompute_indicators(candles, "1h")
+    backtest_engine._compute_scores_series(candles, precomputed)
+
+    assert len(seen_lengths) > 0
+    assert max(seen_lengths) <= backtest_engine._ORDER_BLOCK_LOOKBACK + 3

backend/tests/test_walk_forward.py

@@ -89,7 +89,7 @@ async def test_grid_search_picks_the_best_scoring_combo(monkeypatch):
     best one and reports its stats."""
     good_params = {"strong_threshold": 4.5, "signal_threshold": 1.5, "max_hold_candles": 96}
 
-    def fake_run_combo(candles, precomputed, trade_size, params, active_from_index):
+    def fake_run_combo(candles, scores_series, trade_size, params):
         if params == good_params:
             trades = [{"pnl": 10.0, "status": "CLOSED", "entry_price": 100.0, "exit_price": 110.0} for _ in range(10)]
         else:
@@ -103,7 +103,7 @@ async def test_grid_search_picks_the_best_scoring_combo(monkeypatch):
 
     grid = {"strong_threshold": [3.5, 4.5], "signal_threshold": [1.0, 1.5], "max_hold_candles": [48, 96]}
     best_params, best_score, best_stats = walk_forward._grid_search(
-        candles=[], precomputed={}, param_grid=grid, trade_size=Decimal("10"), active_from_index=0,
+        candles=[], scores_series=[], param_grid=grid, trade_size=Decimal("10"),
     )
 
     assert best_params == good_params
@@ -113,7 +113,7 @@ async def test_grid_search_picks_the_best_scoring_combo(monkeypatch):
 
 @pytest.mark.asyncio
 async def test_grid_search_falls_back_when_every_combo_too_sparse(monkeypatch):
-    def fake_run_combo(candles, precomputed, trade_size, params, active_from_index):
+    def fake_run_combo(candles, scores_series, trade_size, params):
         # 1 trade, below MIN_TRADES_PER_FOLD
         return [], [{"pnl": 1.0, "status": "CLOSED", "entry_price": 100.0, "exit_price": 101.0}]
 
@@ -121,7 +121,7 @@ async def test_grid_search_falls_back_when_every_combo_too_sparse(monkeypatch):
 
     grid = {"strong_threshold": [3.5, 4.5], "signal_threshold": [1.0], "max_hold_candles": [48]}
     best_params, best_score, best_stats = walk_forward._grid_search(
-        candles=[], precomputed={}, param_grid=grid, trade_size=Decimal("10"), active_from_index=0,
+        candles=[], scores_series=[], param_grid=grid, trade_size=Decimal("10"),
     )
 
     # Falls back to the first grid combination rather than raising