Initial commit: Trading Portal - FastAPI + React + PostgreSQL
This commit is contained in:
Executable
+47
@@ -0,0 +1,47 @@
|
||||
#!/bin/bash
|
||||
# Deploy trading-backend container with dual-network setup
|
||||
# Usage: ./deploy-backend.sh [image_tag]
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
IMAGE="${1:-trading-backend:latest}"
|
||||
CONTAINER="trading-backend"
|
||||
NET1="trading-net"
|
||||
NET2="hermes-agent_hermes-net"
|
||||
|
||||
echo "==> Stopping old container (if any)..."
|
||||
docker rm -f "$CONTAINER" 2>/dev/null || true
|
||||
|
||||
echo "==> Creating $CONTAINER with --network $NET1..."
|
||||
docker run -d \
|
||||
--name "$CONTAINER" \
|
||||
--network "$NET1" \
|
||||
-e DATABASE_URL="postgresql+asyncpg://trading:trading_secret@trading-db:5432/trading_portal" \
|
||||
"$IMAGE"
|
||||
|
||||
echo "==> Connecting $CONTAINER to $NET2..."
|
||||
docker network connect "$NET2" "$CONTAINER"
|
||||
|
||||
echo "==> Copying JWT keys..."
|
||||
docker exec -u 0 "$CONTAINER" mkdir -p /run/secrets 2>/dev/null || true
|
||||
docker cp /opt/data/trading-portal/secrets/jwt_private.pem "$CONTAINER":/run/secrets/jwt_private.pem 2>/dev/null || \
|
||||
echo " (jwt_private.pem not found, continuing)"
|
||||
docker cp /opt/data/trading-portal/secrets/jwt_public.pem "$CONTAINER":/run/secrets/jwt_public.pem 2>/dev/null || \
|
||||
echo " (jwt_public.pem not found, continuing)"
|
||||
docker exec -u 0 "$CONTAINER" chown trading:trading /run/secrets/*.pem 2>/dev/null || true
|
||||
|
||||
echo "==> Copying fetch_candles_cron.py..."
|
||||
docker cp /opt/data/trading-portal/backend/scripts/fetch_candles_cron.py "$CONTAINER":/tmp/fc_cron.py 2>/dev/null
|
||||
docker exec -u 0 "$CONTAINER" bash -c "cp /tmp/fc_cron.py /app/scripts/fetch_candles_cron.py && chown trading:trading /app/scripts/fetch_candles_cron.py && chmod +x /app/scripts/fetch_candles_cron.py" 2>/dev/null || \
|
||||
echo " (fetch_candles_cron.py copy skipped, continuing)"
|
||||
|
||||
echo "==> Verifying networks:"
|
||||
docker inspect "$CONTAINER" --format '{{range $k,$v := .NetworkSettings.Networks}}{{$k}} {{$v.IPAddress}}
|
||||
{{end}}'
|
||||
|
||||
echo "==> Checking health..."
|
||||
sleep 2
|
||||
docker exec "$CONTAINER" curl -s http://localhost:8001/health 2>/dev/null || \
|
||||
echo "(health check may need a moment)"
|
||||
|
||||
echo "✅ $CONTAINER deployed successfully on both networks!"
|
||||
Executable
+25
@@ -0,0 +1,25 @@
|
||||
#!/bin/bash
|
||||
# Deploy trading-frontend container with dual-network setup
|
||||
# Usage: ./deploy-frontend.sh [image_tag]
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
IMAGE="${1:-trading-frontend:latest}"
|
||||
CONTAINER="trading-frontend"
|
||||
NET1="trading-net"
|
||||
NET2="hermes-agent_hermes-net"
|
||||
|
||||
echo "==> Stopping old container (if any)..."
|
||||
docker rm -f "$CONTAINER" 2>/dev/null || true
|
||||
|
||||
echo "==> Creating $CONTAINER with --network $NET1..."
|
||||
docker run -d --name "$CONTAINER" --network "$NET1" "$IMAGE"
|
||||
|
||||
echo "==> Connecting $CONTAINER to $NET2..."
|
||||
docker network connect "$NET2" "$CONTAINER"
|
||||
|
||||
echo "==> Verifying networks:"
|
||||
docker inspect "$CONTAINER" --format '{{range $k,$v := .NetworkSettings.Networks}}{{$k}} {{$v.IPAddress}}
|
||||
{{end}}'
|
||||
|
||||
echo "✅ $CONTAINER deployed successfully on both networks!"
|
||||
@@ -0,0 +1,176 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Fetch 3 years of historical OHLCV candles for top100 pairs.
|
||||
Runs inside Docker container — reads DATABASE_URL from environment.
|
||||
"""
|
||||
import asyncio, logging, os, re, sys, time
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from pathlib import Path
|
||||
|
||||
import asyncpg
|
||||
import ccxt.async_support as ccxt_async
|
||||
|
||||
# Config
|
||||
DB_URL = os.environ["DATABASE_URL"].replace("+asyncpg", "")
|
||||
TOP100_FILE = "/app/scripts/top100_volume-v2.txt"
|
||||
TIMEFRAMES = ["1d", "4h", "1h", "30m", "1w", "1M"]
|
||||
YEARS = 3
|
||||
SINCE_MS = int((datetime.now(timezone.utc) - timedelta(days=YEARS * 365)).timestamp() * 1000)
|
||||
FETCH_LIMIT = 1000
|
||||
BATCH_SIZE = 500
|
||||
CCXT_TIMEOUT = 30
|
||||
CONCURRENT = 4
|
||||
|
||||
EXCHANGE_MAP = {"BINANCE": "binance", "BYBIT": "bybit", "MEXC": "mexc", "GATE": "gate", "BINGX": "bingx"}
|
||||
|
||||
# Write to both stdout and a log file for progress monitoring
|
||||
LOG_FILE = "/tmp/fetch_hist.log"
|
||||
logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s",
|
||||
handlers=[logging.StreamHandler(sys.stdout),
|
||||
logging.FileHandler(LOG_FILE, mode='w')])
|
||||
logger = logging.getLogger("fetch")
|
||||
|
||||
|
||||
def parse_top100(path: str) -> dict[str, list[str]]:
|
||||
exs: dict[str, list[str]] = {}
|
||||
cur: str | None = None
|
||||
for line in open(path):
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
for d, c in EXCHANGE_MAP.items():
|
||||
if line == d:
|
||||
cur = c
|
||||
exs.setdefault(cur, [])
|
||||
break
|
||||
else:
|
||||
m = re.match(r"\s*(?:\d+|\+)\.\s+(\S+/USDT)\s+[\d,]+", line)
|
||||
if m and cur and m.group(1) not in exs[cur]:
|
||||
exs[cur].append(m.group(1))
|
||||
return exs
|
||||
|
||||
|
||||
async def get_symbol_ids(pool):
|
||||
rows = await pool.fetch("SELECT s.id, e.name AS en, s.symbol FROM symbols s JOIN exchanges e ON e.id=s.exchange_id WHERE s.is_active=true")
|
||||
return {(r["en"], r["symbol"]): r["id"] for r in rows}
|
||||
|
||||
|
||||
def tf_minutes(tf: str) -> int:
|
||||
m = {"m": 1, "h": 60, "d": 1440, "w": 10080, "M": 43200}
|
||||
return int(tf[:-1]) * m[tf[-1]]
|
||||
|
||||
|
||||
async def process_exchange(name: str, syms: list[str], smap: dict, pool) -> dict:
|
||||
stats = {"candles": 0, "failed": 0}
|
||||
ex = getattr(ccxt_async, name)()
|
||||
await asyncio.wait_for(ex.load_markets(), timeout=30)
|
||||
logger.info(f"[{name}] {len(ex.symbols)} markets, {len(syms)} symbols to fetch")
|
||||
sem = asyncio.Semaphore(CONCURRENT)
|
||||
|
||||
async def fetch_one(symbol, tf):
|
||||
sid = smap.get((name, symbol))
|
||||
if not sid or symbol not in ex.markets:
|
||||
return 0
|
||||
candles = []
|
||||
since = SINCE_MS
|
||||
for page in range(200):
|
||||
async with sem:
|
||||
try:
|
||||
data = await asyncio.wait_for(ex.fetch_ohlcv(symbol, tf, since=since, limit=FETCH_LIMIT), timeout=CCXT_TIMEOUT)
|
||||
except asyncio.TimeoutError:
|
||||
logger.warning(f"[{name}] TIMEOUT {symbol}:{tf}")
|
||||
break
|
||||
except Exception as e:
|
||||
err = str(e)[:120]
|
||||
el = err.lower()
|
||||
if any(k in el for k in ["rate limit", "irtransient", "requesttimeout"]):
|
||||
await asyncio.sleep(10)
|
||||
continue
|
||||
if "maximum" in el and "points" in el:
|
||||
max_ms = 10000 * tf_minutes(tf) * 60 * 1000
|
||||
ns = int(time.time() * 1000) - max_ms
|
||||
if ns > since:
|
||||
since = ns
|
||||
continue
|
||||
if "maximum query range" in el:
|
||||
dm = re.search(r"(\d+)\s*days?", err)
|
||||
if dm:
|
||||
ns = int((datetime.now(timezone.utc) - timedelta(days=int(dm.group(1)))).timestamp() * 1000)
|
||||
if ns > since:
|
||||
since = ns
|
||||
continue
|
||||
if "invalid interval" in el:
|
||||
return 0
|
||||
stats["failed"] += 1
|
||||
return 0
|
||||
if not data:
|
||||
break
|
||||
candles.extend(data)
|
||||
if len(data) < FETCH_LIMIT:
|
||||
break
|
||||
lt = data[-1][0]
|
||||
if lt <= since:
|
||||
break
|
||||
since = lt + 1
|
||||
|
||||
if candles:
|
||||
vals = [(sid, tf, datetime.fromtimestamp(c[0] / 1000, tz=timezone.utc),
|
||||
str(c[1]), str(c[2]), str(c[3]), str(c[4]), str(c[5])) for c in candles]
|
||||
for i in range(0, len(vals), BATCH_SIZE):
|
||||
b = vals[i:i + BATCH_SIZE]
|
||||
async with pool.acquire() as conn:
|
||||
await conn.executemany(
|
||||
"INSERT INTO candles (symbol_id,timeframe,timestamp,open,high,low,close,volume) VALUES ($1,$2,$3,$4::numeric,$5::numeric,$6::numeric,$7::numeric,$8::numeric) ON CONFLICT DO NOTHING",
|
||||
b)
|
||||
return len(candles)
|
||||
|
||||
tasks = [fetch_one(s, tf) for s in syms for tf in TIMEFRAMES]
|
||||
done = [0]
|
||||
total = len(tasks)
|
||||
|
||||
async def tracked(t):
|
||||
r = await t
|
||||
done[0] += 1
|
||||
if done[0] % 50 == 0 or done[0] == total:
|
||||
logger.info(f"[{name}] {done[0]}/{total} tasks done")
|
||||
return r
|
||||
|
||||
results = await asyncio.gather(*[tracked(t) for t in tasks], return_exceptions=True)
|
||||
stats["candles"] = sum(r for r in results if isinstance(r, int) and r > 0)
|
||||
logger.info(f"[{name}] DONE: {stats['candles']:,} candles ({stats['failed']} failed)")
|
||||
await ex.close()
|
||||
return stats
|
||||
|
||||
|
||||
async def main():
|
||||
t0 = time.monotonic()
|
||||
logger.info(f"FETCH {YEARS}Y — {TIMEFRAMES} — DB={DB_URL[:40]}...")
|
||||
if not Path(TOP100_FILE).exists():
|
||||
logger.error(f"Missing {TOP100_FILE}")
|
||||
return
|
||||
|
||||
ex_syms = parse_top100(TOP100_FILE)
|
||||
for ex, syms in ex_syms.items():
|
||||
logger.info(f" {ex}: {len(syms)} symbols")
|
||||
|
||||
pool = await asyncpg.create_pool(DB_URL, min_size=2, max_size=10)
|
||||
smap = await get_symbol_ids(pool)
|
||||
logger.info(f"DB: {len(smap)} active symbols")
|
||||
|
||||
all_stats = []
|
||||
for ex_name, syms in ex_syms.items():
|
||||
existing = [s for s in syms if (ex_name, s) in smap]
|
||||
if not existing:
|
||||
logger.warning(f"[{ex_name}] No symbols in DB — skip")
|
||||
continue
|
||||
logger.info(f"\n{'='*50}\n[{ex_name}] {len(existing)} symbols\n{'='*50}")
|
||||
all_stats.append(await process_exchange(ex_name, existing, smap, pool))
|
||||
|
||||
elapsed = (time.monotonic() - t0) / 60
|
||||
total = sum(s["candles"] for s in all_stats)
|
||||
logger.info(f"\n{'='*50}\nDONE {elapsed:.1f}min — {total:,} candles\n{'='*50}")
|
||||
await pool.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,193 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Fetch 3Y historical OHLCV — fixed: sequential per symbol, low memory.
|
||||
Runs inside scheduler container, reads DATABASE_URL from env.
|
||||
"""
|
||||
import asyncio, logging, os, re, sys, time
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from pathlib import Path
|
||||
|
||||
import asyncpg
|
||||
import ccxt.async_support as ccxt_async
|
||||
|
||||
DB_URL = os.environ["DATABASE_URL"].replace("+asyncpg", "")
|
||||
TOP100_FILE = "/app/scripts/top100_volume-v2.txt"
|
||||
TIMEFRAMES = ["1d", "4h", "1h", "30m", "1w", "1M"]
|
||||
YEARS = 3
|
||||
SINCE_MS = int((datetime.now(timezone.utc) - timedelta(days=YEARS * 365)).timestamp() * 1000)
|
||||
FETCH_LIMIT = 1000
|
||||
BATCH_SIZE = 500
|
||||
CCXT_TIMEOUT = 30
|
||||
|
||||
EXCHANGE_MAP = {"BINANCE": "binance", "BYBIT": "bybit", "MEXC": "mexc", "GATE": "gate", "BINGX": "bingx"}
|
||||
|
||||
LOG_FILE = "/tmp/fetch_full.log"
|
||||
logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s",
|
||||
handlers=[logging.StreamHandler(sys.stdout), logging.FileHandler(LOG_FILE, mode='w')])
|
||||
logger = logging.getLogger("fetch")
|
||||
|
||||
|
||||
def parse_top100(path):
|
||||
exs, cur = {}, None
|
||||
for line in open(path):
|
||||
line = line.strip()
|
||||
if not line: continue
|
||||
for d, c in EXCHANGE_MAP.items():
|
||||
if line == d:
|
||||
cur = c; exs.setdefault(cur, []); break
|
||||
else:
|
||||
m = re.match(r"\s*(?:\d+|\+)\.\s+(\S+/USDT)\s+[\d,]+", line)
|
||||
if m and cur and m.group(1) not in exs[cur]:
|
||||
exs[cur].append(m.group(1))
|
||||
return exs
|
||||
|
||||
|
||||
async def get_symbol_ids(pool):
|
||||
rows = await pool.fetch("""
|
||||
SELECT s.id, e.name AS en, s.symbol
|
||||
FROM symbols s JOIN exchanges e ON e.id=s.exchange_id
|
||||
WHERE s.is_active=true
|
||||
""")
|
||||
return {(r["en"], r["symbol"]): r["id"] for r in rows}
|
||||
|
||||
|
||||
def tf_minutes(tf):
|
||||
m = {"m": 1, "h": 60, "d": 1440, "w": 10080, "M": 43200}
|
||||
return int(tf[:-1]) * m[tf[-1]]
|
||||
|
||||
|
||||
async def fetch_and_save(ex_name, symbol, smap, ex, pool, conn):
|
||||
"""Fetch all 6 TFs for ONE symbol concurrently, insert to DB."""
|
||||
sid = smap.get((ex_name, symbol))
|
||||
if not sid:
|
||||
return {"symbol": symbol, "candles": 0, "error": "not in DB"}
|
||||
|
||||
async def fetch_one_tf(tf):
|
||||
candles = []
|
||||
since = SINCE_MS
|
||||
for page in range(200):
|
||||
try:
|
||||
data = await asyncio.wait_for(
|
||||
ex.fetch_ohlcv(symbol, tf, since=since, limit=FETCH_LIMIT),
|
||||
timeout=CCXT_TIMEOUT,
|
||||
)
|
||||
except asyncio.TimeoutError:
|
||||
logger.warning(f" {symbol}:{tf} TIMEOUT page {page}")
|
||||
break
|
||||
except Exception as e:
|
||||
err = str(e)
|
||||
el = err.lower()
|
||||
if any(k in el for k in ["rate limit", "irtransient", "requesttimeout"]):
|
||||
await asyncio.sleep(5)
|
||||
continue
|
||||
if "maximum" in el and "points" in el:
|
||||
mx = 10000 * tf_minutes(tf) * 60 * 1000
|
||||
ns = int(time.time() * 1000) - mx
|
||||
if ns > since:
|
||||
since = ns; continue
|
||||
if "maximum query range" in el:
|
||||
dm = re.search(r"(\d+)\s*days?", err)
|
||||
if dm:
|
||||
ns = int((datetime.now(timezone.utc) - timedelta(days=int(dm.group(1)))).timestamp() * 1000)
|
||||
if ns > since:
|
||||
since = ns; continue
|
||||
if "invalid interval" in el:
|
||||
return (tf, [])
|
||||
# Other error — skip this TF
|
||||
return (tf, [])
|
||||
if not data:
|
||||
break
|
||||
candles.extend(data)
|
||||
if len(data) < FETCH_LIMIT:
|
||||
break
|
||||
lt = data[-1][0]
|
||||
if lt <= since:
|
||||
break
|
||||
since = lt + 1
|
||||
return (tf, candles)
|
||||
|
||||
# Fetch all 6 TFs concurrently
|
||||
results = await asyncio.gather(*[fetch_one_tf(tf) for tf in TIMEFRAMES], return_exceptions=True)
|
||||
|
||||
total_candles = 0
|
||||
for result in results:
|
||||
if isinstance(result, BaseException):
|
||||
continue
|
||||
tf, candles = result
|
||||
if not candles:
|
||||
continue
|
||||
|
||||
vals = [(sid, tf, datetime.fromtimestamp(c[0] / 1000, tz=timezone.utc),
|
||||
str(c[1]), str(c[2]), str(c[3]), str(c[4]), str(c[5])) for c in candles]
|
||||
|
||||
for i in range(0, len(vals), BATCH_SIZE):
|
||||
b = vals[i:i + BATCH_SIZE]
|
||||
try:
|
||||
await conn.executemany(
|
||||
"INSERT INTO candles (symbol_id,timeframe,timestamp,open,high,low,close,volume) "
|
||||
"VALUES ($1,$2,$3,$4::numeric,$5::numeric,$6::numeric,$7::numeric,$8::numeric) "
|
||||
"ON CONFLICT DO NOTHING", b)
|
||||
except Exception as e:
|
||||
logger.error(f" {symbol}:{tf} DB insert error: {e}")
|
||||
break
|
||||
total_candles += len(candles)
|
||||
|
||||
return {"symbol": symbol, "candles": total_candles}
|
||||
|
||||
|
||||
async def main():
|
||||
t0 = time.monotonic()
|
||||
logger.info(f"FETCH {YEARS}Y — {TIMEFRAMES}")
|
||||
|
||||
ex_syms = parse_top100(TOP100_FILE)
|
||||
for ex, syms in ex_syms.items():
|
||||
logger.info(f" {ex}: {len(syms)} symbols")
|
||||
|
||||
pool = await asyncpg.create_pool(DB_URL, min_size=2, max_size=6)
|
||||
smap = await get_symbol_ids(pool)
|
||||
logger.info(f"DB: {len(smap)} active symbols")
|
||||
|
||||
grand_total = 0
|
||||
|
||||
for ex_name, all_syms in ex_syms.items():
|
||||
existing = [s for s in all_syms if (ex_name, s) in smap]
|
||||
if not existing:
|
||||
logger.warning(f"[{ex_name}] No symbols in DB — skip")
|
||||
continue
|
||||
|
||||
logger.info(f"\n{'='*50}\n[{ex_name}] {len(existing)} symbols\n{'='*50}")
|
||||
|
||||
# Create exchange instance
|
||||
ex = getattr(ccxt_async, ex_name)()
|
||||
try:
|
||||
await asyncio.wait_for(ex.load_markets(), timeout=30)
|
||||
logger.info(f"[{ex_name}] markets loaded")
|
||||
except Exception as e:
|
||||
logger.error(f"[{ex_name}] load_markets failed: {e}")
|
||||
await ex.close()
|
||||
continue
|
||||
|
||||
ex_total = 0
|
||||
for idx, symbol in enumerate(existing):
|
||||
if symbol not in ex.markets:
|
||||
continue
|
||||
|
||||
t1 = time.monotonic()
|
||||
async with pool.acquire() as conn:
|
||||
result = await fetch_and_save(ex_name, symbol, smap, ex, pool, conn)
|
||||
|
||||
elapsed = time.monotonic() - t1
|
||||
ex_total += result["candles"]
|
||||
logger.info(f" [{idx+1}/{len(existing)}] {symbol}: {result['candles']:,} candles ({elapsed:.1f}s)")
|
||||
|
||||
grand_total += ex_total
|
||||
logger.info(f"[{ex_name}] DONE: {ex_total:,} candles")
|
||||
await ex.close()
|
||||
|
||||
elapsed = (time.monotonic() - t0) / 60
|
||||
logger.info(f"\n{'='*50}\nTOTAL: {grand_total:,} candles in {elapsed:.1f} min\n{'='*50}")
|
||||
await pool.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,77 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Minimal working version: fetch Binance 30m for BTC/USDT 3 years."""
|
||||
import asyncio, sys, time, os
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
import asyncpg
|
||||
import ccxt.async_support as ccxt_async
|
||||
|
||||
DB_URL = os.environ["DATABASE_URL"].replace("+asyncpg", "")
|
||||
SINCE_MS = int((datetime.now(timezone.utc) - timedelta(days=3*365)).timestamp() * 1000)
|
||||
|
||||
async def main():
|
||||
t0 = time.monotonic()
|
||||
print(f"DB: {DB_URL[:40]}...", flush=True)
|
||||
|
||||
ex = ccxt_async.binance()
|
||||
await ex.load_markets()
|
||||
print(f"Binance loaded: {len(ex.symbols)} markets", flush=True)
|
||||
|
||||
pool = await asyncpg.create_pool(DB_URL, min_size=1, max_size=4)
|
||||
|
||||
# Get BTC/USDT symbol ID
|
||||
rows = await pool.fetch("SELECT s.id FROM symbols s JOIN exchanges e ON e.id=s.exchange_id WHERE e.name='binance' AND s.symbol='BTC/USDT'")
|
||||
symbol_id = rows[0][0]
|
||||
print(f"BTC/USDT symbol_id={symbol_id}", flush=True)
|
||||
|
||||
# Fetch 30m
|
||||
all_candles = []
|
||||
since = SINCE_MS
|
||||
page = 0
|
||||
while True:
|
||||
page += 1
|
||||
try:
|
||||
data = await asyncio.wait_for(
|
||||
ex.fetch_ohlcv('BTC/USDT', '30m', since=since, limit=1000),
|
||||
timeout=30
|
||||
)
|
||||
except Exception as e:
|
||||
print(f"Page {page} ERROR: {e}", flush=True)
|
||||
break
|
||||
|
||||
if not data:
|
||||
break
|
||||
all_candles.extend(data)
|
||||
print(f"Page {page}: {len(data)} candles (total: {len(all_candles)})", flush=True)
|
||||
|
||||
if len(data) < 1000:
|
||||
break
|
||||
lt = data[-1][0]
|
||||
if lt <= since:
|
||||
break
|
||||
since = lt + 1
|
||||
|
||||
if page > 100:
|
||||
print("Too many pages!", flush=True)
|
||||
break
|
||||
|
||||
print(f"Total: {len(all_candles)} candles in {time.monotonic()-t0:.1f}s", flush=True)
|
||||
|
||||
# Insert
|
||||
values = [(symbol_id, '30m', datetime.fromtimestamp(c[0]/1000, tz=timezone.utc),
|
||||
str(c[1]), str(c[2]), str(c[3]), str(c[4]), str(c[5])) for c in all_candles]
|
||||
|
||||
i0 = time.monotonic()
|
||||
for i in range(0, len(values), 500):
|
||||
batch = values[i:i+500]
|
||||
async with pool.acquire() as conn:
|
||||
await conn.executemany(
|
||||
"INSERT INTO candles (symbol_id,timeframe,timestamp,open,high,low,close,volume) VALUES ($1,$2,$3,$4::numeric,$5::numeric,$6::numeric,$7::numeric,$8::numeric) ON CONFLICT DO NOTHING",
|
||||
batch)
|
||||
print(f"Inserted in {time.monotonic()-i0:.1f}s", flush=True)
|
||||
|
||||
await ex.close()
|
||||
await pool.close()
|
||||
print("DONE", flush=True)
|
||||
|
||||
asyncio.run(main())
|
||||
Executable
+106
@@ -0,0 +1,106 @@
|
||||
#!/usr/bin/env python3
|
||||
"""System Health Check — script-only, zero token cost.
|
||||
|
||||
Checks all 4 trading portal services every 30 minutes.
|
||||
Only outputs when something is BROKEN → triggers Telegram alert.
|
||||
Silent when everything is healthy.
|
||||
"""
|
||||
|
||||
import subprocess, sys, json, os
|
||||
|
||||
PROJECT_DIR = "/opt/data/trading-portal"
|
||||
HEALTH_URL = "http://localhost:8001/health"
|
||||
FRONTEND_URL = "http://localhost:3000"
|
||||
DOCKER = "/usr/bin/docker"
|
||||
|
||||
ALERTS = []
|
||||
|
||||
|
||||
def check(label, ok):
|
||||
if not ok:
|
||||
ALERTS.append(label)
|
||||
|
||||
|
||||
# 1. Docker compose service status
|
||||
try:
|
||||
out = subprocess.check_output(
|
||||
[DOCKER, "compose", "-f", f"{PROJECT_DIR}/docker-compose.yml", "ps", "--format", "json"],
|
||||
cwd=PROJECT_DIR, text=True, timeout=15
|
||||
)
|
||||
services = {}
|
||||
for line in out.strip().split("\n"):
|
||||
if not line.strip():
|
||||
continue
|
||||
s = json.loads(line)
|
||||
# Docker Compose v5.2.0 uses CapitalCase keys
|
||||
name = s.get("Service", s.get("service", "?"))
|
||||
state = s.get("State", s.get("state", "?"))
|
||||
health = s.get("Health", s.get("health", ""))
|
||||
if health:
|
||||
services[name] = f"{state}/{health}"
|
||||
else:
|
||||
services[name] = state
|
||||
|
||||
for svc in ["db", "backend-api", "backend-scheduler", "frontend"]:
|
||||
state = services.get(svc, "MISSING")
|
||||
if "healthy" not in state.lower() and "running" not in state.lower():
|
||||
check(f"Docker {svc}: {state}", False)
|
||||
except Exception as e:
|
||||
check(f"Docker ps failed: {str(e)[:100]}", False)
|
||||
|
||||
|
||||
# 2. API health endpoint
|
||||
try:
|
||||
import urllib.request
|
||||
with urllib.request.urlopen(HEALTH_URL, timeout=10) as resp:
|
||||
data = json.loads(resp.read())
|
||||
if data.get("status") != "healthy":
|
||||
check(f"API health: {data}", False)
|
||||
except Exception as e:
|
||||
check(f"API health unreachable: {e}", False)
|
||||
|
||||
|
||||
# 3. Frontend reachable
|
||||
try:
|
||||
import urllib.request
|
||||
req = urllib.request.Request(FRONTEND_URL, method="HEAD")
|
||||
with urllib.request.urlopen(req, timeout=10) as resp:
|
||||
if resp.status not in (200, 301, 302, 304):
|
||||
check(f"Frontend HTTP {resp.status}", False)
|
||||
except Exception as e:
|
||||
check(f"Frontend unreachable: {e}", False)
|
||||
|
||||
|
||||
# 4. Scheduler recent errors (last 5 minutes)
|
||||
try:
|
||||
logs = subprocess.check_output(
|
||||
["docker", "compose", "logs", "--since", "5m", "backend-scheduler"],
|
||||
cwd=PROJECT_DIR, text=True, stderr=subprocess.STDOUT, timeout=10
|
||||
)
|
||||
errors = [l for l in logs.split("\n") if "ERROR" in l or "CRITICAL" in l or "Traceback" in l]
|
||||
if errors:
|
||||
check(f"Scheduler errors ({len(errors)}): {errors[-1][:200]}", False)
|
||||
except Exception as e:
|
||||
check(f"Scheduler log check failed: {e}", False)
|
||||
|
||||
|
||||
# 5. Check disk space
|
||||
try:
|
||||
stat = os.statvfs(PROJECT_DIR)
|
||||
pct_used = 100 - (stat.f_bavail / stat.f_blocks * 100)
|
||||
if pct_used > 90:
|
||||
check(f"Disk usage {pct_used:.0f}%", False)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
# ── Output (only when there are alerts) ──
|
||||
if ALERTS:
|
||||
print(f"🚨 Trading Portal Health Alert — {len(ALERTS)} issues:")
|
||||
for a in ALERTS:
|
||||
print(f" ❌ {a}")
|
||||
print(f"\nCheck: docker compose -f {PROJECT_DIR}/docker-compose.yml ps")
|
||||
sys.exit(1)
|
||||
else:
|
||||
# All good — silent output = no Telegram delivery
|
||||
pass
|
||||
Reference in New Issue
Block a user