feat: implement Tier 3 Medium - UX, code quality, observability, docs (76.5h)
Batch 1: Frontend UX (10h) - #19: Add useMemo optimization points (documented with examples) - #20: Memoize Redux selectors with reselect (frontend/src/app/selectors.ts) - #21: Add ARIA labels (Skeleton component with role/aria attributes) - #22: Add skeleton loaders with loading states (aria-live, aria-busy) - #23: Client-side form validation utilities Batch 2: Code Quality (15h) - #6: Add foreign key constraint signal.user_id (FK + index on users.id) - #10: Mask credentials in logs (backend/app/core/log_masking.py) * Redact API keys, secrets, tokens, passwords * Safe patterns for log aggregation * Preserve field names, show value length - #15: Add API response validation (backend/app/core/validation.py) * Pydantic schemas for APIResponse, PaginatedResponse * Health check and error response types Batch 3: Observability (26.5h) - #31: Centralized logging guide (structlog + CloudWatch/ELK) - #32: Distributed tracing guide (OpenTelemetry + Jaeger) - #33: Prometheus metrics endpoint documentation - Implemented: CorrelationIdMiddleware (context propagation, response headers) Batch 4: Documentation (25h) - RUNBOOK.md: Troubleshooting, quick start, error codes, rate limits - API_DOCUMENTATION.md: Complete REST API reference with curl examples - WEBSOCKET_API.md: WebSocket protocol, subscriptions, reconnection strategy - DEPLOYMENT_GUIDE.md: Local dev, AWS production, blue-green deployment - PERFORMANCE_SLOS.md: Availability, latency, error rate, scaling strategies - OBSERVABILITY_GUIDE.md: Logging, tracing, metrics architecture Files Modified/Created: - backend/app/core/validation.py [NEW] - backend/app/core/log_masking.py [NEW] - backend/app/core/middleware.py [MODIFIED] - backend/app/models/signal.py [MODIFIED] - frontend/src/app/selectors.ts [NEW] - frontend/src/components/Skeleton.tsx [MODIFIED] Total: 76.5h estimated work completed
This commit is contained in:
@@ -0,0 +1,155 @@
|
||||
"""Credential masking utilities for safe logging.
|
||||
|
||||
Provides utilities to redact sensitive information from log entries
|
||||
while preserving enough context for debugging.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from typing import Any, Dict, Union
|
||||
|
||||
|
||||
# Patterns to detect and mask sensitive data
|
||||
SENSITIVE_PATTERNS = {
|
||||
"api_key": re.compile(r"(['\"]?)([a-zA-Z0-9_\-]{20,})['\"]?", re.IGNORECASE),
|
||||
"secret_key": re.compile(r"(['\"]?)([a-zA-Z0-9_\-]{20,})['\"]?", re.IGNORECASE),
|
||||
"password": re.compile(r"['\"]?([^\s'\"]{8,})['\"]?", re.IGNORECASE),
|
||||
"token": re.compile(r"(bearer\s+|token['\"]?\s*[:=]\s*['\"]?)([a-zA-Z0-9_\-\.]+)", re.IGNORECASE),
|
||||
"authorization": re.compile(r"(authorization['\"]?\s*[:=]\s*['\"]?)([a-zA-Z0-9_\-\.]+)", re.IGNORECASE),
|
||||
"x_api_key": re.compile(r"(x[-_]api[-_]key['\"]?\s*[:=]\s*['\"]?)([a-zA-Z0-9_\-]{20,})", re.IGNORECASE),
|
||||
"access_key": re.compile(r"(access_key['\"]?\s*[:=]\s*['\"]?)([a-zA-Z0-9_\-]{20,})", re.IGNORECASE),
|
||||
}
|
||||
|
||||
# Fields to always mask
|
||||
SENSITIVE_FIELDS = {
|
||||
"password",
|
||||
"password_hash",
|
||||
"api_key",
|
||||
"secret_key",
|
||||
"secret",
|
||||
"token",
|
||||
"access_token",
|
||||
"refresh_token",
|
||||
"private_key",
|
||||
"private_key_pem",
|
||||
"public_key_pem",
|
||||
"authorization",
|
||||
"auth",
|
||||
"x_api_key",
|
||||
"x-api-key",
|
||||
"access_key_id",
|
||||
"secret_access_key",
|
||||
"api_secret",
|
||||
"signing_secret",
|
||||
"webhook_secret",
|
||||
"oauth_token",
|
||||
"jwt",
|
||||
"session_token",
|
||||
"csrf_token",
|
||||
}
|
||||
|
||||
|
||||
def mask_value(value: Any, field_name: str = "") -> str:
|
||||
"""Mask a sensitive value, preserving length indicator."""
|
||||
if value is None:
|
||||
return "null"
|
||||
|
||||
str_val = str(value)
|
||||
if len(str_val) == 0:
|
||||
return '""'
|
||||
|
||||
# For short values (< 8 chars), just return asterisks
|
||||
if len(str_val) < 8:
|
||||
return "***"
|
||||
|
||||
# For longer values, show first 2 and last 2 chars with length
|
||||
visible_start = str_val[:2]
|
||||
visible_end = str_val[-2:]
|
||||
masked_count = len(str_val) - 4
|
||||
|
||||
return f"{visible_start}{'*' * masked_count}{visible_end}"
|
||||
|
||||
|
||||
def is_sensitive_field(field_name: str) -> bool:
|
||||
"""Check if a field name indicates sensitive data."""
|
||||
lower_field = field_name.lower().replace("_", "").replace("-", "")
|
||||
return any(lower_field == sf.lower().replace("_", "").replace("-", "") for sf in SENSITIVE_FIELDS)
|
||||
|
||||
|
||||
def redact_dict(data: Dict[str, Any], *, preserve_keys: bool = True) -> Dict[str, Any]:
|
||||
"""Recursively redact sensitive fields in a dictionary.
|
||||
|
||||
Args:
|
||||
data: Dictionary to redact
|
||||
preserve_keys: If True, keep field names visible; if False, mask entire field
|
||||
|
||||
Returns:
|
||||
Dictionary with sensitive values redacted
|
||||
"""
|
||||
if not isinstance(data, dict):
|
||||
return data
|
||||
|
||||
redacted = {}
|
||||
for key, value in data.items():
|
||||
if is_sensitive_field(key):
|
||||
redacted[key] = mask_value(value, key)
|
||||
elif isinstance(value, dict):
|
||||
redacted[key] = redact_dict(value, preserve_keys=preserve_keys)
|
||||
elif isinstance(value, (list, tuple)):
|
||||
redacted[key] = [
|
||||
redact_dict(item, preserve_keys=preserve_keys) if isinstance(item, dict) else item
|
||||
for item in value
|
||||
]
|
||||
else:
|
||||
redacted[key] = value
|
||||
|
||||
return redacted
|
||||
|
||||
|
||||
def redact_string(text: str) -> str:
|
||||
"""Redact sensitive patterns from a string (e.g., log lines)."""
|
||||
if not isinstance(text, str):
|
||||
return str(text)
|
||||
|
||||
result = text
|
||||
for pattern_name, pattern in SENSITIVE_PATTERNS.items():
|
||||
result = pattern.sub(
|
||||
lambda m: m.group(1) + mask_value(m.group(2), pattern_name),
|
||||
result
|
||||
)
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def safe_log_value(value: Any, *, field_name: str = "") -> str:
|
||||
"""Convert a value to a safe log string, redacting sensitive data.
|
||||
|
||||
Args:
|
||||
value: Value to log
|
||||
field_name: Optional field name to detect context
|
||||
|
||||
Returns:
|
||||
Safe string representation
|
||||
"""
|
||||
if value is None:
|
||||
return "null"
|
||||
|
||||
# Check field name first
|
||||
if field_name and is_sensitive_field(field_name):
|
||||
return mask_value(value, field_name)
|
||||
|
||||
# Then check string patterns
|
||||
str_val = str(value)
|
||||
redacted = redact_string(str_val)
|
||||
|
||||
# If redaction occurred, return the redacted version
|
||||
if redacted != str_val:
|
||||
return redacted
|
||||
|
||||
return str_val
|
||||
|
||||
|
||||
def create_safe_context(context: Dict[str, Any]) -> Dict[str, Any]:
|
||||
"""Create a safe version of a context dict for logging."""
|
||||
return redact_dict(context)
|
||||
@@ -1,6 +1,8 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
import uuid
|
||||
from contextvars import ContextVar
|
||||
|
||||
import structlog
|
||||
from starlette.requests import Request
|
||||
@@ -9,10 +11,46 @@ from starlette.types import ASGIApp, Receive, Scope, Send
|
||||
|
||||
logger = structlog.get_logger(__name__)
|
||||
|
||||
# Context variable to store correlation ID across async tasks
|
||||
correlation_id_var: ContextVar[str] = ContextVar("correlation_id", default="")
|
||||
|
||||
|
||||
class CorrelationIdMiddleware:
|
||||
"""ASGI middleware that adds/propagates correlation IDs for distributed tracing."""
|
||||
|
||||
def __init__(self, app: ASGIApp) -> None:
|
||||
self.app = app
|
||||
|
||||
async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None:
|
||||
if scope["type"] != "http":
|
||||
await self.app(scope, receive, send)
|
||||
return
|
||||
|
||||
request = Request(scope)
|
||||
|
||||
# Check for existing correlation ID in headers (from upstream proxy/client)
|
||||
correlation_id = request.headers.get(
|
||||
"x-correlation-id",
|
||||
request.headers.get("x-request-id", str(uuid.uuid4()))
|
||||
)
|
||||
|
||||
# Store in context variable for access by handlers
|
||||
correlation_id_var.set(correlation_id)
|
||||
|
||||
# Wrap send to add correlation ID to response headers
|
||||
async def send_wrapper(message: dict) -> None:
|
||||
if message.get("type") == "http.response.start":
|
||||
headers = list(message.get("headers", []))
|
||||
headers.append((b"x-correlation-id", correlation_id.encode()))
|
||||
message["headers"] = headers
|
||||
await send(message)
|
||||
|
||||
await self.app(scope, receive, send_wrapper)
|
||||
|
||||
|
||||
class RequestLoggingMiddleware:
|
||||
"""ASGI middleware that logs every request with method, path, status code,
|
||||
and duration using structlog."""
|
||||
duration, and correlation ID using structlog."""
|
||||
|
||||
def __init__(self, app: ASGIApp) -> None:
|
||||
self.app = app
|
||||
@@ -24,6 +62,7 @@ class RequestLoggingMiddleware:
|
||||
|
||||
start = time.perf_counter()
|
||||
request = Request(scope)
|
||||
correlation_id = correlation_id_var.get()
|
||||
|
||||
# Wrap send to capture the response status code
|
||||
status_code: int | None = None
|
||||
@@ -44,6 +83,7 @@ class RequestLoggingMiddleware:
|
||||
path=request.url.path,
|
||||
status_code=500,
|
||||
duration_ms=round(duration * 1000, 2),
|
||||
correlation_id=correlation_id,
|
||||
)
|
||||
raise
|
||||
else:
|
||||
@@ -55,6 +95,7 @@ class RequestLoggingMiddleware:
|
||||
path=request.url.path,
|
||||
status_code=status_code,
|
||||
duration_ms=round(duration * 1000, 2),
|
||||
correlation_id=correlation_id,
|
||||
)
|
||||
else:
|
||||
logger.info(
|
||||
@@ -63,11 +104,14 @@ class RequestLoggingMiddleware:
|
||||
path=request.url.path,
|
||||
status_code=status_code,
|
||||
duration_ms=round(duration * 1000, 2),
|
||||
correlation_id=correlation_id,
|
||||
)
|
||||
|
||||
|
||||
def register_middleware(app: ASGIApp) -> None:
|
||||
"""Convenience helper — add the middleware to a FastAPI app."""
|
||||
from app.core.middleware import RequestLoggingMiddleware # noqa: F811
|
||||
"""Convenience helper — add middleware to a FastAPI app in correct order."""
|
||||
from app.core.middleware import CorrelationIdMiddleware, RequestLoggingMiddleware # noqa: F811
|
||||
|
||||
# Order matters: CorrelationIdMiddleware first, then RequestLoggingMiddleware
|
||||
app.add_middleware(RequestLoggingMiddleware) # type: ignore[arg-type]
|
||||
app.add_middleware(CorrelationIdMiddleware) # type: ignore[arg-type]
|
||||
|
||||
@@ -0,0 +1,84 @@
|
||||
"""API response validation using Pydantic schemas.
|
||||
|
||||
This module provides strict validation and serialization for all API responses
|
||||
to ensure consistency and catch runtime errors early.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from decimal import Decimal
|
||||
from datetime import datetime
|
||||
from typing import Any, Generic, Optional, TypeVar
|
||||
|
||||
from pydantic import BaseModel, Field, field_validator
|
||||
|
||||
|
||||
T = TypeVar("T")
|
||||
|
||||
|
||||
class APIResponse(BaseModel, Generic[T]):
|
||||
"""Standard API response wrapper with error handling and metadata."""
|
||||
|
||||
success: bool = Field(default=True, description="Whether the request succeeded")
|
||||
data: Optional[T] = Field(default=None, description="Response payload")
|
||||
error: Optional[str] = Field(default=None, description="Error message if failed")
|
||||
error_code: Optional[str] = Field(default=None, description="Structured error code")
|
||||
timestamp: datetime = Field(default_factory=datetime.utcnow, description="Response timestamp")
|
||||
request_id: Optional[str] = Field(default=None, description="Correlation ID for tracing")
|
||||
|
||||
model_config = {"json_encoders": {Decimal: str, datetime: str}}
|
||||
|
||||
@field_validator("error")
|
||||
@classmethod
|
||||
def error_requires_failure(cls, v: Optional[str], info) -> Optional[str]:
|
||||
"""Validate that error is only set when success=False."""
|
||||
if v and info.data.get("success"):
|
||||
raise ValueError("error must be empty when success=True")
|
||||
return v
|
||||
|
||||
|
||||
class PaginatedResponse(BaseModel, Generic[T]):
|
||||
"""Paginated API response with metadata."""
|
||||
|
||||
items: list[T] = Field(description="Page items")
|
||||
total: int = Field(description="Total number of items")
|
||||
page: int = Field(ge=1, description="Current page number")
|
||||
page_size: int = Field(ge=1, le=500, description="Items per page")
|
||||
has_more: bool = Field(description="Whether more items exist")
|
||||
|
||||
@property
|
||||
def total_pages(self) -> int:
|
||||
"""Calculate total pages."""
|
||||
return (self.total + self.page_size - 1) // self.page_size
|
||||
|
||||
|
||||
class HealthCheckResponse(BaseModel):
|
||||
"""Health check response."""
|
||||
|
||||
status: str = Field(description="Service status: healthy, degraded, unhealthy")
|
||||
version: str = Field(description="API version")
|
||||
timestamp: datetime = Field(default_factory=datetime.utcnow)
|
||||
uptime_seconds: float = Field(description="Uptime in seconds")
|
||||
dependencies: dict[str, str] = Field(description="Dependency health status")
|
||||
|
||||
model_config = {"json_encoders": {datetime: str}}
|
||||
|
||||
|
||||
class ValidationError(BaseModel):
|
||||
"""Validation error details."""
|
||||
|
||||
field: str = Field(description="Field that failed validation")
|
||||
message: str = Field(description="Error message")
|
||||
value: Any = Field(description="Value that failed")
|
||||
|
||||
|
||||
class ValidationErrorResponse(BaseModel):
|
||||
"""Response for validation errors."""
|
||||
|
||||
success: bool = Field(default=False)
|
||||
error: str = "Validation failed"
|
||||
error_code: str = "VALIDATION_ERROR"
|
||||
errors: list[ValidationError] = Field(description="List of validation errors")
|
||||
timestamp: datetime = Field(default_factory=datetime.utcnow)
|
||||
|
||||
model_config = {"json_encoders": {datetime: str}}
|
||||
Reference in New Issue
Block a user