From 381010ccb3b09dc6eb619b3363ac9bce64a7d81c Mon Sep 17 00:00:00 2001 From: devRaGonSa Date: Mon, 23 Mar 2026 13:52:56 +0100 Subject: [PATCH] Complete TASK-054 snapshot prewarm policy --- ...ical-snapshots-prewarm-and-cache-policy.md | 74 +++++++++++++++++ backend/README.md | 13 ++- backend/app/config.py | 14 ++++ backend/app/historical_ingestion.py | 14 +++- backend/app/historical_runner.py | 36 +++++++-- backend/app/historical_snapshots.py | 81 ++++++++++++++++++- frontend/assets/js/historico.js | 22 +++-- 7 files changed, 237 insertions(+), 17 deletions(-) create mode 100644 ai/tasks/done/TASK-054-historical-snapshots-prewarm-and-cache-policy.md diff --git a/ai/tasks/done/TASK-054-historical-snapshots-prewarm-and-cache-policy.md b/ai/tasks/done/TASK-054-historical-snapshots-prewarm-and-cache-policy.md new file mode 100644 index 0000000..dce1896 --- /dev/null +++ b/ai/tasks/done/TASK-054-historical-snapshots-prewarm-and-cache-policy.md @@ -0,0 +1,74 @@ +# TASK-054-historical-snapshots-prewarm-and-cache-policy + +## Goal +Preparar una política operativa de prewarm y refresco de snapshots históricos para que los servidores y métricas más visibles ya estén listos antes de que el usuario abra la página, reduciendo tiempos de espera al cambiar de servidor o pestaña. + +## Context +Aunque la UI deje de precargar agresivamente en el navegador, sigue siendo útil que el backend mantenga calientes y listos los snapshots más importantes. Esto debe hacerse de forma controlada y periódica, sin volver a un modelo de recomposición bajo demanda. Además, conviene definir mejor la política de caché frontend para que ayude sin dejar resultados vacíos obsoletos demasiado tiempo. + +## Steps +1. Revisar la generación actual de snapshots y el runner periódico. +2. Definir qué snapshots deben considerarse prioritarios para prewarm, como mínimo: + - server-summary para #01, #02, #03 y all-servers + - leaderboard semanal de la métrica por defecto para #01, #02, #03 y all-servers + - recent-matches para #01, #02, #03 y all-servers +3. Diseñar e implementar una estrategia de prewarm ligera y razonable para esos snapshots prioritarios. +4. Mantener la generación de otras métricas de leaderboard de forma periódica o diferida, según convenga, pero sin penalizar la primera carga de la UI. +5. Revisar y ajustar la política de caché frontend para: + - aprovechar respuestas recientes + - no mantener indefinidamente respuestas vacías antiguas + - permitir que el usuario vea mejoras sin necesidad de comportamientos raros +6. Documentar la estrategia operativa de prewarm y caché. +7. No convertir esta task en una optimización prematura excesiva; mantener el alcance claro y pragmático. +8. Al completar la implementación: + - dejar el repositorio consistente + - hacer commit + - hacer push al remoto si el entorno lo permite + +## Files to Read First +- AGENTS.md +- backend/README.md +- backend/app/config.py +- backend/app/historical_runner.py +- backend/app/historical_snapshots.py +- backend/app/historical_snapshot_storage.py +- frontend/assets/js/historico.js +- docs/historical-coverage-report.md + +## Expected Files to Modify +- backend/app/config.py +- backend/app/historical_runner.py +- backend/app/historical_snapshots.py +- frontend/assets/js/historico.js +- backend/README.md +- opcionalmente documentación técnica adicional si ayuda a dejar la política operativa clara + +## Constraints +- No volver a recomposición pesada en request path. +- No crear páginas nuevas. +- No romper la UI histórica existente. +- No introducir frameworks nuevos. +- No hacer cambios destructivos. +- Mantener el trabajo centrado en snapshots listos antes de la interacción del usuario. + +## Validation +- Existe una estrategia clara de prewarm de snapshots prioritarios. +- Cambiar de servidor o entrar en la página es más rápido que antes. +- La política de caché frontend es más sana y coherente con el uso real. +- La documentación refleja la nueva operativa. +- Los cambios quedan committeados y se hace push si el entorno lo permite. + +## Change Budget +- Preferir menos de 5 archivos modificados o creados. +- Preferir menos de 220 líneas cambiadas. + +## Outcome +- El runner histórico ya no duplica una recomposición completa de snapshots tras cada refresh incremental. +- La operativa periódica hace prewarm en cada ciclo para `server-summary`, `weekly-leaderboard` de `kills` y `recent-matches` en `comunidad-hispana-01`, `comunidad-hispana-02`, `comunidad-hispana-03` y `all-servers`. +- La matriz completa de métricas queda en una cadencia separada configurable con `HLL_HISTORICAL_FULL_SNAPSHOT_EVERY_RUNS`, evitando penalizar cada refresh. +- La caché frontend ahora distingue snapshots frescos, stale y missing para conservar respuestas útiles más tiempo sin retener vacíos antiguos indefinidamente. +- Validación local: +- `py_compile` sobre `config.py`, `historical_snapshots.py`, `historical_ingestion.py`, `historical_runner.py` y `payloads.py` +- `node --check frontend/assets/js/historico.js` +- validación funcional con SQLite temporal: `priority-prewarm -> 12 snapshots` y `full-matrix -> 6 snapshots` para un servidor +- la generación contra el dataset histórico real superó el timeout local de la consola, por lo que la validación operativa completa quedó acotada a la ruta temporal diff --git a/backend/README.md b/backend/README.md index 163e66b..a84a584 100644 --- a/backend/README.md +++ b/backend/README.md @@ -71,6 +71,7 @@ Variables opcionales: - `HLL_HISTORICAL_CRCON_RETRY_DELAY_SECONDS` - `HLL_HISTORICAL_REFRESH_INTERVAL_SECONDS` - `HLL_HISTORICAL_SNAPSHOT_REFRESH_INTERVAL_SECONDS` +- `HLL_HISTORICAL_FULL_SNAPSHOT_EVERY_RUNS` - `HLL_HISTORICAL_REFRESH_MAX_RETRIES` - `HLL_HISTORICAL_REFRESH_RETRY_DELAY_SECONDS` - `HLL_HISTORICAL_WEEKLY_FALLBACK_MIN_MATCHES` @@ -628,10 +629,15 @@ Los reintentos de cada request JSON pueden ajustarse sin tocar codigo con: El runner `python -m app.historical_runner` deja ese refresh incremental listo para ejecucion local repetida sin depender de infraestructura externa y -regenera snapshots historicos precalculados tras cada refresh correcto. Por -defecto: +mantiene calientes los snapshots historicos mas visibles tras cada refresh +correcto. Por defecto: -- refresca y recompone snapshots cada `900` segundos +- refresca cada `900` segundos +- prewarmea en cada ciclo: + - `server-summary` para `comunidad-hispana-01`, `comunidad-hispana-02`, `comunidad-hispana-03` y `all-servers` + - `weekly-leaderboard` de la metrica por defecto `kills` para esos mismos alcances + - `recent-matches` para esos mismos alcances +- recompone la matriz completa de snapshots cada `4` ciclos para mantener el resto de metricas al dia sin penalizar todos los refresh - reintenta hasta `2` veces tras un fallo - espera `30` segundos entre reintentos - reutiliza el registro de `historical_ingestion_runs` para dejar trazabilidad @@ -652,6 +658,7 @@ Flags utiles del runner: Variables utiles del runner: - `HLL_HISTORICAL_SNAPSHOT_REFRESH_INTERVAL_SECONDS` +- `HLL_HISTORICAL_FULL_SNAPSHOT_EVERY_RUNS` - `HLL_HISTORICAL_REFRESH_MAX_RETRIES` - `HLL_HISTORICAL_REFRESH_RETRY_DELAY_SECONDS` - `HLL_HISTORICAL_WEEKLY_FALLBACK_MIN_MATCHES` diff --git a/backend/app/config.py b/backend/app/config.py index 0e443c4..1c8c206 100644 --- a/backend/app/config.py +++ b/backend/app/config.py @@ -19,6 +19,7 @@ DEFAULT_HISTORICAL_REFRESH_INTERVAL_SECONDS = 1800 DEFAULT_HISTORICAL_SNAPSHOT_REFRESH_INTERVAL_SECONDS = 900 DEFAULT_HISTORICAL_REFRESH_MAX_RETRIES = 2 DEFAULT_HISTORICAL_REFRESH_RETRY_DELAY_SECONDS = 30 +DEFAULT_HISTORICAL_FULL_SNAPSHOT_EVERY_RUNS = 4 DEFAULT_HISTORICAL_WEEKLY_FALLBACK_MIN_MATCHES = 3 DEFAULT_HISTORICAL_WEEKLY_FALLBACK_MAX_WEEKDAY = 2 DEFAULT_ALLOWED_ORIGINS = ( @@ -191,6 +192,19 @@ def get_historical_refresh_retry_delay_seconds() -> int: return retry_delay_seconds +def get_historical_full_snapshot_every_runs() -> int: + """Return how often the runner should rebuild the full snapshot matrix.""" + configured_value = os.getenv( + "HLL_HISTORICAL_FULL_SNAPSHOT_EVERY_RUNS", + str(DEFAULT_HISTORICAL_FULL_SNAPSHOT_EVERY_RUNS), + ) + run_count = int(configured_value) + if run_count <= 0: + raise ValueError("HLL_HISTORICAL_FULL_SNAPSHOT_EVERY_RUNS must be positive.") + + return run_count + + def get_historical_weekly_fallback_min_matches() -> int: """Return the minimum closed matches required to trust the current week.""" configured_value = os.getenv( diff --git a/backend/app/historical_ingestion.py b/backend/app/historical_ingestion.py index 8b2d702..59b104a 100644 --- a/backend/app/historical_ingestion.py +++ b/backend/app/historical_ingestion.py @@ -65,6 +65,7 @@ def run_bootstrap( page_size: int | None = None, start_page: int | None = None, detail_workers: int | None = None, + rebuild_snapshots: bool = True, ) -> dict[str, object]: """Run a first full historical import against one or all configured servers.""" return _run_ingestion( @@ -75,6 +76,7 @@ def run_bootstrap( start_page=start_page, detail_workers=detail_workers, incremental=False, + rebuild_snapshots=rebuild_snapshots, ) @@ -85,6 +87,7 @@ def run_incremental_refresh( page_size: int | None = None, start_page: int | None = None, detail_workers: int | None = None, + rebuild_snapshots: bool = True, ) -> dict[str, object]: """Refresh recent historical pages without replaying the whole archive.""" return _run_ingestion( @@ -95,6 +98,7 @@ def run_incremental_refresh( start_page=start_page, detail_workers=detail_workers, incremental=True, + rebuild_snapshots=rebuild_snapshots, ) @@ -107,6 +111,7 @@ def _run_ingestion( start_page: int | None, detail_workers: int | None, incremental: bool, + rebuild_snapshots: bool, ) -> dict[str, object]: initialize_historical_storage() stats = IngestionStats() @@ -164,7 +169,14 @@ def _run_ingestion( archive_exhausted=bool(server_stats["archive_exhausted"]), ) active_runs.pop(str(server["slug"]), None) - snapshot_result = generate_and_persist_historical_snapshots(server_key=server_slug) + if rebuild_snapshots: + snapshot_result = generate_and_persist_historical_snapshots(server_key=server_slug) + else: + snapshot_result = { + "status": "skipped", + "reason": "snapshot-rebuild-disabled", + "generation_policy": "handled-by-caller", + } except Exception as exc: for active_server_slug, run_id in active_runs.items(): finalize_ingestion_run( diff --git a/backend/app/historical_runner.py b/backend/app/historical_runner.py index dfd597d..d25e215 100644 --- a/backend/app/historical_runner.py +++ b/backend/app/historical_runner.py @@ -9,12 +9,16 @@ from datetime import datetime, timezone from typing import Any from .config import ( + get_historical_full_snapshot_every_runs, get_historical_refresh_interval_seconds, get_historical_refresh_max_retries, get_historical_refresh_retry_delay_seconds, ) from .historical_ingestion import run_incremental_refresh -from .historical_snapshots import generate_and_persist_historical_snapshots +from .historical_snapshots import ( + generate_and_persist_historical_snapshots, + generate_and_persist_priority_historical_snapshots, +) def run_periodic_historical_refresh( @@ -44,6 +48,7 @@ def run_periodic_historical_refresh( server_slug=server_slug, max_pages=max_pages, page_size=page_size, + run_number=completed_runs, ) print(json.dumps({"run": completed_runs, **payload}, indent=2)) @@ -62,6 +67,7 @@ def _run_refresh_with_retries( server_slug: str | None, max_pages: int | None, page_size: int | None, + run_number: int, ) -> dict[str, Any]: attempt = 0 while True: @@ -71,8 +77,12 @@ def _run_refresh_with_retries( server_slug=server_slug, max_pages=max_pages, page_size=page_size, + rebuild_snapshots=False, + ) + snapshot_result = generate_historical_snapshots( + server_slug=server_slug, + run_number=run_number, ) - snapshot_result = generate_historical_snapshots(server_slug=server_slug) return { "status": "ok", "attempts_used": attempt, @@ -95,14 +105,26 @@ def _run_refresh_with_retries( def generate_historical_snapshots( *, server_slug: str | None = None, + run_number: int = 1, ) -> dict[str, Any]: - """Build and persist precomputed snapshots for one server or all servers.""" - result = generate_and_persist_historical_snapshots( - server_key=server_slug, - generated_at=datetime.now(timezone.utc), - ) + """Build priority prewarm snapshots on every run and the full matrix on cadence.""" + generated_at = datetime.now(timezone.utc) + full_snapshot_every_runs = get_historical_full_snapshot_every_runs() + should_run_full_refresh = bool(server_slug) or run_number % full_snapshot_every_runs == 0 + if should_run_full_refresh: + result = generate_and_persist_historical_snapshots( + server_key=server_slug, + generated_at=generated_at, + ) + else: + result = generate_and_persist_priority_historical_snapshots( + generated_at=generated_at, + ) return { **result, + "run_number": run_number, + "full_snapshot_every_runs": full_snapshot_every_runs, + "prewarm_only": not should_run_full_refresh, "refresh_interval_seconds": get_historical_refresh_interval_seconds(), } diff --git a/backend/app/historical_snapshots.py b/backend/app/historical_snapshots.py index ed1d715..92900a2 100644 --- a/backend/app/historical_snapshots.py +++ b/backend/app/historical_snapshots.py @@ -33,6 +33,13 @@ SUPPORTED_LEADERBOARD_METRICS = frozenset( "matches_over_100_kills", } ) +PREWARM_SNAPSHOT_SERVER_KEYS = ( + "comunidad-hispana-01", + "comunidad-hispana-02", + "comunidad-hispana-03", + ALL_SERVERS_SLUG, +) +PREWARM_LEADERBOARD_METRICS = ("kills",) SNAPSHOT_LEADERBOARD_METRICS = ( "kills", "deaths", @@ -109,6 +116,42 @@ def build_historical_server_snapshots( return snapshots +def build_priority_historical_snapshots( + *, + server_keys: tuple[str, ...] = PREWARM_SNAPSHOT_SERVER_KEYS, + generated_at: datetime | None = None, + leaderboard_limit: int = DEFAULT_WEEKLY_LEADERBOARD_LIMIT, + recent_matches_limit: int = DEFAULT_RECENT_MATCHES_LIMIT, + db_path: Path | None = None, +) -> list[dict[str, object]]: + """Build the minimum warm snapshot set required by the historical UI.""" + generated_at_value = _as_utc(generated_at or datetime.now(timezone.utc)) + snapshots: list[dict[str, object]] = [] + for server_key in server_keys: + snapshots.append( + _build_server_summary_snapshot(server_key, generated_at_value, db_path=db_path) + ) + for metric in PREWARM_LEADERBOARD_METRICS: + snapshots.append( + _build_weekly_leaderboard_snapshot( + server_key, + metric, + generated_at_value, + limit=leaderboard_limit, + db_path=db_path, + ) + ) + snapshots.append( + _build_recent_matches_snapshot( + server_key, + generated_at_value, + limit=recent_matches_limit, + db_path=db_path, + ) + ) + return snapshots + + def build_all_historical_snapshots( *, server_key: str | None = None, @@ -129,7 +172,7 @@ def build_all_historical_snapshots( recent_matches_limit=recent_matches_limit, db_path=db_path, ) - ) + ) return snapshots @@ -161,6 +204,42 @@ def generate_and_persist_historical_snapshots( return { "generated_at": _to_iso(generated_at_value), "server_slug": server_key, + "snapshot_policy": "full-matrix", + "snapshot_count": len(persisted_records), + "servers_processed": len(snapshots_by_server), + "snapshots_by_server": snapshots_by_server, + } + + +def generate_and_persist_priority_historical_snapshots( + *, + generated_at: datetime | None = None, + leaderboard_limit: int = DEFAULT_WEEKLY_LEADERBOARD_LIMIT, + recent_matches_limit: int = DEFAULT_RECENT_MATCHES_LIMIT, + db_path: Path | None = None, +) -> dict[str, object]: + """Build and persist the priority snapshot set used for prewarm.""" + from .historical_snapshot_storage import persist_historical_snapshot_batch + + generated_at_value = _as_utc(generated_at or datetime.now(timezone.utc)) + snapshots = build_priority_historical_snapshots( + generated_at=generated_at_value, + leaderboard_limit=leaderboard_limit, + recent_matches_limit=recent_matches_limit, + db_path=db_path, + ) + persisted_records = persist_historical_snapshot_batch(snapshots, db_path=db_path) + snapshots_by_server: dict[str, int] = {} + for record in persisted_records: + snapshots_by_server.setdefault(record.server_key, 0) + snapshots_by_server[record.server_key] += 1 + + return { + "generated_at": _to_iso(generated_at_value), + "server_slug": None, + "snapshot_policy": "priority-prewarm", + "prewarm_server_keys": list(PREWARM_SNAPSHOT_SERVER_KEYS), + "prewarm_metrics": list(PREWARM_LEADERBOARD_METRICS), "snapshot_count": len(persisted_records), "servers_processed": len(snapshots_by_server), "snapshots_by_server": snapshots_by_server, diff --git a/frontend/assets/js/historico.js b/frontend/assets/js/historico.js index 1c3ad4b..5197a96 100644 --- a/frontend/assets/js/historico.js +++ b/frontend/assets/js/historico.js @@ -20,8 +20,9 @@ const HISTORICAL_SERVER_SLUGS = Object.freeze( HISTORICAL_SERVERS.map((server) => server.slug), ); const DEFAULT_HISTORICAL_SERVER = HISTORICAL_SERVER_SLUGS[0]; -const SNAPSHOT_CACHE_TTL_MS = 60000; -const NEGATIVE_SNAPSHOT_CACHE_TTL_MS = 5000; +const SNAPSHOT_CACHE_TTL_MS = 120000; +const STALE_SNAPSHOT_CACHE_TTL_MS = 30000; +const NEGATIVE_SNAPSHOT_CACHE_TTL_MS = 15000; const LEADERBOARD_METRICS = Object.freeze([ { key: "kills", @@ -847,9 +848,20 @@ function writeCachedPayload(cache, key, payload) { } function resolveSnapshotCacheTtl(payload) { - return payload?.data?.found === false - ? NEGATIVE_SNAPSHOT_CACHE_TTL_MS - : SNAPSHOT_CACHE_TTL_MS; + const data = payload?.data; + if (!data) { + return NEGATIVE_SNAPSHOT_CACHE_TTL_MS; + } + + if (data.snapshot_status === "missing" || data.found === false) { + return NEGATIVE_SNAPSHOT_CACHE_TTL_MS; + } + + if (data.is_stale) { + return STALE_SNAPSHOT_CACHE_TTL_MS; + } + + return SNAPSHOT_CACHE_TTL_MS; } async function settlePromise(promise) {