rgonsal/comunidadhll
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Complete TASK-054 snapshot prewarm policy

Browse Source
This commit is contained in:
devRaGonSa
2026-03-23 13:52:56 +01:00
parent 28a986f2a7
commit 381010ccb3
7 changed files with 237 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
backend/README.md
View File

@@ -71,6 +71,7 @@ Variables opcionales:
- `HLL_HISTORICAL_CRCON_RETRY_DELAY_SECONDS`
- `HLL_HISTORICAL_REFRESH_INTERVAL_SECONDS`
- `HLL_HISTORICAL_SNAPSHOT_REFRESH_INTERVAL_SECONDS`
- `HLL_HISTORICAL_FULL_SNAPSHOT_EVERY_RUNS`
- `HLL_HISTORICAL_REFRESH_MAX_RETRIES`
- `HLL_HISTORICAL_REFRESH_RETRY_DELAY_SECONDS`
- `HLL_HISTORICAL_WEEKLY_FALLBACK_MIN_MATCHES`
@@ -628,10 +629,15 @@ Los reintentos de cada request JSON pueden ajustarse sin tocar codigo con:
El runner `python -m app.historical_runner` deja ese refresh incremental listo
para ejecucion local repetida sin depender de infraestructura externa y
regenera snapshots historicos precalculados tras cada refresh correcto. Por
defecto:
mantiene calientes los snapshots historicos mas visibles tras cada refresh
correcto. Por defecto:
- refresca y recompone snapshots cada `900` segundos
- refresca cada `900` segundos
- prewarmea en cada ciclo:
- `server-summary` para `comunidad-hispana-01`, `comunidad-hispana-02`, `comunidad-hispana-03` y `all-servers`
- `weekly-leaderboard` de la metrica por defecto `kills` para esos mismos alcances
- `recent-matches` para esos mismos alcances
- recompone la matriz completa de snapshots cada `4` ciclos para mantener el resto de metricas al dia sin penalizar todos los refresh
- reintenta hasta `2` veces tras un fallo
- espera `30` segundos entre reintentos
- reutiliza el registro de `historical_ingestion_runs` para dejar trazabilidad
@@ -652,6 +658,7 @@ Flags utiles del runner:
Variables utiles del runner:
- `HLL_HISTORICAL_SNAPSHOT_REFRESH_INTERVAL_SECONDS`
- `HLL_HISTORICAL_FULL_SNAPSHOT_EVERY_RUNS`
- `HLL_HISTORICAL_REFRESH_MAX_RETRIES`
- `HLL_HISTORICAL_REFRESH_RETRY_DELAY_SECONDS`
- `HLL_HISTORICAL_WEEKLY_FALLBACK_MIN_MATCHES`

14
backend/app/config.py
View File

@@ -19,6 +19,7 @@ DEFAULT_HISTORICAL_REFRESH_INTERVAL_SECONDS = 1800
DEFAULT_HISTORICAL_SNAPSHOT_REFRESH_INTERVAL_SECONDS = 900
DEFAULT_HISTORICAL_REFRESH_MAX_RETRIES = 2
DEFAULT_HISTORICAL_REFRESH_RETRY_DELAY_SECONDS = 30
DEFAULT_HISTORICAL_FULL_SNAPSHOT_EVERY_RUNS = 4
DEFAULT_HISTORICAL_WEEKLY_FALLBACK_MIN_MATCHES = 3
DEFAULT_HISTORICAL_WEEKLY_FALLBACK_MAX_WEEKDAY = 2
DEFAULT_ALLOWED_ORIGINS = (
@@ -191,6 +192,19 @@ def get_historical_refresh_retry_delay_seconds() -> int:
return retry_delay_seconds
def get_historical_full_snapshot_every_runs() -> int:
"""Return how often the runner should rebuild the full snapshot matrix."""
configured_value = os.getenv(
"HLL_HISTORICAL_FULL_SNAPSHOT_EVERY_RUNS",
str(DEFAULT_HISTORICAL_FULL_SNAPSHOT_EVERY_RUNS),
)
run_count = int(configured_value)
if run_count <= 0:
raise ValueError("HLL_HISTORICAL_FULL_SNAPSHOT_EVERY_RUNS must be positive.")
return run_count
def get_historical_weekly_fallback_min_matches() -> int:
"""Return the minimum closed matches required to trust the current week."""
configured_value = os.getenv(

14
backend/app/historical_ingestion.py
View File

@@ -65,6 +65,7 @@ def run_bootstrap(
page_size: int | None = None,
start_page: int | None = None,
detail_workers: int | None = None,
rebuild_snapshots: bool = True,
) -> dict[str, object]:
"""Run a first full historical import against one or all configured servers."""
return _run_ingestion(
@@ -75,6 +76,7 @@ def run_bootstrap(
start_page=start_page,
detail_workers=detail_workers,
incremental=False,
rebuild_snapshots=rebuild_snapshots,
)
@@ -85,6 +87,7 @@ def run_incremental_refresh(
page_size: int | None = None,
start_page: int | None = None,
detail_workers: int | None = None,
rebuild_snapshots: bool = True,
) -> dict[str, object]:
"""Refresh recent historical pages without replaying the whole archive."""
return _run_ingestion(
@@ -95,6 +98,7 @@ def run_incremental_refresh(
start_page=start_page,
detail_workers=detail_workers,
incremental=True,
rebuild_snapshots=rebuild_snapshots,
)
@@ -107,6 +111,7 @@ def _run_ingestion(
start_page: int | None,
detail_workers: int | None,
incremental: bool,
rebuild_snapshots: bool,
) -> dict[str, object]:
initialize_historical_storage()
stats = IngestionStats()
@@ -164,7 +169,14 @@ def _run_ingestion(
archive_exhausted=bool(server_stats["archive_exhausted"]),
)
active_runs.pop(str(server["slug"]), None)
snapshot_result = generate_and_persist_historical_snapshots(server_key=server_slug)
if rebuild_snapshots:
snapshot_result = generate_and_persist_historical_snapshots(server_key=server_slug)
else:
snapshot_result = {
"status": "skipped",
"reason": "snapshot-rebuild-disabled",
"generation_policy": "handled-by-caller",
}
except Exception as exc:
for active_server_slug, run_id in active_runs.items():
finalize_ingestion_run(

36
backend/app/historical_runner.py
View File

@@ -9,12 +9,16 @@ from datetime import datetime, timezone
from typing import Any
from .config import (
get_historical_full_snapshot_every_runs,
get_historical_refresh_interval_seconds,
get_historical_refresh_max_retries,
get_historical_refresh_retry_delay_seconds,
)
from .historical_ingestion import run_incremental_refresh
from .historical_snapshots import generate_and_persist_historical_snapshots
from .historical_snapshots import (
generate_and_persist_historical_snapshots,
generate_and_persist_priority_historical_snapshots,
)
def run_periodic_historical_refresh(
@@ -44,6 +48,7 @@ def run_periodic_historical_refresh(
server_slug=server_slug,
max_pages=max_pages,
page_size=page_size,
run_number=completed_runs,
)
print(json.dumps({"run": completed_runs, **payload}, indent=2))
@@ -62,6 +67,7 @@ def _run_refresh_with_retries(
server_slug: str | None,
max_pages: int | None,
page_size: int | None,
run_number: int,
) -> dict[str, Any]:
attempt = 0
while True:
@@ -71,8 +77,12 @@ def _run_refresh_with_retries(
server_slug=server_slug,
max_pages=max_pages,
page_size=page_size,
rebuild_snapshots=False,
)
snapshot_result = generate_historical_snapshots(
server_slug=server_slug,
run_number=run_number,
)
snapshot_result = generate_historical_snapshots(server_slug=server_slug)
return {
"status": "ok",
"attempts_used": attempt,
@@ -95,14 +105,26 @@ def _run_refresh_with_retries(
def generate_historical_snapshots(
*,
server_slug: str | None = None,
run_number: int = 1,
) -> dict[str, Any]:
"""Build and persist precomputed snapshots for one server or all servers."""
result = generate_and_persist_historical_snapshots(
server_key=server_slug,
generated_at=datetime.now(timezone.utc),
)
"""Build priority prewarm snapshots on every run and the full matrix on cadence."""
generated_at = datetime.now(timezone.utc)
full_snapshot_every_runs = get_historical_full_snapshot_every_runs()
should_run_full_refresh = bool(server_slug) or run_number % full_snapshot_every_runs == 0
if should_run_full_refresh:
result = generate_and_persist_historical_snapshots(
server_key=server_slug,
generated_at=generated_at,
)
else:
result = generate_and_persist_priority_historical_snapshots(
generated_at=generated_at,
)
return {
**result,
"run_number": run_number,
"full_snapshot_every_runs": full_snapshot_every_runs,
"prewarm_only": not should_run_full_refresh,
"refresh_interval_seconds": get_historical_refresh_interval_seconds(),
}

81
backend/app/historical_snapshots.py
View File

@@ -33,6 +33,13 @@ SUPPORTED_LEADERBOARD_METRICS = frozenset(
"matches_over_100_kills",
}
)
PREWARM_SNAPSHOT_SERVER_KEYS = (
"comunidad-hispana-01",
"comunidad-hispana-02",
"comunidad-hispana-03",
ALL_SERVERS_SLUG,
)
PREWARM_LEADERBOARD_METRICS = ("kills",)
SNAPSHOT_LEADERBOARD_METRICS = (
"kills",
"deaths",
@@ -109,6 +116,42 @@ def build_historical_server_snapshots(
return snapshots
def build_priority_historical_snapshots(
*,
server_keys: tuple[str, ...] = PREWARM_SNAPSHOT_SERVER_KEYS,
generated_at: datetime | None = None,
leaderboard_limit: int = DEFAULT_WEEKLY_LEADERBOARD_LIMIT,
recent_matches_limit: int = DEFAULT_RECENT_MATCHES_LIMIT,
db_path: Path | None = None,
) -> list[dict[str, object]]:
"""Build the minimum warm snapshot set required by the historical UI."""
generated_at_value = _as_utc(generated_at or datetime.now(timezone.utc))
snapshots: list[dict[str, object]] = []
for server_key in server_keys:
snapshots.append(
_build_server_summary_snapshot(server_key, generated_at_value, db_path=db_path)
)
for metric in PREWARM_LEADERBOARD_METRICS:
snapshots.append(
_build_weekly_leaderboard_snapshot(
server_key,
metric,
generated_at_value,
limit=leaderboard_limit,
db_path=db_path,
)
)
snapshots.append(
_build_recent_matches_snapshot(
server_key,
generated_at_value,
limit=recent_matches_limit,
db_path=db_path,
)
)
return snapshots
def build_all_historical_snapshots(
*,
server_key: str | None = None,
@@ -129,7 +172,7 @@ def build_all_historical_snapshots(
recent_matches_limit=recent_matches_limit,
db_path=db_path,
)
)
)
return snapshots
@@ -161,6 +204,42 @@ def generate_and_persist_historical_snapshots(
return {
"generated_at": _to_iso(generated_at_value),
"server_slug": server_key,
"snapshot_policy": "full-matrix",
"snapshot_count": len(persisted_records),
"servers_processed": len(snapshots_by_server),
"snapshots_by_server": snapshots_by_server,
}
def generate_and_persist_priority_historical_snapshots(
*,
generated_at: datetime | None = None,
leaderboard_limit: int = DEFAULT_WEEKLY_LEADERBOARD_LIMIT,
recent_matches_limit: int = DEFAULT_RECENT_MATCHES_LIMIT,
db_path: Path | None = None,
) -> dict[str, object]:
"""Build and persist the priority snapshot set used for prewarm."""
from .historical_snapshot_storage import persist_historical_snapshot_batch
generated_at_value = _as_utc(generated_at or datetime.now(timezone.utc))
snapshots = build_priority_historical_snapshots(
generated_at=generated_at_value,
leaderboard_limit=leaderboard_limit,
recent_matches_limit=recent_matches_limit,
db_path=db_path,
)
persisted_records = persist_historical_snapshot_batch(snapshots, db_path=db_path)
snapshots_by_server: dict[str, int] = {}
for record in persisted_records:
snapshots_by_server.setdefault(record.server_key, 0)
snapshots_by_server[record.server_key] += 1
return {
"generated_at": _to_iso(generated_at_value),
"server_slug": None,
"snapshot_policy": "priority-prewarm",
"prewarm_server_keys": list(PREWARM_SNAPSHOT_SERVER_KEYS),
"prewarm_metrics": list(PREWARM_LEADERBOARD_METRICS),
"snapshot_count": len(persisted_records),
"servers_processed": len(snapshots_by_server),
"snapshots_by_server": snapshots_by_server,