Complete historical snapshot generation and weekly fallback

2026-03-23 12:35:47 +01:00
parent 14820888a6
commit 1ea1d4b3c6
9 changed files with 372 additions and 37 deletions
--- a/backend/README.md
+++ b/backend/README.md
@@ -73,6 +73,8 @@ Variables opcionales:
 - `HLL_HISTORICAL_SNAPSHOT_REFRESH_INTERVAL_SECONDS`
 - `HLL_HISTORICAL_REFRESH_MAX_RETRIES`
 - `HLL_HISTORICAL_REFRESH_RETRY_DELAY_SECONDS`
+- `HLL_HISTORICAL_WEEKLY_FALLBACK_MIN_MATCHES`
+- `HLL_HISTORICAL_WEEKLY_FALLBACK_MAX_WEEKDAY`

 El `frontend/index.html` viene preparado para volver a consultar el bloque de
 servidores cada `120000` ms (`120s`) sin recargar la pagina completa. La landing
@@ -476,15 +478,16 @@ Parametros opcionales:
 - `player` en `/api/historical/player-profile` aceptando `stable_player_key`,
  `steam_id` o `source_player_id`

-La ventana temporal es siempre la ultima semana movil respecto al momento de la
-request y solo considera partidas cerradas con `ended_at` para no mezclar
-partidas aun en curso ni filas historicas transitorias. El payload devuelve
-servidor, rango temporal, jugador, kills semanales, posicion y numero de
-partidas consideradas.
+La ventana temporal usa semana calendario UTC y solo considera partidas
+cerradas con `ended_at` para no mezclar partidas aun en curso ni filas
+historicas transitorias. El payload devuelve servidor, rango temporal,
+jugador, kills semanales, posicion y numero de partidas consideradas.

 `weekly-leaderboard` generaliza ese bloque para varias metricas semanales por
-servidor usando la misma ventana movil de 7 dias y el mismo filtro de partidas
-cerradas. Metricas soportadas:
+servidor usando el mismo filtro de partidas cerradas. Si la semana actual cae
+entre lunes y miercoles UTC y todavia no acumula al menos `3` partidas
+cerradas, el backend activa un fallback temporal a la semana cerrada anterior.
+Metricas soportadas:

 - `kills`
 - `deaths`
@@ -509,6 +512,15 @@ request. Estos endpoints devuelven payloads ligeros listos para frontend con:
 - `is_stale`
 - `freshness`
 - `found`
+- `window_start`
+- `window_end`
+- `window_kind`
+- `window_label`
+- `uses_fallback`
+- `selection_reason`
+- `current_week_closed_matches`
+- `previous_week_closed_matches`
+- `sufficient_sample`

 `/api/historical/snapshots/server-summary` devuelve `item` con el resumen del
 servidor. `/api/historical/snapshots/weekly-leaderboard` devuelve `items` ya
@@ -547,7 +559,9 @@ Flags utiles:
 La ejecucion `bootstrap` recorre paginas historicas hasta agotar resultados.
 La ejecucion `refresh` usa una ventana de solape sobre la ultima partida
 persistida por servidor para releer solo paginas recientes y absorber updates
-tardios sin reimportar todo el historico.
+tardios sin reimportar todo el historico. Cuando una ejecucion termina
+correctamente, tambien recompone los snapshots historicos precalculados para el
+servidor afectado o para todos los servidores si la ingesta fue global.

 El comando devuelve ademas un resumen de cobertura persistida por servidor. Esto
 ayuda a validar rapidamente cuantos matches reales quedaron importados, el rango
@@ -610,6 +624,8 @@ Variables utiles del runner:
 - `HLL_HISTORICAL_SNAPSHOT_REFRESH_INTERVAL_SECONDS`
 - `HLL_HISTORICAL_REFRESH_MAX_RETRIES`
 - `HLL_HISTORICAL_REFRESH_RETRY_DELAY_SECONDS`
+- `HLL_HISTORICAL_WEEKLY_FALLBACK_MIN_MATCHES`
+- `HLL_HISTORICAL_WEEKLY_FALLBACK_MAX_WEEKDAY`

 Al inicializar la persistencia local, el backend normaliza tambien la identidad
 historica ya guardada:
--- a/backend/app/config.py
+++ b/backend/app/config.py
@@ -19,6 +19,8 @@ DEFAULT_HISTORICAL_REFRESH_INTERVAL_SECONDS = 1800
 DEFAULT_HISTORICAL_SNAPSHOT_REFRESH_INTERVAL_SECONDS = 900
 DEFAULT_HISTORICAL_REFRESH_MAX_RETRIES = 2
 DEFAULT_HISTORICAL_REFRESH_RETRY_DELAY_SECONDS = 30
+DEFAULT_HISTORICAL_WEEKLY_FALLBACK_MIN_MATCHES = 3
+DEFAULT_HISTORICAL_WEEKLY_FALLBACK_MAX_WEEKDAY = 2
 DEFAULT_ALLOWED_ORIGINS = (
    "null",
    "http://127.0.0.1:5500",
@@ -189,6 +191,32 @@ def get_historical_refresh_retry_delay_seconds() -> int:
    return retry_delay_seconds


+def get_historical_weekly_fallback_min_matches() -> int:
+    """Return the minimum closed matches required to trust the current week."""
+    configured_value = os.getenv(
+        "HLL_HISTORICAL_WEEKLY_FALLBACK_MIN_MATCHES",
+        str(DEFAULT_HISTORICAL_WEEKLY_FALLBACK_MIN_MATCHES),
+    )
+    min_matches = int(configured_value)
+    if min_matches <= 0:
+        raise ValueError("HLL_HISTORICAL_WEEKLY_FALLBACK_MIN_MATCHES must be positive.")
+
+    return min_matches
+
+
+def get_historical_weekly_fallback_max_weekday() -> int:
+    """Return the last weekday index where weekly fallback may still apply."""
+    configured_value = os.getenv(
+        "HLL_HISTORICAL_WEEKLY_FALLBACK_MAX_WEEKDAY",
+        str(DEFAULT_HISTORICAL_WEEKLY_FALLBACK_MAX_WEEKDAY),
+    )
+    max_weekday = int(configured_value)
+    if max_weekday < 0 or max_weekday > 6:
+        raise ValueError("HLL_HISTORICAL_WEEKLY_FALLBACK_MAX_WEEKDAY must be between 0 and 6.")
+
+    return max_weekday
+
+
 def get_a2s_targets_payload() -> str | None:
    """Return the optional JSON payload that overrides local A2S targets."""
    raw_payload = os.getenv(DEFAULT_A2S_TARGETS_ENV_VAR)
--- a/backend/app/historical_ingestion.py
+++ b/backend/app/historical_ingestion.py
@@ -19,6 +19,7 @@ from .config import (
    get_historical_crcon_request_timeout_seconds,
    get_historical_crcon_retry_delay_seconds,
 )
+from .historical_snapshots import generate_and_persist_historical_snapshots
 from .historical_storage import (
    finalize_backfill_progress,
    finalize_ingestion_run,
@@ -163,6 +164,7 @@ def _run_ingestion(
                archive_exhausted=bool(server_stats["archive_exhausted"]),
            )
            active_runs.pop(str(server["slug"]), None)
+        snapshot_result = generate_and_persist_historical_snapshots(server_key=server_slug)
    except Exception as exc:
        for active_server_slug, run_id in active_runs.items():
            finalize_ingestion_run(
@@ -193,6 +195,7 @@ def _run_ingestion(
        "detail_workers": detail_workers or get_historical_crcon_detail_workers(),
        "servers": processed_servers,
        "coverage": list_historical_coverage_report(server_slug=server_slug),
+        "snapshot_result": snapshot_result,
        "totals": {
            "pages_processed": stats.pages_processed,
            "matches_seen": stats.matches_seen,
--- a/backend/app/historical_runner.py
+++ b/backend/app/historical_runner.py
@@ -14,8 +14,7 @@ from .config import (
    get_historical_refresh_retry_delay_seconds,
 )
 from .historical_ingestion import run_incremental_refresh
-from .historical_snapshot_storage import persist_historical_snapshot_batch
-from .historical_snapshots import build_all_historical_snapshots
+from .historical_snapshots import generate_and_persist_historical_snapshots


 def run_periodic_historical_refresh(
@@ -98,23 +97,12 @@ def generate_historical_snapshots(
    server_slug: str | None = None,
 ) -> dict[str, Any]:
    """Build and persist precomputed snapshots for one server or all servers."""
-    generated_at = datetime.now(timezone.utc)
-    snapshots = build_all_historical_snapshots(
+    result = generate_and_persist_historical_snapshots(
        server_key=server_slug,
-        generated_at=generated_at,
+        generated_at=datetime.now(timezone.utc),
    )
-    persisted_records = persist_historical_snapshot_batch(snapshots)
-    snapshots_by_server: dict[str, int] = {}
-    for record in persisted_records:
-        snapshots_by_server.setdefault(record.server_key, 0)
-        snapshots_by_server[record.server_key] += 1
-
    return {
-        "generated_at": generated_at.isoformat().replace("+00:00", "Z"),
-        "server_slug": server_slug,
-        "snapshot_count": len(persisted_records),
-        "servers_processed": len(snapshots_by_server),
-        "snapshots_by_server": snapshots_by_server,
+        **result,
        "refresh_interval_seconds": get_historical_refresh_interval_seconds(),
    }

--- a/backend/app/historical_snapshots.py
+++ b/backend/app/historical_snapshots.py
@@ -126,10 +126,44 @@ def build_all_historical_snapshots(
                recent_matches_limit=recent_matches_limit,
                db_path=db_path,
            )
-        )
+    )
    return snapshots


+def generate_and_persist_historical_snapshots(
+    *,
+    server_key: str | None = None,
+    generated_at: datetime | None = None,
+    leaderboard_limit: int = DEFAULT_WEEKLY_LEADERBOARD_LIMIT,
+    recent_matches_limit: int = DEFAULT_RECENT_MATCHES_LIMIT,
+    db_path: Path | None = None,
+) -> dict[str, object]:
+    """Build and persist precomputed snapshots for one server or all servers."""
+    from .historical_snapshot_storage import persist_historical_snapshot_batch
+
+    generated_at_value = _as_utc(generated_at or datetime.now(timezone.utc))
+    snapshots = build_all_historical_snapshots(
+        server_key=server_key,
+        generated_at=generated_at_value,
+        leaderboard_limit=leaderboard_limit,
+        recent_matches_limit=recent_matches_limit,
+        db_path=db_path,
+    )
+    persisted_records = persist_historical_snapshot_batch(snapshots, db_path=db_path)
+    snapshots_by_server: dict[str, int] = {}
+    for record in persisted_records:
+        snapshots_by_server.setdefault(record.server_key, 0)
+        snapshots_by_server[record.server_key] += 1
+
+    return {
+        "generated_at": _to_iso(generated_at_value),
+        "server_slug": server_key,
+        "snapshot_count": len(persisted_records),
+        "servers_processed": len(snapshots_by_server),
+        "snapshots_by_server": snapshots_by_server,
+    }
+
+
 def _build_server_summary_snapshot(
    server_key: str,
    generated_at: datetime,
--- a/backend/app/historical_storage.py
+++ b/backend/app/historical_storage.py
@@ -7,7 +7,11 @@ from datetime import datetime, timedelta, timezone
 from pathlib import Path
 from typing import Mapping

-from .config import get_storage_path
+from .config import (
+    get_historical_weekly_fallback_max_weekday,
+    get_historical_weekly_fallback_min_matches,
+    get_storage_path,
+)
 from .historical_models import HistoricalServerDefinition


@@ -1098,16 +1102,26 @@ def list_weekly_leaderboard(
 ) -> dict[str, object]:
    """Return ranked weekly leaderboard totals from persisted historical match stats."""
    resolved_path = initialize_historical_storage(db_path=db_path)
-    window_end = datetime.now(timezone.utc)
-    window_start = window_end - timedelta(days=DEFAULT_WEEKLY_WINDOW_DAYS)
+    current_time = datetime.now(timezone.utc)
+    current_week_start = _start_of_week(current_time)
+    previous_week_start = current_week_start - timedelta(days=DEFAULT_WEEKLY_WINDOW_DAYS)
    normalized_metric = metric.strip() if isinstance(metric, str) else ""
    if normalized_metric not in SUPPORTED_WEEKLY_LEADERBOARD_METRICS:
        raise ValueError(f"Unsupported weekly leaderboard metric: {metric}")

+    weekly_window = _select_weekly_window(
+        server_id=server_id,
+        current_time=current_time,
+        current_week_start=current_week_start,
+        previous_week_start=previous_week_start,
+        db_path=resolved_path,
+    )
+    window_start = weekly_window["window_start"]
+    window_end = weekly_window["window_end"]
    where_clauses = [
        "historical_matches.ended_at IS NOT NULL",
        "historical_matches.ended_at >= ?",
-        "historical_matches.ended_at <= ?",
+        "historical_matches.ended_at < ?",
    ]
    params: list[object] = [
        window_start.isoformat().replace("+00:00", "Z"),
@@ -1196,6 +1210,21 @@ def list_weekly_leaderboard(
        "metric": normalized_metric,
        "window_start": window_start.isoformat().replace("+00:00", "Z"),
        "window_end": window_end.isoformat().replace("+00:00", "Z"),
+        "window_days": DEFAULT_WEEKLY_WINDOW_DAYS,
+        "window_kind": weekly_window["window_kind"],
+        "window_label": weekly_window["window_label"],
+        "uses_fallback": weekly_window["uses_fallback"],
+        "selection_reason": weekly_window["selection_reason"],
+        "current_week_start": current_week_start.isoformat().replace("+00:00", "Z"),
+        "current_week_closed_matches": weekly_window["current_week_closed_matches"],
+        "previous_week_closed_matches": weekly_window["previous_week_closed_matches"],
+        "sufficient_sample": {
+            "minimum_closed_matches": weekly_window["minimum_closed_matches"],
+            "current_week_closed_matches": weekly_window["current_week_closed_matches"],
+            "current_week_has_sufficient_sample": weekly_window["current_week_has_sufficient_sample"],
+            "is_early_week": weekly_window["is_early_week"],
+            "fallback_max_weekday": weekly_window["fallback_max_weekday"],
+        },
        "items": items,
    }

@@ -2112,6 +2141,105 @@ def _calculate_coverage_days(
    return round(delta.total_seconds() / 86400, 2)


+def _select_weekly_window(
+    *,
+    server_id: str | None,
+    current_time: datetime,
+    current_week_start: datetime,
+    previous_week_start: datetime,
+    db_path: Path,
+) -> dict[str, object]:
+    min_matches = get_historical_weekly_fallback_min_matches()
+    fallback_max_weekday = get_historical_weekly_fallback_max_weekday()
+    current_week_closed_matches = _count_closed_matches_in_window(
+        server_id=server_id,
+        window_start=current_week_start,
+        window_end=current_time,
+        db_path=db_path,
+    )
+    previous_week_closed_matches = _count_closed_matches_in_window(
+        server_id=server_id,
+        window_start=previous_week_start,
+        window_end=current_week_start,
+        db_path=db_path,
+    )
+    is_early_week = current_time.weekday() <= fallback_max_weekday
+    current_week_has_sufficient_sample = current_week_closed_matches >= min_matches
+    uses_fallback = (
+        is_early_week
+        and not current_week_has_sufficient_sample
+        and previous_week_closed_matches > 0
+    )
+
+    if uses_fallback:
+        return {
+            "window_start": previous_week_start,
+            "window_end": current_week_start,
+            "window_kind": "previous-closed-week-fallback",
+            "window_label": "Semana cerrada anterior",
+            "uses_fallback": True,
+            "selection_reason": "insufficient-current-week-sample",
+            "minimum_closed_matches": min_matches,
+            "current_week_closed_matches": current_week_closed_matches,
+            "previous_week_closed_matches": previous_week_closed_matches,
+            "current_week_has_sufficient_sample": False,
+            "is_early_week": is_early_week,
+            "fallback_max_weekday": fallback_max_weekday,
+        }
+
+    return {
+        "window_start": current_week_start,
+        "window_end": current_time,
+        "window_kind": "current-week",
+        "window_label": "Semana actual",
+        "uses_fallback": False,
+        "selection_reason": "current-week",
+        "minimum_closed_matches": min_matches,
+        "current_week_closed_matches": current_week_closed_matches,
+        "previous_week_closed_matches": previous_week_closed_matches,
+        "current_week_has_sufficient_sample": current_week_has_sufficient_sample,
+        "is_early_week": is_early_week,
+        "fallback_max_weekday": fallback_max_weekday,
+    }
+
+
+def _count_closed_matches_in_window(
+    *,
+    server_id: str | None,
+    window_start: datetime,
+    window_end: datetime,
+    db_path: Path,
+) -> int:
+    where_clauses = [
+        "historical_matches.ended_at IS NOT NULL",
+        "historical_matches.ended_at >= ?",
+        "historical_matches.ended_at < ?",
+    ]
+    params: list[object] = [
+        window_start.isoformat().replace("+00:00", "Z"),
+        window_end.isoformat().replace("+00:00", "Z"),
+    ]
+    if server_id:
+        normalized_server_id = server_id.strip()
+        where_clauses.append(
+            "(historical_servers.slug = ? OR CAST(historical_servers.server_number AS TEXT) = ?)"
+        )
+        params.extend([normalized_server_id, normalized_server_id])
+
+    with _connect(db_path) as connection:
+        row = connection.execute(
+            f"""
+            SELECT COUNT(DISTINCT historical_matches.id) AS matches_count
+            FROM historical_matches
+            INNER JOIN historical_servers
+                ON historical_servers.id = historical_matches.historical_server_id
+            WHERE {" AND ".join(where_clauses)}
+            """,
+            params,
+        ).fetchone()
+    return int(row["matches_count"] or 0) if row is not None else 0
+
+
 def _classify_coverage_status(
    matches_count: int,
    coverage_days: float | None,
@@ -2125,6 +2253,12 @@ def _classify_coverage_status(
    return "week-plus"


+def _start_of_week(value: datetime) -> datetime:
+    normalized = value.astimezone(timezone.utc)
+    midnight = normalized.replace(hour=0, minute=0, second=0, microsecond=0)
+    return midnight - timedelta(days=midnight.weekday())
+
+
 def _get_nested(payload: Mapping[str, object], *path: str) -> object:
    current: object = payload
    for key in path:
--- a/backend/app/payloads.py
+++ b/backend/app/payloads.py
@@ -240,10 +240,18 @@ def build_weekly_leaderboard_payload(
            "title": title_by_metric.get(metric, "Ranking semanal por servidor"),
            "context": "historical-weekly-leaderboard",
            "metric": metric,
-            "summary_basis": "closed-matches-last-7-days",
-            "window_days": 7,
+            "summary_basis": "closed-matches-calendar-week",
+            "window_days": result.get("window_days", 7),
            "window_start": result["window_start"],
            "window_end": result["window_end"],
+            "window_kind": result.get("window_kind"),
+            "window_label": result.get("window_label"),
+            "uses_fallback": bool(result.get("uses_fallback")),
+            "selection_reason": result.get("selection_reason"),
+            "current_week_start": result.get("current_week_start"),
+            "current_week_closed_matches": result.get("current_week_closed_matches"),
+            "previous_week_closed_matches": result.get("previous_week_closed_matches"),
+            "sufficient_sample": result.get("sufficient_sample"),
            "limit": limit,
            "items": result["items"],
        },
@@ -331,6 +339,18 @@ def build_weekly_leaderboard_snapshot_payload(
            "window_days": payload.get("window_days") if isinstance(payload, dict) else 7,
            "window_start": payload.get("window_start") if isinstance(payload, dict) else None,
            "window_end": payload.get("window_end") if isinstance(payload, dict) else None,
+            "window_kind": payload.get("window_kind") if isinstance(payload, dict) else None,
+            "window_label": payload.get("window_label") if isinstance(payload, dict) else None,
+            "uses_fallback": bool(payload.get("uses_fallback")) if isinstance(payload, dict) else False,
+            "selection_reason": payload.get("selection_reason") if isinstance(payload, dict) else None,
+            "current_week_start": payload.get("current_week_start") if isinstance(payload, dict) else None,
+            "current_week_closed_matches": (
+                payload.get("current_week_closed_matches") if isinstance(payload, dict) else None
+            ),
+            "previous_week_closed_matches": (
+                payload.get("previous_week_closed_matches") if isinstance(payload, dict) else None
+            ),
+            "sufficient_sample": payload.get("sufficient_sample") if isinstance(payload, dict) else None,
            "snapshot_limit": payload.get("limit") if isinstance(payload, dict) else None,
            "limit": limit,
            "items": sliced_items,