Add historical UI and backend API slice

This commit is contained in:
devRaGonSa
2026-03-20 22:39:17 +01:00
parent f346084c5f
commit 502d11d8b3
19 changed files with 1862 additions and 4 deletions

View File

@@ -26,6 +26,7 @@ backend/
|-- main.py
|-- historical_ingestion.py
|-- historical_models.py
|-- historical_runner.py
|-- historical_storage.py
|-- normalizers.py
|-- payloads.py
@@ -65,6 +66,9 @@ Variables opcionales:
- `HLL_BACKEND_REFRESH_INTERVAL_SECONDS`
- `HLL_HISTORICAL_CRCON_PAGE_SIZE`
- `HLL_HISTORICAL_CRCON_TIMEOUT_SECONDS`
- `HLL_HISTORICAL_REFRESH_INTERVAL_SECONDS`
- `HLL_HISTORICAL_REFRESH_MAX_RETRIES`
- `HLL_HISTORICAL_REFRESH_RETRY_DELAY_SECONDS`
El `frontend/index.html` viene preparado para volver a consultar el bloque de
servidores cada `120000` ms (`120s`) sin recargar la pagina completa. La landing
@@ -115,6 +119,9 @@ normaliza espacios y barras finales para mantener la comparacion con el header
- `GET /api/servers/history?limit=20`
- `GET /api/servers/{id}/history?limit=20`
- `GET /api/historical/weekly-top-kills?limit=10&server=comunidad-hispana-01`
- `GET /api/historical/recent-matches?limit=20&server=comunidad-hispana-01`
- `GET /api/historical/server-summary?server=comunidad-hispana-01`
- `GET /api/historical/player-profile?player=steam%3A76561198000000000`
`GET /api/servers` trata el ultimo snapshot persistido como cache local y lo
reutiliza solo si sigue dentro del objetivo de `120` segundos. Si ese snapshot
@@ -153,6 +160,7 @@ colector. Si todavia no hay snapshots guardados, responden `status: "ok"` con
- `config.py` centraliza host, puerto y allowlist minima de origenes locales.
- `historical_ingestion.py` consulta la capa JSON publica de CRCON para bootstrap y refresh incremental.
- `historical_models.py` fija las entidades historicas minimas del dominio.
- `historical_runner.py` ejecuta refresh incremental periodico con reintentos basicos.
- `historical_storage.py` prepara la persistencia `historical_*` y las consultas agregadas iniciales.
- `main.py` contiene el entrypoint HTTP y la creacion del servidor.
- `normalizers.py` transforma registros crudos o respuestas A2S a un modelo
@@ -413,14 +421,19 @@ consulta historica:
persistido por el colector. El parametro opcional `limit` acepta valores entre
`1` y `100`.
La primera consulta agregada sobre historico CRCON es:
La capa historica propia expone:
- `/api/historical/weekly-top-kills`
- `/api/historical/recent-matches`
- `/api/historical/server-summary`
- `/api/historical/player-profile`
Parametros opcionales:
- `limit` entre `1` y `100`
- `server` con slug historico como `comunidad-hispana-01`
- `player` en `/api/historical/player-profile` aceptando `stable_player_key`,
`steam_id` o `source_player_id`
La ventana temporal es siempre la ultima semana movil respecto al momento de la
request y solo considera partidas cerradas con `ended_at` para no mezclar
@@ -428,6 +441,11 @@ partidas aun en curso ni filas historicas transitorias. El payload devuelve
servidor, rango temporal, jugador, kills semanales, posicion y numero de
partidas consideradas.
`recent-matches` devuelve cierres recientes por servidor con marcador, mapa y
conteo de jugadores. `server-summary` agrega volumen historico, jugadores
unicos, kills, mapas dominantes y rango temporal cubierto. `player-profile`
deja lista la base de consulta agregada por jugador para futuras vistas.
## Ingesta historica CRCON
La ingesta historica no usa A2S ni scraping del HTML de `/games`. Consume la
@@ -444,6 +462,7 @@ Comandos manuales desde `backend/`:
```powershell
python -m app.historical_ingestion bootstrap
python -m app.historical_ingestion refresh
python -m app.historical_runner --interval 1800
```
Flags utiles:
@@ -457,6 +476,24 @@ La ejecucion `refresh` usa una ventana de solape sobre la ultima partida
persistida por servidor para releer solo paginas recientes y absorber updates
tardios sin reimportar todo el historico.
El runner `python -m app.historical_runner` deja ese refresh incremental listo
para ejecucion local repetida sin depender de infraestructura externa. Por
defecto:
- refresca cada `1800` segundos
- reintenta hasta `2` veces tras un fallo
- espera `30` segundos entre reintentos
- reutiliza el registro de `historical_ingestion_runs` para dejar trazabilidad
de ultimo refresh, resultado y errores basicos
Flags utiles del runner:
- `--server comunidad-hispana-01` para limitar a un servidor
- `--interval 900` para aumentar frecuencia local
- `--retries 1` para reducir reintentos
- `--retry-delay 10` para bajar la espera entre fallos
- `--max-runs 1` para una validacion puntual sin bucle indefinido
Al inicializar la persistencia local, el backend normaliza tambien la identidad
historica ya guardada:

View File

@@ -12,6 +12,9 @@ DEFAULT_STORAGE_FILENAME = "hll_vietnam_dev.sqlite3"
DEFAULT_REFRESH_INTERVAL_SECONDS = 120
DEFAULT_HISTORICAL_CRCON_PAGE_SIZE = 50
DEFAULT_HISTORICAL_CRCON_TIMEOUT_SECONDS = 15.0
DEFAULT_HISTORICAL_REFRESH_INTERVAL_SECONDS = 1800
DEFAULT_HISTORICAL_REFRESH_MAX_RETRIES = 2
DEFAULT_HISTORICAL_REFRESH_RETRY_DELAY_SECONDS = 30
DEFAULT_ALLOWED_ORIGINS = (
"null",
"http://127.0.0.1:5500",
@@ -95,6 +98,47 @@ def get_historical_crcon_request_timeout_seconds() -> float:
return timeout_seconds
def get_historical_refresh_interval_seconds() -> int:
"""Return the default interval used by the historical refresh loop."""
configured_value = os.getenv(
"HLL_HISTORICAL_REFRESH_INTERVAL_SECONDS",
str(DEFAULT_HISTORICAL_REFRESH_INTERVAL_SECONDS),
)
interval_seconds = int(configured_value)
if interval_seconds <= 0:
raise ValueError("HLL_HISTORICAL_REFRESH_INTERVAL_SECONDS must be positive.")
return interval_seconds
def get_historical_refresh_max_retries() -> int:
"""Return the retry count used by the historical refresh loop."""
configured_value = os.getenv(
"HLL_HISTORICAL_REFRESH_MAX_RETRIES",
str(DEFAULT_HISTORICAL_REFRESH_MAX_RETRIES),
)
max_retries = int(configured_value)
if max_retries < 0:
raise ValueError("HLL_HISTORICAL_REFRESH_MAX_RETRIES must be zero or positive.")
return max_retries
def get_historical_refresh_retry_delay_seconds() -> int:
"""Return the wait time between historical refresh retries."""
configured_value = os.getenv(
"HLL_HISTORICAL_REFRESH_RETRY_DELAY_SECONDS",
str(DEFAULT_HISTORICAL_REFRESH_RETRY_DELAY_SECONDS),
)
retry_delay_seconds = int(configured_value)
if retry_delay_seconds < 0:
raise ValueError(
"HLL_HISTORICAL_REFRESH_RETRY_DELAY_SECONDS must be zero or positive."
)
return retry_delay_seconds
def get_a2s_targets_payload() -> str | None:
"""Return the optional JSON payload that overrides local A2S targets."""
raw_payload = os.getenv(DEFAULT_A2S_TARGETS_ENV_VAR)

View File

@@ -0,0 +1,160 @@
"""Local development loop for periodic historical CRCON refreshes."""
from __future__ import annotations
import argparse
import json
import time
from typing import Any
from .config import (
get_historical_refresh_interval_seconds,
get_historical_refresh_max_retries,
get_historical_refresh_retry_delay_seconds,
)
from .historical_ingestion import run_incremental_refresh
def run_periodic_historical_refresh(
*,
interval_seconds: int,
max_retries: int,
retry_delay_seconds: int,
server_slug: str | None = None,
max_pages: int | None = None,
page_size: int | None = None,
max_runs: int | None = None,
) -> None:
"""Run periodic historical refreshes until interrupted or the limit is reached."""
completed_runs = 0
print(
"Starting historical refresh loop "
f"(interval={interval_seconds}s, retries={max_retries}, server={server_slug or 'all'})."
)
print("Press Ctrl+C to stop.")
try:
while max_runs is None or completed_runs < max_runs:
completed_runs += 1
payload = _run_refresh_with_retries(
max_retries=max_retries,
retry_delay_seconds=retry_delay_seconds,
server_slug=server_slug,
max_pages=max_pages,
page_size=page_size,
)
print(json.dumps({"run": completed_runs, **payload}, indent=2))
if max_runs is not None and completed_runs >= max_runs:
break
time.sleep(interval_seconds)
except KeyboardInterrupt:
print("\nHistorical refresh loop stopped by user.")
def _run_refresh_with_retries(
*,
max_retries: int,
retry_delay_seconds: int,
server_slug: str | None,
max_pages: int | None,
page_size: int | None,
) -> dict[str, Any]:
attempt = 0
while True:
attempt += 1
try:
result = run_incremental_refresh(
server_slug=server_slug,
max_pages=max_pages,
page_size=page_size,
)
return {
"status": "ok",
"attempts_used": attempt,
"max_retries": max_retries,
"result": result,
}
except Exception as exc:
if attempt > max_retries:
return {
"status": "error",
"attempts_used": attempt,
"max_retries": max_retries,
"error": str(exc),
}
if retry_delay_seconds > 0:
time.sleep(retry_delay_seconds)
def main() -> None:
"""Allow local scheduled historical refresh execution without external infra."""
parser = argparse.ArgumentParser(
description="Run periodic historical CRCON refreshes for HLL Vietnam.",
)
parser.add_argument(
"--interval",
type=int,
default=get_historical_refresh_interval_seconds(),
help="Seconds to wait between incremental refresh runs.",
)
parser.add_argument(
"--retries",
type=int,
default=get_historical_refresh_max_retries(),
help="Retry attempts after a failed incremental refresh.",
)
parser.add_argument(
"--retry-delay",
type=int,
default=get_historical_refresh_retry_delay_seconds(),
help="Seconds to wait between failed attempts.",
)
parser.add_argument(
"--server",
dest="server_slug",
help="Optional historical server slug.",
)
parser.add_argument(
"--max-pages",
type=int,
default=None,
help="Optional page cap for local validation.",
)
parser.add_argument(
"--page-size",
type=int,
default=None,
help="Optional override for CRCON page size.",
)
parser.add_argument(
"--max-runs",
type=int,
default=None,
help="Optional safety limit for the number of refresh cycles to execute.",
)
args = parser.parse_args()
if args.interval <= 0:
raise ValueError("--interval must be a positive integer.")
if args.retries < 0:
raise ValueError("--retries must be zero or positive.")
if args.retry_delay < 0:
raise ValueError("--retry-delay must be zero or positive.")
if args.max_runs is not None and args.max_runs <= 0:
raise ValueError("--max-runs must be positive when provided.")
run_periodic_historical_refresh(
interval_seconds=args.interval,
max_retries=args.retries,
retry_delay_seconds=args.retry_delay,
server_slug=args.server_slug,
max_pages=args.max_pages,
page_size=args.page_size,
max_runs=args.max_runs,
)
if __name__ == "__main__":
main()

View File

@@ -439,7 +439,7 @@ def list_recent_historical_matches(
limit: int = 20,
db_path: Path | None = None,
) -> list[dict[str, object]]:
"""Return recent persisted matches for validation and later API work."""
"""Return recent persisted matches grouped for the historical API layer."""
resolved_path = initialize_historical_storage(db_path=db_path)
where_clause = ""
params: list[object] = []
@@ -474,7 +474,217 @@ def list_recent_historical_matches(
""",
params,
).fetchall()
return [dict(row) for row in rows]
items: list[dict[str, object]] = []
for row in rows:
items.append(
{
"server": {
"slug": row["server_slug"],
"name": row["server_name"],
},
"match_id": row["external_match_id"],
"started_at": row["started_at"],
"ended_at": row["ended_at"],
"closed_at": row["ended_at"] or row["started_at"],
"map": {
"name": row["map_name"],
"pretty_name": row["map_pretty_name"] or row["map_name"],
},
"result": {
"allied_score": _coerce_int(row["allied_score"]),
"axis_score": _coerce_int(row["axis_score"]),
"winner": _resolve_match_winner(
row["allied_score"],
row["axis_score"],
),
},
"player_count": int(row["player_count"] or 0),
}
)
return items
def list_historical_server_summaries(
*,
server_slug: str | None = None,
db_path: Path | None = None,
) -> list[dict[str, object]]:
"""Return aggregate historical metrics per server."""
resolved_path = initialize_historical_storage(db_path=db_path)
where_clause = ""
params: list[object] = []
if server_slug:
where_clause = "WHERE historical_servers.slug = ?"
params.append(server_slug)
with _connect(resolved_path) as connection:
summary_rows = connection.execute(
f"""
SELECT
historical_servers.slug AS server_slug,
historical_servers.display_name AS server_name,
COUNT(DISTINCT historical_matches.id) AS matches_count,
COUNT(DISTINCT historical_players.id) AS unique_players,
COALESCE(SUM(historical_player_match_stats.kills), 0) AS total_kills,
COUNT(DISTINCT COALESCE(historical_matches.map_pretty_name, historical_matches.map_name)) AS map_count,
MIN(COALESCE(historical_matches.ended_at, historical_matches.started_at, historical_matches.created_at_source)) AS first_match_at,
MAX(COALESCE(historical_matches.ended_at, historical_matches.started_at, historical_matches.created_at_source)) AS last_match_at
FROM historical_servers
LEFT JOIN historical_matches
ON historical_matches.historical_server_id = historical_servers.id
LEFT JOIN historical_player_match_stats
ON historical_player_match_stats.historical_match_id = historical_matches.id
LEFT JOIN historical_players
ON historical_players.id = historical_player_match_stats.historical_player_id
{where_clause}
GROUP BY historical_servers.id
ORDER BY historical_servers.server_number ASC, historical_servers.slug ASC
""",
params,
).fetchall()
map_rows = connection.execute(
f"""
SELECT
historical_servers.slug AS server_slug,
COALESCE(historical_matches.map_pretty_name, historical_matches.map_name, 'Mapa no disponible') AS map_name,
COUNT(*) AS matches_count
FROM historical_matches
INNER JOIN historical_servers
ON historical_servers.id = historical_matches.historical_server_id
{where_clause}
GROUP BY historical_servers.slug, COALESCE(historical_matches.map_pretty_name, historical_matches.map_name, 'Mapa no disponible')
ORDER BY historical_servers.slug ASC, matches_count DESC, map_name ASC
""",
params,
).fetchall()
top_maps_by_server: dict[str, list[dict[str, object]]] = {}
for row in map_rows:
server_key = str(row["server_slug"])
top_maps_by_server.setdefault(server_key, [])
if len(top_maps_by_server[server_key]) >= 3:
continue
top_maps_by_server[server_key].append(
{
"map_name": row["map_name"],
"matches_count": int(row["matches_count"] or 0),
}
)
items: list[dict[str, object]] = []
for row in summary_rows:
items.append(
{
"server": {
"slug": row["server_slug"],
"name": row["server_name"],
},
"matches_count": int(row["matches_count"] or 0),
"unique_players": int(row["unique_players"] or 0),
"total_kills": int(row["total_kills"] or 0),
"map_count": int(row["map_count"] or 0),
"top_maps": top_maps_by_server.get(str(row["server_slug"]), []),
"time_range": {
"start": row["first_match_at"],
"end": row["last_match_at"],
},
}
)
return items
def get_historical_player_profile(
player_id: str,
*,
db_path: Path | None = None,
) -> dict[str, object] | None:
"""Return aggregate historical metrics for one player identity."""
resolved_player_id = player_id.strip()
if not resolved_player_id:
return None
resolved_path = initialize_historical_storage(db_path=db_path)
with _connect(resolved_path) as connection:
player_row = connection.execute(
"""
SELECT
historical_players.id,
historical_players.stable_player_key,
historical_players.display_name,
historical_players.steam_id,
historical_players.source_player_id,
COUNT(DISTINCT historical_matches.id) AS matches_count,
COALESCE(SUM(historical_player_match_stats.kills), 0) AS total_kills,
MIN(COALESCE(historical_matches.ended_at, historical_matches.started_at, historical_matches.created_at_source)) AS first_match_at,
MAX(COALESCE(historical_matches.ended_at, historical_matches.started_at, historical_matches.created_at_source)) AS last_match_at
FROM historical_players
LEFT JOIN historical_player_match_stats
ON historical_player_match_stats.historical_player_id = historical_players.id
LEFT JOIN historical_matches
ON historical_matches.id = historical_player_match_stats.historical_match_id
WHERE historical_players.stable_player_key = ?
OR historical_players.steam_id = ?
OR historical_players.source_player_id = ?
GROUP BY historical_players.id
ORDER BY historical_players.display_name ASC
LIMIT 1
""",
(resolved_player_id, resolved_player_id, resolved_player_id),
).fetchone()
if player_row is None:
return None
server_rows = connection.execute(
"""
SELECT
historical_servers.slug AS server_slug,
historical_servers.display_name AS server_name,
COUNT(DISTINCT historical_matches.id) AS matches_count,
COALESCE(SUM(historical_player_match_stats.kills), 0) AS total_kills,
MIN(COALESCE(historical_matches.ended_at, historical_matches.started_at, historical_matches.created_at_source)) AS first_match_at,
MAX(COALESCE(historical_matches.ended_at, historical_matches.started_at, historical_matches.created_at_source)) AS last_match_at
FROM historical_player_match_stats
INNER JOIN historical_matches
ON historical_matches.id = historical_player_match_stats.historical_match_id
INNER JOIN historical_servers
ON historical_servers.id = historical_matches.historical_server_id
WHERE historical_player_match_stats.historical_player_id = ?
GROUP BY historical_servers.id
ORDER BY total_kills DESC, historical_servers.server_number ASC, historical_servers.slug ASC
""",
(player_row["id"],),
).fetchall()
return {
"player": {
"stable_player_key": player_row["stable_player_key"],
"name": player_row["display_name"],
"steam_id": player_row["steam_id"],
"source_player_id": player_row["source_player_id"],
},
"matches_count": int(player_row["matches_count"] or 0),
"total_kills": int(player_row["total_kills"] or 0),
"time_range": {
"start": player_row["first_match_at"],
"end": player_row["last_match_at"],
},
"servers": [
{
"server": {
"slug": row["server_slug"],
"name": row["server_name"],
},
"matches_count": int(row["matches_count"] or 0),
"total_kills": int(row["total_kills"] or 0),
"time_range": {
"start": row["first_match_at"],
"end": row["last_match_at"],
},
}
for row in server_rows
],
}
def list_weekly_top_kills(
@@ -578,6 +788,18 @@ def _connect(db_path: Path) -> sqlite3.Connection:
return connection
def _resolve_match_winner(allied_score: object, axis_score: object) -> str | None:
allied = _coerce_int(allied_score)
axis = _coerce_int(axis_score)
if allied is None or axis is None:
return None
if allied > axis:
return "allies"
if axis > allied:
return "axis"
return "draw"
def _has_legacy_historical_schema(connection: sqlite3.Connection) -> bool:
columns = {
str(row["name"])

View File

@@ -6,7 +6,12 @@ from datetime import datetime, timezone
from .collector import collect_server_snapshots
from .config import get_refresh_interval_seconds
from .historical_storage import list_weekly_top_kills
from .historical_storage import (
get_historical_player_profile,
list_historical_server_summaries,
list_recent_historical_matches,
list_weekly_top_kills,
)
from .normalizers import normalize_map_name
from .server_targets import load_a2s_targets
from .storage import list_latest_snapshots, list_server_history, list_snapshot_history
@@ -205,6 +210,60 @@ def build_weekly_top_kills_payload(
}
def build_recent_historical_matches_payload(
*,
limit: int = 20,
server_slug: str | None = None,
) -> dict[str, object]:
"""Return recent historical matches from persisted CRCON data."""
items = list_recent_historical_matches(limit=limit, server_slug=server_slug)
return {
"status": "ok",
"data": {
"title": "Partidas recientes por servidor",
"context": "historical-recent-matches",
"source": "historical-crcon-storage",
"limit": limit,
"server_slug": server_slug,
"items": items,
},
}
def build_historical_server_summary_payload(
*,
server_slug: str | None = None,
) -> dict[str, object]:
"""Return aggregated historical metrics per server."""
items = list_historical_server_summaries(server_slug=server_slug)
return {
"status": "ok",
"data": {
"title": "Resumen historico por servidor",
"context": "historical-server-summary",
"source": "historical-crcon-storage",
"server_slug": server_slug,
"items": items,
},
}
def build_historical_player_profile_payload(player_id: str) -> dict[str, object]:
"""Return aggregate historical metrics for one player identity."""
profile = get_historical_player_profile(player_id)
return {
"status": "ok",
"data": {
"title": "Perfil historico de jugador",
"context": "historical-player-profile",
"source": "historical-crcon-storage",
"player_id": player_id,
"found": profile is not None,
"profile": profile,
},
}
def _enrich_server_items(items: list[dict[str, object]]) -> list[dict[str, object]]:
target_index = {
target.external_server_id: target

View File

@@ -10,6 +10,9 @@ from .payloads import (
build_discord_payload,
build_error_payload,
build_health_payload,
build_historical_player_profile_payload,
build_historical_server_summary_payload,
build_recent_historical_matches_payload,
build_server_detail_history_payload,
build_server_history_payload,
build_server_latest_payload,
@@ -47,6 +50,26 @@ def resolve_get_payload(path: str) -> tuple[HTTPStatus | None, dict[str, object]
server_id = parse_qs(parsed.query).get("server", [None])[0]
return HTTPStatus.OK, build_weekly_top_kills_payload(limit=limit, server_id=server_id)
if parsed.path == "/api/historical/recent-matches":
limit = _parse_limit(parsed.query)
if limit is None:
return HTTPStatus.BAD_REQUEST, build_error_payload("Invalid limit parameter")
server_slug = parse_qs(parsed.query).get("server", [None])[0]
return HTTPStatus.OK, build_recent_historical_matches_payload(
limit=limit,
server_slug=server_slug,
)
if parsed.path == "/api/historical/server-summary":
server_slug = parse_qs(parsed.query).get("server", [None])[0]
return HTTPStatus.OK, build_historical_server_summary_payload(server_slug=server_slug)
if parsed.path == "/api/historical/player-profile":
player_id = parse_qs(parsed.query).get("player", [None])[0]
if not player_id:
return HTTPStatus.BAD_REQUEST, build_error_payload("Player parameter is required")
return HTTPStatus.OK, build_historical_player_profile_payload(player_id)
builder = GET_ROUTES.get(parsed.path)
if builder is None:
if parsed.path.startswith("/api/servers/") and parsed.path.endswith("/history"):