rgonsal/comunidadhll
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: refresh historical snapshots from runner

Browse Source
This commit is contained in:
devRaGonSa
2026-05-22 14:56:54 +02:00
parent dfdb53d2b6
commit 664f3f456f
8 changed files with 296 additions and 59 deletions
Download Patch File Download Diff File Expand all files Collapse all files

54
backend/app/config.py
View File

@@ -233,20 +233,40 @@ def get_historical_crcon_retry_delay_seconds() -> float:
def get_historical_refresh_interval_seconds() -> int:
"""Return the default interval used by the historical refresh loop."""
configured_value = os.getenv(
return _read_int_env(
"HLL_HISTORICAL_SNAPSHOT_REFRESH_INTERVAL_SECONDS",
os.getenv(
"HLL_HISTORICAL_REFRESH_INTERVAL_SECONDS",
str(DEFAULT_HISTORICAL_SNAPSHOT_REFRESH_INTERVAL_SECONDS),
),
)
interval_seconds = int(configured_value)
if interval_seconds <= 0:
raise ValueError(
"HLL_HISTORICAL_SNAPSHOT_REFRESH_INTERVAL_SECONDS must be positive."
minimum=1,
)
return interval_seconds
def _read_int_env(name: str, default_value: str, *, minimum: int) -> int:
"""Read one integer env var and keep validation errors actionable."""
configured_value = os.getenv(name, default_value)
try:
value = int(configured_value)
except (TypeError, ValueError) as error:
raise ValueError(f"{name} must be an integer.") from error
if value < minimum:
qualifier = "positive" if minimum == 1 else f"at least {minimum}"
raise ValueError(f"{name} must be {qualifier}.")
return value
def _read_float_env(name: str, default_value: str, *, minimum: float) -> float:
"""Read one float env var and keep validation errors actionable."""
configured_value = os.getenv(name, default_value)
try:
value = float(configured_value)
except (TypeError, ValueError) as error:
raise ValueError(f"{name} must be a number.") from error
if value < minimum:
qualifier = "zero or positive" if minimum == 0 else f"at least {minimum}"
raise ValueError(f"{name} must be {qualifier}.")
return value
def get_historical_refresh_overlap_hours() -> int:
@@ -297,30 +317,20 @@ def get_rcon_request_timeout_seconds() -> float:
def get_historical_refresh_max_retries() -> int:
"""Return the retry count used by the historical refresh loop."""
configured_value = os.getenv(
return _read_int_env(
"HLL_HISTORICAL_REFRESH_MAX_RETRIES",
str(DEFAULT_HISTORICAL_REFRESH_MAX_RETRIES),
minimum=0,
)
max_retries = int(configured_value)
if max_retries < 0:
raise ValueError("HLL_HISTORICAL_REFRESH_MAX_RETRIES must be zero or positive.")
return max_retries
def get_historical_refresh_retry_delay_seconds() -> int:
def get_historical_refresh_retry_delay_seconds() -> float:
"""Return the wait time between historical refresh retries."""
configured_value = os.getenv(
return _read_float_env(
"HLL_HISTORICAL_REFRESH_RETRY_DELAY_SECONDS",
str(DEFAULT_HISTORICAL_REFRESH_RETRY_DELAY_SECONDS),
minimum=0,
)
retry_delay_seconds = int(configured_value)
if retry_delay_seconds < 0:
raise ValueError(
"HLL_HISTORICAL_REFRESH_RETRY_DELAY_SECONDS must be zero or positive."
)
return retry_delay_seconds
def get_historical_full_snapshot_every_runs() -> int:

36
backend/app/historical_runner.py
View File

@@ -5,6 +5,7 @@ from __future__ import annotations
import argparse
import json
import time
import traceback
from datetime import datetime, timezone
from typing import Any
@@ -39,7 +40,7 @@ def run_periodic_historical_refresh(
*,
interval_seconds: int,
max_retries: int,
retry_delay_seconds: int,
retry_delay_seconds: float,
server_slug: str | None = None,
max_pages: int | None = None,
page_size: int | None = None,
@@ -73,7 +74,7 @@ def run_periodic_historical_refresh(
page_size=page_size,
run_number=completed_runs,
)
print(json.dumps({"run": completed_runs, **payload}, indent=2), flush=True)
_emit_json_log({"run": completed_runs, **payload})
if max_runs is not None and completed_runs >= max_runs:
break
@@ -86,7 +87,7 @@ def run_periodic_historical_refresh(
def _run_refresh_with_retries(
*,
max_retries: int,
retry_delay_seconds: int,
retry_delay_seconds: float,
server_slug: str | None,
max_pages: int | None,
page_size: int | None,
@@ -182,12 +183,25 @@ def _run_refresh_with_retries(
"elo_mmr_result": elo_mmr_result,
}
except Exception as exc:
failure_payload = {
"event": "historical-refresh-attempt-failed",
"attempt": attempt,
"max_retries": max_retries,
"server_scope": _describe_refresh_scope(server_slug),
"snapshot_scope": _describe_snapshot_scope(server_slug),
"error_type": type(exc).__name__,
"error": str(exc),
"traceback": traceback.format_exc(),
}
_emit_json_log(failure_payload)
if attempt > max_retries:
return {
"status": "error",
"attempts_used": attempt,
"max_retries": max_retries,
"error_type": type(exc).__name__,
"error": str(exc),
"traceback": failure_payload["traceback"],
}
if retry_delay_seconds > 0:
time.sleep(retry_delay_seconds)
@@ -202,6 +216,15 @@ def generate_historical_snapshots(
generated_at = datetime.now(timezone.utc)
full_snapshot_every_runs = get_historical_full_snapshot_every_runs()
should_run_full_refresh = bool(server_slug) or run_number % full_snapshot_every_runs == 0
_emit_json_log(
{
"event": "historical-snapshot-refresh-started",
"run_number": run_number,
"snapshot_step": "full-matrix" if should_run_full_refresh else "priority-prewarm",
"server_slug": server_slug,
"snapshot_scope": _describe_snapshot_scope(server_slug),
}
)
if should_run_full_refresh:
result = generate_and_persist_historical_snapshots(
server_key=server_slug,
@@ -221,6 +244,11 @@ def generate_historical_snapshots(
}
def _emit_json_log(payload: dict[str, Any]) -> None:
"""Print JSON logs that remain safe for Compose and log collectors."""
print(json.dumps(payload, ensure_ascii=True, default=str), flush=True)
def _describe_refresh_scope(server_slug: str | None) -> list[str]:
if server_slug:
return [server_slug]
@@ -353,7 +381,7 @@ def main() -> None:
)
parser.add_argument(
"--retry-delay",
type=int,
type=float,
default=get_historical_refresh_retry_delay_seconds(),
help="Seconds to wait between failed attempts.",
)

68
backend/app/historical_snapshots.py
View File

@@ -2,6 +2,7 @@
from __future__ import annotations
import json
import sqlite3
from datetime import datetime, timezone
from pathlib import Path
@@ -135,9 +136,16 @@ def build_historical_server_snapshots(
) -> list[dict[str, object]]:
"""Build all precomputed historical snapshots required for one server."""
generated_at_value = _as_utc(generated_at or datetime.now(timezone.utc))
leaderboard_limit = _normalize_snapshot_limit("leaderboard_limit", leaderboard_limit)
recent_matches_limit = _normalize_snapshot_limit(
"recent_matches_limit",
recent_matches_limit,
)
_log_snapshot_build_started(server_key, SNAPSHOT_TYPE_SERVER_SUMMARY)
snapshots = [_build_server_summary_snapshot(server_key, generated_at_value, db_path=db_path)]
for metric in SNAPSHOT_LEADERBOARD_METRICS:
_log_snapshot_build_started(server_key, SNAPSHOT_TYPE_WEEKLY_LEADERBOARD, metric=metric)
snapshots.append(
_build_weekly_leaderboard_snapshot(
server_key,
@@ -147,6 +155,7 @@ def build_historical_server_snapshots(
db_path=db_path,
)
)
_log_snapshot_build_started(server_key, SNAPSHOT_TYPE_MONTHLY_LEADERBOARD, metric=metric)
snapshots.append(
_build_monthly_leaderboard_snapshot(
server_key,
@@ -157,6 +166,7 @@ def build_historical_server_snapshots(
)
)
_log_snapshot_build_started(server_key, SNAPSHOT_TYPE_MONTHLY_MVP)
snapshots.append(
_build_monthly_mvp_snapshot(
server_key,
@@ -165,6 +175,7 @@ def build_historical_server_snapshots(
db_path=db_path,
)
)
_log_snapshot_build_started(server_key, SNAPSHOT_TYPE_MONTHLY_MVP_V2)
snapshots.append(
_build_monthly_mvp_v2_snapshot(
server_key,
@@ -174,6 +185,7 @@ def build_historical_server_snapshots(
)
)
for snapshot_type in PLAYER_EVENT_SNAPSHOT_TYPES:
_log_snapshot_build_started(server_key, snapshot_type)
snapshots.append(
_build_player_event_snapshot(
server_key,
@@ -184,6 +196,7 @@ def build_historical_server_snapshots(
)
)
_log_snapshot_build_started(server_key, SNAPSHOT_TYPE_RECENT_MATCHES)
snapshots.append(
_build_recent_matches_snapshot(
server_key,
@@ -205,12 +218,23 @@ def build_priority_historical_snapshots(
) -> list[dict[str, object]]:
"""Build the minimum warm snapshot set required by the historical UI."""
generated_at_value = _as_utc(generated_at or datetime.now(timezone.utc))
leaderboard_limit = _normalize_snapshot_limit("leaderboard_limit", leaderboard_limit)
recent_matches_limit = _normalize_snapshot_limit(
"recent_matches_limit",
recent_matches_limit,
)
snapshots: list[dict[str, object]] = []
for server_key in server_keys:
_log_snapshot_build_started(server_key, SNAPSHOT_TYPE_SERVER_SUMMARY)
snapshots.append(
_build_server_summary_snapshot(server_key, generated_at_value, db_path=db_path)
)
for metric in PREWARM_LEADERBOARD_METRICS:
_log_snapshot_build_started(
server_key,
SNAPSHOT_TYPE_WEEKLY_LEADERBOARD,
metric=metric,
)
snapshots.append(
_build_weekly_leaderboard_snapshot(
server_key,
@@ -220,6 +244,11 @@ def build_priority_historical_snapshots(
db_path=db_path,
)
)
_log_snapshot_build_started(
server_key,
SNAPSHOT_TYPE_MONTHLY_LEADERBOARD,
metric=metric,
)
snapshots.append(
_build_monthly_leaderboard_snapshot(
server_key,
@@ -229,6 +258,7 @@ def build_priority_historical_snapshots(
db_path=db_path,
)
)
_log_snapshot_build_started(server_key, SNAPSHOT_TYPE_MONTHLY_MVP)
snapshots.append(
_build_monthly_mvp_snapshot(
server_key,
@@ -237,6 +267,7 @@ def build_priority_historical_snapshots(
db_path=db_path,
)
)
_log_snapshot_build_started(server_key, SNAPSHOT_TYPE_MONTHLY_MVP_V2)
snapshots.append(
_build_monthly_mvp_v2_snapshot(
server_key,
@@ -246,6 +277,7 @@ def build_priority_historical_snapshots(
)
)
for snapshot_type in PLAYER_EVENT_SNAPSHOT_TYPES:
_log_snapshot_build_started(server_key, snapshot_type)
snapshots.append(
_build_player_event_snapshot(
server_key,
@@ -255,6 +287,7 @@ def build_priority_historical_snapshots(
db_path=db_path,
)
)
_log_snapshot_build_started(server_key, SNAPSHOT_TYPE_RECENT_MATCHES)
snapshots.append(
_build_recent_matches_snapshot(
server_key,
@@ -678,7 +711,7 @@ def _get_latest_player_event_month_key(
with _connect(resolved_path) as connection:
row = connection.execute(
f"""
SELECT MAX(substr(occurred_at, 1, 7)) AS latest_month
SELECT MAX(substr(CAST(occurred_at AS TEXT), 1, 7)) AS latest_month
FROM player_event_raw_ledger
WHERE occurred_at IS NOT NULL
AND {where_sql}
@@ -706,7 +739,7 @@ def _get_player_event_source_range(
MAX(occurred_at) AS source_range_end
FROM player_event_raw_ledger
WHERE occurred_at IS NOT NULL
AND substr(occurred_at, 1, 7) = ?
AND substr(CAST(occurred_at AS TEXT), 1, 7) = ?
AND {where_sql}
""",
[month_key, *params],
@@ -753,3 +786,34 @@ def _as_utc(value: datetime) -> datetime:
def _to_iso(value: datetime) -> str:
return _as_utc(value).isoformat().replace("+00:00", "Z")
def _normalize_snapshot_limit(name: str, value: object) -> int:
try:
limit = int(value)
except (TypeError, ValueError) as error:
raise ValueError(f"{name} must be a positive integer.") from error
if limit <= 0:
raise ValueError(f"{name} must be a positive integer.")
return limit
def _log_snapshot_build_started(
server_key: str,
snapshot_type: str,
*,
metric: str | None = None,
) -> None:
print(
json.dumps(
{
"event": "historical-snapshot-build-started",
"server_key": server_key,
"snapshot_type": snapshot_type,
"metric": metric,
},
ensure_ascii=True,
default=str,
),
flush=True,
)

10
backend/app/historical_storage.py
View File

@@ -1842,7 +1842,7 @@ def list_monthly_mvp_v2_ranking(
FROM player_event_raw_ledger
WHERE event_type = 'player_kill_summary'
AND occurred_at IS NOT NULL
AND substr(occurred_at, 1, 7) = ?
AND substr(CAST(occurred_at AS TEXT), 1, 7) = ?
AND {event_scope_sql}
AND killer_player_key IS NOT NULL
AND victim_player_key IS NOT NULL
@@ -1863,7 +1863,7 @@ def list_monthly_mvp_v2_ranking(
FROM player_event_raw_ledger
WHERE event_type = 'player_death_summary'
AND occurred_at IS NOT NULL
AND substr(occurred_at, 1, 7) = ?
AND substr(CAST(occurred_at AS TEXT), 1, 7) = ?
AND {event_scope_sql}
AND killer_player_key IS NOT NULL
AND victim_player_key IS NOT NULL
@@ -1901,7 +1901,7 @@ def list_monthly_mvp_v2_ranking(
FROM player_event_raw_ledger
WHERE event_type = 'player_kill_summary'
AND occurred_at IS NOT NULL
AND substr(occurred_at, 1, 7) = ?
AND substr(CAST(occurred_at AS TEXT), 1, 7) = ?
AND {event_scope_sql}
AND killer_player_key IS NOT NULL
AND victim_player_key IS NOT NULL
@@ -2020,7 +2020,7 @@ def _get_monthly_player_event_coverage(
with _connect(db_path) as connection:
latest_row = connection.execute(
f"""
SELECT MAX(substr(occurred_at, 1, 7)) AS latest_month_key
SELECT MAX(substr(CAST(occurred_at AS TEXT), 1, 7)) AS latest_month_key
FROM player_event_raw_ledger
WHERE occurred_at IS NOT NULL
AND {scope_sql}
@@ -2035,7 +2035,7 @@ def _get_monthly_player_event_coverage(
MAX(occurred_at) AS source_range_end
FROM player_event_raw_ledger
WHERE occurred_at IS NOT NULL
AND substr(occurred_at, 1, 7) = ?
AND substr(CAST(occurred_at AS TEXT), 1, 7) = ?
AND {scope_sql}
""",
[month_key, *scope_params],

7
backend/app/player_event_aggregates.py
View File

@@ -108,7 +108,10 @@ def list_net_duel_summaries(
COALESCE(SUM(net_value), 0) AS net_duel_value
FROM duel_pairs
GROUP BY player_a_key, player_a_name, player_b_key, player_b_name
ORDER BY ABS(net_duel_value) DESC, total_encounters DESC, player_a_name ASC, player_b_name ASC
ORDER BY ABS(COALESCE(SUM(net_value), 0)) DESC,
COALESCE(SUM(event_value), 0) DESC,
player_a_name ASC,
player_b_name ASC
LIMIT ?
""",
[*params, limit],
@@ -239,7 +242,7 @@ def _build_common_where(
clauses.append("server_slug = ?")
params.append(server_slug.strip())
if month:
clauses.append("substr(COALESCE(occurred_at, ''), 1, 7) = ?")
clauses.append("substr(COALESCE(CAST(occurred_at AS TEXT), ''), 1, 7) = ?")
params.append(month.strip())
if external_match_id:
clauses.append("external_match_id = ?")

13
backend/app/postgres_display_storage.py
View File

@@ -218,7 +218,12 @@ def persist_snapshot_record(snapshot: Mapping[str, object]) -> HistoricalSnapsho
metric = str(snapshot.get("metric") or "")
window = str(snapshot.get("window") or "")
payload = snapshot.get("payload")
payload_json = json.dumps(payload, ensure_ascii=True, separators=(",", ":"))
payload_json = json.dumps(
payload,
ensure_ascii=True,
separators=(",", ":"),
default=_json_payload_default,
)
with connect_postgres() as connection:
connection.execute(
"""
@@ -904,6 +909,12 @@ def _iso(value: object) -> str | None:
return text or None
def _json_payload_default(value: object) -> str:
if isinstance(value, datetime):
return _iso(value) or ""
raise TypeError(f"Object of type {type(value).__name__} is not JSON serializable")
def _parse_datetime(value: str | None) -> datetime | None:
if not value:
return None

35
backend/app/rcon_historical_read_model.py
View File

@@ -582,43 +582,38 @@ def _minutes_since_timestamp(timestamp: str | None) -> int | None:
def _parse_datetime(value: object) -> datetime | None:
if not isinstance(value, str) or not value.strip():
return None
if isinstance(value, datetime):
parsed = value
elif isinstance(value, str) and value.strip():
try:
parsed = datetime.fromisoformat(value.strip().replace("Z", "+00:00"))
except ValueError:
return None
else:
return None
if parsed.tzinfo is None:
parsed = parsed.replace(tzinfo=timezone.utc)
return parsed.astimezone(timezone.utc)
def _calculate_coverage_hours(
first_sample_at: str | None,
last_sample_at: str | None,
first_sample_at: object,
last_sample_at: object,
) -> float | None:
if not first_sample_at or not last_sample_at:
first_point = _parse_datetime(first_sample_at)
last_point = _parse_datetime(last_sample_at)
if first_point is None or last_point is None:
return None
first_point = datetime.fromisoformat(first_sample_at.replace("Z", "+00:00"))
last_point = datetime.fromisoformat(last_sample_at.replace("Z", "+00:00"))
if first_point.tzinfo is None:
first_point = first_point.replace(tzinfo=timezone.utc)
if last_point.tzinfo is None:
last_point = last_point.replace(tzinfo=timezone.utc)
delta = last_point.astimezone(timezone.utc) - first_point.astimezone(timezone.utc)
delta = last_point - first_point
return round(delta.total_seconds() / 3600, 2)
def _calculate_duration_seconds(first_seen_at: object, last_seen_at: object) -> int | None:
if not isinstance(first_seen_at, str) or not isinstance(last_seen_at, str):
first_point = _parse_datetime(first_seen_at)
last_point = _parse_datetime(last_seen_at)
if first_point is None or last_point is None:
return None
first_point = datetime.fromisoformat(first_seen_at.replace("Z", "+00:00"))
last_point = datetime.fromisoformat(last_seen_at.replace("Z", "+00:00"))
if first_point.tzinfo is None:
first_point = first_point.replace(tzinfo=timezone.utc)
if last_point.tzinfo is None:
last_point = last_point.replace(tzinfo=timezone.utc)
return max(0, int((last_point.astimezone(timezone.utc) - first_point.astimezone(timezone.utc)).total_seconds()))
return max(0, int((last_point - first_point).total_seconds()))
def _calculate_match_duration_seconds(item: dict[str, object]) -> int | None:

126
backend/tests/test_historical_snapshot_refresh.py Normal file
View File

@@ -0,0 +1,126 @@
"""Regression coverage for historical snapshot runner refreshes."""
from __future__ import annotations
import io
import json
import os
import unittest
from contextlib import nullcontext, redirect_stdout
from datetime import datetime, timezone
from unittest.mock import patch
from app.config import (
get_historical_refresh_interval_seconds,
get_historical_refresh_max_retries,
get_historical_refresh_retry_delay_seconds,
)
from app.historical_runner import _run_refresh_with_retries, run_periodic_historical_refresh
from app.historical_snapshots import _normalize_snapshot_limit
from app.postgres_display_storage import _json_payload_default
from app.rcon_historical_read_model import (
_calculate_coverage_hours,
_calculate_duration_seconds,
)
class HistoricalSnapshotRefreshTests(unittest.TestCase):
def test_runner_numeric_env_values_are_parsed_before_use(self) -> None:
with patch.dict(
os.environ,
{
"HLL_HISTORICAL_SNAPSHOT_REFRESH_INTERVAL_SECONDS": "300",
"HLL_HISTORICAL_REFRESH_MAX_RETRIES": "4",
"HLL_HISTORICAL_REFRESH_RETRY_DELAY_SECONDS": "0.5",
},
clear=False,
):
self.assertEqual(get_historical_refresh_interval_seconds(), 300)
self.assertEqual(get_historical_refresh_max_retries(), 4)
self.assertEqual(get_historical_refresh_retry_delay_seconds(), 0.5)
def test_runner_numeric_env_values_fail_with_clear_names(self) -> None:
with patch.dict(
os.environ,
{"HLL_HISTORICAL_SNAPSHOT_REFRESH_INTERVAL_SECONDS": "hourly"},
clear=False,
):
with self.assertRaisesRegex(
ValueError,
"HLL_HISTORICAL_SNAPSHOT_REFRESH_INTERVAL_SECONDS must be an integer",
):
get_historical_refresh_interval_seconds()
def test_rcon_coverage_accepts_postgres_datetime_values(self) -> None:
start = datetime(2026, 5, 21, 10, 0, tzinfo=timezone.utc)
end = datetime(2026, 5, 21, 11, 30, tzinfo=timezone.utc)
self.assertEqual(_calculate_coverage_hours(start, end), 1.5)
self.assertEqual(_calculate_duration_seconds(start, end), 5400)
def test_snapshot_limits_are_numeric_before_snapshot_queries(self) -> None:
self.assertEqual(_normalize_snapshot_limit("recent_matches_limit", "10"), 10)
with self.assertRaisesRegex(ValueError, "recent_matches_limit"):
_normalize_snapshot_limit("recent_matches_limit", "ten")
def test_postgres_snapshot_payload_serializes_datetime_values(self) -> None:
payload = {
"captured_at": datetime(2026, 5, 21, 20, 12, 54, tzinfo=timezone.utc),
}
self.assertEqual(
json.loads(json.dumps(payload, default=_json_payload_default)),
{"captured_at": "2026-05-21T20:12:54Z"},
)
def test_runner_failure_log_includes_exception_type_and_traceback(self) -> None:
stream = io.StringIO()
with (
patch("app.historical_runner.backend_writer_lock", return_value=nullcontext()),
patch(
"app.historical_runner._run_primary_rcon_capture",
side_effect=TypeError("bad timestamp"),
),
redirect_stdout(stream),
):
result = _run_refresh_with_retries(
max_retries=0,
retry_delay_seconds=0,
server_slug=None,
max_pages=None,
page_size=None,
run_number=1,
)
self.assertEqual(result["status"], "error")
self.assertEqual(result["error_type"], "TypeError")
self.assertIn("Traceback", result["traceback"])
self.assertIn('"event": "historical-refresh-attempt-failed"', stream.getvalue())
def test_runner_success_log_serializes_datetime_values(self) -> None:
stream = io.StringIO()
with (
patch(
"app.historical_runner._run_refresh_with_retries",
return_value={
"status": "ok",
"rcon_capture_result": {
"captured_at": datetime(2026, 5, 22, tzinfo=timezone.utc),
},
},
),
redirect_stdout(stream),
):
run_periodic_historical_refresh(
interval_seconds=1,
max_retries=0,
retry_delay_seconds=0,
max_runs=1,
)
self.assertIn('"status": "ok"', stream.getvalue())
self.assertIn('"captured_at": "2026-05-22 00:00:00+00:00"', stream.getvalue())
if __name__ == "__main__":
unittest.main()