Restore near-real-time public server status

This commit is contained in:
devRaGonSa
2026-06-10 20:12:00 +02:00
parent ba37000360
commit 68c6b5dac0
7 changed files with 434 additions and 37 deletions

View File

@@ -5,6 +5,7 @@ from __future__ import annotations
from dataclasses import dataclass
from typing import Protocol
from .a2s_client import DEFAULT_A2S_TIMEOUT
from .collector import collect_server_snapshots
from .config import get_historical_data_source_kind, get_live_data_source_kind
from .providers.public_scoreboard_provider import PublicScoreboardHistoricalDataSource
@@ -48,7 +49,12 @@ class LiveDataSource(Protocol):
source_kind: str
def collect_snapshots(self, *, persist: bool) -> dict[str, object]:
def collect_snapshots(
self,
*,
persist: bool,
timeout_seconds: float | None = None,
) -> dict[str, object]:
"""Collect one live snapshot batch."""
def build_target_index(self) -> dict[str | None, object]:
@@ -61,11 +67,17 @@ class A2SLiveDataSource:
source_kind: str = LIVE_SOURCE_A2S
def collect_snapshots(self, *, persist: bool) -> dict[str, object]:
def collect_snapshots(
self,
*,
persist: bool,
timeout_seconds: float | None = None,
) -> dict[str, object]:
return collect_server_snapshots(
source_mode="a2s",
allow_controlled_fallback=False,
persist=persist,
timeout=timeout_seconds if timeout_seconds is not None else DEFAULT_A2S_TIMEOUT,
)
def build_target_index(self) -> dict[str | None, A2SServerTarget]:
@@ -84,12 +96,20 @@ class RconFirstLiveDataSource:
fallback_source: A2SLiveDataSource = A2SLiveDataSource()
source_kind: str = SOURCE_KIND_RCON
def collect_snapshots(self, *, persist: bool) -> dict[str, object]:
def collect_snapshots(
self,
*,
persist: bool,
timeout_seconds: float | None = None,
) -> dict[str, object]:
attempts: list[dict[str, object]] = []
fallback_reason: str | None = None
try:
primary_payload = self.primary_source.collect_snapshots(persist=persist)
primary_payload = self.primary_source.collect_snapshots(
persist=persist,
timeout_seconds=timeout_seconds,
)
except Exception as error: # noqa: BLE001 - source arbitration keeps fallback controlled
attempts.append(
build_source_attempt(
@@ -133,7 +153,10 @@ class RconFirstLiveDataSource:
fallback_reason = "rcon-live-returned-no-usable-snapshots"
try:
fallback_payload = self.fallback_source.collect_snapshots(persist=persist)
fallback_payload = self.fallback_source.collect_snapshots(
persist=persist,
timeout_seconds=timeout_seconds,
)
except Exception as error: # noqa: BLE001 - keep combined failure explicit
attempts.append(
build_source_attempt(

View File

@@ -63,6 +63,8 @@ from .rcon_admin_log_storage import list_current_match_kill_feed, list_current_m
from .scoreboard_origins import get_trusted_public_scoreboard_origin
from .storage import list_latest_snapshots, list_server_history, list_snapshot_history
PUBLIC_SERVER_STATUS_TIMEOUT_SECONDS = 2.5
def build_health_payload() -> dict[str, str]:
"""Return a small status payload without committing to business contracts."""
@@ -118,32 +120,49 @@ def build_discord_payload() -> dict[str, object]:
def build_servers_payload() -> dict[str, object]:
"""Return current server status from persisted snapshots only."""
"""Return current server status, refreshing stale snapshots before responding."""
max_snapshot_age_seconds = get_refresh_interval_seconds()
persisted_items = _select_primary_snapshot_items(
_enrich_server_items(list_latest_snapshots())
)
persisted_snapshot_at = _resolve_last_snapshot_at(persisted_items)
persisted_snapshot_age_seconds = _calculate_snapshot_age_seconds(persisted_snapshot_at)
refresh_attempted = _should_refresh_snapshot(
persisted_items,
persisted_snapshot_age_seconds,
max_snapshot_age_seconds,
)
refresh_errors: list[dict[str, object]] = []
refresh_source_policy = build_source_policy(
primary_source=get_live_data_source_kind(),
selected_source="persisted-snapshot" if persisted_items else "none",
selected_source="none",
fallback_reason=None,
source_attempts=[
build_source_attempt(
source="persisted-snapshot",
role="served-response",
status="success" if persisted_items else "empty",
reason="public-servers-read-is-cache-only",
)
],
source_attempts=[],
)
if refresh_attempted:
refreshed_items, refresh_errors, refresh_source_policy = _try_collect_real_time_snapshot()
if refreshed_items:
refreshed_snapshot_at = _resolve_last_snapshot_at(refreshed_items)
refreshed_snapshot_age_seconds = _calculate_snapshot_age_seconds(refreshed_snapshot_at)
return _build_servers_response(
items=refreshed_items,
response_source=_build_live_response_source(refresh_source_policy),
last_snapshot_at=refreshed_snapshot_at,
snapshot_age_seconds=refreshed_snapshot_age_seconds,
max_snapshot_age_seconds=max_snapshot_age_seconds,
refresh_attempted=True,
refresh_status="success",
refresh_errors=refresh_errors,
source_policy=refresh_source_policy,
)
if persisted_items:
refresh_status = "failed" if refresh_attempted else "not-needed"
response_source = (
"persisted-stale-snapshot"
if _is_snapshot_stale(persisted_snapshot_age_seconds, max_snapshot_age_seconds)
if refresh_attempted
else "persisted-fresh-snapshot"
)
return _build_servers_response(
@@ -152,12 +171,12 @@ def build_servers_payload() -> dict[str, object]:
last_snapshot_at=persisted_snapshot_at,
snapshot_age_seconds=persisted_snapshot_age_seconds,
max_snapshot_age_seconds=max_snapshot_age_seconds,
refresh_attempted=False,
refresh_status="cache-only",
refresh_attempted=refresh_attempted,
refresh_status=refresh_status,
refresh_errors=refresh_errors,
source_policy=_infer_live_source_policy_from_items(
persisted_items,
refresh_attempted=False,
refresh_attempted=refresh_attempted,
refresh_errors=refresh_errors,
),
)
@@ -174,8 +193,8 @@ def build_servers_payload() -> dict[str, object]:
"max_snapshot_age_seconds": max_snapshot_age_seconds,
"is_stale": True,
"freshness": "stale",
"refresh_attempted": False,
"refresh_status": "cache-only",
"refresh_attempted": refresh_attempted,
"refresh_status": "failed" if refresh_attempted else "not-needed",
"refresh_errors": refresh_errors,
**refresh_source_policy,
"items": [],
@@ -2391,7 +2410,39 @@ def _try_collect_real_time_snapshot() -> tuple[
list[dict[str, object]],
dict[str, object],
]:
payload = get_live_data_source().collect_snapshots(persist=False)
try:
payload = get_live_data_source().collect_snapshots(
persist=False,
timeout_seconds=PUBLIC_SERVER_STATUS_TIMEOUT_SECONDS,
)
except Exception as error: # noqa: BLE001 - public server status must degrade cleanly
reason = _public_server_refresh_error_reason(error)
return (
[],
[
{
"source": get_live_data_source_kind(),
"reason": reason,
"error_type": type(error).__name__,
"message": str(error),
}
],
build_source_policy(
primary_source=get_live_data_source_kind(),
selected_source="none",
fallback_used=True,
fallback_reason=reason,
source_attempts=[
build_source_attempt(
source=get_live_data_source_kind(),
role="primary",
status="error",
reason=reason,
message=str(error),
)
],
),
)
snapshots = payload.get("snapshots")
items = _select_primary_snapshot_items(_enrich_server_items(list(snapshots or [])))
errors = payload.get("errors")
@@ -2408,6 +2459,15 @@ def _try_collect_real_time_snapshot() -> tuple[
)
def _public_server_refresh_error_reason(error: Exception) -> str:
message = str(error).lower()
if isinstance(error, TimeoutError) or "timeout" in message or "timed out" in message:
return "live-refresh-timeout"
if "no rcon targets" in message or "no live" in message or "configured" in message:
return "live-refresh-unavailable"
return "live-refresh-failed"
def _build_servers_response(
*,
items: list[dict[str, object]],

View File

@@ -19,7 +19,12 @@ class RconLiveDataSource:
source_kind: str = "rcon"
def collect_snapshots(self, *, persist: bool) -> dict[str, object]:
def collect_snapshots(
self,
*,
persist: bool,
timeout_seconds: float | None = None,
) -> dict[str, object]:
configured_targets = load_rcon_targets()
if not configured_targets:
raise RuntimeError("No RCON targets configured in HLL_BACKEND_RCON_TARGETS.")
@@ -30,7 +35,12 @@ class RconLiveDataSource:
for target in configured_targets:
try:
normalized_records.append(query_live_server_sample(target)["normalized"])
normalized_records.append(
query_live_server_sample(
target,
timeout_seconds=timeout_seconds,
)["normalized"]
)
except Exception as error: # noqa: BLE001 - keep provider failures controlled
errors.append(
{