Add staged RCON handshake diagnostics

This commit is contained in:
devRaGonSa
2026-03-26 09:19:21 +01:00
parent 0779331375
commit 231113d935
3 changed files with 262 additions and 35 deletions

View File

@@ -334,6 +334,7 @@ Variables especificas de captura historica prospectiva RCON:
`HLL_BACKEND_RCON_TARGETS` acepta un array JSON con: `HLL_BACKEND_RCON_TARGETS` acepta un array JSON con:
- `name` - `name`
- `slug` opcional como alias legacy
- `host` - `host`
- `port` - `port`
- `password` - `password`
@@ -343,19 +344,52 @@ Variables especificas de captura historica prospectiva RCON:
- `query_port` opcional - `query_port` opcional
- `source_name` opcional - `source_name` opcional
Compatibilidad operativa del loader:
- si llega `slug` pero no `external_server_id`, el backend reutiliza `slug`
como `external_server_id`
- si falta `name` pero existe `slug`, el backend genera un nombre razonable a
partir del slug en vez de dejar `Unnamed RCON target`
- los errores de validacion indican el campo que falta y las claves
efectivamente recibidas
Timeout recomendado por defecto:
- `HLL_BACKEND_RCON_TIMEOUT_SECONDS=20`
Diagnostico operativo del cliente RCON:
- el cliente informa ahora el stage exacto del fallo cuando puede distinguirlo
- stages observables:
- `tcp_connect`
- `server_connect_request`
- `server_connect_response`
- `xor_key_decode`
- `login_request`
- `login_response`
- `get_server_information_request`
- `get_server_information_response`
- `payload_decode`
- `unexpected_response`
- `timeout`
- esto mejora el diagnostico del protocolo, pero no resuelve por si solo la
conectividad real si el servidor acepta TCP y luego no responde al handshake
RCON o al comando `GetServerInformation`
Ejemplo: Ejemplo:
```powershell ```powershell
$env:HLL_BACKEND_RCON_TARGETS='[ $env:HLL_BACKEND_RCON_TARGETS='[
{ {
"name": "Comunidad Hispana #01", "name": "Comunidad Hispana #01",
"host": "203.0.113.10", "slug": "comunidad-hispana-01",
"port": 28015, "host": "152.114.195.174",
"port": 7779,
"password": "replace-me", "password": "replace-me",
"external_server_id": "comunidad-hispana-01", "external_server_id": "comunidad-hispana-01",
"region": "ES", "region": "ES",
"game_port": 7777, "game_port": null,
"query_port": 7778, "query_port": null,
"source_name": "community-hispana-rcon" "source_name": "community-hispana-rcon"
} }
]' ]'
@@ -423,6 +457,26 @@ La salida del worker incluye:
- `errors` - `errors`
- `storage_status` - `storage_status`
Cuando una captura falla, cada error incluye como minimo:
- `target_key`
- `external_server_id`
- `name`
- `host`
- `port`
- `timeout_seconds`
- `error_type`
- `error_stage`
- `message`
`error_type` intenta clasificar al menos:
- `timeout`
- `auth/login`
- `connection-refused`
- `payload-invalid`
- `other-error`
La persistencia queda separada del historico `historical_*` actual y usa: La persistencia queda separada del historico `historical_*` actual y usa:
- `rcon_historical_targets` - `rcon_historical_targets`
@@ -531,6 +585,17 @@ comparten el mismo SQLite, incluyendo:
- `rcon_historical_storage.py` - `rcon_historical_storage.py`
- `storage.py` - `storage.py`
Politica read-only para historico:
- las rutas de lectura de `historical_storage.py` no ejecutan ya
`initialize_historical_storage()`
- si el SQLite historico todavia no existe, esas lecturas devuelven resultados
vacios o defaults estables sin crear archivo ni correr seed/migraciones
- cuando el archivo ya existe, esas lecturas abren `mode=ro` con
`row_factory = sqlite3.Row` y `PRAGMA busy_timeout`
- la inicializacion, migraciones, seed y normalizaciones siguen reservadas al
writer path explicito
Variable opcional: Variable opcional:
- `HLL_BACKEND_STORAGE_PATH` - `HLL_BACKEND_STORAGE_PATH`
@@ -1448,6 +1513,19 @@ Estado real a fecha de esta fase:
en ventanas derivadas desde persistencia `rcon_historical_*` en ventanas derivadas desde persistencia `rcon_historical_*`
- `server-summary` y `recent-matches` pasan a usar esa capa RCON-backed como - `server-summary` y `recent-matches` pasan a usar esa capa RCON-backed como
camino principal real camino principal real
- en runtime, esas dos rutas solo se sirven como `rcon` cuando la capability
sigue soportada y existe cobertura RCON util para el scope pedido
- cobertura util en esta frontera significa:
- `server-summary`: al menos una fila con `coverage.status != "empty"` y
`window_count` o `sample_count` mayor que cero
- `recent-matches`: al menos una ventana con `match_id`, `closed_at` y
`sample_count > 0`
- si el target persistido quedo con clave legacy `rcon:<host>:<port>` pero el
runtime actual ya conoce su `external_server_id`, la capa read model intenta
resolver ambos aliases antes de caer a fallback
- si no hay coverage suficiente o la lectura RCON falla, el backend mantiene
fallback explicito a `public-scoreboard` con `fallback_used = true` y
`fallback_reason` visible
- `historical_ingestion` intenta primero una captura writer-oriented por RCON y - `historical_ingestion` intenta primero una captura writer-oriented por RCON y
deja esa tentativa visible en su salida deja esa tentativa visible en su salida
- leaderboards semanales/mensuales, MVP V1/V2 y player-events siguen teniendo - leaderboards semanales/mensuales, MVP V1/V2 y player-events siguen teniendo

View File

@@ -40,9 +40,16 @@ class RconServerTarget:
class RconQueryError(RuntimeError): class RconQueryError(RuntimeError):
"""Normalized RCON query failure with a machine-readable error type.""" """Normalized RCON query failure with a machine-readable error type."""
def __init__(self, error_type: str, message: str) -> None: def __init__(
self,
error_type: str,
message: str,
*,
error_stage: str | None = None,
) -> None:
super().__init__(message) super().__init__(message)
self.error_type = error_type self.error_type = error_type
self.error_stage = error_stage
class HllRconConnection: class HllRconConnection:
@@ -54,30 +61,64 @@ class HllRconConnection:
self._xor_key: bytes | None = None self._xor_key: bytes | None = None
self._auth_token: str | None = None self._auth_token: str | None = None
self._request_ids = itertools.count(1) self._request_ids = itertools.count(1)
self._current_stage = "tcp_connect"
def connect(self, *, host: str, port: int, password: str) -> None: def connect(self, *, host: str, port: int, password: str) -> None:
self._socket.connect((host, port)) self._run_socket_stage(
"tcp_connect",
lambda: self._socket.connect((host, port)),
)
server_connect_response = self._exchange("ServerConnect", "") server_connect_response = self._exchange(
"ServerConnect",
"",
request_stage="server_connect_request",
response_stage="server_connect_response",
)
self._current_stage = "xor_key_decode"
xor_key_b64 = _expect_text_content(server_connect_response, command_name="ServerConnect") xor_key_b64 = _expect_text_content(server_connect_response, command_name="ServerConnect")
try: try:
self._xor_key = base64.b64decode(xor_key_b64) self._xor_key = base64.b64decode(xor_key_b64)
except (ValueError, TypeError) as error: except (ValueError, TypeError) as error:
raise RuntimeError("The HLL server returned an invalid RCON XOR key.") from error raise RconQueryError(
"payload-invalid",
"The HLL server returned an invalid RCON XOR key.",
error_stage="xor_key_decode",
) from error
if not self._xor_key: if not self._xor_key:
raise RuntimeError("The HLL server returned an empty RCON XOR key.") raise RconQueryError(
"unexpected-response",
"The HLL server returned an empty RCON XOR key.",
error_stage="xor_key_decode",
)
login_response = self._exchange("Login", password) login_response = self._exchange(
"Login",
password,
request_stage="login_request",
response_stage="login_response",
)
self._auth_token = _expect_text_content(login_response, command_name="Login") self._auth_token = _expect_text_content(login_response, command_name="Login")
if not self._auth_token: if not self._auth_token:
raise RuntimeError("The HLL server returned an empty RCON auth token.") raise RconQueryError(
"unexpected-response",
"The HLL server returned an empty RCON auth token.",
error_stage="login_response",
)
def execute_json( def execute_json(
self, self,
command: str, command: str,
content: dict[str, object] | str = "", content: dict[str, object] | str = "",
) -> dict[str, object]: ) -> dict[str, object]:
response = self._exchange(command, content) stage_prefix = _resolve_command_stage_prefix(command)
response = self._exchange(
command,
content,
request_stage=f"{stage_prefix}_request",
response_stage=f"{stage_prefix}_response",
)
self._current_stage = "payload_decode"
content_body = response.get("contentBody") content_body = response.get("contentBody")
if isinstance(content_body, dict): if isinstance(content_body, dict):
return content_body return content_body
@@ -85,12 +126,18 @@ class HllRconConnection:
try: try:
parsed = json.loads(content_body) parsed = json.loads(content_body)
except json.JSONDecodeError as error: except json.JSONDecodeError as error:
raise RuntimeError( raise RconQueryError(
f"The HLL server returned invalid JSON content for {command}." "payload-invalid",
f"The HLL server returned invalid JSON content for {command}.",
error_stage="payload_decode",
) from error ) from error
if isinstance(parsed, dict): if isinstance(parsed, dict):
return parsed return parsed
raise RuntimeError(f"The HLL server returned an unexpected payload for {command}.") raise RconQueryError(
"unexpected-response",
f"The HLL server returned an unexpected payload for {command}.",
error_stage="unexpected_response",
)
def close(self) -> None: def close(self) -> None:
try: try:
@@ -103,16 +150,26 @@ class HllRconConnection:
self, self,
command: str, command: str,
content: dict[str, object] | str = "", content: dict[str, object] | str = "",
*,
request_stage: str,
response_stage: str,
) -> dict[str, object]: ) -> dict[str, object]:
request_id = next(self._request_ids) request_id = next(self._request_ids)
self._send_request(request_id=request_id, command=command, content=content) self._send_request(
response = self._receive_response() request_id=request_id,
command=command,
content=content,
request_stage=request_stage,
)
response = self._receive_response(response_stage=response_stage)
response_request_id = int(response.get("requestId") or 0) response_request_id = int(response.get("requestId") or 0)
if response_request_id != request_id: if response_request_id != request_id:
raise RuntimeError( raise RconQueryError(
f"Unexpected RCON response id {response_request_id} for request {request_id}." "unexpected-response",
f"Unexpected RCON response id {response_request_id} for request {request_id}.",
error_stage="unexpected_response",
) )
_raise_for_status(response, command_name=command) _raise_for_status(response, command_name=command, error_stage=response_stage)
return response return response
def _send_request( def _send_request(
@@ -121,6 +178,7 @@ class HllRconConnection:
request_id: int, request_id: int,
command: str, command: str,
content: dict[str, object] | str, content: dict[str, object] | str,
request_stage: str,
) -> None: ) -> None:
content_body = ( content_body = (
content content
@@ -137,35 +195,72 @@ class HllRconConnection:
separators=(",", ":"), separators=(",", ":"),
).encode("utf-8") ).encode("utf-8")
header = struct.pack(RCON_HEADER_FORMAT, request_id, len(body)) header = struct.pack(RCON_HEADER_FORMAT, request_id, len(body))
self._socket.sendall(header + self._xor(body)) self._run_socket_stage(
request_stage,
lambda: self._socket.sendall(header + self._xor(body)),
)
def _receive_response(self) -> dict[str, object]: def _receive_response(self, *, response_stage: str) -> dict[str, object]:
header_size = struct.calcsize(RCON_HEADER_FORMAT) header_size = struct.calcsize(RCON_HEADER_FORMAT)
header_bytes = self._recv_exact(header_size) header_bytes = self._recv_exact(header_size, stage=response_stage)
try: try:
request_id, body_length = struct.unpack(RCON_HEADER_FORMAT, header_bytes) request_id, body_length = struct.unpack(RCON_HEADER_FORMAT, header_bytes)
except struct.error as error: except struct.error as error:
raise RuntimeError("The HLL server returned an invalid RCON response header.") from error raise RconQueryError(
"payload-invalid",
"The HLL server returned an invalid RCON response header.",
error_stage=response_stage,
) from error
if body_length <= 0: if body_length <= 0:
raise RuntimeError("The HLL server returned an empty RCON response body.") raise RconQueryError(
"unexpected-response",
"The HLL server returned an empty RCON response body.",
error_stage=response_stage,
)
body = self._xor(self._recv_exact(body_length)) body = self._xor(self._recv_exact(body_length, stage=response_stage))
try: try:
parsed = json.loads(body.decode("utf-8", errors="replace")) parsed = json.loads(body.decode("utf-8", errors="replace"))
except json.JSONDecodeError as error: except json.JSONDecodeError as error:
raise RuntimeError("The HLL server returned malformed RCON JSON.") from error raise RconQueryError(
"payload-invalid",
"The HLL server returned malformed RCON JSON.",
error_stage="payload_decode",
) from error
if not isinstance(parsed, dict): if not isinstance(parsed, dict):
raise RuntimeError("The HLL server returned a non-object RCON response.") raise RconQueryError(
"unexpected-response",
"The HLL server returned a non-object RCON response.",
error_stage="unexpected_response",
)
parsed["requestId"] = request_id parsed["requestId"] = request_id
return parsed return parsed
def _recv_exact(self, expected_length: int) -> bytes: def _recv_exact(self, expected_length: int, *, stage: str) -> bytes:
chunks = bytearray() chunks = bytearray()
while len(chunks) < expected_length: while len(chunks) < expected_length:
chunk = self._socket.recv(min(RCON_BUFFER_SIZE, expected_length - len(chunks))) self._current_stage = stage
try:
chunk = self._socket.recv(min(RCON_BUFFER_SIZE, expected_length - len(chunks)))
except (TimeoutError, socket.timeout) as error:
raise RconQueryError(
"timeout",
f"Timed out during {stage}.",
error_stage=stage,
) from error
except OSError as error:
raise RconQueryError(
_classify_socket_error_type(error),
f"RCON socket error during {stage}: {error}",
error_stage=stage,
) from error
if not chunk: if not chunk:
raise RuntimeError("The HLL RCON connection closed unexpectedly.") raise RconQueryError(
"unexpected-response",
"The HLL RCON connection closed unexpectedly.",
error_stage=stage,
)
chunks.extend(chunk) chunks.extend(chunk)
return bytes(chunks) return bytes(chunks)
@@ -183,6 +278,23 @@ class HllRconConnection:
def __exit__(self, exc_type: object, exc: object, traceback: object) -> None: def __exit__(self, exc_type: object, exc: object, traceback: object) -> None:
self.close() self.close()
def _run_socket_stage(self, stage: str, operation: object) -> object:
self._current_stage = stage
try:
return operation()
except (TimeoutError, socket.timeout) as error:
raise RconQueryError(
"timeout",
f"Timed out during {stage}.",
error_stage=stage,
) from error
except OSError as error:
raise RconQueryError(
_classify_socket_error_type(error),
f"RCON socket error during {stage}: {error}",
error_stage=stage,
) from error
def load_rcon_targets() -> tuple[RconServerTarget, ...]: def load_rcon_targets() -> tuple[RconServerTarget, ...]:
"""Load RCON targets from JSON env payload.""" """Load RCON targets from JSON env payload."""
@@ -240,6 +352,7 @@ def query_live_server_sample(
raise RconQueryError( raise RconQueryError(
_classify_runtime_error_type(error), _classify_runtime_error_type(error),
str(error), str(error),
error_stage=getattr(error, "error_stage", None),
) from error ) from error
resolved_external_id = target.external_server_id or f"rcon:{target.host}:{target.port}" resolved_external_id = target.external_server_id or f"rcon:{target.host}:{target.port}"
@@ -309,7 +422,12 @@ def _coerce_rcon_target(raw_target: dict[str, object]) -> RconServerTarget:
) )
def _raise_for_status(response: dict[str, object], *, command_name: str) -> None: def _raise_for_status(
response: dict[str, object],
*,
command_name: str,
error_stage: str,
) -> None:
status_code = int(response.get("statusCode") or 0) status_code = int(response.get("statusCode") or 0)
if status_code == 200: if status_code == 200:
return return
@@ -318,15 +436,34 @@ def _raise_for_status(response: dict[str, object], *, command_name: str) -> None
raise RconQueryError( raise RconQueryError(
"auth/login", "auth/login",
f"{command_name} failed with RCON status {status_code}: {status_message}", f"{command_name} failed with RCON status {status_code}: {status_message}",
error_stage=error_stage,
) )
raise RuntimeError(f"{command_name} failed with RCON status {status_code}: {status_message}") raise RconQueryError(
"unexpected-response",
f"{command_name} failed with RCON status {status_code}: {status_message}",
error_stage=error_stage,
)
def _expect_text_content(response: dict[str, object], *, command_name: str) -> str: def _expect_text_content(response: dict[str, object], *, command_name: str) -> str:
content = response.get("contentBody") content = response.get("contentBody")
if isinstance(content, str): if isinstance(content, str):
return content return content
raise RuntimeError(f"The HLL server returned unexpected text content for {command_name}.") raise RconQueryError(
"unexpected-response",
f"The HLL server returned unexpected text content for {command_name}.",
error_stage="unexpected_response",
)
def _resolve_command_stage_prefix(command: str) -> str:
normalized_command = str(command or "").strip().lower()
stage_prefix_by_command = {
"serverconnect": "server_connect",
"login": "login",
"getserverinformation": "get_server_information",
}
return stage_prefix_by_command.get(normalized_command, normalized_command or "rcon_command")
def _string_or_none(value: object) -> str | None: def _string_or_none(value: object) -> str | None:

View File

@@ -252,6 +252,7 @@ def _serialize_capture_error(
timeout_seconds: float, timeout_seconds: float,
) -> dict[str, object]: ) -> dict[str, object]:
error_type = _classify_capture_error_type(error) error_type = _classify_capture_error_type(error)
error_stage = _classify_capture_error_stage(error)
return { return {
"target_key": build_rcon_target_key(target), "target_key": build_rcon_target_key(target),
"external_server_id": target.external_server_id, "external_server_id": target.external_server_id,
@@ -260,6 +261,7 @@ def _serialize_capture_error(
"port": target.port, "port": target.port,
"timeout_seconds": timeout_seconds, "timeout_seconds": timeout_seconds,
"error_type": error_type, "error_type": error_type,
"error_stage": error_stage,
"message": str(error), "message": str(error),
} }
@@ -279,8 +281,18 @@ def _classify_capture_error_type(error: Exception) -> str:
return "other-error" return "other-error"
def _classify_capture_error_stage(error: Exception) -> str | None:
if isinstance(error, RconQueryError):
return error.error_stage
return None
def _format_error_message(error: Exception) -> str: def _format_error_message(error: Exception) -> str:
return f"[{_classify_capture_error_type(error)}] {error}" error_type = _classify_capture_error_type(error)
error_stage = _classify_capture_error_stage(error)
if error_stage:
return f"[{error_type}:{error_stage}] {error}"
return f"[{error_type}] {error}"
def build_arg_parser() -> argparse.ArgumentParser: def build_arg_parser() -> argparse.ArgumentParser: