Fix
This commit is contained in:
11
backend/.dockerignore
Normal file
11
backend/.dockerignore
Normal file
@@ -0,0 +1,11 @@
|
||||
.git
|
||||
.gitignore
|
||||
.venv/
|
||||
__pycache__/
|
||||
*.pyc
|
||||
*.pyo
|
||||
*.pyd
|
||||
data/*.sqlite3
|
||||
data/snapshots/**
|
||||
!data/.gitkeep
|
||||
!data/snapshots/.gitkeep
|
||||
21
backend/.env.example
Normal file
21
backend/.env.example
Normal file
@@ -0,0 +1,21 @@
|
||||
HLL_BACKEND_HOST=0.0.0.0
|
||||
HLL_BACKEND_PORT=8000
|
||||
HLL_BACKEND_STORAGE_PATH=/app/data/hll_vietnam_dev.sqlite3
|
||||
HLL_BACKEND_DATABASE_URL=postgresql://hll_vietnam:hll_vietnam_dev@postgres:5432/hll_vietnam
|
||||
HLL_BACKEND_ALLOWED_ORIGINS=http://127.0.0.1,http://127.0.0.1:8080,http://localhost,http://localhost:8080
|
||||
HLL_BACKEND_REFRESH_INTERVAL_SECONDS=120
|
||||
HLL_BACKEND_LIVE_DATA_SOURCE=rcon
|
||||
HLL_BACKEND_HISTORICAL_DATA_SOURCE=rcon
|
||||
HLL_BACKEND_RCON_TIMEOUT_SECONDS=20
|
||||
HLL_BACKEND_RCON_TARGETS=[{"name":"Comunidad Hispana #01","slug":"comunidad-hispana-01","external_server_id":"comunidad-hispana-01","host":"152.114.195.174","port":7779,"password":"replace-me-01","source_name":"community-hispana-rcon","region":"ES","game_port":null,"query_port":null},{"name":"Comunidad Hispana #02","slug":"comunidad-hispana-02","external_server_id":"comunidad-hispana-02","host":"152.114.195.150","port":7879,"password":"replace-me-02","source_name":"community-hispana-rcon","region":"ES","game_port":null,"query_port":null}]
|
||||
HLL_HISTORICAL_CRCON_PAGE_SIZE=50
|
||||
HLL_HISTORICAL_CRCON_TIMEOUT_SECONDS=15
|
||||
HLL_HISTORICAL_CRCON_DETAIL_WORKERS=8
|
||||
HLL_HISTORICAL_CRCON_REQUEST_RETRIES=3
|
||||
HLL_HISTORICAL_CRCON_RETRY_DELAY_SECONDS=0.5
|
||||
HLL_HISTORICAL_SNAPSHOT_REFRESH_INTERVAL_SECONDS=900
|
||||
HLL_HISTORICAL_FULL_SNAPSHOT_EVERY_RUNS=4
|
||||
HLL_HISTORICAL_REFRESH_MAX_RETRIES=2
|
||||
HLL_HISTORICAL_REFRESH_RETRY_DELAY_SECONDS=30
|
||||
HLL_HISTORICAL_WEEKLY_FALLBACK_MIN_MATCHES=3
|
||||
HLL_HISTORICAL_WEEKLY_FALLBACK_MAX_WEEKDAY=2
|
||||
20
backend/Dockerfile
Normal file
20
backend/Dockerfile
Normal file
@@ -0,0 +1,20 @@
|
||||
FROM python:3.12-slim
|
||||
|
||||
ENV PYTHONDONTWRITEBYTECODE=1 \
|
||||
PYTHONUNBUFFERED=1 \
|
||||
HLL_BACKEND_HOST=0.0.0.0 \
|
||||
HLL_BACKEND_PORT=8000 \
|
||||
HLL_BACKEND_STORAGE_PATH=/app/data/hll_vietnam_dev.sqlite3
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
COPY requirements.txt ./
|
||||
RUN pip install --no-cache-dir --requirement requirements.txt
|
||||
|
||||
COPY app ./app
|
||||
|
||||
RUN mkdir -p /app/data/snapshots
|
||||
|
||||
EXPOSE 8000
|
||||
|
||||
CMD ["python", "-m", "app.main"]
|
||||
1685
backend/README.md
Normal file
1685
backend/README.md
Normal file
File diff suppressed because it is too large
Load Diff
59
backend/app/__init__.py
Normal file
59
backend/app/__init__.py
Normal file
@@ -0,0 +1,59 @@
|
||||
"""Minimal bootstrap package for the HLL Vietnam Python backend."""
|
||||
|
||||
from .config import get_allowed_origins, get_bind_address
|
||||
from .main import create_server, run
|
||||
from .normalizers import normalize_a2s_server_info, normalize_server_record
|
||||
from .payloads import build_health_payload
|
||||
from .routes import resolve_get_payload
|
||||
from .snapshots import build_server_snapshot, build_snapshot_batch, utc_now
|
||||
from .storage import initialize_storage, persist_snapshot_batch
|
||||
|
||||
|
||||
def collect_server_snapshots(*args: object, **kwargs: object) -> dict[str, object]:
|
||||
"""Proxy collector access without importing the module during package init."""
|
||||
from .collector import collect_server_snapshots as _collect_server_snapshots
|
||||
|
||||
return _collect_server_snapshots(*args, **kwargs)
|
||||
|
||||
|
||||
def fetch_a2s_probe(*args: object, **kwargs: object) -> dict[str, object]:
|
||||
"""Proxy A2S probe access without importing the collector during package init."""
|
||||
from .collector import fetch_a2s_probe as _fetch_a2s_probe
|
||||
|
||||
return _fetch_a2s_probe(*args, **kwargs)
|
||||
|
||||
|
||||
def query_server_info(*args: object, **kwargs: object) -> object:
|
||||
"""Proxy A2S info queries without importing the module during package init."""
|
||||
from .a2s_client import query_server_info as _query_server_info
|
||||
|
||||
return _query_server_info(*args, **kwargs)
|
||||
|
||||
|
||||
def fetch_controlled_server_source() -> tuple[dict[str, object], ...]:
|
||||
"""Proxy the controlled source without importing the module during package init."""
|
||||
from .collector import (
|
||||
fetch_controlled_server_source as _fetch_controlled_server_source,
|
||||
)
|
||||
|
||||
return tuple(_fetch_controlled_server_source())
|
||||
|
||||
__all__ = [
|
||||
"build_health_payload",
|
||||
"build_server_snapshot",
|
||||
"build_snapshot_batch",
|
||||
"collect_server_snapshots",
|
||||
"create_server",
|
||||
"fetch_a2s_probe",
|
||||
"fetch_controlled_server_source",
|
||||
"get_allowed_origins",
|
||||
"get_bind_address",
|
||||
"initialize_storage",
|
||||
"normalize_a2s_server_info",
|
||||
"normalize_server_record",
|
||||
"persist_snapshot_batch",
|
||||
"query_server_info",
|
||||
"resolve_get_payload",
|
||||
"run",
|
||||
"utc_now",
|
||||
]
|
||||
176
backend/app/a2s_client.py
Normal file
176
backend/app/a2s_client.py
Normal file
@@ -0,0 +1,176 @@
|
||||
"""Minimal Steam A2S info client for development-time HLL server probes."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import socket
|
||||
import struct
|
||||
from dataclasses import asdict, dataclass
|
||||
|
||||
|
||||
DEFAULT_A2S_TIMEOUT = 6.0
|
||||
_A2S_PREFIX = b"\xFF\xFF\xFF\xFF"
|
||||
_A2S_INFO_REQUEST = _A2S_PREFIX + b"\x54Source Engine Query\x00"
|
||||
_A2S_CHALLENGE_RESPONSE = 0x41
|
||||
_A2S_INFO_RESPONSE = 0x49
|
||||
|
||||
|
||||
class A2SError(RuntimeError):
|
||||
"""Base error for A2S query failures."""
|
||||
|
||||
|
||||
class A2STimeoutError(A2SError):
|
||||
"""Raised when an A2S query does not complete before the timeout."""
|
||||
|
||||
|
||||
class A2SProtocolError(A2SError):
|
||||
"""Raised when an A2S server returns an unexpected payload."""
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class A2SServerInfo:
|
||||
"""Minimal metadata returned by an A2S info query."""
|
||||
|
||||
host: str
|
||||
query_port: int
|
||||
server_name: str
|
||||
map_name: str | None
|
||||
players: int
|
||||
max_players: int
|
||||
protocol: int
|
||||
folder: str | None = None
|
||||
game: str | None = None
|
||||
version: str | None = None
|
||||
|
||||
|
||||
def query_server_info(
|
||||
host: str,
|
||||
query_port: int,
|
||||
*,
|
||||
timeout: float = DEFAULT_A2S_TIMEOUT,
|
||||
) -> A2SServerInfo:
|
||||
"""Query one server using A2S_INFO and return minimal reusable metadata."""
|
||||
with socket.socket(socket.AF_INET, socket.SOCK_DGRAM) as udp_socket:
|
||||
udp_socket.settimeout(timeout)
|
||||
address = (host, query_port)
|
||||
|
||||
try:
|
||||
udp_socket.sendto(_A2S_INFO_REQUEST, address)
|
||||
payload = _receive_packet(udp_socket)
|
||||
if _is_challenge_packet(payload):
|
||||
challenge = payload[5:9]
|
||||
udp_socket.sendto(_A2S_INFO_REQUEST + challenge, address)
|
||||
payload = _receive_packet(udp_socket)
|
||||
except socket.timeout as error:
|
||||
raise A2STimeoutError(
|
||||
f"A2S query to {host}:{query_port} timed out after {timeout:.1f}s."
|
||||
) from error
|
||||
except OSError as error:
|
||||
raise A2SError(
|
||||
f"A2S query to {host}:{query_port} failed: {error}."
|
||||
) from error
|
||||
|
||||
return _parse_info_payload(payload, host=host, query_port=query_port)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
"""Allow a direct development-time probe of one A2S target."""
|
||||
parser = argparse.ArgumentParser(description="Probe one server with A2S_INFO.")
|
||||
parser.add_argument("host", help="Server hostname or IPv4 address.")
|
||||
parser.add_argument("query_port", type=int, help="Server Steam query port.")
|
||||
parser.add_argument(
|
||||
"--timeout",
|
||||
type=float,
|
||||
default=DEFAULT_A2S_TIMEOUT,
|
||||
help="Socket timeout in seconds.",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
payload = asdict(
|
||||
query_server_info(args.host, args.query_port, timeout=args.timeout)
|
||||
)
|
||||
print(json.dumps(payload, indent=2))
|
||||
|
||||
|
||||
def _receive_packet(udp_socket: socket.socket) -> bytes:
|
||||
payload, _ = udp_socket.recvfrom(4096)
|
||||
return payload
|
||||
|
||||
|
||||
def _is_challenge_packet(payload: bytes) -> bool:
|
||||
return (
|
||||
len(payload) >= 9
|
||||
and payload.startswith(_A2S_PREFIX)
|
||||
and payload[4] == _A2S_CHALLENGE_RESPONSE
|
||||
)
|
||||
|
||||
|
||||
def _parse_info_payload(
|
||||
payload: bytes,
|
||||
*,
|
||||
host: str,
|
||||
query_port: int,
|
||||
) -> A2SServerInfo:
|
||||
if len(payload) < 6 or not payload.startswith(_A2S_PREFIX):
|
||||
raise A2SProtocolError("A2S response did not include the expected packet header.")
|
||||
if payload[4] != _A2S_INFO_RESPONSE:
|
||||
raise A2SProtocolError(
|
||||
f"A2S response type {payload[4]!r} is not an info response."
|
||||
)
|
||||
|
||||
protocol = payload[5]
|
||||
offset = 6
|
||||
server_name, offset = _read_c_string(payload, offset)
|
||||
map_name, offset = _read_c_string(payload, offset)
|
||||
folder, offset = _read_c_string(payload, offset)
|
||||
game, offset = _read_c_string(payload, offset)
|
||||
offset += 2 # app id
|
||||
players = _read_byte(payload, offset)
|
||||
max_players = _read_byte(payload, offset + 1)
|
||||
offset += 6 # players, max, bots, server type, environment, visibility
|
||||
offset += 1 # vac
|
||||
version, offset = _read_c_string(payload, offset)
|
||||
|
||||
if offset < len(payload):
|
||||
extra_data_flag = payload[offset]
|
||||
offset += 1
|
||||
if extra_data_flag & 0x80:
|
||||
offset += 2
|
||||
if extra_data_flag & 0x10:
|
||||
_, offset = _read_c_string(payload, offset)
|
||||
if extra_data_flag & 0x40:
|
||||
offset += 2
|
||||
offset += 8
|
||||
if extra_data_flag & 0x20:
|
||||
offset += 8
|
||||
|
||||
return A2SServerInfo(
|
||||
host=host,
|
||||
query_port=query_port,
|
||||
server_name=server_name or "Unknown server",
|
||||
map_name=map_name or None,
|
||||
players=players,
|
||||
max_players=max_players,
|
||||
protocol=protocol,
|
||||
folder=folder or None,
|
||||
game=game or None,
|
||||
version=version or None,
|
||||
)
|
||||
|
||||
|
||||
def _read_c_string(payload: bytes, offset: int) -> tuple[str, int]:
|
||||
end = payload.find(b"\x00", offset)
|
||||
if end == -1:
|
||||
raise A2SProtocolError("A2S response ended before a null-terminated string.")
|
||||
return payload[offset:end].decode("utf-8", errors="replace"), end + 1
|
||||
|
||||
|
||||
def _read_byte(payload: bytes, offset: int) -> int:
|
||||
if offset >= len(payload):
|
||||
raise A2SProtocolError("A2S response ended before expected integer fields.")
|
||||
return struct.unpack_from("<B", payload, offset)[0]
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
269
backend/app/collector.py
Normal file
269
backend/app/collector.py
Normal file
@@ -0,0 +1,269 @@
|
||||
"""Minimal collector bootstrap for provisional server snapshots."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Callable, Mapping, Sequence
|
||||
|
||||
from .a2s_client import DEFAULT_A2S_TIMEOUT, query_server_info
|
||||
from .normalizers import normalize_a2s_server_info, normalize_server_record
|
||||
from .server_targets import A2SServerTarget, load_a2s_targets
|
||||
from .snapshots import build_snapshot_batch, utc_now
|
||||
from .storage import persist_snapshot_batch
|
||||
|
||||
|
||||
RawSourceFetcher = Callable[[], Sequence[Mapping[str, object]]]
|
||||
TargetProbe = Callable[[A2SServerTarget, float], Mapping[str, object]]
|
||||
|
||||
|
||||
CONTROLLED_RAW_SERVER_SOURCE: tuple[dict[str, object], ...] = (
|
||||
{
|
||||
"external_server_id": "hll-esp-tactical-rotation",
|
||||
"server_name": "HLL ESP Tactical Rotation",
|
||||
"status": "online",
|
||||
"players": 74,
|
||||
"max_players": 100,
|
||||
"current_map": "Sainte-Marie-du-Mont",
|
||||
"region": "EU",
|
||||
},
|
||||
{
|
||||
"external_server_id": "hll-latam-night-offensive",
|
||||
"server_name": "HLL LATAM Night Offensive",
|
||||
"status": "online",
|
||||
"players": 51,
|
||||
"max_players": 100,
|
||||
"current_map": "Carentan",
|
||||
"region": "LATAM",
|
||||
},
|
||||
{
|
||||
"external_server_id": "hll-community-reserve",
|
||||
"server_name": "HLL Community Reserve",
|
||||
"status": "offline",
|
||||
"players": 0,
|
||||
"max_players": 100,
|
||||
"current_map": None,
|
||||
"region": "EU",
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def fetch_controlled_server_source() -> Sequence[Mapping[str, object]]:
|
||||
"""Return the controlled development source used by the collector bootstrap."""
|
||||
return CONTROLLED_RAW_SERVER_SOURCE
|
||||
|
||||
|
||||
def fetch_a2s_probe(
|
||||
host: str,
|
||||
query_port: int,
|
||||
*,
|
||||
timeout: float = DEFAULT_A2S_TIMEOUT,
|
||||
source_name: str = "a2s-info",
|
||||
external_server_id: str | None = None,
|
||||
region: str | None = None,
|
||||
) -> dict[str, object]:
|
||||
"""Probe one A2S target and normalize its metadata for the collector model."""
|
||||
server_info = query_server_info(host, query_port, timeout=timeout)
|
||||
return normalize_a2s_server_info(
|
||||
server_info,
|
||||
source_name=source_name,
|
||||
external_server_id=external_server_id,
|
||||
region=region,
|
||||
)
|
||||
|
||||
|
||||
def fetch_configured_a2s_probes(
|
||||
*,
|
||||
timeout: float = DEFAULT_A2S_TIMEOUT,
|
||||
probe_target: TargetProbe | None = None,
|
||||
) -> tuple[dict[str, object], ...]:
|
||||
"""Probe the configured A2S targets without hardcoding them in collector logic."""
|
||||
probe = probe_target or _probe_configured_target
|
||||
return tuple(
|
||||
dict(probe(target, timeout))
|
||||
for target in load_a2s_targets()
|
||||
)
|
||||
|
||||
|
||||
def collect_server_snapshots(
|
||||
*,
|
||||
fetch_raw_source: RawSourceFetcher = fetch_controlled_server_source,
|
||||
source_name: str = "controlled-placeholder",
|
||||
source_mode: str = "controlled",
|
||||
timeout: float = DEFAULT_A2S_TIMEOUT,
|
||||
allow_controlled_fallback: bool = True,
|
||||
probe_target: TargetProbe | None = None,
|
||||
persist: bool = False,
|
||||
db_path: Path | None = None,
|
||||
) -> dict[str, object]:
|
||||
"""Collect snapshot batches from controlled data, A2S, or auto mode."""
|
||||
normalized_records, collection_details = _collect_normalized_records(
|
||||
fetch_raw_source=fetch_raw_source,
|
||||
source_name=source_name,
|
||||
source_mode=source_mode,
|
||||
timeout=timeout,
|
||||
allow_controlled_fallback=allow_controlled_fallback,
|
||||
probe_target=probe_target,
|
||||
)
|
||||
captured_at = utc_now()
|
||||
|
||||
payload = {
|
||||
"source_name": collection_details["source_name"],
|
||||
"collection_mode": collection_details["collection_mode"],
|
||||
"fallback_used": collection_details["fallback_used"],
|
||||
"target_count": collection_details["target_count"],
|
||||
"success_count": collection_details["success_count"],
|
||||
"errors": collection_details["errors"],
|
||||
"captured_at": captured_at.isoformat().replace("+00:00", "Z"),
|
||||
"snapshots": build_snapshot_batch(
|
||||
normalized_records,
|
||||
captured_at=captured_at,
|
||||
),
|
||||
}
|
||||
if persist:
|
||||
payload["storage"] = persist_snapshot_batch(
|
||||
payload["snapshots"],
|
||||
source_name=payload["source_name"],
|
||||
captured_at=payload["captured_at"],
|
||||
db_path=db_path,
|
||||
)
|
||||
|
||||
return payload
|
||||
|
||||
|
||||
def main() -> None:
|
||||
"""Allow manual collector execution during development."""
|
||||
parser = argparse.ArgumentParser(description="Collect development server snapshots.")
|
||||
parser.add_argument(
|
||||
"--source",
|
||||
choices=("controlled", "a2s", "auto"),
|
||||
default="auto",
|
||||
help="Choose controlled data, configured A2S targets, or auto with fallback.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--timeout",
|
||||
type=float,
|
||||
default=DEFAULT_A2S_TIMEOUT,
|
||||
help="Socket timeout in seconds for A2S probes.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--no-fallback",
|
||||
action="store_true",
|
||||
help="Disable fallback to controlled data when A2S fails.",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
payload = collect_server_snapshots(
|
||||
source_mode=args.source,
|
||||
timeout=args.timeout,
|
||||
allow_controlled_fallback=not args.no_fallback,
|
||||
persist=True,
|
||||
)
|
||||
print(json.dumps(payload, indent=2))
|
||||
|
||||
|
||||
def _collect_normalized_records(
|
||||
*,
|
||||
fetch_raw_source: RawSourceFetcher,
|
||||
source_name: str,
|
||||
source_mode: str,
|
||||
timeout: float,
|
||||
allow_controlled_fallback: bool,
|
||||
probe_target: TargetProbe | None,
|
||||
) -> tuple[list[dict[str, object]], dict[str, object]]:
|
||||
if source_mode == "controlled":
|
||||
raw_records = fetch_raw_source()
|
||||
return (
|
||||
[
|
||||
normalize_server_record(record, source_name=source_name)
|
||||
for record in raw_records
|
||||
],
|
||||
{
|
||||
"source_name": source_name,
|
||||
"collection_mode": "controlled",
|
||||
"fallback_used": False,
|
||||
"target_count": 0,
|
||||
"success_count": 0,
|
||||
"errors": [],
|
||||
},
|
||||
)
|
||||
|
||||
configured_targets = load_a2s_targets()
|
||||
records: list[dict[str, object]] = []
|
||||
errors: list[dict[str, object]] = []
|
||||
probe = probe_target or _probe_configured_target
|
||||
|
||||
for target in configured_targets:
|
||||
try:
|
||||
records.append(dict(probe(target, timeout)))
|
||||
except Exception as error: # noqa: BLE001 - keep collector failures controlled
|
||||
errors.append(
|
||||
{
|
||||
"target": target.name,
|
||||
"host": target.host,
|
||||
"query_port": target.query_port,
|
||||
"message": str(error),
|
||||
}
|
||||
)
|
||||
|
||||
if records:
|
||||
return (
|
||||
records,
|
||||
{
|
||||
"source_name": "a2s-info",
|
||||
"collection_mode": "a2s",
|
||||
"fallback_used": False,
|
||||
"target_count": len(configured_targets),
|
||||
"success_count": len(records),
|
||||
"errors": errors,
|
||||
},
|
||||
)
|
||||
|
||||
if source_mode == "a2s" or not allow_controlled_fallback:
|
||||
return (
|
||||
[],
|
||||
{
|
||||
"source_name": "a2s-info",
|
||||
"collection_mode": "a2s",
|
||||
"fallback_used": False,
|
||||
"target_count": len(configured_targets),
|
||||
"success_count": 0,
|
||||
"errors": errors,
|
||||
},
|
||||
)
|
||||
|
||||
raw_records = fetch_raw_source()
|
||||
normalized_records = [
|
||||
normalize_server_record(record, source_name=source_name)
|
||||
for record in raw_records
|
||||
]
|
||||
return (
|
||||
normalized_records,
|
||||
{
|
||||
"source_name": source_name,
|
||||
"collection_mode": "controlled-fallback",
|
||||
"fallback_used": True,
|
||||
"target_count": len(configured_targets),
|
||||
"success_count": 0,
|
||||
"errors": errors,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def _probe_configured_target(
|
||||
target: A2SServerTarget,
|
||||
timeout: float,
|
||||
) -> dict[str, object]:
|
||||
return fetch_a2s_probe(
|
||||
target.host,
|
||||
target.query_port,
|
||||
timeout=timeout,
|
||||
source_name=target.source_name,
|
||||
external_server_id=target.external_server_id,
|
||||
region=target.region,
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
659
backend/app/config.py
Normal file
659
backend/app/config.py
Normal file
@@ -0,0 +1,659 @@
|
||||
"""Local development configuration for the HLL Vietnam backend bootstrap."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
DEFAULT_HOST = "127.0.0.1"
|
||||
DEFAULT_PORT = 8000
|
||||
DEFAULT_STORAGE_FILENAME = "hll_vietnam_dev.sqlite3"
|
||||
DEFAULT_REFRESH_INTERVAL_SECONDS = 300
|
||||
DEFAULT_LIVE_DATA_SOURCE = "rcon"
|
||||
DEFAULT_HISTORICAL_DATA_SOURCE = "rcon"
|
||||
DEFAULT_RCON_TIMEOUT_SECONDS = 20.0
|
||||
DEFAULT_HISTORICAL_CRCON_PAGE_SIZE = 50
|
||||
DEFAULT_HISTORICAL_CRCON_TIMEOUT_SECONDS = 15.0
|
||||
DEFAULT_HISTORICAL_CRCON_DETAIL_WORKERS = 8
|
||||
DEFAULT_HISTORICAL_CRCON_REQUEST_RETRIES = 3
|
||||
DEFAULT_HISTORICAL_CRCON_RETRY_DELAY_SECONDS = 0.5
|
||||
DEFAULT_HISTORICAL_REFRESH_INTERVAL_SECONDS = 1800
|
||||
DEFAULT_HISTORICAL_REFRESH_OVERLAP_HOURS = 12
|
||||
DEFAULT_HISTORICAL_SNAPSHOT_REFRESH_INTERVAL_SECONDS = 900
|
||||
DEFAULT_HISTORICAL_REFRESH_MAX_RETRIES = 2
|
||||
DEFAULT_HISTORICAL_REFRESH_RETRY_DELAY_SECONDS = 30
|
||||
DEFAULT_HISTORICAL_FULL_SNAPSHOT_EVERY_RUNS = 4
|
||||
DEFAULT_HISTORICAL_ELO_MMR_REBUILD_INTERVAL_MINUTES = 180
|
||||
DEFAULT_HISTORICAL_ELO_MMR_MIN_NEW_SAMPLES = 12
|
||||
DEFAULT_HISTORICAL_WEEKLY_FALLBACK_MIN_MATCHES = 3
|
||||
DEFAULT_HISTORICAL_WEEKLY_FALLBACK_MAX_WEEKDAY = 2
|
||||
DEFAULT_PLAYER_EVENT_REFRESH_INTERVAL_SECONDS = 1800
|
||||
DEFAULT_PLAYER_EVENT_REFRESH_OVERLAP_HOURS = 12
|
||||
DEFAULT_PLAYER_EVENT_REFRESH_MAX_RETRIES = 2
|
||||
DEFAULT_PLAYER_EVENT_REFRESH_RETRY_DELAY_SECONDS = 30
|
||||
DEFAULT_RCON_HISTORICAL_CAPTURE_INTERVAL_SECONDS = 600
|
||||
DEFAULT_RCON_HISTORICAL_CAPTURE_MAX_RETRIES = 2
|
||||
DEFAULT_RCON_HISTORICAL_CAPTURE_RETRY_DELAY_SECONDS = 15
|
||||
DEFAULT_RCON_CURRENT_MATCH_CAPTURE_INTERVAL_SECONDS = 5
|
||||
DEFAULT_RCON_CURRENT_MATCH_WRITER_LOCK_TIMEOUT_SECONDS = 4.0
|
||||
DEFAULT_RCON_BACKFILL_CHUNK_HOURS = 6
|
||||
DEFAULT_RCON_BACKFILL_SLEEP_SECONDS = 1.0
|
||||
DEFAULT_RCON_BACKFILL_MAX_DAYS_BACK = 45
|
||||
DEFAULT_RECENT_MATCHES_KEEP = 100
|
||||
DEFAULT_ADMIN_LOG_NONCRITICAL_RETENTION_DAYS = 30
|
||||
DEFAULT_ADMIN_LOG_CRITICAL_RETENTION_DAYS = 90
|
||||
DEFAULT_SERVER_SNAPSHOT_RETENTION_DAYS = 14
|
||||
DEFAULT_DB_MAINTENANCE_BATCH_SIZE = 5000
|
||||
DEFAULT_DB_MAINTENANCE_ENABLED = False
|
||||
DEFAULT_DB_MAINTENANCE_INTERVAL_SECONDS = 43200
|
||||
DEFAULT_SQLITE_WRITER_TIMEOUT_SECONDS = 30.0
|
||||
DEFAULT_SQLITE_BUSY_TIMEOUT_MS = 30000
|
||||
DEFAULT_WRITER_LOCK_TIMEOUT_SECONDS = 120.0
|
||||
DEFAULT_WRITER_LOCK_POLL_INTERVAL_SECONDS = 1.0
|
||||
DEFAULT_ALLOWED_ORIGINS = (
|
||||
"null",
|
||||
"http://127.0.0.1",
|
||||
"http://127.0.0.1:5500",
|
||||
"http://127.0.0.1:8080",
|
||||
"http://localhost",
|
||||
"http://localhost:5500",
|
||||
"http://localhost:8080",
|
||||
)
|
||||
DEFAULT_A2S_TARGETS_ENV_VAR = "HLL_BACKEND_A2S_TARGETS"
|
||||
DEFAULT_A2S_SOURCE_NAME = "community-hispana-a2s"
|
||||
DEFAULT_RCON_TARGETS_ENV_VAR = "HLL_BACKEND_RCON_TARGETS"
|
||||
DEFAULT_RCON_SOURCE_NAME = "community-hispana-rcon"
|
||||
|
||||
|
||||
def get_bind_address() -> tuple[str, int]:
|
||||
"""Return the host and port used by the local backend bootstrap."""
|
||||
host = os.getenv("HLL_BACKEND_HOST", DEFAULT_HOST)
|
||||
port = int(os.getenv("HLL_BACKEND_PORT", str(DEFAULT_PORT)))
|
||||
return host, port
|
||||
|
||||
|
||||
def get_allowed_origins() -> tuple[str, ...]:
|
||||
"""Return the small allowlist used for local frontend development."""
|
||||
raw_origins = os.getenv(
|
||||
"HLL_BACKEND_ALLOWED_ORIGINS",
|
||||
",".join(DEFAULT_ALLOWED_ORIGINS),
|
||||
)
|
||||
origins = []
|
||||
for origin in raw_origins.split(","):
|
||||
normalized_origin = _normalize_origin(origin)
|
||||
if normalized_origin:
|
||||
origins.append(normalized_origin)
|
||||
return tuple(origins) or DEFAULT_ALLOWED_ORIGINS
|
||||
|
||||
|
||||
def _normalize_origin(origin: str) -> str:
|
||||
"""Normalize configured origins so env overrides match browser Origin values."""
|
||||
return origin.strip().rstrip("/")
|
||||
|
||||
|
||||
def get_storage_path() -> Path:
|
||||
"""Return the local SQLite path used for development snapshot persistence."""
|
||||
default_path = Path(__file__).resolve().parent.parent / "data" / DEFAULT_STORAGE_FILENAME
|
||||
configured_path = os.getenv("HLL_BACKEND_STORAGE_PATH")
|
||||
return Path(configured_path) if configured_path else default_path
|
||||
|
||||
|
||||
def get_database_url() -> str | None:
|
||||
"""Return the optional PostgreSQL URL for migrated backend storage domains."""
|
||||
configured_url = os.getenv("HLL_BACKEND_DATABASE_URL")
|
||||
if configured_url is None:
|
||||
return None
|
||||
normalized_url = configured_url.strip()
|
||||
return normalized_url or None
|
||||
|
||||
|
||||
def use_postgres_rcon_storage(*, explicit_sqlite_path: Path | None = None) -> bool:
|
||||
"""Return whether phase-1 RCON storage should use PostgreSQL."""
|
||||
return explicit_sqlite_path is None and get_database_url() is not None
|
||||
|
||||
|
||||
def get_sqlite_writer_timeout_seconds() -> float:
|
||||
"""Return the SQLite connection timeout shared by writer-capable storage layers."""
|
||||
configured_value = os.getenv(
|
||||
"HLL_BACKEND_SQLITE_WRITER_TIMEOUT_SECONDS",
|
||||
str(DEFAULT_SQLITE_WRITER_TIMEOUT_SECONDS),
|
||||
)
|
||||
timeout_seconds = float(configured_value)
|
||||
if timeout_seconds <= 0:
|
||||
raise ValueError("HLL_BACKEND_SQLITE_WRITER_TIMEOUT_SECONDS must be positive.")
|
||||
return timeout_seconds
|
||||
|
||||
|
||||
def get_sqlite_busy_timeout_ms() -> int:
|
||||
"""Return the SQLite busy_timeout shared by writer-capable storage layers."""
|
||||
configured_value = os.getenv(
|
||||
"HLL_BACKEND_SQLITE_BUSY_TIMEOUT_MS",
|
||||
str(DEFAULT_SQLITE_BUSY_TIMEOUT_MS),
|
||||
)
|
||||
busy_timeout_ms = int(configured_value)
|
||||
if busy_timeout_ms <= 0:
|
||||
raise ValueError("HLL_BACKEND_SQLITE_BUSY_TIMEOUT_MS must be positive.")
|
||||
return busy_timeout_ms
|
||||
|
||||
|
||||
def get_writer_lock_timeout_seconds() -> float:
|
||||
"""Return how long writer jobs should wait for the shared backend writer lock."""
|
||||
configured_value = os.getenv(
|
||||
"HLL_BACKEND_WRITER_LOCK_TIMEOUT_SECONDS",
|
||||
str(DEFAULT_WRITER_LOCK_TIMEOUT_SECONDS),
|
||||
)
|
||||
timeout_seconds = float(configured_value)
|
||||
if timeout_seconds < 0:
|
||||
raise ValueError("HLL_BACKEND_WRITER_LOCK_TIMEOUT_SECONDS must be zero or positive.")
|
||||
return timeout_seconds
|
||||
|
||||
|
||||
def get_writer_lock_poll_interval_seconds() -> float:
|
||||
"""Return how often writer jobs should poll the shared backend writer lock."""
|
||||
configured_value = os.getenv(
|
||||
"HLL_BACKEND_WRITER_LOCK_POLL_INTERVAL_SECONDS",
|
||||
str(DEFAULT_WRITER_LOCK_POLL_INTERVAL_SECONDS),
|
||||
)
|
||||
poll_interval_seconds = float(configured_value)
|
||||
if poll_interval_seconds <= 0:
|
||||
raise ValueError(
|
||||
"HLL_BACKEND_WRITER_LOCK_POLL_INTERVAL_SECONDS must be positive."
|
||||
)
|
||||
return poll_interval_seconds
|
||||
|
||||
|
||||
def get_refresh_interval_seconds() -> int:
|
||||
"""Return the default interval used by the local refresh loop."""
|
||||
configured_value = os.getenv(
|
||||
"HLL_BACKEND_REFRESH_INTERVAL_SECONDS",
|
||||
str(DEFAULT_REFRESH_INTERVAL_SECONDS),
|
||||
)
|
||||
interval_seconds = int(configured_value)
|
||||
if interval_seconds <= 0:
|
||||
raise ValueError("HLL_BACKEND_REFRESH_INTERVAL_SECONDS must be positive.")
|
||||
|
||||
return interval_seconds
|
||||
|
||||
|
||||
def get_historical_crcon_page_size() -> int:
|
||||
"""Return the default page size used for CRCON historical ingestion."""
|
||||
configured_value = os.getenv(
|
||||
"HLL_HISTORICAL_CRCON_PAGE_SIZE",
|
||||
str(DEFAULT_HISTORICAL_CRCON_PAGE_SIZE),
|
||||
)
|
||||
page_size = int(configured_value)
|
||||
if page_size <= 0:
|
||||
raise ValueError("HLL_HISTORICAL_CRCON_PAGE_SIZE must be positive.")
|
||||
|
||||
return page_size
|
||||
|
||||
|
||||
def get_historical_crcon_request_timeout_seconds() -> float:
|
||||
"""Return the timeout used for CRCON historical JSON requests."""
|
||||
configured_value = os.getenv(
|
||||
"HLL_HISTORICAL_CRCON_TIMEOUT_SECONDS",
|
||||
str(DEFAULT_HISTORICAL_CRCON_TIMEOUT_SECONDS),
|
||||
)
|
||||
timeout_seconds = float(configured_value)
|
||||
if timeout_seconds <= 0:
|
||||
raise ValueError("HLL_HISTORICAL_CRCON_TIMEOUT_SECONDS must be positive.")
|
||||
|
||||
return timeout_seconds
|
||||
|
||||
|
||||
def get_historical_crcon_detail_workers() -> int:
|
||||
"""Return the worker count used for CRCON historical detail requests."""
|
||||
configured_value = os.getenv(
|
||||
"HLL_HISTORICAL_CRCON_DETAIL_WORKERS",
|
||||
str(DEFAULT_HISTORICAL_CRCON_DETAIL_WORKERS),
|
||||
)
|
||||
worker_count = int(configured_value)
|
||||
if worker_count <= 0:
|
||||
raise ValueError("HLL_HISTORICAL_CRCON_DETAIL_WORKERS must be positive.")
|
||||
|
||||
return worker_count
|
||||
|
||||
|
||||
def get_historical_crcon_request_retries() -> int:
|
||||
"""Return the retry count used for CRCON historical JSON requests."""
|
||||
configured_value = os.getenv(
|
||||
"HLL_HISTORICAL_CRCON_REQUEST_RETRIES",
|
||||
str(DEFAULT_HISTORICAL_CRCON_REQUEST_RETRIES),
|
||||
)
|
||||
retry_count = int(configured_value)
|
||||
if retry_count <= 0:
|
||||
raise ValueError("HLL_HISTORICAL_CRCON_REQUEST_RETRIES must be positive.")
|
||||
|
||||
return retry_count
|
||||
|
||||
|
||||
def get_historical_crcon_retry_delay_seconds() -> float:
|
||||
"""Return the base delay used between CRCON request retries."""
|
||||
configured_value = os.getenv(
|
||||
"HLL_HISTORICAL_CRCON_RETRY_DELAY_SECONDS",
|
||||
str(DEFAULT_HISTORICAL_CRCON_RETRY_DELAY_SECONDS),
|
||||
)
|
||||
retry_delay_seconds = float(configured_value)
|
||||
if retry_delay_seconds < 0:
|
||||
raise ValueError(
|
||||
"HLL_HISTORICAL_CRCON_RETRY_DELAY_SECONDS must be zero or positive."
|
||||
)
|
||||
|
||||
return retry_delay_seconds
|
||||
|
||||
|
||||
def get_historical_refresh_interval_seconds() -> int:
|
||||
"""Return the default interval used by the historical refresh loop."""
|
||||
return _read_int_env(
|
||||
"HLL_HISTORICAL_SNAPSHOT_REFRESH_INTERVAL_SECONDS",
|
||||
os.getenv(
|
||||
"HLL_HISTORICAL_REFRESH_INTERVAL_SECONDS",
|
||||
str(DEFAULT_HISTORICAL_SNAPSHOT_REFRESH_INTERVAL_SECONDS),
|
||||
),
|
||||
minimum=1,
|
||||
)
|
||||
|
||||
|
||||
def _read_int_env(name: str, default_value: str, *, minimum: int) -> int:
|
||||
"""Read one integer env var and keep validation errors actionable."""
|
||||
configured_value = os.getenv(name, default_value)
|
||||
try:
|
||||
value = int(configured_value)
|
||||
except (TypeError, ValueError) as error:
|
||||
raise ValueError(f"{name} must be an integer.") from error
|
||||
if value < minimum:
|
||||
qualifier = "positive" if minimum == 1 else f"at least {minimum}"
|
||||
raise ValueError(f"{name} must be {qualifier}.")
|
||||
return value
|
||||
|
||||
|
||||
def _read_float_env(name: str, default_value: str, *, minimum: float) -> float:
|
||||
"""Read one float env var and keep validation errors actionable."""
|
||||
configured_value = os.getenv(name, default_value)
|
||||
try:
|
||||
value = float(configured_value)
|
||||
except (TypeError, ValueError) as error:
|
||||
raise ValueError(f"{name} must be a number.") from error
|
||||
if value < minimum:
|
||||
qualifier = "zero or positive" if minimum == 0 else f"at least {minimum}"
|
||||
raise ValueError(f"{name} must be {qualifier}.")
|
||||
return value
|
||||
|
||||
|
||||
def _read_bool_env(name: str, *, default: bool) -> bool:
|
||||
"""Read one boolean env var using a small set of explicit truthy/falsey values."""
|
||||
configured_value = os.getenv(name)
|
||||
if configured_value is None:
|
||||
return default
|
||||
|
||||
normalized = configured_value.strip().lower()
|
||||
if normalized in {"1", "true", "yes", "on"}:
|
||||
return True
|
||||
if normalized in {"0", "false", "no", "off"}:
|
||||
return False
|
||||
raise ValueError(f"{name} must be a boolean value.")
|
||||
|
||||
|
||||
def get_historical_refresh_overlap_hours() -> int:
|
||||
"""Return the overlap window used by incremental historical refreshes."""
|
||||
configured_value = os.getenv(
|
||||
"HLL_HISTORICAL_REFRESH_OVERLAP_HOURS",
|
||||
str(DEFAULT_HISTORICAL_REFRESH_OVERLAP_HOURS),
|
||||
)
|
||||
overlap_hours = int(configured_value)
|
||||
if overlap_hours < 0:
|
||||
raise ValueError("HLL_HISTORICAL_REFRESH_OVERLAP_HOURS must be zero or positive.")
|
||||
|
||||
return overlap_hours
|
||||
|
||||
|
||||
def get_live_data_source_kind() -> str:
|
||||
"""Return the live provider kind selected for the current environment."""
|
||||
source_kind = os.getenv("HLL_BACKEND_LIVE_DATA_SOURCE", DEFAULT_LIVE_DATA_SOURCE).strip()
|
||||
if source_kind not in {"a2s", "rcon"}:
|
||||
raise ValueError("HLL_BACKEND_LIVE_DATA_SOURCE must be 'a2s' or 'rcon'.")
|
||||
return source_kind
|
||||
|
||||
|
||||
def get_historical_data_source_kind() -> str:
|
||||
"""Return the historical provider kind selected for the current environment."""
|
||||
source_kind = os.getenv(
|
||||
"HLL_BACKEND_HISTORICAL_DATA_SOURCE",
|
||||
DEFAULT_HISTORICAL_DATA_SOURCE,
|
||||
).strip()
|
||||
if source_kind not in {"public-scoreboard", "rcon"}:
|
||||
raise ValueError(
|
||||
"HLL_BACKEND_HISTORICAL_DATA_SOURCE must be 'public-scoreboard' or 'rcon'."
|
||||
)
|
||||
return source_kind
|
||||
|
||||
|
||||
def get_rcon_request_timeout_seconds() -> float:
|
||||
"""Return the timeout used for HLL RCON TCP requests."""
|
||||
configured_value = os.getenv(
|
||||
"HLL_BACKEND_RCON_TIMEOUT_SECONDS",
|
||||
str(DEFAULT_RCON_TIMEOUT_SECONDS),
|
||||
)
|
||||
timeout_seconds = float(configured_value)
|
||||
if timeout_seconds <= 0:
|
||||
raise ValueError("HLL_BACKEND_RCON_TIMEOUT_SECONDS must be positive.")
|
||||
return timeout_seconds
|
||||
|
||||
|
||||
def get_historical_refresh_max_retries() -> int:
|
||||
"""Return the retry count used by the historical refresh loop."""
|
||||
return _read_int_env(
|
||||
"HLL_HISTORICAL_REFRESH_MAX_RETRIES",
|
||||
str(DEFAULT_HISTORICAL_REFRESH_MAX_RETRIES),
|
||||
minimum=0,
|
||||
)
|
||||
|
||||
|
||||
def get_historical_refresh_retry_delay_seconds() -> float:
|
||||
"""Return the wait time between historical refresh retries."""
|
||||
return _read_float_env(
|
||||
"HLL_HISTORICAL_REFRESH_RETRY_DELAY_SECONDS",
|
||||
str(DEFAULT_HISTORICAL_REFRESH_RETRY_DELAY_SECONDS),
|
||||
minimum=0,
|
||||
)
|
||||
|
||||
|
||||
def get_historical_full_snapshot_every_runs() -> int:
|
||||
"""Return how often the runner should rebuild the full snapshot matrix."""
|
||||
configured_value = os.getenv(
|
||||
"HLL_HISTORICAL_FULL_SNAPSHOT_EVERY_RUNS",
|
||||
str(DEFAULT_HISTORICAL_FULL_SNAPSHOT_EVERY_RUNS),
|
||||
)
|
||||
run_count = int(configured_value)
|
||||
if run_count <= 0:
|
||||
raise ValueError("HLL_HISTORICAL_FULL_SNAPSHOT_EVERY_RUNS must be positive.")
|
||||
|
||||
return run_count
|
||||
|
||||
|
||||
def get_historical_elo_mmr_rebuild_interval_minutes() -> int:
|
||||
"""Return the minimum minutes between automatic Elo/MMR rebuilds."""
|
||||
configured_value = os.getenv(
|
||||
"HLL_HISTORICAL_ELO_MMR_REBUILD_INTERVAL_MINUTES",
|
||||
str(DEFAULT_HISTORICAL_ELO_MMR_REBUILD_INTERVAL_MINUTES),
|
||||
)
|
||||
interval_minutes = int(configured_value)
|
||||
if interval_minutes <= 0:
|
||||
raise ValueError("HLL_HISTORICAL_ELO_MMR_REBUILD_INTERVAL_MINUTES must be positive.")
|
||||
return interval_minutes
|
||||
|
||||
|
||||
def get_historical_elo_mmr_min_new_samples() -> int:
|
||||
"""Return the minimum new RCON samples required for an automatic Elo/MMR rebuild."""
|
||||
configured_value = os.getenv(
|
||||
"HLL_HISTORICAL_ELO_MMR_MIN_NEW_SAMPLES",
|
||||
str(DEFAULT_HISTORICAL_ELO_MMR_MIN_NEW_SAMPLES),
|
||||
)
|
||||
min_samples = int(configured_value)
|
||||
if min_samples <= 0:
|
||||
raise ValueError("HLL_HISTORICAL_ELO_MMR_MIN_NEW_SAMPLES must be positive.")
|
||||
return min_samples
|
||||
|
||||
|
||||
def get_historical_weekly_fallback_min_matches() -> int:
|
||||
"""Return the minimum closed matches required to trust the current week."""
|
||||
configured_value = os.getenv(
|
||||
"HLL_HISTORICAL_WEEKLY_FALLBACK_MIN_MATCHES",
|
||||
str(DEFAULT_HISTORICAL_WEEKLY_FALLBACK_MIN_MATCHES),
|
||||
)
|
||||
min_matches = int(configured_value)
|
||||
if min_matches <= 0:
|
||||
raise ValueError("HLL_HISTORICAL_WEEKLY_FALLBACK_MIN_MATCHES must be positive.")
|
||||
|
||||
return min_matches
|
||||
|
||||
|
||||
def get_historical_weekly_fallback_max_weekday() -> int:
|
||||
"""Return the last weekday index where weekly fallback may still apply."""
|
||||
configured_value = os.getenv(
|
||||
"HLL_HISTORICAL_WEEKLY_FALLBACK_MAX_WEEKDAY",
|
||||
str(DEFAULT_HISTORICAL_WEEKLY_FALLBACK_MAX_WEEKDAY),
|
||||
)
|
||||
max_weekday = int(configured_value)
|
||||
if max_weekday < 0 or max_weekday > 6:
|
||||
raise ValueError("HLL_HISTORICAL_WEEKLY_FALLBACK_MAX_WEEKDAY must be between 0 and 6.")
|
||||
|
||||
return max_weekday
|
||||
|
||||
|
||||
def get_player_event_refresh_interval_seconds() -> int:
|
||||
"""Return the default interval used by the player event refresh loop."""
|
||||
configured_value = os.getenv(
|
||||
"HLL_PLAYER_EVENT_REFRESH_INTERVAL_SECONDS",
|
||||
str(DEFAULT_PLAYER_EVENT_REFRESH_INTERVAL_SECONDS),
|
||||
)
|
||||
interval_seconds = int(configured_value)
|
||||
if interval_seconds <= 0:
|
||||
raise ValueError("HLL_PLAYER_EVENT_REFRESH_INTERVAL_SECONDS must be positive.")
|
||||
return interval_seconds
|
||||
|
||||
|
||||
def get_player_event_refresh_overlap_hours() -> int:
|
||||
"""Return the overlap window used by player event refresh runs."""
|
||||
configured_value = os.getenv(
|
||||
"HLL_PLAYER_EVENT_REFRESH_OVERLAP_HOURS",
|
||||
str(DEFAULT_PLAYER_EVENT_REFRESH_OVERLAP_HOURS),
|
||||
)
|
||||
overlap_hours = int(configured_value)
|
||||
if overlap_hours < 0:
|
||||
raise ValueError("HLL_PLAYER_EVENT_REFRESH_OVERLAP_HOURS must be zero or positive.")
|
||||
return overlap_hours
|
||||
|
||||
|
||||
def get_player_event_refresh_max_retries() -> int:
|
||||
"""Return the retry count used by the player event refresh loop."""
|
||||
configured_value = os.getenv(
|
||||
"HLL_PLAYER_EVENT_REFRESH_MAX_RETRIES",
|
||||
str(DEFAULT_PLAYER_EVENT_REFRESH_MAX_RETRIES),
|
||||
)
|
||||
max_retries = int(configured_value)
|
||||
if max_retries < 0:
|
||||
raise ValueError("HLL_PLAYER_EVENT_REFRESH_MAX_RETRIES must be zero or positive.")
|
||||
return max_retries
|
||||
|
||||
|
||||
def get_player_event_refresh_retry_delay_seconds() -> int:
|
||||
"""Return the wait time between player event refresh retries."""
|
||||
configured_value = os.getenv(
|
||||
"HLL_PLAYER_EVENT_REFRESH_RETRY_DELAY_SECONDS",
|
||||
str(DEFAULT_PLAYER_EVENT_REFRESH_RETRY_DELAY_SECONDS),
|
||||
)
|
||||
retry_delay_seconds = int(configured_value)
|
||||
if retry_delay_seconds < 0:
|
||||
raise ValueError(
|
||||
"HLL_PLAYER_EVENT_REFRESH_RETRY_DELAY_SECONDS must be zero or positive."
|
||||
)
|
||||
return retry_delay_seconds
|
||||
|
||||
|
||||
def get_rcon_historical_capture_interval_seconds() -> int:
|
||||
"""Return the default interval used by the prospective RCON capture loop."""
|
||||
configured_value = os.getenv(
|
||||
"HLL_RCON_HISTORICAL_CAPTURE_INTERVAL_SECONDS",
|
||||
str(DEFAULT_RCON_HISTORICAL_CAPTURE_INTERVAL_SECONDS),
|
||||
)
|
||||
interval_seconds = int(configured_value)
|
||||
if interval_seconds <= 0:
|
||||
raise ValueError("HLL_RCON_HISTORICAL_CAPTURE_INTERVAL_SECONDS must be positive.")
|
||||
return interval_seconds
|
||||
|
||||
|
||||
def get_rcon_historical_capture_max_retries() -> int:
|
||||
"""Return the retry count used by the prospective RCON capture loop."""
|
||||
configured_value = os.getenv(
|
||||
"HLL_RCON_HISTORICAL_CAPTURE_MAX_RETRIES",
|
||||
str(DEFAULT_RCON_HISTORICAL_CAPTURE_MAX_RETRIES),
|
||||
)
|
||||
max_retries = int(configured_value)
|
||||
if max_retries < 0:
|
||||
raise ValueError("HLL_RCON_HISTORICAL_CAPTURE_MAX_RETRIES must be zero or positive.")
|
||||
return max_retries
|
||||
|
||||
|
||||
def get_rcon_historical_capture_retry_delay_seconds() -> int:
|
||||
"""Return the wait time between failed prospective RCON capture attempts."""
|
||||
configured_value = os.getenv(
|
||||
"HLL_RCON_HISTORICAL_CAPTURE_RETRY_DELAY_SECONDS",
|
||||
str(DEFAULT_RCON_HISTORICAL_CAPTURE_RETRY_DELAY_SECONDS),
|
||||
)
|
||||
retry_delay_seconds = int(configured_value)
|
||||
if retry_delay_seconds < 0:
|
||||
raise ValueError(
|
||||
"HLL_RCON_HISTORICAL_CAPTURE_RETRY_DELAY_SECONDS must be zero or positive."
|
||||
)
|
||||
return retry_delay_seconds
|
||||
|
||||
|
||||
def get_rcon_capture_mode() -> str:
|
||||
"""Return whether the worker runs the normal historical pipeline or live capture only."""
|
||||
configured_mode = os.getenv("HLL_RCON_CAPTURE_MODE")
|
||||
if configured_mode is not None:
|
||||
normalized_mode = configured_mode.strip().lower()
|
||||
if normalized_mode in {"historical", "current-live"}:
|
||||
return normalized_mode
|
||||
raise ValueError("HLL_RCON_CAPTURE_MODE must be 'historical' or 'current-live'.")
|
||||
|
||||
if _read_bool_env("HLL_RCON_CURRENT_MATCH_MODE", default=False):
|
||||
return "current-live"
|
||||
if _read_bool_env("HLL_RCON_SKIP_HISTORICAL_MATERIALIZATION", default=False):
|
||||
return "current-live"
|
||||
return "historical"
|
||||
|
||||
|
||||
def get_rcon_skip_historical_materialization() -> bool:
|
||||
"""Return whether the worker must skip heavy historical AdminLog materialization."""
|
||||
return get_rcon_capture_mode() == "current-live"
|
||||
|
||||
|
||||
def get_rcon_current_match_capture_interval_seconds() -> int:
|
||||
"""Return the preferred live current-match capture interval."""
|
||||
return _read_int_env(
|
||||
"HLL_RCON_CURRENT_MATCH_CAPTURE_INTERVAL_SECONDS",
|
||||
str(DEFAULT_RCON_CURRENT_MATCH_CAPTURE_INTERVAL_SECONDS),
|
||||
minimum=1,
|
||||
)
|
||||
|
||||
|
||||
def get_rcon_current_match_writer_lock_timeout_seconds() -> float:
|
||||
"""Return the lock wait budget for the lightweight current-match worker."""
|
||||
return _read_float_env(
|
||||
"HLL_RCON_CURRENT_MATCH_WRITER_LOCK_TIMEOUT_SECONDS",
|
||||
str(DEFAULT_RCON_CURRENT_MATCH_WRITER_LOCK_TIMEOUT_SECONDS),
|
||||
minimum=0,
|
||||
)
|
||||
|
||||
|
||||
def get_rcon_backfill_chunk_hours() -> int:
|
||||
"""Return the AdminLog backfill chunk size in hours."""
|
||||
return _read_int_env(
|
||||
"HLL_RCON_BACKFILL_CHUNK_HOURS",
|
||||
str(DEFAULT_RCON_BACKFILL_CHUNK_HOURS),
|
||||
minimum=1,
|
||||
)
|
||||
|
||||
|
||||
def get_rcon_backfill_sleep_seconds() -> float:
|
||||
"""Return the delay between AdminLog backfill RCON requests."""
|
||||
return _read_float_env(
|
||||
"HLL_RCON_BACKFILL_SLEEP_SECONDS",
|
||||
str(DEFAULT_RCON_BACKFILL_SLEEP_SECONDS),
|
||||
minimum=0,
|
||||
)
|
||||
|
||||
|
||||
def get_rcon_backfill_max_days_back() -> int:
|
||||
"""Return the maximum AdminLog backfill lookback horizon in days."""
|
||||
return _read_int_env(
|
||||
"HLL_RCON_BACKFILL_MAX_DAYS_BACK",
|
||||
str(DEFAULT_RCON_BACKFILL_MAX_DAYS_BACK),
|
||||
minimum=1,
|
||||
)
|
||||
|
||||
|
||||
def get_recent_matches_keep() -> int:
|
||||
"""Return how many recent closed materialized matches maintenance must protect."""
|
||||
return _read_int_env(
|
||||
"HLL_RECENT_MATCHES_KEEP",
|
||||
str(DEFAULT_RECENT_MATCHES_KEEP),
|
||||
minimum=1,
|
||||
)
|
||||
|
||||
|
||||
def get_admin_log_noncritical_retention_days() -> int:
|
||||
"""Return retention days for non-critical AdminLog events."""
|
||||
return _read_int_env(
|
||||
"HLL_ADMIN_LOG_NONCRITICAL_RETENTION_DAYS",
|
||||
str(DEFAULT_ADMIN_LOG_NONCRITICAL_RETENTION_DAYS),
|
||||
minimum=1,
|
||||
)
|
||||
|
||||
|
||||
def get_admin_log_critical_retention_days() -> int:
|
||||
"""Return retention days for critical AdminLog events."""
|
||||
return _read_int_env(
|
||||
"HLL_ADMIN_LOG_CRITICAL_RETENTION_DAYS",
|
||||
str(DEFAULT_ADMIN_LOG_CRITICAL_RETENTION_DAYS),
|
||||
minimum=1,
|
||||
)
|
||||
|
||||
|
||||
def get_server_snapshot_retention_days() -> int:
|
||||
"""Return retention days for live server snapshots."""
|
||||
return _read_int_env(
|
||||
"HLL_SERVER_SNAPSHOT_RETENTION_DAYS",
|
||||
str(DEFAULT_SERVER_SNAPSHOT_RETENTION_DAYS),
|
||||
minimum=1,
|
||||
)
|
||||
|
||||
|
||||
def get_db_maintenance_batch_size() -> int:
|
||||
"""Return the delete batch size used by database maintenance."""
|
||||
return _read_int_env(
|
||||
"HLL_DB_MAINTENANCE_BATCH_SIZE",
|
||||
str(DEFAULT_DB_MAINTENANCE_BATCH_SIZE),
|
||||
minimum=1,
|
||||
)
|
||||
|
||||
|
||||
def get_db_maintenance_enabled() -> bool:
|
||||
"""Return whether scheduled database maintenance is enabled."""
|
||||
normalized = os.getenv(
|
||||
"HLL_DB_MAINTENANCE_ENABLED",
|
||||
"true" if DEFAULT_DB_MAINTENANCE_ENABLED else "false",
|
||||
).strip().lower()
|
||||
return normalized in {"1", "true", "yes", "on"}
|
||||
|
||||
|
||||
def get_db_maintenance_interval_seconds() -> int:
|
||||
"""Return the scheduled database maintenance interval in seconds."""
|
||||
return _read_int_env(
|
||||
"HLL_DB_MAINTENANCE_INTERVAL_SECONDS",
|
||||
str(DEFAULT_DB_MAINTENANCE_INTERVAL_SECONDS),
|
||||
minimum=1,
|
||||
)
|
||||
|
||||
|
||||
def get_a2s_targets_payload() -> str | None:
|
||||
"""Return the optional JSON payload that overrides local A2S targets."""
|
||||
raw_payload = os.getenv(DEFAULT_A2S_TARGETS_ENV_VAR)
|
||||
if raw_payload is None:
|
||||
return None
|
||||
|
||||
normalized = raw_payload.strip()
|
||||
return normalized or None
|
||||
|
||||
|
||||
def get_rcon_targets_payload() -> str | None:
|
||||
"""Return the optional JSON payload that defines live RCON targets."""
|
||||
raw_payload = os.getenv(DEFAULT_RCON_TARGETS_ENV_VAR)
|
||||
if raw_payload is None:
|
||||
return None
|
||||
|
||||
normalized = raw_payload.strip()
|
||||
return normalized or None
|
||||
446
backend/app/data_sources.py
Normal file
446
backend/app/data_sources.py
Normal file
@@ -0,0 +1,446 @@
|
||||
"""Data source selection and contracts for live and historical backend flows."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import Protocol
|
||||
|
||||
from .collector import collect_server_snapshots
|
||||
from .config import get_historical_data_source_kind, get_live_data_source_kind
|
||||
from .providers.public_scoreboard_provider import PublicScoreboardHistoricalDataSource
|
||||
from .providers.rcon_provider import RconLiveDataSource
|
||||
from .rcon_historical_read_model import (
|
||||
describe_rcon_historical_read_model,
|
||||
list_rcon_historical_recent_activity,
|
||||
list_rcon_historical_server_summaries,
|
||||
)
|
||||
from .server_targets import A2SServerTarget, load_a2s_targets
|
||||
|
||||
|
||||
LIVE_SOURCE_A2S = "a2s"
|
||||
SOURCE_KIND_PUBLIC_SCOREBOARD = "public-scoreboard"
|
||||
SOURCE_KIND_RCON = "rcon"
|
||||
|
||||
|
||||
class HistoricalDataSource(Protocol):
|
||||
"""Contract for historical providers used by ingestion flows."""
|
||||
|
||||
source_kind: str
|
||||
|
||||
def fetch_public_info(self, *, base_url: str) -> dict[str, object]:
|
||||
"""Fetch provider metadata for one historical source."""
|
||||
|
||||
def fetch_match_page(self, *, base_url: str, page: int, limit: int) -> dict[str, object]:
|
||||
"""Fetch one page of historical matches."""
|
||||
|
||||
def fetch_match_details(
|
||||
self,
|
||||
*,
|
||||
base_url: str,
|
||||
match_ids: list[str],
|
||||
max_workers: int,
|
||||
) -> list[dict[str, object]]:
|
||||
"""Fetch detailed payloads for one batch of matches."""
|
||||
|
||||
|
||||
class LiveDataSource(Protocol):
|
||||
"""Contract for live providers used by API payload builders."""
|
||||
|
||||
source_kind: str
|
||||
|
||||
def collect_snapshots(self, *, persist: bool) -> dict[str, object]:
|
||||
"""Collect one live snapshot batch."""
|
||||
|
||||
def build_target_index(self) -> dict[str | None, object]:
|
||||
"""Return optional server connection metadata keyed by external id."""
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class A2SLiveDataSource:
|
||||
"""Live provider backed by the existing A2S collector flow."""
|
||||
|
||||
source_kind: str = LIVE_SOURCE_A2S
|
||||
|
||||
def collect_snapshots(self, *, persist: bool) -> dict[str, object]:
|
||||
return collect_server_snapshots(
|
||||
source_mode="a2s",
|
||||
allow_controlled_fallback=False,
|
||||
persist=persist,
|
||||
)
|
||||
|
||||
def build_target_index(self) -> dict[str | None, A2SServerTarget]:
|
||||
return {
|
||||
target.external_server_id: target
|
||||
for target in load_a2s_targets()
|
||||
if target.external_server_id
|
||||
}
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class RconFirstLiveDataSource:
|
||||
"""Live source arbitration with RCON as primary and A2S as controlled fallback."""
|
||||
|
||||
primary_source: RconLiveDataSource = RconLiveDataSource()
|
||||
fallback_source: A2SLiveDataSource = A2SLiveDataSource()
|
||||
source_kind: str = SOURCE_KIND_RCON
|
||||
|
||||
def collect_snapshots(self, *, persist: bool) -> dict[str, object]:
|
||||
attempts: list[dict[str, object]] = []
|
||||
fallback_reason: str | None = None
|
||||
|
||||
try:
|
||||
primary_payload = self.primary_source.collect_snapshots(persist=persist)
|
||||
except Exception as error: # noqa: BLE001 - source arbitration keeps fallback controlled
|
||||
attempts.append(
|
||||
build_source_attempt(
|
||||
source=SOURCE_KIND_RCON,
|
||||
role="primary",
|
||||
status="error",
|
||||
reason="rcon-live-request-failed",
|
||||
message=str(error),
|
||||
)
|
||||
)
|
||||
fallback_reason = "rcon-live-request-failed"
|
||||
else:
|
||||
primary_success_count = int(primary_payload.get("success_count") or 0)
|
||||
primary_snapshots = list(primary_payload.get("snapshots") or [])
|
||||
if primary_success_count > 0 and primary_snapshots:
|
||||
attempts.append(
|
||||
build_source_attempt(
|
||||
source=SOURCE_KIND_RCON,
|
||||
role="primary",
|
||||
status="success",
|
||||
)
|
||||
)
|
||||
return attach_source_policy(
|
||||
primary_payload,
|
||||
build_source_policy(
|
||||
primary_source=SOURCE_KIND_RCON,
|
||||
selected_source=SOURCE_KIND_RCON,
|
||||
source_attempts=attempts,
|
||||
),
|
||||
)
|
||||
|
||||
attempts.append(
|
||||
build_source_attempt(
|
||||
source=SOURCE_KIND_RCON,
|
||||
role="primary",
|
||||
status="empty",
|
||||
reason="rcon-live-returned-no-usable-snapshots",
|
||||
message=f"success_count={primary_success_count}",
|
||||
)
|
||||
)
|
||||
fallback_reason = "rcon-live-returned-no-usable-snapshots"
|
||||
|
||||
try:
|
||||
fallback_payload = self.fallback_source.collect_snapshots(persist=persist)
|
||||
except Exception as error: # noqa: BLE001 - keep combined failure explicit
|
||||
attempts.append(
|
||||
build_source_attempt(
|
||||
source=LIVE_SOURCE_A2S,
|
||||
role="fallback",
|
||||
status="error",
|
||||
reason="a2s-live-fallback-failed",
|
||||
message=str(error),
|
||||
)
|
||||
)
|
||||
raise RuntimeError(
|
||||
"RCON-first live collection failed and A2S fallback also failed."
|
||||
) from error
|
||||
|
||||
attempts.append(
|
||||
build_source_attempt(
|
||||
source=LIVE_SOURCE_A2S,
|
||||
role="fallback",
|
||||
status="success",
|
||||
)
|
||||
)
|
||||
return attach_source_policy(
|
||||
fallback_payload,
|
||||
build_source_policy(
|
||||
primary_source=SOURCE_KIND_RCON,
|
||||
selected_source=LIVE_SOURCE_A2S,
|
||||
fallback_used=True,
|
||||
fallback_reason=fallback_reason,
|
||||
source_attempts=attempts,
|
||||
),
|
||||
)
|
||||
|
||||
def build_target_index(self) -> dict[str | None, object]:
|
||||
target_index = dict(self.fallback_source.build_target_index())
|
||||
target_index.update(self.primary_source.build_target_index())
|
||||
return target_index
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class RconHistoricalDataSource:
|
||||
"""Persisted RCON-backed historical read model over captured competitive windows."""
|
||||
|
||||
source_kind: str = SOURCE_KIND_RCON
|
||||
|
||||
def fetch_public_info(self, *, base_url: str) -> dict[str, object]:
|
||||
raise RuntimeError(
|
||||
"RCON historical read mode does not support CRCON ingestion operations."
|
||||
)
|
||||
|
||||
def fetch_match_page(self, *, base_url: str, page: int, limit: int) -> dict[str, object]:
|
||||
raise RuntimeError(
|
||||
"RCON historical read mode does not support CRCON ingestion operations."
|
||||
)
|
||||
|
||||
def fetch_match_details(
|
||||
self,
|
||||
*,
|
||||
base_url: str,
|
||||
match_ids: list[str],
|
||||
max_workers: int,
|
||||
) -> list[dict[str, object]]:
|
||||
raise RuntimeError(
|
||||
"RCON historical read mode does not support CRCON ingestion operations."
|
||||
)
|
||||
|
||||
def list_server_summaries(self, *, server_key: str | None = None) -> list[dict[str, object]]:
|
||||
"""Return coverage and freshness from persisted RCON-backed competitive history."""
|
||||
return list_rcon_historical_server_summaries(server_key=server_key)
|
||||
|
||||
def list_recent_activity(
|
||||
self,
|
||||
*,
|
||||
server_key: str | None = None,
|
||||
limit: int = 20,
|
||||
) -> list[dict[str, object]]:
|
||||
"""Return recent RCON-backed competitive history without on-demand network calls."""
|
||||
return list_rcon_historical_recent_activity(server_key=server_key, limit=limit)
|
||||
|
||||
def has_server_summary_coverage(self, items: list[dict[str, object]]) -> bool:
|
||||
"""Return whether RCON summaries contain usable historical coverage."""
|
||||
for item in items:
|
||||
coverage = item.get("coverage") if isinstance(item, dict) else None
|
||||
if not isinstance(coverage, dict):
|
||||
continue
|
||||
if coverage.get("status") == "available":
|
||||
return True
|
||||
if int(coverage.get("sample_count") or 0) > 0:
|
||||
return True
|
||||
if int(coverage.get("window_count") or 0) > 0:
|
||||
return True
|
||||
if coverage.get("last_sample_at"):
|
||||
return True
|
||||
return False
|
||||
|
||||
def has_recent_activity_coverage(self, items: list[dict[str, object]]) -> bool:
|
||||
"""Return whether RCON recent activity contains at least one usable item."""
|
||||
for item in items:
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
if item.get("closed_at") or item.get("ended_at") or item.get("started_at"):
|
||||
return True
|
||||
if int(item.get("sample_count") or 0) > 0:
|
||||
return True
|
||||
return False
|
||||
|
||||
def describe_capabilities(self) -> dict[str, object]:
|
||||
"""Describe the supported RCON historical read surface."""
|
||||
return describe_rcon_historical_read_model()
|
||||
|
||||
|
||||
def get_historical_data_source() -> HistoricalDataSource:
|
||||
"""Select the historical provider configured for the current environment."""
|
||||
source_kind = get_historical_data_source_kind()
|
||||
if source_kind == SOURCE_KIND_PUBLIC_SCOREBOARD:
|
||||
return PublicScoreboardHistoricalDataSource()
|
||||
if source_kind == SOURCE_KIND_RCON:
|
||||
return RconHistoricalDataSource()
|
||||
raise ValueError(f"Unsupported historical data source: {source_kind}")
|
||||
|
||||
|
||||
def get_live_data_source() -> LiveDataSource:
|
||||
"""Select the live provider configured for the current environment."""
|
||||
source_kind = get_live_data_source_kind()
|
||||
if source_kind == LIVE_SOURCE_A2S:
|
||||
return A2SLiveDataSource()
|
||||
if source_kind == SOURCE_KIND_RCON:
|
||||
return RconFirstLiveDataSource()
|
||||
raise ValueError(f"Unsupported live data source: {source_kind}")
|
||||
|
||||
|
||||
def get_rcon_historical_read_model() -> RconHistoricalDataSource | None:
|
||||
"""Return the persisted RCON-backed historical read model when selected."""
|
||||
if get_historical_data_source_kind() != SOURCE_KIND_RCON:
|
||||
return None
|
||||
return RconHistoricalDataSource()
|
||||
|
||||
|
||||
def describe_historical_runtime_policy() -> dict[str, object]:
|
||||
"""Describe the effective historical runtime policy for the current environment."""
|
||||
if get_historical_data_source_kind() != SOURCE_KIND_RCON:
|
||||
return {
|
||||
"mode": "public-scoreboard-primary",
|
||||
"primary_source": SOURCE_KIND_PUBLIC_SCOREBOARD,
|
||||
"fallback_source": None,
|
||||
"summary": "Historical runtime uses public-scoreboard directly.",
|
||||
}
|
||||
return {
|
||||
"mode": "rcon-first-with-public-scoreboard-fallback",
|
||||
"primary_source": SOURCE_KIND_RCON,
|
||||
"fallback_source": SOURCE_KIND_PUBLIC_SCOREBOARD,
|
||||
"summary": (
|
||||
"Historical runtime attempts the persisted RCON-backed competitive model first "
|
||||
"and falls back to public-scoreboard when the requested operation is unsupported, has "
|
||||
"no coverage yet, or the primary path fails."
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def build_historical_runtime_source_policy(
|
||||
*,
|
||||
operation: str,
|
||||
rcon_status: str,
|
||||
fallback_reason: str | None = None,
|
||||
selected_source: str | None = None,
|
||||
rcon_message: str | None = None,
|
||||
) -> dict[str, object]:
|
||||
"""Build one normalized source-policy block for historical runtime reads."""
|
||||
configured_kind = get_historical_data_source_kind()
|
||||
if configured_kind != SOURCE_KIND_RCON:
|
||||
return build_source_policy(
|
||||
primary_source=SOURCE_KIND_PUBLIC_SCOREBOARD,
|
||||
selected_source=SOURCE_KIND_PUBLIC_SCOREBOARD,
|
||||
source_attempts=[
|
||||
build_source_attempt(
|
||||
source=SOURCE_KIND_PUBLIC_SCOREBOARD,
|
||||
role="primary",
|
||||
status="success",
|
||||
reason=f"{operation}-served-by-public-scoreboard",
|
||||
)
|
||||
],
|
||||
)
|
||||
|
||||
if rcon_status == "success":
|
||||
return build_source_policy(
|
||||
primary_source=SOURCE_KIND_RCON,
|
||||
selected_source=selected_source or SOURCE_KIND_RCON,
|
||||
source_attempts=[
|
||||
build_source_attempt(
|
||||
source=SOURCE_KIND_RCON,
|
||||
role="primary",
|
||||
status="success",
|
||||
reason=f"{operation}-served-by-rcon",
|
||||
)
|
||||
],
|
||||
)
|
||||
|
||||
return build_source_policy(
|
||||
primary_source=SOURCE_KIND_RCON,
|
||||
selected_source=selected_source or SOURCE_KIND_PUBLIC_SCOREBOARD,
|
||||
fallback_used=True,
|
||||
fallback_reason=fallback_reason,
|
||||
source_attempts=[
|
||||
build_source_attempt(
|
||||
source=SOURCE_KIND_RCON,
|
||||
role="primary",
|
||||
status=rcon_status,
|
||||
reason=fallback_reason,
|
||||
message=rcon_message,
|
||||
),
|
||||
build_source_attempt(
|
||||
source=SOURCE_KIND_PUBLIC_SCOREBOARD,
|
||||
role="fallback",
|
||||
status="success",
|
||||
reason=f"{operation}-served-by-public-scoreboard-fallback",
|
||||
),
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
def resolve_historical_ingestion_data_source() -> tuple[HistoricalDataSource, dict[str, object]]:
|
||||
"""Resolve the fallback provider used when classic scoreboard import is required."""
|
||||
configured_kind = get_historical_data_source_kind()
|
||||
if configured_kind in {SOURCE_KIND_PUBLIC_SCOREBOARD, SOURCE_KIND_RCON}:
|
||||
primary_source = (
|
||||
SOURCE_KIND_PUBLIC_SCOREBOARD
|
||||
if configured_kind == SOURCE_KIND_PUBLIC_SCOREBOARD
|
||||
else SOURCE_KIND_RCON
|
||||
)
|
||||
fallback_used = configured_kind == SOURCE_KIND_RCON
|
||||
fallback_reason = (
|
||||
"classic-historical-import-requires-public-scoreboard-fallback"
|
||||
if fallback_used
|
||||
else None
|
||||
)
|
||||
attempts = []
|
||||
if configured_kind == SOURCE_KIND_RCON:
|
||||
attempts.append(
|
||||
build_source_attempt(
|
||||
source=SOURCE_KIND_RCON,
|
||||
role="primary",
|
||||
status="deferred",
|
||||
reason="rcon-primary-writer-attempt-is-handled-by-historical-ingestion",
|
||||
)
|
||||
)
|
||||
attempts.append(
|
||||
build_source_attempt(
|
||||
source=SOURCE_KIND_PUBLIC_SCOREBOARD,
|
||||
role="fallback" if fallback_used else "primary",
|
||||
status="ready",
|
||||
reason="classic-historical-import-provider-ready",
|
||||
)
|
||||
)
|
||||
return (
|
||||
PublicScoreboardHistoricalDataSource(),
|
||||
build_source_policy(
|
||||
primary_source=primary_source,
|
||||
selected_source=SOURCE_KIND_PUBLIC_SCOREBOARD,
|
||||
fallback_used=fallback_used,
|
||||
fallback_reason=fallback_reason,
|
||||
source_attempts=attempts,
|
||||
),
|
||||
)
|
||||
|
||||
raise ValueError(f"Unsupported historical data source: {configured_kind}")
|
||||
|
||||
|
||||
def build_source_attempt(
|
||||
*,
|
||||
source: str,
|
||||
role: str,
|
||||
status: str,
|
||||
reason: str | None = None,
|
||||
message: str | None = None,
|
||||
) -> dict[str, object]:
|
||||
"""Build one normalized trace entry for source arbitration."""
|
||||
return {
|
||||
"source": source,
|
||||
"role": role,
|
||||
"status": status,
|
||||
"reason": reason,
|
||||
"message": message,
|
||||
}
|
||||
|
||||
|
||||
def build_source_policy(
|
||||
*,
|
||||
primary_source: str,
|
||||
selected_source: str,
|
||||
fallback_used: bool = False,
|
||||
fallback_reason: str | None = None,
|
||||
source_attempts: list[dict[str, object]] | None = None,
|
||||
) -> dict[str, object]:
|
||||
"""Build one small source-policy block for API responses and worker output."""
|
||||
return {
|
||||
"primary_source": primary_source,
|
||||
"selected_source": selected_source,
|
||||
"fallback_used": fallback_used,
|
||||
"fallback_reason": fallback_reason,
|
||||
"source_attempts": list(source_attempts or []),
|
||||
}
|
||||
|
||||
|
||||
def attach_source_policy(
|
||||
payload: dict[str, object],
|
||||
source_policy: dict[str, object],
|
||||
) -> dict[str, object]:
|
||||
"""Attach normalized source-policy metadata to an existing payload."""
|
||||
enriched = dict(payload)
|
||||
enriched.update(source_policy)
|
||||
return enriched
|
||||
638
backend/app/database_maintenance.py
Normal file
638
backend/app/database_maintenance.py
Normal file
@@ -0,0 +1,638 @@
|
||||
"""Application-level database maintenance for bounded historical storage."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import sqlite3
|
||||
from contextlib import closing
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any, Iterable, Sequence
|
||||
|
||||
from .config import (
|
||||
get_admin_log_critical_retention_days,
|
||||
get_admin_log_noncritical_retention_days,
|
||||
get_database_url,
|
||||
get_db_maintenance_batch_size,
|
||||
get_historical_weekly_fallback_min_matches,
|
||||
get_recent_matches_keep,
|
||||
get_server_snapshot_retention_days,
|
||||
)
|
||||
from .rcon_admin_log_materialization import MATCH_RESULT_SOURCE
|
||||
from .sqlite_utils import connect_sqlite_writer
|
||||
from .writer_lock import backend_writer_lock, build_writer_lock_holder
|
||||
|
||||
CRITICAL_ADMIN_LOG_EVENT_TYPES = frozenset({"kill", "match_start", "match_end"})
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class MaintenanceOptions:
|
||||
apply: bool
|
||||
recent_matches_keep: int
|
||||
admin_log_noncritical_retention_days: int
|
||||
admin_log_critical_retention_days: int
|
||||
server_snapshot_retention_days: int
|
||||
batch_size: int
|
||||
vacuum_analyze: bool
|
||||
now: datetime
|
||||
|
||||
|
||||
def run_database_maintenance_cleanup(
|
||||
*,
|
||||
apply: bool = False,
|
||||
recent_matches_keep: int | None = None,
|
||||
admin_log_noncritical_retention_days: int | None = None,
|
||||
admin_log_critical_retention_days: int | None = None,
|
||||
server_snapshot_retention_days: int | None = None,
|
||||
batch_size: int | None = None,
|
||||
vacuum_analyze: bool = False,
|
||||
now: str | datetime | None = None,
|
||||
db_path: Path | None = None,
|
||||
) -> dict[str, object]:
|
||||
"""Plan or apply safe bounded cleanup for supported storage tables."""
|
||||
options = MaintenanceOptions(
|
||||
apply=apply,
|
||||
recent_matches_keep=recent_matches_keep or get_recent_matches_keep(),
|
||||
admin_log_noncritical_retention_days=(
|
||||
admin_log_noncritical_retention_days or get_admin_log_noncritical_retention_days()
|
||||
),
|
||||
admin_log_critical_retention_days=(
|
||||
admin_log_critical_retention_days or get_admin_log_critical_retention_days()
|
||||
),
|
||||
server_snapshot_retention_days=(
|
||||
server_snapshot_retention_days or get_server_snapshot_retention_days()
|
||||
),
|
||||
batch_size=batch_size or get_db_maintenance_batch_size(),
|
||||
vacuum_analyze=vacuum_analyze,
|
||||
now=_resolve_now(now),
|
||||
)
|
||||
_emit_json_log(
|
||||
{
|
||||
"event": "database-maintenance-started",
|
||||
"mode": "apply" if options.apply else "dry-run",
|
||||
"database_backend": _database_backend_name(db_path=db_path),
|
||||
"database_url_configured": bool(get_database_url()) and db_path is None,
|
||||
"db_path": str(db_path) if db_path is not None else None,
|
||||
"recent_matches_keep": options.recent_matches_keep,
|
||||
"admin_log_noncritical_retention_days": options.admin_log_noncritical_retention_days,
|
||||
"admin_log_critical_retention_days": options.admin_log_critical_retention_days,
|
||||
"server_snapshot_retention_days": options.server_snapshot_retention_days,
|
||||
"batch_size": options.batch_size,
|
||||
"vacuum_analyze": options.vacuum_analyze,
|
||||
"now": _to_iso(options.now),
|
||||
}
|
||||
)
|
||||
|
||||
try:
|
||||
if options.apply:
|
||||
with backend_writer_lock(
|
||||
holder=build_writer_lock_holder("app.database_maintenance cleanup"),
|
||||
storage_path=db_path,
|
||||
):
|
||||
payload = _run_cleanup(options=options, db_path=db_path)
|
||||
else:
|
||||
payload = _run_cleanup(options=options, db_path=db_path)
|
||||
_emit_json_log(
|
||||
{
|
||||
"event": "database-maintenance-completed",
|
||||
**payload,
|
||||
}
|
||||
)
|
||||
return payload
|
||||
except Exception as exc: # noqa: BLE001 - CLI reports structured diagnostics
|
||||
error_payload = {
|
||||
"status": "error",
|
||||
"mode": "apply" if options.apply else "dry-run",
|
||||
"error_type": type(exc).__name__,
|
||||
"error": str(exc),
|
||||
}
|
||||
_emit_json_log({"event": "database-maintenance-error", **error_payload})
|
||||
return error_payload
|
||||
|
||||
|
||||
def _run_cleanup(*, options: MaintenanceOptions, db_path: Path | None) -> dict[str, object]:
|
||||
with _connect_maintenance(db_path=db_path) as connection:
|
||||
existing_tables = _existing_table_names(connection)
|
||||
plan = _build_cleanup_plan(connection, existing_tables=existing_tables, options=options)
|
||||
_emit_json_log(
|
||||
{
|
||||
"event": "database-maintenance-plan",
|
||||
**plan["summary"],
|
||||
}
|
||||
)
|
||||
|
||||
deleted_counts = {
|
||||
"rcon_match_player_stats": 0,
|
||||
"rcon_materialized_matches": 0,
|
||||
"rcon_admin_log_events": 0,
|
||||
"server_snapshots": 0,
|
||||
}
|
||||
if options.apply:
|
||||
deleted_counts["rcon_match_player_stats"] = _delete_match_player_stats(
|
||||
connection,
|
||||
matches=plan["candidate_matches"],
|
||||
batch_size=options.batch_size,
|
||||
)
|
||||
deleted_counts["rcon_materialized_matches"] = _delete_ids_in_batches(
|
||||
connection,
|
||||
table_name="rcon_materialized_matches",
|
||||
ids=[int(row["id"]) for row in plan["candidate_matches"]],
|
||||
batch_size=options.batch_size,
|
||||
)
|
||||
deleted_counts["rcon_admin_log_events"] = _delete_ids_in_batches(
|
||||
connection,
|
||||
table_name="rcon_admin_log_events",
|
||||
ids=plan["candidate_admin_log_ids"],
|
||||
batch_size=options.batch_size,
|
||||
)
|
||||
deleted_counts["server_snapshots"] = _delete_ids_in_batches(
|
||||
connection,
|
||||
table_name="server_snapshots",
|
||||
ids=plan["candidate_server_snapshot_ids"],
|
||||
batch_size=options.batch_size,
|
||||
)
|
||||
if options.vacuum_analyze:
|
||||
_run_vacuum_analyze(connection)
|
||||
|
||||
return {
|
||||
"status": "ok",
|
||||
"mode": "apply" if options.apply else "dry-run",
|
||||
"deleted_counts": deleted_counts,
|
||||
"plan": plan["summary"],
|
||||
}
|
||||
|
||||
|
||||
def _build_cleanup_plan(
|
||||
connection: sqlite3.Connection | Any,
|
||||
*,
|
||||
existing_tables: set[str],
|
||||
options: MaintenanceOptions,
|
||||
) -> dict[str, object]:
|
||||
candidate_server_snapshot_ids: list[int] = []
|
||||
candidate_admin_log_ids: list[int] = []
|
||||
candidate_matches: list[dict[str, object]] = []
|
||||
protected_match_keys: list[str] = []
|
||||
skipped_tables: list[str] = []
|
||||
|
||||
if "server_snapshots" not in existing_tables:
|
||||
skipped_tables.append("server_snapshots")
|
||||
_emit_skip("server_snapshots", "table-missing")
|
||||
else:
|
||||
cutoff = options.now - timedelta(days=options.server_snapshot_retention_days)
|
||||
for row in connection.execute(
|
||||
"SELECT id, captured_at FROM server_snapshots ORDER BY id ASC"
|
||||
).fetchall():
|
||||
captured_at = _parse_datetime(row["captured_at"])
|
||||
if captured_at is None:
|
||||
continue
|
||||
if captured_at < cutoff:
|
||||
candidate_server_snapshot_ids.append(int(row["id"]))
|
||||
|
||||
protected_ranges: dict[str, list[tuple[int, int]]] = {}
|
||||
if "rcon_materialized_matches" not in existing_tables:
|
||||
skipped_tables.append("rcon_materialized_matches")
|
||||
_emit_skip("rcon_materialized_matches", "table-missing")
|
||||
else:
|
||||
(
|
||||
candidate_matches,
|
||||
protected_matches,
|
||||
protected_ranges,
|
||||
protection_summary,
|
||||
) = _plan_materialized_match_cleanup(connection, options=options)
|
||||
protected_match_keys = [str(row["match_key"]) for row in protected_matches]
|
||||
if "rcon_match_player_stats" not in existing_tables:
|
||||
skipped_tables.append("rcon_match_player_stats")
|
||||
_emit_skip("rcon_match_player_stats", "table-missing")
|
||||
|
||||
if "rcon_admin_log_events" not in existing_tables:
|
||||
skipped_tables.append("rcon_admin_log_events")
|
||||
_emit_skip("rcon_admin_log_events", "table-missing")
|
||||
else:
|
||||
candidate_admin_log_ids = _plan_admin_log_cleanup(
|
||||
connection,
|
||||
options=options,
|
||||
protected_ranges=protected_ranges,
|
||||
)
|
||||
|
||||
candidate_player_stat_rows = 0
|
||||
if candidate_matches and "rcon_match_player_stats" in existing_tables:
|
||||
candidate_player_stat_rows = _count_candidate_player_stats(connection, candidate_matches)
|
||||
|
||||
summary = {
|
||||
"status": "ok",
|
||||
"protected_match_count": len(protected_match_keys),
|
||||
"candidate_match_count": len(candidate_matches),
|
||||
"candidate_match_player_stat_count": candidate_player_stat_rows,
|
||||
"candidate_admin_log_event_count": len(candidate_admin_log_ids),
|
||||
"candidate_server_snapshot_count": len(candidate_server_snapshot_ids),
|
||||
"skipped_tables": skipped_tables,
|
||||
"protected_match_keys_preview": protected_match_keys[:10],
|
||||
}
|
||||
if "protection_summary" in locals():
|
||||
summary["protection_summary"] = protection_summary
|
||||
|
||||
return {
|
||||
"candidate_server_snapshot_ids": candidate_server_snapshot_ids,
|
||||
"candidate_admin_log_ids": candidate_admin_log_ids,
|
||||
"candidate_matches": candidate_matches,
|
||||
"summary": summary,
|
||||
}
|
||||
|
||||
|
||||
def _plan_materialized_match_cleanup(
|
||||
connection: sqlite3.Connection | Any,
|
||||
*,
|
||||
options: MaintenanceOptions,
|
||||
) -> tuple[list[dict[str, object]], list[dict[str, object]], dict[str, list[tuple[int, int]]], dict[str, object]]:
|
||||
rows = [
|
||||
dict(row)
|
||||
for row in connection.execute(
|
||||
"""
|
||||
SELECT id, target_key, match_key, started_at, ended_at,
|
||||
started_server_time, ended_server_time, source_basis
|
||||
FROM rcon_materialized_matches
|
||||
WHERE source_basis = ?
|
||||
""",
|
||||
(MATCH_RESULT_SOURCE,),
|
||||
).fetchall()
|
||||
]
|
||||
closed_rows: list[dict[str, object]] = []
|
||||
protected_rows: list[dict[str, object]] = []
|
||||
current_month_start = options.now.replace(day=1, hour=0, minute=0, second=0, microsecond=0)
|
||||
previous_month_start = (current_month_start - timedelta(days=1)).replace(day=1)
|
||||
current_week_start = (options.now - timedelta(days=options.now.weekday())).replace(
|
||||
hour=0,
|
||||
minute=0,
|
||||
second=0,
|
||||
microsecond=0,
|
||||
)
|
||||
previous_week_start = current_week_start - timedelta(days=7)
|
||||
|
||||
for row in rows:
|
||||
closed_at = _parse_datetime(row.get("ended_at") or row.get("started_at"))
|
||||
if closed_at is None:
|
||||
row["_protect_reason"] = "unparseable-closed-at"
|
||||
protected_rows.append(row)
|
||||
continue
|
||||
row["_closed_at"] = closed_at
|
||||
closed_rows.append(row)
|
||||
|
||||
closed_rows.sort(
|
||||
key=lambda row: (
|
||||
row["_closed_at"],
|
||||
_coerce_int(row.get("ended_server_time")) or _coerce_int(row.get("started_server_time")) or 0,
|
||||
_coerce_int(row.get("id")) or 0,
|
||||
),
|
||||
reverse=True,
|
||||
)
|
||||
latest_ids = {int(row["id"]) for row in closed_rows[: options.recent_matches_keep]}
|
||||
current_week_count = sum(
|
||||
1 for row in closed_rows if current_week_start <= row["_closed_at"] < options.now
|
||||
)
|
||||
previous_week_count = sum(
|
||||
1 for row in closed_rows if previous_week_start <= row["_closed_at"] < current_week_start
|
||||
)
|
||||
protect_previous_week = (
|
||||
current_week_count < get_historical_weekly_fallback_min_matches()
|
||||
and previous_week_count > 0
|
||||
)
|
||||
protect_previous_month = options.now.day <= 7
|
||||
|
||||
candidate_rows: list[dict[str, object]] = []
|
||||
protected_ranges: dict[str, list[tuple[int, int]]] = {}
|
||||
for row in closed_rows:
|
||||
closed_at = row["_closed_at"]
|
||||
should_protect = False
|
||||
if int(row["id"]) in latest_ids:
|
||||
should_protect = True
|
||||
elif closed_at >= current_month_start:
|
||||
should_protect = True
|
||||
elif protect_previous_month and previous_month_start <= closed_at < current_month_start:
|
||||
should_protect = True
|
||||
elif closed_at >= current_week_start:
|
||||
should_protect = True
|
||||
elif protect_previous_week and previous_week_start <= closed_at < current_week_start:
|
||||
should_protect = True
|
||||
|
||||
if should_protect:
|
||||
protected_rows.append(row)
|
||||
lower = _coerce_int(row.get("started_server_time"))
|
||||
upper = _coerce_int(row.get("ended_server_time"))
|
||||
if lower is not None and upper is not None:
|
||||
protected_ranges.setdefault(str(row["target_key"]), []).append((lower, upper))
|
||||
else:
|
||||
candidate_rows.append(row)
|
||||
|
||||
return (
|
||||
candidate_rows,
|
||||
protected_rows,
|
||||
protected_ranges,
|
||||
{
|
||||
"recent_matches_keep": options.recent_matches_keep,
|
||||
"current_week_closed_matches": current_week_count,
|
||||
"previous_week_closed_matches": previous_week_count,
|
||||
"protect_previous_week": protect_previous_week,
|
||||
"protect_previous_month": protect_previous_month,
|
||||
"current_week_start": _to_iso(current_week_start),
|
||||
"previous_week_start": _to_iso(previous_week_start),
|
||||
"current_month_start": _to_iso(current_month_start),
|
||||
"previous_month_start": _to_iso(previous_month_start),
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def _plan_admin_log_cleanup(
|
||||
connection: sqlite3.Connection | Any,
|
||||
*,
|
||||
options: MaintenanceOptions,
|
||||
protected_ranges: dict[str, list[tuple[int, int]]],
|
||||
) -> list[int]:
|
||||
noncritical_cutoff = options.now - timedelta(days=options.admin_log_noncritical_retention_days)
|
||||
critical_cutoff = options.now - timedelta(days=options.admin_log_critical_retention_days)
|
||||
candidate_ids: list[int] = []
|
||||
rows = connection.execute(
|
||||
"""
|
||||
SELECT id, target_key, event_type, event_timestamp, server_time
|
||||
FROM rcon_admin_log_events
|
||||
ORDER BY id ASC
|
||||
"""
|
||||
).fetchall()
|
||||
for row in rows:
|
||||
event_type = str(row["event_type"] or "").strip()
|
||||
event_time = _parse_datetime(row["event_timestamp"])
|
||||
if event_time is None:
|
||||
continue
|
||||
if event_type in CRITICAL_ADMIN_LOG_EVENT_TYPES:
|
||||
if event_time >= critical_cutoff:
|
||||
continue
|
||||
server_time = _coerce_int(row["server_time"])
|
||||
if server_time is None:
|
||||
continue
|
||||
if _server_time_is_protected(
|
||||
target_key=str(row["target_key"] or ""),
|
||||
server_time=server_time,
|
||||
protected_ranges=protected_ranges,
|
||||
):
|
||||
continue
|
||||
candidate_ids.append(int(row["id"]))
|
||||
continue
|
||||
if event_time < noncritical_cutoff:
|
||||
candidate_ids.append(int(row["id"]))
|
||||
return candidate_ids
|
||||
|
||||
|
||||
def _count_candidate_player_stats(
|
||||
connection: sqlite3.Connection | Any,
|
||||
matches: Sequence[dict[str, object]],
|
||||
) -> int:
|
||||
count = 0
|
||||
for batch in _chunked(list(matches), 250):
|
||||
clause, params = _match_pair_clause(batch)
|
||||
row = connection.execute(
|
||||
f"SELECT COUNT(*) AS count FROM rcon_match_player_stats WHERE {clause}",
|
||||
params,
|
||||
).fetchone()
|
||||
count += int(row["count"] or 0)
|
||||
return count
|
||||
|
||||
|
||||
def _delete_match_player_stats(
|
||||
connection: sqlite3.Connection | Any,
|
||||
*,
|
||||
matches: Sequence[dict[str, object]],
|
||||
batch_size: int,
|
||||
) -> int:
|
||||
deleted = 0
|
||||
for batch in _chunked(list(matches), max(1, min(batch_size, 250))):
|
||||
clause, params = _match_pair_clause(batch)
|
||||
deleted_in_batch = int(
|
||||
connection.execute(
|
||||
f"DELETE FROM rcon_match_player_stats WHERE {clause}",
|
||||
params,
|
||||
).rowcount
|
||||
or 0
|
||||
)
|
||||
_commit(connection)
|
||||
deleted += deleted_in_batch
|
||||
_emit_json_log(
|
||||
{
|
||||
"event": "database-maintenance-delete-batch",
|
||||
"table": "rcon_match_player_stats",
|
||||
"deleted_rows": deleted_in_batch,
|
||||
"batch_size": len(batch),
|
||||
}
|
||||
)
|
||||
return deleted
|
||||
|
||||
|
||||
def _delete_ids_in_batches(
|
||||
connection: sqlite3.Connection | Any,
|
||||
*,
|
||||
table_name: str,
|
||||
ids: Sequence[int],
|
||||
batch_size: int,
|
||||
) -> int:
|
||||
deleted = 0
|
||||
for batch in _chunked(list(ids), batch_size):
|
||||
placeholders = ",".join("?" for _ in batch)
|
||||
deleted_in_batch = int(
|
||||
connection.execute(
|
||||
f"DELETE FROM {table_name} WHERE id IN ({placeholders})",
|
||||
batch,
|
||||
).rowcount
|
||||
or 0
|
||||
)
|
||||
_commit(connection)
|
||||
deleted += deleted_in_batch
|
||||
_emit_json_log(
|
||||
{
|
||||
"event": "database-maintenance-delete-batch",
|
||||
"table": table_name,
|
||||
"deleted_rows": deleted_in_batch,
|
||||
"batch_size": len(batch),
|
||||
}
|
||||
)
|
||||
return deleted
|
||||
|
||||
|
||||
def _run_vacuum_analyze(connection: sqlite3.Connection | Any) -> None:
|
||||
raw_connection = _raw_connection(connection)
|
||||
if isinstance(raw_connection, sqlite3.Connection):
|
||||
raw_connection.execute("VACUUM")
|
||||
raw_connection.execute("ANALYZE")
|
||||
raw_connection.commit()
|
||||
return
|
||||
raw_connection.commit()
|
||||
raw_connection.autocommit = True
|
||||
try:
|
||||
raw_connection.execute("VACUUM ANALYZE")
|
||||
finally:
|
||||
raw_connection.autocommit = False
|
||||
|
||||
|
||||
def _match_pair_clause(matches: Sequence[dict[str, object]]) -> tuple[str, list[object]]:
|
||||
clauses: list[str] = []
|
||||
params: list[object] = []
|
||||
for row in matches:
|
||||
clauses.append("(target_key = ? AND match_key = ?)")
|
||||
params.extend([row["target_key"], row["match_key"]])
|
||||
return " OR ".join(clauses), params
|
||||
|
||||
|
||||
def _existing_table_names(connection: sqlite3.Connection | Any) -> set[str]:
|
||||
raw_connection = _raw_connection(connection)
|
||||
if isinstance(raw_connection, sqlite3.Connection):
|
||||
rows = connection.execute(
|
||||
"SELECT name FROM sqlite_master WHERE type = 'table'"
|
||||
).fetchall()
|
||||
return {str(row["name"]) for row in rows}
|
||||
rows = raw_connection.execute(
|
||||
"""
|
||||
SELECT table_name
|
||||
FROM information_schema.tables
|
||||
WHERE table_schema = 'public'
|
||||
"""
|
||||
).fetchall()
|
||||
return {str(row["table_name"]) for row in rows}
|
||||
|
||||
|
||||
def _emit_skip(table_name: str, reason: str) -> None:
|
||||
_emit_json_log(
|
||||
{
|
||||
"event": "database-maintenance-table-skipped",
|
||||
"table": table_name,
|
||||
"reason": reason,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _server_time_is_protected(
|
||||
*,
|
||||
target_key: str,
|
||||
server_time: int,
|
||||
protected_ranges: dict[str, list[tuple[int, int]]],
|
||||
) -> bool:
|
||||
for lower, upper in protected_ranges.get(target_key, []):
|
||||
if lower <= server_time <= upper:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def _connect_maintenance(*, db_path: Path | None):
|
||||
if get_database_url() and db_path is None:
|
||||
from .postgres_rcon_storage import connect_postgres_compat
|
||||
|
||||
return connect_postgres_compat()
|
||||
resolved_path = db_path or Path.cwd() / "backend" / "data" / "hll_vietnam_dev.sqlite3"
|
||||
resolved_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
return closing(connect_sqlite_writer(resolved_path))
|
||||
|
||||
|
||||
def _commit(connection: sqlite3.Connection | Any) -> None:
|
||||
_raw_connection(connection).commit()
|
||||
|
||||
|
||||
def _raw_connection(connection: sqlite3.Connection | Any) -> sqlite3.Connection | Any:
|
||||
return connection.connection if hasattr(connection, "connection") else connection
|
||||
|
||||
|
||||
def _database_backend_name(*, db_path: Path | None) -> str:
|
||||
return "postgres" if get_database_url() and db_path is None else "sqlite"
|
||||
|
||||
|
||||
def _resolve_now(value: str | datetime | None) -> datetime:
|
||||
if value is None:
|
||||
return datetime.now(timezone.utc)
|
||||
if isinstance(value, datetime):
|
||||
return value.astimezone(timezone.utc) if value.tzinfo else value.replace(tzinfo=timezone.utc)
|
||||
parsed = _parse_datetime(value)
|
||||
if parsed is None:
|
||||
raise ValueError("--now must be an ISO 8601 timestamp or date.")
|
||||
return parsed
|
||||
|
||||
|
||||
def _parse_datetime(value: object) -> datetime | None:
|
||||
text = str(value or "").strip()
|
||||
if not text:
|
||||
return None
|
||||
if len(text) == 10:
|
||||
text = f"{text}T00:00:00+00:00"
|
||||
try:
|
||||
parsed = datetime.fromisoformat(text.replace("Z", "+00:00"))
|
||||
except ValueError:
|
||||
return None
|
||||
return parsed.astimezone(timezone.utc) if parsed.tzinfo else parsed.replace(tzinfo=timezone.utc)
|
||||
|
||||
|
||||
def _to_iso(value: datetime) -> str:
|
||||
return value.astimezone(timezone.utc).isoformat().replace("+00:00", "Z")
|
||||
|
||||
|
||||
def _coerce_int(value: object) -> int | None:
|
||||
try:
|
||||
return None if value is None else int(value)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
def _chunked(values: Sequence[Any], size: int) -> Iterable[list[Any]]:
|
||||
for index in range(0, len(values), size):
|
||||
yield list(values[index : index + size])
|
||||
|
||||
|
||||
def _emit_json_log(payload: dict[str, object]) -> None:
|
||||
print(json.dumps(payload, ensure_ascii=True, default=str), flush=True)
|
||||
|
||||
|
||||
def build_arg_parser() -> argparse.ArgumentParser:
|
||||
parser = argparse.ArgumentParser(description="Database maintenance for HLL Vietnam.")
|
||||
subparsers = parser.add_subparsers(dest="command", required=True)
|
||||
cleanup_parser = subparsers.add_parser("cleanup")
|
||||
cleanup_parser.add_argument("--dry-run", action="store_true")
|
||||
cleanup_parser.add_argument("--apply", action="store_true")
|
||||
cleanup_parser.add_argument("--recent-matches-keep", type=int, default=get_recent_matches_keep())
|
||||
cleanup_parser.add_argument(
|
||||
"--admin-log-noncritical-retention-days",
|
||||
type=int,
|
||||
default=get_admin_log_noncritical_retention_days(),
|
||||
)
|
||||
cleanup_parser.add_argument(
|
||||
"--admin-log-critical-retention-days",
|
||||
type=int,
|
||||
default=get_admin_log_critical_retention_days(),
|
||||
)
|
||||
cleanup_parser.add_argument(
|
||||
"--server-snapshot-retention-days",
|
||||
type=int,
|
||||
default=get_server_snapshot_retention_days(),
|
||||
)
|
||||
cleanup_parser.add_argument("--batch-size", type=int, default=get_db_maintenance_batch_size())
|
||||
cleanup_parser.add_argument("--vacuum-analyze", action="store_true")
|
||||
cleanup_parser.add_argument("--now", default=None)
|
||||
return parser
|
||||
|
||||
|
||||
def main(argv: Sequence[str] | None = None) -> int:
|
||||
parser = build_arg_parser()
|
||||
args = parser.parse_args(list(argv) if argv is not None else None)
|
||||
if args.command != "cleanup":
|
||||
raise ValueError("Unsupported command.")
|
||||
if args.apply and args.dry_run:
|
||||
raise ValueError("--apply and --dry-run are mutually exclusive.")
|
||||
payload = run_database_maintenance_cleanup(
|
||||
apply=bool(args.apply),
|
||||
recent_matches_keep=args.recent_matches_keep,
|
||||
admin_log_noncritical_retention_days=args.admin_log_noncritical_retention_days,
|
||||
admin_log_critical_retention_days=args.admin_log_critical_retention_days,
|
||||
server_snapshot_retention_days=args.server_snapshot_retention_days,
|
||||
batch_size=args.batch_size,
|
||||
vacuum_analyze=bool(args.vacuum_analyze),
|
||||
now=args.now,
|
||||
)
|
||||
return 0 if payload.get("status") == "ok" else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
1013
backend/app/elo_mmr_engine.py
Normal file
1013
backend/app/elo_mmr_engine.py
Normal file
File diff suppressed because it is too large
Load Diff
74
backend/app/elo_mmr_models.py
Normal file
74
backend/app/elo_mmr_models.py
Normal file
@@ -0,0 +1,74 @@
|
||||
"""Contracts and capability helpers for the Elo/MMR monthly ranking system."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import asdict, dataclass
|
||||
|
||||
|
||||
CAPABILITY_EXACT = "exact"
|
||||
CAPABILITY_APPROXIMATE = "approximate"
|
||||
CAPABILITY_UNAVAILABLE = "not_available"
|
||||
|
||||
ACCURACY_EXACT = "exact"
|
||||
ACCURACY_APPROXIMATE = "approximate"
|
||||
ACCURACY_PARTIAL = "partial"
|
||||
|
||||
DEFAULT_BASE_MMR = 1000.0
|
||||
ELO_K_FACTOR = 60.0
|
||||
MIN_VALID_MATCH_DURATION_SECONDS = 900
|
||||
MIN_VALID_MATCH_PLAYERS = 20
|
||||
MIN_VALID_PLAYER_PARTICIPATION_SECONDS = 900
|
||||
MIN_VALID_PLAYER_PARTICIPATION_RATIO = 0.45
|
||||
FULL_QUALITY_PLAYER_COUNT = 70
|
||||
FULL_QUALITY_DURATION_SECONDS = 3600
|
||||
MONTHLY_MIN_VALID_MATCHES = 5
|
||||
MONTHLY_MIN_TIME_SECONDS = 21600
|
||||
MONTHLY_ACTIVITY_TARGET_MATCHES = 12
|
||||
MONTHLY_ACTIVITY_TARGET_HOURS = 20.0
|
||||
DEFAULT_MONTHLY_SCOREBOARD_MIN_MATCHES = 3
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class EloSignalAvailability:
|
||||
"""Normalized availability state for one scoring input."""
|
||||
|
||||
name: str
|
||||
status: str
|
||||
detail: str
|
||||
|
||||
def to_dict(self) -> dict[str, object]:
|
||||
"""Return the availability entry as a serializable mapping."""
|
||||
return asdict(self)
|
||||
|
||||
|
||||
def build_signal(name: str, status: str, detail: str) -> dict[str, object]:
|
||||
"""Create a normalized availability block for one signal."""
|
||||
return EloSignalAvailability(name=name, status=status, detail=detail).to_dict()
|
||||
|
||||
|
||||
def summarize_accuracy(signals: list[dict[str, object]]) -> dict[str, object]:
|
||||
"""Summarize exact, approximate and unavailable signals for one calculation."""
|
||||
exact_count = sum(1 for signal in signals if signal.get("status") == CAPABILITY_EXACT)
|
||||
approximate_count = sum(
|
||||
1 for signal in signals if signal.get("status") == CAPABILITY_APPROXIMATE
|
||||
)
|
||||
unavailable_count = sum(
|
||||
1 for signal in signals if signal.get("status") == CAPABILITY_UNAVAILABLE
|
||||
)
|
||||
if unavailable_count > 0:
|
||||
accuracy_mode = ACCURACY_PARTIAL
|
||||
elif approximate_count > 0:
|
||||
accuracy_mode = ACCURACY_APPROXIMATE
|
||||
else:
|
||||
accuracy_mode = ACCURACY_EXACT
|
||||
total = max(1, len(signals))
|
||||
return {
|
||||
"accuracy_mode": accuracy_mode,
|
||||
"exact_count": exact_count,
|
||||
"approximate_count": approximate_count,
|
||||
"unavailable_count": unavailable_count,
|
||||
"exact_ratio": round(exact_count / total, 3),
|
||||
"approximate_ratio": round(approximate_count / total, 3),
|
||||
"unavailable_ratio": round(unavailable_count / total, 3),
|
||||
"signals": list(signals),
|
||||
}
|
||||
578
backend/app/elo_mmr_storage.py
Normal file
578
backend/app/elo_mmr_storage.py
Normal file
@@ -0,0 +1,578 @@
|
||||
"""SQLite storage for persistent Elo/MMR and monthly ranking results."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sqlite3
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
|
||||
from .config import get_storage_path
|
||||
from .sqlite_utils import connect_sqlite_readonly, connect_sqlite_writer
|
||||
|
||||
|
||||
def initialize_elo_mmr_storage(*, db_path: Path | None = None) -> Path:
|
||||
"""Create the Elo/MMR persistence tables in the shared backend SQLite."""
|
||||
resolved_path = _resolve_db_path(db_path)
|
||||
resolved_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with _connect_writer(resolved_path) as connection:
|
||||
connection.executescript(
|
||||
"""
|
||||
CREATE TABLE IF NOT EXISTS elo_mmr_player_ratings (
|
||||
scope_key TEXT NOT NULL,
|
||||
stable_player_key TEXT NOT NULL,
|
||||
player_name TEXT NOT NULL,
|
||||
steam_id TEXT,
|
||||
current_mmr REAL NOT NULL,
|
||||
matches_processed INTEGER NOT NULL DEFAULT 0,
|
||||
wins INTEGER NOT NULL DEFAULT 0,
|
||||
draws INTEGER NOT NULL DEFAULT 0,
|
||||
losses INTEGER NOT NULL DEFAULT 0,
|
||||
last_match_id TEXT,
|
||||
last_match_ended_at TEXT,
|
||||
accuracy_mode TEXT NOT NULL,
|
||||
capabilities_json TEXT NOT NULL,
|
||||
updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
PRIMARY KEY (scope_key, stable_player_key)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS elo_mmr_match_results (
|
||||
scope_key TEXT NOT NULL,
|
||||
month_key TEXT NOT NULL,
|
||||
external_match_id TEXT NOT NULL,
|
||||
stable_player_key TEXT NOT NULL,
|
||||
player_name TEXT NOT NULL,
|
||||
steam_id TEXT,
|
||||
server_slug TEXT NOT NULL,
|
||||
server_name TEXT NOT NULL,
|
||||
match_ended_at TEXT NOT NULL,
|
||||
match_valid INTEGER NOT NULL,
|
||||
quality_factor REAL NOT NULL,
|
||||
quality_bucket TEXT NOT NULL,
|
||||
role_bucket TEXT NOT NULL,
|
||||
role_bucket_mode TEXT NOT NULL,
|
||||
outcome_score REAL NOT NULL,
|
||||
combat_index REAL NOT NULL,
|
||||
objective_index REAL,
|
||||
objective_index_mode TEXT NOT NULL,
|
||||
utility_index REAL,
|
||||
utility_index_mode TEXT NOT NULL,
|
||||
leadership_index REAL,
|
||||
leadership_index_mode TEXT NOT NULL,
|
||||
discipline_index REAL,
|
||||
discipline_index_mode TEXT NOT NULL,
|
||||
impact_score REAL NOT NULL,
|
||||
delta_mmr REAL NOT NULL,
|
||||
mmr_before REAL NOT NULL,
|
||||
mmr_after REAL NOT NULL,
|
||||
match_score REAL NOT NULL,
|
||||
penalty_points REAL NOT NULL,
|
||||
capabilities_json TEXT NOT NULL,
|
||||
updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
PRIMARY KEY (scope_key, external_match_id, stable_player_key)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS elo_mmr_monthly_rankings (
|
||||
scope_key TEXT NOT NULL,
|
||||
month_key TEXT NOT NULL,
|
||||
stable_player_key TEXT NOT NULL,
|
||||
player_name TEXT NOT NULL,
|
||||
steam_id TEXT,
|
||||
current_mmr REAL NOT NULL,
|
||||
baseline_mmr REAL NOT NULL,
|
||||
mmr_gain REAL NOT NULL,
|
||||
avg_match_score REAL NOT NULL,
|
||||
strength_of_schedule REAL NOT NULL,
|
||||
consistency REAL NOT NULL,
|
||||
activity REAL NOT NULL,
|
||||
confidence REAL NOT NULL,
|
||||
penalty_points REAL NOT NULL,
|
||||
monthly_rank_score REAL NOT NULL,
|
||||
valid_matches INTEGER NOT NULL,
|
||||
total_matches INTEGER NOT NULL,
|
||||
total_time_seconds INTEGER NOT NULL,
|
||||
eligible INTEGER NOT NULL,
|
||||
eligibility_reason TEXT,
|
||||
accuracy_mode TEXT NOT NULL,
|
||||
capabilities_json TEXT NOT NULL,
|
||||
component_scores_json TEXT NOT NULL,
|
||||
updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
PRIMARY KEY (scope_key, month_key, stable_player_key)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS elo_mmr_monthly_checkpoints (
|
||||
scope_key TEXT NOT NULL,
|
||||
month_key TEXT NOT NULL,
|
||||
generated_at TEXT NOT NULL,
|
||||
player_count INTEGER NOT NULL,
|
||||
eligible_player_count INTEGER NOT NULL,
|
||||
source_policy_json TEXT NOT NULL,
|
||||
capabilities_summary_json TEXT NOT NULL,
|
||||
updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
PRIMARY KEY (scope_key, month_key)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_elo_mmr_monthly_rankings_scope_month
|
||||
ON elo_mmr_monthly_rankings(scope_key, month_key, eligible, monthly_rank_score DESC);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_elo_mmr_player_ratings_scope
|
||||
ON elo_mmr_player_ratings(scope_key, current_mmr DESC);
|
||||
"""
|
||||
)
|
||||
return resolved_path
|
||||
|
||||
|
||||
def replace_elo_mmr_state(
|
||||
*,
|
||||
player_ratings: list[dict[str, object]],
|
||||
match_results: list[dict[str, object]],
|
||||
monthly_rankings: list[dict[str, object]],
|
||||
monthly_checkpoints: list[dict[str, object]],
|
||||
db_path: Path | None = None,
|
||||
) -> Path:
|
||||
"""Replace the persisted Elo/MMR state with a freshly rebuilt dataset."""
|
||||
resolved_path = initialize_elo_mmr_storage(db_path=db_path)
|
||||
with _connect_writer(resolved_path) as connection:
|
||||
connection.execute("DELETE FROM elo_mmr_monthly_checkpoints")
|
||||
connection.execute("DELETE FROM elo_mmr_monthly_rankings")
|
||||
connection.execute("DELETE FROM elo_mmr_match_results")
|
||||
connection.execute("DELETE FROM elo_mmr_player_ratings")
|
||||
|
||||
connection.executemany(
|
||||
"""
|
||||
INSERT INTO elo_mmr_player_ratings (
|
||||
scope_key,
|
||||
stable_player_key,
|
||||
player_name,
|
||||
steam_id,
|
||||
current_mmr,
|
||||
matches_processed,
|
||||
wins,
|
||||
draws,
|
||||
losses,
|
||||
last_match_id,
|
||||
last_match_ended_at,
|
||||
accuracy_mode,
|
||||
capabilities_json
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
""",
|
||||
[
|
||||
(
|
||||
row["scope_key"],
|
||||
row["stable_player_key"],
|
||||
row["player_name"],
|
||||
row.get("steam_id"),
|
||||
row["current_mmr"],
|
||||
row["matches_processed"],
|
||||
row["wins"],
|
||||
row["draws"],
|
||||
row["losses"],
|
||||
row.get("last_match_id"),
|
||||
row.get("last_match_ended_at"),
|
||||
row["accuracy_mode"],
|
||||
json.dumps(row["capabilities"], ensure_ascii=True, separators=(",", ":")),
|
||||
)
|
||||
for row in player_ratings
|
||||
],
|
||||
)
|
||||
|
||||
connection.executemany(
|
||||
"""
|
||||
INSERT INTO elo_mmr_match_results (
|
||||
scope_key,
|
||||
month_key,
|
||||
external_match_id,
|
||||
stable_player_key,
|
||||
player_name,
|
||||
steam_id,
|
||||
server_slug,
|
||||
server_name,
|
||||
match_ended_at,
|
||||
match_valid,
|
||||
quality_factor,
|
||||
quality_bucket,
|
||||
role_bucket,
|
||||
role_bucket_mode,
|
||||
outcome_score,
|
||||
combat_index,
|
||||
objective_index,
|
||||
objective_index_mode,
|
||||
utility_index,
|
||||
utility_index_mode,
|
||||
leadership_index,
|
||||
leadership_index_mode,
|
||||
discipline_index,
|
||||
discipline_index_mode,
|
||||
impact_score,
|
||||
delta_mmr,
|
||||
mmr_before,
|
||||
mmr_after,
|
||||
match_score,
|
||||
penalty_points,
|
||||
capabilities_json
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
""",
|
||||
[
|
||||
(
|
||||
row["scope_key"],
|
||||
row["month_key"],
|
||||
row["external_match_id"],
|
||||
row["stable_player_key"],
|
||||
row["player_name"],
|
||||
row.get("steam_id"),
|
||||
row["server_slug"],
|
||||
row["server_name"],
|
||||
row["match_ended_at"],
|
||||
1 if row["match_valid"] else 0,
|
||||
row["quality_factor"],
|
||||
row["quality_bucket"],
|
||||
row["role_bucket"],
|
||||
row["role_bucket_mode"],
|
||||
row["outcome_score"],
|
||||
row["combat_index"],
|
||||
row.get("objective_index"),
|
||||
row["objective_index_mode"],
|
||||
row.get("utility_index"),
|
||||
row["utility_index_mode"],
|
||||
row.get("leadership_index"),
|
||||
row["leadership_index_mode"],
|
||||
row.get("discipline_index"),
|
||||
row["discipline_index_mode"],
|
||||
row["impact_score"],
|
||||
row["delta_mmr"],
|
||||
row["mmr_before"],
|
||||
row["mmr_after"],
|
||||
row["match_score"],
|
||||
row["penalty_points"],
|
||||
json.dumps(row["capabilities"], ensure_ascii=True, separators=(",", ":")),
|
||||
)
|
||||
for row in match_results
|
||||
],
|
||||
)
|
||||
|
||||
connection.executemany(
|
||||
"""
|
||||
INSERT INTO elo_mmr_monthly_rankings (
|
||||
scope_key,
|
||||
month_key,
|
||||
stable_player_key,
|
||||
player_name,
|
||||
steam_id,
|
||||
current_mmr,
|
||||
baseline_mmr,
|
||||
mmr_gain,
|
||||
avg_match_score,
|
||||
strength_of_schedule,
|
||||
consistency,
|
||||
activity,
|
||||
confidence,
|
||||
penalty_points,
|
||||
monthly_rank_score,
|
||||
valid_matches,
|
||||
total_matches,
|
||||
total_time_seconds,
|
||||
eligible,
|
||||
eligibility_reason,
|
||||
accuracy_mode,
|
||||
capabilities_json,
|
||||
component_scores_json
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
""",
|
||||
[
|
||||
(
|
||||
row["scope_key"],
|
||||
row["month_key"],
|
||||
row["stable_player_key"],
|
||||
row["player_name"],
|
||||
row.get("steam_id"),
|
||||
row["current_mmr"],
|
||||
row["baseline_mmr"],
|
||||
row["mmr_gain"],
|
||||
row["avg_match_score"],
|
||||
row["strength_of_schedule"],
|
||||
row["consistency"],
|
||||
row["activity"],
|
||||
row["confidence"],
|
||||
row["penalty_points"],
|
||||
row["monthly_rank_score"],
|
||||
row["valid_matches"],
|
||||
row["total_matches"],
|
||||
row["total_time_seconds"],
|
||||
1 if row["eligible"] else 0,
|
||||
row.get("eligibility_reason"),
|
||||
row["accuracy_mode"],
|
||||
json.dumps(row["capabilities"], ensure_ascii=True, separators=(",", ":")),
|
||||
json.dumps(row["component_scores"], ensure_ascii=True, separators=(",", ":")),
|
||||
)
|
||||
for row in monthly_rankings
|
||||
],
|
||||
)
|
||||
|
||||
connection.executemany(
|
||||
"""
|
||||
INSERT INTO elo_mmr_monthly_checkpoints (
|
||||
scope_key,
|
||||
month_key,
|
||||
generated_at,
|
||||
player_count,
|
||||
eligible_player_count,
|
||||
source_policy_json,
|
||||
capabilities_summary_json
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?)
|
||||
""",
|
||||
[
|
||||
(
|
||||
row["scope_key"],
|
||||
row["month_key"],
|
||||
row["generated_at"],
|
||||
row["player_count"],
|
||||
row["eligible_player_count"],
|
||||
json.dumps(row["source_policy"], ensure_ascii=True, separators=(",", ":")),
|
||||
json.dumps(
|
||||
row["capabilities_summary"],
|
||||
ensure_ascii=True,
|
||||
separators=(",", ":"),
|
||||
),
|
||||
)
|
||||
for row in monthly_checkpoints
|
||||
],
|
||||
)
|
||||
return resolved_path
|
||||
|
||||
|
||||
def list_elo_mmr_monthly_rankings(
|
||||
*,
|
||||
scope_key: str,
|
||||
limit: int = 10,
|
||||
month_key: str | None = None,
|
||||
eligible_only: bool = True,
|
||||
db_path: Path | None = None,
|
||||
) -> dict[str, object]:
|
||||
"""Return the persisted monthly Elo/MMR leaderboard for one scope."""
|
||||
resolved_path = _resolve_db_path(db_path)
|
||||
resolved_month_key = month_key or get_latest_elo_mmr_month_key(scope_key=scope_key, db_path=resolved_path)
|
||||
if not resolved_month_key:
|
||||
return {
|
||||
"month_key": None,
|
||||
"found": False,
|
||||
"generated_at": None,
|
||||
"items": [],
|
||||
"source_policy": None,
|
||||
"capabilities_summary": None,
|
||||
}
|
||||
|
||||
where_clauses = ["scope_key = ?", "month_key = ?"]
|
||||
params: list[object] = [scope_key, resolved_month_key]
|
||||
if eligible_only:
|
||||
where_clauses.append("eligible = 1")
|
||||
params.append(limit)
|
||||
try:
|
||||
with _connect_readonly(resolved_path) as connection:
|
||||
checkpoint_row = connection.execute(
|
||||
"""
|
||||
SELECT generated_at, source_policy_json, capabilities_summary_json
|
||||
FROM elo_mmr_monthly_checkpoints
|
||||
WHERE scope_key = ? AND month_key = ?
|
||||
""",
|
||||
(scope_key, resolved_month_key),
|
||||
).fetchone()
|
||||
rows = connection.execute(
|
||||
f"""
|
||||
SELECT *
|
||||
FROM elo_mmr_monthly_rankings
|
||||
WHERE {" AND ".join(where_clauses)}
|
||||
ORDER BY monthly_rank_score DESC, current_mmr DESC, player_name COLLATE NOCASE ASC
|
||||
LIMIT ?
|
||||
""",
|
||||
params,
|
||||
).fetchall()
|
||||
except sqlite3.OperationalError:
|
||||
return {
|
||||
"month_key": None,
|
||||
"found": False,
|
||||
"generated_at": None,
|
||||
"items": [],
|
||||
"source_policy": None,
|
||||
"capabilities_summary": None,
|
||||
}
|
||||
items = []
|
||||
for index, row in enumerate(rows, start=1):
|
||||
items.append(
|
||||
{
|
||||
"ranking_position": index,
|
||||
"player": {
|
||||
"stable_player_key": row["stable_player_key"],
|
||||
"name": row["player_name"],
|
||||
"steam_id": row["steam_id"],
|
||||
},
|
||||
"persistent_rating": {
|
||||
"mmr": round(float(row["current_mmr"] or 0.0), 3),
|
||||
"baseline_mmr": round(float(row["baseline_mmr"] or 0.0), 3),
|
||||
"mmr_gain": round(float(row["mmr_gain"] or 0.0), 3),
|
||||
},
|
||||
"monthly_rank_score": round(float(row["monthly_rank_score"] or 0.0), 3),
|
||||
"components": json.loads(row["component_scores_json"]),
|
||||
"valid_matches": int(row["valid_matches"] or 0),
|
||||
"total_matches": int(row["total_matches"] or 0),
|
||||
"total_time_seconds": int(row["total_time_seconds"] or 0),
|
||||
"eligible": bool(row["eligible"]),
|
||||
"eligibility_reason": row["eligibility_reason"],
|
||||
"accuracy_mode": row["accuracy_mode"],
|
||||
"capabilities": json.loads(row["capabilities_json"]),
|
||||
}
|
||||
)
|
||||
return {
|
||||
"month_key": resolved_month_key,
|
||||
"found": bool(items),
|
||||
"generated_at": checkpoint_row["generated_at"] if checkpoint_row else None,
|
||||
"items": items,
|
||||
"source_policy": json.loads(checkpoint_row["source_policy_json"])
|
||||
if checkpoint_row
|
||||
else None,
|
||||
"capabilities_summary": json.loads(checkpoint_row["capabilities_summary_json"])
|
||||
if checkpoint_row
|
||||
else None,
|
||||
}
|
||||
|
||||
|
||||
def get_elo_mmr_player_profile(
|
||||
*,
|
||||
player_id: str,
|
||||
scope_key: str,
|
||||
month_key: str | None = None,
|
||||
db_path: Path | None = None,
|
||||
) -> dict[str, object] | None:
|
||||
"""Return the persisted rating and monthly ranking profile for one player."""
|
||||
resolved_player_id = player_id.strip()
|
||||
if not resolved_player_id:
|
||||
return None
|
||||
resolved_path = _resolve_db_path(db_path)
|
||||
resolved_month_key = month_key or get_latest_elo_mmr_month_key(scope_key=scope_key, db_path=resolved_path)
|
||||
try:
|
||||
with _connect_readonly(resolved_path) as connection:
|
||||
rating_row = connection.execute(
|
||||
"""
|
||||
SELECT *
|
||||
FROM elo_mmr_player_ratings
|
||||
WHERE scope_key = ?
|
||||
AND (stable_player_key = ? OR steam_id = ?)
|
||||
ORDER BY updated_at DESC
|
||||
LIMIT 1
|
||||
""",
|
||||
(scope_key, resolved_player_id, resolved_player_id),
|
||||
).fetchone()
|
||||
monthly_row = None
|
||||
if resolved_month_key:
|
||||
monthly_row = connection.execute(
|
||||
"""
|
||||
SELECT *
|
||||
FROM elo_mmr_monthly_rankings
|
||||
WHERE scope_key = ?
|
||||
AND month_key = ?
|
||||
AND (stable_player_key = ? OR steam_id = ?)
|
||||
ORDER BY updated_at DESC
|
||||
LIMIT 1
|
||||
""",
|
||||
(scope_key, resolved_month_key, resolved_player_id, resolved_player_id),
|
||||
).fetchone()
|
||||
except sqlite3.OperationalError:
|
||||
return None
|
||||
if rating_row is None and monthly_row is None:
|
||||
return None
|
||||
return {
|
||||
"scope_key": scope_key,
|
||||
"month_key": resolved_month_key,
|
||||
"player": {
|
||||
"stable_player_key": (
|
||||
rating_row["stable_player_key"] if rating_row else monthly_row["stable_player_key"]
|
||||
),
|
||||
"name": rating_row["player_name"] if rating_row else monthly_row["player_name"],
|
||||
"steam_id": rating_row["steam_id"] if rating_row else monthly_row["steam_id"],
|
||||
},
|
||||
"persistent_rating": (
|
||||
{
|
||||
"mmr": round(float(rating_row["current_mmr"] or 0.0), 3),
|
||||
"matches_processed": int(rating_row["matches_processed"] or 0),
|
||||
"wins": int(rating_row["wins"] or 0),
|
||||
"draws": int(rating_row["draws"] or 0),
|
||||
"losses": int(rating_row["losses"] or 0),
|
||||
"last_match_id": rating_row["last_match_id"],
|
||||
"last_match_ended_at": rating_row["last_match_ended_at"],
|
||||
"accuracy_mode": rating_row["accuracy_mode"],
|
||||
"capabilities": json.loads(rating_row["capabilities_json"]),
|
||||
}
|
||||
if rating_row
|
||||
else None
|
||||
),
|
||||
"monthly_ranking": (
|
||||
{
|
||||
"monthly_rank_score": round(float(monthly_row["monthly_rank_score"] or 0.0), 3),
|
||||
"current_mmr": round(float(monthly_row["current_mmr"] or 0.0), 3),
|
||||
"baseline_mmr": round(float(monthly_row["baseline_mmr"] or 0.0), 3),
|
||||
"mmr_gain": round(float(monthly_row["mmr_gain"] or 0.0), 3),
|
||||
"valid_matches": int(monthly_row["valid_matches"] or 0),
|
||||
"total_matches": int(monthly_row["total_matches"] or 0),
|
||||
"total_time_seconds": int(monthly_row["total_time_seconds"] or 0),
|
||||
"eligible": bool(monthly_row["eligible"]),
|
||||
"eligibility_reason": monthly_row["eligibility_reason"],
|
||||
"accuracy_mode": monthly_row["accuracy_mode"],
|
||||
"components": json.loads(monthly_row["component_scores_json"]),
|
||||
"capabilities": json.loads(monthly_row["capabilities_json"]),
|
||||
}
|
||||
if monthly_row
|
||||
else None
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def get_latest_elo_mmr_month_key(
|
||||
*,
|
||||
scope_key: str,
|
||||
db_path: Path | None = None,
|
||||
) -> str | None:
|
||||
"""Return the latest month key available for one Elo/MMR scope."""
|
||||
resolved_path = _resolve_db_path(db_path)
|
||||
try:
|
||||
with _connect_readonly(resolved_path) as connection:
|
||||
row = connection.execute(
|
||||
"""
|
||||
SELECT MAX(month_key) AS latest_month_key
|
||||
FROM elo_mmr_monthly_checkpoints
|
||||
WHERE scope_key = ?
|
||||
""",
|
||||
(scope_key,),
|
||||
).fetchone()
|
||||
except sqlite3.OperationalError:
|
||||
return None
|
||||
return str(row["latest_month_key"]) if row and row["latest_month_key"] else None
|
||||
|
||||
|
||||
def get_latest_elo_mmr_generated_at(*, db_path: Path | None = None) -> datetime | None:
|
||||
"""Return the latest persisted Elo/MMR checkpoint generation time, if any."""
|
||||
resolved_path = _resolve_db_path(db_path)
|
||||
try:
|
||||
with _connect_readonly(resolved_path) as connection:
|
||||
row = connection.execute(
|
||||
"""
|
||||
SELECT MAX(generated_at) AS latest_generated_at
|
||||
FROM elo_mmr_monthly_checkpoints
|
||||
"""
|
||||
).fetchone()
|
||||
except sqlite3.OperationalError:
|
||||
return None
|
||||
latest_generated_at = str(row["latest_generated_at"] or "").strip() if row else ""
|
||||
if not latest_generated_at:
|
||||
return None
|
||||
return datetime.fromisoformat(latest_generated_at.replace("Z", "+00:00"))
|
||||
|
||||
|
||||
def _connect_writer(db_path: Path):
|
||||
return connect_sqlite_writer(db_path)
|
||||
|
||||
|
||||
def _connect_readonly(db_path: Path):
|
||||
return connect_sqlite_readonly(db_path)
|
||||
|
||||
|
||||
def _resolve_db_path(db_path: Path | None) -> Path:
|
||||
return db_path or get_storage_path()
|
||||
714
backend/app/historical_ingestion.py
Normal file
714
backend/app/historical_ingestion.py
Normal file
@@ -0,0 +1,714 @@
|
||||
"""Historical CRCON ingestion bootstrap and incremental refresh."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
from dataclasses import dataclass
|
||||
from typing import Callable, Iterable
|
||||
|
||||
from .config import (
|
||||
get_historical_crcon_detail_workers,
|
||||
get_historical_crcon_page_size,
|
||||
get_historical_data_source_kind,
|
||||
get_historical_refresh_overlap_hours,
|
||||
)
|
||||
from .data_sources import (
|
||||
SOURCE_KIND_PUBLIC_SCOREBOARD,
|
||||
SOURCE_KIND_RCON,
|
||||
HistoricalDataSource,
|
||||
build_historical_runtime_source_policy,
|
||||
resolve_historical_ingestion_data_source,
|
||||
)
|
||||
from .elo_mmr_engine import rebuild_elo_mmr_models
|
||||
from .historical_snapshots import generate_and_persist_historical_snapshots
|
||||
from .historical_storage import (
|
||||
finalize_backfill_progress,
|
||||
finalize_ingestion_run,
|
||||
get_backfill_resume_page,
|
||||
get_refresh_cutoff_for_server,
|
||||
initialize_historical_storage,
|
||||
list_historical_coverage_report,
|
||||
list_historical_servers,
|
||||
mark_backfill_progress_page_completed,
|
||||
mark_backfill_progress_started,
|
||||
start_ingestion_run,
|
||||
upsert_historical_match,
|
||||
)
|
||||
from .rcon_historical_worker import run_rcon_historical_capture_unlocked
|
||||
from .writer_lock import backend_writer_lock, build_writer_lock_holder
|
||||
|
||||
|
||||
ProgressCallback = Callable[[dict[str, object]], None]
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class IngestionStats:
|
||||
"""Mutable counters for one ingestion execution."""
|
||||
|
||||
pages_processed: int = 0
|
||||
matches_seen: int = 0
|
||||
matches_inserted: int = 0
|
||||
matches_updated: int = 0
|
||||
player_rows_inserted: int = 0
|
||||
player_rows_updated: int = 0
|
||||
|
||||
def apply(self, delta: dict[str, int]) -> None:
|
||||
self.matches_inserted += delta.get("matches_inserted", 0)
|
||||
self.matches_updated += delta.get("matches_updated", 0)
|
||||
self.player_rows_inserted += delta.get("player_rows_inserted", 0)
|
||||
self.player_rows_updated += delta.get("player_rows_updated", 0)
|
||||
|
||||
|
||||
def run_bootstrap(
|
||||
*,
|
||||
server_slug: str | None = None,
|
||||
max_pages: int | None = None,
|
||||
page_size: int | None = None,
|
||||
start_page: int | None = None,
|
||||
detail_workers: int | None = None,
|
||||
rebuild_snapshots: bool = True,
|
||||
progress_callback: ProgressCallback | None = None,
|
||||
) -> dict[str, object]:
|
||||
"""Run a first full historical import against one or all configured servers."""
|
||||
with backend_writer_lock(
|
||||
holder=build_writer_lock_holder(
|
||||
f"app.historical_ingestion bootstrap:{server_slug or 'all-servers'}"
|
||||
)
|
||||
):
|
||||
return _run_ingestion(
|
||||
mode="bootstrap",
|
||||
server_slug=server_slug,
|
||||
max_pages=max_pages,
|
||||
page_size=page_size,
|
||||
start_page=start_page,
|
||||
detail_workers=detail_workers,
|
||||
overlap_hours=None,
|
||||
incremental=False,
|
||||
rebuild_snapshots=rebuild_snapshots,
|
||||
progress_callback=progress_callback,
|
||||
)
|
||||
|
||||
|
||||
def run_incremental_refresh(
|
||||
*,
|
||||
server_slug: str | None = None,
|
||||
max_pages: int | None = None,
|
||||
page_size: int | None = None,
|
||||
start_page: int | None = None,
|
||||
detail_workers: int | None = None,
|
||||
overlap_hours: int | None = None,
|
||||
rebuild_snapshots: bool = True,
|
||||
progress_callback: ProgressCallback | None = None,
|
||||
) -> dict[str, object]:
|
||||
"""Refresh recent historical pages without replaying the whole archive."""
|
||||
with backend_writer_lock(
|
||||
holder=build_writer_lock_holder(
|
||||
f"app.historical_ingestion refresh:{server_slug or 'all-servers'}"
|
||||
)
|
||||
):
|
||||
return _run_ingestion(
|
||||
mode="incremental",
|
||||
server_slug=server_slug,
|
||||
max_pages=max_pages,
|
||||
page_size=page_size,
|
||||
start_page=start_page,
|
||||
detail_workers=detail_workers,
|
||||
overlap_hours=overlap_hours,
|
||||
incremental=True,
|
||||
rebuild_snapshots=rebuild_snapshots,
|
||||
progress_callback=progress_callback,
|
||||
)
|
||||
|
||||
|
||||
def _run_ingestion(
|
||||
*,
|
||||
mode: str,
|
||||
server_slug: str | None,
|
||||
max_pages: int | None,
|
||||
page_size: int | None,
|
||||
start_page: int | None,
|
||||
detail_workers: int | None,
|
||||
overlap_hours: int | None,
|
||||
incremental: bool,
|
||||
rebuild_snapshots: bool,
|
||||
progress_callback: ProgressCallback | None,
|
||||
) -> dict[str, object]:
|
||||
initialize_historical_storage()
|
||||
stats = IngestionStats()
|
||||
fallback_data_source, fallback_source_policy = resolve_historical_ingestion_data_source()
|
||||
selected_servers = _select_servers(server_slug)
|
||||
processed_servers: list[dict[str, object]] = []
|
||||
active_runs: dict[str, int] = {}
|
||||
resolved_overlap_hours = (
|
||||
get_historical_refresh_overlap_hours()
|
||||
if overlap_hours is None
|
||||
else overlap_hours
|
||||
)
|
||||
if resolved_overlap_hours < 0:
|
||||
raise ValueError("--overlap-hours must be zero or positive.")
|
||||
|
||||
primary_writer_result = _attempt_primary_rcon_writer(
|
||||
mode=mode,
|
||||
server_slug=server_slug,
|
||||
selected_servers=selected_servers,
|
||||
progress_callback=progress_callback,
|
||||
)
|
||||
source_policy = _resolve_ingestion_source_policy(
|
||||
fallback_source_policy=fallback_source_policy,
|
||||
primary_writer_result=primary_writer_result,
|
||||
)
|
||||
use_classic_fallback = _should_use_classic_fallback(primary_writer_result)
|
||||
_emit_progress(
|
||||
progress_callback,
|
||||
{
|
||||
"event": "historical-ingestion-source-selected",
|
||||
"mode": mode,
|
||||
"primary_source": source_policy.get("primary_source"),
|
||||
"selected_source": source_policy.get("selected_source"),
|
||||
"fallback_used": bool(source_policy.get("fallback_used")),
|
||||
"fallback_reason": source_policy.get("fallback_reason"),
|
||||
},
|
||||
)
|
||||
|
||||
try:
|
||||
if use_classic_fallback:
|
||||
for server in selected_servers:
|
||||
run_id = start_ingestion_run(mode=mode, target_server_slug=str(server["slug"]))
|
||||
active_runs[str(server["slug"])] = run_id
|
||||
mark_backfill_progress_started(
|
||||
server_slug=str(server["slug"]),
|
||||
mode=mode,
|
||||
run_id=run_id,
|
||||
)
|
||||
cutoff = (
|
||||
get_refresh_cutoff_for_server(
|
||||
str(server["slug"]),
|
||||
overlap_hours=resolved_overlap_hours,
|
||||
)
|
||||
if incremental
|
||||
else None
|
||||
)
|
||||
resolved_start_page = _resolve_start_page(
|
||||
start_page=start_page,
|
||||
server_slug=str(server["slug"]),
|
||||
mode=mode,
|
||||
)
|
||||
server_stats = _ingest_server(
|
||||
server=server,
|
||||
mode=mode,
|
||||
run_id=run_id,
|
||||
stats=stats,
|
||||
data_source=fallback_data_source,
|
||||
max_pages=max_pages,
|
||||
page_size=page_size,
|
||||
start_page=resolved_start_page,
|
||||
detail_workers=detail_workers,
|
||||
cutoff=cutoff,
|
||||
progress_callback=progress_callback,
|
||||
source_policy=source_policy,
|
||||
)
|
||||
processed_servers.append(server_stats)
|
||||
finalize_ingestion_run(
|
||||
run_id,
|
||||
status="success",
|
||||
pages_processed=server_stats["pages_processed"],
|
||||
matches_seen=server_stats["matches_seen"],
|
||||
matches_inserted=server_stats["matches_inserted"],
|
||||
matches_updated=server_stats["matches_updated"],
|
||||
player_rows_inserted=server_stats["player_rows_inserted"],
|
||||
player_rows_updated=server_stats["player_rows_updated"],
|
||||
notes=f"public_name={server_stats['public_name']}",
|
||||
)
|
||||
finalize_backfill_progress(
|
||||
server_slug=str(server["slug"]),
|
||||
mode=mode,
|
||||
run_id=run_id,
|
||||
status="success",
|
||||
archive_exhausted=bool(server_stats["archive_exhausted"]),
|
||||
)
|
||||
active_runs.pop(str(server["slug"]), None)
|
||||
if rebuild_snapshots:
|
||||
snapshot_result = generate_and_persist_historical_snapshots(server_key=server_slug)
|
||||
elo_mmr_result = rebuild_elo_mmr_models()
|
||||
else:
|
||||
snapshot_result = {
|
||||
"status": "skipped",
|
||||
"reason": "snapshot-rebuild-disabled",
|
||||
"generation_policy": "handled-by-caller",
|
||||
}
|
||||
elo_mmr_result = {
|
||||
"status": "skipped",
|
||||
"reason": "snapshot-rebuild-disabled",
|
||||
}
|
||||
except Exception as exc:
|
||||
for active_server_slug, run_id in active_runs.items():
|
||||
finalize_ingestion_run(
|
||||
run_id,
|
||||
status="failed",
|
||||
pages_processed=stats.pages_processed,
|
||||
matches_seen=stats.matches_seen,
|
||||
matches_inserted=stats.matches_inserted,
|
||||
matches_updated=stats.matches_updated,
|
||||
player_rows_inserted=stats.player_rows_inserted,
|
||||
player_rows_updated=stats.player_rows_updated,
|
||||
notes=str(exc),
|
||||
)
|
||||
finalize_backfill_progress(
|
||||
server_slug=active_server_slug,
|
||||
mode=mode,
|
||||
run_id=run_id,
|
||||
status="failed",
|
||||
error_message=str(exc),
|
||||
)
|
||||
raise
|
||||
|
||||
return {
|
||||
"status": "ok",
|
||||
"mode": mode,
|
||||
"source_provider": source_policy.get("selected_source"),
|
||||
"source_policy": source_policy,
|
||||
"primary_writer_result": primary_writer_result,
|
||||
"page_size": page_size or get_historical_crcon_page_size(),
|
||||
"start_page": start_page,
|
||||
"detail_workers": detail_workers or get_historical_crcon_detail_workers(),
|
||||
"overlap_hours": resolved_overlap_hours if incremental else None,
|
||||
"servers": processed_servers,
|
||||
"coverage": list_historical_coverage_report(server_slug=server_slug),
|
||||
"snapshot_result": snapshot_result,
|
||||
"elo_mmr_result": elo_mmr_result,
|
||||
"totals": {
|
||||
"pages_processed": stats.pages_processed,
|
||||
"matches_seen": stats.matches_seen,
|
||||
"matches_inserted": stats.matches_inserted,
|
||||
"matches_updated": stats.matches_updated,
|
||||
"player_rows_inserted": stats.player_rows_inserted,
|
||||
"player_rows_updated": stats.player_rows_updated,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _ingest_server(
|
||||
*,
|
||||
server: dict[str, object],
|
||||
mode: str,
|
||||
run_id: int,
|
||||
stats: IngestionStats,
|
||||
data_source: HistoricalDataSource,
|
||||
max_pages: int | None,
|
||||
page_size: int | None,
|
||||
start_page: int,
|
||||
detail_workers: int | None,
|
||||
cutoff: str | None,
|
||||
progress_callback: ProgressCallback | None,
|
||||
source_policy: dict[str, object],
|
||||
) -> dict[str, object]:
|
||||
resolved_page_size = page_size or get_historical_crcon_page_size()
|
||||
resolved_detail_workers = detail_workers or get_historical_crcon_detail_workers()
|
||||
page_limit = max_pages or 1000000
|
||||
start_page = max(1, start_page)
|
||||
local_stats = IngestionStats()
|
||||
public_info = data_source.fetch_public_info(base_url=str(server["scoreboard_base_url"]))
|
||||
discovered_total_matches: int | None = None
|
||||
last_page_processed: int | None = None
|
||||
archive_exhausted = False
|
||||
_emit_progress(
|
||||
progress_callback,
|
||||
{
|
||||
"event": "historical-ingestion-server-started",
|
||||
"mode": mode,
|
||||
"server_slug": server["slug"],
|
||||
"selected_source": source_policy.get("selected_source"),
|
||||
"fallback_used": bool(source_policy.get("fallback_used")),
|
||||
"start_page": start_page,
|
||||
"cutoff": cutoff,
|
||||
},
|
||||
)
|
||||
|
||||
for page_number in range(start_page, start_page + page_limit):
|
||||
payload = data_source.fetch_match_page(
|
||||
base_url=str(server["scoreboard_base_url"]),
|
||||
page=page_number,
|
||||
limit=resolved_page_size,
|
||||
)
|
||||
if discovered_total_matches is None:
|
||||
discovered_total_matches = _coerce_int(payload.get("total"))
|
||||
page_matches = _coerce_match_list(payload.get("maps"))
|
||||
if not page_matches:
|
||||
archive_exhausted = True
|
||||
break
|
||||
|
||||
local_stats.pages_processed += 1
|
||||
stats.pages_processed += 1
|
||||
last_page_processed = page_number
|
||||
stop_after_page = False
|
||||
match_ids_to_fetch: list[str] = []
|
||||
|
||||
for match_summary in page_matches:
|
||||
local_stats.matches_seen += 1
|
||||
stats.matches_seen += 1
|
||||
|
||||
reference_timestamp = _pick_match_timestamp(match_summary)
|
||||
if cutoff and reference_timestamp and reference_timestamp < cutoff:
|
||||
stop_after_page = True
|
||||
continue
|
||||
|
||||
match_id = _stringify(match_summary.get("id"))
|
||||
if match_id:
|
||||
match_ids_to_fetch.append(match_id)
|
||||
|
||||
_emit_progress(
|
||||
progress_callback,
|
||||
{
|
||||
"event": "historical-ingestion-page-loaded",
|
||||
"mode": mode,
|
||||
"server_slug": server["slug"],
|
||||
"page": page_number,
|
||||
"selected_source": source_policy.get("selected_source"),
|
||||
"match_ids_to_detail": len(match_ids_to_fetch),
|
||||
"page_matches": len(page_matches),
|
||||
"cutoff_reached": stop_after_page,
|
||||
},
|
||||
)
|
||||
|
||||
for detail_payload in data_source.fetch_match_details(
|
||||
base_url=str(server["scoreboard_base_url"]),
|
||||
match_ids=match_ids_to_fetch,
|
||||
max_workers=resolved_detail_workers,
|
||||
):
|
||||
delta = upsert_historical_match(
|
||||
server_slug=str(server["slug"]),
|
||||
match_payload=detail_payload,
|
||||
)
|
||||
local_stats.apply(delta)
|
||||
stats.apply(delta)
|
||||
|
||||
mark_backfill_progress_page_completed(
|
||||
server_slug=str(server["slug"]),
|
||||
mode=mode,
|
||||
page_number=page_number,
|
||||
page_size=resolved_page_size,
|
||||
run_id=run_id,
|
||||
discovered_total_matches=discovered_total_matches,
|
||||
)
|
||||
|
||||
if stop_after_page:
|
||||
break
|
||||
|
||||
return {
|
||||
"server_slug": server["slug"],
|
||||
"public_name": _extract_public_name(public_info),
|
||||
"server_number": public_info.get("server_number") or server.get("server_number"),
|
||||
"source_provider": data_source.source_kind,
|
||||
"pages_processed": local_stats.pages_processed,
|
||||
"matches_seen": local_stats.matches_seen,
|
||||
"discovered_total_matches": discovered_total_matches,
|
||||
"matches_inserted": local_stats.matches_inserted,
|
||||
"matches_updated": local_stats.matches_updated,
|
||||
"player_rows_inserted": local_stats.player_rows_inserted,
|
||||
"player_rows_updated": local_stats.player_rows_updated,
|
||||
"start_page": start_page,
|
||||
"last_page_processed": last_page_processed,
|
||||
"cutoff": cutoff,
|
||||
"archive_exhausted": archive_exhausted,
|
||||
}
|
||||
|
||||
|
||||
def _resolve_start_page(
|
||||
*,
|
||||
start_page: int | None,
|
||||
server_slug: str,
|
||||
mode: str,
|
||||
) -> int:
|
||||
if start_page is not None:
|
||||
return max(1, start_page)
|
||||
if mode != "bootstrap":
|
||||
return 1
|
||||
return get_backfill_resume_page(server_slug, mode=mode)
|
||||
|
||||
|
||||
def _attempt_primary_rcon_writer(
|
||||
*,
|
||||
mode: str,
|
||||
server_slug: str | None,
|
||||
selected_servers: list[dict[str, object]],
|
||||
progress_callback: ProgressCallback | None,
|
||||
) -> dict[str, object]:
|
||||
configured_kind = get_historical_data_source_kind()
|
||||
if configured_kind != SOURCE_KIND_RCON:
|
||||
result = {
|
||||
"attempted": False,
|
||||
"status": "skipped",
|
||||
"primary_source": SOURCE_KIND_PUBLIC_SCOREBOARD,
|
||||
"selected_source": SOURCE_KIND_PUBLIC_SCOREBOARD,
|
||||
"fallback_used": False,
|
||||
"fallback_reason": None,
|
||||
"source_attempts": [],
|
||||
}
|
||||
_emit_progress(
|
||||
progress_callback,
|
||||
{
|
||||
"event": "historical-ingestion-rcon-primary-skipped",
|
||||
"mode": mode,
|
||||
"reason": "historical-data-source-configured-for-public-scoreboard",
|
||||
},
|
||||
)
|
||||
return result
|
||||
|
||||
target_scope = server_slug or "all-configured-rcon-targets"
|
||||
_emit_progress(
|
||||
progress_callback,
|
||||
{
|
||||
"event": "historical-ingestion-rcon-primary-started",
|
||||
"mode": mode,
|
||||
"target_scope": target_scope,
|
||||
"servers": [str(server["slug"]) for server in selected_servers],
|
||||
},
|
||||
)
|
||||
try:
|
||||
capture_result = run_rcon_historical_capture_unlocked(target_key=server_slug)
|
||||
except Exception as exc: # noqa: BLE001 - fallback remains explicit and controlled
|
||||
result = {
|
||||
"attempted": True,
|
||||
"status": "error",
|
||||
"primary_source": SOURCE_KIND_RCON,
|
||||
"selected_source": SOURCE_KIND_PUBLIC_SCOREBOARD,
|
||||
"fallback_used": True,
|
||||
"fallback_reason": "rcon-historical-writer-request-failed",
|
||||
"message": str(exc),
|
||||
}
|
||||
_emit_progress(
|
||||
progress_callback,
|
||||
{
|
||||
"event": "historical-ingestion-rcon-primary-failed",
|
||||
"mode": mode,
|
||||
"target_scope": target_scope,
|
||||
"message": str(exc),
|
||||
},
|
||||
)
|
||||
return result
|
||||
|
||||
capture_run_status = str(capture_result.get("run_status") or capture_result.get("status") or "unknown")
|
||||
targets = list(capture_result.get("targets") or [])
|
||||
errors = list(capture_result.get("errors") or [])
|
||||
if targets:
|
||||
result = {
|
||||
"attempted": True,
|
||||
"status": "partial",
|
||||
"primary_source": SOURCE_KIND_RCON,
|
||||
"selected_source": SOURCE_KIND_PUBLIC_SCOREBOARD,
|
||||
"fallback_used": True,
|
||||
"fallback_reason": "rcon-primary-writer-succeeded-but-classic-match-archive-still-needs-fallback",
|
||||
"capture_result": capture_result,
|
||||
}
|
||||
_emit_progress(
|
||||
progress_callback,
|
||||
{
|
||||
"event": "historical-ingestion-rcon-primary-succeeded",
|
||||
"mode": mode,
|
||||
"target_scope": target_scope,
|
||||
"captured_targets": len(targets),
|
||||
"run_status": capture_run_status,
|
||||
"next_step": "classic-public-scoreboard-fallback-required",
|
||||
},
|
||||
)
|
||||
return result
|
||||
|
||||
result = {
|
||||
"attempted": True,
|
||||
"status": "empty",
|
||||
"primary_source": SOURCE_KIND_RCON,
|
||||
"selected_source": SOURCE_KIND_PUBLIC_SCOREBOARD,
|
||||
"fallback_used": True,
|
||||
"fallback_reason": "rcon-historical-writer-returned-no-usable-samples",
|
||||
"capture_result": capture_result,
|
||||
"message": json.dumps(errors, separators=(",", ":")) if errors else None,
|
||||
}
|
||||
_emit_progress(
|
||||
progress_callback,
|
||||
{
|
||||
"event": "historical-ingestion-rcon-primary-empty",
|
||||
"mode": mode,
|
||||
"target_scope": target_scope,
|
||||
"run_status": capture_run_status,
|
||||
"errors": len(errors),
|
||||
},
|
||||
)
|
||||
return result
|
||||
|
||||
|
||||
def _should_use_classic_fallback(primary_writer_result: dict[str, object]) -> bool:
|
||||
selected_source = str(primary_writer_result.get("selected_source") or "")
|
||||
return selected_source == SOURCE_KIND_PUBLIC_SCOREBOARD
|
||||
|
||||
|
||||
def _resolve_ingestion_source_policy(
|
||||
*,
|
||||
fallback_source_policy: dict[str, object],
|
||||
primary_writer_result: dict[str, object],
|
||||
) -> dict[str, object]:
|
||||
configured_kind = get_historical_data_source_kind()
|
||||
if configured_kind != SOURCE_KIND_RCON:
|
||||
return fallback_source_policy
|
||||
|
||||
status = str(primary_writer_result.get("status") or "error")
|
||||
selected_source = str(
|
||||
primary_writer_result.get("selected_source") or SOURCE_KIND_PUBLIC_SCOREBOARD
|
||||
)
|
||||
fallback_reason = primary_writer_result.get("fallback_reason")
|
||||
message = primary_writer_result.get("message")
|
||||
if (
|
||||
fallback_reason
|
||||
== "rcon-primary-writer-succeeded-but-classic-match-archive-still-needs-fallback"
|
||||
):
|
||||
message = (
|
||||
"RCON prospective capture succeeded first, but the classic historical_* "
|
||||
"archive still requires public-scoreboard for match-page import."
|
||||
)
|
||||
return build_historical_runtime_source_policy(
|
||||
operation="historical-ingestion",
|
||||
rcon_status=status,
|
||||
fallback_reason=str(fallback_reason) if fallback_reason else None,
|
||||
selected_source=selected_source,
|
||||
rcon_message=message if isinstance(message, str) else None,
|
||||
)
|
||||
|
||||
|
||||
def _emit_progress(
|
||||
callback: ProgressCallback | None,
|
||||
payload: dict[str, object],
|
||||
) -> None:
|
||||
if callback is None:
|
||||
return
|
||||
callback(payload)
|
||||
|
||||
|
||||
def _select_servers(server_slug: str | None) -> list[dict[str, object]]:
|
||||
servers = list_historical_servers()
|
||||
if server_slug is None:
|
||||
return servers
|
||||
|
||||
normalized = server_slug.strip()
|
||||
selected = [server for server in servers if server["slug"] == normalized]
|
||||
if not selected:
|
||||
raise ValueError(f"Unknown historical server slug: {server_slug}")
|
||||
return selected
|
||||
|
||||
|
||||
def _coerce_match_list(payload: object) -> list[dict[str, object]]:
|
||||
if not isinstance(payload, list):
|
||||
return []
|
||||
return [item for item in payload if isinstance(item, dict)]
|
||||
|
||||
|
||||
def _pick_match_timestamp(match_payload: dict[str, object]) -> str | None:
|
||||
for key in ("end", "start", "creation_time"):
|
||||
value = match_payload.get(key)
|
||||
if isinstance(value, str) and value.strip():
|
||||
return value.strip()
|
||||
return None
|
||||
|
||||
|
||||
def _extract_public_name(public_info: dict[str, object]) -> str | None:
|
||||
name_value = public_info.get("name")
|
||||
if isinstance(name_value, str):
|
||||
return name_value
|
||||
if isinstance(name_value, dict):
|
||||
raw_name = name_value.get("name")
|
||||
return raw_name.strip() if isinstance(raw_name, str) and raw_name.strip() else None
|
||||
return None
|
||||
|
||||
|
||||
def _stringify(value: object) -> str | None:
|
||||
if value is None:
|
||||
return None
|
||||
text = str(value).strip()
|
||||
return text or None
|
||||
|
||||
|
||||
def _coerce_int(value: object) -> int | None:
|
||||
if value in (None, ""):
|
||||
return None
|
||||
try:
|
||||
return int(value)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
def build_arg_parser() -> argparse.ArgumentParser:
|
||||
"""Create the CLI parser for manual historical ingestion runs."""
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Historical CRCON ingestion for HLL Vietnam.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"mode",
|
||||
choices=("bootstrap", "refresh"),
|
||||
help="bootstrap imports the archive, refresh only recent pages",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--server",
|
||||
dest="server_slug",
|
||||
help="optional historical server slug",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--max-pages",
|
||||
type=int,
|
||||
help="optional page cap for local validation",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--page-size",
|
||||
type=int,
|
||||
help="override CRCON page size",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--start-page",
|
||||
type=int,
|
||||
help="override the resume page; bootstrap uses persisted progress when omitted",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--detail-workers",
|
||||
type=int,
|
||||
help="parallel worker count for per-match detail requests",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--overlap-hours",
|
||||
type=int,
|
||||
help="override the incremental overlap window in hours",
|
||||
)
|
||||
return parser
|
||||
|
||||
|
||||
def main(argv: Iterable[str] | None = None) -> int:
|
||||
"""Run the historical ingestion CLI."""
|
||||
parser = build_arg_parser()
|
||||
args = parser.parse_args(list(argv) if argv is not None else None)
|
||||
|
||||
def _print_progress(payload: dict[str, object]) -> None:
|
||||
print(json.dumps(payload, ensure_ascii=True))
|
||||
|
||||
if args.mode == "bootstrap":
|
||||
result = run_bootstrap(
|
||||
server_slug=args.server_slug,
|
||||
max_pages=args.max_pages,
|
||||
page_size=args.page_size,
|
||||
start_page=args.start_page,
|
||||
detail_workers=args.detail_workers,
|
||||
progress_callback=_print_progress,
|
||||
)
|
||||
else:
|
||||
result = run_incremental_refresh(
|
||||
server_slug=args.server_slug,
|
||||
max_pages=args.max_pages,
|
||||
page_size=args.page_size,
|
||||
start_page=args.start_page,
|
||||
detail_workers=args.detail_workers,
|
||||
overlap_hours=args.overlap_hours,
|
||||
progress_callback=_print_progress,
|
||||
)
|
||||
|
||||
print(json.dumps(result, indent=2))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
126
backend/app/historical_models.py
Normal file
126
backend/app/historical_models.py
Normal file
@@ -0,0 +1,126 @@
|
||||
"""Historical domain models for persisted CRCON scoreboard data."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class HistoricalServerDefinition:
|
||||
"""Stable identity for one historical CRCON source."""
|
||||
|
||||
slug: str
|
||||
display_name: str
|
||||
scoreboard_base_url: str
|
||||
server_number: int | None
|
||||
source_kind: str = "crcon-scoreboard-json"
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class HistoricalMapRecord:
|
||||
"""Normalized map metadata reused across historical matches."""
|
||||
|
||||
external_map_id: str | None
|
||||
map_name: str | None
|
||||
pretty_name: str | None
|
||||
game_mode: str | None
|
||||
image_name: str | None
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class HistoricalMatchRecord:
|
||||
"""Normalized match identity and summary."""
|
||||
|
||||
external_match_id: str
|
||||
server_slug: str
|
||||
created_at: datetime | None
|
||||
started_at: datetime | None
|
||||
ended_at: datetime | None
|
||||
map_name: str | None
|
||||
map_pretty_name: str | None
|
||||
map_external_id: str | None
|
||||
game_mode: str | None
|
||||
image_name: str | None
|
||||
allied_score: int | None
|
||||
axis_score: int | None
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class HistoricalPlayerIdentity:
|
||||
"""Stable player identity across historical match stats."""
|
||||
|
||||
stable_player_key: str
|
||||
display_name: str
|
||||
steam_id: str | None
|
||||
source_player_id: str | None
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class HistoricalPlayerMatchStats:
|
||||
"""Metrics persisted per player and match."""
|
||||
|
||||
stable_player_key: str
|
||||
match_player_ref: str | None
|
||||
team_side: str | None
|
||||
level: int | None
|
||||
kills: int | None
|
||||
deaths: int | None
|
||||
teamkills: int | None
|
||||
time_seconds: int | None
|
||||
kills_per_minute: float | None
|
||||
deaths_per_minute: float | None
|
||||
kill_death_ratio: float | None
|
||||
combat: int | None
|
||||
offense: int | None
|
||||
defense: int | None
|
||||
support: int | None
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class HistoricalIngestionRunSummary:
|
||||
"""Outcome metadata recorded for one ingestion execution."""
|
||||
|
||||
mode: str
|
||||
started_at: datetime
|
||||
completed_at: datetime | None
|
||||
status: str
|
||||
pages_processed: int
|
||||
matches_seen: int
|
||||
matches_inserted: int
|
||||
matches_updated: int
|
||||
player_rows_inserted: int
|
||||
player_rows_updated: int
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class HistoricalBackfillProgressSummary:
|
||||
"""Persisted resume checkpoint and last attempt metadata per server."""
|
||||
|
||||
server_slug: str
|
||||
mode: str
|
||||
next_page: int
|
||||
last_completed_page: int | None
|
||||
discovered_total_matches: int | None
|
||||
discovered_total_pages: int | None
|
||||
archive_exhausted: bool
|
||||
last_run_id: int | None
|
||||
last_run_status: str | None
|
||||
last_run_started_at: datetime | None
|
||||
last_run_completed_at: datetime | None
|
||||
last_error: str | None
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class HistoricalSnapshotRecord:
|
||||
"""Persisted precomputed historical snapshot ready for lightweight reads."""
|
||||
|
||||
server_key: str
|
||||
snapshot_type: str
|
||||
metric: str | None
|
||||
window: str | None
|
||||
payload_json: str
|
||||
generated_at: datetime
|
||||
source_range_start: datetime | None
|
||||
source_range_end: datetime | None
|
||||
is_stale: bool
|
||||
529
backend/app/historical_runner.py
Normal file
529
backend/app/historical_runner.py
Normal file
@@ -0,0 +1,529 @@
|
||||
"""Local development loop for periodic historical CRCON refreshes."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import time
|
||||
import traceback
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any
|
||||
|
||||
from .config import (
|
||||
DEFAULT_DB_MAINTENANCE_INTERVAL_SECONDS,
|
||||
get_db_maintenance_enabled,
|
||||
get_db_maintenance_interval_seconds,
|
||||
get_historical_full_snapshot_every_runs,
|
||||
get_historical_elo_mmr_min_new_samples,
|
||||
get_historical_elo_mmr_rebuild_interval_minutes,
|
||||
get_historical_refresh_interval_seconds,
|
||||
get_historical_refresh_max_retries,
|
||||
get_historical_refresh_retry_delay_seconds,
|
||||
get_historical_data_source_kind,
|
||||
)
|
||||
from .database_maintenance import run_database_maintenance_cleanup
|
||||
from .elo_mmr_engine import rebuild_elo_mmr_models
|
||||
from .elo_mmr_storage import get_latest_elo_mmr_generated_at
|
||||
from .historical_ingestion import run_incremental_refresh
|
||||
from .historical_snapshots import (
|
||||
generate_and_persist_historical_snapshots,
|
||||
generate_and_persist_priority_historical_snapshots,
|
||||
)
|
||||
from .rcon_historical_storage import count_rcon_historical_samples_since
|
||||
from .rcon_historical_worker import run_rcon_historical_capture
|
||||
from .writer_lock import backend_writer_lock, build_writer_lock_holder
|
||||
|
||||
HOURLY_INTERVAL_SECONDS = 3600
|
||||
DEFAULT_HISTORICAL_SERVER_SCOPE = (
|
||||
"comunidad-hispana-01",
|
||||
"comunidad-hispana-02",
|
||||
)
|
||||
_LAST_DATABASE_MAINTENANCE_RUN_AT: datetime | None = None
|
||||
|
||||
|
||||
def run_periodic_historical_refresh(
|
||||
*,
|
||||
interval_seconds: int,
|
||||
max_retries: int,
|
||||
retry_delay_seconds: float,
|
||||
server_slug: str | None = None,
|
||||
max_pages: int | None = None,
|
||||
page_size: int | None = None,
|
||||
max_runs: int | None = None,
|
||||
) -> None:
|
||||
"""Run periodic historical refreshes and rebuild persisted snapshots."""
|
||||
completed_runs = 0
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"event": "historical-refresh-loop-started",
|
||||
"interval_seconds": interval_seconds,
|
||||
"max_retries": max_retries,
|
||||
"retry_delay_seconds": retry_delay_seconds,
|
||||
"server_scope": _describe_refresh_scope(server_slug),
|
||||
"snapshot_scope": _describe_snapshot_scope(server_slug),
|
||||
},
|
||||
indent=2,
|
||||
)
|
||||
)
|
||||
print("Press Ctrl+C to stop.")
|
||||
|
||||
try:
|
||||
while max_runs is None or completed_runs < max_runs:
|
||||
completed_runs += 1
|
||||
payload = _run_refresh_with_retries(
|
||||
max_retries=max_retries,
|
||||
retry_delay_seconds=retry_delay_seconds,
|
||||
server_slug=server_slug,
|
||||
max_pages=max_pages,
|
||||
page_size=page_size,
|
||||
run_number=completed_runs,
|
||||
)
|
||||
_emit_json_log({"run": completed_runs, **payload})
|
||||
|
||||
if max_runs is not None and completed_runs >= max_runs:
|
||||
break
|
||||
|
||||
time.sleep(interval_seconds)
|
||||
except KeyboardInterrupt:
|
||||
print("\nHistorical refresh loop stopped by user.")
|
||||
|
||||
|
||||
def _run_refresh_with_retries(
|
||||
*,
|
||||
max_retries: int,
|
||||
retry_delay_seconds: float,
|
||||
server_slug: str | None,
|
||||
max_pages: int | None,
|
||||
page_size: int | None,
|
||||
run_number: int,
|
||||
) -> dict[str, Any]:
|
||||
attempt = 0
|
||||
while True:
|
||||
attempt += 1
|
||||
try:
|
||||
with backend_writer_lock(
|
||||
holder=build_writer_lock_holder(
|
||||
f"app.historical_runner refresh:{server_slug or 'all-servers'}"
|
||||
)
|
||||
):
|
||||
rcon_capture_result = _run_primary_rcon_capture()
|
||||
should_run_classic_fallback, classic_fallback_reason = (
|
||||
_resolve_classic_fallback_policy(
|
||||
server_slug=server_slug,
|
||||
run_number=run_number,
|
||||
rcon_capture_result=rcon_capture_result,
|
||||
)
|
||||
)
|
||||
if should_run_classic_fallback:
|
||||
refresh_result = run_incremental_refresh(
|
||||
server_slug=server_slug,
|
||||
max_pages=max_pages,
|
||||
page_size=page_size,
|
||||
rebuild_snapshots=False,
|
||||
)
|
||||
snapshot_result = generate_historical_snapshots(
|
||||
server_slug=server_slug,
|
||||
run_number=run_number,
|
||||
)
|
||||
elo_mmr_result = rebuild_elo_mmr_models()
|
||||
else:
|
||||
should_generate_snapshots = _rcon_capture_has_new_useful_data(
|
||||
rcon_capture_result
|
||||
)
|
||||
refresh_result = {
|
||||
"status": "skipped",
|
||||
"reason": "rcon-primary-cycle-no-classic-fallback-needed",
|
||||
}
|
||||
if should_generate_snapshots:
|
||||
snapshot_result = generate_historical_snapshots(
|
||||
server_slug=server_slug,
|
||||
run_number=run_number,
|
||||
)
|
||||
snapshot_result = {
|
||||
**snapshot_result,
|
||||
"generation_policy": "rcon-primary-useful-cycle",
|
||||
"reason": "rcon-primary-cycle-produced-new-useful-coverage",
|
||||
}
|
||||
elo_policy = _build_elo_mmr_rebuild_policy(
|
||||
rcon_capture_result=rcon_capture_result
|
||||
)
|
||||
if bool(elo_policy["due"]):
|
||||
elo_mmr_result = {
|
||||
**rebuild_elo_mmr_models(),
|
||||
"generation_policy": "rcon-primary-useful-cycle-elo-rebuild-due",
|
||||
"reason": "rcon-primary-useful-cycle-met-elo-rebuild-threshold",
|
||||
**elo_policy,
|
||||
}
|
||||
else:
|
||||
elo_mmr_result = {
|
||||
"status": "skipped",
|
||||
"reason": "rcon-primary-useful-cycle-elo-rebuild-throttled",
|
||||
"generation_policy": "rcon-primary-useful-cycle-elo-rebuild-throttled",
|
||||
**elo_policy,
|
||||
}
|
||||
else:
|
||||
snapshot_result = {
|
||||
"status": "skipped",
|
||||
"reason": "rcon-primary-cycle-had-no-new-useful-data",
|
||||
"generation_policy": "rcon-primary-no-new-useful-data",
|
||||
}
|
||||
elo_mmr_result = {
|
||||
"status": "skipped",
|
||||
"reason": "rcon-primary-cycle-had-no-new-useful-data",
|
||||
"generation_policy": "rcon-primary-no-new-useful-data",
|
||||
**_build_elo_mmr_rebuild_policy(
|
||||
rcon_capture_result=rcon_capture_result
|
||||
),
|
||||
}
|
||||
maintenance_result = _maybe_run_database_maintenance()
|
||||
return {
|
||||
"status": "ok",
|
||||
"attempts_used": attempt,
|
||||
"max_retries": max_retries,
|
||||
"rcon_capture_result": rcon_capture_result,
|
||||
"classic_fallback_used": should_run_classic_fallback,
|
||||
"classic_fallback_reason": classic_fallback_reason,
|
||||
"refresh_result": refresh_result,
|
||||
"snapshot_result": snapshot_result,
|
||||
"elo_mmr_result": elo_mmr_result,
|
||||
"database_maintenance_result": maintenance_result,
|
||||
}
|
||||
except Exception as exc:
|
||||
failure_payload = {
|
||||
"event": "historical-refresh-attempt-failed",
|
||||
"attempt": attempt,
|
||||
"max_retries": max_retries,
|
||||
"server_scope": _describe_refresh_scope(server_slug),
|
||||
"snapshot_scope": _describe_snapshot_scope(server_slug),
|
||||
"error_type": type(exc).__name__,
|
||||
"error": str(exc),
|
||||
"traceback": traceback.format_exc(),
|
||||
}
|
||||
_emit_json_log(failure_payload)
|
||||
if attempt > max_retries:
|
||||
return {
|
||||
"status": "error",
|
||||
"attempts_used": attempt,
|
||||
"max_retries": max_retries,
|
||||
"error_type": type(exc).__name__,
|
||||
"error": str(exc),
|
||||
"traceback": failure_payload["traceback"],
|
||||
}
|
||||
if retry_delay_seconds > 0:
|
||||
time.sleep(retry_delay_seconds)
|
||||
|
||||
|
||||
def generate_historical_snapshots(
|
||||
*,
|
||||
server_slug: str | None = None,
|
||||
run_number: int = 1,
|
||||
) -> dict[str, Any]:
|
||||
"""Build priority prewarm snapshots on every run and the full matrix on cadence."""
|
||||
generated_at = datetime.now(timezone.utc)
|
||||
full_snapshot_every_runs = get_historical_full_snapshot_every_runs()
|
||||
should_run_full_refresh = bool(server_slug) or run_number % full_snapshot_every_runs == 0
|
||||
_emit_json_log(
|
||||
{
|
||||
"event": "historical-snapshot-refresh-started",
|
||||
"run_number": run_number,
|
||||
"snapshot_step": "full-matrix" if should_run_full_refresh else "priority-prewarm",
|
||||
"server_slug": server_slug,
|
||||
"snapshot_scope": _describe_snapshot_scope(server_slug),
|
||||
}
|
||||
)
|
||||
if should_run_full_refresh:
|
||||
result = generate_and_persist_historical_snapshots(
|
||||
server_key=server_slug,
|
||||
generated_at=generated_at,
|
||||
)
|
||||
else:
|
||||
result = generate_and_persist_priority_historical_snapshots(
|
||||
generated_at=generated_at,
|
||||
)
|
||||
return {
|
||||
**result,
|
||||
"run_number": run_number,
|
||||
"full_snapshot_every_runs": full_snapshot_every_runs,
|
||||
"prewarm_only": not should_run_full_refresh,
|
||||
"refresh_interval_seconds": get_historical_refresh_interval_seconds(),
|
||||
"includes_monthly_mvp_v2": True,
|
||||
}
|
||||
|
||||
|
||||
def _emit_json_log(payload: dict[str, Any]) -> None:
|
||||
"""Print JSON logs that remain safe for Compose and log collectors."""
|
||||
print(json.dumps(payload, ensure_ascii=True, default=str), flush=True)
|
||||
|
||||
|
||||
def _maybe_run_database_maintenance(*, now: datetime | None = None) -> dict[str, Any]:
|
||||
"""Optionally run scheduled database maintenance without crashing the runner."""
|
||||
global _LAST_DATABASE_MAINTENANCE_RUN_AT
|
||||
|
||||
anchor = now.astimezone(timezone.utc) if now else datetime.now(timezone.utc)
|
||||
if not get_db_maintenance_enabled():
|
||||
result = {"status": "skipped", "reason": "disabled", "enabled": False}
|
||||
_emit_json_log({"event": "database-maintenance-scheduler-skipped-disabled", **result})
|
||||
return result
|
||||
|
||||
interval_seconds, interval_source = _resolve_db_maintenance_interval_seconds()
|
||||
if _LAST_DATABASE_MAINTENANCE_RUN_AT is not None:
|
||||
elapsed_seconds = max(
|
||||
0,
|
||||
int((anchor - _LAST_DATABASE_MAINTENANCE_RUN_AT).total_seconds()),
|
||||
)
|
||||
if elapsed_seconds < interval_seconds:
|
||||
result = {
|
||||
"status": "skipped",
|
||||
"reason": "not-due",
|
||||
"enabled": True,
|
||||
"interval_seconds": interval_seconds,
|
||||
"interval_source": interval_source,
|
||||
"elapsed_seconds": elapsed_seconds,
|
||||
"last_run_at": _LAST_DATABASE_MAINTENANCE_RUN_AT.isoformat().replace(
|
||||
"+00:00", "Z"
|
||||
),
|
||||
}
|
||||
_emit_json_log({"event": "database-maintenance-scheduler-skipped-not-due", **result})
|
||||
return result
|
||||
|
||||
_emit_json_log(
|
||||
{
|
||||
"event": "database-maintenance-scheduler-started",
|
||||
"enabled": True,
|
||||
"interval_seconds": interval_seconds,
|
||||
"interval_source": interval_source,
|
||||
"scheduled_at": anchor.isoformat().replace("+00:00", "Z"),
|
||||
}
|
||||
)
|
||||
try:
|
||||
result = run_database_maintenance_cleanup(apply=True, now=anchor)
|
||||
except Exception as exc: # noqa: BLE001 - scheduler must not crash the runner
|
||||
result = {
|
||||
"status": "error",
|
||||
"error_type": type(exc).__name__,
|
||||
"error": str(exc),
|
||||
"enabled": True,
|
||||
"interval_seconds": interval_seconds,
|
||||
"interval_source": interval_source,
|
||||
}
|
||||
_emit_json_log({"event": "database-maintenance-scheduler-failed", **result})
|
||||
return result
|
||||
|
||||
if result.get("status") == "ok":
|
||||
_LAST_DATABASE_MAINTENANCE_RUN_AT = anchor
|
||||
_emit_json_log(
|
||||
{
|
||||
"event": "database-maintenance-scheduler-completed",
|
||||
"enabled": True,
|
||||
"interval_seconds": interval_seconds,
|
||||
"interval_source": interval_source,
|
||||
"result": result,
|
||||
}
|
||||
)
|
||||
return result
|
||||
|
||||
failed_result = {
|
||||
"enabled": True,
|
||||
"interval_seconds": interval_seconds,
|
||||
"interval_source": interval_source,
|
||||
"result": result,
|
||||
}
|
||||
_emit_json_log({"event": "database-maintenance-scheduler-failed", **failed_result})
|
||||
return result
|
||||
|
||||
|
||||
def _resolve_db_maintenance_interval_seconds() -> tuple[int, str]:
|
||||
"""Return a safe maintenance interval even if env configuration is invalid."""
|
||||
try:
|
||||
return get_db_maintenance_interval_seconds(), "env"
|
||||
except ValueError:
|
||||
return DEFAULT_DB_MAINTENANCE_INTERVAL_SECONDS, "default-invalid-env-fallback"
|
||||
|
||||
|
||||
def _describe_refresh_scope(server_slug: str | None) -> list[str]:
|
||||
if server_slug:
|
||||
return [server_slug]
|
||||
return list(DEFAULT_HISTORICAL_SERVER_SCOPE)
|
||||
|
||||
|
||||
def _describe_snapshot_scope(server_slug: str | None) -> list[str]:
|
||||
if server_slug:
|
||||
return [server_slug, "all-servers"]
|
||||
return [*DEFAULT_HISTORICAL_SERVER_SCOPE, "all-servers"]
|
||||
|
||||
|
||||
def _run_primary_rcon_capture() -> dict[str, Any]:
|
||||
if get_historical_data_source_kind() != "rcon":
|
||||
return {
|
||||
"status": "skipped",
|
||||
"reason": "historical-data-source-configured-without-rcon-primary",
|
||||
}
|
||||
return run_rcon_historical_capture()
|
||||
|
||||
|
||||
def _resolve_classic_fallback_policy(
|
||||
*,
|
||||
server_slug: str | None,
|
||||
run_number: int,
|
||||
rcon_capture_result: dict[str, Any],
|
||||
) -> tuple[bool, str]:
|
||||
if get_historical_data_source_kind() != "rcon":
|
||||
return True, "public-scoreboard-configured-as-primary-historical-source"
|
||||
|
||||
if not _rcon_capture_has_usable_results(rcon_capture_result):
|
||||
return True, "rcon-historical-capture-failed-or-returned-no-usable-targets"
|
||||
|
||||
if server_slug:
|
||||
return True, "manual-server-scope-still-needs-classic-historical-fallback"
|
||||
|
||||
if run_number % get_historical_full_snapshot_every_runs() == 0:
|
||||
return True, "periodic-classic-fallback-for-competitive-historical-coverage"
|
||||
|
||||
return False, "rcon-primary-cycle-succeeded-without-needing-classic-fallback"
|
||||
|
||||
|
||||
def _rcon_capture_has_usable_results(rcon_capture_result: dict[str, Any]) -> bool:
|
||||
if rcon_capture_result.get("status") != "ok":
|
||||
return False
|
||||
targets = rcon_capture_result.get("targets")
|
||||
return isinstance(targets, list) and len(targets) > 0
|
||||
|
||||
|
||||
def _rcon_capture_has_new_useful_data(rcon_capture_result: dict[str, Any]) -> bool:
|
||||
if rcon_capture_result.get("status") != "ok":
|
||||
return False
|
||||
totals = rcon_capture_result.get("totals")
|
||||
if isinstance(totals, dict) and int(totals.get("samples_inserted") or 0) > 0:
|
||||
return True
|
||||
if isinstance(totals, dict) and int(totals.get("admin_log_events_inserted") or 0) > 0:
|
||||
return True
|
||||
if isinstance(totals, dict) and int(totals.get("materialized_matches_inserted") or 0) > 0:
|
||||
return True
|
||||
targets = rcon_capture_result.get("targets")
|
||||
if not isinstance(targets, list):
|
||||
return False
|
||||
return any(bool(target.get("sample_inserted")) for target in targets if isinstance(target, dict))
|
||||
|
||||
|
||||
def _build_elo_mmr_rebuild_policy(
|
||||
*,
|
||||
rcon_capture_result: dict[str, Any],
|
||||
) -> dict[str, Any]:
|
||||
interval_minutes = get_historical_elo_mmr_rebuild_interval_minutes()
|
||||
min_new_samples = get_historical_elo_mmr_min_new_samples()
|
||||
last_generated_at = get_latest_elo_mmr_generated_at()
|
||||
last_generated_at_iso = (
|
||||
last_generated_at.astimezone(timezone.utc).isoformat().replace("+00:00", "Z")
|
||||
if last_generated_at is not None
|
||||
else None
|
||||
)
|
||||
minutes_since_last_rebuild = None
|
||||
if last_generated_at is not None:
|
||||
minutes_since_last_rebuild = int(
|
||||
max(
|
||||
0,
|
||||
(
|
||||
datetime.now(timezone.utc) - last_generated_at.astimezone(timezone.utc)
|
||||
).total_seconds() // 60,
|
||||
)
|
||||
)
|
||||
samples_since_last_rebuild = count_rcon_historical_samples_since(last_generated_at_iso)
|
||||
due = (
|
||||
_rcon_capture_has_new_useful_data(rcon_capture_result)
|
||||
and samples_since_last_rebuild >= min_new_samples
|
||||
and (
|
||||
last_generated_at is None
|
||||
or minutes_since_last_rebuild is None
|
||||
or minutes_since_last_rebuild >= interval_minutes
|
||||
)
|
||||
)
|
||||
return {
|
||||
"policy": "min-new-rcon-samples-and-minutes-since-last-successful-rebuild",
|
||||
"due": due,
|
||||
"last_generated_at": last_generated_at_iso,
|
||||
"samples_since_last_rebuild": samples_since_last_rebuild,
|
||||
"minutes_since_last_rebuild": minutes_since_last_rebuild,
|
||||
"rebuild_interval_minutes": interval_minutes,
|
||||
"min_new_samples": min_new_samples,
|
||||
}
|
||||
|
||||
|
||||
def main() -> None:
|
||||
"""Allow local scheduled historical refresh execution without external infra."""
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Run periodic historical refreshes and regenerate snapshots for HLL Vietnam.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--interval",
|
||||
type=int,
|
||||
default=get_historical_refresh_interval_seconds(),
|
||||
help="Seconds to wait between refresh-plus-snapshot runs.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--hourly",
|
||||
action="store_true",
|
||||
help="Shortcut for running the refresh loop every 3600 seconds.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--retries",
|
||||
type=int,
|
||||
default=get_historical_refresh_max_retries(),
|
||||
help="Retry attempts after a failed incremental refresh.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--retry-delay",
|
||||
type=float,
|
||||
default=get_historical_refresh_retry_delay_seconds(),
|
||||
help="Seconds to wait between failed attempts.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--server",
|
||||
dest="server_slug",
|
||||
help="Optional historical server slug.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--max-pages",
|
||||
type=int,
|
||||
default=None,
|
||||
help="Optional page cap for local validation.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--page-size",
|
||||
type=int,
|
||||
default=None,
|
||||
help="Optional override for CRCON page size.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--max-runs",
|
||||
type=int,
|
||||
default=None,
|
||||
help="Optional safety limit for the number of refresh cycles to execute.",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.hourly:
|
||||
args.interval = HOURLY_INTERVAL_SECONDS
|
||||
|
||||
if args.interval <= 0:
|
||||
raise ValueError("--interval must be a positive integer.")
|
||||
if args.retries < 0:
|
||||
raise ValueError("--retries must be zero or positive.")
|
||||
if args.retry_delay < 0:
|
||||
raise ValueError("--retry-delay must be zero or positive.")
|
||||
if args.max_runs is not None and args.max_runs <= 0:
|
||||
raise ValueError("--max-runs must be positive when provided.")
|
||||
|
||||
run_periodic_historical_refresh(
|
||||
interval_seconds=args.interval,
|
||||
max_retries=args.retries,
|
||||
retry_delay_seconds=args.retry_delay,
|
||||
server_slug=args.server_slug,
|
||||
max_pages=args.max_pages,
|
||||
page_size=args.page_size,
|
||||
max_runs=args.max_runs,
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
370
backend/app/historical_snapshot_storage.py
Normal file
370
backend/app/historical_snapshot_storage.py
Normal file
@@ -0,0 +1,370 @@
|
||||
"""File-based persistence for precomputed historical snapshots."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
from .config import get_storage_path, use_postgres_rcon_storage
|
||||
from .historical_models import HistoricalSnapshotRecord
|
||||
from .historical_snapshots import validate_snapshot_identity
|
||||
|
||||
|
||||
SNAPSHOT_DIRECTORY_NAME = "snapshots"
|
||||
|
||||
|
||||
def resolve_historical_snapshot_storage_path(*, db_path: Path | None = None) -> Path:
|
||||
"""Resolve the snapshot directory location without touching SQLite state."""
|
||||
resolved_db_path = db_path or get_storage_path()
|
||||
return resolved_db_path.parent / SNAPSHOT_DIRECTORY_NAME
|
||||
|
||||
|
||||
def initialize_historical_snapshot_storage(*, db_path: Path | None = None) -> Path:
|
||||
"""Create the snapshot directory used by precomputed historical payloads."""
|
||||
snapshots_root = resolve_historical_snapshot_storage_path(db_path=db_path)
|
||||
snapshots_root.mkdir(parents=True, exist_ok=True)
|
||||
return snapshots_root
|
||||
|
||||
|
||||
def persist_historical_snapshot(
|
||||
*,
|
||||
server_key: str,
|
||||
snapshot_type: str,
|
||||
payload: dict[str, object] | list[object],
|
||||
metric: str | None = None,
|
||||
window: str | None = None,
|
||||
generated_at: datetime | None = None,
|
||||
source_range_start: datetime | None = None,
|
||||
source_range_end: datetime | None = None,
|
||||
is_stale: bool = False,
|
||||
db_path: Path | None = None,
|
||||
) -> HistoricalSnapshotRecord:
|
||||
"""Insert or replace one persisted historical snapshot JSON file."""
|
||||
normalized_server_key = server_key.strip()
|
||||
if not normalized_server_key:
|
||||
raise ValueError("server_key is required for historical snapshots.")
|
||||
|
||||
validate_snapshot_identity(snapshot_type=snapshot_type, metric=metric)
|
||||
if use_postgres_rcon_storage(explicit_sqlite_path=db_path):
|
||||
from .postgres_display_storage import persist_snapshot_record
|
||||
|
||||
return persist_snapshot_record(
|
||||
{
|
||||
"server_key": normalized_server_key,
|
||||
"snapshot_type": snapshot_type,
|
||||
"metric": metric,
|
||||
"window": window,
|
||||
"generated_at": generated_at or datetime.now(timezone.utc),
|
||||
"source_range_start": source_range_start,
|
||||
"source_range_end": source_range_end,
|
||||
"is_stale": is_stale,
|
||||
"payload": payload,
|
||||
}
|
||||
)
|
||||
snapshots_root = initialize_historical_snapshot_storage(db_path=db_path)
|
||||
generated_at_value = _as_utc(generated_at or datetime.now(timezone.utc))
|
||||
payload_json = json.dumps(payload, ensure_ascii=True)
|
||||
snapshot_path = _build_snapshot_path(
|
||||
snapshots_root=snapshots_root,
|
||||
server_key=normalized_server_key,
|
||||
snapshot_type=snapshot_type,
|
||||
metric=metric,
|
||||
)
|
||||
snapshot_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
existing_document = _read_snapshot_document(snapshot_path)
|
||||
|
||||
if _should_preserve_existing_snapshot(
|
||||
incoming_payload=payload,
|
||||
snapshot_type=snapshot_type,
|
||||
existing_document=existing_document,
|
||||
):
|
||||
preserved_payload = existing_document.get("payload") if existing_document else payload
|
||||
return HistoricalSnapshotRecord(
|
||||
server_key=normalized_server_key,
|
||||
snapshot_type=snapshot_type,
|
||||
metric=metric,
|
||||
window=window,
|
||||
payload_json=json.dumps(preserved_payload, ensure_ascii=True),
|
||||
generated_at=_parse_optional_datetime(existing_document.get("generated_at"))
|
||||
if existing_document
|
||||
else generated_at_value,
|
||||
source_range_start=_parse_optional_datetime(
|
||||
existing_document.get("source_range_start")
|
||||
)
|
||||
if existing_document
|
||||
else _as_utc(source_range_start),
|
||||
source_range_end=_parse_optional_datetime(existing_document.get("source_range_end"))
|
||||
if existing_document
|
||||
else _as_utc(source_range_end),
|
||||
is_stale=bool(existing_document.get("is_stale", False)) if existing_document else is_stale,
|
||||
)
|
||||
|
||||
snapshot_document = {
|
||||
"server_key": normalized_server_key,
|
||||
"snapshot_type": snapshot_type,
|
||||
"metric": metric,
|
||||
"window": window,
|
||||
"generated_at": _to_iso(generated_at_value),
|
||||
"source_range_start": _to_iso(source_range_start),
|
||||
"source_range_end": _to_iso(source_range_end),
|
||||
"is_stale": is_stale,
|
||||
"payload": payload,
|
||||
}
|
||||
snapshot_path.write_text(
|
||||
json.dumps(snapshot_document, ensure_ascii=True, indent=2) + "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
return HistoricalSnapshotRecord(
|
||||
server_key=normalized_server_key,
|
||||
snapshot_type=snapshot_type,
|
||||
metric=metric,
|
||||
window=window,
|
||||
payload_json=payload_json,
|
||||
generated_at=generated_at_value,
|
||||
source_range_start=_as_utc(source_range_start),
|
||||
source_range_end=_as_utc(source_range_end),
|
||||
is_stale=is_stale,
|
||||
)
|
||||
|
||||
|
||||
def persist_historical_snapshot_batch(
|
||||
snapshots: list[dict[str, object]],
|
||||
*,
|
||||
db_path: Path | None = None,
|
||||
) -> list[HistoricalSnapshotRecord]:
|
||||
"""Persist a batch of snapshots generated in one runner cycle."""
|
||||
records: list[HistoricalSnapshotRecord] = []
|
||||
for snapshot in snapshots:
|
||||
records.append(
|
||||
persist_historical_snapshot(
|
||||
server_key=str(snapshot["server_key"]),
|
||||
snapshot_type=str(snapshot["snapshot_type"]),
|
||||
payload=snapshot["payload"],
|
||||
metric=snapshot.get("metric"),
|
||||
window=snapshot.get("window"),
|
||||
generated_at=snapshot.get("generated_at"),
|
||||
source_range_start=snapshot.get("source_range_start"),
|
||||
source_range_end=snapshot.get("source_range_end"),
|
||||
is_stale=bool(snapshot.get("is_stale", False)),
|
||||
db_path=db_path,
|
||||
)
|
||||
)
|
||||
return records
|
||||
|
||||
|
||||
def get_historical_snapshot(
|
||||
*,
|
||||
server_key: str,
|
||||
snapshot_type: str,
|
||||
metric: str | None = None,
|
||||
window: str | None = None,
|
||||
db_path: Path | None = None,
|
||||
) -> dict[str, object] | None:
|
||||
"""Return one persisted snapshot and decoded payload, if present."""
|
||||
validate_snapshot_identity(snapshot_type=snapshot_type, metric=metric)
|
||||
if use_postgres_rcon_storage(explicit_sqlite_path=db_path):
|
||||
from .postgres_display_storage import get_snapshot
|
||||
|
||||
return get_snapshot(
|
||||
server_key=server_key,
|
||||
snapshot_type=snapshot_type,
|
||||
metric=metric,
|
||||
window=window,
|
||||
)
|
||||
snapshots_root = resolve_historical_snapshot_storage_path(db_path=db_path)
|
||||
snapshot_path = _build_snapshot_path(
|
||||
snapshots_root=snapshots_root,
|
||||
server_key=server_key,
|
||||
snapshot_type=snapshot_type,
|
||||
metric=metric,
|
||||
)
|
||||
if not snapshot_path.exists():
|
||||
return None
|
||||
|
||||
document = json.loads(snapshot_path.read_text(encoding="utf-8"))
|
||||
return {
|
||||
"server_key": document.get("server_key"),
|
||||
"snapshot_type": document.get("snapshot_type"),
|
||||
"metric": document.get("metric"),
|
||||
"window": document.get("window"),
|
||||
"generated_at": document.get("generated_at"),
|
||||
"source_range_start": document.get("source_range_start"),
|
||||
"source_range_end": document.get("source_range_end"),
|
||||
"is_stale": bool(document.get("is_stale", False)),
|
||||
"payload": document.get("payload"),
|
||||
}
|
||||
|
||||
|
||||
def list_historical_snapshots(
|
||||
*,
|
||||
server_key: str | None = None,
|
||||
snapshot_type: str | None = None,
|
||||
db_path: Path | None = None,
|
||||
) -> list[dict[str, object]]:
|
||||
"""List persisted snapshots for validation and operational inspection."""
|
||||
snapshots_root = resolve_historical_snapshot_storage_path(db_path=db_path)
|
||||
if not snapshots_root.exists():
|
||||
return []
|
||||
if snapshot_type:
|
||||
validate_snapshot_identity(snapshot_type=snapshot_type)
|
||||
|
||||
rows: list[dict[str, object]] = []
|
||||
for snapshot_path in snapshots_root.glob("*/*.json"):
|
||||
try:
|
||||
document = json.loads(snapshot_path.read_text(encoding="utf-8"))
|
||||
except (OSError, json.JSONDecodeError):
|
||||
continue
|
||||
|
||||
if server_key and document.get("server_key") != server_key:
|
||||
continue
|
||||
if snapshot_type and document.get("snapshot_type") != snapshot_type:
|
||||
continue
|
||||
|
||||
rows.append(
|
||||
{
|
||||
"server_key": document.get("server_key"),
|
||||
"snapshot_type": document.get("snapshot_type"),
|
||||
"metric": document.get("metric"),
|
||||
"window": document.get("window"),
|
||||
"generated_at": document.get("generated_at"),
|
||||
"source_range_start": document.get("source_range_start"),
|
||||
"source_range_end": document.get("source_range_end"),
|
||||
"is_stale": bool(document.get("is_stale", False)),
|
||||
}
|
||||
)
|
||||
|
||||
return sorted(
|
||||
rows,
|
||||
key=lambda item: (
|
||||
str(item.get("server_key") or ""),
|
||||
str(item.get("snapshot_type") or ""),
|
||||
str(item.get("metric") or ""),
|
||||
str(item.get("generated_at") or ""),
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _should_preserve_existing_snapshot(
|
||||
*,
|
||||
incoming_payload: dict[str, object] | list[object],
|
||||
snapshot_type: str,
|
||||
existing_document: dict[str, object] | None,
|
||||
) -> bool:
|
||||
if not _is_effectively_empty_snapshot_payload(snapshot_type, incoming_payload):
|
||||
return False
|
||||
if existing_document and not _is_effectively_empty_snapshot_payload(
|
||||
snapshot_type,
|
||||
existing_document.get("payload"),
|
||||
):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def _is_effectively_empty_snapshot_payload(
|
||||
snapshot_type: str,
|
||||
payload: object,
|
||||
) -> bool:
|
||||
if not isinstance(payload, dict):
|
||||
return not payload
|
||||
|
||||
if snapshot_type == "server-summary":
|
||||
item = payload.get("item")
|
||||
if not isinstance(item, dict):
|
||||
return True
|
||||
matches_count = item.get("imported_matches_count", item.get("matches_count", 0))
|
||||
return int(matches_count or 0) <= 0
|
||||
|
||||
if snapshot_type == "recent-matches":
|
||||
items = payload.get("items")
|
||||
return not isinstance(items, list) or len(items) == 0
|
||||
|
||||
if snapshot_type in {
|
||||
"weekly-leaderboard",
|
||||
"monthly-leaderboard",
|
||||
"monthly-mvp",
|
||||
"monthly-mvp-v2",
|
||||
}:
|
||||
items = payload.get("items")
|
||||
return not isinstance(items, list) or len(items) == 0
|
||||
|
||||
return False
|
||||
def _read_snapshot_document(snapshot_path: Path) -> dict[str, object] | None:
|
||||
if not snapshot_path.exists():
|
||||
return None
|
||||
try:
|
||||
return json.loads(snapshot_path.read_text(encoding="utf-8"))
|
||||
except (OSError, json.JSONDecodeError):
|
||||
return None
|
||||
|
||||
|
||||
def _build_snapshot_path(
|
||||
*,
|
||||
snapshots_root: Path,
|
||||
server_key: str,
|
||||
snapshot_type: str,
|
||||
metric: str | None,
|
||||
) -> Path:
|
||||
return snapshots_root / server_key / _build_snapshot_filename(
|
||||
snapshot_type=snapshot_type,
|
||||
metric=metric,
|
||||
)
|
||||
|
||||
|
||||
def _build_snapshot_filename(*, snapshot_type: str, metric: str | None) -> str:
|
||||
if snapshot_type == "server-summary":
|
||||
return "server-summary.json"
|
||||
if snapshot_type == "recent-matches":
|
||||
return "recent-matches.json"
|
||||
if snapshot_type == "monthly-mvp-v2":
|
||||
return "monthly-mvp-v2.json"
|
||||
if snapshot_type == "player-event-most-killed":
|
||||
return "player-events-most-killed.json"
|
||||
if snapshot_type == "player-event-death-by":
|
||||
return "player-events-death-by.json"
|
||||
if snapshot_type == "player-event-duels":
|
||||
return "player-events-duels.json"
|
||||
if snapshot_type == "player-event-weapon-kills":
|
||||
return "player-events-weapon-kills.json"
|
||||
if snapshot_type == "player-event-teamkills":
|
||||
return "player-events-teamkills.json"
|
||||
if snapshot_type == "weekly-leaderboard":
|
||||
metric_suffix = "matches-over-100-kills" if metric == "matches_over_100_kills" else _slugify(metric or "unknown")
|
||||
return f"weekly-{metric_suffix}.json"
|
||||
if snapshot_type == "monthly-leaderboard":
|
||||
metric_suffix = "matches-over-100-kills" if metric == "matches_over_100_kills" else _slugify(metric or "unknown")
|
||||
return f"monthly-{metric_suffix}.json"
|
||||
if snapshot_type == "monthly-mvp":
|
||||
return "monthly-mvp.json"
|
||||
metric_suffix = _slugify(metric or "")
|
||||
base_name = _slugify(snapshot_type)
|
||||
return f"{base_name}-{metric_suffix}.json" if metric_suffix else f"{base_name}.json"
|
||||
|
||||
|
||||
def _slugify(value: str) -> str:
|
||||
return value.strip().replace("_", "-").replace(" ", "-").lower()
|
||||
|
||||
|
||||
def _to_iso(value: datetime | None) -> str | None:
|
||||
if value is None:
|
||||
return None
|
||||
return _as_utc(value).isoformat().replace("+00:00", "Z")
|
||||
|
||||
|
||||
def _as_utc(value: datetime | None) -> datetime | None:
|
||||
if value is None:
|
||||
return None
|
||||
if value.tzinfo is None:
|
||||
return value.replace(tzinfo=timezone.utc)
|
||||
return value.astimezone(timezone.utc)
|
||||
|
||||
|
||||
def _parse_optional_datetime(value: object) -> datetime | None:
|
||||
if not isinstance(value, str) or not value.strip():
|
||||
return None
|
||||
normalized = value.strip().replace("Z", "+00:00")
|
||||
parsed = datetime.fromisoformat(normalized)
|
||||
if parsed.tzinfo is None:
|
||||
return parsed.replace(tzinfo=timezone.utc)
|
||||
return parsed.astimezone(timezone.utc)
|
||||
842
backend/app/historical_snapshots.py
Normal file
842
backend/app/historical_snapshots.py
Normal file
@@ -0,0 +1,842 @@
|
||||
"""Definitions for persisted precomputed historical snapshots."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sqlite3
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
from .config import get_database_url, get_historical_data_source_kind
|
||||
from .data_sources import SOURCE_KIND_RCON, get_rcon_historical_read_model
|
||||
from .historical_storage import (
|
||||
ALL_SERVERS_SLUG,
|
||||
list_historical_server_summaries,
|
||||
list_historical_servers,
|
||||
list_monthly_leaderboard,
|
||||
list_monthly_mvp_ranking,
|
||||
list_monthly_mvp_v2_ranking,
|
||||
list_recent_historical_matches,
|
||||
list_weekly_leaderboard,
|
||||
)
|
||||
from .player_event_aggregates import (
|
||||
list_death_by,
|
||||
list_most_killed,
|
||||
list_net_duel_summaries,
|
||||
list_teamkill_summaries,
|
||||
list_weapon_kills,
|
||||
)
|
||||
from .player_event_storage import initialize_player_event_storage
|
||||
|
||||
SNAPSHOT_TYPE_SERVER_SUMMARY = "server-summary"
|
||||
SNAPSHOT_TYPE_WEEKLY_LEADERBOARD = "weekly-leaderboard"
|
||||
SNAPSHOT_TYPE_MONTHLY_LEADERBOARD = "monthly-leaderboard"
|
||||
SNAPSHOT_TYPE_MONTHLY_MVP = "monthly-mvp"
|
||||
SNAPSHOT_TYPE_MONTHLY_MVP_V2 = "monthly-mvp-v2"
|
||||
SNAPSHOT_TYPE_RECENT_MATCHES = "recent-matches"
|
||||
SNAPSHOT_TYPE_PLAYER_EVENT_MOST_KILLED = "player-event-most-killed"
|
||||
SNAPSHOT_TYPE_PLAYER_EVENT_DEATH_BY = "player-event-death-by"
|
||||
SNAPSHOT_TYPE_PLAYER_EVENT_DUELS = "player-event-duels"
|
||||
SNAPSHOT_TYPE_PLAYER_EVENT_WEAPON_KILLS = "player-event-weapon-kills"
|
||||
SNAPSHOT_TYPE_PLAYER_EVENT_TEAMKILLS = "player-event-teamkills"
|
||||
|
||||
SUPPORTED_SNAPSHOT_TYPES = frozenset(
|
||||
{
|
||||
SNAPSHOT_TYPE_SERVER_SUMMARY,
|
||||
SNAPSHOT_TYPE_WEEKLY_LEADERBOARD,
|
||||
SNAPSHOT_TYPE_MONTHLY_LEADERBOARD,
|
||||
SNAPSHOT_TYPE_MONTHLY_MVP,
|
||||
SNAPSHOT_TYPE_MONTHLY_MVP_V2,
|
||||
SNAPSHOT_TYPE_RECENT_MATCHES,
|
||||
SNAPSHOT_TYPE_PLAYER_EVENT_MOST_KILLED,
|
||||
SNAPSHOT_TYPE_PLAYER_EVENT_DEATH_BY,
|
||||
SNAPSHOT_TYPE_PLAYER_EVENT_DUELS,
|
||||
SNAPSHOT_TYPE_PLAYER_EVENT_WEAPON_KILLS,
|
||||
SNAPSHOT_TYPE_PLAYER_EVENT_TEAMKILLS,
|
||||
}
|
||||
)
|
||||
|
||||
SUPPORTED_LEADERBOARD_METRICS = frozenset(
|
||||
{
|
||||
"kills",
|
||||
"deaths",
|
||||
"support",
|
||||
"matches_over_100_kills",
|
||||
}
|
||||
)
|
||||
PREWARM_SNAPSHOT_SERVER_KEYS = (
|
||||
"comunidad-hispana-01",
|
||||
"comunidad-hispana-02",
|
||||
ALL_SERVERS_SLUG,
|
||||
)
|
||||
PREWARM_LEADERBOARD_METRICS = ("kills",)
|
||||
SNAPSHOT_LEADERBOARD_METRICS = (
|
||||
"kills",
|
||||
"deaths",
|
||||
"matches_over_100_kills",
|
||||
"support",
|
||||
)
|
||||
PLAYER_EVENT_SNAPSHOT_TYPES = (
|
||||
SNAPSHOT_TYPE_PLAYER_EVENT_MOST_KILLED,
|
||||
SNAPSHOT_TYPE_PLAYER_EVENT_DEATH_BY,
|
||||
SNAPSHOT_TYPE_PLAYER_EVENT_DUELS,
|
||||
SNAPSHOT_TYPE_PLAYER_EVENT_WEAPON_KILLS,
|
||||
SNAPSHOT_TYPE_PLAYER_EVENT_TEAMKILLS,
|
||||
)
|
||||
|
||||
DEFAULT_SNAPSHOT_WINDOW = "all-time"
|
||||
DEFAULT_WEEKLY_SNAPSHOT_WINDOW = "7d"
|
||||
DEFAULT_MONTHLY_SNAPSHOT_WINDOW = "month"
|
||||
DEFAULT_WEEKLY_LEADERBOARD_LIMIT = 10
|
||||
DEFAULT_RECENT_MATCHES_LIMIT = 20
|
||||
|
||||
|
||||
def validate_snapshot_identity(
|
||||
*,
|
||||
snapshot_type: str,
|
||||
metric: str | None = None,
|
||||
) -> None:
|
||||
"""Validate the persisted snapshot selectors accepted by the storage layer."""
|
||||
if snapshot_type not in SUPPORTED_SNAPSHOT_TYPES:
|
||||
raise ValueError(f"Unsupported historical snapshot type: {snapshot_type}")
|
||||
|
||||
if snapshot_type in {
|
||||
SNAPSHOT_TYPE_WEEKLY_LEADERBOARD,
|
||||
SNAPSHOT_TYPE_MONTHLY_LEADERBOARD,
|
||||
}:
|
||||
if metric not in SUPPORTED_LEADERBOARD_METRICS:
|
||||
raise ValueError(f"Unsupported historical snapshot metric: {metric}")
|
||||
return
|
||||
|
||||
if metric is not None:
|
||||
raise ValueError(
|
||||
"Metric is only supported for weekly-leaderboard and monthly-leaderboard."
|
||||
)
|
||||
|
||||
|
||||
def list_snapshot_server_keys(*, db_path: Path | None = None) -> list[str]:
|
||||
"""Return the historical server slugs that should receive persisted snapshots."""
|
||||
server_keys = [
|
||||
str(item["slug"])
|
||||
for item in list_historical_servers(db_path=db_path)
|
||||
if item.get("slug")
|
||||
]
|
||||
server_keys.append(ALL_SERVERS_SLUG)
|
||||
return server_keys
|
||||
|
||||
|
||||
def build_historical_server_snapshots(
|
||||
*,
|
||||
server_key: str,
|
||||
generated_at: datetime | None = None,
|
||||
leaderboard_limit: int = DEFAULT_WEEKLY_LEADERBOARD_LIMIT,
|
||||
recent_matches_limit: int = DEFAULT_RECENT_MATCHES_LIMIT,
|
||||
db_path: Path | None = None,
|
||||
) -> list[dict[str, object]]:
|
||||
"""Build all precomputed historical snapshots required for one server."""
|
||||
generated_at_value = _as_utc(generated_at or datetime.now(timezone.utc))
|
||||
leaderboard_limit = _normalize_snapshot_limit("leaderboard_limit", leaderboard_limit)
|
||||
recent_matches_limit = _normalize_snapshot_limit(
|
||||
"recent_matches_limit",
|
||||
recent_matches_limit,
|
||||
)
|
||||
_log_snapshot_build_started(server_key, SNAPSHOT_TYPE_SERVER_SUMMARY)
|
||||
snapshots = [_build_server_summary_snapshot(server_key, generated_at_value, db_path=db_path)]
|
||||
|
||||
for metric in SNAPSHOT_LEADERBOARD_METRICS:
|
||||
_log_snapshot_build_started(server_key, SNAPSHOT_TYPE_WEEKLY_LEADERBOARD, metric=metric)
|
||||
snapshots.append(
|
||||
_build_weekly_leaderboard_snapshot(
|
||||
server_key,
|
||||
metric,
|
||||
generated_at_value,
|
||||
limit=leaderboard_limit,
|
||||
db_path=db_path,
|
||||
)
|
||||
)
|
||||
_log_snapshot_build_started(server_key, SNAPSHOT_TYPE_MONTHLY_LEADERBOARD, metric=metric)
|
||||
snapshots.append(
|
||||
_build_monthly_leaderboard_snapshot(
|
||||
server_key,
|
||||
metric,
|
||||
generated_at_value,
|
||||
limit=leaderboard_limit,
|
||||
db_path=db_path,
|
||||
)
|
||||
)
|
||||
|
||||
_log_snapshot_build_started(server_key, SNAPSHOT_TYPE_MONTHLY_MVP)
|
||||
snapshots.append(
|
||||
_build_monthly_mvp_snapshot(
|
||||
server_key,
|
||||
generated_at_value,
|
||||
limit=leaderboard_limit,
|
||||
db_path=db_path,
|
||||
)
|
||||
)
|
||||
_log_snapshot_build_started(server_key, SNAPSHOT_TYPE_MONTHLY_MVP_V2)
|
||||
snapshots.append(
|
||||
_build_monthly_mvp_v2_snapshot(
|
||||
server_key,
|
||||
generated_at_value,
|
||||
limit=leaderboard_limit,
|
||||
db_path=db_path,
|
||||
)
|
||||
)
|
||||
for snapshot_type in PLAYER_EVENT_SNAPSHOT_TYPES:
|
||||
_log_snapshot_build_started(server_key, snapshot_type)
|
||||
snapshots.append(
|
||||
_build_player_event_snapshot(
|
||||
server_key,
|
||||
snapshot_type,
|
||||
generated_at_value,
|
||||
limit=leaderboard_limit,
|
||||
db_path=db_path,
|
||||
)
|
||||
)
|
||||
|
||||
_log_snapshot_build_started(server_key, SNAPSHOT_TYPE_RECENT_MATCHES)
|
||||
snapshots.append(
|
||||
_build_recent_matches_snapshot(
|
||||
server_key,
|
||||
generated_at_value,
|
||||
limit=recent_matches_limit,
|
||||
db_path=db_path,
|
||||
)
|
||||
)
|
||||
return snapshots
|
||||
|
||||
|
||||
def build_priority_historical_snapshots(
|
||||
*,
|
||||
server_keys: tuple[str, ...] = PREWARM_SNAPSHOT_SERVER_KEYS,
|
||||
generated_at: datetime | None = None,
|
||||
leaderboard_limit: int = DEFAULT_WEEKLY_LEADERBOARD_LIMIT,
|
||||
recent_matches_limit: int = DEFAULT_RECENT_MATCHES_LIMIT,
|
||||
db_path: Path | None = None,
|
||||
) -> list[dict[str, object]]:
|
||||
"""Build the minimum warm snapshot set required by the historical UI."""
|
||||
generated_at_value = _as_utc(generated_at or datetime.now(timezone.utc))
|
||||
leaderboard_limit = _normalize_snapshot_limit("leaderboard_limit", leaderboard_limit)
|
||||
recent_matches_limit = _normalize_snapshot_limit(
|
||||
"recent_matches_limit",
|
||||
recent_matches_limit,
|
||||
)
|
||||
snapshots: list[dict[str, object]] = []
|
||||
for server_key in server_keys:
|
||||
_log_snapshot_build_started(server_key, SNAPSHOT_TYPE_SERVER_SUMMARY)
|
||||
snapshots.append(
|
||||
_build_server_summary_snapshot(server_key, generated_at_value, db_path=db_path)
|
||||
)
|
||||
for metric in PREWARM_LEADERBOARD_METRICS:
|
||||
_log_snapshot_build_started(
|
||||
server_key,
|
||||
SNAPSHOT_TYPE_WEEKLY_LEADERBOARD,
|
||||
metric=metric,
|
||||
)
|
||||
snapshots.append(
|
||||
_build_weekly_leaderboard_snapshot(
|
||||
server_key,
|
||||
metric,
|
||||
generated_at_value,
|
||||
limit=leaderboard_limit,
|
||||
db_path=db_path,
|
||||
)
|
||||
)
|
||||
_log_snapshot_build_started(
|
||||
server_key,
|
||||
SNAPSHOT_TYPE_MONTHLY_LEADERBOARD,
|
||||
metric=metric,
|
||||
)
|
||||
snapshots.append(
|
||||
_build_monthly_leaderboard_snapshot(
|
||||
server_key,
|
||||
metric,
|
||||
generated_at_value,
|
||||
limit=leaderboard_limit,
|
||||
db_path=db_path,
|
||||
)
|
||||
)
|
||||
_log_snapshot_build_started(server_key, SNAPSHOT_TYPE_MONTHLY_MVP)
|
||||
snapshots.append(
|
||||
_build_monthly_mvp_snapshot(
|
||||
server_key,
|
||||
generated_at_value,
|
||||
limit=leaderboard_limit,
|
||||
db_path=db_path,
|
||||
)
|
||||
)
|
||||
_log_snapshot_build_started(server_key, SNAPSHOT_TYPE_MONTHLY_MVP_V2)
|
||||
snapshots.append(
|
||||
_build_monthly_mvp_v2_snapshot(
|
||||
server_key,
|
||||
generated_at_value,
|
||||
limit=leaderboard_limit,
|
||||
db_path=db_path,
|
||||
)
|
||||
)
|
||||
for snapshot_type in PLAYER_EVENT_SNAPSHOT_TYPES:
|
||||
_log_snapshot_build_started(server_key, snapshot_type)
|
||||
snapshots.append(
|
||||
_build_player_event_snapshot(
|
||||
server_key,
|
||||
snapshot_type,
|
||||
generated_at_value,
|
||||
limit=leaderboard_limit,
|
||||
db_path=db_path,
|
||||
)
|
||||
)
|
||||
_log_snapshot_build_started(server_key, SNAPSHOT_TYPE_RECENT_MATCHES)
|
||||
snapshots.append(
|
||||
_build_recent_matches_snapshot(
|
||||
server_key,
|
||||
generated_at_value,
|
||||
limit=recent_matches_limit,
|
||||
db_path=db_path,
|
||||
)
|
||||
)
|
||||
return snapshots
|
||||
|
||||
|
||||
def build_all_historical_snapshots(
|
||||
*,
|
||||
server_key: str | None = None,
|
||||
generated_at: datetime | None = None,
|
||||
leaderboard_limit: int = DEFAULT_WEEKLY_LEADERBOARD_LIMIT,
|
||||
recent_matches_limit: int = DEFAULT_RECENT_MATCHES_LIMIT,
|
||||
db_path: Path | None = None,
|
||||
) -> list[dict[str, object]]:
|
||||
"""Build the full snapshot set for one server or for all configured servers."""
|
||||
target_server_keys = _resolve_snapshot_target_keys(server_key=server_key, db_path=db_path)
|
||||
snapshots: list[dict[str, object]] = []
|
||||
for target_server_key in target_server_keys:
|
||||
snapshots.extend(
|
||||
build_historical_server_snapshots(
|
||||
server_key=target_server_key,
|
||||
generated_at=generated_at,
|
||||
leaderboard_limit=leaderboard_limit,
|
||||
recent_matches_limit=recent_matches_limit,
|
||||
db_path=db_path,
|
||||
)
|
||||
)
|
||||
return snapshots
|
||||
|
||||
|
||||
def generate_and_persist_historical_snapshots(
|
||||
*,
|
||||
server_key: str | None = None,
|
||||
generated_at: datetime | None = None,
|
||||
leaderboard_limit: int = DEFAULT_WEEKLY_LEADERBOARD_LIMIT,
|
||||
recent_matches_limit: int = DEFAULT_RECENT_MATCHES_LIMIT,
|
||||
db_path: Path | None = None,
|
||||
) -> dict[str, object]:
|
||||
"""Build and persist precomputed snapshots for one server or all servers."""
|
||||
from .historical_snapshot_storage import persist_historical_snapshot_batch
|
||||
|
||||
generated_at_value = _as_utc(generated_at or datetime.now(timezone.utc))
|
||||
snapshots = build_all_historical_snapshots(
|
||||
server_key=server_key,
|
||||
generated_at=generated_at_value,
|
||||
leaderboard_limit=leaderboard_limit,
|
||||
recent_matches_limit=recent_matches_limit,
|
||||
db_path=db_path,
|
||||
)
|
||||
persisted_records = persist_historical_snapshot_batch(snapshots, db_path=db_path)
|
||||
snapshots_by_server: dict[str, int] = {}
|
||||
for record in persisted_records:
|
||||
snapshots_by_server.setdefault(record.server_key, 0)
|
||||
snapshots_by_server[record.server_key] += 1
|
||||
|
||||
return {
|
||||
"generated_at": _to_iso(generated_at_value),
|
||||
"server_slug": server_key,
|
||||
"snapshot_policy": "full-matrix",
|
||||
"snapshot_count": len(persisted_records),
|
||||
"servers_processed": len(snapshots_by_server),
|
||||
"snapshots_by_server": snapshots_by_server,
|
||||
}
|
||||
|
||||
|
||||
def generate_and_persist_priority_historical_snapshots(
|
||||
*,
|
||||
generated_at: datetime | None = None,
|
||||
leaderboard_limit: int = DEFAULT_WEEKLY_LEADERBOARD_LIMIT,
|
||||
recent_matches_limit: int = DEFAULT_RECENT_MATCHES_LIMIT,
|
||||
db_path: Path | None = None,
|
||||
) -> dict[str, object]:
|
||||
"""Build and persist the priority snapshot set used for prewarm."""
|
||||
from .historical_snapshot_storage import persist_historical_snapshot_batch
|
||||
|
||||
generated_at_value = _as_utc(generated_at or datetime.now(timezone.utc))
|
||||
snapshots = build_priority_historical_snapshots(
|
||||
generated_at=generated_at_value,
|
||||
leaderboard_limit=leaderboard_limit,
|
||||
recent_matches_limit=recent_matches_limit,
|
||||
db_path=db_path,
|
||||
)
|
||||
persisted_records = persist_historical_snapshot_batch(snapshots, db_path=db_path)
|
||||
snapshots_by_server: dict[str, int] = {}
|
||||
for record in persisted_records:
|
||||
snapshots_by_server.setdefault(record.server_key, 0)
|
||||
snapshots_by_server[record.server_key] += 1
|
||||
|
||||
return {
|
||||
"generated_at": _to_iso(generated_at_value),
|
||||
"server_slug": None,
|
||||
"snapshot_policy": "priority-prewarm",
|
||||
"prewarm_server_keys": list(PREWARM_SNAPSHOT_SERVER_KEYS),
|
||||
"prewarm_metrics": list(PREWARM_LEADERBOARD_METRICS),
|
||||
"snapshot_count": len(persisted_records),
|
||||
"servers_processed": len(snapshots_by_server),
|
||||
"snapshots_by_server": snapshots_by_server,
|
||||
}
|
||||
|
||||
|
||||
def _build_server_summary_snapshot(
|
||||
server_key: str,
|
||||
generated_at: datetime,
|
||||
*,
|
||||
db_path: Path | None = None,
|
||||
) -> dict[str, object]:
|
||||
if get_historical_data_source_kind() == SOURCE_KIND_RCON:
|
||||
data_source = get_rcon_historical_read_model()
|
||||
summary_items = (
|
||||
data_source.list_server_summaries(server_key=server_key)
|
||||
if data_source is not None
|
||||
else []
|
||||
)
|
||||
else:
|
||||
summary_items = list_historical_server_summaries(server_slug=server_key, db_path=db_path)
|
||||
summary_item = summary_items[0] if summary_items else {}
|
||||
time_range = summary_item.get("time_range") if isinstance(summary_item, dict) else {}
|
||||
return {
|
||||
"server_key": server_key,
|
||||
"snapshot_type": SNAPSHOT_TYPE_SERVER_SUMMARY,
|
||||
"metric": None,
|
||||
"window": DEFAULT_SNAPSHOT_WINDOW,
|
||||
"generated_at": generated_at,
|
||||
"source_range_start": _parse_optional_timestamp(time_range.get("start")),
|
||||
"source_range_end": _parse_optional_timestamp(time_range.get("end")),
|
||||
"is_stale": False,
|
||||
"payload": {
|
||||
"server_key": server_key,
|
||||
"generated_at": _to_iso(generated_at),
|
||||
"item": summary_item,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _build_weekly_leaderboard_snapshot(
|
||||
server_key: str,
|
||||
metric: str,
|
||||
generated_at: datetime,
|
||||
*,
|
||||
limit: int,
|
||||
db_path: Path | None = None,
|
||||
) -> dict[str, object]:
|
||||
if get_historical_data_source_kind() == SOURCE_KIND_RCON:
|
||||
from .rcon_historical_leaderboards import list_rcon_materialized_leaderboard
|
||||
|
||||
leaderboard_result = list_rcon_materialized_leaderboard(
|
||||
limit=limit,
|
||||
server_key=server_key,
|
||||
metric=metric,
|
||||
timeframe="weekly",
|
||||
db_path=db_path,
|
||||
now=generated_at,
|
||||
)
|
||||
else:
|
||||
leaderboard_result = list_weekly_leaderboard(
|
||||
limit=limit,
|
||||
server_id=server_key,
|
||||
metric=metric,
|
||||
db_path=db_path,
|
||||
)
|
||||
return {
|
||||
"server_key": server_key,
|
||||
"snapshot_type": SNAPSHOT_TYPE_WEEKLY_LEADERBOARD,
|
||||
"metric": metric,
|
||||
"window": DEFAULT_WEEKLY_SNAPSHOT_WINDOW,
|
||||
"generated_at": generated_at,
|
||||
"source_range_start": _parse_optional_timestamp(leaderboard_result.get("window_start")),
|
||||
"source_range_end": _parse_optional_timestamp(leaderboard_result.get("window_end")),
|
||||
"is_stale": False,
|
||||
"payload": {
|
||||
"server_key": server_key,
|
||||
"metric": metric,
|
||||
"limit": limit,
|
||||
"generated_at": _to_iso(generated_at),
|
||||
**leaderboard_result,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _build_monthly_leaderboard_snapshot(
|
||||
server_key: str,
|
||||
metric: str,
|
||||
generated_at: datetime,
|
||||
*,
|
||||
limit: int,
|
||||
db_path: Path | None = None,
|
||||
) -> dict[str, object]:
|
||||
if get_historical_data_source_kind() == SOURCE_KIND_RCON:
|
||||
from .rcon_historical_leaderboards import list_rcon_materialized_leaderboard
|
||||
|
||||
leaderboard_result = list_rcon_materialized_leaderboard(
|
||||
limit=limit,
|
||||
server_key=server_key,
|
||||
metric=metric,
|
||||
timeframe="monthly",
|
||||
db_path=db_path,
|
||||
now=generated_at,
|
||||
)
|
||||
else:
|
||||
leaderboard_result = list_monthly_leaderboard(
|
||||
limit=limit,
|
||||
server_id=server_key,
|
||||
metric=metric,
|
||||
db_path=db_path,
|
||||
)
|
||||
return {
|
||||
"server_key": server_key,
|
||||
"snapshot_type": SNAPSHOT_TYPE_MONTHLY_LEADERBOARD,
|
||||
"metric": metric,
|
||||
"window": DEFAULT_MONTHLY_SNAPSHOT_WINDOW,
|
||||
"generated_at": generated_at,
|
||||
"source_range_start": _parse_optional_timestamp(leaderboard_result.get("window_start")),
|
||||
"source_range_end": _parse_optional_timestamp(leaderboard_result.get("window_end")),
|
||||
"is_stale": False,
|
||||
"payload": {
|
||||
"server_key": server_key,
|
||||
"metric": metric,
|
||||
"limit": limit,
|
||||
"generated_at": _to_iso(generated_at),
|
||||
**leaderboard_result,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _build_recent_matches_snapshot(
|
||||
server_key: str,
|
||||
generated_at: datetime,
|
||||
*,
|
||||
limit: int,
|
||||
db_path: Path | None = None,
|
||||
) -> dict[str, object]:
|
||||
if get_historical_data_source_kind() == SOURCE_KIND_RCON:
|
||||
data_source = get_rcon_historical_read_model()
|
||||
items = (
|
||||
data_source.list_recent_activity(server_key=server_key, limit=limit)
|
||||
if data_source is not None
|
||||
else []
|
||||
)
|
||||
else:
|
||||
items = list_recent_historical_matches(
|
||||
limit=limit,
|
||||
server_slug=server_key,
|
||||
db_path=db_path,
|
||||
)
|
||||
closed_points = [
|
||||
_parse_optional_timestamp(item.get("closed_at"))
|
||||
for item in items
|
||||
if isinstance(item, dict) and item.get("closed_at")
|
||||
]
|
||||
return {
|
||||
"server_key": server_key,
|
||||
"snapshot_type": SNAPSHOT_TYPE_RECENT_MATCHES,
|
||||
"metric": None,
|
||||
"window": DEFAULT_SNAPSHOT_WINDOW,
|
||||
"generated_at": generated_at,
|
||||
"source_range_start": min(closed_points) if closed_points else None,
|
||||
"source_range_end": max(closed_points) if closed_points else None,
|
||||
"is_stale": False,
|
||||
"payload": {
|
||||
"server_key": server_key,
|
||||
"limit": limit,
|
||||
"generated_at": _to_iso(generated_at),
|
||||
"items": items,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _build_player_event_snapshot(
|
||||
server_key: str,
|
||||
snapshot_type: str,
|
||||
generated_at: datetime,
|
||||
*,
|
||||
limit: int,
|
||||
db_path: Path | None = None,
|
||||
) -> dict[str, object]:
|
||||
month_key = _get_latest_player_event_month_key(server_key=server_key, db_path=db_path)
|
||||
source_range_start = None
|
||||
source_range_end = None
|
||||
items: list[dict[str, object]] = []
|
||||
found = False
|
||||
|
||||
if month_key:
|
||||
source_range_start, source_range_end = _get_player_event_source_range(
|
||||
server_key=server_key,
|
||||
month_key=month_key,
|
||||
db_path=db_path,
|
||||
)
|
||||
items = _list_player_event_snapshot_items(
|
||||
snapshot_type=snapshot_type,
|
||||
server_key=server_key,
|
||||
month_key=month_key,
|
||||
limit=limit,
|
||||
db_path=db_path,
|
||||
)
|
||||
found = bool(items or source_range_start or source_range_end)
|
||||
|
||||
return {
|
||||
"server_key": server_key,
|
||||
"snapshot_type": snapshot_type,
|
||||
"metric": None,
|
||||
"window": DEFAULT_MONTHLY_SNAPSHOT_WINDOW,
|
||||
"generated_at": generated_at,
|
||||
"source_range_start": source_range_start,
|
||||
"source_range_end": source_range_end,
|
||||
"is_stale": False,
|
||||
"payload": {
|
||||
"server_key": server_key,
|
||||
"period": "monthly",
|
||||
"month_key": month_key,
|
||||
"limit": limit,
|
||||
"found": found,
|
||||
"generated_at": _to_iso(generated_at),
|
||||
"source_range_start": _to_iso(source_range_start) if source_range_start else None,
|
||||
"source_range_end": _to_iso(source_range_end) if source_range_end else None,
|
||||
"items": items,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _build_monthly_mvp_snapshot(
|
||||
server_key: str,
|
||||
generated_at: datetime,
|
||||
*,
|
||||
limit: int,
|
||||
db_path: Path | None = None,
|
||||
) -> dict[str, object]:
|
||||
ranking_result = list_monthly_mvp_ranking(
|
||||
limit=limit,
|
||||
server_id=server_key,
|
||||
db_path=db_path,
|
||||
)
|
||||
month_key = str(ranking_result.get("window_start") or "")[:7] or None
|
||||
return {
|
||||
"server_key": server_key,
|
||||
"snapshot_type": SNAPSHOT_TYPE_MONTHLY_MVP,
|
||||
"metric": None,
|
||||
"window": DEFAULT_MONTHLY_SNAPSHOT_WINDOW,
|
||||
"generated_at": generated_at,
|
||||
"source_range_start": _parse_optional_timestamp(ranking_result.get("window_start")),
|
||||
"source_range_end": _parse_optional_timestamp(ranking_result.get("window_end")),
|
||||
"is_stale": False,
|
||||
"payload": {
|
||||
"server_key": server_key,
|
||||
"limit": limit,
|
||||
"month_key": month_key,
|
||||
"generated_at": _to_iso(generated_at),
|
||||
**ranking_result,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _build_monthly_mvp_v2_snapshot(
|
||||
server_key: str,
|
||||
generated_at: datetime,
|
||||
*,
|
||||
limit: int,
|
||||
db_path: Path | None = None,
|
||||
) -> dict[str, object]:
|
||||
ranking_result = list_monthly_mvp_v2_ranking(
|
||||
limit=limit,
|
||||
server_id=server_key,
|
||||
db_path=db_path,
|
||||
)
|
||||
month_key = str(ranking_result.get("window_start") or "")[:7] or None
|
||||
event_coverage = ranking_result.get("event_coverage")
|
||||
source_range_start = None
|
||||
source_range_end = None
|
||||
if isinstance(event_coverage, dict):
|
||||
source_range_start = _parse_optional_timestamp(event_coverage.get("source_range_start"))
|
||||
source_range_end = _parse_optional_timestamp(event_coverage.get("source_range_end"))
|
||||
return {
|
||||
"server_key": server_key,
|
||||
"snapshot_type": SNAPSHOT_TYPE_MONTHLY_MVP_V2,
|
||||
"metric": None,
|
||||
"window": DEFAULT_MONTHLY_SNAPSHOT_WINDOW,
|
||||
"generated_at": generated_at,
|
||||
"source_range_start": source_range_start,
|
||||
"source_range_end": source_range_end,
|
||||
"is_stale": False,
|
||||
"payload": {
|
||||
"server_key": server_key,
|
||||
"limit": limit,
|
||||
"month_key": month_key,
|
||||
"found": bool(event_coverage.get("ready")) if isinstance(event_coverage, dict) else False,
|
||||
"generated_at": _to_iso(generated_at),
|
||||
**ranking_result,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _resolve_snapshot_target_keys(
|
||||
*,
|
||||
server_key: str | None,
|
||||
db_path: Path | None = None,
|
||||
) -> list[str]:
|
||||
"""Expand targeted rebuilds so the logical global aggregate stays in sync."""
|
||||
if not server_key:
|
||||
return list_snapshot_server_keys(db_path=db_path)
|
||||
|
||||
normalized_server_key = server_key.strip()
|
||||
if not normalized_server_key:
|
||||
return list_snapshot_server_keys(db_path=db_path)
|
||||
if normalized_server_key == ALL_SERVERS_SLUG:
|
||||
return [ALL_SERVERS_SLUG]
|
||||
|
||||
return [normalized_server_key, ALL_SERVERS_SLUG]
|
||||
|
||||
|
||||
def _list_player_event_snapshot_items(
|
||||
*,
|
||||
snapshot_type: str,
|
||||
server_key: str,
|
||||
month_key: str,
|
||||
limit: int,
|
||||
db_path: Path | None,
|
||||
) -> list[dict[str, object]]:
|
||||
aggregator_by_snapshot_type = {
|
||||
SNAPSHOT_TYPE_PLAYER_EVENT_MOST_KILLED: list_most_killed,
|
||||
SNAPSHOT_TYPE_PLAYER_EVENT_DEATH_BY: list_death_by,
|
||||
SNAPSHOT_TYPE_PLAYER_EVENT_DUELS: list_net_duel_summaries,
|
||||
SNAPSHOT_TYPE_PLAYER_EVENT_WEAPON_KILLS: list_weapon_kills,
|
||||
SNAPSHOT_TYPE_PLAYER_EVENT_TEAMKILLS: list_teamkill_summaries,
|
||||
}
|
||||
aggregator = aggregator_by_snapshot_type[snapshot_type]
|
||||
return aggregator(
|
||||
server_slug=server_key,
|
||||
month=month_key,
|
||||
limit=limit,
|
||||
db_path=db_path,
|
||||
)
|
||||
|
||||
|
||||
def _get_latest_player_event_month_key(
|
||||
*,
|
||||
server_key: str,
|
||||
db_path: Path | None = None,
|
||||
) -> str | None:
|
||||
resolved_path = initialize_player_event_storage(db_path=db_path)
|
||||
where_sql, params = _build_player_event_scope_where(server_key=server_key)
|
||||
with _connect(resolved_path) as connection:
|
||||
row = connection.execute(
|
||||
f"""
|
||||
SELECT MAX(substr(CAST(occurred_at AS TEXT), 1, 7)) AS latest_month
|
||||
FROM player_event_raw_ledger
|
||||
WHERE occurred_at IS NOT NULL
|
||||
AND {where_sql}
|
||||
""",
|
||||
params,
|
||||
).fetchone()
|
||||
if not row or not row["latest_month"]:
|
||||
return None
|
||||
return str(row["latest_month"])
|
||||
|
||||
|
||||
def _get_player_event_source_range(
|
||||
*,
|
||||
server_key: str,
|
||||
month_key: str,
|
||||
db_path: Path | None = None,
|
||||
) -> tuple[datetime | None, datetime | None]:
|
||||
resolved_path = initialize_player_event_storage(db_path=db_path)
|
||||
where_sql, params = _build_player_event_scope_where(server_key=server_key)
|
||||
with _connect(resolved_path) as connection:
|
||||
row = connection.execute(
|
||||
f"""
|
||||
SELECT
|
||||
MIN(occurred_at) AS source_range_start,
|
||||
MAX(occurred_at) AS source_range_end
|
||||
FROM player_event_raw_ledger
|
||||
WHERE occurred_at IS NOT NULL
|
||||
AND substr(CAST(occurred_at AS TEXT), 1, 7) = ?
|
||||
AND {where_sql}
|
||||
""",
|
||||
[month_key, *params],
|
||||
).fetchone()
|
||||
if not row:
|
||||
return None, None
|
||||
return (
|
||||
_parse_optional_timestamp(row["source_range_start"]),
|
||||
_parse_optional_timestamp(row["source_range_end"]),
|
||||
)
|
||||
|
||||
|
||||
def _build_player_event_scope_where(*, server_key: str) -> tuple[str, list[object]]:
|
||||
if server_key == ALL_SERVERS_SLUG:
|
||||
return "1 = 1", []
|
||||
return "server_slug = ?", [server_key]
|
||||
|
||||
|
||||
def _connect(db_path: Path) -> sqlite3.Connection:
|
||||
if get_database_url():
|
||||
from .postgres_display_storage import connect_postgres_compat
|
||||
|
||||
return connect_postgres_compat()
|
||||
connection = sqlite3.connect(db_path)
|
||||
connection.row_factory = sqlite3.Row
|
||||
return connection
|
||||
|
||||
|
||||
def _parse_optional_timestamp(value: object) -> datetime | None:
|
||||
if not isinstance(value, str) or not value.strip():
|
||||
return None
|
||||
normalized = value.strip().replace("Z", "+00:00")
|
||||
parsed = datetime.fromisoformat(normalized)
|
||||
if parsed.tzinfo is None:
|
||||
return parsed.replace(tzinfo=timezone.utc)
|
||||
return parsed.astimezone(timezone.utc)
|
||||
|
||||
|
||||
def _as_utc(value: datetime) -> datetime:
|
||||
if value.tzinfo is None:
|
||||
return value.replace(tzinfo=timezone.utc)
|
||||
return value.astimezone(timezone.utc)
|
||||
|
||||
|
||||
def _to_iso(value: datetime) -> str:
|
||||
return _as_utc(value).isoformat().replace("+00:00", "Z")
|
||||
|
||||
|
||||
def _normalize_snapshot_limit(name: str, value: object) -> int:
|
||||
try:
|
||||
limit = int(value)
|
||||
except (TypeError, ValueError) as error:
|
||||
raise ValueError(f"{name} must be a positive integer.") from error
|
||||
if limit <= 0:
|
||||
raise ValueError(f"{name} must be a positive integer.")
|
||||
return limit
|
||||
|
||||
|
||||
def _log_snapshot_build_started(
|
||||
server_key: str,
|
||||
snapshot_type: str,
|
||||
*,
|
||||
metric: str | None = None,
|
||||
) -> None:
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"event": "historical-snapshot-build-started",
|
||||
"server_key": server_key,
|
||||
"snapshot_type": snapshot_type,
|
||||
"metric": metric,
|
||||
},
|
||||
ensure_ascii=True,
|
||||
default=str,
|
||||
),
|
||||
flush=True,
|
||||
)
|
||||
3325
backend/app/historical_storage.py
Normal file
3325
backend/app/historical_storage.py
Normal file
File diff suppressed because it is too large
Load Diff
90
backend/app/main.py
Normal file
90
backend/app/main.py
Normal file
@@ -0,0 +1,90 @@
|
||||
"""Minimal HTTP entrypoint for the HLL Vietnam backend bootstrap."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from datetime import date, datetime
|
||||
from http import HTTPStatus
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
|
||||
from .config import get_allowed_origins, get_bind_address
|
||||
from .payloads import build_error_payload
|
||||
from .routes import resolve_get_payload
|
||||
|
||||
|
||||
class HealthHandler(BaseHTTPRequestHandler):
|
||||
"""Serve the minimal routes required for the backend bootstrap."""
|
||||
|
||||
server_version = "HLLVietnamBackend/0.1"
|
||||
|
||||
def do_OPTIONS(self) -> None: # noqa: N802 - BaseHTTPRequestHandler interface
|
||||
self.send_response(HTTPStatus.NO_CONTENT)
|
||||
self._send_default_headers()
|
||||
self.send_header("Access-Control-Allow-Methods", "GET, OPTIONS")
|
||||
self.send_header("Access-Control-Allow-Headers", "Content-Type")
|
||||
self.end_headers()
|
||||
|
||||
def do_GET(self) -> None: # noqa: N802 - BaseHTTPRequestHandler interface
|
||||
try:
|
||||
status, payload = resolve_get_payload(self.path)
|
||||
except Exception: # noqa: BLE001 - preserve HTTP/CORS response on route failures
|
||||
self._write_json(
|
||||
HTTPStatus.INTERNAL_SERVER_ERROR,
|
||||
build_error_payload("Unexpected backend error"),
|
||||
)
|
||||
return
|
||||
|
||||
if status is None:
|
||||
self._write_json(
|
||||
HTTPStatus.NOT_FOUND,
|
||||
{"status": "error", "message": "Route not found"},
|
||||
)
|
||||
return
|
||||
|
||||
self._write_json(status, payload)
|
||||
|
||||
def log_message(self, format: str, *args: object) -> None:
|
||||
# Keep local startup output clean unless future tasks need request logging.
|
||||
return
|
||||
|
||||
def _write_json(self, status: HTTPStatus, payload: dict[str, object]) -> None:
|
||||
body = json.dumps(payload, default=_json_default).encode("utf-8")
|
||||
self.send_response(status)
|
||||
self._send_default_headers(content_length=len(body))
|
||||
self.end_headers()
|
||||
self.wfile.write(body)
|
||||
|
||||
def _send_default_headers(self, content_length: int | None = None) -> None:
|
||||
origin = self.headers.get("Origin")
|
||||
if origin in get_allowed_origins():
|
||||
self.send_header("Access-Control-Allow-Origin", origin)
|
||||
self.send_header("Vary", "Origin")
|
||||
|
||||
self.send_header("Content-Type", "application/json; charset=utf-8")
|
||||
if content_length is not None:
|
||||
self.send_header("Content-Length", str(content_length))
|
||||
|
||||
|
||||
def create_server() -> ThreadingHTTPServer:
|
||||
"""Build the HTTP server using the package-supported handler and bind settings."""
|
||||
host, port = get_bind_address()
|
||||
return ThreadingHTTPServer((host, port), HealthHandler)
|
||||
|
||||
|
||||
def _json_default(value: object) -> str:
|
||||
"""Serialize PostgreSQL date/time values before they can abort an HTTP response."""
|
||||
if isinstance(value, (date, datetime)):
|
||||
return value.isoformat()
|
||||
raise TypeError(f"Object of type {type(value).__name__} is not JSON serializable")
|
||||
|
||||
|
||||
def run() -> None:
|
||||
"""Start the local bootstrap server."""
|
||||
host, port = get_bind_address()
|
||||
server = create_server()
|
||||
print(f"HLL Vietnam backend bootstrap listening on http://{host}:{port}")
|
||||
server.serve_forever()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
run()
|
||||
163
backend/app/monthly_mvp.py
Normal file
163
backend/app/monthly_mvp.py
Normal file
@@ -0,0 +1,163 @@
|
||||
"""Monthly MVP V1 scoring helpers."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
from typing import Mapping
|
||||
|
||||
|
||||
MONTHLY_MVP_VERSION = "v1"
|
||||
MONTHLY_MVP_MIN_MATCHES = 6
|
||||
MONTHLY_MVP_MIN_TIME_SECONDS = 21600
|
||||
MONTHLY_MVP_FULL_PARTICIPATION_SECONDS = 28800
|
||||
MONTHLY_MVP_TEAMKILL_PENALTY_CAP = 6.0
|
||||
MONTHLY_MVP_TEAMKILL_PENALTY_PER_KILL = 0.5
|
||||
|
||||
|
||||
def build_monthly_mvp_rankings(
|
||||
aggregated_rows: list[Mapping[str, object]],
|
||||
*,
|
||||
limit: int,
|
||||
) -> dict[str, object]:
|
||||
"""Transform aggregated monthly totals into ranked MVP V1 items."""
|
||||
eligible_rows = [
|
||||
_build_eligible_player_summary(row)
|
||||
for row in aggregated_rows
|
||||
if _is_eligible_player_row(row)
|
||||
]
|
||||
|
||||
if not eligible_rows:
|
||||
return {
|
||||
"ranking_version": MONTHLY_MVP_VERSION,
|
||||
"eligibility": _build_eligibility_metadata(),
|
||||
"items": [],
|
||||
"eligible_players_count": 0,
|
||||
}
|
||||
|
||||
max_total_kills = max(item["totals"]["kills"] for item in eligible_rows)
|
||||
max_total_support = max(item["totals"]["support"] for item in eligible_rows)
|
||||
max_kpm = max(item["derived"]["kpm"] for item in eligible_rows)
|
||||
max_kda = max(item["derived"]["kda"] for item in eligible_rows)
|
||||
|
||||
for item in eligible_rows:
|
||||
component_scores = {
|
||||
"kills_score": _log_normalized_score(item["totals"]["kills"], max_total_kills),
|
||||
"support_score": _log_normalized_score(item["totals"]["support"], max_total_support),
|
||||
"kpm_score": _log_normalized_score(item["derived"]["kpm"], max_kpm),
|
||||
"kda_score": _log_normalized_score(item["derived"]["kda"], max_kda),
|
||||
"participation_score": round(
|
||||
100
|
||||
* min(
|
||||
1.0,
|
||||
item["totals"]["time_seconds"] / MONTHLY_MVP_FULL_PARTICIPATION_SECONDS,
|
||||
),
|
||||
3,
|
||||
),
|
||||
}
|
||||
teamkill_penalty = round(
|
||||
min(
|
||||
MONTHLY_MVP_TEAMKILL_PENALTY_CAP,
|
||||
item["totals"]["teamkills"] * MONTHLY_MVP_TEAMKILL_PENALTY_PER_KILL,
|
||||
),
|
||||
3,
|
||||
)
|
||||
item["component_scores"] = component_scores
|
||||
item["teamkill_penalty"] = teamkill_penalty
|
||||
item["mvp_score"] = round(
|
||||
(0.35 * component_scores["kills_score"])
|
||||
+ (0.20 * component_scores["support_score"])
|
||||
+ (0.20 * component_scores["kpm_score"])
|
||||
+ (0.15 * component_scores["kda_score"])
|
||||
+ (0.10 * component_scores["participation_score"])
|
||||
- teamkill_penalty,
|
||||
3,
|
||||
)
|
||||
|
||||
ranked_items = sorted(
|
||||
eligible_rows,
|
||||
key=lambda item: (
|
||||
-item["mvp_score"],
|
||||
-item["component_scores"]["participation_score"],
|
||||
-item["component_scores"]["kills_score"],
|
||||
-item["component_scores"]["support_score"],
|
||||
item["totals"]["teamkills"],
|
||||
str(item["player"]["name"]).casefold(),
|
||||
str(item["player"]["stable_player_key"]),
|
||||
),
|
||||
)
|
||||
for position, item in enumerate(ranked_items[:limit], start=1):
|
||||
item["ranking_position"] = position
|
||||
|
||||
return {
|
||||
"ranking_version": MONTHLY_MVP_VERSION,
|
||||
"eligibility": _build_eligibility_metadata(),
|
||||
"eligible_players_count": len(eligible_rows),
|
||||
"items": ranked_items[:limit],
|
||||
}
|
||||
|
||||
|
||||
def _is_eligible_player_row(row: Mapping[str, object]) -> bool:
|
||||
matches_count = int(row.get("matches_count") or 0)
|
||||
time_seconds = int(row.get("total_time_seconds") or 0)
|
||||
has_required_fields = all(
|
||||
row.get(field_name) is not None
|
||||
for field_name in ("total_kills", "total_deaths", "total_support", "total_time_seconds")
|
||||
)
|
||||
return (
|
||||
has_required_fields
|
||||
and matches_count >= MONTHLY_MVP_MIN_MATCHES
|
||||
and time_seconds >= MONTHLY_MVP_MIN_TIME_SECONDS
|
||||
)
|
||||
|
||||
|
||||
def _build_eligible_player_summary(row: Mapping[str, object]) -> dict[str, object]:
|
||||
total_kills = int(row.get("total_kills") or 0)
|
||||
total_deaths = int(row.get("total_deaths") or 0)
|
||||
total_support = int(row.get("total_support") or 0)
|
||||
total_teamkills = int(row.get("total_teamkills") or 0)
|
||||
total_time_seconds = int(row.get("total_time_seconds") or 0)
|
||||
total_time_minutes = max(total_time_seconds / 60.0, 1.0)
|
||||
kpm = round(total_kills / total_time_minutes, 6)
|
||||
kda = round(total_kills / max(total_deaths, 1), 6)
|
||||
return {
|
||||
"server": {
|
||||
"slug": row.get("server_slug"),
|
||||
"name": row.get("server_name"),
|
||||
},
|
||||
"player": {
|
||||
"stable_player_key": row.get("stable_player_key"),
|
||||
"name": row.get("player_name"),
|
||||
"steam_id": row.get("steam_id"),
|
||||
},
|
||||
"matches_considered": int(row.get("matches_count") or 0),
|
||||
"totals": {
|
||||
"kills": total_kills,
|
||||
"deaths": total_deaths,
|
||||
"support": total_support,
|
||||
"teamkills": total_teamkills,
|
||||
"time_seconds": total_time_seconds,
|
||||
"time_minutes": round(total_time_seconds / 60.0, 2),
|
||||
},
|
||||
"derived": {
|
||||
"kpm": kpm,
|
||||
"kda": kda,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _log_normalized_score(value: float | int, max_value: float | int) -> float:
|
||||
if value <= 0 or max_value <= 0:
|
||||
return 0.0
|
||||
return round((100 * math.log1p(value)) / math.log1p(max_value), 3)
|
||||
|
||||
|
||||
def _build_eligibility_metadata() -> dict[str, object]:
|
||||
return {
|
||||
"minimum_matches": MONTHLY_MVP_MIN_MATCHES,
|
||||
"minimum_time_seconds": MONTHLY_MVP_MIN_TIME_SECONDS,
|
||||
"minimum_time_hours": round(MONTHLY_MVP_MIN_TIME_SECONDS / 3600, 1),
|
||||
"full_participation_seconds": MONTHLY_MVP_FULL_PARTICIPATION_SECONDS,
|
||||
"full_participation_hours": round(MONTHLY_MVP_FULL_PARTICIPATION_SECONDS / 3600, 1),
|
||||
"teamkill_penalty_per_kill": MONTHLY_MVP_TEAMKILL_PENALTY_PER_KILL,
|
||||
"teamkill_penalty_cap": MONTHLY_MVP_TEAMKILL_PENALTY_CAP,
|
||||
}
|
||||
201
backend/app/monthly_mvp_v2.py
Normal file
201
backend/app/monthly_mvp_v2.py
Normal file
@@ -0,0 +1,201 @@
|
||||
"""Monthly MVP V2 scoring helpers."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
from typing import Mapping
|
||||
|
||||
|
||||
MONTHLY_MVP_V2_VERSION = "v2"
|
||||
MONTHLY_MVP_V2_MIN_MATCHES = 6
|
||||
MONTHLY_MVP_V2_MIN_TIME_SECONDS = 21600
|
||||
MONTHLY_MVP_V2_FULL_PARTICIPATION_SECONDS = 28800
|
||||
MONTHLY_MVP_V2_ADVANCED_CONFIDENCE_KILLS = 35
|
||||
MONTHLY_MVP_V2_TEAMKILL_PENALTY_CAP = 8.0
|
||||
MONTHLY_MVP_V2_TEAMKILL_PENALTY_PER_KILL = 0.75
|
||||
|
||||
|
||||
def build_monthly_mvp_v2_rankings(
|
||||
aggregated_rows: list[Mapping[str, object]],
|
||||
*,
|
||||
limit: int,
|
||||
) -> dict[str, object]:
|
||||
"""Transform aggregated monthly totals plus V2 event signals into rankings."""
|
||||
eligible_rows = [
|
||||
_build_eligible_player_summary(row)
|
||||
for row in aggregated_rows
|
||||
if _is_eligible_player_row(row)
|
||||
]
|
||||
|
||||
if not eligible_rows:
|
||||
return {
|
||||
"ranking_version": MONTHLY_MVP_V2_VERSION,
|
||||
"eligibility": _build_eligibility_metadata(),
|
||||
"eligible_players_count": 0,
|
||||
"items": [],
|
||||
}
|
||||
|
||||
max_total_kills = max(item["totals"]["kills"] for item in eligible_rows)
|
||||
max_total_support = max(item["totals"]["support"] for item in eligible_rows)
|
||||
max_kpm = max(item["derived"]["kpm"] for item in eligible_rows)
|
||||
max_kda = max(item["derived"]["kda"] for item in eligible_rows)
|
||||
max_rivalry_edge = max(item["advanced"]["rivalry_edge_raw"] for item in eligible_rows)
|
||||
max_duel_control = max(item["advanced"]["duel_control_raw"] for item in eligible_rows)
|
||||
|
||||
for item in eligible_rows:
|
||||
component_scores = {
|
||||
"kills_score": _log_normalized_score(item["totals"]["kills"], max_total_kills),
|
||||
"support_score": _log_normalized_score(item["totals"]["support"], max_total_support),
|
||||
"kpm_score": _log_normalized_score(item["derived"]["kpm"], max_kpm),
|
||||
"kda_score": _log_normalized_score(item["derived"]["kda"], max_kda),
|
||||
"participation_score": round(
|
||||
100
|
||||
* min(
|
||||
1.0,
|
||||
item["totals"]["time_seconds"] / MONTHLY_MVP_V2_FULL_PARTICIPATION_SECONDS,
|
||||
),
|
||||
3,
|
||||
),
|
||||
"rivalry_edge_score": _log_normalized_score(
|
||||
item["advanced"]["rivalry_edge_raw"],
|
||||
max_rivalry_edge,
|
||||
),
|
||||
"duel_control_score": _log_normalized_score(
|
||||
item["advanced"]["duel_control_raw"],
|
||||
max_duel_control,
|
||||
),
|
||||
}
|
||||
advanced_confidence = round(
|
||||
min(
|
||||
1.0,
|
||||
item["totals"]["kills"] / MONTHLY_MVP_V2_ADVANCED_CONFIDENCE_KILLS,
|
||||
),
|
||||
3,
|
||||
)
|
||||
teamkill_penalty_v2 = round(
|
||||
min(
|
||||
MONTHLY_MVP_V2_TEAMKILL_PENALTY_CAP,
|
||||
item["totals"]["teamkills"] * MONTHLY_MVP_V2_TEAMKILL_PENALTY_PER_KILL,
|
||||
),
|
||||
3,
|
||||
)
|
||||
item["component_scores"] = component_scores
|
||||
item["advanced_confidence"] = advanced_confidence
|
||||
item["teamkill_penalty_v2"] = teamkill_penalty_v2
|
||||
item["mvp_v2_score"] = round(
|
||||
(0.30 * component_scores["kills_score"])
|
||||
+ (0.18 * component_scores["support_score"])
|
||||
+ (0.18 * component_scores["kpm_score"])
|
||||
+ (0.12 * component_scores["kda_score"])
|
||||
+ (0.10 * component_scores["participation_score"])
|
||||
+ advanced_confidence
|
||||
* (
|
||||
(0.07 * component_scores["rivalry_edge_score"])
|
||||
+ (0.05 * component_scores["duel_control_score"])
|
||||
)
|
||||
- teamkill_penalty_v2,
|
||||
3,
|
||||
)
|
||||
|
||||
ranked_items = sorted(
|
||||
eligible_rows,
|
||||
key=lambda item: (
|
||||
-item["mvp_v2_score"],
|
||||
-item["advanced_confidence"],
|
||||
-item["component_scores"]["participation_score"],
|
||||
-item["component_scores"]["kills_score"],
|
||||
-item["component_scores"]["rivalry_edge_score"],
|
||||
item["totals"]["teamkills"],
|
||||
str(item["player"]["name"]).casefold(),
|
||||
str(item["player"]["stable_player_key"]),
|
||||
),
|
||||
)
|
||||
for position, item in enumerate(ranked_items[:limit], start=1):
|
||||
item["ranking_position"] = position
|
||||
|
||||
return {
|
||||
"ranking_version": MONTHLY_MVP_V2_VERSION,
|
||||
"eligibility": _build_eligibility_metadata(),
|
||||
"eligible_players_count": len(eligible_rows),
|
||||
"items": ranked_items[:limit],
|
||||
}
|
||||
|
||||
|
||||
def _is_eligible_player_row(row: Mapping[str, object]) -> bool:
|
||||
matches_count = int(row.get("matches_count") or 0)
|
||||
time_seconds = int(row.get("total_time_seconds") or 0)
|
||||
has_required_fields = all(
|
||||
row.get(field_name) is not None
|
||||
for field_name in ("total_kills", "total_deaths", "total_support", "total_time_seconds")
|
||||
)
|
||||
return (
|
||||
has_required_fields
|
||||
and matches_count >= MONTHLY_MVP_V2_MIN_MATCHES
|
||||
and time_seconds >= MONTHLY_MVP_V2_MIN_TIME_SECONDS
|
||||
)
|
||||
|
||||
|
||||
def _build_eligible_player_summary(row: Mapping[str, object]) -> dict[str, object]:
|
||||
total_kills = int(row.get("total_kills") or 0)
|
||||
total_deaths = int(row.get("total_deaths") or 0)
|
||||
total_support = int(row.get("total_support") or 0)
|
||||
total_teamkills = int(row.get("total_teamkills") or 0)
|
||||
total_time_seconds = int(row.get("total_time_seconds") or 0)
|
||||
total_time_minutes = max(total_time_seconds / 60.0, 1.0)
|
||||
most_killed_count = int(row.get("most_killed_count") or 0)
|
||||
death_by_count = int(row.get("death_by_count") or 0)
|
||||
duel_control_raw = int(row.get("duel_control_raw") or 0)
|
||||
kpm = round(total_kills / total_time_minutes, 6)
|
||||
kda = round(total_kills / max(total_deaths, 1), 6)
|
||||
return {
|
||||
"server": {
|
||||
"slug": row.get("server_slug"),
|
||||
"name": row.get("server_name"),
|
||||
},
|
||||
"player": {
|
||||
"stable_player_key": row.get("stable_player_key"),
|
||||
"name": row.get("player_name"),
|
||||
"steam_id": row.get("steam_id"),
|
||||
},
|
||||
"matches_considered": int(row.get("matches_count") or 0),
|
||||
"totals": {
|
||||
"kills": total_kills,
|
||||
"deaths": total_deaths,
|
||||
"support": total_support,
|
||||
"teamkills": total_teamkills,
|
||||
"time_seconds": total_time_seconds,
|
||||
"time_minutes": round(total_time_seconds / 60.0, 2),
|
||||
},
|
||||
"derived": {
|
||||
"kpm": kpm,
|
||||
"kda": kda,
|
||||
},
|
||||
"advanced": {
|
||||
"most_killed_count": most_killed_count,
|
||||
"death_by_count": death_by_count,
|
||||
"rivalry_edge_raw": max(0, most_killed_count - death_by_count),
|
||||
"duel_control_raw": duel_control_raw,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _log_normalized_score(value: float | int, max_value: float | int) -> float:
|
||||
if value <= 0 or max_value <= 0:
|
||||
return 0.0
|
||||
return round((100 * math.log1p(value)) / math.log1p(max_value), 3)
|
||||
|
||||
|
||||
def _build_eligibility_metadata() -> dict[str, object]:
|
||||
return {
|
||||
"minimum_matches": MONTHLY_MVP_V2_MIN_MATCHES,
|
||||
"minimum_time_seconds": MONTHLY_MVP_V2_MIN_TIME_SECONDS,
|
||||
"minimum_time_hours": round(MONTHLY_MVP_V2_MIN_TIME_SECONDS / 3600, 1),
|
||||
"full_participation_seconds": MONTHLY_MVP_V2_FULL_PARTICIPATION_SECONDS,
|
||||
"full_participation_hours": round(
|
||||
MONTHLY_MVP_V2_FULL_PARTICIPATION_SECONDS / 3600,
|
||||
1,
|
||||
),
|
||||
"advanced_confidence_kills": MONTHLY_MVP_V2_ADVANCED_CONFIDENCE_KILLS,
|
||||
"teamkill_penalty_per_kill": MONTHLY_MVP_V2_TEAMKILL_PENALTY_PER_KILL,
|
||||
"teamkill_penalty_cap": MONTHLY_MVP_V2_TEAMKILL_PENALTY_CAP,
|
||||
}
|
||||
164
backend/app/normalizers.py
Normal file
164
backend/app/normalizers.py
Normal file
@@ -0,0 +1,164 @@
|
||||
"""Normalization helpers for provisional server collection flows."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
from typing import Mapping
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from .a2s_client import A2SServerInfo
|
||||
|
||||
|
||||
MAP_NAME_ALIASES = {
|
||||
"stmarie": "St. Marie Du Mont",
|
||||
"stmariedumont": "St. Marie Du Mont",
|
||||
"saintemariedumont": "St. Marie Du Mont",
|
||||
"saintemariedumontwarfare": "St. Marie Du Mont",
|
||||
"saintemariedumontoffensiveus": "St. Marie Du Mont",
|
||||
"saintemariedumontoffensiveger": "St. Marie Du Mont",
|
||||
"saintemariedumontnight": "St. Marie Du Mont",
|
||||
"saintemariedumontovercast": "St. Marie Du Mont",
|
||||
"sainte-mariedumont": "St. Marie Du Mont",
|
||||
"sainte-marie-du-mont": "St. Marie Du Mont",
|
||||
"stmereeglise": "St. Mere Eglise",
|
||||
"stmereeglisewarfare": "St. Mere Eglise",
|
||||
"stmereegliseoffensiveus": "St. Mere Eglise",
|
||||
"stmereegliseoffensiveger": "St. Mere Eglise",
|
||||
"saintemereeglise": "St. Mere Eglise",
|
||||
"sainte-mere-eglise": "St. Mere Eglise",
|
||||
"purpleheartlane": "Purple Heart Lane",
|
||||
"utahbeach": "Utah Beach",
|
||||
"omahabeach": "Omaha Beach",
|
||||
"hurtgenforest": "Hurtgen Forest",
|
||||
"hill400": "Hill 400",
|
||||
"foy": "Foy",
|
||||
"kursk": "Kursk",
|
||||
"kharkov": "Kharkov",
|
||||
"kharkiv": "Kharkiv",
|
||||
"stalingrad": "Stalingrad",
|
||||
"remagen": "Remagen",
|
||||
"driel": "Driel",
|
||||
"elalamein": "El Alamein",
|
||||
"mortain": "Mortain",
|
||||
"carentan": "Carentan",
|
||||
"devn": "Elsenborn Ridge",
|
||||
"elsenbornridge": "Elsenborn Ridge",
|
||||
"elsenborn": "Elsenborn Ridge",
|
||||
"smolensk": "Smolensk",
|
||||
"smolenskwarfare": "Smolensk",
|
||||
"smolenskoffensiverus": "Smolensk",
|
||||
"smolenskoffensiveger": "Smolensk",
|
||||
"developertestmap": "Smolensk",
|
||||
"devq": "Smolensk",
|
||||
}
|
||||
|
||||
|
||||
def normalize_server_record(
|
||||
raw_record: Mapping[str, object],
|
||||
*,
|
||||
source_name: str,
|
||||
) -> dict[str, object]:
|
||||
"""Normalize a raw server record into the collector's internal shape."""
|
||||
external_server_id = _string_or_none(raw_record.get("external_server_id"))
|
||||
return {
|
||||
"external_server_id": external_server_id,
|
||||
"server_name": _string_or_default(raw_record.get("server_name"), "Unknown server"),
|
||||
"status": _normalize_status(raw_record.get("status")),
|
||||
"players": _coerce_int(raw_record.get("players")),
|
||||
"max_players": _coerce_int(raw_record.get("max_players")),
|
||||
"current_map": normalize_map_name(raw_record.get("current_map")),
|
||||
"region": _string_or_none(raw_record.get("region")),
|
||||
"source_name": source_name,
|
||||
"snapshot_origin": "controlled-fallback",
|
||||
"source_ref": external_server_id or source_name,
|
||||
}
|
||||
|
||||
|
||||
def normalize_a2s_server_info(
|
||||
server_info: "A2SServerInfo",
|
||||
*,
|
||||
source_name: str,
|
||||
external_server_id: str | None = None,
|
||||
region: str | None = None,
|
||||
) -> dict[str, object]:
|
||||
"""Normalize a probed A2S payload into the collector's internal shape."""
|
||||
resolved_external_id = external_server_id or (
|
||||
f"a2s:{server_info.host}:{server_info.query_port}"
|
||||
)
|
||||
return {
|
||||
"external_server_id": resolved_external_id,
|
||||
"server_name": server_info.server_name or "Unknown server",
|
||||
"status": "online",
|
||||
"players": server_info.players,
|
||||
"max_players": server_info.max_players,
|
||||
"current_map": normalize_map_name(server_info.map_name),
|
||||
"region": region,
|
||||
"source_name": source_name,
|
||||
"snapshot_origin": "real-a2s",
|
||||
"source_ref": f"a2s://{server_info.host}:{server_info.query_port}",
|
||||
}
|
||||
|
||||
|
||||
def normalize_map_name(value: object) -> str | None:
|
||||
"""Normalize internal or abbreviated HLL map labels into a stable display name."""
|
||||
normalized = _string_or_none(value)
|
||||
if normalized is None:
|
||||
return None
|
||||
|
||||
alias_key = "".join(character.lower() for character in normalized if character.isalnum())
|
||||
alias_match = MAP_NAME_ALIASES.get(alias_key)
|
||||
if alias_match:
|
||||
return alias_match
|
||||
|
||||
for candidate_key, candidate_label in MAP_NAME_ALIASES.items():
|
||||
if alias_key.startswith(candidate_key):
|
||||
return candidate_label
|
||||
|
||||
prettified = _prettify_map_name(normalized)
|
||||
return prettified or normalized
|
||||
|
||||
|
||||
def _normalize_status(value: object) -> str:
|
||||
if not isinstance(value, str):
|
||||
return "unknown"
|
||||
|
||||
normalized = value.strip().lower()
|
||||
if normalized in {"online", "offline", "unknown"}:
|
||||
return normalized
|
||||
|
||||
return "unknown"
|
||||
|
||||
|
||||
def _coerce_int(value: object) -> int | None:
|
||||
if value is None:
|
||||
return None
|
||||
|
||||
try:
|
||||
return int(value)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
def _string_or_none(value: object) -> str | None:
|
||||
if not isinstance(value, str):
|
||||
return None
|
||||
|
||||
stripped = value.strip()
|
||||
return stripped or None
|
||||
|
||||
|
||||
def _string_or_default(value: object, default: str) -> str:
|
||||
normalized = _string_or_none(value)
|
||||
return normalized or default
|
||||
|
||||
|
||||
def _prettify_map_name(value: str) -> str:
|
||||
text = value.replace("_", " ").replace("-", " ").strip()
|
||||
compact_text = " ".join(text.split())
|
||||
if not compact_text:
|
||||
return value
|
||||
|
||||
return " ".join(
|
||||
word.upper() if word.isdigit() else word.capitalize()
|
||||
for word in compact_text.split(" ")
|
||||
)
|
||||
2187
backend/app/payloads.py
Normal file
2187
backend/app/payloads.py
Normal file
File diff suppressed because it is too large
Load Diff
261
backend/app/player_event_aggregates.py
Normal file
261
backend/app/player_event_aggregates.py
Normal file
@@ -0,0 +1,261 @@
|
||||
"""Derived duel and weapon aggregates computed from the raw player event ledger."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlite3
|
||||
from pathlib import Path
|
||||
|
||||
from .config import get_database_url, get_storage_path
|
||||
from .player_event_storage import initialize_player_event_storage
|
||||
|
||||
|
||||
def list_most_killed(
|
||||
*,
|
||||
server_slug: str | None = None,
|
||||
month: str | None = None,
|
||||
external_match_id: str | None = None,
|
||||
limit: int = 10,
|
||||
db_path: Path | None = None,
|
||||
) -> list[dict[str, object]]:
|
||||
"""Return strongest killer -> victim summaries from the raw ledger."""
|
||||
return _query_pair_summary(
|
||||
event_type="player_kill_summary",
|
||||
server_slug=server_slug,
|
||||
month=month,
|
||||
external_match_id=external_match_id,
|
||||
limit=limit,
|
||||
db_path=db_path,
|
||||
)
|
||||
|
||||
|
||||
def list_death_by(
|
||||
*,
|
||||
server_slug: str | None = None,
|
||||
month: str | None = None,
|
||||
external_match_id: str | None = None,
|
||||
limit: int = 10,
|
||||
db_path: Path | None = None,
|
||||
) -> list[dict[str, object]]:
|
||||
"""Return strongest killer -> victim summaries from the victim perspective."""
|
||||
return _query_pair_summary(
|
||||
event_type="player_death_summary",
|
||||
server_slug=server_slug,
|
||||
month=month,
|
||||
external_match_id=external_match_id,
|
||||
limit=limit,
|
||||
db_path=db_path,
|
||||
)
|
||||
|
||||
|
||||
def list_net_duel_summaries(
|
||||
*,
|
||||
server_slug: str | None = None,
|
||||
month: str | None = None,
|
||||
external_match_id: str | None = None,
|
||||
limit: int = 10,
|
||||
db_path: Path | None = None,
|
||||
) -> list[dict[str, object]]:
|
||||
"""Return partial net duel summaries using the strongest encounter signals available."""
|
||||
resolved_path = initialize_player_event_storage(db_path=db_path)
|
||||
where_sql, params = _build_common_where(
|
||||
event_type="player_kill_summary",
|
||||
server_slug=server_slug,
|
||||
month=month,
|
||||
external_match_id=external_match_id,
|
||||
)
|
||||
with _connect(resolved_path) as connection:
|
||||
rows = connection.execute(
|
||||
f"""
|
||||
WITH duel_pairs AS (
|
||||
SELECT
|
||||
CASE
|
||||
WHEN COALESCE(killer_player_key, '') <= COALESCE(victim_player_key, '')
|
||||
THEN killer_player_key
|
||||
ELSE victim_player_key
|
||||
END AS player_a_key,
|
||||
CASE
|
||||
WHEN COALESCE(killer_player_key, '') <= COALESCE(victim_player_key, '')
|
||||
THEN killer_display_name
|
||||
ELSE victim_display_name
|
||||
END AS player_a_name,
|
||||
CASE
|
||||
WHEN COALESCE(killer_player_key, '') <= COALESCE(victim_player_key, '')
|
||||
THEN victim_player_key
|
||||
ELSE killer_player_key
|
||||
END AS player_b_key,
|
||||
CASE
|
||||
WHEN COALESCE(killer_player_key, '') <= COALESCE(victim_player_key, '')
|
||||
THEN victim_display_name
|
||||
ELSE killer_display_name
|
||||
END AS player_b_name,
|
||||
CASE
|
||||
WHEN COALESCE(killer_player_key, '') <= COALESCE(victim_player_key, '')
|
||||
THEN event_value
|
||||
ELSE -event_value
|
||||
END AS net_value,
|
||||
event_value
|
||||
FROM player_event_raw_ledger
|
||||
WHERE {where_sql}
|
||||
AND killer_player_key IS NOT NULL
|
||||
AND victim_player_key IS NOT NULL
|
||||
)
|
||||
SELECT
|
||||
player_a_key,
|
||||
player_a_name,
|
||||
player_b_key,
|
||||
player_b_name,
|
||||
COALESCE(SUM(event_value), 0) AS total_encounters,
|
||||
COALESCE(SUM(net_value), 0) AS net_duel_value
|
||||
FROM duel_pairs
|
||||
GROUP BY player_a_key, player_a_name, player_b_key, player_b_name
|
||||
ORDER BY ABS(COALESCE(SUM(net_value), 0)) DESC,
|
||||
COALESCE(SUM(event_value), 0) DESC,
|
||||
player_a_name ASC,
|
||||
player_b_name ASC
|
||||
LIMIT ?
|
||||
""",
|
||||
[*params, limit],
|
||||
).fetchall()
|
||||
return [dict(row) for row in rows]
|
||||
|
||||
|
||||
def list_weapon_kills(
|
||||
*,
|
||||
server_slug: str | None = None,
|
||||
month: str | None = None,
|
||||
external_match_id: str | None = None,
|
||||
limit: int = 10,
|
||||
db_path: Path | None = None,
|
||||
) -> list[dict[str, object]]:
|
||||
"""Return partial weapon summaries derived from top kill events."""
|
||||
resolved_path = initialize_player_event_storage(db_path=db_path)
|
||||
where_sql, params = _build_common_where(
|
||||
event_type="player_weapon_kill_summary",
|
||||
server_slug=server_slug,
|
||||
month=month,
|
||||
external_match_id=external_match_id,
|
||||
)
|
||||
with _connect(resolved_path) as connection:
|
||||
rows = connection.execute(
|
||||
f"""
|
||||
SELECT
|
||||
killer_player_key,
|
||||
killer_display_name,
|
||||
COALESCE(weapon_name, 'unknown') AS weapon_name,
|
||||
COALESCE(SUM(event_value), 0) AS total_kills
|
||||
FROM player_event_raw_ledger
|
||||
WHERE {where_sql}
|
||||
AND killer_player_key IS NOT NULL
|
||||
GROUP BY killer_player_key, killer_display_name, COALESCE(weapon_name, 'unknown')
|
||||
ORDER BY total_kills DESC, killer_display_name ASC, weapon_name ASC
|
||||
LIMIT ?
|
||||
""",
|
||||
[*params, limit],
|
||||
).fetchall()
|
||||
return [dict(row) for row in rows]
|
||||
|
||||
|
||||
def list_teamkill_summaries(
|
||||
*,
|
||||
server_slug: str | None = None,
|
||||
month: str | None = None,
|
||||
external_match_id: str | None = None,
|
||||
limit: int = 10,
|
||||
db_path: Path | None = None,
|
||||
) -> list[dict[str, object]]:
|
||||
"""Return derived teamkill totals per player from the raw ledger."""
|
||||
resolved_path = initialize_player_event_storage(db_path=db_path)
|
||||
where_sql, params = _build_common_where(
|
||||
event_type="player_teamkill_summary",
|
||||
server_slug=server_slug,
|
||||
month=month,
|
||||
external_match_id=external_match_id,
|
||||
)
|
||||
with _connect(resolved_path) as connection:
|
||||
rows = connection.execute(
|
||||
f"""
|
||||
SELECT
|
||||
killer_player_key,
|
||||
killer_display_name,
|
||||
COALESCE(SUM(event_value), 0) AS total_teamkills
|
||||
FROM player_event_raw_ledger
|
||||
WHERE {where_sql}
|
||||
AND killer_player_key IS NOT NULL
|
||||
GROUP BY killer_player_key, killer_display_name
|
||||
ORDER BY total_teamkills DESC, killer_display_name ASC
|
||||
LIMIT ?
|
||||
""",
|
||||
[*params, limit],
|
||||
).fetchall()
|
||||
return [dict(row) for row in rows]
|
||||
|
||||
|
||||
def _query_pair_summary(
|
||||
*,
|
||||
event_type: str,
|
||||
server_slug: str | None,
|
||||
month: str | None,
|
||||
external_match_id: str | None,
|
||||
limit: int,
|
||||
db_path: Path | None,
|
||||
) -> list[dict[str, object]]:
|
||||
resolved_path = initialize_player_event_storage(db_path=db_path)
|
||||
where_sql, params = _build_common_where(
|
||||
event_type=event_type,
|
||||
server_slug=server_slug,
|
||||
month=month,
|
||||
external_match_id=external_match_id,
|
||||
)
|
||||
with _connect(resolved_path) as connection:
|
||||
rows = connection.execute(
|
||||
f"""
|
||||
SELECT
|
||||
killer_player_key,
|
||||
killer_display_name,
|
||||
victim_player_key,
|
||||
victim_display_name,
|
||||
COALESCE(SUM(event_value), 0) AS total_kills
|
||||
FROM player_event_raw_ledger
|
||||
WHERE {where_sql}
|
||||
AND killer_player_key IS NOT NULL
|
||||
AND victim_player_key IS NOT NULL
|
||||
GROUP BY killer_player_key, killer_display_name, victim_player_key, victim_display_name
|
||||
ORDER BY total_kills DESC, killer_display_name ASC, victim_display_name ASC
|
||||
LIMIT ?
|
||||
""",
|
||||
[*params, limit],
|
||||
).fetchall()
|
||||
return [dict(row) for row in rows]
|
||||
|
||||
|
||||
def _build_common_where(
|
||||
*,
|
||||
event_type: str,
|
||||
server_slug: str | None,
|
||||
month: str | None,
|
||||
external_match_id: str | None,
|
||||
) -> tuple[str, list[object]]:
|
||||
clauses = ["event_type = ?"]
|
||||
params: list[object] = [event_type]
|
||||
|
||||
if server_slug and server_slug != "all-servers":
|
||||
clauses.append("server_slug = ?")
|
||||
params.append(server_slug.strip())
|
||||
if month:
|
||||
clauses.append("substr(COALESCE(CAST(occurred_at AS TEXT), ''), 1, 7) = ?")
|
||||
params.append(month.strip())
|
||||
if external_match_id:
|
||||
clauses.append("external_match_id = ?")
|
||||
params.append(external_match_id.strip())
|
||||
|
||||
return " AND ".join(clauses), params
|
||||
|
||||
|
||||
def _connect(db_path: Path) -> sqlite3.Connection:
|
||||
if get_database_url():
|
||||
from .postgres_display_storage import connect_postgres_compat
|
||||
|
||||
return connect_postgres_compat()
|
||||
connection = sqlite3.connect(db_path or get_storage_path())
|
||||
connection.row_factory = sqlite3.Row
|
||||
return connection
|
||||
32
backend/app/player_event_models.py
Normal file
32
backend/app/player_event_models.py
Normal file
@@ -0,0 +1,32 @@
|
||||
"""Normalized player event models for the V2 event pipeline foundation."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import asdict, dataclass
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class PlayerEventRecord:
|
||||
"""Minimal normalized player event contract reused across source and storage."""
|
||||
|
||||
event_id: str
|
||||
event_type: str
|
||||
occurred_at: str | None
|
||||
server_slug: str
|
||||
external_match_id: str
|
||||
source_kind: str
|
||||
source_ref: str | None
|
||||
raw_event_ref: str | None
|
||||
killer_player_key: str | None
|
||||
killer_display_name: str | None
|
||||
victim_player_key: str | None
|
||||
victim_display_name: str | None
|
||||
weapon_name: str | None
|
||||
weapon_category: str | None
|
||||
kill_category: str | None
|
||||
is_teamkill: bool
|
||||
event_value: int = 1
|
||||
|
||||
def to_dict(self) -> dict[str, object]:
|
||||
"""Return the event as a plain dictionary."""
|
||||
return asdict(self)
|
||||
111
backend/app/player_event_source.py
Normal file
111
backend/app/player_event_source.py
Normal file
@@ -0,0 +1,111 @@
|
||||
"""Player event source selection and contracts for the V2 pipeline."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import Protocol
|
||||
|
||||
from .config import get_historical_data_source_kind
|
||||
from .data_sources import (
|
||||
SOURCE_KIND_PUBLIC_SCOREBOARD,
|
||||
SOURCE_KIND_RCON,
|
||||
build_source_attempt,
|
||||
build_source_policy,
|
||||
)
|
||||
from .player_event_models import PlayerEventRecord
|
||||
from .providers.player_event_source_provider import PublicScoreboardPlayerEventSource
|
||||
|
||||
|
||||
class PlayerEventSource(Protocol):
|
||||
"""Contract for adapters that normalize player event signals."""
|
||||
|
||||
source_kind: str
|
||||
|
||||
def extract_match_events(
|
||||
self,
|
||||
*,
|
||||
server_slug: str,
|
||||
match_payload: dict[str, object],
|
||||
source_ref: str | None = None,
|
||||
) -> list[PlayerEventRecord]:
|
||||
"""Normalize one match payload into reusable player event records."""
|
||||
|
||||
def describe_scope(self) -> dict[str, object]:
|
||||
"""Describe what the adapter can and cannot capture today."""
|
||||
|
||||
|
||||
class RconPlayerEventSource:
|
||||
"""Placeholder adapter for a future raw RCON/log feed."""
|
||||
|
||||
source_kind = "rcon-events"
|
||||
|
||||
def extract_match_events(
|
||||
self,
|
||||
*,
|
||||
server_slug: str,
|
||||
match_payload: dict[str, object],
|
||||
source_ref: str | None = None,
|
||||
) -> list[PlayerEventRecord]:
|
||||
raise RuntimeError("Raw RCON player event extraction is not implemented yet.")
|
||||
|
||||
def describe_scope(self) -> dict[str, object]:
|
||||
return {
|
||||
"source_kind": self.source_kind,
|
||||
"supports_raw_kill_events": False,
|
||||
"captures": [],
|
||||
"limitations": [
|
||||
"No raw RCON event or log feed is integrated in this repository yet.",
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class PlayerEventSourceSelection:
|
||||
"""Resolved player-event adapter plus source-policy metadata."""
|
||||
|
||||
source: PlayerEventSource
|
||||
source_policy: dict[str, object]
|
||||
|
||||
|
||||
def resolve_player_event_source() -> PlayerEventSourceSelection:
|
||||
"""Select the event adapter with safe fallback when raw RCON events are unavailable."""
|
||||
source_kind = get_historical_data_source_kind()
|
||||
if source_kind == SOURCE_KIND_PUBLIC_SCOREBOARD:
|
||||
return PlayerEventSourceSelection(
|
||||
source=PublicScoreboardPlayerEventSource(),
|
||||
source_policy=build_source_policy(
|
||||
primary_source=SOURCE_KIND_PUBLIC_SCOREBOARD,
|
||||
selected_source="public-scoreboard-match-summary",
|
||||
source_attempts=[
|
||||
build_source_attempt(
|
||||
source=SOURCE_KIND_PUBLIC_SCOREBOARD,
|
||||
role="primary",
|
||||
status="success",
|
||||
)
|
||||
],
|
||||
),
|
||||
)
|
||||
if source_kind == SOURCE_KIND_RCON:
|
||||
return PlayerEventSourceSelection(
|
||||
source=PublicScoreboardPlayerEventSource(),
|
||||
source_policy=build_source_policy(
|
||||
primary_source=SOURCE_KIND_RCON,
|
||||
selected_source="public-scoreboard-match-summary",
|
||||
fallback_used=True,
|
||||
fallback_reason="rcon-player-events-not-implemented-yet",
|
||||
source_attempts=[
|
||||
build_source_attempt(
|
||||
source=SOURCE_KIND_RCON,
|
||||
role="primary",
|
||||
status="unsupported",
|
||||
reason="rcon-player-events-not-implemented-yet",
|
||||
),
|
||||
build_source_attempt(
|
||||
source="public-scoreboard-match-summary",
|
||||
role="fallback",
|
||||
status="success",
|
||||
),
|
||||
],
|
||||
),
|
||||
)
|
||||
raise ValueError(f"Unsupported player event source: {source_kind}")
|
||||
440
backend/app/player_event_storage.py
Normal file
440
backend/app/player_event_storage.py
Normal file
@@ -0,0 +1,440 @@
|
||||
"""Raw storage and run tracking for the V2 player event pipeline."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlite3
|
||||
from collections.abc import Iterable
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from pathlib import Path
|
||||
|
||||
from .config import (
|
||||
get_player_event_refresh_overlap_hours,
|
||||
get_storage_path,
|
||||
use_postgres_rcon_storage,
|
||||
)
|
||||
from .player_event_models import PlayerEventRecord
|
||||
from .sqlite_utils import connect_sqlite_writer
|
||||
|
||||
|
||||
def initialize_player_event_storage(*, db_path: Path | None = None) -> Path:
|
||||
"""Create the append-only player event ledger and its worker metadata tables."""
|
||||
resolved_path = db_path or get_storage_path()
|
||||
resolved_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
with _connect(resolved_path) as connection:
|
||||
connection.executescript(
|
||||
"""
|
||||
CREATE TABLE IF NOT EXISTS player_event_raw_ledger (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
event_id TEXT NOT NULL UNIQUE,
|
||||
event_type TEXT NOT NULL,
|
||||
occurred_at TEXT,
|
||||
server_slug TEXT NOT NULL,
|
||||
external_match_id TEXT NOT NULL,
|
||||
source_kind TEXT NOT NULL,
|
||||
source_ref TEXT,
|
||||
raw_event_ref TEXT,
|
||||
killer_player_key TEXT,
|
||||
killer_display_name TEXT,
|
||||
victim_player_key TEXT,
|
||||
victim_display_name TEXT,
|
||||
weapon_name TEXT,
|
||||
weapon_category TEXT,
|
||||
kill_category TEXT,
|
||||
is_teamkill INTEGER NOT NULL DEFAULT 0,
|
||||
event_value INTEGER NOT NULL DEFAULT 1,
|
||||
inserted_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS player_event_ingestion_runs (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
mode TEXT NOT NULL,
|
||||
status TEXT NOT NULL,
|
||||
target_server_slug TEXT,
|
||||
started_at TEXT NOT NULL,
|
||||
completed_at TEXT,
|
||||
pages_processed INTEGER NOT NULL DEFAULT 0,
|
||||
matches_seen INTEGER NOT NULL DEFAULT 0,
|
||||
matches_fetched INTEGER NOT NULL DEFAULT 0,
|
||||
events_inserted INTEGER NOT NULL DEFAULT 0,
|
||||
duplicate_events INTEGER NOT NULL DEFAULT 0,
|
||||
notes TEXT,
|
||||
created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS player_event_backfill_progress (
|
||||
server_slug TEXT NOT NULL,
|
||||
mode TEXT NOT NULL,
|
||||
next_page INTEGER NOT NULL DEFAULT 1,
|
||||
last_completed_page INTEGER,
|
||||
cutoff_occurred_at TEXT,
|
||||
discovered_total_matches INTEGER,
|
||||
archive_exhausted INTEGER NOT NULL DEFAULT 0,
|
||||
last_run_id INTEGER,
|
||||
last_run_status TEXT,
|
||||
last_run_started_at TEXT,
|
||||
last_run_completed_at TEXT,
|
||||
last_error TEXT,
|
||||
updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
PRIMARY KEY (server_slug, mode)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_player_event_raw_server_match
|
||||
ON player_event_raw_ledger(server_slug, external_match_id);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_player_event_raw_occurred_at
|
||||
ON player_event_raw_ledger(occurred_at DESC);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_player_event_raw_killer_victim
|
||||
ON player_event_raw_ledger(killer_player_key, victim_player_key);
|
||||
"""
|
||||
)
|
||||
|
||||
return resolved_path
|
||||
|
||||
|
||||
def upsert_player_events(
|
||||
events: Iterable[PlayerEventRecord],
|
||||
*,
|
||||
db_path: Path | None = None,
|
||||
) -> dict[str, int]:
|
||||
"""Insert normalized events idempotently into the raw ledger."""
|
||||
if use_postgres_rcon_storage(explicit_sqlite_path=db_path):
|
||||
from .postgres_display_storage import upsert_player_event_rows
|
||||
|
||||
return upsert_player_event_rows(events)
|
||||
resolved_path = initialize_player_event_storage(db_path=db_path)
|
||||
inserted = 0
|
||||
duplicates = 0
|
||||
with _connect(resolved_path) as connection:
|
||||
for event in events:
|
||||
cursor = connection.execute(
|
||||
"""
|
||||
INSERT OR IGNORE INTO player_event_raw_ledger (
|
||||
event_id,
|
||||
event_type,
|
||||
occurred_at,
|
||||
server_slug,
|
||||
external_match_id,
|
||||
source_kind,
|
||||
source_ref,
|
||||
raw_event_ref,
|
||||
killer_player_key,
|
||||
killer_display_name,
|
||||
victim_player_key,
|
||||
victim_display_name,
|
||||
weapon_name,
|
||||
weapon_category,
|
||||
kill_category,
|
||||
is_teamkill,
|
||||
event_value
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
""",
|
||||
(
|
||||
event.event_id,
|
||||
event.event_type,
|
||||
event.occurred_at,
|
||||
event.server_slug,
|
||||
event.external_match_id,
|
||||
event.source_kind,
|
||||
event.source_ref,
|
||||
event.raw_event_ref,
|
||||
event.killer_player_key,
|
||||
event.killer_display_name,
|
||||
event.victim_player_key,
|
||||
event.victim_display_name,
|
||||
event.weapon_name,
|
||||
event.weapon_category,
|
||||
event.kill_category,
|
||||
1 if event.is_teamkill else 0,
|
||||
max(1, int(event.event_value)),
|
||||
),
|
||||
)
|
||||
if int(cursor.rowcount or 0) > 0:
|
||||
inserted += 1
|
||||
else:
|
||||
duplicates += 1
|
||||
return {
|
||||
"events_inserted": inserted,
|
||||
"duplicate_events": duplicates,
|
||||
}
|
||||
|
||||
|
||||
def start_player_event_ingestion_run(
|
||||
*,
|
||||
mode: str,
|
||||
target_server_slug: str | None = None,
|
||||
db_path: Path | None = None,
|
||||
) -> int:
|
||||
"""Persist one player event ingestion attempt."""
|
||||
resolved_path = initialize_player_event_storage(db_path=db_path)
|
||||
with _connect(resolved_path) as connection:
|
||||
cursor = connection.execute(
|
||||
"""
|
||||
INSERT INTO player_event_ingestion_runs (
|
||||
mode,
|
||||
status,
|
||||
target_server_slug,
|
||||
started_at
|
||||
) VALUES (?, 'running', ?, ?)
|
||||
""",
|
||||
(mode, target_server_slug, _utc_now_iso()),
|
||||
)
|
||||
return int(cursor.lastrowid)
|
||||
|
||||
|
||||
def finalize_player_event_ingestion_run(
|
||||
run_id: int,
|
||||
*,
|
||||
status: str,
|
||||
pages_processed: int,
|
||||
matches_seen: int,
|
||||
matches_fetched: int,
|
||||
events_inserted: int,
|
||||
duplicate_events: int,
|
||||
notes: str | None = None,
|
||||
db_path: Path | None = None,
|
||||
) -> None:
|
||||
"""Update one player event ingestion attempt with final counters."""
|
||||
resolved_path = initialize_player_event_storage(db_path=db_path)
|
||||
with _connect(resolved_path) as connection:
|
||||
connection.execute(
|
||||
"""
|
||||
UPDATE player_event_ingestion_runs
|
||||
SET status = ?,
|
||||
completed_at = ?,
|
||||
pages_processed = ?,
|
||||
matches_seen = ?,
|
||||
matches_fetched = ?,
|
||||
events_inserted = ?,
|
||||
duplicate_events = ?,
|
||||
notes = ?
|
||||
WHERE id = ?
|
||||
""",
|
||||
(
|
||||
status,
|
||||
_utc_now_iso(),
|
||||
pages_processed,
|
||||
matches_seen,
|
||||
matches_fetched,
|
||||
events_inserted,
|
||||
duplicate_events,
|
||||
notes,
|
||||
run_id,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def mark_player_event_progress_started(
|
||||
*,
|
||||
server_slug: str,
|
||||
mode: str,
|
||||
run_id: int,
|
||||
cutoff_occurred_at: str | None,
|
||||
db_path: Path | None = None,
|
||||
) -> None:
|
||||
"""Persist the start state for one server ingestion attempt."""
|
||||
resolved_path = initialize_player_event_storage(db_path=db_path)
|
||||
with _connect(resolved_path) as connection:
|
||||
connection.execute(
|
||||
"""
|
||||
INSERT INTO player_event_backfill_progress (
|
||||
server_slug,
|
||||
mode,
|
||||
next_page,
|
||||
cutoff_occurred_at,
|
||||
archive_exhausted,
|
||||
last_run_id,
|
||||
last_run_status,
|
||||
last_run_started_at,
|
||||
last_run_completed_at,
|
||||
last_error
|
||||
) VALUES (?, ?, 1, ?, 0, ?, 'running', ?, NULL, NULL)
|
||||
ON CONFLICT(server_slug, mode) DO UPDATE SET
|
||||
cutoff_occurred_at = excluded.cutoff_occurred_at,
|
||||
last_run_id = excluded.last_run_id,
|
||||
last_run_status = excluded.last_run_status,
|
||||
last_run_started_at = excluded.last_run_started_at,
|
||||
last_run_completed_at = NULL,
|
||||
last_error = NULL,
|
||||
updated_at = CURRENT_TIMESTAMP
|
||||
""",
|
||||
(server_slug, mode, cutoff_occurred_at, run_id, _utc_now_iso()),
|
||||
)
|
||||
|
||||
|
||||
def mark_player_event_progress_page_completed(
|
||||
*,
|
||||
server_slug: str,
|
||||
mode: str,
|
||||
page_number: int,
|
||||
discovered_total_matches: int | None,
|
||||
run_id: int,
|
||||
db_path: Path | None = None,
|
||||
) -> None:
|
||||
"""Advance the resume checkpoint after one page completes successfully."""
|
||||
resolved_path = initialize_player_event_storage(db_path=db_path)
|
||||
with _connect(resolved_path) as connection:
|
||||
connection.execute(
|
||||
"""
|
||||
INSERT INTO player_event_backfill_progress (
|
||||
server_slug,
|
||||
mode,
|
||||
next_page,
|
||||
last_completed_page,
|
||||
discovered_total_matches,
|
||||
archive_exhausted,
|
||||
last_run_id,
|
||||
last_run_status,
|
||||
last_run_started_at,
|
||||
last_run_completed_at,
|
||||
last_error
|
||||
) VALUES (?, ?, ?, ?, ?, 0, ?, 'running', ?, NULL, NULL)
|
||||
ON CONFLICT(server_slug, mode) DO UPDATE SET
|
||||
next_page = excluded.next_page,
|
||||
last_completed_page = excluded.last_completed_page,
|
||||
discovered_total_matches = COALESCE(
|
||||
excluded.discovered_total_matches,
|
||||
player_event_backfill_progress.discovered_total_matches
|
||||
),
|
||||
archive_exhausted = 0,
|
||||
last_run_id = excluded.last_run_id,
|
||||
last_run_status = excluded.last_run_status,
|
||||
last_run_started_at = excluded.last_run_started_at,
|
||||
last_run_completed_at = NULL,
|
||||
last_error = NULL,
|
||||
updated_at = CURRENT_TIMESTAMP
|
||||
""",
|
||||
(
|
||||
server_slug,
|
||||
mode,
|
||||
page_number + 1,
|
||||
page_number,
|
||||
discovered_total_matches,
|
||||
run_id,
|
||||
_utc_now_iso(),
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def finalize_player_event_progress(
|
||||
*,
|
||||
server_slug: str,
|
||||
mode: str,
|
||||
run_id: int,
|
||||
status: str,
|
||||
archive_exhausted: bool = False,
|
||||
error_message: str | None = None,
|
||||
db_path: Path | None = None,
|
||||
) -> None:
|
||||
"""Persist the final state of one server event ingestion attempt."""
|
||||
resolved_path = initialize_player_event_storage(db_path=db_path)
|
||||
with _connect(resolved_path) as connection:
|
||||
connection.execute(
|
||||
"""
|
||||
INSERT INTO player_event_backfill_progress (
|
||||
server_slug,
|
||||
mode,
|
||||
next_page,
|
||||
archive_exhausted,
|
||||
last_run_id,
|
||||
last_run_status,
|
||||
last_run_started_at,
|
||||
last_run_completed_at,
|
||||
last_error
|
||||
) VALUES (?, ?, 1, ?, ?, ?, ?, ?, ?)
|
||||
ON CONFLICT(server_slug, mode) DO UPDATE SET
|
||||
archive_exhausted = CASE
|
||||
WHEN excluded.last_run_status = 'success' AND excluded.archive_exhausted = 1
|
||||
THEN 1
|
||||
ELSE player_event_backfill_progress.archive_exhausted
|
||||
END,
|
||||
last_run_id = excluded.last_run_id,
|
||||
last_run_status = excluded.last_run_status,
|
||||
last_run_started_at = COALESCE(
|
||||
player_event_backfill_progress.last_run_started_at,
|
||||
excluded.last_run_started_at
|
||||
),
|
||||
last_run_completed_at = excluded.last_run_completed_at,
|
||||
last_error = excluded.last_error,
|
||||
updated_at = CURRENT_TIMESTAMP
|
||||
""",
|
||||
(
|
||||
server_slug,
|
||||
mode,
|
||||
1 if archive_exhausted else 0,
|
||||
run_id,
|
||||
status,
|
||||
_utc_now_iso(),
|
||||
_utc_now_iso(),
|
||||
error_message,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def get_player_event_resume_page(
|
||||
server_slug: str,
|
||||
*,
|
||||
mode: str = "bootstrap",
|
||||
db_path: Path | None = None,
|
||||
) -> int:
|
||||
"""Return the saved resume page for a bootstrap-like event backfill."""
|
||||
resolved_path = initialize_player_event_storage(db_path=db_path)
|
||||
with _connect(resolved_path) as connection:
|
||||
row = connection.execute(
|
||||
"""
|
||||
SELECT next_page
|
||||
FROM player_event_backfill_progress
|
||||
WHERE server_slug = ? AND mode = ?
|
||||
""",
|
||||
(server_slug, mode),
|
||||
).fetchone()
|
||||
return max(1, int(row["next_page"])) if row and row["next_page"] else 1
|
||||
|
||||
|
||||
def get_player_event_refresh_cutoff_for_server(
|
||||
server_slug: str,
|
||||
*,
|
||||
overlap_hours: int | None = None,
|
||||
db_path: Path | None = None,
|
||||
) -> str | None:
|
||||
"""Return the latest occurred_at already persisted for one server."""
|
||||
resolved_overlap_hours = (
|
||||
get_player_event_refresh_overlap_hours()
|
||||
if overlap_hours is None
|
||||
else overlap_hours
|
||||
)
|
||||
if resolved_overlap_hours < 0:
|
||||
raise ValueError("overlap_hours must be zero or positive.")
|
||||
resolved_path = initialize_player_event_storage(db_path=db_path)
|
||||
with _connect(resolved_path) as connection:
|
||||
row = connection.execute(
|
||||
"""
|
||||
SELECT MAX(occurred_at) AS latest_occurred_at
|
||||
FROM player_event_raw_ledger
|
||||
WHERE server_slug = ?
|
||||
""",
|
||||
(server_slug,),
|
||||
).fetchone()
|
||||
latest_occurred_at = str(row["latest_occurred_at"]) if row and row["latest_occurred_at"] else None
|
||||
if not latest_occurred_at:
|
||||
return None
|
||||
|
||||
cutoff = _parse_timestamp(latest_occurred_at) - timedelta(hours=resolved_overlap_hours)
|
||||
return cutoff.astimezone(timezone.utc).isoformat().replace("+00:00", "Z")
|
||||
|
||||
|
||||
def _connect(db_path: Path) -> sqlite3.Connection:
|
||||
return connect_sqlite_writer(db_path)
|
||||
|
||||
|
||||
def _utc_now_iso() -> str:
|
||||
return datetime.now(timezone.utc).isoformat().replace("+00:00", "Z")
|
||||
|
||||
|
||||
def _parse_timestamp(value: str) -> datetime:
|
||||
normalized = value.strip().replace("Z", "+00:00")
|
||||
parsed = datetime.fromisoformat(normalized)
|
||||
if parsed.tzinfo is None:
|
||||
parsed = parsed.replace(tzinfo=timezone.utc)
|
||||
return parsed
|
||||
490
backend/app/player_event_worker.py
Normal file
490
backend/app/player_event_worker.py
Normal file
@@ -0,0 +1,490 @@
|
||||
"""Incremental worker for the V2 player event ingestion pipeline."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from typing import Iterable
|
||||
|
||||
from .config import (
|
||||
get_historical_crcon_detail_workers,
|
||||
get_historical_crcon_page_size,
|
||||
get_player_event_refresh_interval_seconds,
|
||||
get_player_event_refresh_max_retries,
|
||||
get_player_event_refresh_overlap_hours,
|
||||
get_player_event_refresh_retry_delay_seconds,
|
||||
)
|
||||
from .data_sources import resolve_historical_ingestion_data_source
|
||||
from .historical_storage import list_historical_servers
|
||||
from .player_event_source import resolve_player_event_source
|
||||
from .player_event_storage import (
|
||||
finalize_player_event_ingestion_run,
|
||||
finalize_player_event_progress,
|
||||
get_player_event_refresh_cutoff_for_server,
|
||||
get_player_event_resume_page,
|
||||
initialize_player_event_storage,
|
||||
mark_player_event_progress_page_completed,
|
||||
mark_player_event_progress_started,
|
||||
start_player_event_ingestion_run,
|
||||
upsert_player_events,
|
||||
)
|
||||
from .writer_lock import backend_writer_lock, build_writer_lock_holder
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class PlayerEventIngestionStats:
|
||||
pages_processed: int = 0
|
||||
matches_seen: int = 0
|
||||
matches_fetched: int = 0
|
||||
events_inserted: int = 0
|
||||
duplicate_events: int = 0
|
||||
|
||||
def apply(self, delta: dict[str, int]) -> None:
|
||||
self.events_inserted += int(delta.get("events_inserted", 0))
|
||||
self.duplicate_events += int(delta.get("duplicate_events", 0))
|
||||
|
||||
|
||||
def run_player_event_refresh(
|
||||
*,
|
||||
server_slug: str | None = None,
|
||||
max_pages: int | None = None,
|
||||
page_size: int | None = None,
|
||||
start_page: int | None = None,
|
||||
detail_workers: int | None = None,
|
||||
overlap_hours: int | None = None,
|
||||
) -> dict[str, object]:
|
||||
"""Refresh recent player event summaries from the configured historical source."""
|
||||
with backend_writer_lock(
|
||||
holder=build_writer_lock_holder(
|
||||
f"app.player_event_worker refresh:{server_slug or 'all-servers'}"
|
||||
)
|
||||
):
|
||||
initialize_player_event_storage()
|
||||
data_source, data_source_policy = resolve_historical_ingestion_data_source()
|
||||
event_source_selection = resolve_player_event_source()
|
||||
event_source = event_source_selection.source
|
||||
resolved_page_size = page_size or get_historical_crcon_page_size()
|
||||
resolved_detail_workers = detail_workers or get_historical_crcon_detail_workers()
|
||||
resolved_overlap_hours = (
|
||||
get_player_event_refresh_overlap_hours()
|
||||
if overlap_hours is None
|
||||
else overlap_hours
|
||||
)
|
||||
if resolved_overlap_hours < 0:
|
||||
raise ValueError("--overlap-hours must be zero or positive.")
|
||||
selected_servers = _select_servers(server_slug)
|
||||
processed_servers: list[dict[str, object]] = []
|
||||
active_runs: dict[str, int] = {}
|
||||
|
||||
try:
|
||||
for server in selected_servers:
|
||||
current_server_slug = str(server["slug"])
|
||||
run_id = start_player_event_ingestion_run(
|
||||
mode="refresh",
|
||||
target_server_slug=current_server_slug,
|
||||
)
|
||||
active_runs[current_server_slug] = run_id
|
||||
cutoff = get_player_event_refresh_cutoff_for_server(
|
||||
current_server_slug,
|
||||
overlap_hours=resolved_overlap_hours,
|
||||
)
|
||||
mark_player_event_progress_started(
|
||||
server_slug=current_server_slug,
|
||||
mode="refresh",
|
||||
run_id=run_id,
|
||||
cutoff_occurred_at=cutoff,
|
||||
)
|
||||
server_stats = _ingest_server(
|
||||
server=server,
|
||||
run_id=run_id,
|
||||
data_source=data_source,
|
||||
event_source=event_source,
|
||||
page_size=resolved_page_size,
|
||||
max_pages=max_pages,
|
||||
start_page=_resolve_start_page(
|
||||
server_slug=current_server_slug,
|
||||
start_page=start_page,
|
||||
),
|
||||
detail_workers=resolved_detail_workers,
|
||||
cutoff=cutoff,
|
||||
)
|
||||
finalize_player_event_ingestion_run(
|
||||
run_id,
|
||||
status="success",
|
||||
pages_processed=server_stats["pages_processed"],
|
||||
matches_seen=server_stats["matches_seen"],
|
||||
matches_fetched=server_stats["matches_fetched"],
|
||||
events_inserted=server_stats["events_inserted"],
|
||||
duplicate_events=server_stats["duplicate_events"],
|
||||
notes=f"source={data_source.source_kind};adapter={event_source.source_kind}",
|
||||
)
|
||||
finalize_player_event_progress(
|
||||
server_slug=current_server_slug,
|
||||
mode="refresh",
|
||||
run_id=run_id,
|
||||
status="success",
|
||||
archive_exhausted=bool(server_stats["archive_exhausted"]),
|
||||
)
|
||||
processed_servers.append(server_stats)
|
||||
active_runs.pop(current_server_slug, None)
|
||||
except Exception as exc:
|
||||
for active_server_slug, run_id in active_runs.items():
|
||||
finalize_player_event_ingestion_run(
|
||||
run_id,
|
||||
status="failed",
|
||||
pages_processed=0,
|
||||
matches_seen=0,
|
||||
matches_fetched=0,
|
||||
events_inserted=0,
|
||||
duplicate_events=0,
|
||||
notes=str(exc),
|
||||
)
|
||||
finalize_player_event_progress(
|
||||
server_slug=active_server_slug,
|
||||
mode="refresh",
|
||||
run_id=run_id,
|
||||
status="failed",
|
||||
error_message=str(exc),
|
||||
)
|
||||
raise
|
||||
|
||||
return {
|
||||
"status": "ok",
|
||||
"mode": "refresh",
|
||||
"source_provider": data_source.source_kind,
|
||||
"source_policy": data_source_policy,
|
||||
"event_adapter": event_source.source_kind,
|
||||
"event_source_policy": event_source_selection.source_policy,
|
||||
"page_size": resolved_page_size,
|
||||
"detail_workers": resolved_detail_workers,
|
||||
"overlap_hours": resolved_overlap_hours,
|
||||
"scope": event_source.describe_scope(),
|
||||
"servers": processed_servers,
|
||||
}
|
||||
|
||||
|
||||
def run_periodic_player_event_refresh(
|
||||
*,
|
||||
interval_seconds: int,
|
||||
max_retries: int,
|
||||
retry_delay_seconds: int,
|
||||
server_slug: str | None = None,
|
||||
max_pages: int | None = None,
|
||||
page_size: int | None = None,
|
||||
detail_workers: int | None = None,
|
||||
max_runs: int | None = None,
|
||||
) -> None:
|
||||
"""Run the refresh worker repeatedly with bounded retries."""
|
||||
completed_runs = 0
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"event": "player-event-refresh-loop-started",
|
||||
"interval_seconds": interval_seconds,
|
||||
"max_retries": max_retries,
|
||||
"retry_delay_seconds": retry_delay_seconds,
|
||||
"server_scope": [server_slug] if server_slug else [server["slug"] for server in list_historical_servers()],
|
||||
},
|
||||
indent=2,
|
||||
)
|
||||
)
|
||||
print("Press Ctrl+C to stop.")
|
||||
|
||||
try:
|
||||
while max_runs is None or completed_runs < max_runs:
|
||||
completed_runs += 1
|
||||
payload = _run_refresh_with_retries(
|
||||
max_retries=max_retries,
|
||||
retry_delay_seconds=retry_delay_seconds,
|
||||
server_slug=server_slug,
|
||||
max_pages=max_pages,
|
||||
page_size=page_size,
|
||||
detail_workers=detail_workers,
|
||||
)
|
||||
print(json.dumps({"run": completed_runs, **payload}, indent=2))
|
||||
if max_runs is not None and completed_runs >= max_runs:
|
||||
break
|
||||
time.sleep(interval_seconds)
|
||||
except KeyboardInterrupt:
|
||||
print("\nPlayer event refresh loop stopped by user.")
|
||||
|
||||
|
||||
def _run_refresh_with_retries(
|
||||
*,
|
||||
max_retries: int,
|
||||
retry_delay_seconds: int,
|
||||
server_slug: str | None,
|
||||
max_pages: int | None,
|
||||
page_size: int | None,
|
||||
detail_workers: int | None,
|
||||
) -> dict[str, object]:
|
||||
attempt = 0
|
||||
while True:
|
||||
attempt += 1
|
||||
try:
|
||||
return {
|
||||
"status": "ok",
|
||||
"attempts_used": attempt,
|
||||
"refresh_result": run_player_event_refresh(
|
||||
server_slug=server_slug,
|
||||
max_pages=max_pages,
|
||||
page_size=page_size,
|
||||
detail_workers=detail_workers,
|
||||
),
|
||||
}
|
||||
except Exception as exc:
|
||||
if attempt > max_retries:
|
||||
return {
|
||||
"status": "error",
|
||||
"attempts_used": attempt,
|
||||
"error": str(exc),
|
||||
}
|
||||
if retry_delay_seconds > 0:
|
||||
time.sleep(retry_delay_seconds)
|
||||
|
||||
|
||||
def _ingest_server(
|
||||
*,
|
||||
server: dict[str, object],
|
||||
run_id: int,
|
||||
data_source: object,
|
||||
event_source: object,
|
||||
page_size: int,
|
||||
max_pages: int | None,
|
||||
start_page: int,
|
||||
detail_workers: int,
|
||||
cutoff: str | None,
|
||||
) -> dict[str, object]:
|
||||
page_limit = max_pages or 1000000
|
||||
local_stats = PlayerEventIngestionStats()
|
||||
discovered_total_matches: int | None = None
|
||||
archive_exhausted = False
|
||||
|
||||
for page_number in range(start_page, start_page + page_limit):
|
||||
payload = data_source.fetch_match_page(
|
||||
base_url=str(server["scoreboard_base_url"]),
|
||||
page=page_number,
|
||||
limit=page_size,
|
||||
)
|
||||
if discovered_total_matches is None:
|
||||
discovered_total_matches = _coerce_int(payload.get("total"))
|
||||
page_matches = _coerce_match_list(payload.get("maps"))
|
||||
if not page_matches:
|
||||
archive_exhausted = True
|
||||
break
|
||||
|
||||
local_stats.pages_processed += 1
|
||||
stop_after_page = False
|
||||
match_ids_to_fetch: list[str] = []
|
||||
|
||||
for match_summary in page_matches:
|
||||
local_stats.matches_seen += 1
|
||||
reference_timestamp = _pick_match_timestamp(match_summary)
|
||||
if cutoff and reference_timestamp and reference_timestamp < cutoff:
|
||||
stop_after_page = True
|
||||
continue
|
||||
|
||||
match_id = _stringify(match_summary.get("id"))
|
||||
if match_id:
|
||||
match_ids_to_fetch.append(match_id)
|
||||
|
||||
detail_payloads = data_source.fetch_match_details(
|
||||
base_url=str(server["scoreboard_base_url"]),
|
||||
match_ids=match_ids_to_fetch,
|
||||
max_workers=detail_workers,
|
||||
)
|
||||
local_stats.matches_fetched += len(detail_payloads)
|
||||
for detail_payload in detail_payloads:
|
||||
match_id = _stringify(detail_payload.get("id")) or "unknown"
|
||||
source_ref = (
|
||||
f"{server['scoreboard_base_url']}/api/get_map_scoreboard?map_id={match_id}"
|
||||
)
|
||||
normalized_events = event_source.extract_match_events(
|
||||
server_slug=str(server["slug"]),
|
||||
match_payload=detail_payload,
|
||||
source_ref=source_ref,
|
||||
)
|
||||
local_stats.apply(upsert_player_events(normalized_events))
|
||||
|
||||
mark_player_event_progress_page_completed(
|
||||
server_slug=str(server["slug"]),
|
||||
mode="refresh",
|
||||
page_number=page_number,
|
||||
discovered_total_matches=discovered_total_matches,
|
||||
run_id=run_id,
|
||||
)
|
||||
|
||||
if stop_after_page:
|
||||
break
|
||||
|
||||
return {
|
||||
"server_slug": server["slug"],
|
||||
"source_provider": data_source.source_kind,
|
||||
"event_adapter": event_source.source_kind,
|
||||
"pages_processed": local_stats.pages_processed,
|
||||
"matches_seen": local_stats.matches_seen,
|
||||
"matches_fetched": local_stats.matches_fetched,
|
||||
"events_inserted": local_stats.events_inserted,
|
||||
"duplicate_events": local_stats.duplicate_events,
|
||||
"cutoff": cutoff,
|
||||
"archive_exhausted": archive_exhausted,
|
||||
"discovered_total_matches": discovered_total_matches,
|
||||
}
|
||||
|
||||
|
||||
def _resolve_start_page(*, server_slug: str, start_page: int | None) -> int:
|
||||
if start_page is not None:
|
||||
return max(1, start_page)
|
||||
return get_player_event_resume_page(server_slug, mode="refresh")
|
||||
|
||||
|
||||
def _select_servers(server_slug: str | None) -> list[dict[str, object]]:
|
||||
servers = list_historical_servers()
|
||||
if server_slug is None:
|
||||
return servers
|
||||
normalized = server_slug.strip()
|
||||
selected = [server for server in servers if server["slug"] == normalized]
|
||||
if not selected:
|
||||
raise ValueError(f"Unknown historical server slug: {server_slug}")
|
||||
return selected
|
||||
|
||||
|
||||
def _coerce_match_list(payload: object) -> list[dict[str, object]]:
|
||||
if not isinstance(payload, list):
|
||||
return []
|
||||
return [item for item in payload if isinstance(item, dict)]
|
||||
|
||||
|
||||
def _pick_match_timestamp(match_payload: dict[str, object]) -> str | None:
|
||||
for key in ("end", "start", "creation_time"):
|
||||
value = match_payload.get(key)
|
||||
if isinstance(value, str) and value.strip():
|
||||
return value.strip()
|
||||
return None
|
||||
|
||||
|
||||
def _stringify(value: object) -> str | None:
|
||||
if value is None:
|
||||
return None
|
||||
text = str(value).strip()
|
||||
return text or None
|
||||
|
||||
|
||||
def _coerce_int(value: object) -> int | None:
|
||||
if value in (None, ""):
|
||||
return None
|
||||
try:
|
||||
return int(value)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
def build_arg_parser() -> argparse.ArgumentParser:
|
||||
"""Create the CLI parser for manual or periodic player event ingestion."""
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Player event refresh worker for HLL Vietnam.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"mode",
|
||||
choices=("refresh", "loop"),
|
||||
help="refresh runs once; loop keeps the worker running periodically",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--server",
|
||||
dest="server_slug",
|
||||
help="optional historical server slug",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--max-pages",
|
||||
type=int,
|
||||
help="optional page cap for local validation",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--page-size",
|
||||
type=int,
|
||||
help="override CRCON page size",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--start-page",
|
||||
type=int,
|
||||
help="override the saved resume page",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--detail-workers",
|
||||
type=int,
|
||||
help="parallel worker count for per-match detail requests",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--overlap-hours",
|
||||
type=int,
|
||||
help="override the incremental overlap window in hours",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--interval",
|
||||
type=int,
|
||||
default=get_player_event_refresh_interval_seconds(),
|
||||
help="seconds to wait between loop runs",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--retries",
|
||||
type=int,
|
||||
default=get_player_event_refresh_max_retries(),
|
||||
help="retry attempts after a failed refresh",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--retry-delay",
|
||||
type=int,
|
||||
default=get_player_event_refresh_retry_delay_seconds(),
|
||||
help="seconds to wait between failed attempts",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--max-runs",
|
||||
type=int,
|
||||
help="optional safety cap for loop mode",
|
||||
)
|
||||
return parser
|
||||
|
||||
|
||||
def main(argv: Iterable[str] | None = None) -> int:
|
||||
"""Run the player event worker CLI."""
|
||||
parser = build_arg_parser()
|
||||
args = parser.parse_args(list(argv) if argv is not None else None)
|
||||
|
||||
if args.mode == "refresh":
|
||||
result = run_player_event_refresh(
|
||||
server_slug=args.server_slug,
|
||||
max_pages=args.max_pages,
|
||||
page_size=args.page_size,
|
||||
start_page=args.start_page,
|
||||
detail_workers=args.detail_workers,
|
||||
overlap_hours=args.overlap_hours,
|
||||
)
|
||||
print(json.dumps(result, indent=2))
|
||||
return 0
|
||||
|
||||
if args.interval <= 0:
|
||||
raise ValueError("--interval must be a positive integer.")
|
||||
if args.retries < 0:
|
||||
raise ValueError("--retries must be zero or positive.")
|
||||
if args.retry_delay < 0:
|
||||
raise ValueError("--retry-delay must be zero or positive.")
|
||||
if args.max_runs is not None and args.max_runs <= 0:
|
||||
raise ValueError("--max-runs must be positive when provided.")
|
||||
|
||||
run_periodic_player_event_refresh(
|
||||
interval_seconds=args.interval,
|
||||
max_retries=args.retries,
|
||||
retry_delay_seconds=args.retry_delay,
|
||||
server_slug=args.server_slug,
|
||||
max_pages=args.max_pages,
|
||||
page_size=args.page_size,
|
||||
detail_workers=args.detail_workers,
|
||||
max_runs=args.max_runs,
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
65
backend/app/player_external_profiles.py
Normal file
65
backend/app/player_external_profiles.py
Normal file
@@ -0,0 +1,65 @@
|
||||
"""Safe external profile fields derived from captured player identifiers."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
|
||||
_STEAM_ID64_RE = re.compile(r"^\d{17}$")
|
||||
_EPIC_ID_RE = re.compile(r"^[0-9a-f]{32}$", re.IGNORECASE)
|
||||
|
||||
|
||||
def build_external_player_profile_fields(
|
||||
*,
|
||||
player_id: object = None,
|
||||
steam_id: object = None,
|
||||
) -> dict[str, object]:
|
||||
"""Expose external profile links only when a captured identifier is safe."""
|
||||
|
||||
steam_id_64 = normalize_steam_id_64(steam_id) or normalize_steam_id_64(player_id)
|
||||
if steam_id_64:
|
||||
return {
|
||||
"steam_id_64": steam_id_64,
|
||||
"platform": "steam",
|
||||
"external_profile_links": {
|
||||
"steam": f"https://steamcommunity.com/profiles/{steam_id_64}",
|
||||
"hellor": f"https://hellor.pro/player/{steam_id_64}",
|
||||
"hll_records": f"https://hllrecords.com/profiles/{steam_id_64}",
|
||||
"helo": f"https://helo-system.de/statistics/players/{steam_id_64}?series=2024",
|
||||
},
|
||||
}
|
||||
|
||||
epic_id = normalize_epic_id(player_id)
|
||||
if epic_id:
|
||||
return {
|
||||
"epic_id": epic_id,
|
||||
"platform": "epic",
|
||||
"external_profile_links": {
|
||||
"hellor": f"https://hellor.pro/player/{epic_id}",
|
||||
"hll_records": f"https://hllrecords.com/profiles/{epic_id}",
|
||||
},
|
||||
}
|
||||
|
||||
return {
|
||||
"platform": infer_player_platform(player_id=player_id, steam_id=steam_id),
|
||||
"external_profile_links": {},
|
||||
}
|
||||
|
||||
|
||||
def normalize_steam_id_64(value: object) -> str | None:
|
||||
normalized = str(value or "").strip()
|
||||
return normalized if _STEAM_ID64_RE.fullmatch(normalized) else None
|
||||
|
||||
|
||||
def normalize_epic_id(value: object) -> str | None:
|
||||
normalized = str(value or "").strip()
|
||||
return normalized.lower() if _EPIC_ID_RE.fullmatch(normalized) else None
|
||||
|
||||
|
||||
def infer_player_platform(*, player_id: object = None, steam_id: object = None) -> str:
|
||||
normalized_player_id = str(player_id or "").strip()
|
||||
if normalize_steam_id_64(steam_id) or normalize_steam_id_64(normalized_player_id):
|
||||
return "steam"
|
||||
if normalize_epic_id(normalized_player_id):
|
||||
return "epic"
|
||||
return "unknown"
|
||||
929
backend/app/postgres_display_storage.py
Normal file
929
backend/app/postgres_display_storage.py
Normal file
@@ -0,0 +1,929 @@
|
||||
"""PostgreSQL read/write storage for data displayed outside the RCON write path."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from contextlib import contextmanager
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any, Iterable, Mapping
|
||||
|
||||
from .config import get_database_url, get_historical_weekly_fallback_max_weekday
|
||||
from .historical_models import HistoricalSnapshotRecord
|
||||
from .player_external_profiles import build_external_player_profile_fields
|
||||
from .scoreboard_origins import resolve_trusted_scoreboard_match_url
|
||||
|
||||
|
||||
ALL_SERVERS_SLUG = "all-servers"
|
||||
ALL_SERVERS_DISPLAY_NAME = "Todos"
|
||||
SUMMARY_SNAPSHOT_LIMIT = 6
|
||||
|
||||
|
||||
DISPLAY_SCHEMA_SQL = """
|
||||
CREATE TABLE IF NOT EXISTS game_sources (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
slug TEXT NOT NULL UNIQUE,
|
||||
display_name TEXT NOT NULL,
|
||||
provider_kind TEXT NOT NULL,
|
||||
is_active BOOLEAN NOT NULL DEFAULT TRUE,
|
||||
created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS servers (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
game_source_id BIGINT NOT NULL REFERENCES game_sources(id),
|
||||
external_server_id TEXT,
|
||||
server_name TEXT NOT NULL,
|
||||
region TEXT,
|
||||
first_seen_at TEXT NOT NULL,
|
||||
last_seen_at TEXT NOT NULL,
|
||||
created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
UNIQUE(game_source_id, external_server_id)
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS server_snapshots (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
server_id BIGINT NOT NULL REFERENCES servers(id),
|
||||
captured_at TEXT NOT NULL,
|
||||
status TEXT NOT NULL,
|
||||
players INTEGER,
|
||||
max_players INTEGER,
|
||||
current_map TEXT,
|
||||
source_name TEXT NOT NULL,
|
||||
snapshot_origin TEXT,
|
||||
source_ref TEXT,
|
||||
raw_payload_ref TEXT,
|
||||
created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
UNIQUE(server_id, captured_at, source_name, source_ref)
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_pg_server_snapshots_server_time
|
||||
ON server_snapshots(server_id, captured_at DESC);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS historical_servers (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
slug TEXT NOT NULL UNIQUE,
|
||||
display_name TEXT NOT NULL,
|
||||
scoreboard_base_url TEXT NOT NULL UNIQUE,
|
||||
server_number INTEGER,
|
||||
source_kind TEXT NOT NULL,
|
||||
created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS historical_maps (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
external_map_id TEXT UNIQUE,
|
||||
map_name TEXT,
|
||||
pretty_name TEXT,
|
||||
game_mode TEXT,
|
||||
image_name TEXT,
|
||||
created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS historical_matches (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
historical_server_id BIGINT NOT NULL REFERENCES historical_servers(id),
|
||||
external_match_id TEXT NOT NULL,
|
||||
historical_map_id BIGINT REFERENCES historical_maps(id),
|
||||
created_at_source TEXT,
|
||||
started_at TEXT,
|
||||
ended_at TEXT,
|
||||
map_name TEXT,
|
||||
map_pretty_name TEXT,
|
||||
game_mode TEXT,
|
||||
image_name TEXT,
|
||||
allied_score INTEGER,
|
||||
axis_score INTEGER,
|
||||
last_seen_at TEXT NOT NULL,
|
||||
raw_payload_ref TEXT,
|
||||
created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
UNIQUE(historical_server_id, external_match_id)
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS historical_players (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
stable_player_key TEXT NOT NULL UNIQUE,
|
||||
display_name TEXT NOT NULL,
|
||||
steam_id TEXT,
|
||||
source_player_id TEXT,
|
||||
first_seen_at TEXT NOT NULL,
|
||||
last_seen_at TEXT NOT NULL,
|
||||
created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS historical_player_match_stats (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
historical_match_id BIGINT NOT NULL REFERENCES historical_matches(id),
|
||||
historical_player_id BIGINT NOT NULL REFERENCES historical_players(id),
|
||||
match_player_ref TEXT,
|
||||
team_side TEXT,
|
||||
level INTEGER,
|
||||
kills INTEGER,
|
||||
deaths INTEGER,
|
||||
teamkills INTEGER,
|
||||
time_seconds INTEGER,
|
||||
kills_per_minute DOUBLE PRECISION,
|
||||
deaths_per_minute DOUBLE PRECISION,
|
||||
kill_death_ratio DOUBLE PRECISION,
|
||||
combat INTEGER,
|
||||
offense INTEGER,
|
||||
defense INTEGER,
|
||||
support INTEGER,
|
||||
created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
UNIQUE(historical_match_id, historical_player_id)
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_pg_historical_matches_server_end
|
||||
ON historical_matches(historical_server_id, ended_at DESC, started_at DESC);
|
||||
CREATE INDEX IF NOT EXISTS idx_pg_historical_player_stats_match
|
||||
ON historical_player_match_stats(historical_match_id);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS displayed_historical_snapshots (
|
||||
server_key TEXT NOT NULL,
|
||||
snapshot_type TEXT NOT NULL,
|
||||
metric TEXT NOT NULL DEFAULT '',
|
||||
snapshot_window TEXT NOT NULL DEFAULT '',
|
||||
payload_json TEXT NOT NULL,
|
||||
generated_at TEXT NOT NULL,
|
||||
source_range_start TEXT,
|
||||
source_range_end TEXT,
|
||||
is_stale BOOLEAN NOT NULL DEFAULT FALSE,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
PRIMARY KEY(server_key, snapshot_type, metric, snapshot_window)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS player_event_raw_ledger (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
event_id TEXT NOT NULL UNIQUE,
|
||||
event_type TEXT NOT NULL,
|
||||
occurred_at TEXT,
|
||||
server_slug TEXT NOT NULL,
|
||||
external_match_id TEXT NOT NULL,
|
||||
source_kind TEXT NOT NULL,
|
||||
source_ref TEXT,
|
||||
raw_event_ref TEXT,
|
||||
killer_player_key TEXT,
|
||||
killer_display_name TEXT,
|
||||
victim_player_key TEXT,
|
||||
victim_display_name TEXT,
|
||||
weapon_name TEXT,
|
||||
weapon_category TEXT,
|
||||
kill_category TEXT,
|
||||
is_teamkill BOOLEAN NOT NULL DEFAULT FALSE,
|
||||
event_value INTEGER NOT NULL DEFAULT 1,
|
||||
inserted_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_pg_player_event_raw_occurred_at
|
||||
ON player_event_raw_ledger(occurred_at DESC);
|
||||
"""
|
||||
|
||||
|
||||
def initialize_postgres_display_storage() -> None:
|
||||
with connect_postgres() as connection:
|
||||
connection.execute(DISPLAY_SCHEMA_SQL)
|
||||
|
||||
|
||||
def connect_postgres():
|
||||
try:
|
||||
import psycopg
|
||||
from psycopg.rows import dict_row
|
||||
except ImportError as error: # pragma: no cover - environment-specific
|
||||
raise RuntimeError("psycopg is required when HLL_BACKEND_DATABASE_URL is set.") from error
|
||||
database_url = get_database_url()
|
||||
if not database_url:
|
||||
raise RuntimeError("HLL_BACKEND_DATABASE_URL is required for displayed PostgreSQL storage.")
|
||||
return psycopg.connect(database_url, row_factory=dict_row)
|
||||
|
||||
|
||||
class PostgresCompatConnection:
|
||||
"""Small placeholder shim for SQLite-shaped displayed read queries."""
|
||||
|
||||
def __init__(self, connection: Any):
|
||||
self.connection = connection
|
||||
|
||||
def execute(self, sql: str, params: Iterable[object] | None = None):
|
||||
return self.connection.execute(sql.replace("?", "%s"), tuple(params or ()))
|
||||
|
||||
|
||||
@contextmanager
|
||||
def connect_postgres_compat():
|
||||
initialize_postgres_display_storage()
|
||||
with connect_postgres() as connection:
|
||||
yield PostgresCompatConnection(connection)
|
||||
|
||||
|
||||
def persist_snapshot_record(snapshot: Mapping[str, object]) -> HistoricalSnapshotRecord:
|
||||
initialize_postgres_display_storage()
|
||||
generated_at = _iso(snapshot.get("generated_at")) or _utc_now_iso()
|
||||
metric = str(snapshot.get("metric") or "")
|
||||
window = str(snapshot.get("window") or "")
|
||||
payload = snapshot.get("payload")
|
||||
payload_json = json.dumps(
|
||||
payload,
|
||||
ensure_ascii=True,
|
||||
separators=(",", ":"),
|
||||
default=_json_payload_default,
|
||||
)
|
||||
with connect_postgres() as connection:
|
||||
connection.execute(
|
||||
"""
|
||||
INSERT INTO displayed_historical_snapshots (
|
||||
server_key, snapshot_type, metric, snapshot_window, payload_json, generated_at,
|
||||
source_range_start, source_range_end, is_stale
|
||||
) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s)
|
||||
ON CONFLICT(server_key, snapshot_type, metric, snapshot_window) DO UPDATE SET
|
||||
payload_json = EXCLUDED.payload_json,
|
||||
generated_at = EXCLUDED.generated_at,
|
||||
source_range_start = EXCLUDED.source_range_start,
|
||||
source_range_end = EXCLUDED.source_range_end,
|
||||
is_stale = EXCLUDED.is_stale,
|
||||
updated_at = CURRENT_TIMESTAMP
|
||||
""",
|
||||
(
|
||||
str(snapshot["server_key"]),
|
||||
str(snapshot["snapshot_type"]),
|
||||
metric,
|
||||
window,
|
||||
payload_json,
|
||||
generated_at,
|
||||
_iso(snapshot.get("source_range_start")),
|
||||
_iso(snapshot.get("source_range_end")),
|
||||
bool(snapshot.get("is_stale", False)),
|
||||
),
|
||||
)
|
||||
return HistoricalSnapshotRecord(
|
||||
server_key=str(snapshot["server_key"]),
|
||||
snapshot_type=str(snapshot["snapshot_type"]),
|
||||
metric=metric or None,
|
||||
window=window or None,
|
||||
payload_json=payload_json,
|
||||
generated_at=_parse_datetime(generated_at) or datetime.now(timezone.utc),
|
||||
source_range_start=_parse_datetime(_iso(snapshot.get("source_range_start"))),
|
||||
source_range_end=_parse_datetime(_iso(snapshot.get("source_range_end"))),
|
||||
is_stale=bool(snapshot.get("is_stale", False)),
|
||||
)
|
||||
|
||||
|
||||
def get_snapshot(
|
||||
*,
|
||||
server_key: str,
|
||||
snapshot_type: str,
|
||||
metric: str | None,
|
||||
window: str | None,
|
||||
) -> dict[str, object] | None:
|
||||
initialize_postgres_display_storage()
|
||||
with connect_postgres() as connection:
|
||||
row = connection.execute(
|
||||
"""
|
||||
SELECT *
|
||||
FROM displayed_historical_snapshots
|
||||
WHERE server_key = %s AND snapshot_type = %s AND metric = %s AND snapshot_window = %s
|
||||
""",
|
||||
(server_key, snapshot_type, metric or "", window or ""),
|
||||
).fetchone()
|
||||
if not row:
|
||||
return None
|
||||
return {
|
||||
"server_key": row["server_key"],
|
||||
"snapshot_type": row["snapshot_type"],
|
||||
"metric": row["metric"] or None,
|
||||
"window": row["snapshot_window"] or None,
|
||||
"generated_at": row["generated_at"],
|
||||
"source_range_start": row["source_range_start"],
|
||||
"source_range_end": row["source_range_end"],
|
||||
"is_stale": bool(row["is_stale"]),
|
||||
"payload": json.loads(row["payload_json"]),
|
||||
}
|
||||
|
||||
|
||||
def list_latest_server_snapshots() -> list[dict[str, object]]:
|
||||
initialize_postgres_display_storage()
|
||||
with connect_postgres() as connection:
|
||||
rows = connection.execute(
|
||||
"""
|
||||
SELECT s.id AS server_id, s.external_server_id, s.server_name, s.region,
|
||||
g.slug AS context, snap.source_name, snap.snapshot_origin,
|
||||
snap.source_ref, snap.captured_at, snap.status, snap.players,
|
||||
snap.max_players, snap.current_map
|
||||
FROM servers AS s
|
||||
JOIN game_sources AS g ON g.id = s.game_source_id
|
||||
JOIN server_snapshots AS snap ON snap.server_id = s.id
|
||||
JOIN (
|
||||
SELECT server_id, MAX(captured_at) AS captured_at
|
||||
FROM server_snapshots GROUP BY server_id
|
||||
) AS latest ON latest.server_id = snap.server_id
|
||||
AND latest.captured_at = snap.captured_at
|
||||
ORDER BY s.server_name ASC
|
||||
"""
|
||||
).fetchall()
|
||||
return [_attach_server_history(connection, dict(row)) for row in rows]
|
||||
|
||||
|
||||
def persist_server_snapshots(
|
||||
snapshots: Iterable[Mapping[str, object]],
|
||||
*,
|
||||
source_name: str,
|
||||
captured_at: str,
|
||||
game_source: Mapping[str, str],
|
||||
) -> dict[str, object]:
|
||||
initialize_postgres_display_storage()
|
||||
persisted = 0
|
||||
with connect_postgres() as connection:
|
||||
source = connection.execute(
|
||||
"""
|
||||
INSERT INTO game_sources (slug, display_name, provider_kind, is_active)
|
||||
VALUES (%s, %s, %s, TRUE)
|
||||
ON CONFLICT(slug) DO UPDATE SET
|
||||
display_name = EXCLUDED.display_name,
|
||||
provider_kind = EXCLUDED.provider_kind,
|
||||
is_active = TRUE,
|
||||
updated_at = CURRENT_TIMESTAMP
|
||||
RETURNING id
|
||||
""",
|
||||
(game_source["slug"], game_source["display_name"], game_source["provider_kind"]),
|
||||
).fetchone()
|
||||
for snapshot in snapshots:
|
||||
external_server_id = str(snapshot.get("external_server_id") or "").strip()
|
||||
if not external_server_id:
|
||||
external_server_id = _fallback_external_id(snapshot.get("server_name"))
|
||||
server = connection.execute(
|
||||
"""
|
||||
INSERT INTO servers (
|
||||
game_source_id, external_server_id, server_name, region,
|
||||
first_seen_at, last_seen_at
|
||||
) VALUES (%s, %s, %s, %s, %s, %s)
|
||||
ON CONFLICT(game_source_id, external_server_id) DO UPDATE SET
|
||||
server_name = EXCLUDED.server_name,
|
||||
region = EXCLUDED.region,
|
||||
last_seen_at = EXCLUDED.last_seen_at,
|
||||
updated_at = CURRENT_TIMESTAMP
|
||||
RETURNING id
|
||||
""",
|
||||
(
|
||||
source["id"],
|
||||
external_server_id,
|
||||
str(snapshot.get("server_name") or "Unknown server"),
|
||||
snapshot.get("region"),
|
||||
captured_at,
|
||||
captured_at,
|
||||
),
|
||||
).fetchone()
|
||||
connection.execute(
|
||||
"""
|
||||
INSERT INTO server_snapshots (
|
||||
server_id, captured_at, status, players, max_players, current_map,
|
||||
source_name, snapshot_origin, source_ref, raw_payload_ref
|
||||
) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, NULL)
|
||||
ON CONFLICT(server_id, captured_at, source_name, source_ref) DO UPDATE SET
|
||||
status = EXCLUDED.status,
|
||||
players = EXCLUDED.players,
|
||||
max_players = EXCLUDED.max_players,
|
||||
current_map = EXCLUDED.current_map,
|
||||
snapshot_origin = EXCLUDED.snapshot_origin
|
||||
""",
|
||||
(
|
||||
server["id"],
|
||||
captured_at,
|
||||
snapshot.get("status") or "unknown",
|
||||
snapshot.get("players"),
|
||||
snapshot.get("max_players"),
|
||||
snapshot.get("current_map"),
|
||||
snapshot.get("source_name") or source_name,
|
||||
snapshot.get("snapshot_origin"),
|
||||
snapshot.get("source_ref") or snapshot.get("source_name") or source_name,
|
||||
),
|
||||
)
|
||||
persisted += 1
|
||||
return {
|
||||
"db_path": "postgresql",
|
||||
"captured_at": captured_at,
|
||||
"persisted_snapshots": persisted,
|
||||
"game_source_slug": game_source["slug"],
|
||||
}
|
||||
|
||||
|
||||
def upsert_player_event_rows(events: Iterable[object]) -> dict[str, int]:
|
||||
initialize_postgres_display_storage()
|
||||
inserted = 0
|
||||
duplicates = 0
|
||||
with connect_postgres() as connection:
|
||||
for event in events:
|
||||
row = connection.execute(
|
||||
"""
|
||||
INSERT INTO player_event_raw_ledger (
|
||||
event_id, event_type, occurred_at, server_slug, external_match_id,
|
||||
source_kind, source_ref, raw_event_ref, killer_player_key,
|
||||
killer_display_name, victim_player_key, victim_display_name,
|
||||
weapon_name, weapon_category, kill_category, is_teamkill, event_value
|
||||
) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
|
||||
ON CONFLICT(event_id) DO NOTHING
|
||||
RETURNING id
|
||||
""",
|
||||
(
|
||||
event.event_id,
|
||||
event.event_type,
|
||||
event.occurred_at,
|
||||
event.server_slug,
|
||||
event.external_match_id,
|
||||
event.source_kind,
|
||||
event.source_ref,
|
||||
event.raw_event_ref,
|
||||
event.killer_player_key,
|
||||
event.killer_display_name,
|
||||
event.victim_player_key,
|
||||
event.victim_display_name,
|
||||
event.weapon_name,
|
||||
event.weapon_category,
|
||||
event.kill_category,
|
||||
bool(event.is_teamkill),
|
||||
max(1, int(event.event_value)),
|
||||
),
|
||||
).fetchone()
|
||||
inserted += int(bool(row))
|
||||
duplicates += int(not row)
|
||||
return {"events_inserted": inserted, "duplicate_events": duplicates}
|
||||
|
||||
|
||||
def list_server_snapshot_history(*, server_id: str | None = None, limit: int) -> list[dict[str, object]]:
|
||||
initialize_postgres_display_storage()
|
||||
where = ""
|
||||
params: list[object] = []
|
||||
if server_id:
|
||||
if server_id.strip().isdigit():
|
||||
where = "WHERE s.id = %s"
|
||||
params.append(int(server_id))
|
||||
else:
|
||||
where = "WHERE s.external_server_id = %s"
|
||||
params.append(server_id.strip())
|
||||
with connect_postgres() as connection:
|
||||
rows = connection.execute(
|
||||
f"""
|
||||
SELECT s.id AS server_id, s.external_server_id, s.server_name, s.region,
|
||||
g.slug AS context, snap.source_name, snap.snapshot_origin,
|
||||
snap.source_ref, snap.captured_at, snap.status, snap.players,
|
||||
snap.max_players, snap.current_map
|
||||
FROM server_snapshots AS snap
|
||||
JOIN servers AS s ON s.id = snap.server_id
|
||||
JOIN game_sources AS g ON g.id = s.game_source_id
|
||||
{where}
|
||||
ORDER BY snap.captured_at DESC, s.server_name ASC
|
||||
LIMIT %s
|
||||
""",
|
||||
(*params, limit),
|
||||
).fetchall()
|
||||
return [dict(row) for row in rows]
|
||||
|
||||
|
||||
def list_recent_scoreboard_matches(*, server_slug: str | None, limit: int) -> list[dict[str, object]]:
|
||||
initialize_postgres_display_storage()
|
||||
where = ""
|
||||
params: list[object] = []
|
||||
if server_slug and server_slug != ALL_SERVERS_SLUG:
|
||||
where = "WHERE hs.slug = %s"
|
||||
params.append(server_slug)
|
||||
with connect_postgres() as connection:
|
||||
rows = connection.execute(
|
||||
f"""
|
||||
SELECT hs.slug AS server_slug, hs.display_name AS server_name,
|
||||
hm.external_match_id, hm.started_at, hm.ended_at,
|
||||
hm.map_pretty_name, hm.map_name, hm.allied_score, hm.axis_score,
|
||||
hm.raw_payload_ref, COUNT(stats.id) AS player_count
|
||||
FROM historical_matches AS hm
|
||||
JOIN historical_servers AS hs ON hs.id = hm.historical_server_id
|
||||
LEFT JOIN historical_player_match_stats AS stats ON stats.historical_match_id = hm.id
|
||||
{where}
|
||||
GROUP BY hm.id, hs.slug, hs.display_name
|
||||
ORDER BY COALESCE(hm.ended_at, hm.started_at) DESC
|
||||
LIMIT %s
|
||||
""",
|
||||
(*params, limit),
|
||||
).fetchall()
|
||||
return [_recent_match_row(row) for row in rows]
|
||||
|
||||
|
||||
def get_scoreboard_match_detail(*, server_slug: str, match_id: str) -> dict[str, object] | None:
|
||||
initialize_postgres_display_storage()
|
||||
with connect_postgres() as connection:
|
||||
row = connection.execute(
|
||||
"""
|
||||
SELECT hm.id AS match_pk, hs.slug AS server_slug, hs.display_name AS server_name,
|
||||
hm.external_match_id, hm.started_at, hm.ended_at, hm.map_pretty_name,
|
||||
hm.map_name, hm.allied_score, hm.axis_score, hm.raw_payload_ref,
|
||||
COUNT(stats.id) AS player_count,
|
||||
SUM(COALESCE(stats.time_seconds, 0)) AS total_time_seconds
|
||||
FROM historical_matches AS hm
|
||||
JOIN historical_servers AS hs ON hs.id = hm.historical_server_id
|
||||
LEFT JOIN historical_player_match_stats AS stats ON stats.historical_match_id = hm.id
|
||||
WHERE hs.slug = %s AND hm.external_match_id = %s
|
||||
GROUP BY hm.id, hs.slug, hs.display_name
|
||||
LIMIT 1
|
||||
""",
|
||||
(server_slug, match_id),
|
||||
).fetchone()
|
||||
if not row:
|
||||
return None
|
||||
players = connection.execute(
|
||||
"""
|
||||
SELECT hp.display_name, hp.stable_player_key, hp.steam_id, stats.team_side, stats.level,
|
||||
stats.kills, stats.deaths, stats.teamkills, stats.combat, stats.offense,
|
||||
stats.defense, stats.support, stats.time_seconds
|
||||
FROM historical_player_match_stats AS stats
|
||||
JOIN historical_players AS hp ON hp.id = stats.historical_player_id
|
||||
WHERE stats.historical_match_id = %s
|
||||
ORDER BY COALESCE(stats.kills, 0) DESC, hp.display_name ASC
|
||||
""",
|
||||
(row["match_pk"],),
|
||||
).fetchall()
|
||||
started_at = row["started_at"]
|
||||
ended_at = row["ended_at"]
|
||||
return {
|
||||
"server": {"slug": row["server_slug"], "name": row["server_name"]},
|
||||
"match_id": row["external_match_id"],
|
||||
"started_at": started_at,
|
||||
"ended_at": ended_at,
|
||||
"closed_at": ended_at or started_at,
|
||||
"duration_seconds": _duration_seconds(started_at, ended_at),
|
||||
"map": {"name": row["map_name"], "pretty_name": row["map_pretty_name"] or row["map_name"]},
|
||||
"result": _match_result(row["allied_score"], row["axis_score"]),
|
||||
"player_count": int(row["player_count"] or 0),
|
||||
"total_time_seconds": _int(row["total_time_seconds"]),
|
||||
"players": [
|
||||
{
|
||||
"name": player["display_name"],
|
||||
"stable_player_key": player["stable_player_key"],
|
||||
"team_side": player["team_side"],
|
||||
**build_external_player_profile_fields(steam_id=player["steam_id"]),
|
||||
**{
|
||||
key: _int(player[key])
|
||||
for key in (
|
||||
"level", "kills", "deaths", "teamkills", "combat",
|
||||
"offense", "defense", "support", "time_seconds",
|
||||
)
|
||||
},
|
||||
}
|
||||
for player in players
|
||||
],
|
||||
"capture_basis": "public-scoreboard-match",
|
||||
"match_url": resolve_trusted_scoreboard_match_url(row["raw_payload_ref"], row["server_slug"]),
|
||||
}
|
||||
|
||||
|
||||
def list_scoreboard_server_summaries(*, server_slug: str | None) -> list[dict[str, object]]:
|
||||
initialize_postgres_display_storage()
|
||||
if server_slug == ALL_SERVERS_SLUG:
|
||||
rows = list_scoreboard_server_summaries(server_slug=None)
|
||||
return [_all_server_summary(rows)]
|
||||
where = "WHERE hs.slug = %s" if server_slug else ""
|
||||
params = (server_slug,) if server_slug else ()
|
||||
with connect_postgres() as connection:
|
||||
rows = connection.execute(
|
||||
f"""
|
||||
SELECT hs.slug AS server_slug, hs.display_name AS server_name,
|
||||
COUNT(DISTINCT hm.id) AS matches_count,
|
||||
COUNT(DISTINCT hp.id) AS unique_players,
|
||||
COALESCE(SUM(stats.kills), 0) AS total_kills,
|
||||
COUNT(DISTINCT COALESCE(hm.map_pretty_name, hm.map_name)) AS map_count,
|
||||
MIN(COALESCE(hm.ended_at, hm.started_at, hm.created_at_source)) AS first_match_at,
|
||||
MAX(COALESCE(hm.ended_at, hm.started_at, hm.created_at_source)) AS last_match_at
|
||||
FROM historical_servers AS hs
|
||||
LEFT JOIN historical_matches AS hm ON hm.historical_server_id = hs.id
|
||||
LEFT JOIN historical_player_match_stats AS stats ON stats.historical_match_id = hm.id
|
||||
LEFT JOIN historical_players AS hp ON hp.id = stats.historical_player_id
|
||||
{where}
|
||||
GROUP BY hs.id
|
||||
ORDER BY hs.server_number ASC, hs.slug ASC
|
||||
""",
|
||||
params,
|
||||
).fetchall()
|
||||
map_rows = connection.execute(
|
||||
f"""
|
||||
SELECT hs.slug AS server_slug,
|
||||
COALESCE(hm.map_pretty_name, hm.map_name, 'Mapa no disponible') AS map_name,
|
||||
COUNT(*) AS matches_count
|
||||
FROM historical_matches AS hm
|
||||
JOIN historical_servers AS hs ON hs.id = hm.historical_server_id
|
||||
{where}
|
||||
GROUP BY hs.slug, COALESCE(hm.map_pretty_name, hm.map_name, 'Mapa no disponible')
|
||||
ORDER BY hs.slug ASC, matches_count DESC, map_name ASC
|
||||
""",
|
||||
params,
|
||||
).fetchall()
|
||||
maps: dict[str, list[dict[str, object]]] = {}
|
||||
for row in map_rows:
|
||||
maps.setdefault(str(row["server_slug"]), [])
|
||||
if len(maps[str(row["server_slug"])]) < 3:
|
||||
maps[str(row["server_slug"])].append(
|
||||
{"map_name": row["map_name"], "matches_count": int(row["matches_count"] or 0)}
|
||||
)
|
||||
return [_summary_row(row, maps.get(str(row["server_slug"]), [])) for row in rows]
|
||||
|
||||
|
||||
def list_scoreboard_leaderboard(
|
||||
*, timeframe: str, metric: str, server_id: str | None, limit: int
|
||||
) -> dict[str, object]:
|
||||
current = datetime.now(timezone.utc)
|
||||
if timeframe == "monthly":
|
||||
current_start = current.replace(day=1, hour=0, minute=0, second=0, microsecond=0)
|
||||
previous_start = (current_start - timedelta(days=1)).replace(
|
||||
day=1, hour=0, minute=0, second=0, microsecond=0
|
||||
)
|
||||
label = ("current-month", "Mes actual", "previous-closed-month-fallback", "Mes cerrado anterior")
|
||||
else:
|
||||
current_midnight = current.replace(hour=0, minute=0, second=0, microsecond=0)
|
||||
current_start = current_midnight - timedelta(days=current_midnight.weekday())
|
||||
previous_start = current_start - timedelta(days=7)
|
||||
label = ("current-week", "Semana actual", "previous-closed-week-fallback", "Semana cerrada anterior")
|
||||
current_count = _count_scoreboard_matches(server_id, current_start, current)
|
||||
previous_count = _count_scoreboard_matches(server_id, previous_start, current_start)
|
||||
fallback = current_count <= 0 and previous_count > 0
|
||||
start, end = (previous_start, current_start) if fallback else (current_start, current)
|
||||
rows = _leaderboard_rows(server_id=server_id, metric=metric, start=start, end=end, limit=limit)
|
||||
window_days = max(1, int(((end - start).total_seconds() + 86399) // 86400))
|
||||
result = {
|
||||
"metric": metric,
|
||||
"window_start": _iso(start),
|
||||
"window_end": _iso(end),
|
||||
"window_days": window_days,
|
||||
"window_kind": label[2] if fallback else label[0],
|
||||
"window_label": label[3] if fallback else label[1],
|
||||
"uses_fallback": fallback,
|
||||
"selection_reason": (
|
||||
"no-current-month-matches" if fallback and timeframe == "monthly"
|
||||
else "insufficient-current-week-sample" if fallback
|
||||
else label[0]
|
||||
),
|
||||
"items": rows,
|
||||
}
|
||||
if timeframe == "monthly":
|
||||
result.update(
|
||||
{
|
||||
"timeframe": "monthly",
|
||||
"current_month_start": _iso(current_start),
|
||||
"current_month_closed_matches": current_count,
|
||||
"previous_month_closed_matches": previous_count,
|
||||
"sufficient_sample": {
|
||||
"minimum_closed_matches": 1,
|
||||
"current_month_closed_matches": current_count,
|
||||
"current_month_has_sufficient_sample": current_count > 0,
|
||||
"is_early_month": current.day <= 3,
|
||||
},
|
||||
}
|
||||
)
|
||||
else:
|
||||
result.update(
|
||||
{
|
||||
"current_week_start": _iso(current_start),
|
||||
"current_week_closed_matches": current_count,
|
||||
"previous_week_closed_matches": previous_count,
|
||||
"sufficient_sample": {
|
||||
"minimum_closed_matches": 1,
|
||||
"current_week_closed_matches": current_count,
|
||||
"current_week_has_sufficient_sample": current_count > 0,
|
||||
"is_early_week": current.weekday() <= get_historical_weekly_fallback_max_weekday(),
|
||||
"fallback_max_weekday": get_historical_weekly_fallback_max_weekday(),
|
||||
},
|
||||
}
|
||||
)
|
||||
return result
|
||||
|
||||
|
||||
def table_counts() -> dict[str, int]:
|
||||
initialize_postgres_display_storage()
|
||||
tables = (
|
||||
"historical_matches",
|
||||
"historical_player_match_stats",
|
||||
"displayed_historical_snapshots",
|
||||
"player_event_raw_ledger",
|
||||
"server_snapshots",
|
||||
)
|
||||
with connect_postgres() as connection:
|
||||
return {
|
||||
table: int(connection.execute(f"SELECT COUNT(*) AS count FROM {table}").fetchone()["count"] or 0)
|
||||
for table in tables
|
||||
}
|
||||
|
||||
|
||||
def _leaderboard_rows(
|
||||
*, server_id: str | None, metric: str, start: datetime, end: datetime, limit: int
|
||||
) -> list[dict[str, object]]:
|
||||
metric_sql = {
|
||||
"kills": "COALESCE(SUM(stats.kills), 0)",
|
||||
"deaths": "COALESCE(SUM(stats.deaths), 0)",
|
||||
"support": "COALESCE(SUM(stats.support), 0)",
|
||||
"matches_over_100_kills": (
|
||||
"COALESCE(SUM(CASE WHEN COALESCE(stats.kills, 0) >= 100 THEN 1 ELSE 0 END), 0)"
|
||||
),
|
||||
}[metric]
|
||||
aggregate = server_id == ALL_SERVERS_SLUG
|
||||
where, server_params = _server_where(server_id)
|
||||
server_slug = f"'{ALL_SERVERS_SLUG}'" if aggregate else "hs.slug"
|
||||
server_name = f"'{ALL_SERVERS_DISPLAY_NAME}'" if aggregate else "hs.display_name"
|
||||
partition = f"'{ALL_SERVERS_SLUG}'" if aggregate else "hs.slug"
|
||||
group_by = "hp.id" if aggregate else "hs.slug, hs.display_name, hp.id"
|
||||
with connect_postgres() as connection:
|
||||
rows = connection.execute(
|
||||
f"""
|
||||
WITH ranked AS (
|
||||
SELECT {server_slug} AS server_slug, {server_name} AS server_name,
|
||||
hp.stable_player_key, hp.display_name AS player_name, hp.steam_id,
|
||||
COUNT(DISTINCT hm.id) AS matches_count, {metric_sql} AS metric_value,
|
||||
ROW_NUMBER() OVER (
|
||||
PARTITION BY {partition}
|
||||
ORDER BY {metric_sql} DESC, COUNT(DISTINCT hm.id) ASC, hp.display_name ASC
|
||||
) AS ranking_position
|
||||
FROM historical_player_match_stats AS stats
|
||||
JOIN historical_matches AS hm ON hm.id = stats.historical_match_id
|
||||
JOIN historical_servers AS hs ON hs.id = hm.historical_server_id
|
||||
JOIN historical_players AS hp ON hp.id = stats.historical_player_id
|
||||
WHERE hm.ended_at IS NOT NULL AND hm.ended_at >= %s AND hm.ended_at < %s {where}
|
||||
GROUP BY {group_by}
|
||||
)
|
||||
SELECT * FROM ranked WHERE ranking_position <= %s
|
||||
ORDER BY server_slug ASC, ranking_position ASC
|
||||
""",
|
||||
(_iso(start), _iso(end), *server_params, limit),
|
||||
).fetchall()
|
||||
return [
|
||||
{
|
||||
"server": {"slug": row["server_slug"], "name": row["server_name"]},
|
||||
"time_range": {"start": _iso(start), "end": _iso(end), "window_days": max(1, (end - start).days or 1)},
|
||||
"player": {
|
||||
"stable_player_key": row["stable_player_key"],
|
||||
"name": row["player_name"],
|
||||
"steam_id": row["steam_id"],
|
||||
},
|
||||
"metric": metric,
|
||||
"ranking_position": int(row["ranking_position"]),
|
||||
"metric_value": int(row["metric_value"] or 0),
|
||||
"matches_considered": int(row["matches_count"] or 0),
|
||||
}
|
||||
for row in rows
|
||||
]
|
||||
|
||||
|
||||
def _count_scoreboard_matches(server_id: str | None, start: datetime, end: datetime) -> int:
|
||||
where, server_params = _server_where(server_id)
|
||||
with connect_postgres() as connection:
|
||||
row = connection.execute(
|
||||
f"""
|
||||
SELECT COUNT(DISTINCT hm.id) AS count
|
||||
FROM historical_matches AS hm
|
||||
JOIN historical_servers AS hs ON hs.id = hm.historical_server_id
|
||||
JOIN historical_player_match_stats AS stats ON stats.historical_match_id = hm.id
|
||||
WHERE hm.ended_at IS NOT NULL AND hm.ended_at >= %s AND hm.ended_at < %s {where}
|
||||
""",
|
||||
(_iso(start), _iso(end), *server_params),
|
||||
).fetchone()
|
||||
return int(row["count"] or 0)
|
||||
|
||||
|
||||
def _server_where(server_id: str | None) -> tuple[str, tuple[object, ...]]:
|
||||
if not server_id or server_id == ALL_SERVERS_SLUG:
|
||||
return "", ()
|
||||
return "AND (hs.slug = %s OR CAST(hs.server_number AS TEXT) = %s)", (server_id, server_id)
|
||||
|
||||
|
||||
def _recent_match_row(row: Mapping[str, object]) -> dict[str, object]:
|
||||
return {
|
||||
"server": {"slug": row["server_slug"], "name": row["server_name"]},
|
||||
"match_id": row["external_match_id"],
|
||||
"started_at": row["started_at"],
|
||||
"ended_at": row["ended_at"],
|
||||
"closed_at": row["ended_at"] or row["started_at"],
|
||||
"map": {"name": row["map_name"], "pretty_name": row["map_pretty_name"] or row["map_name"]},
|
||||
"result": _match_result(row["allied_score"], row["axis_score"]),
|
||||
"player_count": int(row["player_count"] or 0),
|
||||
"match_url": resolve_trusted_scoreboard_match_url(row["raw_payload_ref"], row["server_slug"]),
|
||||
}
|
||||
|
||||
|
||||
def _summary_row(row: Mapping[str, object], top_maps: list[dict[str, object]]) -> dict[str, object]:
|
||||
first = row["first_match_at"]
|
||||
last = row["last_match_at"]
|
||||
matches = int(row["matches_count"] or 0)
|
||||
return {
|
||||
"server": {"slug": row["server_slug"], "name": row["server_name"]},
|
||||
"matches_count": matches,
|
||||
"imported_matches_count": matches,
|
||||
"unique_players": int(row["unique_players"] or 0),
|
||||
"total_kills": int(row["total_kills"] or 0),
|
||||
"map_count": int(row["map_count"] or 0),
|
||||
"top_maps": top_maps,
|
||||
"coverage": {
|
||||
"basis": "postgres-migrated-public-scoreboard",
|
||||
"status": "available" if matches else "empty",
|
||||
"imported_matches_count": matches,
|
||||
"discovered_total_matches": None,
|
||||
"first_match_at": first,
|
||||
"last_match_at": last,
|
||||
"coverage_days": _coverage_days(first, last),
|
||||
},
|
||||
"backfill": {},
|
||||
"time_range": {"start": first, "end": last},
|
||||
}
|
||||
|
||||
|
||||
def _all_server_summary(items: list[dict[str, object]]) -> dict[str, object]:
|
||||
starts = [item["time_range"]["start"] for item in items if item["time_range"]["start"]]
|
||||
ends = [item["time_range"]["end"] for item in items if item["time_range"]["end"]]
|
||||
return {
|
||||
"server": {"slug": ALL_SERVERS_SLUG, "name": ALL_SERVERS_DISPLAY_NAME},
|
||||
"matches_count": sum(int(item["matches_count"]) for item in items),
|
||||
"imported_matches_count": sum(int(item["imported_matches_count"]) for item in items),
|
||||
"unique_players": None,
|
||||
"total_kills": sum(int(item["total_kills"]) for item in items),
|
||||
"map_count": None,
|
||||
"top_maps": [],
|
||||
"coverage": {"basis": "postgres-migrated-public-scoreboard", "status": "available" if items else "empty"},
|
||||
"backfill": {},
|
||||
"time_range": {"start": min(starts) if starts else None, "end": max(ends) if ends else None},
|
||||
}
|
||||
|
||||
|
||||
def _attach_server_history(connection: Any, item: dict[str, object]) -> dict[str, object]:
|
||||
rows = connection.execute(
|
||||
"""
|
||||
SELECT captured_at, status, players FROM server_snapshots
|
||||
WHERE server_id = %s ORDER BY captured_at DESC LIMIT %s
|
||||
""",
|
||||
(item["server_id"], SUMMARY_SNAPSHOT_LIMIT),
|
||||
).fetchall()
|
||||
players = [int(row["players"]) for row in rows if row["players"] is not None]
|
||||
online = [row for row in rows if row["status"] == "online"]
|
||||
item["history_summary"] = {
|
||||
"window_size": SUMMARY_SNAPSHOT_LIMIT,
|
||||
"recent_capture_count": len(rows),
|
||||
"recent_online_count": len(online),
|
||||
"recent_average_players": round(sum(players) / len(players), 1) if players else None,
|
||||
"recent_peak_players": max(players, default=None),
|
||||
"last_seen_online_at": online[0]["captured_at"] if online else None,
|
||||
"minutes_since_last_capture": _minutes_since(rows[0]["captured_at"]) if rows else None,
|
||||
}
|
||||
return item
|
||||
|
||||
|
||||
def _match_result(allied: object, axis: object) -> dict[str, object]:
|
||||
allied_int, axis_int = _int(allied), _int(axis)
|
||||
winner = None
|
||||
if allied_int is not None and axis_int is not None:
|
||||
winner = "allied" if allied_int > axis_int else "axis" if axis_int > allied_int else "draw"
|
||||
return {"allied_score": allied_int, "axis_score": axis_int, "winner": winner}
|
||||
|
||||
|
||||
def _duration_seconds(start: object, end: object) -> int | None:
|
||||
start_point, end_point = _parse_datetime(_iso(start)), _parse_datetime(_iso(end))
|
||||
return max(0, int((end_point - start_point).total_seconds())) if start_point and end_point else None
|
||||
|
||||
|
||||
def _coverage_days(start: object, end: object) -> int | None:
|
||||
seconds = _duration_seconds(start, end)
|
||||
return max(1, int((seconds + 86399) // 86400)) if seconds is not None else None
|
||||
|
||||
|
||||
def _minutes_since(value: object) -> int | None:
|
||||
point = _parse_datetime(_iso(value))
|
||||
return max(0, int((datetime.now(timezone.utc) - point).total_seconds() // 60)) if point else None
|
||||
|
||||
|
||||
def _int(value: object) -> int | None:
|
||||
try:
|
||||
return None if value is None else int(value)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
def _fallback_external_id(value: object) -> str:
|
||||
normalized = "".join(
|
||||
character.lower() if character.isalnum() else "-"
|
||||
for character in str(value or "unknown-server")
|
||||
)
|
||||
compact = "-".join(part for part in normalized.split("-") if part)
|
||||
return compact or "unknown-server"
|
||||
|
||||
|
||||
def _iso(value: object) -> str | None:
|
||||
if isinstance(value, datetime):
|
||||
point = value if value.tzinfo else value.replace(tzinfo=timezone.utc)
|
||||
return point.astimezone(timezone.utc).isoformat().replace("+00:00", "Z")
|
||||
text = str(value or "").strip()
|
||||
return text or None
|
||||
|
||||
|
||||
def _json_payload_default(value: object) -> str:
|
||||
if isinstance(value, datetime):
|
||||
return _iso(value) or ""
|
||||
raise TypeError(f"Object of type {type(value).__name__} is not JSON serializable")
|
||||
|
||||
|
||||
def _parse_datetime(value: str | None) -> datetime | None:
|
||||
if not value:
|
||||
return None
|
||||
try:
|
||||
point = datetime.fromisoformat(value.replace("Z", "+00:00"))
|
||||
except ValueError:
|
||||
return None
|
||||
return point.astimezone(timezone.utc) if point.tzinfo else point.replace(tzinfo=timezone.utc)
|
||||
|
||||
|
||||
def _utc_now_iso() -> str:
|
||||
return datetime.now(timezone.utc).isoformat().replace("+00:00", "Z")
|
||||
1038
backend/app/postgres_rcon_storage.py
Normal file
1038
backend/app/postgres_rcon_storage.py
Normal file
File diff suppressed because it is too large
Load Diff
415
backend/app/providers/player_event_source_provider.py
Normal file
415
backend/app/providers/player_event_source_provider.py
Normal file
@@ -0,0 +1,415 @@
|
||||
"""Player event adapter backed by public CRCON scoreboard match details."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
from collections.abc import Mapping
|
||||
from dataclasses import dataclass
|
||||
|
||||
from ..player_event_models import PlayerEventRecord
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _PlayerIdentity:
|
||||
stable_player_key: str
|
||||
display_name: str | None
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class PublicScoreboardPlayerEventSource:
|
||||
"""Normalize partial duel and weapon signals from CRCON match detail payloads."""
|
||||
|
||||
source_kind: str = "public-scoreboard-match-summary"
|
||||
|
||||
def extract_match_events(
|
||||
self,
|
||||
*,
|
||||
server_slug: str,
|
||||
match_payload: dict[str, object],
|
||||
source_ref: str | None = None,
|
||||
) -> list[PlayerEventRecord]:
|
||||
match_id = _stringify(match_payload.get("id"))
|
||||
if not match_id:
|
||||
return []
|
||||
|
||||
occurred_at = _pick_match_timestamp(match_payload)
|
||||
player_rows = _coerce_player_rows(match_payload.get("player_stats"))
|
||||
if not player_rows:
|
||||
return []
|
||||
|
||||
identity_index = _build_identity_index(player_rows)
|
||||
events: list[PlayerEventRecord] = []
|
||||
|
||||
for player_row in player_rows:
|
||||
actor = _build_player_identity(player_row)
|
||||
if actor is None:
|
||||
continue
|
||||
|
||||
top_kill_type_name = _extract_primary_name(player_row.get("kills_by_type"))
|
||||
|
||||
for victim_name, victim_count in _extract_named_counts(player_row.get("most_killed")):
|
||||
victim = _find_identity_by_name(identity_index, victim_name)
|
||||
if victim is None or victim_count <= 0:
|
||||
continue
|
||||
events.append(
|
||||
_build_event(
|
||||
event_type="player_kill_summary",
|
||||
occurred_at=occurred_at,
|
||||
server_slug=server_slug,
|
||||
match_id=match_id,
|
||||
source_kind=self.source_kind,
|
||||
source_ref=source_ref,
|
||||
raw_event_ref=(
|
||||
f"match:{match_id}:player:{actor.stable_player_key}:most-killed:{victim.stable_player_key}"
|
||||
),
|
||||
killer=actor,
|
||||
victim=victim,
|
||||
weapon_name=None,
|
||||
kill_category=top_kill_type_name,
|
||||
is_teamkill=False,
|
||||
event_value=victim_count,
|
||||
)
|
||||
)
|
||||
|
||||
for killer_name, killer_count in _extract_named_counts(player_row.get("death_by")):
|
||||
killer = _find_identity_by_name(identity_index, killer_name)
|
||||
if killer is None or killer_count <= 0:
|
||||
continue
|
||||
events.append(
|
||||
_build_event(
|
||||
event_type="player_death_summary",
|
||||
occurred_at=occurred_at,
|
||||
server_slug=server_slug,
|
||||
match_id=match_id,
|
||||
source_kind=self.source_kind,
|
||||
source_ref=source_ref,
|
||||
raw_event_ref=(
|
||||
f"match:{match_id}:player:{actor.stable_player_key}:death-by:{killer.stable_player_key}"
|
||||
),
|
||||
killer=killer,
|
||||
victim=actor,
|
||||
weapon_name=None,
|
||||
kill_category=None,
|
||||
is_teamkill=False,
|
||||
event_value=killer_count,
|
||||
)
|
||||
)
|
||||
|
||||
for weapon_name, weapon_count in _extract_named_counts(player_row.get("weapons")):
|
||||
events.append(
|
||||
_build_event(
|
||||
event_type="player_weapon_kill_summary",
|
||||
occurred_at=occurred_at,
|
||||
server_slug=server_slug,
|
||||
match_id=match_id,
|
||||
source_kind=self.source_kind,
|
||||
source_ref=source_ref,
|
||||
raw_event_ref=(
|
||||
f"match:{match_id}:player:{actor.stable_player_key}:weapons:{weapon_name}"
|
||||
),
|
||||
killer=actor,
|
||||
victim=None,
|
||||
weapon_name=weapon_name,
|
||||
kill_category=top_kill_type_name,
|
||||
is_teamkill=False,
|
||||
event_value=weapon_count,
|
||||
)
|
||||
)
|
||||
|
||||
for weapon_name, weapon_count in _extract_named_counts(player_row.get("death_by_weapons")):
|
||||
events.append(
|
||||
_build_event(
|
||||
event_type="player_weapon_death_summary",
|
||||
occurred_at=occurred_at,
|
||||
server_slug=server_slug,
|
||||
match_id=match_id,
|
||||
source_kind=self.source_kind,
|
||||
source_ref=source_ref,
|
||||
raw_event_ref=(
|
||||
f"match:{match_id}:player:{actor.stable_player_key}:death-by-weapons:{weapon_name}"
|
||||
),
|
||||
killer=None,
|
||||
victim=actor,
|
||||
weapon_name=weapon_name,
|
||||
kill_category=None,
|
||||
is_teamkill=False,
|
||||
event_value=weapon_count,
|
||||
)
|
||||
)
|
||||
|
||||
teamkills = _coerce_int(player_row.get("teamkills")) or 0
|
||||
if teamkills > 0:
|
||||
events.append(
|
||||
_build_event(
|
||||
event_type="player_teamkill_summary",
|
||||
occurred_at=occurred_at,
|
||||
server_slug=server_slug,
|
||||
match_id=match_id,
|
||||
source_kind=self.source_kind,
|
||||
source_ref=source_ref,
|
||||
raw_event_ref=f"match:{match_id}:player:{actor.stable_player_key}:teamkills",
|
||||
killer=actor,
|
||||
victim=None,
|
||||
weapon_name=None,
|
||||
kill_category=top_kill_type_name,
|
||||
is_teamkill=True,
|
||||
event_value=teamkills,
|
||||
)
|
||||
)
|
||||
|
||||
return events
|
||||
|
||||
def describe_scope(self) -> dict[str, object]:
|
||||
return {
|
||||
"source_kind": self.source_kind,
|
||||
"supports_raw_kill_events": False,
|
||||
"captures": [
|
||||
"Encounter summaries per player from most_killed",
|
||||
"Death summaries per player from death_by",
|
||||
"Weapon kill summaries per player from weapons",
|
||||
"Weapon death summaries per player from death_by_weapons",
|
||||
"Aggregated teamkills per player and match",
|
||||
],
|
||||
"limitations": [
|
||||
"The current source is match-summary data, not a true per-kill event feed.",
|
||||
"occurred_at uses the match end/start timestamp, not the exact kill timestamp.",
|
||||
"Only summary counters exposed by the CRCON detail payload are normalized.",
|
||||
"Full killer->victim ledgers, complete weapon breakdowns, and exact per-event teamkills still require a dedicated raw event/log source.",
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def _build_identity_index(player_rows: list[dict[str, object]]) -> dict[str, _PlayerIdentity]:
|
||||
identity_index: dict[str, _PlayerIdentity] = {}
|
||||
for player_row in player_rows:
|
||||
identity = _build_player_identity(player_row)
|
||||
if identity is None or not identity.display_name:
|
||||
continue
|
||||
identity_index[_normalize_name(identity.display_name)] = identity
|
||||
return identity_index
|
||||
|
||||
|
||||
def _build_player_identity(player_row: dict[str, object]) -> _PlayerIdentity | None:
|
||||
display_name = _stringify(player_row.get("player")) or _stringify(player_row.get("name"))
|
||||
source_player_id = _stringify(player_row.get("player_id")) or _stringify(player_row.get("id"))
|
||||
steam_id = _extract_steam_id(player_row.get("steaminfo"))
|
||||
stable_player_key = _build_stable_player_key(steam_id=steam_id, source_player_id=source_player_id)
|
||||
if stable_player_key is None:
|
||||
return None
|
||||
return _PlayerIdentity(
|
||||
stable_player_key=stable_player_key,
|
||||
display_name=display_name or stable_player_key,
|
||||
)
|
||||
|
||||
|
||||
def _find_identity_by_name(
|
||||
identity_index: dict[str, _PlayerIdentity],
|
||||
player_name: str | None,
|
||||
) -> _PlayerIdentity | None:
|
||||
if not player_name:
|
||||
return None
|
||||
return identity_index.get(_normalize_name(player_name))
|
||||
|
||||
|
||||
def _build_event(
|
||||
*,
|
||||
event_type: str,
|
||||
occurred_at: str | None,
|
||||
server_slug: str,
|
||||
match_id: str,
|
||||
source_kind: str,
|
||||
source_ref: str | None,
|
||||
raw_event_ref: str,
|
||||
killer: _PlayerIdentity | None,
|
||||
victim: _PlayerIdentity | None,
|
||||
weapon_name: str | None,
|
||||
kill_category: str | None,
|
||||
is_teamkill: bool,
|
||||
event_value: int,
|
||||
) -> PlayerEventRecord:
|
||||
event_id = _build_event_id(
|
||||
event_type=event_type,
|
||||
occurred_at=occurred_at,
|
||||
server_slug=server_slug,
|
||||
match_id=match_id,
|
||||
killer_player_key=killer.stable_player_key if killer else None,
|
||||
victim_player_key=victim.stable_player_key if victim else None,
|
||||
weapon_name=weapon_name,
|
||||
is_teamkill=is_teamkill,
|
||||
event_value=event_value,
|
||||
)
|
||||
return PlayerEventRecord(
|
||||
event_id=event_id,
|
||||
event_type=event_type,
|
||||
occurred_at=occurred_at,
|
||||
server_slug=server_slug,
|
||||
external_match_id=match_id,
|
||||
source_kind=source_kind,
|
||||
source_ref=source_ref,
|
||||
raw_event_ref=raw_event_ref,
|
||||
killer_player_key=killer.stable_player_key if killer else None,
|
||||
killer_display_name=killer.display_name if killer else None,
|
||||
victim_player_key=victim.stable_player_key if victim else None,
|
||||
victim_display_name=victim.display_name if victim else None,
|
||||
weapon_name=weapon_name,
|
||||
weapon_category=None,
|
||||
kill_category=kill_category,
|
||||
is_teamkill=is_teamkill,
|
||||
event_value=max(1, event_value),
|
||||
)
|
||||
|
||||
|
||||
def _build_event_id(
|
||||
*,
|
||||
event_type: str,
|
||||
occurred_at: str | None,
|
||||
server_slug: str,
|
||||
match_id: str,
|
||||
killer_player_key: str | None,
|
||||
victim_player_key: str | None,
|
||||
weapon_name: str | None,
|
||||
is_teamkill: bool,
|
||||
event_value: int,
|
||||
) -> str:
|
||||
raw_key = "|".join(
|
||||
[
|
||||
event_type,
|
||||
occurred_at or "",
|
||||
server_slug,
|
||||
match_id,
|
||||
killer_player_key or "",
|
||||
victim_player_key or "",
|
||||
weapon_name or "",
|
||||
"1" if is_teamkill else "0",
|
||||
str(event_value),
|
||||
]
|
||||
)
|
||||
return hashlib.sha1(raw_key.encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def _pick_match_timestamp(match_payload: Mapping[str, object]) -> str | None:
|
||||
for key in ("end", "start", "creation_time"):
|
||||
value = _stringify(match_payload.get(key))
|
||||
if value:
|
||||
return value
|
||||
return None
|
||||
|
||||
|
||||
def _extract_primary_name(value: object) -> str | None:
|
||||
named_counts = _extract_named_counts(value)
|
||||
if not named_counts:
|
||||
return None
|
||||
return named_counts[0][0]
|
||||
|
||||
|
||||
def _extract_named_counts(value: object) -> list[tuple[str, int]]:
|
||||
aggregated: dict[str, tuple[str, int]] = {}
|
||||
for name, count in _iter_named_counts(value):
|
||||
normalized_name = _normalize_name(name)
|
||||
existing = aggregated.get(normalized_name)
|
||||
if existing is None:
|
||||
aggregated[normalized_name] = (name, count)
|
||||
continue
|
||||
aggregated[normalized_name] = (existing[0], existing[1] + count)
|
||||
return sorted(
|
||||
aggregated.values(),
|
||||
key=lambda item: (-item[1], item[0].casefold()),
|
||||
)
|
||||
|
||||
|
||||
def _iter_named_counts(value: object) -> list[tuple[str, int]]:
|
||||
if isinstance(value, str):
|
||||
name = _stringify(value)
|
||||
return [(name, 1)] if name else []
|
||||
if isinstance(value, Mapping):
|
||||
named_count = _extract_named_count_mapping(value)
|
||||
if named_count is not None:
|
||||
return [named_count]
|
||||
|
||||
items: list[tuple[str, int]] = []
|
||||
for raw_name, raw_count in value.items():
|
||||
name = _stringify(raw_name)
|
||||
count = _coerce_int(raw_count)
|
||||
if name and count and count > 0:
|
||||
items.append((name, count))
|
||||
return items
|
||||
if isinstance(value, list):
|
||||
items: list[tuple[str, int]] = []
|
||||
for item in value:
|
||||
items.extend(_iter_named_counts(item))
|
||||
return items
|
||||
return []
|
||||
|
||||
|
||||
def _extract_named_count_mapping(value: Mapping[str, object]) -> tuple[str, int] | None:
|
||||
nested_name = None
|
||||
nested_player = value.get("player")
|
||||
if isinstance(nested_player, Mapping):
|
||||
nested_name = _stringify(nested_player.get("name")) or _stringify(nested_player.get("player"))
|
||||
name = (
|
||||
_stringify(value.get("name"))
|
||||
or _stringify(value.get("player"))
|
||||
or _stringify(value.get("victim"))
|
||||
or _stringify(value.get("killer"))
|
||||
or nested_name
|
||||
)
|
||||
if not name:
|
||||
return None
|
||||
count = (
|
||||
_coerce_int(value.get("count"))
|
||||
or _coerce_int(value.get("kills"))
|
||||
or _coerce_int(value.get("deaths"))
|
||||
or _coerce_int(value.get("value"))
|
||||
or _coerce_int(value.get("total"))
|
||||
or 1
|
||||
)
|
||||
return name, max(1, count)
|
||||
|
||||
|
||||
def _extract_steam_id(value: object) -> str | None:
|
||||
if isinstance(value, Mapping):
|
||||
profile = value.get("profile")
|
||||
if isinstance(profile, Mapping):
|
||||
steam_id = _stringify(profile.get("steamid"))
|
||||
if steam_id:
|
||||
return steam_id
|
||||
return _stringify(value.get("id"))
|
||||
return None
|
||||
|
||||
|
||||
def _build_stable_player_key(
|
||||
*,
|
||||
steam_id: str | None,
|
||||
source_player_id: str | None,
|
||||
) -> str | None:
|
||||
if steam_id:
|
||||
return f"steam:{steam_id}"
|
||||
if source_player_id:
|
||||
return f"crcon-player:{source_player_id}"
|
||||
return None
|
||||
|
||||
|
||||
def _coerce_player_rows(value: object) -> list[dict[str, object]]:
|
||||
if not isinstance(value, list):
|
||||
return []
|
||||
return [item for item in value if isinstance(item, dict)]
|
||||
|
||||
|
||||
def _normalize_name(value: str) -> str:
|
||||
return value.strip().casefold()
|
||||
|
||||
|
||||
def _stringify(value: object) -> str | None:
|
||||
if value is None:
|
||||
return None
|
||||
text = str(value).strip()
|
||||
return text or None
|
||||
|
||||
|
||||
def _coerce_int(value: object) -> int | None:
|
||||
if value in (None, ""):
|
||||
return None
|
||||
try:
|
||||
return int(value)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
139
backend/app/providers/public_scoreboard_provider.py
Normal file
139
backend/app/providers/public_scoreboard_provider.py
Normal file
@@ -0,0 +1,139 @@
|
||||
"""Public scoreboard provider adapter for historical HLL data."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import time
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from dataclasses import dataclass
|
||||
from urllib.error import HTTPError, URLError
|
||||
from urllib.parse import urlencode
|
||||
from urllib.request import Request, urlopen
|
||||
|
||||
from ..config import (
|
||||
get_historical_crcon_request_retries,
|
||||
get_historical_crcon_request_timeout_seconds,
|
||||
get_historical_crcon_retry_delay_seconds,
|
||||
)
|
||||
|
||||
|
||||
PUBLIC_INFO_ENDPOINT = "/api/get_public_info"
|
||||
MATCH_LIST_ENDPOINT = "/api/get_scoreboard_maps"
|
||||
MATCH_DETAIL_ENDPOINT = "/api/get_map_scoreboard"
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class PublicScoreboardHistoricalDataSource:
|
||||
"""Historical provider backed by the public CRCON scoreboard JSON API."""
|
||||
|
||||
source_kind: str = "public-scoreboard"
|
||||
|
||||
def fetch_public_info(self, *, base_url: str) -> dict[str, object]:
|
||||
return self._fetch_dict_payload(base_url, PUBLIC_INFO_ENDPOINT)
|
||||
|
||||
def fetch_match_page(self, *, base_url: str, page: int, limit: int) -> dict[str, object]:
|
||||
return self._fetch_dict_payload(
|
||||
base_url,
|
||||
MATCH_LIST_ENDPOINT,
|
||||
{"page": page, "limit": limit},
|
||||
context=f"page={page}",
|
||||
)
|
||||
|
||||
def fetch_match_details(
|
||||
self,
|
||||
*,
|
||||
base_url: str,
|
||||
match_ids: list[str],
|
||||
max_workers: int,
|
||||
) -> list[dict[str, object]]:
|
||||
if not match_ids:
|
||||
return []
|
||||
if max_workers <= 1:
|
||||
return [
|
||||
self._fetch_match_detail(base_url=base_url, match_id=match_id)
|
||||
for match_id in match_ids
|
||||
]
|
||||
|
||||
with ThreadPoolExecutor(max_workers=max_workers) as executor:
|
||||
futures = [
|
||||
executor.submit(self._fetch_match_detail, base_url=base_url, match_id=match_id)
|
||||
for match_id in match_ids
|
||||
]
|
||||
return [future.result() for future in futures]
|
||||
|
||||
def _fetch_match_detail(self, *, base_url: str, match_id: str) -> dict[str, object]:
|
||||
return self._fetch_dict_payload(
|
||||
base_url,
|
||||
MATCH_DETAIL_ENDPOINT,
|
||||
{"map_id": match_id},
|
||||
context=f"match={match_id}",
|
||||
)
|
||||
|
||||
def _fetch_json(
|
||||
self,
|
||||
*,
|
||||
base_url: str,
|
||||
endpoint: str,
|
||||
query: dict[str, object] | None = None,
|
||||
) -> object:
|
||||
url = f"{base_url}{endpoint}"
|
||||
if query:
|
||||
url = f"{url}?{urlencode(query)}"
|
||||
|
||||
request = Request(
|
||||
url,
|
||||
headers={
|
||||
"Accept": "application/json",
|
||||
"User-Agent": "HLL-Vietnam-Historical-Ingestion/0.1",
|
||||
},
|
||||
)
|
||||
try:
|
||||
with urlopen(
|
||||
request,
|
||||
timeout=get_historical_crcon_request_timeout_seconds(),
|
||||
) as response:
|
||||
return json.loads(response.read().decode("utf-8"))
|
||||
except HTTPError as exc:
|
||||
raise RuntimeError(f"Historical provider request failed: {url} ({exc.code})") from exc
|
||||
except URLError as exc:
|
||||
raise RuntimeError(f"Historical provider request failed: {url} ({exc.reason})") from exc
|
||||
|
||||
def _fetch_dict_payload(
|
||||
self,
|
||||
base_url: str,
|
||||
endpoint: str,
|
||||
query: dict[str, object] | None = None,
|
||||
*,
|
||||
context: str = "",
|
||||
retries: int | None = None,
|
||||
) -> dict[str, object]:
|
||||
resolved_retries = retries or get_historical_crcon_request_retries()
|
||||
base_retry_delay_seconds = get_historical_crcon_retry_delay_seconds()
|
||||
last_error: Exception | None = None
|
||||
for attempt in range(1, resolved_retries + 1):
|
||||
try:
|
||||
payload = _unwrap_result(
|
||||
self._fetch_json(base_url=base_url, endpoint=endpoint, query=query)
|
||||
)
|
||||
except Exception as exc: # pragma: no cover - network path
|
||||
last_error = exc
|
||||
else:
|
||||
if isinstance(payload, dict):
|
||||
return payload
|
||||
last_error = ValueError(
|
||||
f"Unexpected payload type for {base_url}{endpoint} {context}".strip()
|
||||
)
|
||||
|
||||
if attempt < resolved_retries:
|
||||
time.sleep(base_retry_delay_seconds * attempt)
|
||||
|
||||
assert last_error is not None
|
||||
raise last_error
|
||||
|
||||
|
||||
def _unwrap_result(payload: object) -> object:
|
||||
if not isinstance(payload, dict):
|
||||
return payload
|
||||
if "result" not in payload:
|
||||
return payload
|
||||
return payload.get("result")
|
||||
67
backend/app/providers/rcon_provider.py
Normal file
67
backend/app/providers/rcon_provider.py
Normal file
@@ -0,0 +1,67 @@
|
||||
"""RCON provider adapter for live HLL server state."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
|
||||
from ..rcon_client import (
|
||||
RconServerTarget,
|
||||
load_rcon_targets,
|
||||
query_live_server_sample,
|
||||
)
|
||||
from ..snapshots import build_snapshot_batch, utc_now
|
||||
from ..storage import persist_snapshot_batch
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class RconLiveDataSource:
|
||||
"""Live provider backed by direct HLL RCON access."""
|
||||
|
||||
source_kind: str = "rcon"
|
||||
|
||||
def collect_snapshots(self, *, persist: bool) -> dict[str, object]:
|
||||
configured_targets = load_rcon_targets()
|
||||
if not configured_targets:
|
||||
raise RuntimeError("No RCON targets configured in HLL_BACKEND_RCON_TARGETS.")
|
||||
|
||||
captured_at = utc_now()
|
||||
normalized_records: list[dict[str, object]] = []
|
||||
errors: list[dict[str, object]] = []
|
||||
|
||||
for target in configured_targets:
|
||||
try:
|
||||
normalized_records.append(query_live_server_sample(target)["normalized"])
|
||||
except Exception as error: # noqa: BLE001 - keep provider failures controlled
|
||||
errors.append(
|
||||
{
|
||||
"target": target.name,
|
||||
"host": target.host,
|
||||
"port": target.port,
|
||||
"message": str(error),
|
||||
}
|
||||
)
|
||||
|
||||
payload = {
|
||||
"source_name": "hll-rcon",
|
||||
"collection_mode": "rcon",
|
||||
"fallback_used": False,
|
||||
"target_count": len(configured_targets),
|
||||
"success_count": len(normalized_records),
|
||||
"errors": errors,
|
||||
"captured_at": captured_at.isoformat().replace("+00:00", "Z"),
|
||||
"snapshots": build_snapshot_batch(normalized_records, captured_at=captured_at),
|
||||
}
|
||||
if persist:
|
||||
payload["storage"] = persist_snapshot_batch(
|
||||
payload["snapshots"],
|
||||
source_name=payload["source_name"],
|
||||
captured_at=payload["captured_at"],
|
||||
)
|
||||
return payload
|
||||
|
||||
def build_target_index(self) -> dict[str | None, RconServerTarget]:
|
||||
return {
|
||||
target.external_server_id: target
|
||||
for target in load_rcon_targets()
|
||||
if target.external_server_id
|
||||
}
|
||||
147
backend/app/rcon_admin_log_ingestion.py
Normal file
147
backend/app/rcon_admin_log_ingestion.py
Normal file
@@ -0,0 +1,147 @@
|
||||
"""Manual ingestion of Hell Let Loose RCON AdminLog events."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
from dataclasses import dataclass
|
||||
|
||||
from .config import get_rcon_request_timeout_seconds
|
||||
from .rcon_admin_log_storage import (
|
||||
list_rcon_admin_log_event_counts,
|
||||
persist_rcon_admin_log_entries,
|
||||
)
|
||||
from .rcon_client import HllRconConnection, build_rcon_target_key, load_rcon_targets
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class AdminLogIngestionStats:
|
||||
targets_seen: int = 0
|
||||
events_seen: int = 0
|
||||
events_inserted: int = 0
|
||||
duplicate_events: int = 0
|
||||
failed_targets: int = 0
|
||||
|
||||
|
||||
def ingest_rcon_admin_logs(
|
||||
*,
|
||||
minutes: int,
|
||||
target_key: str | None = None,
|
||||
) -> dict[str, object]:
|
||||
"""Fetch and persist recent AdminLog entries from configured RCON targets."""
|
||||
selected_targets = _select_targets(target_key)
|
||||
stats = AdminLogIngestionStats()
|
||||
targets: list[dict[str, object]] = []
|
||||
errors: list[dict[str, object]] = []
|
||||
timeout_seconds = get_rcon_request_timeout_seconds()
|
||||
|
||||
for target in selected_targets:
|
||||
stats.targets_seen += 1
|
||||
target_metadata = _serialize_target(target)
|
||||
|
||||
try:
|
||||
with HllRconConnection(timeout_seconds=timeout_seconds) as connection:
|
||||
connection.connect(host=target.host, port=target.port, password=target.password)
|
||||
payload = connection.execute_json(
|
||||
"GetAdminLog",
|
||||
{
|
||||
"LogBackTrackTime": minutes * 60,
|
||||
"Filters": [],
|
||||
},
|
||||
)
|
||||
|
||||
entries = payload.get("entries")
|
||||
if not isinstance(entries, list):
|
||||
entries = []
|
||||
|
||||
normalized_entries = [entry for entry in entries if isinstance(entry, dict)]
|
||||
delta = persist_rcon_admin_log_entries(
|
||||
target=target_metadata,
|
||||
entries=normalized_entries,
|
||||
)
|
||||
|
||||
stats.events_seen += int(delta["events_seen"])
|
||||
stats.events_inserted += int(delta["events_inserted"])
|
||||
stats.duplicate_events += int(delta["duplicate_events"])
|
||||
targets.append(
|
||||
{
|
||||
**target_metadata,
|
||||
"status": "ok",
|
||||
"minutes": minutes,
|
||||
**delta,
|
||||
}
|
||||
)
|
||||
except Exception as exc: # noqa: BLE001 - manual diagnostic command reports per-target failures
|
||||
stats.failed_targets += 1
|
||||
errors.append(
|
||||
{
|
||||
**target_metadata,
|
||||
"status": "error",
|
||||
"error_type": type(exc).__name__,
|
||||
"message": str(exc),
|
||||
}
|
||||
)
|
||||
|
||||
return {
|
||||
"status": "ok" if not errors else ("partial" if targets else "error"),
|
||||
"target_scope": target_key or "all-configured-rcon-targets",
|
||||
"minutes": minutes,
|
||||
"targets": targets,
|
||||
"errors": errors,
|
||||
"totals": {
|
||||
"targets_seen": stats.targets_seen,
|
||||
"events_seen": stats.events_seen,
|
||||
"events_inserted": stats.events_inserted,
|
||||
"duplicate_events": stats.duplicate_events,
|
||||
"failed_targets": stats.failed_targets,
|
||||
},
|
||||
"event_counts": list_rcon_admin_log_event_counts(),
|
||||
}
|
||||
|
||||
|
||||
def _select_targets(target_key: str | None) -> list[object]:
|
||||
configured_targets = list(load_rcon_targets())
|
||||
if not configured_targets:
|
||||
raise RuntimeError("No RCON targets configured in HLL_BACKEND_RCON_TARGETS.")
|
||||
if target_key is None:
|
||||
return configured_targets
|
||||
|
||||
normalized = target_key.strip()
|
||||
selected = [
|
||||
target
|
||||
for target in configured_targets
|
||||
if build_rcon_target_key(target) == normalized
|
||||
]
|
||||
if not selected:
|
||||
raise ValueError(f"Unknown RCON target key: {target_key}")
|
||||
return selected
|
||||
|
||||
|
||||
def _serialize_target(target: object) -> dict[str, object]:
|
||||
return {
|
||||
"target_key": build_rcon_target_key(target),
|
||||
"external_server_id": target.external_server_id,
|
||||
"name": target.name,
|
||||
"host": target.host,
|
||||
"port": target.port,
|
||||
"source_name": target.source_name,
|
||||
}
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--minutes", type=int, default=60)
|
||||
parser.add_argument("--target", default=None)
|
||||
args = parser.parse_args()
|
||||
|
||||
print(
|
||||
json.dumps(
|
||||
ingest_rcon_admin_logs(minutes=args.minutes, target_key=args.target),
|
||||
ensure_ascii=False,
|
||||
indent=2,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
863
backend/app/rcon_admin_log_materialization.py
Normal file
863
backend/app/rcon_admin_log_materialization.py
Normal file
@@ -0,0 +1,863 @@
|
||||
"""Materialize RCON AdminLog events into match and player-stat read models."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import sqlite3
|
||||
from collections import Counter
|
||||
from collections.abc import Iterable
|
||||
from contextlib import closing
|
||||
from pathlib import Path
|
||||
|
||||
from .config import get_storage_path, use_postgres_rcon_storage
|
||||
from .normalizers import normalize_map_name
|
||||
from .rcon_admin_log_storage import initialize_rcon_admin_log_storage
|
||||
from .rcon_historical_storage import list_rcon_historical_competitive_windows
|
||||
from .sqlite_utils import connect_sqlite_readonly, connect_sqlite_writer
|
||||
|
||||
|
||||
MATCH_RESULT_SOURCE = "admin-log-match-ended"
|
||||
SESSION_RESULT_SOURCE = "rcon-session"
|
||||
|
||||
|
||||
def initialize_rcon_materialized_storage(*, db_path: Path | None = None) -> Path:
|
||||
"""Create SQLite structures used by the materialized RCON match pipeline."""
|
||||
if use_postgres_rcon_storage(explicit_sqlite_path=db_path):
|
||||
from .postgres_rcon_storage import initialize_postgres_rcon_storage
|
||||
|
||||
initialize_postgres_rcon_storage()
|
||||
return get_storage_path()
|
||||
|
||||
resolved_path = initialize_rcon_admin_log_storage(db_path=db_path)
|
||||
with closing(connect_sqlite_writer(resolved_path)) as connection:
|
||||
with connection:
|
||||
connection.executescript(
|
||||
"""
|
||||
CREATE TABLE IF NOT EXISTS rcon_materialized_matches (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
target_key TEXT NOT NULL,
|
||||
external_server_id TEXT,
|
||||
match_key TEXT NOT NULL,
|
||||
map_name TEXT,
|
||||
map_pretty_name TEXT,
|
||||
game_mode TEXT,
|
||||
started_server_time INTEGER,
|
||||
ended_server_time INTEGER,
|
||||
started_at TEXT,
|
||||
ended_at TEXT,
|
||||
allied_score INTEGER,
|
||||
axis_score INTEGER,
|
||||
winner TEXT,
|
||||
confidence_mode TEXT NOT NULL,
|
||||
source_basis TEXT NOT NULL,
|
||||
created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
UNIQUE(target_key, match_key)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_rcon_materialized_matches_recent
|
||||
ON rcon_materialized_matches(target_key, ended_at DESC, ended_server_time DESC);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS rcon_match_player_stats (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
target_key TEXT NOT NULL,
|
||||
match_key TEXT NOT NULL,
|
||||
player_id TEXT NOT NULL,
|
||||
player_name TEXT NOT NULL,
|
||||
team TEXT,
|
||||
kills INTEGER NOT NULL DEFAULT 0,
|
||||
deaths INTEGER NOT NULL DEFAULT 0,
|
||||
teamkills INTEGER NOT NULL DEFAULT 0,
|
||||
deaths_by_teamkill INTEGER NOT NULL DEFAULT 0,
|
||||
weapons_json TEXT NOT NULL DEFAULT '{}',
|
||||
death_by_weapons_json TEXT NOT NULL DEFAULT '{}',
|
||||
most_killed_json TEXT NOT NULL DEFAULT '{}',
|
||||
death_by_json TEXT NOT NULL DEFAULT '{}',
|
||||
first_seen_server_time INTEGER,
|
||||
last_seen_server_time INTEGER,
|
||||
created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
UNIQUE(target_key, match_key, player_id)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_rcon_match_player_stats_match
|
||||
ON rcon_match_player_stats(target_key, match_key);
|
||||
"""
|
||||
)
|
||||
return resolved_path
|
||||
|
||||
|
||||
def materialize_rcon_admin_log(*, db_path: Path | None = None) -> dict[str, object]:
|
||||
"""Materialize matches and player stats from stored AdminLog events."""
|
||||
resolved_path = initialize_rcon_materialized_storage(db_path=db_path)
|
||||
if use_postgres_rcon_storage(explicit_sqlite_path=db_path):
|
||||
from .postgres_rcon_storage import connect_postgres_compat
|
||||
|
||||
with connect_postgres_compat() as connection:
|
||||
payload = _materialize_rcon_admin_log_with_connection(
|
||||
connection,
|
||||
session_window_db_path=None,
|
||||
caught_errors=(Exception,),
|
||||
)
|
||||
freshness = summarize_rcon_materialization_status()
|
||||
return {
|
||||
**payload,
|
||||
"latest_materialized_matches": freshness["latest_materialized_matches"],
|
||||
"latest_admin_log_match_end_events": freshness["latest_admin_log_match_end_events"],
|
||||
"match_end_status": freshness["match_end_status"],
|
||||
}
|
||||
|
||||
with closing(connect_sqlite_writer(resolved_path)) as connection:
|
||||
with connection:
|
||||
payload = _materialize_rcon_admin_log_with_connection(
|
||||
connection,
|
||||
session_window_db_path=resolved_path,
|
||||
caught_errors=(sqlite3.Error,),
|
||||
)
|
||||
|
||||
freshness = summarize_rcon_materialization_status(db_path=resolved_path)
|
||||
return {
|
||||
**payload,
|
||||
"latest_materialized_matches": freshness["latest_materialized_matches"],
|
||||
"latest_admin_log_match_end_events": freshness["latest_admin_log_match_end_events"],
|
||||
"match_end_status": freshness["match_end_status"],
|
||||
}
|
||||
|
||||
|
||||
def _materialize_rcon_admin_log_with_connection(
|
||||
connection: object,
|
||||
*,
|
||||
session_window_db_path: Path | None,
|
||||
caught_errors: tuple[type[BaseException], ...],
|
||||
) -> dict[str, object]:
|
||||
errors: list[str] = []
|
||||
matches_seen = 0
|
||||
matches_materialized = 0
|
||||
matches_updated = 0
|
||||
player_stats_seen = 0
|
||||
player_stats_materialized = 0
|
||||
player_stats_updated = 0
|
||||
|
||||
try:
|
||||
match_rows = _derive_admin_log_matches(connection)
|
||||
matches_seen = len(match_rows)
|
||||
for row in match_rows:
|
||||
outcome = _upsert_match(connection, row)
|
||||
matches_materialized += int(outcome == "inserted")
|
||||
matches_updated += int(outcome == "updated")
|
||||
session_rows = _derive_session_fallback_matches(
|
||||
connection,
|
||||
db_path=session_window_db_path,
|
||||
)
|
||||
matches_seen += len(session_rows)
|
||||
for row in session_rows:
|
||||
outcome = _upsert_match(connection, row)
|
||||
matches_materialized += int(outcome == "inserted")
|
||||
matches_updated += int(outcome == "updated")
|
||||
|
||||
persisted_matches = _list_materialized_matches(connection)
|
||||
for match in persisted_matches:
|
||||
stats = _derive_player_stats_for_match(connection, match)
|
||||
player_stats_seen += len(stats)
|
||||
connection.execute(
|
||||
"""
|
||||
DELETE FROM rcon_match_player_stats
|
||||
WHERE target_key = ? AND match_key = ?
|
||||
""",
|
||||
(match["target_key"], match["match_key"]),
|
||||
)
|
||||
for stat in stats:
|
||||
_insert_player_stat(connection, stat)
|
||||
player_stats_materialized += 1
|
||||
except caught_errors as error:
|
||||
errors.append(str(error))
|
||||
return {
|
||||
"matches_seen": matches_seen,
|
||||
"matches_materialized": matches_materialized,
|
||||
"matches_updated": matches_updated,
|
||||
"player_stats_seen": player_stats_seen,
|
||||
"player_stats_materialized": player_stats_materialized,
|
||||
"player_stats_updated": player_stats_updated,
|
||||
"errors": errors,
|
||||
}
|
||||
|
||||
|
||||
def list_materialized_rcon_matches(
|
||||
*,
|
||||
target_key: str | None = None,
|
||||
only_ended: bool = False,
|
||||
limit: int = 20,
|
||||
db_path: Path | None = None,
|
||||
) -> list[dict[str, object]]:
|
||||
"""Return recent materialized RCON matches."""
|
||||
resolved_path = initialize_rcon_materialized_storage(db_path=db_path)
|
||||
clauses: list[str] = []
|
||||
params: list[object] = []
|
||||
if target_key:
|
||||
clauses.append("(m.target_key = ? OR m.external_server_id = ?)")
|
||||
params.extend([target_key, target_key])
|
||||
if only_ended:
|
||||
clauses.append("m.source_basis = ?")
|
||||
params.append(MATCH_RESULT_SOURCE)
|
||||
where = "WHERE " + " AND ".join(clauses) if clauses else ""
|
||||
params.append(limit)
|
||||
if use_postgres_rcon_storage(explicit_sqlite_path=db_path):
|
||||
from .postgres_rcon_storage import connect_postgres_compat
|
||||
|
||||
connection_scope = connect_postgres_compat()
|
||||
else:
|
||||
connection_scope = closing(connect_sqlite_readonly(resolved_path))
|
||||
with connection_scope as connection:
|
||||
rows = connection.execute(
|
||||
f"""
|
||||
SELECT
|
||||
m.*,
|
||||
(
|
||||
SELECT COUNT(*)
|
||||
FROM rcon_match_player_stats AS stats
|
||||
WHERE stats.target_key = m.target_key
|
||||
AND stats.match_key = m.match_key
|
||||
) AS materialized_player_count,
|
||||
(
|
||||
SELECT COUNT(DISTINCT TRIM(stats.player_name))
|
||||
FROM rcon_match_player_stats AS stats
|
||||
WHERE stats.target_key = m.target_key
|
||||
AND stats.match_key = m.match_key
|
||||
AND TRIM(COALESCE(stats.player_name, '')) != ''
|
||||
) AS materialized_distinct_player_count
|
||||
FROM rcon_materialized_matches AS m
|
||||
{where}
|
||||
ORDER BY COALESCE(m.ended_at, m.started_at) DESC,
|
||||
COALESCE(m.ended_server_time, m.started_server_time) DESC
|
||||
LIMIT ?
|
||||
""",
|
||||
params,
|
||||
).fetchall()
|
||||
return [dict(row) for row in rows]
|
||||
|
||||
|
||||
def get_materialized_rcon_match_detail(
|
||||
*,
|
||||
server_key: str,
|
||||
match_key: str,
|
||||
db_path: Path | None = None,
|
||||
) -> dict[str, object] | None:
|
||||
"""Return one materialized match with player stats."""
|
||||
resolved_path = initialize_rcon_materialized_storage(db_path=db_path)
|
||||
if use_postgres_rcon_storage(explicit_sqlite_path=db_path):
|
||||
from .postgres_rcon_storage import connect_postgres_compat
|
||||
|
||||
connection_scope = connect_postgres_compat()
|
||||
else:
|
||||
connection_scope = closing(connect_sqlite_readonly(resolved_path))
|
||||
with connection_scope as connection:
|
||||
match = connection.execute(
|
||||
"""
|
||||
SELECT *
|
||||
FROM rcon_materialized_matches
|
||||
WHERE match_key = ?
|
||||
AND (target_key = ? OR external_server_id = ?)
|
||||
LIMIT 1
|
||||
""",
|
||||
(match_key, server_key, server_key),
|
||||
).fetchone()
|
||||
if match is None and match_key.startswith(f"{server_key}:"):
|
||||
match = connection.execute(
|
||||
"""
|
||||
SELECT *
|
||||
FROM rcon_materialized_matches
|
||||
WHERE match_key = ?
|
||||
LIMIT 1
|
||||
""",
|
||||
(match_key,),
|
||||
).fetchone()
|
||||
if match is None:
|
||||
return None
|
||||
stat_rows = connection.execute(
|
||||
"""
|
||||
SELECT *
|
||||
FROM rcon_match_player_stats
|
||||
WHERE target_key = ? AND match_key = ?
|
||||
ORDER BY kills DESC, deaths ASC, player_name ASC
|
||||
""",
|
||||
(match["target_key"], match["match_key"]),
|
||||
).fetchall()
|
||||
timeline_rows = connection.execute(
|
||||
"""
|
||||
SELECT event_type, COUNT(*) AS event_count
|
||||
FROM rcon_admin_log_events
|
||||
WHERE target_key = ?
|
||||
AND server_time IS NOT NULL
|
||||
AND (? IS NULL OR server_time >= ?)
|
||||
AND (? IS NULL OR server_time <= ?)
|
||||
GROUP BY event_type
|
||||
ORDER BY event_count DESC, event_type ASC
|
||||
""",
|
||||
(
|
||||
match["target_key"],
|
||||
match["started_server_time"],
|
||||
match["started_server_time"],
|
||||
match["ended_server_time"],
|
||||
match["ended_server_time"],
|
||||
),
|
||||
).fetchall()
|
||||
|
||||
return {
|
||||
"match": dict(match),
|
||||
"players": [dict(row) for row in stat_rows],
|
||||
"timeline": [dict(row) for row in timeline_rows],
|
||||
}
|
||||
|
||||
|
||||
def summarize_rcon_materialization_status(*, db_path: Path | None = None) -> dict[str, object]:
|
||||
"""Return a small diagnostic summary for stored RCON materialization state."""
|
||||
resolved_path = initialize_rcon_materialized_storage(db_path=db_path)
|
||||
if use_postgres_rcon_storage(explicit_sqlite_path=db_path):
|
||||
from .postgres_rcon_storage import connect_postgres_compat
|
||||
|
||||
connection_scope = connect_postgres_compat()
|
||||
else:
|
||||
connection_scope = closing(connect_sqlite_readonly(resolved_path))
|
||||
with connection_scope as connection:
|
||||
match_count = connection.execute(
|
||||
"SELECT COUNT(*) AS count FROM rcon_materialized_matches"
|
||||
).fetchone()["count"]
|
||||
stats_match_count = connection.execute(
|
||||
"""
|
||||
SELECT COUNT(*) AS count
|
||||
FROM (
|
||||
SELECT 1
|
||||
FROM rcon_match_player_stats
|
||||
GROUP BY target_key, match_key
|
||||
) AS stats_matches
|
||||
"""
|
||||
).fetchone()["count"]
|
||||
ranges = connection.execute(
|
||||
"""
|
||||
SELECT target_key, MIN(server_time) AS first_server_time, MAX(server_time) AS last_server_time
|
||||
FROM rcon_admin_log_events
|
||||
GROUP BY target_key
|
||||
ORDER BY target_key ASC
|
||||
"""
|
||||
).fetchall()
|
||||
event_counts = connection.execute(
|
||||
"""
|
||||
SELECT target_key, event_type, COUNT(*) AS event_count
|
||||
FROM rcon_admin_log_events
|
||||
GROUP BY target_key, event_type
|
||||
ORDER BY target_key ASC, event_count DESC
|
||||
"""
|
||||
).fetchall()
|
||||
latest_matches = connection.execute(
|
||||
"""
|
||||
SELECT
|
||||
target_key,
|
||||
external_server_id,
|
||||
match_key,
|
||||
map_pretty_name,
|
||||
COALESCE(ended_at, started_at) AS closed_at,
|
||||
ended_at,
|
||||
ended_server_time,
|
||||
source_basis,
|
||||
updated_at
|
||||
FROM (
|
||||
SELECT
|
||||
*,
|
||||
ROW_NUMBER() OVER (
|
||||
PARTITION BY target_key
|
||||
ORDER BY COALESCE(ended_at, started_at) DESC,
|
||||
COALESCE(ended_server_time, started_server_time) DESC,
|
||||
updated_at DESC
|
||||
) AS row_number
|
||||
FROM rcon_materialized_matches
|
||||
WHERE source_basis = ?
|
||||
) AS ranked_matches
|
||||
WHERE row_number = 1
|
||||
ORDER BY target_key ASC
|
||||
""",
|
||||
(MATCH_RESULT_SOURCE,),
|
||||
).fetchall()
|
||||
latest_match_end_events = connection.execute(
|
||||
"""
|
||||
SELECT
|
||||
target_key,
|
||||
external_server_id,
|
||||
MAX(event_timestamp) AS latest_event_timestamp,
|
||||
MAX(server_time) AS latest_server_time,
|
||||
COUNT(*) AS match_end_events
|
||||
FROM rcon_admin_log_events
|
||||
WHERE event_type = 'match_end'
|
||||
GROUP BY target_key, external_server_id
|
||||
ORDER BY target_key ASC
|
||||
"""
|
||||
).fetchall()
|
||||
return {
|
||||
"materialized_matches": int(match_count or 0),
|
||||
"matches_with_player_stats": int(stats_match_count or 0),
|
||||
"server_time_ranges": [dict(row) for row in ranges],
|
||||
"event_counts": [dict(row) for row in event_counts],
|
||||
"latest_materialized_matches": [dict(row) for row in latest_matches],
|
||||
"latest_admin_log_match_end_events": [dict(row) for row in latest_match_end_events],
|
||||
"match_end_status": (
|
||||
"admin-log-match-end-events-available"
|
||||
if latest_match_end_events
|
||||
else "no-admin-log-match-end-events-stored"
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def _derive_admin_log_matches(connection: sqlite3.Connection) -> list[dict[str, object]]:
|
||||
rows = connection.execute(
|
||||
"""
|
||||
SELECT *
|
||||
FROM rcon_admin_log_events
|
||||
WHERE event_type IN ('match_start', 'match_end')
|
||||
ORDER BY target_key ASC, server_time ASC, id ASC
|
||||
"""
|
||||
).fetchall()
|
||||
matches: list[dict[str, object]] = []
|
||||
open_by_target: dict[str, sqlite3.Row] = {}
|
||||
for row in rows:
|
||||
target_key = row["target_key"]
|
||||
payload = _json_object(row["parsed_payload_json"])
|
||||
if row["event_type"] == "match_start":
|
||||
if target_key in open_by_target:
|
||||
matches.append(_build_match_row(open_by_target.pop(target_key), None))
|
||||
open_by_target[target_key] = row
|
||||
continue
|
||||
start_row = open_by_target.pop(target_key, None)
|
||||
matches.append(_build_match_row(start_row, row, end_payload=payload))
|
||||
for start_row in open_by_target.values():
|
||||
matches.append(_build_match_row(start_row, None))
|
||||
return matches
|
||||
|
||||
|
||||
def _derive_session_fallback_matches(
|
||||
connection: sqlite3.Connection,
|
||||
*,
|
||||
db_path: Path | None,
|
||||
) -> list[dict[str, object]]:
|
||||
rows: list[dict[str, object]] = []
|
||||
existing = {
|
||||
(row["target_key"], normalize_map_name(row["map_pretty_name"] or row["map_name"]))
|
||||
for row in connection.execute(
|
||||
"""
|
||||
SELECT target_key, map_name, map_pretty_name
|
||||
FROM rcon_materialized_matches
|
||||
WHERE source_basis = ?
|
||||
""",
|
||||
(MATCH_RESULT_SOURCE,),
|
||||
).fetchall()
|
||||
}
|
||||
for window in list_rcon_historical_competitive_windows(limit=100, db_path=db_path):
|
||||
target_key = str(window.get("target_key") or "")
|
||||
map_name = window.get("map_pretty_name") or window.get("map_name")
|
||||
if (target_key, normalize_map_name(map_name)) in existing:
|
||||
continue
|
||||
session_key = str(window.get("session_key") or "").strip()
|
||||
if not target_key or not session_key:
|
||||
continue
|
||||
rows.append(
|
||||
{
|
||||
"target_key": target_key,
|
||||
"external_server_id": window.get("external_server_id"),
|
||||
"match_key": f"session:{session_key}",
|
||||
"map_name": window.get("map_name"),
|
||||
"map_pretty_name": normalize_map_name(map_name),
|
||||
"game_mode": None,
|
||||
"started_server_time": None,
|
||||
"ended_server_time": None,
|
||||
"started_at": window.get("first_seen_at"),
|
||||
"ended_at": window.get("last_seen_at"),
|
||||
"allied_score": _nested_int(window.get("latest_payload"), "allied_score"),
|
||||
"axis_score": _nested_int(window.get("latest_payload"), "axis_score"),
|
||||
"winner": _resolve_winner(
|
||||
_nested_int(window.get("latest_payload"), "allied_score"),
|
||||
_nested_int(window.get("latest_payload"), "axis_score"),
|
||||
),
|
||||
"confidence_mode": "partial",
|
||||
"source_basis": SESSION_RESULT_SOURCE,
|
||||
}
|
||||
)
|
||||
return rows
|
||||
|
||||
|
||||
def _build_match_row(
|
||||
start_row: sqlite3.Row | None,
|
||||
end_row: sqlite3.Row | None,
|
||||
*,
|
||||
end_payload: dict[str, object] | None = None,
|
||||
) -> dict[str, object]:
|
||||
start_payload = _json_object(start_row["parsed_payload_json"]) if start_row else {}
|
||||
end_payload = end_payload or (_json_object(end_row["parsed_payload_json"]) if end_row else {})
|
||||
target_key = str((end_row or start_row)["target_key"])
|
||||
external_server_id = (end_row or start_row)["external_server_id"]
|
||||
started_server_time = start_row["server_time"] if start_row else None
|
||||
ended_server_time = end_row["server_time"] if end_row else None
|
||||
map_name = end_payload.get("map_name") or start_payload.get("map_name")
|
||||
match_key = _build_match_key(
|
||||
target_key=target_key,
|
||||
started_server_time=started_server_time,
|
||||
ended_server_time=ended_server_time,
|
||||
map_name=map_name,
|
||||
)
|
||||
return {
|
||||
"target_key": target_key,
|
||||
"external_server_id": external_server_id,
|
||||
"match_key": match_key,
|
||||
"map_name": map_name,
|
||||
"map_pretty_name": normalize_map_name(map_name),
|
||||
"game_mode": start_payload.get("game_mode"),
|
||||
"started_server_time": started_server_time,
|
||||
"ended_server_time": ended_server_time,
|
||||
"started_at": start_row["event_timestamp"] if start_row else None,
|
||||
"ended_at": end_row["event_timestamp"] if end_row else None,
|
||||
"allied_score": _coerce_int(end_payload.get("allied_score")),
|
||||
"axis_score": _coerce_int(end_payload.get("axis_score")),
|
||||
"winner": end_payload.get("winner")
|
||||
or _resolve_winner(
|
||||
_coerce_int(end_payload.get("allied_score")),
|
||||
_coerce_int(end_payload.get("axis_score")),
|
||||
),
|
||||
"confidence_mode": "exact" if end_row else "partial",
|
||||
"source_basis": MATCH_RESULT_SOURCE if end_row else "admin-log-match-start",
|
||||
}
|
||||
|
||||
|
||||
def _upsert_match(connection: sqlite3.Connection, row: dict[str, object]) -> str:
|
||||
existing = connection.execute(
|
||||
"""
|
||||
SELECT id
|
||||
FROM rcon_materialized_matches
|
||||
WHERE target_key = ? AND match_key = ?
|
||||
""",
|
||||
(row["target_key"], row["match_key"]),
|
||||
).fetchone()
|
||||
connection.execute(
|
||||
"""
|
||||
INSERT INTO rcon_materialized_matches (
|
||||
target_key, external_server_id, match_key, map_name, map_pretty_name, game_mode,
|
||||
started_server_time, ended_server_time, started_at, ended_at,
|
||||
allied_score, axis_score, winner, confidence_mode, source_basis
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
ON CONFLICT(target_key, match_key) DO UPDATE SET
|
||||
external_server_id = excluded.external_server_id,
|
||||
map_name = excluded.map_name,
|
||||
map_pretty_name = excluded.map_pretty_name,
|
||||
game_mode = excluded.game_mode,
|
||||
started_server_time = excluded.started_server_time,
|
||||
ended_server_time = excluded.ended_server_time,
|
||||
started_at = excluded.started_at,
|
||||
ended_at = excluded.ended_at,
|
||||
allied_score = excluded.allied_score,
|
||||
axis_score = excluded.axis_score,
|
||||
winner = excluded.winner,
|
||||
confidence_mode = excluded.confidence_mode,
|
||||
source_basis = excluded.source_basis,
|
||||
updated_at = CURRENT_TIMESTAMP
|
||||
""",
|
||||
(
|
||||
row["target_key"],
|
||||
row.get("external_server_id"),
|
||||
row["match_key"],
|
||||
row.get("map_name"),
|
||||
row.get("map_pretty_name"),
|
||||
row.get("game_mode"),
|
||||
row.get("started_server_time"),
|
||||
row.get("ended_server_time"),
|
||||
row.get("started_at"),
|
||||
row.get("ended_at"),
|
||||
row.get("allied_score"),
|
||||
row.get("axis_score"),
|
||||
row.get("winner"),
|
||||
row["confidence_mode"],
|
||||
row["source_basis"],
|
||||
),
|
||||
)
|
||||
return "updated" if existing else "inserted"
|
||||
|
||||
|
||||
def _list_materialized_matches(connection: sqlite3.Connection) -> list[dict[str, object]]:
|
||||
rows = connection.execute(
|
||||
"""
|
||||
SELECT *
|
||||
FROM rcon_materialized_matches
|
||||
WHERE started_server_time IS NOT NULL OR ended_server_time IS NOT NULL
|
||||
ORDER BY target_key ASC, COALESCE(started_server_time, ended_server_time) ASC
|
||||
"""
|
||||
).fetchall()
|
||||
return [dict(row) for row in rows]
|
||||
|
||||
|
||||
def _derive_player_stats_for_match(
|
||||
connection: sqlite3.Connection,
|
||||
match: dict[str, object],
|
||||
) -> list[dict[str, object]]:
|
||||
lower = match.get("started_server_time")
|
||||
upper = match.get("ended_server_time")
|
||||
if lower is None and upper is None:
|
||||
return []
|
||||
clauses = ["target_key = ?", "server_time IS NOT NULL"]
|
||||
params: list[object] = [match["target_key"]]
|
||||
if lower is not None:
|
||||
clauses.append("server_time >= ?")
|
||||
params.append(lower)
|
||||
if upper is not None:
|
||||
clauses.append("server_time <= ?")
|
||||
params.append(upper)
|
||||
rows = connection.execute(
|
||||
f"""
|
||||
SELECT *
|
||||
FROM rcon_admin_log_events
|
||||
WHERE {" AND ".join(clauses)}
|
||||
AND event_type IN ('kill', 'team_switch', 'connected', 'disconnected', 'chat')
|
||||
ORDER BY server_time ASC, id ASC
|
||||
""",
|
||||
params,
|
||||
).fetchall()
|
||||
|
||||
players: dict[str, dict[str, object]] = {}
|
||||
team_by_player: dict[str, str] = {}
|
||||
for row in rows:
|
||||
payload = _json_object(row["parsed_payload_json"])
|
||||
server_time = _coerce_int(row["server_time"])
|
||||
event_type = row["event_type"]
|
||||
if event_type == "kill":
|
||||
killer_key = _player_key(payload.get("killer_id"), payload.get("killer_name"))
|
||||
victim_key = _player_key(payload.get("victim_id"), payload.get("victim_name"))
|
||||
killer = _ensure_player(
|
||||
players,
|
||||
player_id=killer_key,
|
||||
player_name=payload.get("killer_name"),
|
||||
team=payload.get("killer_team") or team_by_player.get(killer_key),
|
||||
server_time=server_time,
|
||||
)
|
||||
victim = _ensure_player(
|
||||
players,
|
||||
player_id=victim_key,
|
||||
player_name=payload.get("victim_name"),
|
||||
team=payload.get("victim_team") or team_by_player.get(victim_key),
|
||||
server_time=server_time,
|
||||
)
|
||||
team_by_player[killer_key] = str(payload.get("killer_team") or killer.get("team") or "")
|
||||
team_by_player[victim_key] = str(payload.get("victim_team") or victim.get("team") or "")
|
||||
weapon = str(payload.get("weapon") or "Unknown")
|
||||
same_team = payload.get("killer_team") and payload.get("killer_team") == payload.get("victim_team")
|
||||
if same_team:
|
||||
killer["teamkills"] = int(killer["teamkills"]) + 1
|
||||
victim["deaths_by_teamkill"] = int(victim["deaths_by_teamkill"]) + 1
|
||||
else:
|
||||
killer["kills"] = int(killer["kills"]) + 1
|
||||
victim["deaths"] = int(victim["deaths"]) + 1
|
||||
_counter(killer, "weapons")[weapon] += 1
|
||||
_counter(victim, "death_by_weapons")[weapon] += 1
|
||||
_counter(killer, "most_killed")[str(victim["player_name"])] += 1
|
||||
_counter(victim, "death_by")[str(killer["player_name"])] += 1
|
||||
_touch_player(killer, server_time)
|
||||
_touch_player(victim, server_time)
|
||||
continue
|
||||
|
||||
if event_type == "team_switch" and not payload.get("player_id"):
|
||||
continue
|
||||
player_id = _player_key(payload.get("player_id"), payload.get("player_name"))
|
||||
team = payload.get("to_team") or payload.get("chat_team") or team_by_player.get(player_id)
|
||||
player = _ensure_player(
|
||||
players,
|
||||
player_id=player_id,
|
||||
player_name=payload.get("player_name"),
|
||||
team=team,
|
||||
server_time=server_time,
|
||||
)
|
||||
if team:
|
||||
player["team"] = team
|
||||
team_by_player[player_id] = str(team)
|
||||
_touch_player(player, server_time)
|
||||
|
||||
stats = []
|
||||
for player in players.values():
|
||||
stats.append(
|
||||
{
|
||||
"target_key": match["target_key"],
|
||||
"match_key": match["match_key"],
|
||||
"player_id": player["player_id"],
|
||||
"player_name": player["player_name"],
|
||||
"team": player.get("team"),
|
||||
"kills": player["kills"],
|
||||
"deaths": player["deaths"],
|
||||
"teamkills": player["teamkills"],
|
||||
"deaths_by_teamkill": player["deaths_by_teamkill"],
|
||||
"weapons_json": _dump_counter(player["weapons"]),
|
||||
"death_by_weapons_json": _dump_counter(player["death_by_weapons"]),
|
||||
"most_killed_json": _dump_counter(player["most_killed"]),
|
||||
"death_by_json": _dump_counter(player["death_by"]),
|
||||
"first_seen_server_time": player.get("first_seen_server_time"),
|
||||
"last_seen_server_time": player.get("last_seen_server_time"),
|
||||
}
|
||||
)
|
||||
return stats
|
||||
|
||||
|
||||
def _insert_player_stat(connection: sqlite3.Connection, stat: dict[str, object]) -> None:
|
||||
connection.execute(
|
||||
"""
|
||||
INSERT INTO rcon_match_player_stats (
|
||||
target_key, match_key, player_id, player_name, team,
|
||||
kills, deaths, teamkills, deaths_by_teamkill,
|
||||
weapons_json, death_by_weapons_json, most_killed_json, death_by_json,
|
||||
first_seen_server_time, last_seen_server_time
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
""",
|
||||
(
|
||||
stat["target_key"],
|
||||
stat["match_key"],
|
||||
stat["player_id"],
|
||||
stat["player_name"],
|
||||
stat.get("team"),
|
||||
stat["kills"],
|
||||
stat["deaths"],
|
||||
stat["teamkills"],
|
||||
stat["deaths_by_teamkill"],
|
||||
stat["weapons_json"],
|
||||
stat["death_by_weapons_json"],
|
||||
stat["most_killed_json"],
|
||||
stat["death_by_json"],
|
||||
stat.get("first_seen_server_time"),
|
||||
stat.get("last_seen_server_time"),
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _ensure_player(
|
||||
players: dict[str, dict[str, object]],
|
||||
*,
|
||||
player_id: str,
|
||||
player_name: object,
|
||||
team: object,
|
||||
server_time: int | None,
|
||||
) -> dict[str, object]:
|
||||
if player_id not in players:
|
||||
players[player_id] = {
|
||||
"player_id": player_id,
|
||||
"player_name": str(player_name or player_id),
|
||||
"team": team,
|
||||
"kills": 0,
|
||||
"deaths": 0,
|
||||
"teamkills": 0,
|
||||
"deaths_by_teamkill": 0,
|
||||
"weapons": Counter(),
|
||||
"death_by_weapons": Counter(),
|
||||
"most_killed": Counter(),
|
||||
"death_by": Counter(),
|
||||
"first_seen_server_time": server_time,
|
||||
"last_seen_server_time": server_time,
|
||||
}
|
||||
player = players[player_id]
|
||||
if player_name:
|
||||
player["player_name"] = str(player_name)
|
||||
if team:
|
||||
player["team"] = team
|
||||
_touch_player(player, server_time)
|
||||
return player
|
||||
|
||||
|
||||
def _touch_player(player: dict[str, object], server_time: int | None) -> None:
|
||||
if server_time is None:
|
||||
return
|
||||
first_seen = _coerce_int(player.get("first_seen_server_time"))
|
||||
last_seen = _coerce_int(player.get("last_seen_server_time"))
|
||||
player["first_seen_server_time"] = server_time if first_seen is None else min(first_seen, server_time)
|
||||
player["last_seen_server_time"] = server_time if last_seen is None else max(last_seen, server_time)
|
||||
|
||||
|
||||
def _counter(player: dict[str, object], key: str) -> Counter[str]:
|
||||
value = player[key]
|
||||
if isinstance(value, Counter):
|
||||
return value
|
||||
counter: Counter[str] = Counter()
|
||||
player[key] = counter
|
||||
return counter
|
||||
|
||||
|
||||
def _player_key(player_id: object, player_name: object) -> str:
|
||||
raw_id = str(player_id or "").strip()
|
||||
if raw_id:
|
||||
return raw_id
|
||||
return f"name:{str(player_name or 'unknown').strip().lower()}"
|
||||
|
||||
|
||||
def _build_match_key(
|
||||
*,
|
||||
target_key: str,
|
||||
started_server_time: object,
|
||||
ended_server_time: object,
|
||||
map_name: object,
|
||||
) -> str:
|
||||
map_part = "".join(character.lower() for character in str(map_name or "unknown") if character.isalnum())
|
||||
start_part = "missing" if started_server_time is None else str(started_server_time)
|
||||
end_part = "open" if ended_server_time is None else str(ended_server_time)
|
||||
return f"{target_key}:{start_part}:{end_part}:{map_part}"
|
||||
|
||||
|
||||
def _json_object(raw_value: object) -> dict[str, object]:
|
||||
if not isinstance(raw_value, str) or not raw_value.strip():
|
||||
return {}
|
||||
try:
|
||||
parsed = json.loads(raw_value)
|
||||
except json.JSONDecodeError:
|
||||
return {}
|
||||
return parsed if isinstance(parsed, dict) else {}
|
||||
|
||||
|
||||
def _dump_counter(counter: Counter[str]) -> str:
|
||||
ordered = dict(sorted(counter.items(), key=lambda item: (-item[1], item[0])))
|
||||
return json.dumps(ordered, ensure_ascii=False, separators=(",", ":"))
|
||||
|
||||
|
||||
def _coerce_int(value: object) -> int | None:
|
||||
if value is None:
|
||||
return None
|
||||
try:
|
||||
return int(value)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
def _nested_int(payload: object, key: str) -> int | None:
|
||||
if not isinstance(payload, dict):
|
||||
return None
|
||||
return _coerce_int(payload.get(key))
|
||||
|
||||
|
||||
def _resolve_winner(allied_score: int | None, axis_score: int | None) -> str | None:
|
||||
if allied_score is None or axis_score is None:
|
||||
return None
|
||||
if allied_score > axis_score:
|
||||
return "allied"
|
||||
if axis_score > allied_score:
|
||||
return "axis"
|
||||
return "draw"
|
||||
|
||||
|
||||
def _main(argv: Iterable[str] | None = None) -> int:
|
||||
parser = argparse.ArgumentParser(description="Materialize stored RCON AdminLog events.")
|
||||
parser.add_argument(
|
||||
"command",
|
||||
nargs="?",
|
||||
choices=("materialize", "status"),
|
||||
default="materialize",
|
||||
)
|
||||
parser.add_argument("--db-path", type=Path, default=None)
|
||||
args = parser.parse_args(list(argv) if argv is not None else None)
|
||||
db_path = args.db_path or get_storage_path()
|
||||
payload = (
|
||||
summarize_rcon_materialization_status(db_path=db_path)
|
||||
if args.command == "status"
|
||||
else materialize_rcon_admin_log(db_path=db_path)
|
||||
)
|
||||
print(json.dumps({"status": "ok", "data": payload}, ensure_ascii=False, indent=2))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(_main())
|
||||
464
backend/app/rcon_admin_log_parser.py
Normal file
464
backend/app/rcon_admin_log_parser.py
Normal file
@@ -0,0 +1,464 @@
|
||||
"""Parser for Hell Let Loose RCON admin log messages."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from dataclasses import asdict, dataclass
|
||||
from typing import Literal
|
||||
|
||||
|
||||
RconAdminLogEventType = Literal[
|
||||
"match_start",
|
||||
"match_end",
|
||||
"kill",
|
||||
"team_switch",
|
||||
"connected",
|
||||
"disconnected",
|
||||
"chat",
|
||||
"kick",
|
||||
"ban",
|
||||
"message",
|
||||
"unknown",
|
||||
]
|
||||
|
||||
|
||||
_PREFIX_RE = re.compile(
|
||||
r"^\[(?P<relative>.+?)\s+\((?P<server_time>\d+)\)\]\s+(?P<body>.*)$",
|
||||
re.DOTALL,
|
||||
)
|
||||
|
||||
MATCH_START_RE = re.compile(
|
||||
r"^MATCH START\s+(?P<map_name>.+?)\s+(?P<game_mode>[A-Za-z]+)\s*$",
|
||||
re.DOTALL,
|
||||
)
|
||||
|
||||
MATCH_END_RE = re.compile(
|
||||
r"^MATCH ENDED\s+`(?P<map_name>.+?)`\s+ALLIED\s+\((?P<allied_score>\d+)\s*-\s*(?P<axis_score>\d+)\)\s+AXIS\s*$",
|
||||
re.DOTALL,
|
||||
)
|
||||
|
||||
KILL_RE = re.compile(
|
||||
r"^KILL:\s+"
|
||||
r"(?P<killer_name>.+?)"
|
||||
r"\((?P<killer_team>Allies|Axis|None)/(?P<killer_id>[^)]*)\)"
|
||||
r"\s+->\s+"
|
||||
r"(?P<victim_name>.+?)"
|
||||
r"\((?P<victim_team>Allies|Axis|None)/(?P<victim_id>[^)]*)\)"
|
||||
r"\s+with\s+(?P<weapon>.+?)\s*$",
|
||||
re.DOTALL,
|
||||
)
|
||||
|
||||
TEAM_SWITCH_RE = re.compile(
|
||||
r"^TEAMSWITCH\s+(?P<player_name>.+?)\s+\((?P<from_team>[^>]*)\s+>\s+(?P<to_team>[^)]*)\)\s*$",
|
||||
re.DOTALL,
|
||||
)
|
||||
|
||||
CONNECTED_RE = re.compile(
|
||||
r"^CONNECTED\s+(?P<player_name>.+?)\s+\((?P<player_id>[^)]*)\)\s*$",
|
||||
re.DOTALL,
|
||||
)
|
||||
|
||||
DISCONNECTED_RE = re.compile(
|
||||
r"^DISCONNECTED\s+(?P<player_name>.+?)\s+\((?P<player_id>[^)]*)\)\s*$",
|
||||
re.DOTALL,
|
||||
)
|
||||
|
||||
CHAT_RE = re.compile(
|
||||
r"^CHAT\[(?P<scope>[^\]]+)\]\[(?P<player_name>.+?)\((?P<team>Allies|Axis|None)/(?P<player_id>[^)]*)\)\]:\s*(?P<content>.*)$",
|
||||
re.DOTALL,
|
||||
)
|
||||
|
||||
KICK_RE = re.compile(
|
||||
r"^KICK:\s+\[(?P<player_name>.+?)\]\s+has been kicked\.\s+\[(?P<reason>.*)\]\s*$",
|
||||
re.DOTALL,
|
||||
)
|
||||
|
||||
MESSAGE_RE = re.compile(
|
||||
r"^MESSAGE:\s+player\s+\[(?P<player_name>.+?)\((?P<player_id>[^)]*)\)\],\s+content\s+\[(?P<content>.*)\]\s*$",
|
||||
re.DOTALL,
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ParsedRconAdminLogEvent:
|
||||
event_type: RconAdminLogEventType
|
||||
raw_message: str
|
||||
relative_time: str | None = None
|
||||
server_time: int | None = None
|
||||
map_name: str | None = None
|
||||
game_mode: str | None = None
|
||||
allied_score: int | None = None
|
||||
axis_score: int | None = None
|
||||
winner: str | None = None
|
||||
killer_name: str | None = None
|
||||
killer_team: str | None = None
|
||||
killer_id: str | None = None
|
||||
victim_name: str | None = None
|
||||
victim_team: str | None = None
|
||||
victim_id: str | None = None
|
||||
weapon: str | None = None
|
||||
player_name: str | None = None
|
||||
player_id: str | None = None
|
||||
from_team: str | None = None
|
||||
to_team: str | None = None
|
||||
chat_scope: str | None = None
|
||||
chat_team: str | None = None
|
||||
content: str | None = None
|
||||
reason: str | None = None
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ParsedRconPlayerProfileSnapshot:
|
||||
player_name: str
|
||||
player_id: str
|
||||
source_server_time: int | None
|
||||
event_timestamp: object
|
||||
first_seen: str | None
|
||||
sessions: int | None
|
||||
matches_played: int | None
|
||||
play_time: str | None
|
||||
total_kills: int | None
|
||||
total_deaths: int | None
|
||||
teamkills_done: int | None
|
||||
teamkills_received: int | None
|
||||
kd_ratio: float | None
|
||||
favorite_weapons: dict[str, int]
|
||||
victims: dict[str, int]
|
||||
nemesis: dict[str, int]
|
||||
averages: dict[str, object]
|
||||
sanctions: dict[str, object]
|
||||
raw_content: str
|
||||
|
||||
|
||||
def parse_rcon_admin_log_message(message: str) -> ParsedRconAdminLogEvent:
|
||||
raw_message = str(message or "")
|
||||
prefix_match = _PREFIX_RE.match(raw_message)
|
||||
relative_time = None
|
||||
server_time = None
|
||||
body = raw_message
|
||||
|
||||
if prefix_match:
|
||||
relative_time = prefix_match.group("relative")
|
||||
server_time = _coerce_int(prefix_match.group("server_time"))
|
||||
body = prefix_match.group("body")
|
||||
|
||||
parser_payload = {
|
||||
"raw_message": raw_message,
|
||||
"relative_time": relative_time,
|
||||
"server_time": server_time,
|
||||
}
|
||||
|
||||
if match := MATCH_START_RE.match(body):
|
||||
return ParsedRconAdminLogEvent(
|
||||
event_type="match_start",
|
||||
map_name=_clean(match.group("map_name")),
|
||||
game_mode=_clean(match.group("game_mode")),
|
||||
**parser_payload,
|
||||
)
|
||||
|
||||
if match := MATCH_END_RE.match(body):
|
||||
allied_score = _coerce_int(match.group("allied_score"))
|
||||
axis_score = _coerce_int(match.group("axis_score"))
|
||||
return ParsedRconAdminLogEvent(
|
||||
event_type="match_end",
|
||||
map_name=_clean(match.group("map_name")),
|
||||
allied_score=allied_score,
|
||||
axis_score=axis_score,
|
||||
winner=_resolve_winner(allied_score, axis_score),
|
||||
**parser_payload,
|
||||
)
|
||||
|
||||
if match := KILL_RE.match(body):
|
||||
return ParsedRconAdminLogEvent(
|
||||
event_type="kill",
|
||||
killer_name=_clean(match.group("killer_name")),
|
||||
killer_team=_clean(match.group("killer_team")),
|
||||
killer_id=_clean(match.group("killer_id")),
|
||||
victim_name=_clean(match.group("victim_name")),
|
||||
victim_team=_clean(match.group("victim_team")),
|
||||
victim_id=_clean(match.group("victim_id")),
|
||||
weapon=_clean(match.group("weapon")),
|
||||
**parser_payload,
|
||||
)
|
||||
|
||||
if match := TEAM_SWITCH_RE.match(body):
|
||||
return ParsedRconAdminLogEvent(
|
||||
event_type="team_switch",
|
||||
player_name=_clean(match.group("player_name")),
|
||||
from_team=_clean(match.group("from_team")),
|
||||
to_team=_clean(match.group("to_team")),
|
||||
**parser_payload,
|
||||
)
|
||||
|
||||
if match := CONNECTED_RE.match(body):
|
||||
return ParsedRconAdminLogEvent(
|
||||
event_type="connected",
|
||||
player_name=_clean(match.group("player_name")),
|
||||
player_id=_clean(match.group("player_id")),
|
||||
**parser_payload,
|
||||
)
|
||||
|
||||
if match := DISCONNECTED_RE.match(body):
|
||||
return ParsedRconAdminLogEvent(
|
||||
event_type="disconnected",
|
||||
player_name=_clean(match.group("player_name")),
|
||||
player_id=_clean(match.group("player_id")),
|
||||
**parser_payload,
|
||||
)
|
||||
|
||||
if match := CHAT_RE.match(body):
|
||||
return ParsedRconAdminLogEvent(
|
||||
event_type="chat",
|
||||
player_name=_clean(match.group("player_name")),
|
||||
player_id=_clean(match.group("player_id")),
|
||||
chat_scope=_clean(match.group("scope")),
|
||||
chat_team=_clean(match.group("team")),
|
||||
content=_clean(match.group("content")),
|
||||
**parser_payload,
|
||||
)
|
||||
|
||||
if match := KICK_RE.match(body):
|
||||
return ParsedRconAdminLogEvent(
|
||||
event_type="kick",
|
||||
player_name=_clean(match.group("player_name")),
|
||||
reason=_clean(match.group("reason")),
|
||||
**parser_payload,
|
||||
)
|
||||
|
||||
if body.upper().startswith("BAN"):
|
||||
return ParsedRconAdminLogEvent(event_type="ban", content=_clean(body), **parser_payload)
|
||||
|
||||
if match := MESSAGE_RE.match(body):
|
||||
return ParsedRconAdminLogEvent(
|
||||
event_type="message",
|
||||
player_name=_clean(match.group("player_name")),
|
||||
player_id=_clean(match.group("player_id")),
|
||||
content=_clean(match.group("content")),
|
||||
**parser_payload,
|
||||
)
|
||||
|
||||
return ParsedRconAdminLogEvent(event_type="unknown", content=_clean(body), **parser_payload)
|
||||
|
||||
|
||||
def parse_rcon_admin_log_entry(entry: dict[str, object]) -> dict[str, object]:
|
||||
parsed = parse_rcon_admin_log_message(str(entry.get("message") or ""))
|
||||
payload = asdict(parsed)
|
||||
payload["timestamp"] = entry.get("timestamp")
|
||||
return payload
|
||||
|
||||
|
||||
def parse_rcon_player_profile_snapshot(
|
||||
parsed_event: ParsedRconAdminLogEvent | dict[str, object],
|
||||
*,
|
||||
event_timestamp: object = None,
|
||||
) -> ParsedRconPlayerProfileSnapshot | None:
|
||||
"""Extract long-term player profile data from bot-generated MESSAGE content."""
|
||||
if isinstance(parsed_event, ParsedRconAdminLogEvent):
|
||||
event_type = parsed_event.event_type
|
||||
player_name = parsed_event.player_name
|
||||
player_id = parsed_event.player_id
|
||||
server_time = parsed_event.server_time
|
||||
content = parsed_event.content
|
||||
else:
|
||||
event_type = parsed_event.get("event_type")
|
||||
player_name = parsed_event.get("player_name")
|
||||
player_id = parsed_event.get("player_id")
|
||||
server_time = parsed_event.get("server_time")
|
||||
content = parsed_event.get("content")
|
||||
event_timestamp = event_timestamp if event_timestamp is not None else parsed_event.get("timestamp")
|
||||
|
||||
source_server_time = _coerce_int(server_time)
|
||||
if event_type != "message" or not player_name or not player_id or not content:
|
||||
return None
|
||||
if source_server_time is None:
|
||||
return None
|
||||
|
||||
raw_content = str(content)
|
||||
lines = [_clean_profile_line(line) for line in raw_content.splitlines()]
|
||||
lines = [line for line in lines if line]
|
||||
if not _looks_like_profile_message(lines):
|
||||
return None
|
||||
|
||||
sections = _profile_sections(lines)
|
||||
flat_values = _profile_key_values(lines)
|
||||
total_kills, teamkills_done = _parse_total_with_teamkills(flat_values, "bajas")
|
||||
total_deaths, teamkills_received = _parse_total_with_teamkills(flat_values, "muertes")
|
||||
|
||||
return ParsedRconPlayerProfileSnapshot(
|
||||
player_name=str(player_name),
|
||||
player_id=str(player_id),
|
||||
source_server_time=source_server_time,
|
||||
event_timestamp=event_timestamp,
|
||||
first_seen=_first_value(flat_values, "first seen", "visto por primera vez", "primer visto"),
|
||||
sessions=_first_int(flat_values, "sessions", "sesiones"),
|
||||
matches_played=_first_int(flat_values, "matches played", "partidas jugadas", "partidas"),
|
||||
play_time=_first_value(flat_values, "play time", "tiempo jugado", "tiempo de juego"),
|
||||
total_kills=total_kills,
|
||||
total_deaths=total_deaths,
|
||||
teamkills_done=teamkills_done,
|
||||
teamkills_received=teamkills_received,
|
||||
kd_ratio=_first_float(flat_values, "k/d", "kd"),
|
||||
favorite_weapons=_int_mapping(sections, "armas favoritas", "favorite weapons"),
|
||||
victims=_int_mapping(sections, "victimas", "víctimas", "vãctimas", "victims"),
|
||||
nemesis=_int_mapping(sections, "nemesis", "némesis", "nã©mesis"),
|
||||
averages=_object_mapping(sections, "promedios", "averages"),
|
||||
sanctions=_object_mapping(sections, "sanciones", "sanctions"),
|
||||
raw_content=raw_content,
|
||||
)
|
||||
|
||||
|
||||
def _clean(value: str | None) -> str | None:
|
||||
if value is None:
|
||||
return None
|
||||
normalized = value.strip()
|
||||
return normalized or None
|
||||
|
||||
|
||||
def _coerce_int(value: object) -> int | None:
|
||||
try:
|
||||
return int(value)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
def _coerce_float(value: object) -> float | None:
|
||||
if value is None:
|
||||
return None
|
||||
normalized = str(value).strip().replace(",", ".")
|
||||
match = re.search(r"-?\d+(?:\.\d+)?", normalized)
|
||||
if not match:
|
||||
return None
|
||||
try:
|
||||
return float(match.group(0))
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
|
||||
def _resolve_winner(allied_score: int | None, axis_score: int | None) -> str | None:
|
||||
if allied_score is None or axis_score is None:
|
||||
return None
|
||||
if allied_score > axis_score:
|
||||
return "allied"
|
||||
if axis_score > allied_score:
|
||||
return "axis"
|
||||
return "draw"
|
||||
|
||||
|
||||
def _clean_profile_line(value: str) -> str:
|
||||
cleaned = value.strip().strip("─-").strip()
|
||||
return cleaned.strip("▒").strip()
|
||||
|
||||
|
||||
def _looks_like_profile_message(lines: list[str]) -> bool:
|
||||
labels = {_normalize_profile_label(line.split(":", 1)[0]) for line in lines if ":" in line}
|
||||
section_labels = {_normalize_profile_label(line) for line in lines if ":" not in line}
|
||||
required = {"bajas", "muertes"}
|
||||
known_sections = {
|
||||
"totales",
|
||||
"victimas",
|
||||
"vãctimas",
|
||||
"nemesis",
|
||||
"nã©mesis",
|
||||
"armas favoritas",
|
||||
"promedios",
|
||||
"sanciones",
|
||||
}
|
||||
return required.issubset(labels) and bool(section_labels & known_sections)
|
||||
|
||||
|
||||
def _profile_sections(lines: list[str]) -> dict[str, list[str]]:
|
||||
sections: dict[str, list[str]] = {}
|
||||
current = "root"
|
||||
for line in lines:
|
||||
if ":" not in line:
|
||||
current = _normalize_profile_label(line)
|
||||
sections.setdefault(current, [])
|
||||
continue
|
||||
sections.setdefault(current, []).append(line)
|
||||
return sections
|
||||
|
||||
|
||||
def _profile_key_values(lines: list[str]) -> dict[str, str]:
|
||||
values: dict[str, str] = {}
|
||||
for line in lines:
|
||||
if ":" not in line:
|
||||
continue
|
||||
key, value = line.split(":", 1)
|
||||
values[_normalize_profile_label(key)] = value.strip()
|
||||
return values
|
||||
|
||||
|
||||
def _normalize_profile_label(value: object) -> str:
|
||||
return (
|
||||
str(value or "")
|
||||
.strip()
|
||||
.lower()
|
||||
.replace("\u00ad", "")
|
||||
.replace("í", "i")
|
||||
.replace("é", "e")
|
||||
.replace("ã", "i")
|
||||
.replace("ã©", "e")
|
||||
)
|
||||
|
||||
|
||||
def _first_value(values: dict[str, str], *keys: str) -> str | None:
|
||||
for key in keys:
|
||||
value = values.get(_normalize_profile_label(key))
|
||||
if value:
|
||||
return value
|
||||
return None
|
||||
|
||||
|
||||
def _first_int(values: dict[str, str], *keys: str) -> int | None:
|
||||
return _coerce_int_from_text(_first_value(values, *keys))
|
||||
|
||||
|
||||
def _first_float(values: dict[str, str], *keys: str) -> float | None:
|
||||
return _coerce_float(_first_value(values, *keys))
|
||||
|
||||
|
||||
def _parse_total_with_teamkills(values: dict[str, str], key: str) -> tuple[int | None, int | None]:
|
||||
raw_value = _first_value(values, key)
|
||||
if not raw_value:
|
||||
return None, None
|
||||
return _coerce_int_from_text(raw_value), _coerce_int_from_text(_inside_parentheses(raw_value))
|
||||
|
||||
|
||||
def _inside_parentheses(value: str) -> str | None:
|
||||
match = re.search(r"\((.*?)\)", value)
|
||||
return match.group(1) if match else None
|
||||
|
||||
|
||||
def _int_mapping(sections: dict[str, list[str]], *section_names: str) -> dict[str, int]:
|
||||
mapped: dict[str, int] = {}
|
||||
for line in _section_lines(sections, *section_names):
|
||||
key, value = line.split(":", 1)
|
||||
parsed = _coerce_int_from_text(value)
|
||||
if parsed is not None:
|
||||
mapped[key.strip()] = parsed
|
||||
return mapped
|
||||
|
||||
|
||||
def _object_mapping(sections: dict[str, list[str]], *section_names: str) -> dict[str, object]:
|
||||
mapped: dict[str, object] = {}
|
||||
for line in _section_lines(sections, *section_names):
|
||||
key, value = line.split(":", 1)
|
||||
cleaned = value.strip()
|
||||
mapped[key.strip()] = _coerce_float(cleaned) if re.search(r"\d", cleaned) else cleaned
|
||||
return mapped
|
||||
|
||||
|
||||
def _section_lines(sections: dict[str, list[str]], *section_names: str) -> list[str]:
|
||||
lines: list[str] = []
|
||||
wanted = {_normalize_profile_label(name) for name in section_names}
|
||||
for section_name, section_lines in sections.items():
|
||||
if _normalize_profile_label(section_name) in wanted:
|
||||
lines.extend(section_lines)
|
||||
return lines
|
||||
|
||||
|
||||
def _coerce_int_from_text(value: object) -> int | None:
|
||||
if value is None:
|
||||
return None
|
||||
match = re.search(r"-?\d+", str(value))
|
||||
return _coerce_int(match.group(0)) if match else None
|
||||
1021
backend/app/rcon_admin_log_storage.py
Normal file
1021
backend/app/rcon_admin_log_storage.py
Normal file
File diff suppressed because it is too large
Load Diff
660
backend/app/rcon_client.py
Normal file
660
backend/app/rcon_client.py
Normal file
@@ -0,0 +1,660 @@
|
||||
"""Minimal Hell Let Loose RCON client for live server state queries."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import itertools
|
||||
import json
|
||||
import socket
|
||||
import struct
|
||||
from collections.abc import Mapping
|
||||
from dataclasses import dataclass
|
||||
|
||||
from .config import (
|
||||
DEFAULT_RCON_SOURCE_NAME,
|
||||
get_rcon_request_timeout_seconds,
|
||||
get_rcon_targets_payload,
|
||||
)
|
||||
|
||||
|
||||
RCON_BUFFER_SIZE = 32768
|
||||
RCON_HEADER_FORMAT = "<III"
|
||||
RCON_MAGIC_HEADER_VALUE = 0xDE450508
|
||||
RCON_PROTOCOL_VERSION = 2
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class RconServerTarget:
|
||||
"""Configuration needed to query one HLL RCON endpoint."""
|
||||
|
||||
name: str
|
||||
host: str
|
||||
port: int
|
||||
password: str
|
||||
source_name: str
|
||||
external_server_id: str | None = None
|
||||
region: str | None = None
|
||||
game_port: int | None = None
|
||||
query_port: int | None = None
|
||||
|
||||
|
||||
class RconQueryError(RuntimeError):
|
||||
"""Normalized RCON query failure with a machine-readable error type."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
error_type: str,
|
||||
message: str,
|
||||
*,
|
||||
error_stage: str | None = None,
|
||||
) -> None:
|
||||
super().__init__(message)
|
||||
self.error_type = error_type
|
||||
self.error_stage = error_stage
|
||||
|
||||
|
||||
class HllRconConnection:
|
||||
"""Synchronous HLL RCON v2 connection for lightweight live status queries."""
|
||||
|
||||
def __init__(self, *, timeout_seconds: float) -> None:
|
||||
self._socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
||||
self._socket.settimeout(timeout_seconds)
|
||||
self._xor_key: bytes | None = None
|
||||
self._auth_token: str | None = None
|
||||
self._request_ids = itertools.count(1)
|
||||
self._current_stage = "tcp_connect"
|
||||
|
||||
def connect(self, *, host: str, port: int, password: str) -> None:
|
||||
self._run_socket_stage(
|
||||
"tcp_connect",
|
||||
lambda: self._socket.connect((host, port)),
|
||||
)
|
||||
|
||||
server_connect_response = self._exchange(
|
||||
"ServerConnect",
|
||||
"",
|
||||
request_stage="server_connect_request",
|
||||
response_stage="server_connect_response",
|
||||
)
|
||||
self._current_stage = "xor_key_decode"
|
||||
xor_key_b64 = _expect_text_content(server_connect_response, command_name="ServerConnect")
|
||||
try:
|
||||
self._xor_key = base64.b64decode(xor_key_b64)
|
||||
except (ValueError, TypeError) as error:
|
||||
raise RconQueryError(
|
||||
"payload-invalid",
|
||||
"The HLL server returned an invalid RCON XOR key.",
|
||||
error_stage="xor_key_decode",
|
||||
) from error
|
||||
if not self._xor_key:
|
||||
raise RconQueryError(
|
||||
"unexpected-response",
|
||||
"The HLL server returned an empty RCON XOR key.",
|
||||
error_stage="xor_key_decode",
|
||||
)
|
||||
|
||||
login_response = self._exchange(
|
||||
"Login",
|
||||
password,
|
||||
request_stage="login_request",
|
||||
response_stage="login_response",
|
||||
)
|
||||
self._auth_token = _expect_text_content(login_response, command_name="Login")
|
||||
if not self._auth_token:
|
||||
raise RconQueryError(
|
||||
"unexpected-response",
|
||||
"The HLL server returned an empty RCON auth token.",
|
||||
error_stage="login_response",
|
||||
)
|
||||
|
||||
def execute_json(
|
||||
self,
|
||||
command: str,
|
||||
content: dict[str, object] | str = "",
|
||||
) -> dict[str, object]:
|
||||
stage_prefix = _resolve_command_stage_prefix(command)
|
||||
response = self._exchange(
|
||||
command,
|
||||
content,
|
||||
request_stage=f"{stage_prefix}_request",
|
||||
response_stage=f"{stage_prefix}_response",
|
||||
)
|
||||
self._current_stage = "payload_decode"
|
||||
content_body = response.get("contentBody")
|
||||
if isinstance(content_body, dict):
|
||||
return content_body
|
||||
if isinstance(content_body, str):
|
||||
try:
|
||||
parsed = json.loads(content_body)
|
||||
except json.JSONDecodeError as error:
|
||||
raise RconQueryError(
|
||||
"payload-invalid",
|
||||
f"The HLL server returned invalid JSON content for {command}.",
|
||||
error_stage="payload_decode",
|
||||
) from error
|
||||
if isinstance(parsed, dict):
|
||||
return parsed
|
||||
raise RconQueryError(
|
||||
"unexpected-response",
|
||||
f"The HLL server returned an unexpected payload for {command}.",
|
||||
error_stage="unexpected_response",
|
||||
)
|
||||
|
||||
def close(self) -> None:
|
||||
try:
|
||||
self._socket.shutdown(socket.SHUT_RDWR)
|
||||
except OSError:
|
||||
pass
|
||||
self._socket.close()
|
||||
|
||||
def _exchange(
|
||||
self,
|
||||
command: str,
|
||||
content: dict[str, object] | str = "",
|
||||
*,
|
||||
request_stage: str,
|
||||
response_stage: str,
|
||||
) -> dict[str, object]:
|
||||
request_id = next(self._request_ids)
|
||||
self._send_request(
|
||||
request_id=request_id,
|
||||
command=command,
|
||||
content=content,
|
||||
request_stage=request_stage,
|
||||
)
|
||||
response = self._receive_response(response_stage=response_stage)
|
||||
response_request_id = int(response.get("requestId") or 0)
|
||||
if response_request_id != request_id:
|
||||
raise RconQueryError(
|
||||
"unexpected-response",
|
||||
f"Unexpected RCON response id {response_request_id} for request {request_id}.",
|
||||
error_stage="unexpected_response",
|
||||
)
|
||||
_raise_for_status(response, command_name=command, error_stage=response_stage)
|
||||
return response
|
||||
|
||||
def _send_request(
|
||||
self,
|
||||
*,
|
||||
request_id: int,
|
||||
command: str,
|
||||
content: dict[str, object] | str,
|
||||
request_stage: str,
|
||||
) -> None:
|
||||
content_body = (
|
||||
content
|
||||
if isinstance(content, str)
|
||||
else json.dumps(content, separators=(",", ":"))
|
||||
)
|
||||
body = json.dumps(
|
||||
{
|
||||
"authToken": self._auth_token or "",
|
||||
"version": RCON_PROTOCOL_VERSION,
|
||||
"name": command,
|
||||
"contentBody": content_body,
|
||||
},
|
||||
separators=(",", ":"),
|
||||
).encode("utf-8")
|
||||
header = struct.pack(
|
||||
RCON_HEADER_FORMAT,
|
||||
RCON_MAGIC_HEADER_VALUE,
|
||||
request_id,
|
||||
len(body),
|
||||
)
|
||||
self._run_socket_stage(
|
||||
request_stage,
|
||||
lambda: self._socket.sendall(header + self._xor(body)),
|
||||
)
|
||||
|
||||
def _receive_response(self, *, response_stage: str) -> dict[str, object]:
|
||||
header_size = struct.calcsize(RCON_HEADER_FORMAT)
|
||||
header_bytes = self._recv_exact(
|
||||
header_size,
|
||||
stage=response_stage,
|
||||
receive_context="response header",
|
||||
)
|
||||
try:
|
||||
magic_value, request_id, body_length = struct.unpack(
|
||||
RCON_HEADER_FORMAT,
|
||||
header_bytes,
|
||||
)
|
||||
except struct.error as error:
|
||||
raise RconQueryError(
|
||||
"payload-invalid",
|
||||
"The HLL server returned an invalid RCON response header.",
|
||||
error_stage=response_stage,
|
||||
) from error
|
||||
if magic_value != RCON_MAGIC_HEADER_VALUE:
|
||||
raise RconQueryError(
|
||||
"invalid-magic",
|
||||
(
|
||||
"The HLL server returned an unexpected RCON magic value: "
|
||||
f"{magic_value:#x} (expected {RCON_MAGIC_HEADER_VALUE:#x})."
|
||||
),
|
||||
error_stage=response_stage,
|
||||
)
|
||||
if body_length <= 0:
|
||||
raise RconQueryError(
|
||||
"unexpected-response",
|
||||
"The HLL server returned an empty RCON response body.",
|
||||
error_stage=response_stage,
|
||||
)
|
||||
|
||||
body = self._xor(self._recv_body(body_length, stage=response_stage))
|
||||
try:
|
||||
parsed = json.loads(body.decode("utf-8", errors="replace"))
|
||||
except json.JSONDecodeError as error:
|
||||
raise RconQueryError(
|
||||
"payload-invalid",
|
||||
"The HLL server returned malformed RCON JSON.",
|
||||
error_stage="payload_decode",
|
||||
) from error
|
||||
if not isinstance(parsed, dict):
|
||||
raise RconQueryError(
|
||||
"unexpected-response",
|
||||
"The HLL server returned a non-object RCON response.",
|
||||
error_stage="unexpected_response",
|
||||
)
|
||||
|
||||
parsed["requestId"] = request_id
|
||||
return parsed
|
||||
|
||||
def _recv_body(self, expected_length: int, *, stage: str) -> bytes:
|
||||
chunks = bytearray()
|
||||
original_timeout = self._socket.gettimeout()
|
||||
body_timeout_seconds = min(3.0, original_timeout or 3.0)
|
||||
self._socket.settimeout(body_timeout_seconds)
|
||||
try:
|
||||
while len(chunks) < expected_length:
|
||||
self._current_stage = stage
|
||||
try:
|
||||
chunk = self._socket.recv(
|
||||
min(RCON_BUFFER_SIZE, expected_length - len(chunks))
|
||||
)
|
||||
except (TimeoutError, socket.timeout) as error:
|
||||
raise RconQueryError(
|
||||
"timeout",
|
||||
(
|
||||
f"Timed out during {stage} while waiting for response body "
|
||||
f"({len(chunks)}/{expected_length} bytes received)."
|
||||
),
|
||||
error_stage=stage,
|
||||
) from error
|
||||
except OSError as error:
|
||||
raise RconQueryError(
|
||||
_classify_socket_error_type(error),
|
||||
f"RCON socket error during {stage}: {error}",
|
||||
error_stage=stage,
|
||||
) from error
|
||||
if not chunk:
|
||||
raise RconQueryError(
|
||||
"connection-closed",
|
||||
(
|
||||
"The HLL RCON connection closed unexpectedly while waiting for "
|
||||
f"response body ({len(chunks)}/{expected_length} bytes received)."
|
||||
),
|
||||
error_stage=stage,
|
||||
)
|
||||
chunks.extend(chunk)
|
||||
finally:
|
||||
self._socket.settimeout(original_timeout)
|
||||
return bytes(chunks)
|
||||
|
||||
def _recv_exact(
|
||||
self,
|
||||
expected_length: int,
|
||||
*,
|
||||
stage: str,
|
||||
receive_context: str,
|
||||
) -> bytes:
|
||||
chunks = bytearray()
|
||||
while len(chunks) < expected_length:
|
||||
self._current_stage = stage
|
||||
try:
|
||||
chunk = self._socket.recv(min(RCON_BUFFER_SIZE, expected_length - len(chunks)))
|
||||
except (TimeoutError, socket.timeout) as error:
|
||||
raise RconQueryError(
|
||||
"timeout",
|
||||
(
|
||||
f"Timed out during {stage} while waiting for {receive_context} "
|
||||
f"({len(chunks)}/{expected_length} bytes received)."
|
||||
),
|
||||
error_stage=stage,
|
||||
) from error
|
||||
except OSError as error:
|
||||
raise RconQueryError(
|
||||
_classify_socket_error_type(error),
|
||||
f"RCON socket error during {stage}: {error}",
|
||||
error_stage=stage,
|
||||
) from error
|
||||
if not chunk:
|
||||
raise RconQueryError(
|
||||
"connection-closed",
|
||||
(
|
||||
"The HLL RCON connection closed unexpectedly while waiting for "
|
||||
f"{receive_context} ({len(chunks)}/{expected_length} bytes received)."
|
||||
),
|
||||
error_stage=stage,
|
||||
)
|
||||
chunks.extend(chunk)
|
||||
return bytes(chunks)
|
||||
|
||||
def _xor(self, payload: bytes) -> bytes:
|
||||
if not self._xor_key:
|
||||
return payload
|
||||
return bytes(
|
||||
value ^ self._xor_key[index % len(self._xor_key)]
|
||||
for index, value in enumerate(payload)
|
||||
)
|
||||
|
||||
def __enter__(self) -> HllRconConnection:
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type: object, exc: object, traceback: object) -> None:
|
||||
self.close()
|
||||
|
||||
def _run_socket_stage(self, stage: str, operation: object) -> object:
|
||||
self._current_stage = stage
|
||||
try:
|
||||
return operation()
|
||||
except (TimeoutError, socket.timeout) as error:
|
||||
raise RconQueryError(
|
||||
"timeout",
|
||||
f"Timed out during {stage}.",
|
||||
error_stage=stage,
|
||||
) from error
|
||||
except OSError as error:
|
||||
raise RconQueryError(
|
||||
_classify_socket_error_type(error),
|
||||
f"RCON socket error during {stage}: {error}",
|
||||
error_stage=stage,
|
||||
) from error
|
||||
|
||||
|
||||
def load_rcon_targets() -> tuple[RconServerTarget, ...]:
|
||||
"""Load RCON targets from JSON env payload."""
|
||||
raw_payload = get_rcon_targets_payload()
|
||||
if raw_payload is None:
|
||||
return ()
|
||||
parsed = json.loads(raw_payload)
|
||||
if not isinstance(parsed, list):
|
||||
raise ValueError("HLL_BACKEND_RCON_TARGETS must be a JSON array.")
|
||||
return tuple(_coerce_rcon_target(item) for item in parsed if isinstance(item, dict))
|
||||
|
||||
|
||||
def query_live_server_state(
|
||||
target: RconServerTarget,
|
||||
*,
|
||||
timeout_seconds: float | None = None,
|
||||
) -> dict[str, object]:
|
||||
"""Query one HLL server via RCON and normalize it to the live snapshot shape."""
|
||||
sample = query_live_server_sample(target, timeout_seconds=timeout_seconds)
|
||||
return dict(sample["normalized"])
|
||||
|
||||
|
||||
def query_live_server_sample(
|
||||
target: RconServerTarget,
|
||||
*,
|
||||
timeout_seconds: float | None = None,
|
||||
) -> dict[str, object]:
|
||||
"""Query one HLL server and return both normalized and raw session data."""
|
||||
resolved_timeout = timeout_seconds or get_rcon_request_timeout_seconds()
|
||||
try:
|
||||
with HllRconConnection(timeout_seconds=resolved_timeout) as connection:
|
||||
connection.connect(host=target.host, port=target.port, password=target.password)
|
||||
session = connection.execute_json(
|
||||
"GetServerInformation",
|
||||
{"Name": "session", "Value": ""},
|
||||
)
|
||||
except RconQueryError:
|
||||
raise
|
||||
except (TimeoutError, socket.timeout) as error:
|
||||
raise RconQueryError(
|
||||
"timeout",
|
||||
f"Timed out after {resolved_timeout:.1f}s while querying {target.host}:{target.port}.",
|
||||
) from error
|
||||
except ConnectionRefusedError as error:
|
||||
raise RconQueryError(
|
||||
"connection-refused",
|
||||
f"Connection refused by {target.host}:{target.port}.",
|
||||
) from error
|
||||
except OSError as error:
|
||||
raise RconQueryError(
|
||||
_classify_socket_error_type(error),
|
||||
f"RCON socket error against {target.host}:{target.port}: {error}",
|
||||
) from error
|
||||
except RuntimeError as error:
|
||||
raise RconQueryError(
|
||||
_classify_runtime_error_type(error),
|
||||
str(error),
|
||||
error_stage=getattr(error, "error_stage", None),
|
||||
) from error
|
||||
|
||||
resolved_external_id = target.external_server_id or f"rcon:{target.host}:{target.port}"
|
||||
return {
|
||||
"target": {
|
||||
"target_key": build_rcon_target_key(target),
|
||||
"name": target.name,
|
||||
"host": target.host,
|
||||
"port": target.port,
|
||||
"external_server_id": target.external_server_id,
|
||||
"region": target.region,
|
||||
"game_port": target.game_port,
|
||||
"query_port": target.query_port,
|
||||
"source_name": target.source_name,
|
||||
},
|
||||
"normalized": {
|
||||
"external_server_id": resolved_external_id,
|
||||
"server_name": _string_or_none(session.get("serverName")) or target.name,
|
||||
"status": "online",
|
||||
"players": _coerce_optional_int(session.get("playerCount")),
|
||||
"max_players": _coerce_optional_int(session.get("maxPlayerCount")),
|
||||
"current_map": (
|
||||
_string_or_none(session.get("mapId")) or _string_or_none(session.get("mapName"))
|
||||
),
|
||||
"game_mode": _string_or_none(session.get("gameMode")),
|
||||
"allied_score": _coerce_optional_int(session.get("alliedScore")),
|
||||
"axis_score": _coerce_optional_int(session.get("axisScore")),
|
||||
"winner": _resolve_rcon_winner(
|
||||
_coerce_optional_int(session.get("alliedScore")),
|
||||
_coerce_optional_int(session.get("axisScore")),
|
||||
),
|
||||
"allied_faction": _string_or_none(session.get("alliedFaction")),
|
||||
"axis_faction": _string_or_none(session.get("axisFaction")),
|
||||
"allied_players": _coerce_optional_int(session.get("alliedPlayerCount")),
|
||||
"axis_players": _coerce_optional_int(session.get("axisPlayerCount")),
|
||||
"remaining_match_time_seconds": _coerce_optional_int(session.get("remainingMatchTime")),
|
||||
"match_time_seconds": _coerce_optional_int(session.get("matchTime")),
|
||||
"queue_count": _coerce_optional_int(session.get("queueCount")),
|
||||
"max_queue_count": _coerce_optional_int(session.get("maxQueueCount")),
|
||||
"vip_queue_count": _coerce_optional_int(session.get("vipQueueCount")),
|
||||
"max_vip_queue_count": _coerce_optional_int(session.get("maxVipQueueCount")),
|
||||
"region": target.region,
|
||||
"source_name": target.source_name,
|
||||
"snapshot_origin": "real-rcon",
|
||||
"source_ref": f"rcon://{target.host}:{target.port}",
|
||||
},
|
||||
"raw_session": session,
|
||||
}
|
||||
|
||||
|
||||
def build_rcon_target_key(target: RconServerTarget) -> str:
|
||||
"""Build a stable local key for one configured RCON target."""
|
||||
external_server_id = _string_or_none(target.external_server_id)
|
||||
if external_server_id:
|
||||
return external_server_id
|
||||
return f"rcon:{target.host}:{target.port}"
|
||||
|
||||
|
||||
def _coerce_rcon_target(raw_target: dict[str, object]) -> RconServerTarget:
|
||||
slug = _string_or_none(raw_target.get("slug"))
|
||||
external_server_id = _string_or_none(raw_target.get("external_server_id")) or slug
|
||||
name = _string_or_none(raw_target.get("name")) or _slug_to_display_name(slug) or "Unnamed RCON target"
|
||||
host = _required_string(raw_target, "host")
|
||||
password = _required_string(raw_target, "password")
|
||||
source_name = _string_or_none(raw_target.get("source_name")) or DEFAULT_RCON_SOURCE_NAME
|
||||
port = _required_positive_int(raw_target, "port")
|
||||
if not host:
|
||||
raise ValueError("Each RCON target must define a non-empty 'host'.")
|
||||
if port <= 0:
|
||||
raise ValueError("Each RCON target must define a positive 'port'.")
|
||||
if not password:
|
||||
raise ValueError("Each RCON target must define a non-empty 'password'.")
|
||||
|
||||
return RconServerTarget(
|
||||
name=name,
|
||||
host=host,
|
||||
port=port,
|
||||
password=password,
|
||||
source_name=source_name or DEFAULT_RCON_SOURCE_NAME,
|
||||
external_server_id=external_server_id,
|
||||
region=_string_or_none(raw_target.get("region")),
|
||||
game_port=_coerce_optional_positive_int(raw_target.get("game_port")),
|
||||
query_port=_coerce_optional_positive_int(raw_target.get("query_port")),
|
||||
)
|
||||
|
||||
|
||||
def _raise_for_status(
|
||||
response: dict[str, object],
|
||||
*,
|
||||
command_name: str,
|
||||
error_stage: str,
|
||||
) -> None:
|
||||
status_code = int(response.get("statusCode") or 0)
|
||||
if status_code == 200:
|
||||
return
|
||||
status_message = _string_or_none(response.get("statusMessage")) or "Unknown RCON error."
|
||||
if command_name == "Login" and status_code in {401, 403}:
|
||||
raise RconQueryError(
|
||||
"auth/login",
|
||||
f"{command_name} failed with RCON status {status_code}: {status_message}",
|
||||
error_stage=error_stage,
|
||||
)
|
||||
raise RconQueryError(
|
||||
"unexpected-response",
|
||||
f"{command_name} failed with RCON status {status_code}: {status_message}",
|
||||
error_stage=error_stage,
|
||||
)
|
||||
|
||||
|
||||
def _expect_text_content(response: dict[str, object], *, command_name: str) -> str:
|
||||
content = response.get("contentBody")
|
||||
if isinstance(content, str):
|
||||
return content
|
||||
raise RconQueryError(
|
||||
"unexpected-response",
|
||||
f"The HLL server returned unexpected text content for {command_name}.",
|
||||
error_stage="unexpected_response",
|
||||
)
|
||||
|
||||
|
||||
def _resolve_command_stage_prefix(command: str) -> str:
|
||||
normalized_command = str(command or "").strip().lower()
|
||||
stage_prefix_by_command = {
|
||||
"serverconnect": "server_connect",
|
||||
"login": "login",
|
||||
"getserverinformation": "get_server_information",
|
||||
}
|
||||
return stage_prefix_by_command.get(normalized_command, normalized_command or "rcon_command")
|
||||
|
||||
|
||||
def _string_or_none(value: object) -> str | None:
|
||||
if not isinstance(value, str):
|
||||
return None
|
||||
normalized = value.strip()
|
||||
return normalized or None
|
||||
|
||||
|
||||
def _resolve_rcon_winner(allied_score: int | None, axis_score: int | None) -> str | None:
|
||||
if allied_score is None or axis_score is None:
|
||||
return None
|
||||
if allied_score > axis_score:
|
||||
return "allied"
|
||||
if axis_score > allied_score:
|
||||
return "axis"
|
||||
return "draw"
|
||||
|
||||
|
||||
def _coerce_optional_int(value: object) -> int | None:
|
||||
if value is None:
|
||||
return None
|
||||
try:
|
||||
return int(value)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
def _coerce_optional_positive_int(value: object) -> int | None:
|
||||
if value is None:
|
||||
return None
|
||||
coerced = int(value)
|
||||
if coerced <= 0:
|
||||
raise ValueError("Configured RCON target ports must be positive when defined.")
|
||||
return coerced
|
||||
|
||||
|
||||
def _required_string(raw_target: Mapping[str, object], field_name: str) -> str:
|
||||
value = _string_or_none(raw_target.get(field_name))
|
||||
if value is None:
|
||||
available_fields = ", ".join(sorted(raw_target.keys()))
|
||||
raise ValueError(
|
||||
f"Each RCON target must define a non-empty '{field_name}'. "
|
||||
f"Available fields: {available_fields or 'none'}."
|
||||
)
|
||||
return value
|
||||
|
||||
|
||||
def _required_positive_int(raw_target: Mapping[str, object], field_name: str) -> int:
|
||||
raw_value = raw_target.get(field_name)
|
||||
try:
|
||||
value = int(raw_value)
|
||||
except (TypeError, ValueError) as error:
|
||||
available_fields = ", ".join(sorted(raw_target.keys()))
|
||||
raise ValueError(
|
||||
f"Each RCON target must define a valid integer '{field_name}'. "
|
||||
f"Available fields: {available_fields or 'none'}."
|
||||
) from error
|
||||
if value <= 0:
|
||||
raise ValueError(f"Each RCON target must define a positive '{field_name}'.")
|
||||
return value
|
||||
|
||||
|
||||
def _slug_to_display_name(slug: str | None) -> str | None:
|
||||
normalized_slug = _string_or_none(slug)
|
||||
if normalized_slug is None:
|
||||
return None
|
||||
if normalized_slug.startswith("comunidad-hispana-"):
|
||||
suffix = normalized_slug.removeprefix("comunidad-hispana-")
|
||||
if suffix.isdigit():
|
||||
return f"Comunidad Hispana #{suffix.zfill(2)}"
|
||||
parts = [part for part in normalized_slug.replace("_", "-").split("-") if part]
|
||||
if not parts:
|
||||
return None
|
||||
return " ".join(part.upper() if part.isdigit() else part.capitalize() for part in parts)
|
||||
|
||||
|
||||
def _classify_socket_error_type(error: OSError) -> str:
|
||||
if isinstance(error, TimeoutError):
|
||||
return "timeout"
|
||||
if isinstance(error, ConnectionRefusedError):
|
||||
return "connection-refused"
|
||||
if getattr(error, "errno", None) in {10060, 110, 60}:
|
||||
return "timeout"
|
||||
return "other-error"
|
||||
|
||||
|
||||
def _classify_runtime_error_type(error: RuntimeError) -> str:
|
||||
message = str(error).lower()
|
||||
if "auth token" in message or "login failed" in message or "status 401" in message or "status 403" in message:
|
||||
return "auth/login"
|
||||
if "invalid magic" in message:
|
||||
return "invalid-magic"
|
||||
if "closed unexpectedly" in message or "closed connection" in message:
|
||||
return "connection-closed"
|
||||
if "invalid json" in message or "unexpected payload" in message or "malformed" in message or "invalid rcon" in message:
|
||||
return "payload-invalid"
|
||||
if "timed out" in message:
|
||||
return "timeout"
|
||||
if "unexpected" in message:
|
||||
return "unexpected-response"
|
||||
return "other-error"
|
||||
484
backend/app/rcon_historical_backfill.py
Normal file
484
backend/app/rcon_historical_backfill.py
Normal file
@@ -0,0 +1,484 @@
|
||||
"""Explicit RCON/AdminLog historical backfill command."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from contextlib import closing
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from pathlib import Path
|
||||
from typing import Iterable
|
||||
|
||||
from .config import (
|
||||
get_rcon_backfill_chunk_hours,
|
||||
get_rcon_backfill_max_days_back,
|
||||
get_rcon_backfill_sleep_seconds,
|
||||
get_rcon_request_timeout_seconds,
|
||||
use_postgres_rcon_storage,
|
||||
)
|
||||
from .historical_runner import generate_historical_snapshots
|
||||
from .historical_storage import ALL_SERVERS_SLUG
|
||||
from .rcon_admin_log_materialization import (
|
||||
MATCH_RESULT_SOURCE,
|
||||
initialize_rcon_materialized_storage,
|
||||
materialize_rcon_admin_log,
|
||||
)
|
||||
from .rcon_admin_log_storage import persist_rcon_admin_log_entries
|
||||
from .rcon_client import HllRconConnection, RconServerTarget, build_rcon_target_key, load_rcon_targets
|
||||
from .rcon_historical_leaderboards import list_rcon_materialized_leaderboard
|
||||
from .sqlite_utils import connect_sqlite_readonly
|
||||
from .writer_lock import backend_writer_lock, build_writer_lock_holder
|
||||
|
||||
DEFAULT_ALLOWED_SERVER_KEYS = frozenset({"comunidad-hispana-01", "comunidad-hispana-02"})
|
||||
EXCLUDED_BY_DEFAULT_SERVER_KEYS = frozenset({"comunidad-hispana-03"})
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class BackfillWindow:
|
||||
start: datetime
|
||||
end: datetime
|
||||
|
||||
@property
|
||||
def lookback_seconds(self) -> int:
|
||||
now = datetime.now(timezone.utc)
|
||||
return max(1, int((now - self.start).total_seconds()))
|
||||
|
||||
|
||||
def run_rcon_historical_backfill(
|
||||
*,
|
||||
servers: str | None = None,
|
||||
from_value: str | None = None,
|
||||
to_value: str | None = None,
|
||||
ensure_recent_matches: int | None = None,
|
||||
ensure_current_month: bool = False,
|
||||
ensure_leaderboard_windows: bool = False,
|
||||
chunk_hours: int | None = None,
|
||||
sleep_seconds: float | None = None,
|
||||
max_days_back: int | None = None,
|
||||
dry_run: bool = False,
|
||||
regenerate_snapshots: bool = False,
|
||||
db_path: Path | None = None,
|
||||
) -> dict[str, object]:
|
||||
"""Backfill AdminLog events and materialized RCON matches on explicit operator command."""
|
||||
anchor = datetime.now(timezone.utc)
|
||||
resolved_chunk_hours = chunk_hours or get_rcon_backfill_chunk_hours()
|
||||
resolved_sleep_seconds = (
|
||||
get_rcon_backfill_sleep_seconds() if sleep_seconds is None else sleep_seconds
|
||||
)
|
||||
resolved_max_days_back = max_days_back or get_rcon_backfill_max_days_back()
|
||||
selected_targets = select_backfill_targets(servers)
|
||||
recent_before = count_recent_materialized_closed_matches(db_path=db_path)
|
||||
monthly_before = _window_diagnostic("monthly", db_path=db_path, now=anchor)
|
||||
weekly_before = _window_diagnostic("weekly", db_path=db_path, now=anchor)
|
||||
requested_range = _resolve_requested_range(
|
||||
anchor=anchor,
|
||||
from_value=from_value,
|
||||
to_value=to_value,
|
||||
ensure_recent_matches=ensure_recent_matches,
|
||||
ensure_current_month=ensure_current_month,
|
||||
ensure_leaderboard_windows=ensure_leaderboard_windows,
|
||||
max_days_back=resolved_max_days_back,
|
||||
)
|
||||
windows = _build_backfill_windows(
|
||||
start=requested_range["start"],
|
||||
end=requested_range["end"],
|
||||
chunk_hours=resolved_chunk_hours,
|
||||
)
|
||||
|
||||
result: dict[str, object] = {
|
||||
"status": "dry-run" if dry_run else "ok",
|
||||
"dry_run": dry_run,
|
||||
"servers_processed": [build_rcon_target_key(target) for target in selected_targets],
|
||||
"requested_range": {
|
||||
"from": _to_iso(requested_range["start"]),
|
||||
"to": _to_iso(requested_range["end"]),
|
||||
"reason": requested_range["reason"],
|
||||
"admin_log_api": "lookback-only",
|
||||
},
|
||||
"actual_windows_scanned": [],
|
||||
"events_seen": 0,
|
||||
"events_inserted": 0,
|
||||
"duplicate_events": 0,
|
||||
"matches_materialized": 0,
|
||||
"matches_updated": 0,
|
||||
"player_stats_materialized": 0,
|
||||
"player_stats_updated": 0,
|
||||
"recent_materialized_closed_match_count_before": recent_before,
|
||||
"recent_materialized_closed_match_count_after": recent_before,
|
||||
"monthly_selected_window_before": monthly_before,
|
||||
"monthly_selected_window": monthly_before,
|
||||
"weekly_selected_window_before": weekly_before,
|
||||
"weekly_selected_window": weekly_before,
|
||||
"snapshot_regeneration_result": None,
|
||||
"errors": [],
|
||||
}
|
||||
|
||||
if dry_run:
|
||||
result["actual_windows_scanned"] = [
|
||||
_serialize_window(window) for window in _limit_windows_for_recent_need(
|
||||
windows,
|
||||
ensure_recent_matches=ensure_recent_matches,
|
||||
db_path=db_path,
|
||||
)
|
||||
]
|
||||
return result
|
||||
|
||||
try:
|
||||
with backend_writer_lock(
|
||||
holder=build_writer_lock_holder("app.rcon_historical_backfill")
|
||||
):
|
||||
windows_to_scan = _limit_windows_for_recent_need(
|
||||
windows,
|
||||
ensure_recent_matches=ensure_recent_matches,
|
||||
db_path=db_path,
|
||||
)
|
||||
for window in windows_to_scan:
|
||||
for target in selected_targets:
|
||||
window_result = _scan_target_window(target, window)
|
||||
result["actual_windows_scanned"].append(window_result["window"])
|
||||
result["events_seen"] = int(result["events_seen"]) + int(
|
||||
window_result["events_seen"]
|
||||
)
|
||||
result["events_inserted"] = int(result["events_inserted"]) + int(
|
||||
window_result["events_inserted"]
|
||||
)
|
||||
result["duplicate_events"] = int(result["duplicate_events"]) + int(
|
||||
window_result["duplicate_events"]
|
||||
)
|
||||
if window_result.get("error"):
|
||||
result["errors"].append(window_result["error"])
|
||||
if resolved_sleep_seconds > 0:
|
||||
time.sleep(resolved_sleep_seconds)
|
||||
|
||||
materialized = materialize_rcon_admin_log(db_path=db_path)
|
||||
result["matches_materialized"] = int(result["matches_materialized"]) + int(
|
||||
materialized.get("matches_materialized") or 0
|
||||
)
|
||||
result["matches_updated"] = int(result["matches_updated"]) + int(
|
||||
materialized.get("matches_updated") or 0
|
||||
)
|
||||
result["player_stats_materialized"] = int(
|
||||
result["player_stats_materialized"]
|
||||
) + int(materialized.get("player_stats_materialized") or 0)
|
||||
result["player_stats_updated"] = int(result["player_stats_updated"]) + int(
|
||||
materialized.get("player_stats_updated") or 0
|
||||
)
|
||||
|
||||
if ensure_recent_matches and count_recent_materialized_closed_matches(
|
||||
db_path=db_path
|
||||
) >= ensure_recent_matches:
|
||||
break
|
||||
|
||||
if regenerate_snapshots:
|
||||
result["snapshot_regeneration_result"] = generate_historical_snapshots(
|
||||
server_slug=None,
|
||||
run_number=1,
|
||||
)
|
||||
except Exception as exc: # noqa: BLE001 - CLI reports structured operator diagnostics
|
||||
result["status"] = "error"
|
||||
result["errors"].append({"error_type": type(exc).__name__, "message": str(exc)})
|
||||
|
||||
recent_after = count_recent_materialized_closed_matches(db_path=db_path)
|
||||
result["recent_materialized_closed_match_count_after"] = recent_after
|
||||
result["monthly_selected_window"] = _window_diagnostic("monthly", db_path=db_path, now=anchor)
|
||||
result["weekly_selected_window"] = _window_diagnostic("weekly", db_path=db_path, now=anchor)
|
||||
if result["errors"] and result["status"] == "ok":
|
||||
result["status"] = "partial"
|
||||
return result
|
||||
|
||||
|
||||
def select_backfill_targets(servers: str | None) -> list[RconServerTarget]:
|
||||
"""Load configured RCON targets and apply safe server selection rules."""
|
||||
configured_targets = list(load_rcon_targets())
|
||||
if not configured_targets:
|
||||
raise RuntimeError("No RCON targets configured in HLL_BACKEND_RCON_TARGETS.")
|
||||
by_key = {build_rcon_target_key(target): target for target in configured_targets}
|
||||
requested_keys = _parse_server_keys(servers)
|
||||
if requested_keys:
|
||||
unknown = sorted(key for key in requested_keys if key not in by_key)
|
||||
if unknown:
|
||||
raise ValueError(f"Unknown RCON server key(s): {', '.join(unknown)}")
|
||||
return [by_key[key] for key in requested_keys]
|
||||
selected = [
|
||||
target
|
||||
for key, target in by_key.items()
|
||||
if key in DEFAULT_ALLOWED_SERVER_KEYS and key not in EXCLUDED_BY_DEFAULT_SERVER_KEYS
|
||||
]
|
||||
if not selected:
|
||||
raise RuntimeError(
|
||||
"No default backfill targets selected. Pass --servers with configured keys explicitly."
|
||||
)
|
||||
return selected
|
||||
|
||||
|
||||
def count_recent_materialized_closed_matches(
|
||||
*,
|
||||
server_key: str | None = None,
|
||||
db_path: Path | None = None,
|
||||
) -> int:
|
||||
"""Count materialized closed AdminLog matches available for recent-match UI."""
|
||||
resolved_path = initialize_rcon_materialized_storage(db_path=db_path)
|
||||
scope_sql = ""
|
||||
params: list[object] = [MATCH_RESULT_SOURCE]
|
||||
if server_key and server_key != ALL_SERVERS_SLUG:
|
||||
scope_sql = "AND (target_key = ? OR external_server_id = ?)"
|
||||
params.extend([server_key, server_key])
|
||||
if use_postgres_rcon_storage(explicit_sqlite_path=db_path):
|
||||
from .postgres_rcon_storage import connect_postgres_compat
|
||||
|
||||
connection_scope = connect_postgres_compat()
|
||||
else:
|
||||
connection_scope = closing(connect_sqlite_readonly(resolved_path))
|
||||
with connection_scope as connection:
|
||||
row = connection.execute(
|
||||
f"""
|
||||
SELECT COUNT(*) AS count
|
||||
FROM rcon_materialized_matches
|
||||
WHERE source_basis = ?
|
||||
AND ended_at IS NOT NULL
|
||||
{scope_sql}
|
||||
""",
|
||||
params,
|
||||
).fetchone()
|
||||
return int(row["count"] or 0) if row else 0
|
||||
|
||||
|
||||
def _scan_target_window(target: RconServerTarget, window: BackfillWindow) -> dict[str, object]:
|
||||
target_metadata = _serialize_target(target)
|
||||
serialized_window = _serialize_window(window)
|
||||
try:
|
||||
with HllRconConnection(timeout_seconds=get_rcon_request_timeout_seconds()) as connection:
|
||||
connection.connect(host=target.host, port=target.port, password=target.password)
|
||||
payload = connection.execute_json(
|
||||
"GetAdminLog",
|
||||
{
|
||||
"LogBackTrackTime": window.lookback_seconds,
|
||||
"Filters": [],
|
||||
},
|
||||
)
|
||||
entries = payload.get("entries")
|
||||
if not isinstance(entries, list):
|
||||
entries = []
|
||||
normalized_entries = [entry for entry in entries if isinstance(entry, dict)]
|
||||
delta = persist_rcon_admin_log_entries(
|
||||
target=target_metadata,
|
||||
entries=normalized_entries,
|
||||
)
|
||||
return {"window": serialized_window, "error": None, **delta}
|
||||
except Exception as exc: # noqa: BLE001 - per-window errors must not hide neighboring windows
|
||||
return {
|
||||
"window": serialized_window,
|
||||
"events_seen": 0,
|
||||
"events_inserted": 0,
|
||||
"duplicate_events": 0,
|
||||
"error": {
|
||||
**target_metadata,
|
||||
**serialized_window,
|
||||
"error_type": type(exc).__name__,
|
||||
"message": str(exc),
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _resolve_requested_range(
|
||||
*,
|
||||
anchor: datetime,
|
||||
from_value: str | None,
|
||||
to_value: str | None,
|
||||
ensure_recent_matches: int | None,
|
||||
ensure_current_month: bool,
|
||||
ensure_leaderboard_windows: bool,
|
||||
max_days_back: int,
|
||||
) -> dict[str, object]:
|
||||
end = _parse_datetime_argument(to_value, default=anchor)
|
||||
starts = []
|
||||
reasons = []
|
||||
if from_value:
|
||||
starts.append(_parse_datetime_argument(from_value, default=anchor))
|
||||
reasons.append("explicit-range")
|
||||
if ensure_current_month:
|
||||
starts.append(_month_start(anchor))
|
||||
reasons.append("ensure-current-month")
|
||||
if ensure_leaderboard_windows:
|
||||
starts.append(_previous_month_start(_month_start(anchor)))
|
||||
starts.append(_week_start(anchor) - timedelta(days=7))
|
||||
reasons.append("ensure-leaderboard-windows")
|
||||
if ensure_recent_matches:
|
||||
starts.append(anchor - timedelta(days=max_days_back))
|
||||
reasons.append(f"ensure-recent-matches-{ensure_recent_matches}")
|
||||
if not starts:
|
||||
starts.append(anchor - timedelta(days=max_days_back))
|
||||
reasons.append("default-max-days-back")
|
||||
start = max(min(starts), anchor - timedelta(days=max_days_back))
|
||||
return {"start": start, "end": end, "reason": ",".join(reasons)}
|
||||
|
||||
|
||||
def _build_backfill_windows(
|
||||
*,
|
||||
start: datetime,
|
||||
end: datetime,
|
||||
chunk_hours: int,
|
||||
) -> list[BackfillWindow]:
|
||||
windows: list[BackfillWindow] = []
|
||||
cursor = _as_utc(end)
|
||||
lower = _as_utc(start)
|
||||
chunk = timedelta(hours=chunk_hours)
|
||||
while cursor > lower:
|
||||
window_start = max(lower, cursor - chunk)
|
||||
windows.append(BackfillWindow(start=window_start, end=cursor))
|
||||
cursor = window_start
|
||||
return windows
|
||||
|
||||
|
||||
def _limit_windows_for_recent_need(
|
||||
windows: list[BackfillWindow],
|
||||
*,
|
||||
ensure_recent_matches: int | None,
|
||||
db_path: Path | None,
|
||||
) -> list[BackfillWindow]:
|
||||
if not ensure_recent_matches:
|
||||
return windows
|
||||
if count_recent_materialized_closed_matches(db_path=db_path) >= ensure_recent_matches:
|
||||
return []
|
||||
return windows
|
||||
|
||||
|
||||
def _window_diagnostic(
|
||||
timeframe: str,
|
||||
*,
|
||||
db_path: Path | None,
|
||||
now: datetime,
|
||||
) -> dict[str, object]:
|
||||
payload = list_rcon_materialized_leaderboard(
|
||||
server_key=ALL_SERVERS_SLUG,
|
||||
timeframe=timeframe,
|
||||
metric="kills",
|
||||
limit=1,
|
||||
db_path=db_path,
|
||||
now=now,
|
||||
)
|
||||
return {
|
||||
"window_kind": payload.get("window_kind"),
|
||||
"window_label": payload.get("window_label"),
|
||||
"window_start": payload.get("window_start"),
|
||||
"window_end": payload.get("window_end"),
|
||||
"selection_reason": payload.get("selection_reason"),
|
||||
"current_week_closed_matches": payload.get("current_week_closed_matches"),
|
||||
"previous_week_closed_matches": payload.get("previous_week_closed_matches"),
|
||||
"selected_month_start": payload.get("selected_month_start"),
|
||||
"selected_month_end": payload.get("selected_month_end"),
|
||||
"current_month_closed_matches": payload.get("current_month_closed_matches"),
|
||||
"previous_month_closed_matches": payload.get("previous_month_closed_matches"),
|
||||
"sufficient_sample": payload.get("sufficient_sample"),
|
||||
}
|
||||
|
||||
|
||||
def _parse_server_keys(value: str | None) -> list[str]:
|
||||
return [part.strip() for part in str(value or "").split(",") if part.strip()]
|
||||
|
||||
|
||||
def _parse_datetime_argument(value: str | None, *, default: datetime) -> datetime:
|
||||
if value is None or str(value).strip().lower() == "now":
|
||||
return default
|
||||
raw = str(value).strip()
|
||||
if len(raw) == 10:
|
||||
raw = f"{raw}T00:00:00+00:00"
|
||||
parsed = datetime.fromisoformat(raw.replace("Z", "+00:00"))
|
||||
return _as_utc(parsed)
|
||||
|
||||
|
||||
def _month_start(value: datetime) -> datetime:
|
||||
point = _as_utc(value)
|
||||
return point.replace(day=1, hour=0, minute=0, second=0, microsecond=0)
|
||||
|
||||
|
||||
def _previous_month_start(current_month_start: datetime) -> datetime:
|
||||
return _month_start(current_month_start - timedelta(days=1))
|
||||
|
||||
|
||||
def _week_start(value: datetime) -> datetime:
|
||||
point = _as_utc(value)
|
||||
return (point - timedelta(days=point.weekday())).replace(
|
||||
hour=0,
|
||||
minute=0,
|
||||
second=0,
|
||||
microsecond=0,
|
||||
)
|
||||
|
||||
|
||||
def _as_utc(value: datetime) -> datetime:
|
||||
if value.tzinfo is None:
|
||||
return value.replace(tzinfo=timezone.utc)
|
||||
return value.astimezone(timezone.utc)
|
||||
|
||||
|
||||
def _serialize_target(target: RconServerTarget) -> dict[str, object]:
|
||||
return {
|
||||
"target_key": build_rcon_target_key(target),
|
||||
"external_server_id": target.external_server_id,
|
||||
"name": target.name,
|
||||
"host": target.host,
|
||||
"port": target.port,
|
||||
"source_name": target.source_name,
|
||||
}
|
||||
|
||||
|
||||
def _serialize_window(window: BackfillWindow) -> dict[str, object]:
|
||||
return {
|
||||
"start": _to_iso(window.start),
|
||||
"end": _to_iso(window.end),
|
||||
"requested_log_backtrack_seconds": window.lookback_seconds,
|
||||
}
|
||||
|
||||
|
||||
def _to_iso(value: datetime) -> str:
|
||||
return _as_utc(value).isoformat().replace("+00:00", "Z")
|
||||
|
||||
|
||||
def _main(argv: Iterable[str] | None = None) -> int:
|
||||
parser = argparse.ArgumentParser(description="Backfill RCON AdminLog historical materialized matches.")
|
||||
parser.add_argument("--from", dest="from_value", default=None)
|
||||
parser.add_argument("--to", dest="to_value", default=None)
|
||||
parser.add_argument("--servers", default=None)
|
||||
parser.add_argument("--ensure-recent-matches", type=int, default=None)
|
||||
parser.add_argument("--ensure-current-month", action="store_true")
|
||||
parser.add_argument("--ensure-leaderboard-windows", action="store_true")
|
||||
parser.add_argument("--chunk-hours", type=int, default=get_rcon_backfill_chunk_hours())
|
||||
parser.add_argument("--sleep-seconds", type=float, default=get_rcon_backfill_sleep_seconds())
|
||||
parser.add_argument("--max-days-back", type=int, default=get_rcon_backfill_max_days_back())
|
||||
parser.add_argument("--dry-run", action="store_true")
|
||||
parser.add_argument("--regenerate-snapshots", action="store_true")
|
||||
parser.add_argument("--db-path", type=Path, default=None)
|
||||
args = parser.parse_args(list(argv) if argv is not None else None)
|
||||
|
||||
if args.ensure_recent_matches is not None and args.ensure_recent_matches <= 0:
|
||||
raise ValueError("--ensure-recent-matches must be positive.")
|
||||
if args.chunk_hours <= 0:
|
||||
raise ValueError("--chunk-hours must be positive.")
|
||||
if args.sleep_seconds < 0:
|
||||
raise ValueError("--sleep-seconds must be zero or positive.")
|
||||
if args.max_days_back <= 0:
|
||||
raise ValueError("--max-days-back must be positive.")
|
||||
|
||||
payload = run_rcon_historical_backfill(
|
||||
servers=args.servers,
|
||||
from_value=args.from_value,
|
||||
to_value=args.to_value,
|
||||
ensure_recent_matches=args.ensure_recent_matches,
|
||||
ensure_current_month=args.ensure_current_month,
|
||||
ensure_leaderboard_windows=args.ensure_leaderboard_windows,
|
||||
chunk_hours=args.chunk_hours,
|
||||
sleep_seconds=args.sleep_seconds,
|
||||
max_days_back=args.max_days_back,
|
||||
dry_run=args.dry_run,
|
||||
regenerate_snapshots=args.regenerate_snapshots,
|
||||
db_path=args.db_path,
|
||||
)
|
||||
print(json.dumps(payload, ensure_ascii=False, indent=2, default=str))
|
||||
return 0 if payload.get("status") != "error" else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(_main())
|
||||
173
backend/app/rcon_historical_backfill_operational.py
Normal file
173
backend/app/rcon_historical_backfill_operational.py
Normal file
@@ -0,0 +1,173 @@
|
||||
"""Observable operator backfill for RCON AdminLog.
|
||||
|
||||
This command is intentionally simple and explicit. It is meant to be run after stopping
|
||||
`historical-runner` and `rcon-historical-worker`, so it does not compete with the shared
|
||||
writer lock loop. It prints one JSON line per step, which makes progress visible in
|
||||
PowerShell and Docker logs.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import time
|
||||
from datetime import datetime, timezone
|
||||
from typing import Iterable
|
||||
|
||||
from .historical_runner import generate_historical_snapshots
|
||||
from .rcon_admin_log_ingestion import ingest_rcon_admin_logs
|
||||
from .rcon_admin_log_materialization import materialize_rcon_admin_log
|
||||
from .rcon_historical_backfill import count_recent_materialized_closed_matches, select_backfill_targets
|
||||
from .rcon_client import build_rcon_target_key
|
||||
|
||||
|
||||
def run_operational_backfill(
|
||||
*,
|
||||
ensure_recent_matches: int,
|
||||
servers: str,
|
||||
max_days_back: int,
|
||||
chunk_hours: int,
|
||||
sleep_seconds: float,
|
||||
regenerate_snapshots: bool,
|
||||
) -> dict[str, object]:
|
||||
started_at = datetime.now(timezone.utc)
|
||||
targets = select_backfill_targets(servers)
|
||||
target_keys = [build_rcon_target_key(target) for target in targets]
|
||||
before = count_recent_materialized_closed_matches()
|
||||
result: dict[str, object] = {
|
||||
"status": "ok",
|
||||
"started_at": _iso(started_at),
|
||||
"admin_log_api": "lookback-only",
|
||||
"exact_historical_range_supported": False,
|
||||
"servers_processed": target_keys,
|
||||
"ensure_recent_matches": ensure_recent_matches,
|
||||
"max_days_back": max_days_back,
|
||||
"chunk_hours": chunk_hours,
|
||||
"recent_materialized_closed_match_count_before": before,
|
||||
"recent_materialized_closed_match_count_after": before,
|
||||
"events_seen": 0,
|
||||
"events_inserted": 0,
|
||||
"duplicate_events": 0,
|
||||
"matches_materialized": 0,
|
||||
"matches_updated": 0,
|
||||
"windows_scanned": [],
|
||||
"errors": [],
|
||||
"snapshot_regeneration_result": None,
|
||||
}
|
||||
_log("backfill-started", result=result)
|
||||
|
||||
max_minutes = max_days_back * 24 * 60
|
||||
step_minutes = chunk_hours * 60
|
||||
minutes = step_minutes
|
||||
|
||||
while minutes <= max_minutes:
|
||||
current_count = count_recent_materialized_closed_matches()
|
||||
result["recent_materialized_closed_match_count_after"] = current_count
|
||||
if current_count >= ensure_recent_matches:
|
||||
result["termination_reason"] = "recent-match-target-reached"
|
||||
break
|
||||
|
||||
for target_key in target_keys:
|
||||
_log("target-lookback-started", target_key=target_key, lookback_minutes=minutes)
|
||||
try:
|
||||
ingestion = ingest_rcon_admin_logs(minutes=minutes, target_key=target_key)
|
||||
totals = ingestion.get("totals") if isinstance(ingestion.get("totals"), dict) else {}
|
||||
materialized = materialize_rcon_admin_log()
|
||||
window_summary = {
|
||||
"target_key": target_key,
|
||||
"lookback_minutes": minutes,
|
||||
"events_seen": int(totals.get("events_seen") or 0),
|
||||
"events_inserted": int(totals.get("events_inserted") or 0),
|
||||
"duplicate_events": int(totals.get("duplicate_events") or 0),
|
||||
"matches_materialized": int(materialized.get("matches_materialized") or 0),
|
||||
"matches_updated": int(materialized.get("matches_updated") or 0),
|
||||
}
|
||||
result["windows_scanned"].append(window_summary)
|
||||
_add(result, window_summary)
|
||||
result["recent_materialized_closed_match_count_after"] = count_recent_materialized_closed_matches()
|
||||
_log(
|
||||
"target-lookback-finished",
|
||||
**window_summary,
|
||||
recent_materialized_closed_match_count_after=result["recent_materialized_closed_match_count_after"],
|
||||
)
|
||||
except Exception as exc: # noqa: BLE001 - operator command must continue reporting
|
||||
error = {
|
||||
"target_key": target_key,
|
||||
"lookback_minutes": minutes,
|
||||
"error_type": type(exc).__name__,
|
||||
"message": str(exc),
|
||||
}
|
||||
result["errors"].append(error)
|
||||
_log("target-lookback-failed", error=error)
|
||||
|
||||
if sleep_seconds > 0:
|
||||
time.sleep(sleep_seconds)
|
||||
|
||||
minutes += step_minutes
|
||||
|
||||
if result["recent_materialized_closed_match_count_after"] < ensure_recent_matches:
|
||||
result["status"] = "partial"
|
||||
result.setdefault("termination_reason", "exhausted_available_admin_log_or_max_days_back")
|
||||
|
||||
if regenerate_snapshots:
|
||||
_log("snapshot-regeneration-started")
|
||||
try:
|
||||
result["snapshot_regeneration_result"] = generate_historical_snapshots(server_slug=None, run_number=1)
|
||||
_log("snapshot-regeneration-finished", snapshot_regeneration_result=result["snapshot_regeneration_result"])
|
||||
except Exception as exc: # noqa: BLE001
|
||||
result["status"] = "partial"
|
||||
error = {"phase": "snapshot-regeneration", "error_type": type(exc).__name__, "message": str(exc)}
|
||||
result["errors"].append(error)
|
||||
_log("snapshot-regeneration-failed", error=error)
|
||||
|
||||
result["finished_at"] = _iso(datetime.now(timezone.utc))
|
||||
_log("backfill-finished", result=result)
|
||||
return result
|
||||
|
||||
|
||||
def _add(result: dict[str, object], window_summary: dict[str, object]) -> None:
|
||||
for key in ("events_seen", "events_inserted", "duplicate_events", "matches_materialized", "matches_updated"):
|
||||
result[key] = int(result.get(key) or 0) + int(window_summary.get(key) or 0)
|
||||
|
||||
|
||||
def _log(event: str, **payload: object) -> None:
|
||||
print(json.dumps({"event": event, **payload}, ensure_ascii=False, default=str), flush=True)
|
||||
|
||||
|
||||
def _iso(value: datetime) -> str:
|
||||
return value.astimezone(timezone.utc).isoformat().replace("+00:00", "Z")
|
||||
|
||||
|
||||
def _main(argv: Iterable[str] | None = None) -> int:
|
||||
parser = argparse.ArgumentParser(description="Observable RCON AdminLog backfill operator command.")
|
||||
parser.add_argument("--ensure-recent-matches", type=int, default=100)
|
||||
parser.add_argument("--servers", default="comunidad-hispana-01,comunidad-hispana-02")
|
||||
parser.add_argument("--chunk-hours", type=int, default=3)
|
||||
parser.add_argument("--sleep-seconds", type=float, default=1.0)
|
||||
parser.add_argument("--max-days-back", type=int, default=45)
|
||||
parser.add_argument("--regenerate-snapshots", action="store_true")
|
||||
args = parser.parse_args(list(argv) if argv is not None else None)
|
||||
|
||||
if args.ensure_recent_matches <= 0:
|
||||
raise ValueError("--ensure-recent-matches must be positive.")
|
||||
if args.chunk_hours <= 0:
|
||||
raise ValueError("--chunk-hours must be positive.")
|
||||
if args.sleep_seconds < 0:
|
||||
raise ValueError("--sleep-seconds must be zero or positive.")
|
||||
if args.max_days_back <= 0:
|
||||
raise ValueError("--max-days-back must be positive.")
|
||||
|
||||
payload = run_operational_backfill(
|
||||
ensure_recent_matches=args.ensure_recent_matches,
|
||||
servers=args.servers,
|
||||
chunk_hours=args.chunk_hours,
|
||||
sleep_seconds=args.sleep_seconds,
|
||||
max_days_back=args.max_days_back,
|
||||
regenerate_snapshots=args.regenerate_snapshots,
|
||||
)
|
||||
print(json.dumps(payload, ensure_ascii=False, indent=2, default=str), flush=True)
|
||||
return 0 if payload.get("status") != "error" else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(_main())
|
||||
600
backend/app/rcon_historical_leaderboards.py
Normal file
600
backend/app/rcon_historical_leaderboards.py
Normal file
@@ -0,0 +1,600 @@
|
||||
"""Leaderboard read model over materialized RCON/AdminLog match stats."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from contextlib import closing
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from pathlib import Path
|
||||
from typing import Literal
|
||||
|
||||
from .config import get_storage_path, use_postgres_rcon_storage
|
||||
from .config import get_historical_weekly_fallback_min_matches
|
||||
from .historical_storage import ALL_SERVERS_SLUG
|
||||
from .rcon_admin_log_materialization import (
|
||||
MATCH_RESULT_SOURCE,
|
||||
initialize_rcon_materialized_storage,
|
||||
)
|
||||
from .sqlite_utils import connect_sqlite_readonly
|
||||
|
||||
LeaderboardTimeframe = Literal["weekly", "monthly"]
|
||||
LeaderboardMetric = Literal["kills", "deaths", "matches_over_100_kills", "support"]
|
||||
|
||||
|
||||
def build_rcon_materialized_leaderboard_snapshot_payload(
|
||||
*,
|
||||
server_id: str | None = None,
|
||||
timeframe: str = "weekly",
|
||||
metric: str = "kills",
|
||||
limit: int = 10,
|
||||
) -> dict[str, object]:
|
||||
"""Return an API payload for RCON-backed leaderboard snapshots.
|
||||
|
||||
This is a runtime fast read over the materialized AdminLog tables. It intentionally
|
||||
avoids the old public-scoreboard fallback because the UI is running in RCON mode.
|
||||
"""
|
||||
|
||||
normalized_timeframe = _normalize_timeframe(timeframe)
|
||||
normalized_metric = _normalize_metric(metric)
|
||||
result = list_rcon_materialized_leaderboard(
|
||||
server_key=server_id,
|
||||
timeframe=normalized_timeframe,
|
||||
metric=normalized_metric,
|
||||
limit=limit,
|
||||
)
|
||||
items = list(result.get("items") or [])[:limit]
|
||||
return {
|
||||
"status": "ok",
|
||||
"data": {
|
||||
"title": _build_title(
|
||||
metric=normalized_metric,
|
||||
timeframe=normalized_timeframe,
|
||||
server_id=server_id,
|
||||
),
|
||||
"context": f"historical-{normalized_timeframe}-leaderboard-snapshot",
|
||||
"source": "rcon-materialized-admin-log-leaderboard",
|
||||
"server_slug": server_id,
|
||||
"timeframe": normalized_timeframe,
|
||||
"metric": normalized_metric,
|
||||
"found": True,
|
||||
"snapshot_status": "ready",
|
||||
"missing_reason": None,
|
||||
"request_path_policy": "runtime-rcon-materialized-fast-path",
|
||||
"generation_policy": "runtime-materialized-read",
|
||||
"generated_at": _to_iso(datetime.now(timezone.utc)),
|
||||
"source_range_start": result.get("source_range_start"),
|
||||
"source_range_end": result.get("source_range_end"),
|
||||
"is_stale": False,
|
||||
"freshness": "runtime",
|
||||
"window_days": result.get("window_days"),
|
||||
"window_start": result.get("window_start"),
|
||||
"window_end": result.get("window_end"),
|
||||
"window_kind": result.get("window_kind"),
|
||||
"window_label": result.get("window_label"),
|
||||
"uses_fallback": False,
|
||||
"selection_reason": result.get("selection_reason"),
|
||||
"current_week_start": result.get("current_week_start"),
|
||||
"current_week_closed_matches": result.get("current_week_closed_matches"),
|
||||
"previous_week_closed_matches": result.get("previous_week_closed_matches"),
|
||||
"current_month_start": result.get("current_month_start"),
|
||||
"selected_month_start": result.get("selected_month_start"),
|
||||
"selected_month_end": result.get("selected_month_end"),
|
||||
"current_month_closed_matches": result.get("current_month_closed_matches"),
|
||||
"previous_month_closed_matches": result.get("previous_month_closed_matches"),
|
||||
"sufficient_sample": result.get("sufficient_sample"),
|
||||
"snapshot_limit": result.get("limit"),
|
||||
"limit": limit,
|
||||
"runtime_enrichment": {
|
||||
"applied": False,
|
||||
"reason": None,
|
||||
},
|
||||
"primary_source": "rcon",
|
||||
"selected_source": "rcon",
|
||||
"fallback_used": False,
|
||||
"fallback_reason": None,
|
||||
"source_attempts": [
|
||||
{
|
||||
"source": "rcon",
|
||||
"role": "primary",
|
||||
"status": "success",
|
||||
"reason": "leaderboard-served-by-rcon-materialized-admin-log",
|
||||
"message": None,
|
||||
}
|
||||
],
|
||||
"items": items,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def list_rcon_materialized_leaderboard(
|
||||
*,
|
||||
server_key: str | None = None,
|
||||
timeframe: str = "weekly",
|
||||
metric: str = "kills",
|
||||
limit: int = 10,
|
||||
db_path: Path | None = None,
|
||||
now: datetime | None = None,
|
||||
) -> dict[str, object]:
|
||||
"""Return a leaderboard built from materialized RCON/AdminLog player stats.
|
||||
|
||||
RCON/AdminLog materialization currently has reliable kill/death/teamkill counters,
|
||||
but not public-scoreboard support points. For support, return an explicitly empty
|
||||
supported payload rather than falling back to unrelated public scoreboard storage.
|
||||
"""
|
||||
|
||||
normalized_timeframe = _normalize_timeframe(timeframe)
|
||||
normalized_metric = _normalize_metric(metric)
|
||||
normalized_limit = max(1, int(limit or 10))
|
||||
anchor = _as_utc(now or datetime.now(timezone.utc))
|
||||
|
||||
resolved_path = initialize_rcon_materialized_storage(db_path=db_path)
|
||||
connection_scope = _connect_scope(resolved_path, db_path=db_path)
|
||||
with connection_scope as connection:
|
||||
window = select_leaderboard_window(
|
||||
connection=connection,
|
||||
server_key=server_key,
|
||||
timeframe=normalized_timeframe,
|
||||
now=anchor,
|
||||
)
|
||||
if normalized_metric == "support":
|
||||
return _empty_payload(
|
||||
server_key=server_key,
|
||||
timeframe=normalized_timeframe,
|
||||
metric=normalized_metric,
|
||||
limit=normalized_limit,
|
||||
window=window,
|
||||
reason="rcon-materialized-stats-do-not-include-support-score",
|
||||
)
|
||||
rows = _fetch_leaderboard_rows(
|
||||
connection,
|
||||
server_key=server_key,
|
||||
metric=normalized_metric,
|
||||
limit=normalized_limit,
|
||||
window_start=window["start"],
|
||||
window_end=window["end"],
|
||||
)
|
||||
source_range = _fetch_source_range(
|
||||
connection,
|
||||
server_key=server_key,
|
||||
window_start=window["start"],
|
||||
window_end=window["end"],
|
||||
)
|
||||
|
||||
items = [_build_item(row, index=index + 1) for index, row in enumerate(rows)]
|
||||
return {
|
||||
"source": "rcon-materialized-admin-log-leaderboard",
|
||||
"server_key": server_key,
|
||||
"metric": normalized_metric,
|
||||
"limit": normalized_limit,
|
||||
"window_days": window["days"],
|
||||
"window_start": _to_iso(window["start"]),
|
||||
"window_end": _to_iso(window["end"]),
|
||||
"window_kind": window["kind"],
|
||||
"window_label": window["label"],
|
||||
"uses_fallback": False,
|
||||
"selection_reason": window["selection_reason"],
|
||||
"current_week_start": _to_iso(window["current_week_start"]),
|
||||
"current_week_closed_matches": window["current_week_closed_matches"],
|
||||
"previous_week_closed_matches": window["previous_week_closed_matches"],
|
||||
"current_month_start": _to_iso(window["current_month_start"]),
|
||||
"selected_month_start": _to_iso(window["selected_month_start"]),
|
||||
"selected_month_end": _to_iso(window["selected_month_end"]),
|
||||
"current_month_closed_matches": window["current_month_closed_matches"],
|
||||
"previous_month_closed_matches": window["previous_month_closed_matches"],
|
||||
"sufficient_sample": window["sufficient_sample"],
|
||||
"source_range_start": _to_iso(source_range[0]) if source_range[0] else None,
|
||||
"source_range_end": _to_iso(source_range[1]) if source_range[1] else None,
|
||||
"items": items,
|
||||
}
|
||||
|
||||
|
||||
def _fetch_leaderboard_rows(
|
||||
connection: object,
|
||||
*,
|
||||
server_key: str | None,
|
||||
metric: str,
|
||||
limit: int,
|
||||
window_start: datetime,
|
||||
window_end: datetime,
|
||||
) -> list[dict[str, object]]:
|
||||
scope_sql, scope_params = _build_scope_sql(server_key)
|
||||
metric_sql = {
|
||||
"kills": "SUM(COALESCE(stats.kills, 0))",
|
||||
"deaths": "SUM(COALESCE(stats.deaths, 0))",
|
||||
"matches_over_100_kills": "SUM(CASE WHEN COALESCE(stats.kills, 0) >= 100 THEN 1 ELSE 0 END)",
|
||||
}[metric]
|
||||
having_sql = f"HAVING {metric_sql} > 0"
|
||||
params: list[object] = [
|
||||
_to_iso(window_start),
|
||||
_to_iso(window_end),
|
||||
*scope_params,
|
||||
limit,
|
||||
]
|
||||
rows = connection.execute(
|
||||
f"""
|
||||
SELECT
|
||||
stats.player_id,
|
||||
stats.player_name,
|
||||
{metric_sql} AS metric_value,
|
||||
COUNT(DISTINCT stats.match_key) AS matches_considered,
|
||||
SUM(COALESCE(stats.kills, 0)) AS kills,
|
||||
SUM(COALESCE(stats.deaths, 0)) AS deaths,
|
||||
SUM(COALESCE(stats.teamkills, 0)) AS teamkills
|
||||
FROM rcon_match_player_stats AS stats
|
||||
INNER JOIN rcon_materialized_matches AS matches
|
||||
ON matches.target_key = stats.target_key
|
||||
AND matches.match_key = stats.match_key
|
||||
WHERE matches.source_basis = ?
|
||||
AND COALESCE(CAST(matches.ended_at AS TEXT), CAST(matches.started_at AS TEXT)) >= ?
|
||||
AND COALESCE(CAST(matches.ended_at AS TEXT), CAST(matches.started_at AS TEXT)) <= ?
|
||||
{scope_sql}
|
||||
AND TRIM(COALESCE(stats.player_name, '')) != ''
|
||||
GROUP BY stats.player_id, stats.player_name
|
||||
{having_sql}
|
||||
ORDER BY metric_value DESC, matches_considered DESC, stats.player_name ASC
|
||||
LIMIT ?
|
||||
""",
|
||||
[MATCH_RESULT_SOURCE, *params],
|
||||
).fetchall()
|
||||
return [dict(row) for row in rows]
|
||||
|
||||
|
||||
def _fetch_match_counts(
|
||||
connection: object,
|
||||
*,
|
||||
server_key: str | None,
|
||||
timeframe: str,
|
||||
window_start: datetime,
|
||||
window_end: datetime,
|
||||
) -> dict[str, int]:
|
||||
current_week_start = _week_start(window_end)
|
||||
previous_week_start = current_week_start - timedelta(days=7)
|
||||
current_month_start = _month_start(window_end)
|
||||
previous_month_start = _previous_month_start(current_month_start)
|
||||
return {
|
||||
"current_week_closed_matches": _count_matches(
|
||||
connection,
|
||||
server_key=server_key,
|
||||
start=current_week_start,
|
||||
end=window_end,
|
||||
),
|
||||
"previous_week_closed_matches": _count_matches(
|
||||
connection,
|
||||
server_key=server_key,
|
||||
start=previous_week_start,
|
||||
end=current_week_start,
|
||||
),
|
||||
"current_month_closed_matches": _count_matches(
|
||||
connection,
|
||||
server_key=server_key,
|
||||
start=current_month_start,
|
||||
end=window_end,
|
||||
),
|
||||
"previous_month_closed_matches": _count_matches(
|
||||
connection,
|
||||
server_key=server_key,
|
||||
start=previous_month_start,
|
||||
end=current_month_start,
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def select_leaderboard_window(
|
||||
*,
|
||||
connection: object,
|
||||
server_key: str | None,
|
||||
timeframe: str,
|
||||
now: datetime | None = None,
|
||||
) -> dict[str, object]:
|
||||
"""Select the RCON leaderboard window using weekly/monthly fallback policy."""
|
||||
anchor = _as_utc(now or datetime.now(timezone.utc))
|
||||
current_week_start = _week_start(anchor)
|
||||
previous_week_start = current_week_start - timedelta(days=7)
|
||||
current_month_start = _month_start(anchor)
|
||||
previous_month_start = _previous_month_start(current_month_start)
|
||||
minimum_week_matches = get_historical_weekly_fallback_min_matches()
|
||||
current_week_count = _count_matches(
|
||||
connection,
|
||||
server_key=server_key,
|
||||
start=current_week_start,
|
||||
end=anchor,
|
||||
)
|
||||
previous_week_count = _count_matches(
|
||||
connection,
|
||||
server_key=server_key,
|
||||
start=previous_week_start,
|
||||
end=current_week_start,
|
||||
)
|
||||
current_month_count = _count_matches(
|
||||
connection,
|
||||
server_key=server_key,
|
||||
start=current_month_start,
|
||||
end=anchor,
|
||||
)
|
||||
previous_month_count = _count_matches(
|
||||
connection,
|
||||
server_key=server_key,
|
||||
start=previous_month_start,
|
||||
end=current_month_start,
|
||||
)
|
||||
|
||||
if timeframe == "monthly":
|
||||
use_previous_month = anchor.day <= 7
|
||||
start = previous_month_start if use_previous_month else current_month_start
|
||||
end = current_month_start if use_previous_month else anchor
|
||||
return {
|
||||
"start": start,
|
||||
"end": end,
|
||||
"days": max(1, (end.date() - start.date()).days),
|
||||
"kind": "previous-month" if use_previous_month else "current-month",
|
||||
"label": "Mes anterior" if use_previous_month else "Mes actual",
|
||||
"selection_reason": (
|
||||
"monthly-uses-previous-month-until-day-8"
|
||||
if use_previous_month
|
||||
else "monthly-uses-current-month-after-day-7"
|
||||
),
|
||||
"current_week_start": current_week_start,
|
||||
"current_week_closed_matches": current_week_count,
|
||||
"previous_week_closed_matches": previous_week_count,
|
||||
"current_month_start": current_month_start,
|
||||
"selected_month_start": start,
|
||||
"selected_month_end": end,
|
||||
"current_month_closed_matches": current_month_count,
|
||||
"previous_month_closed_matches": previous_month_count,
|
||||
"sufficient_sample": {
|
||||
"minimum_closed_matches": 1,
|
||||
"current_month_closed_matches": current_month_count,
|
||||
"previous_month_closed_matches": previous_month_count,
|
||||
"current_month_has_sufficient_sample": current_month_count >= 1,
|
||||
"uses_previous_month_until_day": 7,
|
||||
},
|
||||
}
|
||||
|
||||
current_week_has_sample = current_week_count >= minimum_week_matches
|
||||
start = current_week_start if current_week_has_sample else previous_week_start
|
||||
end = anchor if current_week_has_sample else current_week_start
|
||||
return {
|
||||
"start": start,
|
||||
"end": end,
|
||||
"days": max(1, (end.date() - start.date()).days),
|
||||
"kind": "current-week" if current_week_has_sample else "previous-week",
|
||||
"label": "Semana actual" if current_week_has_sample else "Semana anterior",
|
||||
"selection_reason": (
|
||||
"weekly-current-week-has-sufficient-closed-matches"
|
||||
if current_week_has_sample
|
||||
else "weekly-fallback-previous-week-insufficient-current-week-data"
|
||||
),
|
||||
"current_week_start": current_week_start,
|
||||
"current_week_closed_matches": current_week_count,
|
||||
"previous_week_closed_matches": previous_week_count,
|
||||
"current_month_start": current_month_start,
|
||||
"selected_month_start": current_month_start,
|
||||
"selected_month_end": anchor,
|
||||
"current_month_closed_matches": current_month_count,
|
||||
"previous_month_closed_matches": previous_month_count,
|
||||
"sufficient_sample": {
|
||||
"minimum_closed_matches": minimum_week_matches,
|
||||
"current_week_closed_matches": current_week_count,
|
||||
"current_week_has_sufficient_sample": current_week_has_sample,
|
||||
"previous_week_closed_matches": previous_week_count,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _fetch_source_range(
|
||||
connection: object,
|
||||
*,
|
||||
server_key: str | None,
|
||||
window_start: datetime,
|
||||
window_end: datetime,
|
||||
) -> tuple[datetime | None, datetime | None]:
|
||||
scope_sql, scope_params = _build_scope_sql(server_key, table_alias="matches")
|
||||
row = connection.execute(
|
||||
f"""
|
||||
SELECT
|
||||
MIN(COALESCE(CAST(matches.ended_at AS TEXT), CAST(matches.started_at AS TEXT))) AS source_range_start,
|
||||
MAX(COALESCE(CAST(matches.ended_at AS TEXT), CAST(matches.started_at AS TEXT))) AS source_range_end
|
||||
FROM rcon_materialized_matches AS matches
|
||||
WHERE matches.source_basis = ?
|
||||
AND COALESCE(CAST(matches.ended_at AS TEXT), CAST(matches.started_at AS TEXT)) >= ?
|
||||
AND COALESCE(CAST(matches.ended_at AS TEXT), CAST(matches.started_at AS TEXT)) <= ?
|
||||
{scope_sql}
|
||||
""",
|
||||
[MATCH_RESULT_SOURCE, _to_iso(window_start), _to_iso(window_end), *scope_params],
|
||||
).fetchone()
|
||||
if not row:
|
||||
return None, None
|
||||
return _parse_datetime(row["source_range_start"]), _parse_datetime(row["source_range_end"])
|
||||
|
||||
|
||||
def _count_matches(
|
||||
connection: object,
|
||||
*,
|
||||
server_key: str | None,
|
||||
start: datetime,
|
||||
end: datetime,
|
||||
) -> int:
|
||||
scope_sql, scope_params = _build_scope_sql(server_key, table_alias="matches")
|
||||
row = connection.execute(
|
||||
f"""
|
||||
SELECT COUNT(*) AS count
|
||||
FROM rcon_materialized_matches AS matches
|
||||
WHERE matches.source_basis = ?
|
||||
AND COALESCE(CAST(matches.ended_at AS TEXT), CAST(matches.started_at AS TEXT)) >= ?
|
||||
AND COALESCE(CAST(matches.ended_at AS TEXT), CAST(matches.started_at AS TEXT)) < ?
|
||||
{scope_sql}
|
||||
""",
|
||||
[MATCH_RESULT_SOURCE, _to_iso(start), _to_iso(end), *scope_params],
|
||||
).fetchone()
|
||||
return int(row["count"] or 0) if row else 0
|
||||
|
||||
|
||||
def _build_item(row: dict[str, object], *, index: int) -> dict[str, object]:
|
||||
kills = _coerce_int(row.get("kills"))
|
||||
deaths = _coerce_int(row.get("deaths"))
|
||||
return {
|
||||
"ranking_position": index,
|
||||
"player": {
|
||||
"id": row.get("player_id"),
|
||||
"name": row.get("player_name"),
|
||||
},
|
||||
"player_id": row.get("player_id"),
|
||||
"player_name": row.get("player_name"),
|
||||
"metric_value": _coerce_int(row.get("metric_value")),
|
||||
"matches_considered": _coerce_int(row.get("matches_considered")),
|
||||
"kills": kills,
|
||||
"deaths": deaths,
|
||||
"teamkills": _coerce_int(row.get("teamkills")),
|
||||
"kd_ratio": round(kills / deaths, 2) if deaths else float(kills),
|
||||
}
|
||||
|
||||
|
||||
def _build_scope_sql(
|
||||
server_key: str | None,
|
||||
*,
|
||||
table_alias: str = "matches",
|
||||
) -> tuple[str, list[object]]:
|
||||
if not server_key or server_key == ALL_SERVERS_SLUG:
|
||||
return "", []
|
||||
return f"AND ({table_alias}.target_key = ? OR {table_alias}.external_server_id = ?)", [
|
||||
server_key,
|
||||
server_key,
|
||||
]
|
||||
|
||||
|
||||
def _connect_scope(resolved_path: Path, *, db_path: Path | None):
|
||||
if use_postgres_rcon_storage(explicit_sqlite_path=db_path):
|
||||
from .postgres_rcon_storage import connect_postgres_compat
|
||||
|
||||
return connect_postgres_compat()
|
||||
return closing(connect_sqlite_readonly(resolved_path))
|
||||
|
||||
|
||||
def _empty_payload(
|
||||
*,
|
||||
server_key: str | None,
|
||||
timeframe: str,
|
||||
metric: str,
|
||||
limit: int,
|
||||
window: dict[str, object],
|
||||
reason: str,
|
||||
) -> dict[str, object]:
|
||||
return {
|
||||
"source": "rcon-materialized-admin-log-leaderboard",
|
||||
"server_key": server_key,
|
||||
"metric": metric,
|
||||
"limit": limit,
|
||||
"window_days": window["days"],
|
||||
"window_start": _to_iso(window["start"]),
|
||||
"window_end": _to_iso(window["end"]),
|
||||
"window_kind": window["kind"],
|
||||
"window_label": window["label"],
|
||||
"uses_fallback": False,
|
||||
"selection_reason": reason,
|
||||
"current_week_start": _to_iso(window["current_week_start"]),
|
||||
"current_week_closed_matches": window["current_week_closed_matches"],
|
||||
"previous_week_closed_matches": window["previous_week_closed_matches"],
|
||||
"current_month_start": _to_iso(window["current_month_start"]),
|
||||
"selected_month_start": _to_iso(window["selected_month_start"]),
|
||||
"selected_month_end": _to_iso(window["selected_month_end"]),
|
||||
"current_month_closed_matches": window["current_month_closed_matches"],
|
||||
"previous_month_closed_matches": window["previous_month_closed_matches"],
|
||||
"sufficient_sample": window["sufficient_sample"],
|
||||
"source_range_start": None,
|
||||
"source_range_end": None,
|
||||
"items": [],
|
||||
}
|
||||
|
||||
|
||||
def _build_window(timeframe: str) -> dict[str, object]:
|
||||
now = datetime.now(timezone.utc)
|
||||
if timeframe == "monthly":
|
||||
start = _month_start(now)
|
||||
return {
|
||||
"start": start,
|
||||
"end": now,
|
||||
"days": max(1, (now.date() - start.date()).days + 1),
|
||||
"kind": "current-month",
|
||||
"label": "Mes actual",
|
||||
}
|
||||
start = _week_start(now)
|
||||
return {
|
||||
"start": start,
|
||||
"end": now,
|
||||
"days": max(1, (now.date() - start.date()).days + 1),
|
||||
"kind": "current-week",
|
||||
"label": "Semana actual",
|
||||
}
|
||||
|
||||
|
||||
def _as_utc(value: datetime) -> datetime:
|
||||
if value.tzinfo is None:
|
||||
return value.replace(tzinfo=timezone.utc)
|
||||
return value.astimezone(timezone.utc)
|
||||
|
||||
|
||||
def _week_start(value: datetime) -> datetime:
|
||||
point = value.astimezone(timezone.utc)
|
||||
start = point - timedelta(days=point.weekday())
|
||||
return start.replace(hour=0, minute=0, second=0, microsecond=0)
|
||||
|
||||
|
||||
def _month_start(value: datetime) -> datetime:
|
||||
point = value.astimezone(timezone.utc)
|
||||
return point.replace(day=1, hour=0, minute=0, second=0, microsecond=0)
|
||||
|
||||
|
||||
def _previous_month_start(current_month_start: datetime) -> datetime:
|
||||
previous_month_end = current_month_start - timedelta(days=1)
|
||||
return _month_start(previous_month_end)
|
||||
|
||||
|
||||
def _normalize_timeframe(value: str) -> LeaderboardTimeframe:
|
||||
return "monthly" if str(value or "").strip().lower() == "monthly" else "weekly"
|
||||
|
||||
|
||||
def _normalize_metric(value: str) -> LeaderboardMetric:
|
||||
normalized = str(value or "kills").strip().lower()
|
||||
if normalized in {"kills", "deaths", "matches_over_100_kills", "support"}:
|
||||
return normalized # type: ignore[return-value]
|
||||
return "kills"
|
||||
|
||||
|
||||
def _build_title(*, metric: str, timeframe: str, server_id: str | None) -> str:
|
||||
timeframe_label = "mensual" if timeframe == "monthly" else "semanal"
|
||||
scope = "totales" if server_id == ALL_SERVERS_SLUG else "por servidor"
|
||||
metric_label = {
|
||||
"kills": "Top kills",
|
||||
"deaths": "Top muertes",
|
||||
"matches_over_100_kills": "Partidas 100+ kills",
|
||||
"support": "Top soporte",
|
||||
}.get(metric, "Top kills")
|
||||
return f"Snapshot {metric_label} {timeframe_label} {scope}"
|
||||
|
||||
|
||||
def _coerce_int(value: object) -> int:
|
||||
try:
|
||||
return int(value or 0)
|
||||
except (TypeError, ValueError):
|
||||
return 0
|
||||
|
||||
|
||||
def _parse_datetime(value: object) -> datetime | None:
|
||||
if isinstance(value, datetime):
|
||||
parsed = value
|
||||
elif isinstance(value, str) and value.strip():
|
||||
try:
|
||||
parsed = datetime.fromisoformat(value.strip().replace("Z", "+00:00"))
|
||||
except ValueError:
|
||||
return None
|
||||
else:
|
||||
return None
|
||||
if parsed.tzinfo is None:
|
||||
parsed = parsed.replace(tzinfo=timezone.utc)
|
||||
return parsed.astimezone(timezone.utc)
|
||||
|
||||
|
||||
def _to_iso(value: object) -> str:
|
||||
parsed = _parse_datetime(value)
|
||||
if parsed is None:
|
||||
parsed = datetime.now(timezone.utc)
|
||||
return parsed.astimezone(timezone.utc).isoformat().replace("+00:00", "Z")
|
||||
627
backend/app/rcon_historical_read_model.py
Normal file
627
backend/app/rcon_historical_read_model.py
Normal file
@@ -0,0 +1,627 @@
|
||||
"""Read-only minimal HTTP model over prospective RCON historical persistence."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
from .historical_storage import ALL_SERVERS_SLUG
|
||||
from .normalizers import normalize_map_name
|
||||
from .player_external_profiles import build_external_player_profile_fields
|
||||
from .rcon_scoreboard_correlation import resolve_rcon_scoreboard_match_url
|
||||
from .rcon_historical_storage import (
|
||||
find_rcon_historical_competitive_window,
|
||||
get_rcon_historical_competitive_window_by_session,
|
||||
list_rcon_historical_competitive_summary_rows,
|
||||
list_rcon_historical_competitive_windows,
|
||||
)
|
||||
|
||||
MATCH_RESULT_SOURCE = "admin-log-match-ended"
|
||||
SESSION_RESULT_SOURCE = "rcon-session"
|
||||
|
||||
|
||||
def list_rcon_historical_server_summaries(
|
||||
*,
|
||||
server_key: str | None = None,
|
||||
) -> list[dict[str, object]]:
|
||||
"""Return per-target coverage and freshness from RCON-backed competitive storage."""
|
||||
items = list_rcon_historical_competitive_summary_rows()
|
||||
if server_key and server_key != ALL_SERVERS_SLUG:
|
||||
normalized = server_key.strip()
|
||||
items = [
|
||||
item
|
||||
for item in items
|
||||
if item["target_key"] == normalized or item["external_server_id"] == normalized
|
||||
]
|
||||
|
||||
summaries = [_build_server_summary(item) for item in items]
|
||||
if server_key == ALL_SERVERS_SLUG:
|
||||
return [_build_all_servers_summary(summaries)]
|
||||
return summaries
|
||||
|
||||
|
||||
def list_rcon_historical_recent_activity(
|
||||
*,
|
||||
server_key: str | None = None,
|
||||
limit: int = 20,
|
||||
) -> list[dict[str, object]]:
|
||||
"""Return recent RCON-backed competitive windows for one or all targets."""
|
||||
from .rcon_admin_log_materialization import list_materialized_rcon_matches
|
||||
|
||||
normalized_server_key = None if server_key == ALL_SERVERS_SLUG else server_key
|
||||
materialized_items = list_materialized_rcon_matches(
|
||||
target_key=normalized_server_key,
|
||||
only_ended=True,
|
||||
limit=limit,
|
||||
)
|
||||
primary_items = [_build_materialized_recent_item(item) for item in materialized_items]
|
||||
if primary_items:
|
||||
return primary_items[:limit]
|
||||
|
||||
session_items = list_rcon_historical_competitive_windows(
|
||||
target_key=normalized_server_key,
|
||||
limit=limit,
|
||||
)
|
||||
fallback_items = [
|
||||
{
|
||||
"server": {
|
||||
"slug": item["target_key"],
|
||||
"name": item["display_name"],
|
||||
"external_server_id": item["external_server_id"],
|
||||
"region": item["region"],
|
||||
},
|
||||
"match_id": item["session_key"],
|
||||
"internal_detail_match_id": item["session_key"],
|
||||
"started_at": item["first_seen_at"],
|
||||
"ended_at": item["last_seen_at"],
|
||||
"closed_at": item["last_seen_at"],
|
||||
"map": {
|
||||
"name": item.get("map_name"),
|
||||
"pretty_name": normalize_map_name(item.get("map_pretty_name") or item.get("map_name")),
|
||||
},
|
||||
"result": _build_rcon_result(item.get("latest_payload")),
|
||||
"gamestate": _build_rcon_gamestate(item.get("latest_payload")),
|
||||
"player_count": int(round(float(item.get("average_players") or 0))),
|
||||
"peak_players": item.get("peak_players"),
|
||||
"sample_count": item.get("sample_count"),
|
||||
"duration_seconds": item.get("duration_seconds"),
|
||||
"capture_basis": "rcon-competitive-window",
|
||||
"result_source": SESSION_RESULT_SOURCE,
|
||||
"capabilities": item.get("capabilities"),
|
||||
"minutes_since_capture": _minutes_since_timestamp(item.get("last_seen_at")),
|
||||
}
|
||||
for item in session_items
|
||||
]
|
||||
return _merge_recent_items(primary_items, fallback_items, limit=limit)
|
||||
|
||||
|
||||
def describe_rcon_historical_read_model() -> dict[str, object]:
|
||||
"""Describe what the minimal RCON historical read model currently supports."""
|
||||
return {
|
||||
"source": "rcon-historical-competitive-read-model",
|
||||
"supported_endpoints": [
|
||||
"/api/historical/server-summary",
|
||||
"/api/historical/recent-matches",
|
||||
],
|
||||
"unsupported_endpoints": [
|
||||
"/api/historical/weekly-top-kills",
|
||||
"/api/historical/weekly-leaderboard",
|
||||
"/api/historical/leaderboard",
|
||||
"/api/historical/monthly-mvp",
|
||||
"/api/historical/monthly-mvp-v2",
|
||||
"/api/historical/elo-mmr/leaderboard",
|
||||
"/api/historical/elo-mmr/player",
|
||||
"/api/historical/player-events",
|
||||
"/api/historical/player-profile",
|
||||
"/api/historical/snapshots/*",
|
||||
],
|
||||
"capabilities": {
|
||||
"server_summary": "exact",
|
||||
"recent_matches": "exact-when-admin-log-match-ended",
|
||||
"competitive_quality": "partial",
|
||||
"result": "admin-log-match-ended",
|
||||
"gamestate": "session-fallback",
|
||||
"player_stats": "admin-log-derived",
|
||||
},
|
||||
"limitations": [
|
||||
"No retroactive backfill of closed matches.",
|
||||
"No weekly or monthly competitive leaderboards.",
|
||||
"No MVP or player-event parity with public-scoreboard.",
|
||||
"No player-level scoreboard parity from RCON samples alone.",
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def get_rcon_historical_competitive_match_context(
|
||||
*,
|
||||
server_key: str,
|
||||
ended_at: str | None,
|
||||
map_name: str | None = None,
|
||||
) -> dict[str, object] | None:
|
||||
"""Return the closest RCON-backed competitive context for one historical match."""
|
||||
return find_rcon_historical_competitive_window(
|
||||
server_key=server_key,
|
||||
ended_at=ended_at,
|
||||
map_name=map_name,
|
||||
)
|
||||
|
||||
|
||||
def get_rcon_historical_match_detail(
|
||||
*,
|
||||
server_key: str,
|
||||
match_id: str,
|
||||
) -> dict[str, object] | None:
|
||||
"""Return one RCON competitive window as a match-detail compatible payload."""
|
||||
from .rcon_admin_log_materialization import get_materialized_rcon_match_detail
|
||||
|
||||
materialized = get_materialized_rcon_match_detail(server_key=server_key, match_key=match_id)
|
||||
if materialized is not None:
|
||||
return _build_materialized_detail_item(materialized)
|
||||
|
||||
item = get_rcon_historical_competitive_window_by_session(
|
||||
server_key=server_key,
|
||||
session_key=match_id,
|
||||
)
|
||||
if item is None:
|
||||
return None
|
||||
player_count = int(round(float(item.get("average_players") or 0)))
|
||||
server_slug = item["external_server_id"] or item["target_key"]
|
||||
return {
|
||||
"server": {
|
||||
"slug": item["target_key"],
|
||||
"name": item["display_name"],
|
||||
"external_server_id": item["external_server_id"],
|
||||
"region": item["region"],
|
||||
},
|
||||
"match_id": item["session_key"],
|
||||
"started_at": item["first_seen_at"],
|
||||
"ended_at": item["last_seen_at"],
|
||||
"closed_at": item["last_seen_at"],
|
||||
"duration_seconds": item.get("duration_seconds"),
|
||||
"map": {
|
||||
"name": item.get("map_name"),
|
||||
"pretty_name": normalize_map_name(item.get("map_pretty_name") or item.get("map_name")),
|
||||
},
|
||||
"result": _build_rcon_result(item.get("latest_payload")),
|
||||
"gamestate": _build_rcon_gamestate(item.get("latest_payload")),
|
||||
"player_count": int(round(float(item.get("average_players") or 0))),
|
||||
"peak_players": item.get("peak_players"),
|
||||
"sample_count": item.get("sample_count"),
|
||||
"players": [],
|
||||
"capture_basis": "rcon-competitive-window",
|
||||
"confidence": item.get("confidence_mode"),
|
||||
"source_basis": "rcon-session",
|
||||
"result_source": SESSION_RESULT_SOURCE,
|
||||
"capabilities": item.get("capabilities"),
|
||||
"match_url": resolve_rcon_scoreboard_match_url(
|
||||
server_slug=server_slug,
|
||||
map_name=item.get("map_pretty_name") or item.get("map_name"),
|
||||
started_at=item["first_seen_at"],
|
||||
ended_at=item["last_seen_at"],
|
||||
duration_seconds=item.get("duration_seconds"),
|
||||
player_count=player_count,
|
||||
peak_players=item.get("peak_players"),
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def _build_materialized_recent_item(item: dict[str, object]) -> dict[str, object]:
|
||||
timestamps = _build_materialized_timestamp_payload(item)
|
||||
player_count = _resolve_materialized_player_count(item)
|
||||
scoreboard_correlation = build_materialized_scoreboard_correlation_input(item)
|
||||
return {
|
||||
"server": {
|
||||
"slug": item.get("target_key"),
|
||||
"name": _server_display_name(item.get("external_server_id") or item.get("target_key")),
|
||||
"external_server_id": item.get("external_server_id"),
|
||||
"region": None,
|
||||
},
|
||||
"match_id": item.get("match_key"),
|
||||
"internal_detail_match_id": item.get("match_key"),
|
||||
"started_at": timestamps["started_at"],
|
||||
"ended_at": timestamps["ended_at"],
|
||||
"closed_at": timestamps["closed_at"],
|
||||
"timestamp_confidence": timestamps["timestamp_confidence"],
|
||||
"map": {
|
||||
"name": item.get("map_name"),
|
||||
"pretty_name": item.get("map_pretty_name") or normalize_map_name(item.get("map_name")),
|
||||
},
|
||||
"game_mode": item.get("game_mode"),
|
||||
"result": {
|
||||
"allied_score": item.get("allied_score"),
|
||||
"axis_score": item.get("axis_score"),
|
||||
"winner": item.get("winner"),
|
||||
},
|
||||
"winner": item.get("winner"),
|
||||
"player_count": player_count,
|
||||
"peak_players": None,
|
||||
"sample_count": None,
|
||||
"duration_seconds": _calculate_match_duration_seconds(item),
|
||||
"capture_basis": "rcon-materialized-admin-log",
|
||||
"confidence": item.get("confidence_mode"),
|
||||
"source_basis": item.get("source_basis"),
|
||||
"result_source": (
|
||||
MATCH_RESULT_SOURCE
|
||||
if item.get("source_basis") == MATCH_RESULT_SOURCE
|
||||
else SESSION_RESULT_SOURCE
|
||||
),
|
||||
"match_url": resolve_rcon_scoreboard_match_url(
|
||||
**scoreboard_correlation,
|
||||
),
|
||||
"capabilities": describe_rcon_historical_read_model()["capabilities"],
|
||||
}
|
||||
|
||||
|
||||
def _build_materialized_detail_item(materialized: dict[str, object]) -> dict[str, object]:
|
||||
from .rcon_admin_log_storage import get_latest_rcon_player_profile_summaries
|
||||
|
||||
match = materialized["match"]
|
||||
recent_item = _build_materialized_recent_item(match)
|
||||
profile_summaries = get_latest_rcon_player_profile_summaries(
|
||||
target_key=str(match["target_key"]),
|
||||
player_ids=[str(row["player_id"]) for row in materialized["players"] if row.get("player_id")],
|
||||
)
|
||||
players = [
|
||||
_build_player_row(
|
||||
row,
|
||||
profile_summary=profile_summaries.get(str(row.get("player_id"))),
|
||||
)
|
||||
for row in materialized["players"]
|
||||
]
|
||||
player_count = len(players) if players else recent_item.get("player_count")
|
||||
return {
|
||||
**recent_item,
|
||||
"match_id": match["match_key"],
|
||||
"game_mode": match.get("game_mode"),
|
||||
"winner": match.get("winner"),
|
||||
"confidence": match.get("confidence_mode"),
|
||||
"source_basis": match.get("source_basis"),
|
||||
"player_count": player_count,
|
||||
"players": players,
|
||||
"timeline": {
|
||||
"event_counts": materialized.get("timeline", []),
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _resolve_materialized_player_count(item: dict[str, object]) -> int | None:
|
||||
for key in (
|
||||
"player_count",
|
||||
"materialized_player_count",
|
||||
"materialized_distinct_player_count",
|
||||
):
|
||||
value = _coerce_optional_int(item.get(key))
|
||||
if value is not None and value > 0:
|
||||
return value
|
||||
return None
|
||||
|
||||
|
||||
def _build_player_row(
|
||||
row: dict[str, object],
|
||||
*,
|
||||
profile_summary: dict[str, object] | None = None,
|
||||
) -> dict[str, object]:
|
||||
kills = _coerce_optional_int(row.get("kills")) or 0
|
||||
deaths = _coerce_optional_int(row.get("deaths")) or 0
|
||||
player = {
|
||||
"player_name": row.get("player_name"),
|
||||
"team": row.get("team"),
|
||||
"kills": kills,
|
||||
"deaths": deaths,
|
||||
"teamkills": _coerce_optional_int(row.get("teamkills")) or 0,
|
||||
"kd_ratio": round(kills / deaths, 2) if deaths else float(kills),
|
||||
"top_weapons": _top_counter(row.get("weapons_json")),
|
||||
"most_killed": _top_counter(row.get("most_killed_json")),
|
||||
"death_by": _top_counter(row.get("death_by_json")),
|
||||
**build_external_player_profile_fields(player_id=row.get("player_id")),
|
||||
}
|
||||
if profile_summary:
|
||||
player["profile_summary"] = profile_summary
|
||||
return player
|
||||
|
||||
|
||||
def _top_counter(raw_value: object, *, limit: int = 5) -> list[dict[str, object]]:
|
||||
if not isinstance(raw_value, str) or not raw_value.strip():
|
||||
return []
|
||||
try:
|
||||
payload = json.loads(raw_value)
|
||||
except (NameError, ValueError, TypeError):
|
||||
return []
|
||||
if not isinstance(payload, dict):
|
||||
return []
|
||||
rows = [
|
||||
{"name": str(name), "count": int(count)}
|
||||
for name, count in payload.items()
|
||||
if _coerce_optional_int(count) is not None
|
||||
]
|
||||
rows.sort(key=lambda item: (-int(item["count"]), str(item["name"])))
|
||||
return rows[:limit]
|
||||
|
||||
|
||||
def _build_materialized_timestamp_payload(item: dict[str, object]) -> dict[str, object]:
|
||||
started_at = item.get("started_at")
|
||||
ended_at = item.get("ended_at")
|
||||
duration_seconds = _calculate_match_duration_seconds(item)
|
||||
has_server_time_duration = bool(duration_seconds and duration_seconds > 0)
|
||||
if started_at and ended_at and started_at == ended_at and has_server_time_duration:
|
||||
return {
|
||||
"started_at": None,
|
||||
"ended_at": None,
|
||||
"closed_at": ended_at,
|
||||
"timestamp_confidence": "server-time-only",
|
||||
}
|
||||
return {
|
||||
"started_at": started_at,
|
||||
"ended_at": ended_at,
|
||||
"closed_at": ended_at or started_at,
|
||||
"timestamp_confidence": "absolute" if started_at or ended_at else "server-time-only",
|
||||
}
|
||||
|
||||
|
||||
def _build_materialized_scoreboard_correlation_window(
|
||||
item: dict[str, object],
|
||||
timestamps: dict[str, object],
|
||||
) -> dict[str, object]:
|
||||
started_at = timestamps.get("started_at")
|
||||
ended_at = timestamps.get("ended_at")
|
||||
if started_at and ended_at:
|
||||
return {"started_at": started_at, "ended_at": ended_at}
|
||||
|
||||
closed_at = timestamps.get("closed_at") or item.get("ended_at") or item.get("started_at")
|
||||
duration_seconds = _calculate_match_duration_seconds(item)
|
||||
closed_point = _parse_datetime(closed_at)
|
||||
if closed_point is None or not duration_seconds:
|
||||
return {"started_at": started_at, "ended_at": ended_at}
|
||||
|
||||
started_point = closed_point - timedelta(seconds=int(duration_seconds))
|
||||
return {
|
||||
"started_at": started_point.isoformat().replace("+00:00", "Z"),
|
||||
"ended_at": closed_point.isoformat().replace("+00:00", "Z"),
|
||||
}
|
||||
|
||||
|
||||
def build_materialized_scoreboard_correlation_input(
|
||||
item: dict[str, object],
|
||||
) -> dict[str, object]:
|
||||
"""Build safe candidate correlation inputs for one materialized RCON match."""
|
||||
timestamps = _build_materialized_timestamp_payload(item)
|
||||
correlation_window = _build_materialized_scoreboard_correlation_window(item, timestamps)
|
||||
return {
|
||||
"server_slug": item.get("external_server_id") or item.get("target_key"),
|
||||
"map_name": item.get("map_pretty_name") or item.get("map_name"),
|
||||
"started_at": correlation_window["started_at"],
|
||||
"ended_at": correlation_window["ended_at"],
|
||||
"duration_seconds": _calculate_match_duration_seconds(item),
|
||||
"allied_score": item.get("allied_score"),
|
||||
"axis_score": item.get("axis_score"),
|
||||
}
|
||||
|
||||
|
||||
def _merge_recent_items(
|
||||
primary_items: list[dict[str, object]],
|
||||
fallback_items: list[dict[str, object]],
|
||||
*,
|
||||
limit: int,
|
||||
) -> list[dict[str, object]]:
|
||||
merged: list[dict[str, object]] = []
|
||||
seen: set[tuple[object, object]] = set()
|
||||
for item in primary_items + fallback_items:
|
||||
map_payload = item.get("map") if isinstance(item.get("map"), dict) else {}
|
||||
key = (
|
||||
item.get("server", {}).get("slug") if isinstance(item.get("server"), dict) else None,
|
||||
normalize_map_name(map_payload.get("pretty_name") or map_payload.get("name")),
|
||||
)
|
||||
if key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
merged.append(item)
|
||||
merged.sort(key=lambda row: str(row.get("closed_at") or row.get("ended_at") or row.get("started_at") or ""), reverse=True)
|
||||
return merged[:limit]
|
||||
|
||||
|
||||
def _server_display_name(server_slug: object) -> str:
|
||||
slug = str(server_slug or "").strip()
|
||||
if slug == "comunidad-hispana-01":
|
||||
return "Comunidad Hispana #01"
|
||||
if slug == "comunidad-hispana-02":
|
||||
return "Comunidad Hispana #02"
|
||||
return slug or "RCON"
|
||||
|
||||
|
||||
def _build_rcon_result(latest_payload: object) -> dict[str, object]:
|
||||
payload = latest_payload if isinstance(latest_payload, dict) else {}
|
||||
allied_score = _coerce_optional_int(payload.get("allied_score"))
|
||||
axis_score = _coerce_optional_int(payload.get("axis_score"))
|
||||
winner = payload.get("winner")
|
||||
if not isinstance(winner, str) or not winner:
|
||||
winner = _resolve_result_winner(allied_score, axis_score)
|
||||
return {
|
||||
"allied_score": allied_score,
|
||||
"axis_score": axis_score,
|
||||
"winner": winner,
|
||||
}
|
||||
|
||||
|
||||
def _build_rcon_gamestate(latest_payload: object) -> dict[str, object]:
|
||||
payload = latest_payload if isinstance(latest_payload, dict) else {}
|
||||
return {
|
||||
"game_mode": payload.get("game_mode"),
|
||||
"allied_faction": payload.get("allied_faction"),
|
||||
"axis_faction": payload.get("axis_faction"),
|
||||
"allied_players": _coerce_optional_int(payload.get("allied_players")),
|
||||
"axis_players": _coerce_optional_int(payload.get("axis_players")),
|
||||
"remaining_match_time_seconds": _coerce_optional_int(
|
||||
payload.get("remaining_match_time_seconds")
|
||||
),
|
||||
"match_time_seconds": _coerce_optional_int(payload.get("match_time_seconds")),
|
||||
"queue_count": _coerce_optional_int(payload.get("queue_count")),
|
||||
"max_queue_count": _coerce_optional_int(payload.get("max_queue_count")),
|
||||
"vip_queue_count": _coerce_optional_int(payload.get("vip_queue_count")),
|
||||
"max_vip_queue_count": _coerce_optional_int(payload.get("max_vip_queue_count")),
|
||||
}
|
||||
|
||||
|
||||
def _resolve_result_winner(allied_score: int | None, axis_score: int | None) -> str | None:
|
||||
if allied_score is None or axis_score is None:
|
||||
return None
|
||||
if allied_score > axis_score:
|
||||
return "allied"
|
||||
if axis_score > allied_score:
|
||||
return "axis"
|
||||
return "draw"
|
||||
|
||||
|
||||
def _coerce_optional_int(value: object) -> int | None:
|
||||
if value is None:
|
||||
return None
|
||||
try:
|
||||
return int(value)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
def _build_server_summary(item: dict[str, object]) -> dict[str, object]:
|
||||
sample_count = int(item.get("sample_count") or 0)
|
||||
first_last_points = list_rcon_historical_recent_activity(
|
||||
server_key=str(item["target_key"]),
|
||||
limit=1,
|
||||
)
|
||||
last_sample_at = item.get("last_seen_at")
|
||||
latest_activity = first_last_points[0] if first_last_points else None
|
||||
|
||||
return {
|
||||
"server": {
|
||||
"slug": item["target_key"],
|
||||
"name": item["display_name"],
|
||||
"external_server_id": item["external_server_id"],
|
||||
"region": item["region"],
|
||||
},
|
||||
"coverage": {
|
||||
"basis": "rcon-competitive-windows",
|
||||
"status": "available" if int(item.get("window_count") or 0) > 0 else "empty",
|
||||
"window_count": int(item.get("window_count") or 0),
|
||||
"sample_count": sample_count,
|
||||
"first_sample_at": item.get("first_seen_at"),
|
||||
"last_sample_at": last_sample_at,
|
||||
"coverage_hours": _calculate_coverage_hours(item.get("first_seen_at"), last_sample_at),
|
||||
},
|
||||
"freshness": {
|
||||
"last_successful_capture_at": item.get("last_successful_capture_at"),
|
||||
"minutes_since_last_capture": _minutes_since_timestamp(last_sample_at),
|
||||
"last_run_status": item.get("last_run_status"),
|
||||
"last_error": item.get("last_error"),
|
||||
"last_error_at": item.get("last_error_at"),
|
||||
},
|
||||
"activity": {
|
||||
"latest_players": latest_activity.get("player_count") if latest_activity else None,
|
||||
"latest_peak_players": latest_activity.get("peak_players") if latest_activity else None,
|
||||
"latest_map": latest_activity.get("map", {}).get("pretty_name") if latest_activity else None,
|
||||
"latest_status": "captured" if latest_activity else None,
|
||||
},
|
||||
"time_range": {
|
||||
"start": item.get("first_seen_at"),
|
||||
"end": last_sample_at,
|
||||
},
|
||||
"capabilities": describe_rcon_historical_read_model()["capabilities"],
|
||||
}
|
||||
|
||||
|
||||
def _build_all_servers_summary(items: list[dict[str, object]]) -> dict[str, object]:
|
||||
total_samples = sum(int(item["coverage"].get("sample_count") or 0) for item in items)
|
||||
last_points = [
|
||||
item["time_range"].get("end")
|
||||
for item in items
|
||||
if item["time_range"].get("end")
|
||||
]
|
||||
last_capture_at = max(last_points) if last_points else None
|
||||
return {
|
||||
"server": {
|
||||
"slug": ALL_SERVERS_SLUG,
|
||||
"name": "Todos",
|
||||
"external_server_id": None,
|
||||
"region": None,
|
||||
},
|
||||
"coverage": {
|
||||
"basis": "rcon-competitive-windows-aggregate",
|
||||
"status": "available" if total_samples > 0 else "empty",
|
||||
"sample_count": total_samples,
|
||||
"first_sample_at": None,
|
||||
"last_sample_at": last_capture_at,
|
||||
"coverage_hours": None,
|
||||
},
|
||||
"freshness": {
|
||||
"last_successful_capture_at": last_capture_at,
|
||||
"minutes_since_last_capture": _minutes_since_timestamp(last_capture_at),
|
||||
"last_run_status": None,
|
||||
"last_error": None,
|
||||
"last_error_at": None,
|
||||
},
|
||||
"activity": {
|
||||
"latest_players": None,
|
||||
"latest_max_players": None,
|
||||
"latest_map": None,
|
||||
"latest_status": None,
|
||||
},
|
||||
"time_range": {
|
||||
"start": None,
|
||||
"end": last_capture_at,
|
||||
},
|
||||
"server_count": len(items),
|
||||
"capabilities": describe_rcon_historical_read_model()["capabilities"],
|
||||
}
|
||||
|
||||
|
||||
def _minutes_since_timestamp(timestamp: str | None) -> int | None:
|
||||
if not timestamp:
|
||||
return None
|
||||
captured_at = _parse_datetime(timestamp)
|
||||
if captured_at is None:
|
||||
return None
|
||||
delta = datetime.now(timezone.utc) - captured_at.astimezone(timezone.utc)
|
||||
return max(0, int(delta.total_seconds() // 60))
|
||||
|
||||
|
||||
def _parse_datetime(value: object) -> datetime | None:
|
||||
if isinstance(value, datetime):
|
||||
parsed = value
|
||||
elif isinstance(value, str) and value.strip():
|
||||
try:
|
||||
parsed = datetime.fromisoformat(value.strip().replace("Z", "+00:00"))
|
||||
except ValueError:
|
||||
return None
|
||||
else:
|
||||
return None
|
||||
if parsed.tzinfo is None:
|
||||
parsed = parsed.replace(tzinfo=timezone.utc)
|
||||
return parsed.astimezone(timezone.utc)
|
||||
|
||||
|
||||
def _calculate_coverage_hours(
|
||||
first_sample_at: object,
|
||||
last_sample_at: object,
|
||||
) -> float | None:
|
||||
first_point = _parse_datetime(first_sample_at)
|
||||
last_point = _parse_datetime(last_sample_at)
|
||||
if first_point is None or last_point is None:
|
||||
return None
|
||||
delta = last_point - first_point
|
||||
return round(delta.total_seconds() / 3600, 2)
|
||||
|
||||
|
||||
def _calculate_duration_seconds(first_seen_at: object, last_seen_at: object) -> int | None:
|
||||
first_point = _parse_datetime(first_seen_at)
|
||||
last_point = _parse_datetime(last_seen_at)
|
||||
if first_point is None or last_point is None:
|
||||
return None
|
||||
return max(0, int((last_point - first_point).total_seconds()))
|
||||
|
||||
|
||||
def _calculate_match_duration_seconds(item: dict[str, object]) -> int | None:
|
||||
duration = _calculate_duration_seconds(item.get("started_at"), item.get("ended_at"))
|
||||
if duration:
|
||||
return duration
|
||||
started_server_time = _coerce_optional_int(item.get("started_server_time"))
|
||||
ended_server_time = _coerce_optional_int(item.get("ended_server_time"))
|
||||
if started_server_time is None or ended_server_time is None:
|
||||
return duration
|
||||
return max(0, ended_server_time - started_server_time)
|
||||
1109
backend/app/rcon_historical_storage.py
Normal file
1109
backend/app/rcon_historical_storage.py
Normal file
File diff suppressed because it is too large
Load Diff
667
backend/app/rcon_historical_worker.py
Normal file
667
backend/app/rcon_historical_worker.py
Normal file
@@ -0,0 +1,667 @@
|
||||
"""Dedicated prospective RCON historical capture worker."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
from datetime import date, datetime
|
||||
import json
|
||||
import os
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from typing import Iterable
|
||||
|
||||
from .config import (
|
||||
get_rcon_capture_mode,
|
||||
get_rcon_current_match_capture_interval_seconds,
|
||||
get_rcon_current_match_writer_lock_timeout_seconds,
|
||||
get_rcon_historical_capture_interval_seconds,
|
||||
get_rcon_historical_capture_max_retries,
|
||||
get_rcon_historical_capture_retry_delay_seconds,
|
||||
get_rcon_request_timeout_seconds,
|
||||
get_rcon_skip_historical_materialization,
|
||||
)
|
||||
from .rcon_admin_log_ingestion import ingest_rcon_admin_logs
|
||||
from .rcon_admin_log_materialization import materialize_rcon_admin_log
|
||||
from .rcon_client import (
|
||||
RconQueryError,
|
||||
build_rcon_target_key,
|
||||
load_rcon_targets,
|
||||
query_live_server_sample,
|
||||
)
|
||||
from .rcon_historical_storage import (
|
||||
finalize_rcon_historical_capture_run,
|
||||
initialize_rcon_historical_storage,
|
||||
list_rcon_historical_target_statuses,
|
||||
mark_rcon_historical_capture_failure,
|
||||
persist_rcon_historical_sample,
|
||||
start_rcon_historical_capture_run,
|
||||
)
|
||||
from .snapshots import utc_now
|
||||
from .writer_lock import (
|
||||
backend_writer_lock,
|
||||
build_writer_lock_holder,
|
||||
)
|
||||
|
||||
CAPTURE_MODE_HISTORICAL = "historical"
|
||||
CAPTURE_MODE_CURRENT_LIVE = "current-live"
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class RconHistoricalCaptureStats:
|
||||
targets_seen: int = 0
|
||||
samples_inserted: int = 0
|
||||
duplicate_samples: int = 0
|
||||
failed_targets: int = 0
|
||||
admin_log_events_seen: int = 0
|
||||
admin_log_events_inserted: int = 0
|
||||
admin_log_duplicate_events: int = 0
|
||||
admin_log_failed_targets: int = 0
|
||||
materialized_matches_inserted: int = 0
|
||||
materialized_matches_updated: int = 0
|
||||
|
||||
|
||||
def run_rcon_historical_capture(
|
||||
*,
|
||||
target_key: str | None = None,
|
||||
capture_mode: str | None = None,
|
||||
skip_materialization: bool | None = None,
|
||||
writer_lock_timeout_seconds: float | None = None,
|
||||
) -> dict[str, object]:
|
||||
"""Capture one prospective RCON sample for one or all configured targets."""
|
||||
resolved_capture_mode, resolved_skip_materialization = _resolve_capture_options(
|
||||
capture_mode=capture_mode,
|
||||
skip_materialization=skip_materialization,
|
||||
)
|
||||
resolved_lock_timeout = writer_lock_timeout_seconds
|
||||
if resolved_lock_timeout is None and resolved_capture_mode == CAPTURE_MODE_CURRENT_LIVE:
|
||||
resolved_lock_timeout = get_rcon_current_match_writer_lock_timeout_seconds()
|
||||
with backend_writer_lock(
|
||||
holder=build_writer_lock_holder(
|
||||
f"app.rcon_historical_worker capture:{target_key or 'all-targets'}"
|
||||
),
|
||||
timeout_seconds=resolved_lock_timeout,
|
||||
):
|
||||
return run_rcon_historical_capture_unlocked(
|
||||
target_key=target_key,
|
||||
capture_mode=resolved_capture_mode,
|
||||
skip_materialization=resolved_skip_materialization,
|
||||
)
|
||||
|
||||
|
||||
def run_rcon_historical_capture_unlocked(
|
||||
*,
|
||||
target_key: str | None = None,
|
||||
capture_mode: str | None = None,
|
||||
skip_materialization: bool | None = None,
|
||||
) -> dict[str, object]:
|
||||
"""Capture one prospective RCON sample assuming the shared writer lock is already held."""
|
||||
resolved_capture_mode, resolved_skip_materialization = _resolve_capture_options(
|
||||
capture_mode=capture_mode,
|
||||
skip_materialization=skip_materialization,
|
||||
)
|
||||
initialize_rcon_historical_storage()
|
||||
selected_targets = _select_targets(target_key)
|
||||
selected_target_keys = {build_rcon_target_key(target) for target in selected_targets}
|
||||
admin_log_lookback_minutes = get_rcon_admin_log_lookback_minutes()
|
||||
captured_at = utc_now().isoformat().replace("+00:00", "Z")
|
||||
target_scope = target_key or "all-configured-rcon-targets"
|
||||
run_id = start_rcon_historical_capture_run(
|
||||
mode=resolved_capture_mode,
|
||||
target_scope=target_scope,
|
||||
)
|
||||
stats = RconHistoricalCaptureStats()
|
||||
items: list[dict[str, object]] = []
|
||||
errors: list[dict[str, object]] = []
|
||||
admin_log_errors: list[dict[str, object]] = []
|
||||
timeout_seconds = get_rcon_request_timeout_seconds()
|
||||
|
||||
try:
|
||||
for target in selected_targets:
|
||||
target_metadata = _serialize_target(target)
|
||||
stats.targets_seen += 1
|
||||
try:
|
||||
sample = query_live_server_sample(
|
||||
target,
|
||||
timeout_seconds=timeout_seconds,
|
||||
)
|
||||
delta = persist_rcon_historical_sample(
|
||||
run_id=run_id,
|
||||
captured_at=captured_at,
|
||||
target=target_metadata,
|
||||
normalized_payload=sample["normalized"],
|
||||
raw_payload=sample["raw_session"],
|
||||
)
|
||||
stats.samples_inserted += int(delta["samples_inserted"])
|
||||
stats.duplicate_samples += int(delta["duplicate_samples"])
|
||||
items.append(
|
||||
{
|
||||
"target_key": target_metadata["target_key"],
|
||||
"external_server_id": target.external_server_id,
|
||||
"name": target.name,
|
||||
"host": target.host,
|
||||
"port": target.port,
|
||||
"timeout_seconds": timeout_seconds,
|
||||
"captured_at": captured_at,
|
||||
"sample_inserted": bool(delta["samples_inserted"]),
|
||||
"normalized": sample["normalized"],
|
||||
}
|
||||
)
|
||||
except Exception as exc: # noqa: BLE001 - controlled worker failures
|
||||
stats.failed_targets += 1
|
||||
mark_rcon_historical_capture_failure(
|
||||
run_id=run_id,
|
||||
target=target_metadata,
|
||||
error_message=_format_error_message(exc),
|
||||
)
|
||||
errors.append(_serialize_capture_error(target, exc, timeout_seconds=timeout_seconds))
|
||||
|
||||
admin_log_result = _ingest_target_admin_log(
|
||||
target_key=str(target_metadata["target_key"]),
|
||||
minutes=admin_log_lookback_minutes,
|
||||
)
|
||||
_merge_admin_log_result(
|
||||
stats=stats,
|
||||
admin_log_errors=admin_log_errors,
|
||||
target=target_metadata,
|
||||
result=admin_log_result,
|
||||
)
|
||||
|
||||
materialization_result = _run_materialization_if_enabled(
|
||||
skip_materialization=resolved_skip_materialization
|
||||
)
|
||||
if not resolved_skip_materialization:
|
||||
stats.materialized_matches_inserted = int(
|
||||
materialization_result.get("matches_materialized") or 0
|
||||
)
|
||||
stats.materialized_matches_updated = int(
|
||||
materialization_result.get("matches_updated") or 0
|
||||
)
|
||||
|
||||
status = "success" if not errors else ("partial" if items else "failed")
|
||||
finalize_rcon_historical_capture_run(
|
||||
run_id,
|
||||
status=status,
|
||||
targets_seen=stats.targets_seen,
|
||||
samples_inserted=stats.samples_inserted,
|
||||
duplicate_samples=stats.duplicate_samples,
|
||||
failed_targets=stats.failed_targets,
|
||||
notes=None if not errors else json.dumps(errors, separators=(",", ":")),
|
||||
)
|
||||
except Exception as exc:
|
||||
finalize_rcon_historical_capture_run(
|
||||
run_id,
|
||||
status="failed",
|
||||
targets_seen=stats.targets_seen,
|
||||
samples_inserted=stats.samples_inserted,
|
||||
duplicate_samples=stats.duplicate_samples,
|
||||
failed_targets=max(1, stats.failed_targets),
|
||||
notes=str(exc),
|
||||
)
|
||||
raise
|
||||
|
||||
return {
|
||||
"status": "ok" if items else "error",
|
||||
"run_status": status,
|
||||
"captured_at": captured_at,
|
||||
"target_scope": target_scope,
|
||||
"capture_mode": resolved_capture_mode,
|
||||
"materialization_skipped": resolved_skip_materialization,
|
||||
"admin_log_lookback_minutes": admin_log_lookback_minutes,
|
||||
"admin_log_events_seen": stats.admin_log_events_seen,
|
||||
"admin_log_events_inserted": stats.admin_log_events_inserted,
|
||||
"duplicate_events": stats.admin_log_duplicate_events,
|
||||
"samples_inserted": stats.samples_inserted,
|
||||
"targets": items,
|
||||
"errors": errors,
|
||||
"admin_log_errors": admin_log_errors,
|
||||
"materialization_result": materialization_result,
|
||||
"storage_status": [
|
||||
status
|
||||
for status in list_rcon_historical_target_statuses()
|
||||
if status.get("target_key") in selected_target_keys
|
||||
],
|
||||
"totals": {
|
||||
"targets_seen": stats.targets_seen,
|
||||
"samples_inserted": stats.samples_inserted,
|
||||
"duplicate_samples": stats.duplicate_samples,
|
||||
"failed_targets": stats.failed_targets,
|
||||
"admin_log_events_seen": stats.admin_log_events_seen,
|
||||
"admin_log_events_inserted": stats.admin_log_events_inserted,
|
||||
"admin_log_duplicate_events": stats.admin_log_duplicate_events,
|
||||
"admin_log_failed_targets": stats.admin_log_failed_targets,
|
||||
"materialized_matches_inserted": stats.materialized_matches_inserted,
|
||||
"materialized_matches_updated": stats.materialized_matches_updated,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def run_periodic_rcon_historical_capture(
|
||||
*,
|
||||
interval_seconds: int,
|
||||
max_retries: int,
|
||||
retry_delay_seconds: int,
|
||||
target_key: str | None = None,
|
||||
capture_mode: str | None = None,
|
||||
skip_materialization: bool | None = None,
|
||||
max_runs: int | None = None,
|
||||
) -> None:
|
||||
"""Run prospective RCON capture in a local loop."""
|
||||
resolved_capture_mode, resolved_skip_materialization = _resolve_capture_options(
|
||||
capture_mode=capture_mode,
|
||||
skip_materialization=skip_materialization,
|
||||
)
|
||||
completed_runs = 0
|
||||
startup_targets = _describe_loop_targets(target_key)
|
||||
_emit_worker_event(
|
||||
"rcon-historical-capture-worker-started",
|
||||
interval_seconds=interval_seconds,
|
||||
max_retries=max_retries,
|
||||
retry_delay_seconds=retry_delay_seconds,
|
||||
capture_mode=resolved_capture_mode,
|
||||
materialization_skipped=resolved_skip_materialization,
|
||||
target_scope=target_key or "all-configured-rcon-targets",
|
||||
target_count=len(startup_targets),
|
||||
targets=startup_targets,
|
||||
)
|
||||
print("Press Ctrl+C to stop.")
|
||||
|
||||
try:
|
||||
while max_runs is None or completed_runs < max_runs:
|
||||
completed_runs += 1
|
||||
_emit_worker_event(
|
||||
"rcon-historical-capture-cycle-started",
|
||||
run=completed_runs,
|
||||
)
|
||||
payload = _run_capture_with_retries(
|
||||
max_retries=max_retries,
|
||||
retry_delay_seconds=retry_delay_seconds,
|
||||
target_key=target_key,
|
||||
capture_mode=resolved_capture_mode,
|
||||
skip_materialization=resolved_skip_materialization,
|
||||
)
|
||||
_emit_worker_event(
|
||||
"rcon-historical-capture-cycle-finished",
|
||||
run=completed_runs,
|
||||
result=payload,
|
||||
)
|
||||
if max_runs is not None and completed_runs >= max_runs:
|
||||
break
|
||||
_emit_worker_event(
|
||||
"rcon-historical-capture-sleep-started",
|
||||
run=completed_runs,
|
||||
sleep_seconds=interval_seconds,
|
||||
)
|
||||
time.sleep(interval_seconds)
|
||||
except KeyboardInterrupt:
|
||||
print("\nRCON historical capture loop stopped by user.")
|
||||
except Exception as exc:
|
||||
_emit_worker_event(
|
||||
"rcon-historical-capture-worker-exited-unexpectedly",
|
||||
error_type=type(exc).__name__,
|
||||
message=str(exc),
|
||||
)
|
||||
raise
|
||||
|
||||
|
||||
def _run_capture_with_retries(
|
||||
*,
|
||||
max_retries: int,
|
||||
retry_delay_seconds: int,
|
||||
target_key: str | None,
|
||||
capture_mode: str,
|
||||
skip_materialization: bool,
|
||||
) -> dict[str, object]:
|
||||
attempt = 0
|
||||
while True:
|
||||
attempt += 1
|
||||
try:
|
||||
return {
|
||||
"status": "ok",
|
||||
"attempts_used": attempt,
|
||||
"capture_result": run_rcon_historical_capture(
|
||||
target_key=target_key,
|
||||
capture_mode=capture_mode,
|
||||
skip_materialization=skip_materialization,
|
||||
),
|
||||
}
|
||||
except Exception as exc:
|
||||
if attempt > max_retries:
|
||||
_emit_worker_event(
|
||||
"rcon-historical-capture-attempt-failed",
|
||||
attempt=attempt,
|
||||
max_retries=max_retries,
|
||||
error_type=type(exc).__name__,
|
||||
message=str(exc),
|
||||
retries_exhausted=True,
|
||||
)
|
||||
return {
|
||||
"status": "error",
|
||||
"attempts_used": attempt,
|
||||
"error": str(exc),
|
||||
}
|
||||
_emit_worker_event(
|
||||
"rcon-historical-capture-attempt-failed",
|
||||
attempt=attempt,
|
||||
max_retries=max_retries,
|
||||
error_type=type(exc).__name__,
|
||||
message=str(exc),
|
||||
)
|
||||
if retry_delay_seconds > 0:
|
||||
_emit_worker_event(
|
||||
"rcon-historical-capture-retry-sleep-started",
|
||||
attempt=attempt,
|
||||
sleep_seconds=retry_delay_seconds,
|
||||
)
|
||||
time.sleep(retry_delay_seconds)
|
||||
|
||||
|
||||
def _select_targets(target_key: str | None) -> list[object]:
|
||||
configured_targets = list(load_rcon_targets())
|
||||
if not configured_targets:
|
||||
raise RuntimeError("No RCON targets configured in HLL_BACKEND_RCON_TARGETS.")
|
||||
if target_key is None:
|
||||
return configured_targets
|
||||
|
||||
normalized = target_key.strip()
|
||||
selected = [
|
||||
target
|
||||
for target in configured_targets
|
||||
if build_rcon_target_key(target) == normalized
|
||||
]
|
||||
if not selected:
|
||||
raise ValueError(f"Unknown RCON target key: {target_key}")
|
||||
return selected
|
||||
|
||||
|
||||
def _describe_loop_targets(target_key: str | None) -> list[dict[str, str]]:
|
||||
"""Describe configured worker targets without exposing credentials."""
|
||||
try:
|
||||
targets = _select_targets(target_key)
|
||||
except Exception as exc: # noqa: BLE001 - startup logging must not hide capture error
|
||||
return [
|
||||
{
|
||||
"status": "unavailable",
|
||||
"error_type": type(exc).__name__,
|
||||
"message": str(exc),
|
||||
}
|
||||
]
|
||||
return [
|
||||
{
|
||||
"target_key": build_rcon_target_key(target),
|
||||
"external_server_id": str(target.external_server_id or ""),
|
||||
"name": str(target.name or ""),
|
||||
}
|
||||
for target in targets
|
||||
]
|
||||
|
||||
|
||||
def _emit_worker_event(event: str, **fields: object) -> None:
|
||||
"""Print one JSON worker event using safe date/time serialization."""
|
||||
print(
|
||||
json.dumps({"event": event, **fields}, indent=2, default=_json_default),
|
||||
flush=True,
|
||||
)
|
||||
|
||||
|
||||
def _json_default(value: object) -> str:
|
||||
if isinstance(value, (date, datetime)):
|
||||
return value.isoformat()
|
||||
return str(value)
|
||||
|
||||
|
||||
def get_rcon_admin_log_lookback_minutes() -> int:
|
||||
"""Return the AdminLog lookback window used by periodic RCON capture."""
|
||||
configured_value = os.getenv("HLL_BACKEND_RCON_ADMIN_LOG_LOOKBACK_MINUTES", "60")
|
||||
lookback_minutes = int(configured_value)
|
||||
if lookback_minutes <= 0:
|
||||
raise ValueError("HLL_BACKEND_RCON_ADMIN_LOG_LOOKBACK_MINUTES must be positive.")
|
||||
return lookback_minutes
|
||||
|
||||
|
||||
def _ingest_target_admin_log(
|
||||
*,
|
||||
target_key: str,
|
||||
minutes: int,
|
||||
) -> dict[str, object]:
|
||||
try:
|
||||
return ingest_rcon_admin_logs(minutes=minutes, target_key=target_key)
|
||||
except Exception as exc: # noqa: BLE001 - worker reports per-target AdminLog failures
|
||||
return {
|
||||
"status": "error",
|
||||
"errors": [
|
||||
{
|
||||
"target_key": target_key,
|
||||
"status": "error",
|
||||
"error_type": type(exc).__name__,
|
||||
"message": str(exc),
|
||||
}
|
||||
],
|
||||
"totals": {
|
||||
"events_seen": 0,
|
||||
"events_inserted": 0,
|
||||
"duplicate_events": 0,
|
||||
"failed_targets": 1,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _merge_admin_log_result(
|
||||
*,
|
||||
stats: RconHistoricalCaptureStats,
|
||||
admin_log_errors: list[dict[str, object]],
|
||||
target: dict[str, object],
|
||||
result: dict[str, object],
|
||||
) -> None:
|
||||
totals = result.get("totals")
|
||||
if isinstance(totals, dict):
|
||||
stats.admin_log_events_seen += int(totals.get("events_seen") or 0)
|
||||
stats.admin_log_events_inserted += int(totals.get("events_inserted") or 0)
|
||||
stats.admin_log_duplicate_events += int(totals.get("duplicate_events") or 0)
|
||||
stats.admin_log_failed_targets += int(totals.get("failed_targets") or 0)
|
||||
|
||||
errors = result.get("errors")
|
||||
if isinstance(errors, list):
|
||||
for error in errors:
|
||||
if isinstance(error, dict):
|
||||
admin_log_errors.append(
|
||||
{
|
||||
"target_key": target["target_key"],
|
||||
"external_server_id": target.get("external_server_id"),
|
||||
"name": target.get("name"),
|
||||
"status": "error",
|
||||
"error_type": error.get("error_type"),
|
||||
"message": error.get("message"),
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _serialize_target(target: object) -> dict[str, object]:
|
||||
return {
|
||||
"target_key": build_rcon_target_key(target),
|
||||
"external_server_id": target.external_server_id,
|
||||
"name": target.name,
|
||||
"host": target.host,
|
||||
"port": target.port,
|
||||
"region": target.region,
|
||||
"game_port": target.game_port,
|
||||
"query_port": target.query_port,
|
||||
"source_name": target.source_name,
|
||||
}
|
||||
|
||||
|
||||
def _serialize_capture_error(
|
||||
target: object,
|
||||
error: Exception,
|
||||
*,
|
||||
timeout_seconds: float,
|
||||
) -> dict[str, object]:
|
||||
error_type = _classify_capture_error_type(error)
|
||||
error_stage = _classify_capture_error_stage(error)
|
||||
return {
|
||||
"target_key": build_rcon_target_key(target),
|
||||
"external_server_id": target.external_server_id,
|
||||
"name": target.name,
|
||||
"host": target.host,
|
||||
"port": target.port,
|
||||
"timeout_seconds": timeout_seconds,
|
||||
"error_type": error_type,
|
||||
"error_stage": error_stage,
|
||||
"message": str(error),
|
||||
}
|
||||
|
||||
|
||||
def _classify_capture_error_type(error: Exception) -> str:
|
||||
if isinstance(error, RconQueryError):
|
||||
return error.error_type
|
||||
message = str(error).lower()
|
||||
if "timed out" in message or "timeout" in message:
|
||||
return "timeout"
|
||||
if "401" in message or "403" in message or "login" in message or "auth" in message:
|
||||
return "auth/login"
|
||||
if "refused" in message:
|
||||
return "connection-refused"
|
||||
if "payload" in message or "json" in message or "malformed" in message:
|
||||
return "payload-invalid"
|
||||
return "other-error"
|
||||
|
||||
|
||||
def _classify_capture_error_stage(error: Exception) -> str | None:
|
||||
if isinstance(error, RconQueryError):
|
||||
return error.error_stage
|
||||
return None
|
||||
|
||||
|
||||
def _format_error_message(error: Exception) -> str:
|
||||
error_type = _classify_capture_error_type(error)
|
||||
error_stage = _classify_capture_error_stage(error)
|
||||
if error_stage:
|
||||
return f"[{error_type}:{error_stage}] {error}"
|
||||
return f"[{error_type}] {error}"
|
||||
|
||||
|
||||
def _resolve_capture_options(
|
||||
*,
|
||||
capture_mode: str | None,
|
||||
skip_materialization: bool | None,
|
||||
) -> tuple[str, bool]:
|
||||
resolved_capture_mode = capture_mode or get_rcon_capture_mode()
|
||||
if resolved_capture_mode not in {CAPTURE_MODE_HISTORICAL, CAPTURE_MODE_CURRENT_LIVE}:
|
||||
raise ValueError("capture_mode must be 'historical' or 'current-live'.")
|
||||
|
||||
if skip_materialization is None:
|
||||
resolved_skip_materialization = get_rcon_skip_historical_materialization()
|
||||
else:
|
||||
resolved_skip_materialization = skip_materialization
|
||||
|
||||
if resolved_capture_mode == CAPTURE_MODE_CURRENT_LIVE:
|
||||
resolved_skip_materialization = True
|
||||
return resolved_capture_mode, resolved_skip_materialization
|
||||
|
||||
|
||||
def _run_materialization_if_enabled(*, skip_materialization: bool) -> dict[str, object]:
|
||||
if skip_materialization:
|
||||
return {
|
||||
"status": "skipped",
|
||||
"reason": "skip-materialization-enabled",
|
||||
}
|
||||
return materialize_rcon_admin_log()
|
||||
|
||||
|
||||
def build_arg_parser() -> argparse.ArgumentParser:
|
||||
"""Create the CLI parser for manual or periodic prospective RCON capture."""
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Prospective RCON historical capture for HLL Vietnam.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"mode",
|
||||
choices=("capture", "loop"),
|
||||
help="capture runs once; loop keeps collecting periodically",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--target",
|
||||
dest="target_key",
|
||||
help="optional target key; defaults to all configured RCON targets",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--interval",
|
||||
type=int,
|
||||
default=get_rcon_historical_capture_interval_seconds(),
|
||||
help="seconds to wait between loop runs",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--retries",
|
||||
type=int,
|
||||
default=get_rcon_historical_capture_max_retries(),
|
||||
help="retry attempts after a failed capture",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--retry-delay",
|
||||
type=int,
|
||||
default=get_rcon_historical_capture_retry_delay_seconds(),
|
||||
help="seconds to wait between failed attempts",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--max-runs",
|
||||
type=int,
|
||||
help="optional safety cap for loop mode",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--capture-mode",
|
||||
choices=(CAPTURE_MODE_HISTORICAL, CAPTURE_MODE_CURRENT_LIVE),
|
||||
default=get_rcon_capture_mode(),
|
||||
help="historical keeps materialization; current-live only captures lightweight live data",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--skip-materialization",
|
||||
action="store_true",
|
||||
default=None,
|
||||
help="capture AdminLog and live snapshots without running heavy historical materialization",
|
||||
)
|
||||
return parser
|
||||
|
||||
|
||||
def main(argv: Iterable[str] | None = None) -> int:
|
||||
"""Run the prospective RCON historical capture CLI."""
|
||||
parser = build_arg_parser()
|
||||
args = parser.parse_args(list(argv) if argv is not None else None)
|
||||
|
||||
if args.mode == "capture":
|
||||
result = run_rcon_historical_capture(
|
||||
target_key=args.target_key,
|
||||
capture_mode=args.capture_mode,
|
||||
skip_materialization=args.skip_materialization,
|
||||
)
|
||||
print(json.dumps(result, indent=2, default=_json_default))
|
||||
return 0
|
||||
|
||||
default_interval = (
|
||||
get_rcon_current_match_capture_interval_seconds()
|
||||
if args.capture_mode == CAPTURE_MODE_CURRENT_LIVE
|
||||
and "--interval" not in (argv or [])
|
||||
else args.interval
|
||||
)
|
||||
args.interval = default_interval
|
||||
|
||||
if args.interval <= 0:
|
||||
raise ValueError("--interval must be a positive integer.")
|
||||
if args.retries < 0:
|
||||
raise ValueError("--retries must be zero or positive.")
|
||||
if args.retry_delay < 0:
|
||||
raise ValueError("--retry-delay must be zero or positive.")
|
||||
if args.max_runs is not None and args.max_runs <= 0:
|
||||
raise ValueError("--max-runs must be positive when provided.")
|
||||
|
||||
run_periodic_rcon_historical_capture(
|
||||
interval_seconds=args.interval,
|
||||
max_retries=args.retries,
|
||||
retry_delay_seconds=args.retry_delay,
|
||||
target_key=args.target_key,
|
||||
capture_mode=args.capture_mode,
|
||||
skip_materialization=args.skip_materialization,
|
||||
max_runs=args.max_runs,
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
448
backend/app/rcon_scoreboard_correlation.py
Normal file
448
backend/app/rcon_scoreboard_correlation.py
Normal file
@@ -0,0 +1,448 @@
|
||||
"""Correlate RCON competitive windows with trusted persisted scoreboard matches."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlite3
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
from .config import get_storage_path, use_postgres_rcon_storage
|
||||
from .normalizers import normalize_map_name
|
||||
from .scoreboard_origins import resolve_trusted_scoreboard_match_url
|
||||
from .sqlite_utils import connect_sqlite_readonly
|
||||
|
||||
|
||||
MIN_CONFIDENCE_SCORE = 5
|
||||
MAX_CANDIDATES = 200
|
||||
|
||||
|
||||
def resolve_rcon_scoreboard_match_url(
|
||||
*,
|
||||
server_slug: object,
|
||||
map_name: object,
|
||||
started_at: object,
|
||||
ended_at: object,
|
||||
duration_seconds: object = None,
|
||||
player_count: object = None,
|
||||
peak_players: object = None,
|
||||
allied_score: object = None,
|
||||
axis_score: object = None,
|
||||
db_path: Path | None = None,
|
||||
) -> str | None:
|
||||
"""Return a trusted scoreboard URL for an RCON window only on strong evidence."""
|
||||
resolution = resolve_rcon_scoreboard_correlation(
|
||||
server_slug=server_slug,
|
||||
map_name=map_name,
|
||||
started_at=started_at,
|
||||
ended_at=ended_at,
|
||||
duration_seconds=duration_seconds,
|
||||
player_count=player_count,
|
||||
peak_players=peak_players,
|
||||
allied_score=allied_score,
|
||||
axis_score=axis_score,
|
||||
db_path=db_path,
|
||||
)
|
||||
match_url = resolution.get("match_url")
|
||||
return str(match_url) if match_url else None
|
||||
|
||||
|
||||
def resolve_rcon_scoreboard_correlation(
|
||||
*,
|
||||
server_slug: object,
|
||||
map_name: object,
|
||||
started_at: object,
|
||||
ended_at: object,
|
||||
duration_seconds: object = None,
|
||||
player_count: object = None,
|
||||
peak_players: object = None,
|
||||
allied_score: object = None,
|
||||
axis_score: object = None,
|
||||
db_path: Path | None = None,
|
||||
) -> dict[str, object]:
|
||||
"""Return a safe candidate selection summary for one RCON match window."""
|
||||
normalized_server_slug = str(server_slug or "").strip()
|
||||
normalized_map = normalize_map_name(map_name)
|
||||
rcon_start = _parse_timestamp(started_at)
|
||||
rcon_end = _parse_timestamp(ended_at)
|
||||
if not normalized_server_slug or not normalized_map or not rcon_start or not rcon_end:
|
||||
return {"match_url": None, "candidate_count": 0, "reason": "invalid-rcon-window"}
|
||||
if rcon_end < rcon_start:
|
||||
rcon_start, rcon_end = rcon_end, rcon_start
|
||||
|
||||
candidates = _list_persisted_scoreboard_candidates(
|
||||
server_slug=normalized_server_slug,
|
||||
db_path=db_path or get_storage_path(),
|
||||
)
|
||||
scored_candidates = [
|
||||
scored
|
||||
for candidate in candidates
|
||||
if (scored := _score_candidate(
|
||||
candidate,
|
||||
normalized_map=normalized_map,
|
||||
rcon_start=rcon_start,
|
||||
rcon_end=rcon_end,
|
||||
duration_seconds=_coerce_int(duration_seconds),
|
||||
player_count=_coerce_int(player_count),
|
||||
peak_players=_coerce_int(peak_players),
|
||||
allied_score=_coerce_int(allied_score),
|
||||
axis_score=_coerce_int(axis_score),
|
||||
))
|
||||
is not None
|
||||
]
|
||||
if not scored_candidates:
|
||||
return {
|
||||
"match_url": None,
|
||||
"candidate_count": len(candidates),
|
||||
"reason": "no-safe-candidate",
|
||||
}
|
||||
|
||||
scored_candidates.sort(key=lambda item: item["score"], reverse=True)
|
||||
best = scored_candidates[0]
|
||||
if int(best["score"]) < MIN_CONFIDENCE_SCORE:
|
||||
return {
|
||||
"match_url": None,
|
||||
"candidate_count": len(candidates),
|
||||
"reason": "low-confidence",
|
||||
}
|
||||
if len(scored_candidates) > 1 and int(scored_candidates[1]["score"]) >= int(best["score"]):
|
||||
return {
|
||||
"match_url": None,
|
||||
"candidate_count": len(candidates),
|
||||
"reason": "ambiguous-candidate",
|
||||
}
|
||||
return {
|
||||
"match_url": str(best["match_url"]),
|
||||
"candidate_count": len(candidates),
|
||||
"reason": "linked",
|
||||
"selected_candidate": {
|
||||
"external_match_id": best.get("external_match_id"),
|
||||
"correlation_score": int(best["score"]),
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def diagnose_rcon_scoreboard_correlation(
|
||||
*,
|
||||
server_slug: object,
|
||||
map_name: object,
|
||||
started_at: object,
|
||||
ended_at: object,
|
||||
duration_seconds: object = None,
|
||||
player_count: object = None,
|
||||
peak_players: object = None,
|
||||
allied_score: object = None,
|
||||
axis_score: object = None,
|
||||
db_path: Path | None = None,
|
||||
) -> dict[str, object]:
|
||||
"""Describe safe candidate scoring for a single RCON correlation window."""
|
||||
normalized_server_slug = str(server_slug or "").strip()
|
||||
normalized_map = normalize_map_name(map_name)
|
||||
rcon_start = _parse_timestamp(started_at)
|
||||
rcon_end = _parse_timestamp(ended_at)
|
||||
if not normalized_server_slug or not normalized_map or not rcon_start or not rcon_end:
|
||||
return {
|
||||
"candidate_search_window": {
|
||||
"started_at": started_at,
|
||||
"ended_at": ended_at,
|
||||
"candidate_limit": MAX_CANDIDATES,
|
||||
},
|
||||
"candidate_count": 0,
|
||||
"top_candidates": [],
|
||||
"selected_candidate": None,
|
||||
"final_reason": "invalid-rcon-window",
|
||||
}
|
||||
if rcon_end < rcon_start:
|
||||
rcon_start, rcon_end = rcon_end, rcon_start
|
||||
|
||||
candidates = _list_persisted_scoreboard_candidates(
|
||||
server_slug=normalized_server_slug,
|
||||
db_path=db_path or get_storage_path(),
|
||||
)
|
||||
resolution = resolve_rcon_scoreboard_correlation(
|
||||
server_slug=server_slug,
|
||||
map_name=map_name,
|
||||
started_at=started_at,
|
||||
ended_at=ended_at,
|
||||
duration_seconds=duration_seconds,
|
||||
player_count=player_count,
|
||||
peak_players=peak_players,
|
||||
allied_score=allied_score,
|
||||
axis_score=axis_score,
|
||||
db_path=db_path,
|
||||
)
|
||||
summaries = [
|
||||
_diagnostic_candidate_summary(
|
||||
candidate,
|
||||
server_slug=normalized_server_slug,
|
||||
normalized_map=normalized_map,
|
||||
rcon_start=rcon_start,
|
||||
rcon_end=rcon_end,
|
||||
duration_seconds=_coerce_int(duration_seconds),
|
||||
player_count=_coerce_int(player_count),
|
||||
peak_players=_coerce_int(peak_players),
|
||||
allied_score=_coerce_int(allied_score),
|
||||
axis_score=_coerce_int(axis_score),
|
||||
)
|
||||
for candidate in candidates
|
||||
]
|
||||
summaries.sort(
|
||||
key=lambda item: (
|
||||
-int(item["correlation_score"] or -1),
|
||||
str(item.get("external_match_id") or ""),
|
||||
)
|
||||
)
|
||||
selected_id = (
|
||||
resolution.get("selected_candidate", {}).get("external_match_id")
|
||||
if isinstance(resolution.get("selected_candidate"), dict)
|
||||
else None
|
||||
)
|
||||
selected_candidate = next(
|
||||
(item for item in summaries if item.get("external_match_id") == selected_id),
|
||||
None,
|
||||
)
|
||||
return {
|
||||
"candidate_search_window": {
|
||||
"started_at": rcon_start.isoformat().replace("+00:00", "Z"),
|
||||
"ended_at": rcon_end.isoformat().replace("+00:00", "Z"),
|
||||
"candidate_limit": MAX_CANDIDATES,
|
||||
},
|
||||
"candidate_count": len(candidates),
|
||||
"top_candidates": summaries[:5],
|
||||
"selected_candidate": selected_candidate,
|
||||
"final_reason": resolution["reason"],
|
||||
}
|
||||
|
||||
|
||||
def _list_persisted_scoreboard_candidates(
|
||||
*,
|
||||
server_slug: str,
|
||||
db_path: Path,
|
||||
) -> list[dict[str, object]]:
|
||||
if use_postgres_rcon_storage():
|
||||
from .postgres_rcon_storage import list_scoreboard_candidates
|
||||
|
||||
postgres_candidates = list_scoreboard_candidates(
|
||||
server_slug=server_slug,
|
||||
limit=MAX_CANDIDATES,
|
||||
)
|
||||
if postgres_candidates:
|
||||
return postgres_candidates
|
||||
|
||||
try:
|
||||
with connect_sqlite_readonly(db_path) as connection:
|
||||
rows = connection.execute(
|
||||
"""
|
||||
SELECT
|
||||
historical_matches.external_match_id,
|
||||
historical_matches.started_at,
|
||||
historical_matches.ended_at,
|
||||
historical_matches.map_name,
|
||||
historical_matches.map_pretty_name,
|
||||
historical_matches.allied_score,
|
||||
historical_matches.axis_score,
|
||||
historical_matches.raw_payload_ref,
|
||||
historical_servers.slug AS server_slug,
|
||||
COUNT(historical_player_match_stats.id) AS player_count
|
||||
FROM historical_matches
|
||||
INNER JOIN historical_servers
|
||||
ON historical_servers.id = historical_matches.historical_server_id
|
||||
LEFT JOIN historical_player_match_stats
|
||||
ON historical_player_match_stats.historical_match_id = historical_matches.id
|
||||
WHERE historical_servers.slug = ?
|
||||
AND historical_matches.raw_payload_ref IS NOT NULL
|
||||
GROUP BY historical_matches.id
|
||||
ORDER BY COALESCE(historical_matches.ended_at, historical_matches.started_at) DESC
|
||||
LIMIT ?
|
||||
""",
|
||||
(server_slug, MAX_CANDIDATES),
|
||||
).fetchall()
|
||||
except sqlite3.Error:
|
||||
return []
|
||||
|
||||
items: list[dict[str, object]] = []
|
||||
for row in rows:
|
||||
match_url = resolve_trusted_scoreboard_match_url(
|
||||
row["raw_payload_ref"],
|
||||
row["server_slug"],
|
||||
)
|
||||
if not match_url:
|
||||
continue
|
||||
items.append(
|
||||
{
|
||||
"external_match_id": row["external_match_id"],
|
||||
"started_at": row["started_at"],
|
||||
"ended_at": row["ended_at"],
|
||||
"map_name": row["map_name"],
|
||||
"map_pretty_name": row["map_pretty_name"],
|
||||
"allied_score": row["allied_score"],
|
||||
"axis_score": row["axis_score"],
|
||||
"player_count": row["player_count"],
|
||||
"match_url": match_url,
|
||||
}
|
||||
)
|
||||
if items and use_postgres_rcon_storage():
|
||||
from .postgres_rcon_storage import upsert_scoreboard_candidates
|
||||
|
||||
upsert_scoreboard_candidates(server_slug=server_slug, candidates=items)
|
||||
return items
|
||||
|
||||
|
||||
def _score_candidate(
|
||||
candidate: dict[str, object],
|
||||
*,
|
||||
normalized_map: str,
|
||||
rcon_start: datetime,
|
||||
rcon_end: datetime,
|
||||
duration_seconds: int | None,
|
||||
player_count: int | None,
|
||||
peak_players: int | None,
|
||||
allied_score: int | None,
|
||||
axis_score: int | None,
|
||||
) -> dict[str, object] | None:
|
||||
candidate_map = normalize_map_name(
|
||||
candidate.get("map_pretty_name") or candidate.get("map_name")
|
||||
)
|
||||
if candidate_map != normalized_map:
|
||||
return None
|
||||
|
||||
candidate_start = _parse_timestamp(candidate.get("started_at"))
|
||||
candidate_end = _parse_timestamp(candidate.get("ended_at"))
|
||||
if not candidate_start or not candidate_end:
|
||||
return None
|
||||
if candidate_end < candidate_start:
|
||||
candidate_start, candidate_end = candidate_end, candidate_start
|
||||
|
||||
score = 0
|
||||
overlap_seconds = _overlap_seconds(rcon_start, rcon_end, candidate_start, candidate_end)
|
||||
rcon_midpoint = rcon_start + (rcon_end - rcon_start) / 2
|
||||
if overlap_seconds > 0:
|
||||
score += 3
|
||||
if candidate_start <= rcon_midpoint <= candidate_end:
|
||||
score += 2
|
||||
|
||||
closest_edge_distance = min(
|
||||
abs((rcon_start - candidate_start).total_seconds()),
|
||||
abs((rcon_start - candidate_end).total_seconds()),
|
||||
abs((rcon_end - candidate_start).total_seconds()),
|
||||
abs((rcon_end - candidate_end).total_seconds()),
|
||||
)
|
||||
if closest_edge_distance <= 1800:
|
||||
score += 2
|
||||
elif closest_edge_distance <= 3600:
|
||||
score += 1
|
||||
|
||||
candidate_duration = int((candidate_end - candidate_start).total_seconds())
|
||||
if duration_seconds and candidate_duration > 0:
|
||||
if abs(candidate_duration - duration_seconds) <= 1800:
|
||||
score += 1
|
||||
elif overlap_seconds > 0 and duration_seconds <= candidate_duration:
|
||||
score += 1
|
||||
|
||||
candidate_allied_score = _coerce_int(candidate.get("allied_score"))
|
||||
candidate_axis_score = _coerce_int(candidate.get("axis_score"))
|
||||
if (
|
||||
allied_score is not None
|
||||
and axis_score is not None
|
||||
and candidate_allied_score is not None
|
||||
and candidate_axis_score is not None
|
||||
):
|
||||
if candidate_allied_score == allied_score and candidate_axis_score == axis_score:
|
||||
score += 2
|
||||
elif sorted((candidate_allied_score, candidate_axis_score)) == sorted((allied_score, axis_score)):
|
||||
score += 1
|
||||
|
||||
candidate_players = _coerce_int(candidate.get("player_count"))
|
||||
reference_players = peak_players or player_count
|
||||
if candidate_players and reference_players:
|
||||
if abs(candidate_players - reference_players) <= 20:
|
||||
score += 1
|
||||
elif candidate_players >= int(reference_players * 0.75):
|
||||
score += 1
|
||||
|
||||
if score <= 0:
|
||||
return None
|
||||
return {
|
||||
"score": score,
|
||||
"external_match_id": candidate.get("external_match_id"),
|
||||
"match_url": candidate["match_url"],
|
||||
}
|
||||
|
||||
|
||||
def _diagnostic_candidate_summary(
|
||||
candidate: dict[str, object],
|
||||
*,
|
||||
server_slug: str,
|
||||
normalized_map: str,
|
||||
rcon_start: datetime,
|
||||
rcon_end: datetime,
|
||||
duration_seconds: int | None,
|
||||
player_count: int | None,
|
||||
peak_players: int | None,
|
||||
allied_score: int | None,
|
||||
axis_score: int | None,
|
||||
) -> dict[str, object]:
|
||||
match_url = resolve_trusted_scoreboard_match_url(candidate.get("match_url"), server_slug)
|
||||
safe_candidate = {**candidate, "match_url": match_url} if match_url else None
|
||||
scored = (
|
||||
_score_candidate(
|
||||
safe_candidate,
|
||||
normalized_map=normalized_map,
|
||||
rcon_start=rcon_start,
|
||||
rcon_end=rcon_end,
|
||||
duration_seconds=duration_seconds,
|
||||
player_count=player_count,
|
||||
peak_players=peak_players,
|
||||
allied_score=allied_score,
|
||||
axis_score=axis_score,
|
||||
)
|
||||
if safe_candidate
|
||||
else None
|
||||
)
|
||||
map_label = candidate.get("map_pretty_name") or candidate.get("map_name")
|
||||
summary = {
|
||||
"external_match_id": candidate.get("external_match_id"),
|
||||
"started_at": candidate.get("started_at"),
|
||||
"ended_at": candidate.get("ended_at"),
|
||||
"map": map_label,
|
||||
"score": {
|
||||
"allied_score": _coerce_int(candidate.get("allied_score")),
|
||||
"axis_score": _coerce_int(candidate.get("axis_score")),
|
||||
},
|
||||
"match_url": match_url,
|
||||
"correlation_score": int(scored["score"]) if scored else None,
|
||||
}
|
||||
if not match_url:
|
||||
summary["rejection_reason"] = "unsafe-url"
|
||||
elif scored is None:
|
||||
summary["rejection_reason"] = "map-or-window-mismatch"
|
||||
return summary
|
||||
|
||||
|
||||
def _overlap_seconds(
|
||||
first_start: datetime,
|
||||
first_end: datetime,
|
||||
second_start: datetime,
|
||||
second_end: datetime,
|
||||
) -> int:
|
||||
return max(0, int((min(first_end, second_end) - max(first_start, second_start)).total_seconds()))
|
||||
|
||||
|
||||
def _parse_timestamp(value: object) -> datetime | None:
|
||||
if not isinstance(value, str) or not value.strip():
|
||||
return None
|
||||
try:
|
||||
parsed = datetime.fromisoformat(value.strip().replace("Z", "+00:00"))
|
||||
except ValueError:
|
||||
return None
|
||||
if parsed.tzinfo is None:
|
||||
parsed = parsed.replace(tzinfo=timezone.utc)
|
||||
return parsed.astimezone(timezone.utc)
|
||||
|
||||
|
||||
def _coerce_int(value: object) -> int | None:
|
||||
if value is None:
|
||||
return None
|
||||
try:
|
||||
return int(round(float(value)))
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
78
backend/app/rcon_scoreboard_relink.py
Normal file
78
backend/app/rcon_scoreboard_relink.py
Normal file
@@ -0,0 +1,78 @@
|
||||
"""Report safe scoreboard links for existing materialized RCON matches."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
from collections.abc import Iterable
|
||||
from pathlib import Path
|
||||
|
||||
from .rcon_admin_log_materialization import list_materialized_rcon_matches
|
||||
from .rcon_historical_read_model import build_materialized_scoreboard_correlation_input
|
||||
from .rcon_scoreboard_correlation import resolve_rcon_scoreboard_correlation
|
||||
|
||||
|
||||
DEFAULT_LIMIT = 500
|
||||
|
||||
|
||||
def relink_materialized_matches(
|
||||
*,
|
||||
server_key: str | None = None,
|
||||
limit: int = DEFAULT_LIMIT,
|
||||
db_path: Path | None = None,
|
||||
) -> dict[str, object]:
|
||||
"""Scan existing matches against trusted candidates used by the detail read model."""
|
||||
matches = list_materialized_rcon_matches(
|
||||
target_key=server_key,
|
||||
only_ended=True,
|
||||
limit=limit,
|
||||
db_path=db_path,
|
||||
)
|
||||
report: dict[str, object] = {
|
||||
"matches_scanned": len(matches),
|
||||
"candidates_scanned": 0,
|
||||
"matches_linked": 0,
|
||||
"matches_skipped_no_candidate": 0,
|
||||
"matches_skipped_ambiguous": 0,
|
||||
"errors": [],
|
||||
}
|
||||
for match in matches:
|
||||
try:
|
||||
resolution = resolve_rcon_scoreboard_correlation(
|
||||
**build_materialized_scoreboard_correlation_input(match),
|
||||
db_path=db_path,
|
||||
)
|
||||
except Exception as exc:
|
||||
report["errors"].append(
|
||||
{"match_key": match.get("match_key"), "message": str(exc)}
|
||||
)
|
||||
continue
|
||||
report["candidates_scanned"] += int(resolution.get("candidate_count") or 0)
|
||||
if resolution.get("match_url"):
|
||||
report["matches_linked"] += 1
|
||||
elif resolution.get("reason") == "ambiguous-candidate":
|
||||
report["matches_skipped_ambiguous"] += 1
|
||||
else:
|
||||
report["matches_skipped_no_candidate"] += 1
|
||||
return report
|
||||
|
||||
|
||||
def main(argv: Iterable[str] | None = None) -> int:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Resolve trusted scoreboard links for materialized RCON matches."
|
||||
)
|
||||
parser.add_argument("--server", dest="server_key")
|
||||
parser.add_argument("--limit", type=int, default=DEFAULT_LIMIT)
|
||||
parser.add_argument("--db-path", type=Path, default=None)
|
||||
args = parser.parse_args(list(argv) if argv is not None else None)
|
||||
report = relink_materialized_matches(
|
||||
server_key=args.server_key,
|
||||
limit=max(1, args.limit),
|
||||
db_path=args.db_path,
|
||||
)
|
||||
print(json.dumps(report, ensure_ascii=False, indent=2))
|
||||
return 0 if not report["errors"] else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
395
backend/app/routes.py
Normal file
395
backend/app/routes.py
Normal file
@@ -0,0 +1,395 @@
|
||||
"""Route resolution helpers for the HLL Vietnam backend bootstrap."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from http import HTTPStatus
|
||||
from urllib.parse import parse_qs, urlparse
|
||||
|
||||
from .config import get_historical_data_source_kind
|
||||
from .payloads import (
|
||||
build_community_payload,
|
||||
build_current_match_kill_feed_payload,
|
||||
build_current_match_player_stats_payload,
|
||||
build_current_match_payload,
|
||||
build_discord_payload,
|
||||
build_elo_mmr_leaderboard_payload,
|
||||
build_elo_mmr_player_payload,
|
||||
build_error_payload,
|
||||
build_health_payload,
|
||||
build_historical_leaderboard_payload,
|
||||
build_historical_match_detail_payload,
|
||||
build_monthly_mvp_payload,
|
||||
build_monthly_mvp_v2_payload,
|
||||
build_monthly_leaderboard_payload,
|
||||
build_monthly_leaderboard_snapshot_payload,
|
||||
build_monthly_mvp_snapshot_payload,
|
||||
build_monthly_mvp_v2_snapshot_payload,
|
||||
build_player_event_payload,
|
||||
build_player_event_snapshot_payload,
|
||||
build_historical_server_summary_snapshot_payload,
|
||||
build_historical_player_profile_payload,
|
||||
build_historical_server_summary_payload,
|
||||
build_leaderboard_snapshot_payload,
|
||||
build_recent_historical_matches_snapshot_payload,
|
||||
build_recent_historical_matches_payload,
|
||||
build_server_detail_history_payload,
|
||||
build_server_history_payload,
|
||||
build_server_latest_payload,
|
||||
build_servers_payload,
|
||||
build_trailer_payload,
|
||||
build_weekly_leaderboard_snapshot_payload,
|
||||
build_weekly_leaderboard_payload,
|
||||
build_weekly_top_kills_payload,
|
||||
)
|
||||
from .rcon_historical_leaderboards import build_rcon_materialized_leaderboard_snapshot_payload
|
||||
from .scoreboard_origins import get_trusted_public_scoreboard_origin
|
||||
|
||||
|
||||
GET_ROUTES = {
|
||||
"/health": build_health_payload,
|
||||
"/api/community": build_community_payload,
|
||||
"/api/trailer": build_trailer_payload,
|
||||
"/api/discord": build_discord_payload,
|
||||
"/api/servers": build_servers_payload,
|
||||
}
|
||||
|
||||
|
||||
def resolve_get_payload(path: str) -> tuple[HTTPStatus | None, dict[str, object]]:
|
||||
"""Resolve the JSON payload for a supported GET route."""
|
||||
parsed = urlparse(path)
|
||||
if parsed.path == "/api/servers/latest":
|
||||
return HTTPStatus.OK, build_server_latest_payload()
|
||||
|
||||
if parsed.path == "/api/servers/history":
|
||||
limit = _parse_limit(parsed.query)
|
||||
if limit is None:
|
||||
return HTTPStatus.BAD_REQUEST, build_error_payload("Invalid limit parameter")
|
||||
return HTTPStatus.OK, build_server_history_payload(limit=limit)
|
||||
|
||||
if parsed.path == "/api/current-match":
|
||||
server_slug = parse_qs(parsed.query).get("server", [None])[0]
|
||||
if not server_slug:
|
||||
return HTTPStatus.BAD_REQUEST, build_error_payload("Server parameter is required")
|
||||
if get_trusted_public_scoreboard_origin(server_slug) is None:
|
||||
return HTTPStatus.NOT_FOUND, build_error_payload("Current match server is not supported")
|
||||
return HTTPStatus.OK, build_current_match_payload(server_slug=server_slug)
|
||||
|
||||
if parsed.path == "/api/current-match/kills":
|
||||
limit = _parse_limit(parsed.query)
|
||||
if limit is None:
|
||||
return HTTPStatus.BAD_REQUEST, build_error_payload("Invalid limit parameter")
|
||||
params = parse_qs(parsed.query)
|
||||
server_slug = params.get("server", [None])[0]
|
||||
if not server_slug:
|
||||
return HTTPStatus.BAD_REQUEST, build_error_payload("Server parameter is required")
|
||||
if get_trusted_public_scoreboard_origin(server_slug) is None:
|
||||
return HTTPStatus.NOT_FOUND, build_error_payload("Current match server is not supported")
|
||||
return HTTPStatus.OK, build_current_match_kill_feed_payload(
|
||||
server_slug=server_slug,
|
||||
limit=limit,
|
||||
since_event_id=params.get("since_event_id", [None])[0],
|
||||
)
|
||||
|
||||
if parsed.path == "/api/current-match/players":
|
||||
server_slug = parse_qs(parsed.query).get("server", [None])[0]
|
||||
if not server_slug:
|
||||
return HTTPStatus.BAD_REQUEST, build_error_payload("Server parameter is required")
|
||||
if get_trusted_public_scoreboard_origin(server_slug) is None:
|
||||
return HTTPStatus.NOT_FOUND, build_error_payload("Current match server is not supported")
|
||||
return HTTPStatus.OK, build_current_match_player_stats_payload(server_slug=server_slug)
|
||||
|
||||
if parsed.path == "/api/historical/weekly-top-kills":
|
||||
limit = _parse_limit(parsed.query)
|
||||
if limit is None:
|
||||
return HTTPStatus.BAD_REQUEST, build_error_payload("Invalid limit parameter")
|
||||
server_id = parse_qs(parsed.query).get("server", [None])[0]
|
||||
return HTTPStatus.OK, build_weekly_top_kills_payload(limit=limit, server_id=server_id)
|
||||
|
||||
if parsed.path == "/api/historical/leaderboard":
|
||||
limit = _parse_limit(parsed.query)
|
||||
if limit is None:
|
||||
return HTTPStatus.BAD_REQUEST, build_error_payload("Invalid limit parameter")
|
||||
params = parse_qs(parsed.query)
|
||||
server_id = params.get("server", [None])[0]
|
||||
metric = params.get("metric", ["kills"])[0]
|
||||
timeframe = params.get("timeframe", ["weekly"])[0]
|
||||
if metric not in {"kills", "deaths", "support", "matches_over_100_kills"}:
|
||||
return HTTPStatus.BAD_REQUEST, build_error_payload("Invalid metric parameter")
|
||||
if timeframe not in {"weekly", "monthly"}:
|
||||
return HTTPStatus.BAD_REQUEST, build_error_payload("Invalid timeframe parameter")
|
||||
return HTTPStatus.OK, build_historical_leaderboard_payload(
|
||||
limit=limit,
|
||||
server_id=server_id,
|
||||
metric=metric,
|
||||
timeframe=timeframe,
|
||||
)
|
||||
|
||||
if parsed.path == "/api/historical/weekly-leaderboard":
|
||||
limit = _parse_limit(parsed.query)
|
||||
if limit is None:
|
||||
return HTTPStatus.BAD_REQUEST, build_error_payload("Invalid limit parameter")
|
||||
params = parse_qs(parsed.query)
|
||||
server_id = params.get("server", [None])[0]
|
||||
metric = params.get("metric", ["kills"])[0]
|
||||
if metric not in {"kills", "deaths", "support", "matches_over_100_kills"}:
|
||||
return HTTPStatus.BAD_REQUEST, build_error_payload("Invalid metric parameter")
|
||||
return HTTPStatus.OK, build_weekly_leaderboard_payload(
|
||||
limit=limit,
|
||||
server_id=server_id,
|
||||
metric=metric,
|
||||
)
|
||||
|
||||
if parsed.path == "/api/historical/monthly-leaderboard":
|
||||
limit = _parse_limit(parsed.query)
|
||||
if limit is None:
|
||||
return HTTPStatus.BAD_REQUEST, build_error_payload("Invalid limit parameter")
|
||||
params = parse_qs(parsed.query)
|
||||
server_id = params.get("server", [None])[0]
|
||||
metric = params.get("metric", ["kills"])[0]
|
||||
if metric not in {"kills", "deaths", "support", "matches_over_100_kills"}:
|
||||
return HTTPStatus.BAD_REQUEST, build_error_payload("Invalid metric parameter")
|
||||
return HTTPStatus.OK, build_monthly_leaderboard_payload(
|
||||
limit=limit,
|
||||
server_id=server_id,
|
||||
metric=metric,
|
||||
)
|
||||
|
||||
if parsed.path == "/api/historical/monthly-mvp":
|
||||
limit = _parse_limit(parsed.query)
|
||||
if limit is None:
|
||||
return HTTPStatus.BAD_REQUEST, build_error_payload("Invalid limit parameter")
|
||||
server_id = parse_qs(parsed.query).get("server", [None])[0]
|
||||
return HTTPStatus.OK, build_monthly_mvp_payload(
|
||||
limit=limit,
|
||||
server_id=server_id,
|
||||
)
|
||||
|
||||
if parsed.path == "/api/historical/monthly-mvp-v2":
|
||||
limit = _parse_limit(parsed.query)
|
||||
if limit is None:
|
||||
return HTTPStatus.BAD_REQUEST, build_error_payload("Invalid limit parameter")
|
||||
server_id = parse_qs(parsed.query).get("server", [None])[0]
|
||||
return HTTPStatus.OK, build_monthly_mvp_v2_payload(
|
||||
limit=limit,
|
||||
server_id=server_id,
|
||||
)
|
||||
|
||||
if parsed.path == "/api/historical/player-events":
|
||||
limit = _parse_limit(parsed.query)
|
||||
if limit is None:
|
||||
return HTTPStatus.BAD_REQUEST, build_error_payload("Invalid limit parameter")
|
||||
params = parse_qs(parsed.query)
|
||||
server_id = params.get("server", [None])[0]
|
||||
view = params.get("view", ["most-killed"])[0]
|
||||
if view not in {"most-killed", "death-by", "duels", "weapon-kills", "teamkills"}:
|
||||
return HTTPStatus.BAD_REQUEST, build_error_payload("Invalid view parameter")
|
||||
return HTTPStatus.OK, build_player_event_payload(
|
||||
limit=limit,
|
||||
server_id=server_id,
|
||||
view=view,
|
||||
)
|
||||
|
||||
if parsed.path == "/api/historical/snapshots/leaderboard":
|
||||
limit = _parse_limit(parsed.query)
|
||||
if limit is None:
|
||||
return HTTPStatus.BAD_REQUEST, build_error_payload("Invalid limit parameter")
|
||||
params = parse_qs(parsed.query)
|
||||
server_id = params.get("server", [None])[0]
|
||||
metric = params.get("metric", ["kills"])[0]
|
||||
timeframe = params.get("timeframe", ["weekly"])[0]
|
||||
if metric not in {"kills", "deaths", "support", "matches_over_100_kills"}:
|
||||
return HTTPStatus.BAD_REQUEST, build_error_payload("Invalid metric parameter")
|
||||
if timeframe not in {"weekly", "monthly"}:
|
||||
return HTTPStatus.BAD_REQUEST, build_error_payload("Invalid timeframe parameter")
|
||||
if get_historical_data_source_kind() == "rcon":
|
||||
return HTTPStatus.OK, build_rcon_materialized_leaderboard_snapshot_payload(
|
||||
limit=limit,
|
||||
server_id=server_id,
|
||||
metric=metric,
|
||||
timeframe=timeframe,
|
||||
)
|
||||
return HTTPStatus.OK, build_leaderboard_snapshot_payload(
|
||||
limit=limit,
|
||||
server_id=server_id,
|
||||
metric=metric,
|
||||
timeframe=timeframe,
|
||||
)
|
||||
|
||||
if parsed.path == "/api/historical/snapshots/monthly-leaderboard":
|
||||
limit = _parse_limit(parsed.query)
|
||||
if limit is None:
|
||||
return HTTPStatus.BAD_REQUEST, build_error_payload("Invalid limit parameter")
|
||||
params = parse_qs(parsed.query)
|
||||
server_id = params.get("server", [None])[0]
|
||||
metric = params.get("metric", ["kills"])[0]
|
||||
if metric not in {"kills", "deaths", "support", "matches_over_100_kills"}:
|
||||
return HTTPStatus.BAD_REQUEST, build_error_payload("Invalid metric parameter")
|
||||
if get_historical_data_source_kind() == "rcon":
|
||||
return HTTPStatus.OK, build_rcon_materialized_leaderboard_snapshot_payload(
|
||||
limit=limit,
|
||||
server_id=server_id,
|
||||
metric=metric,
|
||||
timeframe="monthly",
|
||||
)
|
||||
return HTTPStatus.OK, build_monthly_leaderboard_snapshot_payload(
|
||||
limit=limit,
|
||||
server_id=server_id,
|
||||
metric=metric,
|
||||
)
|
||||
|
||||
if parsed.path == "/api/historical/snapshots/monthly-mvp":
|
||||
limit = _parse_limit(parsed.query)
|
||||
if limit is None:
|
||||
return HTTPStatus.BAD_REQUEST, build_error_payload("Invalid limit parameter")
|
||||
server_id = parse_qs(parsed.query).get("server", [None])[0]
|
||||
return HTTPStatus.OK, build_monthly_mvp_snapshot_payload(
|
||||
limit=limit,
|
||||
server_id=server_id,
|
||||
)
|
||||
|
||||
if parsed.path == "/api/historical/snapshots/monthly-mvp-v2":
|
||||
limit = _parse_limit(parsed.query)
|
||||
if limit is None:
|
||||
return HTTPStatus.BAD_REQUEST, build_error_payload("Invalid limit parameter")
|
||||
server_id = parse_qs(parsed.query).get("server", [None])[0]
|
||||
return HTTPStatus.OK, build_monthly_mvp_v2_snapshot_payload(
|
||||
limit=limit,
|
||||
server_id=server_id,
|
||||
)
|
||||
|
||||
if parsed.path == "/api/historical/snapshots/player-events":
|
||||
limit = _parse_limit(parsed.query)
|
||||
if limit is None:
|
||||
return HTTPStatus.BAD_REQUEST, build_error_payload("Invalid limit parameter")
|
||||
params = parse_qs(parsed.query)
|
||||
server_id = params.get("server", [None])[0]
|
||||
view = params.get("view", ["most-killed"])[0]
|
||||
if view not in {"most-killed", "death-by", "duels", "weapon-kills", "teamkills"}:
|
||||
return HTTPStatus.BAD_REQUEST, build_error_payload("Invalid view parameter")
|
||||
return HTTPStatus.OK, build_player_event_snapshot_payload(
|
||||
limit=limit,
|
||||
server_id=server_id,
|
||||
view=view,
|
||||
)
|
||||
|
||||
if parsed.path == "/api/historical/snapshots/weekly-leaderboard":
|
||||
limit = _parse_limit(parsed.query)
|
||||
if limit is None:
|
||||
return HTTPStatus.BAD_REQUEST, build_error_payload("Invalid limit parameter")
|
||||
params = parse_qs(parsed.query)
|
||||
server_id = params.get("server", [None])[0]
|
||||
metric = params.get("metric", ["kills"])[0]
|
||||
if metric not in {"kills", "deaths", "support", "matches_over_100_kills"}:
|
||||
return HTTPStatus.BAD_REQUEST, build_error_payload("Invalid metric parameter")
|
||||
if get_historical_data_source_kind() == "rcon":
|
||||
return HTTPStatus.OK, build_rcon_materialized_leaderboard_snapshot_payload(
|
||||
limit=limit,
|
||||
server_id=server_id,
|
||||
metric=metric,
|
||||
timeframe="weekly",
|
||||
)
|
||||
return HTTPStatus.OK, build_weekly_leaderboard_snapshot_payload(
|
||||
limit=limit,
|
||||
server_id=server_id,
|
||||
metric=metric,
|
||||
)
|
||||
|
||||
if parsed.path == "/api/historical/recent-matches":
|
||||
limit = _parse_limit(parsed.query)
|
||||
if limit is None:
|
||||
return HTTPStatus.BAD_REQUEST, build_error_payload("Invalid limit parameter")
|
||||
server_slug = parse_qs(parsed.query).get("server", [None])[0]
|
||||
return HTTPStatus.OK, build_recent_historical_matches_payload(
|
||||
limit=limit,
|
||||
server_slug=server_slug,
|
||||
)
|
||||
|
||||
if parsed.path == "/api/historical/snapshots/recent-matches":
|
||||
limit = _parse_limit(parsed.query)
|
||||
if limit is None:
|
||||
return HTTPStatus.BAD_REQUEST, build_error_payload("Invalid limit parameter")
|
||||
server_slug = parse_qs(parsed.query).get("server", [None])[0]
|
||||
return HTTPStatus.OK, build_recent_historical_matches_snapshot_payload(
|
||||
limit=limit,
|
||||
server_slug=server_slug,
|
||||
)
|
||||
|
||||
if parsed.path == "/api/historical/matches/detail":
|
||||
params = parse_qs(parsed.query)
|
||||
server_slug = params.get("server", [None])[0]
|
||||
match_id = params.get("match", [None])[0]
|
||||
if not server_slug:
|
||||
return HTTPStatus.BAD_REQUEST, build_error_payload("Server parameter is required")
|
||||
if not match_id:
|
||||
return HTTPStatus.BAD_REQUEST, build_error_payload("Match parameter is required")
|
||||
return HTTPStatus.OK, build_historical_match_detail_payload(
|
||||
server_slug=server_slug,
|
||||
match_id=match_id,
|
||||
)
|
||||
|
||||
if parsed.path == "/api/historical/server-summary":
|
||||
server_slug = parse_qs(parsed.query).get("server", [None])[0]
|
||||
return HTTPStatus.OK, build_historical_server_summary_payload(server_slug=server_slug)
|
||||
|
||||
if parsed.path == "/api/historical/snapshots/server-summary":
|
||||
server_slug = parse_qs(parsed.query).get("server", [None])[0]
|
||||
return HTTPStatus.OK, build_historical_server_summary_snapshot_payload(
|
||||
server_slug=server_slug
|
||||
)
|
||||
|
||||
if parsed.path == "/api/historical/player-profile":
|
||||
player_id = parse_qs(parsed.query).get("player", [None])[0]
|
||||
if not player_id:
|
||||
return HTTPStatus.BAD_REQUEST, build_error_payload("Player parameter is required")
|
||||
return HTTPStatus.OK, build_historical_player_profile_payload(player_id)
|
||||
|
||||
if parsed.path == "/api/historical/elo-mmr/leaderboard":
|
||||
limit = _parse_limit(parsed.query)
|
||||
if limit is None:
|
||||
return HTTPStatus.BAD_REQUEST, build_error_payload("Invalid limit parameter")
|
||||
server_id = parse_qs(parsed.query).get("server", [None])[0]
|
||||
return HTTPStatus.OK, build_elo_mmr_leaderboard_payload(
|
||||
limit=limit,
|
||||
server_id=server_id,
|
||||
)
|
||||
|
||||
if parsed.path == "/api/historical/elo-mmr/player":
|
||||
params = parse_qs(parsed.query)
|
||||
player_id = params.get("player", [None])[0]
|
||||
if not player_id:
|
||||
return HTTPStatus.BAD_REQUEST, build_error_payload("Player parameter is required")
|
||||
server_id = params.get("server", [None])[0]
|
||||
return HTTPStatus.OK, build_elo_mmr_player_payload(
|
||||
player_id=player_id,
|
||||
server_id=server_id,
|
||||
)
|
||||
|
||||
builder = GET_ROUTES.get(parsed.path)
|
||||
if builder is None:
|
||||
if parsed.path.startswith("/api/servers/") and parsed.path.endswith("/history"):
|
||||
server_id = parsed.path.removeprefix("/api/servers/").removesuffix("/history")
|
||||
server_id = server_id.strip("/")
|
||||
if not server_id:
|
||||
return HTTPStatus.BAD_REQUEST, build_error_payload("Server id is required")
|
||||
|
||||
limit = _parse_limit(parsed.query)
|
||||
if limit is None:
|
||||
return HTTPStatus.BAD_REQUEST, build_error_payload("Invalid limit parameter")
|
||||
|
||||
return HTTPStatus.OK, build_server_detail_history_payload(server_id, limit=limit)
|
||||
return None, {}
|
||||
|
||||
return HTTPStatus.OK, builder()
|
||||
|
||||
|
||||
def _parse_limit(query: str) -> int | None:
|
||||
raw_limit = parse_qs(query).get("limit", ["20"])[0]
|
||||
try:
|
||||
limit = int(raw_limit)
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
if limit < 1 or limit > 100:
|
||||
return None
|
||||
|
||||
return limit
|
||||
100
backend/app/scheduler.py
Normal file
100
backend/app/scheduler.py
Normal file
@@ -0,0 +1,100 @@
|
||||
"""Local development loop for periodic snapshot refreshes."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import time
|
||||
|
||||
from .a2s_client import DEFAULT_A2S_TIMEOUT
|
||||
from .collector import collect_server_snapshots
|
||||
from .config import get_refresh_interval_seconds
|
||||
|
||||
|
||||
def run_local_refresh_loop(
|
||||
*,
|
||||
interval_seconds: int,
|
||||
source_mode: str,
|
||||
timeout: float,
|
||||
allow_controlled_fallback: bool,
|
||||
max_runs: int | None = None,
|
||||
) -> None:
|
||||
"""Run the collector periodically until interrupted or the run limit is reached."""
|
||||
completed_runs = 0
|
||||
print(
|
||||
"Starting local snapshot refresh loop "
|
||||
f"(interval={interval_seconds}s, source={source_mode}, persist=true)."
|
||||
)
|
||||
print("Press Ctrl+C to stop.")
|
||||
|
||||
try:
|
||||
while max_runs is None or completed_runs < max_runs:
|
||||
completed_runs += 1
|
||||
payload = collect_server_snapshots(
|
||||
source_mode=source_mode,
|
||||
timeout=timeout,
|
||||
allow_controlled_fallback=allow_controlled_fallback,
|
||||
persist=True,
|
||||
)
|
||||
print(json.dumps({"run": completed_runs, **payload}, indent=2))
|
||||
|
||||
if max_runs is not None and completed_runs >= max_runs:
|
||||
break
|
||||
|
||||
time.sleep(interval_seconds)
|
||||
except KeyboardInterrupt:
|
||||
print("\nLocal snapshot refresh loop stopped by user.")
|
||||
|
||||
|
||||
def main() -> None:
|
||||
"""Allow local scheduled refresh execution without adding external infrastructure."""
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Run periodic local snapshot refreshes for development and landing demos.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--interval",
|
||||
type=int,
|
||||
default=get_refresh_interval_seconds(),
|
||||
help="Seconds to wait between persisted refresh runs. Defaults to env value or 60.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--source",
|
||||
choices=("controlled", "a2s", "auto"),
|
||||
default="auto",
|
||||
help="Choose controlled data, configured A2S targets, or auto with fallback.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--timeout",
|
||||
type=float,
|
||||
default=DEFAULT_A2S_TIMEOUT,
|
||||
help="Socket timeout in seconds for A2S probes.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--no-fallback",
|
||||
action="store_true",
|
||||
help="Disable fallback to controlled data when A2S fails.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--max-runs",
|
||||
type=int,
|
||||
default=None,
|
||||
help="Optional safety limit for the number of refresh cycles to execute.",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.interval <= 0:
|
||||
raise ValueError("--interval must be a positive integer.")
|
||||
if args.max_runs is not None and args.max_runs <= 0:
|
||||
raise ValueError("--max-runs must be positive when provided.")
|
||||
|
||||
run_local_refresh_loop(
|
||||
interval_seconds=args.interval,
|
||||
source_mode=args.source,
|
||||
timeout=args.timeout,
|
||||
allow_controlled_fallback=not args.no_fallback,
|
||||
max_runs=args.max_runs,
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
259
backend/app/scoreboard_candidate_backfill.py
Normal file
259
backend/app/scoreboard_candidate_backfill.py
Normal file
@@ -0,0 +1,259 @@
|
||||
"""Backfill public scoreboard candidates for RCON match link correlation."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
from datetime import datetime, timezone
|
||||
from collections.abc import Mapping
|
||||
from typing import Iterable
|
||||
|
||||
from .historical_storage import initialize_historical_storage, list_historical_servers, upsert_historical_match
|
||||
from .postgres_rcon_storage import upsert_scoreboard_candidate
|
||||
from .providers.public_scoreboard_provider import PublicScoreboardHistoricalDataSource
|
||||
from .scoreboard_origins import (
|
||||
build_trusted_scoreboard_match_url,
|
||||
get_trusted_public_scoreboard_origin,
|
||||
list_trusted_public_scoreboard_origins,
|
||||
)
|
||||
|
||||
DEFAULT_MAX_PAGES = 20
|
||||
DEFAULT_PAGE_SIZE = 100
|
||||
DEFAULT_DETAIL_WORKERS = 4
|
||||
|
||||
|
||||
def main(argv: Iterable[str] | None = None) -> int:
|
||||
parser = build_arg_parser()
|
||||
args = parser.parse_args(list(argv) if argv is not None else None)
|
||||
start_at = _parse_timestamp(args.start_at, option_name="--from")
|
||||
end_at = _parse_timestamp(args.end_at, option_name="--to")
|
||||
if end_at <= start_at:
|
||||
parser.error("--to must be later than --from")
|
||||
server = _resolve_server(args.server_slug, parser)
|
||||
report = run_backfill(server=server, start_at=start_at, end_at=end_at, max_pages=args.max_pages, page_size=args.page_size, detail_workers=args.detail_workers)
|
||||
print(json.dumps(report, ensure_ascii=False, indent=2))
|
||||
return 0 if not report["errors"] else 1
|
||||
|
||||
|
||||
def run_backfill(*, server: dict[str, object], start_at: datetime, end_at: datetime, max_pages: int, page_size: int, detail_workers: int) -> dict[str, object]:
|
||||
initialize_historical_storage()
|
||||
provider = PublicScoreboardHistoricalDataSource()
|
||||
server_slug = str(server["slug"])
|
||||
base_url = str(server["scoreboard_base_url"])
|
||||
counters = {
|
||||
"pages_processed": 0,
|
||||
"candidates_seen": 0,
|
||||
"list_candidates_inserted": 0,
|
||||
"list_candidates_updated": 0,
|
||||
"list_candidates_skipped": 0,
|
||||
"candidates_inserted": 0,
|
||||
"candidates_updated": 0,
|
||||
"player_rows_inserted": 0,
|
||||
"player_rows_updated": 0,
|
||||
}
|
||||
errors: list[dict[str, object]] = []
|
||||
skipped_unsafe_urls = 0
|
||||
stopped_after_window = False
|
||||
for page in range(1, max_pages + 1):
|
||||
try:
|
||||
page_payload = provider.fetch_match_page(base_url=base_url, page=page, limit=page_size)
|
||||
except Exception as exc:
|
||||
errors.append({"stage": "fetch_match_page", "page": page, "message": str(exc)})
|
||||
break
|
||||
matches = _coerce_match_list(page_payload.get("maps"))
|
||||
if not matches:
|
||||
break
|
||||
counters["pages_processed"] += 1
|
||||
ids: list[str] = []
|
||||
for match in matches:
|
||||
counters["candidates_seen"] += 1
|
||||
ref_time = _parse_optional_timestamp(_pick_match_timestamp(match))
|
||||
if ref_time and ref_time < start_at:
|
||||
stopped_after_window = True
|
||||
continue
|
||||
if ref_time and ref_time >= end_at:
|
||||
continue
|
||||
candidate = _build_list_candidate(server=server, match=match)
|
||||
if candidate is None:
|
||||
counters["list_candidates_skipped"] += 1
|
||||
skipped_unsafe_urls += int(_list_candidate_url_is_unsafe(server=server, match=match))
|
||||
else:
|
||||
try:
|
||||
outcome = upsert_scoreboard_candidate(
|
||||
server_slug=server_slug,
|
||||
candidate=candidate,
|
||||
)
|
||||
except Exception as exc:
|
||||
counters["list_candidates_skipped"] += 1
|
||||
errors.append(
|
||||
{
|
||||
"stage": "upsert_list_scoreboard_candidate",
|
||||
"match_id": candidate["external_match_id"],
|
||||
"message": str(exc),
|
||||
}
|
||||
)
|
||||
else:
|
||||
counters[f"list_candidates_{outcome}"] += 1
|
||||
match_id = _stringify(match.get("id"))
|
||||
if match_id:
|
||||
ids.append(match_id)
|
||||
if ids:
|
||||
try:
|
||||
details = provider.fetch_match_details(base_url=base_url, match_ids=ids, max_workers=detail_workers)
|
||||
except Exception as exc:
|
||||
errors.append({"stage": "fetch_match_details", "page": page, "message": str(exc)})
|
||||
details = []
|
||||
for detail in details:
|
||||
try:
|
||||
delta = upsert_historical_match(server_slug=server_slug, match_payload=detail)
|
||||
except Exception as exc:
|
||||
errors.append({"stage": "upsert_historical_match", "match_id": _stringify(detail.get("id")), "message": str(exc)})
|
||||
continue
|
||||
counters["candidates_inserted"] += _coerce_int(delta.get("matches_inserted"))
|
||||
counters["candidates_updated"] += _coerce_int(delta.get("matches_updated"))
|
||||
counters["player_rows_inserted"] += _coerce_int(delta.get("player_rows_inserted"))
|
||||
counters["player_rows_updated"] += _coerce_int(delta.get("player_rows_updated"))
|
||||
if stopped_after_window:
|
||||
break
|
||||
return {"status": "ok" if not errors else "partial", "server": server_slug, "scoreboard_base_url": base_url, "requested_window": {"from": _format_timestamp(start_at), "to": _format_timestamp(end_at)}, "stopped_after_window": stopped_after_window, "skipped_unsafe_urls": skipped_unsafe_urls, "errors": errors, **counters}
|
||||
|
||||
|
||||
def build_arg_parser() -> argparse.ArgumentParser:
|
||||
parser = argparse.ArgumentParser(description="Backfill public scoreboard match candidates for RCON link correlation.")
|
||||
parser.add_argument("--server", dest="server_slug", required=True)
|
||||
parser.add_argument("--from", dest="start_at", required=True)
|
||||
parser.add_argument("--to", dest="end_at", required=True)
|
||||
parser.add_argument("--max-pages", type=int, default=DEFAULT_MAX_PAGES)
|
||||
parser.add_argument("--page-size", type=int, default=DEFAULT_PAGE_SIZE)
|
||||
parser.add_argument("--detail-workers", type=int, default=DEFAULT_DETAIL_WORKERS)
|
||||
return parser
|
||||
|
||||
|
||||
def _resolve_server(server_slug: str, parser: argparse.ArgumentParser) -> dict[str, object]:
|
||||
trusted = {origin.slug for origin in list_trusted_public_scoreboard_origins()}
|
||||
if server_slug not in trusted:
|
||||
parser.error(f"unknown or untrusted server '{server_slug}'")
|
||||
for server in list_historical_servers():
|
||||
if server.get("slug") == server_slug:
|
||||
return server
|
||||
parser.error(f"trusted server '{server_slug}' is not present in historical storage")
|
||||
raise AssertionError("unreachable")
|
||||
|
||||
|
||||
def _parse_timestamp(value: str, *, option_name: str) -> datetime:
|
||||
try:
|
||||
parsed = datetime.fromisoformat(value.strip().replace("Z", "+00:00"))
|
||||
except ValueError as exc:
|
||||
raise argparse.ArgumentTypeError(f"{option_name} must be an ISO timestamp") from exc
|
||||
if parsed.tzinfo is None:
|
||||
parsed = parsed.replace(tzinfo=timezone.utc)
|
||||
return parsed.astimezone(timezone.utc)
|
||||
|
||||
|
||||
def _parse_optional_timestamp(value: object) -> datetime | None:
|
||||
if not isinstance(value, str) or not value.strip():
|
||||
return None
|
||||
try:
|
||||
return _parse_timestamp(value, option_name="timestamp")
|
||||
except argparse.ArgumentTypeError:
|
||||
return None
|
||||
|
||||
|
||||
def _format_timestamp(value: datetime) -> str:
|
||||
return value.astimezone(timezone.utc).isoformat().replace("+00:00", "Z")
|
||||
|
||||
|
||||
def _coerce_match_list(payload: object) -> list[dict[str, object]]:
|
||||
return [item for item in payload if isinstance(item, dict)] if isinstance(payload, list) else []
|
||||
|
||||
|
||||
def _pick_match_timestamp(match: dict[str, object]) -> object:
|
||||
for key in ("end", "start", "creation_time"):
|
||||
value = match.get(key)
|
||||
if isinstance(value, str) and value.strip():
|
||||
return value.strip()
|
||||
return None
|
||||
|
||||
|
||||
def _build_list_candidate(
|
||||
*,
|
||||
server: Mapping[str, object],
|
||||
match: Mapping[str, object],
|
||||
) -> dict[str, object] | None:
|
||||
server_slug = _stringify(server.get("slug"))
|
||||
external_match_id = _stringify(match.get("id"))
|
||||
origin = get_trusted_public_scoreboard_origin(server_slug)
|
||||
map_payload = match.get("map")
|
||||
result_payload = match.get("result")
|
||||
if (
|
||||
origin is None
|
||||
or not external_match_id
|
||||
or not external_match_id.isdigit()
|
||||
or str(server.get("scoreboard_base_url") or "").strip() != origin.base_url
|
||||
or _coerce_optional_int(server.get("server_number")) != origin.server_number
|
||||
or _coerce_optional_int(match.get("server_number")) != origin.server_number
|
||||
or not isinstance(map_payload, Mapping)
|
||||
or not isinstance(result_payload, Mapping)
|
||||
):
|
||||
return None
|
||||
|
||||
started_at = _stringify(match.get("start"))
|
||||
ended_at = _stringify(match.get("end"))
|
||||
match_url = build_trusted_scoreboard_match_url(
|
||||
server_slug=server_slug,
|
||||
external_match_id=external_match_id,
|
||||
)
|
||||
if not started_at or not ended_at or not match_url:
|
||||
return None
|
||||
return {
|
||||
"external_match_id": external_match_id,
|
||||
"started_at": started_at,
|
||||
"ended_at": ended_at,
|
||||
"map_name": _stringify(map_payload.get("id") or map_payload.get("name")),
|
||||
"map_pretty_name": _stringify(map_payload.get("pretty_name")),
|
||||
"allied_score": _coerce_optional_int(result_payload.get("allied")),
|
||||
"axis_score": _coerce_optional_int(result_payload.get("axis")),
|
||||
"player_count": _coerce_optional_int(match.get("player_count")),
|
||||
"match_url": match_url,
|
||||
}
|
||||
|
||||
|
||||
def _list_candidate_url_is_unsafe(
|
||||
*,
|
||||
server: Mapping[str, object],
|
||||
match: Mapping[str, object],
|
||||
) -> bool:
|
||||
external_match_id = _stringify(match.get("id"))
|
||||
return bool(
|
||||
external_match_id
|
||||
and build_trusted_scoreboard_match_url(
|
||||
server_slug=server.get("slug"),
|
||||
external_match_id=external_match_id,
|
||||
)
|
||||
is None
|
||||
)
|
||||
|
||||
|
||||
def _stringify(value: object) -> str | None:
|
||||
if value is None:
|
||||
return None
|
||||
text = str(value).strip()
|
||||
return text or None
|
||||
|
||||
|
||||
def _coerce_int(value: object) -> int:
|
||||
try:
|
||||
return int(value or 0)
|
||||
except (TypeError, ValueError):
|
||||
return 0
|
||||
|
||||
|
||||
def _coerce_optional_int(value: object) -> int | None:
|
||||
try:
|
||||
return None if value is None else int(value)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
83
backend/app/scoreboard_correlation_diagnostics.py
Normal file
83
backend/app/scoreboard_correlation_diagnostics.py
Normal file
@@ -0,0 +1,83 @@
|
||||
"""JSON diagnostics for missing materialized RCON scoreboard links."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
from collections.abc import Iterable
|
||||
from pathlib import Path
|
||||
|
||||
from .rcon_admin_log_materialization import get_materialized_rcon_match_detail
|
||||
from .rcon_historical_read_model import build_materialized_scoreboard_correlation_input
|
||||
from .rcon_scoreboard_correlation import diagnose_rcon_scoreboard_correlation
|
||||
|
||||
|
||||
def inspect_materialized_match_correlation(
|
||||
*,
|
||||
server_key: str,
|
||||
match_key: str,
|
||||
db_path: Path | None = None,
|
||||
) -> dict[str, object]:
|
||||
"""Return safe scoreboard correlation diagnostics for one materialized match."""
|
||||
materialized = get_materialized_rcon_match_detail(
|
||||
server_key=server_key,
|
||||
match_key=match_key,
|
||||
db_path=db_path,
|
||||
)
|
||||
if materialized is None:
|
||||
return {
|
||||
"rcon_match_key": match_key,
|
||||
"server": server_key,
|
||||
"candidate_count": 0,
|
||||
"top_candidates": [],
|
||||
"selected_candidate": None,
|
||||
"final_reason": "rcon-match-not-found",
|
||||
}
|
||||
|
||||
match = materialized["match"]
|
||||
correlation_input = build_materialized_scoreboard_correlation_input(match)
|
||||
correlation = diagnose_rcon_scoreboard_correlation(
|
||||
**correlation_input,
|
||||
db_path=db_path,
|
||||
)
|
||||
return {
|
||||
"rcon_match_key": match.get("match_key"),
|
||||
"server": match.get("external_server_id") or match.get("target_key"),
|
||||
"map": match.get("map_pretty_name") or match.get("map_name"),
|
||||
"started_at": match.get("started_at"),
|
||||
"ended_at": match.get("ended_at"),
|
||||
"closed_at": match.get("ended_at") or match.get("started_at"),
|
||||
"duration_seconds": correlation_input.get("duration_seconds"),
|
||||
"score": {
|
||||
"allied_score": match.get("allied_score"),
|
||||
"axis_score": match.get("axis_score"),
|
||||
"winner": match.get("winner"),
|
||||
},
|
||||
**correlation,
|
||||
}
|
||||
|
||||
|
||||
def main(argv: Iterable[str] | None = None) -> int:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Explain scoreboard candidate correlation for one RCON match."
|
||||
)
|
||||
parser.add_argument("--server", required=True)
|
||||
parser.add_argument("--match", dest="match_key", required=True)
|
||||
parser.add_argument("--db-path", type=Path, default=None)
|
||||
args = parser.parse_args(list(argv) if argv is not None else None)
|
||||
print(
|
||||
json.dumps(
|
||||
inspect_materialized_match_correlation(
|
||||
server_key=args.server,
|
||||
match_key=args.match_key,
|
||||
db_path=args.db_path,
|
||||
),
|
||||
ensure_ascii=False,
|
||||
indent=2,
|
||||
)
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
97
backend/app/scoreboard_origins.py
Normal file
97
backend/app/scoreboard_origins.py
Normal file
@@ -0,0 +1,97 @@
|
||||
"""Trusted public scoreboard origins for active community servers."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
import re
|
||||
from urllib.parse import urlparse
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class TrustedScoreboardOrigin:
|
||||
"""Public scoreboard origin trusted for one active community server."""
|
||||
|
||||
slug: str
|
||||
display_name: str
|
||||
base_url: str
|
||||
server_number: int
|
||||
source_kind: str = "crcon-scoreboard-json"
|
||||
|
||||
|
||||
TRUSTED_PUBLIC_SCOREBOARD_ORIGINS = (
|
||||
TrustedScoreboardOrigin(
|
||||
slug="comunidad-hispana-01",
|
||||
display_name="Comunidad Hispana #01",
|
||||
base_url="https://scoreboard.comunidadhll.es",
|
||||
server_number=1,
|
||||
),
|
||||
TrustedScoreboardOrigin(
|
||||
slug="comunidad-hispana-02",
|
||||
display_name="Comunidad Hispana #02",
|
||||
base_url="https://scoreboard.comunidadhll.es:5443",
|
||||
server_number=2,
|
||||
),
|
||||
)
|
||||
|
||||
_TRUSTED_GAME_PATH_RE = re.compile(r"^/games/\d+/?$")
|
||||
|
||||
|
||||
def list_trusted_public_scoreboard_origins() -> tuple[TrustedScoreboardOrigin, ...]:
|
||||
"""Return trusted public scoreboard origins for active default servers."""
|
||||
return TRUSTED_PUBLIC_SCOREBOARD_ORIGINS
|
||||
|
||||
|
||||
def get_trusted_public_scoreboard_origin(
|
||||
server_slug: object,
|
||||
) -> TrustedScoreboardOrigin | None:
|
||||
"""Return the trusted public scoreboard origin for one active server."""
|
||||
normalized_slug = str(server_slug or "").strip()
|
||||
if not normalized_slug:
|
||||
return None
|
||||
for origin in TRUSTED_PUBLIC_SCOREBOARD_ORIGINS:
|
||||
if origin.slug == normalized_slug:
|
||||
return origin
|
||||
return None
|
||||
|
||||
|
||||
def resolve_trusted_scoreboard_match_url(
|
||||
raw_payload_ref: object,
|
||||
server_slug: object,
|
||||
) -> str | None:
|
||||
"""Return a match URL only when it belongs to the trusted server origin."""
|
||||
origin = get_trusted_public_scoreboard_origin(server_slug)
|
||||
candidate = str(raw_payload_ref or "").strip()
|
||||
if origin is None or not candidate:
|
||||
return None
|
||||
|
||||
candidate_parts = urlparse(candidate)
|
||||
origin_parts = urlparse(origin.base_url)
|
||||
if candidate_parts.scheme not in {"http", "https"}:
|
||||
return None
|
||||
if candidate_parts.scheme != origin_parts.scheme:
|
||||
return None
|
||||
if candidate_parts.netloc != origin_parts.netloc:
|
||||
return None
|
||||
if candidate_parts.username or candidate_parts.password:
|
||||
return None
|
||||
if not _TRUSTED_GAME_PATH_RE.match(candidate_parts.path):
|
||||
return None
|
||||
if candidate_parts.params or candidate_parts.query or candidate_parts.fragment:
|
||||
return None
|
||||
return candidate
|
||||
|
||||
|
||||
def build_trusted_scoreboard_match_url(
|
||||
*,
|
||||
server_slug: object,
|
||||
external_match_id: object,
|
||||
) -> str | None:
|
||||
"""Build a trusted scoreboard match URL from one numeric public match id."""
|
||||
origin = get_trusted_public_scoreboard_origin(server_slug)
|
||||
match_id = str(external_match_id or "").strip()
|
||||
if origin is None or not match_id.isdigit():
|
||||
return None
|
||||
return resolve_trusted_scoreboard_match_url(
|
||||
f"{origin.base_url}/games/{match_id}",
|
||||
origin.slug,
|
||||
)
|
||||
106
backend/app/server_targets.py
Normal file
106
backend/app/server_targets.py
Normal file
@@ -0,0 +1,106 @@
|
||||
"""Registry helpers for development-time A2S probe targets."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from dataclasses import dataclass
|
||||
|
||||
from .config import DEFAULT_A2S_SOURCE_NAME, get_a2s_targets_payload
|
||||
|
||||
|
||||
DEFAULT_A2S_TARGETS = (
|
||||
{
|
||||
"name": "Comunidad Hispana #01",
|
||||
"host": "152.114.195.174",
|
||||
"query_port": 7778,
|
||||
"game_port": 7777,
|
||||
"source_name": DEFAULT_A2S_SOURCE_NAME,
|
||||
"external_server_id": "comunidad-hispana-01",
|
||||
"region": "ES",
|
||||
},
|
||||
{
|
||||
"name": "Comunidad Hispana #02",
|
||||
"host": "152.114.195.150",
|
||||
"query_port": 7878,
|
||||
"game_port": 7877,
|
||||
"source_name": DEFAULT_A2S_SOURCE_NAME,
|
||||
"external_server_id": "comunidad-hispana-02",
|
||||
"region": "ES",
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class A2SServerTarget:
|
||||
"""Minimal configuration needed to query one A2S target."""
|
||||
|
||||
name: str
|
||||
host: str
|
||||
query_port: int
|
||||
game_port: int | None
|
||||
source_name: str
|
||||
external_server_id: str | None = None
|
||||
region: str | None = None
|
||||
|
||||
|
||||
def load_a2s_targets() -> tuple[A2SServerTarget, ...]:
|
||||
"""Load configured A2S targets from env JSON or the local default registry."""
|
||||
raw_payload = get_a2s_targets_payload()
|
||||
raw_targets = DEFAULT_A2S_TARGETS if raw_payload is None else _parse_targets(raw_payload)
|
||||
return tuple(_coerce_target(item) for item in raw_targets)
|
||||
|
||||
|
||||
def _parse_targets(raw_payload: str) -> list[dict[str, object]]:
|
||||
try:
|
||||
parsed = json.loads(raw_payload)
|
||||
except json.JSONDecodeError as error:
|
||||
raise ValueError("HLL_BACKEND_A2S_TARGETS must be valid JSON.") from error
|
||||
|
||||
if not isinstance(parsed, list):
|
||||
raise ValueError("HLL_BACKEND_A2S_TARGETS must be a JSON array.")
|
||||
|
||||
return [item for item in parsed if isinstance(item, dict)]
|
||||
|
||||
|
||||
def _coerce_target(raw_target: dict[str, object]) -> A2SServerTarget:
|
||||
name = str(raw_target.get("name") or "Unnamed target").strip()
|
||||
host = str(raw_target.get("host") or "").strip()
|
||||
source_name = str(raw_target.get("source_name") or DEFAULT_A2S_SOURCE_NAME).strip()
|
||||
query_port = int(raw_target.get("query_port") or 0)
|
||||
game_port = _coerce_optional_positive_int(raw_target.get("game_port"))
|
||||
external_server_id = _string_or_none(raw_target.get("external_server_id"))
|
||||
region = _string_or_none(raw_target.get("region"))
|
||||
|
||||
if not host:
|
||||
raise ValueError("Each A2S target must define a non-empty host.")
|
||||
if query_port <= 0:
|
||||
raise ValueError("Each A2S target must define a valid query_port.")
|
||||
|
||||
return A2SServerTarget(
|
||||
name=name,
|
||||
host=host,
|
||||
query_port=query_port,
|
||||
game_port=game_port,
|
||||
source_name=source_name or DEFAULT_A2S_SOURCE_NAME,
|
||||
external_server_id=external_server_id,
|
||||
region=region,
|
||||
)
|
||||
|
||||
|
||||
def _string_or_none(value: object) -> str | None:
|
||||
if not isinstance(value, str):
|
||||
return None
|
||||
|
||||
normalized = value.strip()
|
||||
return normalized or None
|
||||
|
||||
|
||||
def _coerce_optional_positive_int(value: object) -> int | None:
|
||||
if value is None:
|
||||
return None
|
||||
|
||||
coerced = int(value)
|
||||
if coerced <= 0:
|
||||
raise ValueError("Each A2S target game_port must be positive when defined.")
|
||||
|
||||
return coerced
|
||||
54
backend/app/snapshots.py
Normal file
54
backend/app/snapshots.py
Normal file
@@ -0,0 +1,54 @@
|
||||
"""Snapshot builders for normalized provisional server data."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timezone
|
||||
from typing import Iterable, Mapping
|
||||
|
||||
|
||||
def build_server_snapshot(
|
||||
normalized_record: Mapping[str, object],
|
||||
*,
|
||||
captured_at: datetime,
|
||||
) -> dict[str, object]:
|
||||
"""Build a consistent snapshot payload for one normalized server."""
|
||||
timestamp = _as_utc_timestamp(captured_at)
|
||||
return {
|
||||
"external_server_id": normalized_record.get("external_server_id"),
|
||||
"server_name": normalized_record.get("server_name"),
|
||||
"status": normalized_record.get("status"),
|
||||
"players": normalized_record.get("players"),
|
||||
"max_players": normalized_record.get("max_players"),
|
||||
"current_map": normalized_record.get("current_map"),
|
||||
"region": normalized_record.get("region"),
|
||||
"source_name": normalized_record.get("source_name"),
|
||||
"snapshot_origin": normalized_record.get("snapshot_origin"),
|
||||
"source_ref": normalized_record.get("source_ref"),
|
||||
"captured_at": timestamp,
|
||||
}
|
||||
|
||||
|
||||
def build_snapshot_batch(
|
||||
normalized_records: Iterable[Mapping[str, object]],
|
||||
*,
|
||||
captured_at: datetime,
|
||||
) -> list[dict[str, object]]:
|
||||
"""Build snapshots for a batch captured at the same timestamp."""
|
||||
return [
|
||||
build_server_snapshot(record, captured_at=captured_at)
|
||||
for record in normalized_records
|
||||
]
|
||||
|
||||
|
||||
def utc_now() -> datetime:
|
||||
"""Return the current UTC timestamp for snapshot capture."""
|
||||
return datetime.now(timezone.utc)
|
||||
|
||||
|
||||
def _as_utc_timestamp(value: datetime) -> str:
|
||||
if value.tzinfo is None:
|
||||
value = value.replace(tzinfo=timezone.utc)
|
||||
else:
|
||||
value = value.astimezone(timezone.utc)
|
||||
|
||||
return value.isoformat().replace("+00:00", "Z")
|
||||
368
backend/app/sqlite_to_postgres_migration.py
Normal file
368
backend/app/sqlite_to_postgres_migration.py
Normal file
@@ -0,0 +1,368 @@
|
||||
"""Idempotent phase-2 migration from displayed SQLite/files into PostgreSQL."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sqlite3
|
||||
from collections import defaultdict
|
||||
from contextlib import closing
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from .config import get_storage_path
|
||||
from .postgres_display_storage import (
|
||||
connect_postgres as connect_display_postgres,
|
||||
initialize_postgres_display_storage,
|
||||
persist_snapshot_record,
|
||||
)
|
||||
from .postgres_rcon_storage import initialize_postgres_rcon_storage
|
||||
|
||||
|
||||
RCON_TABLES = (
|
||||
"rcon_historical_targets",
|
||||
"rcon_historical_capture_runs",
|
||||
"rcon_historical_samples",
|
||||
"rcon_historical_checkpoints",
|
||||
"rcon_historical_competitive_windows",
|
||||
"rcon_admin_log_events",
|
||||
"rcon_player_profile_snapshots",
|
||||
"rcon_materialized_matches",
|
||||
"rcon_match_player_stats",
|
||||
"rcon_scoreboard_match_candidates",
|
||||
)
|
||||
DISPLAY_TABLES = (
|
||||
"game_sources",
|
||||
"servers",
|
||||
"server_snapshots",
|
||||
"historical_servers",
|
||||
"historical_maps",
|
||||
"historical_matches",
|
||||
"historical_players",
|
||||
"historical_player_match_stats",
|
||||
"player_event_raw_ledger",
|
||||
)
|
||||
SKIP_SLUG = "comunidad-hispana-03"
|
||||
|
||||
|
||||
def migrate_sqlite_to_postgres() -> dict[str, object]:
|
||||
"""Copy displayed legacy data to PostgreSQL without deleting legacy sources."""
|
||||
initialize_postgres_rcon_storage()
|
||||
initialize_postgres_display_storage()
|
||||
summary: dict[str, object] = {
|
||||
"status": "ok",
|
||||
"source_paths": [],
|
||||
"migrated_tables": [],
|
||||
"migrated_domains": [],
|
||||
"rows_read": {},
|
||||
"rows_inserted": {},
|
||||
"rows_updated": {},
|
||||
"rows_skipped": {},
|
||||
"errors": [],
|
||||
}
|
||||
table_totals: dict[str, dict[str, int]] = defaultdict(
|
||||
lambda: {"read": 0, "inserted": 0, "updated": 0, "skipped": 0}
|
||||
)
|
||||
for db_path in _discover_sqlite_paths():
|
||||
summary["source_paths"].append(str(db_path))
|
||||
try:
|
||||
_migrate_sqlite_path(db_path, table_totals)
|
||||
except Exception as error: # noqa: BLE001 - report all source failures
|
||||
summary["errors"].append({"source_path": str(db_path), "error": str(error)})
|
||||
|
||||
snapshots_root = get_storage_path().parent / "snapshots"
|
||||
if snapshots_root.exists():
|
||||
summary["source_paths"].append(str(snapshots_root))
|
||||
_migrate_snapshot_files(snapshots_root, table_totals, summary["errors"])
|
||||
_sync_sequences()
|
||||
summary["migrated_tables"] = sorted(table_totals)
|
||||
summary["migrated_domains"] = [
|
||||
"rcon-admin-log-events",
|
||||
"rcon-player-profile-snapshots",
|
||||
"rcon-historical-capture-samples-and-windows",
|
||||
"rcon-materialized-matches",
|
||||
"rcon-materialized-player-stats",
|
||||
"rcon-safe-scoreboard-candidates",
|
||||
"public-scoreboard-historical-matches-and-player-stats",
|
||||
"weekly-and-monthly-scoreboard-rankings",
|
||||
"displayed-historical-snapshots",
|
||||
"live-server-summary-cache",
|
||||
"player-event-ledger",
|
||||
]
|
||||
for table_name, totals in sorted(table_totals.items()):
|
||||
summary["rows_read"][table_name] = totals["read"]
|
||||
summary["rows_inserted"][table_name] = totals["inserted"]
|
||||
summary["rows_updated"][table_name] = totals["updated"]
|
||||
summary["rows_skipped"][table_name] = totals["skipped"]
|
||||
summary["status"] = "ok" if not summary["errors"] else "completed-with-errors"
|
||||
return summary
|
||||
|
||||
|
||||
def _migrate_sqlite_path(db_path: Path, totals: dict[str, dict[str, int]]) -> None:
|
||||
with closing(sqlite3.connect(db_path)) as sqlite_connection:
|
||||
sqlite_connection.row_factory = sqlite3.Row
|
||||
available_tables = {
|
||||
row["name"]
|
||||
for row in sqlite_connection.execute(
|
||||
"SELECT name FROM sqlite_master WHERE type = 'table'"
|
||||
).fetchall()
|
||||
}
|
||||
tables = [table for table in (*RCON_TABLES, *DISPLAY_TABLES) if table in available_tables]
|
||||
with connect_display_postgres() as postgres_connection:
|
||||
postgres_columns = {
|
||||
table: _postgres_columns(postgres_connection, table)
|
||||
for table in tables
|
||||
}
|
||||
historical_server_ids = _legacy_server03_ids(sqlite_connection)
|
||||
historical_match_ids = _legacy_match_ids(sqlite_connection, historical_server_ids)
|
||||
legacy_rcon_target_ids = _legacy_rcon_target03_ids(sqlite_connection)
|
||||
for table_name in tables:
|
||||
_copy_table(
|
||||
sqlite_connection,
|
||||
postgres_connection,
|
||||
table_name=table_name,
|
||||
postgres_columns=postgres_columns[table_name],
|
||||
totals=totals[table_name],
|
||||
historical_server_ids=historical_server_ids,
|
||||
historical_match_ids=historical_match_ids,
|
||||
legacy_rcon_target_ids=legacy_rcon_target_ids,
|
||||
)
|
||||
|
||||
|
||||
def _copy_table(
|
||||
sqlite_connection: sqlite3.Connection,
|
||||
postgres_connection: Any,
|
||||
*,
|
||||
table_name: str,
|
||||
postgres_columns: list[str],
|
||||
totals: dict[str, int],
|
||||
historical_server_ids: set[int],
|
||||
historical_match_ids: set[int],
|
||||
legacy_rcon_target_ids: set[int],
|
||||
) -> None:
|
||||
sqlite_columns = [
|
||||
str(row["name"])
|
||||
for row in sqlite_connection.execute(f"PRAGMA table_info({table_name})").fetchall()
|
||||
]
|
||||
columns = [column for column in sqlite_columns if column in postgres_columns]
|
||||
if not columns:
|
||||
return
|
||||
rows = sqlite_connection.execute(
|
||||
f"SELECT {', '.join(columns)} FROM {table_name}"
|
||||
).fetchall()
|
||||
placeholders = ", ".join(["%s"] * len(columns))
|
||||
sql = (
|
||||
f"INSERT INTO {table_name} ({', '.join(columns)}) "
|
||||
f"VALUES ({placeholders}) ON CONFLICT DO NOTHING"
|
||||
)
|
||||
values: list[tuple[object, ...]] = []
|
||||
for row in rows:
|
||||
totals["read"] += 1
|
||||
row_dict = dict(row)
|
||||
if _skip_row(
|
||||
table_name,
|
||||
row_dict,
|
||||
historical_server_ids=historical_server_ids,
|
||||
historical_match_ids=historical_match_ids,
|
||||
legacy_rcon_target_ids=legacy_rcon_target_ids,
|
||||
):
|
||||
totals["skipped"] += 1
|
||||
continue
|
||||
values.append(tuple(_postgres_value(column, row_dict[column]) for column in columns))
|
||||
with postgres_connection.cursor() as cursor:
|
||||
for start in range(0, len(values), 1000):
|
||||
batch = values[start : start + 1000]
|
||||
cursor.executemany(sql, batch)
|
||||
inserted = max(0, int(cursor.rowcount or 0))
|
||||
totals["inserted"] += inserted
|
||||
totals["skipped"] += len(batch) - inserted
|
||||
|
||||
|
||||
def _migrate_snapshot_files(
|
||||
snapshots_root: Path,
|
||||
totals: dict[str, dict[str, int]],
|
||||
errors: list[object],
|
||||
) -> None:
|
||||
snapshot_totals = totals["displayed_historical_snapshots"]
|
||||
for snapshot_path in sorted(snapshots_root.glob("*/*.json")):
|
||||
snapshot_totals["read"] += 1
|
||||
try:
|
||||
document = json.loads(snapshot_path.read_text(encoding="utf-8"))
|
||||
if str(document.get("server_key") or "") == SKIP_SLUG:
|
||||
snapshot_totals["skipped"] += 1
|
||||
continue
|
||||
before = _snapshot_exists(document)
|
||||
persist_snapshot_record(document)
|
||||
snapshot_totals["updated" if before else "inserted"] += 1
|
||||
except Exception as error: # noqa: BLE001 - keep migrating neighboring snapshots
|
||||
snapshot_totals["skipped"] += 1
|
||||
errors.append({"source_path": str(snapshot_path), "error": str(error)})
|
||||
|
||||
|
||||
def _snapshot_exists(document: dict[str, object]) -> bool:
|
||||
with connect_display_postgres() as connection:
|
||||
row = connection.execute(
|
||||
"""
|
||||
SELECT 1 FROM displayed_historical_snapshots
|
||||
WHERE server_key = %s AND snapshot_type = %s AND metric = %s AND snapshot_window = %s
|
||||
""",
|
||||
(
|
||||
str(document.get("server_key") or ""),
|
||||
str(document.get("snapshot_type") or ""),
|
||||
str(document.get("metric") or ""),
|
||||
str(document.get("window") or ""),
|
||||
),
|
||||
).fetchone()
|
||||
return bool(row)
|
||||
|
||||
|
||||
def _skip_row(
|
||||
table_name: str,
|
||||
row: dict[str, object],
|
||||
*,
|
||||
historical_server_ids: set[int],
|
||||
historical_match_ids: set[int],
|
||||
legacy_rcon_target_ids: set[int],
|
||||
) -> bool:
|
||||
if row.get("server_slug") == SKIP_SLUG or row.get("slug") == SKIP_SLUG:
|
||||
return True
|
||||
if row.get("external_server_id") == SKIP_SLUG or row.get("target_key") == SKIP_SLUG:
|
||||
return True
|
||||
if table_name == "historical_matches" and row.get("historical_server_id") in historical_server_ids:
|
||||
return True
|
||||
if (
|
||||
table_name == "historical_player_match_stats"
|
||||
and row.get("historical_match_id") in historical_match_ids
|
||||
):
|
||||
return True
|
||||
if table_name == "rcon_historical_samples" and row.get("target_id") in legacy_rcon_target_ids:
|
||||
return True
|
||||
if table_name == "rcon_historical_checkpoints" and row.get("target_id") in legacy_rcon_target_ids:
|
||||
return True
|
||||
if table_name == "rcon_historical_competitive_windows" and row.get("target_id") in legacy_rcon_target_ids:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def _legacy_server03_ids(connection: sqlite3.Connection) -> set[int]:
|
||||
if not _has_table(connection, "historical_servers"):
|
||||
return set()
|
||||
return {
|
||||
int(row["id"])
|
||||
for row in connection.execute(
|
||||
"SELECT id FROM historical_servers WHERE slug = ?",
|
||||
(SKIP_SLUG,),
|
||||
).fetchall()
|
||||
}
|
||||
|
||||
|
||||
def _legacy_rcon_target03_ids(connection: sqlite3.Connection) -> set[int]:
|
||||
if not _has_table(connection, "rcon_historical_targets"):
|
||||
return set()
|
||||
return {
|
||||
int(row["id"])
|
||||
for row in connection.execute(
|
||||
"""
|
||||
SELECT id FROM rcon_historical_targets
|
||||
WHERE external_server_id = ? OR target_key = ?
|
||||
""",
|
||||
(SKIP_SLUG, SKIP_SLUG),
|
||||
).fetchall()
|
||||
}
|
||||
|
||||
|
||||
def _legacy_match_ids(connection: sqlite3.Connection, historical_server_ids: set[int]) -> set[int]:
|
||||
if not historical_server_ids or not _has_table(connection, "historical_matches"):
|
||||
return set()
|
||||
placeholders = ", ".join(["?"] * len(historical_server_ids))
|
||||
return {
|
||||
int(row["id"])
|
||||
for row in connection.execute(
|
||||
f"SELECT id FROM historical_matches WHERE historical_server_id IN ({placeholders})",
|
||||
tuple(sorted(historical_server_ids)),
|
||||
).fetchall()
|
||||
}
|
||||
|
||||
|
||||
def _postgres_columns(connection: Any, table_name: str) -> list[str]:
|
||||
rows = connection.execute(
|
||||
"""
|
||||
SELECT column_name
|
||||
FROM information_schema.columns
|
||||
WHERE table_schema = 'public' AND table_name = %s
|
||||
ORDER BY ordinal_position
|
||||
""",
|
||||
(table_name,),
|
||||
).fetchall()
|
||||
return [str(row["column_name"]) for row in rows]
|
||||
|
||||
|
||||
def _sync_sequences() -> None:
|
||||
tables = (
|
||||
"game_sources",
|
||||
"servers",
|
||||
"server_snapshots",
|
||||
"historical_servers",
|
||||
"historical_maps",
|
||||
"historical_matches",
|
||||
"historical_players",
|
||||
"historical_player_match_stats",
|
||||
"player_event_raw_ledger",
|
||||
"rcon_historical_targets",
|
||||
"rcon_historical_capture_runs",
|
||||
"rcon_historical_samples",
|
||||
"rcon_historical_competitive_windows",
|
||||
"rcon_admin_log_events",
|
||||
"rcon_player_profile_snapshots",
|
||||
"rcon_materialized_matches",
|
||||
"rcon_match_player_stats",
|
||||
"rcon_scoreboard_match_candidates",
|
||||
)
|
||||
with connect_display_postgres() as connection:
|
||||
for table_name in tables:
|
||||
connection.execute(
|
||||
f"""
|
||||
SELECT setval(
|
||||
pg_get_serial_sequence(%s, 'id'),
|
||||
GREATEST(COALESCE((SELECT MAX(id) FROM {table_name}), 1), 1),
|
||||
TRUE
|
||||
)
|
||||
""",
|
||||
(table_name,),
|
||||
)
|
||||
|
||||
|
||||
def _discover_sqlite_paths() -> list[Path]:
|
||||
configured = get_storage_path()
|
||||
candidates = {configured}
|
||||
if configured.parent.exists():
|
||||
candidates.update(configured.parent.glob("*.sqlite*"))
|
||||
return sorted(
|
||||
path
|
||||
for path in candidates
|
||||
if path.exists()
|
||||
and path.is_file()
|
||||
and not str(path).endswith(("-shm", "-wal"))
|
||||
)
|
||||
|
||||
|
||||
def _has_table(connection: sqlite3.Connection, table_name: str) -> bool:
|
||||
return bool(
|
||||
connection.execute(
|
||||
"SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ?",
|
||||
(table_name,),
|
||||
).fetchone()
|
||||
)
|
||||
|
||||
|
||||
def _postgres_value(column: str, value: object) -> object:
|
||||
if column in {"is_active", "is_teamkill"}:
|
||||
return bool(value)
|
||||
return value
|
||||
|
||||
|
||||
def main() -> None:
|
||||
print(json.dumps(migrate_sqlite_to_postgres(), ensure_ascii=False, indent=2))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
41
backend/app/sqlite_utils.py
Normal file
41
backend/app/sqlite_utils.py
Normal file
@@ -0,0 +1,41 @@
|
||||
"""Shared SQLite connection helpers for backend persistence layers."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlite3
|
||||
from pathlib import Path
|
||||
|
||||
from .config import get_sqlite_busy_timeout_ms, get_sqlite_writer_timeout_seconds
|
||||
|
||||
|
||||
def connect_sqlite_writer(
|
||||
db_path: Path,
|
||||
*,
|
||||
timeout_seconds: float | None = None,
|
||||
busy_timeout_ms: int | None = None,
|
||||
) -> sqlite3.Connection:
|
||||
"""Open one SQLite connection with the common writer policy."""
|
||||
resolved_timeout_seconds = (
|
||||
get_sqlite_writer_timeout_seconds()
|
||||
if timeout_seconds is None
|
||||
else timeout_seconds
|
||||
)
|
||||
resolved_busy_timeout_ms = (
|
||||
get_sqlite_busy_timeout_ms()
|
||||
if busy_timeout_ms is None
|
||||
else busy_timeout_ms
|
||||
)
|
||||
|
||||
connection = sqlite3.connect(db_path, timeout=resolved_timeout_seconds)
|
||||
connection.row_factory = sqlite3.Row
|
||||
connection.execute("PRAGMA foreign_keys = ON")
|
||||
connection.execute("PRAGMA journal_mode = WAL")
|
||||
connection.execute(f"PRAGMA busy_timeout = {resolved_busy_timeout_ms}")
|
||||
return connection
|
||||
|
||||
|
||||
def connect_sqlite_readonly(db_path: Path) -> sqlite3.Connection:
|
||||
"""Open one read-only SQLite connection with row access enabled."""
|
||||
connection = sqlite3.connect(f"file:{db_path}?mode=ro", uri=True)
|
||||
connection.row_factory = sqlite3.Row
|
||||
return connection
|
||||
549
backend/app/storage.py
Normal file
549
backend/app/storage.py
Normal file
@@ -0,0 +1,549 @@
|
||||
"""Local SQLite persistence for provisional server snapshots."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlite3
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Iterable, Mapping
|
||||
|
||||
from .config import get_storage_path, use_postgres_rcon_storage
|
||||
from .sqlite_utils import connect_sqlite_readonly, connect_sqlite_writer
|
||||
|
||||
|
||||
DEFAULT_GAME_SOURCE = {
|
||||
"slug": "current-hll",
|
||||
"display_name": "Current Hell Let Loose",
|
||||
"provider_kind": "development",
|
||||
}
|
||||
SUMMARY_SNAPSHOT_LIMIT = 6
|
||||
|
||||
|
||||
def resolve_storage_path(*, db_path: Path | None = None) -> Path:
|
||||
"""Resolve the SQLite path used by live snapshot persistence."""
|
||||
return db_path or get_storage_path()
|
||||
|
||||
|
||||
def initialize_storage(*, db_path: Path | None = None) -> Path:
|
||||
"""Create the local database file and minimal schema when missing."""
|
||||
resolved_path = resolve_storage_path(db_path=db_path)
|
||||
resolved_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
with _connect(resolved_path) as connection:
|
||||
connection.executescript(
|
||||
"""
|
||||
CREATE TABLE IF NOT EXISTS game_sources (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
slug TEXT NOT NULL UNIQUE,
|
||||
display_name TEXT NOT NULL,
|
||||
provider_kind TEXT NOT NULL,
|
||||
is_active INTEGER NOT NULL DEFAULT 1,
|
||||
created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS servers (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
game_source_id INTEGER NOT NULL,
|
||||
external_server_id TEXT,
|
||||
server_name TEXT NOT NULL,
|
||||
region TEXT,
|
||||
first_seen_at TEXT NOT NULL,
|
||||
last_seen_at TEXT NOT NULL,
|
||||
created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
UNIQUE (game_source_id, external_server_id),
|
||||
FOREIGN KEY (game_source_id) REFERENCES game_sources(id)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS server_snapshots (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
server_id INTEGER NOT NULL,
|
||||
captured_at TEXT NOT NULL,
|
||||
status TEXT NOT NULL,
|
||||
players INTEGER,
|
||||
max_players INTEGER,
|
||||
current_map TEXT,
|
||||
source_name TEXT NOT NULL,
|
||||
snapshot_origin TEXT,
|
||||
source_ref TEXT,
|
||||
raw_payload_ref TEXT,
|
||||
created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
FOREIGN KEY (server_id) REFERENCES servers(id)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_server_snapshots_server_time
|
||||
ON server_snapshots(server_id, captured_at);
|
||||
"""
|
||||
)
|
||||
_ensure_server_snapshot_columns(connection)
|
||||
|
||||
return resolved_path
|
||||
|
||||
|
||||
def persist_snapshot_batch(
|
||||
snapshots: Iterable[Mapping[str, object]],
|
||||
*,
|
||||
source_name: str,
|
||||
captured_at: str,
|
||||
game_source: Mapping[str, str] | None = None,
|
||||
db_path: Path | None = None,
|
||||
) -> dict[str, object]:
|
||||
"""Persist a batch of normalized snapshots into local SQLite storage."""
|
||||
source_definition = dict(DEFAULT_GAME_SOURCE)
|
||||
if game_source is not None:
|
||||
source_definition.update(game_source)
|
||||
if use_postgres_rcon_storage(explicit_sqlite_path=db_path):
|
||||
from .postgres_display_storage import persist_server_snapshots
|
||||
|
||||
return persist_server_snapshots(
|
||||
snapshots,
|
||||
source_name=source_name,
|
||||
captured_at=captured_at,
|
||||
game_source=source_definition,
|
||||
)
|
||||
resolved_path = initialize_storage(db_path=db_path)
|
||||
|
||||
persisted = 0
|
||||
with _connect(resolved_path) as connection:
|
||||
game_source_id = _upsert_game_source(connection, source_definition)
|
||||
for snapshot in snapshots:
|
||||
server_id = _upsert_server(
|
||||
connection,
|
||||
game_source_id=game_source_id,
|
||||
snapshot=snapshot,
|
||||
captured_at=captured_at,
|
||||
)
|
||||
connection.execute(
|
||||
"""
|
||||
INSERT INTO server_snapshots (
|
||||
server_id,
|
||||
captured_at,
|
||||
status,
|
||||
players,
|
||||
max_players,
|
||||
current_map,
|
||||
source_name,
|
||||
snapshot_origin,
|
||||
source_ref,
|
||||
raw_payload_ref
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
""",
|
||||
(
|
||||
server_id,
|
||||
captured_at,
|
||||
snapshot.get("status"),
|
||||
snapshot.get("players"),
|
||||
snapshot.get("max_players"),
|
||||
snapshot.get("current_map"),
|
||||
snapshot.get("source_name") or source_name,
|
||||
snapshot.get("snapshot_origin"),
|
||||
snapshot.get("source_ref"),
|
||||
None,
|
||||
),
|
||||
)
|
||||
persisted += 1
|
||||
|
||||
return {
|
||||
"db_path": str(resolved_path),
|
||||
"captured_at": captured_at,
|
||||
"persisted_snapshots": persisted,
|
||||
"game_source_slug": source_definition["slug"],
|
||||
}
|
||||
|
||||
|
||||
def list_latest_snapshots(*, db_path: Path | None = None) -> list[dict[str, object]]:
|
||||
"""Return the latest persisted snapshot for each known server."""
|
||||
if use_postgres_rcon_storage(explicit_sqlite_path=db_path):
|
||||
from .postgres_display_storage import list_latest_server_snapshots
|
||||
|
||||
return list_latest_server_snapshots()
|
||||
resolved_path = resolve_storage_path(db_path=db_path)
|
||||
if not resolved_path.exists():
|
||||
return []
|
||||
with _connect_readonly(resolved_path) as connection:
|
||||
rows = connection.execute(
|
||||
"""
|
||||
SELECT
|
||||
servers.id AS server_id,
|
||||
servers.external_server_id,
|
||||
servers.server_name,
|
||||
servers.region,
|
||||
game_sources.slug AS context,
|
||||
server_snapshots.source_name,
|
||||
server_snapshots.snapshot_origin,
|
||||
server_snapshots.source_ref,
|
||||
server_snapshots.captured_at,
|
||||
server_snapshots.status,
|
||||
server_snapshots.players,
|
||||
server_snapshots.max_players,
|
||||
server_snapshots.current_map
|
||||
FROM servers
|
||||
INNER JOIN game_sources
|
||||
ON game_sources.id = servers.game_source_id
|
||||
INNER JOIN server_snapshots
|
||||
ON server_snapshots.server_id = servers.id
|
||||
INNER JOIN (
|
||||
SELECT server_id, MAX(captured_at) AS latest_captured_at
|
||||
FROM server_snapshots
|
||||
GROUP BY server_id
|
||||
) AS latest
|
||||
ON latest.server_id = server_snapshots.server_id
|
||||
AND latest.latest_captured_at = server_snapshots.captured_at
|
||||
ORDER BY servers.server_name ASC
|
||||
"""
|
||||
).fetchall()
|
||||
items = [_serialize_snapshot_row(row) for row in rows]
|
||||
return _attach_history_summaries(connection, items)
|
||||
|
||||
|
||||
def list_snapshot_history(
|
||||
*,
|
||||
db_path: Path | None = None,
|
||||
limit: int = 20,
|
||||
) -> list[dict[str, object]]:
|
||||
"""Return recent persisted snapshots across all servers."""
|
||||
if use_postgres_rcon_storage(explicit_sqlite_path=db_path):
|
||||
from .postgres_display_storage import list_server_snapshot_history
|
||||
|
||||
return list_server_snapshot_history(limit=limit)
|
||||
resolved_path = resolve_storage_path(db_path=db_path)
|
||||
if not resolved_path.exists():
|
||||
return []
|
||||
with _connect_readonly(resolved_path) as connection:
|
||||
rows = connection.execute(
|
||||
"""
|
||||
SELECT
|
||||
servers.id AS server_id,
|
||||
servers.external_server_id,
|
||||
servers.server_name,
|
||||
servers.region,
|
||||
game_sources.slug AS context,
|
||||
server_snapshots.source_name,
|
||||
server_snapshots.snapshot_origin,
|
||||
server_snapshots.source_ref,
|
||||
server_snapshots.captured_at,
|
||||
server_snapshots.status,
|
||||
server_snapshots.players,
|
||||
server_snapshots.max_players,
|
||||
server_snapshots.current_map
|
||||
FROM server_snapshots
|
||||
INNER JOIN servers
|
||||
ON servers.id = server_snapshots.server_id
|
||||
INNER JOIN game_sources
|
||||
ON game_sources.id = servers.game_source_id
|
||||
ORDER BY server_snapshots.captured_at DESC, servers.server_name ASC
|
||||
LIMIT ?
|
||||
""",
|
||||
(limit,),
|
||||
).fetchall()
|
||||
return [_serialize_snapshot_row(row) for row in rows]
|
||||
|
||||
|
||||
def list_server_history(
|
||||
server_id: str,
|
||||
*,
|
||||
db_path: Path | None = None,
|
||||
limit: int = 20,
|
||||
) -> list[dict[str, object]]:
|
||||
"""Return recent history for one server by numeric id or external id."""
|
||||
if use_postgres_rcon_storage(explicit_sqlite_path=db_path):
|
||||
from .postgres_display_storage import list_server_snapshot_history
|
||||
|
||||
return list_server_snapshot_history(server_id=server_id, limit=limit)
|
||||
resolved_path = resolve_storage_path(db_path=db_path)
|
||||
if not resolved_path.exists():
|
||||
return []
|
||||
server_filter, server_value = _build_server_filter(server_id)
|
||||
with _connect_readonly(resolved_path) as connection:
|
||||
rows = connection.execute(
|
||||
f"""
|
||||
SELECT
|
||||
servers.id AS server_id,
|
||||
servers.external_server_id,
|
||||
servers.server_name,
|
||||
servers.region,
|
||||
game_sources.slug AS context,
|
||||
server_snapshots.source_name,
|
||||
server_snapshots.snapshot_origin,
|
||||
server_snapshots.source_ref,
|
||||
server_snapshots.captured_at,
|
||||
server_snapshots.status,
|
||||
server_snapshots.players,
|
||||
server_snapshots.max_players,
|
||||
server_snapshots.current_map
|
||||
FROM server_snapshots
|
||||
INNER JOIN servers
|
||||
ON servers.id = server_snapshots.server_id
|
||||
INNER JOIN game_sources
|
||||
ON game_sources.id = servers.game_source_id
|
||||
WHERE {server_filter} = ?
|
||||
ORDER BY server_snapshots.captured_at DESC
|
||||
LIMIT ?
|
||||
""",
|
||||
(server_value, limit),
|
||||
).fetchall()
|
||||
return [_serialize_snapshot_row(row) for row in rows]
|
||||
|
||||
|
||||
def _connect(db_path: Path) -> sqlite3.Connection:
|
||||
return connect_sqlite_writer(db_path)
|
||||
|
||||
|
||||
def _connect_readonly(db_path: Path) -> sqlite3.Connection:
|
||||
return connect_sqlite_readonly(db_path)
|
||||
|
||||
|
||||
def _upsert_game_source(
|
||||
connection: sqlite3.Connection,
|
||||
game_source: Mapping[str, str],
|
||||
) -> int:
|
||||
connection.execute(
|
||||
"""
|
||||
INSERT INTO game_sources (slug, display_name, provider_kind, is_active)
|
||||
VALUES (?, ?, ?, 1)
|
||||
ON CONFLICT(slug) DO UPDATE SET
|
||||
display_name = excluded.display_name,
|
||||
provider_kind = excluded.provider_kind,
|
||||
is_active = 1,
|
||||
updated_at = CURRENT_TIMESTAMP
|
||||
""",
|
||||
(
|
||||
game_source["slug"],
|
||||
game_source["display_name"],
|
||||
game_source["provider_kind"],
|
||||
),
|
||||
)
|
||||
row = connection.execute(
|
||||
"SELECT id FROM game_sources WHERE slug = ?",
|
||||
(game_source["slug"],),
|
||||
).fetchone()
|
||||
if row is None:
|
||||
raise RuntimeError("Failed to resolve game source during snapshot persistence.")
|
||||
|
||||
return int(row["id"])
|
||||
|
||||
|
||||
def _upsert_server(
|
||||
connection: sqlite3.Connection,
|
||||
*,
|
||||
game_source_id: int,
|
||||
snapshot: Mapping[str, object],
|
||||
captured_at: str,
|
||||
) -> int:
|
||||
external_server_id = snapshot.get("external_server_id")
|
||||
if not isinstance(external_server_id, str) or not external_server_id.strip():
|
||||
external_server_id = _build_fallback_external_id(snapshot)
|
||||
|
||||
server_name = str(snapshot.get("server_name") or "Unknown server")
|
||||
region = snapshot.get("region")
|
||||
|
||||
connection.execute(
|
||||
"""
|
||||
INSERT INTO servers (
|
||||
game_source_id,
|
||||
external_server_id,
|
||||
server_name,
|
||||
region,
|
||||
first_seen_at,
|
||||
last_seen_at
|
||||
) VALUES (?, ?, ?, ?, ?, ?)
|
||||
ON CONFLICT(game_source_id, external_server_id) DO UPDATE SET
|
||||
server_name = excluded.server_name,
|
||||
region = excluded.region,
|
||||
last_seen_at = excluded.last_seen_at,
|
||||
updated_at = CURRENT_TIMESTAMP
|
||||
""",
|
||||
(
|
||||
game_source_id,
|
||||
external_server_id,
|
||||
server_name,
|
||||
region,
|
||||
captured_at,
|
||||
captured_at,
|
||||
),
|
||||
)
|
||||
row = connection.execute(
|
||||
"""
|
||||
SELECT id
|
||||
FROM servers
|
||||
WHERE game_source_id = ? AND external_server_id = ?
|
||||
""",
|
||||
(game_source_id, external_server_id),
|
||||
).fetchone()
|
||||
if row is None:
|
||||
raise RuntimeError("Failed to resolve server during snapshot persistence.")
|
||||
|
||||
return int(row["id"])
|
||||
|
||||
|
||||
def _build_fallback_external_id(snapshot: Mapping[str, object]) -> str:
|
||||
server_name = str(snapshot.get("server_name") or "unknown-server")
|
||||
normalized = "".join(
|
||||
character.lower() if character.isalnum() else "-"
|
||||
for character in server_name
|
||||
)
|
||||
compact = "-".join(part for part in normalized.split("-") if part)
|
||||
return compact or "unknown-server"
|
||||
|
||||
|
||||
def _ensure_server_snapshot_columns(connection: sqlite3.Connection) -> None:
|
||||
columns = {
|
||||
str(row["name"])
|
||||
for row in connection.execute("PRAGMA table_info(server_snapshots)").fetchall()
|
||||
}
|
||||
|
||||
if "snapshot_origin" not in columns:
|
||||
connection.execute("ALTER TABLE server_snapshots ADD COLUMN snapshot_origin TEXT")
|
||||
if "source_ref" not in columns:
|
||||
connection.execute("ALTER TABLE server_snapshots ADD COLUMN source_ref TEXT")
|
||||
|
||||
connection.execute(
|
||||
"""
|
||||
UPDATE server_snapshots
|
||||
SET snapshot_origin = CASE
|
||||
WHEN source_name = 'controlled-placeholder' THEN 'controlled-fallback'
|
||||
WHEN source_name LIKE '%a2s%' THEN 'real-a2s'
|
||||
ELSE 'unknown'
|
||||
END
|
||||
WHERE snapshot_origin IS NULL OR snapshot_origin = ''
|
||||
"""
|
||||
)
|
||||
connection.execute(
|
||||
"""
|
||||
UPDATE server_snapshots
|
||||
SET source_ref = source_name
|
||||
WHERE source_ref IS NULL OR source_ref = ''
|
||||
"""
|
||||
)
|
||||
_backfill_registered_a2s_source_refs(connection)
|
||||
|
||||
|
||||
def _backfill_registered_a2s_source_refs(connection: sqlite3.Connection) -> None:
|
||||
from .server_targets import load_a2s_targets
|
||||
|
||||
for target in load_a2s_targets():
|
||||
if not target.external_server_id:
|
||||
continue
|
||||
|
||||
connection.execute(
|
||||
"""
|
||||
UPDATE server_snapshots
|
||||
SET source_ref = ?
|
||||
WHERE snapshot_origin = 'real-a2s'
|
||||
AND source_ref = source_name
|
||||
AND server_id IN (
|
||||
SELECT id
|
||||
FROM servers
|
||||
WHERE external_server_id = ?
|
||||
)
|
||||
""",
|
||||
(
|
||||
f"a2s://{target.host}:{target.query_port}",
|
||||
target.external_server_id,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _serialize_snapshot_row(row: sqlite3.Row) -> dict[str, object]:
|
||||
return {
|
||||
"server_id": row["server_id"],
|
||||
"external_server_id": row["external_server_id"],
|
||||
"server_name": row["server_name"],
|
||||
"region": row["region"],
|
||||
"context": row["context"],
|
||||
"source_name": row["source_name"],
|
||||
"snapshot_origin": row["snapshot_origin"],
|
||||
"source_ref": row["source_ref"],
|
||||
"captured_at": row["captured_at"],
|
||||
"status": row["status"],
|
||||
"players": row["players"],
|
||||
"max_players": row["max_players"],
|
||||
"current_map": row["current_map"],
|
||||
}
|
||||
|
||||
|
||||
def _attach_history_summaries(
|
||||
connection: sqlite3.Connection,
|
||||
items: list[dict[str, object]],
|
||||
) -> list[dict[str, object]]:
|
||||
enriched_items: list[dict[str, object]] = []
|
||||
for item in items:
|
||||
enriched = dict(item)
|
||||
enriched["history_summary"] = _build_history_summary(
|
||||
connection,
|
||||
int(item["server_id"]),
|
||||
)
|
||||
enriched_items.append(enriched)
|
||||
|
||||
return enriched_items
|
||||
|
||||
|
||||
def _build_history_summary(
|
||||
connection: sqlite3.Connection,
|
||||
server_id: int,
|
||||
) -> dict[str, object]:
|
||||
rows = connection.execute(
|
||||
"""
|
||||
SELECT
|
||||
captured_at,
|
||||
status,
|
||||
players
|
||||
FROM server_snapshots
|
||||
WHERE server_id = ?
|
||||
ORDER BY captured_at DESC
|
||||
LIMIT ?
|
||||
""",
|
||||
(server_id, SUMMARY_SNAPSHOT_LIMIT),
|
||||
).fetchall()
|
||||
return _summarize_history_rows(rows)
|
||||
|
||||
|
||||
def _summarize_history_rows(rows: list[sqlite3.Row]) -> dict[str, object]:
|
||||
capture_count = len(rows)
|
||||
player_values = [
|
||||
int(row["players"])
|
||||
for row in rows
|
||||
if row["players"] is not None
|
||||
]
|
||||
online_rows = [row for row in rows if row["status"] == "online"]
|
||||
latest_captured_at = str(rows[0]["captured_at"]) if rows else None
|
||||
last_seen_online_at = str(online_rows[0]["captured_at"]) if online_rows else None
|
||||
|
||||
return {
|
||||
"window_size": SUMMARY_SNAPSHOT_LIMIT,
|
||||
"recent_capture_count": capture_count,
|
||||
"recent_online_count": len(online_rows),
|
||||
"recent_average_players": _round_average(player_values),
|
||||
"recent_peak_players": max(player_values, default=None),
|
||||
"last_seen_online_at": last_seen_online_at,
|
||||
"minutes_since_last_capture": _minutes_since_timestamp(latest_captured_at),
|
||||
}
|
||||
|
||||
|
||||
def _round_average(values: list[int]) -> float | None:
|
||||
if not values:
|
||||
return None
|
||||
|
||||
return round(sum(values) / len(values), 1)
|
||||
|
||||
|
||||
def _minutes_since_timestamp(timestamp: str | None) -> int | None:
|
||||
if not timestamp:
|
||||
return None
|
||||
|
||||
normalized = timestamp.replace("Z", "+00:00")
|
||||
captured_at = datetime.fromisoformat(normalized)
|
||||
if captured_at.tzinfo is None:
|
||||
captured_at = captured_at.replace(tzinfo=timezone.utc)
|
||||
|
||||
delta = datetime.now(timezone.utc) - captured_at.astimezone(timezone.utc)
|
||||
return max(0, int(delta.total_seconds() // 60))
|
||||
|
||||
|
||||
def _build_server_filter(server_id: str) -> tuple[str, object]:
|
||||
normalized = server_id.strip()
|
||||
if normalized.isdigit():
|
||||
return "servers.id", int(normalized)
|
||||
|
||||
return "servers.external_server_id", normalized
|
||||
164
backend/app/storage_diagnostics.py
Normal file
164
backend/app/storage_diagnostics.py
Normal file
@@ -0,0 +1,164 @@
|
||||
"""Report active PostgreSQL/displayed storage backend and migration parity counts."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sqlite3
|
||||
from contextlib import closing
|
||||
|
||||
from .config import get_database_url, get_storage_path, use_postgres_rcon_storage
|
||||
from .rcon_admin_log_materialization import summarize_rcon_materialization_status
|
||||
from .rcon_admin_log_storage import initialize_rcon_admin_log_storage
|
||||
from .sqlite_utils import connect_sqlite_readonly
|
||||
|
||||
|
||||
MIGRATED_RCON_TABLES = (
|
||||
"rcon_admin_log_events",
|
||||
"rcon_player_profile_snapshots",
|
||||
"rcon_materialized_matches",
|
||||
"rcon_match_player_stats",
|
||||
"rcon_historical_targets",
|
||||
"rcon_historical_samples",
|
||||
"rcon_historical_competitive_windows",
|
||||
"rcon_scoreboard_match_candidates",
|
||||
)
|
||||
|
||||
|
||||
def build_storage_diagnostics() -> dict[str, object]:
|
||||
"""Return one JSON-safe diagnostic payload for the migrated domains."""
|
||||
if use_postgres_rcon_storage():
|
||||
from .postgres_rcon_storage import count_migrated_tables
|
||||
from .postgres_display_storage import table_counts
|
||||
|
||||
rcon_counts = count_migrated_tables()
|
||||
displayed_counts = table_counts()
|
||||
backend = "postgresql"
|
||||
else:
|
||||
rcon_counts = _count_sqlite_tables()
|
||||
displayed_counts = {}
|
||||
backend = "sqlite-fallback"
|
||||
materialization = summarize_rcon_materialization_status()
|
||||
return {
|
||||
"active_storage_backend": backend,
|
||||
"database_url_configured": bool(get_database_url()),
|
||||
"sqlite_fallback_path": str(get_storage_path()),
|
||||
"migrated_domains": [
|
||||
"rcon-admin-log-events",
|
||||
"rcon-player-profile-snapshots",
|
||||
"rcon-historical-capture-samples-and-windows",
|
||||
"rcon-materialized-matches",
|
||||
"rcon-materialized-player-stats",
|
||||
"rcon-safe-scoreboard-candidates",
|
||||
"public-scoreboard-historical-matches-and-player-stats",
|
||||
"weekly-rankings",
|
||||
"monthly-rankings",
|
||||
"displayed-historical-snapshots",
|
||||
"server-summary-and-live-server-cache",
|
||||
"player-event-ledger",
|
||||
],
|
||||
"table_counts": {
|
||||
**rcon_counts,
|
||||
**displayed_counts,
|
||||
"admin_log_events": rcon_counts.get("rcon_admin_log_events", 0),
|
||||
"materialized_matches": rcon_counts.get("rcon_materialized_matches", 0),
|
||||
"player_stats": rcon_counts.get("rcon_match_player_stats", 0),
|
||||
"public_scoreboard_historical_matches": displayed_counts.get(
|
||||
"historical_matches", 0
|
||||
),
|
||||
"weekly_rankings_source_stats": displayed_counts.get(
|
||||
"historical_player_match_stats", 0
|
||||
),
|
||||
"monthly_rankings_source_stats": displayed_counts.get(
|
||||
"historical_player_match_stats", 0
|
||||
),
|
||||
"server_summary_cache": displayed_counts.get("displayed_historical_snapshots", 0),
|
||||
"player_event_ledger": displayed_counts.get("player_event_raw_ledger", 0),
|
||||
"scoreboard_candidates": rcon_counts.get("rcon_scoreboard_match_candidates", 0),
|
||||
},
|
||||
"latest_materialized_matches": materialization["latest_materialized_matches"],
|
||||
"latest_admin_log_match_end_events": materialization[
|
||||
"latest_admin_log_match_end_events"
|
||||
],
|
||||
"match_end_status": materialization["match_end_status"],
|
||||
"remaining_sqlite_or_file_backed_domains": [
|
||||
{
|
||||
"domain": "public-scoreboard ingestion run and backfill checkpoints",
|
||||
"displayed_in_frontend": False,
|
||||
"reason": "operational import bookkeeping is not read by visible pages",
|
||||
"planned_phase": "phase-3-or-when-scoreboard-import-runs-on-postgresql",
|
||||
},
|
||||
{
|
||||
"domain": "Elo/MMR tables",
|
||||
"displayed_in_frontend": False,
|
||||
"reason": "Elo/MMR remains paused and hidden from visible pages",
|
||||
"planned_phase": "phase-3",
|
||||
},
|
||||
],
|
||||
"sqlite_remaining": [
|
||||
"public-scoreboard ingestion run and backfill checkpoints",
|
||||
"paused Elo/MMR tables",
|
||||
],
|
||||
"scoreboard_correlation": "PostgreSQL safe candidates and migrated trusted historical match URLs are used.",
|
||||
"external_player_ids": _postgres_external_player_id_diagnostics()
|
||||
if backend == "postgresql"
|
||||
else {
|
||||
"available_in_postgresql": False,
|
||||
"reason": "PostgreSQL storage is not active.",
|
||||
},
|
||||
"migration_parity_summary": {
|
||||
"available": backend == "postgresql",
|
||||
"source_command": "python -m app.sqlite_to_postgres_migration",
|
||||
"displayed_historical_storage": (
|
||||
"postgresql" if backend == "postgresql" else "sqlite-or-file-fallback"
|
||||
),
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _count_sqlite_tables() -> dict[str, int]:
|
||||
resolved_path = initialize_rcon_admin_log_storage()
|
||||
counts: dict[str, int] = {}
|
||||
with closing(connect_sqlite_readonly(resolved_path)) as connection:
|
||||
for table_name in MIGRATED_RCON_TABLES:
|
||||
try:
|
||||
row = connection.execute(
|
||||
f"SELECT COUNT(*) AS count FROM {table_name}"
|
||||
).fetchone()
|
||||
except sqlite3.Error:
|
||||
counts[table_name] = 0
|
||||
else:
|
||||
counts[table_name] = int(row["count"] or 0)
|
||||
return counts
|
||||
|
||||
|
||||
def _postgres_external_player_id_diagnostics() -> dict[str, object]:
|
||||
from .postgres_rcon_storage import connect_postgres
|
||||
|
||||
with connect_postgres() as connection:
|
||||
row = connection.execute(
|
||||
"""
|
||||
SELECT
|
||||
(SELECT COUNT(*) FROM rcon_match_player_stats
|
||||
WHERE player_id ~ '^[0-9]{17}$') AS rcon_match_steam_id64_rows,
|
||||
(SELECT COUNT(*) FROM rcon_player_profile_snapshots
|
||||
WHERE player_id ~ '^[0-9]{17}$') AS rcon_profile_steam_id64_rows,
|
||||
(SELECT COUNT(*) FROM historical_players
|
||||
WHERE steam_id ~ '^[0-9]{17}$') AS scoreboard_player_steam_id64_rows
|
||||
"""
|
||||
).fetchone()
|
||||
return {
|
||||
"available_in_postgresql": True,
|
||||
"rcon_match_steam_id64_rows": int(row["rcon_match_steam_id64_rows"] or 0),
|
||||
"rcon_profile_steam_id64_rows": int(row["rcon_profile_steam_id64_rows"] or 0),
|
||||
"scoreboard_player_steam_id64_rows": int(
|
||||
row["scoreboard_player_steam_id64_rows"] or 0
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def main() -> None:
|
||||
print(json.dumps(build_storage_diagnostics(), ensure_ascii=False, indent=2, default=str))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
255
backend/app/writer_lock.py
Normal file
255
backend/app/writer_lock.py
Normal file
@@ -0,0 +1,255 @@
|
||||
"""Shared single-writer lock coordination for backend automation jobs."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import socket
|
||||
import sys
|
||||
import time
|
||||
from contextlib import contextmanager
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from uuid import uuid4
|
||||
|
||||
from .config import (
|
||||
get_storage_path,
|
||||
get_writer_lock_poll_interval_seconds,
|
||||
get_writer_lock_timeout_seconds,
|
||||
)
|
||||
|
||||
|
||||
class BackendWriterLockTimeoutError(RuntimeError):
|
||||
"""Raised when the shared backend writer lock cannot be acquired in time."""
|
||||
|
||||
|
||||
_ACTIVE_LOCK_DEPTH_BY_PATH: dict[Path, int] = {}
|
||||
_ACTIVE_LOCK_TOKEN_BY_PATH: dict[Path, str] = {}
|
||||
CONTAINER_STALE_LOCK_GRACE_SECONDS = 300
|
||||
|
||||
|
||||
def resolve_backend_writer_lock_path(*, storage_path: Path | None = None) -> Path:
|
||||
"""Return the shared lock path derived from the configured SQLite storage path."""
|
||||
resolved_storage_path = storage_path or get_storage_path()
|
||||
return resolved_storage_path.parent / f"{resolved_storage_path.stem}.writer.lock"
|
||||
|
||||
|
||||
@contextmanager
|
||||
def backend_writer_lock(
|
||||
*,
|
||||
holder: str,
|
||||
storage_path: Path | None = None,
|
||||
timeout_seconds: float | None = None,
|
||||
poll_interval_seconds: float | None = None,
|
||||
):
|
||||
"""Acquire the shared backend writer lock with reentrant safety per process."""
|
||||
lock_path = resolve_backend_writer_lock_path(storage_path=storage_path).resolve()
|
||||
if lock_path in _ACTIVE_LOCK_DEPTH_BY_PATH:
|
||||
_ACTIVE_LOCK_DEPTH_BY_PATH[lock_path] += 1
|
||||
try:
|
||||
yield _read_lock_metadata(lock_path)
|
||||
finally:
|
||||
_ACTIVE_LOCK_DEPTH_BY_PATH[lock_path] -= 1
|
||||
if _ACTIVE_LOCK_DEPTH_BY_PATH[lock_path] <= 0:
|
||||
_ACTIVE_LOCK_DEPTH_BY_PATH.pop(lock_path, None)
|
||||
_ACTIVE_LOCK_TOKEN_BY_PATH.pop(lock_path, None)
|
||||
return
|
||||
|
||||
metadata = _acquire_backend_writer_lock(
|
||||
lock_path=lock_path,
|
||||
holder=holder,
|
||||
timeout_seconds=get_writer_lock_timeout_seconds()
|
||||
if timeout_seconds is None
|
||||
else timeout_seconds,
|
||||
poll_interval_seconds=get_writer_lock_poll_interval_seconds()
|
||||
if poll_interval_seconds is None
|
||||
else poll_interval_seconds,
|
||||
)
|
||||
_ACTIVE_LOCK_DEPTH_BY_PATH[lock_path] = 1
|
||||
_ACTIVE_LOCK_TOKEN_BY_PATH[lock_path] = str(metadata["lock_token"])
|
||||
try:
|
||||
yield metadata
|
||||
finally:
|
||||
_release_backend_writer_lock(lock_path)
|
||||
_ACTIVE_LOCK_DEPTH_BY_PATH.pop(lock_path, None)
|
||||
_ACTIVE_LOCK_TOKEN_BY_PATH.pop(lock_path, None)
|
||||
|
||||
|
||||
def build_writer_lock_holder(label: str) -> str:
|
||||
"""Build one readable holder label from the current command line."""
|
||||
argv = " ".join(sys.argv).strip()
|
||||
if argv:
|
||||
return f"{label} [{argv}]"
|
||||
return label
|
||||
|
||||
|
||||
def _acquire_backend_writer_lock(
|
||||
*,
|
||||
lock_path: Path,
|
||||
holder: str,
|
||||
timeout_seconds: float,
|
||||
poll_interval_seconds: float,
|
||||
) -> dict[str, object]:
|
||||
if timeout_seconds < 0:
|
||||
raise ValueError("Writer lock timeout must be zero or positive.")
|
||||
if poll_interval_seconds <= 0:
|
||||
raise ValueError("Writer lock poll interval must be positive.")
|
||||
|
||||
lock_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
deadline = time.monotonic() + timeout_seconds
|
||||
metadata = _build_lock_metadata(holder=holder)
|
||||
|
||||
while True:
|
||||
try:
|
||||
file_descriptor = os.open(
|
||||
lock_path,
|
||||
os.O_CREAT | os.O_EXCL | os.O_WRONLY,
|
||||
)
|
||||
except FileExistsError:
|
||||
existing_metadata = _read_lock_metadata(lock_path)
|
||||
if _can_clear_stale_lock(existing_metadata):
|
||||
_remove_lock_file(lock_path)
|
||||
continue
|
||||
if time.monotonic() >= deadline:
|
||||
raise BackendWriterLockTimeoutError(
|
||||
_build_lock_timeout_message(
|
||||
lock_path=lock_path,
|
||||
holder=holder,
|
||||
timeout_seconds=timeout_seconds,
|
||||
existing_metadata=existing_metadata,
|
||||
)
|
||||
)
|
||||
time.sleep(poll_interval_seconds)
|
||||
continue
|
||||
|
||||
try:
|
||||
with os.fdopen(file_descriptor, "w", encoding="utf-8") as handle:
|
||||
json.dump(metadata, handle, ensure_ascii=True, indent=2)
|
||||
handle.write("\n")
|
||||
return metadata
|
||||
except Exception:
|
||||
_remove_lock_file(lock_path)
|
||||
raise
|
||||
|
||||
|
||||
def _release_backend_writer_lock(lock_path: Path) -> None:
|
||||
expected_token = _ACTIVE_LOCK_TOKEN_BY_PATH.get(lock_path)
|
||||
existing_metadata = _read_lock_metadata(lock_path)
|
||||
if existing_metadata and expected_token and existing_metadata.get("lock_token") != expected_token:
|
||||
return
|
||||
_remove_lock_file(lock_path)
|
||||
|
||||
|
||||
def _remove_lock_file(lock_path: Path) -> None:
|
||||
try:
|
||||
lock_path.unlink()
|
||||
except FileNotFoundError:
|
||||
return
|
||||
|
||||
|
||||
def _build_lock_metadata(*, holder: str) -> dict[str, object]:
|
||||
return {
|
||||
"lock_token": uuid4().hex,
|
||||
"holder": holder,
|
||||
"started_at": _utc_now_iso(),
|
||||
"hostname": socket.gethostname(),
|
||||
"pid": os.getpid(),
|
||||
"cwd": str(Path.cwd()),
|
||||
}
|
||||
|
||||
|
||||
def _read_lock_metadata(lock_path: Path) -> dict[str, object] | None:
|
||||
try:
|
||||
return json.loads(lock_path.read_text(encoding="utf-8"))
|
||||
except (FileNotFoundError, OSError, json.JSONDecodeError):
|
||||
return None
|
||||
|
||||
|
||||
def _can_clear_stale_lock(existing_metadata: dict[str, object] | None) -> bool:
|
||||
if not existing_metadata:
|
||||
return False
|
||||
try:
|
||||
holder_pid = int(existing_metadata.get("pid"))
|
||||
except (TypeError, ValueError):
|
||||
return False
|
||||
if holder_pid <= 0:
|
||||
return False
|
||||
|
||||
holder_hostname = str(existing_metadata.get("hostname") or "").strip()
|
||||
current_hostname = socket.gethostname()
|
||||
if holder_hostname == current_hostname:
|
||||
if _is_process_alive(holder_pid):
|
||||
return False
|
||||
return True
|
||||
if not _looks_like_containerized_holder(existing_metadata):
|
||||
return False
|
||||
lock_age_seconds = _calculate_lock_age_seconds(existing_metadata)
|
||||
if lock_age_seconds is None:
|
||||
return False
|
||||
if lock_age_seconds < CONTAINER_STALE_LOCK_GRACE_SECONDS:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def _is_process_alive(pid: int) -> bool:
|
||||
if pid == os.getpid():
|
||||
return True
|
||||
try:
|
||||
os.kill(pid, 0)
|
||||
except ProcessLookupError:
|
||||
return False
|
||||
except PermissionError:
|
||||
return True
|
||||
except OSError as exc:
|
||||
winerror = getattr(exc, "winerror", None)
|
||||
if winerror in {3, 87} or exc.errno in {3}:
|
||||
return False
|
||||
return True
|
||||
return True
|
||||
|
||||
|
||||
def _build_lock_timeout_message(
|
||||
*,
|
||||
lock_path: Path,
|
||||
holder: str,
|
||||
timeout_seconds: float,
|
||||
existing_metadata: dict[str, object] | None,
|
||||
) -> str:
|
||||
if not existing_metadata:
|
||||
return (
|
||||
f"Writer lock is busy at {lock_path} and could not be acquired within "
|
||||
f"{timeout_seconds:.1f}s for {holder}."
|
||||
)
|
||||
|
||||
existing_holder = existing_metadata.get("holder") or "unknown-holder"
|
||||
started_at = existing_metadata.get("started_at") or "unknown-started-at"
|
||||
hostname = existing_metadata.get("hostname") or "unknown-host"
|
||||
pid = existing_metadata.get("pid") or "unknown-pid"
|
||||
return (
|
||||
f"Writer lock is busy at {lock_path}. Held by {existing_holder} "
|
||||
f"since {started_at} on {hostname} (pid {pid}). "
|
||||
f"Timed out after waiting {timeout_seconds:.1f}s for {holder}."
|
||||
)
|
||||
|
||||
|
||||
def _looks_like_containerized_holder(existing_metadata: dict[str, object]) -> bool:
|
||||
holder_cwd = str(existing_metadata.get("cwd") or "").strip().lower()
|
||||
return holder_cwd.startswith("/app")
|
||||
|
||||
|
||||
def _calculate_lock_age_seconds(existing_metadata: dict[str, object]) -> float | None:
|
||||
started_at_raw = str(existing_metadata.get("started_at") or "").strip()
|
||||
if not started_at_raw:
|
||||
return None
|
||||
try:
|
||||
started_at = datetime.fromisoformat(started_at_raw.replace("Z", "+00:00"))
|
||||
except ValueError:
|
||||
return None
|
||||
if started_at.tzinfo is None:
|
||||
started_at = started_at.replace(tzinfo=timezone.utc)
|
||||
delta = datetime.now(timezone.utc) - started_at.astimezone(timezone.utc)
|
||||
return max(0.0, delta.total_seconds())
|
||||
|
||||
|
||||
def _utc_now_iso() -> str:
|
||||
return datetime.now(timezone.utc).isoformat().replace("+00:00", "Z")
|
||||
1
backend/data/.gitkeep
Normal file
1
backend/data/.gitkeep
Normal file
@@ -0,0 +1 @@
|
||||
|
||||
1
backend/data/snapshots/.gitkeep
Normal file
1
backend/data/snapshots/.gitkeep
Normal file
@@ -0,0 +1 @@
|
||||
|
||||
2
backend/requirements.txt
Normal file
2
backend/requirements.txt
Normal file
@@ -0,0 +1,2 @@
|
||||
# PostgreSQL is used by the phase-1 RCON historical storage migration.
|
||||
psycopg[binary]>=3.2,<4
|
||||
524
backend/tests/test_current_match_payload.py
Normal file
524
backend/tests/test_current_match_payload.py
Normal file
@@ -0,0 +1,524 @@
|
||||
from http import HTTPStatus
|
||||
from datetime import datetime, timezone
|
||||
from unittest.mock import patch
|
||||
|
||||
from app.payloads import build_current_match_payload
|
||||
from app.rcon_admin_log_storage import list_current_match_player_stats, persist_rcon_admin_log_entries
|
||||
from app.rcon_client import RconServerTarget
|
||||
from app.routes import resolve_get_payload
|
||||
|
||||
|
||||
TARGET = RconServerTarget(
|
||||
name="Comunidad Hispana #01",
|
||||
host="127.0.0.1",
|
||||
port=7779,
|
||||
password="test-password",
|
||||
source_name="test-rcon",
|
||||
external_server_id="comunidad-hispana-01",
|
||||
)
|
||||
|
||||
|
||||
def test_current_match_payload_projects_rich_live_rcon_session_fields():
|
||||
data = _build_with_rcon_sample(
|
||||
{
|
||||
"normalized": {
|
||||
"server_name": "Comunidad Hispana #01",
|
||||
"status": "online",
|
||||
"current_map": "carentan_warfare",
|
||||
"game_mode": "Warfare",
|
||||
"allied_score": 2,
|
||||
"axis_score": 2,
|
||||
"allied_players": 0,
|
||||
"axis_players": 0,
|
||||
"players": 0,
|
||||
"max_players": 100,
|
||||
"match_time_seconds": 5400,
|
||||
"remaining_match_time_seconds": 0,
|
||||
},
|
||||
"raw_session": {"mapId": "carentan_warfare", "mapName": "CARENTAN"},
|
||||
}
|
||||
)
|
||||
|
||||
assert data["map"] == "Carentan"
|
||||
assert data["map_id"] == "carentan_warfare"
|
||||
assert data["map_pretty_name"] == "Carentan"
|
||||
assert data["game_mode"] == "Warfare"
|
||||
assert data["allied_score"] == 2
|
||||
assert data["axis_score"] == 2
|
||||
assert data["players"] == 0
|
||||
assert data["player_count_quality"] == "rcon-session-unverified"
|
||||
assert data["player_count_source"] == "rcon-session"
|
||||
assert data["score_source"] == "rcon-session"
|
||||
assert data["map_source"] == "rcon-session"
|
||||
assert data["public_scoreboard_url"] == "https://scoreboard.comunidadhll.es"
|
||||
assert "/games" not in data["public_scoreboard_url"]
|
||||
|
||||
|
||||
def test_current_match_payload_preserves_missing_values_as_null():
|
||||
data = _build_with_rcon_sample(
|
||||
{
|
||||
"normalized": {
|
||||
"server_name": "Comunidad Hispana #01",
|
||||
"status": "online",
|
||||
"current_map": None,
|
||||
"game_mode": None,
|
||||
"players": None,
|
||||
"max_players": None,
|
||||
},
|
||||
"raw_session": {},
|
||||
}
|
||||
)
|
||||
|
||||
assert data["map"] is None
|
||||
assert data["map_id"] is None
|
||||
assert data["game_mode"] is None
|
||||
assert data["allied_score"] is None
|
||||
assert data["axis_score"] is None
|
||||
assert data["players"] is None
|
||||
assert data["player_count_quality"] is None
|
||||
assert data["player_count_source"] is None
|
||||
assert data["score_source"] is None
|
||||
assert data["map_source"] is None
|
||||
|
||||
|
||||
def test_current_match_payload_keeps_explicit_zero_score():
|
||||
data = _build_with_rcon_sample(
|
||||
{
|
||||
"normalized": {
|
||||
"server_name": "Comunidad Hispana #01",
|
||||
"status": "online",
|
||||
"current_map": "stmariedumont_warfare",
|
||||
"allied_score": 0,
|
||||
"axis_score": 0,
|
||||
},
|
||||
"raw_session": {
|
||||
"mapId": "stmariedumont_warfare",
|
||||
"mapName": "ST MARIE DU MONT",
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
assert data["map"] == "St. Marie Du Mont"
|
||||
assert data["allied_score"] == 0
|
||||
assert data["axis_score"] == 0
|
||||
assert data["score_source"] == "rcon-session"
|
||||
|
||||
|
||||
def test_current_match_payload_fallback_resolves_legacy_rcon_external_id_for_01():
|
||||
data = _build_with_snapshot_fallback(
|
||||
"comunidad-hispana-01",
|
||||
{
|
||||
"external_server_id": "rcon:152.114.195.174:7779",
|
||||
"server_name": "#01 [ESP] Comunidad Hispana",
|
||||
"status": "online",
|
||||
"current_map": "St. Marie Du Mont",
|
||||
"players": 0,
|
||||
"max_players": 100,
|
||||
"captured_at": "2026-03-24T14:08:41.008487Z",
|
||||
},
|
||||
)
|
||||
|
||||
assert data["found"] is True
|
||||
assert data["map"] == "St. Marie Du Mont"
|
||||
assert data["map_pretty_name"] == "St. Marie Du Mont"
|
||||
assert data["status"] == "online"
|
||||
assert data["players"] == 0
|
||||
assert data["max_players"] == 100
|
||||
assert data["captured_at"] == "2026-03-24T14:08:41.008487Z"
|
||||
assert data["updated_at"] == "2026-03-24T14:08:41.008487Z"
|
||||
assert data["public_scoreboard_url"] == "https://scoreboard.comunidadhll.es"
|
||||
|
||||
|
||||
def test_current_match_payload_fallback_resolves_legacy_rcon_source_ref_for_02():
|
||||
data = _build_with_snapshot_fallback(
|
||||
"comunidad-hispana-02",
|
||||
{
|
||||
"external_server_id": "snapshot-server-02",
|
||||
"source_ref": "rcon://152.114.195.150:7879",
|
||||
"status": "online",
|
||||
"current_map": "Elsenborn Ridge",
|
||||
"captured_at": "2026-03-24T14:08:41.008487Z",
|
||||
},
|
||||
)
|
||||
|
||||
assert data["found"] is True
|
||||
assert data["server_slug"] == "comunidad-hispana-02"
|
||||
assert data["map"] == "Elsenborn Ridge"
|
||||
assert data["map_pretty_name"] == "Elsenborn Ridge"
|
||||
assert data["public_scoreboard_url"] == "https://scoreboard.comunidadhll.es:5443"
|
||||
|
||||
|
||||
def test_current_match_payload_fallback_resolves_community_server_names():
|
||||
number_first = _build_with_snapshot_fallback(
|
||||
"comunidad-hispana-01",
|
||||
{
|
||||
"external_server_id": "snapshot-server-01",
|
||||
"server_name": "#01 [ESP] Comunidad Hispana - Spa Onl",
|
||||
"current_map": "Mortain",
|
||||
},
|
||||
)
|
||||
community_first = _build_with_snapshot_fallback(
|
||||
"comunidad-hispana-02",
|
||||
{
|
||||
"external_server_id": "snapshot-server-02",
|
||||
"name": "Comunidad Hispana #02",
|
||||
"current_map": "Carentan",
|
||||
},
|
||||
)
|
||||
|
||||
assert number_first["found"] is True
|
||||
assert number_first["map"] == "Mortain"
|
||||
assert community_first["found"] is True
|
||||
assert community_first["map"] == "Carentan"
|
||||
|
||||
|
||||
def test_current_match_payload_fallback_does_not_match_unknown_snapshot():
|
||||
data = _build_with_snapshot_fallback(
|
||||
"comunidad-hispana-01",
|
||||
{
|
||||
"external_server_id": "rcon:203.0.113.10:9000",
|
||||
"source_ref": "rcon://203.0.113.10:9000",
|
||||
"server_name": "#03 Comunidad Hispana",
|
||||
"current_map": "Unknown Match",
|
||||
},
|
||||
)
|
||||
|
||||
assert data["found"] is False
|
||||
assert data["map"] is None
|
||||
assert data["status"] == "unavailable"
|
||||
|
||||
|
||||
def test_current_match_route_rejects_unsupported_server():
|
||||
status, payload = resolve_get_payload("/api/current-match?server=not-trusted")
|
||||
|
||||
assert status == HTTPStatus.NOT_FOUND
|
||||
assert payload["status"] == "error"
|
||||
|
||||
|
||||
def test_current_match_player_route_rejects_unsupported_server():
|
||||
status, payload = resolve_get_payload("/api/current-match/players?server=not-trusted")
|
||||
|
||||
assert status == HTTPStatus.NOT_FOUND
|
||||
assert payload["status"] == "error"
|
||||
|
||||
|
||||
def test_current_match_player_stats_aggregate_safe_admin_log_rows(tmp_path):
|
||||
db_path = tmp_path / "admin-log.sqlite3"
|
||||
persist_rcon_admin_log_entries(
|
||||
target={
|
||||
"target_key": "comunidad-hispana-01",
|
||||
"external_server_id": "comunidad-hispana-01",
|
||||
},
|
||||
entries=[
|
||||
{
|
||||
"timestamp": "2026-05-21T10:00:00Z",
|
||||
"message": "[1:00 min (100)] MATCH START Mortain Warfare",
|
||||
},
|
||||
{
|
||||
"timestamp": "2026-05-21T10:01:00Z",
|
||||
"message": (
|
||||
"[2:00 min (120)] KILL: Bravo(Axis/steam-bravo) -> "
|
||||
"Alpha(Allies/steam-alpha) with MP40"
|
||||
),
|
||||
},
|
||||
{
|
||||
"timestamp": "2026-05-21T10:02:00Z",
|
||||
"message": (
|
||||
"[3:00 min (140)] KILL: Alpha(Allies/steam-alpha) -> "
|
||||
"Charlie(Allies/steam-charlie) with M1 GARAND"
|
||||
),
|
||||
},
|
||||
{
|
||||
"timestamp": "2026-05-21T10:03:00Z",
|
||||
"message": (
|
||||
"[4:00 min (160)] KILL: Alpha(Allies/steam-alpha) -> "
|
||||
"Bravo(Axis/steam-bravo) with M1 GARAND"
|
||||
),
|
||||
},
|
||||
],
|
||||
db_path=db_path,
|
||||
)
|
||||
|
||||
stats = list_current_match_player_stats(
|
||||
server_key="comunidad-hispana-01",
|
||||
db_path=db_path,
|
||||
)
|
||||
|
||||
assert stats["scope"] == "open-admin-log-match-window"
|
||||
assert stats["confidence"] == "admin-log-boundary"
|
||||
assert stats["source"] == "rcon-admin-log-current-match-summary"
|
||||
assert [item["player_name"] for item in stats["items"]] == ["Alpha", "Bravo", "Charlie"]
|
||||
assert stats["items"][0] == {
|
||||
"player_name": "Alpha",
|
||||
"player_id": "steam-alpha",
|
||||
"team": "Allies",
|
||||
"kills": 1,
|
||||
"deaths": 1,
|
||||
"teamkills": 1,
|
||||
"deaths_by_teamkill": 0,
|
||||
"is_connected": None,
|
||||
"connected": None,
|
||||
"last_seen_at": "2026-05-21T10:03:00Z",
|
||||
"favorite_weapon": "M1 GARAND",
|
||||
"source": "kill",
|
||||
"confidence": "admin-log-boundary",
|
||||
}
|
||||
assert "raw_message" not in stats["items"][0]
|
||||
|
||||
|
||||
def test_current_match_player_stats_include_connected_players_without_kills(tmp_path):
|
||||
db_path = tmp_path / "admin-log.sqlite3"
|
||||
persist_rcon_admin_log_entries(
|
||||
target={
|
||||
"target_key": "comunidad-hispana-01",
|
||||
"external_server_id": "comunidad-hispana-01",
|
||||
},
|
||||
entries=[
|
||||
{
|
||||
"timestamp": "2026-05-21T10:00:00Z",
|
||||
"message": "[1:00 min (100)] MATCH START Mortain Warfare",
|
||||
},
|
||||
{
|
||||
"timestamp": "2026-05-21T10:01:00Z",
|
||||
"message": "[2:00 min (120)] CONNECTED Quiet Player (steam-quiet)",
|
||||
},
|
||||
],
|
||||
db_path=db_path,
|
||||
)
|
||||
|
||||
stats = list_current_match_player_stats(
|
||||
server_key="comunidad-hispana-01",
|
||||
db_path=db_path,
|
||||
)
|
||||
|
||||
assert stats["scope"] == "open-admin-log-match-window"
|
||||
assert stats["items"] == [
|
||||
{
|
||||
"player_name": "Quiet Player",
|
||||
"player_id": "steam-quiet",
|
||||
"team": None,
|
||||
"kills": 0,
|
||||
"deaths": 0,
|
||||
"teamkills": 0,
|
||||
"deaths_by_teamkill": 0,
|
||||
"favorite_weapon": None,
|
||||
"last_seen_at": "2026-05-21T10:01:00Z",
|
||||
"is_connected": True,
|
||||
"connected": True,
|
||||
"source": "connected",
|
||||
"confidence": "admin-log-boundary",
|
||||
}
|
||||
]
|
||||
|
||||
|
||||
def test_current_match_player_stats_keep_disconnected_participants_visible(tmp_path):
|
||||
db_path = tmp_path / "admin-log.sqlite3"
|
||||
persist_rcon_admin_log_entries(
|
||||
target={
|
||||
"target_key": "comunidad-hispana-01",
|
||||
"external_server_id": "comunidad-hispana-01",
|
||||
},
|
||||
entries=[
|
||||
{
|
||||
"timestamp": "2026-05-21T10:00:00Z",
|
||||
"message": "[1:00 min (100)] MATCH START Mortain Warfare",
|
||||
},
|
||||
{
|
||||
"timestamp": "2026-05-21T10:01:00Z",
|
||||
"message": "[2:00 min (120)] CONNECTED Brief Player (steam-brief)",
|
||||
},
|
||||
{
|
||||
"timestamp": "2026-05-21T10:05:00Z",
|
||||
"message": "[6:00 min (180)] DISCONNECTED Brief Player (steam-brief)",
|
||||
},
|
||||
],
|
||||
db_path=db_path,
|
||||
)
|
||||
|
||||
stats = list_current_match_player_stats(
|
||||
server_key="comunidad-hispana-01",
|
||||
db_path=db_path,
|
||||
)
|
||||
|
||||
assert stats["items"] == [
|
||||
{
|
||||
"player_name": "Brief Player",
|
||||
"player_id": "steam-brief",
|
||||
"team": None,
|
||||
"kills": 0,
|
||||
"deaths": 0,
|
||||
"teamkills": 0,
|
||||
"deaths_by_teamkill": 0,
|
||||
"favorite_weapon": None,
|
||||
"last_seen_at": "2026-05-21T10:05:00Z",
|
||||
"is_connected": False,
|
||||
"connected": False,
|
||||
"source": "connected,disconnected",
|
||||
"confidence": "admin-log-boundary",
|
||||
}
|
||||
]
|
||||
|
||||
|
||||
def test_current_match_player_stats_include_victim_only_players(tmp_path):
|
||||
db_path = tmp_path / "admin-log.sqlite3"
|
||||
persist_rcon_admin_log_entries(
|
||||
target={
|
||||
"target_key": "comunidad-hispana-01",
|
||||
"external_server_id": "comunidad-hispana-01",
|
||||
},
|
||||
entries=[
|
||||
{
|
||||
"timestamp": "2026-05-21T10:00:00Z",
|
||||
"message": "[1:00 min (100)] MATCH START Mortain Warfare",
|
||||
},
|
||||
{
|
||||
"timestamp": "2026-05-21T10:01:00Z",
|
||||
"message": (
|
||||
"[2:00 min (120)] KILL: Killer One(Axis/steam-killer) -> "
|
||||
"Victim Only(Allies/steam-victim) with MP40"
|
||||
),
|
||||
},
|
||||
],
|
||||
db_path=db_path,
|
||||
)
|
||||
|
||||
stats = list_current_match_player_stats(
|
||||
server_key="comunidad-hispana-01",
|
||||
db_path=db_path,
|
||||
)
|
||||
by_name = {item["player_name"]: item for item in stats["items"]}
|
||||
|
||||
assert by_name["Victim Only"]["kills"] == 0
|
||||
assert by_name["Victim Only"]["deaths"] == 1
|
||||
assert by_name["Victim Only"]["favorite_weapon"] is None
|
||||
|
||||
|
||||
def test_current_match_player_stats_exclude_players_before_open_match_start(tmp_path):
|
||||
db_path = tmp_path / "admin-log.sqlite3"
|
||||
persist_rcon_admin_log_entries(
|
||||
target={
|
||||
"target_key": "comunidad-hispana-01",
|
||||
"external_server_id": "comunidad-hispana-01",
|
||||
},
|
||||
entries=[
|
||||
{
|
||||
"timestamp": "2026-05-21T09:55:00Z",
|
||||
"message": "[0:30 min (90)] CONNECTED Old Match Player (steam-old)",
|
||||
},
|
||||
{
|
||||
"timestamp": "2026-05-21T10:00:00Z",
|
||||
"message": "[1:00 min (100)] MATCH START Mortain Warfare",
|
||||
},
|
||||
{
|
||||
"timestamp": "2026-05-21T10:01:00Z",
|
||||
"message": "[2:00 min (120)] CONNECTED New Match Player (steam-new)",
|
||||
},
|
||||
],
|
||||
db_path=db_path,
|
||||
)
|
||||
|
||||
stats = list_current_match_player_stats(
|
||||
server_key="comunidad-hispana-01",
|
||||
db_path=db_path,
|
||||
)
|
||||
|
||||
assert [item["player_name"] for item in stats["items"]] == ["New Match Player"]
|
||||
|
||||
|
||||
def test_current_match_player_stats_sort_connected_before_disconnected_with_same_stats(tmp_path):
|
||||
db_path = tmp_path / "admin-log.sqlite3"
|
||||
persist_rcon_admin_log_entries(
|
||||
target={
|
||||
"target_key": "comunidad-hispana-01",
|
||||
"external_server_id": "comunidad-hispana-01",
|
||||
},
|
||||
entries=[
|
||||
{
|
||||
"timestamp": "2026-05-21T10:00:00Z",
|
||||
"message": "[1:00 min (100)] MATCH START Mortain Warfare",
|
||||
},
|
||||
{
|
||||
"timestamp": "2026-05-21T10:01:00Z",
|
||||
"message": "[2:00 min (120)] CONNECTED Connected Alpha (steam-connected)",
|
||||
},
|
||||
{
|
||||
"timestamp": "2026-05-21T10:02:00Z",
|
||||
"message": "[3:00 min (140)] CONNECTED Disconnected Bravo (steam-disconnected)",
|
||||
},
|
||||
{
|
||||
"timestamp": "2026-05-21T10:03:00Z",
|
||||
"message": "[4:00 min (160)] DISCONNECTED Disconnected Bravo (steam-disconnected)",
|
||||
},
|
||||
],
|
||||
db_path=db_path,
|
||||
)
|
||||
|
||||
stats = list_current_match_player_stats(
|
||||
server_key="comunidad-hispana-01",
|
||||
db_path=db_path,
|
||||
)
|
||||
|
||||
assert [item["player_name"] for item in stats["items"]] == [
|
||||
"Connected Alpha",
|
||||
"Disconnected Bravo",
|
||||
]
|
||||
|
||||
|
||||
def test_current_match_player_stats_filter_stale_recent_events(tmp_path):
|
||||
db_path = tmp_path / "admin-log.sqlite3"
|
||||
persist_rcon_admin_log_entries(
|
||||
target={
|
||||
"target_key": "comunidad-hispana-01",
|
||||
"external_server_id": "comunidad-hispana-01",
|
||||
},
|
||||
entries=[
|
||||
{
|
||||
"timestamp": "2026-05-21T09:30:00Z",
|
||||
"message": (
|
||||
"[1:00 min (1779355800)] KILL: Old Killer(Allies/steam-old) -> "
|
||||
"Old Victim(Axis/steam-victim-old) with M1 GARAND"
|
||||
),
|
||||
}
|
||||
],
|
||||
db_path=db_path,
|
||||
)
|
||||
|
||||
stats = list_current_match_player_stats(
|
||||
server_key="comunidad-hispana-01",
|
||||
db_path=db_path,
|
||||
now=datetime(2026, 5, 21, 10, 0, tzinfo=timezone.utc),
|
||||
)
|
||||
|
||||
assert stats["scope"] == "no-current-match-events"
|
||||
assert stats["confidence"] == "stale-filtered"
|
||||
assert stats["items"] == []
|
||||
|
||||
|
||||
def _build_with_rcon_sample(sample: dict[str, object]) -> dict[str, object]:
|
||||
with (
|
||||
patch("app.payloads.load_rcon_targets", return_value=(TARGET,)),
|
||||
patch("app.payloads.query_live_server_sample", return_value=sample),
|
||||
):
|
||||
payload = build_current_match_payload(server_slug="comunidad-hispana-01")
|
||||
return payload["data"]
|
||||
|
||||
|
||||
def _build_with_snapshot_fallback(
|
||||
server_slug: str,
|
||||
item: dict[str, object],
|
||||
) -> dict[str, object]:
|
||||
with (
|
||||
patch("app.payloads._query_current_match_rcon_sample", return_value=None),
|
||||
patch(
|
||||
"app.payloads.build_servers_payload",
|
||||
return_value={
|
||||
"status": "ok",
|
||||
"data": {
|
||||
"last_snapshot_at": "2026-03-24T14:08:41.008487Z",
|
||||
"items": [item],
|
||||
},
|
||||
},
|
||||
),
|
||||
):
|
||||
payload = build_current_match_payload(server_slug=server_slug)
|
||||
return payload["data"]
|
||||
448
backend/tests/test_database_maintenance.py
Normal file
448
backend/tests/test_database_maintenance.py
Normal file
@@ -0,0 +1,448 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
import json
|
||||
import sqlite3
|
||||
import tempfile
|
||||
import unittest
|
||||
from contextlib import closing, redirect_stdout
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from pathlib import Path
|
||||
|
||||
from app.database_maintenance import run_database_maintenance_cleanup
|
||||
from app.rcon_admin_log_materialization import MATCH_RESULT_SOURCE, initialize_rcon_materialized_storage
|
||||
from app.rcon_admin_log_storage import initialize_rcon_admin_log_storage
|
||||
from app.storage import initialize_storage
|
||||
|
||||
|
||||
class DatabaseMaintenanceTests(unittest.TestCase):
|
||||
def test_dry_run_does_not_delete(self) -> None:
|
||||
with _temp_db() as db_path:
|
||||
_insert_server_snapshot(db_path, snapshot_id=1, captured_at="2026-05-01T00:00:00Z")
|
||||
|
||||
payload = run_database_maintenance_cleanup(
|
||||
db_path=db_path,
|
||||
now="2026-06-20T12:00:00Z",
|
||||
)
|
||||
|
||||
self.assertEqual(payload["status"], "ok")
|
||||
self.assertEqual(payload["mode"], "dry-run")
|
||||
with closing(sqlite3.connect(db_path)) as connection:
|
||||
self.assertEqual(
|
||||
connection.execute("SELECT COUNT(*) FROM server_snapshots").fetchone()[0],
|
||||
1,
|
||||
)
|
||||
|
||||
def test_apply_deletes_old_server_snapshots(self) -> None:
|
||||
with _temp_db() as db_path:
|
||||
_insert_server_snapshot(db_path, snapshot_id=1, captured_at="2026-05-01T00:00:00Z")
|
||||
_insert_server_snapshot(db_path, snapshot_id=2, captured_at="2026-06-18T00:00:00Z")
|
||||
|
||||
run_database_maintenance_cleanup(
|
||||
apply=True,
|
||||
db_path=db_path,
|
||||
now="2026-06-20T12:00:00Z",
|
||||
recent_matches_keep=1,
|
||||
)
|
||||
|
||||
with closing(sqlite3.connect(db_path)) as connection:
|
||||
ids = [row[0] for row in connection.execute("SELECT id FROM server_snapshots ORDER BY id")]
|
||||
self.assertEqual(ids, [2])
|
||||
|
||||
def test_apply_deletes_old_noncritical_admin_log_events(self) -> None:
|
||||
with _temp_db() as db_path:
|
||||
_insert_admin_log_event(
|
||||
db_path,
|
||||
event_id=1,
|
||||
event_type="chat",
|
||||
event_timestamp="2026-04-01T00:00:00Z",
|
||||
server_time=100,
|
||||
)
|
||||
_insert_admin_log_event(
|
||||
db_path,
|
||||
event_id=2,
|
||||
event_type="chat",
|
||||
event_timestamp="2026-06-15T00:00:00Z",
|
||||
server_time=200,
|
||||
)
|
||||
|
||||
run_database_maintenance_cleanup(
|
||||
apply=True,
|
||||
db_path=db_path,
|
||||
now="2026-06-20T12:00:00Z",
|
||||
)
|
||||
|
||||
with closing(sqlite3.connect(db_path)) as connection:
|
||||
remaining = [
|
||||
tuple(row)
|
||||
for row in connection.execute(
|
||||
"SELECT id, event_type FROM rcon_admin_log_events ORDER BY id"
|
||||
)
|
||||
]
|
||||
self.assertEqual(remaining, [(2, "chat")])
|
||||
|
||||
def test_apply_preserves_critical_events_within_retention(self) -> None:
|
||||
with _temp_db() as db_path:
|
||||
_insert_admin_log_event(
|
||||
db_path,
|
||||
event_id=1,
|
||||
event_type="kill",
|
||||
event_timestamp="2026-06-10T00:00:00Z",
|
||||
server_time=100,
|
||||
)
|
||||
|
||||
run_database_maintenance_cleanup(
|
||||
apply=True,
|
||||
db_path=db_path,
|
||||
now="2026-06-20T12:00:00Z",
|
||||
)
|
||||
|
||||
with closing(sqlite3.connect(db_path)) as connection:
|
||||
count = connection.execute(
|
||||
"SELECT COUNT(*) FROM rcon_admin_log_events WHERE event_type = 'kill'"
|
||||
).fetchone()[0]
|
||||
self.assertEqual(count, 1)
|
||||
|
||||
def test_apply_preserves_latest_100_materialized_matches(self) -> None:
|
||||
with _temp_db() as db_path:
|
||||
for index in range(101):
|
||||
ended_at = (
|
||||
datetime(2026, 1, 1, 12, tzinfo=timezone.utc) + timedelta(days=index)
|
||||
).isoformat().replace("+00:00", "Z")
|
||||
_insert_materialized_match(
|
||||
db_path,
|
||||
match_id=index + 1,
|
||||
match_key=f"match-{index + 1}",
|
||||
ended_at=ended_at,
|
||||
server_time_start=(index + 1) * 10,
|
||||
server_time_end=(index + 1) * 10 + 5,
|
||||
)
|
||||
|
||||
run_database_maintenance_cleanup(
|
||||
apply=True,
|
||||
db_path=db_path,
|
||||
now="2026-06-20T12:00:00Z",
|
||||
)
|
||||
|
||||
with closing(sqlite3.connect(db_path)) as connection:
|
||||
remaining = connection.execute(
|
||||
"SELECT COUNT(*) FROM rcon_materialized_matches"
|
||||
).fetchone()[0]
|
||||
oldest = connection.execute(
|
||||
"SELECT COUNT(*) FROM rcon_materialized_matches WHERE match_key = 'match-1'"
|
||||
).fetchone()[0]
|
||||
self.assertEqual(remaining, 100)
|
||||
self.assertEqual(oldest, 0)
|
||||
|
||||
def test_apply_preserves_current_month_matches(self) -> None:
|
||||
with _temp_db() as db_path:
|
||||
_insert_materialized_match(
|
||||
db_path,
|
||||
match_id=1,
|
||||
match_key="old",
|
||||
ended_at="2026-01-10T12:00:00Z",
|
||||
server_time_start=10,
|
||||
server_time_end=20,
|
||||
)
|
||||
_insert_materialized_match(
|
||||
db_path,
|
||||
match_id=2,
|
||||
match_key="current-month",
|
||||
ended_at="2026-06-03T12:00:00Z",
|
||||
server_time_start=30,
|
||||
server_time_end=40,
|
||||
)
|
||||
|
||||
run_database_maintenance_cleanup(
|
||||
apply=True,
|
||||
db_path=db_path,
|
||||
now="2026-06-20T12:00:00Z",
|
||||
recent_matches_keep=1,
|
||||
)
|
||||
|
||||
with closing(sqlite3.connect(db_path)) as connection:
|
||||
keys = [row[0] for row in connection.execute("SELECT match_key FROM rcon_materialized_matches")]
|
||||
self.assertEqual(keys, ["current-month"])
|
||||
|
||||
def test_apply_preserves_previous_month_when_now_day_is_early(self) -> None:
|
||||
with _temp_db() as db_path:
|
||||
_insert_materialized_match(
|
||||
db_path,
|
||||
match_id=1,
|
||||
match_key="previous-month",
|
||||
ended_at="2026-05-15T12:00:00Z",
|
||||
server_time_start=10,
|
||||
server_time_end=20,
|
||||
)
|
||||
_insert_materialized_match(
|
||||
db_path,
|
||||
match_id=2,
|
||||
match_key="older",
|
||||
ended_at="2026-04-15T12:00:00Z",
|
||||
server_time_start=30,
|
||||
server_time_end=40,
|
||||
)
|
||||
|
||||
run_database_maintenance_cleanup(
|
||||
apply=True,
|
||||
db_path=db_path,
|
||||
now="2026-06-05T12:00:00Z",
|
||||
recent_matches_keep=1,
|
||||
)
|
||||
|
||||
with closing(sqlite3.connect(db_path)) as connection:
|
||||
keys = [row[0] for row in connection.execute("SELECT match_key FROM rcon_materialized_matches")]
|
||||
self.assertEqual(keys, ["previous-month"])
|
||||
|
||||
def test_apply_preserves_current_week(self) -> None:
|
||||
with _temp_db() as db_path:
|
||||
_insert_materialized_match(
|
||||
db_path,
|
||||
match_id=1,
|
||||
match_key="current-week",
|
||||
ended_at="2026-06-10T12:00:00Z",
|
||||
server_time_start=10,
|
||||
server_time_end=20,
|
||||
)
|
||||
_insert_materialized_match(
|
||||
db_path,
|
||||
match_id=2,
|
||||
match_key="older",
|
||||
ended_at="2026-05-01T12:00:00Z",
|
||||
server_time_start=30,
|
||||
server_time_end=40,
|
||||
)
|
||||
|
||||
run_database_maintenance_cleanup(
|
||||
apply=True,
|
||||
db_path=db_path,
|
||||
now="2026-06-10T13:00:00Z",
|
||||
recent_matches_keep=1,
|
||||
)
|
||||
|
||||
with closing(sqlite3.connect(db_path)) as connection:
|
||||
keys = [row[0] for row in connection.execute("SELECT match_key FROM rcon_materialized_matches")]
|
||||
self.assertEqual(keys, ["current-week"])
|
||||
|
||||
def test_apply_preserves_previous_week_when_fallback_may_need_it(self) -> None:
|
||||
with _temp_db() as db_path:
|
||||
_insert_materialized_match(
|
||||
db_path,
|
||||
match_id=1,
|
||||
match_key="previous-week",
|
||||
ended_at="2026-06-03T12:00:00Z",
|
||||
server_time_start=10,
|
||||
server_time_end=20,
|
||||
)
|
||||
_insert_materialized_match(
|
||||
db_path,
|
||||
match_id=2,
|
||||
match_key="current-week-sample",
|
||||
ended_at="2026-06-09T12:00:00Z",
|
||||
server_time_start=30,
|
||||
server_time_end=40,
|
||||
)
|
||||
_insert_materialized_match(
|
||||
db_path,
|
||||
match_id=3,
|
||||
match_key="older",
|
||||
ended_at="2026-05-01T12:00:00Z",
|
||||
server_time_start=50,
|
||||
server_time_end=60,
|
||||
)
|
||||
|
||||
run_database_maintenance_cleanup(
|
||||
apply=True,
|
||||
db_path=db_path,
|
||||
now="2026-06-10T13:00:00Z",
|
||||
recent_matches_keep=1,
|
||||
)
|
||||
|
||||
with closing(sqlite3.connect(db_path)) as connection:
|
||||
keys = {
|
||||
row[0]
|
||||
for row in connection.execute("SELECT match_key FROM rcon_materialized_matches")
|
||||
}
|
||||
self.assertEqual(keys, {"previous-week", "current-week-sample"})
|
||||
|
||||
def test_apply_deletes_old_non_protected_match_and_child_stats(self) -> None:
|
||||
with _temp_db() as db_path:
|
||||
_insert_materialized_match(
|
||||
db_path,
|
||||
match_id=1,
|
||||
match_key="delete-me",
|
||||
ended_at="2026-01-10T12:00:00Z",
|
||||
server_time_start=10,
|
||||
server_time_end=20,
|
||||
)
|
||||
_insert_materialized_match(
|
||||
db_path,
|
||||
match_id=2,
|
||||
match_key="keep-me",
|
||||
ended_at="2026-06-18T12:00:00Z",
|
||||
server_time_start=30,
|
||||
server_time_end=40,
|
||||
)
|
||||
_insert_player_stat(db_path, match_key="delete-me", player_id="player-1")
|
||||
_insert_player_stat(db_path, match_key="keep-me", player_id="player-2")
|
||||
|
||||
run_database_maintenance_cleanup(
|
||||
apply=True,
|
||||
db_path=db_path,
|
||||
now="2026-06-20T12:00:00Z",
|
||||
recent_matches_keep=1,
|
||||
)
|
||||
|
||||
with closing(sqlite3.connect(db_path)) as connection:
|
||||
deleted_match_count = connection.execute(
|
||||
"SELECT COUNT(*) FROM rcon_materialized_matches WHERE match_key = 'delete-me'"
|
||||
).fetchone()[0]
|
||||
deleted_stat_count = connection.execute(
|
||||
"SELECT COUNT(*) FROM rcon_match_player_stats WHERE match_key = 'delete-me'"
|
||||
).fetchone()[0]
|
||||
kept_stat_count = connection.execute(
|
||||
"SELECT COUNT(*) FROM rcon_match_player_stats WHERE match_key = 'keep-me'"
|
||||
).fetchone()[0]
|
||||
self.assertEqual(deleted_match_count, 0)
|
||||
self.assertEqual(deleted_stat_count, 0)
|
||||
self.assertEqual(kept_stat_count, 1)
|
||||
|
||||
def test_missing_optional_tables_are_logged_and_do_not_crash(self) -> None:
|
||||
with _temp_db(create_schema=False) as db_path:
|
||||
stream = io.StringIO()
|
||||
with redirect_stdout(stream):
|
||||
payload = run_database_maintenance_cleanup(
|
||||
db_path=db_path,
|
||||
now="2026-06-20T12:00:00Z",
|
||||
)
|
||||
|
||||
self.assertEqual(payload["status"], "ok")
|
||||
self.assertIn("database-maintenance-table-skipped", stream.getvalue())
|
||||
|
||||
|
||||
def _temp_db(*, create_schema: bool = True):
|
||||
class _TempDbContext:
|
||||
def __enter__(self) -> Path:
|
||||
self._tmpdir = tempfile.TemporaryDirectory(ignore_cleanup_errors=True)
|
||||
self.db_path = Path(self._tmpdir.name) / "maintenance.sqlite3"
|
||||
if create_schema:
|
||||
initialize_storage(db_path=self.db_path)
|
||||
initialize_rcon_admin_log_storage(db_path=self.db_path)
|
||||
initialize_rcon_materialized_storage(db_path=self.db_path)
|
||||
return self.db_path
|
||||
|
||||
def __exit__(self, exc_type, exc, tb) -> None:
|
||||
self._tmpdir.cleanup()
|
||||
|
||||
return _TempDbContext()
|
||||
|
||||
|
||||
def _insert_server_snapshot(db_path: Path, *, snapshot_id: int, captured_at: str) -> None:
|
||||
with closing(sqlite3.connect(db_path)) as connection:
|
||||
connection.execute(
|
||||
"""
|
||||
INSERT OR IGNORE INTO game_sources (
|
||||
id, slug, display_name, provider_kind, is_active, created_at, updated_at
|
||||
) VALUES (1, 'current-hll', 'Current Hell Let Loose', 'development', 1, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)
|
||||
"""
|
||||
)
|
||||
connection.execute(
|
||||
"""
|
||||
INSERT OR IGNORE INTO servers (
|
||||
id, game_source_id, external_server_id, server_name, region, first_seen_at, last_seen_at
|
||||
) VALUES (1, 1, 'server-1', 'Server 1', 'ES', ?, ?)
|
||||
""",
|
||||
(captured_at, captured_at),
|
||||
)
|
||||
connection.execute(
|
||||
"""
|
||||
INSERT INTO server_snapshots (
|
||||
id, server_id, captured_at, status, players, max_players, current_map, source_name
|
||||
) VALUES (?, 1, ?, 'online', 10, 100, 'hurtgen', 'test')
|
||||
""",
|
||||
(snapshot_id, captured_at),
|
||||
)
|
||||
connection.commit()
|
||||
|
||||
|
||||
def _insert_admin_log_event(
|
||||
db_path: Path,
|
||||
*,
|
||||
event_id: int,
|
||||
event_type: str,
|
||||
event_timestamp: str,
|
||||
server_time: int,
|
||||
) -> None:
|
||||
with closing(sqlite3.connect(db_path)) as connection:
|
||||
connection.execute(
|
||||
"""
|
||||
INSERT INTO rcon_admin_log_events (
|
||||
id, target_key, external_server_id, event_timestamp, server_time,
|
||||
relative_time, event_type, raw_message, canonical_message,
|
||||
parsed_payload_json, raw_entry_json
|
||||
) VALUES (?, 'comunidad-hispana-01', 'comunidad-hispana-01', ?, ?, '', ?, '', '', '{}', '{}')
|
||||
""",
|
||||
(event_id, event_timestamp, server_time, event_type),
|
||||
)
|
||||
connection.commit()
|
||||
|
||||
|
||||
def _insert_materialized_match(
|
||||
db_path: Path,
|
||||
*,
|
||||
match_id: int,
|
||||
match_key: str,
|
||||
ended_at: str,
|
||||
server_time_start: int,
|
||||
server_time_end: int,
|
||||
) -> None:
|
||||
started_at = _shift_iso(ended_at, hours=-1)
|
||||
with closing(sqlite3.connect(db_path)) as connection:
|
||||
connection.execute(
|
||||
"""
|
||||
INSERT INTO rcon_materialized_matches (
|
||||
id, target_key, external_server_id, match_key, map_name, map_pretty_name,
|
||||
game_mode, started_server_time, ended_server_time, started_at, ended_at,
|
||||
allied_score, axis_score, winner, confidence_mode, source_basis
|
||||
) VALUES (?, 'comunidad-hispana-01', 'comunidad-hispana-01', ?, 'hurtgen', 'Hurtgen Forest',
|
||||
'warfare', ?, ?, ?, ?, 5, 3, 'allied', 'exact', ?)
|
||||
""",
|
||||
(
|
||||
match_id,
|
||||
match_key,
|
||||
server_time_start,
|
||||
server_time_end,
|
||||
started_at,
|
||||
ended_at,
|
||||
MATCH_RESULT_SOURCE,
|
||||
),
|
||||
)
|
||||
connection.commit()
|
||||
|
||||
|
||||
def _insert_player_stat(db_path: Path, *, match_key: str, player_id: str) -> None:
|
||||
with closing(sqlite3.connect(db_path)) as connection:
|
||||
connection.execute(
|
||||
"""
|
||||
INSERT INTO rcon_match_player_stats (
|
||||
target_key, match_key, player_id, player_name, team,
|
||||
kills, deaths, teamkills, deaths_by_teamkill,
|
||||
weapons_json, death_by_weapons_json, most_killed_json, death_by_json
|
||||
) VALUES (
|
||||
'comunidad-hispana-01', ?, ?, ?, 'Allies',
|
||||
1, 1, 0, 0, '{}', '{}', '{}', '{}'
|
||||
)
|
||||
""",
|
||||
(match_key, player_id, player_id),
|
||||
)
|
||||
connection.commit()
|
||||
|
||||
|
||||
def _shift_iso(value: str, *, hours: int) -> str:
|
||||
point = datetime.fromisoformat(value.replace("Z", "+00:00")).astimezone(timezone.utc)
|
||||
shifted = point + timedelta(hours=hours)
|
||||
return shifted.isoformat().replace("+00:00", "Z")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
153
backend/tests/test_historical_runner_maintenance.py
Normal file
153
backend/tests/test_historical_runner_maintenance.py
Normal file
@@ -0,0 +1,153 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
import os
|
||||
import unittest
|
||||
from contextlib import nullcontext, redirect_stdout
|
||||
from datetime import datetime, timezone
|
||||
from unittest.mock import patch
|
||||
|
||||
import app.historical_runner as historical_runner
|
||||
from app.historical_runner import _maybe_run_database_maintenance, _run_refresh_with_retries
|
||||
|
||||
|
||||
class HistoricalRunnerMaintenanceTests(unittest.TestCase):
|
||||
def setUp(self) -> None:
|
||||
historical_runner._LAST_DATABASE_MAINTENANCE_RUN_AT = None
|
||||
|
||||
def tearDown(self) -> None:
|
||||
historical_runner._LAST_DATABASE_MAINTENANCE_RUN_AT = None
|
||||
|
||||
def test_scheduler_disabled_does_not_call_cleanup(self) -> None:
|
||||
with (
|
||||
patch.dict(os.environ, {"HLL_DB_MAINTENANCE_ENABLED": "false"}, clear=False),
|
||||
patch("app.historical_runner.run_database_maintenance_cleanup") as cleanup,
|
||||
):
|
||||
result = _maybe_run_database_maintenance(
|
||||
now=datetime(2026, 6, 20, 12, tzinfo=timezone.utc)
|
||||
)
|
||||
|
||||
cleanup.assert_not_called()
|
||||
self.assertEqual(result["status"], "skipped")
|
||||
self.assertEqual(result["reason"], "disabled")
|
||||
|
||||
def test_scheduler_enabled_but_not_due_does_not_call_cleanup(self) -> None:
|
||||
with (
|
||||
patch.dict(
|
||||
os.environ,
|
||||
{
|
||||
"HLL_DB_MAINTENANCE_ENABLED": "true",
|
||||
"HLL_DB_MAINTENANCE_INTERVAL_SECONDS": "43200",
|
||||
},
|
||||
clear=False,
|
||||
),
|
||||
patch(
|
||||
"app.historical_runner.run_database_maintenance_cleanup",
|
||||
return_value={"status": "ok"},
|
||||
) as cleanup,
|
||||
):
|
||||
first = _maybe_run_database_maintenance(
|
||||
now=datetime(2026, 6, 20, 0, tzinfo=timezone.utc)
|
||||
)
|
||||
second = _maybe_run_database_maintenance(
|
||||
now=datetime(2026, 6, 20, 1, tzinfo=timezone.utc)
|
||||
)
|
||||
|
||||
self.assertEqual(first["status"], "ok")
|
||||
self.assertEqual(second["status"], "skipped")
|
||||
self.assertEqual(second["reason"], "not-due")
|
||||
cleanup.assert_called_once()
|
||||
|
||||
def test_scheduler_enabled_and_due_calls_cleanup(self) -> None:
|
||||
with (
|
||||
patch.dict(os.environ, {"HLL_DB_MAINTENANCE_ENABLED": "true"}, clear=False),
|
||||
patch(
|
||||
"app.historical_runner.run_database_maintenance_cleanup",
|
||||
return_value={"status": "ok"},
|
||||
) as cleanup,
|
||||
):
|
||||
result = _maybe_run_database_maintenance(
|
||||
now=datetime(2026, 6, 20, 12, tzinfo=timezone.utc)
|
||||
)
|
||||
|
||||
cleanup.assert_called_once()
|
||||
self.assertEqual(result["status"], "ok")
|
||||
|
||||
def test_cleanup_exception_is_logged_and_runner_continues(self) -> None:
|
||||
stream = io.StringIO()
|
||||
with (
|
||||
patch.dict(os.environ, {"HLL_DB_MAINTENANCE_ENABLED": "true"}, clear=False),
|
||||
patch("app.historical_runner.backend_writer_lock", return_value=nullcontext()),
|
||||
patch(
|
||||
"app.historical_runner._run_primary_rcon_capture",
|
||||
return_value={"status": "ok", "targets": []},
|
||||
),
|
||||
patch(
|
||||
"app.historical_runner.run_incremental_refresh",
|
||||
return_value={"status": "ok"},
|
||||
),
|
||||
patch(
|
||||
"app.historical_runner.generate_historical_snapshots",
|
||||
return_value={"status": "ok"},
|
||||
),
|
||||
patch(
|
||||
"app.historical_runner.rebuild_elo_mmr_models",
|
||||
return_value={"status": "ok"},
|
||||
),
|
||||
patch(
|
||||
"app.historical_runner.run_database_maintenance_cleanup",
|
||||
side_effect=RuntimeError("maintenance failed"),
|
||||
),
|
||||
redirect_stdout(stream),
|
||||
):
|
||||
result = _run_refresh_with_retries(
|
||||
max_retries=0,
|
||||
retry_delay_seconds=0,
|
||||
server_slug="comunidad-hispana-01",
|
||||
max_pages=None,
|
||||
page_size=None,
|
||||
run_number=1,
|
||||
)
|
||||
|
||||
self.assertEqual(result["status"], "ok")
|
||||
self.assertEqual(result["database_maintenance_result"]["status"], "error")
|
||||
self.assertIn("database-maintenance-scheduler-failed", stream.getvalue())
|
||||
|
||||
def test_interval_parsing_handles_invalid_values_safely(self) -> None:
|
||||
with patch.dict(
|
||||
os.environ,
|
||||
{
|
||||
"HLL_DB_MAINTENANCE_ENABLED": "true",
|
||||
"HLL_DB_MAINTENANCE_INTERVAL_SECONDS": "bad",
|
||||
},
|
||||
clear=False,
|
||||
):
|
||||
interval_seconds, source = historical_runner._resolve_db_maintenance_interval_seconds()
|
||||
|
||||
self.assertEqual(interval_seconds, 43200)
|
||||
self.assertEqual(source, "default-invalid-env-fallback")
|
||||
|
||||
def test_maintenance_state_is_tracked_in_process(self) -> None:
|
||||
with (
|
||||
patch.dict(
|
||||
os.environ,
|
||||
{
|
||||
"HLL_DB_MAINTENANCE_ENABLED": "true",
|
||||
"HLL_DB_MAINTENANCE_INTERVAL_SECONDS": "3600",
|
||||
},
|
||||
clear=False,
|
||||
),
|
||||
patch(
|
||||
"app.historical_runner.run_database_maintenance_cleanup",
|
||||
return_value={"status": "ok"},
|
||||
),
|
||||
):
|
||||
_maybe_run_database_maintenance(now=datetime(2026, 6, 20, 12, tzinfo=timezone.utc))
|
||||
self.assertEqual(
|
||||
historical_runner._LAST_DATABASE_MAINTENANCE_RUN_AT,
|
||||
datetime(2026, 6, 20, 12, tzinfo=timezone.utc),
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
126
backend/tests/test_historical_snapshot_refresh.py
Normal file
126
backend/tests/test_historical_snapshot_refresh.py
Normal file
@@ -0,0 +1,126 @@
|
||||
"""Regression coverage for historical snapshot runner refreshes."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
import unittest
|
||||
from contextlib import nullcontext, redirect_stdout
|
||||
from datetime import datetime, timezone
|
||||
from unittest.mock import patch
|
||||
|
||||
from app.config import (
|
||||
get_historical_refresh_interval_seconds,
|
||||
get_historical_refresh_max_retries,
|
||||
get_historical_refresh_retry_delay_seconds,
|
||||
)
|
||||
from app.historical_runner import _run_refresh_with_retries, run_periodic_historical_refresh
|
||||
from app.historical_snapshots import _normalize_snapshot_limit
|
||||
from app.postgres_display_storage import _json_payload_default
|
||||
from app.rcon_historical_read_model import (
|
||||
_calculate_coverage_hours,
|
||||
_calculate_duration_seconds,
|
||||
)
|
||||
|
||||
|
||||
class HistoricalSnapshotRefreshTests(unittest.TestCase):
|
||||
def test_runner_numeric_env_values_are_parsed_before_use(self) -> None:
|
||||
with patch.dict(
|
||||
os.environ,
|
||||
{
|
||||
"HLL_HISTORICAL_SNAPSHOT_REFRESH_INTERVAL_SECONDS": "300",
|
||||
"HLL_HISTORICAL_REFRESH_MAX_RETRIES": "4",
|
||||
"HLL_HISTORICAL_REFRESH_RETRY_DELAY_SECONDS": "0.5",
|
||||
},
|
||||
clear=False,
|
||||
):
|
||||
self.assertEqual(get_historical_refresh_interval_seconds(), 300)
|
||||
self.assertEqual(get_historical_refresh_max_retries(), 4)
|
||||
self.assertEqual(get_historical_refresh_retry_delay_seconds(), 0.5)
|
||||
|
||||
def test_runner_numeric_env_values_fail_with_clear_names(self) -> None:
|
||||
with patch.dict(
|
||||
os.environ,
|
||||
{"HLL_HISTORICAL_SNAPSHOT_REFRESH_INTERVAL_SECONDS": "hourly"},
|
||||
clear=False,
|
||||
):
|
||||
with self.assertRaisesRegex(
|
||||
ValueError,
|
||||
"HLL_HISTORICAL_SNAPSHOT_REFRESH_INTERVAL_SECONDS must be an integer",
|
||||
):
|
||||
get_historical_refresh_interval_seconds()
|
||||
|
||||
def test_rcon_coverage_accepts_postgres_datetime_values(self) -> None:
|
||||
start = datetime(2026, 5, 21, 10, 0, tzinfo=timezone.utc)
|
||||
end = datetime(2026, 5, 21, 11, 30, tzinfo=timezone.utc)
|
||||
|
||||
self.assertEqual(_calculate_coverage_hours(start, end), 1.5)
|
||||
self.assertEqual(_calculate_duration_seconds(start, end), 5400)
|
||||
|
||||
def test_snapshot_limits_are_numeric_before_snapshot_queries(self) -> None:
|
||||
self.assertEqual(_normalize_snapshot_limit("recent_matches_limit", "10"), 10)
|
||||
with self.assertRaisesRegex(ValueError, "recent_matches_limit"):
|
||||
_normalize_snapshot_limit("recent_matches_limit", "ten")
|
||||
|
||||
def test_postgres_snapshot_payload_serializes_datetime_values(self) -> None:
|
||||
payload = {
|
||||
"captured_at": datetime(2026, 5, 21, 20, 12, 54, tzinfo=timezone.utc),
|
||||
}
|
||||
|
||||
self.assertEqual(
|
||||
json.loads(json.dumps(payload, default=_json_payload_default)),
|
||||
{"captured_at": "2026-05-21T20:12:54Z"},
|
||||
)
|
||||
|
||||
def test_runner_failure_log_includes_exception_type_and_traceback(self) -> None:
|
||||
stream = io.StringIO()
|
||||
with (
|
||||
patch("app.historical_runner.backend_writer_lock", return_value=nullcontext()),
|
||||
patch(
|
||||
"app.historical_runner._run_primary_rcon_capture",
|
||||
side_effect=TypeError("bad timestamp"),
|
||||
),
|
||||
redirect_stdout(stream),
|
||||
):
|
||||
result = _run_refresh_with_retries(
|
||||
max_retries=0,
|
||||
retry_delay_seconds=0,
|
||||
server_slug=None,
|
||||
max_pages=None,
|
||||
page_size=None,
|
||||
run_number=1,
|
||||
)
|
||||
|
||||
self.assertEqual(result["status"], "error")
|
||||
self.assertEqual(result["error_type"], "TypeError")
|
||||
self.assertIn("Traceback", result["traceback"])
|
||||
self.assertIn('"event": "historical-refresh-attempt-failed"', stream.getvalue())
|
||||
|
||||
def test_runner_success_log_serializes_datetime_values(self) -> None:
|
||||
stream = io.StringIO()
|
||||
with (
|
||||
patch(
|
||||
"app.historical_runner._run_refresh_with_retries",
|
||||
return_value={
|
||||
"status": "ok",
|
||||
"rcon_capture_result": {
|
||||
"captured_at": datetime(2026, 5, 22, tzinfo=timezone.utc),
|
||||
},
|
||||
},
|
||||
),
|
||||
redirect_stdout(stream),
|
||||
):
|
||||
run_periodic_historical_refresh(
|
||||
interval_seconds=1,
|
||||
max_retries=0,
|
||||
retry_delay_seconds=0,
|
||||
max_runs=1,
|
||||
)
|
||||
|
||||
self.assertIn('"status": "ok"', stream.getvalue())
|
||||
self.assertIn('"captured_at": "2026-05-22 00:00:00+00:00"', stream.getvalue())
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
27
backend/tests/test_json_serialization.py
Normal file
27
backend/tests/test_json_serialization.py
Normal file
@@ -0,0 +1,27 @@
|
||||
"""Regression coverage for API JSON encoding of PostgreSQL value types."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import unittest
|
||||
from datetime import date, datetime, timezone
|
||||
|
||||
from app.main import _json_default
|
||||
|
||||
|
||||
class JsonSerializationTests(unittest.TestCase):
|
||||
def test_json_default_serializes_postgres_datetime_and_date_values(self) -> None:
|
||||
payload = {
|
||||
"started_at": datetime(2026, 5, 21, 10, 11, 12, tzinfo=timezone.utc),
|
||||
"day": date(2026, 5, 21),
|
||||
}
|
||||
|
||||
encoded = json.loads(json.dumps(payload, default=_json_default))
|
||||
|
||||
self.assertEqual(
|
||||
encoded,
|
||||
{
|
||||
"started_at": "2026-05-21T10:11:12+00:00",
|
||||
"day": "2026-05-21",
|
||||
},
|
||||
)
|
||||
166
backend/tests/test_rcon_admin_log_parser.py
Normal file
166
backend/tests/test_rcon_admin_log_parser.py
Normal file
@@ -0,0 +1,166 @@
|
||||
from app.rcon_admin_log_parser import parse_rcon_admin_log_message
|
||||
|
||||
|
||||
from app.rcon_admin_log_parser import parse_rcon_player_profile_snapshot
|
||||
|
||||
|
||||
def test_parse_match_start():
|
||||
parsed = parse_rcon_admin_log_message(
|
||||
"[2:09:15 hours (1779178245)] MATCH START UTAH BEACH Warfare"
|
||||
)
|
||||
|
||||
assert parsed.event_type == "match_start"
|
||||
assert parsed.server_time == 1779178245
|
||||
assert parsed.map_name == "UTAH BEACH"
|
||||
assert parsed.game_mode == "Warfare"
|
||||
|
||||
|
||||
def test_parse_match_end():
|
||||
parsed = parse_rcon_admin_log_message(
|
||||
"[20:36:53 hours (1779111786)] MATCH ENDED `ST MARIE DU MONT Warfare` ALLIED (5 - 0) AXIS "
|
||||
)
|
||||
|
||||
assert parsed.event_type == "match_end"
|
||||
assert parsed.map_name == "ST MARIE DU MONT Warfare"
|
||||
assert parsed.allied_score == 5
|
||||
assert parsed.axis_score == 0
|
||||
assert parsed.winner == "allied"
|
||||
|
||||
|
||||
def test_parse_kill():
|
||||
parsed = parse_rcon_admin_log_message(
|
||||
"[1:20:19 hours (1779181181)] KILL: AntonioPruna(Allies/76561198000000000) -> "
|
||||
"[7DV] NEⓇA TACTICAL FEMB✡Y(Axis/76561199000000000) with M1 GARAND"
|
||||
)
|
||||
|
||||
assert parsed.event_type == "kill"
|
||||
assert parsed.killer_name == "AntonioPruna"
|
||||
assert parsed.killer_team == "Allies"
|
||||
assert parsed.killer_id == "76561198000000000"
|
||||
assert parsed.victim_name == "[7DV] NEⓇA TACTICAL FEMB✡Y"
|
||||
assert parsed.victim_team == "Axis"
|
||||
assert parsed.victim_id == "76561199000000000"
|
||||
assert parsed.weapon == "M1 GARAND"
|
||||
|
||||
|
||||
def test_parse_team_switch():
|
||||
parsed = parse_rcon_admin_log_message(
|
||||
"[21:34:19 hours (1779108340)] TEAMSWITCH Ekenef (None > Allies)"
|
||||
)
|
||||
|
||||
assert parsed.event_type == "team_switch"
|
||||
assert parsed.player_name == "Ekenef"
|
||||
assert parsed.from_team == "None"
|
||||
assert parsed.to_team == "Allies"
|
||||
|
||||
|
||||
def test_parse_connected():
|
||||
parsed = parse_rcon_admin_log_message(
|
||||
"[21:34:22 hours (1779108337)] CONNECTED Ekenef (76561198109813520)"
|
||||
)
|
||||
|
||||
assert parsed.event_type == "connected"
|
||||
assert parsed.player_name == "Ekenef"
|
||||
assert parsed.player_id == "76561198109813520"
|
||||
|
||||
|
||||
def test_parse_disconnected():
|
||||
parsed = parse_rcon_admin_log_message(
|
||||
"[21:10:53 hours (1779109746)] DISCONNECTED [BxB] Rab◯l◯k◯ (76561198111111111)"
|
||||
)
|
||||
|
||||
assert parsed.event_type == "disconnected"
|
||||
assert parsed.player_name == "[BxB] Rab◯l◯k◯"
|
||||
assert parsed.player_id == "76561198111111111"
|
||||
|
||||
|
||||
def test_parse_chat():
|
||||
parsed = parse_rcon_admin_log_message(
|
||||
"[18:38:35 hours (1779118884)] CHAT[Team][BXB Ivanxu(Axis/6215e24a1f05c5815ed9e8bf185f94fd)]: !vip"
|
||||
)
|
||||
|
||||
assert parsed.event_type == "chat"
|
||||
assert parsed.chat_scope == "Team"
|
||||
assert parsed.player_name == "BXB Ivanxu"
|
||||
assert parsed.chat_team == "Axis"
|
||||
assert parsed.player_id == "6215e24a1f05c5815ed9e8bf185f94fd"
|
||||
assert parsed.content == "!vip"
|
||||
|
||||
|
||||
def test_parse_kick():
|
||||
parsed = parse_rcon_admin_log_message(
|
||||
"[2:09:10 hours (1779178249)] KICK: [[7DV] NEⓇA TACTICAL FEMB✡Y] has been kicked. "
|
||||
"[Making free spaces for members of the Spanish Discord community.]"
|
||||
)
|
||||
|
||||
assert parsed.event_type == "kick"
|
||||
assert parsed.player_name == "[7DV] NEⓇA TACTICAL FEMB✡Y"
|
||||
assert "Making free spaces" in parsed.reason
|
||||
|
||||
|
||||
def test_parse_message_profile():
|
||||
parsed = parse_rcon_admin_log_message(
|
||||
"[21:34:19 hours (1779108340)] MESSAGE: player [Ekenef(76561198109813520)], "
|
||||
"content [─ Ekenef ─\\n▒ Totales ▒\\nbajas : 141 (6 TKs)\\nmuertes : 268 (5 TKs)]"
|
||||
)
|
||||
|
||||
assert parsed.event_type == "message"
|
||||
assert parsed.player_name == "Ekenef"
|
||||
assert parsed.player_id == "76561198109813520"
|
||||
assert "bajas : 141" in parsed.content
|
||||
|
||||
|
||||
def test_parse_player_profile_snapshot_spanish_sections():
|
||||
parsed = parse_rcon_admin_log_message(
|
||||
"[21:34:19 hours (1779108340)] MESSAGE: player [Jugador Uno(steam-profile-1)], "
|
||||
"content [─ Jugador Uno ─\n"
|
||||
"▒ Totales ▒\n"
|
||||
"Visto por primera vez : 2026-01-01\n"
|
||||
"sesiones : 12\n"
|
||||
"partidas jugadas : 9\n"
|
||||
"tiempo jugado : 18 h 30 min\n"
|
||||
"bajas : 141 (6 TKs)\n"
|
||||
"muertes : 268 (5 TKs)\n"
|
||||
"K/D : 0,53\n"
|
||||
"▒ VÃctimas ▒\n"
|
||||
"Rival Dos : 7\n"
|
||||
"▒ Némesis ▒\n"
|
||||
"Rival Tres : 4\n"
|
||||
"▒ Armas favoritas ▒\n"
|
||||
"M1 GARAND : 31\n"
|
||||
"▒ Promedios ▒\n"
|
||||
"bajas por partida : 15,6\n"
|
||||
"▒ Sanciones ▒\n"
|
||||
"kicks : 1]"
|
||||
)
|
||||
|
||||
snapshot = parse_rcon_player_profile_snapshot(
|
||||
parsed,
|
||||
event_timestamp="2026-05-19T10:00:00Z",
|
||||
)
|
||||
|
||||
assert snapshot is not None
|
||||
assert snapshot.player_name == "Jugador Uno"
|
||||
assert snapshot.player_id == "steam-profile-1"
|
||||
assert snapshot.source_server_time == 1779108340
|
||||
assert snapshot.sessions == 12
|
||||
assert snapshot.matches_played == 9
|
||||
assert snapshot.total_kills == 141
|
||||
assert snapshot.total_deaths == 268
|
||||
assert snapshot.teamkills_done == 6
|
||||
assert snapshot.teamkills_received == 5
|
||||
assert snapshot.kd_ratio == 0.53
|
||||
assert snapshot.favorite_weapons == {"M1 GARAND": 31}
|
||||
assert snapshot.victims == {"Rival Dos": 7}
|
||||
assert snapshot.nemesis == {"Rival Tres": 4}
|
||||
assert snapshot.averages == {"bajas por partida": 15.6}
|
||||
assert snapshot.sanctions == {"kicks": 1.0}
|
||||
|
||||
|
||||
def test_non_profile_message_does_not_parse_as_profile_snapshot():
|
||||
parsed = parse_rcon_admin_log_message(
|
||||
"[21:34:19 hours (1779108340)] MESSAGE: player [Jugador Uno(steam-profile-1)], "
|
||||
"content [Bienvenido al servidor]"
|
||||
)
|
||||
|
||||
assert parse_rcon_player_profile_snapshot(parsed) is None
|
||||
497
backend/tests/test_rcon_admin_log_storage.py
Normal file
497
backend/tests/test_rcon_admin_log_storage.py
Normal file
@@ -0,0 +1,497 @@
|
||||
import gc
|
||||
import json
|
||||
import sqlite3
|
||||
from datetime import datetime, timezone
|
||||
from unittest.mock import patch
|
||||
|
||||
from app.rcon_admin_log_storage import (
|
||||
initialize_rcon_admin_log_storage,
|
||||
list_current_match_kill_feed,
|
||||
list_rcon_admin_log_event_counts,
|
||||
persist_rcon_admin_log_entries,
|
||||
)
|
||||
|
||||
|
||||
TARGET = {
|
||||
"target_key": "test-rcon-target",
|
||||
"external_server_id": "test-rcon-target",
|
||||
}
|
||||
|
||||
|
||||
def test_initialize_rcon_admin_log_storage_creates_event_table(tmp_path):
|
||||
db_path = tmp_path / "admin_log.sqlite3"
|
||||
|
||||
resolved_path = initialize_rcon_admin_log_storage(db_path=db_path)
|
||||
|
||||
assert resolved_path == db_path
|
||||
connection = sqlite3.connect(db_path)
|
||||
try:
|
||||
table_names = {
|
||||
row[0]
|
||||
for row in connection.execute(
|
||||
"SELECT name FROM sqlite_master WHERE type = 'table'"
|
||||
).fetchall()
|
||||
}
|
||||
columns = {
|
||||
row[1]
|
||||
for row in connection.execute("PRAGMA table_info(rcon_admin_log_events)")
|
||||
}
|
||||
finally:
|
||||
connection.close()
|
||||
gc.collect()
|
||||
|
||||
assert "rcon_admin_log_events" in table_names
|
||||
assert "rcon_player_profile_snapshots" in table_names
|
||||
assert {
|
||||
"target_key",
|
||||
"event_type",
|
||||
"raw_message",
|
||||
"canonical_message",
|
||||
"parsed_payload_json",
|
||||
"raw_entry_json",
|
||||
}.issubset(columns)
|
||||
|
||||
|
||||
def test_persist_rcon_admin_log_entries_inserts_then_reports_duplicates(tmp_path):
|
||||
db_path = tmp_path / "admin_log.sqlite3"
|
||||
entries = [
|
||||
{
|
||||
"timestamp": "2026-05-19T10:00:00Z",
|
||||
"message": "[1:00 min (100)] CONNECTED Player One (steam-1)",
|
||||
},
|
||||
{
|
||||
"timestamp": "2026-05-19T10:01:00Z",
|
||||
"message": "[2:00 min (120)] DISCONNECTED Player One (steam-1)",
|
||||
},
|
||||
]
|
||||
|
||||
first_delta = persist_rcon_admin_log_entries(
|
||||
target=TARGET,
|
||||
entries=entries,
|
||||
db_path=db_path,
|
||||
)
|
||||
second_delta = persist_rcon_admin_log_entries(
|
||||
target=TARGET,
|
||||
entries=entries,
|
||||
db_path=db_path,
|
||||
)
|
||||
|
||||
assert first_delta == {
|
||||
"events_seen": 2,
|
||||
"events_inserted": 2,
|
||||
"duplicate_events": 0,
|
||||
}
|
||||
assert second_delta == {
|
||||
"events_seen": 2,
|
||||
"events_inserted": 0,
|
||||
"duplicate_events": 2,
|
||||
}
|
||||
gc.collect()
|
||||
|
||||
|
||||
def test_profile_message_snapshots_are_materialized_and_deduped(tmp_path):
|
||||
db_path = tmp_path / "admin_log.sqlite3"
|
||||
entry = {
|
||||
"timestamp": "2026-05-19T10:00:00Z",
|
||||
"message": (
|
||||
"[21:34:19 hours (1779108340)] MESSAGE: player [Jugador Uno(steam-profile-1)], "
|
||||
"content [─ Jugador Uno ─\n"
|
||||
"▒ Totales ▒\n"
|
||||
"sesiones : 12\n"
|
||||
"partidas jugadas : 9\n"
|
||||
"bajas : 141 (6 TKs)\n"
|
||||
"muertes : 268 (5 TKs)\n"
|
||||
"K/D : 0.53\n"
|
||||
"▒ VÃctimas ▒\n"
|
||||
"Rival Dos : 7\n"
|
||||
"▒ Némesis ▒\n"
|
||||
"Rival Tres : 4\n"
|
||||
"▒ Armas favoritas ▒\n"
|
||||
"M1 GARAND : 31\n"
|
||||
"▒ Promedios ▒\n"
|
||||
"bajas por partida : 15.6\n"
|
||||
"▒ Sanciones ▒\n"
|
||||
"kicks : 1]"
|
||||
),
|
||||
}
|
||||
|
||||
persist_rcon_admin_log_entries(target=TARGET, entries=[entry], db_path=db_path)
|
||||
persist_rcon_admin_log_entries(target=TARGET, entries=[entry], db_path=db_path)
|
||||
|
||||
connection = sqlite3.connect(db_path)
|
||||
connection.row_factory = sqlite3.Row
|
||||
try:
|
||||
rows = connection.execute("SELECT * FROM rcon_player_profile_snapshots").fetchall()
|
||||
finally:
|
||||
connection.close()
|
||||
gc.collect()
|
||||
|
||||
assert len(rows) == 1
|
||||
row = rows[0]
|
||||
assert row["target_key"] == "test-rcon-target"
|
||||
assert row["player_id"] == "steam-profile-1"
|
||||
assert row["source_server_time"] == 1779108340
|
||||
assert row["sessions"] == 12
|
||||
assert row["matches_played"] == 9
|
||||
assert row["total_kills"] == 141
|
||||
assert row["total_deaths"] == 268
|
||||
assert row["teamkills_done"] == 6
|
||||
assert row["teamkills_received"] == 5
|
||||
assert row["kd_ratio"] == 0.53
|
||||
assert json.loads(row["favorite_weapons_json"]) == {"M1 GARAND": 31}
|
||||
assert json.loads(row["victims_json"]) == {"Rival Dos": 7}
|
||||
assert json.loads(row["nemesis_json"]) == {"Rival Tres": 4}
|
||||
assert "bajas : 141" in row["raw_content"]
|
||||
|
||||
|
||||
def test_non_profile_messages_do_not_create_profile_snapshots(tmp_path):
|
||||
db_path = tmp_path / "admin_log.sqlite3"
|
||||
|
||||
persist_rcon_admin_log_entries(
|
||||
target=TARGET,
|
||||
entries=[
|
||||
{
|
||||
"timestamp": "2026-05-19T10:00:00Z",
|
||||
"message": "[1:00 min (100)] MESSAGE: player [Player One(steam-1)], content [hello]",
|
||||
}
|
||||
],
|
||||
db_path=db_path,
|
||||
)
|
||||
|
||||
connection = sqlite3.connect(db_path)
|
||||
try:
|
||||
count = connection.execute(
|
||||
"SELECT COUNT(*) FROM rcon_player_profile_snapshots"
|
||||
).fetchone()[0]
|
||||
finally:
|
||||
connection.close()
|
||||
gc.collect()
|
||||
|
||||
assert count == 0
|
||||
|
||||
|
||||
def test_canonical_message_dedupes_changing_relative_prefixes(tmp_path):
|
||||
db_path = tmp_path / "admin_log.sqlite3"
|
||||
original_entry = {
|
||||
"timestamp": "2026-05-19T10:00:00Z",
|
||||
"message": "[1:00 min (100)] MESSAGE: player [Player One(steam-1)], content [hello]",
|
||||
}
|
||||
repeated_read_entry = {
|
||||
"timestamp": "2026-05-19T10:05:00Z",
|
||||
"message": "[6:00 min (100)] MESSAGE: player [Player One(steam-1)], content [hello]",
|
||||
}
|
||||
|
||||
first_delta = persist_rcon_admin_log_entries(
|
||||
target=TARGET,
|
||||
entries=[original_entry],
|
||||
db_path=db_path,
|
||||
)
|
||||
second_delta = persist_rcon_admin_log_entries(
|
||||
target=TARGET,
|
||||
entries=[repeated_read_entry],
|
||||
db_path=db_path,
|
||||
)
|
||||
|
||||
assert first_delta["events_inserted"] == 1
|
||||
assert second_delta == {
|
||||
"events_seen": 1,
|
||||
"events_inserted": 0,
|
||||
"duplicate_events": 1,
|
||||
}
|
||||
gc.collect()
|
||||
|
||||
|
||||
def test_list_rcon_admin_log_event_counts_groups_by_target_and_event_type(tmp_path):
|
||||
db_path = tmp_path / "admin_log.sqlite3"
|
||||
other_target = {
|
||||
"target_key": "other-rcon-target",
|
||||
"external_server_id": "other-rcon-target",
|
||||
}
|
||||
|
||||
persist_rcon_admin_log_entries(
|
||||
target=TARGET,
|
||||
entries=[
|
||||
{
|
||||
"timestamp": "2026-05-19T10:00:00Z",
|
||||
"message": "[1:00 min (100)] CONNECTED Player One (steam-1)",
|
||||
},
|
||||
{
|
||||
"timestamp": "2026-05-19T10:01:00Z",
|
||||
"message": "[2:00 min (120)] DISCONNECTED Player One (steam-1)",
|
||||
},
|
||||
],
|
||||
db_path=db_path,
|
||||
)
|
||||
persist_rcon_admin_log_entries(
|
||||
target=other_target,
|
||||
entries=[
|
||||
{
|
||||
"timestamp": "2026-05-19T10:02:00Z",
|
||||
"message": "[3:00 min (140)] CONNECTED Player Two (steam-2)",
|
||||
}
|
||||
],
|
||||
db_path=db_path,
|
||||
)
|
||||
|
||||
counts = {
|
||||
(row["target_key"], row["event_type"]): row
|
||||
for row in list_rcon_admin_log_event_counts(db_path=db_path)
|
||||
}
|
||||
|
||||
assert counts == {
|
||||
("other-rcon-target", "connected"): {
|
||||
"target_key": "other-rcon-target",
|
||||
"event_type": "connected",
|
||||
"event_count": 1,
|
||||
"first_server_time": 140,
|
||||
"last_server_time": 140,
|
||||
},
|
||||
("test-rcon-target", "connected"): {
|
||||
"target_key": "test-rcon-target",
|
||||
"event_type": "connected",
|
||||
"event_count": 1,
|
||||
"first_server_time": 100,
|
||||
"last_server_time": 100,
|
||||
},
|
||||
("test-rcon-target", "disconnected"): {
|
||||
"target_key": "test-rcon-target",
|
||||
"event_type": "disconnected",
|
||||
"event_count": 1,
|
||||
"first_server_time": 120,
|
||||
"last_server_time": 120,
|
||||
},
|
||||
}
|
||||
gc.collect()
|
||||
|
||||
|
||||
def test_current_match_kill_feed_prefers_open_match_window_and_normalizes_rows(tmp_path):
|
||||
db_path = tmp_path / "admin_log.sqlite3"
|
||||
persist_rcon_admin_log_entries(
|
||||
target=TARGET,
|
||||
entries=[
|
||||
{
|
||||
"timestamp": "2026-05-19T09:59:00Z",
|
||||
"message": (
|
||||
"[0:59 min (90)] KILL: Old Killer(Allies/steam-old) -> "
|
||||
"Old Victim(Axis/steam-victim-old) with M1 GARAND"
|
||||
),
|
||||
},
|
||||
{
|
||||
"timestamp": "2026-05-19T10:00:00Z",
|
||||
"message": "[1:00 min (100)] MATCH START Mortain Warfare",
|
||||
},
|
||||
{
|
||||
"timestamp": "2026-05-19T10:01:00Z",
|
||||
"message": (
|
||||
"[2:00 min (120)] KILL: Alpha(Allies/steam-alpha) -> "
|
||||
"Bravo(Allies/steam-bravo) with GRENADE"
|
||||
),
|
||||
},
|
||||
],
|
||||
db_path=db_path,
|
||||
)
|
||||
|
||||
feed = list_current_match_kill_feed(
|
||||
server_key="test-rcon-target",
|
||||
db_path=db_path,
|
||||
)
|
||||
|
||||
assert feed["scope"] == "open-admin-log-match-window"
|
||||
assert feed["confidence"] == "admin-log-boundary"
|
||||
assert len(feed["items"]) == 1
|
||||
assert feed["items"][0] == {
|
||||
"event_id": "rcon-admin-log:test-rcon-target:3",
|
||||
"event_timestamp": "2026-05-19T10:01:00Z",
|
||||
"server_time": 120,
|
||||
"killer_name": "Alpha",
|
||||
"killer_team": "Allies",
|
||||
"victim_name": "Bravo",
|
||||
"victim_team": "Allies",
|
||||
"weapon": "GRENADE",
|
||||
"is_teamkill": True,
|
||||
}
|
||||
gc.collect()
|
||||
|
||||
|
||||
def test_current_match_kill_feed_filters_stale_recent_fallback_rows(tmp_path):
|
||||
db_path = tmp_path / "admin_log.sqlite3"
|
||||
persist_rcon_admin_log_entries(
|
||||
target=TARGET,
|
||||
entries=[
|
||||
{
|
||||
"timestamp": "2026-05-21T09:30:00Z",
|
||||
"message": (
|
||||
"[1:00 min (1779355800)] KILL: Old Killer(Allies/steam-old) -> "
|
||||
"Old Victim(Axis/steam-victim-old) with M1 GARAND"
|
||||
),
|
||||
}
|
||||
],
|
||||
db_path=db_path,
|
||||
)
|
||||
|
||||
feed = list_current_match_kill_feed(
|
||||
server_key="test-rcon-target",
|
||||
db_path=db_path,
|
||||
now=datetime(2026, 5, 21, 10, 0, tzinfo=timezone.utc),
|
||||
)
|
||||
|
||||
assert feed["scope"] == "no-current-match-events"
|
||||
assert feed["confidence"] == "stale-filtered"
|
||||
assert feed["stale_events_filtered"] == 1
|
||||
assert feed["items"] == []
|
||||
gc.collect()
|
||||
|
||||
|
||||
def test_current_match_kill_feed_marks_fresh_recent_fallback_rows_partial(tmp_path):
|
||||
db_path = tmp_path / "admin_log.sqlite3"
|
||||
persist_rcon_admin_log_entries(
|
||||
target=TARGET,
|
||||
entries=[
|
||||
{
|
||||
"timestamp": "2026-05-21T09:50:00Z",
|
||||
"message": (
|
||||
"[1:00 min (1779357000)] KILL: Fresh Killer(Allies/steam-fresh) -> "
|
||||
"Fresh Victim(Axis/steam-victim-fresh) with M1 GARAND"
|
||||
),
|
||||
}
|
||||
],
|
||||
db_path=db_path,
|
||||
)
|
||||
|
||||
feed = list_current_match_kill_feed(
|
||||
server_key="test-rcon-target",
|
||||
db_path=db_path,
|
||||
now=datetime(2026, 5, 21, 10, 0, tzinfo=timezone.utc),
|
||||
)
|
||||
|
||||
assert feed["scope"] == "recent-admin-log-window"
|
||||
assert feed["confidence"] == "partial"
|
||||
assert feed["stale_events_filtered"] == 0
|
||||
assert [item["killer_name"] for item in feed["items"]] == ["Fresh Killer"]
|
||||
gc.collect()
|
||||
|
||||
|
||||
def test_current_match_kill_feed_filters_rows_before_incremental_cursor(tmp_path):
|
||||
db_path = tmp_path / "admin_log.sqlite3"
|
||||
persist_rcon_admin_log_entries(
|
||||
target=TARGET,
|
||||
entries=[
|
||||
{
|
||||
"timestamp": "2026-05-21T10:00:00Z",
|
||||
"message": "[1:00 min (100)] MATCH START Mortain Warfare",
|
||||
},
|
||||
{
|
||||
"timestamp": "2026-05-21T10:01:00Z",
|
||||
"message": (
|
||||
"[2:00 min (120)] KILL: First Killer(Allies/steam-first) -> "
|
||||
"First Victim(Axis/steam-first-victim) with M1 GARAND"
|
||||
),
|
||||
},
|
||||
{
|
||||
"timestamp": "2026-05-21T10:02:00Z",
|
||||
"message": (
|
||||
"[3:00 min (140)] KILL: Next Killer(Axis/steam-next) -> "
|
||||
"Next Victim(Allies/steam-next-victim) with MP40"
|
||||
),
|
||||
},
|
||||
],
|
||||
db_path=db_path,
|
||||
)
|
||||
|
||||
feed = list_current_match_kill_feed(
|
||||
server_key="test-rcon-target",
|
||||
db_path=db_path,
|
||||
since_event_id="rcon-admin-log:test-rcon-target:2",
|
||||
)
|
||||
|
||||
assert [item["killer_name"] for item in feed["items"]] == ["Next Killer"]
|
||||
gc.collect()
|
||||
|
||||
|
||||
def test_current_match_kill_feed_without_cursor_omits_nullable_id_predicate(tmp_path):
|
||||
db_path = tmp_path / "admin_log.sqlite3"
|
||||
persist_rcon_admin_log_entries(
|
||||
target=TARGET,
|
||||
entries=[
|
||||
{
|
||||
"timestamp": "2026-05-21T10:00:00Z",
|
||||
"message": "[1:00 min (100)] MATCH START Mortain Warfare",
|
||||
},
|
||||
{
|
||||
"timestamp": "2026-05-21T10:01:00Z",
|
||||
"message": (
|
||||
"[2:00 min (120)] KILL: Cursor Killer(Allies/steam-cursor) -> "
|
||||
"Cursor Victim(Axis/steam-cursor-victim) with M1 GARAND"
|
||||
),
|
||||
},
|
||||
],
|
||||
db_path=db_path,
|
||||
)
|
||||
traced_sql = []
|
||||
connect = sqlite3.connect
|
||||
|
||||
def connect_with_trace(*args, **kwargs):
|
||||
connection = connect(*args, **kwargs)
|
||||
connection.set_trace_callback(traced_sql.append)
|
||||
return connection
|
||||
|
||||
with patch(
|
||||
"app.rcon_admin_log_storage.sqlite3.connect",
|
||||
side_effect=connect_with_trace,
|
||||
):
|
||||
feed = list_current_match_kill_feed(
|
||||
server_key="test-rcon-target",
|
||||
db_path=db_path,
|
||||
)
|
||||
|
||||
kill_queries = [
|
||||
sql
|
||||
for sql in traced_sql
|
||||
if "FROM rcon_admin_log_events" in sql and "event_type = 'kill'" in sql
|
||||
]
|
||||
assert [item["killer_name"] for item in feed["items"]] == ["Cursor Killer"]
|
||||
assert kill_queries
|
||||
assert all("IS NULL OR id >" not in sql for sql in kill_queries)
|
||||
assert all("AND id >" not in sql for sql in kill_queries)
|
||||
gc.collect()
|
||||
|
||||
|
||||
def test_current_match_kill_feed_invalid_cursor_behaves_like_no_cursor(tmp_path):
|
||||
db_path = tmp_path / "admin_log.sqlite3"
|
||||
persist_rcon_admin_log_entries(
|
||||
target=TARGET,
|
||||
entries=[
|
||||
{
|
||||
"timestamp": "2026-05-21T10:00:00Z",
|
||||
"message": "[1:00 min (100)] MATCH START Mortain Warfare",
|
||||
},
|
||||
{
|
||||
"timestamp": "2026-05-21T10:01:00Z",
|
||||
"message": (
|
||||
"[2:00 min (120)] KILL: First Killer(Allies/steam-first) -> "
|
||||
"First Victim(Axis/steam-first-victim) with M1 GARAND"
|
||||
),
|
||||
},
|
||||
{
|
||||
"timestamp": "2026-05-21T10:02:00Z",
|
||||
"message": (
|
||||
"[3:00 min (140)] KILL: Next Killer(Axis/steam-next) -> "
|
||||
"Next Victim(Allies/steam-next-victim) with MP40"
|
||||
),
|
||||
},
|
||||
],
|
||||
db_path=db_path,
|
||||
)
|
||||
|
||||
without_cursor = list_current_match_kill_feed(
|
||||
server_key="test-rcon-target",
|
||||
db_path=db_path,
|
||||
)
|
||||
with_invalid_cursor = list_current_match_kill_feed(
|
||||
server_key="test-rcon-target",
|
||||
db_path=db_path,
|
||||
since_event_id="not-an-admin-log-event",
|
||||
)
|
||||
|
||||
assert with_invalid_cursor == without_cursor
|
||||
gc.collect()
|
||||
171
backend/tests/test_rcon_historical_backfill.py
Normal file
171
backend/tests/test_rcon_historical_backfill.py
Normal file
@@ -0,0 +1,171 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import sqlite3
|
||||
import tempfile
|
||||
import unittest
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from contextlib import closing
|
||||
from unittest.mock import patch
|
||||
|
||||
from app.rcon_admin_log_materialization import (
|
||||
MATCH_RESULT_SOURCE,
|
||||
initialize_rcon_materialized_storage,
|
||||
)
|
||||
from app.rcon_historical_backfill import (
|
||||
count_recent_materialized_closed_matches,
|
||||
run_rcon_historical_backfill,
|
||||
select_backfill_targets,
|
||||
)
|
||||
from app.rcon_historical_leaderboards import list_rcon_materialized_leaderboard
|
||||
|
||||
|
||||
TARGETS_JSON = json.dumps(
|
||||
[
|
||||
{
|
||||
"name": "Comunidad Hispana #01",
|
||||
"slug": "comunidad-hispana-01",
|
||||
"external_server_id": "comunidad-hispana-01",
|
||||
"host": "127.0.0.1",
|
||||
"port": 7779,
|
||||
"password": "secret",
|
||||
},
|
||||
{
|
||||
"name": "Comunidad Hispana #02",
|
||||
"slug": "comunidad-hispana-02",
|
||||
"external_server_id": "comunidad-hispana-02",
|
||||
"host": "127.0.0.1",
|
||||
"port": 7879,
|
||||
"password": "secret",
|
||||
},
|
||||
{
|
||||
"name": "Comunidad Hispana #03",
|
||||
"slug": "comunidad-hispana-03",
|
||||
"external_server_id": "comunidad-hispana-03",
|
||||
"host": "127.0.0.1",
|
||||
"port": 7979,
|
||||
"password": "secret",
|
||||
},
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
class RconHistoricalBackfillTests(unittest.TestCase):
|
||||
def test_monthly_window_selects_previous_month_on_days_1_to_7(self) -> None:
|
||||
with tempfile.TemporaryDirectory(ignore_cleanup_errors=True) as tmpdir:
|
||||
payload = list_rcon_materialized_leaderboard(
|
||||
server_key="all-servers",
|
||||
timeframe="monthly",
|
||||
metric="kills",
|
||||
db_path=Path(tmpdir) / "historical.sqlite3",
|
||||
now=datetime(2026, 5, 7, 12, tzinfo=timezone.utc),
|
||||
)
|
||||
|
||||
self.assertEqual(payload["window_kind"], "previous-month")
|
||||
self.assertEqual(payload["selected_month_start"], "2026-04-01T00:00:00Z")
|
||||
self.assertEqual(payload["selected_month_end"], "2026-05-01T00:00:00Z")
|
||||
|
||||
def test_monthly_window_selects_current_month_on_day_8_plus(self) -> None:
|
||||
with tempfile.TemporaryDirectory(ignore_cleanup_errors=True) as tmpdir:
|
||||
payload = list_rcon_materialized_leaderboard(
|
||||
server_key="all-servers",
|
||||
timeframe="monthly",
|
||||
metric="kills",
|
||||
db_path=Path(tmpdir) / "historical.sqlite3",
|
||||
now=datetime(2026, 5, 8, 12, tzinfo=timezone.utc),
|
||||
)
|
||||
|
||||
self.assertEqual(payload["window_kind"], "current-month")
|
||||
self.assertEqual(payload["selected_month_start"], "2026-05-01T00:00:00Z")
|
||||
|
||||
def test_recent_match_ensure_stops_when_count_is_already_satisfied(self) -> None:
|
||||
with tempfile.TemporaryDirectory(ignore_cleanup_errors=True) as tmpdir, _patched_targets():
|
||||
db_path = Path(tmpdir) / "historical.sqlite3"
|
||||
_insert_closed_matches(db_path, 100)
|
||||
|
||||
payload = run_rcon_historical_backfill(
|
||||
servers="comunidad-hispana-01,comunidad-hispana-02",
|
||||
ensure_recent_matches=100,
|
||||
dry_run=True,
|
||||
db_path=db_path,
|
||||
)
|
||||
|
||||
self.assertEqual(payload["recent_materialized_closed_match_count_before"], 100)
|
||||
self.assertEqual(payload["actual_windows_scanned"], [])
|
||||
|
||||
def test_unknown_server_is_rejected(self) -> None:
|
||||
with _patched_targets():
|
||||
with self.assertRaises(ValueError):
|
||||
select_backfill_targets("unknown-server")
|
||||
|
||||
def test_comunidad_hispana_03_is_not_included_by_default(self) -> None:
|
||||
with _patched_targets():
|
||||
selected = select_backfill_targets(None)
|
||||
|
||||
self.assertEqual(
|
||||
[target.external_server_id for target in selected],
|
||||
["comunidad-hispana-01", "comunidad-hispana-02"],
|
||||
)
|
||||
|
||||
def test_dry_run_does_not_insert_data(self) -> None:
|
||||
with tempfile.TemporaryDirectory(ignore_cleanup_errors=True) as tmpdir, _patched_targets():
|
||||
db_path = Path(tmpdir) / "historical.sqlite3"
|
||||
payload = run_rcon_historical_backfill(
|
||||
servers="comunidad-hispana-01",
|
||||
ensure_current_month=True,
|
||||
dry_run=True,
|
||||
db_path=db_path,
|
||||
)
|
||||
|
||||
count_after = count_recent_materialized_closed_matches(db_path=db_path)
|
||||
|
||||
self.assertEqual(payload["status"], "dry-run")
|
||||
self.assertEqual(payload["events_inserted"], 0)
|
||||
self.assertEqual(count_after, 0)
|
||||
|
||||
def test_backfill_output_is_json_serializable(self) -> None:
|
||||
with tempfile.TemporaryDirectory(ignore_cleanup_errors=True) as tmpdir, _patched_targets():
|
||||
payload = run_rcon_historical_backfill(
|
||||
servers="comunidad-hispana-01",
|
||||
ensure_current_month=True,
|
||||
dry_run=True,
|
||||
db_path=Path(tmpdir) / "historical.sqlite3",
|
||||
)
|
||||
|
||||
json.dumps(payload, ensure_ascii=True)
|
||||
|
||||
|
||||
def _insert_closed_matches(db_path: Path, count: int) -> None:
|
||||
initialize_rcon_materialized_storage(db_path=db_path)
|
||||
with closing(sqlite3.connect(db_path)) as connection:
|
||||
for index in range(count):
|
||||
connection.execute(
|
||||
"""
|
||||
INSERT INTO rcon_materialized_matches (
|
||||
target_key, external_server_id, match_key, map_name, map_pretty_name,
|
||||
started_at, ended_at, confidence_mode, source_basis
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
""",
|
||||
(
|
||||
"comunidad-hispana-01",
|
||||
"comunidad-hispana-01",
|
||||
f"match-{index}",
|
||||
"stmariedumont",
|
||||
"ST MARIE DU MONT",
|
||||
"2026-05-01T10:00:00Z",
|
||||
f"2026-05-{(index % 28) + 1:02d}T12:00:00Z",
|
||||
"exact",
|
||||
MATCH_RESULT_SOURCE,
|
||||
),
|
||||
)
|
||||
connection.commit()
|
||||
|
||||
|
||||
def _patched_targets():
|
||||
return patch.dict(os.environ, {"HLL_BACKEND_RCON_TARGETS": TARGETS_JSON})
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
171
backend/tests/test_rcon_historical_worker.py
Normal file
171
backend/tests/test_rcon_historical_worker.py
Normal file
@@ -0,0 +1,171 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import unittest
|
||||
from contextlib import contextmanager
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import patch
|
||||
|
||||
from app.rcon_historical_worker import (
|
||||
CAPTURE_MODE_CURRENT_LIVE,
|
||||
CAPTURE_MODE_HISTORICAL,
|
||||
build_arg_parser,
|
||||
main,
|
||||
run_rcon_historical_capture,
|
||||
run_rcon_historical_capture_unlocked,
|
||||
)
|
||||
|
||||
|
||||
TARGET = SimpleNamespace(
|
||||
external_server_id="comunidad-hispana-01",
|
||||
name="Comunidad Hispana #01",
|
||||
host="203.0.113.10",
|
||||
port=7779,
|
||||
region="ES",
|
||||
game_port=7777,
|
||||
query_port=7778,
|
||||
source_name="community-hispana-rcon",
|
||||
)
|
||||
|
||||
|
||||
class RconHistoricalWorkerTests(unittest.TestCase):
|
||||
def test_current_live_capture_skips_materialization(self) -> None:
|
||||
with (
|
||||
patch("app.rcon_historical_worker.initialize_rcon_historical_storage"),
|
||||
patch("app.rcon_historical_worker._select_targets", return_value=[TARGET]),
|
||||
patch(
|
||||
"app.rcon_historical_worker.query_live_server_sample",
|
||||
return_value={"normalized": {"players": 10}, "raw_session": {"raw": True}},
|
||||
),
|
||||
patch(
|
||||
"app.rcon_historical_worker.persist_rcon_historical_sample",
|
||||
return_value={"samples_inserted": 1, "duplicate_samples": 0},
|
||||
),
|
||||
patch(
|
||||
"app.rcon_historical_worker._ingest_target_admin_log",
|
||||
return_value={
|
||||
"status": "ok",
|
||||
"errors": [],
|
||||
"totals": {
|
||||
"events_seen": 4,
|
||||
"events_inserted": 2,
|
||||
"duplicate_events": 2,
|
||||
"failed_targets": 0,
|
||||
},
|
||||
},
|
||||
),
|
||||
patch("app.rcon_historical_worker.start_rcon_historical_capture_run", return_value=101),
|
||||
patch("app.rcon_historical_worker.finalize_rcon_historical_capture_run"),
|
||||
patch(
|
||||
"app.rcon_historical_worker.list_rcon_historical_target_statuses",
|
||||
return_value=[{"target_key": "comunidad-hispana-01"}],
|
||||
),
|
||||
patch("app.rcon_historical_worker.materialize_rcon_admin_log") as materialize,
|
||||
):
|
||||
payload = run_rcon_historical_capture_unlocked(capture_mode=CAPTURE_MODE_CURRENT_LIVE)
|
||||
|
||||
materialize.assert_not_called()
|
||||
self.assertEqual(payload["capture_mode"], CAPTURE_MODE_CURRENT_LIVE)
|
||||
self.assertIs(payload["materialization_skipped"], True)
|
||||
self.assertEqual(payload["admin_log_events_seen"], 4)
|
||||
self.assertEqual(payload["admin_log_events_inserted"], 2)
|
||||
self.assertEqual(payload["duplicate_events"], 2)
|
||||
self.assertEqual(payload["samples_inserted"], 1)
|
||||
self.assertEqual(payload["materialization_result"]["status"], "skipped")
|
||||
|
||||
def test_historical_capture_keeps_materialization(self) -> None:
|
||||
with (
|
||||
patch("app.rcon_historical_worker.initialize_rcon_historical_storage"),
|
||||
patch("app.rcon_historical_worker._select_targets", return_value=[TARGET]),
|
||||
patch(
|
||||
"app.rcon_historical_worker.query_live_server_sample",
|
||||
return_value={"normalized": {"players": 10}, "raw_session": {"raw": True}},
|
||||
),
|
||||
patch(
|
||||
"app.rcon_historical_worker.persist_rcon_historical_sample",
|
||||
return_value={"samples_inserted": 1, "duplicate_samples": 0},
|
||||
),
|
||||
patch(
|
||||
"app.rcon_historical_worker._ingest_target_admin_log",
|
||||
return_value={
|
||||
"status": "ok",
|
||||
"errors": [],
|
||||
"totals": {
|
||||
"events_seen": 1,
|
||||
"events_inserted": 1,
|
||||
"duplicate_events": 0,
|
||||
"failed_targets": 0,
|
||||
},
|
||||
},
|
||||
),
|
||||
patch("app.rcon_historical_worker.start_rcon_historical_capture_run", return_value=102),
|
||||
patch("app.rcon_historical_worker.finalize_rcon_historical_capture_run"),
|
||||
patch(
|
||||
"app.rcon_historical_worker.list_rcon_historical_target_statuses",
|
||||
return_value=[{"target_key": "comunidad-hispana-01"}],
|
||||
),
|
||||
patch(
|
||||
"app.rcon_historical_worker.materialize_rcon_admin_log",
|
||||
return_value={"matches_materialized": 3, "matches_updated": 2},
|
||||
) as materialize,
|
||||
):
|
||||
payload = run_rcon_historical_capture_unlocked(capture_mode=CAPTURE_MODE_HISTORICAL)
|
||||
|
||||
materialize.assert_called_once_with()
|
||||
self.assertEqual(payload["capture_mode"], CAPTURE_MODE_HISTORICAL)
|
||||
self.assertIs(payload["materialization_skipped"], False)
|
||||
self.assertEqual(payload["totals"]["materialized_matches_inserted"], 3)
|
||||
self.assertEqual(payload["totals"]["materialized_matches_updated"], 2)
|
||||
|
||||
def test_cli_and_env_can_activate_current_live_mode(self) -> None:
|
||||
with _temporary_env(
|
||||
HLL_RCON_CURRENT_MATCH_MODE="true",
|
||||
HLL_RCON_CURRENT_MATCH_CAPTURE_INTERVAL_SECONDS="5",
|
||||
):
|
||||
args = build_arg_parser().parse_args(["loop"])
|
||||
with patch("app.rcon_historical_worker.run_periodic_rcon_historical_capture") as runner:
|
||||
exit_code = main(["loop"])
|
||||
|
||||
self.assertEqual(args.capture_mode, CAPTURE_MODE_CURRENT_LIVE)
|
||||
self.assertEqual(exit_code, 0)
|
||||
runner.assert_called_once()
|
||||
self.assertEqual(runner.call_args.kwargs["capture_mode"], CAPTURE_MODE_CURRENT_LIVE)
|
||||
self.assertEqual(runner.call_args.kwargs["interval_seconds"], 5)
|
||||
|
||||
explicit = build_arg_parser().parse_args(["capture", "--skip-materialization"])
|
||||
self.assertIs(explicit.skip_materialization, True)
|
||||
|
||||
def test_current_live_capture_uses_short_lock_timeout(self) -> None:
|
||||
seen: dict[str, object] = {}
|
||||
|
||||
@contextmanager
|
||||
def fake_lock(**kwargs):
|
||||
seen.update(kwargs)
|
||||
yield {"holder": kwargs["holder"]}
|
||||
|
||||
with (
|
||||
_temporary_env(HLL_RCON_CURRENT_MATCH_WRITER_LOCK_TIMEOUT_SECONDS="3.5"),
|
||||
patch("app.rcon_historical_worker.backend_writer_lock", side_effect=fake_lock),
|
||||
patch(
|
||||
"app.rcon_historical_worker.run_rcon_historical_capture_unlocked",
|
||||
return_value={"status": "ok"},
|
||||
),
|
||||
):
|
||||
run_rcon_historical_capture(capture_mode=CAPTURE_MODE_CURRENT_LIVE)
|
||||
|
||||
self.assertEqual(seen["timeout_seconds"], 3.5)
|
||||
|
||||
|
||||
@contextmanager
|
||||
def _temporary_env(**values: str):
|
||||
previous = {name: os.environ.get(name) for name in values}
|
||||
try:
|
||||
for name, value in values.items():
|
||||
os.environ[name] = value
|
||||
yield
|
||||
finally:
|
||||
for name, value in previous.items():
|
||||
if value is None:
|
||||
os.environ.pop(name, None)
|
||||
else:
|
||||
os.environ[name] = value
|
||||
400
backend/tests/test_rcon_materialization_pipeline.py
Normal file
400
backend/tests/test_rcon_materialization_pipeline.py
Normal file
@@ -0,0 +1,400 @@
|
||||
"""Regression tests for the materialized RCON AdminLog pipeline."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import gc
|
||||
import os
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
from app.historical_storage import upsert_historical_match
|
||||
from app.payloads import build_recent_historical_matches_payload
|
||||
from app.rcon_admin_log_materialization import (
|
||||
get_materialized_rcon_match_detail,
|
||||
materialize_rcon_admin_log,
|
||||
summarize_rcon_materialization_status,
|
||||
)
|
||||
from app.rcon_admin_log_storage import persist_rcon_admin_log_entries
|
||||
from app.rcon_historical_read_model import (
|
||||
get_rcon_historical_match_detail,
|
||||
list_rcon_historical_recent_activity,
|
||||
)
|
||||
from app.scoreboard_origins import resolve_trusted_scoreboard_match_url
|
||||
|
||||
|
||||
class RconMaterializationPipelineTests(unittest.TestCase):
|
||||
def test_materializes_match_result_and_player_stats_idempotently(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
db_path = Path(tmpdir) / "historical.sqlite3"
|
||||
_persist_admin_log_fixture(db_path)
|
||||
|
||||
first = materialize_rcon_admin_log(db_path=db_path)
|
||||
second = materialize_rcon_admin_log(db_path=db_path)
|
||||
detail = get_materialized_rcon_match_detail(
|
||||
server_key="comunidad-hispana-01",
|
||||
match_key="comunidad-hispana-01:100:500:stmariedumontwarfare",
|
||||
db_path=db_path,
|
||||
)
|
||||
status = summarize_rcon_materialization_status(db_path=db_path)
|
||||
|
||||
self.assertEqual(first["matches_materialized"], 1)
|
||||
self.assertEqual(second["matches_materialized"], 0)
|
||||
self.assertEqual(second["matches_updated"], 1)
|
||||
self.assertIsNotNone(detail)
|
||||
match = detail["match"]
|
||||
self.assertEqual(match["allied_score"], 5)
|
||||
self.assertEqual(match["axis_score"], 0)
|
||||
self.assertEqual(match["winner"], "allied")
|
||||
players = {row["player_name"]: row for row in detail["players"]}
|
||||
self.assertEqual(players["Alpha"]["kills"], 1)
|
||||
self.assertEqual(players["Alpha"]["teamkills"], 1)
|
||||
self.assertEqual(players["Bravo"]["deaths"], 1)
|
||||
self.assertEqual(players["Charlie"]["deaths_by_teamkill"], 1)
|
||||
self.assertEqual(status["materialized_matches"], 1)
|
||||
self.assertEqual(status["matches_with_player_stats"], 1)
|
||||
gc.collect()
|
||||
|
||||
def test_match_detail_read_model_hides_raw_player_ids(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
db_path = Path(tmpdir) / "historical.sqlite3"
|
||||
previous_storage_path = os.environ.get("HLL_BACKEND_STORAGE_PATH")
|
||||
os.environ["HLL_BACKEND_STORAGE_PATH"] = str(db_path)
|
||||
try:
|
||||
_persist_admin_log_fixture(db_path)
|
||||
materialize_rcon_admin_log(db_path=db_path)
|
||||
detail = get_rcon_historical_match_detail(
|
||||
server_key="comunidad-hispana-01",
|
||||
match_id="comunidad-hispana-01:100:500:stmariedumontwarfare",
|
||||
)
|
||||
finally:
|
||||
_restore_env("HLL_BACKEND_STORAGE_PATH", previous_storage_path)
|
||||
|
||||
self.assertIsNotNone(detail)
|
||||
self.assertEqual(detail["result_source"], "admin-log-match-ended")
|
||||
self.assertEqual(detail["result"]["allied_score"], 5)
|
||||
self.assertEqual(detail["timestamp_confidence"], "absolute")
|
||||
players = {row["player_name"]: row for row in detail["players"]}
|
||||
self.assertNotIn("player_id", players["Alpha"])
|
||||
self.assertIn("kd_ratio", players["Alpha"])
|
||||
self.assertEqual(players["Alpha"]["steam_id_64"], "76561198000000001")
|
||||
self.assertEqual(players["Alpha"]["platform"], "steam")
|
||||
self.assertEqual(
|
||||
players["Alpha"]["external_profile_links"]["hellor"],
|
||||
"https://hellor.pro/player/76561198000000001",
|
||||
)
|
||||
self.assertEqual(players["Charlie"]["platform"], "unknown")
|
||||
self.assertNotIn("steam_id_64", players["Charlie"])
|
||||
self.assertNotIn("external_profile_links", players["Charlie"])
|
||||
gc.collect()
|
||||
|
||||
def test_match_detail_marks_equal_materialized_timestamps_as_server_time_only(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
db_path = Path(tmpdir) / "historical.sqlite3"
|
||||
previous_storage_path = os.environ.get("HLL_BACKEND_STORAGE_PATH")
|
||||
os.environ["HLL_BACKEND_STORAGE_PATH"] = str(db_path)
|
||||
try:
|
||||
persist_rcon_admin_log_entries(
|
||||
target={
|
||||
"target_key": "comunidad-hispana-01",
|
||||
"external_server_id": "comunidad-hispana-01",
|
||||
},
|
||||
entries=[
|
||||
{
|
||||
"timestamp": "2026-05-01T12:00:00Z",
|
||||
"message": "[1 min (100)] MATCH START ST MARIE DU MONT Warfare",
|
||||
},
|
||||
{
|
||||
"timestamp": "2026-05-01T12:00:00Z",
|
||||
"message": "[91 min (5500)] MATCH ENDED `ST MARIE DU MONT Warfare` ALLIED (5 - 0) AXIS",
|
||||
},
|
||||
],
|
||||
db_path=db_path,
|
||||
)
|
||||
materialize_rcon_admin_log(db_path=db_path)
|
||||
detail = get_rcon_historical_match_detail(
|
||||
server_key="comunidad-hispana-01",
|
||||
match_id="comunidad-hispana-01:100:5500:stmariedumontwarfare",
|
||||
)
|
||||
finally:
|
||||
_restore_env("HLL_BACKEND_STORAGE_PATH", previous_storage_path)
|
||||
|
||||
self.assertIsNotNone(detail)
|
||||
self.assertIsNone(detail["started_at"])
|
||||
self.assertIsNone(detail["ended_at"])
|
||||
self.assertEqual(detail["closed_at"], "2026-05-01T12:00:00Z")
|
||||
self.assertEqual(detail["timestamp_confidence"], "server-time-only")
|
||||
self.assertEqual(detail["duration_seconds"], 5400)
|
||||
gc.collect()
|
||||
|
||||
def test_equal_timestamp_materialized_detail_uses_closed_at_window_for_scoreboard_link(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
db_path = Path(tmpdir) / "historical.sqlite3"
|
||||
previous_storage_path = os.environ.get("HLL_BACKEND_STORAGE_PATH")
|
||||
os.environ["HLL_BACKEND_STORAGE_PATH"] = str(db_path)
|
||||
try:
|
||||
upsert_historical_match(
|
||||
server_slug="comunidad-hispana-02",
|
||||
match_payload={
|
||||
"id": "1779183861",
|
||||
"creation_time": "2026-05-01T10:30:00Z",
|
||||
"start": "2026-05-01T10:30:00Z",
|
||||
"end": "2026-05-01T12:00:00Z",
|
||||
"map": {"name": "ST MARIE DU MONT Warfare"},
|
||||
"result": {"allied": 5, "axis": 0},
|
||||
"player_stats": [],
|
||||
},
|
||||
db_path=db_path,
|
||||
)
|
||||
persist_rcon_admin_log_entries(
|
||||
target={
|
||||
"target_key": "comunidad-hispana-02",
|
||||
"external_server_id": "comunidad-hispana-02",
|
||||
},
|
||||
entries=[
|
||||
{
|
||||
"timestamp": "2026-05-01T12:00:00Z",
|
||||
"message": "[1 min (100)] MATCH START ST MARIE DU MONT Warfare",
|
||||
},
|
||||
{
|
||||
"timestamp": "2026-05-01T12:00:00Z",
|
||||
"message": "[91 min (5500)] MATCH ENDED `ST MARIE DU MONT Warfare` ALLIED (5 - 0) AXIS",
|
||||
},
|
||||
],
|
||||
db_path=db_path,
|
||||
)
|
||||
materialize_rcon_admin_log(db_path=db_path)
|
||||
detail = get_rcon_historical_match_detail(
|
||||
server_key="comunidad-hispana-02",
|
||||
match_id="comunidad-hispana-02:100:5500:stmariedumontwarfare",
|
||||
)
|
||||
finally:
|
||||
_restore_env("HLL_BACKEND_STORAGE_PATH", previous_storage_path)
|
||||
|
||||
self.assertIsNotNone(detail)
|
||||
self.assertIsNone(detail["started_at"])
|
||||
self.assertIsNone(detail["ended_at"])
|
||||
self.assertEqual(detail["duration_seconds"], 5400)
|
||||
self.assertEqual(
|
||||
detail["match_url"],
|
||||
"https://scoreboard.comunidadhll.es:5443/games/1779183861",
|
||||
)
|
||||
gc.collect()
|
||||
|
||||
def test_match_detail_adds_safe_profile_summary_when_snapshot_exists(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
db_path = Path(tmpdir) / "historical.sqlite3"
|
||||
previous_storage_path = os.environ.get("HLL_BACKEND_STORAGE_PATH")
|
||||
os.environ["HLL_BACKEND_STORAGE_PATH"] = str(db_path)
|
||||
try:
|
||||
_persist_admin_log_fixture(db_path)
|
||||
persist_rcon_admin_log_entries(
|
||||
target={
|
||||
"target_key": "comunidad-hispana-01",
|
||||
"external_server_id": "comunidad-hispana-01",
|
||||
},
|
||||
entries=[
|
||||
{
|
||||
"timestamp": "2026-05-01T10:30:00Z",
|
||||
"message": (
|
||||
"[31 min (300)] MESSAGE: player [Alpha(76561198000000001)], "
|
||||
"content [─ Alpha ─\n"
|
||||
"▒ Totales ▒\n"
|
||||
"sesiones : 12\n"
|
||||
"partidas jugadas : 9\n"
|
||||
"bajas : 141 (6 TKs)\n"
|
||||
"muertes : 268 (5 TKs)\n"
|
||||
"K/D : 0.53\n"
|
||||
"▒ Armas favoritas ▒\n"
|
||||
"M1 Garand : 31]"
|
||||
),
|
||||
}
|
||||
],
|
||||
db_path=db_path,
|
||||
)
|
||||
materialize_rcon_admin_log(db_path=db_path)
|
||||
detail = get_rcon_historical_match_detail(
|
||||
server_key="comunidad-hispana-01",
|
||||
match_id="comunidad-hispana-01:100:500:stmariedumontwarfare",
|
||||
)
|
||||
finally:
|
||||
_restore_env("HLL_BACKEND_STORAGE_PATH", previous_storage_path)
|
||||
|
||||
self.assertIsNotNone(detail)
|
||||
players = {row["player_name"]: row for row in detail["players"]}
|
||||
self.assertIn("profile_summary", players["Alpha"])
|
||||
self.assertNotIn("profile_summary", players["Bravo"])
|
||||
profile_summary = players["Alpha"]["profile_summary"]
|
||||
self.assertEqual(profile_summary["sessions"], 12)
|
||||
self.assertEqual(profile_summary["matches_played"], 9)
|
||||
self.assertEqual(profile_summary["totals"]["kills"], 141)
|
||||
self.assertEqual(profile_summary["favorite_weapons"], {"M1 Garand": 31})
|
||||
self.assertNotIn("raw_content", profile_summary)
|
||||
self.assertNotIn("player_id", players["Alpha"])
|
||||
gc.collect()
|
||||
|
||||
def test_recent_matches_prefer_materialized_rcon_over_scoreboard_fallback(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
db_path = Path(tmpdir) / "historical.sqlite3"
|
||||
previous_storage_path = os.environ.get("HLL_BACKEND_STORAGE_PATH")
|
||||
os.environ["HLL_BACKEND_STORAGE_PATH"] = str(db_path)
|
||||
try:
|
||||
_persist_admin_log_fixture(db_path)
|
||||
materialize_rcon_admin_log(db_path=db_path)
|
||||
_persist_scoreboard_match(db_path)
|
||||
|
||||
payload = build_recent_historical_matches_payload(
|
||||
limit=5,
|
||||
server_slug="comunidad-hispana-01",
|
||||
)
|
||||
recent = list_rcon_historical_recent_activity(
|
||||
server_key="comunidad-hispana-01",
|
||||
limit=5,
|
||||
)
|
||||
finally:
|
||||
_restore_env("HLL_BACKEND_STORAGE_PATH", previous_storage_path)
|
||||
|
||||
self.assertEqual(payload["data"]["selected_source"], "rcon")
|
||||
self.assertEqual(payload["data"]["items"][0]["result_source"], "admin-log-match-ended")
|
||||
self.assertEqual(recent[0]["result_source"], "admin-log-match-ended")
|
||||
self.assertNotEqual(payload["data"]["selected_source"], "public-scoreboard")
|
||||
gc.collect()
|
||||
|
||||
def test_recent_materialized_detail_id_resolves_through_detail_read_model(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
db_path = Path(tmpdir) / "historical.sqlite3"
|
||||
previous_storage_path = os.environ.get("HLL_BACKEND_STORAGE_PATH")
|
||||
os.environ["HLL_BACKEND_STORAGE_PATH"] = str(db_path)
|
||||
try:
|
||||
_persist_admin_log_fixture(db_path)
|
||||
materialize_rcon_admin_log(db_path=db_path)
|
||||
recent = list_rcon_historical_recent_activity(
|
||||
server_key="comunidad-hispana-01",
|
||||
limit=1,
|
||||
)[0]
|
||||
detail = get_rcon_historical_match_detail(
|
||||
server_key="comunidad-hispana-01",
|
||||
match_id=str(recent["internal_detail_match_id"]),
|
||||
)
|
||||
finally:
|
||||
_restore_env("HLL_BACKEND_STORAGE_PATH", previous_storage_path)
|
||||
|
||||
self.assertIsNotNone(detail)
|
||||
self.assertEqual(detail["match_id"], recent["internal_detail_match_id"])
|
||||
gc.collect()
|
||||
|
||||
def test_public_scoreboard_fallback_used_only_without_rcon_activity(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
db_path = Path(tmpdir) / "historical.sqlite3"
|
||||
previous_storage_path = os.environ.get("HLL_BACKEND_STORAGE_PATH")
|
||||
os.environ["HLL_BACKEND_STORAGE_PATH"] = str(db_path)
|
||||
try:
|
||||
_persist_scoreboard_match(db_path)
|
||||
payload = build_recent_historical_matches_payload(
|
||||
limit=5,
|
||||
server_slug="comunidad-hispana-01",
|
||||
)
|
||||
finally:
|
||||
_restore_env("HLL_BACKEND_STORAGE_PATH", previous_storage_path)
|
||||
|
||||
self.assertTrue(payload["data"]["fallback_used"])
|
||||
self.assertEqual(payload["data"]["selected_source"], "public-scoreboard")
|
||||
self.assertEqual(payload["data"]["items"][0]["result_source"], "public-scoreboard-fallback")
|
||||
gc.collect()
|
||||
|
||||
def test_safe_scoreboard_match_url_allowlist_for_active_origins(self) -> None:
|
||||
self.assertEqual(
|
||||
resolve_trusted_scoreboard_match_url(
|
||||
"https://scoreboard.comunidadhll.es/games/1561515",
|
||||
"comunidad-hispana-01",
|
||||
),
|
||||
"https://scoreboard.comunidadhll.es/games/1561515",
|
||||
)
|
||||
self.assertEqual(
|
||||
resolve_trusted_scoreboard_match_url(
|
||||
"https://scoreboard.comunidadhll.es:5443/games/222",
|
||||
"comunidad-hispana-02",
|
||||
),
|
||||
"https://scoreboard.comunidadhll.es:5443/games/222",
|
||||
)
|
||||
self.assertIsNone(
|
||||
resolve_trusted_scoreboard_match_url(
|
||||
"https://example.com/games/222",
|
||||
"comunidad-hispana-02",
|
||||
)
|
||||
)
|
||||
self.assertIsNone(
|
||||
resolve_trusted_scoreboard_match_url(
|
||||
"https://scoreboard.comunidadhll.es:5443/admin/222",
|
||||
"comunidad-hispana-02",
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _persist_admin_log_fixture(db_path: Path) -> None:
|
||||
persist_rcon_admin_log_entries(
|
||||
target={
|
||||
"target_key": "comunidad-hispana-01",
|
||||
"external_server_id": "comunidad-hispana-01",
|
||||
},
|
||||
entries=[
|
||||
{
|
||||
"timestamp": "2026-05-01T10:00:00Z",
|
||||
"message": "[1 min (100)] MATCH START ST MARIE DU MONT Warfare",
|
||||
},
|
||||
{
|
||||
"timestamp": "2026-05-01T10:05:00Z",
|
||||
"message": "[6 min (150)] CONNECTED Alpha (76561198000000001)",
|
||||
},
|
||||
{
|
||||
"timestamp": "2026-05-01T10:06:00Z",
|
||||
"message": "[7 min (160)] TEAMSWITCH Alpha (None > Allies)",
|
||||
},
|
||||
{
|
||||
"timestamp": "2026-05-01T10:10:00Z",
|
||||
"message": (
|
||||
"[11 min (200)] KILL: Alpha(Allies/76561198000000001) -> "
|
||||
"Bravo(Axis/76561198000000002) with M1 Garand"
|
||||
),
|
||||
},
|
||||
{
|
||||
"timestamp": "2026-05-01T10:12:00Z",
|
||||
"message": (
|
||||
"[13 min (220)] KILL: Alpha(Allies/76561198000000001) -> "
|
||||
"Charlie(Allies/nonsteam-local) with M1 Garand"
|
||||
),
|
||||
},
|
||||
{
|
||||
"timestamp": "2026-05-01T11:20:00Z",
|
||||
"message": "[81 min (500)] MATCH ENDED `ST MARIE DU MONT Warfare` ALLIED (5 - 0) AXIS",
|
||||
},
|
||||
],
|
||||
db_path=db_path,
|
||||
)
|
||||
|
||||
|
||||
def _persist_scoreboard_match(db_path: Path) -> None:
|
||||
upsert_historical_match(
|
||||
server_slug="comunidad-hispana-01",
|
||||
match_payload={
|
||||
"id": "1561515",
|
||||
"creation_time": "2026-05-01T10:00:00Z",
|
||||
"start": "2026-05-01T10:00:00Z",
|
||||
"end": "2026-05-01T11:20:00Z",
|
||||
"map": {"name": "ST MARIE DU MONT Warfare"},
|
||||
"result": {"allied": 2, "axis": 3},
|
||||
"player_stats": [],
|
||||
},
|
||||
db_path=db_path,
|
||||
)
|
||||
|
||||
|
||||
def _restore_env(name: str, previous_value: str | None) -> None:
|
||||
if previous_value is None:
|
||||
os.environ.pop(name, None)
|
||||
else:
|
||||
os.environ[name] = previous_value
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
436
backend/tests/test_scoreboard_match_links.py
Normal file
436
backend/tests/test_scoreboard_match_links.py
Normal file
@@ -0,0 +1,436 @@
|
||||
"""Regression checks for persisted public-scoreboard match links."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import gc
|
||||
import os
|
||||
import sqlite3
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
from app.scoreboard_candidate_backfill import run_backfill
|
||||
from app.historical_storage import (
|
||||
get_historical_match_detail,
|
||||
initialize_historical_storage,
|
||||
list_recent_historical_matches,
|
||||
upsert_historical_match,
|
||||
)
|
||||
from app.rcon_historical_storage import initialize_rcon_historical_storage
|
||||
from app.rcon_historical_storage import persist_rcon_historical_sample
|
||||
from app.rcon_historical_storage import start_rcon_historical_capture_run
|
||||
from app.rcon_historical_read_model import get_rcon_historical_match_detail
|
||||
from app.rcon_admin_log_materialization import materialize_rcon_admin_log
|
||||
from app.rcon_admin_log_storage import persist_rcon_admin_log_entries
|
||||
from app.rcon_scoreboard_relink import relink_materialized_matches
|
||||
from app.scoreboard_correlation_diagnostics import inspect_materialized_match_correlation
|
||||
|
||||
|
||||
class PersistedScoreboardMatchLinkTests(unittest.TestCase):
|
||||
def test_list_backfill_persists_foy_candidate_before_detail_fetch_failure(self) -> None:
|
||||
stored: dict[tuple[str, str], dict[str, object]] = {}
|
||||
|
||||
class FoyListProvider:
|
||||
def fetch_match_page(self, *, base_url: str, page: int, limit: int) -> dict[str, object]:
|
||||
return {"maps": [_foy_list_match()]} if page == 1 else {"maps": []}
|
||||
|
||||
def fetch_match_details(
|
||||
self,
|
||||
*,
|
||||
base_url: str,
|
||||
match_ids: list[str],
|
||||
max_workers: int,
|
||||
) -> list[dict[str, object]]:
|
||||
raise RuntimeError("detail endpoint unavailable")
|
||||
|
||||
def fake_upsert(*, server_slug: str, candidate: dict[str, object]) -> str:
|
||||
key = (server_slug, str(candidate["external_match_id"]))
|
||||
outcome = "updated" if key in stored else "inserted"
|
||||
stored[key] = dict(candidate)
|
||||
return outcome
|
||||
|
||||
server = {
|
||||
"slug": "comunidad-hispana-02",
|
||||
"scoreboard_base_url": "https://scoreboard.comunidadhll.es:5443",
|
||||
"server_number": 2,
|
||||
}
|
||||
with (
|
||||
patch("app.scoreboard_candidate_backfill.initialize_historical_storage"),
|
||||
patch(
|
||||
"app.scoreboard_candidate_backfill.PublicScoreboardHistoricalDataSource",
|
||||
return_value=FoyListProvider(),
|
||||
),
|
||||
patch(
|
||||
"app.scoreboard_candidate_backfill.upsert_scoreboard_candidate",
|
||||
side_effect=fake_upsert,
|
||||
),
|
||||
):
|
||||
first = run_backfill(
|
||||
server=server,
|
||||
start_at=_backfill_timestamp("2026-05-20T00:00:00Z"),
|
||||
end_at=_backfill_timestamp("2026-05-21T23:59:59Z"),
|
||||
max_pages=2,
|
||||
page_size=100,
|
||||
detail_workers=1,
|
||||
)
|
||||
second = run_backfill(
|
||||
server=server,
|
||||
start_at=_backfill_timestamp("2026-05-20T00:00:00Z"),
|
||||
end_at=_backfill_timestamp("2026-05-21T23:59:59Z"),
|
||||
max_pages=2,
|
||||
page_size=100,
|
||||
detail_workers=1,
|
||||
)
|
||||
|
||||
candidate = stored[("comunidad-hispana-02", "1562115")]
|
||||
self.assertEqual(
|
||||
candidate["match_url"],
|
||||
"https://scoreboard.comunidadhll.es:5443/games/1562115",
|
||||
)
|
||||
self.assertEqual(first["list_candidates_inserted"], 1)
|
||||
self.assertEqual(first["list_candidates_updated"], 0)
|
||||
self.assertEqual(first["errors"][0]["stage"], "fetch_match_details")
|
||||
self.assertEqual(second["list_candidates_inserted"], 0)
|
||||
self.assertEqual(second["list_candidates_updated"], 1)
|
||||
self.assertEqual(len(stored), 1)
|
||||
|
||||
def test_recent_and_detail_payloads_expose_safe_persisted_match_url(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
db_path = Path(tmpdir) / "historical.sqlite3"
|
||||
match_url = "https://scoreboard.comunidadhll.es:5443/games/12345"
|
||||
_persist_match(db_path, server_slug="comunidad-hispana-02", match_id="12345")
|
||||
|
||||
recent_items = list_recent_historical_matches(
|
||||
server_slug="comunidad-hispana-02",
|
||||
limit=5,
|
||||
db_path=db_path,
|
||||
)
|
||||
detail = get_historical_match_detail(
|
||||
server_slug="comunidad-hispana-02",
|
||||
match_id="12345",
|
||||
db_path=db_path,
|
||||
)
|
||||
|
||||
self.assertEqual(recent_items[0]["match_url"], match_url)
|
||||
self.assertIsNotNone(detail)
|
||||
self.assertEqual(detail["match_url"], match_url)
|
||||
gc.collect()
|
||||
|
||||
def test_untrusted_persisted_match_url_is_not_exposed(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
db_path = Path(tmpdir) / "historical.sqlite3"
|
||||
_persist_match(db_path, server_slug="comunidad-hispana-01", match_id="999")
|
||||
_set_raw_payload_ref(
|
||||
db_path,
|
||||
match_id="999",
|
||||
raw_payload_ref="https://scoreboard.comunidadhll.es:3443/games/999",
|
||||
)
|
||||
|
||||
recent_items = list_recent_historical_matches(
|
||||
server_slug="comunidad-hispana-01",
|
||||
limit=5,
|
||||
db_path=db_path,
|
||||
)
|
||||
detail = get_historical_match_detail(
|
||||
server_slug="comunidad-hispana-01",
|
||||
match_id="999",
|
||||
db_path=db_path,
|
||||
)
|
||||
|
||||
self.assertIsNone(recent_items[0]["match_url"])
|
||||
self.assertIsNotNone(detail)
|
||||
self.assertIsNone(detail["match_url"])
|
||||
gc.collect()
|
||||
|
||||
def test_detail_player_links_use_trusted_scoreboard_steam_id(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
db_path = Path(tmpdir) / "historical.sqlite3"
|
||||
_persist_match(
|
||||
db_path,
|
||||
server_slug="comunidad-hispana-02",
|
||||
match_id="steam-player-match",
|
||||
player_stats=[
|
||||
{
|
||||
"player": "Steam Player",
|
||||
"steaminfo": {"profile": {"steamid": "76561198000000009"}},
|
||||
"team": {"side": "allies"},
|
||||
"kills": 4,
|
||||
"deaths": 2,
|
||||
}
|
||||
],
|
||||
)
|
||||
|
||||
detail = get_historical_match_detail(
|
||||
server_slug="comunidad-hispana-02",
|
||||
match_id="steam-player-match",
|
||||
db_path=db_path,
|
||||
)
|
||||
|
||||
self.assertIsNotNone(detail)
|
||||
player = detail["players"][0]
|
||||
self.assertEqual(player["steam_id_64"], "76561198000000009")
|
||||
self.assertEqual(player["platform"], "steam")
|
||||
self.assertEqual(
|
||||
player["external_profile_links"]["hll_records"],
|
||||
"https://hllrecords.com/profiles/76561198000000009",
|
||||
)
|
||||
gc.collect()
|
||||
|
||||
def test_rcon_match_detail_does_not_fabricate_external_scoreboard_url(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
db_path = Path(tmpdir) / "historical.sqlite3"
|
||||
previous_storage_path = os.environ.get("HLL_BACKEND_STORAGE_PATH")
|
||||
os.environ["HLL_BACKEND_STORAGE_PATH"] = str(db_path)
|
||||
try:
|
||||
initialize_rcon_historical_storage(db_path=db_path)
|
||||
detail = get_rcon_historical_match_detail(
|
||||
server_key="comunidad-hispana-01",
|
||||
match_id="rcon:synthetic-window",
|
||||
)
|
||||
finally:
|
||||
if previous_storage_path is None:
|
||||
os.environ.pop("HLL_BACKEND_STORAGE_PATH", None)
|
||||
else:
|
||||
os.environ["HLL_BACKEND_STORAGE_PATH"] = previous_storage_path
|
||||
|
||||
self.assertIsNone(detail)
|
||||
gc.collect()
|
||||
|
||||
def test_rcon_match_detail_exposes_correlated_scoreboard_url_on_strong_evidence(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
db_path = Path(tmpdir) / "historical.sqlite3"
|
||||
previous_storage_path = os.environ.get("HLL_BACKEND_STORAGE_PATH")
|
||||
os.environ["HLL_BACKEND_STORAGE_PATH"] = str(db_path)
|
||||
try:
|
||||
_persist_match(
|
||||
db_path,
|
||||
server_slug="comunidad-hispana-01",
|
||||
match_id="1561515",
|
||||
map_name="St. Mere Eglise",
|
||||
started_at="2026-04-12T16:20:00Z",
|
||||
ended_at="2026-04-12T17:45:00Z",
|
||||
)
|
||||
session_key = _persist_rcon_window(
|
||||
db_path,
|
||||
map_name="St. Mere Eglise",
|
||||
first_seen_at="2026-04-12T16:28:55.761810Z",
|
||||
last_seen_at="2026-04-12T16:43:55.761810Z",
|
||||
players=94,
|
||||
max_players=98,
|
||||
)
|
||||
|
||||
detail = get_rcon_historical_match_detail(
|
||||
server_key="comunidad-hispana-01",
|
||||
match_id=session_key,
|
||||
)
|
||||
finally:
|
||||
if previous_storage_path is None:
|
||||
os.environ.pop("HLL_BACKEND_STORAGE_PATH", None)
|
||||
else:
|
||||
os.environ["HLL_BACKEND_STORAGE_PATH"] = previous_storage_path
|
||||
|
||||
self.assertIsNotNone(detail)
|
||||
self.assertEqual(
|
||||
detail["match_url"],
|
||||
"https://scoreboard.comunidadhll.es/games/1561515",
|
||||
)
|
||||
gc.collect()
|
||||
|
||||
def test_rcon_match_detail_keeps_low_confidence_correlation_unlinked(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
db_path = Path(tmpdir) / "historical.sqlite3"
|
||||
previous_storage_path = os.environ.get("HLL_BACKEND_STORAGE_PATH")
|
||||
os.environ["HLL_BACKEND_STORAGE_PATH"] = str(db_path)
|
||||
try:
|
||||
_persist_match(
|
||||
db_path,
|
||||
server_slug="comunidad-hispana-01",
|
||||
match_id="1561515",
|
||||
map_name="Carentan",
|
||||
started_at="2026-04-12T10:00:00Z",
|
||||
ended_at="2026-04-12T11:30:00Z",
|
||||
)
|
||||
session_key = _persist_rcon_window(
|
||||
db_path,
|
||||
map_name="St. Mere Eglise",
|
||||
first_seen_at="2026-04-12T16:28:55.761810Z",
|
||||
last_seen_at="2026-04-12T16:43:55.761810Z",
|
||||
players=94,
|
||||
max_players=98,
|
||||
)
|
||||
|
||||
detail = get_rcon_historical_match_detail(
|
||||
server_key="comunidad-hispana-01",
|
||||
match_id=session_key,
|
||||
)
|
||||
finally:
|
||||
if previous_storage_path is None:
|
||||
os.environ.pop("HLL_BACKEND_STORAGE_PATH", None)
|
||||
else:
|
||||
os.environ["HLL_BACKEND_STORAGE_PATH"] = previous_storage_path
|
||||
|
||||
self.assertIsNotNone(detail)
|
||||
self.assertIsNone(detail["match_url"])
|
||||
gc.collect()
|
||||
|
||||
def test_foy_relink_reports_existing_materialized_match_url(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
db_path = Path(tmpdir) / "historical.sqlite3"
|
||||
previous_storage_path = os.environ.get("HLL_BACKEND_STORAGE_PATH")
|
||||
os.environ["HLL_BACKEND_STORAGE_PATH"] = str(db_path)
|
||||
try:
|
||||
_persist_match(
|
||||
db_path,
|
||||
server_slug="comunidad-hispana-02",
|
||||
match_id="1562115",
|
||||
map_name="Foy Warfare",
|
||||
started_at="2026-05-20T20:54:11Z",
|
||||
ended_at="2026-05-20T22:24:11Z",
|
||||
)
|
||||
persist_rcon_admin_log_entries(
|
||||
target={
|
||||
"target_key": "comunidad-hispana-02",
|
||||
"external_server_id": "comunidad-hispana-02",
|
||||
},
|
||||
entries=[
|
||||
{
|
||||
"timestamp": "2026-05-20T20:54:11Z",
|
||||
"message": "[1 min (1779310451)] MATCH START Foy Warfare",
|
||||
},
|
||||
{
|
||||
"timestamp": "2026-05-20T22:24:11Z",
|
||||
"message": "[91 min (1779315851)] MATCH ENDED `Foy Warfare` ALLIED (4 - 1) AXIS",
|
||||
},
|
||||
],
|
||||
db_path=db_path,
|
||||
)
|
||||
materialize_rcon_admin_log(db_path=db_path)
|
||||
report = relink_materialized_matches(
|
||||
server_key="comunidad-hispana-02",
|
||||
db_path=db_path,
|
||||
)
|
||||
detail = get_rcon_historical_match_detail(
|
||||
server_key="comunidad-hispana-02",
|
||||
match_id="comunidad-hispana-02:1779310451:1779315851:foywarfare",
|
||||
)
|
||||
diagnostics = inspect_materialized_match_correlation(
|
||||
server_key="comunidad-hispana-02",
|
||||
match_key="comunidad-hispana-02:1779310451:1779315851:foywarfare",
|
||||
db_path=db_path,
|
||||
)
|
||||
finally:
|
||||
if previous_storage_path is None:
|
||||
os.environ.pop("HLL_BACKEND_STORAGE_PATH", None)
|
||||
else:
|
||||
os.environ["HLL_BACKEND_STORAGE_PATH"] = previous_storage_path
|
||||
|
||||
self.assertEqual(report["matches_scanned"], 1)
|
||||
self.assertEqual(report["matches_linked"], 1)
|
||||
self.assertGreaterEqual(report["candidates_scanned"], 1)
|
||||
self.assertIsNotNone(detail)
|
||||
self.assertEqual(
|
||||
detail["match_url"],
|
||||
"https://scoreboard.comunidadhll.es:5443/games/1562115",
|
||||
)
|
||||
self.assertEqual(diagnostics["final_reason"], "linked")
|
||||
self.assertEqual(diagnostics["selected_candidate"]["external_match_id"], "1562115")
|
||||
self.assertEqual(diagnostics["top_candidates"][0]["map"], "Foy Warfare")
|
||||
gc.collect()
|
||||
|
||||
|
||||
def _persist_match(
|
||||
db_path: Path,
|
||||
*,
|
||||
server_slug: str,
|
||||
match_id: str,
|
||||
map_name: str = "carentan",
|
||||
started_at: str = "2026-05-01T10:00:00Z",
|
||||
ended_at: str = "2026-05-01T11:20:00Z",
|
||||
player_stats: list[dict[str, object]] | None = None,
|
||||
) -> None:
|
||||
upsert_historical_match(
|
||||
server_slug=server_slug,
|
||||
match_payload={
|
||||
"id": match_id,
|
||||
"creation_time": started_at,
|
||||
"start": started_at,
|
||||
"end": ended_at,
|
||||
"map": {"name": map_name},
|
||||
"result": {"allied": 3, "axis": 2},
|
||||
"player_stats": player_stats or [],
|
||||
},
|
||||
db_path=db_path,
|
||||
)
|
||||
|
||||
|
||||
def _foy_list_match() -> dict[str, object]:
|
||||
return {
|
||||
"id": 1562115,
|
||||
"server_number": 2,
|
||||
"start": "2026-05-20T20:54:11+00:00",
|
||||
"end": "2026-05-20T22:24:11+00:00",
|
||||
"map": {"id": "foywarfare", "pretty_name": "Foy Warfare"},
|
||||
"result": {"allied": 4, "axis": 1},
|
||||
}
|
||||
|
||||
|
||||
def _backfill_timestamp(raw_value: str):
|
||||
from app.scoreboard_candidate_backfill import _parse_timestamp
|
||||
|
||||
return _parse_timestamp(raw_value, option_name="test")
|
||||
|
||||
|
||||
def _persist_rcon_window(
|
||||
db_path: Path,
|
||||
*,
|
||||
map_name: str,
|
||||
first_seen_at: str,
|
||||
last_seen_at: str,
|
||||
players: int,
|
||||
max_players: int,
|
||||
) -> str:
|
||||
initialize_rcon_historical_storage(db_path=db_path)
|
||||
run_id = start_rcon_historical_capture_run(
|
||||
mode="test",
|
||||
target_scope="comunidad-hispana-01",
|
||||
db_path=db_path,
|
||||
)
|
||||
target = {
|
||||
"target_key": "comunidad-hispana-01",
|
||||
"external_server_id": "comunidad-hispana-01",
|
||||
"name": "Comunidad Hispana #01",
|
||||
"host": "127.0.0.1",
|
||||
"port": 7779,
|
||||
}
|
||||
for captured_at in (first_seen_at, last_seen_at):
|
||||
persist_rcon_historical_sample(
|
||||
run_id=run_id,
|
||||
captured_at=captured_at,
|
||||
target=target,
|
||||
normalized_payload={
|
||||
"status": "online",
|
||||
"players": players,
|
||||
"max_players": max_players,
|
||||
"current_map": map_name,
|
||||
},
|
||||
raw_payload={},
|
||||
db_path=db_path,
|
||||
)
|
||||
return f"1:{first_seen_at}"
|
||||
|
||||
|
||||
def _set_raw_payload_ref(db_path: Path, *, match_id: str, raw_payload_ref: str) -> None:
|
||||
with sqlite3.connect(db_path) as connection:
|
||||
connection.execute(
|
||||
"""
|
||||
UPDATE historical_matches
|
||||
SET raw_payload_ref = ?
|
||||
WHERE external_match_id = ?
|
||||
""",
|
||||
(raw_payload_ref, match_id),
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user