feat: migrate rcon historical storage to postgres

This commit is contained in:
devRaGonSa
2026-05-21 08:18:45 +02:00
parent b3691e5271
commit 239595052e
19 changed files with 1658 additions and 92 deletions

View File

@@ -85,6 +85,20 @@ def get_storage_path() -> Path:
return Path(configured_path) if configured_path else default_path
def get_database_url() -> str | None:
"""Return the optional PostgreSQL URL for migrated backend storage domains."""
configured_url = os.getenv("HLL_BACKEND_DATABASE_URL")
if configured_url is None:
return None
normalized_url = configured_url.strip()
return normalized_url or None
def use_postgres_rcon_storage(*, explicit_sqlite_path: Path | None = None) -> bool:
"""Return whether phase-1 RCON storage should use PostgreSQL."""
return explicit_sqlite_path is None and get_database_url() is not None
def get_sqlite_writer_timeout_seconds() -> float:
"""Return the SQLite connection timeout shared by writer-capable storage layers."""
configured_value = os.getenv(

View File

@@ -0,0 +1,986 @@
"""PostgreSQL persistence for the phase-1 RCON historical pipeline."""
from __future__ import annotations
import json
from collections.abc import Iterable, Mapping
from contextlib import contextmanager
from datetime import datetime, timezone
from typing import Any
from .config import get_database_url
from .normalizers import normalize_map_name
from .rcon_client import load_rcon_targets
COMPETITIVE_WINDOW_GAP_SECONDS = 1800
COMPETITIVE_MODE_PARTIAL = "partial"
COMPETITIVE_MODE_APPROXIMATE = "approximate"
COMPETITIVE_MODE_EXACT = "exact"
RCON_SCHEMA_SQL = """
CREATE TABLE IF NOT EXISTS rcon_historical_targets (
id BIGSERIAL PRIMARY KEY,
target_key TEXT NOT NULL UNIQUE,
external_server_id TEXT,
display_name TEXT NOT NULL,
host TEXT NOT NULL,
port INTEGER NOT NULL,
region TEXT,
game_port INTEGER,
query_port INTEGER,
source_name TEXT NOT NULL,
last_configured_at TEXT NOT NULL,
created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP
);
CREATE TABLE IF NOT EXISTS rcon_historical_capture_runs (
id BIGSERIAL PRIMARY KEY,
mode TEXT NOT NULL,
status TEXT NOT NULL,
target_scope TEXT,
started_at TEXT NOT NULL,
completed_at TEXT,
targets_seen INTEGER NOT NULL DEFAULT 0,
samples_inserted INTEGER NOT NULL DEFAULT 0,
duplicate_samples INTEGER NOT NULL DEFAULT 0,
failed_targets INTEGER NOT NULL DEFAULT 0,
notes TEXT,
created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP
);
CREATE TABLE IF NOT EXISTS rcon_historical_samples (
id BIGSERIAL PRIMARY KEY,
target_id BIGINT NOT NULL REFERENCES rcon_historical_targets(id),
capture_run_id BIGINT REFERENCES rcon_historical_capture_runs(id),
captured_at TEXT NOT NULL,
source_kind TEXT NOT NULL,
status TEXT NOT NULL,
players INTEGER,
max_players INTEGER,
current_map TEXT,
normalized_payload_json TEXT NOT NULL,
raw_payload_json TEXT,
created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
UNIQUE(target_id, captured_at)
);
CREATE TABLE IF NOT EXISTS rcon_historical_checkpoints (
target_id BIGINT PRIMARY KEY REFERENCES rcon_historical_targets(id),
last_successful_capture_at TEXT,
last_sample_at TEXT,
last_run_id BIGINT REFERENCES rcon_historical_capture_runs(id),
last_run_status TEXT,
last_error TEXT,
last_error_at TEXT,
updated_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP
);
CREATE TABLE IF NOT EXISTS rcon_historical_competitive_windows (
id BIGSERIAL PRIMARY KEY,
target_id BIGINT NOT NULL REFERENCES rcon_historical_targets(id),
session_key TEXT NOT NULL UNIQUE,
source_kind TEXT NOT NULL,
map_name TEXT,
map_pretty_name TEXT,
first_seen_at TEXT NOT NULL,
last_seen_at TEXT NOT NULL,
sample_count INTEGER NOT NULL DEFAULT 0,
total_players INTEGER NOT NULL DEFAULT 0,
peak_players INTEGER NOT NULL DEFAULT 0,
last_players INTEGER,
max_players INTEGER,
status TEXT NOT NULL,
confidence_mode TEXT NOT NULL,
capabilities_json TEXT NOT NULL,
latest_payload_json TEXT,
created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP
);
CREATE TABLE IF NOT EXISTS rcon_admin_log_events (
id BIGSERIAL PRIMARY KEY,
target_key TEXT NOT NULL,
external_server_id TEXT,
event_timestamp TEXT,
server_time BIGINT,
relative_time TEXT,
event_type TEXT NOT NULL,
raw_message TEXT NOT NULL,
canonical_message TEXT NOT NULL,
parsed_payload_json TEXT NOT NULL,
raw_entry_json TEXT NOT NULL,
created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
UNIQUE NULLS NOT DISTINCT(target_key, server_time, canonical_message)
);
CREATE TABLE IF NOT EXISTS rcon_player_profile_snapshots (
id BIGSERIAL PRIMARY KEY,
target_key TEXT NOT NULL,
external_server_id TEXT,
player_id TEXT NOT NULL,
player_name TEXT NOT NULL,
source_server_time BIGINT NOT NULL,
event_timestamp TEXT,
first_seen TEXT,
sessions INTEGER,
matches_played INTEGER,
play_time TEXT,
total_kills INTEGER,
total_deaths INTEGER,
teamkills_done INTEGER,
teamkills_received INTEGER,
kd_ratio DOUBLE PRECISION,
favorite_weapons_json TEXT NOT NULL DEFAULT '{}',
victims_json TEXT NOT NULL DEFAULT '{}',
nemesis_json TEXT NOT NULL DEFAULT '{}',
averages_json TEXT NOT NULL DEFAULT '{}',
sanctions_json TEXT NOT NULL DEFAULT '{}',
raw_content TEXT NOT NULL,
created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
UNIQUE(target_key, player_id, source_server_time)
);
CREATE TABLE IF NOT EXISTS rcon_materialized_matches (
id BIGSERIAL PRIMARY KEY,
target_key TEXT NOT NULL,
external_server_id TEXT,
match_key TEXT NOT NULL,
map_name TEXT,
map_pretty_name TEXT,
game_mode TEXT,
started_server_time BIGINT,
ended_server_time BIGINT,
started_at TEXT,
ended_at TEXT,
allied_score INTEGER,
axis_score INTEGER,
winner TEXT,
confidence_mode TEXT NOT NULL,
source_basis TEXT NOT NULL,
created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
UNIQUE(target_key, match_key)
);
CREATE TABLE IF NOT EXISTS rcon_match_player_stats (
id BIGSERIAL PRIMARY KEY,
target_key TEXT NOT NULL,
match_key TEXT NOT NULL,
player_id TEXT NOT NULL,
player_name TEXT NOT NULL,
team TEXT,
kills INTEGER NOT NULL DEFAULT 0,
deaths INTEGER NOT NULL DEFAULT 0,
teamkills INTEGER NOT NULL DEFAULT 0,
deaths_by_teamkill INTEGER NOT NULL DEFAULT 0,
weapons_json TEXT NOT NULL DEFAULT '{}',
death_by_weapons_json TEXT NOT NULL DEFAULT '{}',
most_killed_json TEXT NOT NULL DEFAULT '{}',
death_by_json TEXT NOT NULL DEFAULT '{}',
first_seen_server_time BIGINT,
last_seen_server_time BIGINT,
created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
UNIQUE(target_key, match_key, player_id)
);
CREATE TABLE IF NOT EXISTS rcon_scoreboard_match_candidates (
id BIGSERIAL PRIMARY KEY,
server_slug TEXT NOT NULL,
external_match_id TEXT NOT NULL,
started_at TEXT,
ended_at TEXT,
map_name TEXT,
map_pretty_name TEXT,
allied_score INTEGER,
axis_score INTEGER,
player_count INTEGER,
match_url TEXT NOT NULL,
created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
UNIQUE(server_slug, external_match_id)
);
CREATE INDEX IF NOT EXISTS idx_rcon_historical_samples_target_time
ON rcon_historical_samples(target_id, captured_at DESC);
CREATE INDEX IF NOT EXISTS idx_rcon_historical_windows_target_time
ON rcon_historical_competitive_windows(target_id, last_seen_at DESC);
CREATE INDEX IF NOT EXISTS idx_rcon_admin_log_events_target_time
ON rcon_admin_log_events(target_key, server_time DESC);
CREATE INDEX IF NOT EXISTS idx_rcon_admin_log_events_type
ON rcon_admin_log_events(event_type);
CREATE INDEX IF NOT EXISTS idx_rcon_player_profile_snapshots_player
ON rcon_player_profile_snapshots(target_key, player_id, source_server_time DESC);
CREATE INDEX IF NOT EXISTS idx_rcon_materialized_matches_recent
ON rcon_materialized_matches(target_key, ended_at DESC, ended_server_time DESC);
CREATE INDEX IF NOT EXISTS idx_rcon_match_player_stats_match
ON rcon_match_player_stats(target_key, match_key);
CREATE INDEX IF NOT EXISTS idx_rcon_scoreboard_candidates_server_end
ON rcon_scoreboard_match_candidates(server_slug, ended_at DESC, started_at DESC);
"""
def initialize_postgres_rcon_storage() -> None:
"""Create deterministic PostgreSQL schema for migrated RCON domains."""
with connect_postgres() as connection:
with connection.cursor() as cursor:
cursor.execute(RCON_SCHEMA_SQL)
@contextmanager
def connect_postgres():
"""Yield one PostgreSQL connection with dict-shaped rows."""
try:
import psycopg
from psycopg.rows import dict_row
except ImportError as error: # pragma: no cover - dependency is environment-specific
raise RuntimeError("psycopg is required when HLL_BACKEND_DATABASE_URL is set.") from error
database_url = get_database_url()
if not database_url:
raise RuntimeError("HLL_BACKEND_DATABASE_URL is required for PostgreSQL RCON storage.")
with psycopg.connect(database_url, row_factory=dict_row) as connection:
yield connection
class PostgresCompatConnection:
"""Small DB-API shim for RCON SQL shared with SQLite functions."""
def __init__(self, connection: Any):
self.connection = connection
def execute(self, sql: str, params: Iterable[object] | None = None):
normalized = sql.replace("server_time IS ?", "server_time IS NOT DISTINCT FROM ?")
normalized = normalized.replace("?", "%s")
return self.connection.execute(normalized, tuple(params or ()))
@contextmanager
def connect_postgres_compat():
"""Yield a query shim that accepts the phase-1 SQLite-style placeholders."""
initialize_postgres_rcon_storage()
with connect_postgres() as connection:
yield PostgresCompatConnection(connection)
def start_capture_run(*, mode: str, target_scope: str) -> int:
initialize_postgres_rcon_storage()
with connect_postgres() as connection:
row = connection.execute(
"""
INSERT INTO rcon_historical_capture_runs (mode, status, target_scope, started_at)
VALUES (%s, 'running', %s, %s)
RETURNING id
""",
(mode, target_scope, _utc_now_iso()),
).fetchone()
return int(row["id"])
def finalize_capture_run(
run_id: int,
*,
status: str,
targets_seen: int,
samples_inserted: int,
duplicate_samples: int,
failed_targets: int,
notes: str | None,
) -> None:
initialize_postgres_rcon_storage()
with connect_postgres() as connection:
connection.execute(
"""
UPDATE rcon_historical_capture_runs
SET status = %s,
completed_at = %s,
targets_seen = %s,
samples_inserted = %s,
duplicate_samples = %s,
failed_targets = %s,
notes = %s
WHERE id = %s
""",
(
status,
_utc_now_iso(),
targets_seen,
samples_inserted,
duplicate_samples,
failed_targets,
notes,
run_id,
),
)
def persist_sample(
*,
run_id: int,
captured_at: str,
target: Mapping[str, object],
normalized_payload: Mapping[str, object],
raw_payload: Mapping[str, object] | None,
) -> dict[str, int]:
initialize_postgres_rcon_storage()
with connect_postgres() as connection:
target_id = _upsert_target(connection, target=target)
row = connection.execute(
"""
INSERT INTO rcon_historical_samples (
target_id, capture_run_id, captured_at, source_kind, status, players,
max_players, current_map, normalized_payload_json, raw_payload_json
) VALUES (%s, %s, %s, 'rcon-live-sample', %s, %s, %s, %s, %s, %s)
ON CONFLICT(target_id, captured_at) DO NOTHING
RETURNING id
""",
(
target_id,
run_id,
captured_at,
normalized_payload.get("status") or "unknown",
normalized_payload.get("players"),
normalized_payload.get("max_players"),
normalized_payload.get("current_map"),
json.dumps(dict(normalized_payload), separators=(",", ":")),
json.dumps(dict(raw_payload), separators=(",", ":")) if raw_payload else None,
),
).fetchone()
inserted = int(row is not None)
_upsert_checkpoint_success(
connection,
target_id=target_id,
run_id=run_id,
captured_at=captured_at,
)
if inserted:
_upsert_competitive_window(
connection,
target_id=target_id,
captured_at=captured_at,
normalized_payload=normalized_payload,
)
return {"samples_inserted": inserted, "duplicate_samples": 0 if inserted else 1}
def mark_capture_failure(
*,
run_id: int,
target: Mapping[str, object],
error_message: str,
) -> None:
initialize_postgres_rcon_storage()
with connect_postgres() as connection:
target_id = _upsert_target(connection, target=target)
connection.execute(
"""
INSERT INTO rcon_historical_checkpoints (
target_id, last_run_id, last_run_status, last_error, last_error_at
) VALUES (%s, %s, 'failed', %s, %s)
ON CONFLICT(target_id) DO UPDATE SET
last_run_id = EXCLUDED.last_run_id,
last_run_status = EXCLUDED.last_run_status,
last_error = EXCLUDED.last_error,
last_error_at = EXCLUDED.last_error_at,
updated_at = CURRENT_TIMESTAMP
""",
(target_id, run_id, error_message, _utc_now_iso()),
)
def list_target_statuses() -> list[dict[str, object]]:
rows = _fetchall(
"""
SELECT
targets.target_key,
targets.external_server_id,
targets.display_name,
targets.host,
targets.port,
targets.region,
targets.source_name,
checkpoints.last_successful_capture_at,
checkpoints.last_sample_at,
checkpoints.last_run_id,
checkpoints.last_run_status,
checkpoints.last_error,
checkpoints.last_error_at,
(SELECT MIN(samples.captured_at) FROM rcon_historical_samples AS samples
WHERE samples.target_id = targets.id) AS first_sample_at,
(SELECT MAX(samples.captured_at) FROM rcon_historical_samples AS samples
WHERE samples.target_id = targets.id) AS latest_sample_at,
(SELECT COUNT(*) FROM rcon_historical_samples AS samples
WHERE samples.target_id = targets.id) AS sample_count
FROM rcon_historical_targets AS targets
LEFT JOIN rcon_historical_checkpoints AS checkpoints
ON checkpoints.target_id = targets.id
ORDER BY targets.display_name ASC, targets.target_key ASC
"""
)
return [
{
**dict(row),
"sample_count": int(row["sample_count"] or 0),
"last_sample_at": row["latest_sample_at"] or row["last_sample_at"],
}
for row in rows
]
def list_recent_samples(*, target_key: str | None, limit: int) -> list[dict[str, object]]:
where_clause, params = _target_where_clause(target_key)
rows = _fetchall(
f"""
SELECT targets.target_key, targets.external_server_id, targets.display_name,
targets.region, samples.captured_at, samples.status, samples.players,
samples.max_players, samples.current_map
FROM rcon_historical_samples AS samples
INNER JOIN rcon_historical_targets AS targets ON targets.id = samples.target_id
{where_clause}
ORDER BY samples.captured_at DESC, targets.display_name ASC
LIMIT %s
""",
[*params, limit],
)
return [dict(row) for row in rows]
def list_competitive_windows(*, target_key: str | None, limit: int) -> list[dict[str, object]]:
where_clause, params = _target_where_clause(target_key)
rows = _fetchall(
f"""
SELECT targets.target_key, targets.external_server_id, targets.display_name,
targets.region, windows.session_key, windows.map_name,
windows.map_pretty_name, windows.first_seen_at, windows.last_seen_at,
windows.sample_count, windows.total_players, windows.peak_players,
windows.last_players, windows.max_players, windows.status,
windows.confidence_mode, windows.capabilities_json,
windows.latest_payload_json
FROM rcon_historical_competitive_windows AS windows
INNER JOIN rcon_historical_targets AS targets ON targets.id = windows.target_id
{where_clause}
ORDER BY windows.last_seen_at DESC, targets.display_name ASC
LIMIT %s
""",
[*params, limit],
)
return [_serialize_window(row) for row in rows]
def count_samples_since(since: str | None) -> int:
if not since:
return 0
row = _fetchone(
"SELECT COUNT(*) AS sample_count FROM rcon_historical_samples WHERE captured_at > %s",
(since,),
)
return int(row["sample_count"] or 0) if row else 0
def list_competitive_summary_rows(*, target_key: str | None) -> list[dict[str, object]]:
where_clause, params = _target_where_clause(target_key)
rows = _fetchall(
f"""
SELECT targets.target_key, targets.external_server_id, targets.display_name,
targets.region, checkpoints.last_successful_capture_at,
checkpoints.last_run_status, checkpoints.last_error,
checkpoints.last_error_at, COUNT(windows.id) AS window_count,
COALESCE(SUM(windows.sample_count), 0) AS sample_count,
MIN(windows.first_seen_at) AS first_seen_at,
MAX(windows.last_seen_at) AS last_seen_at,
COALESCE(MAX(windows.peak_players), 0) AS peak_players
FROM rcon_historical_targets AS targets
LEFT JOIN rcon_historical_checkpoints AS checkpoints ON checkpoints.target_id = targets.id
LEFT JOIN rcon_historical_competitive_windows AS windows ON windows.target_id = targets.id
{where_clause}
GROUP BY targets.id, checkpoints.target_id
ORDER BY targets.display_name ASC, targets.target_key ASC
""",
params,
)
return [
{
**dict(row),
"window_count": int(row["window_count"] or 0),
"sample_count": int(row["sample_count"] or 0),
"peak_players": int(row["peak_players"] or 0),
}
for row in rows
]
def find_competitive_window(
*,
server_key: str,
ended_at: str | None,
map_name: str | None,
) -> dict[str, object] | None:
if not ended_at:
return None
aliases = _expand_target_key_aliases(server_key)
candidates = _fetchall(
"""
SELECT windows.session_key, windows.first_seen_at, windows.last_seen_at,
windows.map_name, windows.map_pretty_name, windows.sample_count,
windows.total_players, windows.peak_players, windows.confidence_mode,
windows.capabilities_json, windows.latest_payload_json
FROM rcon_historical_competitive_windows AS windows
INNER JOIN rcon_historical_targets AS targets ON targets.id = windows.target_id
WHERE targets.target_key = ANY(%s) OR targets.external_server_id = ANY(%s)
ORDER BY windows.last_seen_at DESC
LIMIT 12
""",
(aliases, aliases),
)
ended_point = _parse_timestamp(ended_at)
if ended_point is None:
return None
normalized_map_name = normalize_map_name(map_name)
best_row: dict[str, object] | None = None
best_distance: float | None = None
for row in candidates:
row_map = normalize_map_name(row["map_pretty_name"] or row["map_name"])
if normalized_map_name and row_map and normalized_map_name != row_map:
continue
row_last = _parse_timestamp(row["last_seen_at"])
if row_last is None:
continue
distance = abs((row_last - ended_point).total_seconds())
if best_distance is None or distance < best_distance:
best_row = dict(row)
best_distance = distance
if best_row is None or best_distance is None or best_distance > 21600:
return None
sample_count = int(best_row["sample_count"] or 0)
return {
"session_key": best_row["session_key"],
"first_seen_at": best_row["first_seen_at"],
"last_seen_at": best_row["last_seen_at"],
"duration_seconds": _calculate_duration_seconds(
best_row["first_seen_at"],
best_row["last_seen_at"],
),
"map_name": best_row["map_name"],
"map_pretty_name": best_row["map_pretty_name"] or best_row["map_name"],
"sample_count": sample_count,
"average_players": (
round((int(best_row["total_players"] or 0) / sample_count), 2)
if sample_count > 0
else 0.0
),
"peak_players": int(best_row["peak_players"] or 0),
"confidence_mode": best_row["confidence_mode"],
"capabilities": _deserialize_json_object(best_row["capabilities_json"]),
}
def get_competitive_window_by_session(
*,
server_key: str,
session_key: str,
) -> dict[str, object] | None:
normalized_session_key = str(session_key or "").strip()
if not normalized_session_key:
return None
aliases = _expand_target_key_aliases(server_key)
row = _fetchone(
"""
SELECT targets.target_key, targets.external_server_id, targets.display_name,
targets.region, windows.session_key, windows.map_name,
windows.map_pretty_name, windows.first_seen_at, windows.last_seen_at,
windows.sample_count, windows.total_players, windows.peak_players,
windows.confidence_mode, windows.capabilities_json,
windows.latest_payload_json
FROM rcon_historical_competitive_windows AS windows
INNER JOIN rcon_historical_targets AS targets ON targets.id = windows.target_id
WHERE windows.session_key = %s
AND (targets.target_key = ANY(%s) OR targets.external_server_id = ANY(%s))
LIMIT 1
""",
(normalized_session_key, aliases, aliases),
)
return _serialize_window(row) if row else None
def list_scoreboard_candidates(*, server_slug: str, limit: int) -> list[dict[str, object]]:
rows = _fetchall(
"""
SELECT external_match_id, started_at, ended_at, map_name, map_pretty_name,
allied_score, axis_score, player_count, match_url
FROM rcon_scoreboard_match_candidates
WHERE server_slug = %s
ORDER BY COALESCE(ended_at, started_at) DESC
LIMIT %s
""",
(server_slug, limit),
)
return [dict(row) for row in rows]
def upsert_scoreboard_candidates(
*,
server_slug: str,
candidates: Iterable[Mapping[str, object]],
) -> int:
"""Cache trusted scoreboard correlation candidates in PostgreSQL."""
rows = [candidate for candidate in candidates if candidate.get("match_url")]
if not rows:
return 0
initialize_postgres_rcon_storage()
inserted_or_updated = 0
with connect_postgres() as connection:
for candidate in rows:
connection.execute(
"""
INSERT INTO rcon_scoreboard_match_candidates (
server_slug, external_match_id, started_at, ended_at, map_name,
map_pretty_name, allied_score, axis_score, player_count, match_url
) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
ON CONFLICT(server_slug, external_match_id) DO UPDATE SET
started_at = EXCLUDED.started_at,
ended_at = EXCLUDED.ended_at,
map_name = EXCLUDED.map_name,
map_pretty_name = EXCLUDED.map_pretty_name,
allied_score = EXCLUDED.allied_score,
axis_score = EXCLUDED.axis_score,
player_count = EXCLUDED.player_count,
match_url = EXCLUDED.match_url,
updated_at = CURRENT_TIMESTAMP
""",
(
server_slug,
str(candidate.get("external_match_id") or ""),
candidate.get("started_at"),
candidate.get("ended_at"),
candidate.get("map_name"),
candidate.get("map_pretty_name"),
candidate.get("allied_score"),
candidate.get("axis_score"),
candidate.get("player_count"),
candidate["match_url"],
),
)
inserted_or_updated += 1
return inserted_or_updated
def count_migrated_tables() -> dict[str, int]:
initialize_postgres_rcon_storage()
table_names = (
"rcon_admin_log_events",
"rcon_player_profile_snapshots",
"rcon_materialized_matches",
"rcon_match_player_stats",
"rcon_historical_targets",
"rcon_historical_samples",
"rcon_historical_competitive_windows",
"rcon_scoreboard_match_candidates",
)
with connect_postgres() as connection:
return {
table_name: int(
connection.execute(f"SELECT COUNT(*) AS count FROM {table_name}").fetchone()[
"count"
]
or 0
)
for table_name in table_names
}
def _fetchall(sql: str, params: Iterable[object] = ()) -> list[dict[str, object]]:
initialize_postgres_rcon_storage()
with connect_postgres() as connection:
return [dict(row) for row in connection.execute(sql, tuple(params)).fetchall()]
def _fetchone(sql: str, params: Iterable[object] = ()) -> dict[str, object] | None:
initialize_postgres_rcon_storage()
with connect_postgres() as connection:
row = connection.execute(sql, tuple(params)).fetchone()
return dict(row) if row else None
def _upsert_target(connection: Any, *, target: Mapping[str, object]) -> int:
target_key = str(target.get("target_key") or "").strip()
display_name = str(target.get("name") or target.get("display_name") or target_key).strip()
host = str(target.get("host") or "").strip()
port = int(target.get("port") or 0)
if not target_key or not host or port <= 0:
raise ValueError("Prospective RCON targets require target_key, host and port.")
row = connection.execute(
"""
INSERT INTO rcon_historical_targets (
target_key, external_server_id, display_name, host, port, region,
game_port, query_port, source_name, last_configured_at
) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
ON CONFLICT(target_key) DO UPDATE SET
external_server_id = EXCLUDED.external_server_id,
display_name = EXCLUDED.display_name,
host = EXCLUDED.host,
port = EXCLUDED.port,
region = EXCLUDED.region,
game_port = EXCLUDED.game_port,
query_port = EXCLUDED.query_port,
source_name = EXCLUDED.source_name,
last_configured_at = EXCLUDED.last_configured_at,
updated_at = CURRENT_TIMESTAMP
RETURNING id
""",
(
target_key,
target.get("external_server_id"),
display_name,
host,
port,
target.get("region"),
target.get("game_port"),
target.get("query_port"),
str(target.get("source_name") or "community-hispana-rcon"),
_utc_now_iso(),
),
).fetchone()
return int(row["id"])
def _upsert_checkpoint_success(
connection: Any,
*,
target_id: int,
run_id: int,
captured_at: str,
) -> None:
connection.execute(
"""
INSERT INTO rcon_historical_checkpoints (
target_id, last_successful_capture_at, last_sample_at, last_run_id,
last_run_status, last_error, last_error_at
) VALUES (%s, %s, %s, %s, 'success', NULL, NULL)
ON CONFLICT(target_id) DO UPDATE SET
last_successful_capture_at = EXCLUDED.last_successful_capture_at,
last_sample_at = EXCLUDED.last_sample_at,
last_run_id = EXCLUDED.last_run_id,
last_run_status = EXCLUDED.last_run_status,
last_error = NULL,
last_error_at = NULL,
updated_at = CURRENT_TIMESTAMP
""",
(target_id, captured_at, captured_at, run_id),
)
def _upsert_competitive_window(
connection: Any,
*,
target_id: int,
captured_at: str,
normalized_payload: Mapping[str, object],
) -> None:
current_map_raw = str(normalized_payload.get("current_map") or "").strip()
if not current_map_raw:
return
map_pretty_name = normalize_map_name(current_map_raw) or current_map_raw
players = int(normalized_payload.get("players") or 0)
max_players = normalized_payload.get("max_players")
status = str(normalized_payload.get("status") or "unknown")
latest_window = connection.execute(
"""
SELECT *
FROM rcon_historical_competitive_windows
WHERE target_id = %s
ORDER BY last_seen_at DESC, id DESC
LIMIT 1
""",
(target_id,),
).fetchone()
if latest_window and _should_extend_competitive_window(
latest_window=dict(latest_window),
captured_at=captured_at,
current_map=current_map_raw,
):
connection.execute(
"""
UPDATE rcon_historical_competitive_windows
SET map_name = %s,
map_pretty_name = %s,
last_seen_at = %s,
sample_count = sample_count + 1,
total_players = total_players + %s,
peak_players = GREATEST(peak_players, %s),
last_players = %s,
max_players = %s,
status = %s,
confidence_mode = %s,
capabilities_json = %s,
latest_payload_json = %s,
updated_at = CURRENT_TIMESTAMP
WHERE id = %s
""",
(
current_map_raw,
map_pretty_name,
captured_at,
players,
players,
players,
max_players,
status,
COMPETITIVE_MODE_APPROXIMATE,
json.dumps(_build_competitive_capabilities(), separators=(",", ":")),
json.dumps(dict(normalized_payload), separators=(",", ":")),
latest_window["id"],
),
)
return
connection.execute(
"""
INSERT INTO rcon_historical_competitive_windows (
target_id, session_key, source_kind, map_name, map_pretty_name,
first_seen_at, last_seen_at, sample_count, total_players,
peak_players, last_players, max_players, status, confidence_mode,
capabilities_json, latest_payload_json
) VALUES (%s, %s, 'rcon-historical-samples', %s, %s, %s, %s, 1,
%s, %s, %s, %s, %s, %s, %s, %s)
""",
(
target_id,
f"{target_id}:{captured_at}",
current_map_raw,
map_pretty_name,
captured_at,
captured_at,
players,
players,
players,
max_players,
status,
COMPETITIVE_MODE_APPROXIMATE,
json.dumps(_build_competitive_capabilities(), separators=(",", ":")),
json.dumps(dict(normalized_payload), separators=(",", ":")),
),
)
def _target_where_clause(target_key: str | None) -> tuple[str, list[object]]:
if not target_key:
return "", []
aliases = _expand_target_key_aliases(target_key)
return "WHERE targets.target_key = ANY(%s) OR targets.external_server_id = ANY(%s)", [
aliases,
aliases,
]
def _expand_target_key_aliases(target_key: str) -> list[str]:
normalized_target_key = str(target_key or "").strip()
aliases = {normalized_target_key}
try:
configured_targets = load_rcon_targets()
except Exception:
configured_targets = ()
for target in configured_targets:
external_server_id = str(target.external_server_id or "").strip()
legacy_target_key = f"rcon:{target.host}:{target.port}"
if external_server_id and external_server_id == normalized_target_key:
aliases.update((legacy_target_key, external_server_id))
elif legacy_target_key == normalized_target_key:
aliases.add(legacy_target_key)
if external_server_id:
aliases.add(external_server_id)
return sorted(alias for alias in aliases if alias)
def _serialize_window(row: Mapping[str, object]) -> dict[str, object]:
sample_count = int(row["sample_count"] or 0)
return {
"target_key": row["target_key"],
"external_server_id": row["external_server_id"],
"display_name": row["display_name"],
"region": row["region"],
"session_key": row["session_key"],
"map_name": row["map_name"],
"map_pretty_name": row["map_pretty_name"] or row["map_name"],
"first_seen_at": row["first_seen_at"],
"last_seen_at": row["last_seen_at"],
"duration_seconds": _calculate_duration_seconds(
row["first_seen_at"],
row["last_seen_at"],
),
"sample_count": sample_count,
"average_players": (
round((int(row["total_players"] or 0) / sample_count), 2)
if sample_count > 0
else 0.0
),
"peak_players": int(row["peak_players"] or 0),
"last_players": row.get("last_players"),
"max_players": row.get("max_players"),
"status": row.get("status"),
"confidence_mode": row["confidence_mode"],
"capabilities": _deserialize_json_object(row["capabilities_json"]),
"latest_payload": _deserialize_json_object(row["latest_payload_json"]),
}
def _should_extend_competitive_window(
*,
latest_window: Mapping[str, object],
captured_at: str,
current_map: str,
) -> bool:
if normalize_map_name(latest_window.get("map_name")) != normalize_map_name(current_map):
return False
latest_seen = _parse_timestamp(latest_window.get("last_seen_at"))
captured_point = _parse_timestamp(captured_at)
if latest_seen is None or captured_point is None:
return False
return (captured_point - latest_seen).total_seconds() <= COMPETITIVE_WINDOW_GAP_SECONDS
def _build_competitive_capabilities() -> dict[str, object]:
return {
"recent_matches": COMPETITIVE_MODE_APPROXIMATE,
"server_summary": COMPETITIVE_MODE_EXACT,
"competitive_quality": COMPETITIVE_MODE_PARTIAL,
"result": "session-score",
"gamestate": "session",
"player_stats": "unavailable",
}
def _deserialize_json_object(raw_value: object) -> dict[str, object]:
if isinstance(raw_value, str) and raw_value.strip():
try:
parsed = json.loads(raw_value)
except json.JSONDecodeError:
return {}
return parsed if isinstance(parsed, dict) else {}
return {}
def _calculate_duration_seconds(first_seen_at: object, last_seen_at: object) -> int | None:
first_point = _parse_timestamp(first_seen_at)
last_point = _parse_timestamp(last_seen_at)
if first_point is None or last_point is None:
return None
return max(0, int((last_point - first_point).total_seconds()))
def _parse_timestamp(raw_value: object) -> datetime | None:
if not isinstance(raw_value, str) or not raw_value.strip():
return None
try:
timestamp = datetime.fromisoformat(raw_value.replace("Z", "+00:00"))
except ValueError:
return None
if timestamp.tzinfo is None:
timestamp = timestamp.replace(tzinfo=timezone.utc)
return timestamp.astimezone(timezone.utc)
def _utc_now_iso() -> str:
return datetime.now(timezone.utc).isoformat().replace("+00:00", "Z")

View File

@@ -10,7 +10,7 @@ from collections.abc import Iterable
from contextlib import closing
from pathlib import Path
from .config import get_storage_path
from .config import get_storage_path, use_postgres_rcon_storage
from .normalizers import normalize_map_name
from .rcon_admin_log_storage import initialize_rcon_admin_log_storage
from .rcon_historical_storage import list_rcon_historical_competitive_windows
@@ -23,6 +23,12 @@ SESSION_RESULT_SOURCE = "rcon-session"
def initialize_rcon_materialized_storage(*, db_path: Path | None = None) -> Path:
"""Create SQLite structures used by the materialized RCON match pipeline."""
if use_postgres_rcon_storage(explicit_sqlite_path=db_path):
from .postgres_rcon_storage import initialize_postgres_rcon_storage
initialize_postgres_rcon_storage()
return get_storage_path()
resolved_path = initialize_rcon_admin_log_storage(db_path=db_path)
with closing(connect_sqlite_writer(resolved_path)) as connection:
with connection:
@@ -85,6 +91,46 @@ def initialize_rcon_materialized_storage(*, db_path: Path | None = None) -> Path
def materialize_rcon_admin_log(*, db_path: Path | None = None) -> dict[str, object]:
"""Materialize matches and player stats from stored AdminLog events."""
resolved_path = initialize_rcon_materialized_storage(db_path=db_path)
if use_postgres_rcon_storage(explicit_sqlite_path=db_path):
from .postgres_rcon_storage import connect_postgres_compat
with connect_postgres_compat() as connection:
payload = _materialize_rcon_admin_log_with_connection(
connection,
session_window_db_path=None,
caught_errors=(Exception,),
)
freshness = summarize_rcon_materialization_status()
return {
**payload,
"latest_materialized_matches": freshness["latest_materialized_matches"],
"latest_admin_log_match_end_events": freshness["latest_admin_log_match_end_events"],
"match_end_status": freshness["match_end_status"],
}
with closing(connect_sqlite_writer(resolved_path)) as connection:
with connection:
payload = _materialize_rcon_admin_log_with_connection(
connection,
session_window_db_path=resolved_path,
caught_errors=(sqlite3.Error,),
)
freshness = summarize_rcon_materialization_status(db_path=resolved_path)
return {
**payload,
"latest_materialized_matches": freshness["latest_materialized_matches"],
"latest_admin_log_match_end_events": freshness["latest_admin_log_match_end_events"],
"match_end_status": freshness["match_end_status"],
}
def _materialize_rcon_admin_log_with_connection(
connection: object,
*,
session_window_db_path: Path | None,
caught_errors: tuple[type[BaseException], ...],
) -> dict[str, object]:
errors: list[str] = []
matches_seen = 0
matches_materialized = 0
@@ -93,40 +139,39 @@ def materialize_rcon_admin_log(*, db_path: Path | None = None) -> dict[str, obje
player_stats_materialized = 0
player_stats_updated = 0
with closing(connect_sqlite_writer(resolved_path)) as connection:
with connection:
try:
match_rows = _derive_admin_log_matches(connection)
matches_seen = len(match_rows)
for row in match_rows:
outcome = _upsert_match(connection, row)
matches_materialized += int(outcome == "inserted")
matches_updated += int(outcome == "updated")
session_rows = _derive_session_fallback_matches(connection, db_path=resolved_path)
matches_seen += len(session_rows)
for row in session_rows:
outcome = _upsert_match(connection, row)
matches_materialized += int(outcome == "inserted")
matches_updated += int(outcome == "updated")
try:
match_rows = _derive_admin_log_matches(connection)
matches_seen = len(match_rows)
for row in match_rows:
outcome = _upsert_match(connection, row)
matches_materialized += int(outcome == "inserted")
matches_updated += int(outcome == "updated")
session_rows = _derive_session_fallback_matches(
connection,
db_path=session_window_db_path,
)
matches_seen += len(session_rows)
for row in session_rows:
outcome = _upsert_match(connection, row)
matches_materialized += int(outcome == "inserted")
matches_updated += int(outcome == "updated")
persisted_matches = _list_materialized_matches(connection)
for match in persisted_matches:
stats = _derive_player_stats_for_match(connection, match)
player_stats_seen += len(stats)
connection.execute(
"""
DELETE FROM rcon_match_player_stats
WHERE target_key = ? AND match_key = ?
""",
(match["target_key"], match["match_key"]),
)
for stat in stats:
_insert_player_stat(connection, stat)
player_stats_materialized += 1
except sqlite3.Error as error:
errors.append(str(error))
freshness = summarize_rcon_materialization_status(db_path=resolved_path)
persisted_matches = _list_materialized_matches(connection)
for match in persisted_matches:
stats = _derive_player_stats_for_match(connection, match)
player_stats_seen += len(stats)
connection.execute(
"""
DELETE FROM rcon_match_player_stats
WHERE target_key = ? AND match_key = ?
""",
(match["target_key"], match["match_key"]),
)
for stat in stats:
_insert_player_stat(connection, stat)
player_stats_materialized += 1
except caught_errors as error:
errors.append(str(error))
return {
"matches_seen": matches_seen,
"matches_materialized": matches_materialized,
@@ -135,8 +180,6 @@ def materialize_rcon_admin_log(*, db_path: Path | None = None) -> dict[str, obje
"player_stats_materialized": player_stats_materialized,
"player_stats_updated": player_stats_updated,
"errors": errors,
"latest_materialized_matches": freshness["latest_materialized_matches"],
"latest_admin_log_match_end_events": freshness["latest_admin_log_match_end_events"],
}
@@ -159,7 +202,13 @@ def list_materialized_rcon_matches(
params.append(MATCH_RESULT_SOURCE)
where = "WHERE " + " AND ".join(clauses) if clauses else ""
params.append(limit)
with closing(connect_sqlite_readonly(resolved_path)) as connection:
if use_postgres_rcon_storage(explicit_sqlite_path=db_path):
from .postgres_rcon_storage import connect_postgres_compat
connection_scope = connect_postgres_compat()
else:
connection_scope = closing(connect_sqlite_readonly(resolved_path))
with connection_scope as connection:
rows = connection.execute(
f"""
SELECT
@@ -196,7 +245,13 @@ def get_materialized_rcon_match_detail(
) -> dict[str, object] | None:
"""Return one materialized match with player stats."""
resolved_path = initialize_rcon_materialized_storage(db_path=db_path)
with closing(connect_sqlite_readonly(resolved_path)) as connection:
if use_postgres_rcon_storage(explicit_sqlite_path=db_path):
from .postgres_rcon_storage import connect_postgres_compat
connection_scope = connect_postgres_compat()
else:
connection_scope = closing(connect_sqlite_readonly(resolved_path))
with connection_scope as connection:
match = connection.execute(
"""
SELECT *
@@ -248,7 +303,13 @@ def get_materialized_rcon_match_detail(
def summarize_rcon_materialization_status(*, db_path: Path | None = None) -> dict[str, object]:
"""Return a small diagnostic summary for stored RCON materialization state."""
resolved_path = initialize_rcon_materialized_storage(db_path=db_path)
with closing(connect_sqlite_readonly(resolved_path)) as connection:
if use_postgres_rcon_storage(explicit_sqlite_path=db_path):
from .postgres_rcon_storage import connect_postgres_compat
connection_scope = connect_postgres_compat()
else:
connection_scope = closing(connect_sqlite_readonly(resolved_path))
with connection_scope as connection:
match_count = connection.execute(
"SELECT COUNT(*) AS count FROM rcon_materialized_matches"
).fetchone()["count"]
@@ -259,7 +320,7 @@ def summarize_rcon_materialization_status(*, db_path: Path | None = None) -> dic
SELECT 1
FROM rcon_match_player_stats
GROUP BY target_key, match_key
)
) AS stats_matches
"""
).fetchone()["count"]
ranges = connection.execute(
@@ -301,7 +362,7 @@ def summarize_rcon_materialization_status(*, db_path: Path | None = None) -> dic
) AS row_number
FROM rcon_materialized_matches
WHERE source_basis = ?
)
) AS ranked_matches
WHERE row_number = 1
ORDER BY target_key ASC
""",
@@ -328,6 +389,11 @@ def summarize_rcon_materialization_status(*, db_path: Path | None = None) -> dic
"event_counts": [dict(row) for row in event_counts],
"latest_materialized_matches": [dict(row) for row in latest_matches],
"latest_admin_log_match_end_events": [dict(row) for row in latest_match_end_events],
"match_end_status": (
"admin-log-match-end-events-available"
if latest_match_end_events
else "no-admin-log-match-end-events-stored"
),
}
@@ -360,7 +426,7 @@ def _derive_admin_log_matches(connection: sqlite3.Connection) -> list[dict[str,
def _derive_session_fallback_matches(
connection: sqlite3.Connection,
*,
db_path: Path,
db_path: Path | None,
) -> list[dict[str, object]]:
rows: list[dict[str, object]] = []
existing = {

View File

@@ -8,7 +8,7 @@ import sqlite3
from collections.abc import Mapping
from pathlib import Path
from .config import get_storage_path
from .config import get_storage_path, use_postgres_rcon_storage
from .rcon_admin_log_parser import parse_rcon_admin_log_entry
from .rcon_admin_log_parser import parse_rcon_player_profile_snapshot
from .rcon_historical_storage import initialize_rcon_historical_storage
@@ -17,6 +17,12 @@ from .sqlite_utils import connect_sqlite_writer
def initialize_rcon_admin_log_storage(*, db_path: Path | None = None) -> Path:
"""Create SQLite structures for parsed RCON AdminLog events."""
if use_postgres_rcon_storage(explicit_sqlite_path=db_path):
from .postgres_rcon_storage import initialize_postgres_rcon_storage
initialize_postgres_rcon_storage()
return get_storage_path()
resolved_path = initialize_rcon_historical_storage(db_path=db_path)
with connect_sqlite_writer(resolved_path) as connection:
@@ -90,6 +96,9 @@ def persist_rcon_admin_log_entries(
db_path: Path | None = None,
) -> dict[str, int]:
"""Persist raw and parsed AdminLog entries idempotently."""
if use_postgres_rcon_storage(explicit_sqlite_path=db_path):
return _persist_rcon_admin_log_entries_postgres(target=target, entries=entries)
resolved_path = initialize_rcon_admin_log_storage(db_path=db_path)
target_key = str(target.get("target_key") or target.get("external_server_id") or "")
if not target_key:
@@ -278,6 +287,25 @@ def _ensure_canonical_message_column(connection: sqlite3.Connection) -> None:
def list_rcon_admin_log_event_counts(*, db_path: Path | None = None) -> list[dict[str, object]]:
"""Return event counts grouped by target and event type."""
if use_postgres_rcon_storage(explicit_sqlite_path=db_path):
from .postgres_rcon_storage import connect_postgres_compat
with connect_postgres_compat() as connection:
rows = connection.execute(
"""
SELECT
target_key,
event_type,
COUNT(*) AS event_count,
MIN(server_time) AS first_server_time,
MAX(server_time) AS last_server_time
FROM rcon_admin_log_events
GROUP BY target_key, event_type
ORDER BY target_key ASC, event_count DESC
"""
).fetchall()
return [dict(row) for row in rows]
resolved_path = db_path or get_storage_path()
initialize_rcon_admin_log_storage(db_path=resolved_path)
@@ -310,6 +338,29 @@ def get_latest_rcon_player_profile_summaries(
requested_ids = [str(player_id).strip() for player_id in player_ids if str(player_id).strip()]
if not target_key or not requested_ids:
return {}
if use_postgres_rcon_storage(explicit_sqlite_path=db_path):
from .postgres_rcon_storage import connect_postgres_compat
placeholders = ",".join("?" for _ in requested_ids)
with connect_postgres_compat() as connection:
rows = connection.execute(
f"""
SELECT snapshots.*
FROM rcon_player_profile_snapshots AS snapshots
INNER JOIN (
SELECT player_id, MAX(source_server_time) AS latest_source_server_time
FROM rcon_player_profile_snapshots
WHERE target_key = ?
AND player_id IN ({placeholders})
GROUP BY player_id
) AS latest
ON latest.player_id = snapshots.player_id
AND latest.latest_source_server_time = snapshots.source_server_time
WHERE snapshots.target_key = ?
""",
[target_key, *requested_ids, target_key],
).fetchall()
return {str(row["player_id"]): _build_safe_profile_summary(row) for row in rows}
resolved_path = db_path or get_storage_path()
initialize_rcon_admin_log_storage(db_path=resolved_path)
@@ -369,3 +420,68 @@ def _json_mapping(raw_value: object) -> dict[str, object]:
except json.JSONDecodeError:
return {}
return parsed if isinstance(parsed, dict) else {}
def _persist_rcon_admin_log_entries_postgres(
*,
target: Mapping[str, object],
entries: list[dict[str, object]],
) -> dict[str, int]:
from .postgres_rcon_storage import connect_postgres_compat
target_key = str(target.get("target_key") or target.get("external_server_id") or "")
if not target_key:
raise ValueError("target must include target_key or external_server_id")
external_server_id = target.get("external_server_id")
inserted = 0
duplicates = 0
with connect_postgres_compat() as connection:
for entry in entries:
parsed = parse_rcon_admin_log_entry(entry)
raw_message = str(parsed.get("raw_message") or "")
canonical_message = _canonicalize_admin_log_message(raw_message)
cursor = connection.execute(
"""
INSERT INTO rcon_admin_log_events (
target_key,
external_server_id,
event_timestamp,
server_time,
relative_time,
event_type,
raw_message,
canonical_message,
parsed_payload_json,
raw_entry_json
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT DO NOTHING
""",
(
target_key,
external_server_id,
parsed.get("timestamp"),
parsed.get("server_time"),
parsed.get("relative_time"),
parsed.get("event_type") or "unknown",
raw_message,
canonical_message,
json.dumps(parsed, ensure_ascii=False, separators=(",", ":")),
json.dumps(entry, ensure_ascii=False, separators=(",", ":")),
),
)
if int(cursor.rowcount or 0):
inserted += 1
else:
duplicates += 1
_persist_profile_snapshot_if_present(
connection,
target_key=target_key,
external_server_id=external_server_id,
parsed=parsed,
)
return {
"events_seen": len(entries),
"events_inserted": inserted,
"duplicate_events": duplicates,
}

View File

@@ -8,7 +8,7 @@ from collections.abc import Mapping
from datetime import datetime, timezone
from pathlib import Path
from .config import get_storage_path
from .config import get_storage_path, use_postgres_rcon_storage
from .normalizers import normalize_map_name
from .rcon_client import load_rcon_targets
from .sqlite_utils import connect_sqlite_readonly, connect_sqlite_writer
@@ -22,6 +22,12 @@ COMPETITIVE_MODE_EXACT = "exact"
def initialize_rcon_historical_storage(*, db_path: Path | None = None) -> Path:
"""Create the SQLite structures used by prospective RCON capture."""
if use_postgres_rcon_storage(explicit_sqlite_path=db_path):
from .postgres_rcon_storage import initialize_postgres_rcon_storage
initialize_postgres_rcon_storage()
return get_storage_path()
resolved_path = db_path or get_storage_path()
resolved_path.parent.mkdir(parents=True, exist_ok=True)
@@ -131,6 +137,11 @@ def start_rcon_historical_capture_run(
db_path: Path | None = None,
) -> int:
"""Create one run row for prospective RCON capture."""
if use_postgres_rcon_storage(explicit_sqlite_path=db_path):
from .postgres_rcon_storage import start_capture_run
return start_capture_run(mode=mode, target_scope=target_scope)
resolved_path = initialize_rcon_historical_storage(db_path=db_path)
with _connect(resolved_path) as connection:
cursor = connection.execute(
@@ -159,6 +170,20 @@ def finalize_rcon_historical_capture_run(
db_path: Path | None = None,
) -> None:
"""Finalize one prospective RCON capture run."""
if use_postgres_rcon_storage(explicit_sqlite_path=db_path):
from .postgres_rcon_storage import finalize_capture_run
finalize_capture_run(
run_id,
status=status,
targets_seen=targets_seen,
samples_inserted=samples_inserted,
duplicate_samples=duplicate_samples,
failed_targets=failed_targets,
notes=notes,
)
return
resolved_path = initialize_rcon_historical_storage(db_path=db_path)
with _connect(resolved_path) as connection:
connection.execute(
@@ -196,6 +221,17 @@ def persist_rcon_historical_sample(
db_path: Path | None = None,
) -> dict[str, int]:
"""Persist one prospective RCON sample and refresh its checkpoint."""
if use_postgres_rcon_storage(explicit_sqlite_path=db_path):
from .postgres_rcon_storage import persist_sample
return persist_sample(
run_id=run_id,
captured_at=captured_at,
target=target,
normalized_payload=normalized_payload,
raw_payload=raw_payload,
)
resolved_path = initialize_rcon_historical_storage(db_path=db_path)
with _connect(resolved_path) as connection:
target_id = _upsert_target(connection, target=target)
@@ -254,6 +290,12 @@ def mark_rcon_historical_capture_failure(
db_path: Path | None = None,
) -> None:
"""Persist failure metadata for one target inside a capture run."""
if use_postgres_rcon_storage(explicit_sqlite_path=db_path):
from .postgres_rcon_storage import mark_capture_failure
mark_capture_failure(run_id=run_id, target=target, error_message=error_message)
return
resolved_path = initialize_rcon_historical_storage(db_path=db_path)
with _connect(resolved_path) as connection:
target_id = _upsert_target(connection, target=target)
@@ -282,6 +324,11 @@ def list_rcon_historical_target_statuses(
db_path: Path | None = None,
) -> list[dict[str, object]]:
"""Return per-target coverage and freshness for prospective RCON capture."""
if use_postgres_rcon_storage(explicit_sqlite_path=db_path):
from .postgres_rcon_storage import list_target_statuses
return list_target_statuses()
resolved_path = _resolve_db_path(db_path)
try:
with _connect_readonly(resolved_path) as connection:
@@ -353,6 +400,11 @@ def list_recent_rcon_historical_samples(
db_path: Path | None = None,
) -> list[dict[str, object]]:
"""Return recent prospective RCON samples for one or all configured targets."""
if use_postgres_rcon_storage(explicit_sqlite_path=db_path):
from .postgres_rcon_storage import list_recent_samples
return list_recent_samples(target_key=target_key, limit=limit)
resolved_path = _resolve_db_path(db_path)
where_clause = ""
params: list[object] = [limit]
@@ -413,6 +465,11 @@ def list_rcon_historical_competitive_windows(
db_path: Path | None = None,
) -> list[dict[str, object]]:
"""Return recent RCON-backed competitive windows derived from persisted samples."""
if use_postgres_rcon_storage(explicit_sqlite_path=db_path):
from .postgres_rcon_storage import list_competitive_windows
return list_competitive_windows(target_key=target_key, limit=limit)
resolved_path = _resolve_db_path(db_path)
where_clause = ""
params: list[object] = [limit]
@@ -498,6 +555,11 @@ def count_rcon_historical_samples_since(
db_path: Path | None = None,
) -> int:
"""Return how many RCON samples were captured after one timestamp."""
if use_postgres_rcon_storage(explicit_sqlite_path=db_path):
from .postgres_rcon_storage import count_samples_since
return count_samples_since(since)
if not since:
return 0
resolved_path = _resolve_db_path(db_path)
@@ -522,6 +584,11 @@ def list_rcon_historical_competitive_summary_rows(
db_path: Path | None = None,
) -> list[dict[str, object]]:
"""Return RCON-backed per-target summary rows over competitive windows."""
if use_postgres_rcon_storage(explicit_sqlite_path=db_path):
from .postgres_rcon_storage import list_competitive_summary_rows
return list_competitive_summary_rows(target_key=target_key)
resolved_path = _resolve_db_path(db_path)
where_clause = ""
params: list[object] = []
@@ -593,6 +660,15 @@ def find_rcon_historical_competitive_window(
db_path: Path | None = None,
) -> dict[str, object] | None:
"""Return the closest competitive window for one server/match if coverage exists."""
if use_postgres_rcon_storage(explicit_sqlite_path=db_path):
from .postgres_rcon_storage import find_competitive_window
return find_competitive_window(
server_key=server_key,
ended_at=ended_at,
map_name=map_name,
)
if not ended_at:
return None
resolved_path = _resolve_db_path(db_path)
@@ -672,6 +748,14 @@ def get_rcon_historical_competitive_window_by_session(
db_path: Path | None = None,
) -> dict[str, object] | None:
"""Return one persisted competitive RCON window by its synthetic session key."""
if use_postgres_rcon_storage(explicit_sqlite_path=db_path):
from .postgres_rcon_storage import get_competitive_window_by_session
return get_competitive_window_by_session(
server_key=server_key,
session_key=session_key,
)
normalized_session_key = str(session_key or "").strip()
if not normalized_session_key:
return None

View File

@@ -6,7 +6,7 @@ import sqlite3
from datetime import datetime, timezone
from pathlib import Path
from .config import get_storage_path
from .config import get_storage_path, use_postgres_rcon_storage
from .normalizers import normalize_map_name
from .scoreboard_origins import resolve_trusted_scoreboard_match_url
from .sqlite_utils import connect_sqlite_readonly
@@ -76,6 +76,16 @@ def _list_persisted_scoreboard_candidates(
server_slug: str,
db_path: Path,
) -> list[dict[str, object]]:
if use_postgres_rcon_storage():
from .postgres_rcon_storage import list_scoreboard_candidates
postgres_candidates = list_scoreboard_candidates(
server_slug=server_slug,
limit=MAX_CANDIDATES,
)
if postgres_candidates:
return postgres_candidates
try:
with connect_sqlite_readonly(db_path) as connection:
rows = connection.execute(
@@ -128,6 +138,10 @@ def _list_persisted_scoreboard_candidates(
"match_url": match_url,
}
)
if items and use_postgres_rcon_storage():
from .postgres_rcon_storage import upsert_scoreboard_candidates
upsert_scoreboard_candidates(server_slug=server_slug, candidates=items)
return items

View File

@@ -0,0 +1,89 @@
"""Report active phase-1 RCON storage backend and migrated table counts."""
from __future__ import annotations
import json
import sqlite3
from contextlib import closing
from .config import get_database_url, get_storage_path, use_postgres_rcon_storage
from .rcon_admin_log_materialization import summarize_rcon_materialization_status
from .rcon_admin_log_storage import initialize_rcon_admin_log_storage
from .sqlite_utils import connect_sqlite_readonly
MIGRATED_TABLES = (
"rcon_admin_log_events",
"rcon_player_profile_snapshots",
"rcon_materialized_matches",
"rcon_match_player_stats",
"rcon_historical_targets",
"rcon_historical_samples",
"rcon_historical_competitive_windows",
"rcon_scoreboard_match_candidates",
)
def build_storage_diagnostics() -> dict[str, object]:
"""Return one JSON-safe diagnostic payload for the migrated domains."""
if use_postgres_rcon_storage():
from .postgres_rcon_storage import count_migrated_tables
counts = count_migrated_tables()
backend = "postgresql"
else:
counts = _count_sqlite_tables()
backend = "sqlite-fallback"
materialization = summarize_rcon_materialization_status()
return {
"active_storage_backend": backend,
"database_url_configured": bool(get_database_url()),
"sqlite_fallback_path": str(get_storage_path()),
"migrated_domains": [
"rcon-admin-log-events",
"rcon-player-profile-snapshots",
"rcon-historical-capture-samples-and-windows",
"rcon-materialized-matches",
"rcon-materialized-player-stats",
"rcon-safe-scoreboard-candidates",
],
"table_counts": counts,
"latest_materialized_matches": materialization["latest_materialized_matches"],
"latest_admin_log_match_end_events": materialization[
"latest_admin_log_match_end_events"
],
"match_end_status": materialization["match_end_status"],
"sqlite_remaining": [
"live server snapshots and /api/servers cache",
"public-scoreboard historical_* tables and scoreboard fallback data",
"historical snapshots, player-event ledger and Elo/MMR tables",
],
"scoreboard_correlation": (
"PostgreSQL safe candidates are preferred when present; phase-1 still falls "
"back to trusted persisted historical_* scoreboard rows on SQLite."
),
}
def _count_sqlite_tables() -> dict[str, int]:
resolved_path = initialize_rcon_admin_log_storage()
counts: dict[str, int] = {}
with closing(connect_sqlite_readonly(resolved_path)) as connection:
for table_name in MIGRATED_TABLES:
try:
row = connection.execute(
f"SELECT COUNT(*) AS count FROM {table_name}"
).fetchone()
except sqlite3.Error:
counts[table_name] = 0
else:
counts[table_name] = int(row["count"] or 0)
return counts
def main() -> None:
print(json.dumps(build_storage_diagnostics(), ensure_ascii=False, indent=2))
if __name__ == "__main__":
main()