From 239595052ecc44c9001bfd1573b17f31f7888bf8 Mon Sep 17 00:00:00 2001 From: devRaGonSa Date: Thu, 21 May 2026 08:18:45 +0200 Subject: [PATCH] feat: migrate rcon historical storage to postgres --- ai/tasks/done/TASK-POSTGRES-RCON-PHASE-1.md | 141 +++ backend/.env.example | 1 + backend/README.md | 16 + backend/app/config.py | 14 + backend/app/postgres_rcon_storage.py | 986 ++++++++++++++++++ backend/app/rcon_admin_log_materialization.py | 150 ++- backend/app/rcon_admin_log_storage.py | 118 ++- backend/app/rcon_historical_storage.py | 86 +- backend/app/rcon_scoreboard_correlation.py | 16 +- backend/app/storage_diagnostics.py | 89 ++ backend/requirements.txt | 3 +- docker-compose.yml | 37 +- docs/decisions.md | 28 + .../css/historico-scoreboard-detail.css | 8 - frontend/assets/js/historico-partida.js | 33 +- frontend/assets/js/historico-recent-live.js | 5 - frontend/assets/js/historico.js | 13 - frontend/assets/js/main.js | 2 + .../run-historical-ui-regression-tests.ps1 | 4 +- 19 files changed, 1658 insertions(+), 92 deletions(-) create mode 100644 ai/tasks/done/TASK-POSTGRES-RCON-PHASE-1.md create mode 100644 backend/app/postgres_rcon_storage.py create mode 100644 backend/app/storage_diagnostics.py diff --git a/ai/tasks/done/TASK-POSTGRES-RCON-PHASE-1.md b/ai/tasks/done/TASK-POSTGRES-RCON-PHASE-1.md new file mode 100644 index 0000000..db50de0 --- /dev/null +++ b/ai/tasks/done/TASK-POSTGRES-RCON-PHASE-1.md @@ -0,0 +1,141 @@ +--- +id: TASK-POSTGRES-RCON-PHASE-1 +title: Migrate RCON historical persistence phase 1 +status: done +type: backend +team: Backend Senior +supporting_teams: + - Arquitecto de Base de Datos + - Frontend Senior +roadmap_item: foundation +priority: high +--- + +# TASK-POSTGRES-RCON-PHASE-1 - Migrate RCON historical persistence phase 1 + +## Goal + +Move the lock-prone RCON historical pipeline to PostgreSQL in Docker while +keeping local SQLite fallback and fixing the requested match-detail and server +card frontend regressions. + +## Context + +AdminLog ingestion and materialization currently share SQLite files with +multiple services and materialization can fail with `database is locked`. +PostgreSQL must become authoritative for RCON capture, AdminLog events, +materialized matches, player stats and the detail/recent read model in the +advanced Docker flow. Scoreboard historical persistence may remain on SQLite in +this phase when it is used only as fallback or correlation source. + +## Steps + +1. Inspect the storage, read-model and frontend files named in the request. +2. Add a deterministic PostgreSQL schema and route the phase-1 RCON storage + domains through it when `HLL_BACKEND_DATABASE_URL` is set. +3. Wire Compose diagnostics and required frontend fixes without expanding Elo, + backend scope outside RCON persistence, or Comunidad Hispana #03 targets. +4. Validate the requested commands, document what remains SQLite-backed, and + complete the task record. + +## Files to Read First + +- `ai/architecture-index.md` +- `ai/repo-context.md` +- `backend/app/config.py` +- `backend/app/rcon_admin_log_storage.py` +- `backend/app/rcon_admin_log_materialization.py` +- `frontend/assets/js/historico-partida.js` + +## Expected Files to Modify + +- `backend/app/config.py` +- `backend/app/postgres_rcon_storage.py` +- `backend/app/rcon_admin_log_storage.py` +- `backend/app/rcon_admin_log_materialization.py` +- `backend/app/rcon_historical_storage.py` +- `backend/app/rcon_scoreboard_correlation.py` +- `backend/app/storage_diagnostics.py` +- `backend/README.md` +- `backend/requirements.txt` +- `backend/.env.example` +- `docker-compose.yml` +- `frontend/assets/js/historico-partida.js` +- `frontend/assets/js/historico.js` +- `frontend/assets/js/historico-recent-live.js` +- `frontend/assets/js/main.js` +- `frontend/assets/css/historico-scoreboard-detail.css` +- `docs/decisions.md` +- `scripts/run-historical-ui-regression-tests.ps1` + +## Constraints + +- PostgreSQL is the Docker default for migrated RCON domains. +- SQLite remains only fallback or source material for domains not migrated in + this phase. +- Do not add SQLite lock workarounds as the main solution. +- Keep external scoreboard URLs on the trusted scoreboard allowlist. +- Do not modify unrelated files or add runtime database artifacts. + +## Validation + +- `python -m compileall backend/app` +- `python -m app.storage_diagnostics` +- `node --check frontend/assets/js/main.js` +- `node --check frontend/assets/js/historico.js` +- `node --check frontend/assets/js/historico-recent-live.js` +- `node --check frontend/assets/js/historico-partida.js` +- `powershell -ExecutionPolicy Bypass -File scripts/run-rcon-data-pipeline-tests.ps1` +- `powershell -ExecutionPolicy Bypass -File scripts/run-integration-tests.ps1` +- Requested Docker Compose and endpoint checks when local Docker/RCON + credentials permit them. + +## Outcome + +Implemented phase-1 PostgreSQL delegation for RCON capture samples/windows, +AdminLog events/profile snapshots, materialized RCON matches/player stats, and +safe scoreboard candidate caching. Docker Compose now configures PostgreSQL as +the RCON storage backend for `backend`, `historical-runner` and +`rcon-historical-worker`; local code paths with explicit SQLite paths or no +database URL retain SQLite fallback behavior. + +Frontend changes remove hover-driven player panel expansion in the internal +match detail table, keep the scoreboard link on the internal detail action area +only, and suppress pending-style region placeholders on server cards. + +Validation completed: + +- `python -m compileall backend/app` +- `python -m app.storage_diagnostics` from `backend/` +- `node --check frontend/assets/js/main.js` +- `node --check frontend/assets/js/historico.js` +- `node --check frontend/assets/js/historico-recent-live.js` +- `node --check frontend/assets/js/historico-partida.js` +- `powershell -ExecutionPolicy Bypass -File scripts/run-integration-tests.ps1` +- `docker compose --profile advanced config --quiet` +- local fallback HTTP smoke for `/health`, + `/api/historical/recent-matches`, and the requested known match detail URL +- direct known-match detail payload check confirmed safe URL + `https://scoreboard.comunidadhll.es:5443/games/1562094` + +`scripts/run-rcon-data-pipeline-tests.ps1` completed parser/storage and +unittest work, then exited nonzero when its optional Docker smoke step could +not reach the Docker Desktop Linux engine. The requested `docker compose +--profile advanced up ...` and `docker compose --profile advanced ps` checks +failed for the same local engine-unavailable reason, so container diagnostics, +container RCON commands and Docker-backed HTTP probes remain to rerun when +Docker is available. The Browser plugin is listed but its required JavaScript +control tool was not exposed in this session, so rendered interaction QA remains +manual or follow-up Browser verification. + +Phase boundary documented in `docs/decisions.md` and diagnostics: live server +snapshot cache, public-scoreboard `historical_*` data/rankings, historical +snapshot files, player-event ledger and paused Elo/MMR storage remain SQLite or +file-backed in this phase. + +## Change Budget + +This phase intentionally exceeds the default change budget because PostgreSQL +wiring crosses schema initialization, Compose runtime, RCON storage domains, +diagnostics, docs, and the requested frontend regressions. Keep the migration +surface limited to the RCON path and leave remaining SQLite domains documented. diff --git a/backend/.env.example b/backend/.env.example index f4f5c3d..42cd572 100644 --- a/backend/.env.example +++ b/backend/.env.example @@ -1,6 +1,7 @@ HLL_BACKEND_HOST=0.0.0.0 HLL_BACKEND_PORT=8000 HLL_BACKEND_STORAGE_PATH=/app/data/hll_vietnam_dev.sqlite3 +HLL_BACKEND_DATABASE_URL=postgresql://hll_vietnam:hll_vietnam_dev@postgres:5432/hll_vietnam HLL_BACKEND_ALLOWED_ORIGINS=http://127.0.0.1:8080,http://localhost:8080 HLL_BACKEND_REFRESH_INTERVAL_SECONDS=120 HLL_BACKEND_LIVE_DATA_SOURCE=rcon diff --git a/backend/README.md b/backend/README.md index adbcaac..bd1d7d1 100644 --- a/backend/README.md +++ b/backend/README.md @@ -97,6 +97,7 @@ Variables especialmente relevantes para Docker y Compose: - `HLL_BACKEND_HOST` - `HLL_BACKEND_PORT` +- `HLL_BACKEND_DATABASE_URL` - `HLL_BACKEND_STORAGE_PATH` - `HLL_BACKEND_ALLOWED_ORIGINS` - `HLL_BACKEND_LIVE_DATA_SOURCE` @@ -132,6 +133,21 @@ Dentro del contenedor arranca por defecto con: - `HLL_BACKEND_PORT=8000` - `HLL_BACKEND_STORAGE_PATH=/app/data/hll_vietnam_dev.sqlite3` +Compose configura ademas `HLL_BACKEND_DATABASE_URL` para que PostgreSQL sea el +almacenamiento autoritativo de la fase 1 RCON: muestras/ventanas de captura, +AdminLog, snapshots de perfil y partidas/estadisticas materializadas. Sin esa +variable, la ejecucion local mantiene fallback SQLite para esos dominios. + +Diagnostico rapido del backend activo: + +```powershell +python -m app.storage_diagnostics +``` + +La salida lista el backend RCON activo, counts de las tablas migradas, la ultima +partida materializada por servidor y que superficies siguen temporalmente en +SQLite en esta fase. + Build local: ```powershell diff --git a/backend/app/config.py b/backend/app/config.py index a76ff36..9e668d7 100644 --- a/backend/app/config.py +++ b/backend/app/config.py @@ -85,6 +85,20 @@ def get_storage_path() -> Path: return Path(configured_path) if configured_path else default_path +def get_database_url() -> str | None: + """Return the optional PostgreSQL URL for migrated backend storage domains.""" + configured_url = os.getenv("HLL_BACKEND_DATABASE_URL") + if configured_url is None: + return None + normalized_url = configured_url.strip() + return normalized_url or None + + +def use_postgres_rcon_storage(*, explicit_sqlite_path: Path | None = None) -> bool: + """Return whether phase-1 RCON storage should use PostgreSQL.""" + return explicit_sqlite_path is None and get_database_url() is not None + + def get_sqlite_writer_timeout_seconds() -> float: """Return the SQLite connection timeout shared by writer-capable storage layers.""" configured_value = os.getenv( diff --git a/backend/app/postgres_rcon_storage.py b/backend/app/postgres_rcon_storage.py new file mode 100644 index 0000000..220a84a --- /dev/null +++ b/backend/app/postgres_rcon_storage.py @@ -0,0 +1,986 @@ +"""PostgreSQL persistence for the phase-1 RCON historical pipeline.""" + +from __future__ import annotations + +import json +from collections.abc import Iterable, Mapping +from contextlib import contextmanager +from datetime import datetime, timezone +from typing import Any + +from .config import get_database_url +from .normalizers import normalize_map_name +from .rcon_client import load_rcon_targets + + +COMPETITIVE_WINDOW_GAP_SECONDS = 1800 +COMPETITIVE_MODE_PARTIAL = "partial" +COMPETITIVE_MODE_APPROXIMATE = "approximate" +COMPETITIVE_MODE_EXACT = "exact" + + +RCON_SCHEMA_SQL = """ +CREATE TABLE IF NOT EXISTS rcon_historical_targets ( + id BIGSERIAL PRIMARY KEY, + target_key TEXT NOT NULL UNIQUE, + external_server_id TEXT, + display_name TEXT NOT NULL, + host TEXT NOT NULL, + port INTEGER NOT NULL, + region TEXT, + game_port INTEGER, + query_port INTEGER, + source_name TEXT NOT NULL, + last_configured_at TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP +); + +CREATE TABLE IF NOT EXISTS rcon_historical_capture_runs ( + id BIGSERIAL PRIMARY KEY, + mode TEXT NOT NULL, + status TEXT NOT NULL, + target_scope TEXT, + started_at TEXT NOT NULL, + completed_at TEXT, + targets_seen INTEGER NOT NULL DEFAULT 0, + samples_inserted INTEGER NOT NULL DEFAULT 0, + duplicate_samples INTEGER NOT NULL DEFAULT 0, + failed_targets INTEGER NOT NULL DEFAULT 0, + notes TEXT, + created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP +); + +CREATE TABLE IF NOT EXISTS rcon_historical_samples ( + id BIGSERIAL PRIMARY KEY, + target_id BIGINT NOT NULL REFERENCES rcon_historical_targets(id), + capture_run_id BIGINT REFERENCES rcon_historical_capture_runs(id), + captured_at TEXT NOT NULL, + source_kind TEXT NOT NULL, + status TEXT NOT NULL, + players INTEGER, + max_players INTEGER, + current_map TEXT, + normalized_payload_json TEXT NOT NULL, + raw_payload_json TEXT, + created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, + UNIQUE(target_id, captured_at) +); + +CREATE TABLE IF NOT EXISTS rcon_historical_checkpoints ( + target_id BIGINT PRIMARY KEY REFERENCES rcon_historical_targets(id), + last_successful_capture_at TEXT, + last_sample_at TEXT, + last_run_id BIGINT REFERENCES rcon_historical_capture_runs(id), + last_run_status TEXT, + last_error TEXT, + last_error_at TEXT, + updated_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP +); + +CREATE TABLE IF NOT EXISTS rcon_historical_competitive_windows ( + id BIGSERIAL PRIMARY KEY, + target_id BIGINT NOT NULL REFERENCES rcon_historical_targets(id), + session_key TEXT NOT NULL UNIQUE, + source_kind TEXT NOT NULL, + map_name TEXT, + map_pretty_name TEXT, + first_seen_at TEXT NOT NULL, + last_seen_at TEXT NOT NULL, + sample_count INTEGER NOT NULL DEFAULT 0, + total_players INTEGER NOT NULL DEFAULT 0, + peak_players INTEGER NOT NULL DEFAULT 0, + last_players INTEGER, + max_players INTEGER, + status TEXT NOT NULL, + confidence_mode TEXT NOT NULL, + capabilities_json TEXT NOT NULL, + latest_payload_json TEXT, + created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP +); + +CREATE TABLE IF NOT EXISTS rcon_admin_log_events ( + id BIGSERIAL PRIMARY KEY, + target_key TEXT NOT NULL, + external_server_id TEXT, + event_timestamp TEXT, + server_time BIGINT, + relative_time TEXT, + event_type TEXT NOT NULL, + raw_message TEXT NOT NULL, + canonical_message TEXT NOT NULL, + parsed_payload_json TEXT NOT NULL, + raw_entry_json TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, + UNIQUE NULLS NOT DISTINCT(target_key, server_time, canonical_message) +); + +CREATE TABLE IF NOT EXISTS rcon_player_profile_snapshots ( + id BIGSERIAL PRIMARY KEY, + target_key TEXT NOT NULL, + external_server_id TEXT, + player_id TEXT NOT NULL, + player_name TEXT NOT NULL, + source_server_time BIGINT NOT NULL, + event_timestamp TEXT, + first_seen TEXT, + sessions INTEGER, + matches_played INTEGER, + play_time TEXT, + total_kills INTEGER, + total_deaths INTEGER, + teamkills_done INTEGER, + teamkills_received INTEGER, + kd_ratio DOUBLE PRECISION, + favorite_weapons_json TEXT NOT NULL DEFAULT '{}', + victims_json TEXT NOT NULL DEFAULT '{}', + nemesis_json TEXT NOT NULL DEFAULT '{}', + averages_json TEXT NOT NULL DEFAULT '{}', + sanctions_json TEXT NOT NULL DEFAULT '{}', + raw_content TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, + UNIQUE(target_key, player_id, source_server_time) +); + +CREATE TABLE IF NOT EXISTS rcon_materialized_matches ( + id BIGSERIAL PRIMARY KEY, + target_key TEXT NOT NULL, + external_server_id TEXT, + match_key TEXT NOT NULL, + map_name TEXT, + map_pretty_name TEXT, + game_mode TEXT, + started_server_time BIGINT, + ended_server_time BIGINT, + started_at TEXT, + ended_at TEXT, + allied_score INTEGER, + axis_score INTEGER, + winner TEXT, + confidence_mode TEXT NOT NULL, + source_basis TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, + UNIQUE(target_key, match_key) +); + +CREATE TABLE IF NOT EXISTS rcon_match_player_stats ( + id BIGSERIAL PRIMARY KEY, + target_key TEXT NOT NULL, + match_key TEXT NOT NULL, + player_id TEXT NOT NULL, + player_name TEXT NOT NULL, + team TEXT, + kills INTEGER NOT NULL DEFAULT 0, + deaths INTEGER NOT NULL DEFAULT 0, + teamkills INTEGER NOT NULL DEFAULT 0, + deaths_by_teamkill INTEGER NOT NULL DEFAULT 0, + weapons_json TEXT NOT NULL DEFAULT '{}', + death_by_weapons_json TEXT NOT NULL DEFAULT '{}', + most_killed_json TEXT NOT NULL DEFAULT '{}', + death_by_json TEXT NOT NULL DEFAULT '{}', + first_seen_server_time BIGINT, + last_seen_server_time BIGINT, + created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, + UNIQUE(target_key, match_key, player_id) +); + +CREATE TABLE IF NOT EXISTS rcon_scoreboard_match_candidates ( + id BIGSERIAL PRIMARY KEY, + server_slug TEXT NOT NULL, + external_match_id TEXT NOT NULL, + started_at TEXT, + ended_at TEXT, + map_name TEXT, + map_pretty_name TEXT, + allied_score INTEGER, + axis_score INTEGER, + player_count INTEGER, + match_url TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, + UNIQUE(server_slug, external_match_id) +); + +CREATE INDEX IF NOT EXISTS idx_rcon_historical_samples_target_time +ON rcon_historical_samples(target_id, captured_at DESC); +CREATE INDEX IF NOT EXISTS idx_rcon_historical_windows_target_time +ON rcon_historical_competitive_windows(target_id, last_seen_at DESC); +CREATE INDEX IF NOT EXISTS idx_rcon_admin_log_events_target_time +ON rcon_admin_log_events(target_key, server_time DESC); +CREATE INDEX IF NOT EXISTS idx_rcon_admin_log_events_type +ON rcon_admin_log_events(event_type); +CREATE INDEX IF NOT EXISTS idx_rcon_player_profile_snapshots_player +ON rcon_player_profile_snapshots(target_key, player_id, source_server_time DESC); +CREATE INDEX IF NOT EXISTS idx_rcon_materialized_matches_recent +ON rcon_materialized_matches(target_key, ended_at DESC, ended_server_time DESC); +CREATE INDEX IF NOT EXISTS idx_rcon_match_player_stats_match +ON rcon_match_player_stats(target_key, match_key); +CREATE INDEX IF NOT EXISTS idx_rcon_scoreboard_candidates_server_end +ON rcon_scoreboard_match_candidates(server_slug, ended_at DESC, started_at DESC); +""" + + +def initialize_postgres_rcon_storage() -> None: + """Create deterministic PostgreSQL schema for migrated RCON domains.""" + with connect_postgres() as connection: + with connection.cursor() as cursor: + cursor.execute(RCON_SCHEMA_SQL) + + +@contextmanager +def connect_postgres(): + """Yield one PostgreSQL connection with dict-shaped rows.""" + try: + import psycopg + from psycopg.rows import dict_row + except ImportError as error: # pragma: no cover - dependency is environment-specific + raise RuntimeError("psycopg is required when HLL_BACKEND_DATABASE_URL is set.") from error + + database_url = get_database_url() + if not database_url: + raise RuntimeError("HLL_BACKEND_DATABASE_URL is required for PostgreSQL RCON storage.") + with psycopg.connect(database_url, row_factory=dict_row) as connection: + yield connection + + +class PostgresCompatConnection: + """Small DB-API shim for RCON SQL shared with SQLite functions.""" + + def __init__(self, connection: Any): + self.connection = connection + + def execute(self, sql: str, params: Iterable[object] | None = None): + normalized = sql.replace("server_time IS ?", "server_time IS NOT DISTINCT FROM ?") + normalized = normalized.replace("?", "%s") + return self.connection.execute(normalized, tuple(params or ())) + + +@contextmanager +def connect_postgres_compat(): + """Yield a query shim that accepts the phase-1 SQLite-style placeholders.""" + initialize_postgres_rcon_storage() + with connect_postgres() as connection: + yield PostgresCompatConnection(connection) + + +def start_capture_run(*, mode: str, target_scope: str) -> int: + initialize_postgres_rcon_storage() + with connect_postgres() as connection: + row = connection.execute( + """ + INSERT INTO rcon_historical_capture_runs (mode, status, target_scope, started_at) + VALUES (%s, 'running', %s, %s) + RETURNING id + """, + (mode, target_scope, _utc_now_iso()), + ).fetchone() + return int(row["id"]) + + +def finalize_capture_run( + run_id: int, + *, + status: str, + targets_seen: int, + samples_inserted: int, + duplicate_samples: int, + failed_targets: int, + notes: str | None, +) -> None: + initialize_postgres_rcon_storage() + with connect_postgres() as connection: + connection.execute( + """ + UPDATE rcon_historical_capture_runs + SET status = %s, + completed_at = %s, + targets_seen = %s, + samples_inserted = %s, + duplicate_samples = %s, + failed_targets = %s, + notes = %s + WHERE id = %s + """, + ( + status, + _utc_now_iso(), + targets_seen, + samples_inserted, + duplicate_samples, + failed_targets, + notes, + run_id, + ), + ) + + +def persist_sample( + *, + run_id: int, + captured_at: str, + target: Mapping[str, object], + normalized_payload: Mapping[str, object], + raw_payload: Mapping[str, object] | None, +) -> dict[str, int]: + initialize_postgres_rcon_storage() + with connect_postgres() as connection: + target_id = _upsert_target(connection, target=target) + row = connection.execute( + """ + INSERT INTO rcon_historical_samples ( + target_id, capture_run_id, captured_at, source_kind, status, players, + max_players, current_map, normalized_payload_json, raw_payload_json + ) VALUES (%s, %s, %s, 'rcon-live-sample', %s, %s, %s, %s, %s, %s) + ON CONFLICT(target_id, captured_at) DO NOTHING + RETURNING id + """, + ( + target_id, + run_id, + captured_at, + normalized_payload.get("status") or "unknown", + normalized_payload.get("players"), + normalized_payload.get("max_players"), + normalized_payload.get("current_map"), + json.dumps(dict(normalized_payload), separators=(",", ":")), + json.dumps(dict(raw_payload), separators=(",", ":")) if raw_payload else None, + ), + ).fetchone() + inserted = int(row is not None) + _upsert_checkpoint_success( + connection, + target_id=target_id, + run_id=run_id, + captured_at=captured_at, + ) + if inserted: + _upsert_competitive_window( + connection, + target_id=target_id, + captured_at=captured_at, + normalized_payload=normalized_payload, + ) + return {"samples_inserted": inserted, "duplicate_samples": 0 if inserted else 1} + + +def mark_capture_failure( + *, + run_id: int, + target: Mapping[str, object], + error_message: str, +) -> None: + initialize_postgres_rcon_storage() + with connect_postgres() as connection: + target_id = _upsert_target(connection, target=target) + connection.execute( + """ + INSERT INTO rcon_historical_checkpoints ( + target_id, last_run_id, last_run_status, last_error, last_error_at + ) VALUES (%s, %s, 'failed', %s, %s) + ON CONFLICT(target_id) DO UPDATE SET + last_run_id = EXCLUDED.last_run_id, + last_run_status = EXCLUDED.last_run_status, + last_error = EXCLUDED.last_error, + last_error_at = EXCLUDED.last_error_at, + updated_at = CURRENT_TIMESTAMP + """, + (target_id, run_id, error_message, _utc_now_iso()), + ) + + +def list_target_statuses() -> list[dict[str, object]]: + rows = _fetchall( + """ + SELECT + targets.target_key, + targets.external_server_id, + targets.display_name, + targets.host, + targets.port, + targets.region, + targets.source_name, + checkpoints.last_successful_capture_at, + checkpoints.last_sample_at, + checkpoints.last_run_id, + checkpoints.last_run_status, + checkpoints.last_error, + checkpoints.last_error_at, + (SELECT MIN(samples.captured_at) FROM rcon_historical_samples AS samples + WHERE samples.target_id = targets.id) AS first_sample_at, + (SELECT MAX(samples.captured_at) FROM rcon_historical_samples AS samples + WHERE samples.target_id = targets.id) AS latest_sample_at, + (SELECT COUNT(*) FROM rcon_historical_samples AS samples + WHERE samples.target_id = targets.id) AS sample_count + FROM rcon_historical_targets AS targets + LEFT JOIN rcon_historical_checkpoints AS checkpoints + ON checkpoints.target_id = targets.id + ORDER BY targets.display_name ASC, targets.target_key ASC + """ + ) + return [ + { + **dict(row), + "sample_count": int(row["sample_count"] or 0), + "last_sample_at": row["latest_sample_at"] or row["last_sample_at"], + } + for row in rows + ] + + +def list_recent_samples(*, target_key: str | None, limit: int) -> list[dict[str, object]]: + where_clause, params = _target_where_clause(target_key) + rows = _fetchall( + f""" + SELECT targets.target_key, targets.external_server_id, targets.display_name, + targets.region, samples.captured_at, samples.status, samples.players, + samples.max_players, samples.current_map + FROM rcon_historical_samples AS samples + INNER JOIN rcon_historical_targets AS targets ON targets.id = samples.target_id + {where_clause} + ORDER BY samples.captured_at DESC, targets.display_name ASC + LIMIT %s + """, + [*params, limit], + ) + return [dict(row) for row in rows] + + +def list_competitive_windows(*, target_key: str | None, limit: int) -> list[dict[str, object]]: + where_clause, params = _target_where_clause(target_key) + rows = _fetchall( + f""" + SELECT targets.target_key, targets.external_server_id, targets.display_name, + targets.region, windows.session_key, windows.map_name, + windows.map_pretty_name, windows.first_seen_at, windows.last_seen_at, + windows.sample_count, windows.total_players, windows.peak_players, + windows.last_players, windows.max_players, windows.status, + windows.confidence_mode, windows.capabilities_json, + windows.latest_payload_json + FROM rcon_historical_competitive_windows AS windows + INNER JOIN rcon_historical_targets AS targets ON targets.id = windows.target_id + {where_clause} + ORDER BY windows.last_seen_at DESC, targets.display_name ASC + LIMIT %s + """, + [*params, limit], + ) + return [_serialize_window(row) for row in rows] + + +def count_samples_since(since: str | None) -> int: + if not since: + return 0 + row = _fetchone( + "SELECT COUNT(*) AS sample_count FROM rcon_historical_samples WHERE captured_at > %s", + (since,), + ) + return int(row["sample_count"] or 0) if row else 0 + + +def list_competitive_summary_rows(*, target_key: str | None) -> list[dict[str, object]]: + where_clause, params = _target_where_clause(target_key) + rows = _fetchall( + f""" + SELECT targets.target_key, targets.external_server_id, targets.display_name, + targets.region, checkpoints.last_successful_capture_at, + checkpoints.last_run_status, checkpoints.last_error, + checkpoints.last_error_at, COUNT(windows.id) AS window_count, + COALESCE(SUM(windows.sample_count), 0) AS sample_count, + MIN(windows.first_seen_at) AS first_seen_at, + MAX(windows.last_seen_at) AS last_seen_at, + COALESCE(MAX(windows.peak_players), 0) AS peak_players + FROM rcon_historical_targets AS targets + LEFT JOIN rcon_historical_checkpoints AS checkpoints ON checkpoints.target_id = targets.id + LEFT JOIN rcon_historical_competitive_windows AS windows ON windows.target_id = targets.id + {where_clause} + GROUP BY targets.id, checkpoints.target_id + ORDER BY targets.display_name ASC, targets.target_key ASC + """, + params, + ) + return [ + { + **dict(row), + "window_count": int(row["window_count"] or 0), + "sample_count": int(row["sample_count"] or 0), + "peak_players": int(row["peak_players"] or 0), + } + for row in rows + ] + + +def find_competitive_window( + *, + server_key: str, + ended_at: str | None, + map_name: str | None, +) -> dict[str, object] | None: + if not ended_at: + return None + aliases = _expand_target_key_aliases(server_key) + candidates = _fetchall( + """ + SELECT windows.session_key, windows.first_seen_at, windows.last_seen_at, + windows.map_name, windows.map_pretty_name, windows.sample_count, + windows.total_players, windows.peak_players, windows.confidence_mode, + windows.capabilities_json, windows.latest_payload_json + FROM rcon_historical_competitive_windows AS windows + INNER JOIN rcon_historical_targets AS targets ON targets.id = windows.target_id + WHERE targets.target_key = ANY(%s) OR targets.external_server_id = ANY(%s) + ORDER BY windows.last_seen_at DESC + LIMIT 12 + """, + (aliases, aliases), + ) + ended_point = _parse_timestamp(ended_at) + if ended_point is None: + return None + normalized_map_name = normalize_map_name(map_name) + best_row: dict[str, object] | None = None + best_distance: float | None = None + for row in candidates: + row_map = normalize_map_name(row["map_pretty_name"] or row["map_name"]) + if normalized_map_name and row_map and normalized_map_name != row_map: + continue + row_last = _parse_timestamp(row["last_seen_at"]) + if row_last is None: + continue + distance = abs((row_last - ended_point).total_seconds()) + if best_distance is None or distance < best_distance: + best_row = dict(row) + best_distance = distance + if best_row is None or best_distance is None or best_distance > 21600: + return None + sample_count = int(best_row["sample_count"] or 0) + return { + "session_key": best_row["session_key"], + "first_seen_at": best_row["first_seen_at"], + "last_seen_at": best_row["last_seen_at"], + "duration_seconds": _calculate_duration_seconds( + best_row["first_seen_at"], + best_row["last_seen_at"], + ), + "map_name": best_row["map_name"], + "map_pretty_name": best_row["map_pretty_name"] or best_row["map_name"], + "sample_count": sample_count, + "average_players": ( + round((int(best_row["total_players"] or 0) / sample_count), 2) + if sample_count > 0 + else 0.0 + ), + "peak_players": int(best_row["peak_players"] or 0), + "confidence_mode": best_row["confidence_mode"], + "capabilities": _deserialize_json_object(best_row["capabilities_json"]), + } + + +def get_competitive_window_by_session( + *, + server_key: str, + session_key: str, +) -> dict[str, object] | None: + normalized_session_key = str(session_key or "").strip() + if not normalized_session_key: + return None + aliases = _expand_target_key_aliases(server_key) + row = _fetchone( + """ + SELECT targets.target_key, targets.external_server_id, targets.display_name, + targets.region, windows.session_key, windows.map_name, + windows.map_pretty_name, windows.first_seen_at, windows.last_seen_at, + windows.sample_count, windows.total_players, windows.peak_players, + windows.confidence_mode, windows.capabilities_json, + windows.latest_payload_json + FROM rcon_historical_competitive_windows AS windows + INNER JOIN rcon_historical_targets AS targets ON targets.id = windows.target_id + WHERE windows.session_key = %s + AND (targets.target_key = ANY(%s) OR targets.external_server_id = ANY(%s)) + LIMIT 1 + """, + (normalized_session_key, aliases, aliases), + ) + return _serialize_window(row) if row else None + + +def list_scoreboard_candidates(*, server_slug: str, limit: int) -> list[dict[str, object]]: + rows = _fetchall( + """ + SELECT external_match_id, started_at, ended_at, map_name, map_pretty_name, + allied_score, axis_score, player_count, match_url + FROM rcon_scoreboard_match_candidates + WHERE server_slug = %s + ORDER BY COALESCE(ended_at, started_at) DESC + LIMIT %s + """, + (server_slug, limit), + ) + return [dict(row) for row in rows] + + +def upsert_scoreboard_candidates( + *, + server_slug: str, + candidates: Iterable[Mapping[str, object]], +) -> int: + """Cache trusted scoreboard correlation candidates in PostgreSQL.""" + rows = [candidate for candidate in candidates if candidate.get("match_url")] + if not rows: + return 0 + initialize_postgres_rcon_storage() + inserted_or_updated = 0 + with connect_postgres() as connection: + for candidate in rows: + connection.execute( + """ + INSERT INTO rcon_scoreboard_match_candidates ( + server_slug, external_match_id, started_at, ended_at, map_name, + map_pretty_name, allied_score, axis_score, player_count, match_url + ) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s) + ON CONFLICT(server_slug, external_match_id) DO UPDATE SET + started_at = EXCLUDED.started_at, + ended_at = EXCLUDED.ended_at, + map_name = EXCLUDED.map_name, + map_pretty_name = EXCLUDED.map_pretty_name, + allied_score = EXCLUDED.allied_score, + axis_score = EXCLUDED.axis_score, + player_count = EXCLUDED.player_count, + match_url = EXCLUDED.match_url, + updated_at = CURRENT_TIMESTAMP + """, + ( + server_slug, + str(candidate.get("external_match_id") or ""), + candidate.get("started_at"), + candidate.get("ended_at"), + candidate.get("map_name"), + candidate.get("map_pretty_name"), + candidate.get("allied_score"), + candidate.get("axis_score"), + candidate.get("player_count"), + candidate["match_url"], + ), + ) + inserted_or_updated += 1 + return inserted_or_updated + + +def count_migrated_tables() -> dict[str, int]: + initialize_postgres_rcon_storage() + table_names = ( + "rcon_admin_log_events", + "rcon_player_profile_snapshots", + "rcon_materialized_matches", + "rcon_match_player_stats", + "rcon_historical_targets", + "rcon_historical_samples", + "rcon_historical_competitive_windows", + "rcon_scoreboard_match_candidates", + ) + with connect_postgres() as connection: + return { + table_name: int( + connection.execute(f"SELECT COUNT(*) AS count FROM {table_name}").fetchone()[ + "count" + ] + or 0 + ) + for table_name in table_names + } + + +def _fetchall(sql: str, params: Iterable[object] = ()) -> list[dict[str, object]]: + initialize_postgres_rcon_storage() + with connect_postgres() as connection: + return [dict(row) for row in connection.execute(sql, tuple(params)).fetchall()] + + +def _fetchone(sql: str, params: Iterable[object] = ()) -> dict[str, object] | None: + initialize_postgres_rcon_storage() + with connect_postgres() as connection: + row = connection.execute(sql, tuple(params)).fetchone() + return dict(row) if row else None + + +def _upsert_target(connection: Any, *, target: Mapping[str, object]) -> int: + target_key = str(target.get("target_key") or "").strip() + display_name = str(target.get("name") or target.get("display_name") or target_key).strip() + host = str(target.get("host") or "").strip() + port = int(target.get("port") or 0) + if not target_key or not host or port <= 0: + raise ValueError("Prospective RCON targets require target_key, host and port.") + row = connection.execute( + """ + INSERT INTO rcon_historical_targets ( + target_key, external_server_id, display_name, host, port, region, + game_port, query_port, source_name, last_configured_at + ) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s) + ON CONFLICT(target_key) DO UPDATE SET + external_server_id = EXCLUDED.external_server_id, + display_name = EXCLUDED.display_name, + host = EXCLUDED.host, + port = EXCLUDED.port, + region = EXCLUDED.region, + game_port = EXCLUDED.game_port, + query_port = EXCLUDED.query_port, + source_name = EXCLUDED.source_name, + last_configured_at = EXCLUDED.last_configured_at, + updated_at = CURRENT_TIMESTAMP + RETURNING id + """, + ( + target_key, + target.get("external_server_id"), + display_name, + host, + port, + target.get("region"), + target.get("game_port"), + target.get("query_port"), + str(target.get("source_name") or "community-hispana-rcon"), + _utc_now_iso(), + ), + ).fetchone() + return int(row["id"]) + + +def _upsert_checkpoint_success( + connection: Any, + *, + target_id: int, + run_id: int, + captured_at: str, +) -> None: + connection.execute( + """ + INSERT INTO rcon_historical_checkpoints ( + target_id, last_successful_capture_at, last_sample_at, last_run_id, + last_run_status, last_error, last_error_at + ) VALUES (%s, %s, %s, %s, 'success', NULL, NULL) + ON CONFLICT(target_id) DO UPDATE SET + last_successful_capture_at = EXCLUDED.last_successful_capture_at, + last_sample_at = EXCLUDED.last_sample_at, + last_run_id = EXCLUDED.last_run_id, + last_run_status = EXCLUDED.last_run_status, + last_error = NULL, + last_error_at = NULL, + updated_at = CURRENT_TIMESTAMP + """, + (target_id, captured_at, captured_at, run_id), + ) + + +def _upsert_competitive_window( + connection: Any, + *, + target_id: int, + captured_at: str, + normalized_payload: Mapping[str, object], +) -> None: + current_map_raw = str(normalized_payload.get("current_map") or "").strip() + if not current_map_raw: + return + map_pretty_name = normalize_map_name(current_map_raw) or current_map_raw + players = int(normalized_payload.get("players") or 0) + max_players = normalized_payload.get("max_players") + status = str(normalized_payload.get("status") or "unknown") + latest_window = connection.execute( + """ + SELECT * + FROM rcon_historical_competitive_windows + WHERE target_id = %s + ORDER BY last_seen_at DESC, id DESC + LIMIT 1 + """, + (target_id,), + ).fetchone() + if latest_window and _should_extend_competitive_window( + latest_window=dict(latest_window), + captured_at=captured_at, + current_map=current_map_raw, + ): + connection.execute( + """ + UPDATE rcon_historical_competitive_windows + SET map_name = %s, + map_pretty_name = %s, + last_seen_at = %s, + sample_count = sample_count + 1, + total_players = total_players + %s, + peak_players = GREATEST(peak_players, %s), + last_players = %s, + max_players = %s, + status = %s, + confidence_mode = %s, + capabilities_json = %s, + latest_payload_json = %s, + updated_at = CURRENT_TIMESTAMP + WHERE id = %s + """, + ( + current_map_raw, + map_pretty_name, + captured_at, + players, + players, + players, + max_players, + status, + COMPETITIVE_MODE_APPROXIMATE, + json.dumps(_build_competitive_capabilities(), separators=(",", ":")), + json.dumps(dict(normalized_payload), separators=(",", ":")), + latest_window["id"], + ), + ) + return + connection.execute( + """ + INSERT INTO rcon_historical_competitive_windows ( + target_id, session_key, source_kind, map_name, map_pretty_name, + first_seen_at, last_seen_at, sample_count, total_players, + peak_players, last_players, max_players, status, confidence_mode, + capabilities_json, latest_payload_json + ) VALUES (%s, %s, 'rcon-historical-samples', %s, %s, %s, %s, 1, + %s, %s, %s, %s, %s, %s, %s, %s) + """, + ( + target_id, + f"{target_id}:{captured_at}", + current_map_raw, + map_pretty_name, + captured_at, + captured_at, + players, + players, + players, + max_players, + status, + COMPETITIVE_MODE_APPROXIMATE, + json.dumps(_build_competitive_capabilities(), separators=(",", ":")), + json.dumps(dict(normalized_payload), separators=(",", ":")), + ), + ) + + +def _target_where_clause(target_key: str | None) -> tuple[str, list[object]]: + if not target_key: + return "", [] + aliases = _expand_target_key_aliases(target_key) + return "WHERE targets.target_key = ANY(%s) OR targets.external_server_id = ANY(%s)", [ + aliases, + aliases, + ] + + +def _expand_target_key_aliases(target_key: str) -> list[str]: + normalized_target_key = str(target_key or "").strip() + aliases = {normalized_target_key} + try: + configured_targets = load_rcon_targets() + except Exception: + configured_targets = () + for target in configured_targets: + external_server_id = str(target.external_server_id or "").strip() + legacy_target_key = f"rcon:{target.host}:{target.port}" + if external_server_id and external_server_id == normalized_target_key: + aliases.update((legacy_target_key, external_server_id)) + elif legacy_target_key == normalized_target_key: + aliases.add(legacy_target_key) + if external_server_id: + aliases.add(external_server_id) + return sorted(alias for alias in aliases if alias) + + +def _serialize_window(row: Mapping[str, object]) -> dict[str, object]: + sample_count = int(row["sample_count"] or 0) + return { + "target_key": row["target_key"], + "external_server_id": row["external_server_id"], + "display_name": row["display_name"], + "region": row["region"], + "session_key": row["session_key"], + "map_name": row["map_name"], + "map_pretty_name": row["map_pretty_name"] or row["map_name"], + "first_seen_at": row["first_seen_at"], + "last_seen_at": row["last_seen_at"], + "duration_seconds": _calculate_duration_seconds( + row["first_seen_at"], + row["last_seen_at"], + ), + "sample_count": sample_count, + "average_players": ( + round((int(row["total_players"] or 0) / sample_count), 2) + if sample_count > 0 + else 0.0 + ), + "peak_players": int(row["peak_players"] or 0), + "last_players": row.get("last_players"), + "max_players": row.get("max_players"), + "status": row.get("status"), + "confidence_mode": row["confidence_mode"], + "capabilities": _deserialize_json_object(row["capabilities_json"]), + "latest_payload": _deserialize_json_object(row["latest_payload_json"]), + } + + +def _should_extend_competitive_window( + *, + latest_window: Mapping[str, object], + captured_at: str, + current_map: str, +) -> bool: + if normalize_map_name(latest_window.get("map_name")) != normalize_map_name(current_map): + return False + latest_seen = _parse_timestamp(latest_window.get("last_seen_at")) + captured_point = _parse_timestamp(captured_at) + if latest_seen is None or captured_point is None: + return False + return (captured_point - latest_seen).total_seconds() <= COMPETITIVE_WINDOW_GAP_SECONDS + + +def _build_competitive_capabilities() -> dict[str, object]: + return { + "recent_matches": COMPETITIVE_MODE_APPROXIMATE, + "server_summary": COMPETITIVE_MODE_EXACT, + "competitive_quality": COMPETITIVE_MODE_PARTIAL, + "result": "session-score", + "gamestate": "session", + "player_stats": "unavailable", + } + + +def _deserialize_json_object(raw_value: object) -> dict[str, object]: + if isinstance(raw_value, str) and raw_value.strip(): + try: + parsed = json.loads(raw_value) + except json.JSONDecodeError: + return {} + return parsed if isinstance(parsed, dict) else {} + return {} + + +def _calculate_duration_seconds(first_seen_at: object, last_seen_at: object) -> int | None: + first_point = _parse_timestamp(first_seen_at) + last_point = _parse_timestamp(last_seen_at) + if first_point is None or last_point is None: + return None + return max(0, int((last_point - first_point).total_seconds())) + + +def _parse_timestamp(raw_value: object) -> datetime | None: + if not isinstance(raw_value, str) or not raw_value.strip(): + return None + try: + timestamp = datetime.fromisoformat(raw_value.replace("Z", "+00:00")) + except ValueError: + return None + if timestamp.tzinfo is None: + timestamp = timestamp.replace(tzinfo=timezone.utc) + return timestamp.astimezone(timezone.utc) + + +def _utc_now_iso() -> str: + return datetime.now(timezone.utc).isoformat().replace("+00:00", "Z") diff --git a/backend/app/rcon_admin_log_materialization.py b/backend/app/rcon_admin_log_materialization.py index a720b86..92a76b4 100644 --- a/backend/app/rcon_admin_log_materialization.py +++ b/backend/app/rcon_admin_log_materialization.py @@ -10,7 +10,7 @@ from collections.abc import Iterable from contextlib import closing from pathlib import Path -from .config import get_storage_path +from .config import get_storage_path, use_postgres_rcon_storage from .normalizers import normalize_map_name from .rcon_admin_log_storage import initialize_rcon_admin_log_storage from .rcon_historical_storage import list_rcon_historical_competitive_windows @@ -23,6 +23,12 @@ SESSION_RESULT_SOURCE = "rcon-session" def initialize_rcon_materialized_storage(*, db_path: Path | None = None) -> Path: """Create SQLite structures used by the materialized RCON match pipeline.""" + if use_postgres_rcon_storage(explicit_sqlite_path=db_path): + from .postgres_rcon_storage import initialize_postgres_rcon_storage + + initialize_postgres_rcon_storage() + return get_storage_path() + resolved_path = initialize_rcon_admin_log_storage(db_path=db_path) with closing(connect_sqlite_writer(resolved_path)) as connection: with connection: @@ -85,6 +91,46 @@ def initialize_rcon_materialized_storage(*, db_path: Path | None = None) -> Path def materialize_rcon_admin_log(*, db_path: Path | None = None) -> dict[str, object]: """Materialize matches and player stats from stored AdminLog events.""" resolved_path = initialize_rcon_materialized_storage(db_path=db_path) + if use_postgres_rcon_storage(explicit_sqlite_path=db_path): + from .postgres_rcon_storage import connect_postgres_compat + + with connect_postgres_compat() as connection: + payload = _materialize_rcon_admin_log_with_connection( + connection, + session_window_db_path=None, + caught_errors=(Exception,), + ) + freshness = summarize_rcon_materialization_status() + return { + **payload, + "latest_materialized_matches": freshness["latest_materialized_matches"], + "latest_admin_log_match_end_events": freshness["latest_admin_log_match_end_events"], + "match_end_status": freshness["match_end_status"], + } + + with closing(connect_sqlite_writer(resolved_path)) as connection: + with connection: + payload = _materialize_rcon_admin_log_with_connection( + connection, + session_window_db_path=resolved_path, + caught_errors=(sqlite3.Error,), + ) + + freshness = summarize_rcon_materialization_status(db_path=resolved_path) + return { + **payload, + "latest_materialized_matches": freshness["latest_materialized_matches"], + "latest_admin_log_match_end_events": freshness["latest_admin_log_match_end_events"], + "match_end_status": freshness["match_end_status"], + } + + +def _materialize_rcon_admin_log_with_connection( + connection: object, + *, + session_window_db_path: Path | None, + caught_errors: tuple[type[BaseException], ...], +) -> dict[str, object]: errors: list[str] = [] matches_seen = 0 matches_materialized = 0 @@ -93,40 +139,39 @@ def materialize_rcon_admin_log(*, db_path: Path | None = None) -> dict[str, obje player_stats_materialized = 0 player_stats_updated = 0 - with closing(connect_sqlite_writer(resolved_path)) as connection: - with connection: - try: - match_rows = _derive_admin_log_matches(connection) - matches_seen = len(match_rows) - for row in match_rows: - outcome = _upsert_match(connection, row) - matches_materialized += int(outcome == "inserted") - matches_updated += int(outcome == "updated") - session_rows = _derive_session_fallback_matches(connection, db_path=resolved_path) - matches_seen += len(session_rows) - for row in session_rows: - outcome = _upsert_match(connection, row) - matches_materialized += int(outcome == "inserted") - matches_updated += int(outcome == "updated") + try: + match_rows = _derive_admin_log_matches(connection) + matches_seen = len(match_rows) + for row in match_rows: + outcome = _upsert_match(connection, row) + matches_materialized += int(outcome == "inserted") + matches_updated += int(outcome == "updated") + session_rows = _derive_session_fallback_matches( + connection, + db_path=session_window_db_path, + ) + matches_seen += len(session_rows) + for row in session_rows: + outcome = _upsert_match(connection, row) + matches_materialized += int(outcome == "inserted") + matches_updated += int(outcome == "updated") - persisted_matches = _list_materialized_matches(connection) - for match in persisted_matches: - stats = _derive_player_stats_for_match(connection, match) - player_stats_seen += len(stats) - connection.execute( - """ - DELETE FROM rcon_match_player_stats - WHERE target_key = ? AND match_key = ? - """, - (match["target_key"], match["match_key"]), - ) - for stat in stats: - _insert_player_stat(connection, stat) - player_stats_materialized += 1 - except sqlite3.Error as error: - errors.append(str(error)) - - freshness = summarize_rcon_materialization_status(db_path=resolved_path) + persisted_matches = _list_materialized_matches(connection) + for match in persisted_matches: + stats = _derive_player_stats_for_match(connection, match) + player_stats_seen += len(stats) + connection.execute( + """ + DELETE FROM rcon_match_player_stats + WHERE target_key = ? AND match_key = ? + """, + (match["target_key"], match["match_key"]), + ) + for stat in stats: + _insert_player_stat(connection, stat) + player_stats_materialized += 1 + except caught_errors as error: + errors.append(str(error)) return { "matches_seen": matches_seen, "matches_materialized": matches_materialized, @@ -135,8 +180,6 @@ def materialize_rcon_admin_log(*, db_path: Path | None = None) -> dict[str, obje "player_stats_materialized": player_stats_materialized, "player_stats_updated": player_stats_updated, "errors": errors, - "latest_materialized_matches": freshness["latest_materialized_matches"], - "latest_admin_log_match_end_events": freshness["latest_admin_log_match_end_events"], } @@ -159,7 +202,13 @@ def list_materialized_rcon_matches( params.append(MATCH_RESULT_SOURCE) where = "WHERE " + " AND ".join(clauses) if clauses else "" params.append(limit) - with closing(connect_sqlite_readonly(resolved_path)) as connection: + if use_postgres_rcon_storage(explicit_sqlite_path=db_path): + from .postgres_rcon_storage import connect_postgres_compat + + connection_scope = connect_postgres_compat() + else: + connection_scope = closing(connect_sqlite_readonly(resolved_path)) + with connection_scope as connection: rows = connection.execute( f""" SELECT @@ -196,7 +245,13 @@ def get_materialized_rcon_match_detail( ) -> dict[str, object] | None: """Return one materialized match with player stats.""" resolved_path = initialize_rcon_materialized_storage(db_path=db_path) - with closing(connect_sqlite_readonly(resolved_path)) as connection: + if use_postgres_rcon_storage(explicit_sqlite_path=db_path): + from .postgres_rcon_storage import connect_postgres_compat + + connection_scope = connect_postgres_compat() + else: + connection_scope = closing(connect_sqlite_readonly(resolved_path)) + with connection_scope as connection: match = connection.execute( """ SELECT * @@ -248,7 +303,13 @@ def get_materialized_rcon_match_detail( def summarize_rcon_materialization_status(*, db_path: Path | None = None) -> dict[str, object]: """Return a small diagnostic summary for stored RCON materialization state.""" resolved_path = initialize_rcon_materialized_storage(db_path=db_path) - with closing(connect_sqlite_readonly(resolved_path)) as connection: + if use_postgres_rcon_storage(explicit_sqlite_path=db_path): + from .postgres_rcon_storage import connect_postgres_compat + + connection_scope = connect_postgres_compat() + else: + connection_scope = closing(connect_sqlite_readonly(resolved_path)) + with connection_scope as connection: match_count = connection.execute( "SELECT COUNT(*) AS count FROM rcon_materialized_matches" ).fetchone()["count"] @@ -259,7 +320,7 @@ def summarize_rcon_materialization_status(*, db_path: Path | None = None) -> dic SELECT 1 FROM rcon_match_player_stats GROUP BY target_key, match_key - ) + ) AS stats_matches """ ).fetchone()["count"] ranges = connection.execute( @@ -301,7 +362,7 @@ def summarize_rcon_materialization_status(*, db_path: Path | None = None) -> dic ) AS row_number FROM rcon_materialized_matches WHERE source_basis = ? - ) + ) AS ranked_matches WHERE row_number = 1 ORDER BY target_key ASC """, @@ -328,6 +389,11 @@ def summarize_rcon_materialization_status(*, db_path: Path | None = None) -> dic "event_counts": [dict(row) for row in event_counts], "latest_materialized_matches": [dict(row) for row in latest_matches], "latest_admin_log_match_end_events": [dict(row) for row in latest_match_end_events], + "match_end_status": ( + "admin-log-match-end-events-available" + if latest_match_end_events + else "no-admin-log-match-end-events-stored" + ), } @@ -360,7 +426,7 @@ def _derive_admin_log_matches(connection: sqlite3.Connection) -> list[dict[str, def _derive_session_fallback_matches( connection: sqlite3.Connection, *, - db_path: Path, + db_path: Path | None, ) -> list[dict[str, object]]: rows: list[dict[str, object]] = [] existing = { diff --git a/backend/app/rcon_admin_log_storage.py b/backend/app/rcon_admin_log_storage.py index da57346..8b00ea6 100644 --- a/backend/app/rcon_admin_log_storage.py +++ b/backend/app/rcon_admin_log_storage.py @@ -8,7 +8,7 @@ import sqlite3 from collections.abc import Mapping from pathlib import Path -from .config import get_storage_path +from .config import get_storage_path, use_postgres_rcon_storage from .rcon_admin_log_parser import parse_rcon_admin_log_entry from .rcon_admin_log_parser import parse_rcon_player_profile_snapshot from .rcon_historical_storage import initialize_rcon_historical_storage @@ -17,6 +17,12 @@ from .sqlite_utils import connect_sqlite_writer def initialize_rcon_admin_log_storage(*, db_path: Path | None = None) -> Path: """Create SQLite structures for parsed RCON AdminLog events.""" + if use_postgres_rcon_storage(explicit_sqlite_path=db_path): + from .postgres_rcon_storage import initialize_postgres_rcon_storage + + initialize_postgres_rcon_storage() + return get_storage_path() + resolved_path = initialize_rcon_historical_storage(db_path=db_path) with connect_sqlite_writer(resolved_path) as connection: @@ -90,6 +96,9 @@ def persist_rcon_admin_log_entries( db_path: Path | None = None, ) -> dict[str, int]: """Persist raw and parsed AdminLog entries idempotently.""" + if use_postgres_rcon_storage(explicit_sqlite_path=db_path): + return _persist_rcon_admin_log_entries_postgres(target=target, entries=entries) + resolved_path = initialize_rcon_admin_log_storage(db_path=db_path) target_key = str(target.get("target_key") or target.get("external_server_id") or "") if not target_key: @@ -278,6 +287,25 @@ def _ensure_canonical_message_column(connection: sqlite3.Connection) -> None: def list_rcon_admin_log_event_counts(*, db_path: Path | None = None) -> list[dict[str, object]]: """Return event counts grouped by target and event type.""" + if use_postgres_rcon_storage(explicit_sqlite_path=db_path): + from .postgres_rcon_storage import connect_postgres_compat + + with connect_postgres_compat() as connection: + rows = connection.execute( + """ + SELECT + target_key, + event_type, + COUNT(*) AS event_count, + MIN(server_time) AS first_server_time, + MAX(server_time) AS last_server_time + FROM rcon_admin_log_events + GROUP BY target_key, event_type + ORDER BY target_key ASC, event_count DESC + """ + ).fetchall() + return [dict(row) for row in rows] + resolved_path = db_path or get_storage_path() initialize_rcon_admin_log_storage(db_path=resolved_path) @@ -310,6 +338,29 @@ def get_latest_rcon_player_profile_summaries( requested_ids = [str(player_id).strip() for player_id in player_ids if str(player_id).strip()] if not target_key or not requested_ids: return {} + if use_postgres_rcon_storage(explicit_sqlite_path=db_path): + from .postgres_rcon_storage import connect_postgres_compat + + placeholders = ",".join("?" for _ in requested_ids) + with connect_postgres_compat() as connection: + rows = connection.execute( + f""" + SELECT snapshots.* + FROM rcon_player_profile_snapshots AS snapshots + INNER JOIN ( + SELECT player_id, MAX(source_server_time) AS latest_source_server_time + FROM rcon_player_profile_snapshots + WHERE target_key = ? + AND player_id IN ({placeholders}) + GROUP BY player_id + ) AS latest + ON latest.player_id = snapshots.player_id + AND latest.latest_source_server_time = snapshots.source_server_time + WHERE snapshots.target_key = ? + """, + [target_key, *requested_ids, target_key], + ).fetchall() + return {str(row["player_id"]): _build_safe_profile_summary(row) for row in rows} resolved_path = db_path or get_storage_path() initialize_rcon_admin_log_storage(db_path=resolved_path) @@ -369,3 +420,68 @@ def _json_mapping(raw_value: object) -> dict[str, object]: except json.JSONDecodeError: return {} return parsed if isinstance(parsed, dict) else {} + + +def _persist_rcon_admin_log_entries_postgres( + *, + target: Mapping[str, object], + entries: list[dict[str, object]], +) -> dict[str, int]: + from .postgres_rcon_storage import connect_postgres_compat + + target_key = str(target.get("target_key") or target.get("external_server_id") or "") + if not target_key: + raise ValueError("target must include target_key or external_server_id") + + external_server_id = target.get("external_server_id") + inserted = 0 + duplicates = 0 + with connect_postgres_compat() as connection: + for entry in entries: + parsed = parse_rcon_admin_log_entry(entry) + raw_message = str(parsed.get("raw_message") or "") + canonical_message = _canonicalize_admin_log_message(raw_message) + cursor = connection.execute( + """ + INSERT INTO rcon_admin_log_events ( + target_key, + external_server_id, + event_timestamp, + server_time, + relative_time, + event_type, + raw_message, + canonical_message, + parsed_payload_json, + raw_entry_json + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT DO NOTHING + """, + ( + target_key, + external_server_id, + parsed.get("timestamp"), + parsed.get("server_time"), + parsed.get("relative_time"), + parsed.get("event_type") or "unknown", + raw_message, + canonical_message, + json.dumps(parsed, ensure_ascii=False, separators=(",", ":")), + json.dumps(entry, ensure_ascii=False, separators=(",", ":")), + ), + ) + if int(cursor.rowcount or 0): + inserted += 1 + else: + duplicates += 1 + _persist_profile_snapshot_if_present( + connection, + target_key=target_key, + external_server_id=external_server_id, + parsed=parsed, + ) + return { + "events_seen": len(entries), + "events_inserted": inserted, + "duplicate_events": duplicates, + } diff --git a/backend/app/rcon_historical_storage.py b/backend/app/rcon_historical_storage.py index 8369da3..30d75b4 100644 --- a/backend/app/rcon_historical_storage.py +++ b/backend/app/rcon_historical_storage.py @@ -8,7 +8,7 @@ from collections.abc import Mapping from datetime import datetime, timezone from pathlib import Path -from .config import get_storage_path +from .config import get_storage_path, use_postgres_rcon_storage from .normalizers import normalize_map_name from .rcon_client import load_rcon_targets from .sqlite_utils import connect_sqlite_readonly, connect_sqlite_writer @@ -22,6 +22,12 @@ COMPETITIVE_MODE_EXACT = "exact" def initialize_rcon_historical_storage(*, db_path: Path | None = None) -> Path: """Create the SQLite structures used by prospective RCON capture.""" + if use_postgres_rcon_storage(explicit_sqlite_path=db_path): + from .postgres_rcon_storage import initialize_postgres_rcon_storage + + initialize_postgres_rcon_storage() + return get_storage_path() + resolved_path = db_path or get_storage_path() resolved_path.parent.mkdir(parents=True, exist_ok=True) @@ -131,6 +137,11 @@ def start_rcon_historical_capture_run( db_path: Path | None = None, ) -> int: """Create one run row for prospective RCON capture.""" + if use_postgres_rcon_storage(explicit_sqlite_path=db_path): + from .postgres_rcon_storage import start_capture_run + + return start_capture_run(mode=mode, target_scope=target_scope) + resolved_path = initialize_rcon_historical_storage(db_path=db_path) with _connect(resolved_path) as connection: cursor = connection.execute( @@ -159,6 +170,20 @@ def finalize_rcon_historical_capture_run( db_path: Path | None = None, ) -> None: """Finalize one prospective RCON capture run.""" + if use_postgres_rcon_storage(explicit_sqlite_path=db_path): + from .postgres_rcon_storage import finalize_capture_run + + finalize_capture_run( + run_id, + status=status, + targets_seen=targets_seen, + samples_inserted=samples_inserted, + duplicate_samples=duplicate_samples, + failed_targets=failed_targets, + notes=notes, + ) + return + resolved_path = initialize_rcon_historical_storage(db_path=db_path) with _connect(resolved_path) as connection: connection.execute( @@ -196,6 +221,17 @@ def persist_rcon_historical_sample( db_path: Path | None = None, ) -> dict[str, int]: """Persist one prospective RCON sample and refresh its checkpoint.""" + if use_postgres_rcon_storage(explicit_sqlite_path=db_path): + from .postgres_rcon_storage import persist_sample + + return persist_sample( + run_id=run_id, + captured_at=captured_at, + target=target, + normalized_payload=normalized_payload, + raw_payload=raw_payload, + ) + resolved_path = initialize_rcon_historical_storage(db_path=db_path) with _connect(resolved_path) as connection: target_id = _upsert_target(connection, target=target) @@ -254,6 +290,12 @@ def mark_rcon_historical_capture_failure( db_path: Path | None = None, ) -> None: """Persist failure metadata for one target inside a capture run.""" + if use_postgres_rcon_storage(explicit_sqlite_path=db_path): + from .postgres_rcon_storage import mark_capture_failure + + mark_capture_failure(run_id=run_id, target=target, error_message=error_message) + return + resolved_path = initialize_rcon_historical_storage(db_path=db_path) with _connect(resolved_path) as connection: target_id = _upsert_target(connection, target=target) @@ -282,6 +324,11 @@ def list_rcon_historical_target_statuses( db_path: Path | None = None, ) -> list[dict[str, object]]: """Return per-target coverage and freshness for prospective RCON capture.""" + if use_postgres_rcon_storage(explicit_sqlite_path=db_path): + from .postgres_rcon_storage import list_target_statuses + + return list_target_statuses() + resolved_path = _resolve_db_path(db_path) try: with _connect_readonly(resolved_path) as connection: @@ -353,6 +400,11 @@ def list_recent_rcon_historical_samples( db_path: Path | None = None, ) -> list[dict[str, object]]: """Return recent prospective RCON samples for one or all configured targets.""" + if use_postgres_rcon_storage(explicit_sqlite_path=db_path): + from .postgres_rcon_storage import list_recent_samples + + return list_recent_samples(target_key=target_key, limit=limit) + resolved_path = _resolve_db_path(db_path) where_clause = "" params: list[object] = [limit] @@ -413,6 +465,11 @@ def list_rcon_historical_competitive_windows( db_path: Path | None = None, ) -> list[dict[str, object]]: """Return recent RCON-backed competitive windows derived from persisted samples.""" + if use_postgres_rcon_storage(explicit_sqlite_path=db_path): + from .postgres_rcon_storage import list_competitive_windows + + return list_competitive_windows(target_key=target_key, limit=limit) + resolved_path = _resolve_db_path(db_path) where_clause = "" params: list[object] = [limit] @@ -498,6 +555,11 @@ def count_rcon_historical_samples_since( db_path: Path | None = None, ) -> int: """Return how many RCON samples were captured after one timestamp.""" + if use_postgres_rcon_storage(explicit_sqlite_path=db_path): + from .postgres_rcon_storage import count_samples_since + + return count_samples_since(since) + if not since: return 0 resolved_path = _resolve_db_path(db_path) @@ -522,6 +584,11 @@ def list_rcon_historical_competitive_summary_rows( db_path: Path | None = None, ) -> list[dict[str, object]]: """Return RCON-backed per-target summary rows over competitive windows.""" + if use_postgres_rcon_storage(explicit_sqlite_path=db_path): + from .postgres_rcon_storage import list_competitive_summary_rows + + return list_competitive_summary_rows(target_key=target_key) + resolved_path = _resolve_db_path(db_path) where_clause = "" params: list[object] = [] @@ -593,6 +660,15 @@ def find_rcon_historical_competitive_window( db_path: Path | None = None, ) -> dict[str, object] | None: """Return the closest competitive window for one server/match if coverage exists.""" + if use_postgres_rcon_storage(explicit_sqlite_path=db_path): + from .postgres_rcon_storage import find_competitive_window + + return find_competitive_window( + server_key=server_key, + ended_at=ended_at, + map_name=map_name, + ) + if not ended_at: return None resolved_path = _resolve_db_path(db_path) @@ -672,6 +748,14 @@ def get_rcon_historical_competitive_window_by_session( db_path: Path | None = None, ) -> dict[str, object] | None: """Return one persisted competitive RCON window by its synthetic session key.""" + if use_postgres_rcon_storage(explicit_sqlite_path=db_path): + from .postgres_rcon_storage import get_competitive_window_by_session + + return get_competitive_window_by_session( + server_key=server_key, + session_key=session_key, + ) + normalized_session_key = str(session_key or "").strip() if not normalized_session_key: return None diff --git a/backend/app/rcon_scoreboard_correlation.py b/backend/app/rcon_scoreboard_correlation.py index fd170eb..2f9a5f2 100644 --- a/backend/app/rcon_scoreboard_correlation.py +++ b/backend/app/rcon_scoreboard_correlation.py @@ -6,7 +6,7 @@ import sqlite3 from datetime import datetime, timezone from pathlib import Path -from .config import get_storage_path +from .config import get_storage_path, use_postgres_rcon_storage from .normalizers import normalize_map_name from .scoreboard_origins import resolve_trusted_scoreboard_match_url from .sqlite_utils import connect_sqlite_readonly @@ -76,6 +76,16 @@ def _list_persisted_scoreboard_candidates( server_slug: str, db_path: Path, ) -> list[dict[str, object]]: + if use_postgres_rcon_storage(): + from .postgres_rcon_storage import list_scoreboard_candidates + + postgres_candidates = list_scoreboard_candidates( + server_slug=server_slug, + limit=MAX_CANDIDATES, + ) + if postgres_candidates: + return postgres_candidates + try: with connect_sqlite_readonly(db_path) as connection: rows = connection.execute( @@ -128,6 +138,10 @@ def _list_persisted_scoreboard_candidates( "match_url": match_url, } ) + if items and use_postgres_rcon_storage(): + from .postgres_rcon_storage import upsert_scoreboard_candidates + + upsert_scoreboard_candidates(server_slug=server_slug, candidates=items) return items diff --git a/backend/app/storage_diagnostics.py b/backend/app/storage_diagnostics.py new file mode 100644 index 0000000..fc001ab --- /dev/null +++ b/backend/app/storage_diagnostics.py @@ -0,0 +1,89 @@ +"""Report active phase-1 RCON storage backend and migrated table counts.""" + +from __future__ import annotations + +import json +import sqlite3 +from contextlib import closing + +from .config import get_database_url, get_storage_path, use_postgres_rcon_storage +from .rcon_admin_log_materialization import summarize_rcon_materialization_status +from .rcon_admin_log_storage import initialize_rcon_admin_log_storage +from .sqlite_utils import connect_sqlite_readonly + + +MIGRATED_TABLES = ( + "rcon_admin_log_events", + "rcon_player_profile_snapshots", + "rcon_materialized_matches", + "rcon_match_player_stats", + "rcon_historical_targets", + "rcon_historical_samples", + "rcon_historical_competitive_windows", + "rcon_scoreboard_match_candidates", +) + + +def build_storage_diagnostics() -> dict[str, object]: + """Return one JSON-safe diagnostic payload for the migrated domains.""" + if use_postgres_rcon_storage(): + from .postgres_rcon_storage import count_migrated_tables + + counts = count_migrated_tables() + backend = "postgresql" + else: + counts = _count_sqlite_tables() + backend = "sqlite-fallback" + materialization = summarize_rcon_materialization_status() + return { + "active_storage_backend": backend, + "database_url_configured": bool(get_database_url()), + "sqlite_fallback_path": str(get_storage_path()), + "migrated_domains": [ + "rcon-admin-log-events", + "rcon-player-profile-snapshots", + "rcon-historical-capture-samples-and-windows", + "rcon-materialized-matches", + "rcon-materialized-player-stats", + "rcon-safe-scoreboard-candidates", + ], + "table_counts": counts, + "latest_materialized_matches": materialization["latest_materialized_matches"], + "latest_admin_log_match_end_events": materialization[ + "latest_admin_log_match_end_events" + ], + "match_end_status": materialization["match_end_status"], + "sqlite_remaining": [ + "live server snapshots and /api/servers cache", + "public-scoreboard historical_* tables and scoreboard fallback data", + "historical snapshots, player-event ledger and Elo/MMR tables", + ], + "scoreboard_correlation": ( + "PostgreSQL safe candidates are preferred when present; phase-1 still falls " + "back to trusted persisted historical_* scoreboard rows on SQLite." + ), + } + + +def _count_sqlite_tables() -> dict[str, int]: + resolved_path = initialize_rcon_admin_log_storage() + counts: dict[str, int] = {} + with closing(connect_sqlite_readonly(resolved_path)) as connection: + for table_name in MIGRATED_TABLES: + try: + row = connection.execute( + f"SELECT COUNT(*) AS count FROM {table_name}" + ).fetchone() + except sqlite3.Error: + counts[table_name] = 0 + else: + counts[table_name] = int(row["count"] or 0) + return counts + + +def main() -> None: + print(json.dumps(build_storage_diagnostics(), ensure_ascii=False, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/backend/requirements.txt b/backend/requirements.txt index a975eab..6ab0fc9 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -1 +1,2 @@ -# El bootstrap actual usa solo la libreria estandar de Python. +# PostgreSQL is used by the phase-1 RCON historical storage migration. +psycopg[binary]>=3.2,<4 diff --git a/docker-compose.yml b/docker-compose.yml index 4ff58cd..19e6c3d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,4 +1,22 @@ services: + postgres: + image: postgres:16-alpine + container_name: hll-vietnam-postgres + environment: + POSTGRES_DB: hll_vietnam + POSTGRES_USER: hll_vietnam + POSTGRES_PASSWORD: hll_vietnam_dev + ports: + - "5432:5432" + volumes: + - postgres-data:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U hll_vietnam -d hll_vietnam"] + interval: 5s + timeout: 5s + retries: 12 + restart: unless-stopped + backend: build: context: ./backend @@ -6,6 +24,7 @@ services: env_file: - ./backend/.env.example environment: + HLL_BACKEND_DATABASE_URL: ${HLL_BACKEND_DATABASE_URL:-postgresql://hll_vietnam:hll_vietnam_dev@postgres:5432/hll_vietnam} HLL_BACKEND_LIVE_DATA_SOURCE: ${HLL_BACKEND_LIVE_DATA_SOURCE:-rcon} HLL_BACKEND_HISTORICAL_DATA_SOURCE: ${HLL_BACKEND_HISTORICAL_DATA_SOURCE:-rcon} HLL_BACKEND_RCON_TIMEOUT_SECONDS: ${HLL_BACKEND_RCON_TIMEOUT_SECONDS:-20} @@ -13,6 +32,9 @@ services: ${HLL_BACKEND_RCON_TARGETS:-[{"name":"Comunidad Hispana #01","slug":"comunidad-hispana-01","external_server_id":"comunidad-hispana-01","host":"152.114.195.174","port":7779,"password":"replace-me-01","source_name":"community-hispana-rcon","region":"ES","game_port":null,"query_port":null},{"name":"Comunidad Hispana #02","slug":"comunidad-hispana-02","external_server_id":"comunidad-hispana-02","host":"152.114.195.150","port":7879,"password":"replace-me-02","source_name":"community-hispana-rcon","region":"ES","game_port":null,"query_port":null}]} ports: - "8000:8000" + depends_on: + postgres: + condition: service_healthy volumes: - ./backend/data:/app/data restart: unless-stopped @@ -27,13 +49,17 @@ services: env_file: - ./backend/.env.example environment: + HLL_BACKEND_DATABASE_URL: ${HLL_BACKEND_DATABASE_URL:-postgresql://hll_vietnam:hll_vietnam_dev@postgres:5432/hll_vietnam} HLL_BACKEND_LIVE_DATA_SOURCE: ${HLL_BACKEND_LIVE_DATA_SOURCE:-rcon} HLL_BACKEND_HISTORICAL_DATA_SOURCE: ${HLL_BACKEND_HISTORICAL_DATA_SOURCE:-rcon} HLL_BACKEND_RCON_TIMEOUT_SECONDS: ${HLL_BACKEND_RCON_TIMEOUT_SECONDS:-20} HLL_BACKEND_RCON_TARGETS: >- ${HLL_BACKEND_RCON_TARGETS:-[{"name":"Comunidad Hispana #01","slug":"comunidad-hispana-01","external_server_id":"comunidad-hispana-01","host":"152.114.195.174","port":7779,"password":"replace-me-01","source_name":"community-hispana-rcon","region":"ES","game_port":null,"query_port":null},{"name":"Comunidad Hispana #02","slug":"comunidad-hispana-02","external_server_id":"comunidad-hispana-02","host":"152.114.195.150","port":7879,"password":"replace-me-02","source_name":"community-hispana-rcon","region":"ES","game_port":null,"query_port":null}]} depends_on: - - backend + postgres: + condition: service_healthy + backend: + condition: service_started volumes: - ./backend/data:/app/data restart: unless-stopped @@ -48,6 +74,7 @@ services: env_file: - ./backend/.env.example environment: + HLL_BACKEND_DATABASE_URL: ${HLL_BACKEND_DATABASE_URL:-postgresql://hll_vietnam:hll_vietnam_dev@postgres:5432/hll_vietnam} HLL_BACKEND_LIVE_DATA_SOURCE: ${HLL_BACKEND_LIVE_DATA_SOURCE:-rcon} HLL_BACKEND_HISTORICAL_DATA_SOURCE: ${HLL_BACKEND_HISTORICAL_DATA_SOURCE:-rcon} HLL_BACKEND_RCON_TIMEOUT_SECONDS: ${HLL_BACKEND_RCON_TIMEOUT_SECONDS:-20} @@ -57,7 +84,10 @@ services: HLL_RCON_HISTORICAL_CAPTURE_MAX_RETRIES: ${HLL_RCON_HISTORICAL_CAPTURE_MAX_RETRIES:-2} HLL_RCON_HISTORICAL_CAPTURE_RETRY_DELAY_SECONDS: ${HLL_RCON_HISTORICAL_CAPTURE_RETRY_DELAY_SECONDS:-15} depends_on: - - backend + postgres: + condition: service_healthy + backend: + condition: service_started volumes: - ./backend/data:/app/data restart: unless-stopped @@ -71,3 +101,6 @@ services: ports: - "8080:8080" restart: unless-stopped + +volumes: + postgres-data: diff --git a/docs/decisions.md b/docs/decisions.md index e93b019..cff44e4 100644 --- a/docs/decisions.md +++ b/docs/decisions.md @@ -177,3 +177,31 @@ son confiables `comunidad-hispana-01`, con origen defecto nuevos. Los datos historicos ya persistidos no se eliminan, pero las URLs publicas de partidas solo se aceptan si el `raw_payload_ref` usa HTTP(S), apunta al origen confiable del servidor activo y mantiene una ruta `/games/`. + +## Decision 017: PostgreSQL phase 1 for RCON historical persistence + +La primera migracion de persistencia a PostgreSQL cubre el camino que sufria +contencion SQLite entre `backend`, `historical-runner` y +`rcon-historical-worker`: + +- captura prospectiva RCON, muestras y ventanas competitivas +- eventos AdminLog deduplicados y snapshots de perfil derivados +- partidas RCON materializadas y estadisticas por jugador +- candidatos confiables de URL de scoreboard que puedan poblarse para + correlacion de detalle + +Docker Compose configura `HLL_BACKEND_DATABASE_URL` y usa PostgreSQL como +backend autoritativo para esas tablas. La ejecucion local sin esa variable sigue +usando SQLite como fallback temporal para preservar comandos y tests locales. + +Quedan SQLite-backed en esta fase porque no forman parte del lock-prone writer +path migrado y siguen cubriendo fallback publico o caches locales: + +- snapshots live y cache de `/api/servers` +- tablas `historical_*` de scoreboard publico, rankings y correlacion legacy +- snapshots historicos precalculados, ledger player-event y Elo/MMR pausado + +La correlacion de URL publica en detalle usa primero candidatos PostgreSQL +confiables cuando existan y puede seguir leyendo filas `historical_*` +persistidas en SQLite durante la transicion. El diagnostico operativo se expone +con `python -m app.storage_diagnostics`. diff --git a/frontend/assets/css/historico-scoreboard-detail.css b/frontend/assets/css/historico-scoreboard-detail.css index f1148ce..3b2afbf 100644 --- a/frontend/assets/css/historico-scoreboard-detail.css +++ b/frontend/assets/css/historico-scoreboard-detail.css @@ -39,18 +39,15 @@ } .historical-player-row { - cursor: pointer; outline: 0; } .historical-player-row:hover td, -.historical-player-row:focus td, .historical-player-row:focus-within td, .historical-player-row.is-expanded td { background: rgba(210, 182, 118, 0.06); } -.historical-player-row:focus-visible td, .historical-player-row__details-button:focus-visible { outline: 2px solid rgba(210, 182, 118, 0.78); outline-offset: -2px; @@ -99,11 +96,6 @@ display: none; } -.historical-player-row:hover + .historical-player-detail-row, -.historical-player-row:focus + .historical-player-detail-row, -.historical-player-row:focus-within + .historical-player-detail-row, -.historical-player-detail-row:hover, -.historical-player-detail-row:focus-within, .historical-player-detail-row.is-open { display: table-row; } diff --git a/frontend/assets/js/historico-partida.js b/frontend/assets/js/historico-partida.js index 5f67703..74a0d63 100644 --- a/frontend/assets/js/historico-partida.js +++ b/frontend/assets/js/historico-partida.js @@ -230,9 +230,6 @@ function renderPlayerRows(player, item, index) {