From 73a6c1afe13bbe500a11d7538fa695fb55b818de Mon Sep 17 00:00:00 2001 From: devRaGonSa Date: Tue, 9 Jun 2026 10:31:21 +0200 Subject: [PATCH] Add player search read model --- .../TASK-198-add-player-search-read-model.md | 163 ++++++ backend/app/payloads.py | 1 + backend/app/postgres_rcon_storage.py | 23 + backend/app/rcon_historical_player_stats.py | 546 +++++++++++++++++- docs/player-search-read-model-plan.md | 126 ++++ scripts/run-stats-validation.ps1 | 288 +++++++++ 6 files changed, 1144 insertions(+), 3 deletions(-) create mode 100644 ai/tasks/done/TASK-198-add-player-search-read-model.md create mode 100644 docs/player-search-read-model-plan.md diff --git a/ai/tasks/done/TASK-198-add-player-search-read-model.md b/ai/tasks/done/TASK-198-add-player-search-read-model.md new file mode 100644 index 0000000..75b77b5 --- /dev/null +++ b/ai/tasks/done/TASK-198-add-player-search-read-model.md @@ -0,0 +1,163 @@ +--- +id: TASK-198-add-player-search-read-model +title: Add player search read model +status: done +type: backend +team: Backend Senior +supporting_teams: + - Arquitecto de Base de Datos + - Arquitecto Python +roadmap_item: foundation +priority: high +--- + +# TASK-198 - Add player search read model + +## Goal + +Add the first PostgreSQL read model for player search so `/api/stats/players/search` can serve compact search results without aggregating large RCON historical tables on every public request. + +## Context + +Weekly and monthly ranking snapshots are already implemented and validated in PostgreSQL with runtime fallback preserved. The next high-cost public path is the player search flow, which currently reads directly from `rcon_match_player_stats` and `rcon_materialized_matches` at request time. + +This task adds a dedicated regenerable read model for player lookup, refreshed manually out of band from materialized RCON PostgreSQL tables. The public search endpoint must prefer that read model when it exists and has rows, while preserving controlled runtime fallback if the read model is missing, empty or temporarily unreadable. + +Preserve the current product identity: Spanish-speaking HLL Vietnam community, military/Vietnam/tactical/sober visual direction and controlled repository evolution. + +## Steps + +1. Read the listed files first. +2. Add a dedicated player search index table as a regenerable read model. +3. Implement a refresh function that rebuilds the index from materialized RCON PostgreSQL tables, with current-year counters and recent player rows only. +4. Add a manual CLI command to rebuild the read model on demand. +5. Make `/api/stats/players/search` prefer the read model and keep runtime fallback when needed. +6. Preserve the current frontend contract consumed by `frontend/assets/js/stats.js`. +7. Document the operational command, fallback behavior and current limitations. + +## Files to Read First + +- `AGENTS.md` +- `ai/repo-context.md` +- `ai/architecture-index.md` +- `backend/app/payloads.py` +- `backend/app/rcon_historical_player_stats.py` +- `backend/app/postgres_rcon_storage.py` +- `backend/app/rcon_admin_log_materialization.py` +- `docs/stats-section-functional-plan.md` +- `docs/ranking-snapshot-read-model-plan.md` +- `ai/tasks/done/TASK-197-automate-ranking-snapshot-refresh.md` + +## Expected Files to Modify + +- `backend/app/rcon_historical_player_stats.py` +- `backend/app/postgres_rcon_storage.py` +- `scripts/run-stats-validation.ps1` +- `docs/player-search-read-model-plan.md` +- `ai/tasks/done/TASK-198-add-player-search-read-model.md` + +## Constraints + +- Keep the change minimal and backend-only. +- Do not execute `ai-platform run`. +- Do not modify frontend. +- Do not change design. +- Do not touch images or assets. +- Do not reactivate Elo/MMR. +- Do not reintroduce Comunidad Hispana `#03`. +- Do not mix visual corrections into this task. +- PostgreSQL is the operational storage target. +- SQLite may remain only as explicit local compatibility where the current backend architecture already supports it. +- The new table is a regenerable read model, not a canonical source of truth. +- Do not require PostgreSQL extensions such as `pg_trgm`. +- Keep the current frontend contract for player search results compatible with `frontend/assets/js/stats.js`. + +## Validation + +Before completing the task ensure: + +- `powershell -ExecutionPolicy Bypass -File scripts/run-stats-validation.ps1` +- `powershell -ExecutionPolicy Bypass -File scripts/run-integration-tests.ps1` +- local validation proves the `player_search_index` table can be initialized +- local validation proves `refresh_player_search_index(...)` generates rows from materialized test data +- local validation proves player search uses `player_search_index` when data exists +- local validation proves search falls back to the runtime aggregation path when the index is empty or unavailable +- the public search payload remains compatible with `frontend/assets/js/stats.js` +- `git diff --name-only` matches the expected scope + +## Outcome + +Implemented: + +- `backend/app/rcon_historical_player_stats.py` + - added `player_search_index` initialization for SQLite compatibility and PostgreSQL-first operational mode + - added `refresh_player_search_index(...)` + - added snapshot-first search logic for `/api/stats/players/search` + - preserved runtime fallback when the read model is empty or unavailable + - added manual CLI: + - `python -m app.rcon_historical_player_stats refresh-player-search-index` +- `backend/app/postgres_rcon_storage.py` + - added PostgreSQL schema for `player_search_index` + - added scope-aware indexes for normalized name, last seen and player id +- `backend/app/payloads.py` + - preserved the public search payload contract and added backward-compatible source metadata +- `scripts/run-stats-validation.ps1` + - validates PostgreSQL schema wiring + - validates CLI defaults + - validates fixture-driven refresh generation + - validates read-model-first search + - validates fallback when the index is empty or unavailable +- `docs/player-search-read-model-plan.md` + - documents objective, fields, CLI, PostgreSQL role, fallback and current limitations + +Created table: + +- `player_search_index` + +Table behavior: + +- regenerable read model only +- one row per `(server_id, player_id)` public search scope +- supported scopes: + - `all-servers` + - `comunidad-hispana-01` + - `comunidad-hispana-02` +- aggregates only current UTC year counters +- stores Python-normalized accent-insensitive `normalized_player_name` + +CLI added: + +- `python -m app.rcon_historical_player_stats refresh-player-search-index` + +Production validation path: + +- refresh manually with the CLI above +- confirm row counts are reported for `all-servers`, `comunidad-hispana-01` and `comunidad-hispana-02` +- call `/api/stats/players/search?q=&limit=5` +- verify `data.source.read_model=player-search-index` +- verify `data.source.fallback_used=false` when the read model is populated +- verify fallback metadata appears only when the read model is empty or unavailable + +Validations executed: + +- `python -m py_compile backend/app/rcon_historical_player_stats.py backend/app/payloads.py backend/app/postgres_rcon_storage.py` +- `powershell -ExecutionPolicy Bypass -File scripts/run-stats-validation.ps1` +- `powershell -ExecutionPolicy Bypass -File scripts/run-integration-tests.ps1` +- local CLI smoke check: + - `python -m app.rcon_historical_player_stats refresh-player-search-index --sqlite-path backend/data/hll_vietnam_dev.sqlite3` + +Validation notes: + +- live backend HTTP at `http://127.0.0.1:8000` was not available in this environment +- route-contract validation still passed through local Python imports +- no frontend, design, asset, Elo/MMR or Comunidad Hispana `#03` changes were made + +Pending follow-up kept out of scope: + +- player profile and personal stats still read from runtime aggregates; only the search path is snapshot-backed in this task + +## Change Budget + +- Prefer fewer than 5 modified files. +- Prefer changes under 200 lines when feasible. +- Split the work into follow-up tasks if limits are exceeded. diff --git a/backend/app/payloads.py b/backend/app/payloads.py index 5ff2892..7ec4145 100644 --- a/backend/app/payloads.py +++ b/backend/app/payloads.py @@ -671,6 +671,7 @@ def build_stats_player_search_payload( "data": { "query": result["query"], "server_id": result["server_id"], + "source": result.get("source"), "items": result["items"], }, } diff --git a/backend/app/postgres_rcon_storage.py b/backend/app/postgres_rcon_storage.py index d508b3d..e7d7dbb 100644 --- a/backend/app/postgres_rcon_storage.py +++ b/backend/app/postgres_rcon_storage.py @@ -258,6 +258,23 @@ CREATE TABLE IF NOT EXISTS ranking_snapshot_items ( UNIQUE(snapshot_id, player_id) ); +CREATE TABLE IF NOT EXISTS player_search_index ( + id BIGSERIAL PRIMARY KEY, + server_id TEXT NOT NULL, + player_id TEXT NOT NULL, + player_name TEXT NOT NULL, + normalized_player_name TEXT NOT NULL, + first_seen_at TEXT, + last_seen_at TEXT, + servers_seen TEXT NOT NULL DEFAULT '[]', + matches_current_year INTEGER NOT NULL DEFAULT 0, + kills_current_year INTEGER NOT NULL DEFAULT 0, + deaths_current_year INTEGER NOT NULL DEFAULT 0, + teamkills_current_year INTEGER NOT NULL DEFAULT 0, + updated_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, + UNIQUE(server_id, player_id) +); + CREATE TABLE IF NOT EXISTS rcon_scoreboard_match_candidates ( id BIGSERIAL PRIMARY KEY, server_slug TEXT NOT NULL, @@ -322,6 +339,12 @@ CREATE INDEX IF NOT EXISTS idx_ranking_snapshot_items_snapshot ON ranking_snapshot_items(snapshot_id, ranking_position); CREATE INDEX IF NOT EXISTS idx_ranking_snapshot_items_player ON ranking_snapshot_items(snapshot_id, player_id); +CREATE INDEX IF NOT EXISTS idx_player_search_index_name +ON player_search_index(server_id, normalized_player_name); +CREATE INDEX IF NOT EXISTS idx_player_search_index_last_seen +ON player_search_index(server_id, last_seen_at DESC); +CREATE INDEX IF NOT EXISTS idx_player_search_index_player +ON player_search_index(server_id, player_id); CREATE INDEX IF NOT EXISTS idx_rcon_scoreboard_candidates_server_end ON rcon_scoreboard_match_candidates(server_slug, ended_at DESC, started_at DESC); """ diff --git a/backend/app/rcon_historical_player_stats.py b/backend/app/rcon_historical_player_stats.py index cadd739..1f3a098 100644 --- a/backend/app/rcon_historical_player_stats.py +++ b/backend/app/rcon_historical_player_stats.py @@ -2,17 +2,115 @@ from __future__ import annotations +import argparse +import json from contextlib import closing -from datetime import datetime, timezone +from datetime import date, datetime, timezone from pathlib import Path +import unicodedata from .config import use_postgres_rcon_storage from .historical_storage import ALL_SERVERS_SLUG from .rcon_admin_log_materialization import MATCH_RESULT_SOURCE, initialize_rcon_materialized_storage from .rcon_historical_leaderboards import select_leaderboard_window -from .sqlite_utils import connect_sqlite_readonly +from .sqlite_utils import connect_sqlite_readonly, connect_sqlite_writer from .rcon_historical_leaderboards import _to_iso +PLAYER_SEARCH_INDEX_SERVER_KEYS = ( + ALL_SERVERS_SLUG, + "comunidad-hispana-01", + "comunidad-hispana-02", +) + +PLAYER_SEARCH_INDEX_SQLITE_SCHEMA_SQL = """ +CREATE TABLE IF NOT EXISTS player_search_index ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + server_id TEXT NOT NULL, + player_id TEXT NOT NULL, + player_name TEXT NOT NULL, + normalized_player_name TEXT NOT NULL, + first_seen_at TEXT, + last_seen_at TEXT, + servers_seen TEXT NOT NULL DEFAULT '[]', + matches_current_year INTEGER NOT NULL DEFAULT 0, + kills_current_year INTEGER NOT NULL DEFAULT 0, + deaths_current_year INTEGER NOT NULL DEFAULT 0, + teamkills_current_year INTEGER NOT NULL DEFAULT 0, + updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, + UNIQUE(server_id, player_id) +); + +CREATE INDEX IF NOT EXISTS idx_player_search_index_name +ON player_search_index(server_id, normalized_player_name); + +CREATE INDEX IF NOT EXISTS idx_player_search_index_last_seen +ON player_search_index(server_id, last_seen_at DESC); + +CREATE INDEX IF NOT EXISTS idx_player_search_index_player +ON player_search_index(server_id, player_id); +""" + + +def initialize_player_search_index_storage(*, db_path: Path | None = None) -> Path: + """Create the player search read model storage if it does not exist.""" + resolved_path = initialize_rcon_materialized_storage(db_path=db_path) + if use_postgres_rcon_storage(explicit_sqlite_path=db_path): + from .postgres_rcon_storage import initialize_postgres_rcon_storage + + initialize_postgres_rcon_storage() + return resolved_path + + with closing(connect_sqlite_writer(resolved_path)) as connection: + with connection: + connection.executescript(PLAYER_SEARCH_INDEX_SQLITE_SCHEMA_SQL) + return resolved_path + + +def refresh_player_search_index( + *, + db_path: Path | None = None, + now: datetime | None = None, +) -> dict[str, object]: + """Rebuild the player search read model from current-year materialized matches.""" + anchor = _as_utc(now or datetime.now(timezone.utc)) + year_start = anchor.replace(month=1, day=1, hour=0, minute=0, second=0, microsecond=0) + next_year_start = year_start.replace(year=year_start.year + 1) + resolved_path = initialize_player_search_index_storage(db_path=db_path) + + results: list[dict[str, object]] = [] + total_rows = 0 + with _connect_write_scope(resolved_path, db_path=db_path) as connection: + for server_id in PLAYER_SEARCH_INDEX_SERVER_KEYS: + rows = _build_player_search_index_rows( + connection=connection, + server_id=server_id, + year_start=year_start, + next_year_start=next_year_start, + ) + _replace_player_search_index_scope( + connection=connection, + server_id=server_id, + rows=rows, + updated_at=_to_iso(anchor), + ) + results.append( + { + "server_id": server_id, + "row_count": len(rows), + } + ) + total_rows += len(rows) + + return { + "status": "ok", + "generated_at": _to_iso(anchor), + "year": year_start.year, + "source": "rcon-materialized-admin-log", + "server_ids": list(PLAYER_SEARCH_INDEX_SERVER_KEYS), + "total_rows": total_rows, + "results": results, + } + def search_rcon_materialized_players( *, @@ -21,11 +119,43 @@ def search_rcon_materialized_players( limit: int = 10, db_path: Path | None = None, ) -> dict[str, object]: - """Search players in materialized RCON/AdminLog statistics.""" + """Search players from the read model first, with runtime fallback.""" normalized_query = query.strip() if not normalized_query: raise ValueError("Query cannot be empty.") + read_model_result, fallback_reason = _search_player_search_index( + query=normalized_query, + server_id=server_id, + limit=limit, + db_path=db_path, + ) + if read_model_result is not None: + return read_model_result + + runtime_result = _search_rcon_materialized_players_runtime( + query=normalized_query, + server_id=server_id, + limit=limit, + db_path=db_path, + ) + runtime_result["source"] = { + "read_model": "rcon-materialized-admin-log-player-stats", + "fallback_used": True, + "fallback_reason": fallback_reason or "player-search-index-unavailable", + } + return runtime_result + + +def _search_rcon_materialized_players_runtime( + *, + query: str, + server_id: str | None = None, + limit: int = 10, + db_path: Path | None = None, +) -> dict[str, object]: + """Search players directly in materialized RCON/AdminLog statistics.""" + normalized_query = query.strip() resolved_path = initialize_rcon_materialized_storage(db_path=db_path) resolved_server_id = _normalize_server_id(server_id) normalized_pattern = _escape_like_pattern(normalized_query) @@ -98,6 +228,11 @@ def search_rcon_materialized_players( "server_id": resolved_server_id, "query": normalized_query, "items": items, + "source": { + "read_model": "rcon-materialized-admin-log-player-stats", + "fallback_used": False, + "fallback_reason": None, + }, } @@ -386,6 +521,247 @@ def _to_datetime_or_none(value: object) -> datetime | None: return parsed.astimezone(timezone.utc) +def _build_player_search_index_rows( + *, + connection: object, + server_id: str, + year_start: datetime, + next_year_start: datetime, +) -> list[dict[str, object]]: + scope_sql, scope_params = _build_scope_sql(server_id) + rows = connection.execute( + f""" + SELECT + stats.player_id, + MIN(COALESCE(CAST(matches.ended_at AS TEXT), CAST(matches.started_at AS TEXT))) AS first_seen_at, + MAX(COALESCE(CAST(matches.ended_at AS TEXT), CAST(matches.started_at AS TEXT))) AS last_seen_at, + COUNT(DISTINCT stats.match_key) AS matches_current_year, + SUM(COALESCE(stats.kills, 0)) AS kills_current_year, + SUM(COALESCE(stats.deaths, 0)) AS deaths_current_year, + SUM(COALESCE(stats.teamkills, 0)) AS teamkills_current_year + FROM rcon_match_player_stats AS stats + INNER JOIN rcon_materialized_matches AS matches + ON matches.target_key = stats.target_key + AND matches.match_key = stats.match_key + WHERE matches.source_basis = ? + AND COALESCE(CAST(matches.ended_at AS TEXT), CAST(matches.started_at AS TEXT)) >= ? + AND COALESCE(CAST(matches.ended_at AS TEXT), CAST(matches.started_at AS TEXT)) < ? + {scope_sql} + GROUP BY stats.player_id + HAVING COUNT(DISTINCT stats.match_key) > 0 + ORDER BY MAX(COALESCE(CAST(matches.ended_at AS TEXT), CAST(matches.started_at AS TEXT))) DESC, + COUNT(DISTINCT stats.match_key) DESC, + stats.player_id ASC + """, + [MATCH_RESULT_SOURCE, _to_iso(year_start), _to_iso(next_year_start), *scope_params], + ).fetchall() + if not rows: + return [] + + player_ids = [ + str(row["player_id"]).strip() + for row in rows + if str(row["player_id"] or "").strip() + ] + latest_names = _lookup_latest_player_names_with_connection( + connection=connection, + player_ids=player_ids, + server_id=server_id, + start=year_start, + end=next_year_start, + ) + servers_seen = _lookup_servers_seen_with_connection( + connection=connection, + player_ids=player_ids, + server_id=server_id, + start=year_start, + end=next_year_start, + ) + + index_rows: list[dict[str, object]] = [] + for row in rows: + player_id = str(row["player_id"] or "").strip() + if not player_id: + continue + player_name = latest_names.get(player_id) or player_id + normalized_player_name = _normalize_player_search_text(player_name) or player_id.casefold() + index_rows.append( + { + "player_id": player_id, + "player_name": player_name, + "normalized_player_name": normalized_player_name, + "first_seen_at": row["first_seen_at"], + "last_seen_at": row["last_seen_at"], + "servers_seen": servers_seen.get(player_id, []), + "matches_current_year": int(row["matches_current_year"] or 0), + "kills_current_year": int(row["kills_current_year"] or 0), + "deaths_current_year": int(row["deaths_current_year"] or 0), + "teamkills_current_year": int(row["teamkills_current_year"] or 0), + } + ) + return index_rows + + +def _replace_player_search_index_scope( + *, + connection: object, + server_id: str, + rows: list[dict[str, object]], + updated_at: str, +) -> None: + connection.execute( + """ + DELETE FROM player_search_index + WHERE server_id = ? + """, + [server_id], + ) + for row in rows: + connection.execute( + """ + INSERT INTO player_search_index ( + server_id, + player_id, + player_name, + normalized_player_name, + first_seen_at, + last_seen_at, + servers_seen, + matches_current_year, + kills_current_year, + deaths_current_year, + teamkills_current_year, + updated_at + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + """, + [ + server_id, + row["player_id"], + row["player_name"], + row["normalized_player_name"], + row["first_seen_at"], + row["last_seen_at"], + json.dumps(row["servers_seen"], ensure_ascii=True, separators=(",", ":")), + row["matches_current_year"], + row["kills_current_year"], + row["deaths_current_year"], + row["teamkills_current_year"], + updated_at, + ], + ) + + +def _search_player_search_index( + *, + query: str, + server_id: str | None = None, + limit: int = 10, + db_path: Path | None = None, +) -> tuple[dict[str, object] | None, str | None]: + resolved_path = initialize_player_search_index_storage(db_path=db_path) + resolved_server_id = _normalize_server_id(server_id) + normalized_query = query.strip() + normalized_name_query = _normalize_player_search_text(normalized_query) + anchor = datetime.now(timezone.utc) + year_start = anchor.replace(month=1, day=1, hour=0, minute=0, second=0, microsecond=0) + escaped_name_contains = f"%{_escape_like_pattern(normalized_name_query)}%" + escaped_name_prefix = f"{_escape_like_pattern(normalized_name_query)}%" + escaped_id_contains = f"%{_escape_like_pattern(normalized_query.casefold())}%" + escaped_id_exact = normalized_query.casefold() + + try: + with _connect_scope(resolved_path, db_path=db_path) as connection: + has_rows = connection.execute( + """ + SELECT 1 + FROM player_search_index + WHERE server_id = ? + LIMIT 1 + """, + [resolved_server_id], + ).fetchone() + if has_rows is None: + return None, "player-search-index-empty" + + rows = connection.execute( + """ + SELECT + player_id, + player_name, + first_seen_at, + last_seen_at, + servers_seen, + matches_current_year, + kills_current_year, + deaths_current_year, + teamkills_current_year, + updated_at + FROM player_search_index + WHERE server_id = ? + AND ( + normalized_player_name LIKE ? ESCAPE '\\' + OR LOWER(player_id) LIKE LOWER(?) + ) + ORDER BY + CASE + WHEN LOWER(player_id) = LOWER(?) THEN 0 + WHEN normalized_player_name = ? THEN 1 + WHEN normalized_player_name LIKE ? ESCAPE '\\' THEN 2 + WHEN normalized_player_name LIKE ? ESCAPE '\\' THEN 3 + ELSE 4 + END, + matches_current_year DESC, + last_seen_at DESC, + player_name ASC + LIMIT ? + """, + [ + resolved_server_id, + escaped_name_contains, + escaped_id_contains, + escaped_id_exact, + normalized_name_query, + escaped_name_prefix, + escaped_name_contains, + limit, + ], + ).fetchall() + except Exception: + return None, "player-search-index-unavailable" + + items: list[dict[str, object]] = [] + generated_at = None + for row in rows: + if generated_at is None: + generated_at = row["updated_at"] + items.append( + { + "player_id": str(row["player_id"] or "").strip(), + "player_name": str(row["player_name"] or row["player_id"] or "").strip(), + "matches_considered": int(row["matches_current_year"] or 0), + "last_seen_at": row["last_seen_at"], + "servers_seen": _deserialize_string_list(row["servers_seen"]), + } + ) + + return ( + { + "server_id": resolved_server_id, + "query": normalized_query, + "items": items, + "source": { + "read_model": "player-search-index", + "fallback_used": False, + "fallback_reason": None, + "generated_at": generated_at, + "window_start": _to_iso(year_start), + "window_end": _to_iso(anchor), + }, + }, + None, + ) + + def _lookup_latest_player_names( *, db_path: Path, @@ -426,6 +802,45 @@ def _lookup_latest_player_names( return latest_names +def _lookup_latest_player_names_with_connection( + *, + connection: object, + player_ids: list[str], + server_id: str | None, + start: datetime, + end: datetime, +) -> dict[str, str]: + if not player_ids: + return {} + placeholders = ", ".join(["?"] * len(player_ids)) + scope_sql, scope_params = _build_scope_sql(server_id) + rows = connection.execute( + f""" + SELECT + stats.player_id, + stats.player_name, + COALESCE(matches.ended_at, matches.started_at) AS last_seen_at + FROM rcon_match_player_stats AS stats + INNER JOIN rcon_materialized_matches AS matches + ON matches.target_key = stats.target_key + AND matches.match_key = stats.match_key + WHERE stats.player_id IN ({placeholders}) + AND matches.source_basis = ? + AND COALESCE(CAST(matches.ended_at AS TEXT), CAST(matches.started_at AS TEXT)) >= ? + AND COALESCE(CAST(matches.ended_at AS TEXT), CAST(matches.started_at AS TEXT)) < ? + {scope_sql} + ORDER BY COALESCE(matches.ended_at, matches.started_at) DESC, stats.player_name ASC + """, + [*player_ids, MATCH_RESULT_SOURCE, _to_iso(start), _to_iso(end), *scope_params], + ).fetchall() + latest_names: dict[str, str] = {} + for row in rows: + player_id = str(row["player_id"] or "").strip() + if player_id and player_id not in latest_names: + latest_names[player_id] = str(row["player_name"] or player_id) + return latest_names + + def _lookup_servers_seen( *, db_path: Path, @@ -465,6 +880,50 @@ def _lookup_servers_seen( return by_player +def _lookup_servers_seen_with_connection( + *, + connection: object, + player_ids: list[str], + server_id: str | None, + start: datetime, + end: datetime, +) -> dict[str, list[str]]: + if not player_ids: + return {} + placeholders = ", ".join(["?"] * len(player_ids)) + scope_sql, scope_params = _build_scope_sql(server_id) + rows = connection.execute( + f""" + SELECT + stats.player_id, + COALESCE(matches.external_server_id, matches.target_key) AS server_id + FROM rcon_match_player_stats AS stats + INNER JOIN rcon_materialized_matches AS matches + ON matches.target_key = stats.target_key + AND matches.match_key = stats.match_key + WHERE stats.player_id IN ({placeholders}) + AND matches.source_basis = ? + AND COALESCE(CAST(matches.ended_at AS TEXT), CAST(matches.started_at AS TEXT)) >= ? + AND COALESCE(CAST(matches.ended_at AS TEXT), CAST(matches.started_at AS TEXT)) < ? + {scope_sql} + GROUP BY stats.player_id, COALESCE(matches.external_server_id, matches.target_key) + ORDER BY stats.player_id, server_id ASC + """, + [*player_ids, MATCH_RESULT_SOURCE, _to_iso(start), _to_iso(end), *scope_params], + ).fetchall() + + by_player: dict[str, list[str]] = {} + for row in rows: + player_id = str(row["player_id"] or "").strip() + if not player_id: + continue + by_player.setdefault(player_id, []) + candidate_server_id = str(row["server_id"] or "").strip() + if candidate_server_id and candidate_server_id not in by_player[player_id]: + by_player[player_id].append(candidate_server_id) + return by_player + + def _build_scope_sql( server_id: str | None, *, @@ -511,3 +970,84 @@ def _connect_scope(resolved_path: Path, *, db_path: Path | None = None): return connect_postgres_compat() return closing(connect_sqlite_readonly(resolved_path)) + + +def _connect_write_scope(resolved_path: Path, *, db_path: Path | None = None): + if use_postgres_rcon_storage(explicit_sqlite_path=db_path): + from .postgres_rcon_storage import connect_postgres_compat + + return connect_postgres_compat() + return connect_sqlite_writer(resolved_path) + + +def _normalize_player_search_text(value: object) -> str: + text = str(value or "").strip().casefold() + if not text: + return "" + decomposed = unicodedata.normalize("NFKD", text) + without_accents = "".join(character for character in decomposed if not unicodedata.combining(character)) + compact = " ".join(without_accents.split()) + return compact + + +def _deserialize_string_list(value: object) -> list[str]: + if isinstance(value, list): + return [str(item).strip() for item in value if str(item).strip()] + if not isinstance(value, str) or not value.strip(): + return [] + try: + parsed = json.loads(value) + except json.JSONDecodeError: + return [] + if not isinstance(parsed, list): + return [] + return [str(item).strip() for item in parsed if str(item).strip()] + + +def _as_utc(value: datetime) -> datetime: + if value.tzinfo is None: + return value.replace(tzinfo=timezone.utc) + return value.astimezone(timezone.utc) + + +def _json_default(value: object) -> str: + if isinstance(value, datetime): + return _to_iso(value) + if isinstance(value, date): + return value.isoformat() + raise TypeError(f"Object of type {type(value).__name__} is not JSON serializable") + + +def _main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser( + description="Search and refresh player-search read models over materialized RCON stats.", + ) + subparsers = parser.add_subparsers(dest="command") + + refresh_parser = subparsers.add_parser("refresh-player-search-index") + refresh_parser.add_argument( + "--sqlite-path", + type=Path, + default=None, + help="explicit local SQLite override; default operational mode uses PostgreSQL when configured", + ) + + args = parser.parse_args(argv) + if args.command == "refresh-player-search-index": + payload = refresh_player_search_index(db_path=args.sqlite_path) + print( + json.dumps( + {"status": "ok", "data": payload}, + ensure_ascii=True, + indent=2, + default=_json_default, + ) + ) + return 0 + + parser.print_help() + return 2 + + +if __name__ == "__main__": + raise SystemExit(_main()) diff --git a/docs/player-search-read-model-plan.md b/docs/player-search-read-model-plan.md new file mode 100644 index 0000000..2f518a8 --- /dev/null +++ b/docs/player-search-read-model-plan.md @@ -0,0 +1,126 @@ +# Player Search Read Model + +## Objective + +Define the first dedicated read model for player search so `/api/stats/players/search` does not aggregate large RCON historical tables on every public request. + +## Table + +Primary table: + +- `player_search_index` + +Operational scope: + +- PostgreSQL is the default operational store +- SQLite remains available only as explicit local compatibility for validation and isolated maintenance runs + +Read-model nature: + +- this table is regenerable +- it is not the canonical source of truth +- canonical historical data remains in `rcon_materialized_matches` and `rcon_match_player_stats` + +## Stored Fields + +Each row stores one player projection for one public search scope: + +- `server_id` +- `player_id` +- `player_name` +- `normalized_player_name` +- `first_seen_at` +- `last_seen_at` +- `servers_seen` +- `matches_current_year` +- `kills_current_year` +- `deaths_current_year` +- `teamkills_current_year` +- `updated_at` + +Current scope rows: + +- `all-servers` +- `comunidad-hispana-01` +- `comunidad-hispana-02` + +Why `server_id` exists: + +- the public search endpoint already supports `server_id` +- keeping one row per scope preserves the existing frontend contract without recalculating large runtime aggregates for server-filtered searches + +## Refresh Command + +Manual command: + +```bash +python -m app.rcon_historical_player_stats refresh-player-search-index +``` + +SQLite-only local override: + +```bash +python -m app.rcon_historical_player_stats refresh-player-search-index --sqlite-path backend/data/hll_vietnam_dev.sqlite3 +``` + +Refresh policy: + +- rebuild from materialized RCON/AdminLog tables +- replace rows scope by scope +- aggregate only the current UTC year +- keep the latest current-year player name +- store accent-insensitive normalized names in Python + +## Public Read Path + +Priority for `/api/stats/players/search`: + +1. use `player_search_index` when the requested scope has rows +2. return read-model results directly, including empty query results when the index is populated but the query does not match +3. fall back to runtime aggregation only when: + - the read model table is unavailable + - the requested scope has no rows yet + - a controlled read error occurs + +Returned compatibility: + +- the payload still returns `player_id` +- the payload still returns `player_name` +- the payload still returns `matches_considered` +- the payload still returns `last_seen_at` +- the payload still returns `servers_seen` + +`matches_considered` remains compatible by mapping from `matches_current_year`. + +## PostgreSQL Notes + +No extra PostgreSQL extensions are required. + +Specifically: + +- no `pg_trgm` +- no custom text-search extension + +Search tolerance is implemented with: + +- normalized lowercase names +- accent stripping in Python +- indexed scope + normalized-name reads +- runtime fallback preserved as a safety net + +## Current Limitations + +- this read model is focused on player search only, not personal profile totals +- counts are current-year only by design +- historical players with no activity in the current UTC year are not intentionally prioritized in this first model +- profile and personal stats still use runtime aggregation over materialized tables + +## Production Validation + +Recommended checks after refresh: + +- run `python -m app.rcon_historical_player_stats refresh-player-search-index` +- confirm the command reports rows for `all-servers`, `comunidad-hispana-01` and `comunidad-hispana-02` +- call `/api/stats/players/search?q=&limit=5` +- verify response metadata reports `read_model=player-search-index` +- verify fallback metadata only appears when the read model is empty or unavailable diff --git a/scripts/run-stats-validation.ps1 b/scripts/run-stats-validation.ps1 index 6db9bd2..4701151 100644 --- a/scripts/run-stats-validation.ps1 +++ b/scripts/run-stats-validation.ps1 @@ -128,11 +128,16 @@ from app.routes import resolve_get_payload from app.config import use_postgres_rcon_storage import app.postgres_rcon_storage as postgres_rcon_storage import app.rcon_historical_leaderboards as ranking_leaderboards +import app.rcon_historical_player_stats as player_search_stats initialize_ranking_snapshot_storage = ranking_leaderboards.initialize_ranking_snapshot_storage generate_ranking_snapshot = ranking_leaderboards.generate_ranking_snapshot get_latest_ranking_snapshot = ranking_leaderboards.get_latest_ranking_snapshot refresh_ranking_snapshots = ranking_leaderboards.refresh_ranking_snapshots +initialize_player_search_index_storage = player_search_stats.initialize_player_search_index_storage +refresh_player_search_index = player_search_stats.refresh_player_search_index +search_rcon_materialized_players = player_search_stats.search_rcon_materialized_players +MATCH_RESULT_SOURCE = player_search_stats.MATCH_RESULT_SOURCE def require(condition, message): @@ -539,6 +544,283 @@ def validate_ranking_snapshot_postgres_selection(): postgres_rcon_storage.connect_postgres_compat = original_connect_postgres_compat +def validate_postgres_player_search_index_schema_path(): + require( + "CREATE TABLE IF NOT EXISTS player_search_index" in postgres_rcon_storage.RCON_SCHEMA_SQL, + "PostgreSQL schema should define player_search_index.", + ) + require( + "CREATE INDEX IF NOT EXISTS idx_player_search_index_name" in postgres_rcon_storage.RCON_SCHEMA_SQL, + "PostgreSQL schema should define the player_search_index normalized name index.", + ) + + original_initialize_materialized = player_search_stats.initialize_rcon_materialized_storage + original_use_postgres_storage = player_search_stats.use_postgres_rcon_storage + original_initialize_postgres_storage = postgres_rcon_storage.initialize_postgres_rcon_storage + calls = {"postgres_init": 0} + + player_search_stats.initialize_rcon_materialized_storage = ( + lambda db_path=None: Path("backend/data/hll_vietnam_dev.sqlite3") + ) + player_search_stats.use_postgres_rcon_storage = lambda explicit_sqlite_path=None: True + + def fake_initialize_postgres_storage(): + calls["postgres_init"] += 1 + + postgres_rcon_storage.initialize_postgres_rcon_storage = fake_initialize_postgres_storage + + try: + initialize_player_search_index_storage() + finally: + player_search_stats.initialize_rcon_materialized_storage = original_initialize_materialized + player_search_stats.use_postgres_rcon_storage = original_use_postgres_storage + postgres_rcon_storage.initialize_postgres_rcon_storage = original_initialize_postgres_storage + + require( + calls["postgres_init"] == 1, + "Player search index initialization should delegate to initialize_postgres_rcon_storage exactly once.", + ) + + +def validate_player_search_index_cli_defaults(): + original_refresh_player_search_index = player_search_stats.refresh_player_search_index + captured = {} + + def fake_refresh_player_search_index(**kwargs): + captured.update(kwargs) + return { + "status": "ok", + "generated_at": datetime(2026, 6, 9, 8, 0, 0, tzinfo=timezone.utc), + "year": 2026, + "source": "rcon-materialized-admin-log", + "server_ids": ["all-servers", "comunidad-hispana-01", "comunidad-hispana-02"], + "total_rows": 3, + "results": [{"server_id": "all-servers", "row_count": 3}], + } + + player_search_stats.refresh_player_search_index = fake_refresh_player_search_index + try: + stdout_buffer = StringIO() + with redirect_stdout(stdout_buffer): + exit_code = player_search_stats._main(["refresh-player-search-index"]) + require(exit_code == 0, "Player search index CLI should exit 0 for a valid command.") + require( + captured.get("db_path") is None, + "Player search index CLI should use PostgreSQL-compatible db_path=None by default.", + ) + serialized_default_payload = json.loads(stdout_buffer.getvalue()) + require( + serialized_default_payload.get("data", {}).get("generated_at") == "2026-06-09T08:00:00Z", + "Player search index CLI should serialize datetime values as ISO strings.", + ) + + captured.clear() + stdout_buffer = StringIO() + with redirect_stdout(stdout_buffer): + exit_code = player_search_stats._main([ + "refresh-player-search-index", + "--sqlite-path", "backend/data/hll_vietnam_dev.sqlite3", + ]) + require(exit_code == 0, "Player search index CLI with --sqlite-path should exit 0.") + require( + captured.get("db_path") == Path("backend/data/hll_vietnam_dev.sqlite3"), + "Player search index CLI should pass the explicit --sqlite-path override through to refresh_player_search_index.", + ) + finally: + player_search_stats.refresh_player_search_index = original_refresh_player_search_index + + +def build_player_search_fixture(): + stamp = datetime.now(timezone.utc).strftime("%Y%m%d%H%M%S%f") + db_path = Path(f"backend/data/hll_vietnam_player_search_validation_{stamp}.sqlite3") + if db_path.exists(): + try: + db_path.unlink() + except PermissionError: + pass + initialize_player_search_index_storage(db_path=db_path) + connection = sqlite3.connect(db_path) + connection.row_factory = sqlite3.Row + with connection: + connection.execute( + """ + INSERT INTO rcon_materialized_matches ( + target_key, external_server_id, match_key, map_name, map_pretty_name, game_mode, + started_server_time, ended_server_time, started_at, ended_at, + allied_score, axis_score, winner, confidence_mode, source_basis + ) VALUES + ('comunidad-hispana-01', 'comunidad-hispana-01', 'm1', 'stmarie', 'St. Marie Du Mont', 'warfare', 1, 2, '2026-03-01T18:00:00Z', '2026-03-01T19:00:00Z', 5, 3, 'allied', 'exact', ?), + ('comunidad-hispana-02', 'comunidad-hispana-02', 'm2', 'foy', 'Foy', 'warfare', 3, 4, '2026-04-01T18:00:00Z', '2026-04-01T19:00:00Z', 2, 5, 'axis', 'exact', ?), + ('comunidad-hispana-01', 'comunidad-hispana-01', 'm3', 'kursk', 'Kursk', 'warfare', 5, 6, '2025-12-01T18:00:00Z', '2025-12-01T19:00:00Z', 1, 0, 'allied', 'exact', ?) + """, + (MATCH_RESULT_SOURCE, MATCH_RESULT_SOURCE, MATCH_RESULT_SOURCE), + ) + connection.execute( + """ + INSERT INTO rcon_match_player_stats ( + target_key, match_key, player_id, player_name, team, + kills, deaths, teamkills, deaths_by_teamkill, + weapons_json, death_by_weapons_json, most_killed_json, death_by_json, + first_seen_server_time, last_seen_server_time + ) VALUES + ('comunidad-hispana-01', 'm1', 'player-rambo', 'Rámbó', 'allied', 12, 4, 1, 0, '{}', '{}', '{}', '{}', 1, 2), + ('comunidad-hispana-02', 'm2', 'player-rambo', 'Rambo', 'axis', 8, 6, 0, 0, '{}', '{}', '{}', '{}', 3, 4), + ('comunidad-hispana-01', 'm3', 'player-rambo', 'Old Rambo', 'allied', 99, 1, 0, 0, '{}', '{}', '{}', '{}', 5, 6), + ('comunidad-hispana-01', 'm1', 'player-ghost', 'Ghost', 'axis', 3, 9, 0, 0, '{}', '{}', '{}', '{}', 1, 2) + """ + ) + connection.close() + return db_path + + +def validate_player_search_index_refresh_and_search(): + db_path = build_player_search_fixture() + try: + payload = refresh_player_search_index( + db_path=db_path, + now=datetime(2026, 6, 9, 8, 0, 0, tzinfo=timezone.utc), + ) + require(payload.get("status") == "ok", "Player search index refresh should return ok.") + require(payload.get("year") == 2026, "Player search index refresh should use the current UTC year.") + require_int(payload.get("total_rows"), "Player search index refresh should report numeric total_rows.") + + connection = sqlite3.connect(db_path) + connection.row_factory = sqlite3.Row + rows = connection.execute( + """ + SELECT server_id, player_id, player_name, normalized_player_name, + matches_current_year, kills_current_year, deaths_current_year, + teamkills_current_year, servers_seen + FROM player_search_index + ORDER BY server_id ASC, player_id ASC + """ + ).fetchall() + connection.close() + require(len(rows) >= 4, "Player search index refresh should materialize rows for all and per-server scopes.") + all_scope_row = next( + ( + dict(row) + for row in rows + if row["server_id"] == "all-servers" and row["player_id"] == "player-rambo" + ), + None, + ) + require(all_scope_row is not None, "Player search index should contain an all-servers row for player-rambo.") + require( + all_scope_row["player_name"] == "Rambo", + "Player search index should preserve the latest current-year player display name.", + ) + require( + all_scope_row["normalized_player_name"] == "rambo", + "Player search index should persist an accent-insensitive normalized player name.", + ) + require( + int(all_scope_row["matches_current_year"] or 0) == 2, + "Player search index should only count current-year matches.", + ) + require( + int(all_scope_row["kills_current_year"] or 0) == 20, + "Player search index should aggregate current-year kills.", + ) + require( + json.loads(all_scope_row["servers_seen"]) == ["comunidad-hispana-01", "comunidad-hispana-02"], + "Player search index should preserve the list of servers seen in scope order.", + ) + + read_model_result = search_rcon_materialized_players( + query="Rambo", + server_id="all", + limit=10, + db_path=db_path, + ) + require( + (read_model_result.get("source") or {}).get("read_model") == "player-search-index", + "Player search should prefer player_search_index when populated.", + ) + require( + (read_model_result.get("source") or {}).get("fallback_used") is False, + "Player search should not report fallback when player_search_index is used.", + ) + items = read_model_result.get("items") or [] + require(len(items) >= 1, "Player search read model should return at least one item for Rambo.") + require( + items[0].get("player_id") == "player-rambo", + "Player search read model should return player-rambo first for a direct-name query.", + ) + require( + items[0].get("matches_considered") == 2, + "Player search read model should preserve the current-year match count in the public contract.", + ) + + server_scoped_result = search_rcon_materialized_players( + query="Rambo", + server_id="comunidad-hispana-01", + limit=10, + db_path=db_path, + ) + server_items = server_scoped_result.get("items") or [] + require( + len(server_items) >= 1 and server_items[0].get("matches_considered") == 1, + "Server-scoped player search should use the server-specific index row when available.", + ) + finally: + if db_path.exists(): + try: + db_path.unlink() + except PermissionError: + pass + + +def validate_player_search_index_fallbacks(): + db_path = build_player_search_fixture() + try: + fallback_result = search_rcon_materialized_players( + query="Rambo", + server_id="all", + limit=10, + db_path=db_path, + ) + require( + (fallback_result.get("source") or {}).get("fallback_used") is True, + "Player search should fall back to runtime when player_search_index is empty.", + ) + require( + (fallback_result.get("source") or {}).get("fallback_reason") == "player-search-index-empty", + "Player search should expose an explicit empty-index fallback reason.", + ) + + original_search_player_search_index = player_search_stats._search_player_search_index + + def fake_search_player_search_index(**kwargs): + return None, "player-search-index-unavailable" + + player_search_stats._search_player_search_index = fake_search_player_search_index + try: + unavailable_result = search_rcon_materialized_players( + query="Rambo", + server_id="all", + limit=10, + db_path=db_path, + ) + finally: + player_search_stats._search_player_search_index = original_search_player_search_index + + require( + (unavailable_result.get("source") or {}).get("fallback_used") is True, + "Player search should preserve runtime fallback when the read model is unavailable.", + ) + require( + (unavailable_result.get("source") or {}).get("fallback_reason") == "player-search-index-unavailable", + "Player search should expose an explicit unavailable-index fallback reason.", + ) + finally: + if db_path.exists(): + try: + db_path.unlink() + except PermissionError: + pass + + def cleanup_snapshot_fixture(db_path): connection = sqlite3.connect(db_path) with connection: @@ -584,6 +866,10 @@ validate_ranking_snapshot_cli_defaults() validate_ranking_snapshot_postgres_selection() validate_ranking_snapshot_bulk_refresh() validate_ranking_snapshot_bulk_partial_failure() +validate_postgres_player_search_index_schema_path() +validate_player_search_index_cli_defaults() +validate_player_search_index_refresh_and_search() +validate_player_search_index_fallbacks() kd_metric_sql, _, _ = ranking_leaderboards._resolve_metric_sql("kd_ratio") require( @@ -611,6 +897,8 @@ search_data = search_payload.get("data") or {} require(search_payload.get("status") == "ok", "Stats player search should return ok status.") require(search_data.get("query") == "regression-check", "Stats player search should preserve query.") require(search_data.get("server_id"), "Stats player search should include server_id.") +require(isinstance(search_data.get("source"), dict), "Stats player search should expose source metadata.") +require((search_data.get("source") or {}).get("read_model") in {"player-search-index", "rcon-materialized-admin-log-player-stats"}, "Stats player search should expose a recognized read model.") require(isinstance(search_data.get("items"), list), "Stats player search items must be a list.") for item in search_data["items"]: