From 18bb1564342ca66f08a0bbf939aae4406dfa2357 Mon Sep 17 00:00:00 2001 From: devRaGonSa Date: Tue, 2 Jun 2026 11:21:15 +0200 Subject: [PATCH] feat: add database maintenance and update home copy --- ...SK-database-maintenance-cleanup-command.md | 0 .../TASK-database-maintenance-docs.md | 0 .../TASK-db-retention-audit.md | 0 ...K-final-maintenance-and-copy-validation.md | 0 ...historical-runner-maintenance-scheduler.md | 0 ...home-remove-vietnam-copy-except-trailer.md | 0 .../review/TASK-db-retention-audit-report.md | 518 ++++++++++++++ ...-maintenance-and-copy-validation-report.md | 122 ++++ backend/app/config.py | 70 ++ backend/app/database_maintenance.py | 638 ++++++++++++++++++ backend/app/historical_runner.py | 92 +++ backend/tests/test_database_maintenance.py | 448 ++++++++++++ .../test_historical_runner_maintenance.py | 153 +++++ docs/database-maintenance.md | 307 +++++++++ frontend/index.html | 8 +- 15 files changed, 2352 insertions(+), 4 deletions(-) rename ai/tasks/{pending => done}/TASK-database-maintenance-cleanup-command.md (100%) rename ai/tasks/{pending => done}/TASK-database-maintenance-docs.md (100%) rename ai/tasks/{pending => done}/TASK-db-retention-audit.md (100%) rename ai/tasks/{pending => done}/TASK-final-maintenance-and-copy-validation.md (100%) rename ai/tasks/{pending => done}/TASK-historical-runner-maintenance-scheduler.md (100%) rename ai/tasks/{pending => done}/TASK-home-remove-vietnam-copy-except-trailer.md (100%) create mode 100644 ai/tasks/review/TASK-db-retention-audit-report.md create mode 100644 ai/tasks/review/TASK-final-maintenance-and-copy-validation-report.md create mode 100644 backend/app/database_maintenance.py create mode 100644 backend/tests/test_database_maintenance.py create mode 100644 backend/tests/test_historical_runner_maintenance.py create mode 100644 docs/database-maintenance.md diff --git a/ai/tasks/pending/TASK-database-maintenance-cleanup-command.md b/ai/tasks/done/TASK-database-maintenance-cleanup-command.md similarity index 100% rename from ai/tasks/pending/TASK-database-maintenance-cleanup-command.md rename to ai/tasks/done/TASK-database-maintenance-cleanup-command.md diff --git a/ai/tasks/pending/TASK-database-maintenance-docs.md b/ai/tasks/done/TASK-database-maintenance-docs.md similarity index 100% rename from ai/tasks/pending/TASK-database-maintenance-docs.md rename to ai/tasks/done/TASK-database-maintenance-docs.md diff --git a/ai/tasks/pending/TASK-db-retention-audit.md b/ai/tasks/done/TASK-db-retention-audit.md similarity index 100% rename from ai/tasks/pending/TASK-db-retention-audit.md rename to ai/tasks/done/TASK-db-retention-audit.md diff --git a/ai/tasks/pending/TASK-final-maintenance-and-copy-validation.md b/ai/tasks/done/TASK-final-maintenance-and-copy-validation.md similarity index 100% rename from ai/tasks/pending/TASK-final-maintenance-and-copy-validation.md rename to ai/tasks/done/TASK-final-maintenance-and-copy-validation.md diff --git a/ai/tasks/pending/TASK-historical-runner-maintenance-scheduler.md b/ai/tasks/done/TASK-historical-runner-maintenance-scheduler.md similarity index 100% rename from ai/tasks/pending/TASK-historical-runner-maintenance-scheduler.md rename to ai/tasks/done/TASK-historical-runner-maintenance-scheduler.md diff --git a/ai/tasks/pending/TASK-home-remove-vietnam-copy-except-trailer.md b/ai/tasks/done/TASK-home-remove-vietnam-copy-except-trailer.md similarity index 100% rename from ai/tasks/pending/TASK-home-remove-vietnam-copy-except-trailer.md rename to ai/tasks/done/TASK-home-remove-vietnam-copy-except-trailer.md diff --git a/ai/tasks/review/TASK-db-retention-audit-report.md b/ai/tasks/review/TASK-db-retention-audit-report.md new file mode 100644 index 0000000..c6f8a67 --- /dev/null +++ b/ai/tasks/review/TASK-db-retention-audit-report.md @@ -0,0 +1,518 @@ +--- +id: TASK-db-retention-audit-report +source_task: TASK-db-retention-audit +status: review +type: report +--- + +# Database Retention Audit + +## Scope + +This report audits the current storage model used by HLL Vietnam for: + +- live server snapshots; +- public-scoreboard historical matches and player stats; +- RCON AdminLog events and materialized matches; +- displayed historical snapshots; +- player event raw ledger; +- paused Elo/MMR storage surfaces. + +Confirmed facts below are based on the current repository code, not on assumptions. + +## Files Inspected + +- `AGENTS.md` +- `ai/repo-context.md` +- `ai/architecture-index.md` +- `backend/app/config.py` +- `backend/app/historical_runner.py` +- `backend/app/historical_storage.py` +- `backend/app/historical_snapshot_storage.py` +- `backend/app/player_event_aggregates.py` +- `backend/app/player_event_storage.py` +- `backend/app/postgres_display_storage.py` +- `backend/app/postgres_rcon_storage.py` +- `backend/app/rcon_admin_log_materialization.py` +- `backend/app/rcon_admin_log_storage.py` +- `backend/app/rcon_historical_backfill.py` +- `backend/app/rcon_historical_leaderboards.py` +- `backend/app/rcon_historical_read_model.py` +- `backend/app/rcon_historical_worker.py` +- `backend/app/writer_lock.py` + +## Confirmed Storage Areas + +### Live and displayed data + +- `server_snapshots` + - Present in SQLite live storage and PostgreSQL displayed storage. + - Used for live/current server status history. +- `displayed_historical_snapshots` + - PostgreSQL primary key is `(server_key, snapshot_type, metric, snapshot_window)`. + - Persistence uses upsert semantics, so normal operation is already bounded by logical identity. + +### Public-scoreboard historical fallback + +- `historical_servers` +- `historical_maps` +- `historical_matches` +- `historical_players` +- `historical_player_match_stats` +- `historical_ingestion_runs` +- `historical_backfill_progress` + +These remain part of the displayed historical fallback/read path. + +### RCON historical primary path + +- `rcon_historical_targets` +- `rcon_historical_capture_runs` +- `rcon_historical_samples` +- `rcon_historical_checkpoints` +- `rcon_historical_competitive_windows` +- `rcon_admin_log_events` +- `rcon_player_profile_snapshots` +- `rcon_materialized_matches` +- `rcon_match_player_stats` +- `rcon_scoreboard_match_candidates` + +### Player-event ledger + +- `player_event_raw_ledger` +- `player_event_ingestion_runs` +- `player_event_backfill_progress` + +### Elo/MMR paused surfaces + +Confirmed in backend documentation and runtime payloads as preserved but paused. +This audit did not find cleanup code or explicit retention logic for Elo/MMR tables in the task’s read-first files, so they should stay out of cleanup scope. + +## Snapshot Boundedness Findings + +### `displayed_historical_snapshots` + +Confirmed bounded in PostgreSQL by primary key plus upsert: + +- same logical snapshot identity overwrites previous payload; +- normal scheduled refresh does not append unlimited rows; +- routine cleanup is not required for healthy records. + +Safe policy: + +- do not bulk-delete this table in the first maintenance command; +- only report suspicious legacy/corrupt rows if found later. + +### File-based snapshots + +Confirmed bounded by deterministic file paths under `backend/data/snapshots//...`. +Normal writes replace files in place. + +Safe policy: + +- do not add file cleanup in the first database-maintenance command. + +## Retention Windows Confirmed In Runtime Code + +Two different historical paths currently exist, and they do not use identical monthly rules. + +### Public-scoreboard fallback (`historical_*`) + +Confirmed in `backend/app/historical_storage.py`: + +- weekly fallback can use previous week when current-week sample is insufficient; +- monthly fallback uses previous month only when current month has zero closed matches; +- `is_early_month` is computed with `current_time.day <= 3`, but selection still depends on zero current-month matches. + +### RCON materialized leaderboards (`rcon_materialized_matches`) + +Confirmed in `backend/app/rcon_historical_leaderboards.py`: + +- monthly leaderboard selection uses previous month through day 7 inclusive; +- weekly selection falls back to previous week when current week lacks the configured minimum closed matches. + +## Cleanup Policy Recommendation + +Because the new maintenance task explicitly requires: + +- latest 100 materialized matches; +- current month; +- previous month during first 7 days of a new month; +- current week; +- previous week when weekly fallback may need it; + +the cleanup command should implement the stricter policy directly instead of copying the older `historical_storage.py` month fallback behavior. + +## Safe-To-Delete Areas + +### 1. `server_snapshots` + +Safe to delete old rows by `captured_at` age once outside retention. + +Why: + +- live UI only needs recent history; +- no dependent child tables were found; +- latest snapshot queries use recent rows only. + +Recommended initial retention: + +- keep last 14 days by default. + +### 2. Non-critical `rcon_admin_log_events` + +Safe to delete by event age after retention. + +Confirmed critical event types: + +- `kill` +- `match_start` +- `match_end` + +Everything else should be treated as non-critical for first-pass cleanup: + +- `chat` +- `message` +- `connected` +- `disconnected` +- `team_switch` +- `unknown` +- `kick` +- `ban` +- any other non-materialization event type. + +Recommended initial retention: + +- non-critical: 30 days. + +### 3. Critical `rcon_admin_log_events` + +Conditionally safe to delete only when both are true: + +- older than the critical retention threshold; +- not needed by protected materialized matches/windows. + +Recommended initial retention: + +- critical: 90 days. + +### 4. Old `rcon_materialized_matches` + +Conditionally safe to delete when all of the following are true: + +- not in latest N closed matches; +- not in current month; +- not in previous month during the first 7 days of a new month; +- not in current week; +- not in previous week when weekly fallback may still be needed. + +### 5. Child rows in `rcon_match_player_stats` + +Safe to delete only as a dependent delete for non-protected materialized matches. + +Delete these before deleting parent matches because no cascade constraint was found. + +## Areas That Must Be Protected + +### RCON materialized read path + +Protect: + +- latest 100 closed `rcon_materialized_matches`; +- all `rcon_match_player_stats` for protected matches; +- `kill`, `match_start`, `match_end` events that still support protected matches or currently relevant windows. + +### Live/current match surfaces + +Protect recent/current RCON data: + +- recent `rcon_admin_log_events` still used by current-match kill feed and current-match player stats; +- recent `server_snapshots`. + +Reason: + +- current-match pages read recent AdminLog windows directly; +- premature deletion would break kill feed or partial player stats. + +### Displayed snapshots + +Protect: + +- `displayed_historical_snapshots` as a bounded read cache; +- file-based snapshots under `backend/data/snapshots/`. + +### Public-scoreboard `historical_*` + +Do not delete in the first cleanup command. + +Reason: + +- these tables still back visible historical endpoints and fallback behavior; +- the task’s cleanup requirements are expressed around materialized matches and AdminLog, not around legacy scoreboard imports; +- deleting this domain now would widen scope materially. + +### Player-event ledger + +Do not delete in the first cleanup command. + +Reason: + +- it feeds visible player-event snapshots and monthly MVP V2 support; +- no task requirement currently defines safe retention windows for this ledger; +- deleting it without product retention rules is risky. + +### Elo/MMR paused data + +Do not delete. + +Reason: + +- task explicitly warns against unrelated Elo/MMR deletions; +- runtime is paused, but persisted surfaces are intentionally preserved. + +## Foreign Keys, Cascades, and Orphan Risk + +### Confirmed foreign keys + +Public-scoreboard historical tables have foreign keys: + +- `historical_matches.historical_server_id -> historical_servers.id` +- `historical_matches.historical_map_id -> historical_maps.id` +- `historical_player_match_stats.historical_match_id -> historical_matches.id` +- `historical_player_match_stats.historical_player_id -> historical_players.id` + +RCON/postgres tables also define references for some capture-oriented tables. + +### Missing cascade behavior relevant to cleanup + +No `ON DELETE CASCADE` behavior was found for: + +- `rcon_materialized_matches` +- `rcon_match_player_stats` + +The materialization code explicitly deletes/rebuilds child player rows itself, which is another signal that cleanup must manage delete order manually. + +### Required delete order + +For materialized RCON cleanup: + +1. identify protected match keys; +2. identify candidate old matches outside protected windows; +3. delete `rcon_match_player_stats` for candidate match keys; +4. delete candidate rows from `rcon_materialized_matches`; +5. delete eligible `rcon_admin_log_events`; +6. delete eligible `server_snapshots`. + +This order avoids orphaned player-stat rows. + +## Risks and Edge Cases + +### Timestamp parsing risk + +Some match rows may have: + +- missing `started_at`; +- missing `ended_at`; +- only server-time fields; +- malformed or non-ISO timestamps. + +Safe rule: + +- if a materialized match timestamp cannot be interpreted safely, protect it and skip deletion. + +### Cross-path monthly rule mismatch + +There is a confirmed mismatch today: + +- public-scoreboard fallback code does not strictly use previous month through day 7; +- RCON leaderboard code does. + +Cleanup should follow the task requirement, not the older fallback implementation. + +### Current-match dependency on recent AdminLog + +Current match pages query recent AdminLog rows directly, not only materialized matches. +Deleting recent critical events too aggressively could break live/current-match UI even if historical leaderboards still work. + +### Optional table presence + +Some environments may not have every migrated or optional table yet. +Maintenance code should treat absent tables as controlled skips, not hard failures. + +### SQLite versus PostgreSQL + +The repo still supports SQLite fallback when `HLL_BACKEND_DATABASE_URL` is unset. +Maintenance code should either: + +- support SQLite safely for the targeted tables; or +- emit a clear unsupported message. + +Given the codebase still keeps local SQLite testability, SQLite support is preferable for the cleanup command and its tests. + +## Recommended Cleanup Scope For First Implementation + +Implement only: + +- `server_snapshots`; +- `rcon_admin_log_events`; +- `rcon_materialized_matches`; +- `rcon_match_player_stats`. + +Explicitly skip in first implementation: + +- `displayed_historical_snapshots`; +- `historical_*`; +- `player_event_raw_ledger` and related worker tables; +- Elo/MMR tables; +- `rcon_historical_samples`, `rcon_historical_competitive_windows`, `rcon_player_profile_snapshots`, `rcon_scoreboard_match_candidates`. + +Reason: + +- keeps scope aligned to the task; +- avoids deleting sources still needed for visible history fallback or paused domains; +- stays within the repository’s change-budget expectations. + +## Implementation Plan For `app.database_maintenance` + +1. Add maintenance config accessors for all retention values and batch size. +2. Add a CLI with `cleanup --dry-run` and `cleanup --apply`. +3. Build one plan phase that calculates: + - protected materialized match keys; + - candidate match keys; + - candidate admin-log rows by criticality and age; + - candidate server snapshots by age. +4. Log the plan in structured JSON before any deletes. +5. On apply: + - acquire the shared backend writer lock; + - delete `rcon_match_player_stats` in batches for candidate match keys; + - delete `rcon_materialized_matches` in batches; + - delete `rcon_admin_log_events` in batches; + - delete `server_snapshots` in batches; + - optionally run `VACUUM ANALYZE` only when explicitly requested. +6. Treat missing optional tables as logged skips. +7. Keep dry-run as default behavior. + +## Recommended SQL Audit Commands + +### Table-size audit + +```sql +select + schemaname, + relname as table_name, + pg_size_pretty(pg_total_relation_size(relid)) as total_size, + pg_size_pretty(pg_relation_size(relid)) as table_size, + pg_size_pretty(pg_total_relation_size(relid) - pg_relation_size(relid)) as indexes_size, + n_live_tup as estimated_rows, + n_dead_tup as estimated_dead_rows +from pg_stat_user_tables +order by pg_total_relation_size(relid) desc; +``` + +### AdminLog by type and age + +```sql +select + event_type, + count(*) as row_count, + min(event_timestamp) as first_event_timestamp, + max(event_timestamp) as last_event_timestamp, + min(server_time) as first_server_time, + max(server_time) as last_server_time +from rcon_admin_log_events +group by event_type +order by row_count desc, event_type asc; +``` + +### Materialized matches by server and closure time + +```sql +select + target_key, + source_basis, + count(*) as matches, + min(coalesce(ended_at, started_at)) as first_closed_at, + max(coalesce(ended_at, started_at)) as last_closed_at +from rcon_materialized_matches +group by target_key, source_basis +order by target_key asc, source_basis asc; +``` + +### Materialized player stats coverage + +```sql +select + target_key, + count(*) as player_stat_rows, + count(distinct match_key) as match_count +from rcon_match_player_stats +group by target_key +order by target_key asc; +``` + +### Live server snapshots by age + +```sql +select + server_id, + min(captured_at) as first_captured_at, + max(captured_at) as last_captured_at, + count(*) as snapshot_rows +from server_snapshots +group by server_id +order by last_captured_at desc; +``` + +### Displayed snapshot coverage + +```sql +select + snapshot_type, + metric, + snapshot_window, + count(*) as snapshot_rows, + min(generated_at) as first_generated_at, + max(generated_at) as last_generated_at +from displayed_historical_snapshots +group by snapshot_type, metric, snapshot_window +order by snapshot_type asc, metric asc, snapshot_window asc; +``` + +### Public-scoreboard fallback historical matches + +```sql +select + hs.slug as server_slug, + count(*) as matches, + min(coalesce(hm.ended_at, hm.started_at, hm.created_at_source)) as first_match_at, + max(coalesce(hm.ended_at, hm.started_at, hm.created_at_source)) as last_match_at +from historical_matches hm +join historical_servers hs on hs.id = hm.historical_server_id +group by hs.slug +order by hs.slug asc; +``` + +### Player-event ledger age + +```sql +select + event_type, + count(*) as row_count, + min(occurred_at) as first_occurred_at, + max(occurred_at) as last_occurred_at +from player_event_raw_ledger +group by event_type +order by row_count desc, event_type asc; +``` + +## Summary + +The first cleanup command can be implemented safely if it stays narrow: + +- clean `server_snapshots`; +- clean aged `rcon_admin_log_events`; +- clean non-protected `rcon_materialized_matches`; +- delete `rcon_match_player_stats` first for those matches; +- skip bounded snapshot caches and broader fallback domains for now. + +The main correctness risk is not delete syntax. It is retention-window selection and preserving current/live dependencies. The cleanup command should therefore be plan-first, dry-run-first, and lock-protected. diff --git a/ai/tasks/review/TASK-final-maintenance-and-copy-validation-report.md b/ai/tasks/review/TASK-final-maintenance-and-copy-validation-report.md new file mode 100644 index 0000000..ea8de3c --- /dev/null +++ b/ai/tasks/review/TASK-final-maintenance-and-copy-validation-report.md @@ -0,0 +1,122 @@ +--- +id: TASK-final-maintenance-and-copy-validation-report +title: Final maintenance and copy validation report +status: review +type: report +team: PM +supporting_teams: [Backend Senior, Frontend Senior, Arquitecto de Base de Datos] +roadmap_item: validation +priority: high +--- + +# TASK-final-maintenance-and-copy-validation-report + +## Scope + +Validation-only report for: + +- database maintenance cleanup command +- historical-runner maintenance scheduler +- database maintenance operations documentation +- home copy update removing `Vietnam` outside the trailer block + +No runtime code was changed during this validation task. + +## Validation Commands Run + +```powershell +python -m compileall backend/app +PYTHONPATH=backend python -m unittest backend.tests.test_database_maintenance +PYTHONPATH=backend python -m unittest backend.tests.test_historical_runner_maintenance +PYTHONPATH=backend python -m unittest discover -s backend/tests -p "*historical*" +node --check frontend/assets/js/main.js +node --check frontend/assets/js/historico.js +node --check frontend/assets/js/partida-actual.js +git diff --check +Invoke-RestMethod "http://127.0.0.1:8000/api/historical/snapshots/recent-matches?server=all-servers&limit=10" | ConvertTo-Json -Depth 10 +Invoke-RestMethod "http://127.0.0.1:8000/api/historical/snapshots/leaderboard?server=all-servers&timeframe=weekly&metric=kills&limit=10" | ConvertTo-Json -Depth 10 +Invoke-RestMethod "http://127.0.0.1:8000/api/historical/snapshots/leaderboard?server=all-servers&timeframe=monthly&metric=kills&limit=10" | ConvertTo-Json -Depth 10 +$env:PYTHONPATH='backend'; python -m app.database_maintenance cleanup --dry-run +Invoke-WebRequest "http://127.0.0.1:8080/?v=maintenance-copy-validation" +Invoke-WebRequest "http://127.0.0.1:8080/historico.html?v=maintenance-copy-validation" +Invoke-WebRequest "http://127.0.0.1:8080/partida-actual.html?server=comunidad-hispana-01&v=maintenance-copy-validation" +``` + +## Result + +- `compileall`: passed +- `test_database_maintenance`: passed +- `test_historical_runner_maintenance`: passed +- `*historical*` discovery suite: passed +- frontend JavaScript syntax checks: passed +- `git diff --check`: passed + +## Docker Status + +Docker smoke validation could not run because the local Docker daemon was unavailable. + +Observed error: + +`failed to connect to the docker API at npipe:////./pipe/dockerDesktopLinuxEngine` + +This prevented: + +- `docker compose up -d --build backend frontend postgres` +- `docker compose --profile advanced up -d --build historical-runner rcon-historical-worker` +- `docker compose --profile advanced ps` +- `docker compose logs --tail=200 historical-runner` +- `docker compose exec backend python -m app.database_maintenance cleanup --dry-run` + +## Local Smoke Checks + +Fallback local processes were started instead: + +- backend: `python -m app.main` +- frontend: `python -m http.server 8080` + +Endpoint summary: + +- recent matches endpoint returned `200` with snapshot data for `all-servers` +- weekly leaderboard endpoint returned `200` with a valid empty weekly result for the current dataset +- monthly leaderboard endpoint returned `200` with valid leaderboard rows + +Maintenance dry-run summary: + +- backend mode: `sqlite` +- protected materialized matches: `22` +- candidate materialized matches: `0` +- candidate player-stat rows: `0` +- candidate admin-log rows: `0` +- candidate server snapshots: `298` +- skipped tables: none + +Frontend HTTP checks: + +- `/`: `200` +- `/historico.html`: `200` +- `/partida-actual.html?server=comunidad-hispana-01`: `200` + +## Copy Validation + +Confirmed in the diff for `frontend/index.html`: + +- meta description changed from `HLL Vietnam` to `HLL` +- page title changed from `Comunidad Hispana - HLL Vietnam` to `Comunidad Hispana - HLL` +- logo alt text changed from `HLL Vietnam` to `HLL` +- hero accent changed from `HLL Vietnam` to `HLL` +- trailer text `Primer vistazo a HLL Vietnam` was preserved +- iframe title `Trailer HLL Vietnam` was preserved + +No browser-based visual QA was performed in this task. + +## Unrelated Notes + +Python test runs emitted existing `ResourceWarning` messages about unclosed SQLite connections from current initialization helpers. The test suites still passed, and this task did not change that behavior. + +An unintended local runtime artifact at `backend/backend/data/hll_vietnam_dev.sqlite3` was created during smoke setup and removed before final scope review. + +## Branch/Push Status + +- branch push: not performed +- commit: not performed + diff --git a/backend/app/config.py b/backend/app/config.py index fa8f447..49aefcb 100644 --- a/backend/app/config.py +++ b/backend/app/config.py @@ -38,6 +38,13 @@ DEFAULT_RCON_HISTORICAL_CAPTURE_RETRY_DELAY_SECONDS = 15 DEFAULT_RCON_BACKFILL_CHUNK_HOURS = 6 DEFAULT_RCON_BACKFILL_SLEEP_SECONDS = 1.0 DEFAULT_RCON_BACKFILL_MAX_DAYS_BACK = 45 +DEFAULT_RECENT_MATCHES_KEEP = 100 +DEFAULT_ADMIN_LOG_NONCRITICAL_RETENTION_DAYS = 30 +DEFAULT_ADMIN_LOG_CRITICAL_RETENTION_DAYS = 90 +DEFAULT_SERVER_SNAPSHOT_RETENTION_DAYS = 14 +DEFAULT_DB_MAINTENANCE_BATCH_SIZE = 5000 +DEFAULT_DB_MAINTENANCE_ENABLED = False +DEFAULT_DB_MAINTENANCE_INTERVAL_SECONDS = 43200 DEFAULT_SQLITE_WRITER_TIMEOUT_SECONDS = 30.0 DEFAULT_SQLITE_BUSY_TIMEOUT_MS = 30000 DEFAULT_WRITER_LOCK_TIMEOUT_SECONDS = 120.0 @@ -514,6 +521,69 @@ def get_rcon_backfill_max_days_back() -> int: ) +def get_recent_matches_keep() -> int: + """Return how many recent closed materialized matches maintenance must protect.""" + return _read_int_env( + "HLL_RECENT_MATCHES_KEEP", + str(DEFAULT_RECENT_MATCHES_KEEP), + minimum=1, + ) + + +def get_admin_log_noncritical_retention_days() -> int: + """Return retention days for non-critical AdminLog events.""" + return _read_int_env( + "HLL_ADMIN_LOG_NONCRITICAL_RETENTION_DAYS", + str(DEFAULT_ADMIN_LOG_NONCRITICAL_RETENTION_DAYS), + minimum=1, + ) + + +def get_admin_log_critical_retention_days() -> int: + """Return retention days for critical AdminLog events.""" + return _read_int_env( + "HLL_ADMIN_LOG_CRITICAL_RETENTION_DAYS", + str(DEFAULT_ADMIN_LOG_CRITICAL_RETENTION_DAYS), + minimum=1, + ) + + +def get_server_snapshot_retention_days() -> int: + """Return retention days for live server snapshots.""" + return _read_int_env( + "HLL_SERVER_SNAPSHOT_RETENTION_DAYS", + str(DEFAULT_SERVER_SNAPSHOT_RETENTION_DAYS), + minimum=1, + ) + + +def get_db_maintenance_batch_size() -> int: + """Return the delete batch size used by database maintenance.""" + return _read_int_env( + "HLL_DB_MAINTENANCE_BATCH_SIZE", + str(DEFAULT_DB_MAINTENANCE_BATCH_SIZE), + minimum=1, + ) + + +def get_db_maintenance_enabled() -> bool: + """Return whether scheduled database maintenance is enabled.""" + normalized = os.getenv( + "HLL_DB_MAINTENANCE_ENABLED", + "true" if DEFAULT_DB_MAINTENANCE_ENABLED else "false", + ).strip().lower() + return normalized in {"1", "true", "yes", "on"} + + +def get_db_maintenance_interval_seconds() -> int: + """Return the scheduled database maintenance interval in seconds.""" + return _read_int_env( + "HLL_DB_MAINTENANCE_INTERVAL_SECONDS", + str(DEFAULT_DB_MAINTENANCE_INTERVAL_SECONDS), + minimum=1, + ) + + def get_a2s_targets_payload() -> str | None: """Return the optional JSON payload that overrides local A2S targets.""" raw_payload = os.getenv(DEFAULT_A2S_TARGETS_ENV_VAR) diff --git a/backend/app/database_maintenance.py b/backend/app/database_maintenance.py new file mode 100644 index 0000000..6cce252 --- /dev/null +++ b/backend/app/database_maintenance.py @@ -0,0 +1,638 @@ +"""Application-level database maintenance for bounded historical storage.""" + +from __future__ import annotations + +import argparse +import json +import sqlite3 +from contextlib import closing +from dataclasses import dataclass +from datetime import datetime, timedelta, timezone +from pathlib import Path +from typing import Any, Iterable, Sequence + +from .config import ( + get_admin_log_critical_retention_days, + get_admin_log_noncritical_retention_days, + get_database_url, + get_db_maintenance_batch_size, + get_historical_weekly_fallback_min_matches, + get_recent_matches_keep, + get_server_snapshot_retention_days, +) +from .rcon_admin_log_materialization import MATCH_RESULT_SOURCE +from .sqlite_utils import connect_sqlite_writer +from .writer_lock import backend_writer_lock, build_writer_lock_holder + +CRITICAL_ADMIN_LOG_EVENT_TYPES = frozenset({"kill", "match_start", "match_end"}) + + +@dataclass(frozen=True, slots=True) +class MaintenanceOptions: + apply: bool + recent_matches_keep: int + admin_log_noncritical_retention_days: int + admin_log_critical_retention_days: int + server_snapshot_retention_days: int + batch_size: int + vacuum_analyze: bool + now: datetime + + +def run_database_maintenance_cleanup( + *, + apply: bool = False, + recent_matches_keep: int | None = None, + admin_log_noncritical_retention_days: int | None = None, + admin_log_critical_retention_days: int | None = None, + server_snapshot_retention_days: int | None = None, + batch_size: int | None = None, + vacuum_analyze: bool = False, + now: str | datetime | None = None, + db_path: Path | None = None, +) -> dict[str, object]: + """Plan or apply safe bounded cleanup for supported storage tables.""" + options = MaintenanceOptions( + apply=apply, + recent_matches_keep=recent_matches_keep or get_recent_matches_keep(), + admin_log_noncritical_retention_days=( + admin_log_noncritical_retention_days or get_admin_log_noncritical_retention_days() + ), + admin_log_critical_retention_days=( + admin_log_critical_retention_days or get_admin_log_critical_retention_days() + ), + server_snapshot_retention_days=( + server_snapshot_retention_days or get_server_snapshot_retention_days() + ), + batch_size=batch_size or get_db_maintenance_batch_size(), + vacuum_analyze=vacuum_analyze, + now=_resolve_now(now), + ) + _emit_json_log( + { + "event": "database-maintenance-started", + "mode": "apply" if options.apply else "dry-run", + "database_backend": _database_backend_name(db_path=db_path), + "database_url_configured": bool(get_database_url()) and db_path is None, + "db_path": str(db_path) if db_path is not None else None, + "recent_matches_keep": options.recent_matches_keep, + "admin_log_noncritical_retention_days": options.admin_log_noncritical_retention_days, + "admin_log_critical_retention_days": options.admin_log_critical_retention_days, + "server_snapshot_retention_days": options.server_snapshot_retention_days, + "batch_size": options.batch_size, + "vacuum_analyze": options.vacuum_analyze, + "now": _to_iso(options.now), + } + ) + + try: + if options.apply: + with backend_writer_lock( + holder=build_writer_lock_holder("app.database_maintenance cleanup"), + storage_path=db_path, + ): + payload = _run_cleanup(options=options, db_path=db_path) + else: + payload = _run_cleanup(options=options, db_path=db_path) + _emit_json_log( + { + "event": "database-maintenance-completed", + **payload, + } + ) + return payload + except Exception as exc: # noqa: BLE001 - CLI reports structured diagnostics + error_payload = { + "status": "error", + "mode": "apply" if options.apply else "dry-run", + "error_type": type(exc).__name__, + "error": str(exc), + } + _emit_json_log({"event": "database-maintenance-error", **error_payload}) + return error_payload + + +def _run_cleanup(*, options: MaintenanceOptions, db_path: Path | None) -> dict[str, object]: + with _connect_maintenance(db_path=db_path) as connection: + existing_tables = _existing_table_names(connection) + plan = _build_cleanup_plan(connection, existing_tables=existing_tables, options=options) + _emit_json_log( + { + "event": "database-maintenance-plan", + **plan["summary"], + } + ) + + deleted_counts = { + "rcon_match_player_stats": 0, + "rcon_materialized_matches": 0, + "rcon_admin_log_events": 0, + "server_snapshots": 0, + } + if options.apply: + deleted_counts["rcon_match_player_stats"] = _delete_match_player_stats( + connection, + matches=plan["candidate_matches"], + batch_size=options.batch_size, + ) + deleted_counts["rcon_materialized_matches"] = _delete_ids_in_batches( + connection, + table_name="rcon_materialized_matches", + ids=[int(row["id"]) for row in plan["candidate_matches"]], + batch_size=options.batch_size, + ) + deleted_counts["rcon_admin_log_events"] = _delete_ids_in_batches( + connection, + table_name="rcon_admin_log_events", + ids=plan["candidate_admin_log_ids"], + batch_size=options.batch_size, + ) + deleted_counts["server_snapshots"] = _delete_ids_in_batches( + connection, + table_name="server_snapshots", + ids=plan["candidate_server_snapshot_ids"], + batch_size=options.batch_size, + ) + if options.vacuum_analyze: + _run_vacuum_analyze(connection) + + return { + "status": "ok", + "mode": "apply" if options.apply else "dry-run", + "deleted_counts": deleted_counts, + "plan": plan["summary"], + } + + +def _build_cleanup_plan( + connection: sqlite3.Connection | Any, + *, + existing_tables: set[str], + options: MaintenanceOptions, +) -> dict[str, object]: + candidate_server_snapshot_ids: list[int] = [] + candidate_admin_log_ids: list[int] = [] + candidate_matches: list[dict[str, object]] = [] + protected_match_keys: list[str] = [] + skipped_tables: list[str] = [] + + if "server_snapshots" not in existing_tables: + skipped_tables.append("server_snapshots") + _emit_skip("server_snapshots", "table-missing") + else: + cutoff = options.now - timedelta(days=options.server_snapshot_retention_days) + for row in connection.execute( + "SELECT id, captured_at FROM server_snapshots ORDER BY id ASC" + ).fetchall(): + captured_at = _parse_datetime(row["captured_at"]) + if captured_at is None: + continue + if captured_at < cutoff: + candidate_server_snapshot_ids.append(int(row["id"])) + + protected_ranges: dict[str, list[tuple[int, int]]] = {} + if "rcon_materialized_matches" not in existing_tables: + skipped_tables.append("rcon_materialized_matches") + _emit_skip("rcon_materialized_matches", "table-missing") + else: + ( + candidate_matches, + protected_matches, + protected_ranges, + protection_summary, + ) = _plan_materialized_match_cleanup(connection, options=options) + protected_match_keys = [str(row["match_key"]) for row in protected_matches] + if "rcon_match_player_stats" not in existing_tables: + skipped_tables.append("rcon_match_player_stats") + _emit_skip("rcon_match_player_stats", "table-missing") + + if "rcon_admin_log_events" not in existing_tables: + skipped_tables.append("rcon_admin_log_events") + _emit_skip("rcon_admin_log_events", "table-missing") + else: + candidate_admin_log_ids = _plan_admin_log_cleanup( + connection, + options=options, + protected_ranges=protected_ranges, + ) + + candidate_player_stat_rows = 0 + if candidate_matches and "rcon_match_player_stats" in existing_tables: + candidate_player_stat_rows = _count_candidate_player_stats(connection, candidate_matches) + + summary = { + "status": "ok", + "protected_match_count": len(protected_match_keys), + "candidate_match_count": len(candidate_matches), + "candidate_match_player_stat_count": candidate_player_stat_rows, + "candidate_admin_log_event_count": len(candidate_admin_log_ids), + "candidate_server_snapshot_count": len(candidate_server_snapshot_ids), + "skipped_tables": skipped_tables, + "protected_match_keys_preview": protected_match_keys[:10], + } + if "protection_summary" in locals(): + summary["protection_summary"] = protection_summary + + return { + "candidate_server_snapshot_ids": candidate_server_snapshot_ids, + "candidate_admin_log_ids": candidate_admin_log_ids, + "candidate_matches": candidate_matches, + "summary": summary, + } + + +def _plan_materialized_match_cleanup( + connection: sqlite3.Connection | Any, + *, + options: MaintenanceOptions, +) -> tuple[list[dict[str, object]], list[dict[str, object]], dict[str, list[tuple[int, int]]], dict[str, object]]: + rows = [ + dict(row) + for row in connection.execute( + """ + SELECT id, target_key, match_key, started_at, ended_at, + started_server_time, ended_server_time, source_basis + FROM rcon_materialized_matches + WHERE source_basis = ? + """, + (MATCH_RESULT_SOURCE,), + ).fetchall() + ] + closed_rows: list[dict[str, object]] = [] + protected_rows: list[dict[str, object]] = [] + current_month_start = options.now.replace(day=1, hour=0, minute=0, second=0, microsecond=0) + previous_month_start = (current_month_start - timedelta(days=1)).replace(day=1) + current_week_start = (options.now - timedelta(days=options.now.weekday())).replace( + hour=0, + minute=0, + second=0, + microsecond=0, + ) + previous_week_start = current_week_start - timedelta(days=7) + + for row in rows: + closed_at = _parse_datetime(row.get("ended_at") or row.get("started_at")) + if closed_at is None: + row["_protect_reason"] = "unparseable-closed-at" + protected_rows.append(row) + continue + row["_closed_at"] = closed_at + closed_rows.append(row) + + closed_rows.sort( + key=lambda row: ( + row["_closed_at"], + _coerce_int(row.get("ended_server_time")) or _coerce_int(row.get("started_server_time")) or 0, + _coerce_int(row.get("id")) or 0, + ), + reverse=True, + ) + latest_ids = {int(row["id"]) for row in closed_rows[: options.recent_matches_keep]} + current_week_count = sum( + 1 for row in closed_rows if current_week_start <= row["_closed_at"] < options.now + ) + previous_week_count = sum( + 1 for row in closed_rows if previous_week_start <= row["_closed_at"] < current_week_start + ) + protect_previous_week = ( + current_week_count < get_historical_weekly_fallback_min_matches() + and previous_week_count > 0 + ) + protect_previous_month = options.now.day <= 7 + + candidate_rows: list[dict[str, object]] = [] + protected_ranges: dict[str, list[tuple[int, int]]] = {} + for row in closed_rows: + closed_at = row["_closed_at"] + should_protect = False + if int(row["id"]) in latest_ids: + should_protect = True + elif closed_at >= current_month_start: + should_protect = True + elif protect_previous_month and previous_month_start <= closed_at < current_month_start: + should_protect = True + elif closed_at >= current_week_start: + should_protect = True + elif protect_previous_week and previous_week_start <= closed_at < current_week_start: + should_protect = True + + if should_protect: + protected_rows.append(row) + lower = _coerce_int(row.get("started_server_time")) + upper = _coerce_int(row.get("ended_server_time")) + if lower is not None and upper is not None: + protected_ranges.setdefault(str(row["target_key"]), []).append((lower, upper)) + else: + candidate_rows.append(row) + + return ( + candidate_rows, + protected_rows, + protected_ranges, + { + "recent_matches_keep": options.recent_matches_keep, + "current_week_closed_matches": current_week_count, + "previous_week_closed_matches": previous_week_count, + "protect_previous_week": protect_previous_week, + "protect_previous_month": protect_previous_month, + "current_week_start": _to_iso(current_week_start), + "previous_week_start": _to_iso(previous_week_start), + "current_month_start": _to_iso(current_month_start), + "previous_month_start": _to_iso(previous_month_start), + }, + ) + + +def _plan_admin_log_cleanup( + connection: sqlite3.Connection | Any, + *, + options: MaintenanceOptions, + protected_ranges: dict[str, list[tuple[int, int]]], +) -> list[int]: + noncritical_cutoff = options.now - timedelta(days=options.admin_log_noncritical_retention_days) + critical_cutoff = options.now - timedelta(days=options.admin_log_critical_retention_days) + candidate_ids: list[int] = [] + rows = connection.execute( + """ + SELECT id, target_key, event_type, event_timestamp, server_time + FROM rcon_admin_log_events + ORDER BY id ASC + """ + ).fetchall() + for row in rows: + event_type = str(row["event_type"] or "").strip() + event_time = _parse_datetime(row["event_timestamp"]) + if event_time is None: + continue + if event_type in CRITICAL_ADMIN_LOG_EVENT_TYPES: + if event_time >= critical_cutoff: + continue + server_time = _coerce_int(row["server_time"]) + if server_time is None: + continue + if _server_time_is_protected( + target_key=str(row["target_key"] or ""), + server_time=server_time, + protected_ranges=protected_ranges, + ): + continue + candidate_ids.append(int(row["id"])) + continue + if event_time < noncritical_cutoff: + candidate_ids.append(int(row["id"])) + return candidate_ids + + +def _count_candidate_player_stats( + connection: sqlite3.Connection | Any, + matches: Sequence[dict[str, object]], +) -> int: + count = 0 + for batch in _chunked(list(matches), 250): + clause, params = _match_pair_clause(batch) + row = connection.execute( + f"SELECT COUNT(*) AS count FROM rcon_match_player_stats WHERE {clause}", + params, + ).fetchone() + count += int(row["count"] or 0) + return count + + +def _delete_match_player_stats( + connection: sqlite3.Connection | Any, + *, + matches: Sequence[dict[str, object]], + batch_size: int, +) -> int: + deleted = 0 + for batch in _chunked(list(matches), max(1, min(batch_size, 250))): + clause, params = _match_pair_clause(batch) + deleted_in_batch = int( + connection.execute( + f"DELETE FROM rcon_match_player_stats WHERE {clause}", + params, + ).rowcount + or 0 + ) + _commit(connection) + deleted += deleted_in_batch + _emit_json_log( + { + "event": "database-maintenance-delete-batch", + "table": "rcon_match_player_stats", + "deleted_rows": deleted_in_batch, + "batch_size": len(batch), + } + ) + return deleted + + +def _delete_ids_in_batches( + connection: sqlite3.Connection | Any, + *, + table_name: str, + ids: Sequence[int], + batch_size: int, +) -> int: + deleted = 0 + for batch in _chunked(list(ids), batch_size): + placeholders = ",".join("?" for _ in batch) + deleted_in_batch = int( + connection.execute( + f"DELETE FROM {table_name} WHERE id IN ({placeholders})", + batch, + ).rowcount + or 0 + ) + _commit(connection) + deleted += deleted_in_batch + _emit_json_log( + { + "event": "database-maintenance-delete-batch", + "table": table_name, + "deleted_rows": deleted_in_batch, + "batch_size": len(batch), + } + ) + return deleted + + +def _run_vacuum_analyze(connection: sqlite3.Connection | Any) -> None: + raw_connection = _raw_connection(connection) + if isinstance(raw_connection, sqlite3.Connection): + raw_connection.execute("VACUUM") + raw_connection.execute("ANALYZE") + raw_connection.commit() + return + raw_connection.commit() + raw_connection.autocommit = True + try: + raw_connection.execute("VACUUM ANALYZE") + finally: + raw_connection.autocommit = False + + +def _match_pair_clause(matches: Sequence[dict[str, object]]) -> tuple[str, list[object]]: + clauses: list[str] = [] + params: list[object] = [] + for row in matches: + clauses.append("(target_key = ? AND match_key = ?)") + params.extend([row["target_key"], row["match_key"]]) + return " OR ".join(clauses), params + + +def _existing_table_names(connection: sqlite3.Connection | Any) -> set[str]: + raw_connection = _raw_connection(connection) + if isinstance(raw_connection, sqlite3.Connection): + rows = connection.execute( + "SELECT name FROM sqlite_master WHERE type = 'table'" + ).fetchall() + return {str(row["name"]) for row in rows} + rows = raw_connection.execute( + """ + SELECT table_name + FROM information_schema.tables + WHERE table_schema = 'public' + """ + ).fetchall() + return {str(row["table_name"]) for row in rows} + + +def _emit_skip(table_name: str, reason: str) -> None: + _emit_json_log( + { + "event": "database-maintenance-table-skipped", + "table": table_name, + "reason": reason, + } + ) + + +def _server_time_is_protected( + *, + target_key: str, + server_time: int, + protected_ranges: dict[str, list[tuple[int, int]]], +) -> bool: + for lower, upper in protected_ranges.get(target_key, []): + if lower <= server_time <= upper: + return True + return False + + +def _connect_maintenance(*, db_path: Path | None): + if get_database_url() and db_path is None: + from .postgres_rcon_storage import connect_postgres_compat + + return connect_postgres_compat() + resolved_path = db_path or Path.cwd() / "backend" / "data" / "hll_vietnam_dev.sqlite3" + resolved_path.parent.mkdir(parents=True, exist_ok=True) + return closing(connect_sqlite_writer(resolved_path)) + + +def _commit(connection: sqlite3.Connection | Any) -> None: + _raw_connection(connection).commit() + + +def _raw_connection(connection: sqlite3.Connection | Any) -> sqlite3.Connection | Any: + return connection.connection if hasattr(connection, "connection") else connection + + +def _database_backend_name(*, db_path: Path | None) -> str: + return "postgres" if get_database_url() and db_path is None else "sqlite" + + +def _resolve_now(value: str | datetime | None) -> datetime: + if value is None: + return datetime.now(timezone.utc) + if isinstance(value, datetime): + return value.astimezone(timezone.utc) if value.tzinfo else value.replace(tzinfo=timezone.utc) + parsed = _parse_datetime(value) + if parsed is None: + raise ValueError("--now must be an ISO 8601 timestamp or date.") + return parsed + + +def _parse_datetime(value: object) -> datetime | None: + text = str(value or "").strip() + if not text: + return None + if len(text) == 10: + text = f"{text}T00:00:00+00:00" + try: + parsed = datetime.fromisoformat(text.replace("Z", "+00:00")) + except ValueError: + return None + return parsed.astimezone(timezone.utc) if parsed.tzinfo else parsed.replace(tzinfo=timezone.utc) + + +def _to_iso(value: datetime) -> str: + return value.astimezone(timezone.utc).isoformat().replace("+00:00", "Z") + + +def _coerce_int(value: object) -> int | None: + try: + return None if value is None else int(value) + except (TypeError, ValueError): + return None + + +def _chunked(values: Sequence[Any], size: int) -> Iterable[list[Any]]: + for index in range(0, len(values), size): + yield list(values[index : index + size]) + + +def _emit_json_log(payload: dict[str, object]) -> None: + print(json.dumps(payload, ensure_ascii=True, default=str), flush=True) + + +def build_arg_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description="Database maintenance for HLL Vietnam.") + subparsers = parser.add_subparsers(dest="command", required=True) + cleanup_parser = subparsers.add_parser("cleanup") + cleanup_parser.add_argument("--dry-run", action="store_true") + cleanup_parser.add_argument("--apply", action="store_true") + cleanup_parser.add_argument("--recent-matches-keep", type=int, default=get_recent_matches_keep()) + cleanup_parser.add_argument( + "--admin-log-noncritical-retention-days", + type=int, + default=get_admin_log_noncritical_retention_days(), + ) + cleanup_parser.add_argument( + "--admin-log-critical-retention-days", + type=int, + default=get_admin_log_critical_retention_days(), + ) + cleanup_parser.add_argument( + "--server-snapshot-retention-days", + type=int, + default=get_server_snapshot_retention_days(), + ) + cleanup_parser.add_argument("--batch-size", type=int, default=get_db_maintenance_batch_size()) + cleanup_parser.add_argument("--vacuum-analyze", action="store_true") + cleanup_parser.add_argument("--now", default=None) + return parser + + +def main(argv: Sequence[str] | None = None) -> int: + parser = build_arg_parser() + args = parser.parse_args(list(argv) if argv is not None else None) + if args.command != "cleanup": + raise ValueError("Unsupported command.") + if args.apply and args.dry_run: + raise ValueError("--apply and --dry-run are mutually exclusive.") + payload = run_database_maintenance_cleanup( + apply=bool(args.apply), + recent_matches_keep=args.recent_matches_keep, + admin_log_noncritical_retention_days=args.admin_log_noncritical_retention_days, + admin_log_critical_retention_days=args.admin_log_critical_retention_days, + server_snapshot_retention_days=args.server_snapshot_retention_days, + batch_size=args.batch_size, + vacuum_analyze=bool(args.vacuum_analyze), + now=args.now, + ) + return 0 if payload.get("status") == "ok" else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/backend/app/historical_runner.py b/backend/app/historical_runner.py index e14442a..10edced 100644 --- a/backend/app/historical_runner.py +++ b/backend/app/historical_runner.py @@ -10,6 +10,9 @@ from datetime import datetime, timezone from typing import Any from .config import ( + DEFAULT_DB_MAINTENANCE_INTERVAL_SECONDS, + get_db_maintenance_enabled, + get_db_maintenance_interval_seconds, get_historical_full_snapshot_every_runs, get_historical_elo_mmr_min_new_samples, get_historical_elo_mmr_rebuild_interval_minutes, @@ -18,6 +21,7 @@ from .config import ( get_historical_refresh_retry_delay_seconds, get_historical_data_source_kind, ) +from .database_maintenance import run_database_maintenance_cleanup from .elo_mmr_engine import rebuild_elo_mmr_models from .elo_mmr_storage import get_latest_elo_mmr_generated_at from .historical_ingestion import run_incremental_refresh @@ -34,6 +38,7 @@ DEFAULT_HISTORICAL_SERVER_SCOPE = ( "comunidad-hispana-01", "comunidad-hispana-02", ) +_LAST_DATABASE_MAINTENANCE_RUN_AT: datetime | None = None def run_periodic_historical_refresh( @@ -171,6 +176,7 @@ def _run_refresh_with_retries( rcon_capture_result=rcon_capture_result ), } + maintenance_result = _maybe_run_database_maintenance() return { "status": "ok", "attempts_used": attempt, @@ -181,6 +187,7 @@ def _run_refresh_with_retries( "refresh_result": refresh_result, "snapshot_result": snapshot_result, "elo_mmr_result": elo_mmr_result, + "database_maintenance_result": maintenance_result, } except Exception as exc: failure_payload = { @@ -249,6 +256,91 @@ def _emit_json_log(payload: dict[str, Any]) -> None: print(json.dumps(payload, ensure_ascii=True, default=str), flush=True) +def _maybe_run_database_maintenance(*, now: datetime | None = None) -> dict[str, Any]: + """Optionally run scheduled database maintenance without crashing the runner.""" + global _LAST_DATABASE_MAINTENANCE_RUN_AT + + anchor = now.astimezone(timezone.utc) if now else datetime.now(timezone.utc) + if not get_db_maintenance_enabled(): + result = {"status": "skipped", "reason": "disabled", "enabled": False} + _emit_json_log({"event": "database-maintenance-scheduler-skipped-disabled", **result}) + return result + + interval_seconds, interval_source = _resolve_db_maintenance_interval_seconds() + if _LAST_DATABASE_MAINTENANCE_RUN_AT is not None: + elapsed_seconds = max( + 0, + int((anchor - _LAST_DATABASE_MAINTENANCE_RUN_AT).total_seconds()), + ) + if elapsed_seconds < interval_seconds: + result = { + "status": "skipped", + "reason": "not-due", + "enabled": True, + "interval_seconds": interval_seconds, + "interval_source": interval_source, + "elapsed_seconds": elapsed_seconds, + "last_run_at": _LAST_DATABASE_MAINTENANCE_RUN_AT.isoformat().replace( + "+00:00", "Z" + ), + } + _emit_json_log({"event": "database-maintenance-scheduler-skipped-not-due", **result}) + return result + + _emit_json_log( + { + "event": "database-maintenance-scheduler-started", + "enabled": True, + "interval_seconds": interval_seconds, + "interval_source": interval_source, + "scheduled_at": anchor.isoformat().replace("+00:00", "Z"), + } + ) + try: + result = run_database_maintenance_cleanup(apply=True, now=anchor) + except Exception as exc: # noqa: BLE001 - scheduler must not crash the runner + result = { + "status": "error", + "error_type": type(exc).__name__, + "error": str(exc), + "enabled": True, + "interval_seconds": interval_seconds, + "interval_source": interval_source, + } + _emit_json_log({"event": "database-maintenance-scheduler-failed", **result}) + return result + + if result.get("status") == "ok": + _LAST_DATABASE_MAINTENANCE_RUN_AT = anchor + _emit_json_log( + { + "event": "database-maintenance-scheduler-completed", + "enabled": True, + "interval_seconds": interval_seconds, + "interval_source": interval_source, + "result": result, + } + ) + return result + + failed_result = { + "enabled": True, + "interval_seconds": interval_seconds, + "interval_source": interval_source, + "result": result, + } + _emit_json_log({"event": "database-maintenance-scheduler-failed", **failed_result}) + return result + + +def _resolve_db_maintenance_interval_seconds() -> tuple[int, str]: + """Return a safe maintenance interval even if env configuration is invalid.""" + try: + return get_db_maintenance_interval_seconds(), "env" + except ValueError: + return DEFAULT_DB_MAINTENANCE_INTERVAL_SECONDS, "default-invalid-env-fallback" + + def _describe_refresh_scope(server_slug: str | None) -> list[str]: if server_slug: return [server_slug] diff --git a/backend/tests/test_database_maintenance.py b/backend/tests/test_database_maintenance.py new file mode 100644 index 0000000..e0af8aa --- /dev/null +++ b/backend/tests/test_database_maintenance.py @@ -0,0 +1,448 @@ +from __future__ import annotations + +import io +import json +import sqlite3 +import tempfile +import unittest +from contextlib import closing, redirect_stdout +from datetime import datetime, timedelta, timezone +from pathlib import Path + +from app.database_maintenance import run_database_maintenance_cleanup +from app.rcon_admin_log_materialization import MATCH_RESULT_SOURCE, initialize_rcon_materialized_storage +from app.rcon_admin_log_storage import initialize_rcon_admin_log_storage +from app.storage import initialize_storage + + +class DatabaseMaintenanceTests(unittest.TestCase): + def test_dry_run_does_not_delete(self) -> None: + with _temp_db() as db_path: + _insert_server_snapshot(db_path, snapshot_id=1, captured_at="2026-05-01T00:00:00Z") + + payload = run_database_maintenance_cleanup( + db_path=db_path, + now="2026-06-20T12:00:00Z", + ) + + self.assertEqual(payload["status"], "ok") + self.assertEqual(payload["mode"], "dry-run") + with closing(sqlite3.connect(db_path)) as connection: + self.assertEqual( + connection.execute("SELECT COUNT(*) FROM server_snapshots").fetchone()[0], + 1, + ) + + def test_apply_deletes_old_server_snapshots(self) -> None: + with _temp_db() as db_path: + _insert_server_snapshot(db_path, snapshot_id=1, captured_at="2026-05-01T00:00:00Z") + _insert_server_snapshot(db_path, snapshot_id=2, captured_at="2026-06-18T00:00:00Z") + + run_database_maintenance_cleanup( + apply=True, + db_path=db_path, + now="2026-06-20T12:00:00Z", + recent_matches_keep=1, + ) + + with closing(sqlite3.connect(db_path)) as connection: + ids = [row[0] for row in connection.execute("SELECT id FROM server_snapshots ORDER BY id")] + self.assertEqual(ids, [2]) + + def test_apply_deletes_old_noncritical_admin_log_events(self) -> None: + with _temp_db() as db_path: + _insert_admin_log_event( + db_path, + event_id=1, + event_type="chat", + event_timestamp="2026-04-01T00:00:00Z", + server_time=100, + ) + _insert_admin_log_event( + db_path, + event_id=2, + event_type="chat", + event_timestamp="2026-06-15T00:00:00Z", + server_time=200, + ) + + run_database_maintenance_cleanup( + apply=True, + db_path=db_path, + now="2026-06-20T12:00:00Z", + ) + + with closing(sqlite3.connect(db_path)) as connection: + remaining = [ + tuple(row) + for row in connection.execute( + "SELECT id, event_type FROM rcon_admin_log_events ORDER BY id" + ) + ] + self.assertEqual(remaining, [(2, "chat")]) + + def test_apply_preserves_critical_events_within_retention(self) -> None: + with _temp_db() as db_path: + _insert_admin_log_event( + db_path, + event_id=1, + event_type="kill", + event_timestamp="2026-06-10T00:00:00Z", + server_time=100, + ) + + run_database_maintenance_cleanup( + apply=True, + db_path=db_path, + now="2026-06-20T12:00:00Z", + ) + + with closing(sqlite3.connect(db_path)) as connection: + count = connection.execute( + "SELECT COUNT(*) FROM rcon_admin_log_events WHERE event_type = 'kill'" + ).fetchone()[0] + self.assertEqual(count, 1) + + def test_apply_preserves_latest_100_materialized_matches(self) -> None: + with _temp_db() as db_path: + for index in range(101): + ended_at = ( + datetime(2026, 1, 1, 12, tzinfo=timezone.utc) + timedelta(days=index) + ).isoformat().replace("+00:00", "Z") + _insert_materialized_match( + db_path, + match_id=index + 1, + match_key=f"match-{index + 1}", + ended_at=ended_at, + server_time_start=(index + 1) * 10, + server_time_end=(index + 1) * 10 + 5, + ) + + run_database_maintenance_cleanup( + apply=True, + db_path=db_path, + now="2026-06-20T12:00:00Z", + ) + + with closing(sqlite3.connect(db_path)) as connection: + remaining = connection.execute( + "SELECT COUNT(*) FROM rcon_materialized_matches" + ).fetchone()[0] + oldest = connection.execute( + "SELECT COUNT(*) FROM rcon_materialized_matches WHERE match_key = 'match-1'" + ).fetchone()[0] + self.assertEqual(remaining, 100) + self.assertEqual(oldest, 0) + + def test_apply_preserves_current_month_matches(self) -> None: + with _temp_db() as db_path: + _insert_materialized_match( + db_path, + match_id=1, + match_key="old", + ended_at="2026-01-10T12:00:00Z", + server_time_start=10, + server_time_end=20, + ) + _insert_materialized_match( + db_path, + match_id=2, + match_key="current-month", + ended_at="2026-06-03T12:00:00Z", + server_time_start=30, + server_time_end=40, + ) + + run_database_maintenance_cleanup( + apply=True, + db_path=db_path, + now="2026-06-20T12:00:00Z", + recent_matches_keep=1, + ) + + with closing(sqlite3.connect(db_path)) as connection: + keys = [row[0] for row in connection.execute("SELECT match_key FROM rcon_materialized_matches")] + self.assertEqual(keys, ["current-month"]) + + def test_apply_preserves_previous_month_when_now_day_is_early(self) -> None: + with _temp_db() as db_path: + _insert_materialized_match( + db_path, + match_id=1, + match_key="previous-month", + ended_at="2026-05-15T12:00:00Z", + server_time_start=10, + server_time_end=20, + ) + _insert_materialized_match( + db_path, + match_id=2, + match_key="older", + ended_at="2026-04-15T12:00:00Z", + server_time_start=30, + server_time_end=40, + ) + + run_database_maintenance_cleanup( + apply=True, + db_path=db_path, + now="2026-06-05T12:00:00Z", + recent_matches_keep=1, + ) + + with closing(sqlite3.connect(db_path)) as connection: + keys = [row[0] for row in connection.execute("SELECT match_key FROM rcon_materialized_matches")] + self.assertEqual(keys, ["previous-month"]) + + def test_apply_preserves_current_week(self) -> None: + with _temp_db() as db_path: + _insert_materialized_match( + db_path, + match_id=1, + match_key="current-week", + ended_at="2026-06-10T12:00:00Z", + server_time_start=10, + server_time_end=20, + ) + _insert_materialized_match( + db_path, + match_id=2, + match_key="older", + ended_at="2026-05-01T12:00:00Z", + server_time_start=30, + server_time_end=40, + ) + + run_database_maintenance_cleanup( + apply=True, + db_path=db_path, + now="2026-06-10T13:00:00Z", + recent_matches_keep=1, + ) + + with closing(sqlite3.connect(db_path)) as connection: + keys = [row[0] for row in connection.execute("SELECT match_key FROM rcon_materialized_matches")] + self.assertEqual(keys, ["current-week"]) + + def test_apply_preserves_previous_week_when_fallback_may_need_it(self) -> None: + with _temp_db() as db_path: + _insert_materialized_match( + db_path, + match_id=1, + match_key="previous-week", + ended_at="2026-06-03T12:00:00Z", + server_time_start=10, + server_time_end=20, + ) + _insert_materialized_match( + db_path, + match_id=2, + match_key="current-week-sample", + ended_at="2026-06-09T12:00:00Z", + server_time_start=30, + server_time_end=40, + ) + _insert_materialized_match( + db_path, + match_id=3, + match_key="older", + ended_at="2026-05-01T12:00:00Z", + server_time_start=50, + server_time_end=60, + ) + + run_database_maintenance_cleanup( + apply=True, + db_path=db_path, + now="2026-06-10T13:00:00Z", + recent_matches_keep=1, + ) + + with closing(sqlite3.connect(db_path)) as connection: + keys = { + row[0] + for row in connection.execute("SELECT match_key FROM rcon_materialized_matches") + } + self.assertEqual(keys, {"previous-week", "current-week-sample"}) + + def test_apply_deletes_old_non_protected_match_and_child_stats(self) -> None: + with _temp_db() as db_path: + _insert_materialized_match( + db_path, + match_id=1, + match_key="delete-me", + ended_at="2026-01-10T12:00:00Z", + server_time_start=10, + server_time_end=20, + ) + _insert_materialized_match( + db_path, + match_id=2, + match_key="keep-me", + ended_at="2026-06-18T12:00:00Z", + server_time_start=30, + server_time_end=40, + ) + _insert_player_stat(db_path, match_key="delete-me", player_id="player-1") + _insert_player_stat(db_path, match_key="keep-me", player_id="player-2") + + run_database_maintenance_cleanup( + apply=True, + db_path=db_path, + now="2026-06-20T12:00:00Z", + recent_matches_keep=1, + ) + + with closing(sqlite3.connect(db_path)) as connection: + deleted_match_count = connection.execute( + "SELECT COUNT(*) FROM rcon_materialized_matches WHERE match_key = 'delete-me'" + ).fetchone()[0] + deleted_stat_count = connection.execute( + "SELECT COUNT(*) FROM rcon_match_player_stats WHERE match_key = 'delete-me'" + ).fetchone()[0] + kept_stat_count = connection.execute( + "SELECT COUNT(*) FROM rcon_match_player_stats WHERE match_key = 'keep-me'" + ).fetchone()[0] + self.assertEqual(deleted_match_count, 0) + self.assertEqual(deleted_stat_count, 0) + self.assertEqual(kept_stat_count, 1) + + def test_missing_optional_tables_are_logged_and_do_not_crash(self) -> None: + with _temp_db(create_schema=False) as db_path: + stream = io.StringIO() + with redirect_stdout(stream): + payload = run_database_maintenance_cleanup( + db_path=db_path, + now="2026-06-20T12:00:00Z", + ) + + self.assertEqual(payload["status"], "ok") + self.assertIn("database-maintenance-table-skipped", stream.getvalue()) + + +def _temp_db(*, create_schema: bool = True): + class _TempDbContext: + def __enter__(self) -> Path: + self._tmpdir = tempfile.TemporaryDirectory(ignore_cleanup_errors=True) + self.db_path = Path(self._tmpdir.name) / "maintenance.sqlite3" + if create_schema: + initialize_storage(db_path=self.db_path) + initialize_rcon_admin_log_storage(db_path=self.db_path) + initialize_rcon_materialized_storage(db_path=self.db_path) + return self.db_path + + def __exit__(self, exc_type, exc, tb) -> None: + self._tmpdir.cleanup() + + return _TempDbContext() + + +def _insert_server_snapshot(db_path: Path, *, snapshot_id: int, captured_at: str) -> None: + with closing(sqlite3.connect(db_path)) as connection: + connection.execute( + """ + INSERT OR IGNORE INTO game_sources ( + id, slug, display_name, provider_kind, is_active, created_at, updated_at + ) VALUES (1, 'current-hll', 'Current Hell Let Loose', 'development', 1, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP) + """ + ) + connection.execute( + """ + INSERT OR IGNORE INTO servers ( + id, game_source_id, external_server_id, server_name, region, first_seen_at, last_seen_at + ) VALUES (1, 1, 'server-1', 'Server 1', 'ES', ?, ?) + """, + (captured_at, captured_at), + ) + connection.execute( + """ + INSERT INTO server_snapshots ( + id, server_id, captured_at, status, players, max_players, current_map, source_name + ) VALUES (?, 1, ?, 'online', 10, 100, 'hurtgen', 'test') + """, + (snapshot_id, captured_at), + ) + connection.commit() + + +def _insert_admin_log_event( + db_path: Path, + *, + event_id: int, + event_type: str, + event_timestamp: str, + server_time: int, +) -> None: + with closing(sqlite3.connect(db_path)) as connection: + connection.execute( + """ + INSERT INTO rcon_admin_log_events ( + id, target_key, external_server_id, event_timestamp, server_time, + relative_time, event_type, raw_message, canonical_message, + parsed_payload_json, raw_entry_json + ) VALUES (?, 'comunidad-hispana-01', 'comunidad-hispana-01', ?, ?, '', ?, '', '', '{}', '{}') + """, + (event_id, event_timestamp, server_time, event_type), + ) + connection.commit() + + +def _insert_materialized_match( + db_path: Path, + *, + match_id: int, + match_key: str, + ended_at: str, + server_time_start: int, + server_time_end: int, +) -> None: + started_at = _shift_iso(ended_at, hours=-1) + with closing(sqlite3.connect(db_path)) as connection: + connection.execute( + """ + INSERT INTO rcon_materialized_matches ( + id, target_key, external_server_id, match_key, map_name, map_pretty_name, + game_mode, started_server_time, ended_server_time, started_at, ended_at, + allied_score, axis_score, winner, confidence_mode, source_basis + ) VALUES (?, 'comunidad-hispana-01', 'comunidad-hispana-01', ?, 'hurtgen', 'Hurtgen Forest', + 'warfare', ?, ?, ?, ?, 5, 3, 'allied', 'exact', ?) + """, + ( + match_id, + match_key, + server_time_start, + server_time_end, + started_at, + ended_at, + MATCH_RESULT_SOURCE, + ), + ) + connection.commit() + + +def _insert_player_stat(db_path: Path, *, match_key: str, player_id: str) -> None: + with closing(sqlite3.connect(db_path)) as connection: + connection.execute( + """ + INSERT INTO rcon_match_player_stats ( + target_key, match_key, player_id, player_name, team, + kills, deaths, teamkills, deaths_by_teamkill, + weapons_json, death_by_weapons_json, most_killed_json, death_by_json + ) VALUES ( + 'comunidad-hispana-01', ?, ?, ?, 'Allies', + 1, 1, 0, 0, '{}', '{}', '{}', '{}' + ) + """, + (match_key, player_id, player_id), + ) + connection.commit() + + +def _shift_iso(value: str, *, hours: int) -> str: + point = datetime.fromisoformat(value.replace("Z", "+00:00")).astimezone(timezone.utc) + shifted = point + timedelta(hours=hours) + return shifted.isoformat().replace("+00:00", "Z") + + +if __name__ == "__main__": + unittest.main() diff --git a/backend/tests/test_historical_runner_maintenance.py b/backend/tests/test_historical_runner_maintenance.py new file mode 100644 index 0000000..10d0513 --- /dev/null +++ b/backend/tests/test_historical_runner_maintenance.py @@ -0,0 +1,153 @@ +from __future__ import annotations + +import io +import os +import unittest +from contextlib import nullcontext, redirect_stdout +from datetime import datetime, timezone +from unittest.mock import patch + +import app.historical_runner as historical_runner +from app.historical_runner import _maybe_run_database_maintenance, _run_refresh_with_retries + + +class HistoricalRunnerMaintenanceTests(unittest.TestCase): + def setUp(self) -> None: + historical_runner._LAST_DATABASE_MAINTENANCE_RUN_AT = None + + def tearDown(self) -> None: + historical_runner._LAST_DATABASE_MAINTENANCE_RUN_AT = None + + def test_scheduler_disabled_does_not_call_cleanup(self) -> None: + with ( + patch.dict(os.environ, {"HLL_DB_MAINTENANCE_ENABLED": "false"}, clear=False), + patch("app.historical_runner.run_database_maintenance_cleanup") as cleanup, + ): + result = _maybe_run_database_maintenance( + now=datetime(2026, 6, 20, 12, tzinfo=timezone.utc) + ) + + cleanup.assert_not_called() + self.assertEqual(result["status"], "skipped") + self.assertEqual(result["reason"], "disabled") + + def test_scheduler_enabled_but_not_due_does_not_call_cleanup(self) -> None: + with ( + patch.dict( + os.environ, + { + "HLL_DB_MAINTENANCE_ENABLED": "true", + "HLL_DB_MAINTENANCE_INTERVAL_SECONDS": "43200", + }, + clear=False, + ), + patch( + "app.historical_runner.run_database_maintenance_cleanup", + return_value={"status": "ok"}, + ) as cleanup, + ): + first = _maybe_run_database_maintenance( + now=datetime(2026, 6, 20, 0, tzinfo=timezone.utc) + ) + second = _maybe_run_database_maintenance( + now=datetime(2026, 6, 20, 1, tzinfo=timezone.utc) + ) + + self.assertEqual(first["status"], "ok") + self.assertEqual(second["status"], "skipped") + self.assertEqual(second["reason"], "not-due") + cleanup.assert_called_once() + + def test_scheduler_enabled_and_due_calls_cleanup(self) -> None: + with ( + patch.dict(os.environ, {"HLL_DB_MAINTENANCE_ENABLED": "true"}, clear=False), + patch( + "app.historical_runner.run_database_maintenance_cleanup", + return_value={"status": "ok"}, + ) as cleanup, + ): + result = _maybe_run_database_maintenance( + now=datetime(2026, 6, 20, 12, tzinfo=timezone.utc) + ) + + cleanup.assert_called_once() + self.assertEqual(result["status"], "ok") + + def test_cleanup_exception_is_logged_and_runner_continues(self) -> None: + stream = io.StringIO() + with ( + patch.dict(os.environ, {"HLL_DB_MAINTENANCE_ENABLED": "true"}, clear=False), + patch("app.historical_runner.backend_writer_lock", return_value=nullcontext()), + patch( + "app.historical_runner._run_primary_rcon_capture", + return_value={"status": "ok", "targets": []}, + ), + patch( + "app.historical_runner.run_incremental_refresh", + return_value={"status": "ok"}, + ), + patch( + "app.historical_runner.generate_historical_snapshots", + return_value={"status": "ok"}, + ), + patch( + "app.historical_runner.rebuild_elo_mmr_models", + return_value={"status": "ok"}, + ), + patch( + "app.historical_runner.run_database_maintenance_cleanup", + side_effect=RuntimeError("maintenance failed"), + ), + redirect_stdout(stream), + ): + result = _run_refresh_with_retries( + max_retries=0, + retry_delay_seconds=0, + server_slug="comunidad-hispana-01", + max_pages=None, + page_size=None, + run_number=1, + ) + + self.assertEqual(result["status"], "ok") + self.assertEqual(result["database_maintenance_result"]["status"], "error") + self.assertIn("database-maintenance-scheduler-failed", stream.getvalue()) + + def test_interval_parsing_handles_invalid_values_safely(self) -> None: + with patch.dict( + os.environ, + { + "HLL_DB_MAINTENANCE_ENABLED": "true", + "HLL_DB_MAINTENANCE_INTERVAL_SECONDS": "bad", + }, + clear=False, + ): + interval_seconds, source = historical_runner._resolve_db_maintenance_interval_seconds() + + self.assertEqual(interval_seconds, 43200) + self.assertEqual(source, "default-invalid-env-fallback") + + def test_maintenance_state_is_tracked_in_process(self) -> None: + with ( + patch.dict( + os.environ, + { + "HLL_DB_MAINTENANCE_ENABLED": "true", + "HLL_DB_MAINTENANCE_INTERVAL_SECONDS": "3600", + }, + clear=False, + ), + patch( + "app.historical_runner.run_database_maintenance_cleanup", + return_value={"status": "ok"}, + ), + ): + _maybe_run_database_maintenance(now=datetime(2026, 6, 20, 12, tzinfo=timezone.utc)) + self.assertEqual( + historical_runner._LAST_DATABASE_MAINTENANCE_RUN_AT, + datetime(2026, 6, 20, 12, tzinfo=timezone.utc), + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/docs/database-maintenance.md b/docs/database-maintenance.md new file mode 100644 index 0000000..ce0a521 --- /dev/null +++ b/docs/database-maintenance.md @@ -0,0 +1,307 @@ +# Database Maintenance + +## Overview + +HLL Vietnam keeps database cleanup at the application level. + +The current maintenance scope is intentionally narrow: + +- old `server_snapshots`; +- old non-critical `rcon_admin_log_events`; +- old critical `rcon_admin_log_events` only after retention and protected-match checks; +- old non-protected `rcon_materialized_matches`; +- dependent `rcon_match_player_stats` for deleted matches. + +The first maintenance pass does not routinely delete: + +- `displayed_historical_snapshots`; +- file-based snapshots under `backend/data/snapshots/`; +- public-scoreboard `historical_*` fallback tables; +- `player_event_raw_ledger` and its worker metadata; +- Elo/MMR tables; +- Comunidad Hispana #03 data reactivation or targets. + +## Why Application-Level And Not `pg_cron` + +Cleanup is versioned in backend code instead of delegated to `pg_cron`, host cron, or a separate container because the retention logic depends on product rules: + +- keep the latest 100 closed materialized matches; +- keep the current month; +- keep the previous month during the first 7 days of a new month; +- keep the current week; +- keep the previous week when weekly fallback may still need it; +- keep child stats for protected matches; +- avoid breaking current/live pages that still read recent AdminLog data. + +Those rules belong with the application’s read and write model, not inside database-only scheduling. + +## Scheduled Cleanup Inside `historical-runner` + +Database maintenance is scheduled inside `app.historical_runner`. + +Behavior: + +- disabled by default; +- no extra Docker service is added for maintenance; +- the runner checks whether maintenance is due; +- when enabled and due, the runner invokes `python -m app.database_maintenance cleanup --apply` behavior through the shared Python function; +- failures are logged and do not crash the historical runner loop; +- cleanup runs under the same writer-lock coordination used by the historical writer flows. + +Relevant structured log events: + +- `database-maintenance-scheduler-skipped-disabled` +- `database-maintenance-scheduler-skipped-not-due` +- `database-maintenance-scheduler-started` +- `database-maintenance-scheduler-completed` +- `database-maintenance-scheduler-failed` + +## Environment Variables + +Required maintenance-related variables: + +```text +HLL_DB_MAINTENANCE_ENABLED=false +HLL_DB_MAINTENANCE_INTERVAL_SECONDS=43200 +HLL_RECENT_MATCHES_KEEP=100 +HLL_ADMIN_LOG_NONCRITICAL_RETENTION_DAYS=30 +HLL_ADMIN_LOG_CRITICAL_RETENTION_DAYS=90 +HLL_SERVER_SNAPSHOT_RETENTION_DAYS=14 +HLL_DB_MAINTENANCE_BATCH_SIZE=5000 +``` + +Meaning: + +- `HLL_DB_MAINTENANCE_ENABLED` + Enables scheduled apply mode inside `historical-runner`. +- `HLL_DB_MAINTENANCE_INTERVAL_SECONDS` + Default scheduler interval. `43200` means every 12 hours. +- `HLL_RECENT_MATCHES_KEEP` + Number of latest closed materialized matches that must always be protected. +- `HLL_ADMIN_LOG_NONCRITICAL_RETENTION_DAYS` + Retention for non-critical AdminLog events such as chat/connect/disconnect. +- `HLL_ADMIN_LOG_CRITICAL_RETENTION_DAYS` + Retention for critical AdminLog events such as `kill`, `match_start`, `match_end`. +- `HLL_SERVER_SNAPSHOT_RETENTION_DAYS` + Retention for live server snapshots. +- `HLL_DB_MAINTENANCE_BATCH_SIZE` + Delete batch size for apply mode. + +## Protected Data + +The cleanup command protects: + +- latest 100 closed materialized matches by default; +- current month materialized matches; +- previous month materialized matches when the current day is `1` through `7`; +- current week materialized matches; +- previous week materialized matches when weekly fallback may still need them; +- `rcon_match_player_stats` belonging to protected matches; +- current/live AdminLog data required for visible current-match surfaces; +- `displayed_historical_snapshots`; +- file snapshots in `backend/data/snapshots/`. + +If a match timestamp cannot be interpreted safely, that match is skipped and protected instead of deleted. + +## Deleted Data + +Apply mode is currently allowed to delete: + +- `server_snapshots` older than retention; +- non-critical `rcon_admin_log_events` older than retention; +- critical `rcon_admin_log_events` older than retention only when they are not required by protected materialized match ranges; +- non-protected `rcon_materialized_matches`; +- dependent `rcon_match_player_stats` for deleted matches. + +Current critical AdminLog event types: + +- `kill` +- `match_start` +- `match_end` + +## Dry-Run Command + +From `backend/`: + +```powershell +python -m app.database_maintenance cleanup --dry-run +``` + +From the repository root with the backend package on `PYTHONPATH`: + +```powershell +$env:PYTHONPATH='backend' +python -m app.database_maintenance cleanup --dry-run +``` + +Inside Docker Compose: + +```powershell +docker compose exec backend python -m app.database_maintenance cleanup --dry-run +``` + +Useful dry-run options: + +```powershell +docker compose exec backend python -m app.database_maintenance cleanup --dry-run ` + --recent-matches-keep 100 ` + --admin-log-noncritical-retention-days 30 ` + --admin-log-critical-retention-days 90 ` + --server-snapshot-retention-days 14 ` + --batch-size 5000 +``` + +Dry-run is the safe preview path and should be reviewed before any production apply. + +## Apply Command + +Local module execution: + +```powershell +python -m app.database_maintenance cleanup --apply +``` + +Docker Compose: + +```powershell +docker compose exec backend python -m app.database_maintenance cleanup --apply +``` + +One-off local validation with a fixed time anchor: + +```powershell +python -m app.database_maintenance cleanup --apply --now 2026-06-20T12:00:00Z +``` + +Optional maintenance vacuum/analyze: + +```powershell +python -m app.database_maintenance cleanup --apply --vacuum-analyze +``` + +## Table-Size Audit SQL + +```sql +select + schemaname, + relname as table_name, + pg_size_pretty(pg_total_relation_size(relid)) as total_size, + pg_size_pretty(pg_relation_size(relid)) as table_size, + pg_size_pretty(pg_total_relation_size(relid) - pg_relation_size(relid)) as indexes_size, + n_live_tup as estimated_rows, + n_dead_tup as estimated_dead_rows +from pg_stat_user_tables +order by pg_total_relation_size(relid) desc; +``` + +## Row-Count And Age Audit SQL + +### AdminLog events by type/date + +```sql +select + event_type, + count(*) as row_count, + min(event_timestamp) as first_event_timestamp, + max(event_timestamp) as last_event_timestamp, + min(server_time) as first_server_time, + max(server_time) as last_server_time +from rcon_admin_log_events +group by event_type +order by row_count desc, event_type asc; +``` + +### Materialized matches by server/date + +```sql +select + target_key, + source_basis, + count(*) as matches, + min(coalesce(ended_at, started_at)) as first_closed_at, + max(coalesce(ended_at, started_at)) as last_closed_at +from rcon_materialized_matches +group by target_key, source_basis +order by target_key asc, source_basis asc; +``` + +### Server snapshots by date + +```sql +select + server_id, + min(captured_at) as first_captured_at, + max(captured_at) as last_captured_at, + count(*) as snapshot_rows +from server_snapshots +group by server_id +order by last_captured_at desc; +``` + +### Displayed snapshots count + +```sql +select + snapshot_type, + metric, + snapshot_window, + count(*) as snapshot_rows, + min(generated_at) as first_generated_at, + max(generated_at) as last_generated_at +from displayed_historical_snapshots +group by snapshot_type, metric, snapshot_window +order by snapshot_type asc, metric asc, snapshot_window asc; +``` + +## Logs To Inspect + +The cleanup command emits JSON logs. Minimum events to look for: + +- `database-maintenance-started` +- `database-maintenance-plan` +- `database-maintenance-table-skipped` +- `database-maintenance-delete-batch` +- `database-maintenance-completed` +- `database-maintenance-error` + +Examples: + +```powershell +docker compose logs --tail=200 backend +docker compose logs --tail=200 historical-runner +``` + +If scheduled cleanup is enabled: + +```powershell +docker compose logs --tail=200 historical-runner +``` + +## Docker And Portainer Warnings + +- Never use `docker compose down -v` unless you intentionally want to delete PostgreSQL and mounted volume data. +- Always review dry-run output before enabling apply in production. +- Do not manually delete protected match or player-stat rows from PostgreSQL. +- Keep backups before changing retention settings. +- Do not add Comunidad Hispana #03 back into RCON targets in this task. +- Do not add a separate maintenance container, host cron, or `pg_cron` job for this feature. + +For Portainer-style operations the same warning applies: + +- deleting volumes is destructive; +- maintenance should run through the application command, not through manual table purges. + +## Rollback And Restore Considerations + +- Retention changes are destructive when apply mode runs. +- Keep a PostgreSQL backup before enabling scheduled apply in production. +- If cleanup removes too much data, recovery is restore-based, not “undo last delete.” +- Favor dry-run, smaller batch sizes, and reviewed retention values before long-running scheduled apply. + +## Safe Operator Flow + +1. Audit table size and row ages with the SQL above. +2. Run dry-run locally or in Compose. +3. Review protected counts and candidate counts in JSON output. +4. Enable `HLL_DB_MAINTENANCE_ENABLED=true` only after dry-run review. +5. Monitor `historical-runner` logs for scheduler events and cleanup completion. diff --git a/frontend/index.html b/frontend/index.html index bf01812..3b79278 100644 --- a/frontend/index.html +++ b/frontend/index.html @@ -5,9 +5,9 @@ - Comunidad Hispana - HLL Vietnam + Comunidad Hispana - HLL @@ -23,7 +23,7 @@
Logo oficial de la comunidad HLL Vietnam

Comunidad Hispana - HLL Vietnam + HLL

Centro de reunion para escuadras, eventos y acceso directo al