Přidáno:

nová stránka /portal/incidents s title CSBot Services Portal - Incidenty
menu Provoz -> Incidenty
aktivní incidenty a historie posledních 100 incidentů
barevné stavy: open červeně, resolved zeleně
karta Aktivní incidenty a seznam Poslední incidenty na přehledu
realtime napojení přes existující operations-live.js a snapshot
sekce Incidenty služby v detailu služby, posledních 20 incidentů
DB helpery pro service_incidents
migrace/indexy pro lokální prostředí, kde tabulka ještě neexistuje
This commit is contained in:
JiriUhlir
2026-06-01 13:38:06 +02:00
parent 994171be1a
commit 6aa5e69ee4
9 changed files with 374 additions and 9 deletions
+93
View File
@@ -0,0 +1,93 @@
from app.db.database import get_connection
from app.db.migrations import run_migrations
def _incident_select() -> str:
return """
SELECT
i.id,
i.service_id,
COALESCE(a.name, i.service_id) AS service_name,
i.status,
i.started_at,
i.ended_at,
CASE
WHEN i.duration_seconds IS NOT NULL THEN i.duration_seconds
WHEN LOWER(COALESCE(i.status, '')) = 'open' THEN CAST(strftime('%s', 'now') - strftime('%s', i.started_at) AS INTEGER)
ELSE NULL
END AS duration_seconds,
i.title,
i.description
FROM service_incidents i
LEFT JOIN apps a ON a.id = i.service_id
"""
def get_open_incidents(limit: int | None = None):
run_migrations()
con = get_connection()
sql = f"""
{_incident_select()}
WHERE LOWER(COALESCE(i.status, '')) = 'open'
ORDER BY i.started_at DESC, i.id DESC
"""
params = ()
if limit is not None:
sql += " LIMIT ?"
params = (limit,)
rows = con.execute(sql, params).fetchall()
con.close()
return [dict(row) for row in rows]
def get_recent_incidents(limit: int = 100):
run_migrations()
con = get_connection()
rows = con.execute(
f"""
{_incident_select()}
ORDER BY i.started_at DESC, i.id DESC
LIMIT ?
""",
(limit,),
).fetchall()
con.close()
return [dict(row) for row in rows]
def get_service_incidents(service_id: str, limit: int = 20):
run_migrations()
con = get_connection()
rows = con.execute(
f"""
{_incident_select()}
WHERE i.service_id = ?
ORDER BY i.started_at DESC, i.id DESC
LIMIT ?
""",
(service_id, limit),
).fetchall()
con.close()
return [dict(row) for row in rows]
def count_open_incidents() -> int:
run_migrations()
con = get_connection()
row = con.execute(
"""
SELECT COUNT(*) AS count
FROM service_incidents
WHERE LOWER(COALESCE(status, '')) = 'open'
"""
).fetchone()
con.close()
return row["count"] if row else 0
+16
View File
@@ -110,6 +110,20 @@ def run_migrations():
) )
""" """
) )
con.execute(
"""
CREATE TABLE IF NOT EXISTS service_incidents (
id INTEGER PRIMARY KEY AUTOINCREMENT,
service_id TEXT NOT NULL,
status TEXT NOT NULL,
started_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
ended_at TEXT,
duration_seconds INTEGER,
title TEXT,
description TEXT
)
"""
)
for statement in ( for statement in (
"ALTER TABLE app_templates ADD COLUMN runtime TEXT", "ALTER TABLE app_templates ADD COLUMN runtime TEXT",
"ALTER TABLE app_templates ADD COLUMN description TEXT", "ALTER TABLE app_templates ADD COLUMN description TEXT",
@@ -123,6 +137,8 @@ def run_migrations():
except Exception: except Exception:
pass pass
con.execute("CREATE INDEX IF NOT EXISTS idx_app_variables_app_id ON app_variables(app_id)") con.execute("CREATE INDEX IF NOT EXISTS idx_app_variables_app_id ON app_variables(app_id)")
con.execute("CREATE INDEX IF NOT EXISTS idx_service_incidents_status_started ON service_incidents(status, started_at)")
con.execute("CREATE INDEX IF NOT EXISTS idx_service_incidents_service_started ON service_incidents(service_id, started_at)")
con.commit() con.commit()
con.close() con.close()
+9
View File
@@ -1,5 +1,6 @@
from app.db.database import get_connection from app.db.database import get_connection
from app.db.health import get_health_summary, get_problem_service_health from app.db.health import get_health_summary, get_problem_service_health
from app.db.incidents import count_open_incidents, get_open_incidents, get_recent_incidents
from app.db.migrations import run_migrations from app.db.migrations import run_migrations
from app.db.workers import is_worker_online from app.db.workers import is_worker_online
@@ -200,6 +201,9 @@ def get_operations_snapshot():
).fetchone()["count"] ).fetchone()["count"]
health_summary = get_health_summary() health_summary = get_health_summary()
health_problems = get_problem_service_health() health_problems = get_problem_service_health()
open_incidents_count = count_open_incidents()
open_incidents = get_open_incidents(limit=10)
recent_incidents = get_recent_incidents(limit=10)
jobs_summary = con.execute( jobs_summary = con.execute(
""" """
@@ -305,6 +309,11 @@ def get_operations_snapshot():
"unreachable": health_summary["unreachable"], "unreachable": health_summary["unreachable"],
"problems": health_problems, "problems": health_problems,
}, },
"incidents": {
"open": open_incidents_count,
"active": open_incidents,
"recent": recent_incidents,
},
"workers": { "workers": {
"online": workers_online, "online": workers_online,
"offline": max(0, total_workers - workers_online), "offline": max(0, total_workers - workers_online),
+2 -1
View File
@@ -5,7 +5,7 @@ from fastapi.staticfiles import StaticFiles
from starlette.middleware.sessions import SessionMiddleware from starlette.middleware.sessions import SessionMiddleware
from .config import read_env_value from .config import read_env_value
from .routes import apps, audit, auth, backups, deployments, health, jobs, operations, workers from .routes import apps, audit, auth, backups, deployments, health, incidents, jobs, operations, workers
def create_app() -> FastAPI: def create_app() -> FastAPI:
@@ -27,6 +27,7 @@ def create_app() -> FastAPI:
app.include_router(apps.router) app.include_router(apps.router)
app.include_router(backups.router) app.include_router(backups.router)
app.include_router(deployments.router) app.include_router(deployments.router)
app.include_router(incidents.router)
app.include_router(operations.router) app.include_router(operations.router)
app.include_router(jobs.router) app.include_router(jobs.router)
app.include_router(workers.router) app.include_router(workers.router)
+17
View File
@@ -29,8 +29,10 @@ from ..db.apps import (
) )
from ..db.audit import log_audit_event from ..db.audit import log_audit_event
from ..db.health import get_latest_service_health, get_service_health, get_service_health_history from ..db.health import get_latest_service_health, get_service_health, get_service_health_history
from ..db.incidents import get_service_incidents
from ..db.jobs import create_job, get_jobs, has_active_deploy_job from ..db.jobs import create_job, get_jobs, has_active_deploy_job
from ..routes.deployments import render_status_pill from ..routes.deployments import render_status_pill
from ..routes.incidents import render_incident_history_rows
from ..shell import run_command from ..shell import run_command
from ..templates.layout import page, render_result from ..templates.layout import page, render_result
@@ -415,6 +417,7 @@ def app_detail(app_id: str, request: Request, user=Depends(require_user)):
templates = get_app_templates() templates = get_app_templates()
template_options = render_template_options(templates, app.get("template", "") or "") template_options = render_template_options(templates, app.get("template", "") or "")
variables = get_app_variables(app.get("id", "")) variables = get_app_variables(app.get("id", ""))
incidents = get_service_incidents(app.get("id", ""), limit=20)
current_health = get_service_health(app.get("id", "")) current_health = get_service_health(app.get("id", ""))
health_history = get_service_health_history(app.get("id", ""), limit=50) health_history = get_service_health_history(app.get("id", ""), limit=50)
@@ -653,6 +656,20 @@ def app_detail(app_id: str, request: Request, user=Depends(require_user)):
</table> </table>
</div> </div>
<div class="card">
<h2>Incidenty slu&zcaron;by</h2>
<table>
<tr>
<th>N&aacute;zev</th>
<th>Za&ccaron;&aacute;tek</th>
<th>Konec</th>
<th>Trv&aacute;n&iacute;</th>
<th>Stav</th>
</tr>
{render_incident_history_rows(incidents, include_service=False)}
</table>
</div>
<div class="card"> <div class="card">
<h2>Historie nasazení</h2> <h2>Historie nasazení</h2>
<table> <table>
+148
View File
@@ -0,0 +1,148 @@
import html
from urllib.parse import quote
from fastapi import APIRouter, Depends, Request
from fastapi.responses import HTMLResponse
from app.auth import require_user
from app.db.audit import log_audit_event
from app.db.incidents import get_open_incidents, get_recent_incidents
from app.templates.layout import page
router = APIRouter()
def format_duration(seconds) -> str:
if seconds is None or seconds == "":
return ""
try:
total = max(0, int(seconds))
except (TypeError, ValueError):
return ""
hours, remainder = divmod(total, 3600)
minutes, secs = divmod(remainder, 60)
if hours:
return f"{hours} h {minutes} min"
if minutes:
return f"{minutes} min {secs} s"
return f"{secs} s"
def render_incident_status(status: str | None) -> str:
value = status or ""
normalized = value.lower()
labels = {
"open": "otev\u0159en\u00fd",
"resolved": "vy\u0159e\u0161en\u00fd",
}
class_name = "pill pill-muted"
if normalized == "open":
class_name = "pill pill-danger"
elif normalized == "resolved":
class_name = "pill pill-success"
return f'<span class="{class_name}">{html.escape(labels.get(normalized, value))}</span>'
def render_open_incident_rows(incidents: list[dict]) -> str:
rows = ""
for incident in incidents:
service_id = incident.get("service_id", "") or ""
service_name = html.escape(incident.get("service_name") or service_id)
title = html.escape(incident.get("title", "") or "")
started_at = html.escape(incident.get("started_at", "") or "")
duration = format_duration(incident.get("duration_seconds"))
if not duration:
duration = "prob\u00edh\u00e1"
rows += f"""
<tr>
<td><a href="/portal/apps/{quote(service_id, safe='')}">{service_name}</a></td>
<td>{title}</td>
<td>{started_at}</td>
<td>{html.escape(duration)}</td>
</tr>
"""
if not rows:
rows = '<tr><td colspan="4">\u017d\u00e1dn\u00e9 aktivn\u00ed incidenty.</td></tr>'
return rows
def render_incident_history_rows(incidents: list[dict], include_service: bool = True) -> str:
rows = ""
for incident in incidents:
service_id = incident.get("service_id", "") or ""
service_name = html.escape(incident.get("service_name") or service_id)
title = html.escape(incident.get("title", "") or "")
started_at = html.escape(incident.get("started_at", "") or "")
ended_at = html.escape(incident.get("ended_at", "") or "")
duration = html.escape(format_duration(incident.get("duration_seconds")))
status = render_incident_status(incident.get("status"))
service_cell = f'<td><a href="/portal/apps/{quote(service_id, safe="")}">{service_name}</a></td>' if include_service else ""
rows += f"""
<tr>
{service_cell}
<td>{title}</td>
<td>{started_at}</td>
<td>{ended_at}</td>
<td>{duration}</td>
<td>{status}</td>
</tr>
"""
colspan = 6 if include_service else 5
if not rows:
rows = f'<tr><td colspan="{colspan}">Zat\u00edm nejsou evidovan\u00e9 \u017e\u00e1dn\u00e9 incidenty.</td></tr>'
return rows
@router.get("/incidents", response_class=HTMLResponse)
def incidents_page(request: Request, user=Depends(require_user)):
open_incidents = get_open_incidents()
recent_incidents = get_recent_incidents(limit=100)
log_audit_event(
user,
action="incidents.view",
target_type="incidents",
metadata={"open": len(open_incidents), "history": len(recent_incidents)},
)
return page(
"Incidenty",
f"""
<div class="card">
<h2>Incidenty</h2>
<p class="muted">P\u0159ehled incident\u016f vytv\u00e1\u0159en\u00fdch a uzav\u00edran\u00fdch monitorem.</p>
</div>
<div class="card">
<h2>Aktivn\u00ed incidenty</h2>
<table>
<tr>
<th>Slu\u017eba</th>
<th>N\u00e1zev</th>
<th>Za\u010d\u00e1tek</th>
<th>Trv\u00e1n\u00ed</th>
</tr>
<tbody data-live-table="incidents.active">{render_open_incident_rows(open_incidents)}</tbody>
</table>
</div>
<div class="card">
<h2>Historie incident\u016f</h2>
<table>
<tr>
<th>Slu\u017eba</th>
<th>N\u00e1zev</th>
<th>Za\u010d\u00e1tek</th>
<th>Konec</th>
<th>Trv\u00e1n\u00ed</th>
<th>Stav</th>
</tr>
<tbody data-live-table="incidents.recent">{render_incident_history_rows(recent_incidents)}</tbody>
</table>
</div>
<script src="/portal/static/operations-live.js"></script>
""",
user=user,
)
+26 -8
View File
@@ -11,6 +11,7 @@ from app.db.operations import (
get_operations_snapshot, get_operations_snapshot,
) )
from app.routes.deployments import render_status_pill from app.routes.deployments import render_status_pill
from app.routes.incidents import render_incident_history_rows, render_open_incident_rows
from app.routes.jobs import render_job_status from app.routes.jobs import render_job_status
from app.templates.layout import page from app.templates.layout import page
@@ -171,6 +172,9 @@ def operations_dashboard(request: Request, user=Depends(require_user)):
"running": snapshot["deployments"]["running"], "running": snapshot["deployments"]["running"],
"failed_24h": snapshot["deployments"]["failed_24h"], "failed_24h": snapshot["deployments"]["failed_24h"],
}, },
"incidents": {
"open": snapshot["incidents"]["open"],
},
}, },
) )
@@ -188,6 +192,7 @@ def operations_dashboard(request: Request, user=Depends(require_user)):
<div class="stat-card stat-success"><span>Zdravé služby</span><strong data-live-count="health.healthy">{snapshot["health"]["healthy"]}</strong></div> <div class="stat-card stat-success"><span>Zdravé služby</span><strong data-live-count="health.healthy">{snapshot["health"]["healthy"]}</strong></div>
<div class="stat-card stat-warning"><span>Nezdravé služby</span><strong data-live-count="health.unhealthy">{snapshot["health"]["unhealthy"]}</strong></div> <div class="stat-card stat-warning"><span>Nezdravé služby</span><strong data-live-count="health.unhealthy">{snapshot["health"]["unhealthy"]}</strong></div>
<div class="stat-card stat-danger"><span>Nedostupné služby</span><strong data-live-count="health.unreachable">{snapshot["health"]["unreachable"]}</strong></div> <div class="stat-card stat-danger"><span>Nedostupné služby</span><strong data-live-count="health.unreachable">{snapshot["health"]["unreachable"]}</strong></div>
<div class="stat-card stat-danger"><span>Aktivní incidenty</span><strong data-live-count="incidents.open">{snapshot["incidents"]["open"]}</strong></div>
<div class="stat-card stat-warning"><span>Aktivní úlohy</span><strong data-live-count="jobs.running">{snapshot["jobs"]["running"]}</strong></div> <div class="stat-card stat-warning"><span>Aktivní úlohy</span><strong data-live-count="jobs.running">{snapshot["jobs"]["running"]}</strong></div>
<div class="stat-card stat-total"><span>Nasazení dnes</span><strong data-live-count="deployments.today">{snapshot["deployments"]["today"]}</strong></div> <div class="stat-card stat-total"><span>Nasazení dnes</span><strong data-live-count="deployments.today">{snapshot["deployments"]["today"]}</strong></div>
<div class="stat-card stat-success"><span>Online workery</span><strong data-live-count="workers.online">{snapshot["workers"]["online"]}</strong></div> <div class="stat-card stat-success"><span>Online workery</span><strong data-live-count="workers.online">{snapshot["workers"]["online"]}</strong></div>
@@ -209,17 +214,15 @@ def operations_dashboard(request: Request, user=Depends(require_user)):
</div> </div>
<div class="card"> <div class="card">
<h2>Poslední incidenty</h2> <h2>Aktivní incidenty</h2>
<table> <table>
<tr> <tr>
<th>ID</th> <th>Služba</th>
<th>Status</th> <th>Název</th>
<th>Typ</th> <th>Začátek</th>
<th>Cíl</th> <th>Trvání</th>
<th>Zdroj</th>
<th>Vytvořeno</th>
</tr> </tr>
<tbody data-live-table="jobs.failed">{render_recent_jobs(snapshot["jobs"]["recent_failed"])}</tbody> <tbody data-live-table="incidents.active">{render_open_incident_rows(snapshot["incidents"]["active"])}</tbody>
</table> </table>
</div> </div>
@@ -239,6 +242,21 @@ def operations_dashboard(request: Request, user=Depends(require_user)):
</table> </table>
</div> </div>
<div class="card">
<h2>Poslední incidenty</h2>
<table>
<tr>
<th>Služba</th>
<th>Název</th>
<th>Začátek</th>
<th>Konec</th>
<th>Trvání</th>
<th>Stav</th>
</tr>
<tbody data-live-table="incidents.recent">{render_incident_history_rows(snapshot["incidents"]["recent"])}</tbody>
</table>
</div>
<div class="card"> <div class="card">
<h2>Poslední auditní události</h2> <h2>Poslední auditní události</h2>
<table> <table>
+62
View File
@@ -59,6 +59,32 @@
return `<span class="${className}">${escapeHtml(labels[normalized] || value)}</span>`; return `<span class="${className}">${escapeHtml(labels[normalized] || value)}</span>`;
}; };
const incidentBadge = (status) => {
const value = String(status || "");
const normalized = value.toLowerCase();
const labels = {
open: "otevřený",
resolved: "vyřešený",
};
let className = "pill pill-muted";
if (normalized === "open") className = "pill pill-danger";
if (normalized === "resolved") className = "pill pill-success";
return `<span class="${className}">${escapeHtml(labels[normalized] || value)}</span>`;
};
const formatDuration = (seconds) => {
if (seconds === null || seconds === undefined || seconds === "") {
return "";
}
const total = Math.max(0, Number.parseInt(seconds, 10) || 0);
const hours = Math.floor(total / 3600);
const minutes = Math.floor((total % 3600) / 60);
const secs = total % 60;
if (hours) return `${hours} h ${minutes} min`;
if (minutes) return `${minutes} min ${secs} s`;
return `${secs} s`;
};
const setText = (selector, value) => { const setText = (selector, value) => {
document.querySelectorAll(selector).forEach((el) => { document.querySelectorAll(selector).forEach((el) => {
el.textContent = value ?? 0; el.textContent = value ?? 0;
@@ -145,6 +171,39 @@
return rows || '<tr><td colspan="5">Žádné služby nevyžadují pozornost.</td></tr>'; return rows || '<tr><td colspan="5">Žádné služby nevyžadují pozornost.</td></tr>';
}; };
const renderOpenIncidents = (items) => {
const rows = (items || []).map((item) => {
const serviceName = item.service_name || item.service_id || "";
const duration = formatDuration(item.duration_seconds) || "probíhá";
return `
<tr>
<td><a href="/portal/apps/${encodeURIComponent(item.service_id || "")}">${escapeHtml(serviceName)}</a></td>
<td>${escapeHtml(item.title)}</td>
<td>${escapeHtml(item.started_at)}</td>
<td>${escapeHtml(duration)}</td>
</tr>
`;
}).join("");
return rows || '<tr><td colspan="4">Žádné aktivní incidenty.</td></tr>';
};
const renderIncidents = (items) => {
const rows = (items || []).map((item) => {
const serviceName = item.service_name || item.service_id || "";
return `
<tr>
<td><a href="/portal/apps/${encodeURIComponent(item.service_id || "")}">${escapeHtml(serviceName)}</a></td>
<td>${escapeHtml(item.title)}</td>
<td>${escapeHtml(item.started_at)}</td>
<td>${escapeHtml(item.ended_at)}</td>
<td>${escapeHtml(formatDuration(item.duration_seconds))}</td>
<td>${incidentBadge(item.status)}</td>
</tr>
`;
}).join("");
return rows || '<tr><td colspan="6">Zatím nejsou evidované žádné incidenty.</td></tr>';
};
const renderSnapshot = (snapshot) => { const renderSnapshot = (snapshot) => {
setText("[data-live-count='jobs.queued']", snapshot.jobs?.queued); setText("[data-live-count='jobs.queued']", snapshot.jobs?.queued);
setText("[data-live-count='jobs.running']", snapshot.jobs?.running); setText("[data-live-count='jobs.running']", snapshot.jobs?.running);
@@ -155,6 +214,7 @@
setText("[data-live-count='health.healthy']", snapshot.health?.healthy); setText("[data-live-count='health.healthy']", snapshot.health?.healthy);
setText("[data-live-count='health.unhealthy']", snapshot.health?.unhealthy); setText("[data-live-count='health.unhealthy']", snapshot.health?.unhealthy);
setText("[data-live-count='health.unreachable']", snapshot.health?.unreachable); setText("[data-live-count='health.unreachable']", snapshot.health?.unreachable);
setText("[data-live-count='incidents.open']", snapshot.incidents?.open);
setText("[data-live-count='workers.online']", snapshot.workers?.online); setText("[data-live-count='workers.online']", snapshot.workers?.online);
setText("[data-live-count='workers.offline']", snapshot.workers?.offline); setText("[data-live-count='workers.offline']", snapshot.workers?.offline);
setText("[data-live-count='workers.total']", (snapshot.workers?.online || 0) + (snapshot.workers?.offline || 0)); setText("[data-live-count='workers.total']", (snapshot.workers?.online || 0) + (snapshot.workers?.offline || 0));
@@ -170,6 +230,8 @@
document.querySelectorAll("[data-live-table='deployments.failed']").forEach((el) => { el.innerHTML = renderDeployments(snapshot.deployments?.recent_failed); }); document.querySelectorAll("[data-live-table='deployments.failed']").forEach((el) => { el.innerHTML = renderDeployments(snapshot.deployments?.recent_failed); });
document.querySelectorAll("[data-live-table='audit.recent']").forEach((el) => { el.innerHTML = renderAudit(snapshot.audit?.recent); }); document.querySelectorAll("[data-live-table='audit.recent']").forEach((el) => { el.innerHTML = renderAudit(snapshot.audit?.recent); });
document.querySelectorAll("[data-live-table='health.problems']").forEach((el) => { el.innerHTML = renderHealthProblems(snapshot.health?.problems); }); document.querySelectorAll("[data-live-table='health.problems']").forEach((el) => { el.innerHTML = renderHealthProblems(snapshot.health?.problems); });
document.querySelectorAll("[data-live-table='incidents.active']").forEach((el) => { el.innerHTML = renderOpenIncidents(snapshot.incidents?.active); });
document.querySelectorAll("[data-live-table='incidents.recent']").forEach((el) => { el.innerHTML = renderIncidents(snapshot.incidents?.recent); });
(snapshot.workers?.recent || []).forEach((worker) => { (snapshot.workers?.recent || []).forEach((worker) => {
const row = Array.from(document.querySelectorAll("tr[data-worker-id]")) const row = Array.from(document.querySelectorAll("tr[data-worker-id]"))
+1
View File
@@ -21,6 +21,7 @@ def page(title: str, body: str, user=None) -> str:
<div class="nav-menu-panel"> <div class="nav-menu-panel">
<a href="/portal/jobs">Úlohy</a> <a href="/portal/jobs">Úlohy</a>
<a href="/portal/deployments">Nasazení</a> <a href="/portal/deployments">Nasazení</a>
<a href="/portal/incidents">Incidenty</a>
<a href="/portal/workers">Workery</a> <a href="/portal/workers">Workery</a>
</div> </div>
</div> </div>