diff --git a/app/db/incidents.py b/app/db/incidents.py new file mode 100644 index 0000000..694c7b9 --- /dev/null +++ b/app/db/incidents.py @@ -0,0 +1,93 @@ +from app.db.database import get_connection +from app.db.migrations import run_migrations + + +def _incident_select() -> str: + return """ + SELECT + i.id, + i.service_id, + COALESCE(a.name, i.service_id) AS service_name, + i.status, + i.started_at, + i.ended_at, + CASE + WHEN i.duration_seconds IS NOT NULL THEN i.duration_seconds + WHEN LOWER(COALESCE(i.status, '')) = 'open' THEN CAST(strftime('%s', 'now') - strftime('%s', i.started_at) AS INTEGER) + ELSE NULL + END AS duration_seconds, + i.title, + i.description + FROM service_incidents i + LEFT JOIN apps a ON a.id = i.service_id + """ + + +def get_open_incidents(limit: int | None = None): + run_migrations() + con = get_connection() + + sql = f""" + {_incident_select()} + WHERE LOWER(COALESCE(i.status, '')) = 'open' + ORDER BY i.started_at DESC, i.id DESC + """ + params = () + if limit is not None: + sql += " LIMIT ?" + params = (limit,) + + rows = con.execute(sql, params).fetchall() + con.close() + return [dict(row) for row in rows] + + +def get_recent_incidents(limit: int = 100): + run_migrations() + con = get_connection() + + rows = con.execute( + f""" + {_incident_select()} + ORDER BY i.started_at DESC, i.id DESC + LIMIT ? + """, + (limit,), + ).fetchall() + + con.close() + return [dict(row) for row in rows] + + +def get_service_incidents(service_id: str, limit: int = 20): + run_migrations() + con = get_connection() + + rows = con.execute( + f""" + {_incident_select()} + WHERE i.service_id = ? + ORDER BY i.started_at DESC, i.id DESC + LIMIT ? + """, + (service_id, limit), + ).fetchall() + + con.close() + return [dict(row) for row in rows] + + +def count_open_incidents() -> int: + run_migrations() + con = get_connection() + + row = con.execute( + """ + SELECT COUNT(*) AS count + FROM service_incidents + WHERE LOWER(COALESCE(status, '')) = 'open' + """ + ).fetchone() + + con.close() + return row["count"] if row else 0 diff --git a/app/db/migrations.py b/app/db/migrations.py index b182d17..a1ffe68 100644 --- a/app/db/migrations.py +++ b/app/db/migrations.py @@ -110,6 +110,20 @@ def run_migrations(): ) """ ) + con.execute( + """ + CREATE TABLE IF NOT EXISTS service_incidents ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + service_id TEXT NOT NULL, + status TEXT NOT NULL, + started_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, + ended_at TEXT, + duration_seconds INTEGER, + title TEXT, + description TEXT + ) + """ + ) for statement in ( "ALTER TABLE app_templates ADD COLUMN runtime TEXT", "ALTER TABLE app_templates ADD COLUMN description TEXT", @@ -123,6 +137,8 @@ def run_migrations(): except Exception: pass con.execute("CREATE INDEX IF NOT EXISTS idx_app_variables_app_id ON app_variables(app_id)") + con.execute("CREATE INDEX IF NOT EXISTS idx_service_incidents_status_started ON service_incidents(status, started_at)") + con.execute("CREATE INDEX IF NOT EXISTS idx_service_incidents_service_started ON service_incidents(service_id, started_at)") con.commit() con.close() diff --git a/app/db/operations.py b/app/db/operations.py index 88ecc08..d275cbc 100644 --- a/app/db/operations.py +++ b/app/db/operations.py @@ -1,5 +1,6 @@ from app.db.database import get_connection from app.db.health import get_health_summary, get_problem_service_health +from app.db.incidents import count_open_incidents, get_open_incidents, get_recent_incidents from app.db.migrations import run_migrations from app.db.workers import is_worker_online @@ -200,6 +201,9 @@ def get_operations_snapshot(): ).fetchone()["count"] health_summary = get_health_summary() health_problems = get_problem_service_health() + open_incidents_count = count_open_incidents() + open_incidents = get_open_incidents(limit=10) + recent_incidents = get_recent_incidents(limit=10) jobs_summary = con.execute( """ @@ -305,6 +309,11 @@ def get_operations_snapshot(): "unreachable": health_summary["unreachable"], "problems": health_problems, }, + "incidents": { + "open": open_incidents_count, + "active": open_incidents, + "recent": recent_incidents, + }, "workers": { "online": workers_online, "offline": max(0, total_workers - workers_online), diff --git a/app/main.py b/app/main.py index 4aa05e4..6d1cacb 100644 --- a/app/main.py +++ b/app/main.py @@ -5,7 +5,7 @@ from fastapi.staticfiles import StaticFiles from starlette.middleware.sessions import SessionMiddleware from .config import read_env_value -from .routes import apps, audit, auth, backups, deployments, health, jobs, operations, workers +from .routes import apps, audit, auth, backups, deployments, health, incidents, jobs, operations, workers def create_app() -> FastAPI: @@ -27,6 +27,7 @@ def create_app() -> FastAPI: app.include_router(apps.router) app.include_router(backups.router) app.include_router(deployments.router) + app.include_router(incidents.router) app.include_router(operations.router) app.include_router(jobs.router) app.include_router(workers.router) diff --git a/app/routes/apps.py b/app/routes/apps.py index dcdb107..321854d 100644 --- a/app/routes/apps.py +++ b/app/routes/apps.py @@ -29,8 +29,10 @@ from ..db.apps import ( ) from ..db.audit import log_audit_event from ..db.health import get_latest_service_health, get_service_health, get_service_health_history +from ..db.incidents import get_service_incidents from ..db.jobs import create_job, get_jobs, has_active_deploy_job from ..routes.deployments import render_status_pill +from ..routes.incidents import render_incident_history_rows from ..shell import run_command from ..templates.layout import page, render_result @@ -415,6 +417,7 @@ def app_detail(app_id: str, request: Request, user=Depends(require_user)): templates = get_app_templates() template_options = render_template_options(templates, app.get("template", "") or "") variables = get_app_variables(app.get("id", "")) + incidents = get_service_incidents(app.get("id", ""), limit=20) current_health = get_service_health(app.get("id", "")) health_history = get_service_health_history(app.get("id", ""), limit=50) @@ -653,6 +656,20 @@ def app_detail(app_id: str, request: Request, user=Depends(require_user)): +
+

Incidenty služby

+ + + + + + + + + {render_incident_history_rows(incidents, include_service=False)} +
NázevZačátekKonecTrváníStav
+
+

Historie nasazení

diff --git a/app/routes/incidents.py b/app/routes/incidents.py new file mode 100644 index 0000000..0deb907 --- /dev/null +++ b/app/routes/incidents.py @@ -0,0 +1,148 @@ +import html +from urllib.parse import quote + +from fastapi import APIRouter, Depends, Request +from fastapi.responses import HTMLResponse + +from app.auth import require_user +from app.db.audit import log_audit_event +from app.db.incidents import get_open_incidents, get_recent_incidents +from app.templates.layout import page + +router = APIRouter() + + +def format_duration(seconds) -> str: + if seconds is None or seconds == "": + return "" + try: + total = max(0, int(seconds)) + except (TypeError, ValueError): + return "" + + hours, remainder = divmod(total, 3600) + minutes, secs = divmod(remainder, 60) + if hours: + return f"{hours} h {minutes} min" + if minutes: + return f"{minutes} min {secs} s" + return f"{secs} s" + + +def render_incident_status(status: str | None) -> str: + value = status or "" + normalized = value.lower() + labels = { + "open": "otev\u0159en\u00fd", + "resolved": "vy\u0159e\u0161en\u00fd", + } + class_name = "pill pill-muted" + if normalized == "open": + class_name = "pill pill-danger" + elif normalized == "resolved": + class_name = "pill pill-success" + return f'{html.escape(labels.get(normalized, value))}' + + +def render_open_incident_rows(incidents: list[dict]) -> str: + rows = "" + for incident in incidents: + service_id = incident.get("service_id", "") or "" + service_name = html.escape(incident.get("service_name") or service_id) + title = html.escape(incident.get("title", "") or "") + started_at = html.escape(incident.get("started_at", "") or "") + duration = format_duration(incident.get("duration_seconds")) + if not duration: + duration = "prob\u00edh\u00e1" + rows += f""" + + + + + + + """ + + if not rows: + rows = '' + return rows + + +def render_incident_history_rows(incidents: list[dict], include_service: bool = True) -> str: + rows = "" + for incident in incidents: + service_id = incident.get("service_id", "") or "" + service_name = html.escape(incident.get("service_name") or service_id) + title = html.escape(incident.get("title", "") or "") + started_at = html.escape(incident.get("started_at", "") or "") + ended_at = html.escape(incident.get("ended_at", "") or "") + duration = html.escape(format_duration(incident.get("duration_seconds"))) + status = render_incident_status(incident.get("status")) + service_cell = f'' if include_service else "" + rows += f""" + + {service_cell} + + + + + + + """ + + colspan = 6 if include_service else 5 + if not rows: + rows = f'' + return rows + + +@router.get("/incidents", response_class=HTMLResponse) +def incidents_page(request: Request, user=Depends(require_user)): + open_incidents = get_open_incidents() + recent_incidents = get_recent_incidents(limit=100) + log_audit_event( + user, + action="incidents.view", + target_type="incidents", + metadata={"open": len(open_incidents), "history": len(recent_incidents)}, + ) + + return page( + "Incidenty", + f""" +
+

Incidenty

+

P\u0159ehled incident\u016f vytv\u00e1\u0159en\u00fdch a uzav\u00edran\u00fdch monitorem.

+
+ +
+

Aktivn\u00ed incidenty

+
{service_name}{title}{started_at}{html.escape(duration)}
\u017d\u00e1dn\u00e9 aktivn\u00ed incidenty.
{service_name}
{title}{started_at}{ended_at}{duration}{status}
Zat\u00edm nejsou evidovan\u00e9 \u017e\u00e1dn\u00e9 incidenty.
+ + + + + + + {render_open_incident_rows(open_incidents)} +
Slu\u017ebaN\u00e1zevZa\u010d\u00e1tekTrv\u00e1n\u00ed
+
+ +
+

Historie incident\u016f

+ + + + + + + + + + {render_incident_history_rows(recent_incidents)} +
Slu\u017ebaN\u00e1zevZa\u010d\u00e1tekKonecTrv\u00e1n\u00edStav
+
+ + """, + user=user, + ) diff --git a/app/routes/operations.py b/app/routes/operations.py index 583bff7..07bb4a5 100644 --- a/app/routes/operations.py +++ b/app/routes/operations.py @@ -11,6 +11,7 @@ from app.db.operations import ( get_operations_snapshot, ) from app.routes.deployments import render_status_pill +from app.routes.incidents import render_incident_history_rows, render_open_incident_rows from app.routes.jobs import render_job_status from app.templates.layout import page @@ -171,6 +172,9 @@ def operations_dashboard(request: Request, user=Depends(require_user)): "running": snapshot["deployments"]["running"], "failed_24h": snapshot["deployments"]["failed_24h"], }, + "incidents": { + "open": snapshot["incidents"]["open"], + }, }, ) @@ -188,6 +192,7 @@ def operations_dashboard(request: Request, user=Depends(require_user)):
Zdravé služby{snapshot["health"]["healthy"]}
Nezdravé služby{snapshot["health"]["unhealthy"]}
Nedostupné služby{snapshot["health"]["unreachable"]}
+
Aktivní incidenty{snapshot["incidents"]["open"]}
Aktivní úlohy{snapshot["jobs"]["running"]}
Nasazení dnes{snapshot["deployments"]["today"]}
Online workery{snapshot["workers"]["online"]}
@@ -209,17 +214,15 @@ def operations_dashboard(request: Request, user=Depends(require_user)):
-

Poslední incidenty

+

Aktivní incidenty

- - - - - - + + + + - {render_recent_jobs(snapshot["jobs"]["recent_failed"])} + {render_open_incident_rows(snapshot["incidents"]["active"])}
IDStatusTypCílZdrojVytvořenoSlužbaNázevZačátekTrvání
@@ -239,6 +242,21 @@ def operations_dashboard(request: Request, user=Depends(require_user)): +
+

Poslední incidenty

+ + + + + + + + + + {render_incident_history_rows(snapshot["incidents"]["recent"])} +
SlužbaNázevZačátekKonecTrváníStav
+
+

Poslední auditní události

diff --git a/app/static/operations-live.js b/app/static/operations-live.js index a25e376..314886e 100644 --- a/app/static/operations-live.js +++ b/app/static/operations-live.js @@ -59,6 +59,32 @@ return `${escapeHtml(labels[normalized] || value)}`; }; + const incidentBadge = (status) => { + const value = String(status || ""); + const normalized = value.toLowerCase(); + const labels = { + open: "otevřený", + resolved: "vyřešený", + }; + let className = "pill pill-muted"; + if (normalized === "open") className = "pill pill-danger"; + if (normalized === "resolved") className = "pill pill-success"; + return `${escapeHtml(labels[normalized] || value)}`; + }; + + const formatDuration = (seconds) => { + if (seconds === null || seconds === undefined || seconds === "") { + return ""; + } + const total = Math.max(0, Number.parseInt(seconds, 10) || 0); + const hours = Math.floor(total / 3600); + const minutes = Math.floor((total % 3600) / 60); + const secs = total % 60; + if (hours) return `${hours} h ${minutes} min`; + if (minutes) return `${minutes} min ${secs} s`; + return `${secs} s`; + }; + const setText = (selector, value) => { document.querySelectorAll(selector).forEach((el) => { el.textContent = value ?? 0; @@ -145,6 +171,39 @@ return rows || ''; }; + const renderOpenIncidents = (items) => { + const rows = (items || []).map((item) => { + const serviceName = item.service_name || item.service_id || ""; + const duration = formatDuration(item.duration_seconds) || "probíhá"; + return ` + + + + + + + `; + }).join(""); + return rows || ''; + }; + + const renderIncidents = (items) => { + const rows = (items || []).map((item) => { + const serviceName = item.service_name || item.service_id || ""; + return ` + + + + + + + + + `; + }).join(""); + return rows || ''; + }; + const renderSnapshot = (snapshot) => { setText("[data-live-count='jobs.queued']", snapshot.jobs?.queued); setText("[data-live-count='jobs.running']", snapshot.jobs?.running); @@ -155,6 +214,7 @@ setText("[data-live-count='health.healthy']", snapshot.health?.healthy); setText("[data-live-count='health.unhealthy']", snapshot.health?.unhealthy); setText("[data-live-count='health.unreachable']", snapshot.health?.unreachable); + setText("[data-live-count='incidents.open']", snapshot.incidents?.open); setText("[data-live-count='workers.online']", snapshot.workers?.online); setText("[data-live-count='workers.offline']", snapshot.workers?.offline); setText("[data-live-count='workers.total']", (snapshot.workers?.online || 0) + (snapshot.workers?.offline || 0)); @@ -170,6 +230,8 @@ document.querySelectorAll("[data-live-table='deployments.failed']").forEach((el) => { el.innerHTML = renderDeployments(snapshot.deployments?.recent_failed); }); document.querySelectorAll("[data-live-table='audit.recent']").forEach((el) => { el.innerHTML = renderAudit(snapshot.audit?.recent); }); document.querySelectorAll("[data-live-table='health.problems']").forEach((el) => { el.innerHTML = renderHealthProblems(snapshot.health?.problems); }); + document.querySelectorAll("[data-live-table='incidents.active']").forEach((el) => { el.innerHTML = renderOpenIncidents(snapshot.incidents?.active); }); + document.querySelectorAll("[data-live-table='incidents.recent']").forEach((el) => { el.innerHTML = renderIncidents(snapshot.incidents?.recent); }); (snapshot.workers?.recent || []).forEach((worker) => { const row = Array.from(document.querySelectorAll("tr[data-worker-id]")) diff --git a/app/templates/layout.py b/app/templates/layout.py index 7703910..259dad3 100644 --- a/app/templates/layout.py +++ b/app/templates/layout.py @@ -21,6 +21,7 @@ def page(title: str, body: str, user=None) -> str:
Žádné služby nevyžadují pozornost.
${escapeHtml(serviceName)}${escapeHtml(item.title)}${escapeHtml(item.started_at)}${escapeHtml(duration)}
Žádné aktivní incidenty.
${escapeHtml(serviceName)}${escapeHtml(item.title)}${escapeHtml(item.started_at)}${escapeHtml(item.ended_at)}${escapeHtml(formatDuration(item.duration_seconds))}${incidentBadge(item.status)}
Zatím nejsou evidované žádné incidenty.