diff --git a/app/db/health.py b/app/db/health.py new file mode 100644 index 0000000..45a9843 --- /dev/null +++ b/app/db/health.py @@ -0,0 +1,102 @@ +from app.db.database import get_connection + + +HEALTH_STATUSES = ("healthy", "unhealthy", "unreachable") + + +def get_latest_service_health(): + con = get_connection() + + try: + rows = con.execute( + """ + SELECT + h.service_id, + COALESCE(a.name, h.service_id) AS service_name, + h.status, + h.http_status, + h.response_time_ms, + h.error_text, + h.checked_at + FROM service_health h + LEFT JOIN apps a ON a.id = h.service_id + INNER JOIN ( + SELECT service_id, MAX(id) AS max_id + FROM service_health + GROUP BY service_id + ) latest ON latest.max_id = h.id + """ + ).fetchall() + except Exception: + rows = [] + + con.close() + return {row["service_id"]: dict(row) for row in rows} + + +def get_service_health(service_id: str): + con = get_connection() + + try: + row = con.execute( + """ + SELECT + h.service_id, + COALESCE(a.name, h.service_id) AS service_name, + h.status, + h.http_status, + h.response_time_ms, + h.error_text, + h.checked_at + FROM service_health h + LEFT JOIN apps a ON a.id = h.service_id + WHERE h.service_id = ? + ORDER BY h.id DESC + LIMIT 1 + """, + (service_id,), + ).fetchone() + except Exception: + row = None + + con.close() + return dict(row) if row else None + + +def get_service_health_history(service_id: str, limit: int = 50): + con = get_connection() + + try: + rows = con.execute( + """ + SELECT id, service_id, status, http_status, response_time_ms, error_text, checked_at + FROM service_health + WHERE service_id = ? + ORDER BY id DESC + LIMIT ? + """, + (service_id, limit), + ).fetchall() + except Exception: + rows = [] + + con.close() + return [dict(row) for row in rows] + + +def get_health_summary(): + latest = get_latest_service_health() + summary = {"healthy": 0, "unhealthy": 0, "unreachable": 0} + for item in latest.values(): + status = item.get("status") + if status in summary: + summary[status] += 1 + return summary + + +def get_problem_service_health(): + latest = get_latest_service_health() + return [ + item for item in latest.values() + if item.get("status") in {"unhealthy", "unreachable"} + ] diff --git a/app/db/operations.py b/app/db/operations.py index 323dc72..88ecc08 100644 --- a/app/db/operations.py +++ b/app/db/operations.py @@ -1,4 +1,5 @@ from app.db.database import get_connection +from app.db.health import get_health_summary, get_problem_service_health from app.db.migrations import run_migrations from app.db.workers import is_worker_online @@ -197,6 +198,8 @@ def get_operations_snapshot(): WHERE LOWER(COALESCE(status, '')) IN ('failed', 'error', 'disabled', 'deleted', 'archived') """ ).fetchone()["count"] + health_summary = get_health_summary() + health_problems = get_problem_service_health() jobs_summary = con.execute( """ @@ -296,6 +299,12 @@ def get_operations_snapshot(): "active": active_applications, "problematic": problematic_applications, }, + "health": { + "healthy": health_summary["healthy"], + "unhealthy": health_summary["unhealthy"], + "unreachable": health_summary["unreachable"], + "problems": health_problems, + }, "workers": { "online": workers_online, "offline": max(0, total_workers - workers_online), diff --git a/app/routes/apps.py b/app/routes/apps.py index d99af2f..ff3ae65 100644 --- a/app/routes/apps.py +++ b/app/routes/apps.py @@ -17,6 +17,7 @@ from ..config import ( ) from ..db.apps import get_app, get_app_deployments, get_apps, update_app_resources from ..db.audit import log_audit_event +from ..db.health import get_latest_service_health, get_service_health, get_service_health_history from ..db.jobs import create_job, get_jobs, has_active_deploy_job from ..routes.deployments import render_status_pill from ..shell import run_command @@ -26,6 +27,24 @@ router = APIRouter() DEFAULT_PAGE_SIZE = 20 +def render_health_status(status: str | None) -> str: + value = status or "" + normalized = value.lower() + labels = { + "healthy": "zdravá", + "unhealthy": "nezdravá", + "unreachable": "nedostupná", + } + class_name = "pill pill-muted" + if normalized == "healthy": + class_name = "pill pill-success" + elif normalized == "unhealthy": + class_name = "pill pill-warning" + elif normalized == "unreachable": + class_name = "pill pill-danger" + return f'{html.escape(labels.get(normalized, value or "neznámá"))}' + + @router.get("/") def portal_home(user=Depends(require_user)): return RedirectResponse(url="/portal/operations", status_code=303) @@ -40,6 +59,7 @@ def apps_page( user=Depends(require_user), ): apps = get_apps() + latest_health = get_latest_service_health() query = q.strip() selected_status = status.strip() if query: @@ -50,6 +70,16 @@ def apps_page( ] if selected_status: apps = [item for item in apps if (item.get("status", "") or "") == selected_status] + sort = request.query_params.get("sort", "").strip() + if sort == "health": + order = {"unreachable": 0, "unhealthy": 1, "healthy": 2} + apps = sorted( + apps, + key=lambda item: ( + order.get((latest_health.get(item.get("id", "")) or {}).get("status"), 3), + item.get("id", ""), + ), + ) status_values = sorted({item.get("status", "") for item in get_apps() if item.get("status")}) total_apps = len(apps) total_pages = max(1, math.ceil(total_apps / DEFAULT_PAGE_SIZE)) @@ -68,6 +98,9 @@ def apps_page( app_id = html.escape(item.get("id", "")) app_url_id = quote(item.get("id", ""), safe="") status = html.escape(item.get("status", "")) + health = latest_health.get(item.get("id", "")) or {} + health_status = render_health_status(health.get("status")) + health_checked_at = html.escape(health.get("checked_at", "") or "") docs = html.escape(item.get("docs", f"/apps/{app_id}/docs")) memory = item.get("memory", "") cpus = item.get("cpus", "") @@ -104,6 +137,7 @@ def apps_page( /apps/{app_id} {status} + {health_status}
{health_checked_at} Swagger
@@ -150,7 +184,7 @@ def apps_page( """ if not rows: - rows = 'Zatím nejsou nasazené žádné služby.' + rows = 'Zatím nejsou nasazené žádné služby.' status_options = [''] for value in status_values: selected = " selected" if selected_status == value else "" @@ -165,6 +199,8 @@ def apps_page( params["q"] = query if selected_status: params["status"] = selected_status + if sort: + params["sort"] = sort return f"/portal/apps?{urlencode(params)}" previous_link = ( @@ -216,6 +252,10 @@ def apps_page( + Reset @@ -225,6 +265,7 @@ def apps_page( Služba Status + Zdraví Dokumentace Prostředky Git @@ -245,6 +286,13 @@ def app_detail(app_id: str, request: Request, user=Depends(require_user)): if not app: raise HTTPException(status_code=404, detail="App not found") + log_audit_event( + user, + action="service.health.view", + target_type="service", + target_id=app_id, + ) + escaped_app_id = html.escape(app.get("id", "")) app_url_id = quote(app.get("id", ""), safe="") name = html.escape(app.get("name", "") or "") @@ -254,6 +302,8 @@ def app_detail(app_id: str, request: Request, user=Depends(require_user)): memory = html.escape(app.get("memory", "") or "") cpus = html.escape(app.get("cpus", "") or "") updated_at = html.escape(app.get("updated_at", "") or "") + current_health = get_service_health(app.get("id", "")) + health_history = get_service_health_history(app.get("id", ""), limit=50) rows = "" for deployment in get_app_deployments(app.get("id", ""), limit=10): @@ -294,8 +344,29 @@ def app_detail(app_id: str, request: Request, user=Depends(require_user)): if not job_rows: job_rows = 'Zatím nejsou evidované žádné úlohy této služby.' + health_status = render_health_status(current_health.get("status") if current_health else None) + health_http_status = html.escape(str(current_health.get("http_status") or "")) if current_health else "" + health_response_time = html.escape(str(current_health.get("response_time_ms") or "")) if current_health else "" + health_checked_at = html.escape(current_health.get("checked_at", "") or "") if current_health else "" + health_rows = "" + for item in health_history: + error_text = item.get("error_text", "") or "" + error_preview = error_text if len(error_text) <= 140 else f"{error_text[:137]}..." + health_rows += f""" + + {html.escape(item.get("checked_at", "") or "")} + {render_health_status(item.get("status"))} + {html.escape(str(item.get("http_status") or ""))} + {html.escape(str(item.get("response_time_ms") or ""))} + {html.escape(error_preview)} + + """ + + if not health_rows: + health_rows = 'Zatím nejsou evidované žádné kontroly zdraví.' + return page( - f"Služba {escaped_app_id}", + name or escaped_app_id, f"""

{escaped_app_id}

@@ -325,6 +396,30 @@ def app_detail(app_id: str, request: Request, user=Depends(require_user)):
+
+

Zdraví služby

+ + + + + +
Aktuální status{health_status}
HTTP status{health_http_status}
Odezva{health_response_time}
Poslední kontrola{health_checked_at}
+
+ +
+

Historie kontrol

+ + + + + + + + + {health_rows} +
ČasStatusHTTP statusOdezvaChyba
+
+

Historie nasazení

diff --git a/app/routes/operations.py b/app/routes/operations.py index 9e6e4e8..583bff7 100644 --- a/app/routes/operations.py +++ b/app/routes/operations.py @@ -110,6 +110,46 @@ def render_recent_audit_events(events: list[dict]) -> str: return rows +def render_health_status(status: str | None) -> str: + value = status or "" + normalized = value.lower() + labels = { + "healthy": "zdravá", + "unhealthy": "nezdravá", + "unreachable": "nedostupná", + } + class_name = "pill pill-muted" + if normalized == "healthy": + class_name = "pill pill-success" + elif normalized == "unhealthy": + class_name = "pill pill-warning" + elif normalized == "unreachable": + class_name = "pill pill-danger" + return f'{html.escape(labels.get(normalized, value))}' + + +def render_health_problems(items: list[dict]) -> str: + rows = "" + for item in items: + service_id = html.escape(item.get("service_id", "") or "") + service_name = html.escape(item.get("service_name") or item.get("service_id", "") or "") + error_text = item.get("error_text", "") or "" + error_preview = error_text if len(error_text) <= 120 else f"{error_text[:117]}..." + rows += f""" + + + + + + + + """ + + if not rows: + rows = '' + return rows + + @router.get("/operations", response_class=HTMLResponse) def operations_dashboard(request: Request, user=Depends(require_user)): snapshot = get_operations_snapshot() @@ -145,12 +185,29 @@ def operations_dashboard(request: Request, user=Depends(require_user)):
Běžící služby{snapshot["apps"]["active"]}
Problémové služby{snapshot["apps"]["problematic"]}
+
Zdravé služby{snapshot["health"]["healthy"]}
+
Nezdravé služby{snapshot["health"]["unhealthy"]}
+
Nedostupné služby{snapshot["health"]["unreachable"]}
Aktivní úlohy{snapshot["jobs"]["running"]}
Nasazení dnes{snapshot["deployments"]["today"]}
Online workery{snapshot["workers"]["online"]}
Selhané úlohy (24 h){snapshot["jobs"]["failed_24h"]}
+
+

Služby vyžadující pozornost

+
{service_name}{render_health_status(item.get("status"))}{html.escape(item.get("checked_at", "") or "")}{html.escape(str(item.get("response_time_ms") or ""))}{html.escape(error_preview)}
Žádné služby nevyžadují pozornost.
+ + + + + + + + {render_health_problems(snapshot["health"]["problems"])} +
SlužbaStatusPoslední kontrolaOdezvaChyba
+
+

Poslední incidenty

diff --git a/app/static/operations-live.js b/app/static/operations-live.js index 6a2f5b3..a25e376 100644 --- a/app/static/operations-live.js +++ b/app/static/operations-live.js @@ -44,6 +44,21 @@ ? 'online' : 'offline'; + const healthBadge = (status) => { + const value = String(status || ""); + const normalized = value.toLowerCase(); + const labels = { + healthy: "zdravá", + unhealthy: "nezdravá", + unreachable: "nedostupná", + }; + let className = "pill pill-muted"; + if (normalized === "healthy") className = "pill pill-success"; + if (normalized === "unhealthy") className = "pill pill-warning"; + if (normalized === "unreachable") className = "pill pill-danger"; + return `${escapeHtml(labels[normalized] || value)}`; + }; + const setText = (selector, value) => { document.querySelectorAll(selector).forEach((el) => { el.textContent = value ?? 0; @@ -112,6 +127,24 @@ return rows || ''; }; + const renderHealthProblems = (items) => { + const rows = (items || []).map((item) => { + const errorText = String(item.error_text || ""); + const errorPreview = errorText.length > 120 ? `${errorText.slice(0, 117)}...` : errorText; + const serviceName = item.service_name || item.service_id || ""; + return ` + + + + + + + + `; + }).join(""); + return rows || ''; + }; + const renderSnapshot = (snapshot) => { setText("[data-live-count='jobs.queued']", snapshot.jobs?.queued); setText("[data-live-count='jobs.running']", snapshot.jobs?.running); @@ -119,6 +152,9 @@ setText("[data-live-count='jobs.success_24h']", snapshot.jobs?.success_24h); setText("[data-live-count='apps.active']", snapshot.apps?.active); setText("[data-live-count='apps.problematic']", snapshot.apps?.problematic); + setText("[data-live-count='health.healthy']", snapshot.health?.healthy); + setText("[data-live-count='health.unhealthy']", snapshot.health?.unhealthy); + setText("[data-live-count='health.unreachable']", snapshot.health?.unreachable); setText("[data-live-count='workers.online']", snapshot.workers?.online); setText("[data-live-count='workers.offline']", snapshot.workers?.offline); setText("[data-live-count='workers.total']", (snapshot.workers?.online || 0) + (snapshot.workers?.offline || 0)); @@ -133,6 +169,7 @@ document.querySelectorAll("[data-live-table='deployments.recent']").forEach((el) => { el.innerHTML = renderDeployments(snapshot.deployments?.recent); }); document.querySelectorAll("[data-live-table='deployments.failed']").forEach((el) => { el.innerHTML = renderDeployments(snapshot.deployments?.recent_failed); }); document.querySelectorAll("[data-live-table='audit.recent']").forEach((el) => { el.innerHTML = renderAudit(snapshot.audit?.recent); }); + document.querySelectorAll("[data-live-table='health.problems']").forEach((el) => { el.innerHTML = renderHealthProblems(snapshot.health?.problems); }); (snapshot.workers?.recent || []).forEach((worker) => { const row = Array.from(document.querySelectorAll("tr[data-worker-id]")) diff --git a/app/templates/layout.py b/app/templates/layout.py index 1c44b0e..7703910 100644 --- a/app/templates/layout.py +++ b/app/templates/layout.py @@ -46,7 +46,7 @@ def page(title: str, body: str, user=None) -> str: return f""" - {html.escape(title)} | {APP_NAME} + {APP_NAME} - {html.escape(title)}
Zatím nejsou evidované žádné auditní události.
${escapeHtml(serviceName)}${healthBadge(item.status)}${escapeHtml(item.checked_at)}${escapeHtml(item.response_time_ms ?? "")}${escapeHtml(errorPreview)}
Žádné služby nevyžadují pozornost.