diff --git a/app/main.py b/app/main.py index 6b06ca0..5e7e83d 100644 --- a/app/main.py +++ b/app/main.py @@ -37,9 +37,9 @@ def get_app_services(): rows = con.execute( """ - SELECT id + SELECT id, health_url FROM apps - WHERE status IS NOT NULL + WHERE is_enabled = 1 ORDER BY id """ ).fetchall() @@ -50,7 +50,12 @@ def get_app_services(): for row in rows: app_id = row["id"] - services[app_id] = f"http://appfactory-caddy/apps/{app_id}/health" + health_url = row["health_url"] or "/health" + + if health_url.startswith("http://") or health_url.startswith("https://"): + services[app_id] = health_url + else: + services[app_id] = f"http://appfactory-caddy/apps/{app_id}{health_url}" return services @@ -76,10 +81,10 @@ def save_health_result( http_status: int | None, response_time_ms: int | None, error_text: str | None, -): +) -> int: con = get_connection() - con.execute( + cur = con.execute( """ INSERT INTO service_health ( service_id, @@ -100,9 +105,157 @@ def save_health_result( ), ) + health_id = cur.lastrowid + con.commit() con.close() + return int(health_id) + + +def get_open_incident(service_id: str): + con = get_connection() + + row = con.execute( + """ + SELECT * + FROM service_incidents + WHERE service_id = ? + AND status = 'open' + ORDER BY id DESC + LIMIT 1 + """, + (service_id,), + ).fetchone() + + con.close() + + return row + + +def open_incident(service_id: str, health_id: int, status: str, error_text: str | None): + if get_open_incident(service_id): + return + + title = f"Služba {service_id} není zdravá" + description = error_text or f"Health status: {status}" + + con = get_connection() + + con.execute( + """ + INSERT INTO service_incidents ( + service_id, + status, + started_at, + start_health_id, + title, + description + ) + VALUES (?, 'open', CURRENT_TIMESTAMP, ?, ?, ?) + """, + ( + service_id, + health_id, + title, + description, + ), + ) + + con.execute( + """ + INSERT INTO audit_events ( + username, + action, + target_type, + target_id, + source, + metadata, + created_at + ) + VALUES ( + 'system', + 'service.incident.opened', + 'service', + ?, + 'monitor', + ?, + CURRENT_TIMESTAMP + ) + """, + ( + service_id, + f'{{"health_id": {health_id}, "status": "{status}"}}', + ), + ) + + con.commit() + con.close() + + +def resolve_incident(service_id: str, health_id: int): + incident = get_open_incident(service_id) + + if not incident: + return + + con = get_connection() + + con.execute( + """ + UPDATE service_incidents + SET status = 'resolved', + ended_at = CURRENT_TIMESTAMP, + end_health_id = ?, + duration_seconds = CAST( + (julianday(CURRENT_TIMESTAMP) - julianday(started_at)) * 86400 + AS INTEGER + ) + WHERE id = ? + """, + ( + health_id, + incident["id"], + ), + ) + + con.execute( + """ + INSERT INTO audit_events ( + username, + action, + target_type, + target_id, + source, + metadata, + created_at + ) + VALUES ( + 'system', + 'service.incident.resolved', + 'service', + ?, + 'monitor', + ?, + CURRENT_TIMESTAMP + ) + """, + ( + service_id, + f'{{"incident_id": {incident["id"]}, "health_id": {health_id}}}', + ), + ) + + con.commit() + con.close() + + +def update_incident_state(service_id: str, health_id: int, status: str, error_text: str | None): + if status == "healthy": + resolve_incident(service_id, health_id) + else: + open_incident(service_id, health_id, status, error_text) + def check_service(service_id: str, url: str): start = time.monotonic() @@ -116,31 +269,55 @@ def check_service(service_id: str, url: str): elapsed_ms = int((time.monotonic() - start) * 1000) if 200 <= response.status_code < 300: - save_health_result( + health_id = save_health_result( service_id=service_id, status="healthy", http_status=response.status_code, response_time_ms=elapsed_ms, error_text=None, ) + + update_incident_state( + service_id=service_id, + health_id=health_id, + status="healthy", + error_text=None, + ) else: - save_health_result( + error_text = response.text[:500] + + health_id = save_health_result( service_id=service_id, status="unhealthy", http_status=response.status_code, response_time_ms=elapsed_ms, - error_text=response.text[:500], + error_text=error_text, + ) + + update_incident_state( + service_id=service_id, + health_id=health_id, + status="unhealthy", + error_text=error_text, ) except Exception as exc: elapsed_ms = int((time.monotonic() - start) * 1000) + error_text = str(exc) - save_health_result( + health_id = save_health_result( service_id=service_id, status="unreachable", http_status=None, response_time_ms=elapsed_ms, - error_text=str(exc), + error_text=error_text, + ) + + update_incident_state( + service_id=service_id, + health_id=health_id, + status="unreachable", + error_text=error_text, )