Add incident creation and resolution

app/main.py

@@ -37,9 +37,9 @@ def get_app_services():
 
     rows = con.execute(
         """
-        SELECT id
+        SELECT id, health_url
         FROM apps
-        WHERE status IS NOT NULL
+        WHERE is_enabled = 1
         ORDER BY id
         """
     ).fetchall()
@@ -50,7 +50,12 @@ def get_app_services():
 
     for row in rows:
         app_id = row["id"]
-        services[app_id] = f"http://appfactory-caddy/apps/{app_id}/health"
+        health_url = row["health_url"] or "/health"
+
+        if health_url.startswith("http://") or health_url.startswith("https://"):
+            services[app_id] = health_url
+        else:
+            services[app_id] = f"http://appfactory-caddy/apps/{app_id}{health_url}"
 
     return services
 
@@ -76,10 +81,10 @@ def save_health_result(
     http_status: int | None,
     response_time_ms: int | None,
     error_text: str | None,
-):
+) -> int:
     con = get_connection()
 
-    con.execute(
+    cur = con.execute(
         """
         INSERT INTO service_health (
             service_id,
@@ -100,9 +105,157 @@ def save_health_result(
         ),
     )
 
+    health_id = cur.lastrowid
+
     con.commit()
     con.close()
 
+    return int(health_id)
+
+
+def get_open_incident(service_id: str):
+    con = get_connection()
+
+    row = con.execute(
+        """
+        SELECT *
+        FROM service_incidents
+        WHERE service_id = ?
+          AND status = 'open'
+        ORDER BY id DESC
+        LIMIT 1
+        """,
+        (service_id,),
+    ).fetchone()
+
+    con.close()
+
+    return row
+
+
+def open_incident(service_id: str, health_id: int, status: str, error_text: str | None):
+    if get_open_incident(service_id):
+        return
+
+    title = f"Služba {service_id} není zdravá"
+    description = error_text or f"Health status: {status}"
+
+    con = get_connection()
+
+    con.execute(
+        """
+        INSERT INTO service_incidents (
+            service_id,
+            status,
+            started_at,
+            start_health_id,
+            title,
+            description
+        )
+        VALUES (?, 'open', CURRENT_TIMESTAMP, ?, ?, ?)
+        """,
+        (
+            service_id,
+            health_id,
+            title,
+            description,
+        ),
+    )
+
+    con.execute(
+        """
+        INSERT INTO audit_events (
+            username,
+            action,
+            target_type,
+            target_id,
+            source,
+            metadata,
+            created_at
+        )
+        VALUES (
+            'system',
+            'service.incident.opened',
+            'service',
+            ?,
+            'monitor',
+            ?,
+            CURRENT_TIMESTAMP
+        )
+        """,
+        (
+            service_id,
+            f'{{"health_id": {health_id}, "status": "{status}"}}',
+        ),
+    )
+
+    con.commit()
+    con.close()
+
+
+def resolve_incident(service_id: str, health_id: int):
+    incident = get_open_incident(service_id)
+
+    if not incident:
+        return
+
+    con = get_connection()
+
+    con.execute(
+        """
+        UPDATE service_incidents
+        SET status = 'resolved',
+            ended_at = CURRENT_TIMESTAMP,
+            end_health_id = ?,
+            duration_seconds = CAST(
+                (julianday(CURRENT_TIMESTAMP) - julianday(started_at)) * 86400
+                AS INTEGER
+            )
+        WHERE id = ?
+        """,
+        (
+            health_id,
+            incident["id"],
+        ),
+    )
+
+    con.execute(
+        """
+        INSERT INTO audit_events (
+            username,
+            action,
+            target_type,
+            target_id,
+            source,
+            metadata,
+            created_at
+        )
+        VALUES (
+            'system',
+            'service.incident.resolved',
+            'service',
+            ?,
+            'monitor',
+            ?,
+            CURRENT_TIMESTAMP
+        )
+        """,
+        (
+            service_id,
+            f'{{"incident_id": {incident["id"]}, "health_id": {health_id}}}',
+        ),
+    )
+
+    con.commit()
+    con.close()
+
+
+def update_incident_state(service_id: str, health_id: int, status: str, error_text: str | None):
+    if status == "healthy":
+        resolve_incident(service_id, health_id)
+    else:
+        open_incident(service_id, health_id, status, error_text)
+
 
 def check_service(service_id: str, url: str):
     start = time.monotonic()
@@ -116,31 +269,55 @@ def check_service(service_id: str, url: str):
         elapsed_ms = int((time.monotonic() - start) * 1000)
 
         if 200 <= response.status_code < 300:
-            save_health_result(
+            health_id = save_health_result(
                 service_id=service_id,
                 status="healthy",
                 http_status=response.status_code,
                 response_time_ms=elapsed_ms,
                 error_text=None,
             )
+
+            update_incident_state(
+                service_id=service_id,
+                health_id=health_id,
+                status="healthy",
+                error_text=None,
+            )
         else:
-            save_health_result(
+            error_text = response.text[:500]
+
+            health_id = save_health_result(
                 service_id=service_id,
                 status="unhealthy",
                 http_status=response.status_code,
                 response_time_ms=elapsed_ms,
-                error_text=response.text[:500],
+                error_text=error_text,
+            )
+
+            update_incident_state(
+                service_id=service_id,
+                health_id=health_id,
+                status="unhealthy",
+                error_text=error_text,
             )
 
     except Exception as exc:
         elapsed_ms = int((time.monotonic() - start) * 1000)
+        error_text = str(exc)
 
-        save_health_result(
+        health_id = save_health_result(
             service_id=service_id,
             status="unreachable",
             http_status=None,
             response_time_ms=elapsed_ms,
-            error_text=str(exc),
+            error_text=error_text,
+        )
+
+        update_incident_state(
+            service_id=service_id,
+            health_id=health_id,
+            status="unreachable",
+            error_text=error_text,
         )