Add incident creation and resolution
This commit is contained in:
AppFactory Bot
+187
-10
@@ -37,9 +37,9 @@ def get_app_services():
|
||||
|
||||
rows = con.execute(
|
||||
"""
|
||||
SELECT id
|
||||
SELECT id, health_url
|
||||
FROM apps
|
||||
WHERE status IS NOT NULL
|
||||
WHERE is_enabled = 1
|
||||
ORDER BY id
|
||||
"""
|
||||
).fetchall()
|
||||
@@ -50,7 +50,12 @@ def get_app_services():
|
||||
|
||||
for row in rows:
|
||||
app_id = row["id"]
|
||||
services[app_id] = f"http://appfactory-caddy/apps/{app_id}/health"
|
||||
health_url = row["health_url"] or "/health"
|
||||
|
||||
if health_url.startswith("http://") or health_url.startswith("https://"):
|
||||
services[app_id] = health_url
|
||||
else:
|
||||
services[app_id] = f"http://appfactory-caddy/apps/{app_id}{health_url}"
|
||||
|
||||
return services
|
||||
|
||||
@@ -76,10 +81,10 @@ def save_health_result(
|
||||
http_status: int | None,
|
||||
response_time_ms: int | None,
|
||||
error_text: str | None,
|
||||
):
|
||||
) -> int:
|
||||
con = get_connection()
|
||||
|
||||
con.execute(
|
||||
cur = con.execute(
|
||||
"""
|
||||
INSERT INTO service_health (
|
||||
service_id,
|
||||
@@ -100,9 +105,157 @@ def save_health_result(
|
||||
),
|
||||
)
|
||||
|
||||
health_id = cur.lastrowid
|
||||
|
||||
con.commit()
|
||||
con.close()
|
||||
|
||||
return int(health_id)
|
||||
|
||||
|
||||
def get_open_incident(service_id: str):
|
||||
con = get_connection()
|
||||
|
||||
row = con.execute(
|
||||
"""
|
||||
SELECT *
|
||||
FROM service_incidents
|
||||
WHERE service_id = ?
|
||||
AND status = 'open'
|
||||
ORDER BY id DESC
|
||||
LIMIT 1
|
||||
""",
|
||||
(service_id,),
|
||||
).fetchone()
|
||||
|
||||
con.close()
|
||||
|
||||
return row
|
||||
|
||||
|
||||
def open_incident(service_id: str, health_id: int, status: str, error_text: str | None):
|
||||
if get_open_incident(service_id):
|
||||
return
|
||||
|
||||
title = f"Služba {service_id} není zdravá"
|
||||
description = error_text or f"Health status: {status}"
|
||||
|
||||
con = get_connection()
|
||||
|
||||
con.execute(
|
||||
"""
|
||||
INSERT INTO service_incidents (
|
||||
service_id,
|
||||
status,
|
||||
started_at,
|
||||
start_health_id,
|
||||
title,
|
||||
description
|
||||
)
|
||||
VALUES (?, 'open', CURRENT_TIMESTAMP, ?, ?, ?)
|
||||
""",
|
||||
(
|
||||
service_id,
|
||||
health_id,
|
||||
title,
|
||||
description,
|
||||
),
|
||||
)
|
||||
|
||||
con.execute(
|
||||
"""
|
||||
INSERT INTO audit_events (
|
||||
username,
|
||||
action,
|
||||
target_type,
|
||||
target_id,
|
||||
source,
|
||||
metadata,
|
||||
created_at
|
||||
)
|
||||
VALUES (
|
||||
'system',
|
||||
'service.incident.opened',
|
||||
'service',
|
||||
?,
|
||||
'monitor',
|
||||
?,
|
||||
CURRENT_TIMESTAMP
|
||||
)
|
||||
""",
|
||||
(
|
||||
service_id,
|
||||
f'{{"health_id": {health_id}, "status": "{status}"}}',
|
||||
),
|
||||
)
|
||||
|
||||
con.commit()
|
||||
con.close()
|
||||
|
||||
|
||||
def resolve_incident(service_id: str, health_id: int):
|
||||
incident = get_open_incident(service_id)
|
||||
|
||||
if not incident:
|
||||
return
|
||||
|
||||
con = get_connection()
|
||||
|
||||
con.execute(
|
||||
"""
|
||||
UPDATE service_incidents
|
||||
SET status = 'resolved',
|
||||
ended_at = CURRENT_TIMESTAMP,
|
||||
end_health_id = ?,
|
||||
duration_seconds = CAST(
|
||||
(julianday(CURRENT_TIMESTAMP) - julianday(started_at)) * 86400
|
||||
AS INTEGER
|
||||
)
|
||||
WHERE id = ?
|
||||
""",
|
||||
(
|
||||
health_id,
|
||||
incident["id"],
|
||||
),
|
||||
)
|
||||
|
||||
con.execute(
|
||||
"""
|
||||
INSERT INTO audit_events (
|
||||
username,
|
||||
action,
|
||||
target_type,
|
||||
target_id,
|
||||
source,
|
||||
metadata,
|
||||
created_at
|
||||
)
|
||||
VALUES (
|
||||
'system',
|
||||
'service.incident.resolved',
|
||||
'service',
|
||||
?,
|
||||
'monitor',
|
||||
?,
|
||||
CURRENT_TIMESTAMP
|
||||
)
|
||||
""",
|
||||
(
|
||||
service_id,
|
||||
f'{{"incident_id": {incident["id"]}, "health_id": {health_id}}}',
|
||||
),
|
||||
)
|
||||
|
||||
con.commit()
|
||||
con.close()
|
||||
|
||||
|
||||
def update_incident_state(service_id: str, health_id: int, status: str, error_text: str | None):
|
||||
if status == "healthy":
|
||||
resolve_incident(service_id, health_id)
|
||||
else:
|
||||
open_incident(service_id, health_id, status, error_text)
|
||||
|
||||
|
||||
def check_service(service_id: str, url: str):
|
||||
start = time.monotonic()
|
||||
@@ -116,31 +269,55 @@ def check_service(service_id: str, url: str):
|
||||
elapsed_ms = int((time.monotonic() - start) * 1000)
|
||||
|
||||
if 200 <= response.status_code < 300:
|
||||
save_health_result(
|
||||
health_id = save_health_result(
|
||||
service_id=service_id,
|
||||
status="healthy",
|
||||
http_status=response.status_code,
|
||||
response_time_ms=elapsed_ms,
|
||||
error_text=None,
|
||||
)
|
||||
|
||||
update_incident_state(
|
||||
service_id=service_id,
|
||||
health_id=health_id,
|
||||
status="healthy",
|
||||
error_text=None,
|
||||
)
|
||||
else:
|
||||
save_health_result(
|
||||
error_text = response.text[:500]
|
||||
|
||||
health_id = save_health_result(
|
||||
service_id=service_id,
|
||||
status="unhealthy",
|
||||
http_status=response.status_code,
|
||||
response_time_ms=elapsed_ms,
|
||||
error_text=response.text[:500],
|
||||
error_text=error_text,
|
||||
)
|
||||
|
||||
update_incident_state(
|
||||
service_id=service_id,
|
||||
health_id=health_id,
|
||||
status="unhealthy",
|
||||
error_text=error_text,
|
||||
)
|
||||
|
||||
except Exception as exc:
|
||||
elapsed_ms = int((time.monotonic() - start) * 1000)
|
||||
error_text = str(exc)
|
||||
|
||||
save_health_result(
|
||||
health_id = save_health_result(
|
||||
service_id=service_id,
|
||||
status="unreachable",
|
||||
http_status=None,
|
||||
response_time_ms=elapsed_ms,
|
||||
error_text=str(exc),
|
||||
error_text=error_text,
|
||||
)
|
||||
|
||||
update_incident_state(
|
||||
service_id=service_id,
|
||||
health_id=health_id,
|
||||
status="unreachable",
|
||||
error_text=error_text,
|
||||
)
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user