AppFactory Bot
379 lines
8.3 KiB
Python
379 lines
8.3 KiB
Python
import os
|
|
import sqlite3
|
|
import threading
|
|
import time
|
|
from pathlib import Path
|
|
|
|
import requests
|
|
from fastapi import FastAPI
|
|
|
|
DB_FILE = Path("/opt/appfactory/data/appfactory/appfactory.db")
|
|
|
|
CHECK_INTERVAL_SECONDS = int(os.getenv("APPFACTORY_MONITOR_INTERVAL_SECONDS", "30"))
|
|
REQUEST_TIMEOUT_SECONDS = int(os.getenv("APPFACTORY_MONITOR_TIMEOUT_SECONDS", "5"))
|
|
HEALTH_RETENTION_DAYS = int(os.getenv("APPFACTORY_HEALTH_RETENTION_DAYS", "7"))
|
|
|
|
CORE_SERVICES = {
|
|
"appfactory-portal": "http://appfactory-portal:9100/health",
|
|
"appfactory-webhook": "http://appfactory-webhook:9000/health",
|
|
"appfactory-worker": "http://appfactory-worker:9200/health",
|
|
"appfactory-monitor": "http://appfactory-monitor:9300/health",
|
|
}
|
|
|
|
app = FastAPI(title="AppFactory Monitor")
|
|
|
|
_monitor_thread = None
|
|
_monitor_started = False
|
|
|
|
|
|
def get_connection():
|
|
con = sqlite3.connect(DB_FILE, timeout=30)
|
|
con.row_factory = sqlite3.Row
|
|
return con
|
|
|
|
|
|
def get_app_services():
|
|
con = get_connection()
|
|
|
|
rows = con.execute(
|
|
"""
|
|
SELECT id, health_url
|
|
FROM apps
|
|
WHERE is_enabled = 1
|
|
ORDER BY id
|
|
"""
|
|
).fetchall()
|
|
|
|
con.close()
|
|
|
|
services = {}
|
|
|
|
for row in rows:
|
|
app_id = row["id"]
|
|
health_url = row["health_url"] or "/health"
|
|
|
|
if health_url.startswith("http://") or health_url.startswith("https://"):
|
|
services[app_id] = health_url
|
|
else:
|
|
services[app_id] = f"http://appfactory-caddy/apps/{app_id}{health_url}"
|
|
|
|
return services
|
|
|
|
|
|
def cleanup_old_health_records():
|
|
con = get_connection()
|
|
|
|
con.execute(
|
|
"""
|
|
DELETE FROM service_health
|
|
WHERE checked_at < datetime('now', ?)
|
|
""",
|
|
(f"-{HEALTH_RETENTION_DAYS} days",),
|
|
)
|
|
|
|
con.commit()
|
|
con.close()
|
|
|
|
|
|
def save_health_result(
|
|
service_id: str,
|
|
status: str,
|
|
http_status: int | None,
|
|
response_time_ms: int | None,
|
|
error_text: str | None,
|
|
) -> int:
|
|
con = get_connection()
|
|
|
|
cur = con.execute(
|
|
"""
|
|
INSERT INTO service_health (
|
|
service_id,
|
|
status,
|
|
http_status,
|
|
response_time_ms,
|
|
error_text,
|
|
checked_at
|
|
)
|
|
VALUES (?, ?, ?, ?, ?, CURRENT_TIMESTAMP)
|
|
""",
|
|
(
|
|
service_id,
|
|
status,
|
|
http_status,
|
|
response_time_ms,
|
|
error_text,
|
|
),
|
|
)
|
|
|
|
health_id = cur.lastrowid
|
|
|
|
con.commit()
|
|
con.close()
|
|
|
|
return int(health_id)
|
|
|
|
|
|
def get_open_incident(service_id: str):
|
|
con = get_connection()
|
|
|
|
row = con.execute(
|
|
"""
|
|
SELECT *
|
|
FROM service_incidents
|
|
WHERE service_id = ?
|
|
AND status = 'open'
|
|
ORDER BY id DESC
|
|
LIMIT 1
|
|
""",
|
|
(service_id,),
|
|
).fetchone()
|
|
|
|
con.close()
|
|
|
|
return row
|
|
|
|
|
|
def open_incident(service_id: str, health_id: int, status: str, error_text: str | None):
|
|
if get_open_incident(service_id):
|
|
return
|
|
|
|
title = f"Služba {service_id} není zdravá"
|
|
description = error_text or f"Health status: {status}"
|
|
|
|
con = get_connection()
|
|
|
|
con.execute(
|
|
"""
|
|
INSERT INTO service_incidents (
|
|
service_id,
|
|
status,
|
|
started_at,
|
|
start_health_id,
|
|
title,
|
|
description
|
|
)
|
|
VALUES (?, 'open', CURRENT_TIMESTAMP, ?, ?, ?)
|
|
""",
|
|
(
|
|
service_id,
|
|
health_id,
|
|
title,
|
|
description,
|
|
),
|
|
)
|
|
|
|
con.execute(
|
|
"""
|
|
INSERT INTO audit_events (
|
|
username,
|
|
action,
|
|
target_type,
|
|
target_id,
|
|
source,
|
|
metadata,
|
|
created_at
|
|
)
|
|
VALUES (
|
|
'system',
|
|
'service.incident.opened',
|
|
'service',
|
|
?,
|
|
'monitor',
|
|
?,
|
|
CURRENT_TIMESTAMP
|
|
)
|
|
""",
|
|
(
|
|
service_id,
|
|
f'{{"health_id": {health_id}, "status": "{status}"}}',
|
|
),
|
|
)
|
|
|
|
con.commit()
|
|
con.close()
|
|
|
|
|
|
def resolve_incident(service_id: str, health_id: int):
|
|
incident = get_open_incident(service_id)
|
|
|
|
if not incident:
|
|
return
|
|
|
|
con = get_connection()
|
|
|
|
con.execute(
|
|
"""
|
|
UPDATE service_incidents
|
|
SET status = 'resolved',
|
|
ended_at = CURRENT_TIMESTAMP,
|
|
end_health_id = ?,
|
|
duration_seconds = CAST(
|
|
(julianday(CURRENT_TIMESTAMP) - julianday(started_at)) * 86400
|
|
AS INTEGER
|
|
)
|
|
WHERE id = ?
|
|
""",
|
|
(
|
|
health_id,
|
|
incident["id"],
|
|
),
|
|
)
|
|
|
|
con.execute(
|
|
"""
|
|
INSERT INTO audit_events (
|
|
username,
|
|
action,
|
|
target_type,
|
|
target_id,
|
|
source,
|
|
metadata,
|
|
created_at
|
|
)
|
|
VALUES (
|
|
'system',
|
|
'service.incident.resolved',
|
|
'service',
|
|
?,
|
|
'monitor',
|
|
?,
|
|
CURRENT_TIMESTAMP
|
|
)
|
|
""",
|
|
(
|
|
service_id,
|
|
f'{{"incident_id": {incident["id"]}, "health_id": {health_id}}}',
|
|
),
|
|
)
|
|
|
|
con.commit()
|
|
con.close()
|
|
|
|
|
|
def update_incident_state(service_id: str, health_id: int, status: str, error_text: str | None):
|
|
if status == "healthy":
|
|
resolve_incident(service_id, health_id)
|
|
else:
|
|
open_incident(service_id, health_id, status, error_text)
|
|
|
|
|
|
def check_service(service_id: str, url: str):
|
|
start = time.monotonic()
|
|
|
|
try:
|
|
response = requests.get(
|
|
url,
|
|
timeout=REQUEST_TIMEOUT_SECONDS,
|
|
)
|
|
|
|
elapsed_ms = int((time.monotonic() - start) * 1000)
|
|
|
|
if 200 <= response.status_code < 300:
|
|
health_id = save_health_result(
|
|
service_id=service_id,
|
|
status="healthy",
|
|
http_status=response.status_code,
|
|
response_time_ms=elapsed_ms,
|
|
error_text=None,
|
|
)
|
|
|
|
update_incident_state(
|
|
service_id=service_id,
|
|
health_id=health_id,
|
|
status="healthy",
|
|
error_text=None,
|
|
)
|
|
else:
|
|
error_text = response.text[:500]
|
|
|
|
health_id = save_health_result(
|
|
service_id=service_id,
|
|
status="unhealthy",
|
|
http_status=response.status_code,
|
|
response_time_ms=elapsed_ms,
|
|
error_text=error_text,
|
|
)
|
|
|
|
update_incident_state(
|
|
service_id=service_id,
|
|
health_id=health_id,
|
|
status="unhealthy",
|
|
error_text=error_text,
|
|
)
|
|
|
|
except Exception as exc:
|
|
elapsed_ms = int((time.monotonic() - start) * 1000)
|
|
error_text = str(exc)
|
|
|
|
health_id = save_health_result(
|
|
service_id=service_id,
|
|
status="unreachable",
|
|
http_status=None,
|
|
response_time_ms=elapsed_ms,
|
|
error_text=error_text,
|
|
)
|
|
|
|
update_incident_state(
|
|
service_id=service_id,
|
|
health_id=health_id,
|
|
status="unreachable",
|
|
error_text=error_text,
|
|
)
|
|
|
|
|
|
def run_check_cycle():
|
|
cleanup_old_health_records()
|
|
|
|
services = {}
|
|
services.update(CORE_SERVICES)
|
|
services.update(get_app_services())
|
|
|
|
for service_id, url in services.items():
|
|
check_service(service_id, url)
|
|
|
|
|
|
def monitor_loop():
|
|
while True:
|
|
try:
|
|
run_check_cycle()
|
|
except Exception as exc:
|
|
print(f"Monitor loop error: {exc}", flush=True)
|
|
|
|
time.sleep(CHECK_INTERVAL_SECONDS)
|
|
|
|
|
|
@app.on_event("startup")
|
|
def start_monitor():
|
|
global _monitor_thread
|
|
global _monitor_started
|
|
|
|
if _monitor_started:
|
|
return
|
|
|
|
_monitor_started = True
|
|
|
|
_monitor_thread = threading.Thread(
|
|
target=monitor_loop,
|
|
daemon=True,
|
|
)
|
|
_monitor_thread.start()
|
|
|
|
|
|
@app.get("/health")
|
|
def health():
|
|
return {
|
|
"status": "ok",
|
|
"interval_seconds": CHECK_INTERVAL_SECONDS,
|
|
"timeout_seconds": REQUEST_TIMEOUT_SECONDS,
|
|
"retention_days": HEALTH_RETENTION_DAYS,
|
|
}
|
|
|
|
|
|
@app.get("/services")
|
|
def services():
|
|
result = {}
|
|
result.update(CORE_SERVICES)
|
|
result.update(get_app_services())
|
|
|
|
return result
|