Add service health monitoring

app/main.py

@@ -1,9 +1,182 @@
+import os
+import sqlite3
+import threading
+import time
+from pathlib import Path
+
+import requests
 from fastapi import FastAPI
 
+DB_FILE = Path("/opt/appfactory/data/appfactory/appfactory.db")
+
+CHECK_INTERVAL_SECONDS = int(os.getenv("APPFACTORY_MONITOR_INTERVAL_SECONDS", "30"))
+REQUEST_TIMEOUT_SECONDS = int(os.getenv("APPFACTORY_MONITOR_TIMEOUT_SECONDS", "5"))
+
+CORE_SERVICES = {
+    "appfactory-portal": "http://appfactory-portal:9100/health",
+    "appfactory-webhook": "http://appfactory-webhook:9000/health",
+    "appfactory-worker": "http://appfactory-worker:9200/health",
+    "appfactory-monitor": "http://appfactory-monitor:9300/health",
+}
+
 app = FastAPI(title="AppFactory Monitor")
 
+_monitor_thread = None
+_monitor_started = False
+
+
+def get_connection():
+    con = sqlite3.connect(DB_FILE, timeout=30)
+    con.row_factory = sqlite3.Row
+    return con
+
+
+def get_app_services():
+    con = get_connection()
+
+    rows = con.execute(
+        """
+        SELECT id
+        FROM apps
+        WHERE status IS NOT NULL
+        ORDER BY id
+        """
+    ).fetchall()
+
+    con.close()
+
+    services = {}
+
+    for row in rows:
+        app_id = row["id"]
+        services[app_id] = f"http://appfactory-caddy/apps/{app_id}/health"
+
+    return services
+
+
+def save_health_result(
+    service_id: str,
+    status: str,
+    http_status: int | None,
+    response_time_ms: int | None,
+    error_text: str | None,
+):
+    con = get_connection()
+
+    con.execute(
+        """
+        INSERT INTO service_health (
+            service_id,
+            status,
+            http_status,
+            response_time_ms,
+            error_text,
+            checked_at
+        )
+        VALUES (?, ?, ?, ?, ?, CURRENT_TIMESTAMP)
+        """,
+        (
+            service_id,
+            status,
+            http_status,
+            response_time_ms,
+            error_text,
+        ),
+    )
+
+    con.commit()
+    con.close()
+
+
+def check_service(service_id: str, url: str):
+    start = time.monotonic()
+
+    try:
+        response = requests.get(
+            url,
+            timeout=REQUEST_TIMEOUT_SECONDS,
+        )
+
+        elapsed_ms = int((time.monotonic() - start) * 1000)
+
+        if 200 <= response.status_code < 300:
+            save_health_result(
+                service_id=service_id,
+                status="healthy",
+                http_status=response.status_code,
+                response_time_ms=elapsed_ms,
+                error_text=None,
+            )
+        else:
+            save_health_result(
+                service_id=service_id,
+                status="unhealthy",
+                http_status=response.status_code,
+                response_time_ms=elapsed_ms,
+                error_text=response.text[:500],
+            )
+
+    except Exception as exc:
+        elapsed_ms = int((time.monotonic() - start) * 1000)
+
+        save_health_result(
+            service_id=service_id,
+            status="unreachable",
+            http_status=None,
+            response_time_ms=elapsed_ms,
+            error_text=str(exc),
+        )
+
+
+def run_check_cycle():
+    services = {}
+    services.update(CORE_SERVICES)
+    services.update(get_app_services())
+
+    for service_id, url in services.items():
+        check_service(service_id, url)
+
+
+def monitor_loop():
+    while True:
+        try:
+            run_check_cycle()
+        except Exception as exc:
+            print(f"Monitor loop error: {exc}", flush=True)
+
+        time.sleep(CHECK_INTERVAL_SECONDS)
+
+
+@app.on_event("startup")
+def start_monitor():
+    global _monitor_thread
+    global _monitor_started
+
+    if _monitor_started:
+        return
+
+    _monitor_started = True
+
+    _monitor_thread = threading.Thread(
+        target=monitor_loop,
+        daemon=True,
+    )
+    _monitor_thread.start()
+
+
 @app.get("/health")
 def health():
     return {
-        "status": "ok"
+        "status": "ok",
+        "interval_seconds": CHECK_INTERVAL_SECONDS,
+        "timeout_seconds": REQUEST_TIMEOUT_SECONDS,
     }
+
+
+@app.get("/services")
+def services():
+    result = {}
+    result.update(CORE_SERVICES)
+    result.update(get_app_services())
+
+    return result