import os import sqlite3 import threading import time from pathlib import Path import requests from fastapi import FastAPI DB_FILE = Path("/opt/appfactory/data/appfactory/appfactory.db") CHECK_INTERVAL_SECONDS = int(os.getenv("APPFACTORY_MONITOR_INTERVAL_SECONDS", "30")) REQUEST_TIMEOUT_SECONDS = int(os.getenv("APPFACTORY_MONITOR_TIMEOUT_SECONDS", "5")) HEALTH_RETENTION_DAYS = int(os.getenv("APPFACTORY_HEALTH_RETENTION_DAYS", "7")) CORE_SERVICES = { "appfactory-portal": "http://appfactory-portal:9100/health", "appfactory-webhook": "http://appfactory-webhook:9000/health", "appfactory-worker": "http://appfactory-worker:9200/health", "appfactory-monitor": "http://appfactory-monitor:9300/health", } app = FastAPI(title="AppFactory Monitor") _monitor_thread = None _monitor_started = False def get_connection(): con = sqlite3.connect(DB_FILE, timeout=30) con.row_factory = sqlite3.Row return con def get_app_services(): con = get_connection() rows = con.execute( """ SELECT id, health_url FROM apps WHERE is_enabled = 1 ORDER BY id """ ).fetchall() con.close() services = {} for row in rows: app_id = row["id"] health_url = row["health_url"] or "/health" if health_url.startswith("http://") or health_url.startswith("https://"): services[app_id] = health_url else: services[app_id] = f"http://appfactory-caddy/apps/{app_id}{health_url}" return services def cleanup_old_health_records(): con = get_connection() con.execute( """ DELETE FROM service_health WHERE checked_at < datetime('now', ?) """, (f"-{HEALTH_RETENTION_DAYS} days",), ) con.commit() con.close() def save_health_result( service_id: str, status: str, http_status: int | None, response_time_ms: int | None, error_text: str | None, ) -> int: con = get_connection() cur = con.execute( """ INSERT INTO service_health ( service_id, status, http_status, response_time_ms, error_text, checked_at ) VALUES (?, ?, ?, ?, ?, CURRENT_TIMESTAMP) """, ( service_id, status, http_status, response_time_ms, error_text, ), ) health_id = cur.lastrowid con.commit() con.close() return int(health_id) def get_open_incident(service_id: str): con = get_connection() row = con.execute( """ SELECT * FROM service_incidents WHERE service_id = ? AND status = 'open' ORDER BY id DESC LIMIT 1 """, (service_id,), ).fetchone() con.close() return row def open_incident(service_id: str, health_id: int, status: str, error_text: str | None): if get_open_incident(service_id): return title = f"Služba {service_id} není zdravá" description = error_text or f"Health status: {status}" con = get_connection() con.execute( """ INSERT INTO service_incidents ( service_id, status, started_at, start_health_id, title, description ) VALUES (?, 'open', CURRENT_TIMESTAMP, ?, ?, ?) """, ( service_id, health_id, title, description, ), ) con.execute( """ INSERT INTO audit_events ( username, action, target_type, target_id, source, metadata, created_at ) VALUES ( 'system', 'service.incident.opened', 'service', ?, 'monitor', ?, CURRENT_TIMESTAMP ) """, ( service_id, f'{{"health_id": {health_id}, "status": "{status}"}}', ), ) con.commit() con.close() def resolve_incident(service_id: str, health_id: int): incident = get_open_incident(service_id) if not incident: return con = get_connection() con.execute( """ UPDATE service_incidents SET status = 'resolved', ended_at = CURRENT_TIMESTAMP, end_health_id = ?, duration_seconds = CAST( (julianday(CURRENT_TIMESTAMP) - julianday(started_at)) * 86400 AS INTEGER ) WHERE id = ? """, ( health_id, incident["id"], ), ) con.execute( """ INSERT INTO audit_events ( username, action, target_type, target_id, source, metadata, created_at ) VALUES ( 'system', 'service.incident.resolved', 'service', ?, 'monitor', ?, CURRENT_TIMESTAMP ) """, ( service_id, f'{{"incident_id": {incident["id"]}, "health_id": {health_id}}}', ), ) con.commit() con.close() def update_incident_state(service_id: str, health_id: int, status: str, error_text: str | None): if status == "healthy": resolve_incident(service_id, health_id) else: open_incident(service_id, health_id, status, error_text) def check_service(service_id: str, url: str): start = time.monotonic() try: response = requests.get( url, timeout=REQUEST_TIMEOUT_SECONDS, ) elapsed_ms = int((time.monotonic() - start) * 1000) if 200 <= response.status_code < 300: health_id = save_health_result( service_id=service_id, status="healthy", http_status=response.status_code, response_time_ms=elapsed_ms, error_text=None, ) update_incident_state( service_id=service_id, health_id=health_id, status="healthy", error_text=None, ) else: error_text = response.text[:500] health_id = save_health_result( service_id=service_id, status="unhealthy", http_status=response.status_code, response_time_ms=elapsed_ms, error_text=error_text, ) update_incident_state( service_id=service_id, health_id=health_id, status="unhealthy", error_text=error_text, ) except Exception as exc: elapsed_ms = int((time.monotonic() - start) * 1000) error_text = str(exc) health_id = save_health_result( service_id=service_id, status="unreachable", http_status=None, response_time_ms=elapsed_ms, error_text=error_text, ) update_incident_state( service_id=service_id, health_id=health_id, status="unreachable", error_text=error_text, ) def run_check_cycle(): cleanup_old_health_records() services = {} services.update(CORE_SERVICES) services.update(get_app_services()) for service_id, url in services.items(): check_service(service_id, url) def monitor_loop(): while True: try: run_check_cycle() except Exception as exc: print(f"Monitor loop error: {exc}", flush=True) time.sleep(CHECK_INTERVAL_SECONDS) @app.on_event("startup") def start_monitor(): global _monitor_thread global _monitor_started if _monitor_started: return _monitor_started = True _monitor_thread = threading.Thread( target=monitor_loop, daemon=True, ) _monitor_thread.start() @app.get("/health") def health(): return { "status": "ok", "interval_seconds": CHECK_INTERVAL_SECONDS, "timeout_seconds": REQUEST_TIMEOUT_SECONDS, "retention_days": HEALTH_RETENTION_DAYS, } @app.get("/services") def services(): result = {} result.update(CORE_SERVICES) result.update(get_app_services()) return result