AppFactory Bot
183 lines
4.0 KiB
Python
183 lines
4.0 KiB
Python
import os
|
|
import sqlite3
|
|
import threading
|
|
import time
|
|
from pathlib import Path
|
|
|
|
import requests
|
|
from fastapi import FastAPI
|
|
|
|
DB_FILE = Path("/opt/appfactory/data/appfactory/appfactory.db")
|
|
|
|
CHECK_INTERVAL_SECONDS = int(os.getenv("APPFACTORY_MONITOR_INTERVAL_SECONDS", "30"))
|
|
REQUEST_TIMEOUT_SECONDS = int(os.getenv("APPFACTORY_MONITOR_TIMEOUT_SECONDS", "5"))
|
|
|
|
CORE_SERVICES = {
|
|
"appfactory-portal": "http://appfactory-portal:9100/health",
|
|
"appfactory-webhook": "http://appfactory-webhook:9000/health",
|
|
"appfactory-worker": "http://appfactory-worker:9200/health",
|
|
"appfactory-monitor": "http://appfactory-monitor:9300/health",
|
|
}
|
|
|
|
app = FastAPI(title="AppFactory Monitor")
|
|
|
|
_monitor_thread = None
|
|
_monitor_started = False
|
|
|
|
|
|
def get_connection():
|
|
con = sqlite3.connect(DB_FILE, timeout=30)
|
|
con.row_factory = sqlite3.Row
|
|
return con
|
|
|
|
|
|
def get_app_services():
|
|
con = get_connection()
|
|
|
|
rows = con.execute(
|
|
"""
|
|
SELECT id
|
|
FROM apps
|
|
WHERE status IS NOT NULL
|
|
ORDER BY id
|
|
"""
|
|
).fetchall()
|
|
|
|
con.close()
|
|
|
|
services = {}
|
|
|
|
for row in rows:
|
|
app_id = row["id"]
|
|
services[app_id] = f"http://appfactory-caddy/apps/{app_id}/health"
|
|
|
|
return services
|
|
|
|
|
|
def save_health_result(
|
|
service_id: str,
|
|
status: str,
|
|
http_status: int | None,
|
|
response_time_ms: int | None,
|
|
error_text: str | None,
|
|
):
|
|
con = get_connection()
|
|
|
|
con.execute(
|
|
"""
|
|
INSERT INTO service_health (
|
|
service_id,
|
|
status,
|
|
http_status,
|
|
response_time_ms,
|
|
error_text,
|
|
checked_at
|
|
)
|
|
VALUES (?, ?, ?, ?, ?, CURRENT_TIMESTAMP)
|
|
""",
|
|
(
|
|
service_id,
|
|
status,
|
|
http_status,
|
|
response_time_ms,
|
|
error_text,
|
|
),
|
|
)
|
|
|
|
con.commit()
|
|
con.close()
|
|
|
|
|
|
def check_service(service_id: str, url: str):
|
|
start = time.monotonic()
|
|
|
|
try:
|
|
response = requests.get(
|
|
url,
|
|
timeout=REQUEST_TIMEOUT_SECONDS,
|
|
)
|
|
|
|
elapsed_ms = int((time.monotonic() - start) * 1000)
|
|
|
|
if 200 <= response.status_code < 300:
|
|
save_health_result(
|
|
service_id=service_id,
|
|
status="healthy",
|
|
http_status=response.status_code,
|
|
response_time_ms=elapsed_ms,
|
|
error_text=None,
|
|
)
|
|
else:
|
|
save_health_result(
|
|
service_id=service_id,
|
|
status="unhealthy",
|
|
http_status=response.status_code,
|
|
response_time_ms=elapsed_ms,
|
|
error_text=response.text[:500],
|
|
)
|
|
|
|
except Exception as exc:
|
|
elapsed_ms = int((time.monotonic() - start) * 1000)
|
|
|
|
save_health_result(
|
|
service_id=service_id,
|
|
status="unreachable",
|
|
http_status=None,
|
|
response_time_ms=elapsed_ms,
|
|
error_text=str(exc),
|
|
)
|
|
|
|
|
|
def run_check_cycle():
|
|
services = {}
|
|
services.update(CORE_SERVICES)
|
|
services.update(get_app_services())
|
|
|
|
for service_id, url in services.items():
|
|
check_service(service_id, url)
|
|
|
|
|
|
def monitor_loop():
|
|
while True:
|
|
try:
|
|
run_check_cycle()
|
|
except Exception as exc:
|
|
print(f"Monitor loop error: {exc}", flush=True)
|
|
|
|
time.sleep(CHECK_INTERVAL_SECONDS)
|
|
|
|
|
|
@app.on_event("startup")
|
|
def start_monitor():
|
|
global _monitor_thread
|
|
global _monitor_started
|
|
|
|
if _monitor_started:
|
|
return
|
|
|
|
_monitor_started = True
|
|
|
|
_monitor_thread = threading.Thread(
|
|
target=monitor_loop,
|
|
daemon=True,
|
|
)
|
|
_monitor_thread.start()
|
|
|
|
|
|
@app.get("/health")
|
|
def health():
|
|
return {
|
|
"status": "ok",
|
|
"interval_seconds": CHECK_INTERVAL_SECONDS,
|
|
"timeout_seconds": REQUEST_TIMEOUT_SECONDS,
|
|
}
|
|
|
|
|
|
@app.get("/services")
|
|
def services():
|
|
result = {}
|
|
result.update(CORE_SERVICES)
|
|
result.update(get_app_services())
|
|
|
|
return result
|