502 lines
16 KiB
Bash
Executable File
502 lines
16 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
set -u
|
|
|
|
APPFACTORY_ROOT="/opt/appfactory"
|
|
ENV_FILE="$APPFACTORY_ROOT/config/appfactory.env"
|
|
|
|
BACKUP_WARN_HOURS="${BACKUP_WARN_HOURS:-24}"
|
|
BACKUP_FAIL_HOURS="${BACKUP_FAIL_HOURS:-72}"
|
|
|
|
DISK_WARN_PERCENT="${DISK_WARN_PERCENT:-80}"
|
|
DISK_FAIL_PERCENT="${DISK_FAIL_PERCENT:-90}"
|
|
|
|
OK_COUNT=0
|
|
WARN_COUNT=0
|
|
FAIL_COUNT=0
|
|
|
|
ok(){ OK_COUNT=$((OK_COUNT+1)); echo "[OK] $1"; }
|
|
warn(){ WARN_COUNT=$((WARN_COUNT+1)); echo "[WARN] $1"; }
|
|
fail(){ FAIL_COUNT=$((FAIL_COUNT+1)); echo "[FAIL] $1"; }
|
|
section(){ echo; echo "== $1 =="; }
|
|
|
|
check_command() {
|
|
command -v "$1" >/dev/null 2>&1 && ok "Command available: $1" || fail "Command missing: $1"
|
|
}
|
|
|
|
load_env() {
|
|
section "Environment"
|
|
|
|
if [ ! -f "$ENV_FILE" ]; then
|
|
fail "Env file missing: $ENV_FILE"
|
|
return
|
|
fi
|
|
|
|
ok "Env file exists: $ENV_FILE"
|
|
|
|
set -a
|
|
# shellcheck disable=SC1090
|
|
source "$ENV_FILE"
|
|
set +a
|
|
|
|
for key in APPFACTORY_UID APPFACTORY_GID APPFACTORY_DOCKER_GID; do
|
|
[ -n "${!key:-}" ] && ok "$key=${!key}" || fail "$key is missing"
|
|
done
|
|
|
|
for key in APPFACTORY_DOMAIN APPFACTORY_PORTAL_DOMAIN APPFACTORY_GITEA_DOMAIN APPFACTORY_REGISTRY_DOMAIN; do
|
|
[ -n "${!key:-}" ] && ok "$key=${!key}" || warn "$key is empty"
|
|
done
|
|
|
|
[ -n "${APPFACTORY_ENABLE_HTTPS:-}" ] && ok "APPFACTORY_ENABLE_HTTPS=$APPFACTORY_ENABLE_HTTPS" || warn "APPFACTORY_ENABLE_HTTPS is missing"
|
|
|
|
for key in SMTP_HOST SMTP_PORT SMTP_USERNAME SMTP_FROM SMTP_PASSWORD; do
|
|
[ -n "${!key:-}" ] && ok "$key configured" || warn "$key is empty"
|
|
done
|
|
|
|
if [ -n "${GOOGLE_CLIENT_ID:-}" ] && [ -n "${GOOGLE_CLIENT_SECRET:-}" ]; then
|
|
ok "Google OAuth configured"
|
|
else
|
|
warn "Google OAuth not configured"
|
|
fi
|
|
}
|
|
|
|
check_directories() {
|
|
section "Directories"
|
|
|
|
for dir in apps backups config data deploy gateway services templates tools workspace; do
|
|
path="$APPFACTORY_ROOT/$dir"
|
|
[ -d "$path" ] && ok "Directory exists: $path" || fail "Directory missing: $path"
|
|
done
|
|
}
|
|
|
|
check_ownership() {
|
|
section "Ownership"
|
|
|
|
uid="${APPFACTORY_UID:-}"
|
|
gid="${APPFACTORY_GID:-}"
|
|
|
|
if [ -z "$uid" ] || [ -z "$gid" ]; then
|
|
fail "Cannot check ownership because APPFACTORY_UID/GID is missing"
|
|
return
|
|
fi
|
|
|
|
for path in "$APPFACTORY_ROOT" "$APPFACTORY_ROOT/data" "$APPFACTORY_ROOT/workspace" "$APPFACTORY_ROOT/backups"; do
|
|
if [ ! -e "$path" ]; then
|
|
fail "Path missing for ownership check: $path"
|
|
continue
|
|
fi
|
|
|
|
actual_uid="$(stat -c '%u' "$path")"
|
|
actual_gid="$(stat -c '%g' "$path")"
|
|
|
|
if [ "$actual_uid" = "$uid" ] && [ "$actual_gid" = "$gid" ]; then
|
|
ok "Ownership OK: $path ($actual_uid:$actual_gid)"
|
|
else
|
|
fail "Ownership mismatch: $path is $actual_uid:$actual_gid, expected $uid:$gid"
|
|
fi
|
|
done
|
|
}
|
|
|
|
check_docker() {
|
|
section "Docker"
|
|
|
|
check_command docker
|
|
|
|
docker compose version >/dev/null 2>&1 && ok "docker compose available" || fail "docker compose not available"
|
|
|
|
[ -S /var/run/docker.sock ] && ok "Docker socket exists" || fail "Docker socket missing"
|
|
|
|
if getent group docker >/dev/null 2>&1; then
|
|
docker_group_gid="$(getent group docker | cut -d: -f3)"
|
|
ok "Docker group exists in current runtime with gid=$docker_group_gid"
|
|
else
|
|
warn "Docker group does not exist in current runtime"
|
|
fi
|
|
|
|
if [ -S /var/run/docker.sock ]; then
|
|
docker_sock_gid="$(stat -c '%g' /var/run/docker.sock)"
|
|
ok "Docker socket gid=$docker_sock_gid"
|
|
|
|
if [ -n "${APPFACTORY_DOCKER_GID:-}" ] && [ "$docker_sock_gid" = "$APPFACTORY_DOCKER_GID" ]; then
|
|
ok "APPFACTORY_DOCKER_GID matches docker socket gid"
|
|
else
|
|
fail "APPFACTORY_DOCKER_GID mismatch. env=${APPFACTORY_DOCKER_GID:-empty}, docker_sock_gid=$docker_sock_gid"
|
|
fi
|
|
fi
|
|
|
|
docker ps >/dev/null 2>&1 && ok "docker ps works from host/runtime" || fail "docker ps failed from host/runtime"
|
|
}
|
|
|
|
check_containers() {
|
|
section "Containers"
|
|
|
|
for name in appfactory-portal appfactory-worker appfactory-monitor appfactory-webhook appfactory-gitea appfactory-registry appfactory-caddy; do
|
|
if ! docker inspect "$name" >/dev/null 2>&1; then
|
|
warn "Container missing: $name"
|
|
continue
|
|
fi
|
|
|
|
state="$(docker inspect -f '{{.State.Status}}' "$name" 2>/dev/null || true)"
|
|
health="$(docker inspect -f '{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' "$name" 2>/dev/null || true)"
|
|
|
|
if [ "$state" != "running" ]; then
|
|
fail "Container not running: $name status=$state"
|
|
elif [ "$health" = "unhealthy" ]; then
|
|
fail "Container unhealthy: $name"
|
|
else
|
|
ok "Container running: $name health=$health"
|
|
fi
|
|
done
|
|
}
|
|
|
|
check_container_docker_access() {
|
|
section "Container Docker Access"
|
|
|
|
for name in appfactory-portal appfactory-worker appfactory-webhook; do
|
|
if ! docker inspect "$name" >/dev/null 2>&1; then
|
|
warn "Skipping docker access check, container missing: $name"
|
|
continue
|
|
fi
|
|
|
|
if docker exec "$name" docker ps >/dev/null 2>&1; then
|
|
ok "Docker access from container works: $name"
|
|
else
|
|
fail "Docker access from container failed: $name"
|
|
fi
|
|
done
|
|
}
|
|
|
|
find_portal_db() {
|
|
candidates="$(find "$APPFACTORY_ROOT" -maxdepth 7 -type f \( -name '*.db' -o -name '*.sqlite' -o -name '*.sqlite3' \) 2>/dev/null || true)"
|
|
|
|
for db in $candidates; do
|
|
has_apps="$(sqlite3 "$db" "select name from sqlite_master where type='table' and name='apps';" 2>/dev/null || true)"
|
|
has_jobs="$(sqlite3 "$db" "select name from sqlite_master where type='table' and name='jobs';" 2>/dev/null || true)"
|
|
has_templates="$(sqlite3 "$db" "select name from sqlite_master where type='table' and name='app_templates';" 2>/dev/null || true)"
|
|
|
|
if [ "$has_apps" = "apps" ] && [ "$has_jobs" = "jobs" ] && [ "$has_templates" = "app_templates" ]; then
|
|
echo "$db"
|
|
return 0
|
|
fi
|
|
done
|
|
|
|
return 1
|
|
}
|
|
|
|
check_sqlite() {
|
|
section "SQLite"
|
|
|
|
check_command sqlite3
|
|
|
|
db_path="$(find_portal_db || true)"
|
|
|
|
if [ -z "$db_path" ]; then
|
|
fail "Portal SQLite DB not found. No DB contains apps + jobs + app_templates."
|
|
return
|
|
fi
|
|
|
|
ok "Portal SQLite DB found: $db_path"
|
|
|
|
[ -r "$db_path" ] && ok "DB readable" || fail "DB not readable: $db_path"
|
|
[ -w "$db_path" ] && ok "DB writable" || fail "DB not writable: $db_path"
|
|
|
|
for table in apps app_templates jobs scheduled_scripts alert_rules alert_events; do
|
|
exists="$(sqlite3 "$db_path" "select name from sqlite_master where type='table' and name='$table';" 2>/dev/null || true)"
|
|
[ "$exists" = "$table" ] && ok "Table exists: $table" || fail "Table missing: $table"
|
|
done
|
|
|
|
template_count="$(sqlite3 "$db_path" "select count(*) from app_templates where is_enabled=1;" 2>/dev/null || echo 0)"
|
|
if [ "$template_count" -gt 0 ] 2>/dev/null; then
|
|
ok "Enabled app templates: $template_count"
|
|
sqlite3 "$db_path" "select '- ' || coalesce(name, template) || ' [' || coalesce(template, '') || ']' from app_templates where is_enabled=1 order by 1;" 2>/dev/null || true
|
|
else
|
|
fail "No enabled app templates found"
|
|
fi
|
|
}
|
|
|
|
check_alerting() {
|
|
section "Alerting"
|
|
|
|
alerts_dir="$APPFACTORY_ROOT/workspace/appfactory-tools/alerts"
|
|
|
|
[ -d "$alerts_dir" ] && ok "Alert scripts directory exists: $alerts_dir" || { fail "Alert scripts directory missing: $alerts_dir"; return; }
|
|
|
|
count="$(find "$alerts_dir" -maxdepth 1 -type f -name '*.sh' | wc -l | tr -d ' ')"
|
|
if [ "$count" -gt 0 ]; then
|
|
ok "Alert scripts found: $count"
|
|
find "$alerts_dir" -maxdepth 1 -type f -name '*.sh' -printf '%f\n' | sort | sed 's/^/- /'
|
|
else
|
|
warn "No alert scripts found"
|
|
fi
|
|
}
|
|
|
|
check_scheduler() {
|
|
section "Scheduler"
|
|
|
|
maintenance_dir="$APPFACTORY_ROOT/workspace/appfactory-tools/maintenance"
|
|
|
|
[ -d "$maintenance_dir" ] && ok "Maintenance scripts directory exists: $maintenance_dir" || { fail "Maintenance scripts directory missing: $maintenance_dir"; return; }
|
|
|
|
count="$(find "$maintenance_dir" -maxdepth 1 -type f -name '*.sh' | wc -l | tr -d ' ')"
|
|
if [ "$count" -gt 0 ]; then
|
|
ok "Maintenance scripts found: $count"
|
|
find "$maintenance_dir" -maxdepth 1 -type f -name '*.sh' -printf '%f\n' | sort | sed 's/^/- /'
|
|
else
|
|
warn "No maintenance scripts found"
|
|
fi
|
|
}
|
|
|
|
check_disk_space() {
|
|
section "Disk Space"
|
|
|
|
for path in "$APPFACTORY_ROOT" "$APPFACTORY_ROOT/data" "$APPFACTORY_ROOT/backups"; do
|
|
if [ ! -d "$path" ]; then
|
|
fail "Cannot check disk space, path missing: $path"
|
|
continue
|
|
fi
|
|
|
|
used_percent="$(df -P "$path" | awk 'NR==2 {gsub("%","",$5); print $5}')"
|
|
available="$(df -hP "$path" | awk 'NR==2 {print $4}')"
|
|
mountpoint="$(df -P "$path" | awk 'NR==2 {print $6}')"
|
|
|
|
if [ -z "$used_percent" ]; then
|
|
warn "Could not read disk usage for $path"
|
|
continue
|
|
fi
|
|
|
|
if [ "$used_percent" -ge "$DISK_FAIL_PERCENT" ] 2>/dev/null; then
|
|
fail "Disk usage critical for $path: ${used_percent}% used, available=$available, mount=$mountpoint"
|
|
elif [ "$used_percent" -ge "$DISK_WARN_PERCENT" ] 2>/dev/null; then
|
|
warn "Disk usage high for $path: ${used_percent}% used, available=$available, mount=$mountpoint"
|
|
else
|
|
ok "Disk usage OK for $path: ${used_percent}% used, available=$available, mount=$mountpoint"
|
|
fi
|
|
done
|
|
}
|
|
|
|
check_backup_freshness() {
|
|
section "Backup Freshness"
|
|
|
|
backup_dir="$APPFACTORY_ROOT/backups"
|
|
|
|
if [ ! -d "$backup_dir" ]; then
|
|
fail "Backup directory missing: $backup_dir"
|
|
return
|
|
fi
|
|
|
|
latest_backup="$(find "$backup_dir" -type f \
|
|
\( -name '*.tar' -o -name '*.tar.gz' -o -name '*.tgz' -o -name '*.zip' -o -name '*.db' -o -name '*.sqlite' -o -name '*.sqlite3' \) \
|
|
-printf '%T@ %p\n' 2>/dev/null | sort -nr | head -n 1 | cut -d' ' -f2- || true)"
|
|
|
|
if [ -z "$latest_backup" ]; then
|
|
warn "No backup files found in $backup_dir"
|
|
return
|
|
fi
|
|
|
|
now_epoch="$(date +%s)"
|
|
backup_epoch="$(stat -c '%Y' "$latest_backup")"
|
|
age_seconds=$((now_epoch - backup_epoch))
|
|
age_hours=$((age_seconds / 3600))
|
|
|
|
backup_size="$(du -h "$latest_backup" 2>/dev/null | awk '{print $1}')"
|
|
backup_time="$(date -d "@$backup_epoch" '+%Y-%m-%d %H:%M:%S %z' 2>/dev/null || date -r "$backup_epoch" '+%Y-%m-%d %H:%M:%S' 2>/dev/null || echo "$backup_epoch")"
|
|
|
|
if [ "$age_hours" -ge "$BACKUP_FAIL_HOURS" ]; then
|
|
warn "Latest backup is too old: ${age_hours}h, file=$latest_backup, size=$backup_size, time=$backup_time"
|
|
elif [ "$age_hours" -ge "$BACKUP_WARN_HOURS" ]; then
|
|
warn "Latest backup is older than warning threshold: ${age_hours}h, file=$latest_backup, size=$backup_size, time=$backup_time"
|
|
else
|
|
ok "Latest backup fresh: ${age_hours}h, file=$latest_backup, size=$backup_size, time=$backup_time"
|
|
fi
|
|
}
|
|
|
|
check_gitea_readiness() {
|
|
section "Gitea Readiness"
|
|
|
|
gitea_container="appfactory-gitea"
|
|
|
|
if docker inspect "$gitea_container" >/dev/null 2>&1; then
|
|
state="$(docker inspect -f '{{.State.Status}}' "$gitea_container" 2>/dev/null || true)"
|
|
[ "$state" = "running" ] && ok "Gitea container running" || fail "Gitea container not running: $state"
|
|
else
|
|
fail "Gitea container missing: $gitea_container"
|
|
fi
|
|
|
|
gitea_db_candidates=(
|
|
"$APPFACTORY_ROOT/data/gitea/gitea/gitea.db"
|
|
"$APPFACTORY_ROOT/data/gitea/gitea.db"
|
|
)
|
|
|
|
gitea_db=""
|
|
for candidate in "${gitea_db_candidates[@]}"; do
|
|
if [ -f "$candidate" ]; then
|
|
gitea_db="$candidate"
|
|
break
|
|
fi
|
|
done
|
|
|
|
if [ -n "$gitea_db" ]; then
|
|
ok "Gitea DB exists: $gitea_db"
|
|
[ -r "$gitea_db" ] && ok "Gitea DB readable" || fail "Gitea DB not readable: $gitea_db"
|
|
[ -w "$gitea_db" ] && ok "Gitea DB writable" || warn "Gitea DB not writable from current runtime: $gitea_db"
|
|
|
|
if command -v sqlite3 >/dev/null 2>&1; then
|
|
repo_table="$(sqlite3 "$gitea_db" "select name from sqlite_master where type='table' and name='repository';" 2>/dev/null || true)"
|
|
[ "$repo_table" = "repository" ] && ok "Gitea repository table exists" || warn "Gitea repository table not found"
|
|
fi
|
|
else
|
|
warn "Gitea DB not found in expected paths"
|
|
fi
|
|
|
|
repo_dirs=(
|
|
"$APPFACTORY_ROOT/data/gitea/git/repositories"
|
|
"$APPFACTORY_ROOT/data/gitea/gitea-repositories"
|
|
"$APPFACTORY_ROOT/data/gitea/repositories"
|
|
)
|
|
|
|
repo_dir_found=""
|
|
for candidate in "${repo_dirs[@]}"; do
|
|
if [ -d "$candidate" ]; then
|
|
repo_dir_found="$candidate"
|
|
break
|
|
fi
|
|
done
|
|
|
|
if [ -n "$repo_dir_found" ]; then
|
|
ok "Gitea repository storage exists: $repo_dir_found"
|
|
else
|
|
warn "Gitea repository storage not found in expected paths"
|
|
fi
|
|
}
|
|
|
|
check_registry_readiness() {
|
|
section "Registry Readiness"
|
|
|
|
registry_container="appfactory-registry"
|
|
|
|
if docker inspect "$registry_container" >/dev/null 2>&1; then
|
|
state="$(docker inspect -f '{{.State.Status}}' "$registry_container" 2>/dev/null || true)"
|
|
[ "$state" = "running" ] && ok "Registry container running" || fail "Registry container not running: $state"
|
|
else
|
|
fail "Registry container missing: $registry_container"
|
|
fi
|
|
|
|
registry_dirs=(
|
|
"$APPFACTORY_ROOT/data/registry"
|
|
"$APPFACTORY_ROOT/data/docker-registry"
|
|
"$APPFACTORY_ROOT/registry"
|
|
)
|
|
|
|
registry_dir_found=""
|
|
for candidate in "${registry_dirs[@]}"; do
|
|
if [ -d "$candidate" ]; then
|
|
registry_dir_found="$candidate"
|
|
break
|
|
fi
|
|
done
|
|
|
|
if [ -n "$registry_dir_found" ]; then
|
|
ok "Registry storage exists: $registry_dir_found"
|
|
[ -r "$registry_dir_found" ] && ok "Registry storage readable" || fail "Registry storage not readable: $registry_dir_found"
|
|
[ -w "$registry_dir_found" ] && ok "Registry storage writable" || warn "Registry storage not writable from current runtime: $registry_dir_found"
|
|
else
|
|
warn "Registry storage not found in expected paths"
|
|
fi
|
|
}
|
|
|
|
check_caddy_readiness() {
|
|
section "Caddy Readiness"
|
|
|
|
caddyfile="$APPFACTORY_ROOT/gateway/Caddyfile"
|
|
|
|
if [ ! -f "$caddyfile" ]; then
|
|
fail "Caddyfile missing: $caddyfile"
|
|
return
|
|
fi
|
|
|
|
ok "Caddyfile exists: $caddyfile"
|
|
|
|
if docker inspect appfactory-caddy >/dev/null 2>&1; then
|
|
state="$(docker inspect -f '{{.State.Status}}' appfactory-caddy 2>/dev/null || true)"
|
|
[ "$state" = "running" ] && ok "Caddy container running" || fail "Caddy container not running: $state"
|
|
|
|
if docker exec appfactory-caddy caddy validate --config /etc/caddy/Caddyfile >/tmp/appfactory-caddy-validate.log 2>&1; then
|
|
ok "Caddyfile validates inside container"
|
|
else
|
|
fail "Caddyfile validation failed"
|
|
sed 's/^/[caddy] /' /tmp/appfactory-caddy-validate.log || true
|
|
fi
|
|
else
|
|
fail "Caddy container missing: appfactory-caddy"
|
|
fi
|
|
|
|
if [ "${APPFACTORY_ENABLE_HTTPS:-false}" = "true" ]; then
|
|
if [ -n "${APPFACTORY_PORTAL_DOMAIN:-}" ] || [ -n "${APPFACTORY_GITEA_DOMAIN:-}" ] || [ -n "${APPFACTORY_REGISTRY_DOMAIN:-}" ]; then
|
|
ok "HTTPS mode requested with at least one domain configured"
|
|
else
|
|
fail "HTTPS mode enabled but no domains are configured"
|
|
fi
|
|
|
|
if grep -q '^:80[[:space:]]*{' "$caddyfile"; then
|
|
fail "HTTPS mode enabled but Caddyfile still uses :80 catch-all block"
|
|
else
|
|
ok "Caddyfile is not using :80-only mode"
|
|
fi
|
|
else
|
|
if grep -q '^:80[[:space:]]*{' "$caddyfile"; then
|
|
ok "HTTP-only Caddyfile mode active"
|
|
else
|
|
warn "HTTPS disabled but Caddyfile does not contain :80 catch-all block"
|
|
fi
|
|
fi
|
|
}
|
|
|
|
check_readiness_flags() {
|
|
section "Readiness Flags"
|
|
|
|
[ "${APPFACTORY_ENABLE_HTTPS:-false}" = "true" ] && ok "HTTPS enabled" || warn "HTTPS disabled"
|
|
|
|
[ -n "${APPFACTORY_DOMAIN:-}" ] && ok "Base domain configured: $APPFACTORY_DOMAIN" || warn "Base domain not configured"
|
|
[ -n "${APPFACTORY_PORTAL_DOMAIN:-}" ] && ok "Portal domain configured: $APPFACTORY_PORTAL_DOMAIN" || warn "Portal domain not configured"
|
|
[ -n "${APPFACTORY_GITEA_DOMAIN:-}" ] && ok "Gitea domain configured: $APPFACTORY_GITEA_DOMAIN" || warn "Gitea domain not configured"
|
|
[ -n "${APPFACTORY_REGISTRY_DOMAIN:-}" ] && ok "Registry domain configured: $APPFACTORY_REGISTRY_DOMAIN" || warn "Registry domain not configured"
|
|
}
|
|
|
|
main() {
|
|
echo "APPFACTORY PREFLIGHT CHECK"
|
|
echo "Root: $APPFACTORY_ROOT"
|
|
|
|
load_env
|
|
check_directories
|
|
check_ownership
|
|
check_docker
|
|
check_containers
|
|
check_container_docker_access
|
|
check_sqlite
|
|
check_alerting
|
|
check_scheduler
|
|
check_disk_space
|
|
check_backup_freshness
|
|
check_gitea_readiness
|
|
check_registry_readiness
|
|
check_caddy_readiness
|
|
check_readiness_flags
|
|
|
|
echo
|
|
echo "== Summary =="
|
|
echo "OK: $OK_COUNT"
|
|
echo "WARN: $WARN_COUNT"
|
|
echo "FAIL: $FAIL_COUNT"
|
|
|
|
if [ "$FAIL_COUNT" -eq 0 ]; then
|
|
echo
|
|
echo "APPFACTORY PREFLIGHT: READY"
|
|
exit 0
|
|
fi
|
|
|
|
echo
|
|
echo "APPFACTORY PREFLIGHT: NOT READY"
|
|
exit 1
|
|
}
|
|
|
|
main "$@"
|