Files
appfactory-portal/app/routes/jobs.py
T
2026-06-17 11:39:52 +02:00

585 lines
23 KiB
Python

import asyncio
import html
import json
import math
from urllib.parse import quote, urlencode
from fastapi import APIRouter, Depends, HTTPException, Query, Request, WebSocket, WebSocketDisconnect
from fastapi.responses import HTMLResponse, RedirectResponse
from app.auth import current_user, require_user
from app.db.audit import log_audit_event
from app.db.jobs import cancel_job, count_jobs, get_job, get_job_logs, get_job_logs_after, get_job_stats, get_jobs, retry_failed_job
from app.routes.deployments import calculate_duration, render_status_pill
from app.templates.layout import page
router = APIRouter()
LIVE_STATUSES = {"queued", "running", "cancelled_requested"}
FINISHED_STATUSES = {"success", "failed", "cancelled"}
JOB_STATUSES = {"queued", "running", "cancelled_requested", "cancelled", "success", "failed"}
JOB_FILTER_STATUSES = ("queued", "running", "success", "failed", "cancelled")
JOB_FILTER_TYPES = ("deploy_app", "deploy_core_service", "run_script")
IGNORED_RETRY_REPOSITORIES = {"appfactory-tools", "appfactory-infrastructure"}
DEFAULT_PAGE_SIZE = 20
def render_job_status(status: str | None) -> str:
status_value = status or ""
normalized = status_value.lower()
if normalized not in JOB_STATUSES:
return render_status_pill(status)
if normalized == "success":
class_name = "pill pill-success"
elif normalized in {"queued", "running", "cancelled_requested"}:
class_name = "pill pill-warning"
elif normalized in {"failed", "cancelled"}:
class_name = "pill pill-danger"
else:
class_name = "pill pill-muted"
labels = {
"queued": "čeká",
"running": "běží",
"cancelled_requested": "žádost o zrušení",
"cancelled": "zrušeno",
"success": "úspěšné",
"failed": "selhalo",
}
return f'<span class="{class_name}">{html.escape(labels.get(normalized, status_value))}</span>'
def pretty_json(value: str | None) -> str:
if not value:
return "{}"
try:
return json.dumps(json.loads(value), ensure_ascii=False, indent=2, sort_keys=True)
except (TypeError, ValueError):
return value
def render_options(values: tuple[str, ...], selected: str, empty_label: str) -> str:
options = [f'<option value="">{html.escape(empty_label)}</option>']
for value in values:
selected_attr = " selected" if selected == value else ""
escaped_value = html.escape(value)
options.append(f'<option value="{escaped_value}"{selected_attr}>{escaped_value}</option>')
return "".join(options)
def can_retry_job(job: dict) -> bool:
job_type = job.get("type") or ""
target_id = job.get("target_id") or ""
if job_type == "deploy_core_service":
return True
if job_type == "deploy_app" and target_id not in IGNORED_RETRY_REPOSITORIES:
return True
return False
def log_message_payload(log: dict) -> dict:
return {
"type": "log",
"id": log.get("id"),
"stream": log.get("stream") or "system",
"message": log.get("message") or "",
"created_at": log.get("created_at") or "",
}
@router.websocket("/ws/jobs/{job_id}/logs")
async def job_logs_websocket(websocket: WebSocket, job_id: int):
user = current_user(websocket)
if not user:
await websocket.close(code=1008)
return
job = get_job(job_id)
if not job:
await websocket.close(code=1008)
return
await websocket.accept()
log_audit_event(
user,
action="job.logs.live.view",
target_type="job",
target_id=job_id,
)
last_log_id = 0
try:
while True:
while True:
logs = get_job_logs_after(job_id, last_log_id)
if not logs:
break
for log in logs:
last_log_id = max(last_log_id, int(log.get("id") or 0))
await websocket.send_json(log_message_payload(log))
job = get_job(job_id)
status_value = (job.get("status") or "").lower() if job else ""
if status_value in FINISHED_STATUSES:
await websocket.send_json({"type": "job_finished", "status": status_value})
await websocket.close()
return
await asyncio.sleep(1)
except WebSocketDisconnect:
return
@router.get("/jobs", response_class=HTMLResponse)
def jobs_page(
request: Request,
status: str = Query(""),
job_type: str = Query("", alias="type"),
target: str = Query(""),
page_number: int = Query(1, alias="page", ge=1),
user=Depends(require_user),
):
selected_status = status.strip().lower()
selected_type = job_type.strip()
selected_target = target.strip()
if selected_status not in JOB_FILTER_STATUSES:
selected_status = ""
if selected_type not in JOB_FILTER_TYPES:
selected_type = ""
total_jobs = count_jobs(
status=selected_status or None,
job_type=selected_type or None,
target=selected_target or None,
)
total_pages = max(1, math.ceil(total_jobs / DEFAULT_PAGE_SIZE))
if page_number > total_pages:
page_number = total_pages
offset = (page_number - 1) * DEFAULT_PAGE_SIZE
jobs = get_jobs(
limit=DEFAULT_PAGE_SIZE,
offset=offset,
status=selected_status or None,
job_type=selected_type or None,
target=selected_target or None,
)
stats = get_job_stats()
refresh = ""
rows = ""
for job in jobs:
job_id = html.escape(str(job.get("id", "")))
status = render_job_status(job.get("status"))
source = html.escape(job.get("source", "") or "")
target_type = html.escape(job.get("target_type", "") or "")
target_id = html.escape(job.get("target_id", "") or "")
job_type = html.escape(job.get("type", "") or "")
created_by = html.escape(job.get("created_by_display_name") or job.get("created_by_username") or "")
created_at = html.escape(job.get("created_at", "") or "")
started_at = html.escape(job.get("started_at", "") or "")
finished_at = html.escape(job.get("finished_at", "") or "")
row_class = ' class="running-row"' if (job.get("status") or "").lower() == "running" else ""
rows += f"""
<tr{row_class}>
<td>
<strong><a href="/portal/jobs/{job_id}">#{job_id}</a></strong><br>
<span class="muted">{job_type}</span>
</td>
<td>{status}</td>
<td>{source}</td>
<td>{target_type}: <strong>{target_id}</strong></td>
<td>{created_by}</td>
<td>{created_at}</td>
<td>{started_at}</td>
<td>{finished_at}</td>
<td class="actions-cell"><a class="icon-action" href="/portal/jobs/{job_id}" title="Detail" aria-label="Detail"><i class="fa-solid fa-eye" aria-hidden="true"></i></a></td>
</tr>
"""
if not rows:
rows = '<tr><td colspan="9">Zatím nejsou evidované žádné úlohy.</td></tr>'
status_options = render_options(JOB_FILTER_STATUSES, selected_status, "Všechny statusy")
type_options = render_options(JOB_FILTER_TYPES, selected_type, "Všechny typy")
target_value = html.escape(selected_target)
first_item = offset + 1 if total_jobs else 0
last_item = min(offset + len(jobs), total_jobs)
def page_url(page: int) -> str:
params = {"page": page}
if selected_status:
params["status"] = selected_status
if selected_type:
params["type"] = selected_type
if selected_target:
params["target"] = selected_target
return f"/portal/jobs?{urlencode(params)}"
previous_link = (
f'<a class="btn btn-secondary" href="{page_url(page_number - 1)}"><i class="fa-solid fa-chevron-left" aria-hidden="true"></i> Předchozí</a>'
if page_number > 1
else ""
)
next_link = (
f'<a class="btn btn-secondary" href="{page_url(page_number + 1)}">Další <i class="fa-solid fa-chevron-right" aria-hidden="true"></i></a>'
if page_number < total_pages
else ""
)
pagination = ""
if total_pages > 1:
pagination = f"""
<div class="pagination">
<span>Zobrazeno {first_item}-{last_item} z {total_jobs}</span>
<div class="pagination-actions">
{previous_link}
<span class="page-indicator">Strana {page_number} / {total_pages}</span>
{next_link}
</div>
</div>
"""
return page(
"Úlohy",
f"""
{refresh}
<div class="card">
<h2><i class="fa-solid fa-list-check" aria-hidden="true"></i> Úlohy</h2>
<p class="muted">Fronta portálových a webhook úloh připravená pro centrální worker.</p>
</div>
<div class="stats-grid">
<div class="stat-card stat-warning"><span>Čekající</span><strong data-live-count="jobs.queued">{stats.get("queued") or 0}</strong></div>
<div class="stat-card stat-warning"><span>Běžící</span><strong data-live-count="jobs.running">{stats.get("running") or 0}</strong></div>
<div class="stat-card stat-danger"><span>Selhané (24 h)</span><strong data-live-count="jobs.failed_24h">{stats.get("failed_24h") or 0}</strong></div>
<div class="stat-card stat-success"><span>Úspěšné (24 h)</span><strong data-live-count="jobs.success_24h">{stats.get("success_24h") or 0}</strong></div>
</div>
<div class="card">
<h2>Filtry</h2>
<form method="get" action="/portal/jobs" class="filter-form">
<select name="status">{status_options}</select>
<select name="type">{type_options}</select>
<input name="target" value="{target_value}" placeholder="Cíl">
<input type="hidden" name="page" value="1">
<button type="submit"><i class="fa-solid fa-filter" aria-hidden="true"></i> Filtrovat</button>
<a class="btn btn-secondary" href="/portal/jobs"><i class="fa-solid fa-arrow-rotate-left" aria-hidden="true"></i> Reset</a>
</form>
</div>
<div class="card">
<h2>Poslední úlohy</h2>
<table>
<tr>
<th>ID</th>
<th>Status</th>
<th>Typ</th>
<th>Cíl</th>
<th>Zdroj</th>
<th>Vytvořeno</th>
</tr>
<tbody data-live-table="jobs.recent"></tbody>
</table>
</div>
<div class="card">
<h2>Fronta</h2>
{pagination}
<table>
<tr>
<th>ID</th>
<th>Status</th>
<th>Zdroj</th>
<th>Cíl</th>
<th>Vytvořil</th>
<th>Vytvořeno</th>
<th>Started at</th>
<th>Finished at</th>
<th>Akce</th>
</tr>
<tbody>{rows}</tbody>
</table>
{pagination}
</div>
<script src="/portal/static/operations-live.js"></script>
""",
user=user,
)
@router.get("/jobs/{job_id}", response_class=HTMLResponse)
def job_detail_page(job_id: int, request: Request, user=Depends(require_user)):
job = get_job(job_id)
if not job:
raise HTTPException(status_code=404, detail="Job not found")
status_value = (job.get("status") or "").lower()
refresh = ""
title = f"Úloha #{html.escape(str(job.get('id', job_id)))}"
status = render_job_status(job.get("status"))
target_type = job.get("target_type", "") or ""
target_id = job.get("target_id", "") or ""
target_id_html = html.escape(target_id)
# Kontextová navigace: "Zpět" vede tam, odkud úloha typicky vznikla, a tlačítko na detail cíle
# se ukáže jen tehdy, když cíl má vlastní stránku (jinak nedávalo smysl a vedlo zpět na seznam úloh).
back_url = "/portal/jobs"
back_label = "Zpět na úlohy"
target_detail_button = ""
if target_type == "app" and target_id:
app_url = f"/portal/apps/{quote(target_id, safe='')}"
target_detail_button = (
f'<a class="btn btn-secondary" href="{app_url}">'
'<i class="fa-solid fa-arrow-up-right-from-square" aria-hidden="true"></i> Detail služby</a>'
)
elif target_type == "scheduled_script":
back_url = "/portal/scheduled-scripts"
back_label = "Zpět na plánované skripty"
try:
payload_obj = json.loads(job.get("payload_json") or "{}")
except (TypeError, ValueError):
payload_obj = {}
scheduled_script_id = payload_obj.get("scheduled_script_id")
if scheduled_script_id:
sid_html = html.escape(str(scheduled_script_id))
target_detail_button = (
f'<a class="btn btn-secondary" href="/portal/scheduled-scripts/{sid_html}">'
'<i class="fa-solid fa-arrow-up-right-from-square" aria-hidden="true"></i> Detail skriptu</a>'
)
duration = html.escape(calculate_duration(job.get("started_at"), job.get("finished_at")))
payload = html.escape(pretty_json(job.get("payload_json")))
result = html.escape(pretty_json(job.get("result_json")))
error_text = html.escape(job.get("error_text", "") or "")
actions = ""
retry_blocked_notice = ""
if status_value == "failed" and can_retry_job(job):
actions += f"""
<form method="post" action="/portal/jobs/{html.escape(str(job_id))}/retry" onsubmit="return confirm('Retry job #{html.escape(str(job_id))}?');">
<button type="submit"><i class="fa-solid fa-rotate-right" aria-hidden="true"></i> Spustit znovu</button>
</form>
"""
elif status_value == "failed" and (job.get("target_id") or "") in IGNORED_RETRY_REPOSITORIES:
retry_blocked_notice = '<p class="muted">Tento repozitář není deployovatelná služba.</p>'
if status_value in {"queued", "running"}:
actions += f"""
<form method="post" action="/portal/jobs/{html.escape(str(job_id))}/cancel" onsubmit="return confirm('Cancel job #{html.escape(str(job_id))}?');">
<button type="submit" class="danger"><i class="fa-solid fa-ban" aria-hidden="true"></i> Zrušit úlohu</button>
</form>
"""
if actions:
actions = f"""
<div class="inline-form">
{actions}
</div>
"""
log_blocks = ""
for log in get_job_logs(job_id):
log_id = html.escape(str(log.get("id", "")))
stream = html.escape(log.get("stream", "") or "system")
created_at = html.escape(log.get("created_at", "") or "")
message = html.escape(log.get("message", "") or "")
if stream == "stderr":
class_name = "log-stderr"
elif stream == "system":
class_name = "log-system"
else:
class_name = "log-stdout"
# id="log-{id}" umožní proklik z globálního prohlížeče logů přímo na konkrétní řádek.
log_blocks += f"""
<div class="job-log-entry" id="log-{log_id}">
<div class="muted">{created_at} · {stream}</div>
<pre class="log-viewer {class_name}">{message}</pre>
</div>
"""
if not log_blocks:
log_blocks = '<p class="muted">Zatím nejsou uložené žádné logy.</p>'
live_logs_script = f"""
<script>
(() => {{
const statusEl = document.getElementById("live-log-status");
const panel = document.getElementById("live-log-panel");
const banner = document.getElementById("job-finished-banner");
// Pokud úloha při načtení stránky ještě běžela, po jejím dokončení stránku obnovíme,
// aby se přepsal i horní stav a souhrn (jinak by zůstal viset na "běží").
const jobWasLive = {"true" if status_value in ("queued", "running", "cancelled_requested") else "false"};
if (!statusEl || !panel || !window.WebSocket) {{
if (statusEl) statusEl.textContent = "Disconnected";
return;
}}
const protocol = window.location.protocol === "https:" ? "wss:" : "ws:";
const socket = new WebSocket(`${{protocol}}//${{window.location.host}}/portal/ws/jobs/{html.escape(str(job_id))}/logs`);
const appendLog = (item) => {{
const entry = document.createElement("div");
entry.className = "job-log-entry";
const meta = document.createElement("div");
meta.className = "muted";
meta.textContent = `${{item.created_at || ""}} · ${{item.stream || "system"}}`;
const pre = document.createElement("pre");
const stream = item.stream || "system";
const streamClass = stream === "stderr" ? "log-stderr" : (stream === "system" ? "log-system" : "log-stdout");
pre.className = `log-viewer ${{streamClass}}`;
pre.dataset.stream = stream;
pre.textContent = item.message || "";
entry.appendChild(meta);
entry.appendChild(pre);
panel.appendChild(entry);
panel.scrollTop = panel.scrollHeight;
}};
socket.addEventListener("open", () => {{
statusEl.textContent = "Connected";
}});
socket.addEventListener("message", (event) => {{
const item = JSON.parse(event.data);
if (item.type === "log") {{
appendLog(item);
}}
if (item.type === "job_finished") {{
statusEl.textContent = `Finished: ${{item.status}}`;
if (banner) {{
const labels = {{ success: "úspěšně", failed: "se selháním", cancelled: "zrušena" }};
banner.classList.toggle("alert-danger", item.status === "failed" || item.status === "cancelled");
banner.textContent = jobWasLive
? `Úloha doběhla (${{labels[item.status] || item.status}}). Načítám výsledek…`
: `Úloha doběhla (${{labels[item.status] || item.status}}).`;
banner.style.display = "block";
}}
socket.close();
if (jobWasLive) {{
setTimeout(() => window.location.reload(), 1200);
}}
}}
}});
socket.addEventListener("close", () => {{
if (!statusEl.textContent.startsWith("Finished:")) {{
statusEl.textContent = "Disconnected";
}}
}});
socket.addEventListener("error", () => {{
statusEl.textContent = "Disconnected";
}});
}})();
</script>
"""
return page(
title,
f"""
{refresh}
<div class="card">
<h2>{title}</h2>
<p>
<a class="btn" href="{back_url}">&larr; {back_label}</a>
{target_detail_button}
</p>
<p id="job-finished-banner" class="alert" style="display:none;"></p>
{retry_blocked_notice}
{actions}
</div>
<div class="card">
<h2>Souhrn</h2>
<table>
<tr><th>Typ</th><td>{html.escape(job.get("type", "") or "")}</td></tr>
<tr><th>Status</th><td>{status}</td></tr>
<tr><th>Zdroj</th><td>{html.escape(job.get("source", "") or "")}</td></tr>
<tr><th>Cíl</th><td>{html.escape(target_type)}: {target_id_html}</td></tr>
<tr><th>Vytvořil</th><td>{html.escape(job.get("created_by_display_name") or job.get("created_by_username") or "")}</td></tr>
<tr><th>Vytvořeno</th><td>{html.escape(job.get("created_at", "") or "")}</td></tr>
<tr><th>Started at</th><td>{html.escape(job.get("started_at", "") or "")}</td></tr>
<tr><th>Finished at</th><td>{html.escape(job.get("finished_at", "") or "")}</td></tr>
<tr><th>Duration</th><td>{duration}</td></tr>
<tr><th>Worker</th><td>{html.escape(job.get("worker_id", "") or "")}</td></tr>
</table>
</div>
<div class="grid">
<div class="card">
<h2>Payload</h2>
<pre class="log-viewer log-stdout">{payload}</pre>
</div>
<div class="card">
<h2>Result / error</h2>
<pre class="log-viewer log-stdout">{result}</pre>
<pre class="log-viewer log-stderr">{error_text}</pre>
</div>
</div>
<div class="card">
<h2>Živé logy</h2>
<p class="muted">WebSocket: <span id="live-log-status">Připojování</span></p>
<div id="live-log-panel" class="log-viewer log-stdout"></div>
</div>
<div class="card">
<h2>Logy z DB</h2>
{log_blocks}
</div>
{live_logs_script}
""",
user=user,
)
@router.post("/jobs/{job_id}/retry")
def retry_job_action(job_id: int, user=Depends(require_user)):
job = get_job(job_id)
if not job:
raise HTTPException(status_code=404, detail="Job not found")
if (job.get("status") or "").lower() != "failed":
raise HTTPException(status_code=400, detail="Only failed jobs can be retried")
if not can_retry_job(job):
raise HTTPException(status_code=400, detail="This job cannot be retried")
new_job_id = retry_failed_job(job_id, user)
if not new_job_id:
raise HTTPException(status_code=409, detail="Job can no longer be retried")
log_audit_event(
user,
action="job.retry",
target_type="job",
target_id=job_id,
metadata={"new_job_id": new_job_id},
)
return RedirectResponse(url=f"/portal/jobs/{new_job_id}", status_code=303)
@router.post("/jobs/{job_id}/cancel")
def cancel_job_action(job_id: int, user=Depends(require_user)):
job = get_job(job_id)
if not job:
raise HTTPException(status_code=404, detail="Job not found")
status_value = (job.get("status") or "").lower()
if status_value not in {"queued", "running"}:
raise HTTPException(status_code=400, detail="Only queued or running jobs can be cancelled")
result = cancel_job(job_id)
if not result:
raise HTTPException(status_code=409, detail="Job can no longer be cancelled")
log_audit_event(
user,
action="job.cancel",
target_type="job",
target_id=job_id,
metadata=result,
)
return RedirectResponse(url=f"/portal/jobs/{job_id}", status_code=303)