JiriUhlir
530 lines
20 KiB
Python
530 lines
20 KiB
Python
import asyncio
|
|
import html
|
|
import json
|
|
import math
|
|
from urllib.parse import quote, urlencode
|
|
|
|
from fastapi import APIRouter, Depends, HTTPException, Query, Request, WebSocket, WebSocketDisconnect
|
|
from fastapi.responses import HTMLResponse, RedirectResponse
|
|
|
|
from app.auth import current_user, require_user
|
|
from app.db.audit import log_audit_event
|
|
from app.db.jobs import cancel_job, count_jobs, get_job, get_job_logs, get_job_logs_after, get_job_stats, get_jobs, retry_failed_job
|
|
from app.routes.deployments import calculate_duration, render_status_pill
|
|
from app.templates.layout import page
|
|
|
|
router = APIRouter()
|
|
|
|
LIVE_STATUSES = {"queued", "running", "cancelled_requested"}
|
|
FINISHED_STATUSES = {"success", "failed", "cancelled"}
|
|
JOB_STATUSES = {"queued", "running", "cancelled_requested", "cancelled", "success", "failed"}
|
|
JOB_FILTER_STATUSES = ("queued", "running", "success", "failed", "cancelled")
|
|
JOB_FILTER_TYPES = ("deploy_app", "deploy_core_service")
|
|
IGNORED_RETRY_REPOSITORIES = {"appfactory-tools", "appfactory-infrastructure"}
|
|
DEFAULT_PAGE_SIZE = 20
|
|
|
|
|
|
def render_job_status(status: str | None) -> str:
|
|
status_value = status or ""
|
|
normalized = status_value.lower()
|
|
if normalized not in JOB_STATUSES:
|
|
return render_status_pill(status)
|
|
|
|
if normalized == "success":
|
|
class_name = "pill pill-success"
|
|
elif normalized in {"queued", "running", "cancelled_requested"}:
|
|
class_name = "pill pill-warning"
|
|
elif normalized in {"failed", "cancelled"}:
|
|
class_name = "pill pill-danger"
|
|
else:
|
|
class_name = "pill pill-muted"
|
|
|
|
labels = {
|
|
"queued": "čeká",
|
|
"running": "běží",
|
|
"cancelled_requested": "žádost o zrušení",
|
|
"cancelled": "zrušeno",
|
|
"success": "úspěšné",
|
|
"failed": "selhalo",
|
|
}
|
|
return f'<span class="{class_name}">{html.escape(labels.get(normalized, status_value))}</span>'
|
|
|
|
|
|
def pretty_json(value: str | None) -> str:
|
|
if not value:
|
|
return "{}"
|
|
|
|
try:
|
|
return json.dumps(json.loads(value), ensure_ascii=False, indent=2, sort_keys=True)
|
|
except (TypeError, ValueError):
|
|
return value
|
|
|
|
|
|
def render_options(values: tuple[str, ...], selected: str, empty_label: str) -> str:
|
|
options = [f'<option value="">{html.escape(empty_label)}</option>']
|
|
for value in values:
|
|
selected_attr = " selected" if selected == value else ""
|
|
escaped_value = html.escape(value)
|
|
options.append(f'<option value="{escaped_value}"{selected_attr}>{escaped_value}</option>')
|
|
return "".join(options)
|
|
|
|
|
|
def can_retry_job(job: dict) -> bool:
|
|
job_type = job.get("type") or ""
|
|
target_id = job.get("target_id") or ""
|
|
|
|
if job_type == "deploy_core_service":
|
|
return True
|
|
if job_type == "deploy_app" and target_id not in IGNORED_RETRY_REPOSITORIES:
|
|
return True
|
|
return False
|
|
|
|
|
|
def log_message_payload(log: dict) -> dict:
|
|
return {
|
|
"type": "log",
|
|
"id": log.get("id"),
|
|
"stream": log.get("stream") or "system",
|
|
"message": log.get("message") or "",
|
|
"created_at": log.get("created_at") or "",
|
|
}
|
|
|
|
|
|
@router.websocket("/ws/jobs/{job_id}/logs")
|
|
async def job_logs_websocket(websocket: WebSocket, job_id: int):
|
|
user = current_user(websocket)
|
|
if not user:
|
|
await websocket.close(code=1008)
|
|
return
|
|
|
|
job = get_job(job_id)
|
|
if not job:
|
|
await websocket.close(code=1008)
|
|
return
|
|
|
|
await websocket.accept()
|
|
log_audit_event(
|
|
user,
|
|
action="job.logs.live.view",
|
|
target_type="job",
|
|
target_id=job_id,
|
|
)
|
|
|
|
last_log_id = 0
|
|
try:
|
|
while True:
|
|
while True:
|
|
logs = get_job_logs_after(job_id, last_log_id)
|
|
if not logs:
|
|
break
|
|
|
|
for log in logs:
|
|
last_log_id = max(last_log_id, int(log.get("id") or 0))
|
|
await websocket.send_json(log_message_payload(log))
|
|
|
|
job = get_job(job_id)
|
|
status_value = (job.get("status") or "").lower() if job else ""
|
|
if status_value in FINISHED_STATUSES:
|
|
await websocket.send_json({"type": "job_finished", "status": status_value})
|
|
await websocket.close()
|
|
return
|
|
|
|
await asyncio.sleep(1)
|
|
except WebSocketDisconnect:
|
|
return
|
|
|
|
|
|
@router.get("/jobs", response_class=HTMLResponse)
|
|
def jobs_page(
|
|
request: Request,
|
|
status: str = Query(""),
|
|
job_type: str = Query("", alias="type"),
|
|
target: str = Query(""),
|
|
page_number: int = Query(1, alias="page", ge=1),
|
|
user=Depends(require_user),
|
|
):
|
|
selected_status = status.strip().lower()
|
|
selected_type = job_type.strip()
|
|
selected_target = target.strip()
|
|
if selected_status not in JOB_FILTER_STATUSES:
|
|
selected_status = ""
|
|
if selected_type not in JOB_FILTER_TYPES:
|
|
selected_type = ""
|
|
|
|
total_jobs = count_jobs(
|
|
status=selected_status or None,
|
|
job_type=selected_type or None,
|
|
target=selected_target or None,
|
|
)
|
|
total_pages = max(1, math.ceil(total_jobs / DEFAULT_PAGE_SIZE))
|
|
if page_number > total_pages:
|
|
page_number = total_pages
|
|
offset = (page_number - 1) * DEFAULT_PAGE_SIZE
|
|
|
|
jobs = get_jobs(
|
|
limit=DEFAULT_PAGE_SIZE,
|
|
offset=offset,
|
|
status=selected_status or None,
|
|
job_type=selected_type or None,
|
|
target=selected_target or None,
|
|
)
|
|
stats = get_job_stats()
|
|
refresh = ""
|
|
|
|
rows = ""
|
|
for job in jobs:
|
|
job_id = html.escape(str(job.get("id", "")))
|
|
status = render_job_status(job.get("status"))
|
|
source = html.escape(job.get("source", "") or "")
|
|
target_type = html.escape(job.get("target_type", "") or "")
|
|
target_id = html.escape(job.get("target_id", "") or "")
|
|
job_type = html.escape(job.get("type", "") or "")
|
|
created_by = html.escape(job.get("created_by_display_name") or job.get("created_by_username") or "")
|
|
created_at = html.escape(job.get("created_at", "") or "")
|
|
started_at = html.escape(job.get("started_at", "") or "")
|
|
finished_at = html.escape(job.get("finished_at", "") or "")
|
|
row_class = ' class="running-row"' if (job.get("status") or "").lower() == "running" else ""
|
|
|
|
rows += f"""
|
|
<tr{row_class}>
|
|
<td>
|
|
<strong><a href="/portal/jobs/{job_id}">#{job_id}</a></strong><br>
|
|
<span class="muted">{job_type}</span>
|
|
</td>
|
|
<td>{status}</td>
|
|
<td>{source}</td>
|
|
<td>{target_type}: <strong>{target_id}</strong></td>
|
|
<td>{created_by}</td>
|
|
<td>{created_at}</td>
|
|
<td>{started_at}</td>
|
|
<td>{finished_at}</td>
|
|
<td class="actions-cell"><a class="btn" href="/portal/jobs/{job_id}">Detail</a></td>
|
|
</tr>
|
|
"""
|
|
|
|
if not rows:
|
|
rows = '<tr><td colspan="9">Zatím nejsou evidované žádné úlohy.</td></tr>'
|
|
|
|
status_options = render_options(JOB_FILTER_STATUSES, selected_status, "Všechny statusy")
|
|
type_options = render_options(JOB_FILTER_TYPES, selected_type, "Všechny typy")
|
|
target_value = html.escape(selected_target)
|
|
first_item = offset + 1 if total_jobs else 0
|
|
last_item = min(offset + len(jobs), total_jobs)
|
|
|
|
def page_url(page: int) -> str:
|
|
params = {"page": page}
|
|
if selected_status:
|
|
params["status"] = selected_status
|
|
if selected_type:
|
|
params["type"] = selected_type
|
|
if selected_target:
|
|
params["target"] = selected_target
|
|
return f"/portal/jobs?{urlencode(params)}"
|
|
|
|
pagination = f"""
|
|
<div class="pagination">
|
|
<span>Zobrazeno {first_item}-{last_item} z {total_jobs}</span>
|
|
<div class="pagination-actions">
|
|
<a class="btn btn-secondary{' disabled' if page_number <= 1 else ''}" href="{page_url(max(1, page_number - 1))}">Předchozí</a>
|
|
<span class="page-indicator">Strana {page_number} / {total_pages}</span>
|
|
<a class="btn btn-secondary{' disabled' if page_number >= total_pages else ''}" href="{page_url(min(total_pages, page_number + 1))}">Další</a>
|
|
</div>
|
|
</div>
|
|
"""
|
|
|
|
return page(
|
|
"Úlohy",
|
|
f"""
|
|
{refresh}
|
|
<div class="card">
|
|
<h2>Úlohy</h2>
|
|
<p class="muted">Fronta portálových a webhook úloh připravená pro centrální worker.</p>
|
|
</div>
|
|
|
|
<div class="stats-grid">
|
|
<div class="stat-card stat-warning"><span>Čekající</span><strong data-live-count="jobs.queued">{stats.get("queued") or 0}</strong></div>
|
|
<div class="stat-card stat-warning"><span>Běžící</span><strong data-live-count="jobs.running">{stats.get("running") or 0}</strong></div>
|
|
<div class="stat-card stat-danger"><span>Selhané (24 h)</span><strong data-live-count="jobs.failed_24h">{stats.get("failed_24h") or 0}</strong></div>
|
|
<div class="stat-card stat-success"><span>Úspěšné (24 h)</span><strong data-live-count="jobs.success_24h">{stats.get("success_24h") or 0}</strong></div>
|
|
</div>
|
|
|
|
<div class="card">
|
|
<h2>Filtry</h2>
|
|
<form method="get" action="/portal/jobs" class="filter-form">
|
|
<select name="status">{status_options}</select>
|
|
<select name="type">{type_options}</select>
|
|
<input name="target" value="{target_value}" placeholder="Cíl">
|
|
<input type="hidden" name="page" value="1">
|
|
<button type="submit">Filtrovat</button>
|
|
<a class="btn btn-secondary" href="/portal/jobs">Reset</a>
|
|
</form>
|
|
</div>
|
|
|
|
<div class="card">
|
|
<h2>Poslední úlohy</h2>
|
|
<table>
|
|
<tr>
|
|
<th>ID</th>
|
|
<th>Status</th>
|
|
<th>Typ</th>
|
|
<th>Cíl</th>
|
|
<th>Zdroj</th>
|
|
<th>Vytvořeno</th>
|
|
</tr>
|
|
<tbody data-live-table="jobs.recent"></tbody>
|
|
</table>
|
|
</div>
|
|
|
|
<div class="card">
|
|
<h2>Fronta</h2>
|
|
{pagination}
|
|
<table>
|
|
<tr>
|
|
<th>ID</th>
|
|
<th>Status</th>
|
|
<th>Zdroj</th>
|
|
<th>Cíl</th>
|
|
<th>Vytvořil</th>
|
|
<th>Vytvořeno</th>
|
|
<th>Started at</th>
|
|
<th>Finished at</th>
|
|
<th>Akce</th>
|
|
</tr>
|
|
<tbody>{rows}</tbody>
|
|
</table>
|
|
{pagination}
|
|
</div>
|
|
<script src="/portal/static/operations-live.js"></script>
|
|
""",
|
|
user=user,
|
|
)
|
|
|
|
|
|
@router.get("/jobs/{job_id}", response_class=HTMLResponse)
|
|
def job_detail_page(job_id: int, request: Request, user=Depends(require_user)):
|
|
job = get_job(job_id)
|
|
if not job:
|
|
raise HTTPException(status_code=404, detail="Job not found")
|
|
|
|
status_value = (job.get("status") or "").lower()
|
|
refresh = ""
|
|
title = f"Úloha #{html.escape(str(job.get('id', job_id)))}"
|
|
status = render_job_status(job.get("status"))
|
|
target_type = job.get("target_type", "") or ""
|
|
target_id = job.get("target_id", "") or ""
|
|
target_id_html = html.escape(target_id)
|
|
target_url = f"/portal/apps/{quote(target_id, safe='')}" if target_type == "app" else "/portal/jobs"
|
|
duration = html.escape(calculate_duration(job.get("started_at"), job.get("finished_at")))
|
|
payload = html.escape(pretty_json(job.get("payload_json")))
|
|
result = html.escape(pretty_json(job.get("result_json")))
|
|
error_text = html.escape(job.get("error_text", "") or "")
|
|
actions = ""
|
|
retry_blocked_notice = ""
|
|
if status_value == "failed" and can_retry_job(job):
|
|
actions += f"""
|
|
<form method="post" action="/portal/jobs/{html.escape(str(job_id))}/retry" onsubmit="return confirm('Retry job #{html.escape(str(job_id))}?');">
|
|
<button type="submit">Spustit znovu</button>
|
|
</form>
|
|
"""
|
|
elif status_value == "failed" and (job.get("target_id") or "") in IGNORED_RETRY_REPOSITORIES:
|
|
retry_blocked_notice = '<p class="muted">Tento repozitář není deployovatelná služba.</p>'
|
|
if status_value in {"queued", "running"}:
|
|
actions += f"""
|
|
<form method="post" action="/portal/jobs/{html.escape(str(job_id))}/cancel" onsubmit="return confirm('Cancel job #{html.escape(str(job_id))}?');">
|
|
<button type="submit" class="danger">Zrušit úlohu</button>
|
|
</form>
|
|
"""
|
|
if actions:
|
|
actions = f"""
|
|
<div class="inline-form">
|
|
{actions}
|
|
</div>
|
|
"""
|
|
|
|
log_blocks = ""
|
|
for log in get_job_logs(job_id):
|
|
stream = html.escape(log.get("stream", "") or "system")
|
|
created_at = html.escape(log.get("created_at", "") or "")
|
|
message = html.escape(log.get("message", "") or "")
|
|
if stream == "stderr":
|
|
class_name = "log-stderr"
|
|
elif stream == "system":
|
|
class_name = "log-system"
|
|
else:
|
|
class_name = "log-stdout"
|
|
log_blocks += f"""
|
|
<div class="job-log-entry">
|
|
<div class="muted">{created_at} · {stream}</div>
|
|
<pre class="log-viewer {class_name}">{message}</pre>
|
|
</div>
|
|
"""
|
|
|
|
if not log_blocks:
|
|
log_blocks = '<p class="muted">Zatím nejsou uložené žádné logy.</p>'
|
|
|
|
live_logs_script = f"""
|
|
<script>
|
|
(() => {{
|
|
const statusEl = document.getElementById("live-log-status");
|
|
const panel = document.getElementById("live-log-panel");
|
|
if (!statusEl || !panel || !window.WebSocket) {{
|
|
if (statusEl) statusEl.textContent = "Disconnected";
|
|
return;
|
|
}}
|
|
|
|
const protocol = window.location.protocol === "https:" ? "wss:" : "ws:";
|
|
const socket = new WebSocket(`${{protocol}}//${{window.location.host}}/portal/ws/jobs/{html.escape(str(job_id))}/logs`);
|
|
|
|
const appendLog = (item) => {{
|
|
const entry = document.createElement("div");
|
|
entry.className = "job-log-entry";
|
|
|
|
const meta = document.createElement("div");
|
|
meta.className = "muted";
|
|
meta.textContent = `${{item.created_at || ""}} · ${{item.stream || "system"}}`;
|
|
|
|
const pre = document.createElement("pre");
|
|
const stream = item.stream || "system";
|
|
const streamClass = stream === "stderr" ? "log-stderr" : (stream === "system" ? "log-system" : "log-stdout");
|
|
pre.className = `log-viewer ${{streamClass}}`;
|
|
pre.dataset.stream = stream;
|
|
pre.textContent = item.message || "";
|
|
|
|
entry.appendChild(meta);
|
|
entry.appendChild(pre);
|
|
panel.appendChild(entry);
|
|
panel.scrollTop = panel.scrollHeight;
|
|
}};
|
|
|
|
socket.addEventListener("open", () => {{
|
|
statusEl.textContent = "Connected";
|
|
}});
|
|
socket.addEventListener("message", (event) => {{
|
|
const item = JSON.parse(event.data);
|
|
if (item.type === "log") {{
|
|
appendLog(item);
|
|
}}
|
|
if (item.type === "job_finished") {{
|
|
statusEl.textContent = `Finished: ${{item.status}}`;
|
|
socket.close();
|
|
}}
|
|
}});
|
|
socket.addEventListener("close", () => {{
|
|
if (!statusEl.textContent.startsWith("Finished:")) {{
|
|
statusEl.textContent = "Disconnected";
|
|
}}
|
|
}});
|
|
socket.addEventListener("error", () => {{
|
|
statusEl.textContent = "Disconnected";
|
|
}});
|
|
}})();
|
|
</script>
|
|
"""
|
|
|
|
return page(
|
|
title,
|
|
f"""
|
|
{refresh}
|
|
<div class="card">
|
|
<h2>{title}</h2>
|
|
<p>
|
|
<a class="btn" href="/portal/jobs">← Zpět na úlohy</a>
|
|
<a class="btn btn-secondary" href="{target_url}">Detail cíle</a>
|
|
</p>
|
|
{retry_blocked_notice}
|
|
{actions}
|
|
</div>
|
|
|
|
<div class="card">
|
|
<h2>Souhrn</h2>
|
|
<table>
|
|
<tr><th>Typ</th><td>{html.escape(job.get("type", "") or "")}</td></tr>
|
|
<tr><th>Status</th><td>{status}</td></tr>
|
|
<tr><th>Zdroj</th><td>{html.escape(job.get("source", "") or "")}</td></tr>
|
|
<tr><th>Cíl</th><td>{html.escape(target_type)}: {target_id_html}</td></tr>
|
|
<tr><th>Vytvořil</th><td>{html.escape(job.get("created_by_display_name") or job.get("created_by_username") or "")}</td></tr>
|
|
<tr><th>Vytvořeno</th><td>{html.escape(job.get("created_at", "") or "")}</td></tr>
|
|
<tr><th>Started at</th><td>{html.escape(job.get("started_at", "") or "")}</td></tr>
|
|
<tr><th>Finished at</th><td>{html.escape(job.get("finished_at", "") or "")}</td></tr>
|
|
<tr><th>Duration</th><td>{duration}</td></tr>
|
|
<tr><th>Worker</th><td>{html.escape(job.get("worker_id", "") or "")}</td></tr>
|
|
</table>
|
|
</div>
|
|
|
|
<div class="grid">
|
|
<div class="card">
|
|
<h2>Payload</h2>
|
|
<pre class="log-viewer log-stdout">{payload}</pre>
|
|
</div>
|
|
<div class="card">
|
|
<h2>Result / error</h2>
|
|
<pre class="log-viewer log-stdout">{result}</pre>
|
|
<pre class="log-viewer log-stderr">{error_text}</pre>
|
|
</div>
|
|
</div>
|
|
|
|
<div class="card">
|
|
<h2>Živé logy</h2>
|
|
<p class="muted">WebSocket: <span id="live-log-status">Připojování</span></p>
|
|
<div id="live-log-panel" class="log-viewer log-stdout"></div>
|
|
</div>
|
|
|
|
<div class="card">
|
|
<h2>Logy z DB</h2>
|
|
{log_blocks}
|
|
</div>
|
|
{live_logs_script}
|
|
""",
|
|
user=user,
|
|
)
|
|
|
|
|
|
@router.post("/jobs/{job_id}/retry")
|
|
def retry_job_action(job_id: int, user=Depends(require_user)):
|
|
job = get_job(job_id)
|
|
if not job:
|
|
raise HTTPException(status_code=404, detail="Job not found")
|
|
if (job.get("status") or "").lower() != "failed":
|
|
raise HTTPException(status_code=400, detail="Only failed jobs can be retried")
|
|
if not can_retry_job(job):
|
|
raise HTTPException(status_code=400, detail="This job cannot be retried")
|
|
|
|
new_job_id = retry_failed_job(job_id, user)
|
|
if not new_job_id:
|
|
raise HTTPException(status_code=409, detail="Job can no longer be retried")
|
|
|
|
log_audit_event(
|
|
user,
|
|
action="job.retry",
|
|
target_type="job",
|
|
target_id=job_id,
|
|
metadata={"new_job_id": new_job_id},
|
|
)
|
|
|
|
return RedirectResponse(url=f"/portal/jobs/{new_job_id}", status_code=303)
|
|
|
|
|
|
@router.post("/jobs/{job_id}/cancel")
|
|
def cancel_job_action(job_id: int, user=Depends(require_user)):
|
|
job = get_job(job_id)
|
|
if not job:
|
|
raise HTTPException(status_code=404, detail="Job not found")
|
|
|
|
status_value = (job.get("status") or "").lower()
|
|
if status_value not in {"queued", "running"}:
|
|
raise HTTPException(status_code=400, detail="Only queued or running jobs can be cancelled")
|
|
|
|
result = cancel_job(job_id)
|
|
if not result:
|
|
raise HTTPException(status_code=409, detail="Job can no longer be cancelled")
|
|
|
|
log_audit_event(
|
|
user,
|
|
action="job.cancel",
|
|
target_type="job",
|
|
target_id=job_id,
|
|
metadata=result,
|
|
)
|
|
|
|
return RedirectResponse(url=f"/portal/jobs/{job_id}", status_code=303)
|