appfactory-portal/app/routes/jobs.py

import asyncio
import html
import json
import math
from urllib.parse import quote, urlencode

from fastapi import APIRouter, Depends, HTTPException, Query, Request, WebSocket, WebSocketDisconnect
from fastapi.responses import HTMLResponse, RedirectResponse

from app.auth import current_user, require_user
from app.db.audit import log_audit_event
from app.db.jobs import cancel_job, count_jobs, get_job, get_job_logs, get_job_logs_after, get_job_stats, get_jobs, retry_failed_job
from app.routes.deployments import calculate_duration, render_status_pill
from app.templates.layout import page

router = APIRouter()

LIVE_STATUSES = {"queued", "running", "cancelled_requested"}
FINISHED_STATUSES = {"success", "failed", "cancelled"}
JOB_STATUSES = {"queued", "running", "cancelled_requested", "cancelled", "success", "failed"}
JOB_FILTER_STATUSES = ("queued", "running", "success", "failed", "cancelled")
JOB_FILTER_TYPES = ("deploy_app", "deploy_core_service")
IGNORED_RETRY_REPOSITORIES = {"appfactory-tools", "appfactory-infrastructure"}
DEFAULT_PAGE_SIZE = 20


def render_job_status(status: str | None) -> str:
    status_value = status or ""
    normalized = status_value.lower()
    if normalized not in JOB_STATUSES:
        return render_status_pill(status)

    if normalized == "success":
        class_name = "pill pill-success"
    elif normalized in {"queued", "running", "cancelled_requested"}:
        class_name = "pill pill-warning"
    elif normalized in {"failed", "cancelled"}:
        class_name = "pill pill-danger"
    else:
        class_name = "pill pill-muted"

    return f'<span class="{class_name}">{html.escape(status_value)}</span>'


def pretty_json(value: str | None) -> str:
    if not value:
        return "{}"

    try:
        return json.dumps(json.loads(value), ensure_ascii=False, indent=2, sort_keys=True)
    except (TypeError, ValueError):
        return value


def render_options(values: tuple[str, ...], selected: str, empty_label: str) -> str:
    options = [f'<option value="">{html.escape(empty_label)}</option>']
    for value in values:
        selected_attr = " selected" if selected == value else ""
        escaped_value = html.escape(value)
        options.append(f'<option value="{escaped_value}"{selected_attr}>{escaped_value}</option>')
    return "".join(options)


def can_retry_job(job: dict) -> bool:
    job_type = job.get("type") or ""
    target_id = job.get("target_id") or ""

    if job_type == "deploy_core_service":
        return True
    if job_type == "deploy_app" and target_id not in IGNORED_RETRY_REPOSITORIES:
        return True
    return False


def log_message_payload(log: dict) -> dict:
    return {
        "type": "log",
        "id": log.get("id"),
        "stream": log.get("stream") or "system",
        "message": log.get("message") or "",
        "created_at": log.get("created_at") or "",
    }


@router.websocket("/ws/jobs/{job_id}/logs")
async def job_logs_websocket(websocket: WebSocket, job_id: int):
    user = current_user(websocket)
    if not user:
        await websocket.close(code=1008)
        return

    job = get_job(job_id)
    if not job:
        await websocket.close(code=1008)
        return

    await websocket.accept()
    log_audit_event(
        user,
        action="job.logs.live.view",
        target_type="job",
        target_id=job_id,
    )

    last_log_id = 0
    try:
        while True:
            while True:
                logs = get_job_logs_after(job_id, last_log_id)
                if not logs:
                    break

                for log in logs:
                    last_log_id = max(last_log_id, int(log.get("id") or 0))
                    await websocket.send_json(log_message_payload(log))

            job = get_job(job_id)
            status_value = (job.get("status") or "").lower() if job else ""
            if status_value in FINISHED_STATUSES:
                await websocket.send_json({"type": "job_finished", "status": status_value})
                await websocket.close()
                return

            await asyncio.sleep(1)
    except WebSocketDisconnect:
        return


@router.get("/jobs", response_class=HTMLResponse)
def jobs_page(
    request: Request,
    status: str = Query(""),
    job_type: str = Query("", alias="type"),
    target: str = Query(""),
    page_number: int = Query(1, alias="page", ge=1),
    user=Depends(require_user),
):
    selected_status = status.strip().lower()
    selected_type = job_type.strip()
    selected_target = target.strip()
    if selected_status not in JOB_FILTER_STATUSES:
        selected_status = ""
    if selected_type not in JOB_FILTER_TYPES:
        selected_type = ""

    total_jobs = count_jobs(
        status=selected_status or None,
        job_type=selected_type or None,
        target=selected_target or None,
    )
    total_pages = max(1, math.ceil(total_jobs / DEFAULT_PAGE_SIZE))
    if page_number > total_pages:
        page_number = total_pages
    offset = (page_number - 1) * DEFAULT_PAGE_SIZE

    jobs = get_jobs(
        limit=DEFAULT_PAGE_SIZE,
        offset=offset,
        status=selected_status or None,
        job_type=selected_type or None,
        target=selected_target or None,
    )
    stats = get_job_stats()
    refresh = ""

    rows = ""
    for job in jobs:
        job_id = html.escape(str(job.get("id", "")))
        status = render_job_status(job.get("status"))
        source = html.escape(job.get("source", "") or "")
        target_type = html.escape(job.get("target_type", "") or "")
        target_id = html.escape(job.get("target_id", "") or "")
        job_type = html.escape(job.get("type", "") or "")
        created_by = html.escape(job.get("created_by_display_name") or job.get("created_by_username") or "")
        created_at = html.escape(job.get("created_at", "") or "")
        started_at = html.escape(job.get("started_at", "") or "")
        finished_at = html.escape(job.get("finished_at", "") or "")
        row_class = ' class="running-row"' if (job.get("status") or "").lower() == "running" else ""

        rows += f"""
        <tr{row_class}>
            <td>
                <strong><a href="/portal/jobs/{job_id}">#{job_id}</a></strong><br>
                <span class="muted">{job_type}</span>
            </td>
            <td>{status}</td>
            <td>{source}</td>
            <td>{target_type}: <strong>{target_id}</strong></td>
            <td>{created_by}</td>
            <td>{created_at}</td>
            <td>{started_at}</td>
            <td>{finished_at}</td>
            <td class="actions-cell"><a class="btn" href="/portal/jobs/{job_id}">Detail</a></td>
        </tr>
        """

    if not rows:
        rows = '<tr><td colspan="9">Zatím nejsou evidované žádné joby.</td></tr>'

    status_options = render_options(JOB_FILTER_STATUSES, selected_status, "Všechny statusy")
    type_options = render_options(JOB_FILTER_TYPES, selected_type, "Všechny typy")
    target_value = html.escape(selected_target)
    first_item = offset + 1 if total_jobs else 0
    last_item = min(offset + len(jobs), total_jobs)

    def page_url(page: int) -> str:
        params = {"page": page}
        if selected_status:
            params["status"] = selected_status
        if selected_type:
            params["type"] = selected_type
        if selected_target:
            params["target"] = selected_target
        return f"/portal/jobs?{urlencode(params)}"

    pagination = f"""
        <div class="pagination">
            <span>Zobrazeno {first_item}-{last_item} z {total_jobs}</span>
            <div class="pagination-actions">
                <a class="btn btn-secondary{' disabled' if page_number <= 1 else ''}" href="{page_url(max(1, page_number - 1))}">Předchozí</a>
                <span class="page-indicator">Strana {page_number} / {total_pages}</span>
                <a class="btn btn-secondary{' disabled' if page_number >= total_pages else ''}" href="{page_url(min(total_pages, page_number + 1))}">Další</a>
            </div>
        </div>
    """

    return page(
        "Joby",
        f"""
        {refresh}
        <div class="card">
            <h2>Joby</h2>
            <p class="muted">Fronta portálových a webhook úloh připravená pro centrální worker.</p>
        </div>

        <div class="stats-grid">
            <div class="stat-card stat-warning"><span>Queued</span><strong data-live-count="jobs.queued">{stats.get("queued") or 0}</strong></div>
            <div class="stat-card stat-warning"><span>Running</span><strong data-live-count="jobs.running">{stats.get("running") or 0}</strong></div>
            <div class="stat-card stat-danger"><span>Failed (24h)</span><strong data-live-count="jobs.failed_24h">{stats.get("failed_24h") or 0}</strong></div>
            <div class="stat-card stat-success"><span>Success (24h)</span><strong data-live-count="jobs.success_24h">{stats.get("success_24h") or 0}</strong></div>
        </div>

        <div class="card">
            <h2>Filtry</h2>
            <form method="get" action="/portal/jobs" class="filter-form">
                <select name="status">{status_options}</select>
                <select name="type">{type_options}</select>
                <input name="target" value="{target_value}" placeholder="Target">
                <input type="hidden" name="page" value="1">
                <button type="submit">Filtrovat</button>
                <a class="btn btn-secondary" href="/portal/jobs">Reset</a>
            </form>
        </div>

        <div class="card">
            <h2>Recent Jobs</h2>
            <table>
                <tr>
                    <th>ID</th>
                    <th>Status</th>
                    <th>Type</th>
                    <th>Target</th>
                    <th>Source</th>
                    <th>Created At</th>
                </tr>
                <tbody data-live-table="jobs.recent"></tbody>
            </table>
        </div>

        <div class="card">
            <h2>Fronta</h2>
            {pagination}
            <table>
                <tr>
                    <th>ID</th>
                    <th>Status</th>
                    <th>Source</th>
                    <th>Target</th>
                    <th>Created by</th>
                    <th>Created at</th>
                    <th>Started at</th>
                    <th>Finished at</th>
                    <th>Akce</th>
                </tr>
                <tbody>{rows}</tbody>
            </table>
            {pagination}
        </div>
        <script src="/portal/static/operations-live.js"></script>
        """,
        user=user,
    )


@router.get("/jobs/{job_id}", response_class=HTMLResponse)
def job_detail_page(job_id: int, request: Request, user=Depends(require_user)):
    job = get_job(job_id)
    if not job:
        raise HTTPException(status_code=404, detail="Job not found")

    status_value = (job.get("status") or "").lower()
    refresh = ""
    title = f"Job #{html.escape(str(job.get('id', job_id)))}"
    status = render_job_status(job.get("status"))
    target_type = job.get("target_type", "") or ""
    target_id = job.get("target_id", "") or ""
    target_id_html = html.escape(target_id)
    target_url = f"/portal/apps/{quote(target_id, safe='')}" if target_type == "app" else "/portal/jobs"
    duration = html.escape(calculate_duration(job.get("started_at"), job.get("finished_at")))
    payload = html.escape(pretty_json(job.get("payload_json")))
    result = html.escape(pretty_json(job.get("result_json")))
    error_text = html.escape(job.get("error_text", "") or "")
    actions = ""
    retry_blocked_notice = ""
    if status_value == "failed" and can_retry_job(job):
        actions += f"""
                <form method="post" action="/portal/jobs/{html.escape(str(job_id))}/retry" onsubmit="return confirm('Retry job #{html.escape(str(job_id))}?');">
                    <button type="submit">Retry Job</button>
                </form>
        """
    elif status_value == "failed" and (job.get("target_id") or "") in IGNORED_RETRY_REPOSITORIES:
        retry_blocked_notice = '<p class="muted">Tento repozitář není deployovatelná služba.</p>'
    if status_value in {"queued", "running"}:
        actions += f"""
                <form method="post" action="/portal/jobs/{html.escape(str(job_id))}/cancel" onsubmit="return confirm('Cancel job #{html.escape(str(job_id))}?');">
                    <button type="submit" class="danger">Cancel Job</button>
                </form>
        """
    if actions:
        actions = f"""
            <div class="inline-form">
                {actions}
            </div>
        """

    log_blocks = ""
    for log in get_job_logs(job_id):
        stream = html.escape(log.get("stream", "") or "system")
        created_at = html.escape(log.get("created_at", "") or "")
        message = html.escape(log.get("message", "") or "")
        if stream == "stderr":
            class_name = "log-stderr"
        elif stream == "system":
            class_name = "log-system"
        else:
            class_name = "log-stdout"
        log_blocks += f"""
        <div class="job-log-entry">
            <div class="muted">{created_at} · {stream}</div>
            <pre class="log-viewer {class_name}">{message}</pre>
        </div>
        """

    if not log_blocks:
        log_blocks = '<p class="muted">Zatím nejsou uložené žádné logy.</p>'

    live_logs_script = f"""
        <script>
        (() => {{
            const statusEl = document.getElementById("live-log-status");
            const panel = document.getElementById("live-log-panel");
            if (!statusEl || !panel || !window.WebSocket) {{
                if (statusEl) statusEl.textContent = "Disconnected";
                return;
            }}

            const protocol = window.location.protocol === "https:" ? "wss:" : "ws:";
            const socket = new WebSocket(`${{protocol}}//${{window.location.host}}/portal/ws/jobs/{html.escape(str(job_id))}/logs`);

            const appendLog = (item) => {{
                const entry = document.createElement("div");
                entry.className = "job-log-entry";

                const meta = document.createElement("div");
                meta.className = "muted";
                meta.textContent = `${{item.created_at || ""}} · ${{item.stream || "system"}}`;

                const pre = document.createElement("pre");
                const stream = item.stream || "system";
                const streamClass = stream === "stderr" ? "log-stderr" : (stream === "system" ? "log-system" : "log-stdout");
                pre.className = `log-viewer ${{streamClass}}`;
                pre.dataset.stream = stream;
                pre.textContent = item.message || "";

                entry.appendChild(meta);
                entry.appendChild(pre);
                panel.appendChild(entry);
                panel.scrollTop = panel.scrollHeight;
            }};

            socket.addEventListener("open", () => {{
                statusEl.textContent = "Connected";
            }});
            socket.addEventListener("message", (event) => {{
                const item = JSON.parse(event.data);
                if (item.type === "log") {{
                    appendLog(item);
                }}
                if (item.type === "job_finished") {{
                    statusEl.textContent = `Finished: ${{item.status}}`;
                    socket.close();
                }}
            }});
            socket.addEventListener("close", () => {{
                if (!statusEl.textContent.startsWith("Finished:")) {{
                    statusEl.textContent = "Disconnected";
                }}
            }});
            socket.addEventListener("error", () => {{
                statusEl.textContent = "Disconnected";
            }});
        }})();
        </script>
    """

    return page(
        title,
        f"""
        {refresh}
        <div class="card">
            <h2>{title}</h2>
            <p>
                <a class="btn" href="/portal/jobs">&larr; Zpět na joby</a>
                <a class="btn btn-secondary" href="{target_url}">Detail targetu</a>
            </p>
            {retry_blocked_notice}
            {actions}
        </div>

        <div class="card">
            <h2>Souhrn</h2>
            <table>
                <tr><th>Typ</th><td>{html.escape(job.get("type", "") or "")}</td></tr>
                <tr><th>Status</th><td>{status}</td></tr>
                <tr><th>Source</th><td>{html.escape(job.get("source", "") or "")}</td></tr>
                <tr><th>Target</th><td>{html.escape(target_type)}: {target_id_html}</td></tr>
                <tr><th>Created by</th><td>{html.escape(job.get("created_by_display_name") or job.get("created_by_username") or "")}</td></tr>
                <tr><th>Created at</th><td>{html.escape(job.get("created_at", "") or "")}</td></tr>
                <tr><th>Started at</th><td>{html.escape(job.get("started_at", "") or "")}</td></tr>
                <tr><th>Finished at</th><td>{html.escape(job.get("finished_at", "") or "")}</td></tr>
                <tr><th>Duration</th><td>{duration}</td></tr>
                <tr><th>Worker</th><td>{html.escape(job.get("worker_id", "") or "")}</td></tr>
            </table>
        </div>

        <div class="grid">
            <div class="card">
                <h2>Payload</h2>
                <pre class="log-viewer log-stdout">{payload}</pre>
            </div>
            <div class="card">
                <h2>Result / error</h2>
                <pre class="log-viewer log-stdout">{result}</pre>
                <pre class="log-viewer log-stderr">{error_text}</pre>
            </div>
        </div>

        <div class="card">
            <h2>Live logy</h2>
            <p class="muted">WebSocket: <span id="live-log-status">Connecting</span></p>
            <div id="live-log-panel" class="log-viewer log-stdout"></div>
        </div>

        <div class="card">
            <h2>Logy z DB</h2>
            {log_blocks}
        </div>
        {live_logs_script}
        """,
        user=user,
    )


@router.post("/jobs/{job_id}/retry")
def retry_job_action(job_id: int, user=Depends(require_user)):
    job = get_job(job_id)
    if not job:
        raise HTTPException(status_code=404, detail="Job not found")
    if (job.get("status") or "").lower() != "failed":
        raise HTTPException(status_code=400, detail="Only failed jobs can be retried")
    if not can_retry_job(job):
        raise HTTPException(status_code=400, detail="This job cannot be retried")

    new_job_id = retry_failed_job(job_id, user)
    if not new_job_id:
        raise HTTPException(status_code=409, detail="Job can no longer be retried")

    log_audit_event(
        user,
        action="job.retry",
        target_type="job",
        target_id=job_id,
        metadata={"new_job_id": new_job_id},
    )

    return RedirectResponse(url=f"/portal/jobs/{new_job_id}", status_code=303)


@router.post("/jobs/{job_id}/cancel")
def cancel_job_action(job_id: int, user=Depends(require_user)):
    job = get_job(job_id)
    if not job:
        raise HTTPException(status_code=404, detail="Job not found")

    status_value = (job.get("status") or "").lower()
    if status_value not in {"queued", "running"}:
        raise HTTPException(status_code=400, detail="Only queued or running jobs can be cancelled")

    result = cancel_job(job_id)
    if not result:
        raise HTTPException(status_code=409, detail="Job can no longer be cancelled")

    log_audit_event(
        user,
        action="job.cancel",
        target_type="job",
        target_id=job_id,
        metadata=result,
    )

    return RedirectResponse(url=f"/portal/jobs/{job_id}", status_code=303)