added jobs handling.
This commit is contained in:
+115
@@ -56,6 +56,121 @@ def create_job(
|
|||||||
return job_id
|
return job_id
|
||||||
|
|
||||||
|
|
||||||
|
def retry_failed_job(job_id: int, user: dict[str, Any]):
|
||||||
|
run_migrations()
|
||||||
|
created_by_user_id = user.get("id") if user else None
|
||||||
|
created_by_username = user.get("username") if user else None
|
||||||
|
created_by_display_name = user.get("display_name") if user else None
|
||||||
|
|
||||||
|
con = get_connection()
|
||||||
|
source_job = con.execute(
|
||||||
|
"""
|
||||||
|
SELECT id
|
||||||
|
FROM jobs
|
||||||
|
WHERE id = ?
|
||||||
|
AND status = 'failed'
|
||||||
|
""",
|
||||||
|
(job_id,),
|
||||||
|
).fetchone()
|
||||||
|
if not source_job:
|
||||||
|
con.close()
|
||||||
|
return None
|
||||||
|
|
||||||
|
cur = con.execute(
|
||||||
|
"""
|
||||||
|
INSERT INTO jobs (
|
||||||
|
type,
|
||||||
|
target_type,
|
||||||
|
target_id,
|
||||||
|
payload_json,
|
||||||
|
status,
|
||||||
|
created_by_user_id,
|
||||||
|
created_by_username,
|
||||||
|
created_by_display_name,
|
||||||
|
source
|
||||||
|
)
|
||||||
|
SELECT
|
||||||
|
type,
|
||||||
|
target_type,
|
||||||
|
target_id,
|
||||||
|
payload_json,
|
||||||
|
'queued',
|
||||||
|
?,
|
||||||
|
?,
|
||||||
|
?,
|
||||||
|
'portal_retry'
|
||||||
|
FROM jobs
|
||||||
|
WHERE id = ?
|
||||||
|
AND status = 'failed'
|
||||||
|
""",
|
||||||
|
(created_by_user_id, created_by_username, created_by_display_name, job_id),
|
||||||
|
)
|
||||||
|
if cur.rowcount != 1:
|
||||||
|
con.close()
|
||||||
|
return None
|
||||||
|
|
||||||
|
new_job_id = cur.lastrowid
|
||||||
|
con.commit()
|
||||||
|
con.close()
|
||||||
|
|
||||||
|
append_job_log(new_job_id, "system", f"Job queued as retry of #{job_id}.")
|
||||||
|
return new_job_id
|
||||||
|
|
||||||
|
|
||||||
|
def cancel_job(job_id: int):
|
||||||
|
run_migrations()
|
||||||
|
con = get_connection()
|
||||||
|
|
||||||
|
row = con.execute(
|
||||||
|
"""
|
||||||
|
SELECT status
|
||||||
|
FROM jobs
|
||||||
|
WHERE id = ?
|
||||||
|
""",
|
||||||
|
(job_id,),
|
||||||
|
).fetchone()
|
||||||
|
if not row:
|
||||||
|
con.close()
|
||||||
|
return None
|
||||||
|
|
||||||
|
status = (row["status"] or "").lower()
|
||||||
|
if status == "queued":
|
||||||
|
new_status = "cancelled"
|
||||||
|
cur = con.execute(
|
||||||
|
"""
|
||||||
|
UPDATE jobs
|
||||||
|
SET status = 'cancelled',
|
||||||
|
finished_at = CURRENT_TIMESTAMP
|
||||||
|
WHERE id = ?
|
||||||
|
AND status = 'queued'
|
||||||
|
""",
|
||||||
|
(job_id,),
|
||||||
|
)
|
||||||
|
elif status == "running":
|
||||||
|
new_status = "cancelled_requested"
|
||||||
|
cur = con.execute(
|
||||||
|
"""
|
||||||
|
UPDATE jobs
|
||||||
|
SET status = 'cancelled_requested'
|
||||||
|
WHERE id = ?
|
||||||
|
AND status = 'running'
|
||||||
|
""",
|
||||||
|
(job_id,),
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
con.close()
|
||||||
|
return None
|
||||||
|
|
||||||
|
if cur.rowcount != 1:
|
||||||
|
con.close()
|
||||||
|
return None
|
||||||
|
|
||||||
|
con.commit()
|
||||||
|
con.close()
|
||||||
|
append_job_log(job_id, "system", f"Job status changed from {status} to {new_status}.")
|
||||||
|
return {"previous_status": status, "status": new_status}
|
||||||
|
|
||||||
|
|
||||||
def update_job_status(
|
def update_job_status(
|
||||||
job_id: int,
|
job_id: int,
|
||||||
status: str,
|
status: str,
|
||||||
|
|||||||
+87
-3
@@ -3,21 +3,37 @@ import json
|
|||||||
from urllib.parse import quote
|
from urllib.parse import quote
|
||||||
|
|
||||||
from fastapi import APIRouter, Depends, HTTPException, Request
|
from fastapi import APIRouter, Depends, HTTPException, Request
|
||||||
from fastapi.responses import HTMLResponse
|
from fastapi.responses import HTMLResponse, RedirectResponse
|
||||||
|
|
||||||
from app.auth import require_user
|
from app.auth import require_user
|
||||||
from app.db.jobs import get_job, get_job_logs, get_jobs
|
from app.db.audit import log_audit_event
|
||||||
|
from app.db.jobs import cancel_job, get_job, get_job_logs, get_jobs, retry_failed_job
|
||||||
from app.routes.deployments import calculate_duration, render_status_pill
|
from app.routes.deployments import calculate_duration, render_status_pill
|
||||||
from app.templates.layout import page
|
from app.templates.layout import page
|
||||||
|
|
||||||
router = APIRouter()
|
router = APIRouter()
|
||||||
|
|
||||||
LIVE_STATUSES = {"queued", "running"}
|
LIVE_STATUSES = {"queued", "running", "cancelled_requested"}
|
||||||
|
JOB_STATUSES = {"queued", "running", "cancelled_requested", "cancelled", "success", "failed"}
|
||||||
|
|
||||||
|
|
||||||
def render_job_status(status: str | None) -> str:
|
def render_job_status(status: str | None) -> str:
|
||||||
|
status_value = status or ""
|
||||||
|
normalized = status_value.lower()
|
||||||
|
if normalized not in JOB_STATUSES:
|
||||||
return render_status_pill(status)
|
return render_status_pill(status)
|
||||||
|
|
||||||
|
if normalized == "success":
|
||||||
|
class_name = "pill pill-success"
|
||||||
|
elif normalized in {"queued", "running", "cancelled_requested"}:
|
||||||
|
class_name = "pill pill-warning"
|
||||||
|
elif normalized in {"failed", "cancelled"}:
|
||||||
|
class_name = "pill pill-danger"
|
||||||
|
else:
|
||||||
|
class_name = "pill pill-muted"
|
||||||
|
|
||||||
|
return f'<span class="{class_name}">{html.escape(status_value)}</span>'
|
||||||
|
|
||||||
|
|
||||||
def pretty_json(value: str | None) -> str:
|
def pretty_json(value: str | None) -> str:
|
||||||
if not value:
|
if not value:
|
||||||
@@ -119,6 +135,25 @@ def job_detail_page(job_id: int, request: Request, user=Depends(require_user)):
|
|||||||
payload = html.escape(pretty_json(job.get("payload_json")))
|
payload = html.escape(pretty_json(job.get("payload_json")))
|
||||||
result = html.escape(pretty_json(job.get("result_json")))
|
result = html.escape(pretty_json(job.get("result_json")))
|
||||||
error_text = html.escape(job.get("error_text", "") or "")
|
error_text = html.escape(job.get("error_text", "") or "")
|
||||||
|
actions = ""
|
||||||
|
if status_value == "failed":
|
||||||
|
actions += f"""
|
||||||
|
<form method="post" action="/portal/jobs/{html.escape(str(job_id))}/retry" onsubmit="return confirm('Retry job #{html.escape(str(job_id))}?');">
|
||||||
|
<button type="submit">Retry Job</button>
|
||||||
|
</form>
|
||||||
|
"""
|
||||||
|
if status_value in {"queued", "running"}:
|
||||||
|
actions += f"""
|
||||||
|
<form method="post" action="/portal/jobs/{html.escape(str(job_id))}/cancel" onsubmit="return confirm('Cancel job #{html.escape(str(job_id))}?');">
|
||||||
|
<button type="submit" class="danger">Cancel Job</button>
|
||||||
|
</form>
|
||||||
|
"""
|
||||||
|
if actions:
|
||||||
|
actions = f"""
|
||||||
|
<div class="inline-form">
|
||||||
|
{actions}
|
||||||
|
</div>
|
||||||
|
"""
|
||||||
|
|
||||||
log_blocks = ""
|
log_blocks = ""
|
||||||
for log in get_job_logs(job_id):
|
for log in get_job_logs(job_id):
|
||||||
@@ -146,6 +181,7 @@ def job_detail_page(job_id: int, request: Request, user=Depends(require_user)):
|
|||||||
<a class="btn" href="/portal/jobs">← Zpět na joby</a>
|
<a class="btn" href="/portal/jobs">← Zpět na joby</a>
|
||||||
<a class="btn btn-secondary" href="{target_url}">Detail targetu</a>
|
<a class="btn btn-secondary" href="{target_url}">Detail targetu</a>
|
||||||
</p>
|
</p>
|
||||||
|
{actions}
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
<div class="card">
|
<div class="card">
|
||||||
@@ -183,3 +219,51 @@ def job_detail_page(job_id: int, request: Request, user=Depends(require_user)):
|
|||||||
""",
|
""",
|
||||||
user=user,
|
user=user,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/jobs/{job_id}/retry")
|
||||||
|
def retry_job_action(job_id: int, user=Depends(require_user)):
|
||||||
|
job = get_job(job_id)
|
||||||
|
if not job:
|
||||||
|
raise HTTPException(status_code=404, detail="Job not found")
|
||||||
|
if (job.get("status") or "").lower() != "failed":
|
||||||
|
raise HTTPException(status_code=400, detail="Only failed jobs can be retried")
|
||||||
|
|
||||||
|
new_job_id = retry_failed_job(job_id, user)
|
||||||
|
if not new_job_id:
|
||||||
|
raise HTTPException(status_code=409, detail="Job can no longer be retried")
|
||||||
|
|
||||||
|
log_audit_event(
|
||||||
|
user,
|
||||||
|
action="job.retry",
|
||||||
|
target_type="job",
|
||||||
|
target_id=job_id,
|
||||||
|
metadata={"new_job_id": new_job_id},
|
||||||
|
)
|
||||||
|
|
||||||
|
return RedirectResponse(url=f"/portal/jobs/{new_job_id}", status_code=303)
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/jobs/{job_id}/cancel")
|
||||||
|
def cancel_job_action(job_id: int, user=Depends(require_user)):
|
||||||
|
job = get_job(job_id)
|
||||||
|
if not job:
|
||||||
|
raise HTTPException(status_code=404, detail="Job not found")
|
||||||
|
|
||||||
|
status_value = (job.get("status") or "").lower()
|
||||||
|
if status_value not in {"queued", "running"}:
|
||||||
|
raise HTTPException(status_code=400, detail="Only queued or running jobs can be cancelled")
|
||||||
|
|
||||||
|
result = cancel_job(job_id)
|
||||||
|
if not result:
|
||||||
|
raise HTTPException(status_code=409, detail="Job can no longer be cancelled")
|
||||||
|
|
||||||
|
log_audit_event(
|
||||||
|
user,
|
||||||
|
action="job.cancel",
|
||||||
|
target_type="job",
|
||||||
|
target_id=job_id,
|
||||||
|
metadata=result,
|
||||||
|
)
|
||||||
|
|
||||||
|
return RedirectResponse(url=f"/portal/jobs/{job_id}", status_code=303)
|
||||||
|
|||||||
Reference in New Issue
Block a user