Add alert script job type
This commit is contained in:
+85
-11
@@ -13,6 +13,7 @@ from fastapi import FastAPI
|
|||||||
DB_FILE = Path("/opt/appfactory/data/appfactory/appfactory.db")
|
DB_FILE = Path("/opt/appfactory/data/appfactory/appfactory.db")
|
||||||
TOOLS_DIR = Path("/tools")
|
TOOLS_DIR = Path("/tools")
|
||||||
MAINTENANCE_DIR = Path("/opt/appfactory/workspace/appfactory-tools/maintenance")
|
MAINTENANCE_DIR = Path("/opt/appfactory/workspace/appfactory-tools/maintenance")
|
||||||
|
ALERTS_DIR = Path("/opt/appfactory/workspace/appfactory-tools/alerts")
|
||||||
|
|
||||||
WORKER_ID = os.getenv("APPFACTORY_WORKER_ID", socket.gethostname())
|
WORKER_ID = os.getenv("APPFACTORY_WORKER_ID", socket.gethostname())
|
||||||
POLL_SECONDS = int(os.getenv("APPFACTORY_WORKER_POLL_SECONDS", "3"))
|
POLL_SECONDS = int(os.getenv("APPFACTORY_WORKER_POLL_SECONDS", "3"))
|
||||||
@@ -348,9 +349,28 @@ def build_env(job: dict):
|
|||||||
env["APPFACTORY_PUSHER"] = str(payload.get("pusher") or "")
|
env["APPFACTORY_PUSHER"] = str(payload.get("pusher") or "")
|
||||||
env["APPFACTORY_COMMIT_SHA"] = str(payload.get("commit_sha") or "")
|
env["APPFACTORY_COMMIT_SHA"] = str(payload.get("commit_sha") or "")
|
||||||
|
|
||||||
|
if job.get("type") == "run_alert_script":
|
||||||
|
env["APPFACTORY_ALERT_EVENT_ID"] = str(payload.get("alert_event_id") or "")
|
||||||
|
env["APPFACTORY_ALERT_RULE_ID"] = str(payload.get("rule_id") or "")
|
||||||
|
env["APPFACTORY_ALERT_EVENT_TYPE"] = str(payload.get("event_type") or "")
|
||||||
|
env["APPFACTORY_ALERT_SERVICE_ID"] = str(payload.get("service_id") or "")
|
||||||
|
env["APPFACTORY_ALERT_INCIDENT_ID"] = str(payload.get("incident_id") or "")
|
||||||
|
env["APPFACTORY_ALERT_PAYLOAD_JSON"] = json.dumps(payload.get("alert_payload") or {}, ensure_ascii=False)
|
||||||
|
|
||||||
return env
|
return env
|
||||||
|
|
||||||
|
|
||||||
|
def validate_script_name(script_name: str):
|
||||||
|
if not script_name:
|
||||||
|
raise ValueError("Missing script_name")
|
||||||
|
|
||||||
|
if "/" in script_name or "\\" in script_name or ".." in script_name:
|
||||||
|
raise ValueError("Invalid script name")
|
||||||
|
|
||||||
|
if not script_name.endswith(".sh"):
|
||||||
|
raise ValueError("Script must end with .sh")
|
||||||
|
|
||||||
|
|
||||||
def command_for_job(job: dict):
|
def command_for_job(job: dict):
|
||||||
job_type = job["type"]
|
job_type = job["type"]
|
||||||
target_id = job["target_id"]
|
target_id = job["target_id"]
|
||||||
@@ -365,14 +385,7 @@ def command_for_job(job: dict):
|
|||||||
payload = json.loads(job.get("payload_json") or "{}")
|
payload = json.loads(job.get("payload_json") or "{}")
|
||||||
script_name = payload.get("script_name") or target_id
|
script_name = payload.get("script_name") or target_id
|
||||||
|
|
||||||
if not script_name:
|
validate_script_name(script_name)
|
||||||
raise ValueError("Missing script_name")
|
|
||||||
|
|
||||||
if "/" in script_name or "\\" in script_name or ".." in script_name:
|
|
||||||
raise ValueError("Invalid script name")
|
|
||||||
|
|
||||||
if not script_name.endswith(".sh"):
|
|
||||||
raise ValueError("Script must end with .sh")
|
|
||||||
|
|
||||||
script_path = MAINTENANCE_DIR / script_name
|
script_path = MAINTENANCE_DIR / script_name
|
||||||
|
|
||||||
@@ -384,9 +397,63 @@ def command_for_job(job: dict):
|
|||||||
|
|
||||||
return [str(script_path)]
|
return [str(script_path)]
|
||||||
|
|
||||||
|
if job_type == "run_alert_script":
|
||||||
|
payload = json.loads(job.get("payload_json") or "{}")
|
||||||
|
script_name = payload.get("script_name") or target_id
|
||||||
|
|
||||||
|
validate_script_name(script_name)
|
||||||
|
|
||||||
|
script_path = ALERTS_DIR / script_name
|
||||||
|
|
||||||
|
if not script_path.exists():
|
||||||
|
raise ValueError(f"Alert script not found: {script_name}")
|
||||||
|
|
||||||
|
if not script_path.is_file():
|
||||||
|
raise ValueError(f"Alert script is not a file: {script_name}")
|
||||||
|
|
||||||
|
return [str(script_path)]
|
||||||
|
|
||||||
raise ValueError(f"Unsupported job type: {job_type}")
|
raise ValueError(f"Unsupported job type: {job_type}")
|
||||||
|
|
||||||
|
|
||||||
|
def update_alert_event_from_job(job: dict, status: str, error_text: str | None):
|
||||||
|
if job.get("type") != "run_alert_script":
|
||||||
|
return
|
||||||
|
|
||||||
|
payload = json.loads(job.get("payload_json") or "{}")
|
||||||
|
alert_event_id = payload.get("alert_event_id")
|
||||||
|
|
||||||
|
if not alert_event_id:
|
||||||
|
return
|
||||||
|
|
||||||
|
con = get_connection()
|
||||||
|
|
||||||
|
if status == "success":
|
||||||
|
alert_status = "success"
|
||||||
|
elif status == "cancelled":
|
||||||
|
alert_status = "cancelled"
|
||||||
|
else:
|
||||||
|
alert_status = "failed"
|
||||||
|
|
||||||
|
con.execute(
|
||||||
|
"""
|
||||||
|
UPDATE alert_events
|
||||||
|
SET status = ?,
|
||||||
|
error_text = ?,
|
||||||
|
processed_at = CURRENT_TIMESTAMP
|
||||||
|
WHERE id = ?
|
||||||
|
""",
|
||||||
|
(
|
||||||
|
alert_status,
|
||||||
|
error_text,
|
||||||
|
alert_event_id,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
con.commit()
|
||||||
|
con.close()
|
||||||
|
|
||||||
|
|
||||||
def terminate_process_group(process: subprocess.Popen, job_id: int):
|
def terminate_process_group(process: subprocess.Popen, job_id: int):
|
||||||
append_job_log(job_id, "system", "Cancel requested. Terminating process group...")
|
append_job_log(job_id, "system", "Cancel requested. Terminating process group...")
|
||||||
|
|
||||||
@@ -448,6 +515,7 @@ def execute_job(job: dict):
|
|||||||
error_text="Job cancelled by user request",
|
error_text="Job cancelled by user request",
|
||||||
result={"cancelled": True},
|
result={"cancelled": True},
|
||||||
)
|
)
|
||||||
|
update_alert_event_from_job(job, "cancelled", "Job cancelled by user request")
|
||||||
append_job_log(job_id, "system", "Job cancelled.")
|
append_job_log(job_id, "system", "Job cancelled.")
|
||||||
return
|
return
|
||||||
|
|
||||||
@@ -469,6 +537,7 @@ def execute_job(job: dict):
|
|||||||
error_text="Job cancelled by user request",
|
error_text="Job cancelled by user request",
|
||||||
result={"returncode": returncode, "cancelled": True},
|
result={"returncode": returncode, "cancelled": True},
|
||||||
)
|
)
|
||||||
|
update_alert_event_from_job(job, "cancelled", "Job cancelled by user request")
|
||||||
append_job_log(job_id, "system", "Job cancelled after process finished.")
|
append_job_log(job_id, "system", "Job cancelled after process finished.")
|
||||||
return
|
return
|
||||||
|
|
||||||
@@ -482,19 +551,24 @@ def execute_job(job: dict):
|
|||||||
"stderr_length": len(stderr or ""),
|
"stderr_length": len(stderr or ""),
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
|
update_alert_event_from_job(job, "success", None)
|
||||||
append_job_log(job_id, "system", "Job finished successfully")
|
append_job_log(job_id, "system", "Job finished successfully")
|
||||||
else:
|
else:
|
||||||
|
error_text = f"Command failed with return code {returncode}"
|
||||||
update_job_status(
|
update_job_status(
|
||||||
job_id,
|
job_id,
|
||||||
"failed",
|
"failed",
|
||||||
error_text=f"Command failed with return code {returncode}",
|
error_text=error_text,
|
||||||
result={"returncode": returncode},
|
result={"returncode": returncode},
|
||||||
)
|
)
|
||||||
|
update_alert_event_from_job(job, "failed", error_text)
|
||||||
append_job_log(job_id, "system", f"Job failed with return code {returncode}")
|
append_job_log(job_id, "system", f"Job failed with return code {returncode}")
|
||||||
|
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
update_job_status(job_id, "failed", error_text=str(exc))
|
error_text = str(exc)
|
||||||
append_job_log(job_id, "stderr", str(exc))
|
update_job_status(job_id, "failed", error_text=error_text)
|
||||||
|
update_alert_event_from_job(job, "failed", error_text)
|
||||||
|
append_job_log(job_id, "stderr", error_text)
|
||||||
|
|
||||||
finally:
|
finally:
|
||||||
update_worker_status(current_job_id=None)
|
update_worker_status(current_job_id=None)
|
||||||
|
|||||||
Reference in New Issue
Block a user