oprava blokace vlaken

app/main.py

@@ -20,6 +20,8 @@ WORKER_ID = os.getenv("APPFACTORY_WORKER_ID", socket.gethostname())
 POLL_SECONDS = int(os.getenv("APPFACTORY_WORKER_POLL_SECONDS", "3"))
 HEARTBEAT_SECONDS = int(os.getenv("APPFACTORY_WORKER_HEARTBEAT_SECONDS", "10"))
 CANCEL_CHECK_SECONDS = int(os.getenv("APPFACTORY_CANCEL_CHECK_SECONDS", "2"))
+LOG_FLUSH_CHARS = int(os.getenv("APPFACTORY_LOG_FLUSH_CHARS", "8000"))
+STREAM_JOIN_TIMEOUT = int(os.getenv("APPFACTORY_STREAM_JOIN_TIMEOUT", "60"))
 
 app = FastAPI(title="AppFactory Worker")
 
@@ -474,6 +476,89 @@ def update_alert_event_from_job(job: dict, status: str, error_text: str | None):
     con.close()
 
 
+class StreamPump:
+    """Reads one subprocess stream in a background thread and stores it in the job log.
+
+    The subprocess writes into an OS pipe with a limited buffer. If nobody reads
+    the pipe while the process runs, the child blocks on write and never exits,
+    so the worker would wait forever. This pump keeps both pipes drained from
+    the moment the process starts.
+    """
+
+    def __init__(self, job_id: int, stream_name: str, stream):
+        self.job_id = job_id
+        self.stream_name = stream_name
+        self.stream = stream
+        self.length = 0
+        self._buffer = []
+        self._buffer_chars = 0
+        self._lock = threading.Lock()
+        self._thread = threading.Thread(
+            target=self._run,
+            name=f"job-{job_id}-{stream_name}",
+            daemon=True,
+        )
+
+    def start(self):
+        self._thread.start()
+
+    def is_alive(self) -> bool:
+        return self._thread.is_alive()
+
+    def join(self, timeout: float | None = None):
+        self._thread.join(timeout)
+
+    def flush(self):
+        # The lock is held across the write so that the reader thread and the
+        # main loop cannot store two chunks of the same stream out of order.
+        with self._lock:
+            if not self._buffer:
+                return
+
+            message = "".join(self._buffer)
+            self._buffer = []
+            self._buffer_chars = 0
+
+            try:
+                append_job_log(self.job_id, self.stream_name, message)
+            except Exception as exc:
+                print(f"Job log write error ({self.stream_name}): {exc}", flush=True)
+
+    def _run(self):
+        try:
+            for chunk in iter(self.stream.readline, ""):
+                with self._lock:
+                    self.length += len(chunk)
+                    self._buffer.append(chunk)
+                    self._buffer_chars += len(chunk)
+                    should_flush = self._buffer_chars >= LOG_FLUSH_CHARS
+
+                if should_flush:
+                    self.flush()
+
+        except Exception as exc:
+            print(f"Stream reader error ({self.stream_name}): {exc}", flush=True)
+            try:
+                append_job_log(
+                    self.job_id,
+                    "system",
+                    f"Error while reading {self.stream_name}: {exc}",
+                )
+            except Exception:
+                pass
+
+        finally:
+            try:
+                self.flush()
+            except Exception:
+                pass
+
+            try:
+                self.stream.close()
+            except Exception:
+                pass
+
+
 def terminate_process_group(process: subprocess.Popen, job_id: int):
     append_job_log(job_id, "system", "Cancel requested. Terminating process group...")
 
@@ -521,34 +606,59 @@ def execute_job(job: dict):
             start_new_session=True,
         )
 
-        while process.poll() is None:
-            extend_lock(job)
+        stdout_pump = StreamPump(job_id, "stdout", process.stdout)
+        stderr_pump = StreamPump(job_id, "stderr", process.stderr)
 
-            status = get_job_status(job_id)
+        stdout_pump.start()
+        stderr_pump.start()
 
-            if status == "cancelled_requested":
+        cancelled = False
+
+        try:
+            while process.poll() is None:
+                extend_lock(job)
+
+                stdout_pump.flush()
+                stderr_pump.flush()
+
+                status = get_job_status(job_id)
+
+                if status == "cancelled_requested":
+                    terminate_process_group(process, job_id)
+                    cancelled = True
+                    break
+
+                time.sleep(CANCEL_CHECK_SECONDS)
+
+        finally:
+            if process.poll() is None:
                 terminate_process_group(process, job_id)
 
-                update_job_status(
-                    job_id,
-                    "cancelled",
-                    error_text="Job cancelled by user request",
-                    result={"cancelled": True},
-                )
-                update_alert_event_from_job(job, "cancelled", "Job cancelled by user request")
-                append_job_log(job_id, "system", "Job cancelled.")
-                return
+            returncode = process.wait()
 
-            time.sleep(CANCEL_CHECK_SECONDS)
+            for pump in (stdout_pump, stderr_pump):
+                pump.join(STREAM_JOIN_TIMEOUT)
 
-        stdout, stderr = process.communicate()
-        returncode = process.returncode
+                if pump.is_alive():
+                    append_job_log(
+                        job_id,
+                        "system",
+                        f"Stream {pump.stream_name} did not reach EOF within {STREAM_JOIN_TIMEOUT} s."
+                        " Some output may be missing from the log.",
+                    )
 
-        if stdout:
-            append_job_log(job_id, "stdout", stdout)
+                pump.flush()
 
-        if stderr:
-            append_job_log(job_id, "stderr", stderr)
+        if cancelled:
+            update_job_status(
+                job_id,
+                "cancelled",
+                error_text="Job cancelled by user request",
+                result={"cancelled": True},
+            )
+            update_alert_event_from_job(job, "cancelled", "Job cancelled by user request")
+            append_job_log(job_id, "system", "Job cancelled.")
+            return
 
         if get_job_status(job_id) == "cancelled_requested":
             update_job_status(
@@ -567,8 +677,8 @@ def execute_job(job: dict):
                 "success",
                 result={
                     "returncode": returncode,
-                    "stdout_length": len(stdout or ""),
-                    "stderr_length": len(stderr or ""),
+                    "stdout_length": stdout_pump.length,
+                    "stderr_length": stderr_pump.length,
                 },
             )
             update_alert_event_from_job(job, "success", None)