Implementace prevodu HTML na PDF
Sluzba prijme adresu HTML dokumentu nebo HTML v tele requestu a vrati PDF. Navrzena pro dokumenty o stovkach az tisicich stranek. Rendering: - WeasyPrint jako vychozi engine, spravne CSS Paged Media, nizka pametova narocnost, bez JavaScriptu - Chromium pres Playwright pro dokumenty dokreslovane skripty - rezim auto s detekci skriptu a fallbackem pri selhani WeasyPrintu Velke dokumenty: - deleni na casti na strukturalnich hranicich, rez nikdy uvnitr tabulky nebo odstavce - dvoupruchodovy render obsahu se skutecnymi cisly stranek, pozice nadpisu se ctou z kotev hlasenych u kazde stranky - cislovani stranek bud pres CSS countery, nebo pres cislovaci vrstvu nastampovanou na hotove PDF, rozmer stranky se cte z vysledneho souboru - Chromium se restartuje po N jobech, nikdy vsak behem beziciho renderu API: - POST /convert synchronne, POST /jobs asynchronne se sledovanim stavu, stahovanim vysledku, rusenim a volitelnym callbackem - GET /health s overenim dostupnosti obou enginu a stavem fronty - OpenAPI respektuje prefix reverse proxy pres root_path Bezpecnost a provoz: - SSRF kontrola po DNS resolvu, na kazdem presmerovani a u vsech pozadavku prohlizece - nedostupne assety render nezastavi, ale hlasi se v odpovedi i v logu - fronta s omezenym poctem workeru, rozpracovane joby se pri ukonceni oznaci jako failed, nezmizi potichu - strukturovane JSON logovani s job_id - vsechny limity vypnute ve vychozim stavu Dockerfile je dvoufazovy, obsahuje zavislosti WeasyPrintu, Chromium a fonty s ceskou diakritikou. Autentizace zamerne neni implementovana, zpusob predavani neni domluveny. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
JiriUhlir
co-authored by
Claude Opus 5
Claude Opus 5
parent
e3cc8f418b
commit
156289fe2d
@@ -0,0 +1,181 @@
|
||||
"""Fetching of the source document and of its assets.
|
||||
|
||||
Both paths go through UrlGuard. Asset failures never abort the render, they are
|
||||
collected and reported back to the caller so nobody silently receives a PDF with
|
||||
missing images.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
import logging
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
import httpx
|
||||
|
||||
from ..config import Settings
|
||||
from ..errors import LimitExceededError, SourceUnavailableError
|
||||
from ..models import MissingAsset
|
||||
from .security import UrlGuard
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
USER_AGENT = "html-to-pdf/1.0 (AppFactory)"
|
||||
|
||||
|
||||
@dataclass
|
||||
class FetchedDocument:
|
||||
html: str
|
||||
base_url: str
|
||||
|
||||
|
||||
@dataclass
|
||||
class AssetReport:
|
||||
"""Collects assets that could not be loaded during a render."""
|
||||
|
||||
missing: list[MissingAsset] = field(default_factory=list)
|
||||
_seen: set[str] = field(default_factory=set)
|
||||
|
||||
def add(self, url: str, reason: str) -> None:
|
||||
if url in self._seen:
|
||||
return
|
||||
self._seen.add(url)
|
||||
self.missing.append(MissingAsset(url=url, reason=reason))
|
||||
logger.warning("Asset could not be loaded", extra={"asset_url": url, "reason": reason})
|
||||
|
||||
|
||||
def fetch_document(url: str, guard: UrlGuard, settings: Settings) -> FetchedDocument:
|
||||
"""Download the source HTML, validating every redirect hop."""
|
||||
|
||||
current = url
|
||||
timeout = httpx.Timeout(settings.fetch_timeout_seconds)
|
||||
|
||||
with httpx.Client(follow_redirects=False, timeout=timeout, headers={"User-Agent": USER_AGENT}) as client:
|
||||
for hop in range(settings.max_redirects + 1):
|
||||
guard.check(current)
|
||||
try:
|
||||
response = client.get(current)
|
||||
except httpx.HTTPError as exc:
|
||||
raise SourceUnavailableError(
|
||||
"Zdrojovy dokument se nepodarilo stahnout.",
|
||||
{"url": current, "reason": str(exc)},
|
||||
) from exc
|
||||
|
||||
if response.is_redirect:
|
||||
location = response.headers.get("location")
|
||||
if not location:
|
||||
raise SourceUnavailableError(
|
||||
"Zdroj vratil presmerovani bez hlavicky Location.", {"url": current}
|
||||
)
|
||||
current = str(response.url.join(location))
|
||||
logger.info("Following redirect", extra={"hop": hop + 1, "target": current})
|
||||
continue
|
||||
|
||||
if response.status_code >= 400:
|
||||
raise SourceUnavailableError(
|
||||
f"Zdrojovy dokument vratil HTTP {response.status_code}.",
|
||||
{"url": current, "status_code": response.status_code},
|
||||
)
|
||||
|
||||
_check_size(len(response.content), settings)
|
||||
return FetchedDocument(html=response.text, base_url=str(response.url))
|
||||
|
||||
raise SourceUnavailableError(
|
||||
"Prekrocen maximalni pocet presmerovani.",
|
||||
{"url": url, "max_redirects": settings.max_redirects},
|
||||
)
|
||||
|
||||
|
||||
def _check_size(size: int, settings: Settings) -> None:
|
||||
if settings.max_html_bytes and size > settings.max_html_bytes:
|
||||
raise LimitExceededError(
|
||||
"Zdrojovy dokument je vetsi nez nakonfigurovany limit MAX_HTML_BYTES.",
|
||||
{"size_bytes": size, "limit_bytes": settings.max_html_bytes},
|
||||
)
|
||||
|
||||
|
||||
def build_url_fetcher(guard: UrlGuard, report: AssetReport, allow_remote: bool, timeout_seconds: int):
|
||||
"""Return a WeasyPrint url_fetcher with SSRF checks and failure reporting."""
|
||||
|
||||
from weasyprint import default_url_fetcher
|
||||
|
||||
def fetcher(url: str, timeout: int = 10, ssl_context=None): # noqa: ARG001
|
||||
if url.startswith("data:"):
|
||||
return default_url_fetcher(url)
|
||||
|
||||
if not allow_remote:
|
||||
report.add(url, "stahovani externich assetu je vypnute")
|
||||
return _empty_asset()
|
||||
|
||||
try:
|
||||
guard.check(url)
|
||||
except Exception as exc: # noqa: BLE001 - reported, never silent
|
||||
report.add(url, f"zablokovano: {exc}")
|
||||
return _empty_asset()
|
||||
|
||||
try:
|
||||
response = httpx.get(
|
||||
url,
|
||||
timeout=timeout_seconds,
|
||||
follow_redirects=False,
|
||||
headers={"User-Agent": USER_AGENT},
|
||||
)
|
||||
while response.is_redirect:
|
||||
location = response.headers.get("location")
|
||||
if not location:
|
||||
raise httpx.HTTPError("presmerovani bez hlavicky Location")
|
||||
target = str(response.url.join(location))
|
||||
guard.check(target)
|
||||
response = httpx.get(
|
||||
target,
|
||||
timeout=timeout_seconds,
|
||||
follow_redirects=False,
|
||||
headers={"User-Agent": USER_AGENT},
|
||||
)
|
||||
|
||||
if response.status_code >= 400:
|
||||
report.add(url, f"HTTP {response.status_code}")
|
||||
return _empty_asset()
|
||||
|
||||
return {
|
||||
"string": response.content,
|
||||
"mime_type": response.headers.get("content-type", "").split(";")[0] or None,
|
||||
"redirected_url": str(response.url),
|
||||
}
|
||||
except Exception as exc: # noqa: BLE001 - reported, never silent
|
||||
report.add(url, str(exc))
|
||||
return _empty_asset()
|
||||
|
||||
return fetcher
|
||||
|
||||
|
||||
def _empty_asset() -> dict:
|
||||
"""Placeholder returned instead of a failed asset so the render continues."""
|
||||
return {"file_obj": io.BytesIO(b""), "mime_type": "application/octet-stream"}
|
||||
|
||||
|
||||
@dataclass
|
||||
class AssetGate:
|
||||
"""Per job gateway for everything the render pulls from the network."""
|
||||
|
||||
guard: UrlGuard
|
||||
report: AssetReport
|
||||
allow_remote: bool = True
|
||||
timeout_seconds: int = 10
|
||||
|
||||
def weasy_fetcher(self):
|
||||
return build_url_fetcher(self.guard, self.report, self.allow_remote, self.timeout_seconds)
|
||||
|
||||
def allowed(self, url: str) -> tuple[bool, str]:
|
||||
"""Decide whether a browser initiated request may proceed."""
|
||||
if url.startswith("data:") or url.startswith("blob:") or url.startswith("about:"):
|
||||
return True, ""
|
||||
if not url.startswith(("http://", "https://")):
|
||||
return False, "nepovolene schema"
|
||||
if not self.allow_remote:
|
||||
return False, "stahovani externich assetu je vypnute"
|
||||
try:
|
||||
self.guard.check(url)
|
||||
except Exception as exc: # noqa: BLE001 - reported, never silent
|
||||
return False, str(exc)
|
||||
return True, ""
|
||||
@@ -0,0 +1,238 @@
|
||||
"""In memory job queue.
|
||||
|
||||
The queue is deliberately in process. There is no broker and no database, which
|
||||
means a restart loses queued work. Such jobs are marked as failed with an
|
||||
explicit reason instead of silently disappearing.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import uuid
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
import httpx
|
||||
|
||||
from ..config import Settings
|
||||
from ..errors import ConversionError, JobNotFoundError, QueueFullError
|
||||
from ..logging_setup import current_job_id
|
||||
from ..models import ConvertRequest, ErrorInfo, JobProgress, JobState
|
||||
from .pipeline import ConversionPipeline
|
||||
from .storage import Storage
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _now() -> datetime:
|
||||
return datetime.now(timezone.utc)
|
||||
|
||||
|
||||
class Job:
|
||||
def __init__(self, job_id: str, request: ConvertRequest) -> None:
|
||||
self.id = job_id
|
||||
self.request = request
|
||||
self.state = JobState(job_id=job_id, status="queued", created_at=_now())
|
||||
self.task = None
|
||||
self.done = asyncio.Event()
|
||||
|
||||
|
||||
class JobManager:
|
||||
def __init__(self, pipeline: ConversionPipeline, storage: Storage, settings: Settings) -> None:
|
||||
self._pipeline = pipeline
|
||||
self._storage = storage
|
||||
self._settings = settings
|
||||
self._jobs = {}
|
||||
self._queue = asyncio.Queue(maxsize=settings.queue_max_size)
|
||||
self._workers = []
|
||||
self._cleaner = None
|
||||
|
||||
async def start(self) -> None:
|
||||
self._storage.sweep_orphans(set(self._jobs))
|
||||
for index in range(max(1, self._settings.workers)):
|
||||
self._workers.append(asyncio.create_task(self._worker(index), name=f"htp-worker-{index}"))
|
||||
self._cleaner = asyncio.create_task(self._cleanup_loop(), name="htp-cleanup")
|
||||
logger.info("Job manager started", extra={"workers": len(self._workers)})
|
||||
|
||||
async def stop(self) -> None:
|
||||
for task in self._workers:
|
||||
task.cancel()
|
||||
if self._cleaner is not None:
|
||||
self._cleaner.cancel()
|
||||
|
||||
for job in self._jobs.values():
|
||||
if job.state.status in ("queued", "running"):
|
||||
self._fail(
|
||||
job,
|
||||
ErrorInfo(
|
||||
error_code="service_restarted",
|
||||
message="Sluzba byla ukoncena drive, nez job dobehl. Odeslete pozadavek znovu.",
|
||||
),
|
||||
)
|
||||
logger.info("Job manager stopped")
|
||||
|
||||
def submit(self, request: ConvertRequest) -> Job:
|
||||
job = Job(str(uuid.uuid4()), request)
|
||||
self._jobs[job.id] = job
|
||||
try:
|
||||
self._queue.put_nowait(job.id)
|
||||
except asyncio.QueueFull as exc:
|
||||
del self._jobs[job.id]
|
||||
raise QueueFullError(
|
||||
"Fronta je plna, zkuste to prosim za chvili.",
|
||||
{"queue_max_size": self._settings.queue_max_size},
|
||||
) from exc
|
||||
|
||||
logger.info("Job queued", extra={"job_id": job.id, "queue_size": self._queue.qsize()})
|
||||
return job
|
||||
|
||||
def get(self, job_id: str) -> Job:
|
||||
job = self._jobs.get(job_id)
|
||||
if job is None:
|
||||
raise JobNotFoundError("Job s timto identifikatorem neexistuje nebo uz expiroval.")
|
||||
return job
|
||||
|
||||
def cancel(self, job_id: str) -> Job:
|
||||
job = self.get(job_id)
|
||||
if job.task is not None and not job.task.done():
|
||||
job.task.cancel()
|
||||
else:
|
||||
job.state.status = "cancelled"
|
||||
job.state.finished_at = _now()
|
||||
self._storage.discard(job.id)
|
||||
job.done.set()
|
||||
job.state.expires_at = _now() + timedelta(seconds=self._settings.job_result_ttl_seconds)
|
||||
logger.info("Job cancelled", extra={"job_id": job_id})
|
||||
return job
|
||||
|
||||
def stats(self) -> dict:
|
||||
counts = {"queued": 0, "running": 0, "done": 0, "failed": 0, "cancelled": 0, "expired": 0}
|
||||
for job in self._jobs.values():
|
||||
counts[job.state.status] = counts.get(job.state.status, 0) + 1
|
||||
counts["workers"] = len(self._workers)
|
||||
return counts
|
||||
|
||||
async def _worker(self, index: int) -> None:
|
||||
while True:
|
||||
job_id = await self._queue.get()
|
||||
job = self._jobs.get(job_id)
|
||||
if job is None or job.state.status != "queued":
|
||||
self._queue.task_done()
|
||||
continue
|
||||
|
||||
job.task = asyncio.current_task()
|
||||
token = current_job_id.set(job.id)
|
||||
try:
|
||||
await self._execute(job)
|
||||
except asyncio.CancelledError:
|
||||
job.state.status = "cancelled"
|
||||
job.state.finished_at = _now()
|
||||
job.state.expires_at = _now() + timedelta(seconds=self._settings.job_result_ttl_seconds)
|
||||
self._storage.discard(job.id)
|
||||
logger.info("Job execution cancelled", extra={"job_id": job.id})
|
||||
finally:
|
||||
current_job_id.reset(token)
|
||||
job.task = None
|
||||
job.done.set()
|
||||
self._queue.task_done()
|
||||
await self._notify_callback(job)
|
||||
|
||||
async def _execute(self, job: Job) -> None:
|
||||
job.state.status = "running"
|
||||
job.state.started_at = _now()
|
||||
logger.info("Job started")
|
||||
|
||||
def progress(pages, chunks_done, chunks_total, pass_number):
|
||||
job.state.progress = JobProgress(
|
||||
pages_rendered=pages,
|
||||
chunks_done=chunks_done,
|
||||
chunks_total=chunks_total,
|
||||
pass_number=pass_number,
|
||||
)
|
||||
|
||||
workdir = self._storage.job_dir(job.id)
|
||||
try:
|
||||
result = await self._pipeline.run(job.request, workdir, progress)
|
||||
except ConversionError as exc:
|
||||
logger.error("Job failed", extra={"error_code": exc.error_code}, exc_info=exc)
|
||||
self._fail(job, ErrorInfo(error_code=exc.error_code, message=exc.message, detail=exc.detail or None))
|
||||
return
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception as exc:
|
||||
logger.exception("Job failed with an unexpected error")
|
||||
self._fail(
|
||||
job,
|
||||
ErrorInfo(
|
||||
error_code="internal_error",
|
||||
message="Pri generovani PDF doslo k neocekavane chybe.",
|
||||
detail={"reason": str(exc)},
|
||||
),
|
||||
)
|
||||
return
|
||||
|
||||
self._storage.publish(job.id, result.path)
|
||||
job.state.status = "done"
|
||||
job.state.finished_at = _now()
|
||||
job.state.expires_at = _now() + timedelta(seconds=self._settings.job_result_ttl_seconds)
|
||||
job.state.page_count = result.page_count
|
||||
job.state.engine_used = result.engine_used
|
||||
job.state.missing_assets = result.missing_assets
|
||||
job.state.warnings = result.warnings
|
||||
logger.info(
|
||||
"Job finished",
|
||||
extra={
|
||||
"pages": result.page_count,
|
||||
"engine": result.engine_used,
|
||||
"missing_assets": len(result.missing_assets),
|
||||
},
|
||||
)
|
||||
|
||||
def _fail(self, job: Job, error: ErrorInfo) -> None:
|
||||
job.state.status = "failed"
|
||||
job.state.finished_at = _now()
|
||||
job.state.expires_at = _now() + timedelta(seconds=self._settings.job_result_ttl_seconds)
|
||||
job.state.error = error
|
||||
self._storage.discard(job.id)
|
||||
|
||||
async def _notify_callback(self, job: Job) -> None:
|
||||
url = job.request.callback_url
|
||||
if not url or job.state.status not in ("done", "failed"):
|
||||
return
|
||||
|
||||
payload = job.state.model_dump(mode="json")
|
||||
for attempt in range(1, max(1, self._settings.callback_retries) + 1):
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=self._settings.callback_timeout_seconds) as client:
|
||||
response = await client.post(url, json=payload)
|
||||
if response.status_code < 400:
|
||||
logger.info("Callback delivered", extra={"attempt": attempt})
|
||||
return
|
||||
logger.warning(
|
||||
"Callback returned an error status",
|
||||
extra={"attempt": attempt, "status_code": response.status_code},
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.warning("Callback delivery failed", extra={"attempt": attempt}, exc_info=exc)
|
||||
await asyncio.sleep(min(2 ** attempt, 10))
|
||||
|
||||
logger.error("Callback could not be delivered, job result stays available over the API")
|
||||
|
||||
async def _cleanup_loop(self) -> None:
|
||||
while True:
|
||||
await asyncio.sleep(60)
|
||||
try:
|
||||
self._expire_old_jobs()
|
||||
except Exception:
|
||||
logger.exception("Cleanup loop failed")
|
||||
|
||||
def _expire_old_jobs(self) -> None:
|
||||
now = _now()
|
||||
for job_id, job in list(self._jobs.items()):
|
||||
expires_at = job.state.expires_at
|
||||
if expires_at is None or expires_at > now:
|
||||
continue
|
||||
job.state.status = "expired"
|
||||
self._storage.discard(job_id)
|
||||
del self._jobs[job_id]
|
||||
logger.info("Job result expired and was removed", extra={"job_id": job_id})
|
||||
@@ -0,0 +1,350 @@
|
||||
"""The conversion pipeline.
|
||||
|
||||
Order of operations:
|
||||
|
||||
1. load the source HTML, validating the target address
|
||||
2. pick the engine
|
||||
3. decide how page numbers will be produced
|
||||
4. build the document, inject the page stylesheet, optionally insert the table
|
||||
of contents placeholder
|
||||
5. split into chunks
|
||||
6. first render pass, which yields the real page count and anchor positions
|
||||
7. second render pass when a table of contents needs real page numbers
|
||||
8. merge the chunks
|
||||
9. stamp the numbering overlay when CSS counters cannot be used
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import re
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Callable
|
||||
|
||||
from ..config import Settings, get_settings
|
||||
from ..errors import (
|
||||
ConversionError,
|
||||
LimitExceededError,
|
||||
RenderTimeoutError,
|
||||
UnsupportedCombinationError,
|
||||
)
|
||||
from ..models import ConvertRequest, MissingAsset
|
||||
from ..pdf import merger, paginator
|
||||
from ..pdf.chunker import split_document
|
||||
from ..pdf.document import SourceDocument
|
||||
from ..pdf.styles import build_page_css
|
||||
from .fetcher import AssetGate, AssetReport, fetch_document
|
||||
from .security import UrlGuard
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
SCRIPT_TAG = re.compile(r"<script\b[^>]*>(.*?)</script>", re.IGNORECASE | re.DOTALL)
|
||||
SCRIPT_SRC = re.compile(r"<script\b[^>]*\bsrc\s*=", re.IGNORECASE)
|
||||
|
||||
ProgressCallback = Callable[[int, int, int, int], None]
|
||||
|
||||
|
||||
@dataclass
|
||||
class ConversionResult:
|
||||
path: Path
|
||||
page_count: int
|
||||
engine_used: str
|
||||
missing_assets: list[MissingAsset] = field(default_factory=list)
|
||||
warnings: list[str] = field(default_factory=list)
|
||||
|
||||
|
||||
class ConversionPipeline:
|
||||
def __init__(self, engines: dict, settings: Settings | None = None) -> None:
|
||||
self._engines = engines
|
||||
self._settings = settings or get_settings()
|
||||
self._guard = UrlGuard(self._settings)
|
||||
|
||||
async def run(
|
||||
self,
|
||||
request: ConvertRequest,
|
||||
workdir: Path,
|
||||
progress: ProgressCallback | None = None,
|
||||
) -> ConversionResult:
|
||||
workdir.mkdir(parents=True, exist_ok=True)
|
||||
timeout = self._settings.max_render_seconds
|
||||
|
||||
coroutine = self._run_with_fallback(request, workdir, progress)
|
||||
if timeout:
|
||||
try:
|
||||
return await asyncio.wait_for(coroutine, timeout=timeout)
|
||||
except asyncio.TimeoutError as exc:
|
||||
raise RenderTimeoutError(
|
||||
"Render prekrocil nakonfigurovany limit MAX_RENDER_SECONDS.",
|
||||
{"limit_seconds": timeout},
|
||||
) from exc
|
||||
return await coroutine
|
||||
|
||||
async def _run_with_fallback(
|
||||
self, request: ConvertRequest, workdir: Path, progress: ProgressCallback | None
|
||||
) -> ConversionResult:
|
||||
raw_html, base_url = await self._load_source(request)
|
||||
requested = request.engine
|
||||
engine_name = self._select_engine(requested, raw_html)
|
||||
|
||||
try:
|
||||
return await self._execute(request, raw_html, base_url, engine_name, workdir, progress)
|
||||
except ConversionError:
|
||||
raise
|
||||
except Exception as exc:
|
||||
if requested != "auto" or engine_name != "weasyprint" or "chromium" not in self._engines:
|
||||
raise
|
||||
logger.warning(
|
||||
"WeasyPrint render failed, falling back to Chromium",
|
||||
exc_info=exc,
|
||||
extra={"engine": engine_name},
|
||||
)
|
||||
result = await self._execute(request, raw_html, base_url, "chromium", workdir, progress)
|
||||
result.warnings.append(
|
||||
"Render pres WeasyPrint selhal, dokument byl vygenerovan pres Chromium."
|
||||
)
|
||||
return result
|
||||
|
||||
def _select_engine(self, requested: str, raw_html: str) -> str:
|
||||
if requested != "auto":
|
||||
if requested not in self._engines:
|
||||
raise UnsupportedCombinationError(
|
||||
f"Engine {requested} neni v teto instanci k dispozici.",
|
||||
{"available": sorted(self._engines)},
|
||||
)
|
||||
return requested
|
||||
|
||||
if self._has_active_scripts(raw_html) and "chromium" in self._engines:
|
||||
logger.info("Auto engine selected chromium because the document contains scripts")
|
||||
return "chromium"
|
||||
|
||||
if "weasyprint" in self._engines:
|
||||
return "weasyprint"
|
||||
|
||||
return next(iter(self._engines))
|
||||
|
||||
@staticmethod
|
||||
def _has_active_scripts(raw_html: str) -> bool:
|
||||
if SCRIPT_SRC.search(raw_html):
|
||||
return True
|
||||
return any(body.strip() for body in SCRIPT_TAG.findall(raw_html))
|
||||
|
||||
async def _load_source(self, request: ConvertRequest) -> tuple:
|
||||
if request.source.html is not None:
|
||||
html = request.source.html
|
||||
limit = self._settings.max_html_bytes
|
||||
if limit and len(html.encode("utf-8")) > limit:
|
||||
raise LimitExceededError(
|
||||
"Zdrojove HTML je vetsi nez nakonfigurovany limit MAX_HTML_BYTES.",
|
||||
{"limit_bytes": limit},
|
||||
)
|
||||
return html, request.source.base_url
|
||||
|
||||
fetched = await asyncio.to_thread(
|
||||
fetch_document, request.source.url, self._guard, self._settings
|
||||
)
|
||||
return fetched.html, request.source.base_url or fetched.base_url
|
||||
|
||||
async def _execute(
|
||||
self,
|
||||
request: ConvertRequest,
|
||||
raw_html: str,
|
||||
base_url: str | None,
|
||||
engine_name: str,
|
||||
workdir: Path,
|
||||
progress: ProgressCallback | None,
|
||||
) -> ConversionResult:
|
||||
engine = self._engines[engine_name]
|
||||
report = AssetReport()
|
||||
gate = AssetGate(
|
||||
guard=self._guard,
|
||||
report=report,
|
||||
allow_remote=request.assets.allow_remote,
|
||||
timeout_seconds=request.assets.timeout_seconds,
|
||||
)
|
||||
warnings: list[str] = []
|
||||
|
||||
numbering_mode = self._numbering_mode(request, engine_name)
|
||||
|
||||
if request.toc.enabled and not getattr(engine, "supports_anchor_pages", False):
|
||||
raise UnsupportedCombinationError(
|
||||
"Generovani obsahu s cisly stranek podporuje pouze engine weasyprint.",
|
||||
{"engine": engine_name},
|
||||
)
|
||||
|
||||
document = SourceDocument(raw_html)
|
||||
document.set_base_url(base_url)
|
||||
document.append_stylesheet(
|
||||
build_page_css(
|
||||
request.page,
|
||||
request.page_numbers if numbering_mode == "css" else None,
|
||||
total_pages=None,
|
||||
outline=request.outline,
|
||||
)
|
||||
)
|
||||
|
||||
headings = document.collect_headings(request.toc.depth) if request.toc.enabled else []
|
||||
if request.toc.enabled:
|
||||
if not headings:
|
||||
warnings.append("Dokument neobsahuje zadne nadpisy, obsah nebyl vygenerovan.")
|
||||
request.toc.enabled = False
|
||||
else:
|
||||
document.insert_toc(headings, request.toc.title, pages=None)
|
||||
|
||||
chunks = self._split(document, request)
|
||||
direct_navigation = self._can_navigate_directly(request, engine_name, chunks)
|
||||
if direct_navigation:
|
||||
logger.info("Chromium will navigate to the source URL directly so its scripts run in context")
|
||||
|
||||
renders = await self._render_pass(
|
||||
engine, chunks, base_url, request, workdir, gate, progress, 1, direct_navigation
|
||||
)
|
||||
|
||||
if request.toc.enabled:
|
||||
anchor_pages = self._absolute_anchor_pages(renders)
|
||||
missing = [item.anchor for item in headings if item.anchor not in anchor_pages]
|
||||
if missing:
|
||||
logger.warning(
|
||||
"Some headings have no anchor position, their page numbers stay empty",
|
||||
extra={"missing_anchors": len(missing)},
|
||||
)
|
||||
document.remove_toc()
|
||||
document.insert_toc(headings, request.toc.title, pages=anchor_pages)
|
||||
chunks = self._split(document, request)
|
||||
renders = await self._render_pass(
|
||||
engine, chunks, base_url, request, workdir, gate, progress, 2
|
||||
)
|
||||
|
||||
if len(chunks) > 1:
|
||||
warnings.append(
|
||||
"Dokument byl rozdelen na casti, odkazy v obsahu proto nejsou klikatelne. "
|
||||
"Cisla stranek jsou spravna."
|
||||
)
|
||||
|
||||
merged_path = workdir / "merged.pdf"
|
||||
total_pages = merger.merge([item.path for item in renders], merged_path)
|
||||
self._check_page_limit(total_pages)
|
||||
|
||||
final_path = merged_path
|
||||
if request.page_numbers.enabled and numbering_mode == "overlay":
|
||||
final_path = await asyncio.to_thread(
|
||||
self._stamp_numbers, merged_path, workdir, request, total_pages
|
||||
)
|
||||
|
||||
on_job_finished = getattr(engine, "on_job_finished", None)
|
||||
if callable(on_job_finished):
|
||||
on_job_finished()
|
||||
|
||||
return ConversionResult(
|
||||
path=final_path,
|
||||
page_count=total_pages,
|
||||
engine_used=engine_name,
|
||||
missing_assets=report.missing,
|
||||
warnings=warnings,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _can_navigate_directly(request: ConvertRequest, engine_name: str, chunks: list) -> bool:
|
||||
"""Chromium renders a foreign page best when it loads the URL itself.
|
||||
|
||||
Only possible when the document is not split and needs no injected
|
||||
markup, otherwise the modified HTML has to be pushed into the page.
|
||||
"""
|
||||
return (
|
||||
engine_name == "chromium"
|
||||
and request.source.url is not None
|
||||
and not request.toc.enabled
|
||||
and len(chunks) == 1
|
||||
)
|
||||
|
||||
def _numbering_mode(self, request: ConvertRequest, engine_name: str) -> str:
|
||||
mode = request.page_numbers.mode
|
||||
if mode == "auto":
|
||||
if engine_name == "weasyprint" and not request.chunking.enabled:
|
||||
return "css"
|
||||
return "overlay"
|
||||
|
||||
if mode == "css" and engine_name != "weasyprint":
|
||||
raise UnsupportedCombinationError(
|
||||
"Rezim cislovani css funguje pouze s enginem weasyprint. Pouzijte overlay nebo auto.",
|
||||
{"engine": engine_name},
|
||||
)
|
||||
if mode == "css" and request.chunking.enabled:
|
||||
raise UnsupportedCombinationError(
|
||||
"Rezim cislovani css nelze kombinovat s chunkovanim, protoze citac stranek se v kazde "
|
||||
"casti restartuje. Vypnete chunking nebo pouzijte overlay.",
|
||||
)
|
||||
return mode
|
||||
|
||||
def _split(self, document: SourceDocument, request: ConvertRequest) -> list:
|
||||
if not request.chunking.enabled:
|
||||
return [document.to_html()]
|
||||
return split_document(document, request.chunking.pages_per_chunk)
|
||||
|
||||
async def _render_pass(
|
||||
self,
|
||||
engine,
|
||||
chunks: list,
|
||||
base_url: str | None,
|
||||
request: ConvertRequest,
|
||||
workdir: Path,
|
||||
gate: AssetGate,
|
||||
progress: ProgressCallback | None,
|
||||
pass_number: int,
|
||||
direct_navigation: bool = False,
|
||||
) -> list:
|
||||
renders = []
|
||||
pages_rendered = 0
|
||||
|
||||
for index, chunk_html in enumerate(chunks):
|
||||
output = workdir / f"pass{pass_number}-chunk{index:04d}.pdf"
|
||||
render = await engine.render_chunk(
|
||||
None if direct_navigation else chunk_html,
|
||||
base_url,
|
||||
request,
|
||||
output,
|
||||
total_pages=None,
|
||||
asset_gate=gate,
|
||||
)
|
||||
renders.append(render)
|
||||
pages_rendered += render.page_count
|
||||
|
||||
if progress is not None:
|
||||
progress(pages_rendered, index + 1, len(chunks), pass_number)
|
||||
|
||||
self._check_page_limit(pages_rendered)
|
||||
|
||||
logger.info(
|
||||
"Render pass finished",
|
||||
extra={"pass_number": pass_number, "chunks": len(chunks), "pages": pages_rendered},
|
||||
)
|
||||
return renders
|
||||
|
||||
@staticmethod
|
||||
def _absolute_anchor_pages(renders: list) -> dict:
|
||||
pages: dict = {}
|
||||
offset = 0
|
||||
for render in renders:
|
||||
for anchor, local_page in render.anchor_pages.items():
|
||||
pages.setdefault(anchor, offset + local_page + 1)
|
||||
offset += render.page_count
|
||||
return pages
|
||||
|
||||
def _check_page_limit(self, pages: int) -> None:
|
||||
if self._settings.max_pages and pages > self._settings.max_pages:
|
||||
raise LimitExceededError(
|
||||
"Dokument ma vice stranek nez nakonfigurovany limit MAX_PAGES.",
|
||||
{"pages": pages, "limit": self._settings.max_pages},
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _stamp_numbers(merged_path: Path, workdir: Path, request: ConvertRequest, total_pages: int) -> Path:
|
||||
width, height = merger.first_page_size(merged_path)
|
||||
overlay_path = workdir / "overlay.pdf"
|
||||
paginator.build_overlay(
|
||||
total_pages, width, height, request.page, request.page_numbers, overlay_path
|
||||
)
|
||||
numbered_path = workdir / "numbered.pdf"
|
||||
paginator.apply_overlay(merged_path, overlay_path, numbered_path)
|
||||
return numbered_path
|
||||
@@ -0,0 +1,132 @@
|
||||
"""SSRF protection.
|
||||
|
||||
The service fetches arbitrary URLs on request, which is exactly the shape of an
|
||||
SSRF vulnerability. Every URL is validated after DNS resolution, not on the
|
||||
string alone, and the check is repeated on every redirect hop.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import ipaddress
|
||||
import logging
|
||||
import socket
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from ..config import Settings
|
||||
from ..errors import BlockedTargetError
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
ALLOWED_SCHEMES = ("http", "https")
|
||||
|
||||
# Ranges that must never be reachable from a user supplied URL.
|
||||
PRIVATE_NETWORKS = [
|
||||
ipaddress.ip_network("127.0.0.0/8"),
|
||||
ipaddress.ip_network("10.0.0.0/8"),
|
||||
ipaddress.ip_network("172.16.0.0/12"),
|
||||
ipaddress.ip_network("192.168.0.0/16"),
|
||||
ipaddress.ip_network("169.254.0.0/16"),
|
||||
ipaddress.ip_network("0.0.0.0/8"),
|
||||
ipaddress.ip_network("100.64.0.0/10"),
|
||||
ipaddress.ip_network("192.0.0.0/24"),
|
||||
ipaddress.ip_network("198.18.0.0/15"),
|
||||
ipaddress.ip_network("224.0.0.0/4"),
|
||||
ipaddress.ip_network("240.0.0.0/4"),
|
||||
ipaddress.ip_network("::1/128"),
|
||||
ipaddress.ip_network("fc00::/7"),
|
||||
ipaddress.ip_network("fe80::/10"),
|
||||
ipaddress.ip_network("::/128"),
|
||||
]
|
||||
|
||||
|
||||
class UrlGuard:
|
||||
"""Validates URLs against the configured policy."""
|
||||
|
||||
def __init__(self, settings: Settings) -> None:
|
||||
self._block_private = settings.ssrf_block_private
|
||||
self._allowed_hosts = {host.lower() for host in settings.ssrf_allowed_hosts}
|
||||
self._extra_blocked: list[ipaddress._BaseNetwork] = []
|
||||
|
||||
for cidr in settings.ssrf_extra_blocked_cidrs:
|
||||
try:
|
||||
self._extra_blocked.append(ipaddress.ip_network(cidr, strict=False))
|
||||
except ValueError:
|
||||
logger.warning("Ignoring invalid CIDR in SSRF_EXTRA_BLOCKED_CIDRS", extra={"cidr": cidr})
|
||||
|
||||
def check(self, url: str) -> str:
|
||||
"""Raise BlockedTargetError when the URL must not be fetched.
|
||||
|
||||
Returns the hostname so callers can reuse it without parsing again.
|
||||
"""
|
||||
parsed = urlparse(url)
|
||||
scheme = (parsed.scheme or "").lower()
|
||||
|
||||
if scheme not in ALLOWED_SCHEMES:
|
||||
raise BlockedTargetError(
|
||||
"Povolena jsou pouze schemata http a https.",
|
||||
{"url": url, "scheme": scheme or None},
|
||||
)
|
||||
|
||||
host = parsed.hostname
|
||||
if not host:
|
||||
raise BlockedTargetError("Adresa neobsahuje hostname.", {"url": url})
|
||||
|
||||
if host.lower() in self._allowed_hosts:
|
||||
logger.info("Host explicitly allowlisted", extra={"host": host})
|
||||
return host
|
||||
|
||||
for address in self._resolve(host, url):
|
||||
self._check_address(address, host, url)
|
||||
|
||||
return host
|
||||
|
||||
def _resolve(self, host: str, url: str) -> list[ipaddress.IPv4Address | ipaddress.IPv6Address]:
|
||||
# A literal IP address needs no lookup.
|
||||
try:
|
||||
return [ipaddress.ip_address(host)]
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
try:
|
||||
infos = socket.getaddrinfo(host, None, proto=socket.IPPROTO_TCP)
|
||||
except socket.gaierror as exc:
|
||||
raise BlockedTargetError(
|
||||
f"Hostname {host} se nepodarilo prelozit na IP adresu.",
|
||||
{"url": url, "reason": str(exc)},
|
||||
) from exc
|
||||
|
||||
addresses = []
|
||||
for info in infos:
|
||||
try:
|
||||
addresses.append(ipaddress.ip_address(info[4][0]))
|
||||
except ValueError:
|
||||
continue
|
||||
|
||||
if not addresses:
|
||||
raise BlockedTargetError(f"Hostname {host} nema zadnou pouzitelnou IP adresu.", {"url": url})
|
||||
|
||||
return addresses
|
||||
|
||||
def _check_address(self, address, host: str, url: str) -> None:
|
||||
if self._block_private:
|
||||
for network in PRIVATE_NETWORKS:
|
||||
if address.version == network.version and address in network:
|
||||
logger.warning(
|
||||
"Blocked request to private address",
|
||||
extra={"host": host, "address": str(address), "network": str(network)},
|
||||
)
|
||||
raise BlockedTargetError(
|
||||
"Cilova adresa smeruje do privatniho nebo vyhrazeneho rozsahu a je zablokovana.",
|
||||
{"url": url, "host": host, "address": str(address)},
|
||||
)
|
||||
|
||||
for network in self._extra_blocked:
|
||||
if address.version == network.version and address in network:
|
||||
logger.warning(
|
||||
"Blocked request by configured CIDR",
|
||||
extra={"host": host, "address": str(address), "network": str(network)},
|
||||
)
|
||||
raise BlockedTargetError(
|
||||
"Cilova adresa je v konfigurovanem seznamu blokovanych rozsahu.",
|
||||
{"url": url, "host": host, "address": str(address)},
|
||||
)
|
||||
@@ -0,0 +1,65 @@
|
||||
"""Temporary storage of job working directories and results.
|
||||
|
||||
Nothing is kept longer than needed. A result lives until it is picked up or
|
||||
until its TTL expires, whichever comes first.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import shutil
|
||||
from pathlib import Path
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
RESULT_NAME = "result.pdf"
|
||||
|
||||
|
||||
class Storage:
|
||||
def __init__(self, root: str) -> None:
|
||||
self.root = Path(root)
|
||||
self.root.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
def job_dir(self, job_id: str) -> Path:
|
||||
path = self.root / job_id
|
||||
path.mkdir(parents=True, exist_ok=True)
|
||||
return path
|
||||
|
||||
def result_path(self, job_id: str) -> Path:
|
||||
return self.root / job_id / RESULT_NAME
|
||||
|
||||
def publish(self, job_id: str, produced: Path) -> Path:
|
||||
"""Move the produced file to its final name and drop the intermediates."""
|
||||
target = self.result_path(job_id)
|
||||
if produced != target:
|
||||
produced.replace(target)
|
||||
|
||||
for item in self.job_dir(job_id).iterdir():
|
||||
if item.name == RESULT_NAME:
|
||||
continue
|
||||
self._remove(item)
|
||||
|
||||
return target
|
||||
|
||||
def discard(self, job_id: str) -> None:
|
||||
self._remove(self.root / job_id)
|
||||
|
||||
def _remove(self, path: Path) -> None:
|
||||
try:
|
||||
if path.is_dir():
|
||||
shutil.rmtree(path, ignore_errors=False)
|
||||
elif path.exists():
|
||||
path.unlink()
|
||||
except OSError as exc:
|
||||
logger.warning("Could not remove temporary path", extra={"path": str(path)}, exc_info=exc)
|
||||
|
||||
def sweep_orphans(self, known_job_ids: set[str]) -> int:
|
||||
"""Remove directories that belong to no known job, for example after a restart."""
|
||||
removed = 0
|
||||
for item in self.root.iterdir():
|
||||
if item.is_dir() and item.name not in known_job_ids:
|
||||
self._remove(item)
|
||||
removed += 1
|
||||
if removed:
|
||||
logger.info("Removed orphaned job directories", extra={"count": removed})
|
||||
return removed
|
||||
Reference in New Issue
Block a user