from __future__ import annotations import base64 import json from datetime import datetime, timezone from typing import Any from urllib.parse import quote_plus import httpx from fastapi import HTTPException, status from ..core.config import get_settings from ..models.base import utcnow from ..schemas.flower_monitor import ( FlowerTaskItem, FlowerWorkerItem, FlowerWorkerTaskOverviewResponse, FlowerWorkerTaskSummary, FlowerWorkersOverviewResponse, FlowerWorkersSummary, ) def build_workers_overview(*, force_refresh: bool) -> FlowerWorkersOverviewResponse: refresh = "true" if force_refresh else "false" workers_map = _call_flower_json(f"/api/workers?refresh={refresh}") status_map = _call_flower_json("/api/workers?status=true") if not isinstance(workers_map, dict): workers_map = {} if not isinstance(status_map, dict): status_map = {} worker_names = sorted(set(workers_map.keys()) | set(status_map.keys())) workers: list[FlowerWorkerItem] = [] for worker_name in worker_names: worker_raw = _as_record(workers_map.get(worker_name)) is_online = bool(status_map.get(worker_name)) workers.append(_normalize_worker(worker_name, worker_raw, is_online)) workers.sort(key=lambda item: (0 if item.status == "ONLINE" else 1, item.worker)) summary = FlowerWorkersSummary( total=len(workers), online=sum(1 for item in workers if item.status == "ONLINE"), offline=sum(1 for item in workers if item.status != "ONLINE"), ) return FlowerWorkersOverviewResponse( generated_at=utcnow(), workers=workers, summary=summary, ) def build_worker_task_overview( *, worker: str, force_refresh: bool, recent_limit: int, ) -> FlowerWorkerTaskOverviewResponse: normalized_worker = worker.strip() if not normalized_worker: raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="worker is required") refresh = "true" if force_refresh else "false" safe_recent_limit = max(1, min(200, int(recent_limit))) workers_map = _call_flower_json( f"/api/workers?refresh={refresh}&workername={quote_plus(normalized_worker)}" ) tasks_map = _call_flower_json( f"/api/tasks?limit={safe_recent_limit}&workername={quote_plus(normalized_worker)}" ) worker_raw = _pick_worker_raw(_as_record(workers_map), normalized_worker) active_tasks = _build_snapshot_tasks(normalized_worker, worker_raw, "ACTIVE") reserved_tasks = _build_snapshot_tasks(normalized_worker, worker_raw, "RESERVED") scheduled_tasks = _build_snapshot_tasks(normalized_worker, worker_raw, "SCHEDULED") recent_tasks = _build_recent_tasks(normalized_worker, _as_record(tasks_map)) summary = FlowerWorkerTaskSummary( active=len(active_tasks), reserved=len(reserved_tasks), scheduled=len(scheduled_tasks), recent=len(recent_tasks), ) return FlowerWorkerTaskOverviewResponse( generated_at=utcnow(), worker=normalized_worker, active_tasks=active_tasks, reserved_tasks=reserved_tasks, scheduled_tasks=scheduled_tasks, recent_tasks=recent_tasks, summary=summary, ) def _call_flower_json(path: str) -> Any: settings = get_settings() url = f"{settings.resolved_flower_api_base_url}{path}" headers = {"Accept": "application/json"} basic_auth = settings.resolved_flower_basic_auth if basic_auth: token = base64.b64encode(basic_auth.encode("utf-8")).decode("ascii") headers["Authorization"] = f"Basic {token}" timeout = max(3, int(settings.flower_api_timeout_seconds)) try: with httpx.Client(timeout=timeout) as client: response = client.get(url, headers=headers) except httpx.TimeoutException as exc: raise HTTPException(status_code=status.HTTP_504_GATEWAY_TIMEOUT, detail=f"flower request timeout: {path}") from exc except httpx.HTTPError as exc: raise HTTPException(status_code=status.HTTP_502_BAD_GATEWAY, detail=f"flower request failed: {path}: {exc}") from exc if response.status_code >= 400: detail = (response.text or "").strip() or response.reason_phrase or "flower error" raise HTTPException(status_code=status.HTTP_502_BAD_GATEWAY, detail=f"flower error {response.status_code}: {detail}") try: return response.json() except ValueError: raise HTTPException(status_code=status.HTTP_502_BAD_GATEWAY, detail=f"flower returned non-json payload: {path}") def _normalize_worker(worker_name: str, worker_raw: dict[str, Any], is_online: bool) -> FlowerWorkerItem: stats = _as_record(worker_raw.get("stats")) return FlowerWorkerItem( worker=worker_name, status="ONLINE" if is_online else "OFFLINE", queue_names=_parse_active_queue_names(worker_raw.get("active_queues")), registered_count=len(_as_array(worker_raw.get("registered"))), processed_count=_parse_processed_count(stats.get("total")), concurrency=_safe_int(_as_record(stats.get("pool")).get("max-concurrency") or _as_record(stats.get("pool")).get("max_concurrency")), prefetch_count=_safe_int(stats.get("prefetch_count")), active_count=len(_as_array(worker_raw.get("active"))), reserved_count=len(_as_array(worker_raw.get("reserved"))), scheduled_count=len(_as_array(worker_raw.get("scheduled"))), last_heartbeat_at=_parse_datetime(worker_raw.get("timestamp")), ) def _pick_worker_raw(workers_map: dict[str, Any], worker_name: str) -> dict[str, Any]: if worker_name in workers_map: return _as_record(workers_map.get(worker_name)) entries = list(workers_map.items()) if len(entries) == 1: return _as_record(entries[0][1]) normalized = worker_name.strip().lower() for name, value in entries: if str(name).strip().lower() == normalized: return _as_record(value) return {} def _build_snapshot_tasks(worker_name: str, worker_raw: dict[str, Any], source: str) -> list[FlowerTaskItem]: key = source.lower() snapshot = _as_array(worker_raw.get(key)) items: list[FlowerTaskItem] = [] for index, raw in enumerate(snapshot, start=1): items.append( _normalize_task_item( raw, source=source, fallback_task_id=f"{worker_name}:{source}:{index}", default_worker=worker_name, ) ) return items def _build_recent_tasks(worker_name: str, tasks_map: dict[str, Any]) -> list[FlowerTaskItem]: items: list[FlowerTaskItem] = [] for task_id, raw in tasks_map.items(): item = _normalize_task_item(raw, source="RECENT", fallback_task_id=str(task_id), default_worker=worker_name) if item.worker.strip().lower() != worker_name.strip().lower(): continue items.append(item) items.sort( key=lambda item: _sortable_timestamp( item.received_at or item.started_at or item.finished_at or item.eta ), reverse=True, ) return items def _normalize_task_item( raw_task: Any, *, source: str, fallback_task_id: str, default_worker: str, ) -> FlowerTaskItem: raw = _as_record(raw_task) request = _as_record(raw.get("request")) task_data = dict(raw) if request: task_data.update(request) task_id = _safe_text(task_data.get("uuid") or task_data.get("id") or fallback_task_id) or fallback_task_id task_name = _safe_text(task_data.get("name") or task_data.get("type")) or "-" worker_name = _safe_text(task_data.get("worker") or task_data.get("hostname")) or default_worker state_raw = _safe_text(task_data.get("state")) state = state_raw.upper() if state_raw else ("UNKNOWN" if source == "RECENT" else source) queue_name = _extract_queue_name(task_data) return FlowerTaskItem( task_id=task_id, name=task_name, state=state, source=source, worker=worker_name, queue_name=queue_name, args_text=_stringify_payload(task_data.get("args")), kwargs_text=_stringify_payload(task_data.get("kwargs")), eta=_parse_datetime(task_data.get("eta")), received_at=_parse_datetime(task_data.get("received") or task_data.get("timestamp")), started_at=_parse_datetime(task_data.get("started") or task_data.get("time_start")), finished_at=_parse_datetime(task_data.get("succeeded") or task_data.get("failed") or task_data.get("revoked")), runtime_seconds=_safe_float(task_data.get("runtime")), result_text=_stringify_payload(task_data.get("result")), exception_text=_stringify_payload(task_data.get("exception")), ) def _parse_active_queue_names(raw: Any) -> list[str]: values = [] for item in _as_array(raw): name = _safe_text(_as_record(item).get("name")) if name: values.append(name) return sorted(set(values)) def _parse_processed_count(raw_total: Any) -> int: total = _as_record(raw_total) count = 0 for value in total.values(): count += _safe_int(value) return count def _extract_queue_name(task_data: dict[str, Any]) -> str | None: delivery = _as_record(task_data.get("delivery_info")) routing_key = _safe_text(delivery.get("routing_key")) if routing_key: return routing_key exchange = _safe_text(task_data.get("exchange") or delivery.get("exchange")) return exchange def _parse_datetime(value: Any) -> datetime | None: if value is None: return None if isinstance(value, datetime): if value.tzinfo is None: return value.replace(tzinfo=timezone.utc) return value.astimezone(timezone.utc) if isinstance(value, (int, float)): numeric = float(value) if numeric <= 0: return None if numeric < 1e11: numeric *= 1000 try: return datetime.fromtimestamp(numeric / 1000, timezone.utc) except (OverflowError, OSError, ValueError): return None text = str(value).strip() if not text: return None if text.replace(".", "", 1).isdigit(): try: return _parse_datetime(float(text)) except ValueError: return None text = text.replace("Z", "+00:00") try: parsed = datetime.fromisoformat(text) except ValueError: return None if parsed.tzinfo is None: return parsed.replace(tzinfo=timezone.utc) return parsed.astimezone(timezone.utc) def _stringify_payload(value: Any) -> str | None: if value is None: return None if isinstance(value, str): text = value.strip() else: try: text = json.dumps(value, ensure_ascii=False) except TypeError: text = str(value) text = text.strip() if not text: return None if len(text) > 1000: return text[:997] + "..." return text def _safe_text(value: Any) -> str: return str(value or "").strip() def _safe_int(value: Any) -> int: try: return int(value) except (TypeError, ValueError): return 0 def _safe_float(value: Any) -> float | None: try: parsed = float(value) except (TypeError, ValueError): return None if parsed < 0: return None return parsed def _as_record(value: Any) -> dict[str, Any]: if isinstance(value, dict): return dict(value) return {} def _as_array(value: Any) -> list[Any]: if isinstance(value, list): return list(value) return [] def _sortable_timestamp(value: datetime | None) -> float: if value is None: return 0.0 if value.tzinfo is None: value = value.replace(tzinfo=timezone.utc) return value.timestamp()