fix: persist Chatwoot outbound message timeline

app/persistence.py

@@ -1,3 +1,4 @@
+from datetime import datetime, timezone
 from typing import Any, Dict, Optional, Tuple
 import json
 
@@ -32,6 +33,95 @@ def _lock_source_identity(conn: Any, source_system: str, source_event_id: str |
         })
 
 
+def _message_created_at(value: Any) -> datetime | None:
+    """Normalize a source message timestamp without substituting processing time."""
+    if value in (None, ""):
+        return None
+    if isinstance(value, datetime):
+        return value if value.tzinfo else value.replace(tzinfo=timezone.utc)
+    if isinstance(value, (int, float)) or str(value).strip().replace(".", "", 1).isdigit():
+        try:
+            return datetime.fromtimestamp(float(value), tz=timezone.utc)
+        except (OverflowError, TypeError, ValueError):
+            return None
+    try:
+        parsed = datetime.fromisoformat(str(value).strip().replace("Z", "+00:00"))
+    except ValueError:
+        return None
+    return parsed if parsed.tzinfo else parsed.replace(tzinfo=timezone.utc)
+
+
+def save_factual_chatwoot_message(
+    *,
+    raw_event_id: str,
+    source_event_id: str | None,
+    conversation_id: str | None,
+    contact_id: str | None,
+    direction: str,
+    raw_body: str,
+    clean_body: str | None = None,
+    source_created_at: Any = None,
+    metadata: Dict[str, Any] | None = None,
+) -> tuple[str, bool]:
+    """Persist one factual public Chatwoot message, without workflow effects.
+
+    Returns ``(message_id, inserted)``. Canonical source identity and the
+    transaction advisory lock make webhook delivery and backfill idempotent.
+    """
+    if not settings.clientflow_persist:
+        return "", False
+    normalized_direction = str(direction or "").strip().lower()
+    if normalized_direction not in {"inbound", "outbound"}:
+        raise ValueError("direction must be inbound or outbound")
+    created_at = _message_created_at(source_created_at)
+    with engine.begin() as conn:
+        _lock_source_identity(conn, "chatwoot", source_event_id)
+        existing = None
+        if source_event_id:
+            existing = conn.execute(text("""
+                SELECT id::text FROM messages
+                WHERE source_system='chatwoot' AND source_event_id=:source_event_id
+                LIMIT 1
+            """), {"source_event_id": source_event_id}).first()
+        if not existing:
+            existing = conn.execute(text("""
+                SELECT id::text FROM messages
+                WHERE raw_event_id=CAST(:raw_event_id AS UUID) LIMIT 1
+            """), {"raw_event_id": raw_event_id}).first()
+        inserted = existing is None
+        if inserted:
+            row = conn.execute(text("""
+                INSERT INTO messages (
+                    raw_event_id, source_system, source_event_id, conversation_id,
+                    contact_id, direction, raw_body, clean_body, previous_context,
+                    metadata, created_at
+                ) VALUES (
+                    CAST(:raw_event_id AS UUID), 'chatwoot', :source_event_id,
+                    :conversation_id, :contact_id, :direction, :raw_body, :clean_body,
+                    NULL, CAST(:metadata AS JSONB),
+                    COALESCE(CAST(:created_at AS TIMESTAMPTZ), now())
+                ) RETURNING id::text
+            """), {
+                "raw_event_id": raw_event_id,
+                "source_event_id": source_event_id,
+                "conversation_id": conversation_id,
+                "contact_id": contact_id,
+                "direction": normalized_direction,
+                "raw_body": raw_body,
+                "clean_body": clean_body if clean_body is not None else raw_body,
+                "metadata": _json(metadata),
+                "created_at": created_at,
+            }).first()
+            message_id = str(row[0])
+        else:
+            message_id = str(existing[0])
+        conn.execute(text("""
+            UPDATE raw_events SET message_id=CAST(:message_id AS UUID)
+            WHERE id=CAST(:raw_event_id AS UUID) AND message_id IS NULL
+        """), {"message_id": message_id, "raw_event_id": raw_event_id})
+    return message_id, inserted
+
+
 def save_inbound_message(
     *,
     request: AnalyzeRequest,

app/webhooks_chatwoot.py

@@ -17,6 +17,7 @@ from app.persistence import (
     get_state_for_conversation,
     mark_raw_event_error,
     mark_raw_event_processed,
+    save_factual_chatwoot_message,
     save_raw_event,
 )
 from app.schemas import AnalyzeRequest
@@ -32,6 +33,12 @@ def as_str(value: Any) -> Optional[str]:
     return str(value)
 
 
+def as_bool(value: Any) -> bool:
+    if isinstance(value, str):
+        return value.strip().lower() in {"1", "true", "yes"}
+    return bool(value)
+
+
 def get_nested(data: Dict[str, Any], *keys: str) -> Any:
     current: Any = data
 
@@ -103,6 +110,11 @@ def message_timestamp(message: Dict[str, Any]) -> Any:
     )
 
 
+def source_message_timestamp(payload: Dict[str, Any]) -> Any:
+    message = payload.get("message") if isinstance(payload.get("message"), dict) else payload
+    return message.get("created_at") or payload.get("created_at") or message.get("timestamp") or payload.get("timestamp")
+
+
 def extract_message_content_for_context(message: Dict[str, Any]) -> str:
     try:
         content = extract_chatwoot_content(message, message)
@@ -261,9 +273,21 @@ def extract_chatwoot_event(payload: Dict[str, Any]) -> Dict[str, Any]:
     message_type_text = str(message_type).lower()
     is_outgoing = message_type_text in {"outgoing", "outbound", "1"}
 
+    # The sender of an outgoing message is normally the agent, not the customer.
+    # Retain a contact only when Chatwoot supplied an explicit conversation/contact
+    # identity rather than falling back to the sender object.
+    if (
+        is_outgoing
+        and payload.get("contact_id") is None
+        and not isinstance(payload.get("contact"), dict)
+        and not isinstance(conversation.get("contact"), dict)
+        and not isinstance(get_nested(conversation, "contact_inbox", "contact"), dict)
+    ):
+        contact_id = None
+
     is_private = bool(
-        message.get("private")
-        or payload.get("private")
+        as_bool(message.get("private"))
+        or as_bool(payload.get("private"))
         or message.get("content_type") == "input_select"
     )
 
@@ -292,6 +316,7 @@ def extract_chatwoot_event(payload: Dict[str, Any]) -> Dict[str, Any]:
         "is_private": is_private,
         "sender_name": as_str(sender_name),
         "sender_type": as_str(sender_type),
+        "created_at": source_message_timestamp(payload),
     }
 
 
@@ -375,6 +400,25 @@ async def process_saved_chatwoot_raw_event(raw_event_id: str, payload: Dict[str,
         }
 
     if extracted["is_outgoing"]:
+        factual_message_id = None
+        if extracted.get("event_type") == "message_created" and not extracted.get("is_private"):
+            factual_message_id, _ = save_factual_chatwoot_message(
+                raw_event_id=raw_event_id,
+                source_event_id=extracted.get("source_event_id"),
+                conversation_id=extracted.get("conversation_id"),
+                contact_id=extracted.get("contact_id"),
+                direction="outbound",
+                raw_body=extracted["content"],
+                clean_body=extracted["content"],
+                source_created_at=extracted.get("created_at"),
+                metadata={
+                    "message_type": extracted.get("message_type"),
+                    "public": True,
+                    "private": False,
+                    "sender_name": extracted.get("sender_name"),
+                    "sender_type": extracted.get("sender_type"),
+                },
+            )
         auto_complete_result = auto_complete_task_from_outgoing_message(
             conversation_id=extracted["conversation_id"],
             contact_id=extracted["contact_id"],
@@ -400,6 +444,7 @@ async def process_saved_chatwoot_raw_event(raw_event_id: str, payload: Dict[str,
             "status": "outgoing_processed" if completed else "ignored",
             "reason": auto_complete_result.get("status"),
             "raw_event_id": raw_event_id,
+            "message_id": factual_message_id,
             "conversation_id": extracted["conversation_id"],
             "contact_id": extracted["contact_id"],
             "auto_complete": auto_complete_result,