admin/clientflow_backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: preserve Chatwoot message visibility and identity

Browse Source
This commit is contained in:
plx
2026-08-14 14:08:23 +00:00
parent 475ff15b9b
commit c4d55d37ed
10 changed files with 531 additions and 71 deletions
Download Patch File Download Diff File Expand all files Collapse all files

55
app/analyzer.py
View File

@@ -4,13 +4,32 @@ from app.action_decider import decide_action
from app.action_mapper import map_action_decision
from app.config import settings
from app.persistence import save_action_run
from app.persistence import save_inbound_message
from app.schemas import ActionDecision, AnalyzeRequest, AnalyzeResponse, UsageInfo
from app.task_service import create_task_from_action_result
async def analyze(request: AnalyzeRequest, raw_event_id: str | None = None) -> AnalyzeResponse:
async def analyze(
request: AnalyzeRequest,
raw_event_id: str | None = None,
source_event_id: str | None = None,
) -> AnalyzeResponse:
needs_review = False
# The inbound fact is durable before any LLM/policy decision. This keeps
# ignored classifications and decision failures visible and replay-safe.
message_id, source_event_id = save_inbound_message(
request=request, raw_event_id=raw_event_id, source_event_id=source_event_id,
raw_body=request.last_customer_message, clean_body=request.last_customer_message,
)
if (request.source or "").lower() == "chatwoot" and source_event_id:
from app.communication_service import upsert_chatwoot_inbound_communication
upsert_chatwoot_inbound_communication(
source_message_id=source_event_id, conversation_id=request.conversation_id,
contact_id=request.contact_id, body=request.last_customer_message,
metadata={"raw_event_id": raw_event_id, "message_id": message_id},
)
try:
decision, action_result, usage, decision_source = await decide_action(request)
except Exception as exc:
@@ -32,24 +51,11 @@ async def analyze(request: AnalyzeRequest, raw_event_id: str | None = None) -> A
)
decision_source = "fallback"
if str(action_result.action_code or "").upper() == "IGNORE_BOUNCE":
# v4.8.5: NDR/bounce emails are already readable in Chatwoot/Thunderbird.
# They must not create ClientFlow work, opportunities or fiscal customers.
return AnalyzeResponse(
app="ClientFlow",
model=settings.openrouter_model,
action_decision=decision,
action_result=action_result,
usage=usage,
needs_review=False,
action_run_id=None,
message_id=None,
task_id=None,
)
if not action_result.safe_to_post:
needs_review = True
is_ignored_bounce = str(action_result.action_code or "").upper() == "IGNORE_BOUNCE"
action_run_id, message_id = save_action_run(
request=request,
action_decision=decision,
@@ -61,6 +67,8 @@ async def analyze(request: AnalyzeRequest, raw_event_id: str | None = None) -> A
raw_body=request.last_customer_message,
clean_body=request.last_customer_message,
raw_event_id=raw_event_id,
source_event_id=source_event_id,
message_id=message_id,
)
normalized_message_for_idem = re.sub(r"\s+", " ", str(request.last_customer_message or "").strip().casefold())
@@ -74,7 +82,7 @@ async def analyze(request: AnalyzeRequest, raw_event_id: str | None = None) -> A
conversation_id=request.conversation_id,
contact_id=request.contact_id,
source_system=request.source or "manual",
source_event_id=message_id,
source_event_id=source_event_id or message_id,
metadata={
"created_from_analyzer": True,
"content_fingerprint": message_fingerprint,
@@ -89,6 +97,19 @@ async def analyze(request: AnalyzeRequest, raw_event_id: str | None = None) -> A
},
)
if (request.source or "").lower() == "chatwoot" and source_event_id:
from app.communication_service import enrich_chatwoot_communication_from_task
action_code = str(action_result.action_code or "").upper()
enrich_chatwoot_communication_from_task(
source_message_id=source_event_id, classification=action_code,
confidence=float(getattr(decision, "confidence", 0.0) or 0.0),
ignored=is_ignored_bounce or action_code in {"IGNORE_SPAM", "SPAM"},
conversation_id=request.conversation_id, contact_id=request.contact_id,
body=request.last_customer_message, task_id=task_id,
metadata={"raw_event_id": raw_event_id, "message_id": message_id,
"action_run_id": action_run_id, "decision_source": decision_source},
)
return AnalyzeResponse(
app="ClientFlow",
model=settings.openrouter_model,

70
app/communication_service.py
View File

@@ -477,6 +477,76 @@ def link_communication_to_opportunity(communication_id: str, opportunity_id: Opt
"""), {"id": communication_id, "opportunity_id": opportunity_id or ""})
def upsert_chatwoot_inbound_communication(
*, source_message_id: str, conversation_id: Optional[str], contact_id: Optional[str],
body: str, classification: Optional[str] = None, confidence: Optional[float] = None,
status: str = "new", customer_id: Optional[str] = None,
opportunity_id: Optional[str] = None, task_id: Optional[str] = None,
metadata: Optional[Dict[str, Any]] = None,
) -> Optional[str]:
"""Materialize one inbound Chatwoot message without schema or work creation."""
if not str(source_message_id or "").strip():
return None
with engine.begin() as conn:
row = conn.execute(text("""
INSERT INTO communications (
source_system, source_message_id, conversation_id, contact_id,
direction, body, classification, confidence, status,
customer_id, opportunity_id, task_id, metadata
) VALUES (
'chatwoot', :source_message_id, :conversation_id, :contact_id,
'inbound', :body, :classification, :confidence, :status,
CAST(:customer_id AS UUID), CAST(:opportunity_id AS UUID),
CAST(:task_id AS UUID), CAST(:metadata AS JSONB)
)
ON CONFLICT (source_system, source_message_id) WHERE source_message_id IS NOT NULL
DO UPDATE SET
conversation_id=COALESCE(NULLIF(EXCLUDED.conversation_id,''), communications.conversation_id),
contact_id=COALESCE(NULLIF(EXCLUDED.contact_id,''), communications.contact_id),
body=COALESCE(NULLIF(EXCLUDED.body,''), communications.body),
classification=COALESCE(EXCLUDED.classification, communications.classification),
confidence=COALESCE(EXCLUDED.confidence, communications.confidence),
status=CASE WHEN EXCLUDED.classification IS NULL THEN communications.status ELSE EXCLUDED.status END,
customer_id=COALESCE(EXCLUDED.customer_id, communications.customer_id),
opportunity_id=COALESCE(EXCLUDED.opportunity_id, communications.opportunity_id),
task_id=COALESCE(EXCLUDED.task_id, communications.task_id),
metadata=COALESCE(communications.metadata,'{}'::jsonb) || EXCLUDED.metadata,
updated_at=now()
RETURNING id::text
"""), {"source_message_id": str(source_message_id),
"conversation_id": conversation_id or "", "contact_id": contact_id or "",
"body": body or "", "classification": classification,
"confidence": confidence, "status": status or "new",
"customer_id": customer_id or None, "opportunity_id": opportunity_id or None,
"task_id": task_id or None,
"metadata": json.dumps(metadata or {}, ensure_ascii=False, default=str)}).first()
return str(row[0]) if row else None
def enrich_chatwoot_communication_from_task(
*, source_message_id: str, classification: str, confidence: Optional[float],
ignored: bool, conversation_id: Optional[str], contact_id: Optional[str], body: str,
task_id: Optional[str], metadata: Optional[Dict[str, Any]] = None,
) -> Optional[str]:
customer_id = opportunity_id = None
if task_id:
with engine.begin() as conn:
linked = conn.execute(text("""
SELECT NULLIF(t.customer_id, '') AS customer_id, t.opportunity_id::text
FROM tasks t WHERE t.id=CAST(:task_id AS UUID)
"""), {"task_id": task_id}).mappings().first()
if linked:
customer_id = linked.get("customer_id")
opportunity_id = linked.get("opportunity_id")
return upsert_chatwoot_inbound_communication(
source_message_id=source_message_id, conversation_id=conversation_id,
contact_id=contact_id, body=body, classification=classification,
confidence=confidence, status="ignored" if ignored else "classified",
customer_id=customer_id, opportunity_id=opportunity_id, task_id=task_id,
metadata=metadata,
)
def record_outbound_communication(

5
app/db.py
View File

@@ -296,6 +296,11 @@ def ensure_core_schema() -> None:
conn.execute(text("CREATE INDEX IF NOT EXISTS idx_raw_events_conversation ON raw_events(conversation_id)"))
conn.execute(text("CREATE INDEX IF NOT EXISTS idx_messages_conversation ON messages(conversation_id)"))
conn.execute(text("CREATE INDEX IF NOT EXISTS idx_messages_raw_event ON messages(raw_event_id)"))
conn.execute(text("""
CREATE UNIQUE INDEX IF NOT EXISTS ux_messages_source_event
ON messages(source_system, source_event_id)
WHERE source_event_id IS NOT NULL
"""))
conn.execute(text("CREATE INDEX IF NOT EXISTS idx_raw_events_message ON raw_events(message_id)"))
conn.execute(text("CREATE INDEX IF NOT EXISTS idx_action_runs_message ON action_runs(message_id)"))
conn.execute(text("CREATE INDEX IF NOT EXISTS idx_action_runs_raw_event ON action_runs(raw_event_id)"))

143
app/persistence.py
View File

@@ -25,6 +25,92 @@ def _uuid_or_none(value):
return value or None
def _lock_source_identity(conn: Any, source_system: str, source_event_id: str | None) -> None:
if source_event_id:
conn.execute(text("SELECT pg_advisory_xact_lock(hashtext(:identity))"), {
"identity": f"message:{source_system}:{source_event_id}",
})
def save_inbound_message(
*,
request: AnalyzeRequest,
raw_event_id: Optional[str] = None,
source_event_id: Optional[str] = None,
raw_body: Optional[str] = None,
clean_body: Optional[str] = None,
) -> Tuple[str, Optional[str]]:
"""Persist the canonical inbound message before classification.
Chatwoot identity comes from its raw event when callers do not pass it.
The advisory lock makes retries safe even while older databases are waiting
for the canonical identity index migration.
"""
if not settings.clientflow_persist:
return "", source_event_id
source_system = request.source or "manual"
conversation_id = request.conversation_id or "manual"
with engine.begin() as conn:
if raw_event_id:
raw = conn.execute(text("""
SELECT source_system, source_event_id
FROM raw_events WHERE id = CAST(:raw_event_id AS UUID)
"""), {"raw_event_id": raw_event_id}).mappings().first()
if raw:
source_system = str(raw.get("source_system") or source_system)
source_event_id = source_event_id or raw.get("source_event_id")
_lock_source_identity(conn, source_system, source_event_id)
existing = None
if source_event_id:
existing = conn.execute(text("""
SELECT id::text FROM messages
WHERE source_system=:source_system AND source_event_id=:source_event_id
LIMIT 1
"""), {"source_system": source_system, "source_event_id": source_event_id}).first()
if not existing and raw_event_id:
existing = conn.execute(text("""
SELECT id::text FROM messages WHERE raw_event_id=CAST(:raw_event_id AS UUID) LIMIT 1
"""), {"raw_event_id": raw_event_id}).first()
if existing:
message_id = str(existing[0])
conn.execute(text("""
UPDATE messages SET
source_event_id=COALESCE(source_event_id, :source_event_id),
raw_event_id=COALESCE(raw_event_id, CAST(:raw_event_id AS UUID)),
conversation_id=COALESCE(NULLIF(conversation_id,''), :conversation_id),
contact_id=COALESCE(NULLIF(contact_id,''), :contact_id),
raw_body=COALESCE(NULLIF(raw_body,''), :raw_body),
clean_body=COALESCE(NULLIF(clean_body,''), :clean_body)
WHERE id=CAST(:message_id AS UUID)
"""), {"message_id": message_id, "source_event_id": source_event_id,
"raw_event_id": raw_event_id, "conversation_id": conversation_id,
"contact_id": request.contact_id, "raw_body": raw_body or request.last_customer_message,
"clean_body": clean_body or request.last_customer_message})
else:
row = conn.execute(text("""
INSERT INTO messages (
raw_event_id, source_system, source_event_id, conversation_id,
contact_id, direction, raw_body, clean_body, previous_context, metadata
) VALUES (
CAST(:raw_event_id AS UUID), :source_system, :source_event_id,
:conversation_id, :contact_id, 'inbound', :raw_body, :clean_body,
:previous_context, CAST(:metadata AS JSONB)
) RETURNING id::text
"""), {"raw_event_id": raw_event_id, "source_system": source_system,
"source_event_id": source_event_id, "conversation_id": conversation_id,
"contact_id": request.contact_id, "raw_body": raw_body or request.last_customer_message,
"clean_body": clean_body or request.last_customer_message,
"previous_context": request.previous_context,
"metadata": _json({"current_state": request.current_state.model_dump()})}).first()
message_id = str(row[0])
if raw_event_id:
conn.execute(text("""UPDATE raw_events SET message_id=CAST(:message_id AS UUID)
WHERE id=CAST(:raw_event_id AS UUID)"""), {
"message_id": message_id, "raw_event_id": raw_event_id,
})
return message_id, source_event_id
def save_action_run(
*,
request: AnalyzeRequest,
@@ -37,53 +123,30 @@ def save_action_run(
raw_body: Optional[str] = None,
clean_body: Optional[str] = None,
raw_event_id: Optional[str] = None,
source_event_id: Optional[str] = None,
message_id: Optional[str] = None,
) -> Tuple[str, str]:
if not settings.clientflow_persist:
return "", ""
conversation_id = request.conversation_id or "manual"
with engine.begin() as conn:
message_row = conn.execute(text("""
INSERT INTO messages (
raw_event_id,
source_system,
source_event_id,
conversation_id,
contact_id,
direction,
raw_body,
clean_body,
previous_context,
metadata
)
VALUES (
CAST(:raw_event_id AS UUID),
:source_system,
NULL,
:conversation_id,
:contact_id,
'inbound',
:raw_body,
:clean_body,
:previous_context,
CAST(:metadata AS JSONB)
)
RETURNING id::text
"""), {
"raw_event_id": raw_event_id,
"source_system": request.source or "manual",
"conversation_id": conversation_id,
"contact_id": request.contact_id,
"raw_body": raw_body or request.last_customer_message,
"clean_body": clean_body or request.last_customer_message,
"previous_context": request.previous_context,
"metadata": _json({
"current_state": request.current_state.model_dump(),
}),
}).fetchone()
if not message_id:
message_id, source_event_id = save_inbound_message(
request=request, raw_event_id=raw_event_id, source_event_id=source_event_id,
raw_body=raw_body, clean_body=clean_body,
)
message_id = message_row[0]
with engine.begin() as conn:
_lock_source_identity(conn, request.source or "manual", source_event_id)
existing_run = conn.execute(text("""
SELECT id::text FROM action_runs
WHERE message_id=CAST(:message_id AS UUID)
OR (CAST(:raw_event_id AS UUID) IS NOT NULL AND raw_event_id=CAST(:raw_event_id AS UUID))
ORDER BY created_at LIMIT 1
"""), {"message_id": message_id, "raw_event_id": raw_event_id}).first()
if existing_run:
return str(existing_run[0]), str(message_id)
run_row = conn.execute(text("""
INSERT INTO action_runs (

12
app/task_service.py
View File

@@ -191,6 +191,18 @@ def create_task_from_action_result(
action_code = data.get("action_code") or "REVIEW_MANUALLY"
if str(action_code or "").strip().upper() == "IGNORE_BOUNCE":
return None
if raw_event_id:
# A retry may re-run classification after the action_run was already
# persisted. Raw event identity wins over a potentially different LLM
# answer and prevents a second work item for the same inbound fact.
with engine.begin() as conn:
existing = conn.execute(text("""
SELECT id::text FROM tasks
WHERE raw_event_id=CAST(:raw_event_id AS UUID)
ORDER BY created_at LIMIT 1
"""), {"raw_event_id": raw_event_id}).first()
if existing:
return str(existing[0])
config = get_action_config(action_code)
route = data.get("route") or config["route"]

11
app/webhooks_chatwoot.py
View File

@@ -416,7 +416,11 @@ async def process_saved_chatwoot_raw_event(raw_event_id: str, payload: Dict[str,
contact_id=extracted["contact_id"],
)
response = await analyze(analyze_request, raw_event_id=raw_event_id)
response = await analyze(
analyze_request,
raw_event_id=raw_event_id,
source_event_id=extracted.get("source_event_id"),
)
chatwoot_note_result = {
"status": "skipped",
@@ -429,11 +433,14 @@ async def process_saved_chatwoot_raw_event(raw_event_id: str, payload: Dict[str,
action_result=response.action_result,
)
ignored_action = str(response.action_result.action_code or "").upper() in {
"IGNORE_BOUNCE", "IGNORE_SPAM", "SPAM",
}
mark_raw_event_processed(
raw_event_id=raw_event_id,
action_run_id=response.action_run_id,
message_id=response.message_id,
ignored=False,
ignored=ignored_action,
error=None,
)