Import ClientFlow production v4928.1.5.132.4

This commit is contained in:
plx
2026-07-29 13:11:01 +00:00
parent 6445044ac6
commit 261d342057
405 changed files with 48373 additions and 1401 deletions

View File

@@ -0,0 +1,505 @@
#!/usr/bin/env python3
"""Audita mensagens que existem no Chatwoot mas não aparecem no ClientFlow.
Caso de uso: ClientFlow esteve em baixo e o Chatwoot não conseguiu entregar
webhooks. Nessa situação não existe raw_event para auditar; por isso este script
lê conversas/mensagens diretamente da API do Chatwoot e compara cada mensagem
inbound pública com raw_events/messages/communications no PostgreSQL.
Uso seguro:
PYTHONPATH=. python scripts/audit_chatwoot_remote_missing_messages.py --hours 72
PYTHONPATH=. python scripts/audit_chatwoot_remote_missing_messages.py --hours 72 --include-ok
PYTHONPATH=. python scripts/audit_chatwoot_remote_missing_messages.py --hours 72 --post-missing --limit 25
Variáveis necessárias:
CHATWOOT_BASE_URL
CHATWOOT_ACCOUNT_ID
CHATWOOT_API_TOKEN
DATABASE_URL
Para --post-missing também:
CLIENTFLOW_WEBHOOK_SECRET
CLIENTFLOW_WEBHOOK_URL opcional; por defeito http://127.0.0.1:8020/webhooks/chatwoot
"""
from __future__ import annotations
import argparse
import csv
import hashlib
import hmac
import json
import os
import sys
import time
import urllib.error
import urllib.parse
import urllib.request
from dataclasses import dataclass
from datetime import datetime, timedelta, timezone
from pathlib import Path
from typing import Any, Iterable
PROJECT_ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(PROJECT_ROOT))
os.chdir(PROJECT_ROOT)
from sqlalchemy import text
from app.db import engine
OUT_CSV = Path("/tmp/clientflow_chatwoot_remote_missing_messages.csv")
OUT_MD = Path("/tmp/clientflow_chatwoot_remote_missing_messages.md")
def env(name: str, default: str = "") -> str:
return os.getenv(name, default).strip()
CHATWOOT_BASE_URL = env("CHATWOOT_BASE_URL").rstrip("/")
CHATWOOT_ACCOUNT_ID = env("CHATWOOT_ACCOUNT_ID")
CHATWOOT_API_TOKEN = env("CHATWOOT_API_TOKEN")
CLIENTFLOW_WEBHOOK_SECRET = env("CLIENTFLOW_WEBHOOK_SECRET")
CLIENTFLOW_WEBHOOK_URL = env("CLIENTFLOW_WEBHOOK_URL", "http://127.0.0.1:8020/webhooks/chatwoot")
@dataclass
class RemoteMessage:
conversation: dict[str, Any]
message: dict[str, Any]
recent_context: list[dict[str, Any]]
def clean(value: Any, limit: int = 240) -> str:
txt = " ".join(str(value or "").replace("\r", " ").replace("\n", " ").split())
return txt if len(txt) <= limit else txt[: limit - 1].rstrip() + ""
def ts_to_datetime(value: Any) -> datetime | None:
if value is None:
return None
try:
if isinstance(value, (int, float)):
return datetime.fromtimestamp(float(value), tz=timezone.utc)
s = str(value).strip().replace("Z", "+00:00")
if not s:
return None
return datetime.fromisoformat(s).astimezone(timezone.utc)
except Exception:
return None
def dt_to_display(value: Any) -> str:
dt = ts_to_datetime(value)
return dt.isoformat() if dt else str(value or "")
def message_created_at(message: dict[str, Any]) -> datetime:
return ts_to_datetime(message.get("created_at")) or datetime.fromtimestamp(0, tz=timezone.utc)
def conversation_last_activity(conversation: dict[str, Any]) -> datetime | None:
for key in ["last_activity_at", "updated_at", "created_at"]:
dt = ts_to_datetime(conversation.get(key))
if dt:
return dt
return None
def request_json(method: str, url: str, *, body: dict[str, Any] | None = None, headers: dict[str, str] | None = None) -> dict[str, Any]:
data = None
final_headers = dict(headers or {})
if body is not None:
data = json.dumps(body, ensure_ascii=False, separators=(",", ":")).encode("utf-8")
final_headers["Content-Type"] = "application/json"
req = urllib.request.Request(url, data=data, headers=final_headers, method=method)
try:
with urllib.request.urlopen(req, timeout=45) as resp:
raw = resp.read().decode("utf-8", errors="replace")
try:
parsed: Any = json.loads(raw) if raw else {}
except json.JSONDecodeError:
parsed = {}
return {"ok": 200 <= resp.status < 300, "status": resp.status, "json": parsed, "raw": raw}
except urllib.error.HTTPError as e:
raw = e.read().decode("utf-8", errors="replace")
return {"ok": False, "status": e.code, "json": None, "raw": raw}
except urllib.error.URLError as e:
return {"ok": False, "status": 0, "json": None, "raw": str(e)}
def chatwoot_headers() -> dict[str, str]:
return {"api_access_token": CHATWOOT_API_TOKEN, "Accept": "application/json"}
def payload_list(data: Any) -> list[dict[str, Any]]:
if isinstance(data, list):
return [item for item in data if isinstance(item, dict)]
if not isinstance(data, dict):
return []
candidates = [
data.get("payload"),
data.get("data", {}).get("payload") if isinstance(data.get("data"), dict) else None,
data.get("data"),
data.get("messages"),
]
for item in candidates:
if isinstance(item, list):
return [row for row in item if isinstance(row, dict)]
return []
def fetch_conversations(status: str, *, max_pages: int) -> list[dict[str, Any]]:
all_items: list[dict[str, Any]] = []
for page in range(1, max_pages + 1):
query = urllib.parse.urlencode({"status": status, "page": page})
url = f"{CHATWOOT_BASE_URL}/api/v1/accounts/{CHATWOOT_ACCOUNT_ID}/conversations?{query}"
result = request_json("GET", url, headers=chatwoot_headers())
if not result["ok"]:
print(f"ERRO Chatwoot conversations status={status} page={page}: {result['status']} {result['raw'][:300]}")
break
items = payload_list(result["json"])
if not items:
break
all_items.extend(items)
if len(items) < 10:
break
return all_items
def fetch_messages(conversation_id: str) -> list[dict[str, Any]]:
url = f"{CHATWOOT_BASE_URL}/api/v1/accounts/{CHATWOOT_ACCOUNT_ID}/conversations/{conversation_id}/messages"
result = request_json("GET", url, headers=chatwoot_headers())
if not result["ok"]:
print(f"ERRO Chatwoot messages conversation={conversation_id}: {result['status']} {result['raw'][:300]}")
return []
return payload_list(result["json"])
def is_incoming(message: dict[str, Any]) -> bool:
mt = message.get("message_type")
return mt == "incoming" or mt == 0 or str(mt).lower() == "incoming"
def message_content(message: dict[str, Any]) -> str:
return str(message.get("content") or message.get("text") or message.get("body") or "").strip()
def message_id(message: dict[str, Any]) -> str:
return str(message.get("id") or message.get("message_id") or "").strip()
def sender_from_conversation(conversation: dict[str, Any], message: dict[str, Any]) -> dict[str, Any]:
sender: dict[str, Any] = {}
meta = conversation.get("meta") or {}
if isinstance(meta, dict) and isinstance(meta.get("sender"), dict):
sender.update(meta.get("sender") or {})
if isinstance(conversation.get("contact"), dict):
sender.update({k: v for k, v in conversation["contact"].items() if v is not None})
if isinstance(message.get("sender"), dict):
sender.update({k: v for k, v in message["sender"].items() if v is not None})
return sender
def iter_remote_messages(
*,
statuses: Iterable[str],
cutoff: datetime,
max_pages: int,
max_context_messages: int,
) -> list[RemoteMessage]:
seen_conversations: set[str] = set()
selected: list[RemoteMessage] = []
for status in statuses:
conversations = fetch_conversations(status, max_pages=max_pages)
print(f"--- status={status} conversations={len(conversations)}")
for conv in conversations:
conv_id = str(conv.get("id") or "").strip()
if not conv_id or conv_id in seen_conversations:
continue
seen_conversations.add(conv_id)
last_activity = conversation_last_activity(conv)
if last_activity and last_activity < cutoff:
continue
messages = fetch_messages(conv_id)
messages.sort(key=message_created_at)
public_messages = [m for m in messages if not m.get("private") and message_content(m)]
for index, msg in enumerate(public_messages):
if not is_incoming(msg):
continue
if message_created_at(msg) < cutoff:
continue
mid = message_id(msg)
if not mid:
continue
start = max(0, index - max_context_messages)
context = public_messages[start : index + 1]
selected.append(RemoteMessage(conversation=conv, message=msg, recent_context=context))
selected.sort(key=lambda item: message_created_at(item.message))
return selected
def load_clientflow_status(source_event_ids: list[str]) -> dict[str, dict[str, Any]]:
if not source_event_ids:
return {}
with engine.begin() as conn:
rows = conn.execute(text("""
WITH ids AS (
SELECT unnest(CAST(:ids AS text[])) AS source_event_id
)
SELECT
ids.source_event_id,
re.id::text AS raw_event_id,
re.created_at AS raw_event_created_at,
re.processed,
re.ignored,
re.processing_error,
re.message_id::text AS raw_event_message_id,
re.action_run_id::text AS raw_event_action_run_id,
m.id::text AS message_id,
c.id::text AS communication_id,
o.id::text AS opportunity_id
FROM ids
LEFT JOIN raw_events re
ON re.source_system = 'chatwoot'
AND re.source_event_id = ids.source_event_id
LEFT JOIN messages m
ON (
(re.message_id IS NOT NULL AND m.id = re.message_id)
OR (re.id IS NOT NULL AND m.raw_event_id = re.id)
OR (m.source_system = 'chatwoot' AND m.source_event_id = ids.source_event_id)
)
LEFT JOIN communications c
ON c.source_system = 'chatwoot'
AND c.source_message_id = ids.source_event_id
LEFT JOIN opportunities o
ON o.conversation_id = COALESCE(re.conversation_id, m.conversation_id)
"""), {"ids": source_event_ids}).mappings().all()
return {str(row["source_event_id"]): dict(row) for row in rows}
def classify(row: dict[str, Any] | None) -> str:
row = row or {}
visible_message_id = row.get("message_id") or row.get("raw_event_message_id")
if not row.get("raw_event_id") and not visible_message_id and not row.get("communication_id"):
return "MISSING_IN_CLIENTFLOW"
if row.get("processing_error"):
return "RAW_PROCESSING_ERROR"
if row.get("raw_event_id") and not row.get("processed") and not row.get("ignored"):
return "RAW_PENDING"
if row.get("ignored"):
return "RAW_IGNORED"
if row.get("processed") and not visible_message_id and not row.get("communication_id"):
return "PROCESSED_BUT_NOT_VISIBLE"
if not row.get("opportunity_id"):
return "VISIBLE_BUT_UNLINKED_CONVERSATION"
return "OK"
def build_webhook_payload(remote: RemoteMessage) -> dict[str, Any]:
conv = remote.conversation
msg = remote.message
conv_id = str(conv.get("id") or msg.get("conversation_id") or "")
sender = sender_from_conversation(conv, msg)
contact_id = str(sender.get("id") or conv.get("contact_id") or conv.get("contact", {}).get("id") or msg.get("contact_id") or "")
normalized_message = dict(msg)
normalized_message["id"] = message_id(msg)
normalized_message["content"] = message_content(msg)
normalized_message["message_type"] = "incoming"
normalized_message["conversation_id"] = conv_id
normalized_message["sender"] = sender
normalized_message["private"] = bool(msg.get("private"))
conversation_payload = dict(conv)
conversation_payload["id"] = conv_id
conversation_payload["messages"] = remote.recent_context
conversation_payload["contact"] = conv.get("contact") if isinstance(conv.get("contact"), dict) else sender
meta = conversation_payload.get("meta") if isinstance(conversation_payload.get("meta"), dict) else {}
meta = dict(meta)
meta.setdefault("sender", sender)
conversation_payload["meta"] = meta
return {
"event": "message_created",
"id": normalized_message["id"],
"message_id": normalized_message["id"],
"conversation_id": conv_id,
"contact_id": contact_id,
"content": normalized_message["content"],
"message_type": "incoming",
"private": bool(msg.get("private")),
"message": normalized_message,
"conversation": conversation_payload,
"sender": sender,
"backfill": {
"source": "chatwoot_remote_missing_messages",
"reason": "clientflow_down_or_webhook_not_delivered",
"audited_at": datetime.now(timezone.utc).isoformat(),
},
}
def sign_body(body_raw: str) -> dict[str, str]:
ts = str(int(time.time()))
msg = ts.encode("utf-8") + b"." + body_raw.encode("utf-8")
sig = "sha256=" + hmac.new(CLIENTFLOW_WEBHOOK_SECRET.encode("utf-8"), msg, hashlib.sha256).hexdigest()
return {"Content-Type": "application/json", "X-Chatwoot-Timestamp": ts, "X-Chatwoot-Signature": sig}
def post_to_clientflow(payload: dict[str, Any]) -> dict[str, Any]:
body_raw = json.dumps(payload, ensure_ascii=False, separators=(",", ":"))
headers = sign_body(body_raw)
req = urllib.request.Request(CLIENTFLOW_WEBHOOK_URL, data=body_raw.encode("utf-8"), headers=headers, method="POST")
try:
with urllib.request.urlopen(req, timeout=60) as resp:
raw = resp.read().decode("utf-8", errors="replace")
return {"ok": 200 <= resp.status < 300, "status": resp.status, "raw": raw}
except urllib.error.HTTPError as e:
raw = e.read().decode("utf-8", errors="replace")
return {"ok": False, "status": e.code, "raw": raw}
except urllib.error.URLError as e:
return {"ok": False, "status": 0, "raw": str(e)}
def write_reports(rows: list[dict[str, Any]], counts: dict[str, int], *, hours: int) -> None:
fieldnames = list(rows[0].keys()) if rows else [
"status", "chatwoot_created_at", "conversation_id", "source_event_id", "email", "content",
"raw_event_id", "processed", "ignored", "processing_error", "message_id", "communication_id", "opportunity_id", "post_status",
]
with OUT_CSV.open("w", encoding="utf-8", newline="") as f:
writer = csv.DictWriter(f, fieldnames=fieldnames)
writer.writeheader()
writer.writerows(rows)
with OUT_MD.open("w", encoding="utf-8") as f:
f.write("# Auditoria remota Chatwoot → ClientFlow\n\n")
f.write(f"Janela: últimas {hours} horas\n\n")
f.write("## Resumo\n\n")
if not counts:
f.write("- OK: sem gaps inbound encontrados\n")
for status, total in sorted(counts.items(), key=lambda item: (-item[1], item[0])):
f.write(f"- {status}: {total}\n")
f.write("\n## Casos\n\n")
for row in rows[:300]:
f.write(f"### {row['status']} · conv #{row['conversation_id']} · msg {row['source_event_id']}\n\n")
f.write(f"- Data Chatwoot: {row['chatwoot_created_at']}\n")
f.write(f"- Email: {row['email'] or '-'}\n")
f.write(f"- Raw event: {row['raw_event_id'] or '-'}\n")
f.write(f"- Erro: {row['processing_error'] or '-'}\n")
if row.get("post_status"):
f.write(f"- Post recovery: {row['post_status']}\n")
f.write(f"- Excerto: {row['content'] or '-'}\n\n")
def main() -> int:
parser = argparse.ArgumentParser()
parser.add_argument("--hours", type=int, default=int(env("CHATWOOT_AUDIT_HOURS", "168")))
parser.add_argument("--statuses", default=env("CHATWOOT_AUDIT_STATUSES", "open,pending,resolved,snoozed"))
parser.add_argument("--max-pages", type=int, default=int(env("CHATWOOT_AUDIT_MAX_PAGES", "50")))
parser.add_argument("--include-ok", action="store_true")
parser.add_argument("--post-missing", action="store_true", help="Reenvia para /webhooks/chatwoot apenas MISSING_IN_CLIENTFLOW")
parser.add_argument("--limit", type=int, default=0, help="Limita mensagens a auditar/repostar; 0 = sem limite")
parser.add_argument("--max-context-messages", type=int, default=5)
args = parser.parse_args()
required = {
"CHATWOOT_BASE_URL": CHATWOOT_BASE_URL,
"CHATWOOT_ACCOUNT_ID": CHATWOOT_ACCOUNT_ID,
"CHATWOOT_API_TOKEN": CHATWOOT_API_TOKEN,
}
if args.post_missing:
required["CLIENTFLOW_WEBHOOK_SECRET"] = CLIENTFLOW_WEBHOOK_SECRET
missing = [k for k, v in required.items() if not v]
if missing:
raise SystemExit(f"Faltam variáveis: {', '.join(missing)}")
statuses = [s.strip() for s in str(args.statuses or "").split(",") if s.strip()]
cutoff = datetime.now(timezone.utc) - timedelta(hours=int(args.hours))
print(f"CHATWOOT_AUDIT_HOURS={args.hours}")
print(f"CHATWOOT_AUDIT_STATUSES={statuses}")
print(f"CHATWOOT_AUDIT_MAX_PAGES={args.max_pages}")
print(f"CUTOFF={cutoff.isoformat()}")
remote_messages = iter_remote_messages(
statuses=statuses,
cutoff=cutoff,
max_pages=int(args.max_pages),
max_context_messages=max(1, int(args.max_context_messages)),
)
if args.limit and args.limit > 0:
remote_messages = remote_messages[: int(args.limit)]
ids = [message_id(item.message) for item in remote_messages]
status_map = load_clientflow_status(ids)
rows: list[dict[str, Any]] = []
counts: dict[str, int] = {}
posted = 0
failed = 0
for item in remote_messages:
msg = item.message
conv = item.conversation
mid = message_id(msg)
cf = status_map.get(mid, {})
status = classify(cf)
if status == "OK" and not args.include_ok:
continue
post_status = ""
if args.post_missing and status == "MISSING_IN_CLIENTFLOW":
result = post_to_clientflow(build_webhook_payload(item))
if result["ok"]:
post_status = f"POSTED {result['status']}"
posted += 1
else:
post_status = f"FAILED {result['status']}: {clean(result['raw'], 300)}"
failed += 1
counts[status] = counts.get(status, 0) + 1
sender = sender_from_conversation(conv, msg)
rows.append({
"status": status,
"chatwoot_created_at": dt_to_display(msg.get("created_at")),
"conversation_id": str(conv.get("id") or msg.get("conversation_id") or ""),
"source_event_id": mid,
"email": str(sender.get("email") or ""),
"content": clean(message_content(msg)),
"raw_event_id": cf.get("raw_event_id") or "",
"processed": cf.get("processed") if cf.get("raw_event_id") else "",
"ignored": cf.get("ignored") if cf.get("raw_event_id") else "",
"processing_error": cf.get("processing_error") or "",
"message_id": cf.get("message_id") or cf.get("raw_event_message_id") or "",
"communication_id": cf.get("communication_id") or "",
"opportunity_id": cf.get("opportunity_id") or "",
"post_status": post_status,
})
write_reports(rows, counts, hours=int(args.hours))
print("Auditoria remota Chatwoot inbound concluída.")
print(f"Mensagens inbound públicas lidas do Chatwoot: {len(remote_messages)}")
print(f"CSV: {OUT_CSV}")
print(f"Markdown: {OUT_MD}")
print("Resumo:")
if not counts:
print("OK: sem gaps inbound encontrados")
for status, total in sorted(counts.items(), key=lambda item: (-item[1], item[0])):
print(f"{status}: {total}")
if args.post_missing:
print(f"posted={posted}")
print(f"failed={failed}")
return 0 if failed == 0 else 1
if __name__ == "__main__":
raise SystemExit(main())