admin/clientflow_backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Import ClientFlow production v4928.1.5.132.4

Browse Source
This commit is contained in:
plx
2026-07-29 13:11:01 +00:00
parent 6445044ac6
commit 261d342057
405 changed files with 48373 additions and 1401 deletions
Download Patch File Download Diff File Expand all files Collapse all files

247
app/operations_service.py
View File

@@ -6,6 +6,7 @@ health checks and unified opportunity timeline without changing business state.
from __future__ import annotations
import os
import re
import subprocess
from typing import Any, Dict, List, Optional
@@ -14,6 +15,12 @@ from sqlalchemy import text
from app.config import settings
from app.db import engine
from app.operation_noise import is_low_value_no_opportunity_item, is_noise_operation_item
from app.work_center_action_policy import (
RECONSTRUCTED_SENSITIVE_ACTIONS,
association_review_required,
canonical_action_code,
reconstructed_review_required,
)
def _int(value: Any) -> int:
@@ -77,6 +84,197 @@ def build_chatwoot_conversation_url(conversation_id: Optional[str]) -> str:
return f"{base_url}/app/accounts/{account_id}/conversations/{conversation_id}"
_IDENTITY_STOPWORDS = {
"da", "de", "do", "dos", "das", "e", "lda", "ltda", "unipessoal",
"sa", "s.a", "s.a.", "email", "mail", "geral", "info", "office",
"frontoffice", "comercial", "vendas", "admin", "contacto", "contact",
}
def _norm_identity(value: Any) -> str:
return " ".join(re.sub(r"[^0-9a-zA-ZÀ-ÿ]+", " ", str(value or "").casefold()).split())
def _identity_tokens(value: Any) -> set[str]:
return {
token
for token in _norm_identity(value).split()
if len(token) >= 3 and token not in _IDENTITY_STOPWORDS
}
def _email_tokens(value: Any) -> set[str]:
email = str(value or "").strip().casefold()
local = email.split("@", 1)[0]
return {
token
for token in re.split(r"[^0-9a-zA-ZÀ-ÿ]+", local)
if len(token) >= 3 and token not in _IDENTITY_STOPWORDS
}
def _name_matches_email(name: Any, email: Any) -> bool:
name_tokens = _identity_tokens(name)
email_tokens = _email_tokens(email)
if not name_tokens or not email_tokens:
return False
if name_tokens & email_tokens:
return True
return any(nt in et or et in nt for nt in name_tokens for et in email_tokens)
def _looks_like_campaign_process(title: Any) -> bool:
title_norm = _norm_identity(title)
return "carregadores" in title_norm and "para" in title_norm
def _process_customer_hint(title: Any) -> str:
"""Extract the intended customer/company from common campaign/process titles.
This is only a UI safety hint. It must never create/link customers. It helps
avoid displaying a contaminated Chatwoot contact name as the card owner.
"""
value = str(title or "").strip()
if not value:
return ""
# Examples: "Re: Carregadores ... para a Nova Maquiambiente" and
# "Processo reconstruído · F.S. Motors".
if "·" in value:
tail = value.rsplit("·", 1)[-1].strip()
if tail and not tail.upper().startswith(("ORC.", "S0")):
return tail
match = re.search(r"\bpara\s+(?:a|o|as|os|à|ao)?\s*(.+)$", value, re.IGNORECASE)
if match:
candidate = re.sub(r"\s+", " ", match.group(1)).strip(" .:-–—")
# Avoid returning a long quoted email/thread suffix.
candidate = re.split(r"\s+(?:de:|from:|enviada:|sent:)", candidate, maxsplit=1, flags=re.IGNORECASE)[0].strip()
if 2 <= len(candidate) <= 120:
return candidate
return ""
def _sanitize_operation_identities(items: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
"""Prevent polluted Chatwoot sender names from becoming card titles.
Chatwoot contact names can be reused or contaminated after campaign replies.
The workbench must show the real process/customer context, not blindly trust
``payload.sender.name``. This function is deliberately UI-only: it does not
change tasks, contacts, customers or opportunities.
"""
name_to_emails: Dict[str, set[str]] = {}
for item in items:
name = str(item.get("sender_name") or item.get("contact_display_name") or "").strip()
email = str(item.get("sender_email") or item.get("customer_email") or "").strip().casefold()
key = _norm_identity(name)
if key and email:
name_to_emails.setdefault(key, set()).add(email)
repeated_names = {key for key, emails in name_to_emails.items() if len(emails) > 1}
for item in items:
fiscal = str(item.get("fiscal_customer_name") or "").strip()
if fiscal:
item["contact_identity_status"] = "fiscal_customer"
continue
sender_name = str(item.get("sender_name") or item.get("contact_display_name") or "").strip()
sender_email = str(item.get("sender_email") or item.get("customer_email") or "").strip()
opportunity_title = str(item.get("opportunity_title") or "").strip()
hint = _process_customer_hint(opportunity_title)
if hint:
item["process_customer_hint"] = hint
key = _norm_identity(sender_name)
reused = bool(key and key in repeated_names)
email_match = _name_matches_email(sender_name, sender_email)
campaign_context = _looks_like_campaign_process(opportunity_title)
unsafe = False
reason = ""
if sender_name and reused:
unsafe = True
reason = "sender_name_reused_across_emails"
elif sender_name and campaign_context and hint and not email_match:
unsafe = True
reason = "sender_name_not_supported_by_email_or_process"
if unsafe:
item["contact_identity_status"] = "unsafe"
item["contact_identity_reason"] = reason
# Force operation_card_title to prefer the process/company hint or
# email instead of a possibly contaminated person name.
item["contact_display_name"] = ""
item["customer_name"] = hint or sender_email or "Contacto sem identificação"
else:
item["contact_identity_status"] = "trusted" if sender_name else "missing"
if not sender_name and hint:
item["customer_name"] = hint
return items
def _is_reconstructed_mode(value: Any) -> bool:
return str(value or "").strip().lower() in {
"reconstructed_invoice_review",
"historical_reconstructed",
"legacy_review",
}
def _looks_like_association_review(item: Dict[str, Any]) -> bool:
"""Return True only for an explicitly persisted association blocker.
v132 deliberately ignores opportunity titles such as "sem oportunidade".
Those titles are historical provenance and must not invent a current blocker.
"""
return association_review_required(
action_code=item.get("action_code"),
linking_status=item.get("opportunity_linking_status"),
)
def _normalise_work_item_intent(items: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
"""Expose one first safe operator action from persisted blockers/tasks.
No blocker is inferred from titles or free text. Association review and
reconstructed review must be explicit in persisted state. The underlying
task remains available through ``original_action_code`` for auditability.
"""
sensitive_codes = {canonical_action_code(code) for code in RECONSTRUCTED_SENSITIVE_ACTIONS}
for item in items:
original_code = canonical_action_code(item.get("action_code"))
item["original_action_code"] = original_code
if _looks_like_association_review(item):
item["action_code"] = "ASSOCIATE_OPPORTUNITY"
item["queue"] = "rever"
item["priority"] = "alta"
item["opportunity_linking_status"] = item.get("opportunity_linking_status") or "review_required"
source = str(item.get("source_system") or "").strip().lower()
item["title"] = "Validar associação do documento" if source == "jasmin" else "Validar associação operacional"
item["detail"] = (
"Confirmar primeiro a que processo pertence a evidência. "
"Ligar a uma oportunidade existente, criar uma nova ou ignorar; "
"só depois executar ações comerciais, fiscais ou financeiras."
)
continue
metadata = item.get("opportunity_metadata") or {}
if not isinstance(metadata, dict):
metadata = {}
if reconstructed_review_required(metadata) and original_code in sensitive_codes:
item["action_code"] = "REVIEW_RECONSTRUCTED_PROCESS"
item["queue"] = "rever"
item["priority"] = "alta"
item["title"] = "Validar processo reconstruído"
item["detail"] = (
"Confirmar cliente, documento principal, valor e evidência de pagamento "
"antes de executar a ação financeira, fiscal ou logística sugerida."
)
item["reconstructed_review_required"] = True
return items
def _attach_operation_urls(items: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
cleaned: List[Dict[str, Any]] = []
for item in items:
@@ -94,7 +292,7 @@ def _attach_operation_urls(items: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
else:
item["chatwoot_url"] = ""
cleaned.append(item)
return cleaned
return _sanitize_operation_identities(cleaned)
def get_operations_summary(limit: int = 24) -> Dict[str, Any]:
@@ -110,6 +308,7 @@ def get_operations_summary(limit: int = 24) -> Dict[str, Any]:
SELECT
(SELECT COUNT(*) FROM opportunities WHERE status = 'open')::int AS open_opportunities,
(SELECT COUNT(*) FROM tasks WHERE status = 'pending')::int AS pending_tasks,
(SELECT COUNT(*) FROM tasks WHERE status = 'pending' AND action_code LIKE 'FOLLOW_UP_%' AND due_at IS NOT NULL AND due_at > now())::int AS scheduled_followups,
(SELECT COUNT(*) FROM integration_outbox WHERE status = 'pending')::int AS outbox_pending,
(SELECT COUNT(*) FROM integration_outbox WHERE status = 'failed' AND COALESCE(last_error,'') NOT ILIKE '%limpo manualmente%' AND COALESCE(last_error,'') NOT ILIKE '%resolvido manualmente%')::int AS outbox_failed,
(SELECT COUNT(*) FROM commercial_documents WHERE document_kind = 'quotation' AND status NOT IN ('failed','cancelled','converted'))::int AS open_quotations,
@@ -125,7 +324,7 @@ def get_operations_summary(limit: int = 24) -> Dict[str, Any]:
(SELECT COUNT(*) FROM tasks WHERE status = 'pending' AND (route = 'rever' OR action_code = 'REVIEW_MANUALLY'))::int AS review_tasks,
(SELECT COUNT(*) FROM integration_outbox WHERE status IN ('failed','blocked') AND COALESCE(last_error,'') NOT ILIKE '%limpo manualmente%' AND COALESCE(last_error,'') NOT ILIKE '%resolvido manualmente%')::int AS blocked_outbox,
(
(SELECT COUNT(*) FROM tasks WHERE status = 'pending')
(SELECT COUNT(*) FROM tasks WHERE status = 'pending' AND NOT (action_code LIKE 'FOLLOW_UP_%' AND due_at IS NOT NULL AND due_at > now()))
+ (SELECT COUNT(*) FROM integration_outbox WHERE status IN ('failed','blocked') AND COALESCE(last_error,'') NOT ILIKE '%limpo manualmente%' AND COALESCE(last_error,'') NOT ILIKE '%resolvido manualmente%')
+ (SELECT COUNT(*) FROM communications WHERE status = 'needs_review')
)::int AS work_queue_total
@@ -186,7 +385,7 @@ def get_operations_summary(limit: int = 24) -> Dict[str, Any]:
work_items = conn.execute(text("""
SELECT * FROM (
SELECT 'task' AS source, t.id::text AS id, t.created_at,
SELECT 'task' AS source, t.id::text AS id, t.created_at, t.due_at,
CASE
WHEN COALESCE(t.metadata->>'opportunity_linking_status','') = 'ambiguous' THEN 'alta'
ELSE COALESCE(t.priority, CASE WHEN t.due_at < now() THEN 'alta' ELSE 'normal' END)
@@ -215,6 +414,8 @@ def get_operations_summary(limit: int = 24) -> Dict[str, Any]:
t.opportunity_id::text,
COALESCE(o.title, '') AS opportunity_title,
COALESCE(NULLIF(re.payload->'sender'->>'name',''), NULLIF(re.payload->'sender'->>'email',''), NULLIF(t.contact_id,''), '') AS customer_name,
NULLIF(re.payload->'sender'->>'name','') AS sender_name,
NULLIF(re.payload->'sender'->>'email','') AS sender_email,
COALESCE(cu_opp.name, cu_task.name, '') AS fiscal_customer_name,
COALESCE(cu_opp.email, cu_task.email, '') AS fiscal_customer_email,
COALESCE(cu_opp.tax_id, cu_task.tax_id, '') AS fiscal_customer_tax_id,
@@ -222,9 +423,16 @@ def get_operations_summary(limit: int = 24) -> Dict[str, Any]:
COALESCE(cu_opp.postal_zone, cu_task.postal_zone, '') AS fiscal_customer_postal_zone,
COALESCE(cu_opp.city_name, cu_task.city_name, '') AS fiscal_customer_city_name,
COALESCE(NULLIF(re.payload->'sender'->>'name',''), NULLIF(re.payload->'sender'->>'email',''), NULLIF(t.contact_id,''), '') AS contact_display_name,
COALESCE(NULLIF(re.payload->'sender'->>'email',''), '') AS customer_email,
COALESCE(t.metadata->>'no_opportunity_reason', '') AS no_opportunity_reason,
'/tasks/' || t.id::text AS href,
'Abrir' AS action_label
'Abrir' AS action_label,
COALESCE(t.metadata->>'opportunity_linking_status','') AS opportunity_linking_status,
COALESCE(t.metadata, '{}'::jsonb) AS item_metadata,
COALESCE(o.metadata, '{}'::jsonb) AS opportunity_metadata,
COALESCE(o.stage, '') AS opportunity_stage,
COALESCE(o.value_amount, 0) AS opportunity_value_amount,
COALESCE(o.currency, 'EUR') AS opportunity_currency
FROM tasks t
LEFT JOIN opportunities o ON o.id = t.opportunity_id
LEFT JOIN customers cu_opp ON cu_opp.id = o.local_customer_id
@@ -232,10 +440,11 @@ def get_operations_summary(limit: int = 24) -> Dict[str, Any]:
LEFT JOIN messages m ON m.id = t.message_id
LEFT JOIN raw_events re ON re.id = t.raw_event_id
WHERE t.status = 'pending'
AND NOT (t.action_code LIKE 'FOLLOW_UP_%' AND t.due_at IS NOT NULL AND t.due_at > now())
UNION ALL
SELECT 'outbox' AS source, io.id::text AS id, io.created_at,
SELECT 'outbox' AS source, io.id::text AS id, io.created_at, NULL::timestamptz AS due_at,
CASE WHEN io.status = 'failed' THEN 'alta' ELSE 'normal' END AS priority,
'sistema' AS queue,
upper(io.target_system || '_' || io.action_type) AS action_code,
@@ -249,6 +458,8 @@ def get_operations_summary(limit: int = 24) -> Dict[str, Any]:
NULLIF(io.payload->>'opportunity_id','') AS opportunity_id,
COALESCE(o.title, '') AS opportunity_title,
COALESCE(cu.name, '') AS customer_name,
''::text AS sender_name,
''::text AS sender_email,
COALESCE(cu.name, '') AS fiscal_customer_name,
COALESCE(cu.email, '') AS fiscal_customer_email,
COALESCE(cu.tax_id, '') AS fiscal_customer_tax_id,
@@ -256,9 +467,16 @@ def get_operations_summary(limit: int = 24) -> Dict[str, Any]:
COALESCE(cu.postal_zone, '') AS fiscal_customer_postal_zone,
COALESCE(cu.city_name, '') AS fiscal_customer_city_name,
''::text AS contact_display_name,
''::text AS customer_email,
''::text AS no_opportunity_reason,
'/outbox/' || io.id::text AS href,
CASE WHEN io.status = 'failed' THEN 'Reprocessar' ELSE 'Abrir' END AS action_label
CASE WHEN io.status = 'failed' THEN 'Reprocessar' ELSE 'Abrir' END AS action_label,
''::text AS opportunity_linking_status,
COALESCE(io.payload, '{}'::jsonb) AS item_metadata,
COALESCE(o.metadata, '{}'::jsonb) AS opportunity_metadata,
COALESCE(o.stage, '') AS opportunity_stage,
COALESCE(o.value_amount, 0) AS opportunity_value_amount,
COALESCE(o.currency, 'EUR') AS opportunity_currency
FROM integration_outbox io
LEFT JOIN opportunities o ON o.id::text = NULLIF(io.payload->>'opportunity_id','')
LEFT JOIN customers cu ON cu.id = o.local_customer_id
@@ -267,7 +485,7 @@ def get_operations_summary(limit: int = 24) -> Dict[str, Any]:
UNION ALL
SELECT 'communication' AS source, c.id::text AS id, c.created_at,
SELECT 'communication' AS source, c.id::text AS id, c.created_at, NULL::timestamptz AS due_at,
CASE WHEN c.status = 'needs_review' THEN 'normal' ELSE 'normal' END AS priority,
CASE
WHEN c.classification IN ('comprovativo_pagamento','pedido_fatura','aceitacao_orcamento','dados_fiscais') THEN 'financeiro'
@@ -287,6 +505,8 @@ def get_operations_summary(limit: int = 24) -> Dict[str, Any]:
c.opportunity_id::text,
COALESCE(o.title, '') AS opportunity_title,
COALESCE(cu.name, c.sender_name, c.sender_email, '') AS customer_name,
COALESCE(c.sender_name, '') AS sender_name,
COALESCE(c.sender_email, '') AS sender_email,
COALESCE(cu.name, '') AS fiscal_customer_name,
COALESCE(cu.email, '') AS fiscal_customer_email,
COALESCE(cu.tax_id, '') AS fiscal_customer_tax_id,
@@ -294,9 +514,16 @@ def get_operations_summary(limit: int = 24) -> Dict[str, Any]:
COALESCE(cu.postal_zone, '') AS fiscal_customer_postal_zone,
COALESCE(cu.city_name, '') AS fiscal_customer_city_name,
COALESCE(c.sender_name, c.sender_email, c.contact_id, '') AS contact_display_name,
COALESCE(c.sender_email, '') AS customer_email,
''::text AS no_opportunity_reason,
'/communications/' || c.id::text AS href,
CASE WHEN c.customer_id IS NULL THEN 'Associar cliente' ELSE 'Abrir' END AS action_label
CASE WHEN c.customer_id IS NULL THEN 'Associar cliente' ELSE 'Abrir' END AS action_label,
''::text AS opportunity_linking_status,
COALESCE(c.metadata, '{}'::jsonb) AS item_metadata,
COALESCE(o.metadata, '{}'::jsonb) AS opportunity_metadata,
COALESCE(o.stage, '') AS opportunity_stage,
COALESCE(o.value_amount, 0) AS opportunity_value_amount,
COALESCE(o.currency, 'EUR') AS opportunity_currency
FROM communications c
LEFT JOIN customers cu ON cu.id = c.customer_id
LEFT JOIN opportunities o ON o.id = c.opportunity_id
@@ -308,7 +535,7 @@ def get_operations_summary(limit: int = 24) -> Dict[str, Any]:
LIMIT :limit
"""), {"limit": int(limit)}).mappings().all()
cleaned_work_items = _attach_operation_urls([dict(r) for r in work_items])
cleaned_work_items = _attach_operation_urls(_normalise_work_item_intent([dict(r) for r in work_items]))
cleaned_counts = {k: _int(v) for k, v in dict(counts).items()}
# v4.9.0: the visible Operations total should match the queue the
# operator can actually act on, not raw pending tasks that include mailbox
@@ -375,6 +602,8 @@ def get_system_health_summary() -> Dict[str, Any]:
(SELECT COUNT(*) FROM opportunities o JOIN customers c ON c.id = o.local_customer_id WHERE o.status = 'open' AND (COALESCE(c.tax_id,'') = '' OR COALESCE(c.email,'') = '' OR COALESCE(c.street_name,'') = '' OR COALESCE(c.postal_zone,'') = '' OR COALESCE(c.city_name,'') = ''))::int AS active_incomplete_fiscal_customers,
(SELECT COUNT(*) FROM products WHERE active = TRUE AND COALESCE(jasmin_sales_item,'') = '')::int AS products_missing_external_code,
(SELECT COUNT(*) FROM raw_events WHERE source_system = 'chatwoot')::int AS chatwoot_events_total,
(SELECT COUNT(*) FROM raw_events WHERE source_system = 'chatwoot' AND event_type = 'message_created' AND processed = false AND ignored = false AND processing_error IS NULL AND COALESCE(payload #>> '{message_type}', '') = 'incoming')::int AS chatwoot_incoming_pending,
(SELECT COUNT(*) FROM raw_events WHERE source_system = 'chatwoot' AND event_type = 'message_created' AND COALESCE(payload #>> '{message_type}', '') = 'incoming' AND created_at >= now() - interval '24 hours')::int AS chatwoot_incoming_24h,
(SELECT EXTRACT(EPOCH FROM (now() - max(created_at)))::int FROM raw_events WHERE source_system = 'chatwoot')::int AS seconds_since_last_chatwoot_webhook,
(SELECT COUNT(*) FROM tasks WHERE status = 'pending' AND route = 'vendas')::int AS tasks_pending_vendas,
(SELECT COUNT(*) FROM tasks WHERE status = 'pending' AND route = 'financeiro')::int AS tasks_pending_financeiro,