Import ClientFlow production v4928.1.5.132.4

2026-07-29 13:11:01 +00:00
parent 6445044ac6
commit 261d342057
405 changed files with 48373 additions and 1401 deletions
--- a/app/fiscal_enrichment_service.py
+++ b/app/fiscal_enrichment_service.py
@@ -55,7 +55,6 @@ VERY_STRONG_MATCH_TYPES = {
    "email_exato_empresa_inferida",
    "email_exato_empresa_associada",
    "empresa_email_principal_exato",
-    "email_identity_company_internal",
 }
 STRONG_MATCH_TYPES = VERY_STRONG_MATCH_TYPES | {
    "nome_exato",
@@ -484,7 +483,7 @@ def _lookup_external_by_domain(domain: str) -> Optional[Dict[str, Any]]:

 def _lookup_external_by_name(name: str) -> Optional[Dict[str, Any]]:
    name = _clean(name)
-    if len(name) < 3:
+    if len(name) < 3 or not _is_valid_company_lookup_value(name) or _looks_like_person_name(name):
        return None
    normalized = normalize_fiscal_name(name)
    cached = _cache_lookup("name", normalized)
@@ -518,8 +517,8 @@ def _source_signals_from_opportunity(opportunity: Dict[str, Any]) -> List[Tuple[
        nif = normalize_tax_id(value)
        if nif:
            signals.append(("nif", nif))
-    name = _clean(opportunity.get("customer_name") or metadata.get("customer_name") or opportunity.get("title"))
-    if name:
+    name = _clean(opportunity.get("customer_name") or metadata.get("customer_name"))
+    if name and _is_likely_company_name_signal(name):
        signals.append(("name", name))
    # stable de-dup preserving order
    result: List[Tuple[str, str]] = []
@@ -569,21 +568,158 @@ def _normalized_identity_mentions(identity: Optional[Dict[str, Any]]) -> List[st
    return [normalize_fiscal_name(x) for x in _identity_company_mentions(identity) if normalize_fiscal_name(x)]


+FISCAL_NAME_WEAK_TOKENS = {
+    # Tokens that are common legal/geographic/sector descriptors and must never
+    # be enough to associate a fiscal customer by themselves.  Real audits found
+    # false links such as Verifone Portugal -> ERT and Feteira/Torrão Engenharia
+    # -> HUASI when generic tokens were treated as strong evidence.
+    "portugal", "portuguesa", "portugues", "pt",
+    "lda", "ltda", "limitada", "unipessoal", "sociedade", "empresa",
+    "grupo", "group", "holding", "sgps", "sa", "s", "a",
+    "comercial", "comercio", "comércio", "servicos", "servico", "serviço", "serviços",
+    "engenharia", "engineer", "engineering", "construcao", "construção", "construcoes", "construções",
+    "seguros", "seguro", "mediacao", "mediação", "contabilidade", "contabilista",
+    "solucoes", "soluções", "solutions", "sistemas", "systems", "industrial", "industriais",
+    "energy", "energia", "power", "electric", "eletrica", "elétrica", "tecnica", "técnica",
+}
+
+INVALID_COMPANY_LOOKUP_VALUES = {
+    "pt", "com", "www", "mail", "email", "geral", "info", "contacto", "contato",
+    "administrativo", "contabilidade", "financeiro", "support", "suporte", "noreply", "no-reply",
+}
+
+COMPANY_LEGAL_OR_ORG_TOKENS = {
+    "lda", "ltda", "limitada", "unipessoal", "sa", "sgps", "sociedade",
+    "empresa", "grupo", "holding", "associacao", "associação", "fundacao", "fundação",
+}
+
+
+def _strong_company_tokens(name_norm: str) -> set[str]:
+    """Return meaningful tokens for fiscal-name matching.
+
+    These tokens are used only as a fallback after exact/substring checks.
+    They intentionally exclude weak legal/geographic words such as Portugal,
+    LDA or S.A. so fuzzy identity extraction cannot auto-link unrelated
+    customers that share only generic descriptors.
+    """
+    return {
+        token
+        for token in (name_norm or "").split()
+        if len(token) >= 4 and token not in FISCAL_NAME_WEAK_TOKENS
+    }
+
+
+def _has_company_legal_or_org_signal(value: Any) -> bool:
+    normalized = normalize_fiscal_name(value)
+    tokens = set((normalized or "").split())
+    return bool(tokens & COMPANY_LEGAL_OR_ORG_TOKENS)
+
+
+def _is_valid_company_lookup_value(value: Any) -> bool:
+    """Return False for fragments that should never trigger fiscal lookup.
+
+    The worker used to create historical suggestions for values like "pt".
+    Those are evidence of an email/domain, not a company identity.
+    """
+    normalized = normalize_fiscal_name(value)
+    if not normalized:
+        return False
+    tokens = [t for t in normalized.split() if t]
+    if not tokens:
+        return False
+    if len(tokens) == 1:
+        token = tokens[0]
+        if token in INVALID_COMPANY_LOOKUP_VALUES or token in FISCAL_NAME_WEAK_TOKENS or len(token) < 3:
+            return False
+    strong = _strong_company_tokens(normalized)
+    if not strong:
+        return False
+    return True
+
+
+def _looks_like_person_name(value: Any) -> bool:
+    """Heuristic guard: avoid fiscal lookup by ordinary personal names."""
+    raw = _clean(value)
+    normalized = normalize_fiscal_name(raw)
+    if not normalized:
+        return False
+    tokens = [t for t in normalized.split() if t]
+    if len(tokens) < 2 or len(tokens) > 4:
+        return False
+    if _has_company_legal_or_org_signal(raw):
+        return False
+    if any(t in FISCAL_NAME_WEAK_TOKENS for t in tokens):
+        return False
+    # Names written as normal title-case, e.g. "Nuno Silva" or "Bárbara Gonçalves",
+    # are more likely contacts than companies.  Uppercase fiscal names still pass.
+    letters = [ch for ch in raw if ch.isalpha()]
+    upper_ratio = (sum(1 for ch in letters if ch.isupper()) / len(letters)) if letters else 0.0
+    if upper_ratio < 0.75:
+        return True
+    return False
+
+
+def _is_likely_company_name_signal(value: Any) -> bool:
+    """Return True only when a name is safe enough to use as fiscal-name signal."""
+    raw = _clean(value)
+    if not _is_valid_company_lookup_value(raw):
+        return False
+    if _looks_like_person_name(raw):
+        return False
+    return _has_company_legal_or_org_signal(raw) or len(_strong_company_tokens(normalize_fiscal_name(raw))) >= 2
+
+
 def _company_name_matches_mention(mention_norm: str, candidate_norm: str) -> bool:
    if not mention_norm or not candidate_norm:
        return False
    if len(mention_norm) < 4 or len(candidate_norm) < 4:
        return False
+    mention_tokens = _strong_company_tokens(mention_norm)
+    candidate_tokens = _strong_company_tokens(candidate_norm)
+    if not mention_tokens or not candidate_tokens:
+        return False
    if mention_norm == candidate_norm:
        return True
+    if mention_tokens == candidate_tokens:
+        return True
+    shared = mention_tokens & candidate_tokens
+    if shared:
+        return True
    # Substring matching is useful for variants like "Dietimport S.A" vs
-    # "DIETIMPORT, S.A.", but dangerous for tiny tokens such as "pt".
-    if len(mention_norm) >= 5 and len(candidate_norm) >= 5:
-        if mention_norm in candidate_norm or candidate_norm in mention_norm:
-            return True
-    mention_tokens = {t for t in mention_norm.split() if len(t) >= 5 and t not in {"unipessoal", "limitada"}}
-    candidate_tokens = {t for t in candidate_norm.split() if len(t) >= 5 and t not in {"unipessoal", "limitada"}}
-    return bool(mention_tokens & candidate_tokens)
+    # "DIETIMPORT, S.A.", but it must never bypass strong-token evidence.
+    # This prevents generic sector overlaps such as Engenharia/Construções.
+    return False
+
+
+def _has_manually_rejected_suggestion(opportunity_id: str, *, suggested_nif: str = "", lookup_value: str = "") -> bool:
+    """Avoid re-applying the same fiscal suggestion that an operator rejected.
+
+    A rejected wrong NIF must not block a future correct NIF for the same company
+    mention.  Therefore, when a NIF exists, the guard is scoped to that NIF only.
+    Lookup-value matching is used only for suggestions without NIF.
+    """
+    if not _clean(opportunity_id):
+        return False
+    nif = normalize_tax_id(suggested_nif)
+    lookup = _clean(lookup_value)
+    if not nif and not lookup:
+        return False
+    params = {"opportunity_id": opportunity_id, "nif": nif, "lookup_pattern": f"%{lookup}%"}
+    if nif:
+        where_clause = "suggested_nif = :nif"
+    else:
+        where_clause = "lookup_value ILIKE :lookup_pattern"
+    with engine.begin() as conn:
+        row = conn.execute(text(f"""
+            SELECT 1
+            FROM fiscal_customer_suggestions
+            WHERE opportunity_id = CAST(:opportunity_id AS UUID)
+              AND status = 'rejected'
+              AND ({where_clause})
+            LIMIT 1
+        """), params).first()
+    return bool(row)
+

 def _find_internal_customer_by_identity(identity: Optional[Dict[str, Any]], opportunity: Dict[str, Any]) -> Optional[Dict[str, Any]]:
    """Prefer explicit company evidence found in the email body/signature.
@@ -596,7 +732,12 @@ def _find_internal_customer_by_identity(identity: Optional[Dict[str, Any]], oppo
        return None
    mentions = _identity_company_mentions(identity)
    normalized_mentions = _normalized_identity_mentions(identity)
-    domain = normalize_domain(identity.get("domain") or _domain_from_email(opportunity.get("customer_email")))
+    raw_domain = normalize_domain(identity.get("domain") or _domain_from_email(opportunity.get("customer_email")))
+    # v4928.1.5.76: public/ISP domains (sapo.pt, gmail.com, outlook.com, ...)
+    # are valid contact channels but must never be treated as company identity
+    # evidence.  Two unrelated Portuguese companies can both use sapo.pt; using
+    # that as an internal-customer domain match caused false confirmations.
+    domain = "" if is_public_email_domain(raw_domain) else raw_domain
    if not normalized_mentions and not domain:
        return None

@@ -613,28 +754,36 @@ def _find_internal_customer_by_identity(identity: Optional[Dict[str, Any]], oppo

    best: Optional[Dict[str, Any]] = None
    best_score = 0.0
+    best_match_type = "email_identity_company_internal"
    for row in rows:
        name_norm = normalize_fiscal_name(row.get("name"))
        email_domain = _domain_from_email(row.get("email"))
        score = 0.0
+        match_type = "email_identity_company_internal"
        for mention in normalized_mentions:
            if mention and name_norm:
                if mention == name_norm:
                    score = max(score, 98.0)
+                    match_type = "email_identity_company_internal_exact"
                elif _company_name_matches_mention(mention, name_norm):
-                    score = max(score, 92.0)
-        if domain and email_domain and domain == email_domain:
-            score += 3.0 if score else 70.0
+                    # Name-only overlap is useful for operator suggestions, but
+                    # it is not safe enough for auto-application.
+                    score = max(score, 88.0)
+                    match_type = "email_identity_company_internal"
+        if domain and email_domain and not is_public_email_domain(email_domain) and domain == email_domain:
+            score = max(score + 6.0 if score else 96.0, 96.0)
+            match_type = "email_identity_company_internal_domain"
        # same address reinforces an explicit company mention
        if score >= 90 and _clean(identity.get("address")) and _clean(row.get("street_name")):
            if normalize_fiscal_name(row.get("street_name")) in normalize_fiscal_name(identity.get("address")):
                score += 2.0
        if score > best_score:
            best_score = min(score, 99.0)
+            best_match_type = match_type
            best = dict(row)
-    if not best or best_score < 90:
+    if not best or best_score < 88:
        return None
-    return _company_from_customer_row(best, score=best_score)
+    return _company_from_customer_row(best, match_type=best_match_type, score=best_score)


 def _identity_company_conflict(identity: Optional[Dict[str, Any]], company: Dict[str, Any]) -> bool:
@@ -783,6 +932,18 @@ def assist_email_identity_enrichment(opportunity_id: str, *, refresh: bool = Tru
                "status": "email_identity_matches_current_fiscal_customer",
            }

+        lookup_value = ", ".join(identity.get("company_mentions") or [])
+        if _has_manually_rejected_suggestion(
+            opportunity_id,
+            suggested_nif=identity_company.get("nif"),
+            lookup_value=lookup_value,
+        ):
+            return {
+                "seen": 1, "suggested": 0, "auto_applied": 0, "skipped": 1,
+                "identity_used": True, "conflict": conflict,
+                "reason": "manual_rejection_exists",
+            }
+
        status = "pending"
        auto_applied = False
        if apply_safe and not conflict and not linked_customer_id and suggested_customer_id and _should_auto_apply(opportunity, identity_company, confidence=confidence):
@@ -793,7 +954,7 @@ def assist_email_identity_enrichment(opportunity_id: str, *, refresh: bool = Tru
            opportunity_id,
            identity_company,
            lookup_type="email_identity",
-            lookup_value=", ".join(identity.get("company_mentions") or []),
+            lookup_value=lookup_value,
            confidence=confidence,
            status=status,
            suggested_customer_id=suggested_customer_id,
@@ -835,6 +996,15 @@ def assist_email_identity_enrichment(opportunity_id: str, *, refresh: bool = Tru
        confidence = _apply_identity_confidence_guard(company, identity=identity, confidence=_confidence_for_company(company, lookup_type="name"))
        if confidence < 75:
            continue
+        if _has_manually_rejected_suggestion(
+            opportunity_id,
+            suggested_nif=company.get("nif") or company.get("tax_id"),
+            lookup_value=mention,
+        ):
+            return {
+                "seen": 1, "suggested": 0, "auto_applied": 0, "skipped": 1,
+                "identity_used": True, "reason": "manual_rejection_exists",
+            }
        suggestion = _upsert_suggestion(
            opportunity_id,
            company,
@@ -869,6 +1039,38 @@ def assist_email_identity_enrichment(opportunity_id: str, *, refresh: bool = Tru
    return {"seen": 1, "suggested": 0, "auto_applied": 0, "skipped": 0, "identity_used": bool(identity), "status": "identity_only"}


+def _apply_source_name_confidence_guard(
+    opportunity: Dict[str, Any],
+    company: Dict[str, Any],
+    *,
+    lookup_type: str,
+    confidence: float,
+) -> float:
+    """Block unsafe auto-application when an exact NIF maps to another company name.
+
+    The external pipeline is a candidate source, not the fiscal source of truth.
+    A syntactically exact NIF response can still be attached to the wrong row.
+    When the operational source already carries a company-like legal name and
+    the returned company has no strong name overlap, keep the result only as a
+    manual suggestion.
+    """
+    adjusted = float(confidence or 0.0)
+    if str(lookup_type or "").strip().lower() != "nif":
+        return adjusted
+    source_name = _clean(opportunity.get("customer_name") or opportunity.get("title"))
+    candidate_name = _clean(company.get("nome") or company.get("legal_name") or company.get("name"))
+    if not (_is_likely_company_name_signal(source_name) and candidate_name):
+        return adjusted
+    source_norm = normalize_fiscal_name(source_name)
+    candidate_norm = normalize_fiscal_name(candidate_name)
+    if _company_name_matches_mention(source_norm, candidate_norm):
+        return adjusted
+    company["source_name_conflict"] = True
+    company["source_company_name"] = source_name
+    company["external_company_name"] = candidate_name
+    return min(adjusted, 70.0)
+
+
 def _apply_identity_confidence_guard(company: Dict[str, Any], *, identity: Optional[Dict[str, Any]], confidence: float) -> float:
    """Lower confidence when the endpoint only matched a domain and the email mentions another company."""
    match_type = _clean(company.get("match_type") or company.get("empresa_resolution_type"))
@@ -992,9 +1194,17 @@ def _should_auto_apply(opportunity: Dict[str, Any], company: Dict[str, Any], *,
        return False
    if _has_conflicting_customer(opportunity, tax_id):
        return False
+    if company.get("identity_conflict") or company.get("source_name_conflict"):
+        return False
    match_type = _clean(company.get("match_type") or company.get("empresa_resolution_type"))
    if match_type in DOMAIN_ONLY_MATCH_TYPES:
        return False
+    if match_type == "email_identity_company_internal":
+        return False
+    if match_type == "email_identity_company_internal_exact":
+        return False
+    if match_type == "email_identity_company_internal_domain":
+        return confidence >= _auto_threshold()
    return confidence >= _auto_threshold() and match_type in VERY_STRONG_MATCH_TYPES


@@ -1112,6 +1322,18 @@ def enrich_opportunity(opportunity_id: str, *, apply_safe: bool = True) -> Dict[
    if identity_company:
        confidence = _confidence_for_company(identity_company, lookup_type="email_identity")
        existing_customer_id = _clean(identity_company.get("clientflow_customer_id")) or None
+        lookup_value = ", ".join(identity.get("company_mentions") or []) if identity else ""
+        if _has_manually_rejected_suggestion(
+            opportunity_id,
+            suggested_nif=identity_company.get("nif"),
+            lookup_value=lookup_value,
+        ):
+            return {
+                "seen": 1, "enriched": 0, "suggested": 0, "auto_applied": 0,
+                "skipped": 1, "reason": "manual_rejection_exists",
+                "errors": errors, "identity_used": True,
+            }
+
        auto_applied = False
        status = "pending"
        if apply_safe and existing_customer_id and _should_auto_apply(opportunity, identity_company, confidence=confidence):
@@ -1122,7 +1344,7 @@ def enrich_opportunity(opportunity_id: str, *, apply_safe: bool = True) -> Dict[
            opportunity_id,
            identity_company,
            lookup_type="email_identity",
-            lookup_value=", ".join(identity.get("company_mentions") or []) if identity else "",
+            lookup_value=lookup_value,
            confidence=confidence,
            status=status,
            suggested_customer_id=existing_customer_id,
@@ -1155,6 +1377,9 @@ def enrich_opportunity(opportunity_id: str, *, apply_safe: bool = True) -> Dict[
            continue
        confidence = _confidence_for_company(company, lookup_type=lookup_type)
        confidence = _apply_identity_confidence_guard(company, identity=identity, confidence=confidence)
+        confidence = _apply_source_name_confidence_guard(
+            opportunity, company, lookup_type=lookup_type, confidence=confidence
+        )
        match_type = _clean(company.get("match_type") or company.get("empresa_resolution_type"))
        # v4.9.25.1: keep the enrichment queue operationally clean.
        # Fuzzy/name-only matches below 75 or explicit approximate-name matches
@@ -1168,6 +1393,12 @@ def enrich_opportunity(opportunity_id: str, *, apply_safe: bool = True) -> Dict[
            existing = get_customer_by_tax_id(tax_id)
            if existing:
                existing_customer_id = str(existing.get("id") or "") or None
+        if _has_manually_rejected_suggestion(opportunity_id, suggested_nif=tax_id, lookup_value=lookup_value):
+            return {
+                "seen": 1, "enriched": 0, "suggested": 0, "auto_applied": 0,
+                "skipped": 1, "reason": "manual_rejection_exists", "errors": errors,
+            }
+
        auto_applied = False
        status = "pending"
        if apply_safe and _should_auto_apply(opportunity, company, confidence=confidence):