perf: optimize reconciliation candidate lookup
This commit is contained in:
plx
@@ -12,6 +12,7 @@ import json
|
||||
|
||||
from sqlalchemy import text
|
||||
|
||||
from app.company_opportunity_linking import is_public_email_domain
|
||||
from app.db import engine
|
||||
from app.reconciliation_service import (
|
||||
_apply_jasmin_documents_to_opportunity, # noqa: PLC2701 - deliberate operator maintenance helper
|
||||
@@ -466,7 +467,8 @@ def find_jasmin_document_candidates_for_opportunity(opportunity_id: str, *, limi
|
||||
fiscal_customer_id = _as_text(opp.get("local_customer_id"))
|
||||
fiscal_name = _as_text(opp.get("fiscal_customer_name"))
|
||||
fiscal_email = _as_text(opp.get("fiscal_customer_email") or opp.get("customer_email")).lower()
|
||||
domain = _email_domain(fiscal_email)
|
||||
raw_domain = _email_domain(fiscal_email)
|
||||
domain = "" if is_public_email_domain(raw_domain) else raw_domain
|
||||
|
||||
current_docs = conn.execute(text("""
|
||||
SELECT
|
||||
@@ -479,8 +481,7 @@ def find_jasmin_document_candidates_for_opportunity(opportunity_id: str, *, limi
|
||||
ORDER BY document_date DESC NULLS LAST, created_at DESC
|
||||
"""), {"opportunity_id": opportunity_id}).mappings().all()
|
||||
|
||||
rows = conn.execute(text("""
|
||||
SELECT
|
||||
candidate_columns = """
|
||||
ri.id::text,
|
||||
ri.source_system,
|
||||
ri.external_type,
|
||||
@@ -502,52 +503,159 @@ def find_jasmin_document_candidates_for_opportunity(opportunity_id: str, *, limi
|
||||
CASE
|
||||
WHEN CAST(:fiscal_customer_id AS TEXT) <> '' AND ri.customer_id::text = CAST(:fiscal_customer_id AS TEXT) THEN 100
|
||||
WHEN CAST(:fiscal_tax_id AS TEXT) <> '' AND ri.customer_tax_id = CAST(:fiscal_tax_id AS TEXT) THEN 98
|
||||
WHEN CAST(:fiscal_tax_id AS TEXT) <> '' AND ri.payload::text ILIKE '%' || CAST(:fiscal_tax_id AS TEXT) || '%' THEN 96
|
||||
{payload_nif_score}
|
||||
WHEN CAST(:fiscal_email AS TEXT) <> '' AND LOWER(COALESCE(ri.customer_email, '')) = CAST(:fiscal_email AS TEXT) THEN 90
|
||||
WHEN CAST(:domain AS TEXT) <> '' AND ri.payload::text ILIKE '%' || CAST(:domain AS TEXT) || '%' THEN 78
|
||||
{payload_domain_score}
|
||||
WHEN CAST(:fiscal_name AS TEXT) <> '' AND LOWER(COALESCE(ri.customer_name, '')) = LOWER(CAST(:fiscal_name AS TEXT)) THEN 75
|
||||
ELSE 0
|
||||
END AS match_score,
|
||||
CASE
|
||||
WHEN CAST(:fiscal_customer_id AS TEXT) <> '' AND ri.customer_id::text = CAST(:fiscal_customer_id AS TEXT) THEN 'customer_id'
|
||||
WHEN CAST(:fiscal_tax_id AS TEXT) <> '' AND ri.customer_tax_id = CAST(:fiscal_tax_id AS TEXT) THEN 'nif'
|
||||
WHEN CAST(:fiscal_tax_id AS TEXT) <> '' AND ri.payload::text ILIKE '%' || CAST(:fiscal_tax_id AS TEXT) || '%' THEN 'payload_nif'
|
||||
{payload_nif_reason}
|
||||
WHEN CAST(:fiscal_email AS TEXT) <> '' AND LOWER(COALESCE(ri.customer_email, '')) = CAST(:fiscal_email AS TEXT) THEN 'email'
|
||||
WHEN CAST(:domain AS TEXT) <> '' AND ri.payload::text ILIKE '%' || CAST(:domain AS TEXT) || '%' THEN 'domain_payload'
|
||||
{payload_domain_reason}
|
||||
WHEN CAST(:fiscal_name AS TEXT) <> '' AND LOWER(COALESCE(ri.customer_name, '')) = LOWER(CAST(:fiscal_name AS TEXT)) THEN 'exact_name'
|
||||
ELSE 'none'
|
||||
END AS match_reason
|
||||
"""
|
||||
common_from = """
|
||||
FROM reconciliation_items ri
|
||||
WHERE ri.source_system = 'jasmin'
|
||||
AND ri.external_type IN ('jasmin_quotation', 'jasmin_proforma', 'jasmin_invoice')
|
||||
AND (ri.opportunity_id IS NULL OR ri.opportunity_id = CAST(:opportunity_id AS UUID))
|
||||
AND NOT (
|
||||
CAST(:fiscal_tax_id AS TEXT) <> ''
|
||||
AND COALESCE(ri.customer_tax_id, '') <> ''
|
||||
AND ri.customer_tax_id <> CAST(:fiscal_tax_id AS TEXT)
|
||||
)
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM commercial_documents cd
|
||||
WHERE cd.opportunity_id = CAST(:opportunity_id AS UUID)
|
||||
AND (
|
||||
(ri.document_number IS NOT NULL AND cd.document_number = ri.document_number)
|
||||
OR (ri.external_id IS NOT NULL AND cd.external_id = ri.external_id)
|
||||
)
|
||||
AND ri.document_number IS NOT NULL
|
||||
AND cd.document_number = ri.document_number
|
||||
)
|
||||
AND (
|
||||
(CAST(:fiscal_customer_id AS TEXT) <> '' AND ri.customer_id::text = CAST(:fiscal_customer_id AS TEXT))
|
||||
OR (CAST(:fiscal_tax_id AS TEXT) <> '' AND ri.customer_tax_id = CAST(:fiscal_tax_id AS TEXT))
|
||||
OR (CAST(:fiscal_tax_id AS TEXT) <> '' AND ri.payload::text ILIKE '%' || CAST(:fiscal_tax_id AS TEXT) || '%')
|
||||
OR (CAST(:fiscal_email AS TEXT) <> '' AND LOWER(COALESCE(ri.customer_email, '')) = CAST(:fiscal_email AS TEXT))
|
||||
OR (CAST(:domain AS TEXT) <> '' AND ri.payload::text ILIKE '%' || CAST(:domain AS TEXT) || '%')
|
||||
OR (CAST(:fiscal_name AS TEXT) <> '' AND LOWER(COALESCE(ri.customer_name, '')) = LOWER(CAST(:fiscal_name AS TEXT)))
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM commercial_documents cd
|
||||
WHERE cd.opportunity_id = CAST(:opportunity_id AS UUID)
|
||||
AND ri.external_id IS NOT NULL
|
||||
AND cd.external_id = ri.external_id
|
||||
)
|
||||
ORDER BY match_score DESC, ri.document_date DESC NULLS LAST, ri.updated_at DESC
|
||||
LIMIT :limit
|
||||
"""), {
|
||||
"""
|
||||
params = {
|
||||
"opportunity_id": opportunity_id,
|
||||
"fiscal_customer_id": fiscal_customer_id,
|
||||
"fiscal_tax_id": fiscal_tax_id,
|
||||
"fiscal_email": fiscal_email,
|
||||
"domain": domain,
|
||||
"fiscal_name": fiscal_name,
|
||||
"limit": int(limit),
|
||||
}).mappings().all()
|
||||
"limit": max(1, int(limit)),
|
||||
}
|
||||
structured_columns = candidate_columns.format(
|
||||
payload_nif_score="",
|
||||
payload_domain_score="",
|
||||
payload_nif_reason="",
|
||||
payload_domain_reason="",
|
||||
)
|
||||
structured_sql = """
|
||||
WITH candidate_ids AS (
|
||||
SELECT ri.id FROM reconciliation_items ri
|
||||
WHERE CAST(:fiscal_customer_id AS TEXT) <> ''
|
||||
AND ri.customer_id IS NOT NULL
|
||||
AND ri.customer_id = CAST(NULLIF(:fiscal_customer_id, '') AS UUID)
|
||||
UNION
|
||||
SELECT ri.id FROM reconciliation_items ri
|
||||
WHERE CAST(:fiscal_tax_id AS TEXT) <> ''
|
||||
AND ri.customer_tax_id IS NOT NULL AND ri.customer_tax_id <> ''
|
||||
AND ri.customer_tax_id = CAST(:fiscal_tax_id AS TEXT)
|
||||
UNION
|
||||
SELECT ri.id FROM reconciliation_items ri
|
||||
WHERE CAST(:fiscal_email AS TEXT) <> ''
|
||||
AND ri.customer_email IS NOT NULL AND ri.customer_email <> ''
|
||||
AND lower(ri.customer_email) = CAST(:fiscal_email AS TEXT)
|
||||
UNION
|
||||
SELECT ri.id FROM reconciliation_items ri
|
||||
WHERE CAST(:fiscal_name AS TEXT) <> ''
|
||||
AND ri.customer_name IS NOT NULL AND ri.customer_name <> ''
|
||||
AND lower(ri.customer_name) = lower(CAST(:fiscal_name AS TEXT))
|
||||
)
|
||||
SELECT {columns}
|
||||
{common_from}
|
||||
AND ri.id IN (SELECT id FROM candidate_ids)
|
||||
ORDER BY match_score DESC, ri.document_date DESC NULLS LAST, ri.updated_at DESC
|
||||
LIMIT :limit
|
||||
""".format(columns=structured_columns, common_from=common_from)
|
||||
structured_rows = [dict(row) for row in conn.execute(text(structured_sql), params).mappings().all()]
|
||||
|
||||
# Payload NIF has score 96. It cannot displace a full page made only of
|
||||
# customer-id/NIF matches (100/98), so that is the semantics-preserving
|
||||
# fast path. Email/name matches still require the legacy pass because a
|
||||
# payload NIF candidate may outrank them.
|
||||
strong_page_complete = (
|
||||
len(structured_rows) >= params["limit"]
|
||||
and int(structured_rows[params["limit"] - 1].get("match_score") or 0) >= 96
|
||||
)
|
||||
if strong_page_complete:
|
||||
rows = structured_rows[:params["limit"]]
|
||||
else:
|
||||
payload_predicates: list[str] = []
|
||||
payload_nif_score = payload_nif_reason = ""
|
||||
payload_domain_score = payload_domain_reason = ""
|
||||
if fiscal_tax_id:
|
||||
payload_predicates.append(
|
||||
"ri.payload::text ILIKE '%' || CAST(:fiscal_tax_id AS TEXT) || '%'"
|
||||
)
|
||||
payload_nif_score = "WHEN ri.payload::text ILIKE '%' || CAST(:fiscal_tax_id AS TEXT) || '%' THEN 96"
|
||||
payload_nif_reason = "WHEN ri.payload::text ILIKE '%' || CAST(:fiscal_tax_id AS TEXT) || '%' THEN 'payload_nif'"
|
||||
if domain:
|
||||
payload_predicates.append(
|
||||
"ri.payload::text ILIKE '%' || CAST(:domain AS TEXT) || '%'"
|
||||
)
|
||||
payload_domain_score = "WHEN ri.payload::text ILIKE '%' || CAST(:domain AS TEXT) || '%' THEN 78"
|
||||
payload_domain_reason = "WHEN ri.payload::text ILIKE '%' || CAST(:domain AS TEXT) || '%' THEN 'domain_payload'"
|
||||
|
||||
if not payload_predicates:
|
||||
rows = structured_rows[:params["limit"]]
|
||||
payload_rows = []
|
||||
else:
|
||||
payload_columns = candidate_columns.format(
|
||||
payload_nif_score=payload_nif_score,
|
||||
payload_domain_score=payload_domain_score,
|
||||
payload_nif_reason=payload_nif_reason,
|
||||
payload_domain_reason=payload_domain_reason,
|
||||
)
|
||||
payload_sql = """
|
||||
SELECT {columns}
|
||||
{common_from}
|
||||
AND ({payload_where})
|
||||
ORDER BY match_score DESC, ri.document_date DESC NULLS LAST, ri.updated_at DESC
|
||||
LIMIT :limit
|
||||
""".format(
|
||||
columns=payload_columns,
|
||||
common_from=common_from,
|
||||
payload_where=" OR ".join(f"({predicate})" for predicate in payload_predicates),
|
||||
)
|
||||
payload_rows = [dict(row) for row in conn.execute(text(payload_sql), params).mappings().all()]
|
||||
|
||||
if payload_predicates:
|
||||
merged = {str(row.get("id")): row for row in structured_rows}
|
||||
merged.update({str(row.get("id")): row for row in payload_rows})
|
||||
rows = sorted(
|
||||
merged.values(),
|
||||
key=lambda row: (
|
||||
int(row.get("match_score") or 0),
|
||||
str(row.get("document_date") or ""),
|
||||
str(row.get("updated_at") or ""),
|
||||
),
|
||||
reverse=True,
|
||||
)[:params["limit"]]
|
||||
|
||||
if fiscal_tax_id:
|
||||
rows = [
|
||||
row for row in rows
|
||||
if not _as_text(row.get("customer_tax_id"))
|
||||
or _as_text(row.get("customer_tax_id")) == fiscal_tax_id
|
||||
]
|
||||
|
||||
current_doc_items = [_commercial_doc_as_jasmin_item(dict(row)) for row in current_docs]
|
||||
current_latest_key = max((_jasmin_item_recency_tuple(item) for item in current_doc_items), default=None)
|
||||
|
||||
Reference in New Issue
Block a user