Files
clientflow_backend/app/external_reconciliation_sync.py

1109 lines
45 KiB
Python

"""External API sync for operational reconciliation.
This module is deliberately conservative: it fetches loose evidence from
Jasmin/Odoo/Packlink and creates reconciliation candidates. It does not create
opportunities, confirm payments, issue invoices or close processes without an
operator decision.
"""
from __future__ import annotations
import asyncio
import json
import re
from datetime import date, datetime, timedelta, timezone
from decimal import Decimal, InvalidOperation
from typing import Any, Dict, Iterable, List, Optional
from sqlalchemy import text
from app.config import settings
from app.db import engine
from app.commercial_service import upsert_customer
from app.reconciliation_service import upsert_reconciliation_item
JASMIN_QUOTATION_TYPE = "jasmin_quotation"
JASMIN_INVOICE_TYPE = "jasmin_invoice"
JASMIN_PROFORMA_TYPE = "jasmin_proforma"
ODOO_SALE_ORDER_TYPE = "odoo_sale_order"
PACKLINK_SHIPMENT_TYPE = "packlink_shipment"
def _clean(value: Any) -> str:
return str(value or "").strip()
def _normalize_tax_id(value: Any) -> str:
raw = _clean(value).upper().replace(" ", "").replace("-", "").replace(".", "")
if raw.startswith("PT"):
raw = raw[2:]
return raw
def _first(record: Dict[str, Any], *keys: str) -> Any:
for key in keys:
if key in record and record.get(key) not in (None, ""):
return record.get(key)
return None
def _as_list(data: Any) -> List[Dict[str, Any]]:
"""Normalize common REST/OData payload shapes into a list of records."""
if data is None:
return []
if isinstance(data, list):
return [x for x in data if isinstance(x, dict)]
if isinstance(data, dict):
for key in ("value", "items", "data", "results", "shipments"):
value = data.get(key)
if isinstance(value, list):
return [x for x in value if isinstance(x, dict)]
return []
def _decimal_or_none(value: Any) -> Optional[str]:
raw = _clean(value).replace("", "").replace(" ", "").replace(",", ".")
if not raw:
return None
try:
return str(Decimal(raw).quantize(Decimal("0.01")))
except (InvalidOperation, ValueError):
return None
def _date_or_none(value: Any) -> Optional[str]:
if not value:
return None
if isinstance(value, datetime):
return value.date().isoformat()
if isinstance(value, date):
return value.isoformat()
text_value = _clean(value)
if not text_value:
return None
# Jasmin/Odoo commonly return ISO timestamps. PostgreSQL DATE accepts the
# YYYY-MM-DD prefix and rejecting unclear values keeps the sync safe.
if len(text_value) >= 10 and text_value[4:5] == "-" and text_value[7:8] == "-":
return text_value[:10]
return None
def _date_sort_key(value: Optional[str]) -> str:
# Keep undated records last when sorting newest first.
return value or "0000-00-00"
def _recent_window_start(days: int, *, today: Optional[date] = None) -> date:
"""Return the inclusive start date for a human "last N days" window.
Operators read "últimos 3 dias" as today plus the previous two days.
For example, on 2026-06-05, days=3 starts at 2026-06-03, not
2026-06-02.
"""
days = max(int(days or 1), 1)
return (today or datetime.now(timezone.utc).date()) - timedelta(days=days - 1)
def _record_date(record: Dict[str, Any]) -> Optional[str]:
return _date_or_none(_first(record, "documentDate", "date", "creationDate", "postingDate"))
def _within_days(record: Dict[str, Any], *, days: int, today: Optional[date] = None) -> bool:
days = max(int(days or 0), 1)
date_value = _record_date(record)
if not date_value:
return False
start = _recent_window_start(days, today=today)
return date_value >= start.isoformat()
def _currency(record: Dict[str, Any]) -> str:
value = _first(record, "currency", "currencyKey", "currency_id", "currencyCode")
if isinstance(value, (list, tuple)) and value:
value = value[1] if len(value) > 1 else value[0]
return _clean(value) or "EUR"
def _external_id(record: Dict[str, Any], *, fallback_prefix: str) -> str:
value = _first(
record,
"id",
"key",
"naturalKey",
"documentKey",
"documentId",
"externalId",
"name",
"reference",
"shipmentReference",
"tracking_number",
)
if value:
return _clean(value)
digest = json.dumps(record, ensure_ascii=False, default=str, sort_keys=True)[:120]
return f"{fallback_prefix}:{abs(hash(digest))}"
def _document_number(record: Dict[str, Any]) -> str:
composed = _first(record, "documentNumber", "number", "naturalKey", "name", "reference")
if composed:
return _clean(composed)
doc_type = _clean(_first(record, "documentType", "documentTypeKey"))
serie = _clean(_first(record, "serie", "serieKey", "series"))
number = _clean(_first(record, "seriesNumber", "sequenceNumber"))
return " ".join(part for part in [doc_type, serie, number] if part)
def _customer_name(record: Dict[str, Any]) -> str:
value = _first(
record,
"customerName",
"partyName",
"buyerCustomerPartyName",
"sellerSupplierPartyName",
"name",
"partner_name",
"recipient_name",
)
if isinstance(value, (list, tuple)) and value:
value = value[1] if len(value) > 1 else value[0]
return _clean(value)
def _customer_email(record: Dict[str, Any]) -> str:
return _clean(_first(record, "customerEmail", "email", "electronicMail", "recipient_email", "partner_email"))
def _customer_tax_id(record: Dict[str, Any]) -> str:
"""Extract NIF/VAT from common Jasmin/Odoo/Packlink payload shapes."""
direct = _first(
record,
"customerTaxId",
"customerTaxID",
"customer_tax_id",
"companyTaxID",
"companyTaxId",
"partyTaxId",
"partyTaxID",
"buyerCustomerPartyTaxId",
"buyerCustomerPartyTaxID",
"taxId",
"taxID",
"vat",
"partner_vat",
"nif",
)
if direct:
return _normalize_tax_id(direct)
for container_key in ("customer", "customerParty", "buyerCustomerParty", "party", "partner", "recipient"):
nested = record.get(container_key)
if isinstance(nested, dict):
nested_value = _first(nested, "tax_id", "taxId", "taxID", "companyTaxID", "vat", "nif")
if nested_value:
return _normalize_tax_id(nested_value)
return ""
COMPANY_NAME_MARKERS = (
"lda", "limitada", "unipessoal", "s.a", " sa", "sociedade anónima",
"sociedade anonima", "sgps", "ace", "crl", "sucursal", "empresa",
"construções", "construcoes", "soluções", "solucoes", "indústria",
"industria", "comércio", "comercio", "portugal",
)
def _looks_like_company_name(value: Any) -> bool:
text_value = f" {_clean(value).casefold()} "
if not text_value.strip():
return False
return any(marker in text_value for marker in COMPANY_NAME_MARKERS)
def _external_customer_key(record: Dict[str, Any], *, source_system: str) -> str:
if source_system == "jasmin":
return _clean(_first(record, "partyKey", "customerPartyKey", "naturalKey", "key", "id"))
if source_system == "odoo":
return _clean(_first(record, "partner_external_id", "id"))
return _clean(_first(record, "id", "key", "externalId"))
def _looks_like_odoo_sale_order_reference(value: Any) -> bool:
"""Return True for Odoo sale order numbers such as S00265.
Sale order references identify the commercial process, not the fiscal
customer. They must never be stored in customers.name.
"""
return bool(re.fullmatch(r"S\d{4,}", _clean(value).upper()))
def _odoo_fiscal_customer_name(record: Dict[str, Any]) -> str:
"""Prefer Odoo partner legal name over sale.order.name.
Recent Odoo sale orders have ``name`` equal to S00xxx and the real fiscal
entity in ``partner_name`` or in the many2one ``partner_id`` display name.
"""
name = _clean(record.get("partner_name"))
if not name:
partner_id = record.get("partner_id")
if isinstance(partner_id, (list, tuple)) and partner_id:
name = _clean(partner_id[1] if len(partner_id) > 1 else partner_id[0])
if name and not _looks_like_odoo_sale_order_reference(name):
return name
# res.partner customer seeding records use ``name`` as the partner name.
raw_name = _customer_name(record)
if raw_name and not _looks_like_odoo_sale_order_reference(raw_name):
return raw_name
return ""
def _country_code(value: Any) -> str:
if isinstance(value, (list, tuple)) and value:
return _clean(value[1] if len(value) > 1 else value[0]) or "PT"
return _clean(value) or "PT"
def _customer_seed_data_from_record(record: Dict[str, Any], *, source_system: str) -> Optional[Dict[str, Any]]:
if source_system == "odoo":
name = _odoo_fiscal_customer_name(record)
else:
name = _customer_name(record) or _clean(record.get("partner_name"))
tax_id = _customer_tax_id(record)
if not name:
return None
# Never create/update a fiscal customer with an Odoo sale order reference
# (S00265, S00277, ...). That reference belongs to the purchase process.
if source_system == "odoo" and _looks_like_odoo_sale_order_reference(name):
return None
# Jasmin customer parties are fiscal/commercial customers by definition.
# Odoo res.partner can also contain people/contacts, so require VAT, an
# explicit company flag, or a clear legal-company name before creating a
# fiscal customer in ClientFlow.
is_company = bool(record.get("is_company")) or _clean(record.get("company_type")) == "company"
if source_system == "odoo" and not (tax_id or is_company or _looks_like_company_name(name)):
return None
external_key = _external_customer_key(record, source_system=source_system)
metadata = {
"seeded_by_reconciliation": True,
"source_system": source_system,
"external_customer_key": external_key,
"raw_customer_payload": record,
}
data = {
"name": name,
"tax_id": tax_id,
"email": _customer_email(record),
"phone": _clean(_first(record, "phone", "telephone", "mobile", "phoneNumber")),
"street_name": _clean(_first(record, "streetName", "street", "address", "street_name")),
"postal_zone": _clean(_first(record, "postalZone", "zip", "postal_code", "postalCode")),
"city_name": _clean(_first(record, "cityName", "city")),
"country": _country_code(_first(record, "country", "country_id", "countryCode")),
"metadata": metadata,
}
if source_system == "jasmin":
data["jasmin_customer_party_key"] = external_key
data["jasmin_customer_id"] = _clean(_first(record, "id", "uuid")) or None
return data
def _upsert_fiscal_customer_from_external_record(record: Dict[str, Any], *, source_system: str) -> Optional[str]:
data = _customer_seed_data_from_record(record, source_system=source_system)
if not data:
return None
try:
customer = upsert_customer(data)
return str(customer.get("id") or "") or None
except Exception:
# Customer seeding must never block candidate creation. A document can
# still be reviewed manually if the fiscal customer import fails.
return None
def _already_linked_to_opportunity(*, source_system: str, external_id: str) -> bool:
if not external_id:
return False
try:
with engine.begin() as conn:
value = conn.execute(text("""
SELECT EXISTS (
SELECT 1 FROM commercial_documents
WHERE system = :source_system
AND external_id = :external_id
AND opportunity_id IS NOT NULL
)
"""), {"source_system": source_system, "external_id": external_id}).scalar()
return bool(value)
except Exception:
# commercial_documents may not exist yet on very old deployments. The
# sync must not fail because of the optional local cache.
return False
def _jasmin_external_type(record: Dict[str, Any], default_type: str) -> str:
text_blob = " ".join(
_clean(_first(record, key))
for key in ("documentType", "documentTypeKey", "documentKind", "naturalKey", "documentNumber")
).lower()
if "proforma" in text_blob or "pró-forma" in text_blob or "pro-forma" in text_blob:
return JASMIN_PROFORMA_TYPE
if "invoice" in text_blob or "fatura" in text_blob or "factura" in text_blob or "ft" in text_blob.split():
return JASMIN_INVOICE_TYPE
return default_type
def _jasmin_amount(record: Dict[str, Any]) -> Optional[str]:
return _decimal_or_none(_first(record, "payableAmount", "totalAmount", "total", "grossAmount", "amount"))
def _jasmin_candidate_from_record(record: Dict[str, Any], *, default_type: str) -> Optional[Dict[str, Any]]:
external_id = _external_id(record, fallback_prefix="jasmin")
if _already_linked_to_opportunity(source_system="jasmin", external_id=external_id):
return None
external_type = _jasmin_external_type(record, default_type)
document_number = _document_number(record) or external_id
customer_name = _customer_name(record)
amount = _jasmin_amount(record)
date_value = _date_or_none(_first(record, "documentDate", "date", "creationDate", "postingDate"))
label = {
JASMIN_QUOTATION_TYPE: "Orçamento Jasmin sem oportunidade",
JASMIN_PROFORMA_TYPE: "Pró-forma Jasmin sem oportunidade",
JASMIN_INVOICE_TYPE: "Fatura Jasmin sem oportunidade",
}.get(external_type, "Documento Jasmin sem oportunidade")
suggested_action = {
JASMIN_QUOTATION_TYPE: "SEND_PROFORMA",
JASMIN_PROFORMA_TYPE: "CONFIRM_PAYMENT",
JASMIN_INVOICE_TYPE: "CONFIRM_PAYMENT",
}.get(external_type, "REVIEW_MANUALLY")
priority = "alta" if external_type in {JASMIN_PROFORMA_TYPE, JASMIN_INVOICE_TYPE} else "normal"
customer_id = _upsert_fiscal_customer_from_external_record(record, source_system="jasmin")
return {
"source_system": "jasmin",
"external_type": external_type,
"external_id": external_id,
"title": f"{label} · {document_number}",
"description": "Documento encontrado via API Jasmin. Operador deve ligar a oportunidade existente, criar oportunidade ou ignorar.",
"priority": priority,
"suggested_action": suggested_action,
"confidence": 0.75 if customer_name else 0.55,
"customer_id": customer_id,
"customer_name": customer_name,
"customer_email": _customer_email(record),
"customer_tax_id": _customer_tax_id(record),
"document_number": document_number,
"document_date": date_value,
"amount": amount,
"currency": _currency(record),
"payload": {"source": "jasmin_api", "record": record},
"idempotency_key": f"api-sync:jasmin:{external_type}:{external_id}",
}
async def sync_jasmin_reconciliation_candidates(*, limit: int = 100, days: int = 3) -> Dict[str, Any]:
"""Fetch recent Jasmin quotations/invoices and stage unlinked candidates.
`days` is intentionally enforced locally after the API call. Some Jasmin
tenants accept OData `$filter`, while others return broader result sets.
The reconciliation queue must only show the recent window requested by the
operator, never historic invoices just because they are first in the API
default order.
"""
if not bool(settings.jasmin_enabled):
return {"source": "jasmin", "enabled": False, "seen": 0, "created_or_updated": 0, "skipped": "JASMIN_ENABLED=false"}
from app.jasmin_client import JasminClient
client = JasminClient()
seen = 0
created = 0
errors: List[str] = []
limit = min(max(int(limit or 100), 1), 500)
days = max(int(days or 3), 1)
since = _recent_window_start(days).isoformat()
async def _fetch(kind: str) -> List[Dict[str, Any]]:
# Fetch a wider first page per document family because final `limit` is
# global after merging quotations and invoices. Ask Jasmin for recent
# documents first, then still validate the date locally.
top = min(max(limit * 2, 20), 100)
odata_filter = f"documentDate ge {since}"
try:
if kind == "quotation":
data = await client.list_quotations(top=top, skip=0, filter=odata_filter, orderby="documentDate desc")
else:
data = await client.list_invoices(top=top, skip=0, filter=odata_filter, orderby="documentDate desc")
return _as_list(data)
except Exception as exc:
errors.append(f"{kind} filtered: {exc}")
try:
# Compatibility fallback for tenants where OData filtering over
# documentDate is not supported. Local filtering still applies.
if kind == "quotation":
data = await client.list_quotations(top=top, skip=0, orderby="documentDate desc")
else:
data = await client.list_invoices(top=top, skip=0, orderby="documentDate desc")
return _as_list(data)
except Exception as fallback_exc:
errors.append(f"{kind}: {fallback_exc}")
return []
staged: List[tuple[str, Dict[str, Any]]] = []
for default_type, records in [
(JASMIN_QUOTATION_TYPE, await _fetch("quotation")),
(JASMIN_INVOICE_TYPE, await _fetch("invoice")),
]:
for record in records:
if not _within_days(record, days=days):
continue
staged.append((default_type, record))
staged.sort(key=lambda item: _date_sort_key(_record_date(item[1])), reverse=True)
for default_type, record in staged[:limit]:
seen += 1
candidate = _jasmin_candidate_from_record(record, default_type=default_type)
if not candidate:
continue
upsert_reconciliation_item(**candidate)
created += 1
return {
"source": "jasmin",
"enabled": True,
"seen": seen,
"created_or_updated": created,
"days": days,
"since": since,
"errors": errors,
}
def _odoo_partner_name(value: Any) -> str:
if isinstance(value, (list, tuple)) and value:
return _clean(value[1] if len(value) > 1 else value[0])
return _clean(value)
def _odoo_currency(value: Any) -> str:
if isinstance(value, (list, tuple)) and value:
return _clean(value[1] if len(value) > 1 else value[0]) or "EUR"
return _clean(value) or "EUR"
def _odoo_m2o_id(value: Any) -> Optional[int]:
if isinstance(value, (list, tuple)) and value:
try:
return int(value[0])
except (TypeError, ValueError):
return None
try:
return int(value)
except (TypeError, ValueError):
return None
def _odoo_m2o_name(value: Any) -> str:
if isinstance(value, (list, tuple)) and value:
return _clean(value[1] if len(value) > 1 else value[0])
return _clean(value)
def _odoo_search_read_safe(client: Any, model: str, domain: List[Any], fields: List[str], *, limit: int = 100, order: str = "id asc") -> List[Dict[str, Any]]:
try:
rows = client.search_read(model, domain=domain, fields=fields, limit=limit, order=order)
return [dict(row) for row in rows]
except Exception:
return []
def _odoo_fetch_sale_lines(client: Any, sale_id: int) -> List[Dict[str, Any]]:
if not sale_id:
return []
base_fields = ["id", "order_id", "product_id", "name", "product_uom_qty", "price_unit", "price_total"]
extended_fields = base_fields + ["qty_delivered", "qty_invoiced"]
rows = _odoo_search_read_safe(client, "sale.order.line", [["order_id", "=", sale_id]], extended_fields, limit=200, order="id asc")
if not rows:
rows = _odoo_search_read_safe(client, "sale.order.line", [["order_id", "=", sale_id]], base_fields, limit=200, order="id asc")
normalized = []
for row in rows:
normalized.append({
"id": row.get("id"),
"product_id": _odoo_m2o_id(row.get("product_id")),
"product_name": _odoo_m2o_name(row.get("product_id")) or _clean(row.get("name")),
"description": _clean(row.get("name")),
"qty_ordered": row.get("product_uom_qty"),
"qty_delivered": row.get("qty_delivered"),
"qty_invoiced": row.get("qty_invoiced"),
"price_unit": row.get("price_unit"),
"price_total": row.get("price_total"),
})
return normalized
def _odoo_fetch_pickings(client: Any, sale_name: str) -> List[Dict[str, Any]]:
if not sale_name:
return []
fields = ["id", "name", "state", "origin", "picking_type_id", "scheduled_date", "date_done"]
rows = _odoo_search_read_safe(client, "stock.picking", [["origin", "ilike", sale_name]], fields, limit=100, order="id desc")
return [
{
"id": row.get("id"),
"name": row.get("name"),
"state": row.get("state"),
"origin": row.get("origin"),
"type": _odoo_m2o_name(row.get("picking_type_id")),
"scheduled_date": row.get("scheduled_date"),
"date_done": row.get("date_done"),
}
for row in rows
]
def _odoo_fetch_productions(client: Any, sale_name: str) -> List[Dict[str, Any]]:
if not sale_name:
return []
fields = ["id", "name", "state", "origin", "product_id", "product_qty", "date_start", "date_finished"]
rows = _odoo_search_read_safe(client, "mrp.production", [["origin", "ilike", sale_name]], fields, limit=100, order="id desc")
return [
{
"id": row.get("id"),
"name": row.get("name"),
"state": row.get("state"),
"origin": row.get("origin"),
"product_name": _odoo_m2o_name(row.get("product_id")),
"qty": row.get("product_qty"),
"date_start": row.get("date_start"),
"date_finished": row.get("date_finished"),
}
for row in rows
]
def _odoo_is_outgoing_picking(picking: Dict[str, Any]) -> bool:
name = _clean(picking.get("name")).upper()
picking_type = _clean(picking.get("type")).lower()
if "/OUT/" in name or name.startswith("WH/OUT"):
return True
return any(keyword in picking_type for keyword in ["delivery", "outgoing", "entrega", "expedi", "saída", "saida"])
def _odoo_derive_fulfilment(record: Dict[str, Any]) -> Dict[str, Any]:
lines = record.get("order_lines") if isinstance(record.get("order_lines"), list) else []
pickings = record.get("pickings") if isinstance(record.get("pickings"), list) else []
productions = record.get("productions") if isinstance(record.get("productions"), list) else []
outgoing = [p for p in pickings if _odoo_is_outgoing_picking(p)] or pickings
delivery_done = bool(outgoing) and all(_clean(p.get("state")) == "done" for p in outgoing)
delivery_ready = any(_clean(p.get("state")) == "assigned" for p in outgoing)
production_active = any(_clean(p.get("state")) in {"progress", "to_close", "confirmed"} for p in productions)
invoice_status = _clean(record.get("invoice_status"))
has_uninvoiced_lines = False
for line in lines:
ordered = line.get("qty_ordered") or 0
invoiced = line.get("qty_invoiced")
try:
if invoiced is not None and float(invoiced) < float(ordered or 0):
has_uninvoiced_lines = True
except (TypeError, ValueError):
pass
invoice_pending = invoice_status in {"to invoice", "no"} or has_uninvoiced_lines
physical_status = "order_created"
label = "Venda criada"
stage = "ODOO_ORDER_CREATED"
if delivery_done:
physical_status = "shipped"
label = "Entrega concluída no Odoo"
stage = "SHIPPED"
elif delivery_ready:
physical_status = "picking_assigned"
label = "Picking reservado — validação física pendente"
stage = "ORDER_PREPARATION"
elif production_active:
physical_status = "in_production"
label = "Em produção/preparação"
stage = "IN_PRODUCTION"
return {
"physical_status": physical_status,
"label": label,
"stage": stage,
"delivery_done": delivery_done,
"delivery_ready": delivery_ready,
"invoice_pending": invoice_pending,
"invoice_status": invoice_status,
"next_action": "SEND_INVOICE" if invoice_pending else "REVIEW_MANUALLY",
"outgoing_pickings": outgoing,
"lines": lines,
"productions": productions,
}
def _odoo_candidate_from_record(record: Dict[str, Any]) -> Dict[str, Any]:
external_id = _clean(_first(record, "id", "name"))
order_name = _clean(_first(record, "name", "client_order_ref")) or f"Odoo #{external_id}"
customer_name = _clean(record.get("partner_name")) or _odoo_partner_name(record.get("partner_id"))
state = _clean(record.get("state"))
invoice_status = _clean(record.get("invoice_status"))
amount = _decimal_or_none(record.get("amount_total"))
fulfilment = record.get("fulfilment") if isinstance(record.get("fulfilment"), dict) else _odoo_derive_fulfilment(record)
invoice_pending = bool(fulfilment.get("invoice_pending")) or invoice_status in {"to invoice", "no"}
customer_id = _upsert_fiscal_customer_from_external_record(record, source_system="odoo")
return {
"source_system": "odoo",
"external_type": ODOO_SALE_ORDER_TYPE,
"external_id": external_id or order_name,
"title": f"Venda Odoo sem oportunidade · {order_name}",
"description": "Venda/encomenda encontrada via API Odoo. Operador deve ligar a oportunidade existente, criar oportunidade ou ignorar.",
"priority": "alta" if state in {"sale", "done"} and invoice_pending else "normal",
"suggested_action": "SEND_INVOICE" if invoice_pending else "REVIEW_MANUALLY",
"confidence": 0.70 if customer_name else 0.50,
"customer_id": customer_id,
"customer_name": customer_name,
"customer_email": _customer_email(record),
"customer_tax_id": _customer_tax_id(record),
"document_number": order_name,
"document_date": _date_or_none(record.get("date_order") or record.get("create_date")),
"amount": amount,
"currency": _odoo_currency(record.get("currency_id")),
"payload": {"source": "odoo_api", "record": record},
"idempotency_key": f"api-sync:odoo:sale_order:{external_id or order_name}",
}
def _existing_odoo_sale_links(external_id: Any, order_name: Any) -> List[Dict[str, Any]]:
"""Return exact open opportunity links for one Odoo sale order.
Matching is deliberately strict: numeric Odoo id or exact sale name. Name,
customer and amount suggestions belong to the operator review path and must
never auto-resolve a reconciliation item.
"""
external_id = _clean(external_id)
order_name = _clean(order_name)
if not external_id and not order_name:
return []
try:
begin = engine.begin
except Exception:
return []
try:
with begin() as conn:
rows = conn.execute(text("""
SELECT DISTINCT
ol.opportunity_id::text AS opportunity_id,
o.title,
o.stage,
o.status,
ol.external_id,
ol.external_name
FROM operation_links ol
JOIN opportunities o ON o.id = ol.opportunity_id
WHERE ol.system = 'odoo'
AND ol.external_type = 'sale_order'
AND o.status = 'open'
AND (
(NULLIF(:external_id, '') IS NOT NULL AND ol.external_id = :external_id)
OR (NULLIF(:order_name, '') IS NOT NULL AND UPPER(COALESCE(ol.external_name, '')) = UPPER(:order_name))
)
ORDER BY ol.opportunity_id
"""), {"external_id": external_id, "order_name": order_name}).mappings().all()
except Exception:
# Reconciliation sync must remain conservative when local schema access
# is unavailable: keep staging the candidate rather than auto-resolving.
return []
return [dict(row) for row in rows]
def _resolve_existing_odoo_reconciliation_item(
*,
external_id: Any,
order_name: Any,
opportunity_id: str,
) -> int:
"""Close stale open candidates without changing opportunity stage or tasks."""
external_id = _clean(external_id)
order_name = _clean(order_name)
with engine.begin() as conn:
result = conn.execute(text("""
UPDATE reconciliation_items ri
SET opportunity_id = CAST(:opportunity_id AS UUID),
status = 'linked',
resolution_note = 'Resolvido automaticamente: venda Odoo já ligada à oportunidade',
resolved_at = now(),
updated_at = now(),
payload = COALESCE(ri.payload, '{}'::jsonb) || jsonb_build_object(
'resolved_as_existing_operation_link', TRUE,
'resolved_by', 'odoo_reconciliation_sync_v129',
'resolved_sale_order', COALESCE(NULLIF(:order_name, ''), NULLIF(:external_id, ''))
)
WHERE ri.source_system = 'odoo'
AND ri.external_type = 'odoo_sale_order'
AND ri.status IN ('open', 'needs_review', 'conflict')
AND (
(NULLIF(:external_id, '') IS NOT NULL AND ri.external_id = :external_id)
OR (NULLIF(:order_name, '') IS NOT NULL AND UPPER(COALESCE(ri.document_number, '')) = UPPER(:order_name))
)
"""), {
"external_id": external_id,
"order_name": order_name,
"opportunity_id": opportunity_id,
})
return int(result.rowcount or 0)
def sync_odoo_reconciliation_candidates(*, limit: int = 100, days: int = 3) -> Dict[str, Any]:
"""Fetch recent Odoo sale orders and stage unlinked candidates."""
if not bool(settings.odoo_enabled):
return {"source": "odoo", "enabled": False, "seen": 0, "created_or_updated": 0, "skipped": "ODOO_ENABLED=false"}
from app.odoo_client import OdooClient
client = OdooClient()
days = max(int(days or 3), 1)
since = _recent_window_start(days).strftime("%Y-%m-%d")
domain = [["date_order", ">=", since], ["state", "in", ["sale", "done"]]]
fields = [
"id",
"name",
"partner_id",
"amount_total",
"currency_id",
"date_order",
"state",
"invoice_status",
"client_order_ref",
"create_date",
"order_line",
]
records = client.search_read("sale.order", domain=domain, fields=fields, limit=limit, order="date_order desc")
# sale.order does not reliably expose partner VAT/NIF in all Odoo editions.
# Fetch it from res.partner and enrich the staging record before matching.
partner_ids: List[int] = []
for record in records:
partner = record.get("partner_id")
if isinstance(partner, (list, tuple)) and partner:
try:
partner_ids.append(int(partner[0]))
except (TypeError, ValueError):
pass
partner_vat_by_id: Dict[int, str] = {}
partner_email_by_id: Dict[int, str] = {}
partner_name_by_id: Dict[int, str] = {}
if partner_ids:
partners = client.search_read(
"res.partner",
domain=[["id", "in", sorted(set(partner_ids))]],
fields=["id", "vat", "email", "name"],
limit=len(set(partner_ids)),
order="id asc",
)
for partner in partners:
try:
partner_id = int(partner.get("id"))
except (TypeError, ValueError):
continue
partner_vat_by_id[partner_id] = _normalize_tax_id(partner.get("vat"))
partner_email_by_id[partner_id] = _clean(partner.get("email"))
partner_name_by_id[partner_id] = _clean(partner.get("name"))
seen = 0
created = 0
already_linked = 0
resolved_existing = 0
link_conflicts = 0
for record in records:
partner = record.get("partner_id")
if isinstance(partner, (list, tuple)) and partner:
try:
partner_id = int(partner[0])
record["partner_vat"] = partner_vat_by_id.get(partner_id, "")
record["partner_email"] = partner_email_by_id.get(partner_id, "")
record["partner_name"] = partner_name_by_id.get(partner_id, "")
record["partner_external_id"] = partner_id
except (TypeError, ValueError):
pass
try:
sale_id = int(record.get("id"))
except (TypeError, ValueError):
sale_id = 0
sale_name = _clean(record.get("name"))
record["order_lines"] = _odoo_fetch_sale_lines(client, sale_id)
record["pickings"] = _odoo_fetch_pickings(client, sale_name)
record["productions"] = _odoo_fetch_productions(client, sale_name)
record["fulfilment"] = _odoo_derive_fulfilment(record)
seen += 1
candidate = _odoo_candidate_from_record(record)
existing_links = _existing_odoo_sale_links(candidate.get("external_id"), candidate.get("document_number"))
if len(existing_links) == 1:
already_linked += 1
resolved_existing += _resolve_existing_odoo_reconciliation_item(
external_id=candidate.get("external_id"),
order_name=candidate.get("document_number"),
opportunity_id=existing_links[0]["opportunity_id"],
)
continue
if len(existing_links) > 1:
link_conflicts += 1
candidate["status"] = "conflict"
candidate["priority"] = "alta"
candidate["description"] = (
"A venda Odoo aparece ligada a mais de uma oportunidade. "
"Requer correção manual das ligações antes de reconciliar."
)
candidate["payload"] = {
**(candidate.get("payload") or {}),
"existing_operation_link_conflict": existing_links,
}
upsert_reconciliation_item(**candidate)
created += 1
return {
"source": "odoo",
"enabled": True,
"seen": seen,
"created_or_updated": created,
"already_linked": already_linked,
"resolved_existing": resolved_existing,
"link_conflicts": link_conflicts,
"days": max(int(days), 1),
"since": since,
}
def _packlink_candidate_from_record(record: Dict[str, Any]) -> Dict[str, Any]:
external_id = _external_id(record, fallback_prefix="packlink")
reference = _clean(_first(record, "reference", "shipmentReference", "id", "tracking_number")) or external_id
recipient = record.get("to") if isinstance(record.get("to"), dict) else {}
customer_name = _customer_name(record) or _clean(_first(recipient, "name", "contactName", "company"))
email = _customer_email(record) or _clean(_first(recipient, "email"))
amount = _decimal_or_none(_first(record, "price", "totalPrice", "amount"))
customer_id = _upsert_fiscal_customer_from_external_record(record, source_system="packlink")
return {
"source_system": "packlink",
"external_type": PACKLINK_SHIPMENT_TYPE,
"external_id": external_id,
"title": f"Envio Packlink sem oportunidade · {reference}",
"description": "Envio/tracking encontrado via API Packlink. Operador deve ligar a oportunidade existente ou ignorar.",
"priority": "normal",
"suggested_action": "REVIEW_MANUALLY",
"confidence": 0.55,
"customer_id": customer_id,
"customer_name": customer_name,
"customer_email": email,
"customer_tax_id": _customer_tax_id(record),
"document_number": reference,
"document_date": _date_or_none(_first(record, "createdAt", "created_at", "date")),
"amount": amount,
"currency": _currency(record),
"payload": {"source": "packlink_api", "record": record},
"idempotency_key": f"api-sync:packlink:shipment:{external_id}",
}
async def sync_packlink_reconciliation_candidates(*, limit: int = 100, days: int = 3) -> Dict[str, Any]:
"""Fetch recent Packlink shipments if the API/list endpoint is available."""
if not bool(settings.packlink_enabled):
return {"source": "packlink", "enabled": False, "seen": 0, "created_or_updated": 0, "skipped": "PACKLINK_ENABLED=false"}
from app.packlink_client import PacklinkClient
client = PacklinkClient()
try:
if hasattr(client, "list_shipments"):
data = await client.list_shipments(limit=limit)
else:
data = await client._request("GET", "/shipments", params={"limit": limit}) # type: ignore[attr-defined]
except Exception as exc:
return {"source": "packlink", "enabled": True, "seen": 0, "created_or_updated": 0, "errors": [str(exc)]}
days = max(int(days or 3), 1)
records = []
for record in _as_list(data):
if _within_days(record, days=days):
records.append(record)
if len(records) >= int(limit):
break
created = 0
for record in records:
upsert_reconciliation_item(**_packlink_candidate_from_record(record))
created += 1
return {"source": "packlink", "enabled": True, "seen": len(records), "created_or_updated": created, "days": days}
async def sync_jasmin_fiscal_customers_for_reconciliation(*, limit: int = 200) -> Dict[str, Any]:
"""Seed ClientFlow fiscal customers from Jasmin before document matching."""
if not bool(settings.jasmin_enabled):
return {"source": "jasmin_customers", "enabled": False, "seen": 0, "created_or_updated": 0, "skipped": "JASMIN_ENABLED=false"}
from app.jasmin_client import JasminClient
client = JasminClient()
seen = 0
created = 0
errors: List[str] = []
try:
data = await client.list_customers_odata(top=min(max(int(limit or 200), 1), 100), skip=0)
records = _as_list(data)
except Exception as exc:
errors.append(str(exc))
records = []
for record in records[: max(int(limit or 200), 1)]:
seen += 1
if _upsert_fiscal_customer_from_external_record(record, source_system="jasmin"):
created += 1
return {"source": "jasmin_customers", "enabled": True, "seen": seen, "created_or_updated": created, "errors": errors}
def _odoo_model_field_names(client: Any, model: str, errors: List[str]) -> Optional[set]:
"""Return available Odoo model fields, or None if discovery is unavailable.
Odoo installations can differ by edition/module set. In the production
instance that triggered v4.9.25.2, res.partner did not expose ``mobile``;
asking search_read for that optional field made the entire customer seeding
phase fail even though quotations/sales could still sync.
"""
try:
result = client.execute_kw(model, "fields_get", [], {"attributes": ["type"]})
except Exception as exc:
errors.append(f"Odoo {model}.fields_get falhou; a usar campos compatíveis por defeito: {exc}")
return None
if isinstance(result, dict):
return set(result.keys())
return None
def _odoo_partner_fields_for_available_schema(available_fields: Optional[set]) -> List[str]:
requested = [
"id",
"name",
"vat",
"email",
"phone",
"mobile",
"street",
"zip",
"city",
"country_id",
"is_company",
"company_type",
]
if available_fields is None:
# Most compatible fallback: do not request mobile unless we confirmed it
# exists. This prevents "Invalid field 'mobile' on 'res.partner'".
return [field for field in requested if field != "mobile"]
return [field for field in requested if field in available_fields]
def _odoo_partner_domains_for_available_schema(available_fields: Optional[set]) -> List[List[Any]]:
domains: List[List[Any]] = []
if available_fields is None or {"is_company", "vat"}.issubset(available_fields):
domains.append(["|", ["is_company", "=", True], ["vat", "!=", False]])
elif "vat" in available_fields:
domains.append([["vat", "!=", False]])
if available_fields is None or "customer_rank" in available_fields:
domains.append([["customer_rank", ">", 0]])
# Final fallback: fetch recent partners and let _customer_seed_data... filter
# people/contacts out before creating fiscal customers.
domains.append([])
return domains
def sync_odoo_fiscal_customers_for_reconciliation(*, limit: int = 200) -> Dict[str, Any]:
"""Seed ClientFlow fiscal customers from Odoo companies/partners.
v4.9.25.2 makes this phase schema-compatible with Odoo installs that do
not have optional partner fields such as ``mobile``.
"""
if not bool(settings.odoo_enabled):
return {"source": "odoo_customers", "enabled": False, "seen": 0, "created_or_updated": 0, "skipped": "ODOO_ENABLED=false"}
from app.odoo_client import OdooClient
client = OdooClient()
errors: List[str] = []
available_fields = _odoo_model_field_names(client, "res.partner", errors)
fields = _odoo_partner_fields_for_available_schema(available_fields)
domains = _odoo_partner_domains_for_available_schema(available_fields)
ignored_optional_fields = []
if available_fields is not None:
ignored_optional_fields = [field for field in ("mobile",) if field not in available_fields]
records: List[Dict[str, Any]] = []
for domain in domains:
try:
records = client.search_read("res.partner", domain=domain, fields=fields, limit=limit, order="write_date desc")
break
except Exception as exc:
errors.append(str(exc))
records = []
seen = 0
created = 0
for record in records[: max(int(limit or 200), 1)]:
seen += 1
record["partner_external_id"] = record.get("id")
if _upsert_fiscal_customer_from_external_record(record, source_system="odoo"):
created += 1
return {
"source": "odoo_customers",
"enabled": True,
"seen": seen,
"created_or_updated": created,
"errors": errors,
"ignored_optional_fields": ignored_optional_fields,
}
async def sync_external_fiscal_customers_for_reconciliation(*, limit: int = 200) -> Dict[str, Any]:
"""Phase 1 of reconciliation: create/update fiscal customers first."""
jasmin_result = await sync_jasmin_fiscal_customers_for_reconciliation(limit=limit)
odoo_result = await asyncio.to_thread(sync_odoo_fiscal_customers_for_reconciliation, limit=limit)
results = [jasmin_result, odoo_result]
return {
"seen": sum(int(r.get("seen") or 0) for r in results),
"created_or_updated": sum(int(r.get("created_or_updated") or 0) for r in results),
"results": results,
}
async def sync_all_external_reconciliation_candidates(*, limit: int = 100, days: int = 3) -> Dict[str, Any]:
"""Run the ordered reconciliation pipeline.
v4.9.25 adds Phase 0: enrich open ClientFlow opportunities with fiscal
customers before trying to match Jasmin/Odoo documents. This makes the
rest of the pipeline use stronger keys (NIF/fiscal name/customer id)
instead of relying on contact names.
Phase 1 seeds fiscal customers from Jasmin/Odoo. Phase 2 stages Jasmin
quotations/proformas/invoices. Phase 3 stages Odoo sales/orders. Phase 4
stages shipment evidence. Process cards are then reconstructed by the
reconciliation service from customer identity + purchase-operation anchors.
"""
try:
from app.fiscal_enrichment_service import enrich_open_opportunities
enrichment_result = await asyncio.to_thread(
enrich_open_opportunities,
limit=max(min(limit, 200), 50),
apply_safe=True,
mode="pre_reconciliation",
)
except Exception as exc:
enrichment_result = {"seen": 0, "enriched": 0, "suggested": 0, "auto_applied": 0, "errors": [str(exc)]}
customer_result = await sync_external_fiscal_customers_for_reconciliation(limit=max(limit, 200))
jasmin_result = await sync_jasmin_reconciliation_candidates(limit=limit, days=days)
# Odoo client is synchronous XML-RPC; run it in a thread to keep async routes responsive.
odoo_result = await asyncio.to_thread(sync_odoo_reconciliation_candidates, limit=limit, days=days)
packlink_result = await sync_packlink_reconciliation_candidates(limit=limit, days=days)
document_results = [jasmin_result, odoo_result, packlink_result]
return {
"seen": sum(int(r.get("seen") or 0) for r in document_results),
"created_or_updated": sum(int(r.get("created_or_updated") or 0) for r in document_results),
"enrichment_seen": int(enrichment_result.get("seen") or 0),
"enrichment_suggested": int(enrichment_result.get("suggested") or 0),
"enrichment_auto_applied": int(enrichment_result.get("auto_applied") or 0),
"enrichment_result": enrichment_result,
"customer_seen": int(customer_result.get("seen") or 0),
"customers_created_or_updated": int(customer_result.get("created_or_updated") or 0),
"customer_results": customer_result.get("results") or [],
"results": document_results,
}